pixie-lab 0.1.0__tar.gz → 0.3.0__tar.gz

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/.github/workflows/cicd.yaml

@@ -17,6 +17,14 @@ name: cicd
 #     version cannot be re-uploaded -- so the job gates on every
 #     prior job (test + integration + build-container + build-wheel
 #     + build-media + every PXE chain test) succeeding first.
+#   * ``publish-container``: build + push the appliance image to
+#     ghcr.io/<owner>/pixie (``:<version>`` + ``:latest``), v* tags
+#     only. Before this the image was smoke-built and thrown away, so
+#     the documented ghcr image was never populated by CI.
+#   * ``publish-release``: create the GitHub Release for the tag and
+#     attach the media ``build-media`` produces -- the netboot-pc
+#     live-env bake + the usbboot-pc bootable image + checksums. These
+#     used to live only as ephemeral, login-gated workflow artifacts.
 #
 # First publish (claiming the pixie-lab name) requires a one-time
 # PyPI setup: register a Pending Publisher on https://pypi.org
@@ -602,3 +610,127 @@ jobs:
           # later job is retrying), succeed silently rather than
           # fail.
           skip-existing: true
+
+  publish-container:
+    name: Publish container image to ghcr
+    # Build + push the appliance image to ghcr.io/<owner>/pixie on a
+    # v* tag: ``:<version>`` (the tag minus its leading ``v``) plus
+    # ``:latest``. This is the image the compose deploy runs; before
+    # this the ``build-container`` job only smoke-built it and threw it
+    # away, so ghcr.io/safl/pixie was never populated by CI. Gated on
+    # the whole pipeline like publish-pypi -- never ship an image the
+    # boot-chain tests did not pass against.
+    needs:
+      - test
+      - integration
+      - build-container
+      - build-wheel
+      - build-media
+      - test-pxe
+      - test-pxe-nbdboot
+      - test-pxe-inventory
+      - test-pxe-tui
+      - test-pxe-flash
+      - test-pxe-flash-always
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v6
+      - name: Log in to ghcr.io
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" \
+            | podman login ghcr.io -u "${{ github.actor }}" --password-stdin
+      - name: Build + push image
+        run: |
+          set -eu
+          IMAGE=ghcr.io/${{ github.repository_owner }}/pixie
+          VERSION=${GITHUB_REF_NAME#v}
+          podman build \
+            --build-arg PIXIE_VERSION="$VERSION" \
+            -t "$IMAGE:$VERSION" \
+            -t "$IMAGE:latest" \
+            -f Containerfile .
+          podman push "$IMAGE:$VERSION"
+          podman push "$IMAGE:latest"
+
+  publish-release:
+    name: Publish GitHub Release (media assets)
+    # Attach the media CI already builds -- the netboot-pc live-env
+    # bake (vmlinuz + initrd + squashfs) and the usbboot-pc bootable
+    # image -- to a GitHub Release on v* tags. Before this they only
+    # existed as ephemeral, login-gated workflow artifacts, so an
+    # operator had no released live-env or USB image to download. Same
+    # full-pipeline gate as the other publish jobs.
+    needs:
+      - test
+      - integration
+      - build-container
+      - build-wheel
+      - build-media
+      - test-pxe
+      - test-pxe-nbdboot
+      - test-pxe-inventory
+      - test-pxe-tui
+      - test-pxe-flash
+      - test-pxe-flash-always
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+      - name: Download media artifacts
+        uses: actions/download-artifact@v8
+        with:
+          # Both build-media matrix legs upload under this shape:
+          # ``pixie-netboot-pc-x86_64`` + ``pixie-usbboot-pc-x86_64``.
+          # merge-multiple flattens both into one directory.
+          pattern: pixie-*-pc-x86_64
+          merge-multiple: true
+          path: release-assets
+      - name: List release assets
+        run: ls -la release-assets
+      - name: Assemble live-env fetch tarball
+        # pixie's in-app "Fetch live-env" pulls ONE tarball that unpacks
+        # to vmlinuz + initrd + live.squashfs (see catalog/_live_env.py).
+        # Repackage the netboot-pc trio under those stable names so
+        # ``.../releases/latest/download/pixie-live-env-x86_64.tar.gz``
+        # -- the default PIXIE_LIVE_ENV_SRC -- resolves.
+        run: |
+          set -eu
+          d=$(mktemp -d)
+          cp release-assets/pixie-netboot-pc-x86_64-*.vmlinuz  "$d/vmlinuz"
+          cp release-assets/pixie-netboot-pc-x86_64-*.initrd   "$d/initrd"
+          cp release-assets/pixie-netboot-pc-x86_64-*.squashfs "$d/live.squashfs"
+          tar -C "$d" -czf release-assets/pixie-live-env-x86_64.tar.gz \
+            vmlinuz initrd live.squashfs
+          ( cd release-assets && sha256sum pixie-live-env-x86_64.tar.gz \
+              > pixie-live-env-x86_64.tar.gz.sha256 )
+          # Ship the curated catalog (the netboot-capable nosi subset)
+          # so DEFAULT_CATALOG_URL (.../releases/latest/download/catalog.toml)
+          # resolves and pixie's Import + live-env TUI default work.
+          cp src/pixie/catalog/catalog.toml release-assets/catalog.toml
+          ls -la release-assets
+      - name: Create / update GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: release-assets/*
+          # Fail loud if the glob matched nothing -- a release with no
+          # media is the bug this job exists to prevent.
+          fail_on_unmatched_files: true
+          generate_release_notes: true
+          body: |
+            ## Distribution
+
+            - Container image: `ghcr.io/${{ github.repository_owner }}/pixie:latest` and the matching version tag (this tag without the leading `v`).
+            - Python CLI: `uv tool install pixie-lab` (or `pip install pixie-lab`).
+
+            ## Media assets attached here
+
+            - `pixie-live-env-x86_64.tar.gz` — the live-env trio packaged for pixie's in-app **Fetch live-env** action (the default `PIXIE_LIVE_ENV_SRC`). Click Fetch live-env on the dashboard and pixie pulls + stages this itself; no manual bake or copy.
+            - `pixie-netboot-pc-x86_64-*.{vmlinuz,initrd,squashfs}` — the same live-env (netboot-pc bake) as loose files, if you prefer to stage `PIXIE_LIVE_ENV_DIR` (`data/live-env/`) by hand. Used by the `pixie-flash-once` / `pixie-flash-always` / `pixie-inventory` / `pixie-tui` boot modes.
+            - `pixie-usbboot-pc-x86_64-*.iso.gz` — bootable USB / ISO media for the pixie live env.
+            - `*.sha256` — checksums for the above.

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/CHANGELOG.md

@@ -7,15 +7,140 @@ perceives, defaults that survived a `pip install -U`, and gates that
 landed in CI.
 
 Per-release commit history lives in `git log`; this file is the
-operator-facing summary. Nothing between the 0.1.0 skeleton and the
-next real release has been tagged: the intermediate work all lands
-under `[Unreleased]` until an operator can drive the full flow end-
-to-end on real hardware.
+operator-facing summary.
 
 ## [Unreleased]
 
+## [0.3.0] - 2026-07-23
+
+Operator-facing surface + release plumbing. A fleet overlay-management
+page, a tightened inventory view, and a dedicated live-env pane; the
+release now ships the container image + boot media + a curated catalog,
+and pixie can fetch its own live-env.
+
+### Added
+
+**Fleet-wide overlay management page.** A new **Overlays** page
+(`/ui/overlays`) lists every persistent nbdboot qcow2 across the fleet
+with disk-used, last-modified, serving port, and a state
+(active / idle / orphaned / missing); per-row Reset and a Prune that
+reclaims only the orphaned + missing ones. The machine-detail Inventory
+view was tightened from stacked stat-cards into one dense summary.
+
+**Pixie ships a curated catalog and defaults to it, not nosi's.** A
+fresh (empty) catalog is seeded on first start from a `catalog.toml`
+bundled in the package: a strict subset of the upstream nosi catalog
+restricted to the four netboot-capable images pixie's nbdboot + live-env
+chains actually test and support (debian-13-headless, ubuntu-2404 /
+2604-headless, fedora-44-headless, each with its netboot bundle). The
+desktop / proxmox / rpios / freebsd variants that have no netboot bundle
+are omitted. The "Import catalog" field and the live-env TUI now default
+to the pixie release copy of this curated catalog rather than the full
+nosi catalog; both remain overridable by URL. Seeding is one-shot,
+never clobbers an operator-populated catalog, and is disabled with
+`PIXIE_SEED_CATALOG=0`.
+
+**Pixie can fetch its own live-env, from a dedicated Live-env pane.** A
+new **Live env** page (`/ui/live-env`) is the one place the live env is
+managed: staged-media readiness, a **Fetch live env** action, the fetch
+source, and the extra-kernel-cmdline override. Fetch downloads the
+netboot-pc bake as a single tarball (`PIXIE_LIVE_ENV_SRC`, defaulting to
+the latest GitHub release's `pixie-live-env-x86_64.tar.gz`) and stages
+`vmlinuz` + `initrd` + `live.squashfs` under `PIXIE_LIVE_ENV_DIR`,
+reusing the catalog fetch's curl transport. This replaces the only
+artifact an operator previously had to bake locally
+(`make build VARIANT=netboot-pc`) or hand-copy. The source is overridable
+per deploy (point at a mirror for air-gapped installs). The dashboard
+Live-env card is now status-only and links to the pane; the live-env
+knobs moved off the Settings page. The `publish-release` job assembles
+that tarball so the default source resolves.
+
+**Releases now ship the container image and the boot media, not just
+the PyPI package.** Tagging `v*` publishes the appliance image to
+`ghcr.io/safl/pixie` (`:<version>` + `:latest`) and creates a GitHub
+Release with the boot media attached: the netboot-pc live-env bake
+(`vmlinuz` + `initrd` + `squashfs`) that the `pixie-flash-once` /
+`pixie-flash-always` / `pixie-inventory` / `pixie-tui` modes chain
+into, and the `usbboot-pc` bootable `.iso.gz`, each with a `.sha256`.
+Previously a tag published only to PyPI, so the ghcr image that
+`pixie-lab deploy` pins and the live-env media both existed only as
+ephemeral CI artifacts or a local `make build` -- a fresh
+`pixie-lab deploy` pulled an image that was never pushed.
+
+## [0.2.0] - 2026-07-22
+
+First real release after the 0.1.0 skeleton: nbdboot (ephemeral +
+persistent per-machine overlays) validated end-to-end on real
+hardware, plus a bty-lab-shaped deploy CLI and an operator-managed
+event log.
+
 ### Added
 
+**Event-log Acknowledge + Clear actions.** The `/ui/events` page grows
+two bulk actions: **Acknowledge** advances the ack cursor so the
+dashboard's unacknowledged-error count zeros without touching the log,
+and **Clear** wipes the whole log (behind a confirm) and drops one
+`events.cleared` marker so the reset itself stays on the record.
+Mirrors the ack/clear affordances bty's event log carried.
+
+**`pixie-lab purge` reworked to match `bty-lab`.** `purge` now prints
+a plan and gates the destructive parts behind a `y/N` confirmation
+(`-y`/`--yes` to skip; a non-TTY refuses without `--yes`). Flags:
+`--data` deletes the on-disk state (previously `--wipe-data`, which
+never actually removed the bind-mounted `data/`), `--images` removes
+the container image, and `--all` also removes the deploy directory
+(implies `--data`).
+
+**Per-machine persistent qcow2 overlays under `nbdboot`.** A new
+`overlay_profile` field on the machine binding flips one target from
+the default ephemeral-tmpfs behaviour to a per-machine writable
+overlay without changing anything else about the bind. A non-blank
+profile maps to a `data/overlays/<mac>/<image_sha>/<profile>.qcow2`
+file with the image's base blob as `backing_file`, served over NBD by
+`qemu-nbd`; the target mounts it read-write and system changes (apt
+installs, kernel modules, hardware-specific config) survive reboots.
+Overlays are keyed by `(mac, image_sha, profile)`, so different
+machines have fully independent files under the same profile name and
+rebinding to a different image leaves the old image's overlays on
+disk for a later resume. A Reset button on the machine detail page
+tears down `qemu-nbd`, unlinks the qcow2, and lets the next boot
+lazy-create a fresh overlay from the base. New `overlays` table on
+state.db (idempotent additive migration), new `overlay.created`,
+`overlay.reset`, `overlay.booted` events. Concurrency is by
+construction (a MAC boots one target at a time), so there is no
+holder tracking or force-reclaim.
+
+**Slick Inventory card viz.** The machine detail Inventory card was
+rewritten to consume a normalised view of the stored `lshw -json`
+blob. CPU renders one stat-block per socket with the model as
+headline, an architecture badge, and Bootstrap `display-6` big-type
+numbers for cores over threads plus max clock. Memory shows a total
+headline plus a per-DIMM slot-fill row (filled blocks for populated
+SMBIOS type-17 bank records, outlined blocks for empty slots,
+hover-title tooltip per slot showing size / speed / type), with a
+total-only fallback for firmwares that skimp on bank records. The
+extractor lives in `pixie.web._inventory.normalise_inventory` and
+runs at render time, so a wire-format change or a new lshw quirk
+touches one function. Two new Jinja filters: `humanize_bytes` and
+`humanize_hz`.
+
+### Fixed
+
+**Truncated `.img.gz` fetches now fail at the download stage.**
+Operators saw "decompress img.gz failed: Compressed file ended before
+the end-of-stream marker was reached" on the catalog page when a
+ghcr download was interrupted mid-transfer. Root cause: the fetch
+pipeline's byte-copy loop treated `resp.read()` returning an empty
+chunk as "done" but urllib does not raise when a peer closes the
+connection early, so a short body was accepted and the gzip trailer
+check surfaced the truncation several minutes downstream of the
+actual failure. `_stream_to_tmpfile` now cross-checks bytes-written
+against `Content-Length` and raises a clear `download truncated for
+<url>: got X of Y bytes` `FetchError` at the point of cause. A
+related cleanup leak (the `finally` block only unlinked `.inflight`
+files for `tar.gz`, so failing `img.gz` fetches left multi-GB
+orphans in `data/tmp/` forever) is fixed as part of the same change.
+
 **Settings pane with per-operator display picks.** New top-nav
 pill `/ui/settings` with two knobs: display timezone (IANA zone
 name) and datetime format (strftime pattern). Both resolve override

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/Containerfile

@@ -44,6 +44,9 @@ LABEL org.opencontainers.image.title="pixie" \
 #                    /usr/lib/ipxe/, which we copy into pixie's
 #                    TFTP root so the daemon can serve them.
 #   qemu-utils:      qemu-img info + qemu-nbd for qcow2 handling.
+#   fdisk:           sfdisk for pixie._partition partition-offset lookup
+#                    on persistent-overlay renders. On Ubuntu 26.04
+#                    sfdisk moved out of util-linux into this package.
 #   ca-certificates: HTTPS fetch of ORAS + release assets.
 #
 # Verified against ubuntu:26.04's package list in CI on every build;
@@ -64,6 +67,7 @@ RUN apt-get update \
         tftpd-hpa \
         ipxe \
         qemu-utils \
+        fdisk \
  && rm -rf /var/lib/apt/lists/*
 
 # App under /app with editable-style install from the source we copy in.

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pixie-lab
-Version: 0.1.0
+Version: 0.3.0
 Summary: Bare-metal netboot appliance: catalog + fetch + NBD + PXE + TFTP + operator TUI in one container
 Project-URL: Homepage, https://github.com/safl/pixie
 Project-URL: Repository, https://github.com/safl/pixie
@@ -81,11 +81,13 @@ top nav has Machines, Catalog, Events, and Settings.
 
 Everything writable lives under `PIXIE_DATA_DIR` (default
 `/var/lib/pixie`). `state.db` holds catalog rows, machine rows,
-event log, settings, and NBD-export records. `blobs/<sha>/blob`
-holds fetched disk images. `artifacts/<sha>/{vmlinuz,initrd,
-manifest.json}` holds unpacked netboot bundles. `live-env/` holds
-the pixie live env kernel + initrd + squashfs the netboot-pc bake
-produced.
+event log, settings, NBD-export records, and persistent-overlay
+records. `blobs/<sha>/blob` holds fetched disk images.
+`artifacts/<sha>/{vmlinuz,initrd,manifest.json}` holds unpacked
+netboot bundles. `overlays/<mac>/<image_sha>/<profile>.qcow2` holds
+per-machine writable overlays for the `nbdboot` boot mode.
+`live-env/` holds the pixie live env kernel + initrd + squashfs the
+netboot-pc bake produced.
 
 Both admin password and display timezone / strftime pattern have
 env-var overrides plus DB overrides via `/ui/settings`; env wins

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/README.md

@@ -53,11 +53,13 @@ top nav has Machines, Catalog, Events, and Settings.
 
 Everything writable lives under `PIXIE_DATA_DIR` (default
 `/var/lib/pixie`). `state.db` holds catalog rows, machine rows,
-event log, settings, and NBD-export records. `blobs/<sha>/blob`
-holds fetched disk images. `artifacts/<sha>/{vmlinuz,initrd,
-manifest.json}` holds unpacked netboot bundles. `live-env/` holds
-the pixie live env kernel + initrd + squashfs the netboot-pc bake
-produced.
+event log, settings, NBD-export records, and persistent-overlay
+records. `blobs/<sha>/blob` holds fetched disk images.
+`artifacts/<sha>/{vmlinuz,initrd,manifest.json}` holds unpacked
+netboot bundles. `overlays/<mac>/<image_sha>/<profile>.qcow2` holds
+per-machine writable overlays for the `nbdboot` boot mode.
+`live-env/` holds the pixie live env kernel + initrd + squashfs the
+netboot-pc bake produced.
 
 Both admin password and display timezone / strftime pattern have
 env-var overrides plus DB overrides via `/ui/settings`; env wins

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/cijoe/configs/test-pxe-nbdboot.toml

@@ -88,3 +88,62 @@ needle = "systemd["
 [[test.pxe.chain_markers]]
 key = "debian-userspace"
 needle = "Debian"
+
+# ---- Positive userland proof ---------------------------------------
+#
+# Getty printed the login prompt on the serial console: this is what
+# "operator-usable userland" actually looks like from outside the VM.
+# Getty runs as part of getty.target, which is pulled in by
+# multi-user.target, so a login prompt on serial is unambiguous
+# proof that multi-user.target was reached. Chosen over the
+# "Reached target multi-user.target" systemd banner because Debian
+# 13 boots in ``quiet`` mode by default and does not print that
+# banner on the console -- getty's login prompt is always printed.
+# Catches the "boots to systemd but a hanging unit blocks userspace"
+# shape where the earlier markers (systemd-started, debian-userspace)
+# fire but the target never reaches operator-usable userland.
+[[test.pxe.chain_markers]]
+key = "login-prompt"
+needle = "login:"
+
+# ---- Forbidden markers ---------------------------------------------
+#
+# Anything below here MUST NOT appear on the client serial log or
+# in pixie's container logs. The check runs alongside the positive-
+# marker poll and fails the whole test the moment any needle hits.
+# Kept scoped to failure signals that would leave a booted-looking
+# target with a broken userland (booted markers OK, but the console
+# spells out the bad news).
+
+# systemd-resolved / networkd starvation with no fallback resolv.conf.
+# The mount hook's mask of systemd-networkd + NetworkManager killed
+# both DHCP daemons on the pivoted rootfs; if the initrd's
+# /etc/resolv.conf write did not stick, apt / curl on the target log
+# this literal string on serial. Nosi 2026.W30's mount hook unlinks
+# the /etc/resolv.conf symlink before writing so glibc's resolver
+# reads the file directly -- if a regression puts the symlink back,
+# this marker fires.
+[[test.pxe.forbidden_markers]]
+key = "dns-broken"
+needle = "Temporary failure resolving"
+
+# sshd itself failed to start (its unit went into failed state during
+# multi-user.target activation). Distinct from "sshd-started not
+# seen" so the operator gets a clear "start failure" vs "start never
+# attempted" diagnosis in the CI log.
+[[test.pxe.forbidden_markers]]
+key = "sshd-failed"
+needle = "Failed to start ssh.service"
+
+# Dracut emergency shell: the mount hook died with _pixie_die and
+# systemd-emergency published the emergency-shell prompt. Every
+# persist-branch regression this session ended here.
+[[test.pxe.forbidden_markers]]
+key = "dracut-emergency"
+needle = "dracut-emergency"
+
+# Post-pivot emergency mode (something failed in userspace's own
+# init and systemd fell back to emergency.target).
+[[test.pxe.forbidden_markers]]
+key = "emergency-mode"
+needle = "Emergency mode"

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/cijoe/scripts/pxe_run_chain_test.py

@@ -243,6 +243,7 @@ def main(args, cijoe) -> int:
         client = _start_client_vm(workspace, cfg, client_log, firmware)
 
         markers = _build_markers(cfg)
+        forbidden = [(e["key"], e["needle"]) for e in cfg.get("forbidden_markers", [])]
         # ``mode=flash`` runs the full real-image pipeline (oras
         # pull + img.gz decompress + dd of ~10 GiB) inside the live
         # env; ``mode=nbdboot`` also pulls the real nosi image but
@@ -252,7 +253,20 @@ def main(args, cijoe) -> int:
         # blob store, similar order of magnitude. Everything else
         # finishes well within the 5 min short-mode budget.
         chain_deadline = CHAIN_TIMEOUT_FLASH if mode in ("flash", "nbdboot") else CHAIN_TIMEOUT
-        seen = _wait_for_chain_markers(client_log, markers, chain_deadline)
+        try:
+            seen = _wait_for_chain_markers(client_log, markers, chain_deadline, forbidden)
+        except ForbiddenMarkerSeen as exc:
+            log.error(
+                f"PXE chain hit forbidden marker {exc.key!r} ({exc.needle!r}); "
+                "a known-bad userland signal appeared on the client serial log "
+                "or in pixie's container logs. Dumping context."
+            )
+            qemu_log = client_log.with_suffix(".qemu.log")
+            _dump_tail(qemu_log, 60)
+            _dump_tail(workspace / "dnsmasq.log", 60)
+            _dump_tail(client_log, 200)
+            _dump_container_logs()
+            return errno.EPROTO
         missing = [k for k, ok in seen.items() if not ok]
         if missing:
             log.error(f"PXE chain incomplete; missing markers: {', '.join(missing)}")
@@ -620,30 +634,52 @@ def _build_markers(cfg):
     return out
 
 
-def _wait_for_chain_markers(log_path: Path, markers, timeout: int):
+def _wait_for_chain_markers(log_path: Path, markers, timeout: int, forbidden=None):
+    """Poll the client serial log + pixie's container logs for the
+    positive markers, checking the same buffers for any ``forbidden``
+    substrings each round. A forbidden hit shortcuts the wait with
+    ``ForbiddenMarkerSeen`` so the caller fails the test loud instead
+    of timing out looking for a positive marker that would never
+    arrive (the DNS / sshd / emergency-shell regressions come with
+    highly recognisable serial output and belong in this list rather
+    than in the reactive "check what's missing" branch)."""
     seen = {key: False for key, _ in markers}
+    forbidden = forbidden or []
     deadline = time.monotonic() + timeout
     while time.monotonic() < deadline and not all(seen.values()):
-        # Check the serial log for iPXE-side markers.
+        body = ""
         if log_path.exists():
             body = log_path.read_text(encoding="utf-8", errors="replace")
             for key, needle in markers:
                 if not seen[key] and needle in body:
                     log.info(f"  + {key}: matched {needle!r}")
                     seen[key] = True
-        # Also mirror-check container logs so a server-side hit counts
-        # even when the client's console doesn't spell the fetch out.
         cont = _container_log_snapshot()
         for key, needle in markers:
             if not seen[key] and needle in cont:
                 log.info(f"  + {key}: matched {needle!r} in container logs")
                 seen[key] = True
+        for key, needle in forbidden:
+            if needle in body or needle in cont:
+                raise ForbiddenMarkerSeen(key, needle)
         if all(seen.values()):
             break
         time.sleep(2)
     return seen
 
 
+class ForbiddenMarkerSeen(RuntimeError):
+    """A ``[[test.pxe.forbidden_markers]]`` entry was observed on the
+    client serial log or in pixie's container logs. Raised by the
+    marker-polling loop so the caller can log the exact key + needle
+    that fired and fail with a distinct error code."""
+
+    def __init__(self, key: str, needle: str) -> None:
+        super().__init__(f"forbidden marker {key!r} matched {needle!r}")
+        self.key = key
+        self.needle = needle
+
+
 def _container_log_snapshot() -> str:
     res = subprocess.run(
         ["podman", "logs", "--tail", "500", CONTAINER_NAME],

{pixie_lab-0.1.0 → pixie_lab-0.3.0}/docs/src/boot-modes.md

@@ -49,11 +49,50 @@ known-clean image each cycle.
 ### `nbdboot`
 
 The target boots the image's OWN kernel (extracted from the sibling
-netboot bundle) and mounts the image over NBD. Root is an
+netboot bundle) and mounts the image over NBD. By default, root is an
 overlay-on-tmpfs: writes go to RAM, nothing propagates back to the
 source blob. Multiple targets can nbdboot the same image
 simultaneously because they each get their own overlay.
 
+**Persistent overlays** flip a single target from ephemeral to dev
+mode without changing anything else about the bind. On the machine
+detail page, the `Overlay profile` field is blank by default
+(ephemeral, unchanged behaviour) or names a per-machine profile
+(e.g. `simon`, `karl`, `ci-with-nvme-tools`). A non-blank profile
+maps to a per-machine qcow2 file with the image's base blob as
+`backing_file`, served by `qemu-nbd` at a dedicated port. The target
+mounts the NBD device read-write; system-level changes (apt-installed
+packages, hardware-specific config, kernel modules) land on the qcow2
+and survive reboots.
+
+Overlays are keyed by `(mac, image_content_sha256, profile)`:
+different machines have fully independent files even under the same
+profile name, and rebinding a machine to a different image leaves the
+old image's overlays on disk (a rebind back resumes them). Storage is
+`data/overlays/<mac>/<image_sha>/<profile>.qcow2`. The **Reset**
+button on the machine detail page tears down `qemu-nbd`, unlinks the
+qcow2, and lets the next boot lazy-create a fresh overlay from the
+base.
+
+Concurrency is by construction: a MAC boots one target at a time, so
+two machines can never contend for the same qcow2. There is no
+holder-tracking or force-reclaim.
+
+**Kexec into a locally-installed kernel.** The netboot bundle owns
+the kernel and initrd pixie serves. Installing `linux-image-*` on the
+target's persistent overlay writes files to `/boot` but the next
+power-cycle refetches pixie's kernel and those files sit unused.
+`kexec` bridges that gap: the netboot kernel comes up, then the
+operator runs
+`kexec -l /boot/vmlinuz-<v> --initrd=/boot/initrd.img-<v>
+--reuse-cmdline && systemctl kexec` to switch to the local kernel
+without going through firmware. Every netboot-shipping nosi variant
+now bakes `kexec-tools` in; see nosi's [Custom kernel under netboot
+(kexec)](https://safl.github.io/nosi/kexec.html) for the full
+workflow. Recovery from a bad kernel is a power-cycle back to
+pixie's kernel; nothing kexec's automatically, so a broken install
+never becomes a boot loop.
+
 ## Prerequisites at a glance
 
 | Mode | Needs image | Needs inventory | Needs target disk |