pixie-lab 0.1.0__py3-none-any.whl

pixie/events/_routes.py

@@ -0,0 +1,42 @@
+"""HTTP routes for the events log.
+
+Read routes are OPEN because the on-call operator glances at
+``GET /events`` from a workstation curl; the events themselves are
+just names + timestamps + already-visible fields, no secrets.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query, Request
+
+from pixie.events._log import EventsLog
+
+router = APIRouter()
+
+
+def _get_log(request: Request) -> EventsLog:
+    log: EventsLog | None = getattr(request.app.state, "events_log", None)
+    if log is None:
+        raise HTTPException(status_code=503, detail="events log not initialised")
+    return log
+
+
+@router.get("/events")
+def list_events(
+    request: Request,
+    kind: str = Query("", description="filter by exact kind (empty = all kinds)"),
+    subject_kind: str = Query("", description="filter by subject_kind"),
+    subject_id: str = Query("", description="filter by subject_id"),
+    since_id: int = Query(0, ge=0, description="return only events with id > since_id"),
+    limit: int = Query(100, ge=1, le=1000, description="max rows"),
+) -> dict[str, list[dict[str, Any]]]:
+    rows = _get_log(request).list(
+        kind=kind,
+        subject_kind=subject_kind,
+        subject_id=subject_id,
+        since_id=since_id,
+        limit=limit,
+    )
+    return {"events": [r.to_dict() for r in rows]}

pixie/exports/__init__.py

@@ -0,0 +1,17 @@
+"""NBD-export supervisor + persistence.
+
+An **export** is a name-plus-content_sha256 pair that pixie serves over
+NBD. The bytes come from the catalog blob at
+``<state_dir>/blobs/<content_sha256>/blob``; the name is a short
+identifier the operator can type + iPXE can reference.
+
+One nbdkit subprocess per export, one TCP port each, one filter chain
+(``--filter=cow``, plus ``--filter=partition`` when the blob has an
+MBR/GPT). Ports are allocated from a base + scan; the assigned port
+lands on the export row so ``GET /exports`` surfaces it.
+
+Ported from nbdmux 0.9.2's ``server.py`` NbdServer + friends on
+2026-07-13. Simplified for pixie's content-addressed model: no more
+``images_dir`` step (the catalog store owns the blob path); exports
+reference catalog entries by content_sha256 instead of by src_url.
+"""

pixie/exports/_routes.py

@@ -0,0 +1,211 @@
+"""HTTP routes for the exports surface.
+
+Read routes (``GET /exports``, ``GET /export/{name}``) are OPEN so
+in-network monitoring can poll without a session. Write routes
+(``POST /exports``, ``DELETE /exports/{name}``) require the pixie
+session cookie.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.responses import Response
+from pydantic import BaseModel, Field
+
+from pixie.catalog._store import CatalogStore
+from pixie.events._kinds import (
+    EXPORT_DELETED,
+    EXPORT_NBDKIT_EXITED,
+    EXPORT_NBDKIT_SPAWNED,
+    EXPORT_REGISTERED,
+)
+from pixie.exports._store import Export, ExportsStore
+from pixie.exports._supervisor import NbdServer
+from pixie.web._auth import require_auth
+
+_log = logging.getLogger(__name__)
+
+# Same allowlist as nbdmux 0.9.2; the export name lands in nbdkit's
+# ``-e <name>`` argv and (indirectly) in operator-facing URLs.
+_EXPORT_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
+
+
+class RegisterBody(BaseModel):
+    """Register an export against an already-fetched catalog entry.
+
+    The ``content_sha256`` refers to the catalog entry's fetched
+    content sha (which the catalog list surfaces once the entry has
+    been Fetched). The name is a short operator-typable identifier
+    used both as the nbdkit ``-e`` param and as the URL segment.
+    """
+
+    name: str = Field(..., min_length=1, max_length=64)
+    content_sha256: str = Field(..., min_length=64, max_length=64, pattern="^[0-9a-f]{64}$")
+
+
+router = APIRouter()
+
+
+def _get_exports(request: Request) -> ExportsStore:
+    store: ExportsStore | None = getattr(request.app.state, "exports_store", None)
+    if store is None:
+        raise HTTPException(status_code=503, detail="exports store not initialised")
+    return store
+
+
+def _get_catalog(request: Request) -> CatalogStore:
+    store: CatalogStore | None = getattr(request.app.state, "catalog_store", None)
+    if store is None:
+        raise HTTPException(status_code=503, detail="catalog store not initialised")
+    return store
+
+
+def _get_nbd(request: Request) -> NbdServer:
+    nbd: NbdServer | None = getattr(request.app.state, "nbd_server", None)
+    if nbd is None:
+        raise HTTPException(status_code=503, detail="nbd supervisor not initialised")
+    return nbd
+
+
+def _refresh_row(
+    export: Export,
+    nbd: NbdServer,
+    exports: ExportsStore,
+    events_log: Any = None,
+) -> Export:
+    """Merge live supervisor state into a stored row before returning
+    it to the operator. Doesn't persist the port -- we do that from
+    the spawn path -- but keeps ``GET /exports`` honest when nbdkit
+    has died out of band."""
+    port = nbd.port_for(export.name)
+    if port is None and export.status == "running":
+        # Supervisor lost track: row still says running but no proc.
+        # Log the transition as a distinct event so an operator can
+        # grep for spontaneous nbdkit deaths without having to diff
+        # /exports over time. The row's own status still moves to
+        # ``error`` so the /ui/catalog table renders it.
+        previous_port = export.nbd_port
+        exports.update_runtime(export.name, nbd_port=0, status="error", error="nbdkit exited")
+        export.nbd_port = 0
+        export.status = "error"
+        export.error = "nbdkit exited"
+        if events_log is not None:
+            events_log.emit(
+                EXPORT_NBDKIT_EXITED,
+                subject_kind="export",
+                subject_id=export.name,
+                summary=f"nbdkit for {export.name} exited",
+                details={"previous_port": previous_port},
+            )
+    elif port is not None and export.nbd_port != port:
+        exports.update_runtime(export.name, nbd_port=port, status="running", error="")
+        export.nbd_port = port
+        export.status = "running"
+        export.error = ""
+    return export
+
+
+@router.get("/exports")
+def list_exports(request: Request) -> dict[str, list[dict[str, Any]]]:
+    exports = _get_exports(request)
+    nbd = _get_nbd(request)
+    events_log = getattr(request.app.state, "events_log", None)
+    rows = [_refresh_row(e, nbd, exports, events_log).to_dict() for e in exports.list()]
+    return {"exports": rows}
+
+
+@router.get("/exports/{name}")
+def get_export(request: Request, name: str) -> dict[str, Any]:
+    exports = _get_exports(request)
+    row = exports.get(name)
+    if row is None:
+        raise HTTPException(status_code=404, detail=f"no export named {name!r}")
+    nbd = _get_nbd(request)
+    return _refresh_row(row, nbd, exports, getattr(request.app.state, "events_log", None)).to_dict()
+
+
+@router.post("/exports", status_code=201)
+def register_export(
+    request: Request,
+    body: RegisterBody,
+    _auth: None = Depends(require_auth),
+) -> dict[str, Any]:
+    """Register + spawn nbdkit for the given ``content_sha256``.
+
+    Fails with 400 if the catalog blob is missing (operator hit Register
+    before Fetch), 409 if the name is already taken, 500 with
+    ``last_error`` set on the row if nbdkit refused to start.
+    """
+    if not _EXPORT_NAME_RE.match(body.name):
+        raise HTTPException(
+            status_code=400,
+            detail="name must match [A-Za-z0-9][A-Za-z0-9._-]{0,63}",
+        )
+    exports = _get_exports(request)
+    catalog = _get_catalog(request)
+    nbd = _get_nbd(request)
+
+    if exports.get(body.name) is not None:
+        raise HTTPException(status_code=409, detail=f"export {body.name!r} already exists")
+
+    blob_path = catalog.blob_path(body.content_sha256)
+    if not blob_path.is_file():
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                f"no blob on disk for content_sha256={body.content_sha256[:12]!r}; "
+                "Fetch the catalog entry first"
+            ),
+        )
+
+    export = Export(name=body.name, content_sha256=body.content_sha256)
+    exports.upsert(export)
+
+    try:
+        port = nbd.spawn(body.name, blob_path)
+    except RuntimeError as exc:
+        exports.update_runtime(body.name, nbd_port=0, status="error", error=str(exc))
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+    exports.update_runtime(body.name, nbd_port=port, status="running", error="")
+    log = getattr(request.app.state, "events_log", None)
+    if log is not None:
+        log.emit(
+            EXPORT_REGISTERED,
+            subject_kind="export",
+            subject_id=body.name,
+            summary=f"{body.name} on port {port}",
+            details={"content_sha256": body.content_sha256, "nbd_port": port},
+        )
+        log.emit(
+            EXPORT_NBDKIT_SPAWNED,
+            subject_kind="export",
+            subject_id=body.name,
+            summary=f"nbdkit spawned on port {port}",
+            details={"nbd_port": port},
+        )
+    row = exports.get(body.name)
+    assert row is not None
+    return row.to_dict()
+
+
+@router.delete("/exports/{name}", status_code=204)
+def delete_export(
+    request: Request,
+    name: str,
+    _auth: None = Depends(require_auth),
+) -> Response:
+    exports = _get_exports(request)
+    nbd = _get_nbd(request)
+    if exports.get(name) is None:
+        raise HTTPException(status_code=404, detail=f"no export named {name!r}")
+    nbd.terminate(name)
+    exports.delete(name)
+    log = getattr(request.app.state, "events_log", None)
+    if log is not None:
+        log.emit(EXPORT_DELETED, subject_kind="export", subject_id=name, summary=name)
+    return Response(status_code=204)

pixie/exports/_store.py

@@ -0,0 +1,156 @@
+"""Exports table + typed row.
+
+An export row is name + content_sha256 (points at the catalog blob) +
+transient runtime fields (nbd_port, status). The row is persistent so
+a restart can respawn the NBD supervisor without operator help; the
+port + status are refreshed at spawn time.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import sqlite3
+import threading
+from collections.abc import Generator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from pixie._util import now_iso
+
+_DB_WRITE_LOCK = threading.Lock()
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS exports (
+    name           TEXT PRIMARY KEY,
+    content_sha256 TEXT NOT NULL,
+    nbd_port       INTEGER NOT NULL DEFAULT 0,
+    status         TEXT NOT NULL DEFAULT 'idle',
+    error          TEXT NOT NULL DEFAULT '',
+    created_at     TEXT NOT NULL,
+    updated_at     TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_exports_content_sha
+    ON exports(content_sha256);
+"""
+
+
+@dataclass
+class Export:
+    """One registered NBD export."""
+
+    name: str
+    content_sha256: str
+    nbd_port: int = 0
+    status: str = "idle"  # "idle" | "running" | "error"
+    error: str = ""
+    created_at: str = field(default_factory=now_iso)
+    updated_at: str = field(default_factory=now_iso)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "content_sha256": self.content_sha256,
+            "nbd_port": self.nbd_port or None,
+            "status": self.status,
+            "error": self.error or None,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+        }
+
+
+class ExportsStore:
+    """Repository over the ``exports`` table on the shared state.db.
+
+    Shares the state.db with :class:`pixie.catalog.CatalogStore`;
+    pixie's design is one state.db per deploy. Callers pass the DB
+    path in explicitly so tests can point at a fresh tmpdir.
+    """
+
+    def __init__(self, db_path: Path) -> None:
+        self.db_path = Path(db_path)
+        self._ensure_schema()
+
+    def _ensure_schema(self) -> None:
+        with self._conn() as conn:
+            conn.executescript(_SCHEMA)
+
+    @contextlib.contextmanager
+    def _conn(self) -> Generator[sqlite3.Connection]:
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        try:
+            yield conn
+            conn.commit()
+        finally:
+            conn.close()
+
+    # ---------- CRUD ------------------------------------------------
+
+    def list(self) -> list[Export]:
+        with self._conn() as conn:
+            rows = conn.execute("SELECT * FROM exports ORDER BY name").fetchall()
+        return [_row_to_export(r) for r in rows]
+
+    def get(self, name: str) -> Export | None:
+        with self._conn() as conn:
+            row = conn.execute("SELECT * FROM exports WHERE name = ?", (name,)).fetchone()
+        return _row_to_export(row) if row else None
+
+    def upsert(self, export: Export) -> None:
+        with _DB_WRITE_LOCK, self._conn() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO exports (
+                    name, content_sha256, nbd_port, status, error,
+                    created_at, updated_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    export.name,
+                    export.content_sha256,
+                    export.nbd_port,
+                    export.status,
+                    export.error,
+                    export.created_at,
+                    now_iso(),
+                ),
+            )
+
+    def delete(self, name: str) -> bool:
+        with _DB_WRITE_LOCK, self._conn() as conn:
+            cur = conn.execute("DELETE FROM exports WHERE name = ?", (name,))
+            return cur.rowcount > 0
+
+    def update_runtime(
+        self,
+        name: str,
+        *,
+        nbd_port: int,
+        status: str,
+        error: str = "",
+    ) -> None:
+        """Refresh the transient fields (port + status + error) after
+        a spawn attempt. Silent on missing row (raced with a delete)."""
+        with _DB_WRITE_LOCK, self._conn() as conn:
+            conn.execute(
+                """
+                UPDATE exports
+                SET nbd_port = ?, status = ?, error = ?, updated_at = ?
+                WHERE name = ?
+                """,
+                (nbd_port, status, error, now_iso(), name),
+            )
+
+
+def _row_to_export(row: sqlite3.Row) -> Export:
+    return Export(
+        name=row["name"],
+        content_sha256=row["content_sha256"],
+        nbd_port=row["nbd_port"] or 0,
+        status=row["status"],
+        error=row["error"],
+        created_at=row["created_at"],
+        updated_at=row["updated_at"],
+    )

pixie/exports/_supervisor.py

@@ -0,0 +1,240 @@
+"""NbdServer: one nbdkit subprocess per registered export.
+
+Port allocation: :attr:`port_base` is scanned upward; a running nbdkit
+holds its port for the export's lifetime. :meth:`reload` diff-syncs the
+running processes against a desired export list (spawn missing, kill
+dropped). :meth:`stop` kills everything.
+
+Idempotent + thread-safe: :meth:`spawn` on an already-alive export is
+a no-op; :meth:`terminate` on an unknown export is a no-op.
+
+Filter chain:
+* ``--filter=cow`` always. Under nbdkit >= 1.44 this is safe with
+  per-connection named exports; earlier nbdkit silently corrupts
+  under this combination (the base container image pins ubuntu:26.04
+  which ships 1.46 for that reason).
+* ``--filter=partition`` when the blob's boot sector has an MBR/GPT
+  magic. Full-disk images (nosi's ``img.gz`` shape) need this so
+  ``nbd0p1`` shows up; raw filesystem images do not.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import socket
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+_log = logging.getLogger(__name__)
+
+DEFAULT_NBDKIT_BIN = "nbdkit"
+DEFAULT_PORT_BASE = 10809
+_PORT_SCAN_WIDTH = 256
+_SPAWN_STARTUP_GRACE = 0.2  # seconds -- give nbdkit a moment to bind + fail
+
+
+def file_looks_partitioned(path: Path) -> bool:
+    """True iff the file at ``path`` has an MBR/GPT partition table.
+
+    Checks the classic boot-sector magic ``0x55 0xAA`` at bytes
+    510-511. Covers MBR + protective-MBR-for-GPT + hybrid disks.
+    Files smaller than 512 bytes, unreadable, or read-erroring return
+    False (safer default: skip the partition filter -- if we're
+    wrong, the boot fails loudly on mount instead of silently
+    stripping to partition 1 of a non-partitioned image).
+    """
+    try:
+        with open(path, "rb") as f:
+            head = f.read(512)
+    except OSError:
+        return False
+    return len(head) == 512 and head[510:512] == b"\x55\xaa"
+
+
+def _port_available(bind: str, port: int) -> bool:
+    """True iff ``bind:port`` accepts a fresh bind. The ~1ms race
+    between this check and nbdkit's own bind is fine: nbdkit exits
+    loudly on bind failure and the caller reports the error."""
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.bind((bind, port))
+    except OSError:
+        return False
+    else:
+        return True
+    finally:
+        with contextlib.suppress(OSError):
+            s.close()
+
+
+class NbdServer:
+    """Manages nbdkit subprocesses for a collection of exports.
+
+    ``port_base`` is the first port scanned; each spawn takes the
+    next free port in a window of :data:`_PORT_SCAN_WIDTH`. The
+    server does NOT persist ports; callers who need to survive
+    process restart write ``NbdServer.port_for(name)`` to their
+    own store post-spawn.
+    """
+
+    def __init__(
+        self,
+        *,
+        port_base: int = DEFAULT_PORT_BASE,
+        bind: str = "0.0.0.0",
+        nbdkit_bin: str = DEFAULT_NBDKIT_BIN,
+    ) -> None:
+        self.port_base = port_base
+        self.bind = bind
+        self.bin = nbdkit_bin
+        self._procs: dict[str, subprocess.Popen[bytes]] = {}
+        self._ports: dict[str, int] = {}
+        self._paths: dict[str, Path] = {}
+        self._lock = threading.Lock()
+
+    # ---------- public API ------------------------------------------
+
+    def spawn(self, name: str, blob_path: Path) -> int:
+        """Spawn nbdkit for ``name`` serving ``blob_path``. Idempotent.
+
+        Returns the port the export listens on (allocated fresh or
+        reused if already running). Raises :class:`RuntimeError` if
+        nbdkit refuses to start (bad binary, port collision, missing
+        file).
+        """
+        with self._lock:
+            return self._spawn_locked(name, blob_path)
+
+    def terminate(self, name: str) -> bool:
+        """Kill the nbdkit for ``name``. Returns True iff a process
+        was actually killed. No-op if the export isn't running."""
+        with self._lock:
+            return self._terminate_locked(name)
+
+    def reload(self, desired: dict[str, Path]) -> None:
+        """Diff-sync against ``desired``: spawn every name we're not
+        already serving, kill every running export that dropped out of
+        the map. Order-independent."""
+        with self._lock:
+            # Kill dropped
+            for name in list(self._procs):
+                if name not in desired:
+                    self._terminate_locked(name)
+            # Spawn desired
+            for name, path in desired.items():
+                self._spawn_locked(name, path)
+
+    def stop(self) -> None:
+        """Kill every running nbdkit. Called on app shutdown."""
+        with self._lock:
+            for name in list(self._procs):
+                self._terminate_locked(name)
+
+    def is_running(self, name: str) -> bool:
+        with self._lock:
+            proc = self._procs.get(name)
+            return proc is not None and proc.poll() is None
+
+    def port_for(self, name: str) -> int | None:
+        with self._lock:
+            return self._ports.get(name)
+
+    def running_exports(self) -> dict[str, int]:
+        """Snapshot of currently-running exports and their ports."""
+        with self._lock:
+            return {n: p for n, p in self._ports.items() if self._procs[n].poll() is None}
+
+    # ---------- internals -------------------------------------------
+
+    def _spawn_locked(self, name: str, blob_path: Path) -> int:
+        """Requires ``self._lock``. Idempotent per name."""
+        existing = self._procs.get(name)
+        if existing is not None and existing.poll() is None:
+            return self._ports[name]
+
+        # Reap dead entry so its port slot is available for reuse.
+        if existing is not None:
+            self._procs.pop(name, None)
+            self._ports.pop(name, None)
+            self._paths.pop(name, None)
+
+        if not blob_path.is_file():
+            raise RuntimeError(f"blob {blob_path!s} does not exist")
+
+        port = self._allocate_port_locked()
+
+        argv: list[str] = [
+            self.bin,
+            "-p",
+            str(port),
+            "--ipaddr",
+            self.bind,
+            "-f",
+            "--newstyle",
+            "-e",
+            name,
+            "--filter=cow",
+        ]
+        # --filter=partition must sit BELOW --filter=cow (nearer the
+        # plugin) so client writes land in the cow overlay, not in a
+        # partition-filter-managed slice of the backing.
+        partitioned = file_looks_partitioned(blob_path)
+        if partitioned:
+            argv.append("--filter=partition")
+        # nbdkit's arg parser treats the first non-flag as the plugin
+        # name and everything after as KEY=VALUE plugin params.
+        # Filter params come after the plugin.
+        argv.extend(["file", f"file={blob_path!s}"])
+        if partitioned:
+            argv.append("partition=1")
+
+        _log.info("nbdkit spawn %r on port %d: %s", name, port, blob_path)
+        try:
+            proc = subprocess.Popen(argv, stdout=sys.stderr, stderr=sys.stderr)
+        except FileNotFoundError as exc:
+            raise RuntimeError(f"nbdkit binary not found: {self.bin!r}") from exc
+
+        # Give the child a moment to bind or fail loudly.
+        time.sleep(_SPAWN_STARTUP_GRACE)
+        if proc.poll() is not None:
+            rc = proc.returncode
+            raise RuntimeError(
+                f"nbdkit for export {name!r} exited immediately (rc={rc}, port={port}, "
+                f"file={blob_path!s}); check the binary + file exist and the port is free"
+            )
+
+        self._procs[name] = proc
+        self._ports[name] = port
+        self._paths[name] = blob_path
+        return port
+
+    def _terminate_locked(self, name: str) -> bool:
+        """Requires ``self._lock``. Returns True iff we killed a proc."""
+        proc = self._procs.pop(name, None)
+        self._ports.pop(name, None)
+        self._paths.pop(name, None)
+        if proc is None:
+            return False
+        if proc.poll() is None:
+            proc.terminate()
+            with contextlib.suppress(subprocess.TimeoutExpired):
+                proc.wait(timeout=3)
+            if proc.poll() is None:
+                proc.kill()
+        return True
+
+    def _allocate_port_locked(self) -> int:
+        """Return the next free port at or above ``self.port_base``."""
+        used = set(self._ports.values())
+        for p in range(self.port_base, self.port_base + _PORT_SCAN_WIDTH):
+            if p in used:
+                continue
+            if _port_available(self.bind, p):
+                return p
+        raise RuntimeError(
+            f"no free TCP port in range {self.port_base}..{self.port_base + _PORT_SCAN_WIDTH}"
+        )