pixie-lab 0.1.0__py3-none-any.whl

pixie/catalog/_store.py

@@ -0,0 +1,234 @@
+"""Catalog store: SQLite-backed metadata + content-addressed blobs
+and artifacts on the filesystem.
+
+One ``state.db`` under ``<state_dir>`` holds every catalog entry;
+blobs land at ``<state_dir>/blobs/<content_sha256>/blob`` and the
+tar.gz-unpacked netboot artifacts at
+``<state_dir>/artifacts/<content_sha256>/{vmlinuz,initrd,manifest.json}``.
+
+Both paths are content-addressed: the same bytes served under
+different catalog names share on-disk storage, and an entry's blob
+URL (``/b/<sha>/<name>``) never changes across renames as long as the
+content is stable. This is a deliberate departure from withcache's
+URL-addressed store; pixie is an operator-curated library, not a
+URL->bytes cache.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import sqlite3
+import threading
+from collections.abc import Generator
+from pathlib import Path
+
+from pixie._util import now_iso
+from pixie.catalog._schema import CatalogEntry
+
+_DB_WRITE_LOCK = threading.Lock()
+
+# SQLite schema. Migrations for pixie land as ``CREATE TABLE IF NOT
+# EXISTS`` at ``open_store`` time, per bty's pattern. Pre-1.0: if a
+# schema change is required and the migration would be complex, we
+# rotate ``state.db`` -> ``state.db.<oldver>.<ts>.bak`` and start
+# clean rather than write migration SQL. (Documented ergonomics
+# tradeoff -- Cf. bty v0.25.0.)
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS catalog_entries (
+    name           TEXT PRIMARY KEY,
+    src            TEXT NOT NULL,
+    format         TEXT NOT NULL,
+    arch           TEXT NOT NULL DEFAULT '',
+    description    TEXT NOT NULL DEFAULT '',
+    netboot_src    TEXT NOT NULL DEFAULT '',
+    content_sha256 TEXT NOT NULL DEFAULT '',
+    size_bytes     INTEGER NOT NULL DEFAULT 0,
+    fetched_at     TEXT NOT NULL DEFAULT '',
+    added_at       TEXT NOT NULL,
+    extra_json     TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_catalog_entries_src ON catalog_entries(src);
+CREATE INDEX IF NOT EXISTS idx_catalog_entries_content_sha
+    ON catalog_entries(content_sha256);
+"""
+
+
+class CatalogStore:
+    """Thin repository over ``state.db`` for catalog rows.
+
+    Blob + artifact storage on the filesystem is siblings of the DB
+    (``<state_dir>/{blobs,artifacts}/``); the store owns their
+    directory-shape but does NOT own the fetch pipeline that fills
+    them -- that lives in :mod:`pixie.catalog._fetcher`.
+    """
+
+    def __init__(self, state_dir: Path) -> None:
+        self.state_dir = Path(state_dir).resolve()
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        self.db_path = self.state_dir / "state.db"
+        self.blobs_dir = self.state_dir / "blobs"
+        self.artifacts_dir = self.state_dir / "artifacts"
+        self.blobs_dir.mkdir(exist_ok=True)
+        self.artifacts_dir.mkdir(exist_ok=True)
+        self._ensure_schema()
+
+    # ---------- schema init ---------------------------------------
+
+    def _ensure_schema(self) -> None:
+        with self._conn() as conn:
+            conn.executescript(_SCHEMA)
+
+    @contextlib.contextmanager
+    def _conn(self) -> Generator[sqlite3.Connection]:
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        try:
+            yield conn
+            conn.commit()
+        finally:
+            conn.close()
+
+    # ---------- entry CRUD ----------------------------------------
+
+    def list_entries(self) -> list[CatalogEntry]:
+        with self._conn() as conn:
+            rows = conn.execute("SELECT * FROM catalog_entries ORDER BY LOWER(name)").fetchall()
+        return [_row_to_entry(r) for r in rows]
+
+    def get_entry(self, name: str) -> CatalogEntry | None:
+        with self._conn() as conn:
+            row = conn.execute("SELECT * FROM catalog_entries WHERE name = ?", (name,)).fetchone()
+        return _row_to_entry(row) if row else None
+
+    def get_entry_by_src(self, src: str) -> CatalogEntry | None:
+        """Look up an entry by its ``src`` URL. This is the pairing key
+        for ``netboot_src`` cross-reference: pixie resolves a
+        disk-image entry's netboot bundle by matching src, never by
+        name."""
+        with self._conn() as conn:
+            row = conn.execute("SELECT * FROM catalog_entries WHERE src = ?", (src,)).fetchone()
+        return _row_to_entry(row) if row else None
+
+    def upsert(self, entry: CatalogEntry) -> None:
+        """Insert or update an entry by name. Uses INSERT OR REPLACE so
+        editing an entry doesn't leave orphan rows; a full row write
+        is cheap at catalog sizes."""
+        import json as _json
+
+        with _DB_WRITE_LOCK, self._conn() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO catalog_entries (
+                    name, src, format, arch, description, netboot_src,
+                    content_sha256, size_bytes, fetched_at, added_at,
+                    extra_json
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    entry.name,
+                    entry.src,
+                    entry.format,
+                    entry.arch,
+                    entry.description,
+                    entry.netboot_src,
+                    entry.content_sha256,
+                    entry.size_bytes,
+                    entry.fetched_at,
+                    entry.added_at,
+                    _json.dumps(entry.extra),
+                ),
+            )
+
+    def delete(self, name: str) -> bool:
+        """Remove an entry by name. Returns True iff a row was removed.
+        Blob + artifact bytes are NOT deleted here; they may be shared
+        by other entries with the same content_sha256. GC (refcount
+        walk) lives on a caller in the routes layer."""
+        with _DB_WRITE_LOCK, self._conn() as conn:
+            cur = conn.execute("DELETE FROM catalog_entries WHERE name = ?", (name,))
+            return cur.rowcount > 0
+
+    def mark_unfetched(self, name: str) -> None:
+        """Reverse of :meth:`mark_fetched`: clear the row's fetched
+        fields (content sha, size, timestamp) without deleting the
+        row itself. Used by the "delete blob but keep the entry"
+        path so a subsequent Fetch re-runs the pipeline instead of
+        needing the operator to re-Import the URL."""
+        with _DB_WRITE_LOCK, self._conn() as conn:
+            conn.execute(
+                """
+                UPDATE catalog_entries
+                SET content_sha256 = '', size_bytes = 0, fetched_at = ''
+                WHERE name = ?
+                """,
+                (name,),
+            )
+
+    def mark_fetched(
+        self,
+        name: str,
+        *,
+        content_sha256: str,
+        size_bytes: int,
+    ) -> None:
+        """Post-fetch update: record content sha + size + timestamp.
+        Failure (no matching row) is silent because the fetch pipeline
+        may race with a concurrent delete; the fetched blob is
+        content-addressed so it's not orphaned in a bad way."""
+        with _DB_WRITE_LOCK, self._conn() as conn:
+            conn.execute(
+                """
+                UPDATE catalog_entries
+                SET content_sha256 = ?, size_bytes = ?, fetched_at = ?
+                WHERE name = ?
+                """,
+                (content_sha256, size_bytes, now_iso(), name),
+            )
+
+    # ---------- content-addressed storage paths -------------------
+
+    def blob_path(self, content_sha256: str) -> Path:
+        """Where a fetched disk-image blob lives on disk."""
+        return self.blobs_dir / content_sha256 / "blob"
+
+    def artifact_dir(self, content_sha256: str) -> Path:
+        """Where an unpacked netboot bundle's files live on disk."""
+        return self.artifacts_dir / content_sha256
+
+    def artifact_path(self, content_sha256: str, filename: str) -> Path:
+        return self.artifact_dir(content_sha256) / filename
+
+    def content_shas_in_use(self) -> set[str]:
+        """The union of content_sha256 values referenced by any
+        catalog row. Used to compute GC candidates."""
+        with self._conn() as conn:
+            rows = conn.execute(
+                "SELECT DISTINCT content_sha256 FROM catalog_entries WHERE content_sha256 != ''"
+            ).fetchall()
+        return {str(r["content_sha256"]) for r in rows}
+
+
+def _row_to_entry(row: sqlite3.Row) -> CatalogEntry:
+    import json as _json
+
+    try:
+        extra = _json.loads(row["extra_json"] or "{}")
+        if not isinstance(extra, dict):
+            extra = {}
+    except _json.JSONDecodeError:
+        extra = {}
+
+    return CatalogEntry(
+        name=row["name"],
+        src=row["src"],
+        format=row["format"],
+        arch=row["arch"],
+        description=row["description"],
+        netboot_src=row["netboot_src"],
+        content_sha256=row["content_sha256"],
+        size_bytes=row["size_bytes"] or 0,
+        fetched_at=row["fetched_at"],
+        added_at=row["added_at"],
+        extra=extra,
+    )

pixie/deploy/__init__.py

@@ -0,0 +1,26 @@
+"""pixie-lab: deploy generator + operator convenience CLI.
+
+``pixie-lab init [dest]`` emits a ready-to-run compose deployment for
+pixie: a ``compose.yml`` with one service on ``--network=host``, an
+``envvars.example`` with the settings an operator MUST fill (admin
+password, host address for LAN visibility), a ``data/`` scaffold and
+a README pointing at ``PLAN.md``.
+
+``pixie-lab deploy [dest]`` builds on init: auto-fills envvars with a
+detected LAN IP + generated password, then runs ``podman compose
+up -d`` and waits for ``/healthz`` to answer 200.
+
+``pixie-lab purge [dest]`` tears the stack down (``podman compose
+down -v``) so a fresh ``deploy`` on the same dir starts clean.
+
+Kept deliberately shallower than bty-lab: pixie is one container, so
+the whole file is measured in hundreds of LOC, not thousands. The
+extra knobs (Quadlet emission, upgrade paths, backup / restore) land
+in a follow-up if operators ask.
+"""
+
+from __future__ import annotations
+
+from pixie.deploy._main import main
+
+__all__ = ["main"]

pixie/deploy/_main.py

@@ -0,0 +1,321 @@
+"""pixie-lab CLI entry.
+
+Subcommands mirror bty-lab's shape to keep operator muscle memory:
+
+    pixie-lab init [dest]     write compose.yml + envvars + README
+    pixie-lab deploy [dest]   init + auto-fill envvars + podman compose up + wait /healthz
+    pixie-lab purge [dest]    podman compose down -v; optionally --wipe-data
+
+Every subcommand takes an optional ``dest`` argument (defaults to the
+current directory). The ``deploy`` and ``purge`` subcommands shell out
+to ``podman compose`` (or ``docker compose``, or ``podman-compose``,
+whichever is on PATH) so pixie-lab does not carry its own container
+runtime.
+"""
+
+from __future__ import annotations
+
+import argparse
+import http.client
+import os
+import secrets
+import shutil
+import socket
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import pixie
+from pixie.deploy._templates import (
+    DEFAULT_ADMIN_PASSWORD,
+    DEFAULT_IMAGE_REPO,
+    compose_yaml,
+    envvars_example,
+    readme_md,
+)
+
+DEFAULT_DEST = Path.cwd()
+_HEALTHZ_TIMEOUT = 60.0
+
+
+# ---------- filesystem writes ------------------------------------------
+
+
+def _write(path: Path, body: str, *, force: bool) -> Path:
+    """Write ``body`` to ``path``, refusing to overwrite unless
+    ``force`` is set. Creates parent dirs on the way. Returns the
+    absolute path so the caller can print a friendly summary."""
+    if path.exists() and not force:
+        raise FileExistsError(f"{path} already exists; pass --force to overwrite")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(body, encoding="utf-8")
+    return path
+
+
+def _emit_files(dest: Path, *, image: str, admin_password: str, force: bool) -> list[Path]:
+    """Lay down the three canonical files + create the ``data/``
+    scaffold. Returns the list of written paths."""
+    compose = _write(
+        dest / "compose.yml", compose_yaml(image=image, admin_password=admin_password), force=force
+    )
+    envvars = _write(
+        dest / "envvars.example", envvars_example(admin_password=admin_password), force=force
+    )
+    readme = _write(dest / "README.md", readme_md(), force=force)
+    (dest / "data").mkdir(parents=True, exist_ok=True)
+    return [compose, envvars, readme]
+
+
+# ---------- host address + password detection --------------------------
+
+
+def detect_host_addr() -> str:
+    """LAN-facing IP by UDP-connect probe (no packet is sent; the
+    kernel just chooses an outbound interface). Falls back to
+    127.0.0.1 when no route is available. Same trick bty uses."""
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    s.settimeout(2)
+    try:
+        s.connect(("198.51.100.1", 80))
+        addr: str = s.getsockname()[0]
+        return addr
+    except OSError:
+        return "127.0.0.1"
+    finally:
+        s.close()
+
+
+def gen_admin_password(nbytes: int = 24) -> str:
+    """URL-safe random token to hand the operator on first deploy."""
+    return secrets.token_urlsafe(nbytes)
+
+
+# ---------- compose runner detection -----------------------------------
+
+
+def _compose_cmd() -> list[str] | None:
+    """Return the compose invocation to shell out to, or ``None`` when
+    no supported runner is on PATH."""
+    if shutil.which("podman-compose"):
+        return ["podman-compose"]
+    if shutil.which("podman") and shutil.which("docker-compose") is None:
+        # ``podman compose`` shells out to a provider; keep it simple.
+        return ["podman", "compose"]
+    if shutil.which("docker"):
+        return ["docker", "compose"]
+    return None
+
+
+def _compose_up(dest: Path, envvars: Path) -> None:
+    cmd = _compose_cmd()
+    if cmd is None:
+        raise RuntimeError(
+            "no compose runner on PATH (need one of: podman-compose, podman, docker)"
+        )
+    try:
+        subprocess.run(
+            [*cmd, "--env-file", str(envvars), "up", "-d"],
+            cwd=str(dest),
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+    except subprocess.CalledProcessError as exc:
+        raise RuntimeError(
+            f"compose up failed (rc={exc.returncode}):\n"
+            f"--- stdout ---\n{exc.stdout}\n--- stderr ---\n{exc.stderr}"
+        ) from exc
+
+
+def _compose_down(dest: Path, envvars: Path, *, wipe_data: bool) -> None:
+    cmd = _compose_cmd()
+    if cmd is None:
+        raise RuntimeError(
+            "no compose runner on PATH (need one of: podman-compose, podman, docker)"
+        )
+    args = [*cmd, "--env-file", str(envvars), "down"]
+    if wipe_data:
+        args.append("-v")
+    subprocess.run(args, cwd=str(dest), check=False, capture_output=True)
+
+
+def _wait_healthz(host: str, port: int, deadline: float) -> None:
+    url = f"http://{host}:{port}/healthz"
+    last_err = ""
+    while time.monotonic() < deadline:
+        try:
+            with urllib.request.urlopen(url, timeout=2.0) as resp:
+                if resp.status == 200:
+                    return
+                last_err = f"HTTP {resp.status}"
+        except (urllib.error.URLError, http.client.HTTPException, OSError) as exc:
+            last_err = str(exc)
+        time.sleep(1.0)
+    raise RuntimeError(f"healthz timeout on {url}: {last_err}")
+
+
+# ---------- subcommands ------------------------------------------------
+
+
+def _cmd_init(args: argparse.Namespace) -> int:
+    dest = Path(args.dest).resolve()
+    image = args.image or f"{DEFAULT_IMAGE_REPO}:{pixie.__version__}"
+    admin_pw = args.admin_password or DEFAULT_ADMIN_PASSWORD
+    try:
+        written = _emit_files(dest, image=image, admin_password=admin_pw, force=args.force)
+    except FileExistsError as exc:
+        print(f"pixie-lab init: {exc}", file=sys.stderr)
+        return 1
+    for p in written:
+        print(f"  {p.relative_to(dest.parent) if dest.parent != Path() else p}", file=sys.stderr)
+    print(
+        f"\nNext:\n"
+        f"  cd {dest}\n"
+        f'  cp envvars.example envvars && "${{EDITOR:-vi}}" envvars\n'
+        f"  COMPOSE_ENV_FILES=envvars podman compose up -d\n",
+        file=sys.stderr,
+    )
+    return 0
+
+
+def _cmd_deploy(args: argparse.Namespace) -> int:
+    dest = Path(args.dest).resolve()
+    image = args.image or f"{DEFAULT_IMAGE_REPO}:{pixie.__version__}"
+    admin_pw = args.admin_password or gen_admin_password()
+    host_addr = args.host_addr or detect_host_addr()
+
+    try:
+        _emit_files(dest, image=image, admin_password=admin_pw, force=args.force)
+    except FileExistsError as exc:
+        print(f"pixie-lab deploy: {exc}", file=sys.stderr)
+        return 1
+
+    # Realise envvars from envvars.example + operator-detected
+    # host_addr + admin password.
+    envvars = dest / "envvars"
+    envvars.write_text(
+        f"PIXIE_HOST_ADDR={host_addr}\nPIXIE_ADMIN_PASSWORD={admin_pw}\n",
+        encoding="utf-8",
+    )
+
+    print(
+        f"pixie-lab deploy:\n"
+        f"  dest        : {dest}\n"
+        f"  image       : {image}\n"
+        f"  host_addr   : {host_addr}\n"
+        f"  admin_pass  : {admin_pw}\n",
+        file=sys.stderr,
+    )
+
+    try:
+        _compose_up(dest, envvars)
+        _wait_healthz("127.0.0.1", 8080, time.monotonic() + _HEALTHZ_TIMEOUT)
+    except RuntimeError as exc:
+        print(f"pixie-lab deploy: {exc}", file=sys.stderr)
+        return 1
+    print(f"pixie is up at http://{host_addr}:8080/ui/", file=sys.stderr)
+    return 0
+
+
+def _cmd_purge(args: argparse.Namespace) -> int:
+    dest = Path(args.dest).resolve()
+    envvars = dest / "envvars"
+    if not envvars.exists():
+        # No envvars means no compose to pull down. Still let the
+        # operator wipe data if they asked for it.
+        if args.wipe_data:
+            data_dir = dest / "data"
+            if data_dir.exists():
+                shutil.rmtree(data_dir)
+                print(f"pixie-lab purge: removed {data_dir}", file=sys.stderr)
+        return 0
+    try:
+        _compose_down(dest, envvars, wipe_data=args.wipe_data)
+    except RuntimeError as exc:
+        print(f"pixie-lab purge: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+# ---------- CLI wiring -------------------------------------------------
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="pixie-lab",
+        description=(
+            "pixie deploy generator + convenience CLI. See PLAN.md at "
+            "github.com/safl/pixie for the design rationale."
+        ),
+    )
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    def _add_common(sp: argparse.ArgumentParser) -> None:
+        sp.add_argument(
+            "dest",
+            nargs="?",
+            default=str(DEFAULT_DEST),
+            help="deploy directory (default: cwd)",
+        )
+        sp.add_argument(
+            "--image",
+            default=None,
+            help=f"container image (default: {DEFAULT_IMAGE_REPO}:<pixie-version>)",
+        )
+        sp.add_argument(
+            "--admin-password",
+            default=None,
+            help="operator UI + write-route password (init: default pixie; deploy: random)",
+        )
+        sp.add_argument(
+            "--force",
+            action="store_true",
+            help="overwrite existing files under dest",
+        )
+
+    p_init = sub.add_parser("init", help="write compose.yml + envvars.example + README + data/")
+    _add_common(p_init)
+    p_init.set_defaults(func=_cmd_init)
+
+    p_deploy = sub.add_parser(
+        "deploy",
+        help="init + fill envvars + podman compose up -d + wait /healthz",
+    )
+    _add_common(p_deploy)
+    p_deploy.add_argument(
+        "--host-addr",
+        default=None,
+        help="LAN address pixie advertises (default: auto-detected)",
+    )
+    p_deploy.set_defaults(func=_cmd_deploy)
+
+    p_purge = sub.add_parser("purge", help="podman compose down [--wipe-data drops the volume]")
+    p_purge.add_argument(
+        "dest",
+        nargs="?",
+        default=str(DEFAULT_DEST),
+        help="deploy directory (default: cwd)",
+    )
+    p_purge.add_argument(
+        "--wipe-data",
+        action="store_true",
+        help="also delete the ``data/`` subdirectory",
+    )
+    p_purge.set_defaults(func=_cmd_purge)
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv if argv is not None else sys.argv[1:])
+    return int(args.func(args))
+
+
+if __name__ == "__main__":  # pragma: no cover -- exercised via console-script
+    sys.exit(main())
+
+
+_ = os.environ  # silence unused-import warnings in case future paths grow env reads