testerkit 0.4.0__py3-none-any.whl

testerkit/data/files/catalog.py

@@ -0,0 +1,218 @@
+"""Files catalog — warm DuckDB index over FileStore sidecar metadata.
+
+The catalog daemon owns this index so the UI / API / materializer query
+it instead of walking the date-partitioned tree per call (req 2). It is
+a derived cache: the on-disk blobs and their ``.meta.json`` sidecars are
+the durable truth, so the catalog is built in-memory and rebuilt by
+scanning sidecars on every daemon start, then kept warm by a do_put from
+``FileStore.write`` after each artifact lands durably.
+
+Mirrors the channels Opt-1 model (daemon indexes producer files; live
+writes push; restart rebuilds from disk). Per the plan, the low-level
+``FileStore`` walk stays as a fallback until Phase E removes it.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import duckdb
+import pyarrow as pa
+
+from testerkit.data.files.models import FileArtifactMetadata
+from testerkit.data.schema_dispatch import SchemaVersionRefused, dispatch, report_schema_refusal
+from testerkit.data.schema_versions import SchemaStore
+
+_SIDECAR_SUFFIX = ".meta.json"
+
+CATALOG_DDL = """
+CREATE TABLE IF NOT EXISTS file_catalog (
+    uri VARCHAR PRIMARY KEY,
+    session_id VARCHAR,
+    run_id VARCHAR,
+    name VARCHAR,
+    path VARCHAR,
+    mime VARCHAR,
+    extension VARCHAR,
+    size_bytes BIGINT,
+    instrument_role VARCHAR,
+    resource VARCHAR,
+    created_at TIMESTAMPTZ,
+    attributes VARCHAR
+)
+"""
+
+# ``uri`` is the natural key (``file://{date}/{session}/{name}`` is 1:1 with a
+# blob), so the catalog is its own ingest ledger: the startup scan skips
+# any sidecar whose uri is already present (incremental — no rebuild from
+# every sidecar), and the live ``do_put`` from ``FileStore.write`` upserts.
+# A file rewritten under the same uri refreshes its row instead of
+# duplicating it (the old plain INSERT double-counted).
+_CATALOG_COLUMNS = (
+    "uri",
+    "session_id",
+    "run_id",
+    "name",
+    "path",
+    "mime",
+    "extension",
+    "size_bytes",
+    "instrument_role",
+    "resource",
+    "created_at",
+    "attributes",
+)
+# Column-explicit (not ``SELECT *``) so a catalog upgraded via ALTER (which
+# appends run_id at the end) still aligns with the source table by name.
+_INSERT_COLS = ", ".join(_CATALOG_COLUMNS)
+_UPSERT_SQL = (
+    f"INSERT INTO file_catalog ({_INSERT_COLS}) SELECT {_INSERT_COLS} FROM {{src}} "
+    "ON CONFLICT (uri) DO UPDATE SET "
+    + ", ".join(f"{c}=excluded.{c}" for c in _CATALOG_COLUMNS if c != "uri")
+)
+
+CATALOG_ARROW_SCHEMA = pa.schema(
+    [
+        ("uri", pa.utf8()),
+        ("session_id", pa.utf8()),
+        ("run_id", pa.utf8()),
+        ("name", pa.utf8()),
+        ("path", pa.utf8()),
+        ("mime", pa.utf8()),
+        ("extension", pa.utf8()),
+        ("size_bytes", pa.int64()),
+        ("instrument_role", pa.utf8()),
+        ("resource", pa.utf8()),
+        ("created_at", pa.timestamp("us", tz="UTC")),
+        ("attributes", pa.utf8()),
+    ]
+)
+
+
+# Ephemeral live-stream frames. NOT a durable event (the committed design
+# keeps file events lifecycle-only — FileStarted / FileEnded — to
+# avoid flooding the EventStore at kHz/30 fps rates). Frames ride a
+# fan-out-only Flight db so live consumers get the new bytes push-style, no
+# poll. The frame carries the chunk PAYLOAD (like channels push the sample
+# value) so a live consumer never has to read a still-growing object — which
+# an object-store backend cannot serve anyway. ``payload`` is nullable:
+# raw/jsonl fill it; format sinks (tdms/h5) leave it null and a subscriber
+# rejoins via a library reload at the next boundary (byte-drop can't decode
+# mid-container). ``byte_offset``/``length`` stay for ordering + at-rest seek.
+FRAMES_DB = "file_frames"
+
+FRAME_ARROW_SCHEMA = pa.schema(
+    [
+        ("file_id", pa.utf8()),
+        ("uri", pa.utf8()),
+        ("byte_offset", pa.int64()),
+        ("length", pa.int64()),
+        ("payload", pa.binary()),
+    ]
+)
+
+
+def ensure_schema(conn: duckdb.DuckDBPyConnection) -> None:
+    """Idempotently align the on-disk catalog schema (additive open)."""
+    conn.execute(CATALOG_DDL)
+    # Additive upgrade for catalogs created before run_id existed.
+    conn.execute("ALTER TABLE file_catalog ADD COLUMN IF NOT EXISTS run_id VARCHAR")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_file_catalog_created ON file_catalog(created_at)")
+
+
+def upsert_rows(conn: duckdb.DuckDBPyConnection, table: pa.Table) -> None:
+    """Upsert catalog rows by ``uri`` (idempotent for scan + live do_put)."""
+    if table.num_rows == 0:
+        return
+    conn.register("_upsert", table)
+    conn.execute(_UPSERT_SQL.format(src="_upsert"))
+    conn.unregister("_upsert")
+
+
+def catalog_row(
+    *,
+    uri: str,
+    session_id: str,
+    name: str,
+    key: str,
+    meta: FileArtifactMetadata,
+    created_at: datetime,
+) -> dict[str, Any]:
+    """Build one catalog row from sidecar metadata + backend key.
+
+    ``key`` is the backend-relative physical locator
+    (``{date}/{session_id}/{filename}``) — NOT an absolute path — so the
+    catalog stays backend-agnostic and a resolve hands back a key the
+    blob backend (local or remote) can read.
+    """
+    return {
+        "uri": uri,
+        "session_id": session_id,
+        "run_id": meta.run_id,
+        "name": name,
+        "path": key,
+        "mime": meta.mime,
+        "extension": meta.extension,
+        "size_bytes": meta.size_bytes,
+        "instrument_role": meta.instrument_role,
+        "resource": meta.resource,
+        "created_at": created_at,
+        "attributes": json.dumps(meta.attributes),
+    }
+
+
+def scan_sidecars(conn: duckdb.DuckDBPyConnection, files_dir: Path) -> int:
+    """Fold sidecars not yet cataloged into the (persistent) catalog.
+
+    Layout: ``{files_dir}/{date}/{session_id}/{filename}`` with a
+    ``{filename}.meta.json`` sidecar alongside. **Incremental**: a sidecar
+    whose uri is already in the catalog is skipped, so a daemon restart
+    reads only new sidecars rather than rebuilding from every one. Returns
+    the count of newly-ingested rows.
+    """
+    known = {row[0] for row in conn.execute("SELECT uri FROM file_catalog").fetchall()}
+    rows: list[dict[str, Any]] = []
+    for sidecar in files_dir.glob(f"*/*/*{_SIDECAR_SUFFIX}"):
+        blob = sidecar.with_name(sidecar.name[: -len(_SIDECAR_SUFFIX)])
+        if not blob.exists():
+            continue
+        session_id = blob.parent.name
+        name = blob.name
+        # URI carries the full backend key incl. date ({date}/{session_id}/{name}),
+        # matching FileStore.write so _resolve_key strips the right key on rescan.
+        uri = f"file://{blob.parent.parent.name}/{session_id}/{name}"
+        if uri in known:
+            continue
+        try:
+            # Read the raw dict first so an ABSENT ``schema_version`` is
+            # detectable — the Pydantic default would otherwise mask a pre-1.0
+            # sidecar (no key) as the current version. Dispatch, then validate.
+            raw = json.loads(sidecar.read_text())
+            adapter = dispatch(SchemaStore.FILES, raw.get("schema_version"))
+            meta = adapter(FileArtifactMetadata.model_validate(raw))
+            created_at = datetime.fromtimestamp(blob.stat().st_mtime, tz=UTC)
+        except (OSError, ValueError):
+            continue
+        except SchemaVersionRefused as exc:
+            # Surface it (permanent → warn-once; deferrable → debug), then skip
+            # WITHOUT cataloging so a deferrable (newer) sidecar is re-read next
+            # scan and a newer daemon ingests it (#43). Presence-only catalog →
+            # an absent/pre-1.0 sidecar is re-read too; the warning is deduped.
+            report_schema_refusal(exc, str(sidecar))
+            continue
+        rows.append(
+            catalog_row(
+                uri=uri,
+                session_id=session_id,
+                name=name,
+                key=f"{blob.parent.parent.name}/{session_id}/{name}",
+                meta=meta,
+                created_at=created_at,
+            )
+        )
+    if rows:
+        upsert_rows(conn, pa.Table.from_pylist(rows, schema=CATALOG_ARROW_SCHEMA))
+    return len(rows)

testerkit/data/files/catalog_manager.py

@@ -0,0 +1,325 @@
+"""Files catalog daemon manager + client helpers.
+
+Subclasses ``DaemonManager`` for the files catalog DuckDB daemon and
+provides the client-side query/push/discovery helpers consumers use to
+reach the daemon's warm catalog (req 2) instead of walking the tree.
+
+Blobs + ``.meta.json`` sidecars are the durable truth; the in-memory
+catalog is rebuilt from them on every daemon start.
+"""
+
+from __future__ import annotations
+
+import json
+import warnings
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+import pyarrow as pa
+import pyarrow.flight as flight
+
+from testerkit.data._daemon_lifecycle import DaemonManager, _pid_alive, wait_for_location
+from testerkit.data._flight_query import (
+    FlightQueryClient,
+    _drop_pooled_client,
+    _get_pooled_client,
+    call_options,
+    probe_sql,
+)
+from testerkit.data._flight_subscribe import subscribe
+from testerkit.data._push_relay import PushRelay
+from testerkit.data.files.catalog import (
+    CATALOG_ARROW_SCHEMA,
+    FRAME_ARROW_SCHEMA,
+    FRAMES_DB,
+)
+from testerkit.models.data_options import FileOptions
+
+
+class FilesCatalogManager(DaemonManager):
+    """Manages the files catalog DuckDB daemon."""
+
+    _state_name = "_files_catalog.json"
+    _lock_name = "_files_catalog.lock"
+    _ready_name = "_files_catalog_ready"
+    _pid_name = "_files_catalog_pid"
+    _daemon_module = "testerkit.data.files._catalog_daemon"
+    _port_file = "_files_catalog_flight_port"
+
+
+def acquire(files_dir: Path) -> str:
+    """Acquire a ref to the catalog daemon, starting it if needed.
+
+    Returns the gRPC location string for Flight queries. Probes the daemon
+    after acquiring: a wedged or dead Flight thread (PID alive but not
+    responding) is killed and respawned so callers get a working connection.
+    """
+    mgr = FilesCatalogManager(files_dir)
+    mgr.acquire()
+    location = wait_for_location(mgr, files_dir, "files")
+    if not probe_sql(location, "files"):
+        warnings.warn(
+            f"Files catalog daemon at {location} is not responding — killing and respawning.",
+            stacklevel=2,
+        )
+        mgr.force_restart()
+        mgr.acquire()
+        location = wait_for_location(mgr, files_dir, "files")
+    return location
+
+
+def release(files_dir: Path) -> None:
+    """Release our reference to the catalog daemon."""
+    FilesCatalogManager(files_dir).release()
+
+
+def is_running(files_dir: Path) -> bool:
+    """Return True iff a live catalog daemon is serving ``files_dir``.
+
+    Inspection only — never spawns. Consumers use this to prefer the
+    warm catalog when a daemon is up, and fall back to the local walk
+    otherwise (so unit tests with throwaway data dirs don't spawn a
+    per-test daemon). Phase E removes the walk fallback entirely.
+    """
+    state = files_dir / FilesCatalogManager._state_name
+    if not state.exists():
+        return False
+    try:
+        data = json.loads(state.read_text())
+        pid = data.get("pid")
+    except (json.JSONDecodeError, OSError):
+        return False
+    return isinstance(pid, int) and _pid_alive(pid)
+
+
+def _sql_str(value: str) -> str:
+    return value.replace("'", "''")
+
+
+def query_catalog(files_dir: Path, sql: str) -> list[dict[str, Any]]:
+    """Run a catalog SQL query against the daemon (acquires + releases)."""
+    location = acquire(files_dir)
+    try:
+        client = FlightQueryClient(
+            location,
+            "files",
+            reacquire=lambda: acquire(files_dir),
+            label="FileCatalog",
+        )
+        return client.query(sql)
+    finally:
+        release(files_dir)
+
+
+def resolve_uri(files_dir: Path, uri: str) -> str | None:
+    """Resolve a ``file://`` URI to its on-disk path via the warm catalog."""
+    rows = query_catalog(
+        files_dir,
+        f"SELECT path FROM file_catalog WHERE uri = '{_sql_str(uri)}' LIMIT 1",
+    )
+    return rows[0]["path"] if rows else None
+
+
+def list_recent(files_dir: Path, limit: int) -> list[dict[str, Any]]:
+    """Return the ``limit`` most-recent catalog rows, newest first."""
+    return query_catalog(
+        files_dir,
+        f"SELECT * FROM file_catalog ORDER BY created_at DESC LIMIT {int(limit)}",
+    )
+
+
+def list_artifacts(
+    files_dir: Path,
+    *,
+    uri: str | None = None,
+    session_id: str | None = None,
+    run_id: str | None = None,
+    limit: int = 50,
+) -> list[dict[str, Any]]:
+    """Return catalog rows newest-first, optionally filtered.
+
+    ``uri`` returns the single matching artifact; ``session_id`` /
+    ``run_id`` filter the listing. SQL is built here, inside the files
+    store, so callers (HTTP API, MCP tool) never touch the catalog
+    table directly.
+    """
+    clauses = []
+    if uri:
+        clauses.append(f"uri = '{_sql_str(uri)}'")
+    if session_id:
+        clauses.append(f"session_id = '{_sql_str(session_id)}'")
+    if run_id:
+        clauses.append(f"run_id = '{_sql_str(run_id)}'")
+    where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
+    return query_catalog(
+        files_dir,
+        f"SELECT * FROM file_catalog {where} ORDER BY created_at DESC LIMIT {int(limit)}",
+    )
+
+
+def push_artifact(files_dir: Path, row: dict[str, Any]) -> None:
+    """Push one catalog row to the daemon (best-effort, non-fatal).
+
+    Called by ``FileStore.write`` after the blob + sidecar land durably.
+    Non-fatal: the sidecar is the durable truth, so a failed push just
+    means the artifact isn't in the warm catalog until the next daemon
+    restart rescans it. Skips silently when no daemon is running, so
+    plain writes never spawn one.
+    """
+    if not is_running(files_dir):
+        return
+    location = acquire(files_dir)
+    try:
+        client = _get_pooled_client(location)
+        tbl = pa.Table.from_pylist([row], schema=CATALOG_ARROW_SCHEMA)
+        descriptor = flight.FlightDescriptor.for_command(b"files\0file_catalog")
+        writer, reader = client.do_put(descriptor, tbl.schema, options=call_options())
+        writer.write_table(tbl)
+        # Drain the server ACK(s) before returning: each ACK confirms the daemon
+        # committed one batch, so a resolve_uri right after write() is guaranteed
+        # to see the row (read-after-write) instead of racing the insert.
+        for _ in tbl.to_batches():
+            reader.read()
+        writer.close()
+    except (OSError, RuntimeError, pa.ArrowException) as exc:
+        _drop_pooled_client(location)
+        warnings.warn(f"Files catalog push failed (non-fatal): {exc}", stacklevel=2)
+    finally:
+        release(files_dir)
+
+
+def publish_frame(
+    files_dir: Path,
+    *,
+    file_id: str,
+    uri: str,
+    byte_offset: int,
+    length: int,
+    payload: bytes | None = None,
+) -> None:
+    """Publish one ephemeral stream-frame (best-effort).
+
+    Called by a streaming sink after each ``write``. Fans out to live
+    subscribers push-style (req 5: no poll). The frame carries the chunk
+    ``payload`` so a consumer receives the new bytes directly — never
+    range-reading a still-growing object (an object-store backend can't
+    serve one). raw/jsonl pass the bytes; format sinks (tdms/h5) pass
+    ``None`` and a subscriber rejoins via a library reload at the next
+    boundary. NOT persisted — the on-disk artifact is the durable record.
+    Skips silently when no daemon runs, so plain streaming never spawns one.
+    """
+    if not is_running(files_dir):
+        return
+    location = acquire(files_dir)
+    try:
+        client = _get_pooled_client(location)
+        tbl = pa.Table.from_pylist(
+            [
+                {
+                    "file_id": file_id,
+                    "uri": uri,
+                    "byte_offset": byte_offset,
+                    "length": length,
+                    "payload": payload,
+                }
+            ],
+            schema=FRAME_ARROW_SCHEMA,
+        )
+        descriptor = flight.FlightDescriptor.for_command(f"{FRAMES_DB}\0frames".encode())
+        writer, _ = client.do_put(descriptor, tbl.schema, options=call_options())
+        writer.write_table(tbl)
+        writer.close()
+    except (OSError, RuntimeError, pa.ArrowException) as exc:
+        _drop_pooled_client(location)
+        warnings.warn(f"Files frame publish failed (non-fatal): {exc}", stacklevel=2)
+    finally:
+        release(files_dir)
+
+
+class _FrameTransport:
+    """The files-frame codec + held transport behind a :class:`PushRelay`.
+
+    Coalesces a drained burst of frame dicts into one ``RecordBatch`` and
+    ``do_put``s it to the catalog daemon's hook-only frames db on a pooled,
+    held client. On error it drops + reacquires the pooled client; the on-disk
+    byte stream is the durable record, so a failed push is non-fatal.
+    """
+
+    _DESCRIPTOR = flight.FlightDescriptor.for_command(f"{FRAMES_DB}\0frames".encode())
+
+    def __init__(self, location: str) -> None:
+        self._location = location
+        self._client = _get_pooled_client(location)
+
+    def flush(self, _key: object, rows: list[dict[str, Any]]) -> None:
+        try:
+            tbl = pa.Table.from_pylist(rows, schema=FRAME_ARROW_SCHEMA)
+            writer, _ = self._client.do_put(self._DESCRIPTOR, tbl.schema, options=call_options())
+            writer.write_table(tbl)
+            writer.close()
+        except (OSError, RuntimeError, pa.ArrowException) as exc:
+            _drop_pooled_client(self._location)
+            self._client = _get_pooled_client(self._location)
+            warnings.warn(f"Files frame relay do_put failed (non-fatal): {exc}", stacklevel=2)
+
+
+def open_frame_relay(files_dir: Path, options: FileOptions | None = None) -> PushRelay | None:
+    """Start a non-blocking frame relay iff a catalog daemon is already serving.
+
+    Resolved ONCE per stream (not per chunk). Returns ``None`` when no daemon
+    runs — the common no-subscriber / benchmark case — so the writer's hot path
+    skips all frame work. Never spawns a daemon. The shared :class:`PushRelay`
+    owns the queue + drain + drop-oldest overflow; :class:`_FrameTransport`
+    supplies the frame codec + held ``do_put``. Coalescing tuning comes from
+    ``options`` (testerkit.yaml ``files:``).
+    """
+    if not is_running(files_dir):
+        return None
+    opts = options or FileOptions()
+    transport = _FrameTransport(acquire(files_dir))
+    return PushRelay(
+        flush=transport.flush,
+        max_weight=opts.frame_push_max_rows,
+        max_wait=opts.frame_push_max_wait,
+        queue_max=opts.frame_push_queue_max,
+        thread_name="files-frame-relay",
+        on_close=lambda: release(files_dir),
+    )
+
+
+def subscribe_frames(
+    files_dir: Path,
+    callback: Callable[[dict[str, Any]], None],
+) -> Callable[[], None]:
+    """Subscribe to live stream-frame notifications. Returns an unsub callable.
+
+    Spawns a reader thread that calls ``callback`` with each frame dict
+    (``file_id``/``uri``/``byte_offset``/``length``). Holds a daemon
+    ref for the subscription's lifetime; ``unsub`` closes the stream and
+    releases it. Uses a dedicated client so closing it cleanly interrupts
+    the held-open ``do_get``.
+    """
+    location = acquire(files_dir)
+    client = flight.connect(location)
+
+    def _on_batch(batch: pa.RecordBatch) -> None:
+        for i in range(batch.num_rows):
+            callback({c: batch.column(c)[i].as_py() for c in batch.schema.names})
+
+    def _close() -> None:
+        try:
+            client.close()
+        except (OSError, RuntimeError, pa.ArrowException):
+            pass
+        release(files_dir)
+
+    unsub, _thread = subscribe(
+        client,
+        flight.Ticket(f"{FRAMES_DB}\0__SUBSCRIBE__".encode()),
+        _on_batch,
+        name="files-frame-sub",
+        swallow_errors=True,
+        on_close=_close,
+    )
+    return unsub

testerkit/data/files/models.py

@@ -0,0 +1,60 @@
+"""Models for FileStore artifact metadata (build item 1c)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from testerkit.data.schema_versions import CURRENT_SCHEMA_VERSION, SchemaStore
+
+# FileStore sidecar stamp. The sidecar ({filename}.meta.json) is a published,
+# directly-readable consumer surface — version it so readers can detect format
+# changes. Sourced from the central registry (one home); see
+# ``testerkit.data.schema_versions``.
+FILE_METADATA_SCHEMA_VERSION = CURRENT_SCHEMA_VERSION[SchemaStore.FILES]
+
+
+class FileArtifactMetadata(BaseModel):
+    """Metadata persisted next to a FileStore artifact as a sidecar.
+
+    Written at put time alongside the artifact at
+    ``{filename}.meta.json``. Captures the kind of file that landed,
+    its size on disk, and any user-supplied attributes routed through
+    :meth:`FileStore.write`'s ``attributes`` kwarg.
+
+    Per build item 13, the ``mime`` value follows the TesterKit
+    convention table — see :mod:`testerkit.data.files.serializers`.
+
+    Per build item 17, the metadata-bag field is called
+    ``attributes`` (matching :class:`ChannelDescriptor.attributes`
+    and :class:`Waveform.attributes`).
+
+    Format-specific extraction (image dimensions, audio duration,
+    video duration) is not in the initial cut — that lands when the
+    relevant format library exposes the data inexpensively. Today
+    callers carry whatever they want via the ``attributes`` kwarg.
+    """
+
+    mime: str
+    extension: str
+    size_bytes: int
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    # Optional provenance — populated when a Waveform / channel-shaped
+    # value falls through to FileStore (no ChannelStore wired). Without
+    # these, the FileStore fallback path silently loses the instrument
+    # context that the ChannelStore descriptor would have captured
+    # (first-write provenance). Consumers joining back to the run
+    # record can use these to reconstruct the same context that the
+    # ChannelStore path would have provided.
+    instrument_role: str = ""
+    resource: str = ""
+    # The run that produced this artifact (when written inside a run), so the
+    # catalog can be filtered by run and the UI can link back. Persisted in the
+    # sidecar so a daemon-restart rescan recovers it. ``None`` for run-less writes.
+    run_id: str | None = None
+    # Schema version stamp — included in every sidecar so readers can detect
+    # format changes without scanning field presence. Defaults to the current
+    # version so old sidecars (missing this field) still validate: Pydantic
+    # fills in the default, giving backward-tolerant reads.
+    schema_version: str = Field(default=FILE_METADATA_SCHEMA_VERSION)