feather-control-plane 0.1.0__py3-none-any.whl

control_plane/__init__.py

@@ -0,0 +1,69 @@
+"""control_plane — durable failure observability for an extractor fleet.
+
+A WAL for crash durability, a Postgres control plane for run lifecycle / events / budget
+telemetry, and a lease for deterministic death detection. The data plane lives elsewhere.
+
+Usage:
+    from control_plane import ControlPlane, status
+    cp = ControlPlane("gofrugal", dsn=PG_DSN, wal_dir="control_wal")
+    with cp.run("nightly", run_id=run_id) as run:
+        run.event(entity="sales_item_wise", unit="1001", business_date=d,
+                  status=status.EV_OK, http_status=200, rows=42,
+                  source_attempts=1, sink_attempts=1, pull_ms=1200, load_ms=800)
+        run.budget(status.BUDGET_WALLTIME, limit=1800, used=1800, remaining=0, hit=True)
+        run.finish(status.RUN_PARTIAL, stop_reason=status.STOP_BUDGET_TIME,
+                   n_ok=47, n_err=1, n_skipped=0)
+
+The facade above is resolved LAZILY (PEP 562), and that is load-bearing rather than tidy.
+`control_plane.business_day` and `control_plane.status` are pure stdlib, and consumers want them
+on their own — a cron tick asking what the business day is has no database in the picture. A
+plain `from .run import ControlPlane` at the top of this file runs on `import
+control_plane.business_day` too, because a submodule import executes its package first, so every
+such consumer was forced to have psycopg installed. That tax is why the same `business_day.py`
+ended up hand-copied into scripts that only needed `ist_today()`.
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+__all__ = ["ControlPlane", "ControlPlaneError", "RunContext", "new_run_id", "status"]
+
+# Name -> the submodule that defines it. `status` is absent deliberately: `from control_plane
+# import status` is a submodule import, which Python's import machinery already resolves on its
+# own without help from here.
+_LAZY = {
+    "ControlPlane": ".run",
+    "ControlPlaneError": ".run",
+    "RunContext": ".run",
+    "new_run_id": ".run",
+}
+
+if TYPE_CHECKING:  # the lazy names, visible to type checkers and IDEs
+    from . import status as status
+    from .run import ControlPlane as ControlPlane
+    from .run import ControlPlaneError as ControlPlaneError
+    from .run import RunContext as RunContext
+    from .run import new_run_id as new_run_id
+
+
+def __getattr__(name: str):
+    module = _LAZY.get(name)
+    if module is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    import importlib
+
+    try:
+        return getattr(importlib.import_module(module, __name__), name)
+    except ModuleNotFoundError as exc:
+        if exc.name != "psycopg":
+            raise
+        raise ModuleNotFoundError(
+            f"control_plane.{name} talks to Postgres and needs the psycopg driver, which is an "
+            "optional dependency: install `feather-control-plane[postgres]`. The stdlib-only half "
+            "(control_plane.business_day, control_plane.status) needs no extra.",
+            name="psycopg",
+        ) from exc
+
+
+def __dir__() -> list[str]:
+    return sorted(__all__)

control_plane/business_day.py

@@ -0,0 +1,54 @@
+"""The warehouse's business day, in IST (#3854).
+
+Two clocks, and confusing them is the bug this module exists to prevent:
+
+  * a **business day** — a load window, a grid key, an audit window, a report "as of" —
+    is a partition of the *business*, and the business runs on IST;
+  * an **instant** — `_loaded_at`, a run id, a log stamp, an elapsed/deadline comparison,
+    a cron hour compared against `REPAIR_UTC_HOURS` — is UTC (ADR 0027).
+
+This module is the first half only. Never reach for it to stamp an instant.
+
+Import it rather than writing `ZoneInfo("Asia/Kolkata")` inline: this docstring is the one
+place the reason is recorded, and a literal carries no reason with it.
+"""
+from __future__ import annotations
+
+import datetime as dt
+from zoneinfo import ZoneInfo
+
+IST = ZoneInfo("Asia/Kolkata")
+
+
+def ist_now() -> dt.datetime:
+    """The current instant as an IST-aware datetime — for the callers that need the HOUR.
+
+    A tick that asks "is it past 22:00 IST?" needs a datetime, not a date, and building
+    `now_utc().astimezone(ZoneInfo(...))` inline is how a module ends up with two spellings of IST
+    and no rule about which to use. Use this, then `.hour` / `.date()` off it.
+
+    This is still a *business* clock, not an instant to store: stamp `_loaded_at` with
+    `datetime.now(timezone.utc)` (ADR 0027), never with this.
+    """
+    return dt.datetime.now(IST)
+
+
+def ist_today() -> dt.date:
+    """The calendar date it is *now in India* — the warehouse's business day.
+
+    Never `date.today()`. No service sets `TZ`, so every container runs UTC, and between
+    00:00 and 05:30 IST `date.today()` returns the PREVIOUS business day. Every window
+    derived from it then shifts a day, and the run still reports success: a GoFrugal
+    nightly started at 04:48 IST on 20-Sep-2026 (23:18 UTC on the 19th) pulled 18/17/16-Sep
+    and lost 19-Sep entirely, with a clean `control.run` row and an all-`ok` grid (#3854).
+    """
+    return ist_now().date()
+
+
+def ist_yesterday() -> dt.date:
+    """The business day before `ist_today()` — the usual edge / nightly anchor.
+
+    Plain calendar arithmetic, deliberately: the group's stores trade seven days a week, so
+    there is no trading-day calendar to skip weekends or holidays against.
+    """
+    return ist_today() - dt.timedelta(days=1)

control_plane/recovery.py

@@ -0,0 +1,33 @@
+"""Startup recovery: replay un-shipped WAL tails of prior runs (issue #204).
+
+ARIES-simplified. Before a run does any work it replays the WAL tails left by *prior* runs
+of this extractor on this host — pushing their last-logged events (incl. a dead run's cause)
+into Postgres. Reconciling those runs to `died` is done by the caller, *after* it acquires
+the lease (holding the lease ⇒ no prior run is alive ⇒ unfinished runs are dead), so death
+detection stays deterministic — never a runtime guess.
+"""
+from __future__ import annotations
+
+import glob
+import os
+
+import psycopg
+
+from . import shipper
+
+
+def replay_prior(conn: psycopg.Connection, extractor: str, wal_dir: str, this_run_id: str) -> int:
+    """Ship the un-shipped tail of every prior run's WAL for this extractor. Returns total
+    records shipped. Best-effort per file: one unreadable WAL never blocks the others."""
+    shipped = 0
+    pattern = os.path.join(wal_dir, extractor, "*.jsonl")
+    for path in sorted(glob.glob(pattern)):
+        run_id = os.path.basename(path)[: -len(".jsonl")]
+        if run_id == this_run_id:  # @spec prior_wal_replayed_before_lease
+            continue
+        try:
+            shipped += shipper.ship(conn, path)
+        except Exception as exc:  # noqa: BLE001 — one bad WAL must not abort recovery
+            print(f"[control_plane] WAL replay failed for {path}: "
+                  f"{type(exc).__name__}: {str(exc)[:160]}", flush=True)
+    return shipped

control_plane/run.py

@@ -0,0 +1,237 @@
+"""The consumer-facing facade (issue #204).
+
+An extractor wraps its work in `ControlPlane(...).run(mode) as run:` and calls `run.event(...)`
+per unit, `run.budget(...)` for quota telemetry, and `run.finish(...)` with final counts. The
+facade handles WAL durability, ship-as-you-go to Postgres, startup recovery, the lease, and a
+background renewer thread.
+
+Postgres is a **hard dependency** (ADR 0028): if a DSN is configured and unreachable at startup,
+the run **aborts** — no blind runs (a silent load with no provenance is worse than no load). The
+WAL is a local **crash redo-log** *under* the Postgres writes, not an outage buffer: every event
+hits the WAL first (fsync), then ships to PG; if the process dies, the next run replays the tail so
+the death's breadcrumbs survive ("died after table X / chunk Y"), and the lease supplies the fact
+of death. `dsn=None` is the explicit no-control-plane mode for dev/test (WAL-only).
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import os
+import secrets
+import socket
+import threading
+from contextlib import contextmanager
+
+import psycopg
+
+from . import recovery, shipper, status, store
+from .status import ERROR_DETAIL_CAP
+from .wal import Wal
+
+
+def _now() -> _dt.datetime:
+    return _dt.datetime.now(_dt.timezone.utc)
+
+
+def new_run_id(extractor: str) -> str:  # @spec run_id_unique_across_extractors
+    """`<UTC second>-<extractor>-<6 hex>`. `control.run` is keyed on run_id alone, and every Railway
+    container is pid 1, so the old `<second>-<pid>` gave two services starting in the same second
+    one id (#4614). The timestamp stays first: SAP names backups `run_id[:8]`."""
+    return f"{_now().strftime('%Y%m%dT%H%M%SZ')}-{extractor}-{secrets.token_hex(3)}"
+
+
+class ControlPlaneError(RuntimeError):
+    pass
+
+
+class _Renewer:
+    """Renews the lease on its OWN connection, decoupled from the work loop — so a 14h job or
+    a 90-min SAP chunk keeps proving liveness and is never falsely declared dead. Swallows
+    transient PG errors and retries on the next tick (reconnecting if needed)."""
+
+    def __init__(self, dsn: str, extractor: str, run_id: str, ttl_s: int, interval_s: int):
+        self._dsn = dsn
+        self._extractor = extractor
+        self._run_id = run_id
+        self._ttl = ttl_s
+        self._interval = interval_s
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._loop, name="cp-lease-renewer", daemon=True)
+        self._conn: psycopg.Connection | None = None
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+        self._thread.join(timeout=self._interval + 5)
+        if self._conn is not None:
+            try:
+                self._conn.close()
+            except Exception:  # noqa: BLE001
+                pass
+
+    def _loop(self) -> None:
+        while not self._stop.wait(self._interval):  # @spec lease_renewed_in_background
+            try:
+                if self._conn is None or self._conn.closed:
+                    self._conn = psycopg.connect(self._dsn, connect_timeout=10)
+                if not store.lease_renew(self._conn, self._extractor, self._run_id, self._ttl):
+                    # @spec lease_loss_does_not_stop_run
+                    print(f"[control_plane] lease lost for {self._extractor}/{self._run_id} "
+                          "(another run reclaimed it?)", flush=True)
+            except Exception as exc:  # noqa: BLE001 — renew must never crash the run  # @spec false_death_of_live_run
+                print(f"[control_plane] lease renew failed (will retry): "
+                      f"{type(exc).__name__}: {str(exc)[:160]}", flush=True)
+                if self._conn is not None:
+                    try:
+                        self._conn.close()
+                    except Exception:  # noqa: BLE001
+                        pass
+                self._conn = None
+
+
+class RunContext:
+    def __init__(self, cp: "ControlPlane", run_id: str, wal: Wal, conn: psycopg.Connection | None):
+        self.cp = cp
+        self.run_id = run_id
+        self.wal = wal
+        self.conn = conn            # None ⇒ PG was down at start; WAL-only degraded mode
+        self._seq = 0
+        self._finished = False
+
+    # --- internal ---
+    def _ship_now(self) -> None:
+        if self.conn is None:
+            return
+        try:
+            shipper.ship(self.conn, self.wal.path)
+        except Exception as exc:  # noqa: BLE001 — WAL already holds it; recovery ships later  # @spec ship_as_you_go
+            print(f"[control_plane] ship failed (buffered in WAL): "
+                  f"{type(exc).__name__}: {str(exc)[:160]}", flush=True)
+
+    def _emit(self, record: dict) -> None:
+        self.wal.append(record)   # durable first  # @spec wal_before_network
+        self._ship_now()          # then best-effort to PG
+
+    # --- public API ---
+    def event(self, *, entity: str, status: str, unit: str | None = None,
+              business_date=None, http_status: int | None = None, rows: int | None = None,
+              source_attempts: int | None = None, sink_attempts: int | None = None,
+              pull_ms: int | None = None, load_ms: int | None = None,
+              error_detail: str | None = None, extra: dict | None = None) -> None:
+        """Record one unit/cell outcome (one row in control.event). `status` here is the
+        event-status string (status.EV_*), not the module. `extra` is a small JSONB bag for
+        source-specific metrics that don't warrant a shared column (e.g. Zakya's n_requests /
+        gen_ms / rate_remaining); leave it None when there's nothing source-specific to keep."""
+        self._emit({
+            "kind": "event", "extractor": self.cp.extractor, "run_id": self.run_id,
+            "seq": self._seq, "ran_at": _now(), "entity": entity, "unit": unit,
+            "business_date": business_date, "status": status, "http_status": http_status,
+            "rows": rows, "source_attempts": source_attempts, "sink_attempts": sink_attempts,
+            "pull_ms": pull_ms, "load_ms": load_ms,
+            "error_detail": (error_detail[:ERROR_DETAIL_CAP] if error_detail else None),  # @spec error_detail_capped
+            "extra": extra,
+        })
+        self._seq += 1  # @spec event_one_row_per_unit
+
+    def budget(self, name: str, *, limit: int | None, used: int | None,
+               remaining: int | None, hit: bool) -> None:
+        """Record quota utilisation (every run, not only on a hit) → headroom is queryable."""
+        self._emit({
+            "kind": "budget", "extractor": self.cp.extractor, "run_id": self.run_id,
+            "budget_name": name, "limit_val": limit, "used": used, "remaining": remaining,
+            "hit": hit, "ran_at": _now(),
+        })
+
+    def finish(self, run_status: str, *, stop_reason: str, n_ok: int | None = None,
+               n_err: int | None = None, n_skipped: int | None = None) -> None:
+        """Close the run with its final disposition. Idempotent (first call wins)."""
+        if self._finished:  # @spec run_finish_first_call_wins
+            return
+        self._emit({
+            "kind": "run_finish", "extractor": self.cp.extractor, "run_id": self.run_id,
+            "started_at": self.cp._started_at, "finished_at": _now(),
+            "status": run_status, "stop_reason": stop_reason,
+            "n_ok": n_ok, "n_err": n_err, "n_skipped": n_skipped,
+        })
+        self._finished = True
+
+
+class ControlPlane:
+    def __init__(self, extractor: str, dsn: str | None, wal_dir: str, *,
+                 lease_ttl_s: int = 300, renew_interval_s: int = 60, host: str | None = None):
+        self.extractor = extractor
+        self.dsn = dsn
+        self.wal_dir = wal_dir
+        self.lease_ttl_s = lease_ttl_s
+        self.renew_interval_s = renew_interval_s
+        self.host = host or socket.gethostname()
+        self._started_at: str | None = None
+
+    @contextmanager
+    def run(self, mode: str, *, run_id: str | None = None):
+        run_id = run_id or new_run_id(self.extractor)
+        self._started_at = _now().isoformat()
+        wal = Wal(os.path.join(self.wal_dir, self.extractor, run_id + ".jsonl"))
+        conn: psycopg.Connection | None = None
+        renewer: _Renewer | None = None
+
+        # Postgres is a HARD dependency (ADR 0028): if a DSN is configured, the control plane
+        # MUST be reachable or the run aborts — never run blind. A failure here propagates out of
+        # the `with` before any work begins. (dsn=None is the explicit no-control-plane mode for
+        # dev/test → WAL-only.)
+        if self.dsn:  # @spec hard_startup_dependency
+            conn = psycopg.connect(self.dsn, connect_timeout=10)
+            try:
+                store.ensure_schema(conn)
+                recovery.replay_prior(conn, self.extractor, self.wal_dir, run_id)
+                # @spec lease_refuses_second_writer
+                if not store.lease_acquire(conn, self.extractor, run_id, self.lease_ttl_s):
+                    raise ControlPlaneError(
+                        f"another live run holds the lease for {self.extractor!r} "
+                        "(fresh lease present) — refusing to start a second writer")
+                store.reconcile_dead(conn, self.extractor, run_id)  # @spec death_visible_only_at_next_start
+                renewer = _Renewer(self.dsn, self.extractor, run_id,
+                                   self.lease_ttl_s, self.renew_interval_s)
+                renewer.start()
+            except BaseException:
+                # setup failed — clean up and abort; never degrade to a blind run
+                if renewer is not None:
+                    renewer.stop()
+                try:
+                    conn.close()
+                except Exception:  # noqa: BLE001
+                    pass
+                raise
+
+        rc = RunContext(self, run_id, wal, conn)
+        rc._emit({  # @spec run_start_recorded_before_work
+            "kind": "run_start", "run_id": run_id, "extractor": self.extractor,
+            "mode": mode, "host": self.host, "pid": os.getpid(),
+            "started_at": self._started_at, "status": status.RUN_RUNNING,
+            "wal_path": wal.path,
+        })
+
+        try:
+            yield rc
+        except BaseException:
+            if not rc._finished:
+                rc.finish(status.RUN_FAILED, stop_reason=status.STOP_ERROR)  # @spec implicit_finish_on_exit.sys_exit_inside_block
+            raise
+        finally:
+            if not rc._finished:
+                rc.finish(status.RUN_OK, stop_reason=status.STOP_CLEAN)  # @spec implicit_finish_on_exit
+            if renewer is not None:
+                renewer.stop()
+            if conn is not None:
+                try:
+                    store.lease_release(conn, self.extractor, run_id)  # @spec lease_released_on_exit
+                    shipper.ship(conn, wal.path)  # final flush  # @spec final_flush_on_exit
+                except Exception as exc:  # noqa: BLE001
+                    print(f"[control_plane] final flush/release failed (WAL retained): "
+                          f"{type(exc).__name__}: {str(exc)[:160]}", flush=True)
+                finally:
+                    try:
+                        conn.close()
+                    except Exception:  # noqa: BLE001
+                        pass

control_plane/shipper.py

@@ -0,0 +1,56 @@
+"""WAL → Postgres projection (issue #204).
+
+Tails a WAL file from a byte offset (sidecar `<wal>.off`) and applies each complete record
+to Postgres in one transaction, then advances the offset. At-least-once + idempotent upserts
+(store.apply) = effectively-once, with no distributed transaction. Re-shipping after a lost
+offset is harmless (ON CONFLICT), so the offset is an optimisation, not a correctness anchor.
+"""
+from __future__ import annotations
+
+import os
+
+import psycopg
+
+from . import store, wal
+
+
+def _off_path(wal_path: str) -> str:
+    return wal_path + ".off"
+
+
+def _read_off(wal_path: str) -> int:
+    p = _off_path(wal_path)
+    try:
+        with open(p, encoding="utf-8") as f:
+            return int(f.read().strip() or 0)
+    except (FileNotFoundError, ValueError):
+        return 0
+
+
+def _write_off(wal_path: str, offset: int) -> None:
+    p = _off_path(wal_path)
+    tmp = p + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        f.write(str(offset))
+    os.replace(tmp, p)  # atomic
+
+
+def ship(conn: psycopg.Connection, wal_path: str) -> int:
+    """Ship the un-shipped tail of `wal_path` to Postgres. Returns records shipped.
+
+    One transaction for the whole tail: if any apply fails we roll back and leave the offset
+    untouched, so the next attempt re-ships the same tail cleanly (idempotent)."""
+    start = _read_off(wal_path)
+    records, new_off = wal.read_new(wal_path, start)
+    if not records:
+        return 0
+    try:
+        with conn.cursor() as cur:
+            for rec in records:
+                store.apply(cur, rec)
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+    _write_off(wal_path, new_off)  # @spec whole_tail_one_transaction
+    return len(records)

control_plane/status.py

@@ -0,0 +1,60 @@
+"""Control-plane status vocabulary (issue #204).
+
+One place for the words written to `control.run.status`, `control.run.stop_reason`,
+`control.event.status`, and `control.budget_usage.budget_name`, so every extractor speaks
+the same language and the #206 cross-source health query never special-cases per source.
+
+Vocabulary is a **blend** (locked 2026-06-24): API extractors (gofrugal, zakya) split the
+source leg from the sink leg (`source_error` vs `sink_error`); DB extractors (sap, ptp,
+stylehr) use the single fused `error` (read+load happen in one retry). The `run` lifecycle
+states and budget `stop_reason` are standard across all five.
+"""
+from __future__ import annotations
+
+# --- control.run.status (run lifecycle) ---
+RUN_RUNNING = "running"   # inserted at start; live
+RUN_OK = "ok"             # finished, every unit succeeded
+RUN_PARTIAL = "partial"   # finished, some units errored (or budget cut it short)
+RUN_FAILED = "failed"     # finished by an unhandled exception
+RUN_DIED = "died"         # never finished itself; reconciled by a later run (lease lapsed)
+
+RUN_STATES = frozenset({RUN_RUNNING, RUN_OK, RUN_PARTIAL, RUN_FAILED, RUN_DIED})  # @spec run_vocabulary_unvalidated
+
+# --- control.run.stop_reason (why the run ended) ---
+STOP_CLEAN = "clean"               # ran to natural completion
+STOP_BUDGET_API = "budget_api"     # hit a per-day API quota
+STOP_BUDGET_EXPORT = "budget_export"  # hit a per-hour export quota
+STOP_BUDGET_TIME = "budget_time"   # walltime budget cut it short (still healthy)
+STOP_TOKEN_REFRESH_FAILED = "token_refresh_failed"  # no usable OAuth token: the token endpoint
+                                   # refused or is cooling down, or a forced refresh was withheld.
+                                   # NOT a quota stop — budget_usage hit stays false (#4245)
+STOP_TOKEN_REJECTED = "token_rejected"  # Zoho kept answering 401 to freshly minted tokens: a
+                                   # scope / org / grant problem, not the token endpoint (#4245)
+STOP_ERROR = "error"               # unhandled exception
+STOP_DIED = "died"                 # set by the reconciler on a dead run
+
+STOP_REASONS = frozenset({
+    STOP_CLEAN, STOP_BUDGET_API, STOP_BUDGET_EXPORT, STOP_BUDGET_TIME, STOP_TOKEN_REFRESH_FAILED,
+    STOP_TOKEN_REJECTED, STOP_ERROR, STOP_DIED,
+})
+
+# --- control.event.status (per unit/cell outcome) ---
+EV_OK = "ok"                    # landed rows  # @spec event_vocabulary_unvalidated
+EV_ZERO_ROWS = "zero_rows"      # source reachable, genuinely empty (http 200)
+EV_SOURCE_ERROR = "source_error"  # source returned non-200 / bad payload (API extractors)
+EV_TIMEOUT = "timeout"          # source did not respond at all (http=0, all attempts threw)
+EV_SINK_ERROR = "sink_error"    # the load/sink (MotherDuck) failed after retries (API extractors)
+EV_RATE_LIMITED = "rate_limited"  # source 429 (Zakya)
+EV_OK_PARTIAL = "ok_partial"    # landed, but some records quarantined (Zakya, ADR 0021)
+EV_SOURCE_NOT_SETTLED = "source_not_settled"  # source answered, but as-of a moment before the
+                                # business date closed, so the answer cannot be complete —
+                                # NOT terminal, retry once the source has caught up (#2957)
+EV_ERROR = "error"              # DB extractors: fused read-or-load failure (single leg)
+
+# --- control.budget_usage.budget_name ---
+BUDGET_WALLTIME = "walltime"          # per-run wall-clock budget (seconds)
+BUDGET_API_DAILY = "api_daily"        # Zakya 25k calls/day
+BUDGET_EXPORT_HOURLY = "export_hourly"  # Zakya 10 exports/hour
+
+# error_detail is truncated to this (matches #108).
+ERROR_DETAIL_CAP = 2000

control_plane/store.py

@@ -0,0 +1,466 @@
+"""Postgres control plane: schema, idempotent upserts, the liveness lease (issue #204).
+
+This is the *control* plane — run lifecycle, the event log, budget telemetry, and the lease.
+The *data* plane stays in MotherDuck. Control is a transactional/concurrent/locking workload
+(the opposite of columnar analytics), and you cannot durably log a MotherDuck failure into
+MotherDuck — hence a separate Postgres (supersedes ADR 0005 + #65).
+
+All writes are idempotent so the WAL shipper can replay at-least-once and converge:
+- `event` keyed `(extractor, run_id, seq)` → ON CONFLICT DO NOTHING
+- `run` keyed `run_id` → start DO NOTHING, finish is an UPDATE
+- `budget_usage` keyed `(run_id, extractor, budget_name)` → ON CONFLICT DO UPDATE
+
+`ran_at` is the canonical run-health timestamp (ADR 0027). `limit_val` avoids the reserved
+word `limit`.
+"""
+from __future__ import annotations
+
+import psycopg
+from psycopg.types.json import Json
+
+from . import status
+
+SCHEMA_DDL = """
+CREATE SCHEMA IF NOT EXISTS control;
+
+CREATE TABLE IF NOT EXISTS control.run (
+    run_id        TEXT PRIMARY KEY,
+    extractor     TEXT NOT NULL,
+    mode          TEXT,
+    host          TEXT,
+    pid           INTEGER,
+    started_at    TIMESTAMPTZ NOT NULL,
+    finished_at   TIMESTAMPTZ,
+    status        TEXT NOT NULL,
+    stop_reason   TEXT,
+    n_ok          INTEGER,
+    n_err         INTEGER,
+    n_skipped     INTEGER,
+    reconciled_by TEXT,
+    wal_path      TEXT,
+    extra         JSONB
+);
+CREATE INDEX IF NOT EXISTS run_extractor_unfinished
+    ON control.run (extractor) WHERE finished_at IS NULL;
+
+CREATE TABLE IF NOT EXISTS control.lease (
+    extractor        TEXT PRIMARY KEY,
+    run_id           TEXT NOT NULL,
+    lease_expires_at TIMESTAMPTZ NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS control.event (
+    extractor       TEXT NOT NULL,
+    run_id          TEXT NOT NULL,
+    seq             BIGINT NOT NULL,
+    ran_at          TIMESTAMPTZ NOT NULL,
+    entity          TEXT,
+    unit            TEXT,
+    business_date   DATE,
+    status          TEXT NOT NULL,
+    http_status     INTEGER,
+    rows            BIGINT,
+    source_attempts INTEGER,
+    sink_attempts   INTEGER,
+    pull_ms         INTEGER,
+    load_ms         INTEGER,
+    error_detail    TEXT,
+    extra           JSONB,
+    PRIMARY KEY (extractor, run_id, seq)
+);
+CREATE INDEX IF NOT EXISTS event_run ON control.event (run_id);
+
+CREATE TABLE IF NOT EXISTS control.budget_usage (
+    run_id      TEXT NOT NULL,
+    extractor   TEXT NOT NULL,
+    budget_name TEXT NOT NULL,
+    limit_val   BIGINT,
+    used        BIGINT,
+    remaining   BIGINT,
+    hit         BOOLEAN,
+    ran_at      TIMESTAMPTZ,
+    PRIMARY KEY (run_id, extractor, budget_name)
+);
+"""
+
+
+# Advisory-lock key serializing the DDL path across processes (#3091). An arbitrary but FIXED
+# 64-bit constant: every writer must pick the same number or the lock serializes nothing. Recorded
+# here so a future advisory lock in this repo picks a different one.
+_SCHEMA_ADVISORY_LOCK = 3_204_198_401
+
+# The shape ensure_schema guarantees. Checked as a READ before any DDL runs, because DDL on this
+# path is what deadlocked the fleet: `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` takes an
+# AccessExclusiveLock *even when the column already exists*, so running it on every startup gave
+# every pair of concurrently-starting writers a chance to deadlock (#3091).
+_REQUIRED_TABLES = ("run", "lease", "event", "budget_usage")
+_REQUIRED_COLUMNS = (("event", "extra"), ("run", "extra"))
+
+# The dbt-ready notification (#3884). dbt-runner's worker LISTENs on this channel and runs a tick
+# the moment Zakya's coherent-batch marker is recorded, instead of waiting for its next slot.
+# AFTER INSERT ... FOR EACH ROW fires only for a row actually inserted, so a WAL replay that hits
+# `ON CONFLICT DO NOTHING` notifies nobody; a duplicate would be harmless anyway (the tick is
+# idempotent). NOTIFY is delivered at COMMIT, and the shipper commits each shipped tail at once
+# (ship-as-you-go), so the listener hears it seconds after the marker lands. Not durable: a
+# listener that is down misses it, which is why the worker also ticks on its registry slots.
+# To CHANGE the function body or filter, rename the trigger (the presence check below is by name).
+DBT_READY_CHANNEL = "dbt_ready"  # @spec ready_event_notify
+DBT_READY_TRIGGER = "event_dbt_ready_notify"
+DBT_READY_SOURCE = ("zakya", "transform_ready", status.EV_OK)  # extractor, entity, status
+DBT_READY_DDL = f"""
+CREATE OR REPLACE FUNCTION control.notify_dbt_ready() RETURNS trigger
+LANGUAGE plpgsql AS $fn$
+BEGIN
+    PERFORM pg_notify('{DBT_READY_CHANNEL}', json_build_object(
+        'extractor', NEW.extractor, 'run_id', NEW.run_id,
+        'business_date', NEW.business_date)::text);
+    RETURN NULL;
+END
+$fn$;
+DO $do$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_trigger t
+        JOIN pg_class c ON c.oid = t.tgrelid
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = 'control' AND c.relname = 'event' AND t.tgname = '{DBT_READY_TRIGGER}'
+    ) THEN
+        CREATE TRIGGER {DBT_READY_TRIGGER}
+            AFTER INSERT ON control.event FOR EACH ROW
+            WHEN (NEW.extractor = '{DBT_READY_SOURCE[0]}' AND NEW.entity = '{DBT_READY_SOURCE[1]}'
+                  AND NEW.status = '{DBT_READY_SOURCE[2]}')
+            EXECUTE FUNCTION control.notify_dbt_ready();
+    END IF;
+END
+$do$;
+"""
+
+# The run-finish notification (#4024), for every OTHER source. GoFrugal and SAP have no marker
+# event; their proof of a complete load is their own `control.run` row finishing `ok`, which the
+# worker used to find only at its 15-minute check. This trigger fires on the TRANSITION into
+# (status ok, finished_at set) -- on the INSERT a `run_finish` makes when no `run_start` landed, or
+# on the UPDATE from running/died -- and never again for that row: a WAL replay of `run_finish` is
+# `ON CONFLICT DO UPDATE` on a row already `ok`, so the body drops it. Every extractor except `dbt`
+# (the listener's own rows) notifies; the WORKER owns which extractors it acts on
+# (listen_worker.RUN_NOTIFY_EXTRACTORS, derived from the admitted sources), so admitting a source
+# never needs this trigger renamed. Its OWN channel, not `dbt_ready`: a worker from before #4024
+# treats every `dbt_ready` payload as a Zakya marker and exits 76 for a run_id it cannot find, and
+# with the CDC instances finishing a run every 5 minutes that retry chain never runs out -- so on
+# the shared channel a rolled-back or not-yet-redeployed dbt-runner would stop building while any
+# other writer had installed this trigger (review of #4025). On its own channel an old worker
+# never hears it. The payload carries `kind: run` as well, so a listener on both channels can route
+# by payload alone.
+DBT_RUN_FINISHED_CHANNEL = "dbt_run_finished"  # @spec run_finish_notify
+RUN_FINISH_TRIGGER = "run_dbt_ready_notify"
+RUN_FINISH_DDL = f"""
+CREATE OR REPLACE FUNCTION control.notify_dbt_run_finished() RETURNS trigger
+LANGUAGE plpgsql AS $fn$
+BEGIN
+    IF TG_OP = 'UPDATE' AND OLD.status = '{status.RUN_OK}' AND OLD.finished_at IS NOT NULL THEN
+        RETURN NULL;
+    END IF;
+    PERFORM pg_notify('{DBT_RUN_FINISHED_CHANNEL}', json_build_object(
+        'kind', 'run', 'extractor', NEW.extractor, 'run_id', NEW.run_id,
+        'finished_at', NEW.finished_at)::text);
+    RETURN NULL;
+END
+$fn$;
+DO $do$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_trigger t
+        JOIN pg_class c ON c.oid = t.tgrelid
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = 'control' AND c.relname = 'run' AND t.tgname = '{RUN_FINISH_TRIGGER}'
+    ) THEN
+        CREATE TRIGGER {RUN_FINISH_TRIGGER}
+            AFTER INSERT OR UPDATE ON control.run FOR EACH ROW
+            WHEN (NEW.status = '{status.RUN_OK}' AND NEW.finished_at IS NOT NULL
+                  AND NEW.extractor <> 'dbt')
+            EXECUTE FUNCTION control.notify_dbt_run_finished();
+    END IF;
+END
+$do$;
+"""
+# (trigger name, table) for every notification trigger a current plane carries; the presence check
+# in `_schema_state` and the create-if-absent DDL are both by name, so changing a body means a
+# new name here.
+_NOTIFY_TRIGGERS = ((DBT_READY_TRIGGER, "event"), (RUN_FINISH_TRIGGER, "run"))
+_NOTIFY_DDLS = (DBT_READY_DDL, RUN_FINISH_DDL)
+
+
+# How long the trigger-only migration may WAIT for its table before it gives up for this start.
+# A shipper transaction holds RowExclusive on control.event / control.run; CREATE TRIGGER needs
+# ShareRowExclusive and must queue behind it. Bounded so a writer's startup is delayed seconds at
+# worst, never stalled -- and never part of a lock cycle, because each trigger is installed in its
+# OWN transaction that takes no other table lock (#3884). Installing both in one transaction would
+# lock control.event then control.run while a replaying shipper locks control.run (run_start) then
+# control.event (events): a cycle, reproduced in the review of #4025 as DeadlockDetected.
+_TRIGGER_LOCK_TIMEOUT = "5s"
+
+
+def _schema_state(conn: psycopg.Connection) -> str:
+    """'current', 'trigger_missing' (every table and column present, a notification trigger --
+    #3884's on control.event or #4024's on control.run -- absent) or 'stale' -- pure catalog
+    reads, no lock on any control table."""
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT table_name FROM information_schema.tables WHERE table_schema = 'control'")
+        tables = {r[0] for r in cur.fetchall()}
+        if not set(_REQUIRED_TABLES) <= tables:
+            return "stale"
+        cur.execute(
+            "SELECT table_name, column_name FROM information_schema.columns "
+            "WHERE table_schema = 'control' AND table_name = ANY(%s)",
+            ([t for t, _ in _REQUIRED_COLUMNS],))
+        columns = {(r[0], r[1]) for r in cur.fetchall()}
+        if not set(_REQUIRED_COLUMNS) <= columns:
+            return "stale"
+        for trigger, table in _NOTIFY_TRIGGERS:
+            cur.execute(
+                "SELECT 1 FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid "
+                "JOIN pg_namespace n ON n.oid = c.relnamespace "
+                "WHERE n.nspname = 'control' AND c.relname = %s AND t.tgname = %s",
+                (table, trigger))
+            if cur.fetchone() is None:
+                return "trigger_missing"
+        return "current"
+
+
+def _schema_is_current(conn: psycopg.Connection) -> bool:
+    """True when every table, migrated column and both dbt-ready triggers exist — no locks.
+
+    This is the steady state on a live control plane, so the common path of ensure_schema() must
+    reach exactly here and stop: no CREATE, no ALTER, nothing that can block another session."""
+    return _schema_state(conn) == "current"
+
+
+def ensure_schema(conn: psycopg.Connection) -> None:
+    """Create the control schema if it is not already there. Safe to call concurrently.
+
+    Called at the start of EVERY run by every writer, so its steady-state cost is the thing that
+    matters: a single catalog read and no lock at all. Before #3091 it ran `CREATE TABLE IF NOT
+    EXISTS` + two `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` unconditionally, and because the ALTER
+    takes an AccessExclusiveLock whether or not the column is missing, two writers starting in the
+    same moment deadlocked -- each holding the CREATE's weaker lock while requesting the exclusive
+    one. Postgres killed them, and under ADR 0028's hard dependency that aborts the run BEFORE it
+    reads its source: on 04-Sep-2026 two of PTP's three instances did not load at all, killed by
+    the featherbase drainer's startup 27 seconds earlier. With featherbase ticking every 5 minutes
+    that was a dice-roll at every startup, fleet-wide.
+    """
+    state = _schema_state(conn)
+    if state == "current":  # @spec schema_ensured_without_locks
+        conn.rollback()   # end the read's implicit transaction; hold no locks on the way out
+        return
+    if state == "trigger_missing":
+        # Every live plane on first start after #3884 (and again after #4024). NOT the full DDL
+        # path below: its CREATE INDEX (ShareLock on control.run) then ALTERs (AccessExclusive on
+        # control.event) form a cycle with a shipper holding control.event and updating
+        # control.run -- the #3091 deadlock again, reproduced 4/4 in review. Only the triggers,
+        # lock-bounded, each created only if absent (the one already there costs a catalog read
+        # and no table lock), and each in its OWN transaction so this path never holds two table
+        # locks at once (see _TRIGGER_LOCK_TIMEOUT); a start that cannot get a lock in time, or
+        # loses a deadlock race, carries on without that trigger and a later start installs it.
+        conn.rollback()
+        for trigger, ddl in zip((t for t, _ in _NOTIFY_TRIGGERS), _NOTIFY_DDLS):
+            try:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT pg_advisory_xact_lock(%s)", (_SCHEMA_ADVISORY_LOCK,))
+                    cur.execute(f"SET LOCAL lock_timeout = '{_TRIGGER_LOCK_TIMEOUT}'")
+                    cur.execute(ddl)
+                conn.commit()
+            except (psycopg.errors.LockNotAvailable, psycopg.errors.DeadlockDetected) as exc:
+                conn.rollback()
+                print(f"[control_plane] trigger {trigger} not installed this start "
+                      f"({type(exc).__name__}: its table was busy > {_TRIGGER_LOCK_TIMEOUT}); "
+                      "a later start will.", flush=True)
+        return
+
+    with conn.cursor() as cur:
+        # First run, or a genuine migration. Serialize the DDL so concurrent first-runs queue
+        # instead of deadlocking; the lock is released with the transaction below.
+        cur.execute("SELECT pg_advisory_xact_lock(%s)", (_SCHEMA_ADVISORY_LOCK,))
+        cur.execute(SCHEMA_DDL)
+        # Migration for control planes created before control.event.extra existed (e.g. the
+        # live GoFrugal table): CREATE TABLE IF NOT EXISTS above won't add a column to it.
+        cur.execute("ALTER TABLE control.event ADD COLUMN IF NOT EXISTS extra JSONB")
+        # Same for control.run.extra, added for dbt (#2640): dbt's run row carries fields no
+        # extractor has -- bronze_watermark, action, dbt_rc -- and bolting five dbt-specific
+        # columns onto the table five writers share would be worse than one JSONB bag.
+        cur.execute("ALTER TABLE control.run ADD COLUMN IF NOT EXISTS extra JSONB")
+        for ddl in _NOTIFY_DDLS:  # #3884, #4024 — created only if absent, so a re-run takes no lock
+            cur.execute(ddl)
+    conn.commit()
+
+
+# --- WAL-record projection (the shipper calls apply() per record, in file order) ---
+
+class RunIdClash(ValueError):
+    """A run record met a `control.run` row of the same run_id owned by another extractor."""
+
+
+def _refuse_other_extractors_run(cur: psycopg.Cursor, rec: dict) -> None:  # @spec run_id_clash_refused
+    """Raise, rolling the shipper's transaction back, rather than drop this run's start or fold
+    its finish into another extractor's row: both hid a finished GoFrugal load from the build gate
+    (#4614). The record stays in the WAL, so nothing is lost."""
+    cur.execute("SELECT extractor FROM control.run WHERE run_id = %s", (rec["run_id"],))
+    row = cur.fetchone()
+    if row is not None and row[0] != rec["extractor"]:
+        raise RunIdClash(f"run_id {rec['run_id']!r} of {rec['extractor']!r} is already "
+                         f"{row[0]!r}'s run; refusing to {rec['kind']} over it")
+
+def apply(cur: psycopg.Cursor, rec: dict) -> None:  # @spec shipping_idempotent
+    """Project one WAL record into Postgres. Idempotent; dispatched by `kind`."""
+    kind = rec["kind"]
+    if kind == "run_start":
+        run_extra = rec.get("extra")
+        cur.execute(
+            "INSERT INTO control.run "
+            "(run_id, extractor, mode, host, pid, started_at, status, wal_path, extra) "
+            "VALUES (%s,%s,%s,%s,%s,%s::timestamptz,%s,%s,%s) ON CONFLICT (run_id) DO NOTHING",
+            (rec["run_id"], rec["extractor"], rec.get("mode"), rec.get("host"),
+             rec.get("pid"), rec["started_at"], rec["status"], rec.get("wal_path"),
+             Json(run_extra) if run_extra is not None else None),
+        )
+        if cur.rowcount == 0:
+            _refuse_other_extractors_run(cur, rec)
+    elif kind == "run_finish":
+        # Upsert: if PG was down at start (no run_start row landed), still record the run.
+        fin_extra = rec.get("extra")
+        cur.execute(
+            "INSERT INTO control.run "
+            "(run_id, extractor, started_at, finished_at, status, stop_reason, n_ok, n_err, n_skipped, extra) "
+            "VALUES (%s,%s,%s::timestamptz,%s::timestamptz,%s,%s,%s,%s,%s,%s) "
+            "ON CONFLICT (run_id) DO UPDATE SET "
+            "finished_at=EXCLUDED.finished_at, status=EXCLUDED.status, "  # @spec died_overwritten_by_late_finish
+            "stop_reason=EXCLUDED.stop_reason, n_ok=EXCLUDED.n_ok, n_err=EXCLUDED.n_err, "
+            "n_skipped=EXCLUDED.n_skipped, "
+            # COALESCE, not overwrite: a finish that carries no extra must never erase what
+            # run_start recorded (dbt's bronze_watermark lands at start).
+            "extra=COALESCE(EXCLUDED.extra, control.run.extra) "  # @spec run_extra_preserved_on_finish
+            "WHERE control.run.extractor = EXCLUDED.extractor",
+            (rec["run_id"], rec["extractor"], rec.get("started_at", rec["finished_at"]),
+             rec["finished_at"], rec["status"], rec.get("stop_reason"),
+             rec.get("n_ok"), rec.get("n_err"), rec.get("n_skipped"),
+             Json(fin_extra) if fin_extra is not None else None),
+        )
+        if cur.rowcount == 0:
+            _refuse_other_extractors_run(cur, rec)
+    elif kind == "event":
+        extra = rec.get("extra")
+        cur.execute(
+            "INSERT INTO control.event "
+            "(extractor, run_id, seq, ran_at, entity, unit, business_date, status, "
+            " http_status, rows, source_attempts, sink_attempts, pull_ms, load_ms, error_detail, extra) "
+            "VALUES (%s,%s,%s,%s::timestamptz,%s,%s,%s::date,%s,%s,%s,%s,%s,%s,%s,%s,%s) "
+            "ON CONFLICT (extractor, run_id, seq) DO NOTHING",
+            (rec["extractor"], rec["run_id"], rec["seq"], rec["ran_at"], rec.get("entity"),
+             rec.get("unit"), rec.get("business_date"), rec["status"], rec.get("http_status"),
+             rec.get("rows"), rec.get("source_attempts"), rec.get("sink_attempts"),
+             rec.get("pull_ms"), rec.get("load_ms"), rec.get("error_detail"),
+             Json(extra) if extra is not None else None),
+        )
+    elif kind == "budget":
+        cur.execute(
+            "INSERT INTO control.budget_usage "
+            "(run_id, extractor, budget_name, limit_val, used, remaining, hit, ran_at) "
+            "VALUES (%s,%s,%s,%s,%s,%s,%s,%s::timestamptz) "
+            "ON CONFLICT (run_id, extractor, budget_name) DO UPDATE SET "  # @spec budget_recorded_every_run
+            "limit_val=EXCLUDED.limit_val, used=EXCLUDED.used, remaining=EXCLUDED.remaining, "
+            "hit=EXCLUDED.hit, ran_at=EXCLUDED.ran_at",
+            (rec["run_id"], rec["extractor"], rec["budget_name"], rec.get("limit_val"),
+             rec.get("used"), rec.get("remaining"), rec.get("hit"), rec.get("ran_at")),
+        )
+    else:
+        raise ValueError(f"unknown WAL record kind: {kind!r}")
+
+
+def apply_events_batch(cur: psycopg.Cursor, recs: list[dict]) -> None:
+    """Batch form of apply() for a homogeneous list of kind='event' records.
+
+    Same SQL and same idempotency key as the per-record path -- executemany just sends it in
+    one pipelined batch instead of one round-trip per row. Added for #2640: dbt's run_results
+    projection can carry tens of thousands of nodes in a catch-up window, and at ~250ms RTT over
+    the public proxy the per-record path turns 28k rows into two hours. executemany does it in
+    seconds, and on Railway's private network the difference is merely large.
+    """
+    if not recs:
+        return
+    bad = [r["kind"] for r in recs if r.get("kind") != "event"]
+    if bad:
+        raise ValueError(f"apply_events_batch only takes kind='event' records, got {bad[:3]}")
+    cur.executemany(
+        "INSERT INTO control.event "
+        "(extractor, run_id, seq, ran_at, entity, unit, business_date, status, "
+        " http_status, rows, source_attempts, sink_attempts, pull_ms, load_ms, error_detail, extra) "
+        "VALUES (%s,%s,%s,%s::timestamptz,%s,%s,%s::date,%s,%s,%s,%s,%s,%s,%s,%s,%s) "
+        "ON CONFLICT (extractor, run_id, seq) DO NOTHING",
+        [(r["extractor"], r["run_id"], r["seq"], r["ran_at"], r.get("entity"),
+          r.get("unit"), r.get("business_date"), r["status"], r.get("http_status"),
+          r.get("rows"), r.get("source_attempts"), r.get("sink_attempts"),
+          r.get("pull_ms"), r.get("load_ms"), r.get("error_detail"),
+          Json(r["extra"]) if r.get("extra") is not None else None)
+         for r in recs],
+    )
+
+
+# --- the liveness lease (the deterministic death-detection primitive) ---
+
+def lease_acquire(conn: psycopg.Connection, extractor: str, run_id: str, ttl_s: int) -> bool:
+    """Acquire this extractor's lease. Steal-only-if-expired: the ON CONFLICT UPDATE fires
+    *only* when the existing lease has lapsed, so a genuinely-live run is never displaced.
+    Returns True if acquired (no live prior run), False if another run holds a fresh lease.
+    """
+    with conn.cursor() as cur:
+        cur.execute(
+            "INSERT INTO control.lease (extractor, run_id, lease_expires_at) "
+            "VALUES (%s, %s, now() + make_interval(secs => %s)) "
+            "ON CONFLICT (extractor) DO UPDATE SET "
+            "run_id = EXCLUDED.run_id, lease_expires_at = EXCLUDED.lease_expires_at "
+            "WHERE control.lease.lease_expires_at < now() "  # @spec lease_steal_only_if_expired
+            "RETURNING run_id",
+            (extractor, run_id, ttl_s),
+        )
+        acquired = cur.fetchone() is not None
+    conn.commit()
+    return acquired
+
+
+def lease_renew(conn: psycopg.Connection, extractor: str, run_id: str, ttl_s: int) -> bool:
+    """Push the lease forward (renewer thread). Returns False if we no longer own it."""
+    with conn.cursor() as cur:
+        cur.execute(
+            "UPDATE control.lease SET lease_expires_at = now() + make_interval(secs => %s) "
+            "WHERE extractor = %s AND run_id = %s",
+            (ttl_s, extractor, run_id),
+        )
+        ok = cur.rowcount > 0
+    conn.commit()
+    return ok
+
+
+def lease_release(conn: psycopg.Connection, extractor: str, run_id: str) -> None:
+    """Expire our lease immediately so the next run can reclaim without waiting out the TTL."""
+    with conn.cursor() as cur:
+        cur.execute(
+            "UPDATE control.lease SET lease_expires_at = now() "
+            "WHERE extractor = %s AND run_id = %s",
+            (extractor, run_id),
+        )
+    conn.commit()
+
+
+def reconcile_dead(conn: psycopg.Connection, extractor: str, run_id: str) -> int:
+    """Mark every unfinished prior run of this extractor as `died`. Safe to call only after
+    we hold the lease (⇒ no other run of this extractor is alive). The dead run's cause is
+    already in control.event via WAL replay. Returns the number reconciled."""
+    with conn.cursor() as cur:
+        cur.execute(
+            "UPDATE control.run SET status = 'died', stop_reason = 'died', "
+            "finished_at = now(), reconciled_by = %s "
+            "WHERE extractor = %s AND finished_at IS NULL AND run_id <> %s",  # @spec unfinished_prior_runs_marked_died
+            (run_id, extractor, run_id),
+        )
+        n = cur.rowcount
+    conn.commit()
+    return n

control_plane/wal.py

@@ -0,0 +1,63 @@
+"""Local write-ahead log: the durability boundary (issue #204).
+
+Every control record is appended here as one line of JSON, fsync'd, *before* any network
+write to Postgres. On a crash the tail survives on disk (box: persistent; Railway: the
+in-flight buffer), and the next run's recovery ships it. This already beats GoFrugal's old
+"buffer events in memory, flush at end-of-date" — a mid-date death now loses nothing.
+
+The file is `O_APPEND`, so concurrent appends never interleave a partial line, and a reader
+that stops at the last newline never sees a half-written record (the durability invariant
+recovery relies on).
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import json
+import os
+
+
+def _default(o):
+    if isinstance(o, (_dt.datetime, _dt.date)):
+        return o.isoformat()
+    raise TypeError(f"not JSON-serialisable for the WAL: {type(o).__name__}")
+
+
+class Wal:
+    """Append-only newline-JSON log for one run."""
+
+    def __init__(self, path: str):
+        self.path = path
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+
+    def append(self, record: dict) -> None:
+        """Serialise + fsync one record. Durable on return."""
+        line = (json.dumps(record, default=_default, separators=(",", ":")) + "\n").encode("utf-8")
+        fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644)
+        try:
+            os.write(fd, line)
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+
+
+def read_new(path: str, start_offset: int = 0) -> tuple[list[dict], int]:
+    """Parse complete (newline-terminated) records from `start_offset` onward.
+
+    Returns (records, new_offset). A partial trailing line (no newline yet — a crash
+    mid-write) is left unconsumed, so new_offset never advances past durable data.
+    """
+    if not os.path.exists(path):
+        return [], start_offset
+    with open(path, "rb") as f:
+        f.seek(start_offset)
+        data = f.read()
+    records: list[dict] = []
+    consumed = 0
+    for line in data.splitlines(keepends=True):
+        if not line.endswith(b"\n"):  # @spec wal_torn_tail_not_consumed
+            break  # partial trailing line — not durable, stop here
+        consumed += len(line)
+        s = line.strip()
+        if s:
+            records.append(json.loads(s))
+    return records, start_offset + consumed

feather_control_plane-0.1.0.dist-info/METADATA

@@ -0,0 +1,107 @@
+Metadata-Version: 2.5
+Name: feather-control-plane
+Version: 0.1.0
+Summary: Durable failure observability for an extractor fleet: WAL + Postgres control plane + liveness lease
+Project-URL: Homepage, https://github.com/siraj-samsudeen/feather-flow
+Project-URL: Repository, https://github.com/siraj-samsudeen/feather-flow
+Project-URL: Issues, https://github.com/siraj-samsudeen/feather-flow/issues
+Author-email: Siraj Samsudeen <mailsiraj@gmail.com>
+License-Expression: MIT
+Keywords: control-plane,etl,observability,postgres,write-ahead-log
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: System :: Monitoring
+Requires-Python: >=3.12
+Requires-Dist: tzdata>=2026.1
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.1; extra == 'postgres'
+Description-Content-Type: text/markdown
+
+# feather-control-plane
+
+Durable failure observability for an extractor fleet.
+
+A long-running extractor that dies mid-load leaves nothing behind: no row, no error, no
+"it got as far as table X". This library gives it three things that survive the death —
+
+- a **write-ahead log** on local disk, fsynced before anything is shipped, so a crash leaves
+  the breadcrumbs of how far the run got;
+- a **Postgres control plane** holding run lifecycle, per-unit events and budget telemetry,
+  shipped as the run goes rather than written at the end;
+- a **liveness lease**, renewed by a background thread, so a process that stops renewing is
+  known to be dead instead of being assumed still running.
+
+The data plane is somewhere else entirely — this records what happened to the load, not the
+load's data.
+
+## Install
+
+```bash
+pip install "feather-control-plane[postgres]"    # the full facade
+pip install feather-control-plane                # pure-stdlib helpers only
+```
+
+**`psycopg` is an optional extra, and that is deliberate.** Two of the modules here —
+`control_plane.business_day` and `control_plane.status` — are pure stdlib, and consumers import
+them on their own, with no database anywhere in the picture. Making a database driver mandatory
+for a date helper is what drove nine services to copy `business_day.py` around by hand instead
+of depending on the library. Anything that actually opens a connection needs the `postgres`
+extra; `__init__.py` resolves those names lazily (PEP 562), so importing the stdlib half never
+pulls the driver in.
+
+## Import name
+
+The distribution is `feather-control-plane`; the **import name is `control_plane`**:
+
+```python
+from control_plane import ControlPlane, status
+from control_plane.business_day import ist_today
+```
+
+They differ on purpose. The nine services adopting this already have ~40 call sites spelled
+`from control_plane import ...`, so keeping the import name turns adoption into a dependency
+substitution with no source edits.
+
+## Usage
+
+```python
+from control_plane import ControlPlane, status
+
+cp = ControlPlane("gofrugal", dsn=PG_DSN, wal_dir="control_wal")
+with cp.run("nightly", run_id=run_id) as run:
+    run.event(entity="sales_item_wise", unit="1001", business_date=d,
+              status=status.EV_OK, http_status=200, rows=42,
+              source_attempts=1, sink_attempts=1, pull_ms=1200, load_ms=800)
+    run.budget(status.BUDGET_WALLTIME, limit=1800, used=1800, remaining=0, hit=True)
+    run.finish(status.RUN_PARTIAL, stop_reason=status.STOP_BUDGET_TIME,
+               n_ok=47, n_err=1, n_skipped=0)
+```
+
+`dsn=None` is the explicit no-control-plane mode for dev and test: the WAL still records
+everything, nothing ships. With a DSN configured, Postgres is a **hard dependency of the run** —
+if it is unreachable at startup the run aborts rather than loading blind, because a load with no
+provenance is worse than no load.
+
+## Tests
+
+```bash
+cd packages/feather-control-plane && uv run --package feather-control-plane pytest
+# or, from the repo root, naming the path:
+uv run --package feather-control-plane pytest packages/feather-control-plane/tests
+```
+
+`--package` picks the *environment*; it does not pick the tests. pytest takes its config from the
+nearest `pyproject.toml` above the working directory, so a bare `pytest` run from the repo root
+collects feather-flow's suite no matter what `--package` says — green, 1050 tests, and not one of
+them this package's. Either `cd` in or name the path.
+
+The pure-logic tests (WAL framing, status vocabulary, business day) run anywhere. The rest need a
+real Postgres and find one in this order: `CONTROL_PG_DSN` if set → a `postgres:16` Docker
+container → a local Postgres binary (`pg_ctl` on `PATH`, else Homebrew's keg-only
+`postgresql@N`), `initdb`'d into a temp directory on a throwaway port and torn down afterwards.
+With none of the three available they skip.

feather_control_plane-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+control_plane/__init__.py,sha256=eWZer0yYYWCZNL5gsO2T2sF5zvmHVl_ZRqjkamSr91s,3075
+control_plane/business_day.py,sha256=IVH78mmBINa9jZC29AJTr-sGQ8ycAed_BTne7hIzXuI,2325
+control_plane/recovery.py,sha256=iNkgroXBgHGGnFRBRkJA23c7T5CVguQaxpoHnOWmiqQ,1472
+control_plane/run.py,sha256=vLVReod1OmJWJ9KD53Eo12sdpfQFrbvrB0q8QBCZwIM,11336
+control_plane/shipper.py,sha256=Wq3VGRjskqC_7ao2LguuV_9wnApNmQLL8a2EKfTV4Hk,1750
+control_plane/status.py,sha256=FWQjWDcNKSb-3E7PD3sv68Mx3UZ_t-TIsrHfGFs5iHQ,3649
+control_plane/store.py,sha256=U_czsnV7yQOiMiMKHanTTmc9B0JmADoH7D0XbC9HxT4,23794
+control_plane/wal.py,sha256=gvlYg_v5Ed7N0XkFYwVRxUunIlFgf4UmmWRVeGSJKYc,2341
+feather_control_plane-0.1.0.dist-info/METADATA,sha256=o8hInhfCKAu-nhX_QLScU-zMnPG9g4o7Xk8z_IJZir0,4978
+feather_control_plane-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+feather_control_plane-0.1.0.dist-info/RECORD,,

feather_control_plane-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any