duckstring 0.3.0__tar.gz → 0.4.0__tar.gz

{duckstring-0.3.0/src/duckstring.egg-info → duckstring-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckstring
-Version: 0.3.0
+Version: 0.4.0
 Summary: Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically.
 Author-email: Duckstring <dev@duckstring.com>
 License:                                  Apache License

{duckstring-0.3.0 → duckstring-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "duckstring"
-version = "0.3.0"
+version = "0.4.0"
 description = "Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically."
 readme = "README.md"
 requires-python = ">=3.10"

duckstring-0.4.0/src/duckstring/alerts/__init__.py

@@ -0,0 +1,21 @@
+"""Alerts — failure & freshness notifications to external channels. See plans/alerts.md.
+
+The observability sibling of a Spout: operational config + a scheme-selected notifier seam +
+``${env:}``/``${secret:}`` credentials + an async delivery worker that never cascades a failure back into
+the engine. Alerting *observes* the state the engine already computes; it adds no orchestration state.
+"""
+
+from __future__ import annotations
+
+from .base import Notifier, NotifierError, get_notifier, parse_notifier_destination
+from .event import KNOWN_EVENTS, AlertEvent, normalise_events
+
+__all__ = [
+    "AlertEvent",
+    "KNOWN_EVENTS",
+    "Notifier",
+    "NotifierError",
+    "get_notifier",
+    "normalise_events",
+    "parse_notifier_destination",
+]

duckstring-0.4.0/src/duckstring/alerts/base.py

@@ -0,0 +1,94 @@
+"""The notifier seam (see plans/alerts.md) — mirrors :mod:`duckstring.egress.base`.
+
+A small, scheme-selected interface the alert worker delivers an :class:`AlertEvent` through. The channel
+destination is a URI whose scheme picks the notifier; ``get_notifier(destination)`` resolves it, exactly
+like ``get_egress``. Credentials travel inside the URI as ``${env:NAME}``/``${secret:NAME}`` references and
+are resolved only at send time (:mod:`duckstring.egress.credentials`) — never persisted or logged resolved.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Protocol, runtime_checkable
+from urllib.parse import urlparse
+
+from ..egress import credentials
+from .event import AlertEvent
+
+# Schemes a channel may target. http/https → a webhook (Slack-incoming-webhook compatible); mailto → SMTP.
+WEBHOOK_SCHEMES = {"http", "https"}
+EMAIL_SCHEMES = {"mailto"}
+KNOWN_SCHEMES = WEBHOOK_SCHEMES | EMAIL_SCHEMES
+
+
+class NotifierError(ValueError):
+    """An invalid channel destination, or a delivery/connectivity failure (sanitised — never a credential)."""
+
+
+@dataclass(frozen=True)
+class Destination:
+    scheme: str
+    raw: str  # the original URI, with ${...} references intact (resolved only at send time)
+
+
+def parse_notifier_destination(uri: str) -> Destination:
+    """Validate a channel destination URI: a known scheme + well-formed ``${...}`` references. Does **not**
+    resolve credentials (so a channel can be created before its secrets are present). Raises NotifierError."""
+    if not uri or not uri.strip():
+        raise NotifierError("destination must not be empty")
+    try:
+        credentials.references(uri)  # validates ${env:}/${secret:} syntax
+    except credentials.CredentialError as exc:
+        raise NotifierError(str(exc)) from exc
+    scheme = urlparse(uri).scheme.lower()
+    if not scheme:
+        raise NotifierError(f"destination {uri!r} has no scheme — expected e.g. https://…, mailto:…")
+    if scheme not in KNOWN_SCHEMES:
+        raise NotifierError(
+            f"unsupported alert destination scheme {scheme!r} — supported: {', '.join(sorted(KNOWN_SCHEMES))}"
+        )
+    return Destination(scheme=scheme, raw=uri)
+
+
+@runtime_checkable
+class Notifier(Protocol):
+    def send(self, event: AlertEvent) -> None:
+        """Deliver ``event`` to the destination. Raises :class:`NotifierError` (sanitised) on failure."""
+        ...
+
+    def test(self) -> None:
+        """Probe connectivity/credentials without delivering a real alert (the ``alert test`` command).
+        Returns on success; raises :class:`NotifierError` (sanitised) on failure."""
+        ...
+
+
+_REGISTRY: dict[str, Callable[[Destination], Notifier]] = {}
+
+
+def register(scheme: str, factory: Callable[[Destination], Notifier]) -> None:
+    _REGISTRY[scheme] = factory
+
+
+def get_notifier(destination: str) -> Notifier:
+    """Resolve the notifier for a channel destination by its scheme. Raises :class:`NotifierError` for an
+    unknown scheme, or a known scheme whose notifier is not built."""
+    dest = parse_notifier_destination(destination)
+    factory = _REGISTRY.get(dest.scheme)
+    if factory is None:
+        raise NotifierError(
+            f"notifier for scheme {dest.scheme!r} is not implemented yet (built: "
+            f"{', '.join(sorted(_REGISTRY)) or 'none'})"
+        )
+    return factory(dest)
+
+
+def _register_builtins() -> None:
+    from .email import EmailNotifier
+    from .webhook import WebhookNotifier
+
+    for driver in (WebhookNotifier, EmailNotifier):
+        for scheme in driver.SCHEMES:
+            register(scheme, driver)
+
+
+_register_builtins()

duckstring-0.4.0/src/duckstring/alerts/email.py

@@ -0,0 +1,106 @@
+"""The email notifier (``mailto:``) — the simplest floor.
+
+``mailto:ops@x.com,dev@x.com?smtp=host:587&from=alerts@x.com&user=${env:SMTP_USER}&password=${secret:SMTP_PASS}&tls=1``
+
+SMTP host/port/user/password/from come from the URI query, or the ``DUCKSTRING_SMTP_*`` environment as a
+fallback (so a Catchment can carry one SMTP config for every mailto channel). Credentials are resolved from
+``${env:}``/``${secret:}`` only at send time. Uses the stdlib ``smtplib`` — no new dependency.
+"""
+
+from __future__ import annotations
+
+import os
+from email.message import EmailMessage
+from urllib.parse import parse_qs, unquote, urlparse
+
+from .base import Destination, NotifierError
+from .event import AlertEvent
+
+_TIMEOUT = 20.0
+
+
+class EmailNotifier:
+    SCHEMES = ("mailto",)
+
+    def __init__(self, dest: Destination):
+        parsed = urlparse(dest.raw)
+        self.recipients = [r.strip() for r in unquote(parsed.path).split(",") if r.strip()]
+        if not self.recipients:
+            raise NotifierError("mailto: destination has no recipient — use mailto:you@example.com")
+        q = {k: v[0] for k, v in parse_qs(parsed.query).items()}
+        self._raw = q  # references intact; resolved at send time
+        self.smtp = q.get("smtp") or os.environ.get("DUCKSTRING_SMTP_HOST", "")
+        self.sender = q.get("from") or os.environ.get("DUCKSTRING_SMTP_FROM", "duckstring@localhost")
+        tls = q.get("tls")
+        self.tls = (tls not in ("0", "false", "no")) if tls is not None else \
+            (os.environ.get("DUCKSTRING_SMTP_TLS", "1") not in ("0", "false", "no"))
+        if not self.smtp:
+            raise NotifierError(
+                "mailto: destination needs an SMTP server — add ?smtp=host:port or set DUCKSTRING_SMTP_HOST"
+            )
+
+    def _host_port(self) -> tuple[str, int]:
+        host, _, port = self.smtp.partition(":")
+        return host, int(port) if port else 587
+
+    def _credentials(self) -> tuple[str | None, str | None]:
+        from ..egress import credentials
+
+        user = self._raw.get("user") or os.environ.get("DUCKSTRING_SMTP_USER")
+        password = self._raw.get("password") or os.environ.get("DUCKSTRING_SMTP_PASSWORD")
+        # Resolve ${env:}/${secret:} refs at call time — never persisted/logged resolved.
+        user = credentials.resolve(user) if user else None
+        password = credentials.resolve(password) if password else None
+        return user, password
+
+    def _connect(self):
+        import smtplib
+
+        host, port = self._host_port()
+        server = smtplib.SMTP(host, port, timeout=_TIMEOUT)
+        try:
+            server.ehlo()
+            if self.tls:
+                server.starttls()
+                server.ehlo()
+            user, password = self._credentials()
+            if user and password:
+                server.login(user, password)
+        except Exception:
+            server.close()
+            raise
+        return server
+
+    def send(self, event: AlertEvent) -> None:
+        msg = EmailMessage()
+        msg["Subject"] = event.summary()
+        msg["From"] = self.sender
+        msg["To"] = ", ".join(self.recipients)
+        lines = [event.message]
+        if event.pond:
+            lines.append(f"\nPond: {event.pond}")
+        if event.f:
+            lines.append(f"Freshness: {event.f}")
+        if event.detail:
+            for k, v in event.detail.items():
+                lines.append(f"{k}: {v}")
+        msg.set_content("\n".join(lines))
+        try:
+            server = self._connect()
+            try:
+                server.send_message(msg)
+            finally:
+                server.quit()
+        except NotifierError:
+            raise
+        except Exception as exc:  # noqa: BLE001 — sanitise: type only, never the credential/host detail
+            raise NotifierError(f"email delivery failed: {type(exc).__name__}") from None
+
+    def test(self) -> None:
+        try:
+            server = self._connect()  # connect + STARTTLS + login, deliver nothing
+            server.quit()
+        except NotifierError:
+            raise
+        except Exception as exc:  # noqa: BLE001
+            raise NotifierError(f"SMTP connection failed: {type(exc).__name__}") from None

duckstring-0.4.0/src/duckstring/alerts/event.py

@@ -0,0 +1,97 @@
+"""The alert event model + the event-kind vocabulary (see plans/alerts.md).
+
+An :class:`AlertEvent` is the rendered, **sanitised** payload a notifier delivers. It reuses what
+``/api/runs`` surfaces (error message, freshness, pond) but never a raw traceback — a channel destination
+can be third-party, and a traceback can leak paths/connection strings (the same concern behind the API's
+``_redact_tracebacks``). Keep it JSON-serialisable: it is stored verbatim in the ``alert_delivery`` outbox.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+# The event vocabulary. `freshness` is tick-driven (the SLA sweep); the rest are transition-driven.
+KNOWN_EVENTS = ("failure", "contract", "spout", "recovery", "freshness")
+
+# Default severity per kind (a channel filters by kind, not severity — this is only for the payload/label).
+SEVERITY = {
+    "failure": "error",
+    "contract": "error",
+    "spout": "error",
+    "freshness": "warning",
+    "recovery": "info",
+}
+
+
+def normalise_events(events: str | None) -> tuple[str, ...]:
+    """Parse a channel's ``events`` CSV (or ``all``/empty) into a validated tuple of known kinds.
+
+    ``all`` / empty → every kind. Raises :class:`ValueError` naming an unknown kind."""
+    if not events or events.strip().lower() == "all":
+        return KNOWN_EVENTS
+    out = []
+    for raw in events.split(","):
+        kind = raw.strip().lower()
+        if not kind:
+            continue
+        if kind not in KNOWN_EVENTS:
+            raise ValueError(f"unknown alert event {kind!r} — choose from {', '.join(KNOWN_EVENTS)} (or 'all')")
+        out.append(kind)
+    if not out:
+        return KNOWN_EVENTS
+    return tuple(dict.fromkeys(out))  # de-duplicated, order preserved
+
+
+@dataclass
+class AlertEvent:
+    """A rendered notification. ``detail`` carries kind-specific extras (e.g. the blocked-downstream blast
+    radius for a failure); never put a traceback in it."""
+
+    kind: str
+    pond: str | None
+    title: str
+    message: str
+    severity: str = ""
+    f: str | None = None
+    catchment: str | None = None
+    ts: str = ""
+    detail: dict = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.severity:
+            self.severity = SEVERITY.get(self.kind, "info")
+        if not self.ts:
+            self.ts = datetime.now(timezone.utc).isoformat()
+
+    def summary(self) -> str:
+        """A one-line human summary (the webhook ``text`` / the email subject line)."""
+        where = f" [{self.catchment}]" if self.catchment else ""
+        return f"{self.severity.upper()}{where}: {self.title}"
+
+    def to_payload(self) -> dict:
+        return {
+            "kind": self.kind,
+            "severity": self.severity,
+            "pond": self.pond,
+            "f": self.f,
+            "title": self.title,
+            "message": self.message,
+            "catchment": self.catchment,
+            "ts": self.ts,
+            "detail": self.detail,
+        }
+
+    @classmethod
+    def from_payload(cls, payload: dict) -> "AlertEvent":
+        return cls(
+            kind=payload.get("kind", ""),
+            pond=payload.get("pond"),
+            title=payload.get("title", ""),
+            message=payload.get("message", ""),
+            severity=payload.get("severity", ""),
+            f=payload.get("f"),
+            catchment=payload.get("catchment"),
+            ts=payload.get("ts", ""),
+            detail=payload.get("detail") or {},
+        )

duckstring-0.4.0/src/duckstring/alerts/webhook.py

@@ -0,0 +1,46 @@
+"""The webhook notifier (``http``/``https``) — the highest-leverage channel.
+
+POSTs a JSON body that is both a plain structured event **and** Slack-incoming-webhook compatible: a
+top-level ``text`` summary (which Slack renders, and a generic receiver can read) plus the full structured
+event. So one driver covers Slack, a generic webhook receiver, and (via a proxy) PagerDuty's Events API.
+Any credential/token in the URL is a ``${env:}``/``${secret:}`` reference, resolved only at send time.
+"""
+
+from __future__ import annotations
+
+from .base import Destination, NotifierError
+from .event import AlertEvent
+
+_TIMEOUT = 15.0
+
+
+class WebhookNotifier:
+    SCHEMES = ("http", "https")
+
+    def __init__(self, dest: Destination):
+        self.dest = dest
+
+    def _post(self, event: AlertEvent) -> None:
+        import httpx
+
+        from ..egress import credentials
+
+        url = credentials.resolve(self.dest.raw)  # resolve any ${env:}/${secret:} token — never logged
+        body = {"text": event.summary(), **event.to_payload()}  # `text` for Slack; the rest for generic receivers
+        try:
+            resp = httpx.post(url, json=body, timeout=_TIMEOUT)
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            # Sanitise: report status + reason, never the URL (it may carry a token in the path/query).
+            raise NotifierError(f"webhook returned {exc.response.status_code}") from None
+        except httpx.HTTPError as exc:
+            raise NotifierError(f"webhook delivery failed: {type(exc).__name__}") from None
+
+    def send(self, event: AlertEvent) -> None:
+        self._post(event)
+
+    def test(self) -> None:
+        self._post(AlertEvent(
+            kind="recovery", pond=None, title="Duckstring alert channel test",
+            message="This is a test notification — your alert channel is configured correctly.",
+        ))

duckstring-0.4.0/src/duckstring/catchment/alert_worker.py

@@ -0,0 +1,58 @@
+"""The alert worker — delivers queued notifications to their channels. See plans/alerts.md.
+
+One async task in the Catchment process (outbound I/O, the exact shape of the egress worker): each pass it
+drains the pending ``alert_delivery`` rows the engine enqueued (:meth:`Driver.take_alert_deliveries`),
+resolves each channel's notifier by scheme, ``send``s it in a threadpool with a per-send timeout, and marks
+the row sent (:meth:`Driver.mark_delivery_sent`) or bumps attempts / parks it failed at the cap
+(:meth:`Driver.mark_delivery_failed`).
+
+**A send failure never propagates into the engine** — it is recorded on the delivery row (auditable via
+``alert log``) and retried on the next tick until it succeeds or hits ``MAX_ATTEMPTS``. A permanently-broken
+channel therefore stops retrying but leaves a visible failed row, never a Pond failure.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from fastapi.concurrency import run_in_threadpool
+
+_RECONCILE_INTERVAL = 5.0  # self-healing tick — retries pending deliveries a previous send left behind
+_PER_SEND_TIMEOUT = 30.0   # ceiling on one delivery, so a slow channel can't starve the others
+MAX_ATTEMPTS = 6           # after this many failed sends, park the delivery 'failed' (stop retrying)
+
+
+def _deliver(destination: str, payload: dict) -> None:
+    """Send one notification (blocking — runs in the thread pool). Raises on any failure."""
+    from ..alerts import AlertEvent, get_notifier
+
+    notifier = get_notifier(destination)  # resolves the scheme's driver (+ validates the URI)
+    notifier.send(AlertEvent.from_payload(payload))
+
+
+async def _drain(driver) -> None:
+    for row in driver.take_alert_deliveries():
+        try:
+            await asyncio.wait_for(
+                run_in_threadpool(_deliver, row["destination"], row["payload"]),
+                timeout=_PER_SEND_TIMEOUT,
+            )
+        except Exception as exc:  # noqa: BLE001 — any delivery error is recorded, never raised into the engine
+            driver.mark_delivery_failed(row["id"], f"{type(exc).__name__}: {exc}", MAX_ATTEMPTS)
+        else:
+            driver.mark_delivery_sent(row["id"])
+
+
+async def run_alert_worker(driver, wake: asyncio.Event) -> None:
+    """Drain queued alert deliveries on each wake (a delivery was enqueued) or the reconcile tick.
+    Cancelled on shutdown."""
+    while True:
+        try:
+            await asyncio.wait_for(wake.wait(), timeout=_RECONCILE_INTERVAL)
+        except asyncio.TimeoutError:
+            pass  # periodic reconcile — retry anything still pending
+        wake.clear()
+        try:
+            await _drain(driver)
+        except Exception as exc:  # keep the loop alive
+            print(f"[catchment] alert worker error: {exc}", flush=True)

duckstring-0.4.0/src/duckstring/catchment/app.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import asyncio
+import os
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+
+from . import auth
+from .db import connect, ensure_identity, migrate
+from .driver import Driver
+from .launcher import NoopLauncher, SubprocessLauncher
+from .routes import router
+
+_STATIC_DIR = Path(__file__).parent / "static"
+
+
+async def _scheduler(driver: Driver) -> None:
+    """Drive clock processes (Tide deadlines, window boundaries, Wave-on-idle) at next_wake."""
+    while True:
+        nw = driver.next_wake()
+        now = datetime.now(timezone.utc)
+        delay = (nw - now).total_seconds() if nw else 1.0
+        await asyncio.sleep(max(0.05, min(delay, 5.0)))
+        try:
+            driver.scheduler_tick()
+        except Exception as exc:  # keep the loop alive
+            print(f"[catchment] scheduler error: {exc}", flush=True)
+
+
+@asynccontextmanager
+async def _lifespan(app: FastAPI):
+    base_url = app.state.base_url
+    if os.environ.get("DUCKSTRING_DISABLE_DUCKS"):
+        launcher = NoopLauncher()
+    else:
+        # Ducks dial back over the duck channel with the internal worker token (decoupled from the user
+        # keys, so rotating those never disrupts running Ducks).
+        # base_url None = unknown (the platform picked the bind address): the launcher defers spawns
+        # until the dial-back middleware learns the address from the first request.
+        launcher = SubprocessLauncher(
+            app.state.root, base_url, token=app.state.duck_token, data_root=app.state.data_root
+        )
+    driver = Driver(app.state.db, app.state.root, base_url, launcher, data_root=app.state.data_root)
+    app.state.driver = driver
+    app.state.launcher = launcher
+
+    # Restore: resume any Pond Runs that were in flight when the Catchment last stopped.
+    driver.resume_incomplete()
+
+    from .alert_worker import run_alert_worker
+    from .egress_worker import run_egress_worker
+    from .poller import run_poller
+
+    # The poller wakes immediately when a Draw acquires demand (so it solicits its upstream at once),
+    # instead of waiting for its next cycle. The driver signals across threads via the running loop.
+    wake = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    driver.set_notify(lambda: loop.call_soon_threadsafe(wake.set))
+
+    # The egress worker wakes when a Pond publishes (its Spouts may have work) or a Spout is resynced.
+    egress_wake = asyncio.Event()
+    driver.set_egress_notify(lambda: loop.call_soon_threadsafe(egress_wake.set))
+
+    # The alert worker wakes when a failure/freshness event enqueues a notification delivery.
+    alert_wake = asyncio.Event()
+    driver.set_alert_notify(lambda: loop.call_soon_threadsafe(alert_wake.set))
+
+    from .state_sync import checkpoint_full, run_checkpoint_worker
+
+    scheduler = asyncio.create_task(_scheduler(driver))
+    poller = asyncio.create_task(run_poller(driver, app.state.root, wake))
+    egress = asyncio.create_task(run_egress_worker(driver, app.state.root, egress_wake))
+    alerts = asyncio.create_task(run_alert_worker(driver, alert_wake))
+    # Tier-1 state backup: push a duck.db snapshot to DUCKSTRING_STATE_BACKUP_URI on an interval (no-op
+    # when unset). The Tier-2 warm bundle is flushed once below, after the Ducks are stopped (quiescent).
+    checkpointer = asyncio.create_task(
+        run_checkpoint_worker(app.state.root, app.state.state_backup, app.state.checkpoint_every)
+    )
+    # Renew the data-root writer lease so a live Catchment's ownership never lapses (external data root only).
+    tasks = [scheduler, poller, egress, alerts, checkpointer]
+    if app.state.data_lease is not None:
+        from .data_lease import run_lease_renewer
+
+        lease_store, owner_id = app.state.data_lease
+        tasks.append(asyncio.create_task(run_lease_renewer(lease_store, owner_id)))
+    try:
+        yield
+    finally:
+        for task in tasks:
+            task.cancel()
+            try:
+                await task
+            except asyncio.CancelledError:
+                pass
+        launcher.shutdown_all()
+        if app.state.data_lease is not None:
+            from .data_lease import release_lease
+
+            lease_store, owner_id = app.state.data_lease
+            try:
+                release_lease(lease_store, owner_id)
+            except Exception:  # pragma: no cover - releasing the lease must not break shutdown
+                pass
+        # Ducks are now stopped → the registries/ledgers are quiescent: flush the warm Tier-1+2 bundle so a
+        # scaled-to-zero restart comes back warm (and engine state survives even a hard next crash).
+        if app.state.state_backup:
+            from fastapi.concurrency import run_in_threadpool
+
+            await run_in_threadpool(checkpoint_full, app.state.root, app.state.state_backup)
+
+
+def create_app(
+    root: Path, api_key: str | None = None, base_url: str | None = None, name: str | None = None,
+    *, data_root: str | None = None, state_backup: str | None = None, checkpoint_every: str | None = None,
+) -> FastAPI:
+    # Data plane location (object store / Volume / path) and the Tier-1 state-backup target. Explicit
+    # arguments (the CLI passes the registration's values) win, else the platform-hosting env vars.
+    data_root = data_root or os.environ.get("DUCKSTRING_DATA_ROOT") or None
+    state_backup = state_backup or os.environ.get("DUCKSTRING_STATE_BACKUP_URI") or None
+    checkpoint_every = checkpoint_every or os.environ.get("DUCKSTRING_CHECKPOINT_INTERVAL") or "60s"
+
+    # Restore Tier-1 state (duck.db, ledgers) from the backup before opening the DB, if the state root is
+    # empty (a fresh/scaled-to-zero node) and a backup exists.
+    from .state_sync import restore_state_if_empty
+
+    restore_state_if_empty(root, state_backup)
+
+    root.mkdir(parents=True, exist_ok=True)
+    con = connect(root / "duck.db")
+    migrate(con)
+    ensure_identity(con, name or os.environ.get("DUCKSTRING_CATCHMENT_NAME"))
+
+    # Writer lease on an external data root — refuse to start if a *different* live Catchment owns it (two
+    # Catchments racing one lake's Iceberg catalog would dangle its pointer). A same-id restart reclaims
+    # instantly; only engaged for an external DUCKSTRING_DATA_ROOT, so the local default is untouched.
+    data_lease = None
+    if data_root:
+        from ..storage import get_storage
+        from .data_lease import acquire_lease
+
+        cid = con.execute("SELECT value FROM catchment_meta WHERE key = 'id'").fetchone()
+        owner_id = cid[0] if cid else "unknown"
+        lease_store = get_storage(data_root)
+        acquire_lease(lease_store, owner_id)  # raises LeaseConflict → the server does not start
+        data_lease = (lease_store, owner_id)
+
+    app = FastAPI(title="Duckstring Catchment", lifespan=_lifespan)
+    app.state.root = root
+    app.state.data_lease = data_lease
+    app.state.data_root = data_root
+    app.state.state_backup = state_backup
+    app.state.checkpoint_every = checkpoint_every
+    app.state.db = con
+    # Built-in single API key (legacy / bare self-hosting): explicit argument or the environment. It
+    # means full access. The tiered read/demand/full keys live in `catchment_key` (see auth.py); either
+    # may gate the API. With neither configured, the Catchment is fully open.
+    app.state.api_key = api_key or os.environ.get("DUCKSTRING_API_KEY") or None
+    # The internal token Ducks present on the duck channel (persisted, decoupled from the user keys).
+    app.state.duck_token = auth.ensure_duck_token(con)
+    # The write-only secret store (at the root, archive-excluded). Injected into the egress credential
+    # resolver so a ${secret:NAME} reference resolves at egress time.
+    from ..egress import credentials
+    from .secrets import SecretStore
+    app.state.secret_store = SecretStore(root)
+    credentials.set_secret_provider(app.state.secret_store.get)
+    # The address Ducks dial back to: explicit argument (the CLI passes its bind address), or the
+    # environment, or None — unknown, because the host platform picks the bind address (e.g. Posit
+    # Connect). When None it is learned from the first request's ASGI scope below.
+    app.state.base_url = base_url or os.environ.get("DUCKSTRING_CATCHMENT_URL") or None
+
+    @app.middleware("http")
+    async def _learn_dialback_address(request, call_next):
+        launcher = getattr(app.state, "launcher", None)
+        if launcher is not None and getattr(launcher, "base_url", "") is None:
+            server = request.scope.get("server")  # the server's bound (host, port) per the ASGI spec
+            if server and server[1]:  # a unix socket has port None — nothing TCP to dial
+                host = "127.0.0.1" if server[0] in ("0.0.0.0", "::") else server[0]
+                url = f"http://{host}:{server[1]}"
+                app.state.base_url = url
+                app.state.driver.base_url = url
+                launcher.set_base_url(url)  # spawns any Ducks that were waiting on the address
+        return await call_next(request)
+
+    app.include_router(router, prefix="/api")
+    auth.audit_routes(app)  # fail-closed: every /api route must declare an access level
+
+    # Prometheus scrape endpoint at the ROOT (unauthenticated, the exporter convention) — mounted before
+    # the static "/" catch-all so it resolves, and outside "/api" so the access-level audit doesn't cover it.
+    from .routes.metrics import router as metrics_router
+    app.include_router(metrics_router)
+
+    if _STATIC_DIR.exists():
+        app.mount("/", StaticFiles(directory=_STATIC_DIR, html=True), name="frontend")
+
+    return app