duckstring 0.2.0__tar.gz → 0.4.0__tar.gz

duckstring-0.4.0/MANIFEST.in

@@ -0,0 +1,8 @@
+# The sdist ships the installable package only — keep internal design notes and the test suite
+# out of the PyPI source tarball (they're in the git repo for contributors).
+prune tests
+prune plans
+prune experiment
+prune docs
+prune frontend
+prune playground

{duckstring-0.2.0 → duckstring-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckstring
-Version: 0.2.0
+Version: 0.4.0
 Summary: Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically.
 Author-email: Duckstring <dev@duckstring.com>
 License:                                  Apache License
@@ -221,6 +221,10 @@ Requires-Dist: uvicorn[standard]>=0.29
 Requires-Dist: httpx>=0.25
 Requires-Dist: tomli>=2.0; python_version < "3.11"
 Requires-Dist: tomli-w>=1.0
+Requires-Dist: pyiceberg>=0.7
+Requires-Dist: pyarrow>=14
+Requires-Dist: pytz
+Provides-Extra: iceberg
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-timeout>=0.5; extra == "dev"
@@ -244,6 +248,7 @@ The main elements:
 - **Catchment**: Control environment (FastAPI + UI + CLI)
 - **Pond**: Versioned container with declared upstream dependencies
 - **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
+- **Trickle**: DBSP-based incremental engine operating within a Ripple
 
 Ponds are typed or referred to in context:
 
@@ -292,10 +297,6 @@ Ponds execute on demand signals sent to an Outlet, in two flavours — **push**
 
 A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
 
-## Scope & maturity
-
-Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
-
 ## Going further
 
 Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:

{duckstring-0.2.0 → duckstring-0.4.0}/README.md

@@ -14,6 +14,7 @@ The main elements:
 - **Catchment**: Control environment (FastAPI + UI + CLI)
 - **Pond**: Versioned container with declared upstream dependencies
 - **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
+- **Trickle**: DBSP-based incremental engine operating within a Ripple
 
 Ponds are typed or referred to in context:
 
@@ -62,10 +63,6 @@ Ponds execute on demand signals sent to an Outlet, in two flavours — **push**
 
 A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
 
-## Scope & maturity
-
-Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
-
 ## Going further
 
 Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:

{duckstring-0.2.0 → duckstring-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "duckstring"
-version = "0.2.0"
+version = "0.4.0"
 description = "Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -19,9 +19,22 @@ dependencies = [
   "httpx>=0.25",
   "tomli>=2.0; python_version < '3.11'",
   "tomli-w>=1.0",
+  # The default data plane: an Apache Iceberg base layer (snapshots + schema metadata over the
+  # Parquet files we already write). Core deps because Iceberg is default-on; set
+  # DUCKSTRING_DATA_PLANE=parquet to opt out (the lighter fallback, also for offline Catchments that
+  # can't fetch DuckDB's iceberg extension). NB: we use our own file-backed catalog
+  # (duckstring.iceberg_catalog) rather than pyiceberg's SqlCatalog, so SQLAlchemy is NOT pulled in.
+  "pyiceberg>=0.7",
+  "pyarrow>=14",
+  # DuckDB requires pytz to convert TIMESTAMPTZ columns to Python datetimes on fetch
+  # (our _duckstring_f freshness column is TIMESTAMPTZ, read back in dataplane/trickle export).
+  # It is otherwise only present transitively (pandas), so a clean install lacks it.
+  "pytz",
 ]
 
 [project.optional-dependencies]
+# Retained for back-compat: the Iceberg deps are in core now (Iceberg is the default plane).
+iceberg = []
 dev = [
   "pytest>=8.0",
   "pytest-timeout>=0.5",

{duckstring-0.2.0 → duckstring-0.4.0}/src/duckstring/__init__.py

@@ -6,7 +6,6 @@ from .core import (
     Pond,
     Puddle,
     Ripple,
-    Trickle,
     puddle,
     ripple,
 )
@@ -21,7 +20,6 @@ __all__ = [
     "Pond",
     "Puddle",
     "Ripple",
-    "Trickle",
     "puddle",
     "ripple",
     "__version__",

duckstring-0.4.0/src/duckstring/acc.py

@@ -0,0 +1,13 @@
+"""Backwards-compatible alias — scan (order-dependent) metric specs live in :mod:`duckstring.trickle.acc`.
+
+Kept so ``from duckstring import acc`` works alongside ``from duckstring import agg``. See
+:mod:`duckstring.trickle_io` for the rationale.
+"""
+
+from __future__ import annotations
+
+from .trickle import acc as _acc
+
+
+def __getattr__(name):  # PEP 562: forward all names to the relocated module
+    return getattr(_acc, name)

duckstring-0.4.0/src/duckstring/agg.py

@@ -0,0 +1,12 @@
+"""Backwards-compatible alias — aggregate metric specs now live in :mod:`duckstring.trickle.agg`.
+
+Kept so ``from duckstring import agg`` continues to work. See :mod:`duckstring.trickle_io` for the rationale.
+"""
+
+from __future__ import annotations
+
+from .trickle import agg as _agg
+
+
+def __getattr__(name):  # PEP 562: forward all names to the relocated module
+    return getattr(_agg, name)

duckstring-0.4.0/src/duckstring/alerts/__init__.py

@@ -0,0 +1,21 @@
+"""Alerts — failure & freshness notifications to external channels. See plans/alerts.md.
+
+The observability sibling of a Spout: operational config + a scheme-selected notifier seam +
+``${env:}``/``${secret:}`` credentials + an async delivery worker that never cascades a failure back into
+the engine. Alerting *observes* the state the engine already computes; it adds no orchestration state.
+"""
+
+from __future__ import annotations
+
+from .base import Notifier, NotifierError, get_notifier, parse_notifier_destination
+from .event import KNOWN_EVENTS, AlertEvent, normalise_events
+
+__all__ = [
+    "AlertEvent",
+    "KNOWN_EVENTS",
+    "Notifier",
+    "NotifierError",
+    "get_notifier",
+    "normalise_events",
+    "parse_notifier_destination",
+]

duckstring-0.4.0/src/duckstring/alerts/base.py

@@ -0,0 +1,94 @@
+"""The notifier seam (see plans/alerts.md) — mirrors :mod:`duckstring.egress.base`.
+
+A small, scheme-selected interface the alert worker delivers an :class:`AlertEvent` through. The channel
+destination is a URI whose scheme picks the notifier; ``get_notifier(destination)`` resolves it, exactly
+like ``get_egress``. Credentials travel inside the URI as ``${env:NAME}``/``${secret:NAME}`` references and
+are resolved only at send time (:mod:`duckstring.egress.credentials`) — never persisted or logged resolved.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Protocol, runtime_checkable
+from urllib.parse import urlparse
+
+from ..egress import credentials
+from .event import AlertEvent
+
+# Schemes a channel may target. http/https → a webhook (Slack-incoming-webhook compatible); mailto → SMTP.
+WEBHOOK_SCHEMES = {"http", "https"}
+EMAIL_SCHEMES = {"mailto"}
+KNOWN_SCHEMES = WEBHOOK_SCHEMES | EMAIL_SCHEMES
+
+
+class NotifierError(ValueError):
+    """An invalid channel destination, or a delivery/connectivity failure (sanitised — never a credential)."""
+
+
+@dataclass(frozen=True)
+class Destination:
+    scheme: str
+    raw: str  # the original URI, with ${...} references intact (resolved only at send time)
+
+
+def parse_notifier_destination(uri: str) -> Destination:
+    """Validate a channel destination URI: a known scheme + well-formed ``${...}`` references. Does **not**
+    resolve credentials (so a channel can be created before its secrets are present). Raises NotifierError."""
+    if not uri or not uri.strip():
+        raise NotifierError("destination must not be empty")
+    try:
+        credentials.references(uri)  # validates ${env:}/${secret:} syntax
+    except credentials.CredentialError as exc:
+        raise NotifierError(str(exc)) from exc
+    scheme = urlparse(uri).scheme.lower()
+    if not scheme:
+        raise NotifierError(f"destination {uri!r} has no scheme — expected e.g. https://…, mailto:…")
+    if scheme not in KNOWN_SCHEMES:
+        raise NotifierError(
+            f"unsupported alert destination scheme {scheme!r} — supported: {', '.join(sorted(KNOWN_SCHEMES))}"
+        )
+    return Destination(scheme=scheme, raw=uri)
+
+
+@runtime_checkable
+class Notifier(Protocol):
+    def send(self, event: AlertEvent) -> None:
+        """Deliver ``event`` to the destination. Raises :class:`NotifierError` (sanitised) on failure."""
+        ...
+
+    def test(self) -> None:
+        """Probe connectivity/credentials without delivering a real alert (the ``alert test`` command).
+        Returns on success; raises :class:`NotifierError` (sanitised) on failure."""
+        ...
+
+
+_REGISTRY: dict[str, Callable[[Destination], Notifier]] = {}
+
+
+def register(scheme: str, factory: Callable[[Destination], Notifier]) -> None:
+    _REGISTRY[scheme] = factory
+
+
+def get_notifier(destination: str) -> Notifier:
+    """Resolve the notifier for a channel destination by its scheme. Raises :class:`NotifierError` for an
+    unknown scheme, or a known scheme whose notifier is not built."""
+    dest = parse_notifier_destination(destination)
+    factory = _REGISTRY.get(dest.scheme)
+    if factory is None:
+        raise NotifierError(
+            f"notifier for scheme {dest.scheme!r} is not implemented yet (built: "
+            f"{', '.join(sorted(_REGISTRY)) or 'none'})"
+        )
+    return factory(dest)
+
+
+def _register_builtins() -> None:
+    from .email import EmailNotifier
+    from .webhook import WebhookNotifier
+
+    for driver in (WebhookNotifier, EmailNotifier):
+        for scheme in driver.SCHEMES:
+            register(scheme, driver)
+
+
+_register_builtins()

duckstring-0.4.0/src/duckstring/alerts/email.py

@@ -0,0 +1,106 @@
+"""The email notifier (``mailto:``) — the simplest floor.
+
+``mailto:ops@x.com,dev@x.com?smtp=host:587&from=alerts@x.com&user=${env:SMTP_USER}&password=${secret:SMTP_PASS}&tls=1``
+
+SMTP host/port/user/password/from come from the URI query, or the ``DUCKSTRING_SMTP_*`` environment as a
+fallback (so a Catchment can carry one SMTP config for every mailto channel). Credentials are resolved from
+``${env:}``/``${secret:}`` only at send time. Uses the stdlib ``smtplib`` — no new dependency.
+"""
+
+from __future__ import annotations
+
+import os
+from email.message import EmailMessage
+from urllib.parse import parse_qs, unquote, urlparse
+
+from .base import Destination, NotifierError
+from .event import AlertEvent
+
+_TIMEOUT = 20.0
+
+
+class EmailNotifier:
+    SCHEMES = ("mailto",)
+
+    def __init__(self, dest: Destination):
+        parsed = urlparse(dest.raw)
+        self.recipients = [r.strip() for r in unquote(parsed.path).split(",") if r.strip()]
+        if not self.recipients:
+            raise NotifierError("mailto: destination has no recipient — use mailto:you@example.com")
+        q = {k: v[0] for k, v in parse_qs(parsed.query).items()}
+        self._raw = q  # references intact; resolved at send time
+        self.smtp = q.get("smtp") or os.environ.get("DUCKSTRING_SMTP_HOST", "")
+        self.sender = q.get("from") or os.environ.get("DUCKSTRING_SMTP_FROM", "duckstring@localhost")
+        tls = q.get("tls")
+        self.tls = (tls not in ("0", "false", "no")) if tls is not None else \
+            (os.environ.get("DUCKSTRING_SMTP_TLS", "1") not in ("0", "false", "no"))
+        if not self.smtp:
+            raise NotifierError(
+                "mailto: destination needs an SMTP server — add ?smtp=host:port or set DUCKSTRING_SMTP_HOST"
+            )
+
+    def _host_port(self) -> tuple[str, int]:
+        host, _, port = self.smtp.partition(":")
+        return host, int(port) if port else 587
+
+    def _credentials(self) -> tuple[str | None, str | None]:
+        from ..egress import credentials
+
+        user = self._raw.get("user") or os.environ.get("DUCKSTRING_SMTP_USER")
+        password = self._raw.get("password") or os.environ.get("DUCKSTRING_SMTP_PASSWORD")
+        # Resolve ${env:}/${secret:} refs at call time — never persisted/logged resolved.
+        user = credentials.resolve(user) if user else None
+        password = credentials.resolve(password) if password else None
+        return user, password
+
+    def _connect(self):
+        import smtplib
+
+        host, port = self._host_port()
+        server = smtplib.SMTP(host, port, timeout=_TIMEOUT)
+        try:
+            server.ehlo()
+            if self.tls:
+                server.starttls()
+                server.ehlo()
+            user, password = self._credentials()
+            if user and password:
+                server.login(user, password)
+        except Exception:
+            server.close()
+            raise
+        return server
+
+    def send(self, event: AlertEvent) -> None:
+        msg = EmailMessage()
+        msg["Subject"] = event.summary()
+        msg["From"] = self.sender
+        msg["To"] = ", ".join(self.recipients)
+        lines = [event.message]
+        if event.pond:
+            lines.append(f"\nPond: {event.pond}")
+        if event.f:
+            lines.append(f"Freshness: {event.f}")
+        if event.detail:
+            for k, v in event.detail.items():
+                lines.append(f"{k}: {v}")
+        msg.set_content("\n".join(lines))
+        try:
+            server = self._connect()
+            try:
+                server.send_message(msg)
+            finally:
+                server.quit()
+        except NotifierError:
+            raise
+        except Exception as exc:  # noqa: BLE001 — sanitise: type only, never the credential/host detail
+            raise NotifierError(f"email delivery failed: {type(exc).__name__}") from None
+
+    def test(self) -> None:
+        try:
+            server = self._connect()  # connect + STARTTLS + login, deliver nothing
+            server.quit()
+        except NotifierError:
+            raise
+        except Exception as exc:  # noqa: BLE001
+            raise NotifierError(f"SMTP connection failed: {type(exc).__name__}") from None

duckstring-0.4.0/src/duckstring/alerts/event.py

@@ -0,0 +1,97 @@
+"""The alert event model + the event-kind vocabulary (see plans/alerts.md).
+
+An :class:`AlertEvent` is the rendered, **sanitised** payload a notifier delivers. It reuses what
+``/api/runs`` surfaces (error message, freshness, pond) but never a raw traceback — a channel destination
+can be third-party, and a traceback can leak paths/connection strings (the same concern behind the API's
+``_redact_tracebacks``). Keep it JSON-serialisable: it is stored verbatim in the ``alert_delivery`` outbox.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+# The event vocabulary. `freshness` is tick-driven (the SLA sweep); the rest are transition-driven.
+KNOWN_EVENTS = ("failure", "contract", "spout", "recovery", "freshness")
+
+# Default severity per kind (a channel filters by kind, not severity — this is only for the payload/label).
+SEVERITY = {
+    "failure": "error",
+    "contract": "error",
+    "spout": "error",
+    "freshness": "warning",
+    "recovery": "info",
+}
+
+
+def normalise_events(events: str | None) -> tuple[str, ...]:
+    """Parse a channel's ``events`` CSV (or ``all``/empty) into a validated tuple of known kinds.
+
+    ``all`` / empty → every kind. Raises :class:`ValueError` naming an unknown kind."""
+    if not events or events.strip().lower() == "all":
+        return KNOWN_EVENTS
+    out = []
+    for raw in events.split(","):
+        kind = raw.strip().lower()
+        if not kind:
+            continue
+        if kind not in KNOWN_EVENTS:
+            raise ValueError(f"unknown alert event {kind!r} — choose from {', '.join(KNOWN_EVENTS)} (or 'all')")
+        out.append(kind)
+    if not out:
+        return KNOWN_EVENTS
+    return tuple(dict.fromkeys(out))  # de-duplicated, order preserved
+
+
+@dataclass
+class AlertEvent:
+    """A rendered notification. ``detail`` carries kind-specific extras (e.g. the blocked-downstream blast
+    radius for a failure); never put a traceback in it."""
+
+    kind: str
+    pond: str | None
+    title: str
+    message: str
+    severity: str = ""
+    f: str | None = None
+    catchment: str | None = None
+    ts: str = ""
+    detail: dict = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.severity:
+            self.severity = SEVERITY.get(self.kind, "info")
+        if not self.ts:
+            self.ts = datetime.now(timezone.utc).isoformat()
+
+    def summary(self) -> str:
+        """A one-line human summary (the webhook ``text`` / the email subject line)."""
+        where = f" [{self.catchment}]" if self.catchment else ""
+        return f"{self.severity.upper()}{where}: {self.title}"
+
+    def to_payload(self) -> dict:
+        return {
+            "kind": self.kind,
+            "severity": self.severity,
+            "pond": self.pond,
+            "f": self.f,
+            "title": self.title,
+            "message": self.message,
+            "catchment": self.catchment,
+            "ts": self.ts,
+            "detail": self.detail,
+        }
+
+    @classmethod
+    def from_payload(cls, payload: dict) -> "AlertEvent":
+        return cls(
+            kind=payload.get("kind", ""),
+            pond=payload.get("pond"),
+            title=payload.get("title", ""),
+            message=payload.get("message", ""),
+            severity=payload.get("severity", ""),
+            f=payload.get("f"),
+            catchment=payload.get("catchment"),
+            ts=payload.get("ts", ""),
+            detail=payload.get("detail") or {},
+        )

duckstring-0.4.0/src/duckstring/alerts/webhook.py

@@ -0,0 +1,46 @@
+"""The webhook notifier (``http``/``https``) — the highest-leverage channel.
+
+POSTs a JSON body that is both a plain structured event **and** Slack-incoming-webhook compatible: a
+top-level ``text`` summary (which Slack renders, and a generic receiver can read) plus the full structured
+event. So one driver covers Slack, a generic webhook receiver, and (via a proxy) PagerDuty's Events API.
+Any credential/token in the URL is a ``${env:}``/``${secret:}`` reference, resolved only at send time.
+"""
+
+from __future__ import annotations
+
+from .base import Destination, NotifierError
+from .event import AlertEvent
+
+_TIMEOUT = 15.0
+
+
+class WebhookNotifier:
+    SCHEMES = ("http", "https")
+
+    def __init__(self, dest: Destination):
+        self.dest = dest
+
+    def _post(self, event: AlertEvent) -> None:
+        import httpx
+
+        from ..egress import credentials
+
+        url = credentials.resolve(self.dest.raw)  # resolve any ${env:}/${secret:} token — never logged
+        body = {"text": event.summary(), **event.to_payload()}  # `text` for Slack; the rest for generic receivers
+        try:
+            resp = httpx.post(url, json=body, timeout=_TIMEOUT)
+            resp.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            # Sanitise: report status + reason, never the URL (it may carry a token in the path/query).
+            raise NotifierError(f"webhook returned {exc.response.status_code}") from None
+        except httpx.HTTPError as exc:
+            raise NotifierError(f"webhook delivery failed: {type(exc).__name__}") from None
+
+    def send(self, event: AlertEvent) -> None:
+        self._post(event)
+
+    def test(self) -> None:
+        self._post(AlertEvent(
+            kind="recovery", pond=None, title="Duckstring alert channel test",
+            message="This is a test notification — your alert channel is configured correctly.",
+        ))

duckstring-0.4.0/src/duckstring/catchment/alert_worker.py

@@ -0,0 +1,58 @@
+"""The alert worker — delivers queued notifications to their channels. See plans/alerts.md.
+
+One async task in the Catchment process (outbound I/O, the exact shape of the egress worker): each pass it
+drains the pending ``alert_delivery`` rows the engine enqueued (:meth:`Driver.take_alert_deliveries`),
+resolves each channel's notifier by scheme, ``send``s it in a threadpool with a per-send timeout, and marks
+the row sent (:meth:`Driver.mark_delivery_sent`) or bumps attempts / parks it failed at the cap
+(:meth:`Driver.mark_delivery_failed`).
+
+**A send failure never propagates into the engine** — it is recorded on the delivery row (auditable via
+``alert log``) and retried on the next tick until it succeeds or hits ``MAX_ATTEMPTS``. A permanently-broken
+channel therefore stops retrying but leaves a visible failed row, never a Pond failure.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from fastapi.concurrency import run_in_threadpool
+
+_RECONCILE_INTERVAL = 5.0  # self-healing tick — retries pending deliveries a previous send left behind
+_PER_SEND_TIMEOUT = 30.0   # ceiling on one delivery, so a slow channel can't starve the others
+MAX_ATTEMPTS = 6           # after this many failed sends, park the delivery 'failed' (stop retrying)
+
+
+def _deliver(destination: str, payload: dict) -> None:
+    """Send one notification (blocking — runs in the thread pool). Raises on any failure."""
+    from ..alerts import AlertEvent, get_notifier
+
+    notifier = get_notifier(destination)  # resolves the scheme's driver (+ validates the URI)
+    notifier.send(AlertEvent.from_payload(payload))
+
+
+async def _drain(driver) -> None:
+    for row in driver.take_alert_deliveries():
+        try:
+            await asyncio.wait_for(
+                run_in_threadpool(_deliver, row["destination"], row["payload"]),
+                timeout=_PER_SEND_TIMEOUT,
+            )
+        except Exception as exc:  # noqa: BLE001 — any delivery error is recorded, never raised into the engine
+            driver.mark_delivery_failed(row["id"], f"{type(exc).__name__}: {exc}", MAX_ATTEMPTS)
+        else:
+            driver.mark_delivery_sent(row["id"])
+
+
+async def run_alert_worker(driver, wake: asyncio.Event) -> None:
+    """Drain queued alert deliveries on each wake (a delivery was enqueued) or the reconcile tick.
+    Cancelled on shutdown."""
+    while True:
+        try:
+            await asyncio.wait_for(wake.wait(), timeout=_RECONCILE_INTERVAL)
+        except asyncio.TimeoutError:
+            pass  # periodic reconcile — retry anything still pending
+        wake.clear()
+        try:
+            await _drain(driver)
+        except Exception as exc:  # keep the loop alive
+            print(f"[catchment] alert worker error: {exc}", flush=True)