interlaced 2.5.0__tar.gz → 2.7.0__tar.gz

{interlaced-2.5.0/src/interlaced.egg-info → interlaced-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.5.0
+Version: 2.7.0
 Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT
@@ -272,7 +272,7 @@ fast lane when possible). Contract: `docs/architecture/MULTI_ENGINE.md`.
   with column-level tracing, models, plan/apply with SQL diffs, live runs, query console,
   streams, checks, environments, and system — live over SSE;
 - the **HTTP API** (Litestar + msgspec, OpenAPI at `/schema/scalar`) with the same surface as
-  the CLI: plan/apply, runs, checks, streams, engines, schedules, lineage, query, gc;
+  the CLI: plan/apply, runs, checks, streams, engines, schedules, lineage, query, gc, reset;
 - the **scheduler**: cron/interval triggers enqueue onto a **durable run queue** (leases,
   retries, cooperative cancellation — `interlace cancel <id>` or `POST /runs/{id}/cancel`);
 - **stream ingestion** and retention.

{interlaced-2.5.0 → interlaced-2.7.0}/README.md

@@ -206,7 +206,7 @@ fast lane when possible). Contract: `docs/architecture/MULTI_ENGINE.md`.
   with column-level tracing, models, plan/apply with SQL diffs, live runs, query console,
   streams, checks, environments, and system — live over SSE;
 - the **HTTP API** (Litestar + msgspec, OpenAPI at `/schema/scalar`) with the same surface as
-  the CLI: plan/apply, runs, checks, streams, engines, schedules, lineage, query, gc;
+  the CLI: plan/apply, runs, checks, streams, engines, schedules, lineage, query, gc, reset;
 - the **scheduler**: cron/interval triggers enqueue onto a **durable run queue** (leases,
   retries, cooperative cancellation — `interlace cancel <id>` or `POST /runs/{id}/cancel`);
 - **stream ingestion** and retention.

{interlaced-2.5.0 → interlaced-2.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.5.0"
+version = "2.7.0"
 description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -143,6 +143,7 @@ module = [
     "adbc_driver_bigquery.*",  # optional 'adbc-bigquery' extra — absent in the base env
     "pyspark.*",  # optional 'spark' extra — absent in the base env
     "delta.*",  # optional 'spark' extra (delta-spark) — absent in the base env
+    "psycopg.*",  # optional 'postgres' extra — absent in the base env
 ]
 ignore_missing_imports = true
 

interlaced-2.7.0/src/interlace/cdc/__init__.py

@@ -0,0 +1,6 @@
+"""Postgres logical replication into a durable ``@stream``."""
+
+from interlace.cdc.decode import Change, decode_message
+from interlace.cdc.publish import confirm_flushed, publish_changes
+
+__all__ = ["Change", "confirm_flushed", "decode_message", "publish_changes"]

interlaced-2.7.0/src/interlace/cdc/decode.py

@@ -0,0 +1,152 @@
+"""Decode ``pgoutput`` logical-replication messages into row changes.
+
+The slot reader passes each message's WAL start as ``lsn``. Relation messages
+update ``relations`` and return None. Insert, update, and delete return a
+:class:`Change` whose ``row`` is text values keyed by the relation's column names.
+"""
+
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class Relation:
+    namespace: str
+    name: str
+    columns: list[str]
+
+    @property
+    def qualified(self) -> str:
+        return f"{self.namespace}.{self.name}"
+
+
+@dataclass
+class Change:
+    """One row change. ``kind`` is ``insert``, ``update``, or ``delete``."""
+
+    lsn: str
+    kind: str
+    table: str
+    row: dict[str, Any]
+    before: dict[str, Any] | None = None
+
+
+@dataclass
+class _Cursor:
+    data: bytes
+    pos: int = 0
+
+    def byte(self) -> bytes:
+        value = self.data[self.pos : self.pos + 1]
+        self.pos += 1
+        return value
+
+    def i16(self) -> int:
+        value = struct.unpack_from("!h", self.data, self.pos)[0]
+        self.pos += 2
+        return int(value)
+
+    def i32(self) -> int:
+        value = struct.unpack_from("!i", self.data, self.pos)[0]
+        self.pos += 4
+        return int(value)
+
+    def i64(self) -> int:
+        value = struct.unpack_from("!q", self.data, self.pos)[0]
+        self.pos += 8
+        return int(value)
+
+    def text(self) -> str:
+        end = self.data.index(b"\x00", self.pos)
+        value = self.data[self.pos : end].decode()
+        self.pos = end + 1
+        return value
+
+    def tuple(self) -> list[str | None]:
+        values: list[str | None] = []
+        for _ in range(self.i16()):
+            kind = self.byte()
+            if kind in (b"n", b"u"):
+                values.append(None)
+                continue
+            if kind != b"t":
+                raise ValueError(f"unsupported pgoutput tuple marker {kind!r}")
+            length = self.i32()
+            values.append(self.data[self.pos : self.pos + length].decode())
+            self.pos += length
+        return values
+
+
+def decode_message(payload: bytes, relations: dict[int, Relation], lsn: str) -> Change | None:
+    """Decode one ``pgoutput`` message. Begin, commit, and relation messages return None."""
+    if not payload:
+        return None
+    cur = _Cursor(payload)
+    kind = cur.byte()
+    if kind == b"R":
+        oid = cur.i32()
+        namespace = cur.text()
+        name = cur.text()
+        cur.byte()  # replica identity
+        columns: list[str] = []
+        for _ in range(cur.i16()):
+            cur.byte()  # flags
+            columns.append(cur.text())
+            cur.i32()  # type oid
+            cur.i32()  # typemod
+        relations[oid] = Relation(namespace, name, columns)
+        return None
+    if kind == b"I":
+        return _row_change(cur, relations, lsn, "insert")
+    if kind == b"U":
+        return _update(cur, relations, lsn)
+    if kind == b"D":
+        return _row_change(cur, relations, lsn, "delete")
+    return None  # begin, commit, truncate, type, origin: no row
+
+
+def _relation(cur: _Cursor, relations: dict[int, Relation]) -> Relation:
+    oid = cur.i32()
+    found = relations.get(oid)
+    if found is None:
+        raise ValueError(f"pgoutput message for unknown relation oid {oid}")
+    return found
+
+
+def _as_row(relation: Relation, values: list[str | None]) -> dict[str, Any]:
+    return dict(zip(relation.columns, values, strict=False))
+
+
+def _row_change(cur: _Cursor, relations: dict[int, Relation], lsn: str, kind: str) -> Change:
+    relation = _relation(cur, relations)
+    cur.byte()  # 'N' for insert, 'K' or 'O' for delete
+    return Change(lsn=lsn, kind=kind, table=relation.qualified, row=_as_row(relation, cur.tuple()))
+
+
+def _update(cur: _Cursor, relations: dict[int, Relation], lsn: str) -> Change:
+    relation = _relation(cur, relations)
+    marker = cur.byte()
+    before: dict[str, Any] | None = None
+    if marker in (b"K", b"O"):
+        before = _as_row(relation, cur.tuple())
+        marker = cur.byte()
+    if marker != b"N":
+        raise ValueError(f"pgoutput update expected a new tuple, got {marker!r}")
+    return Change(
+        lsn=lsn,
+        kind="update",
+        table=relation.qualified,
+        row=_as_row(relation, cur.tuple()),
+        before=before,
+    )
+
+
+def payload_of(change: Change) -> dict[str, Any]:
+    """Stream payload: the row, plus ``_change`` so a merge model can drop deletes."""
+    body = dict(change.row)
+    body["_change"] = change.kind
+    body["_table"] = change.table
+    return body

interlaced-2.7.0/src/interlace/cdc/publish.py

@@ -0,0 +1,27 @@
+"""Append decoded changes to a stream, and confirm their LSN only after a flush."""
+
+from __future__ import annotations
+
+from interlace.cdc.decode import Change, payload_of
+from interlace.state.store import SqliteStateStore
+from interlace.streaming.log import Event, StreamLog
+
+
+async def publish_changes(log: StreamLog, store: SqliteStateStore, stream: str, changes: list[Change]) -> list[int]:
+    """Append ``changes`` to ``stream``. A repeated LSN is deduped by the log.
+
+    The LSN is recorded against the log offset and is not confirmed here.
+    """
+    if not changes:
+        return []
+    result = await log.append(
+        stream,
+        [Event(payload=payload_of(change), idempotency_key=f"cdc:{change.lsn}") for change in changes],
+    )
+    await store.cdc_note_pending(stream, list(zip(result.offsets, (change.lsn for change in changes), strict=True)))
+    return result.offsets
+
+
+async def confirm_flushed(store: SqliteStateStore, stream: str, watermark: int) -> str | None:
+    """Advance the stored LSN up to the offset ``flush_streams`` has committed."""
+    return await store.cdc_advance(stream, watermark)

interlaced-2.7.0/src/interlace/cdc/slot.py

@@ -0,0 +1,97 @@
+"""Read a Postgres logical replication slot with ``psycopg`` and decode ``pgoutput``.
+
+The connection is opened only when CDC is configured. Feedback to the slot
+(which lets Postgres drop WAL) is sent by :meth:`SlotReader.feedback` after
+:func:`interlace.cdc.publish.confirm_flushed` has stored the LSN — never when
+the message is merely appended to the stream log.
+"""
+
+from __future__ import annotations
+
+from typing import cast
+
+from interlace.cdc.decode import Change, Relation, decode_message
+from interlace.config.config import CdcConfig
+from interlace.exceptions import ConfigurationError
+
+
+class SlotReader:
+    """One replication connection for one slot. ``poll`` does not advance the slot."""
+
+    def __init__(self, dsn: str, source: CdcConfig) -> None:
+        self._dsn = dsn
+        self._source = source
+        self._conn: object | None = None
+        self._cur: object | None = None
+        self._relations: dict[int, Relation] = {}
+        self._started = False
+        self._tables = set(source.tables)
+
+    def poll(self, confirmed_lsn: str | None, *, limit: int = 100) -> list[Change]:
+        """Read up to ``limit`` decoded row changes. Blocks briefly when the slot is idle."""
+        cursor = self._cursor(confirmed_lsn)
+        changes: list[Change] = []
+        while len(changes) < limit:
+            message = self._read_one(cursor)
+            if message is None:
+                break
+            payload = getattr(message, "payload", b"")
+            data = payload if isinstance(payload, bytes) else bytes(payload)
+            lsn = str(getattr(message, "data_start", ""))
+            change = decode_message(data, self._relations, lsn)
+            if change is None or not self._wanted(change.table):
+                continue
+            changes.append(change)
+        return changes
+
+    def feedback(self, lsn: str) -> None:
+        """Tell the slot that ``lsn`` is flushed. Safe to call only after confirm_flushed."""
+        cursor = self._cur
+        if cursor is None or not lsn:
+            return
+        send = getattr(cursor, "send_feedback", None)
+        if send is not None:
+            send(flush_lsn=lsn, force=True)
+
+    def close(self) -> None:
+        conn = self._conn
+        self._cur = None
+        self._conn = None
+        self._started = False
+        close = getattr(conn, "close", None)
+        if close is not None:
+            close()
+
+    def _wanted(self, table: str) -> bool:
+        if table in self._tables:
+            return True
+        return any(item == table.rsplit(".", 1)[-1] for item in self._tables)
+
+    def _cursor(self, confirmed_lsn: str | None) -> object:
+        if self._cur is not None:
+            return self._cur
+        try:
+            import psycopg  # optional postgres extra
+        except ImportError as exc:
+            raise ConfigurationError("CDC needs the postgres extra (psycopg). Install interlaced[postgres].") from exc
+        conn = psycopg.connect(self._dsn, autocommit=True, replication="database")
+        cursor = conn.cursor()
+        options = {"proto_version": "1", "publication_names": self._source.publication}
+        start = confirmed_lsn if confirmed_lsn else None
+        cursor.start_replication(slot_name=self._source.slot, decode=False, start_lsn=start, options=options)
+        self._conn = conn
+        self._cur = cursor
+        self._started = True
+        return cursor
+
+    def _read_one(self, cursor: object) -> object | None:
+        read = getattr(cursor, "read_message", None)
+        if read is None:
+            raise ConfigurationError("this psycopg build has no replication cursor")
+        try:
+            return cast(object | None, read(timeout=0.2))
+        except Exception as exc:
+            # psycopg raises a timeout-ish error or returns None when the slot is idle
+            if type(exc).__name__ in {"ReplicationTimeout", "QueryCanceled"} or "timeout" in str(exc).lower():
+                return None
+            raise

{interlaced-2.5.0 → interlaced-2.7.0}/src/interlace/checks/builtin.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 
 import re
 from collections.abc import Callable
-from typing import Any
+from typing import Any, cast
 
 from sqlglot import exp, parse_one
 
@@ -65,33 +65,23 @@ def _interval(max_age: str) -> exp.Interval:
     return exp.Interval(this=exp.Literal.string(match.group(1)), unit=exp.Var(this=_AGE_UNITS[match.group(2)]))
 
 
-def build_check_query(
-    spec: CheckSpec, table: TableRef, model: str, dialect: str, resolve: ResolveTable
-) -> exp.Expression:
-    """Compile ``spec`` against ``table`` into a query returning ``failures``."""
+def _row_predicate(spec: CheckSpec, model: str, dialect: str, resolve: ResolveTable) -> exp.Expression | None:
+    """A WHERE predicate that is true for a violating row.
+
+    ``None`` for checks that are not a per-row filter (``unique`` counts groups,
+    ``row_count`` / ``freshness`` judge the table, ``sql`` is its own query)."""
     cols = [exp.column(c) for c in spec.columns]
     params: dict[str, Any] = spec.params
 
     if spec.type == "not_null":
         _require(spec, model, columns=1)
-        condition: exp.Expression = exp.or_(*[col.is_(exp.null()) for col in cols])
-        return _count_where(table, condition)
-
-    if spec.type == "unique":
-        _require(spec, model, columns=1)
-        grouped = (
-            exp.select(*cols)
-            .from_(_table(table))
-            .group_by(*cols)
-            .having(exp.GT(this=exp.Count(this=exp.Star()), expression=exp.Literal.number(1)))
-        )
-        return exp.select(exp.alias_(exp.Count(this=exp.Star()), _FAILURES)).from_(grouped.subquery("dup"))
+        return exp.or_(*[col.is_(exp.null()) for col in cols])
 
     if spec.type == "accepted_values":
         _require(spec, model, columns=1, params=("values",))
         values = [exp.Literal.string(v) if isinstance(v, str) else exp.Literal.number(v) for v in params["values"]]
         col = cols[0]
-        return _count_where(table, exp.and_(col.is_(exp.null()).not_(), exp.In(this=col, expressions=values).not_()))
+        return exp.and_(col.is_(exp.null()).not_(), exp.In(this=col, expressions=values).not_())
 
     if spec.type == "range":
         _require(spec, model, columns=1)
@@ -103,25 +93,47 @@ def build_check_query(
             bounds.append(exp.LT(this=col, expression=exp.Literal.number(params["min"])))
         if "max" in params:
             bounds.append(exp.GT(this=col, expression=exp.Literal.number(params["max"])))
-        return _count_where(table, exp.or_(*bounds))
+        return exp.or_(*bounds)
 
     if spec.type == "pattern":
         _require(spec, model, columns=1, params=("regex",))
         col = cols[0]
         matches = exp.RegexpLike(this=col, expression=exp.Literal.string(str(params["regex"])))
-        return _count_where(table, exp.and_(col.is_(exp.null()).not_(), matches.not_()))
+        return exp.and_(col.is_(exp.null()).not_(), matches.not_())
 
     if spec.type == "expression":
         _require(spec, model, params=("expression",))
-        predicate = _parse_expr(str(params["expression"]), dialect)
-        return _count_where(table, exp.paren(predicate).not_())
+        return cast(exp.Expression, exp.paren(_parse_expr(str(params["expression"]), dialect)).not_())
 
     if spec.type == "relationships":
         _require(spec, model, columns=1, params=("to", "field"))
         col = cols[0]
         parent = exp.select(exp.column(str(params["field"]))).from_(_table(resolve(str(params["to"]))))
         orphan = exp.In(this=col, query=exp.Subquery(this=parent)).not_()
-        return _count_where(table, exp.and_(col.is_(exp.null()).not_(), orphan))
+        return exp.and_(col.is_(exp.null()).not_(), orphan)
+
+    return None
+
+
+def build_check_query(
+    spec: CheckSpec, table: TableRef, model: str, dialect: str, resolve: ResolveTable
+) -> exp.Expression:
+    """Compile ``spec`` against ``table`` into a query returning ``failures``."""
+    cols = [exp.column(c) for c in spec.columns]
+    params: dict[str, Any] = spec.params
+    predicate = _row_predicate(spec, model, dialect, resolve)
+    if predicate is not None:
+        return _count_where(table, predicate)
+
+    if spec.type == "unique":
+        _require(spec, model, columns=1)
+        grouped = (
+            exp.select(*cols)
+            .from_(_table(table))
+            .group_by(*cols)
+            .having(exp.GT(this=exp.Count(this=exp.Star()), expression=exp.Literal.number(1)))
+        )
+        return exp.select(exp.alias_(exp.Count(this=exp.Star()), _FAILURES)).from_(grouped.subquery("dup"))
 
     if spec.type == "row_count":
         if "min" not in params and "max" not in params:
@@ -150,3 +162,34 @@ def build_check_query(
         )
 
     raise DefinitionError(f"unknown check type {spec.type!r} on {model!r}")
+
+
+def build_failing_rows(
+    spec: CheckSpec, table: TableRef, model: str, dialect: str, resolve: ResolveTable
+) -> exp.Expression | None:
+    """The rows a check rejected, or ``None`` when the check has no row set.
+
+    ``row_count`` and ``freshness`` judge the whole table. ``unique`` returns the
+    duplicate rows (the failure *count* stays the number of duplicate groups).
+    A ``sql`` check's query is already the failing rows."""
+    if spec.type in ("row_count", "freshness"):
+        return None
+    if spec.type == "sql":
+        _require(spec, model, params=("query",))
+        sql = str(spec.params["query"]).replace("{table}", _table(table).sql(dialect=dialect))
+        return _parse_expr(sql, dialect)
+    if spec.type == "unique":
+        _require(spec, model, columns=1)
+        cols = [exp.column(c) for c in spec.columns]
+        grouped = (
+            exp.select(*cols)
+            .from_(_table(table))
+            .group_by(*cols)
+            .having(exp.GT(this=exp.Count(this=exp.Star()), expression=exp.Literal.number(1)))
+        )
+        key: exp.Expression = exp.Tuple(expressions=cols) if len(cols) > 1 else cols[0]
+        return exp.select(exp.Star()).from_(_table(table)).where(exp.In(this=key, query=exp.Subquery(this=grouped)))
+    predicate = _row_predicate(spec, model, dialect, resolve)
+    if predicate is None:
+        return None
+    return exp.select(exp.Star()).from_(_table(table)).where(predicate)