interlaced 2.8.0__tar.gz → 2.8.2__tar.gz

{interlaced-2.8.0/src/interlaced.egg-info → interlaced-2.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.8.0
+Version: 2.8.2
 Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT
@@ -22,7 +22,7 @@ Classifier: Operating System :: POSIX :: Linux
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: sqlglot<31.0,>=25.0
+Requires-Dist: sqlglot<31.0,>=30.0
 Requires-Dist: duckdb>=1.5.3
 Requires-Dist: pyarrow>=17.0
 Requires-Dist: pydantic<3.0,>=2.5

{interlaced-2.8.0 → interlaced-2.8.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.8.0"
+version = "2.8.2"
 description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -30,7 +30,7 @@ classifiers = [
 # Core: the IR + engine + state + plan spine (Phase 1). Everything is Arrow- and
 # sqlglot-native. See docs/architecture/architecture.md.
 dependencies = [
-    "sqlglot>=25.0,<31.0",        # canonical IR, transpilation, semantic diff, column lineage
+    "sqlglot>=30.0,<31.0",        # canonical IR, transpilation, semantic diff, column lineage
     "duckdb>=1.5.3",              # default engine + federation hub; DuckLake storage + quack serving
     "pyarrow>=17.0",              # the wire format: RecordBatchReader everywhere
     "pydantic>=2.5,<3.0",         # config + manifest validation (cold paths only)

{interlaced-2.8.0 → interlaced-2.8.2}/src/interlace/cdc/publish.py

@@ -18,10 +18,10 @@ async def publish_changes(log: StreamLog, store: SqliteStateStore, stream: str,
         stream,
         [Event(payload=payload_of(change), idempotency_key=f"cdc:{change.lsn}") for change in changes],
     )
-    await store.cdc_note_pending(stream, list(zip(result.offsets, (change.lsn for change in changes), strict=True)))
+    await store.cdc.cdc_note_pending(stream, list(zip(result.offsets, (change.lsn for change in changes), strict=True)))
     return result.offsets
 
 
 async def confirm_flushed(store: SqliteStateStore, stream: str, watermark: int) -> str | None:
     """Advance the stored LSN up to the offset ``flush_streams`` has committed."""
-    return await store.cdc_advance(stream, watermark)
+    return await store.cdc.cdc_advance(stream, watermark)

interlaced-2.8.2/src/interlace/cdc/slot.py

@@ -0,0 +1,110 @@
+"""Read a Postgres logical replication slot with ``psycopg`` and decode ``pgoutput``.
+
+psycopg 3 has no replication cursor, so this peeks the slot with
+``pg_logical_slot_peek_binary_changes`` (Postgres 14+) and does not move it.
+:meth:`SlotReader.feedback` advances the slot only after
+:func:`interlace.cdc.publish.confirm_flushed` has stored the LSN — never when
+the message is merely appended to the stream log.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from interlace.cdc.decode import Change, Relation, decode_message
+from interlace.config.config import CdcConfig
+from interlace.exceptions import ConfigurationError
+
+
+class SlotReader:
+    """One Postgres connection for one slot. ``poll`` does not advance the slot."""
+
+    def __init__(self, dsn: str, source: CdcConfig) -> None:
+        self._dsn = dsn
+        self._source = source
+        self._conn: Any = None
+        self._relations: dict[int, Relation] = {}
+        self._tables = set(source.tables)
+
+    def poll(self, confirmed_lsn: str | None, *, limit: int = 100) -> list[Change]:
+        """Read up to ``limit`` decoded row changes. Returns immediately when the slot is idle.
+
+        ``confirmed_lsn`` is the highest LSN already flushed. Changes at or below it are
+        skipped, so a peek that still contains them (feedback has not advanced the slot yet)
+        does not publish them again.
+        """
+        confirmed = _lsn_key(confirmed_lsn) if confirmed_lsn else None
+        changes: list[Change] = []
+        # Begin, relation, and commit messages sit beside each row change.
+        for lsn, data in self._peek(max(limit * 4, limit)):
+            payload = bytes(data)
+            change = decode_message(payload, self._relations, lsn)
+            if change is None or not self._wanted(change.table):
+                continue
+            if confirmed is not None and _lsn_key(change.lsn) <= confirmed:
+                continue
+            changes.append(change)
+            if len(changes) >= limit:
+                break
+        return changes
+
+    def feedback(self, lsn: str) -> None:
+        """Advance the slot to ``lsn``. Safe to call only after confirm_flushed."""
+        if not lsn or self._conn is None:
+            return
+        self._fetch("SELECT pg_replication_slot_advance(%s, %s::pg_lsn)", (self._source.slot, lsn))
+
+    def close(self) -> None:
+        conn = self._conn
+        self._conn = None
+        close = getattr(conn, "close", None)
+        if close is not None:
+            close()
+
+    def _wanted(self, table: str) -> bool:
+        if table in self._tables:
+            return True
+        return any(item == table.rsplit(".", 1)[-1] for item in self._tables)
+
+    def _peek(self, nchanges: int) -> list[tuple[str, bytes]]:
+        rows = self._fetch(
+            """
+            SELECT lsn::text, data
+            FROM pg_logical_slot_peek_binary_changes(
+                %s, NULL, %s, 'proto_version', '1', 'publication_names', %s
+            )
+            """,
+            (self._source.slot, nchanges, self._source.publication),
+        )
+        return [(str(row[0]), bytes(row[1])) for row in rows]
+
+    def _fetch(self, sql: str, params: tuple[object, ...]) -> list[Any]:
+        conn = self._connection()
+        try:
+            with conn.cursor() as cur:
+                cur.execute(sql, params)
+                return list(cur.fetchall())
+        except Exception as exc:
+            if "pg_logical_slot_peek_binary_changes" in str(exc):
+                raise ConfigurationError(
+                    "CDC needs Postgres 14 or newer (pg_logical_slot_peek_binary_changes)."
+                ) from exc
+            raise
+
+    def _connection(self) -> Any:
+        if self._conn is not None:
+            return self._conn
+        try:
+            import psycopg  # optional postgres extra
+        except ImportError as exc:
+            raise ConfigurationError("CDC needs the postgres extra (psycopg). Install interlaced[postgres].") from exc
+        self._conn = psycopg.connect(self._dsn, autocommit=True)
+        return self._conn
+
+
+def _lsn_key(lsn: str) -> tuple[int, int]:
+    """Order key for a ``pg_lsn`` text value (``high/low``, both hex)."""
+    high, sep, low = lsn.partition("/")
+    if not sep:
+        raise ValueError(f"not a pg_lsn: {lsn!r}")
+    return (int(high, 16), int(low, 16))

{interlaced-2.8.0 → interlaced-2.8.2}/src/interlace/checks/runner.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 
 import asyncio
 import inspect
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 
 import pyarrow as pa
@@ -21,10 +21,13 @@ from interlace.checks.builtin import build_check_query
 from interlace.checks.spec import CheckSpec
 from interlace.dsl.decorators import CheckDef
 from interlace.engines.base import EngineAdapter
-from interlace.exceptions import DefinitionError
+from interlace.engines.registry import EngineRegistry
+from interlace.exceptions import DefinitionError, PlanError
 from interlace.graph.project import CompiledModel, CompiledProject
 from interlace.ir.relation import TableRef
 from interlace.runtime.handles import RelationHandle
+from interlace.sinks import target_ref
+from interlace.state.store import SqliteStateStore
 
 
 @dataclass(frozen=True)
@@ -101,3 +104,45 @@ async def run_checks(
     outcomes = [await _run_declared(spec, model, compiled, engine, table, physical) for spec in model.checks]
     outcomes += [await _run_python(check, engine, table, model.name) for check in python_checks]
     return outcomes
+
+
+async def run_promoted_checks(
+    compiled: CompiledProject,
+    store: SqliteStateStore,
+    engines: EngineRegistry,
+    environment: str,
+    selectors: Sequence[str] = (),
+) -> tuple[list[CheckOutcome], list[str]]:
+    """Re-check an environment's promoted tables. ``state:`` selectors see the ledger.
+
+    Returns ``(outcomes, skipped)``. Models that were never promoted in this
+    environment are skipped. A missing     environment is a plan error.
+    """
+    from interlace.plan.orchestrate import resolve_selection
+
+    chosen = await resolve_selection(compiled, store, environment, selectors)
+    promoted = await store.get_environment(environment)
+    if not promoted:
+        raise PlanError(f"no environment {environment!r} — run `interlace apply` first")
+    snapshots = await store.get_snapshots(promoted.items())
+    physical = {name: snapshot.physical_table for (name, _), snapshot in snapshots.items()}
+    outcomes: list[CheckOutcome] = []
+    skipped: list[str] = []
+    for name, model in compiled.models.items():
+        if chosen is not None and name not in chosen:
+            continue
+        if not model.checks and not compiled.python_checks.get(name):
+            continue
+        if model.materialise == "file" or (model.is_terminal and environment not in model.environments):
+            continue
+        snapshot = snapshots.get((name, promoted.get(name, "")))
+        if snapshot is None:
+            skipped.append(name)
+            continue
+        engine = engines.require(snapshot.engine, model=name)
+        check_table = target_ref(model.target or "") if model.materialise == "table" else snapshot.physical_table
+        results = await run_checks(model, compiled, engine, check_table, compiled.python_checks.get(name, ()), physical)
+        if results:
+            await store.record_check_results(environment, snapshot.fingerprint, results)
+        outcomes.extend(results)
+    return outcomes, skipped