feather-cdc-postgres 0.1.0__py3-none-any.whl

feather_cdc_postgres/snapshot.py

@@ -0,0 +1,448 @@
+"""Consistent baselines with NO staging tables on the source (#3079 Deviation D-1).
+
+The slot's exported snapshot is read directly inside one REPEATABLE READ transaction, so the
+source gains no `_dlt_*` tables (50 stray ones found live, #2874). Ordering invariant, proved
+against PG 17 on 04-Sep-2026: `SET TRANSACTION SNAPSHOT` must run while the exporting
+replication connection is still open (it fails "snapshot does not exist" otherwise); once the
+import succeeds the exporter may close and the read continues.
+Convergence: the publication is reconciled BEFORE the slot exists, so any concurrent change is
+also in the stream and the merge write-disposition dedups on the declared key.
+A baseline taken LATER (a table added mid-life, a resumed snapshot, an operator resnapshot) is
+`consistent_baseline`: a throwaway slot hands us PG's own consistent point + exported snapshot,
+so "visible in the baseline" ⟺ "commit LSN < boundary" by construction, and the drain drops
+what the baseline already saw. That is what makes an append_only overlap exact (#3079 review).
+"""
+from __future__ import annotations
+
+import contextlib
+import threading
+from dataclasses import dataclass, field
+from typing import Iterator, Sequence
+
+import dlt
+import psycopg
+import psycopg2
+from psycopg import sql
+from psycopg2.extras import LogicalReplicationConnection
+
+from pg_replication.decoders import ColumnType
+from pg_replication.helpers import published_shape
+from pg_replication.schema_types import _PG_TYPES, _to_dlt_column_schema
+
+Relation = tuple[str, str]
+
+BASELINE_WAIT_S_DEFAULT = 300
+
+
+class BaselineBlocked(RuntimeError):
+    """CREATE_REPLICATION_SLOT ... EXPORT_SNAPSHOT did not complete within `baseline_wait_s`
+    because it is waiting for every in-flight source transaction to finish. See
+    `create_slot_with_snapshot` for why `statement_timeout` cannot be used here, and
+    docs/agents/pg-md-cdc-runbook.md §c for the remedy."""
+
+
+def _regclass(schema: str, table: str) -> str:
+    """A quoted `schema.table` for `to_regclass`. Every catalogue read of a baseline resolves the
+    name the way `LOCK`/`SELECT` do — through CURRENT name resolution — never by joining
+    `pg_class` on the name, which an imported REPEATABLE READ snapshot answers from older
+    catalogue rows: after a swap the two disagree about which table a name means (#4346)."""
+    return '"' + schema.replace('"', '""') + '"."' + table.replace('"', '""') + '"'
+
+
+def lsn_hex_to_int(lsn: str) -> int:
+    hi, lo = lsn.split("/")
+    return (int(hi, 16) << 32) + int(lo, 16)
+
+
+@dataclass
+class ExportedSnapshot:
+    """Handle on a freshly created slot + its exported snapshot. Keep it open until the
+    importing transaction has run SET TRANSACTION SNAPSHOT (see module docstring)."""
+
+    slot: str
+    consistent_point_lsn: int
+    snapshot_name: str
+    _conn: object | None = field(default=None, repr=False)
+
+    def close(self) -> None:
+        if self._conn is not None:
+            with contextlib.suppress(Exception):
+                self._conn.close()
+            self._conn = None
+
+    def __iter__(self):  # so `lsn, name = handle` reads naturally at call sites
+        return iter((self.consistent_point_lsn, self.snapshot_name))
+
+
+def drop_slot_if_exists(dsn: str, slot: str) -> bool:
+    with psycopg.connect(dsn, autocommit=True, connect_timeout=10) as conn:
+        row = conn.execute("select 1 from pg_replication_slots where slot_name = %s",
+                           (slot,)).fetchone()
+        if row:
+            conn.execute("select pg_drop_replication_slot(%s)", (slot,))
+    return bool(row)
+
+
+def temp_slot_name(slot: str) -> str:
+    """The throwaway slot a consistent baseline is taken from. Ours by name; the caller holds
+    the registration's owner lock, so a leftover (an older build's persistent one) is safe to
+    drop. Created TEMPORARY, so a crash mid-baseline leaks nothing."""
+    return (slot + "_bl")[:63]
+
+
+def _describe_blocker(dsn: str, waiting_pid: int) -> str:
+    """Best-effort context for `BaselineBlocked`: the oldest session `pg_blocking_pids` names
+    as blocking `waiting_pid`, read on a plain connection (the stuck replication connection is
+    still busy). Never raises — a failure to identify the blocker must not hide the timeout."""
+    try:
+        with psycopg.connect(dsn, autocommit=True, connect_timeout=10) as c:
+            rows = c.execute(
+                # NOT `query`: the blocker's last statement is arbitrary source SQL — an
+                # `UPDATE person SET password_hash = '…'` would carry the hash into the
+                # exception, control.event and the log (#4346, Codex round 9).
+                "select pid, state, xact_start, usename, application_name from pg_stat_activity "
+                "where pid = any(pg_blocking_pids(%s)) order by xact_start asc nulls last",
+                (waiting_pid,)).fetchall()
+    except Exception as exc:  # noqa: BLE001 — context only, the timeout itself is the finding
+        return f"could not identify the blocking session: {type(exc).__name__}: {exc}"
+    if not rows:
+        return "no blocking session found in pg_stat_activity (it may have just ended)"
+    pid, state, xact_start, user, app = rows[0]
+    return (f"oldest blocker pid={pid} state={state!r} xact_start={xact_start} "
+            f"user={user!r} application={app!r}")
+
+
+def create_slot_with_snapshot(dsn: str, slot: str, *, temporary: bool = False,
+                              baseline_wait_s: float = BASELINE_WAIT_S_DEFAULT) -> ExportedSnapshot:
+    """Create `slot` with pgoutput and KEEP the replication connection open so its exported
+    snapshot stays importable. Raises if the slot already exists (the caller drops it first).
+    `temporary`: PostgreSQL drops the slot itself when this session ends — a killed container
+    cannot leave it behind retaining WAL. Only the main slot must outlive the process.
+
+    `CREATE_REPLICATION_SLOT ... EXPORT_SNAPSHOT` blocks until every transaction in flight on
+    the source ends (pg_stat_activity shows it waiting `Lock`/`transactionid`) — a long-running
+    or idle-in-transaction session can block it indefinitely. `statement_timeout` does NOT
+    bound this: verified empirically on PG 17 + psycopg2's `LogicalReplicationConnection`,
+    04-Sep-2026 — neither `SET statement_timeout` on the replication cursor nor
+    `options=-c statement_timeout=...` in the DSN cancels a blocked CREATE_REPLICATION_SLOT,
+    even though the session's own `SHOW statement_timeout` confirms the GUC took. Replication
+    protocol commands run through PostgreSQL's `exec_replication_command()` path, not the
+    ordinary simple-query path that arms the statement timer, so the GUC is simply never
+    consulted there. The bound is therefore enforced from the CLIENT instead: the call runs on
+    a background thread and, if it has not returned within `baseline_wait_s`, this function
+    looks up the blocking session with `pg_blocking_pids()`, cancels the stuck backend with
+    `pg_cancel_backend()` (proven to unblock it), and raises `BaselineBlocked` naming the
+    oldest blocker's pid/state/xact_start — see docs/agents/pg-md-cdc-runbook.md §c for the
+    remedy (end that session; the next tick retries from `snapshotting`)."""
+    conn = psycopg2.connect(dsn, connection_factory=LogicalReplicationConnection)
+    pid = conn.get_backend_pid()
+    outcome: dict = {}
+
+    def _create() -> None:
+        try:
+            cur = conn.cursor()
+            if temporary:
+                quoted = '"' + slot.replace('"', '""') + '"'
+                cur.execute(f"CREATE_REPLICATION_SLOT {quoted} TEMPORARY LOGICAL pgoutput "
+                            "EXPORT_SNAPSHOT")
+            else:
+                cur.create_replication_slot(slot, output_plugin="pgoutput")
+            outcome["row"] = cur.fetchone()
+        except BaseException as exc:  # noqa: BLE001 — re-raised on the calling thread below
+            outcome["exc"] = exc
+
+    worker = threading.Thread(target=_create, daemon=True)
+    worker.start()
+    worker.join(timeout=baseline_wait_s)
+    if worker.is_alive():
+        blocker = _describe_blocker(dsn, pid)
+        with contextlib.suppress(Exception):
+            with psycopg.connect(dsn, autocommit=True, connect_timeout=10) as c:
+                c.execute("select pg_cancel_backend(%s)", (pid,))
+        worker.join(timeout=10)  # let the cancelled thread unwind before we close its connection
+        with contextlib.suppress(Exception):
+            conn.close()
+        raise BaselineBlocked(
+            f"CREATE_REPLICATION_SLOT for slot {slot!r} did not complete within "
+            f"{baseline_wait_s:g}s: {blocker}. Remedy: end the blocking session "
+            "(docs/agents/pg-md-cdc-runbook.md §c) — the table stays `snapshotting` and "
+            "the next tick retries.")
+    if "exc" in outcome:
+        conn.close()
+        raise outcome["exc"]
+    _, consistent_point, snapshot_name, _ = outcome["row"]
+    return ExportedSnapshot(slot, lsn_hex_to_int(consistent_point), snapshot_name, conn)
+
+
+def column_hints(conn: psycopg.Connection, schema: str, table: str, keys: Sequence[str],
+                 exclude: Sequence[str] = ()) -> dict:
+    """Destination column types compiled from the source catalog through the SAME
+    OID→dlt mapper the CDC path uses — otherwise int4 lands BIGINT on one path and INTEGER
+    on the other (observed live in bronze_featherbase, plan Appendix A). A denied column
+    (`exclude`, #4346) is not even declared to dlt."""
+    rows = conn.execute(
+        "select a.attname, a.atttypid, a.atttypmod from pg_attribute a "
+        "where a.attrelid = to_regclass(%s) and a.attnum > 0 and not a.attisdropped "
+        "order by a.attnum",
+        (_regclass(schema, table),),
+    ).fetchall()
+    key_set = set(keys or ())
+    denied = set(exclude or ())
+    return {
+        name: _to_dlt_column_schema(ColumnType(int(name in key_set), name, type_id, typmod))
+        for name, type_id, typmod in rows
+        if name not in denied
+    }
+
+
+def _select_list(conn: psycopg.Connection, schema: str, table: str, exclude: Sequence[str] = ()):
+    """`select *` except that every column with no declared OID mapping is cast to text.
+
+    Those columns compile to `text_fallback` (schema_registry), and the CDC path already
+    receives pgoutput's text rendering — but psycopg hands the SNAPSHOT path a native object
+    (`inet` → IPv4Address), which dlt cannot serialise. Casting here makes both paths agree.
+    A denied column (`exclude`, #4346) is never read; a list the deny would leave empty raises
+    rather than fall back to `select *`, which would read exactly what was denied.
+    """
+    rows = conn.execute(
+        "select a.attname, a.atttypid, t.typcategory = 'A' as is_array from pg_attribute a "
+        "join pg_type t on t.oid = a.atttypid "
+        "where a.attrelid = to_regclass(%s) and a.attnum > 0 and not a.attisdropped "
+        "order by a.attnum",
+        (_regclass(schema, table),),
+    ).fetchall()
+    denied = set(exclude or ())
+    parts = []
+    for name, type_oid, is_array in rows:
+        if name in denied:
+            continue
+        if is_array or int(type_oid) not in _PG_TYPES:
+            parts.append(sql.SQL("{}::text as {}").format(sql.Identifier(name),
+                                                          sql.Identifier(name)))
+        else:
+            parts.append(sql.Identifier(name))
+    if not parts and rows:
+        raise ValueError(f"{schema}.{table}: every column is denied — nothing to baseline")
+    return sql.SQL(", ").join(parts) if parts else sql.SQL("*")
+
+
+def _rows(conn: psycopg.Connection, schema: str, table: str, cursor_name: str, itersize: int,
+          select_list):
+    """Server-side cursor so a large table never materialises in memory."""
+    def gen():
+        with conn.cursor(name=cursor_name) as cur:
+            cur.itersize = itersize
+            cur.execute(
+                sql.SQL("select {} from {}.{}").format(
+                    select_list, sql.Identifier(schema), sql.Identifier(table))
+            )
+            cols = [d.name for d in cur.description]
+            for row in cur:
+                yield dict(zip(cols, row))
+    return gen
+
+
+def _resource(conn, schema, table, keys, key_policy, cursor_name, itersize, exclude=(),
+              catalog=None):
+    """`catalog`: the connection column names are read from — a fresh one taken under the
+    relation locks for a snapshot read (see snapshot_resources); else `conn` itself."""
+    hints = column_hints(catalog or conn, schema, table, keys, exclude)
+    select_list = _select_list(catalog or conn, schema, table, exclude)
+    merge = bool(keys) and key_policy != "append_only"
+    return dlt.resource(
+        _rows(conn, schema, table, cursor_name, itersize, select_list),
+        name=table,
+        write_disposition="merge" if merge else "append",
+        primary_key=list(keys) if merge else None,
+        columns=hints,
+    )
+
+
+@contextlib.contextmanager
+def snapshot_resources(
+    dsn: str,
+    snapshot_name: str,
+    relations: Sequence[Relation],
+    keys: dict[Relation, Sequence[str]],
+    *,
+    key_policies: dict[Relation, str] | None = None,
+    itersize: int = 5000,
+    deny_columns: dict[Relation, Sequence[str]] | None = None,
+    oids_out: dict | None = None,
+    deny_by_oid: dict[int, Sequence[str]] | None = None,
+    deny_attnums_by_oid: dict[int, Sequence[int]] | None = None,
+    layouts_out: dict | None = None,
+    bound_out: dict | None = None,
+) -> Iterator[dict[str, list]]:
+    """Yields {source_schema: [dlt resources]} reading the exported snapshot. The importing
+    transaction is opened EAGERLY (before this returns) so the caller may then close the
+    exporting replication connection; the reads themselves stay lazy.
+
+    #4346: `layouts_out[(schema, table)] = (oid, ReviewedLayout | None)` — the description
+    every later Relation message of the table must match exactly, read under the locks; None
+    when a column was dropped between the export and the lock.
+    `bound_out[(schema, table)] = [(column, attnum)]` binds each denied name under the locks.
+    A denied name that is not a column there, on a table with no bound attnum, refuses."""
+    conn = psycopg.connect(dsn, connect_timeout=10)
+    cat = None
+    try:
+        conn.execute("begin isolation level repeatable read")
+        conn.execute(sql.SQL("set transaction snapshot {}").format(sql.Literal(snapshot_name)))
+        by_schema: dict[str, list] = {}
+        policies = key_policies or {}
+        # Pin each name to its relation for the whole read (#4346, Codex round 8): PostgreSQL
+        # resolves a name in every later statement against the CURRENT catalogue, not this
+        # snapshot, so a rename/swap between the OID lookup below and the lazy SELECTs would
+        # hand a protected table's rows to an unprotected projection. ACCESS SHARE conflicts
+        # with the ACCESS EXCLUSIVE a rename needs, so the rename waits for this transaction.
+        for schema, table in relations:
+            conn.execute(sql.SQL("lock table {}.{} in access share mode").format(
+                sql.Identifier(schema), sql.Identifier(table)))
+        # Column NAMES come from a fresh catalogue read, taken while the locks are held (#4346,
+        # Codex round 14): the SELECT resolves names against the CURRENT catalogue, but a
+        # pg_attribute query inside this snapshot answers with the snapshot's names — a column
+        # swap between export and lock would split them. The locks forbid any later rename, so
+        # these names are exactly the ones the SELECT will use.
+        cat = psycopg.connect(dsn, autocommit=True, connect_timeout=10)
+        denied = {k: set(v) for k, v in (deny_columns or {}).items()}
+        # The OID each name has INSIDE the snapshot the baseline reads (#4346): exactly the
+        # relation whose rows are read. Its OID deny is unioned in HERE, so a rename or swap
+        # after the run resolved its map cannot hand a protected relation's rows to a name
+        # with no deny; the OID is also reported for the caller to persist.
+        for schema, table in relations:
+            row = conn.execute("select to_regclass(%s)::oid",
+                               (_regclass(schema, table),)).fetchone()
+            if not row or row[0] is None:
+                continue
+            oid = int(row[0])
+            if oids_out is not None:
+                oids_out[(schema, table)] = oid
+            if deny_by_oid and deny_by_oid.get(oid):
+                denied.setdefault((schema, table), set()).update(deny_by_oid[oid])
+            if deny_attnums_by_oid and deny_attnums_by_oid.get(oid):
+                # A denied COLUMN's identity (Sonnet review F1): whatever it is called in the
+                # catalogue this read uses, the column at that attnum is withheld.
+                names = cat.execute(
+                    "select attname from pg_attribute where attrelid = %s "
+                    "and attnum = any(%s) and not attisdropped",
+                    (oid, list(deny_attnums_by_oid[oid]))).fetchall()
+                denied.setdefault((schema, table), set()).update(n for (n,) in names)
+            # The reviewed layout (#4346, Codex rounds 16-17): the full published description,
+            # fresh, under the locks — every later description of the table must match it
+            # exactly. A column live in the snapshot (the boundary) but dropped by now went in
+            # the window, and WAL written before that drop could carry it: no layout.
+            fresh, _ = published_shape(cat.cursor(), oid)
+            at_boundary, _ = published_shape(conn.cursor(), oid)
+            vanished = {c[0] for c in at_boundary.columns} - {c[0] for c in fresh.columns}
+            if layouts_out is not None:
+                layouts_out[(schema, table)] = (oid, None if vanished else fresh)
+            want = denied.get((schema, table), set())
+            if want:
+                got = dict(cat.execute(
+                    "select attname, attnum from pg_attribute where attrelid = %s "
+                    "and attnum > 0 and not attisdropped and attname = any(%s)",
+                    (oid, sorted(want))).fetchall())
+                missing = sorted(want - set(got))
+                if missing and not (deny_attnums_by_oid or {}).get(oid):
+                    # Never bound, and not there under its name: it may have been renamed
+                    # before anyone bound it, so no column can be read safely (Codex round 16).
+                    raise ValueError(
+                        f"{schema}.{table}: denied column(s) {missing} are not columns of the "
+                        "table and were never bound to one — refusing to baseline (#4346)")
+                if bound_out is not None:
+                    bound_out[(schema, table)] = sorted((c, int(a)) for c, a in got.items())
+        for (schema, table) in relations:
+            clash = set(keys.get((schema, table), [])) & denied.get((schema, table), set())
+            if clash:
+                raise ValueError(f"{schema}.{table}: merge key column(s) {sorted(clash)} are "
+                                 "denied — refusing to baseline (#4346)")
+        for i, (schema, table) in enumerate(relations):
+            res = _resource(conn, schema, table, keys.get((schema, table), []),
+                            policies.get((schema, table), "merge"), f"cdc_snap_{i}", itersize,
+                            denied.get((schema, table), ()), catalog=cat)
+            by_schema.setdefault(schema, []).append(res)
+        cat.close()
+        yield by_schema
+    finally:
+        if cat is not None:
+            with contextlib.suppress(Exception):
+                cat.close()
+        with contextlib.suppress(Exception):
+            conn.rollback()
+        with contextlib.suppress(Exception):
+            conn.close()
+
+
+@contextlib.contextmanager
+def consistent_baseline(
+    dsn: str,
+    slot: str,
+    relation: Relation,
+    keys: Sequence[str],
+    *,
+    key_policy: str = "merge",
+    itersize: int = 5000,
+    baseline_wait_s: float = BASELINE_WAIT_S_DEFAULT,
+    deny_columns: Sequence[str] = (),
+    oids_out: dict | None = None,
+    deny_by_oid: dict[int, Sequence[str]] | None = None,
+    deny_attnums_by_oid: dict[int, Sequence[int]] | None = None,
+    layouts_out: dict | None = None,
+    bound_out: dict | None = None,
+) -> Iterator[tuple[object, int]]:
+    """Yields (resource, boundary_lsn) for ONE relation that is already published.
+
+    The publication add happened before this, so the table's changes are in the main slot's
+    WAL from that moment — and a plain snapshot read overlaps that WAL by however long the
+    read takes. A keyed merge hides the overlap; an append_only row has no key and lands twice.
+    A throwaway slot `<slot>_bl` gives the two things a plain read cannot: PG waits for every
+    transaction in flight before exporting the snapshot, and the consistent point it returns is
+    the exact LSN below which every transaction is in that snapshot (the decoder's own skip
+    predicate, strict `<`). The runner persists the boundary per relation and the drain drops
+    that relation's transactions with commit LSN < boundary. The slot is TEMPORARY — it dies
+    with its session, so a kill mid-baseline leaks nothing — and is dropped on exit anyway.
+    """
+    schema, table = relation
+    temp = temp_slot_name(slot)
+    drop_slot_if_exists(dsn, temp)   # a persistent leftover from an older build
+    handle = create_slot_with_snapshot(dsn, temp, temporary=True,
+                                       baseline_wait_s=baseline_wait_s)
+    try:
+        with snapshot_resources(dsn, handle.snapshot_name, [relation], {relation: list(keys)},
+                                key_policies={relation: key_policy}, itersize=itersize,
+                                deny_columns={relation: list(deny_columns)},
+                                oids_out=oids_out, deny_by_oid=deny_by_oid,
+                                deny_attnums_by_oid=deny_attnums_by_oid,
+                                layouts_out=layouts_out, bound_out=bound_out) as by_schema:
+            handle.close()  # the import already happened; the exporter is no longer needed
+            (resource,) = by_schema[schema]
+            yield resource, handle.consistent_point_lsn
+    finally:
+        handle.close()
+        drop_slot_if_exists(dsn, temp)
+
+
+@contextlib.contextmanager
+def baseline_resource(
+    dsn: str,
+    relation: Relation,
+    keys: Sequence[str],
+    *,
+    key_policy: str = "merge",
+    itersize: int = 5000,
+    deny_columns: Sequence[str] = (),
+) -> Iterator[object]:
+    """Plain REPEATABLE READ read of one table — a reader for tests and telemetry. The runtime
+    never baselines with it: a row committed between the publication add and this read would
+    land from both paths (see `consistent_baseline`)."""
+    schema, table = relation
+    conn = psycopg.connect(dsn, connect_timeout=10)
+    try:
+        conn.execute("begin isolation level repeatable read")
+        yield _resource(conn, schema, table, keys, key_policy, "cdc_base_0", itersize,
+                        deny_columns)
+    finally:
+        with contextlib.suppress(Exception):
+            conn.rollback()
+        with contextlib.suppress(Exception):
+            conn.close()

feather_cdc_postgres/stream.py

@@ -0,0 +1,162 @@
+"""Slot consumption in committed batches, grouped by source schema (#3079).
+
+Yields `Batch`es the caller loads and only THEN checkpoints — the checkpoint is runtime-owned
+(`_cdc_runtime.checkpoint`), never dlt resource state, so an LSN is never discarded before its
+destination load commits. A transaction larger than `target_batch_size` still lands whole
+(pgoutput never splits a commit). Kept from the #2851 pilot via the vendored consumer: true-PK
+override, unchanged-TOAST splice, `lsn` dedup_sort desc, `deleted_ts` hard_delete. An
+`append_only` relation merges on `lsn` instead (no key of its own), so a replay is exact.
+Two review additions (#3079): a relation's transactions below its baseline boundary are dropped
+(the baseline already holds them), and a TRUNCATE arrives on `Batch.truncates` for the runner
+to apply BEFORE the batch's rows.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Iterable, Iterator, Mapping, Sequence
+
+from dlt.sources.credentials import ConnectionStringCredentials
+
+from pg_replication.helpers import ItemGenerator, ReviewedLayout, advance_slot, get_max_lsn
+
+Relation = tuple[str, str]
+
+
+@dataclass
+class Batch:
+    """One drained batch: source schema -> [meta_item, marked_items, ...]."""
+
+    items_by_namespace: dict[str, list]
+    last_commit_lsn: int
+    consumed_all: bool
+    # TRUNCATEs in WAL order: {namespace, table, lsn, commit_lsn, commit_ts, applied}. Any rows
+    # of a keyed relation that preceded its truncate in this batch are already gone from
+    # `items_by_namespace`, so "delete, then load the batch" reproduces the source order.
+    truncates: list = field(default_factory=list)
+    # (schema, table) -> why a record's merge key could not be honoured (#4338); the runner
+    # reports each as an error event.
+    key_problems: dict = field(default_factory=dict)
+    # (oid, schema, table, column) the decoder matched to a denied NAME (#4346); the runner
+    # persists them before checkpointing so the deny follows that OID through a later rename.
+    denied_seen: set = field(default_factory=set)
+    # (schema, table) -> columns withheld because no reviewed signature names them (#4346).
+    unreviewed: dict = field(default_factory=dict)
+    # relation OID -> (schema, table, why) for each relation quarantined in this batch (#4346):
+    # a change arrived under a description that did not match its reviewed layout; nothing
+    # more of it loads. Keyed by OID (Codex round 17); the names are reporting metadata.
+    quarantined: dict = field(default_factory=dict)
+
+    @property
+    def quarantined_names(self) -> set:
+        """(schema, table) of every relation quarantined in this batch, for reports."""
+        return {(s, t) for s, t, _why in self.quarantined.values()}
+
+    @property
+    def is_empty(self) -> bool:
+        return not any(self.items_by_namespace.values()) and not self.truncates
+
+
+def credentials(dsn: str) -> ConnectionStringCredentials:
+    return ConnectionStringCredentials(dsn)
+
+
+def max_lsn(dsn: str, slot: str, publication: str) -> int | None:
+    """Highest LSN sitting in the slot, or None when the slot is empty. Does not consume."""
+    return get_max_lsn(slot, _options(publication), credentials(dsn))
+
+
+def _options(publication: str) -> dict[str, str]:
+    return {"publication_names": publication, "proto_version": "1"}
+
+
+def drain(
+    dsn: str,
+    slot: str,
+    publication: str,
+    start_lsn: int = 0,
+    upto_lsn: int | None = None,
+    *,
+    target_batch_size: int = 1000,
+    keys_by_relation: Mapping[Relation, Sequence[str]] | None = None,
+    policies_by_relation: Mapping[Relation, str] | None = None,
+    baseline_lsn_by_relation: Mapping[Relation, int] | None = None,
+    deny_columns_by_relation: Mapping[Relation, Sequence[str]] | None = None,
+    deny_columns_by_oid: Mapping[int, Sequence[str]] | None = None,
+    approved_columns_by_relation: Mapping[Relation, Sequence[str]] | None = None,
+    approved_columns_by_oid: Mapping[int, Sequence[str]] | None = None,
+    deny_attnums_by_oid: Mapping[int, Sequence[int]] | None = None,
+    reviewed_layout_by_oid: Mapping[int, ReviewedLayout] | None = None,
+    bound_oids: Iterable[int] | None = None,
+    quarantined_oids: Iterable[int] = (),
+) -> Iterator[Batch]:
+    """Iterate committed batches from `start_lsn` up to the slot's current max LSN.
+
+    Does NOT advance the slot — `runner` does that after the load commits. Restarting at a
+    checkpoint re-delivers the boundary commit (measured on PG 17); merge makes that harmless.
+    `deny_columns_by_relation` (#4346) is dropped by name from every record and from the
+    hints, so a denied column is never declared to dlt nor loaded. `deny_columns_by_oid` seeds
+    the per-OID deny that follows a renamed table; it is shared by every batch of this drain.
+    On a relation that carries a deny, each description must match its
+    `reviewed_layout_by_oid` entry exactly (helpers `_prove_layout`); any difference, or no
+    entry, quarantines it at its next change. `bound_oids` (the runner's: OIDs bound at run start or by a baseline)
+    quarantines any other OID met under a denied name; None leaves that check off.
+    `quarantined_oids` are skipped from the first batch on.
+    """
+    creds = credentials(dsn)
+    options = _options(publication)
+    if upto_lsn is None:
+        upto_lsn = get_max_lsn(slot, options, creds)
+    if upto_lsn is None:
+        return
+    cursor_lsn = start_lsn
+    by_oid = {int(k): frozenset(v) for k, v in (deny_columns_by_oid or {}).items() if v}
+    approved_by_oid = {int(k): frozenset(v) for k, v in (approved_columns_by_oid or {}).items()}
+    attnums_by_oid = {int(k): frozenset(v) for k, v in (deny_attnums_by_oid or {}).items() if v}
+    layouts = {int(k): v for k, v in (reviewed_layout_by_oid or {}).items()}
+    bound = None if bound_oids is None else frozenset(int(o) for o in bound_oids)
+    quarantined = {int(o) for o in quarantined_oids}   # shared: a quarantine is sticky
+    while True:
+        gen = ItemGenerator(
+            credentials=creds,
+            slot_name=slot,
+            options=options,
+            upto_lsn=upto_lsn,
+            start_lsn=cursor_lsn,
+            target_batch_size=target_batch_size,
+            include_columns=None,
+            columns=None,
+            keys_by_relation=dict(keys_by_relation or {}),
+            policies_by_relation=dict(policies_by_relation or {}),
+            baseline_lsn_by_relation=dict(baseline_lsn_by_relation or {}),
+            exclude_columns={k: tuple(v) for k, v in (deny_columns_by_relation or {}).items()},
+            exclude_columns_by_relid=by_oid,
+            approved_columns={k: frozenset(v)
+                              for k, v in (approved_columns_by_relation or {}).items()},
+            approved_by_relid=approved_by_oid,
+            exclude_attnums_by_relid=attnums_by_oid,
+            reviewed_layout_by_relid=layouts,
+            bound_relids=bound,
+            quarantined_relids=quarantined,
+        )
+        grouped = gen.collect()
+        if gen.last_commit_lsn is None:
+            # Backstop (#4345): the consumer stops only at a commit, so this cannot happen
+            # today. If it ever does, the batch is a partial transaction and there is no LSN to
+            # restart at — fail before the runner loads it, never loop on a `None` restart.
+            raise RuntimeError(
+                f"slot {slot}: replication stopped without reaching a commit (restarted at LSN "
+                f"{cursor_lsn}); refusing to load a partial transaction — see issue #4345")
+        yield Batch(grouped, gen.last_commit_lsn, gen.generated_all,
+                    truncates=list(gen.truncates),
+                    key_problems=dict(getattr(gen, "key_problems", {}) or {}),
+                    denied_seen=set(getattr(gen, "denied_seen", set()) or ()),
+                    unreviewed=dict(getattr(gen, "unreviewed", {}) or {}),
+                    quarantined=dict(getattr(gen, "quarantined", {}) or {}))
+        if gen.generated_all:
+            return
+        cursor_lsn = gen.last_commit_lsn
+
+
+def advance(dsn: str, slot: str, upto_lsn: int) -> None:
+    """Flush the slot up to an LSN whose destination load has ALREADY committed."""
+    advance_slot(upto_lsn, slot, credentials(dsn))