smart-data-engine-sdk 0.1.0.dev0__py3-none-any.whl

sde/engines/postgres.py

@@ -0,0 +1,672 @@
+"""PostgreSQL adapter.
+
+Two rules run through everything here, and both come straight from the requirements rather than
+from taste.
+
+**A failed write is reported, never worked around.** No retry into another engine, no swallowing,
+no "eventually consistent" story invented on the spot. If the source engine for a group will not
+take the write, the client's code finds out (:class:`~sde.errors.EngineError`). The library
+swallows its *own* internal problems - routing, telemetry - because a bug of ours must not take
+down someone's application, but a write that did not happen is not our internal problem and
+reporting success for it would be the single worst thing this library could do.
+
+**Identifiers are quoted, always.** Not for injection - identifiers come from the placement map,
+not from user input - but because entity names may contain non-ASCII characters, and an unquoted
+identifier in PostgreSQL is folded to lower case in a way that is lossy for some of them.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping, Sequence
+from contextlib import contextmanager
+from typing import Any
+
+from ..errors import EngineError
+from ..explain import (
+    Cost,
+    QueryPlan,
+    QueryPlanRefused,
+    postgres_findings,
+)
+from ..logging import log
+from ..migration import key_columns, same_width
+from ..placement import BACKFILL_TABLE, WATERMARK_TABLE, PhysicalLayout
+from ..schema import QUOTE, schema_statements
+
+__all__ = ["PostgresEngine"]
+
+
+# Bound from the one definition in sde.schema, so that DDL and DML cannot disagree about
+# how an identifier is escaped.
+_quote = QUOTE["postgres"]
+
+
+# Seconds. Not a guess about networks: this bounds *opening* a connection, which either completes
+# in milliseconds on a healthy link or is not going to complete. Ten seconds leaves room for a
+# saturated cross-region hop and still fails long before a request timeout that a caller sets. It
+# is a default rather than a rule - a `connect_timeout` in the DSN wins - and it exists because the
+# alternative, measured, is a call that never returns.
+CONNECT_TIMEOUT_SECONDS = 10
+
+
+class PostgresEngine:
+    """A thin adapter over psycopg. Deliberately thin: it executes decisions, it makes none."""
+
+    dialect = "postgres"
+
+    def __init__(self, dsn: str) -> None:
+        try:
+            import psycopg
+        except ImportError as exc:  # pragma: no cover - depends on the install extra
+            raise EngineError(
+                "the PostgreSQL adapter needs the 'postgres' extra: "
+                "pip install 'smart-data-engine-sdk[postgres]'. "
+                "The core library has no dependencies, because it goes into your application and "
+                "every dependency here would be one you inherit."
+            ) from exc
+        self._psycopg = psycopg
+        self._dsn = dsn
+        self._conn: Any = None
+
+    # --- connection ------------------------------------------------------------------------
+
+    def connect(self) -> None:
+        """Open the connection, with a bound on how long that may take.
+
+        Measured before this bound existed: a host that accepts the TCP connection and never
+        answers hung the call **for as long as the test was willing to wait**, because libpq has no
+        default ``connect_timeout`` and neither did we. That is not an exotic case - it is a
+        firewall that accepts, a load balancer with no healthy backend, a server mid-restart - and
+        without a bound it happens inside the caller's request path with nothing to time out.
+
+        The default is only applied when the caller has not chosen one. A ``connect_timeout`` in
+        the DSN is their decision about their own network and this must not override it.
+        """
+        if self._conn is None:
+            options: dict[str, Any] = {}
+            if "connect_timeout" not in self._dsn:
+                options["connect_timeout"] = CONNECT_TIMEOUT_SECONDS
+            try:
+                self._conn = self._psycopg.connect(self._dsn, autocommit=True, **options)
+            except Exception as exc:
+                raise EngineError(f"could not connect to PostgreSQL: {exc}") from exc
+
+    def close(self) -> None:
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    def __enter__(self) -> PostgresEngine:
+        self.connect()
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()
+
+    @property
+    def _cx(self) -> Any:
+        if self._conn is None:
+            raise EngineError("not connected; call connect() first")
+        return self._conn
+
+    def _explain(self, exc: Exception) -> str:
+        """The driver's message, plus the one sentence it cannot know to add.
+
+        Measured: cut the connection under a live session and the first failing call reports what
+        the server said ("terminating connection due to administrator command"), which is right.
+        **Every call after it reports "the connection is closed"** - true, unhelpful, and the point
+        at which a reader needs to be told that this library holds the connection it was handed and
+        does not reopen it. Reconnecting is one line and it is the caller's, because a library that
+        silently reconnected would also silently retry, and requirement 6.4 allows a retry only for
+        an operation known to be idempotent.
+        """
+        message = str(exc)
+        if self._conn is not None and getattr(self._conn, "closed", False):
+            message += (
+                ". The connection is gone and this library does not reopen one it was handed: "
+                "call close() then connect() on the engine, or hand the session a new one. "
+                "Nothing was retried, so no write reached the engine twice."
+            )
+        return message
+
+    # --- schema ----------------------------------------------------------------------------
+
+    def ensure_schema(self, layout: PhysicalLayout, *, keys: Mapping[str, Sequence[str]]) -> None:
+        """Create what is missing, change nothing that exists.
+
+        Idempotent on purpose: an application restarting must not reapply DDL, and two instances
+        starting at once must not race. Anything beyond creation - altering a column, dropping an
+        index - is a migration, which is the orchestrator's job and carries a safety
+        classification. A library that quietly altered a live column would be doing the one thing
+        this product promises never to do without a rollback path.
+        """
+        statements = schema_statements(layout, keys=keys, dialect=self.dialect)
+
+        with self._cx.cursor() as cur:
+            for statement in statements:
+                try:
+                    cur.execute(statement)
+                except Exception as exc:
+                    raise EngineError(f"schema statement failed: {statement}: {exc}") from exc
+        log("sde.schema.applied", engine=self.dialect, statements=len(statements))
+        self._verify_schema(layout)
+
+    def _verify_schema(self, layout: PhysicalLayout) -> None:
+        """Check that what exists is what the map describes, because IF NOT EXISTS does not.
+
+        `CREATE TABLE IF NOT EXISTS` accepts a table of that name whatever shape it is in, so a
+        table left over from something else - an older map, another application, a hand-run
+        migration - is silently kept and the first insert fails with `column "at" does not exist`.
+        That error names a column and not the cause, and it arrives in the client's request path
+        rather than at startup.
+
+        A **missing** column is refused: writes through this map cannot work. An **extra** column
+        is logged and allowed - a client may have added one outside SDE, the map does not name it,
+        writes are unaffected, and refusing would make this library an obstacle to work it has no
+        opinion about.
+
+        **Types as well as names, since 7 September 2026, and the reason the earlier version
+        checked names only was a true statement about the wrong catalogue.** It read
+        `information_schema.data_type`, which reports `numeric` for a `numeric(8,2)` column, and
+        concluded that comparing types would report differences that are not differences.
+        `pg_catalog.format_type(atttypid, atttypmod)` reports the canonical type *with* its
+        modifier - measured against every type this library renders, eleven of thirteen come back
+        as the exact string we wrote, and the two that do not are the timestamp aliases, which the
+        server itself will resolve for us.
+
+        What that cost while it was names-only, measured: a table whose `at` column is **`text`**
+        where the map says `timestamptz` passed this check and was reported as a good schema.
+        """
+        expected = {
+            table: dict(layout.columns.get(entity, {}))
+            for entity, table in sorted(layout.tables.items())
+        }
+        if not expected:
+            return
+
+        with self._cx.cursor() as cur:
+            # `pg_attribute` rather than `information_schema`, for `format_type`: the canonical
+            # spelling *including* the modifier, which is the whole reason this can compare types.
+            cur.execute(
+                "SELECT c.relname, a.attname, format_type(a.atttypid, a.atttypmod) "
+                "FROM pg_attribute a "
+                "JOIN pg_class c ON c.oid = a.attrelid "
+                "JOIN pg_namespace n ON n.oid = c.relnamespace "
+                "WHERE n.nspname = current_schema() AND c.relname = ANY(%s) "
+                "AND a.attnum > 0 AND NOT a.attisdropped",
+                [sorted(expected)],
+            )
+            found: dict[str, dict[str, str]] = {}
+            for table_name, column_name, column_type in cur.fetchall():
+                found.setdefault(str(table_name), {})[str(column_name)] = str(column_type)
+
+        for table, columns in sorted(expected.items()):
+            actual = found.get(table)
+            if actual is None:
+                raise EngineError(
+                    f"{table!r} does not exist after applying the schema. The statement reported "
+                    f"success, so this is a permissions or search_path problem rather than a bad "
+                    f"map."
+                )
+            missing = sorted(set(columns) - set(actual))
+            if missing:
+                raise EngineError(
+                    f"{table!r} already existed with a different shape: the map needs {missing} "
+                    f"and the table has {sorted(actual)}. `CREATE TABLE IF NOT EXISTS` keeps "
+                    f"whatever is there, so this table came from somewhere else - an older map, "
+                    f"another application, a migration run by hand. Refusing here rather than at "
+                    f"the first insert, which would fail in your request path with an error naming "
+                    f"a column and not the cause."
+                )
+            for column, declared in sorted(columns.items()):
+                reported = actual[column]
+                if self._same_type(declared, reported):
+                    continue
+                raise EngineError(
+                    f"{table}.{column} is {reported!r} and this map declares it {declared!r}. "
+                    f"`CREATE TABLE IF NOT EXISTS` keeps a table of that name whatever shape it "
+                    f"is in, and this library never alters a column's type - so the table came "
+                    f"from somewhere else, or from a map that rendered this column differently. "
+                    f"Refusing rather than writing into it: a type that differs is either a write "
+                    f"that fails in your request path or, worse, one that succeeds and hands the "
+                    f"value back as something else."
+                )
+            extra = sorted(set(actual) - set(columns))
+            if extra:
+                log("sde.schema.extra_columns", table=table, columns=extra)
+
+    def _same_type(self, declared: str, reported: str) -> bool:
+        """Whether two PostgreSQL type spellings denote the same type. Asked of the server.
+
+        Literal first, because that is the answer for every type this library renders except the
+        two timestamps. When it fails, the server is asked - `to_regtype` resolves an alias to the
+        type it names, so `timestamptz` and `timestamp with time zone` come back equal without this
+        module holding a table of aliases that could fall behind the renderer.
+
+        **A modifier makes a literal mismatch a real one.** `to_regtype` discards modifiers, so
+        `numeric(12,2)` and `numeric(8,2)` would both resolve to `numeric` and a precision change
+        would read as agreement. That is the one difference this check exists to catch, so a
+        parenthesis on either side ends the question here.
+        """
+        if declared == reported:
+            return True
+        if "(" in declared or "(" in reported:
+            return False
+        with self._cx.cursor() as cur:
+            cur.execute("SELECT to_regtype(%s)::text, to_regtype(%s)::text", [declared, reported])
+            row = cur.fetchone()
+        return bool(row is not None and row[0] is not None and row[0] == row[1])
+
+
+    # --- data ------------------------------------------------------------------------------
+
+    def explain_plan(self, sql: str) -> QueryPlan:
+        """Plan an analyst's query without running it, inside a read-only transaction.
+
+        Requirement 19.4. Two ``EXPLAIN``s in one transaction: ``FORMAT JSON`` for the numbers and
+        the node types, and the plain one for the text a person reads - the engine's own rendering
+        rather than mine, because a plan reformatted by us is a plan whose wording somebody will
+        compare against the documentation and not find.
+
+        ``SET TRANSACTION READ ONLY``, and a **corrected** account of what it is for. The first
+        version of this docstring said planning could reach a write, because constant folding
+        evaluates immutable functions - ``EXPLAIN SELECT 1/0`` raises at plan time - so an
+        immutable function that lied about being immutable would run. Measured: it cannot.
+        PostgreSQL refuses the write itself, with *"INSERT is not allowed in a non-volatile
+        function"*, in a read-only transaction and outside one alike. And a ``VOLATILE`` function
+        is not folded at all. So **planning cannot write, and the read-only transaction guards
+        against nothing reachable through ``EXPLAIN`` today.**
+
+        It is kept, and the reason is an edit inside this block rather than a query outside it. One
+        word - ``ANALYZE`` - turns either statement below into an execution, and with the
+        transaction read-only that mistake fails loudly instead of running an analyst's
+        data-modifying CTE. A guard whose present hazard is zero and whose future hazard is one
+        keyword is worth a statement.
+
+        ``force_rollback=True`` has **no observable effect through EXPLAIN** and is kept for the
+        same reason one level along: it covers what read-only does not - a temporary table, a
+        ``SET LOCAL``, an advisory lock - none of which anything here creates today. Said plainly
+        rather than counted as a tested guarantee, because a mutation of it changes no output and
+        a mutation that changes no output is not one.
+
+        What gets reported is the transaction, not a claim about the query: this library has no SQL
+        parser and cannot say it checked the SQL.
+        """
+        with self._cx.transaction(force_rollback=True) as _tx, self._cx.cursor() as cursor:
+            cursor.execute("SET TRANSACTION READ ONLY")
+            # Read it back, because a statement that says nothing does not mean something
+            # happened - the lesson three GitHub API endpoints taught this project by returning
+            # 200 and changing nothing. It also turns this guard from unverifiable into checked:
+            # without the read-back, deleting the line above changes no observable behaviour, so
+            # nothing could fail when somebody did.
+            cursor.execute("SHOW transaction_read_only")
+            state = cursor.fetchone()
+            if not state or str(state[0]).strip().lower() != "on":
+                raise EngineError(
+                    f"this connection would not go read-only for planning: "
+                    f"transaction_read_only is {state[0] if state else 'unreadable'!r}. Refusing "
+                    f"to plan anything: requirement 19.2 keeps this product out of the data path, "
+                    f"and the one keyword between EXPLAIN and execution is ANALYZE - measured, an "
+                    f"EXPLAIN (ANALYZE) of a write runs in a read-write transaction and is "
+                    f"refused in a read-only one."
+                )
+            try:
+                cursor.execute(f"EXPLAIN (FORMAT JSON) {sql}")
+                raw = cursor.fetchone()
+                cursor.execute(f"EXPLAIN {sql}")
+                text = tuple(str(row[0]) for row in cursor.fetchall())
+            except Exception as exc:
+                # The refusal *is* the validation. Re-raised with the engine's own wording,
+                # because the case requirement 19.4 exists for - a column our map says exists
+                # and this database says does not - is one where the engine names the column
+                # and any summary of ours would lose it.
+                raise QueryPlanRefused(
+                    f"{self.dialect} would not plan this query: {exc} Nothing was executed. "
+                    f"This is what validating against a live engine means: the control plane "
+                    f"checked the query against the schema it authored, and this is the schema "
+                    f"that exists."
+                ) from exc
+        if not raw or not raw[0]:
+            raise QueryPlanRefused(
+                f"{self.dialect} returned no plan for this query and did not say why"
+            )
+        node = dict(raw[0][0]).get("Plan") or {}
+        return QueryPlan(
+            engine=self.dialect,
+            dialect=self.dialect,
+            plan=text or ("(the engine returned an empty plan)",),
+            cost=Cost(
+                units="PostgreSQL planner cost units",
+                basis=(
+                    "arbitrary units, not seconds and not rows. The figure depends on this "
+                    "machine and on this database's planner settings (seq_page_cost, "
+                    "random_page_cost, the cost constants), so it is comparable between two plans "
+                    "on this database and meaningless against a number from anywhere else. "
+                    "plan_rows is the planner's estimate of rows returned, from statistics that "
+                    "are as fresh as the last ANALYZE."
+                ),
+                values={
+                    "total_cost": str(node.get("Total Cost", "")),
+                    "startup_cost": str(node.get("Startup Cost", "")),
+                    "plan_rows": str(node.get("Plan Rows", "")),
+                    "plan_width": str(node.get("Plan Width", "")),
+                    "root_node": str(node.get("Node Type", "")),
+                },
+            ),
+            findings=postgres_findings(node),
+            read_only_enforced=(
+                "SET TRANSACTION READ ONLY, and the transaction is rolled back. PostgreSQL "
+                "refuses a write in such a transaction whatever the statement looks like - "
+                "measured: a data-modifying CTE fails with 'cannot execute SELECT in a read-only "
+                "transaction'."
+            ),
+        )
+
+    def insert(self, table: str, values: Mapping[str, Any]) -> None:
+        if not values:
+            raise EngineError("nothing to insert")
+        cols = sorted(values)
+        placeholders = ", ".join(["%s"] * len(cols))
+        sql = (
+            f"INSERT INTO {_quote(table)} ({', '.join(_quote(c) for c in cols)}) "
+            f"VALUES ({placeholders})"
+        )
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(sql, [values[c] for c in cols])
+        except Exception as exc:
+            # Surfaced, not swallowed and not rerouted. See the module docstring.
+            log("sde.write.failed", table=table, error=type(exc).__name__)
+            raise EngineError(f"insert into {table} failed: {self._explain(exc)}") from exc
+
+    def get(self, table: str, key: Mapping[str, Any]) -> dict[str, Any] | None:
+        where = " AND ".join(f"{_quote(c)} = %s" for c in sorted(key))
+        sql = f"SELECT * FROM {_quote(table)} WHERE {where}"
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(sql, [key[c] for c in sorted(key)])
+                row = cur.fetchone()
+                if row is None:
+                    return None
+                names = [d.name for d in cur.description or ()]
+                return dict(zip(names, row, strict=True))
+        except Exception as exc:
+            raise EngineError(f"select from {table} failed: {self._explain(exc)}") from exc
+
+    # --- rollback protection ------------------------------------------------------------------
+    #
+    # Two methods that satisfy `sde.watermark.WatermarkStore`, which is a separate optional
+    # protocol rather than part of `Engine`: adding these to `Engine` would break every adapter
+    # anybody has written, for a capability our own orderbook engine cannot provide.
+
+    def map_watermark(self) -> int | None:
+        """The highest map version applied against this engine, creating the table if missing.
+
+        Creating on read rather than on write, and that closes a real gap: with the table appearing
+        only on the first write, the first load of a signed map has nothing to compare against and a
+        file swapped immediately after a deployment goes unnoticed. Reading is also the cheaper of
+        the two paths to make idempotent - `CREATE TABLE IF NOT EXISTS` here costs one statement per
+        session, once per process.
+        """
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(
+                    f"CREATE TABLE IF NOT EXISTS {_quote(WATERMARK_TABLE)} ("
+                    f"{_quote('map_version')} bigint NOT NULL, "
+                    f"{_quote('model_version')} text NOT NULL, "
+                    f"{_quote('seen_at')} timestamptz NOT NULL DEFAULT now())"
+                )
+                cur.execute(f"SELECT max({_quote('map_version')}) FROM {_quote(WATERMARK_TABLE)}")
+                row = cur.fetchone()
+        except Exception as exc:
+            raise EngineError(f"reading {WATERMARK_TABLE} failed: {self._explain(exc)}") from exc
+        if row is None or row[0] is None:
+            return None
+        return int(row[0])
+
+    def record_map_version(self, version: int, *, model_version: str) -> None:
+        """Append. Never update, so there is nothing to contend over and nothing to lose.
+
+        The timestamp comes from the engine's own `now()` rather than from this process: an audit
+        column wants the clock of the thing being audited, and this library reading a clock is a
+        thing its tests would then have to work around.
+        """
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(
+                    f"INSERT INTO {_quote(WATERMARK_TABLE)} "
+                    f"({_quote('map_version')}, {_quote('model_version')}) VALUES (%s, %s)",
+                    [version, model_version],
+                )
+        except Exception as exc:
+            raise EngineError(
+                f"recording a map version in {WATERMARK_TABLE} failed: {exc}"
+            ) from exc
+
+    def range(
+        self,
+        table: str,
+        column: str,
+        *,
+        low: Any = None,
+        high: Any = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        clauses: list[str] = []
+        params: list[Any] = []
+        if low is not None:
+            clauses.append(f"{_quote(column)} >= %s")
+            params.append(low)
+        if high is not None:
+            clauses.append(f"{_quote(column)} < %s")
+            params.append(high)
+        where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
+        order = f" ORDER BY {_quote(column)}"
+        cap = ""
+        if limit is not None:
+            cap = " LIMIT %s"
+            params.append(limit)
+        sql = f"SELECT * FROM {_quote(table)}{where}{order}{cap}"
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(sql, params)
+                names = [d.name for d in cur.description or ()]
+                return [dict(zip(names, row, strict=True)) for row in cur.fetchall()]
+        except Exception as exc:
+            raise EngineError(f"range select from {table} failed: {exc}") from exc
+
+    def count(self, table: str) -> int:
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(f"SELECT count(*) FROM {_quote(table)}")
+                row = cur.fetchone()
+                return int(row[0]) if row else 0
+        except Exception as exc:
+            raise EngineError(f"count on {table} failed: {exc}") from exc
+
+    # --- migration ------------------------------------------------------------------------------
+    #
+    # Five methods that satisfy `sde.migration.Migratable`, which like `WatermarkStore` is a
+    # separate optional protocol. Same reason: our own orderbook engine cannot offer any of them -
+    # its schema is fixed in its own source and it has nowhere to keep a marker - and an engine that
+    # cannot take part in a migration should be a named refusal rather than a broken adapter.
+
+    def key_range(
+        self,
+        table: str,
+        order: Sequence[str],
+        *,
+        after: Sequence[Any] | None = None,
+        upto: Sequence[Any] | None = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        """Rows in key order, strictly after one key and up to another inclusive.
+
+        Row-value comparison - ``(a, b) > (%s, %s)`` - rather than a hand-rolled disjunction over
+        the key's columns. The disjunction is where composite-key pagination goes wrong, and it goes
+        wrong by skipping rows.
+
+        The bounds are asymmetric on purpose. ``after`` is exclusive because it is a resume point:
+        the row it names has been dealt with. ``upto`` is inclusive because it names the last row of
+        a chunk read from somewhere else, and that row is one this range has to include.
+        """
+        cols = key_columns(order, table)
+        clauses: list[str] = []
+        params: list[Any] = []
+        tuple_expr = f"({', '.join(_quote(c) for c in cols)})"
+        if after is not None:
+            same_width(after, cols, "after")
+            clauses.append(f"{tuple_expr} > ({', '.join(['%s'] * len(cols))})")
+            params.extend(after)
+        if upto is not None:
+            same_width(upto, cols, "upto")
+            clauses.append(f"{tuple_expr} <= ({', '.join(['%s'] * len(cols))})")
+            params.extend(upto)
+        where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
+        cap = ""
+        if limit is not None:
+            cap = " LIMIT %s"
+            params.append(int(limit))
+        sql = f"SELECT * FROM {_quote(table)}{where} ORDER BY {tuple_expr}{cap}"
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(sql, params)
+                names = [d.name for d in cur.description or ()]
+                return [dict(zip(names, row, strict=True)) for row in cur.fetchall()]
+        except Exception as exc:
+            raise EngineError(f"key range select from {table} failed: {exc}") from exc
+
+    def nth_key(
+        self, table: str, order: Sequence[str], *, position: int
+    ) -> tuple[Any, ...] | None:
+        """The key of the ``position``-th row in key order, one-based, or None if there is no such
+        row.
+
+        An ``OFFSET`` scan, which is the expensive kind of query, and it is here because it is paid
+        **once per resume** rather than once per chunk. See :mod:`sde.migration` for why the marker
+        is a row count and not a key.
+        """
+        cols = key_columns(order, table)
+        if position < 1:
+            raise EngineError(f"position is one-based; {position} is not a row")
+        projection = ", ".join(_quote(c) for c in cols)
+        sql = (
+            f"SELECT {projection} FROM {_quote(table)} ORDER BY ({projection}) "
+            f"OFFSET %s LIMIT 1"
+        )
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(sql, [position - 1])
+                row = cur.fetchone()
+        except Exception as exc:
+            raise EngineError(f"reading row {position} of {table} failed: {exc}") from exc
+        return None if row is None else tuple(row)
+
+    def copy_in(self, table: str, rows: Sequence[Mapping[str, Any]]) -> None:
+        """Insert rows, skipping any whose key is already there.
+
+        ``ON CONFLICT DO NOTHING`` is what makes a backfill chunk **idempotent**, and idempotence is
+        what makes it resumable: the marker is written after the chunk, so a crash in between costs
+        a recopy and never a lost row. Without it the recopy would be a primary-key violation and
+        the safe failure mode would become the loud one.
+
+        The bare form, with no conflict target, so it covers the primary key and any unique index
+        the layout asked for. Naming the key here would mean deriving it a second time, and two
+        derivations of one key is how they come to disagree.
+
+        Returns nothing on purpose. PostgreSQL can say how many rows it actually wrote and
+        ClickHouse cannot, so a count here would mean different things in different engines - and
+        the caller needs "how much of the source have I consumed", which it already knows.
+        """
+        if not rows:
+            return
+        cols = sorted(rows[0])
+        for row in rows:
+            if sorted(row) != cols:
+                raise EngineError(
+                    f"copy_in into {table} was given rows with different columns "
+                    f"({cols} and {sorted(row)}). A chunk comes from one table, so this is a "
+                    f"caller assembling it from two."
+                )
+        placeholders = ", ".join(f"({', '.join(['%s'] * len(cols))})" for _ in rows)
+        sql = (
+            f"INSERT INTO {_quote(table)} ({', '.join(_quote(c) for c in cols)}) "
+            f"VALUES {placeholders} ON CONFLICT DO NOTHING"
+        )
+        params: list[Any] = []
+        for row in rows:
+            params.extend(row[c] for c in cols)
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(sql, params)
+        except Exception as exc:
+            log("sde.write.failed", table=table, error=type(exc).__name__)
+            raise EngineError(f"copying {len(rows)} rows into {table} failed: {exc}") from exc
+
+    def backfill_marker(self, *, materialization: str, entity: str) -> int:
+        """How many rows of this entity have been copied into this engine. Zero if none.
+
+        `max()` over an append-only table, exactly like the map watermark, and for the same reason:
+        no row to update, nothing to contend over, and identical semantics in an engine with no
+        unique constraint. A stale row can never lower the marker.
+        """
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(
+                    f"CREATE TABLE IF NOT EXISTS {_quote(BACKFILL_TABLE)} ("
+                    f"{_quote('materialization')} text NOT NULL, "
+                    f"{_quote('entity')} text NOT NULL, "
+                    f"{_quote('rows_copied')} bigint NOT NULL, "
+                    f"{_quote('at')} timestamptz NOT NULL DEFAULT now())"
+                )
+                cur.execute(
+                    f"SELECT max({_quote('rows_copied')}) FROM {_quote(BACKFILL_TABLE)} "
+                    f"WHERE {_quote('materialization')} = %s AND {_quote('entity')} = %s",
+                    [materialization, entity],
+                )
+                row = cur.fetchone()
+        except Exception as exc:
+            raise EngineError(f"reading {BACKFILL_TABLE} failed: {exc}") from exc
+        if row is None or row[0] is None:
+            return 0
+        return int(row[0])
+
+    def record_backfill_marker(
+        self, *, materialization: str, entity: str, rows: int
+    ) -> None:
+        """Append the new marker. Never update, so an interrupted run leaves a readable trail."""
+        try:
+            with self._cx.cursor() as cur:
+                cur.execute(
+                    f"INSERT INTO {_quote(BACKFILL_TABLE)} ({_quote('materialization')}, "
+                    f"{_quote('entity')}, {_quote('rows_copied')}) VALUES (%s, %s, %s)",
+                    [materialization, entity, int(rows)],
+                )
+        except Exception as exc:
+            raise EngineError(
+                f"recording backfill progress in {BACKFILL_TABLE} failed: {exc}"
+            ) from exc
+
+    # --- transactions ----------------------------------------------------------------------
+
+    @contextmanager
+    def transaction(self) -> Iterator[PostgresEngine]:
+        """One engine, one transaction, that engine's semantics.
+
+        There is no distributed transaction here and there will not be one. A client needing two
+        entities to commit together declares that, and the planner puts them in the same group and
+        therefore the same engine - so the requirement turns into a placement constraint instead
+        of a two-phase commit. That is the trade this product makes, and it is why this method is
+        four lines rather than a subsystem.
+        """
+        cx = self._cx
+        previous = cx.autocommit
+        cx.autocommit = False
+        try:
+            yield self
+            cx.commit()
+        except Exception:
+            cx.rollback()
+            raise
+        finally:
+            cx.autocommit = previous