smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/engines/clickhouse.py

@@ -0,0 +1,1115 @@
+"""ClickHouse adapter.
+
+The second engine, and the reason there is a product at all: between PostgreSQL and ClickHouse
+lies the decision clients get wrong once, at the start, and never revisit. With one adapter the
+planner chooses from a set of size one.
+
+Three things here are genuinely different from the PostgreSQL adapter, and none of them is a
+detail. Each is a place where "the same operation" means something else because the engine is
+different, and the product's position is that such a difference must be *stated* rather than
+smoothed over.
+
+**A naive datetime is UTC. Always, explicitly, here.**
+
+A fix for a divergence that was measured rather than imagined. `datetime(2026, 8, 27, 12, 0)` with
+no tzinfo, written through this library:
+
+- into PostgreSQL `timestamptz`, it arrives as ``12:00:00+00:00`` - into ClickHouse
+``DateTime64(3, 'UTC')`` with the driver left to its own devices, it arrived as
+  ``10:00:00`` - because `clickhouse-connect` reads a naive datetime as *local* time and converts
+
+Two hours apart, from one call, in one library, with no error anywhere. After a migration from one
+engine to the other every timestamp in the client's analytics would shift by the offset of
+whichever machine happened to write the row. So naive datetimes are given `timezone.utc` before
+they reach the driver, which is what the PostgreSQL path already does in effect. If you mean a
+different zone, pass an aware datetime and it is respected.
+
+**There are no transactions, and this adapter says so instead of pretending.**
+
+`transaction()` raises. Not a no-op context manager: a caller who believes they are in a
+transaction and is not has been lied to at the worst possible moment. A client who needs two
+entities to commit together declares that, and the planner then places them in the same group and
+therefore the same engine - the requirement becomes a placement constraint. This method existing
+and refusing is how that constraint is discovered at the point of use rather than inferred from a
+rollback that never happened.
+
+**Keys are not enforced, so the table is a `ReplacingMergeTree` and point reads use `FINAL`.**
+
+A MergeTree does not enforce uniqueness on its `ORDER BY`. So a second `save()` of the same key
+does not raise the way PostgreSQL's primary key does - and that divergence cannot be removed, only
+chosen. Plain `MergeTree` would leave two rows and make every aggregate over that group quietly
+wrong; `ReplacingMergeTree` keeps the newest and collapses the rest at merge time. Of the two,
+"the newest row for a key wins" is something a client can reason about. `FINAL` on point reads and
+counts is what makes the collapse visible immediately rather than eventually, and it is not free -
+which is the trade, stated.
+
+The residue is worth naming plainly, because a client will meet it: in PostgreSQL, saving the same
+key twice is an error; here it is an overwrite. That belongs in a placement decision rather than
+in an adapter, and there is no declaration for it yet.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import math
+import os
+import re
+import ssl
+import stat
+from collections.abc import Iterator, Mapping, Sequence
+from importlib.metadata import version
+from typing import Any, cast
+
+from .._usage import UsageGate, guarded
+from ..bulk import batch_columns
+from ..errors import EngineError
+from ..explain import (
+    Cost,
+    QueryPlan,
+    QueryPlanRefused,
+    replacing_merge_tree_finding,
+)
+from ..logging import log
+from ..migration import key_columns, same_width
+from ..physical import (
+    PhysicalFinding,
+    declared_tables,
+    parse_identifier_list,
+    parse_partition_key,
+)
+from ..placement import BACKFILL_TABLE, WATERMARK_TABLE, PhysicalLayout
+from ..query import ReadColumn, ReadPlan, read_row, read_sql, summary_sql
+from ..schema import QUOTE, schema_statements
+from ..write_fence import WriteFence
+from ._clickhouse_connection import (
+    CONNECT_TIMEOUT_SECONDS as CONNECT_TIMEOUT_SECONDS,
+)
+from ._clickhouse_connection import (
+    HANDSHAKE_TIMEOUT_SECONDS as HANDSHAKE_TIMEOUT_SECONDS,
+)
+from ._clickhouse_connection import (
+    ConnectionParameters,
+    parse_dsn,
+)
+from ._storage import STORAGE_COLUMNS
+from ._write_fences import ClickHouseFences
+
+__all__ = ["STORAGE_COLUMNS", "ClickHouseEngine"]
+
+
+# Bound from the one definition in sde.schema, so that DDL and DML cannot disagree about
+# how an identifier is escaped.
+_quote = QUOTE["clickhouse"]
+
+
+def _as_utc(value: Any) -> Any:
+    """A naive datetime is UTC. See the module docstring for the measurement behind this."""
+    if isinstance(value, _dt.datetime) and value.tzinfo is None:
+        return value.replace(tzinfo=_dt.UTC)
+    return value
+
+
+def _query_parameter(name: str, value: Any, parameters: dict[str, Any]) -> str:
+    """Bind a scalar without letting the driver's datetime formatter discard its fraction.
+
+    The plain datetime formatter emits seconds. Use a bound string and an explicit UTC
+    DateTime64 conversion for temporal comparisons, independent of the driver's formatting zone.
+    Data values still go through the driver's parameter binding; none are interpolated into SQL.
+    """
+    placeholder = f"%({name})s"
+    if isinstance(value, _dt.datetime):
+        utc = _as_utc(value).astimezone(_dt.UTC)
+        parameters[name] = utc.replace(tzinfo=None).isoformat(" ", timespec="microseconds")
+        return f"toDateTime64({placeholder}, 6, 'UTC')"
+    parameters[name] = value
+    return placeholder
+
+
+def _row(names: Sequence[str], types: Sequence[Any], row: Sequence[Any]) -> dict[str, Any]:
+    """One result row as a dict, with the timezone put back on values that had one in the schema.
+
+    The write side of this module makes a naive datetime UTC. Without this, the read side would
+    not give it back: `clickhouse-connect` returns a *naive* datetime for a `DateTime64(3, 'UTC')`
+    column, while psycopg returns an aware one for a `timestamptz`. So the same entity read from
+    the two engines produces two datetimes that Python refuses to compare - ``can't compare
+    offset-naive and offset-aware datetimes`` - which is a `TypeError` in a client's code that
+    appears on the day a group is moved and not before.
+
+    The column type carries the zone, so it is read from there rather than assumed: a column
+    declared with a timezone yields aware values, one declared without yields naive, which is
+    exactly the distinction the neutral vocabulary makes between `timestamptz` and `timestamp`.
+    """
+    out: dict[str, Any] = {}
+    for name, column_type, value in zip(names, types, row, strict=True):
+        zone = getattr(column_type, "tzinfo", None)
+        if zone is not None and isinstance(value, _dt.datetime) and value.tzinfo is None:
+            value = value.replace(tzinfo=zone)
+        out[name] = value
+    return out
+
+
+class _NoReplayTransport:
+    """Keep the driver's TLS/proxy pool but forbid ambiguous transport replay for this client.
+
+    clickhouse-connect retries RemoteDisconnected even when its retry count is zero. In a live
+    probe that error followed an accepted INSERT: the driver replayed it and returned success.
+    Translate transport errors before that retry handler, and disable urllib3's own retry layer.
+    The shared pool is never patched; only this client's reference is wrapped.
+    """
+
+    def __init__(self, pool: Any) -> None:
+        self.pool = pool
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self.pool, name)
+
+    # The driver keys pool-expiration bookkeeping by manager. Preserve that identity instead of
+    # registering a new shared-pool owner on every connection (and clearing its active sockets).
+    def __hash__(self) -> int:
+        return hash(self.pool)
+
+    def __eq__(self, other: object) -> bool:
+        return self.pool is (other.pool if isinstance(other, _NoReplayTransport) else other)
+
+    def request(self, method: str, url: str, **options: Any) -> Any:
+        from urllib3.exceptions import HTTPError
+
+        options["retries"] = False
+        options["redirect"] = False
+        try:
+            return self.pool.request(method, url, **options)
+        except HTTPError as exc:
+            raise EngineError(
+                "ClickHouse transport failed; the operation was not replayed and its outcome "
+                "may be unknown"
+            ) from exc
+
+
+MIN_CLICKHOUSE_CONNECT = "1.7.2"
+
+
+def _require_driver_version(installed: str) -> None:
+    parts = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:\.post\d+)?", installed)
+    if parts is None or tuple(int(part) for part in parts.groups()) < tuple(
+        int(part) for part in MIN_CLICKHOUSE_CONNECT.split(".")
+    ):
+        raise EngineError(
+            f"ClickHouse requires stable clickhouse-connect >= {MIN_CLICKHOUSE_CONNECT}; "
+            f"installed {installed}. Older drivers do not meet the timestamp and structured "
+            "error-code contract. Install the SDK clickhouse extra in this environment."
+        )
+
+
+MAX_CA_BYTES = 1024 * 1024
+
+
+def _ca_context(path: str | None) -> ssl.SSLContext | None:
+    """Read and qualify private per-client trust before creating any driver connection."""
+    if path is None:
+        return None
+    try:
+        flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_CLOEXEC", 0)
+        fd = os.open(path, flags)
+        try:
+            info = os.fstat(fd)
+            if not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= MAX_CA_BYTES:
+                raise ValueError("CA must be a bounded regular file")
+            pieces: list[bytes] = []
+            size = 0
+            while size <= MAX_CA_BYTES:
+                piece = os.read(fd, min(65536, MAX_CA_BYTES + 1 - size))
+                if not piece:
+                    break
+                pieces.append(piece)
+                size += len(piece)
+            payload = b"".join(pieces)
+        finally:
+            os.close(fd)
+        if not payload or len(payload) > MAX_CA_BYTES:
+            raise ValueError("CA must be nonempty and at most 1 MiB")
+        context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        context.load_verify_locations(cadata=payload.decode("ascii"))
+        if not context.cert_store_stats()["x509"]:
+            raise ValueError("empty CA store")
+        return context
+    except (OSError, ValueError, UnicodeError):
+        raise EngineError(
+            "ClickHouse CA file must contain readable valid PEM certificates within 1 MiB"
+        ) from None
+
+
+def _connect_client(parameters: ConnectionParameters) -> Any:
+    from clickhouse_connect.driver.httpclient import HttpClient
+    from clickhouse_connect.driver.query import TzSource
+    from urllib3 import Timeout
+
+    context = _ca_context(parameters.ca_cert)
+
+    class GuardedHttpClient(HttpClient):
+        def _init_common_settings(self, tz_source: TzSource) -> None:
+            # HttpClient calls this after constructing its native pool/backend and before the
+            # initial version/settings exchange. Change only this client's pool reference.
+            try:
+                self.http = cast(Any, _NoReplayTransport(self.http))
+                self.timeout = Timeout(
+                    connect=parameters.connect_timeout, read=parameters.send_receive_timeout
+                )
+                if context is not None:
+                    if not self._owns_pool_manager:
+                        raise EngineError("The driver did not create an isolated CA pool")
+                    # The native factory owns this pool; pin the already validated CA bytes so
+                    # later sockets do not reread a changed path. Never alter the shared pool.
+                    pool_options = self.http.connection_pool_kw
+                    pool_options["ssl_context"] = context
+                    pool_options.pop("ca_certs", None)
+                super()._init_common_settings(tz_source)
+            except BaseException as exc:
+                try:
+                    self.close()
+                except BaseException:
+                    exc.add_note("ClickHouse client cleanup also failed after startup.")
+                raise
+
+    # Driver 1.7.2 coerces timeout inputs to integers before constructing its backend. Positive
+    # placeholders permit construction; the scoped hook installs the exact requested fractions
+    # before I/O. The URI itself is never reparsed or used as an arbitrary kwargs channel.
+    host = f"[{parameters.host}]" if ":" in parameters.host else parameters.host
+    return GuardedHttpClient(
+        interface=parameters.interface,
+        host=host,
+        port=parameters.port,
+        username=parameters.username,
+        password=parameters.password,
+        database=parameters.database,
+        verify=True,
+        ca_cert=parameters.ca_cert,
+        query_retries=0,
+        connect_timeout=math.ceil(parameters.connect_timeout),
+        send_receive_timeout=math.ceil(parameters.send_receive_timeout),
+    )
+
+
+class ClickHouseEngine:
+    """A thin adapter over clickhouse-connect. Executes decisions, makes none."""
+
+    dialect = "clickhouse"
+
+    def __init__(self, dsn: str) -> None:
+        self._connection = parse_dsn(dsn)
+        try:
+            import clickhouse_connect  # noqa: F401 -- validate the optional driver extra
+        except ImportError as exc:  # pragma: no cover - depends on the install extra
+            raise EngineError(
+                "the ClickHouse adapter needs the 'clickhouse' extra: "
+                "pip install 'smart-data-engine-sdk[clickhouse]'. "
+                "The core library has no dependencies, because it goes into your application and "
+                "every dependency here would be one you inherit."
+            ) from exc
+        _require_driver_version(version("clickhouse-connect"))
+        self._dsn = dsn
+        self._client: Any = None
+        self._usage = UsageGate()
+
+    # --- connection ------------------------------------------------------------------------
+
+    @guarded
+    def connect(self) -> None:
+        """Open the client, with a bound on how long that may take.
+
+        Same measurement as the PostgreSQL adapter, and the same absence: a host that accepts the
+        connection and never answers hung for as long as the test would wait. Here the reason is
+        one layer up - the TCP connect succeeds, so ``connect_timeout`` never fires, and what is
+        left is ``send_receive_timeout``, whose driver default is 300 seconds.
+
+        Both are transport timers, with distinct defaults for opening and the initial exchange.
+        Exact caller-supplied seconds survive the driver's integer constructor coercion. Socket
+        inactivity is not an absolute query deadline: progress received from a long-running query
+        can keep that exchange alive under the driver's existing transport behavior.
+        """
+        if self._client is None:
+            try:
+                self._client = _connect_client(self._connection)
+            except EngineError as exc:
+                raise EngineError("could not connect to ClickHouse: " + str(exc)) from None
+            except Exception:
+                # Driver/parser exceptions may echo URL, credentials or server content. The
+                # connection error conveys no proof of rollback and never weakens TLS to retry.
+                raise EngineError(
+                    "could not connect to ClickHouse with the configured transport"
+                ) from None
+
+    @guarded
+    def close(self) -> None:
+        if self._client is not None:
+            self._client.close()
+            self._client = None
+
+    def __enter__(self) -> ClickHouseEngine:
+        self.connect()
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()
+
+    @property
+    def _cx(self) -> Any:
+        if self._client is None:
+            raise EngineError("not connected; call connect() first")
+        return self._client
+
+    def write_fence(self, table: str, *, project_id: str) -> WriteFence:
+        """DDL capability; use a dedicated connection separate from application traffic."""
+        return WriteFence(ClickHouseFences(self._cx), table, project_id=project_id)
+
+    # --- schema ----------------------------------------------------------------------------
+
+    @guarded
+    def ensure_schema(
+        self, layout: PhysicalLayout, *, keys: Mapping[str, Sequence[str]]
+    ) -> tuple[PhysicalFinding, ...]:
+        """Create what is missing, change nothing that exists.
+
+        `ORDER BY` is the declared key, in declared order. That order is positional and carries
+        meaning: it decides which prefixes of the key can prune granules, so sorting it would
+        change the physical performance of the table while leaving the map looking identical.
+
+        Data-skipping indexes, when a layout declares them (map contract 5), are part of the
+        `CREATE TABLE` statement: `IF NOT EXISTS` never adds one to a table that already exists.
+
+        Returns how existing tables differ from the declared physical design - sort key, partition,
+        indexes - without refusing: the caller decides whether a difference is a refusal (a person
+        provisioning a map) or a report (a running session). Columns and types still refuse here.
+        """
+        statements = schema_statements(layout, keys=keys, dialect=self.dialect)
+
+        for statement in statements:
+            try:
+                self._cx.command(statement)
+            except Exception as exc:
+                raise EngineError(f"schema statement failed: {statement}: {exc}") from exc
+        log("sde.schema.applied", engine=self.dialect, statements=len(statements))
+        self._verify_schema(layout)
+        return self._physical_findings(layout, keys)
+
+    @guarded
+    def validate_schema(
+        self, layout: PhysicalLayout, *, keys: Mapping[str, Sequence[str]] | None = None
+    ) -> tuple[PhysicalFinding, ...]:
+        """Check the existing physical columns without issuing DDL; report the physical design."""
+        self._verify_schema(layout)
+        return () if keys is None else self._physical_findings(layout, keys)
+
+    def _physical_findings(
+        self, layout: PhysicalLayout, keys: Mapping[str, Sequence[str]]
+    ) -> tuple[PhysicalFinding, ...]:
+        """Sort key, partition and data-skipping indexes as the catalogue reports them.
+
+        The catalogue formats expressions, so they are parsed into names rather than compared as
+        strings we predict (:func:`sde.physical.parse_identifier_list`). A key or partition the
+        parser does not understand is reported as it stands - an expression we cannot read is not
+        evidence that the table matches.
+        """
+        declared = declared_tables(layout, keys)
+        if not declared:
+            return ()
+        tables = [entry.table for entry in declared]
+        described = self._cx.query(
+            "SELECT name, sorting_key, partition_key FROM system.tables "
+            "WHERE database = currentDatabase() AND name IN %(tables)s",
+            parameters={"tables": tables},
+        ).result_rows
+        keys_found = {str(name): (str(sort), str(part)) for name, sort, part in described}
+        present: dict[str, dict[str, tuple[str, str, int]]] = {}
+        unreadable = ""
+        indexed = sorted(entry.table for entry in declared if entry.indexes)
+        # Read only when something is declared. `system.tables` and `system.columns` are filtered
+        # by the login's own table privileges, so a restricted runtime login reads them without a
+        # grant; `system.data_skipping_indices` needs an explicit one (measured, 24.8: code 497).
+        # Reading it for every table would have made every session of an existing deployment
+        # fail to start after an upgrade, for a check about performance.
+        if indexed:
+            try:
+                index_rows = self._cx.query(
+                    "SELECT table, name, type_full, expr, granularity "
+                    "FROM system.data_skipping_indices "
+                    "WHERE database = currentDatabase() AND table IN %(tables)s",
+                    parameters={"tables": indexed},
+                ).result_rows
+            except Exception as exc:
+                if getattr(exc, "code", None) != 497:
+                    raise
+                # Unverified, not matching: a running session reports this and goes on; a person
+                # provisioning refuses on it, because "could not look" is not "looked and agreed".
+                unreadable = (
+                    "unverified: this login cannot read system.data_skipping_indices "
+                    "(GRANT SELECT ON system.data_skipping_indices to verify it)"
+                )
+                index_rows = []
+            for table_name, index_name, type_full, expr, granularity in index_rows:
+                present.setdefault(str(table_name), {})[str(index_name)] = (
+                    str(type_full),
+                    str(expr),
+                    int(granularity),
+                )
+        findings: list[PhysicalFinding] = []
+        for entry in declared:
+            if entry.table not in keys_found:
+                continue  # _verify_schema has already refused a missing table
+            sorting, partition = keys_found[entry.table]
+            try:
+                found_key: tuple[str, ...] | str = parse_identifier_list(sorting)
+            except ValueError:
+                found_key = repr(sorting)
+            if found_key != entry.key:
+                findings.append(
+                    PhysicalFinding(entry.table, "sort key", repr(list(entry.key)), str(found_key))
+                )
+            try:
+                found_partition: tuple[str, str] | str | None = parse_partition_key(partition)
+            except ValueError:
+                found_partition = repr(partition)
+            if found_partition != entry.partition:
+                findings.append(
+                    PhysicalFinding(
+                        entry.table, "partition", str(entry.partition), str(found_partition)
+                    )
+                )
+            for index in entry.indexes:
+                got = present.get(entry.table, {}).get(index.name)
+                wanted = (
+                    f"{index.type_full} on {list(index.columns)} granularity {index.granularity}"
+                )
+                if got is None:
+                    findings.append(
+                        PhysicalFinding(
+                            entry.table, f"index {index.name}", wanted, unreadable or "absent"
+                        )
+                    )
+                    continue
+                type_full, expr, granularity = got
+                try:
+                    columns: tuple[str, ...] | str = parse_identifier_list(expr)
+                except ValueError:
+                    columns = repr(expr)
+                if (
+                    type_full != index.type_full
+                    or columns != index.columns
+                    or granularity != index.granularity
+                ):
+                    findings.append(
+                        PhysicalFinding(
+                            entry.table,
+                            f"index {index.name}",
+                            wanted,
+                            f"{type_full} on {columns} granularity {granularity}",
+                        )
+                    )
+        return tuple(findings)
+
+    def _verify_schema(self, layout: PhysicalLayout) -> None:
+        """The same check as the PostgreSQL adapter, for the same reason.
+
+        `CREATE TABLE IF NOT EXISTS` keeps whatever table is already there, so a leftover from an
+        older map is accepted in silence and the first insert fails naming a column rather than
+        the cause. Missing columns are refused, extra ones are logged: a client may have added one
+        outside SDE and the map has no opinion about it.
+
+        **Types too, and here they need no normalising at all.** Measured against every type this
+        library renders: `system.columns.type` returns the exact string we wrote, down to the space
+        in `Decimal(12, 2)` and the quotes in `DateTime64(6, 'UTC')`. So the comparison is literal,
+        and it is literal on purpose - a renderer that started emitting a different spelling of the
+        same type would fail this check, which is the right way round for a document we sign.
+        """
+        expected = {
+            table: dict(layout.columns.get(entity, {}))
+            for entity, table in sorted(layout.tables.items())
+        }
+        if not expected:
+            return
+
+        result = self._cx.query(
+            "SELECT table, name, type FROM system.columns "
+            "WHERE database = currentDatabase() AND table IN %(tables)s",
+            parameters={"tables": sorted(expected)},
+        )
+        found: dict[str, dict[str, str]] = {}
+        for table_name, column_name, column_type in result.result_rows:
+            found.setdefault(str(table_name), {})[str(column_name)] = str(column_type)
+
+        for table, columns in sorted(expected.items()):
+            actual = found.get(table)
+            if actual is None:
+                raise EngineError(
+                    f"{table!r} does not exist after applying the schema. The statement reported "
+                    f"success, so this is a permissions or database-selection problem rather "
+                    f"than a bad map."
+                )
+            missing = sorted(set(columns) - set(actual))
+            if missing:
+                raise EngineError(
+                    f"{table!r} already existed with a different shape: the map needs "
+                    f"{missing} and the table has {sorted(actual)}. `CREATE TABLE IF NOT "
+                    f"EXISTS` keeps whatever "
+                    f"is there, so this table came from somewhere else. Refusing here rather than "
+                    f"at the first insert, which would fail in your request path with an error "
+                    f"naming a column and not the cause."
+                )
+            for column, declared in sorted(columns.items()):
+                reported = actual[column]
+                if reported == declared:
+                    continue
+                raise EngineError(
+                    f"{table}.{column} is {reported!r} and this map declares it {declared!r}. "
+                    f"`CREATE TABLE IF NOT EXISTS` keeps a table of that name whatever shape it "
+                    f"is in, and this library never alters a column's type - so the table came "
+                    f"from somewhere else, or from a map that rendered this column differently. "
+                    f"Refusing rather than writing into it: with a timestamp the difference is "
+                    f"usually precision, and a write that succeeds and comes back rounded is "
+                    f"worse than one that fails."
+                )
+            extra = sorted(set(actual) - set(columns))
+            if extra:
+                log("sde.schema.extra_columns", table=table, columns=extra)
+
+
+    # --- data ------------------------------------------------------------------------------
+
+    @guarded
+    def explain_plan(self, sql: str) -> QueryPlan:
+        """Plan an analyst's query without running it, with ``readonly=1`` on every statement.
+
+        Requirement 19.4. Four questions to the server, all under ``readonly=1`` - which ClickHouse
+        refuses a mutation under (code 164) and which leaves the rest working, both halves
+        measured, because a write protection that also blocked the measurement would be one nobody
+        would keep switched on.
+
+        The plan tree, ``EXPLAIN ESTIMATE`` for the numbers, ``EXPLAIN QUERY TREE`` for the table
+        names, and ``system.tables`` for each table's engine.
+
+        **The third question exists because of a measurement that broke the obvious design.** The
+        table names came from ``EXPLAIN ESTIMATE`` first, and for ``SELECT count() FROM t`` that
+        returns **no rows at all** - ClickHouse answers a trivial count from part metadata and
+        reads nothing, so there is no table in the estimate. That is precisely the query where
+        9.7's hazard is worst: a count over a ``ReplacingMergeTree`` without ``FINAL`` returns the
+        uncollapsed number. So the one query that most needs the warning was the one the estimate
+        could not see. ``EXPLAIN QUERY TREE`` names every table in the join tree whatever the read
+        optimisation does.
+        """
+        settings = {"readonly": 1}
+        # Read it back. Same reason as the PostgreSQL side: without this, dropping the setting
+        # changes nothing observable - nothing EXPLAIN does would write - so the guard would be
+        # unverifiable and its removal silent.
+        confirmed = self._cx.query("SELECT getSetting('readonly')", settings=settings)
+        answer = confirmed.result_rows[0][0] if confirmed.result_rows else None
+        if int(answer or 0) < 1:
+            raise EngineError(
+                f"this server would not accept readonly=1 for planning: getSetting('readonly') "
+                f"came back {answer!r}. Refusing to plan anything: requirement 19.2 keeps this "
+                f"product out of the data path, and readonly=1 is the whole of what stops a "
+                f"statement here from writing - measured, it refuses a mutation with code 164 and "
+                f"still answers EXPLAIN ESTIMATE."
+            )
+        try:
+            tree = self._cx.query(f"EXPLAIN {sql}", settings=settings)
+            plan = tuple(str(row[0]) for row in tree.result_rows)
+            estimate = self._cx.query(f"EXPLAIN ESTIMATE {sql}", settings=settings)
+        except Exception as exc:
+            raise QueryPlanRefused(
+                f"{self.dialect} would not plan this query: {exc} Nothing was executed. This is "
+                f"what validating against a live engine means: the control plane checked the query "
+                f"against the schema it authored, and this is the schema that exists."
+            ) from exc
+
+        columns = list(estimate.column_names)
+        rows = [dict(zip(columns, row, strict=False)) for row in estimate.result_rows]
+        values: dict[str, str] = {}
+        for name in ("parts", "rows", "marks"):
+            if name in columns:
+                values[name] = str(sum(int(row.get(name) or 0) for row in rows))
+        values["tables"] = str(len(rows))
+
+        named = self._tables_in(sql, settings=settings) or [
+            # The estimate's own names, as a fallback for a server whose analyzer does not answer
+            # EXPLAIN QUERY TREE. Both parts come from the answer rather than from this
+            # connection's default database: a query can read a table elsewhere, and the default
+            # would name the wrong one quietly, because system.tables would simply find nothing.
+            (str(row.get("table", "")), str(row.get("database", "")))
+            for row in rows
+            if row.get("table") and row.get("database")
+        ]
+        engines: list[tuple[str, str]] = []
+        for table, database in named:
+            try:
+                answer = self._cx.query(
+                    "SELECT engine FROM system.tables WHERE database = {db:String} "
+                    "AND name = {tb:String}",
+                    parameters={"db": database, "tb": table},
+                    settings=settings,
+                )
+            except Exception:  # pragma: no cover - a client without system.tables access
+                # Not fatal and not silent: the plan is still worth having, and the finding this
+                # would have produced is a property of the table rather than of the query, so its
+                # absence costs a warning rather than a guarantee.
+                continue
+            for row in answer.result_rows:
+                engines.append((table, str(row[0])))
+
+        return QueryPlan(
+            engine=self.dialect,
+            dialect=self.dialect,
+            plan=plan or ("(the engine returned an empty plan)",),
+            cost=Cost(
+                units="parts, rows and marks to read",
+                basis=(
+                    "real counts rather than arbitrary units, and coarse below one granule: a "
+                    "mark covers 8192 rows by default, so a small table reports every row and one "
+                    "mark whether or not the key filter prunes anything - measured on a "
+                    "thousand-row table. The figures are what the primary key can rule out before "
+                    "reading, so they move when the query filters on a key prefix and not when it "
+                    "filters on anything else. Zero across the board means the engine reads "
+                    "nothing at all: a trivial count() is answered from part metadata, and the "
+                    "estimate correctly reports no rows read - which is not the same as a query "
+                    "that returns nothing."
+                ),
+                values=values,
+            ),
+            findings=replacing_merge_tree_finding(engines),
+            read_only_enforced=(
+                "every statement sent with readonly=1. ClickHouse refuses a mutation under it "
+                "(code 164, READONLY) and still answers EXPLAIN ESTIMATE - both measured. There "
+                "is no transaction here to roll back, which is the same absence transaction() "
+                "refuses rather than pretends about."
+            ),
+        )
+
+    def _tables_in(self, sql: str, *, settings: Mapping[str, Any]) -> list[tuple[str, str]]:
+        """Every table the query reads, from the analyzer's own tree. ``(table, database)`` pairs.
+
+        A regex over ``EXPLAIN QUERY TREE`` output, which is the engine's own rendering and could
+        change between versions. The cost of that is a **missing** warning rather than a wrong one:
+        the finding this feeds is a property of a table, so losing it costs an analyst a sentence
+        and costs no guarantee. Worth having anyway, because the alternative loses it for the one
+        query where it matters most - see :meth:`explain_plan`.
+        """
+        import re
+
+        try:
+            tree = self._cx.query(f"EXPLAIN QUERY TREE {sql}", settings=dict(settings))
+        except Exception as exc:
+            # No pragma here any more, and that is the point. The marker that used to say "no test
+            # comes here" sat one line above a log() call whose event name was missing from the
+            # vocabulary, and log() raised on an unknown name - so the graceful path was the one
+            # that crashed, in the client's process, on an older server.
+            log("sde.explain.no_query_tree", reason=str(exc)[:120])
+            return []
+        found: list[tuple[str, str]] = []
+        for row in tree.result_rows:
+            for match in re.finditer(r"table_name:\s*(\S+)", str(row[0])):
+                qualified = match.group(1)
+                database, _, table = qualified.rpartition(".")
+                if table and (table, database) not in found:
+                    found.append((table, database))
+        return found
+
+    @guarded
+    def insert(self, table: str, values: Mapping[str, Any]) -> None:
+        if not values:
+            raise EngineError("nothing to insert")
+        cols = sorted(values)
+        row = [_as_utc(values[c]) for c in cols]
+        try:
+            self._cx.insert(_quote(table), [row], column_names=cols)
+        except Exception as exc:
+            # Surfaced, not swallowed and not rerouted, as in the PostgreSQL adapter. A write that
+            # did not happen is not our internal problem to absorb.
+            log("sde.write.failed", table=table, error=type(exc).__name__)
+            raise EngineError(f"insert into {table} failed: {exc}") from exc
+
+    @guarded
+    def insert_many(self, table: str, rows: Sequence[Mapping[str, Any]]) -> None:
+        """One native batch; no promise of cross-block transactional atomicity."""
+        cols = batch_columns(rows)
+        if not cols:
+            return
+        data = [[_as_utc(row[c]) for c in cols] for row in rows]
+        try:
+            self._cx.insert(_quote(table), data, column_names=cols)
+        except Exception as exc:
+            log("sde.write.failed", table=table, error=type(exc).__name__)
+            raise EngineError(f"batch insert into {table} failed: {exc}") from exc
+
+    @guarded
+    def storage_sizes(self, tables: Sequence[str]) -> dict[str, tuple[int, int]]:
+        """Each table's bytes and its secondary index bytes, from active parts. Numbers only.
+
+        One statement: ``system.tables`` says which of the names exist - readable by a runtime
+        login without any grant, and only for its own tables - and ``system.parts`` what their
+        active parts occupy, which needs the column grant in :data:`STORAGE_COLUMNS`. An empty
+        table has no parts and reads 0 bytes; a missing one is absent from the answer, because a
+        missing table is not an empty one.
+        """
+        if not tables:
+            return {}
+        sql = (
+            "SELECT t.name, sum(p.bytes_on_disk), "
+            "sum(p.secondary_indices_compressed_bytes + p.secondary_indices_marks_bytes) "
+            "FROM system.tables AS t LEFT JOIN ("
+            "SELECT table, bytes_on_disk, secondary_indices_compressed_bytes, "
+            "secondary_indices_marks_bytes FROM system.parts "
+            "WHERE active AND database = currentDatabase()"
+            ") AS p ON p.table = t.name "
+            "WHERE t.database = currentDatabase() AND t.name IN {tables:Array(String)} "
+            "GROUP BY t.name"
+        )
+        try:
+            rows = self._cx.query(
+                sql, parameters={"tables": list(tables)}, settings={"join_use_nulls": 0}
+            ).result_rows
+        except Exception as exc:
+            raise EngineError(f"storage sizes could not be read: {exc}") from exc
+        return {str(name): (int(total), int(secondary)) for name, total, secondary in rows}
+
+    @guarded
+    def get(self, table: str, key: Mapping[str, Any]) -> dict[str, Any] | None:
+        """One row by key, with `FINAL` so a superseded row is never returned.
+
+        Without `FINAL` a key that has been saved twice returns whichever duplicate the scan
+        reaches first until a merge happens - which is to say, nondeterministically the old value.
+        Paying for `FINAL` on a point read is the cheaper half of that trade.
+        """
+        parameters: dict[str, Any] = {}
+        where = " AND ".join(
+            f"{_quote(column)} = {_query_parameter(f'key_{index}', key[column], parameters)}"
+            for index, column in enumerate(sorted(key))
+        )
+        sql = f"SELECT * FROM {_quote(table)} FINAL WHERE {where} LIMIT 1"
+        try:
+            result = self._cx.query(sql, parameters=parameters)
+        except Exception as exc:
+            raise EngineError(f"select from {table} failed: {exc}") from exc
+        if not result.result_rows:
+            return None
+        return _row(result.column_names, result.column_types, result.result_rows[0])
+
+    # --- rollback protection ------------------------------------------------------------------
+    #
+    # Append-only and `max()`, which is what makes this identical in both engines: no key to
+    # enforce, no row to update, nothing for this engine's lack of a unique constraint to spoil.
+
+    @guarded
+    def map_watermark(self) -> int | None:
+        """The highest map version applied against this engine, creating the table if missing.
+
+        A plain `MergeTree` is right here, and it is the one place in this adapter where that is
+        true without qualification: the table is append-only by design and the answer is an
+        aggregate, so there are no duplicates to collapse and no reason to pay for `FINAL`.
+        """
+        try:
+            existing = self._cx.query(f"EXISTS TABLE {_quote(WATERMARK_TABLE)}").result_rows
+            if not existing or existing[0][0] not in (0, 1):
+                raise EngineError("watermark catalog lookup returned no presence result")
+            if existing[0][0] == 0:
+                self._cx.command(
+                    f"CREATE TABLE IF NOT EXISTS {_quote(WATERMARK_TABLE)} ("
+                    f"{_quote('map_version')} Int64, "
+                    f"{_quote('model_version')} String, "
+                    f"{_quote('seen_at')} DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')) "
+                    f"ENGINE = MergeTree ORDER BY ({_quote('map_version')})"
+                )
+            result = self._cx.query(
+                f"SELECT max({_quote('map_version')}) FROM {_quote(WATERMARK_TABLE)}"
+            )
+        except Exception as exc:
+            raise EngineError(f"reading {WATERMARK_TABLE} failed: {exc}") from exc
+        if not result.result_rows or result.result_rows[0][0] is None:
+            return None
+        highest = int(result.result_rows[0][0])
+        # An empty MergeTree answers max() with 0 rather than with null, so zero here means either
+        # "no map has been applied" or "map version 0 has been". Map version 0 is what this library
+        # reads when a hand-written map omits the field, and a hand-written map is unsigned - so it
+        # never reaches this path. Reported as absent, which is the honest reading of the two.
+        return highest if highest > 0 else None
+
+    @guarded
+    def record_map_version(self, version: int, *, model_version: str) -> None:
+        try:
+            self._cx.insert(
+                WATERMARK_TABLE,
+                [[version, model_version]],
+                column_names=["map_version", "model_version"],
+            )
+        except Exception as exc:
+            raise EngineError(
+                f"recording a map version in {WATERMARK_TABLE} failed: {exc}"
+            ) from exc
+
+    @guarded
+    def select_rows(self, table: str, plan: ReadPlan) -> list[dict[str, Any]]:
+        parameters: dict[str, Any] = {}
+        def parameter(value: Any) -> str:
+            return _query_parameter(f"read_{len(parameters)}", value, parameters)
+        statement = read_sql(table, plan, dialect=self.dialect, parameter=parameter)
+        try:
+            result = self._cx.query(statement, parameters=parameters)
+            rows = [_row(result.column_names, result.column_types, row)
+                    for row in result.result_rows]
+            for row in rows:
+                for column in plan.columns:
+                    value = row[column.name]
+                    if value is not None and column.type in ("timestamp", "timestamptz"):
+                        instant = (
+                            value.replace(tzinfo=_dt.UTC)
+                            if value.tzinfo is None else value.astimezone(_dt.UTC)
+                        )
+                        row[column.name] = (
+                            instant.replace(tzinfo=None) if column.type == "timestamp" else instant
+                        )
+            return [read_row(plan.columns, row) for row in rows]
+        except Exception as exc:
+            raise EngineError(f"logical scan of {table} failed: {exc}") from exc
+
+    @guarded
+    def count_rows(self, table: str, plan: ReadPlan) -> int:
+        parameters: dict[str, Any] = {}
+        def parameter(value: Any) -> str:
+            return _query_parameter(f"read_{len(parameters)}", value, parameters)
+        statement = read_sql(table, plan, dialect=self.dialect, parameter=parameter, count=True)
+        try:
+            result = self._cx.query(statement, parameters=parameters)
+            if not result.result_rows:
+                raise EngineError("count query returned no result")
+            return int(result.result_rows[0][0])
+        except Exception as exc:
+            raise EngineError(f"logical count of {table} failed: {exc}") from exc
+
+    @guarded
+    def summarize_rows(
+        self, table: str, plan: ReadPlan, column: ReadColumn,
+    ) -> Mapping[str, Any]:
+        parameters: dict[str, Any] = {}
+        def parameter(value: Any) -> str:
+            return _query_parameter(f"read_{len(parameters)}", value, parameters)
+        statement = summary_sql(table, plan, column, dialect=self.dialect, parameter=parameter)
+        try:
+            result = self._cx.query(statement, parameters=parameters)
+            if not result.result_rows:
+                raise EngineError("summary query returned no result")
+            return dict(zip(result.column_names, result.result_rows[0], strict=True))
+        except Exception as exc:
+            raise EngineError(f"logical summary of {table} failed: {exc}") from exc
+
+    @guarded
+    def range(
+        self,
+        table: str,
+        column: str,
+        *,
+        low: Any = None,
+        high: Any = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        clauses: list[str] = []
+        parameters: dict[str, Any] = {}
+        if low is not None:
+            bound = _query_parameter("low", low, parameters)
+            clauses.append(f"{_quote(column)} >= {bound}")
+        if high is not None:
+            bound = _query_parameter("high", high, parameters)
+            clauses.append(f"{_quote(column)} < {bound}")
+        where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
+        cap = ""
+        if limit is not None:
+            cap = " LIMIT %(limit)s"
+            parameters["limit"] = int(limit)
+        sql = f"SELECT * FROM {_quote(table)} FINAL{where} ORDER BY {_quote(column)}{cap}"
+        try:
+            result = self._cx.query(sql, parameters=parameters)
+        except Exception as exc:
+            raise EngineError(f"range select from {table} failed: {exc}") from exc
+        return [_row(result.column_names, result.column_types, row) for row in result.result_rows]
+
+    @guarded
+    def count(self, table: str) -> int:
+        """`FINAL` here too, so this counts entities rather than stored rows.
+
+        The two differ between a save and the next merge. A count that drifts and then settles is
+        worse than a slower count, because it makes a test flaky and a dashboard untrustworthy in
+        the same way.
+        """
+        try:
+            result = self._cx.query(f"SELECT count() FROM {_quote(table)} FINAL")
+        except Exception as exc:
+            raise EngineError(f"count on {table} failed: {exc}") from exc
+        return int(result.result_rows[0][0]) if result.result_rows else 0
+
+    # --- migration ------------------------------------------------------------------------------
+    #
+    # `sde.migration.Migratable`, the same optional protocol the PostgreSQL adapter satisfies. Two
+    # things are different here and neither is smoothed over: reads take `FINAL`, because a key
+    # saved twice is an overwrite in this engine rather than an error, and `copy_in` has no
+    # conflict clause to write - the collapse *is* the idempotence.
+
+    @guarded
+    def key_range(
+        self,
+        table: str,
+        order: Sequence[str],
+        *,
+        after: Sequence[Any] | None = None,
+        upto: Sequence[Any] | None = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        """Rows in key order, strictly after one key and up to another inclusive, with `FINAL`.
+
+        `FINAL` is what makes keyset pagination correct here rather than merely fast enough. Without
+        it a key saved twice returns two rows until a merge happens, and the next page starts
+        strictly after that key - so one of the duplicates is read and the other is not, which for a
+        backfill means copying a row this engine considers superseded.
+        """
+        cols = key_columns(order, table)
+        clauses: list[str] = []
+        parameters: dict[str, Any] = {}
+        tuple_expr = f"({', '.join(_quote(c) for c in cols)})"
+        if after is not None:
+            same_width(after, cols, "after")
+            bound = [
+                _query_parameter(f"after_{index}", value, parameters)
+                for index, value in enumerate(after)
+            ]
+            clauses.append(f"{tuple_expr} > ({', '.join(bound)})")
+        if upto is not None:
+            same_width(upto, cols, "upto")
+            bound = [
+                _query_parameter(f"upto_{index}", value, parameters)
+                for index, value in enumerate(upto)
+            ]
+            clauses.append(f"{tuple_expr} <= ({', '.join(bound)})")
+        where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
+        cap = ""
+        if limit is not None:
+            cap = " LIMIT %(row_limit)s"
+            parameters["row_limit"] = int(limit)
+        sql = f"SELECT * FROM {_quote(table)} FINAL{where} ORDER BY {tuple_expr}{cap}"
+        try:
+            result = self._cx.query(sql, parameters=parameters)
+        except Exception as exc:
+            raise EngineError(f"key range select from {table} failed: {exc}") from exc
+        return [_row(result.column_names, result.column_types, row) for row in result.result_rows]
+
+    @guarded
+    def nth_key(
+        self, table: str, order: Sequence[str], *, position: int
+    ) -> tuple[Any, ...] | None:
+        """The key of the ``position``-th row in key order, one-based, or None if there is none."""
+        cols = key_columns(order, table)
+        if position < 1:
+            raise EngineError(f"position is one-based; {position} is not a row")
+        projection = ", ".join(_quote(c) for c in cols)
+        sql = (
+            f"SELECT {projection} FROM {_quote(table)} FINAL ORDER BY ({projection}) "
+            f"LIMIT 1 OFFSET %(skip)s"
+        )
+        try:
+            result = self._cx.query(sql, parameters={"skip": position - 1})
+        except Exception as exc:
+            raise EngineError(f"reading row {position} of {table} failed: {exc}") from exc
+        if not result.result_rows:
+            return None
+        row = _row(result.column_names, result.column_types, result.result_rows[0])
+        return tuple(row[column] for column in cols)
+
+    @guarded
+    def copy_in(self, table: str, rows: Sequence[Mapping[str, Any]]) -> None:
+        """Insert rows. Duplicates are collapsed by the table rather than rejected by it.
+
+        There is no `ON CONFLICT` to write and none is needed: the tables this library creates here
+        are `ReplacingMergeTree` ordered by the key, so a row copied twice leaves two parts that
+        collapse to the newest at merge time and read as one under `FINAL`. That is the same
+        idempotence the PostgreSQL path gets from a conflict clause, arrived at from the opposite
+        direction - and it is why a recopied chunk is free in both engines.
+        """
+        if not rows:
+            return
+        cols = sorted(rows[0])
+        for row in rows:
+            if sorted(row) != cols:
+                raise EngineError(
+                    f"copy_in into {table} was given rows with different columns "
+                    f"({cols} and {sorted(row)}). A chunk comes from one table, so this is a "
+                    f"caller assembling it from two."
+                )
+        data = [[_as_utc(row[c]) for c in cols] for row in rows]
+        try:
+            self._cx.insert(_quote(table), data, column_names=cols)
+        except Exception as exc:
+            log("sde.write.failed", table=table, error=type(exc).__name__)
+            raise EngineError(f"copying {len(rows)} rows into {table} failed: {exc}") from exc
+
+    @guarded
+    def backfill_marker(self, *, materialization: str, entity: str) -> int:
+        """How many rows of this entity have been copied into this engine. Zero if none.
+
+        A plain `MergeTree` and `max()`, as the map watermark is - append-only by design, so there
+        are no duplicates to collapse and no reason to pay for `FINAL`. The quirk that needed a
+        comment there is harmless here: an empty aggregate answers 0 rather than null, and 0 is
+        exactly what "nothing has been copied" means, so the two readings coincide.
+        """
+        try:
+            self._cx.command(
+                f"CREATE TABLE IF NOT EXISTS {_quote(BACKFILL_TABLE)} ("
+                f"{_quote('materialization')} String, "
+                f"{_quote('entity')} String, "
+                f"{_quote('rows_copied')} Int64, "
+                f"{_quote('at')} DateTime64(3, 'UTC') DEFAULT now64(3, 'UTC')) "
+                f"ENGINE = MergeTree ORDER BY ({_quote('materialization')}, {_quote('entity')})"
+            )
+            result = self._cx.query(
+                f"SELECT max({_quote('rows_copied')}) FROM {_quote(BACKFILL_TABLE)} "
+                f"WHERE {_quote('materialization')} = %(m)s AND {_quote('entity')} = %(e)s",
+                parameters={"m": materialization, "e": entity},
+            )
+        except Exception as exc:
+            raise EngineError(f"reading {BACKFILL_TABLE} failed: {exc}") from exc
+        if not result.result_rows or result.result_rows[0][0] is None:
+            return 0
+        return int(result.result_rows[0][0])
+
+    @guarded
+    def record_backfill_marker(self, *, materialization: str, entity: str, rows: int) -> None:
+        """Append the new marker. Never update, so an interrupted run leaves a readable trail."""
+        try:
+            self._cx.insert(
+                BACKFILL_TABLE,
+                [[materialization, entity, int(rows)]],
+                column_names=["materialization", "entity", "rows_copied"],
+            )
+        except Exception as exc:
+            raise EngineError(
+                f"recording backfill progress in {BACKFILL_TABLE} failed: {exc}"
+            ) from exc
+
+    # --- transactions ----------------------------------------------------------------------
+
+    def transaction(self) -> Iterator[ClickHouseEngine]:
+        """Refuses. There is no transaction here to give you.
+
+        A no-op context manager would be the friendlier signature and the worse library: the
+        caller would believe a group of writes was atomic, and would find out otherwise from the
+        state of the data rather than from an exception.
+
+        The way out is not a flag. Declare the atomicity - `atomic_with` on the entity - and the
+        planner is then obliged to place those entities in one group, and one group is one engine,
+        so it will not be this one.
+
+        Not decorated with `@contextmanager`, unlike the PostgreSQL one. A decorated generator
+        would need a `yield` after the `raise` to keep the type honest, and that statement is
+        unreachable - `mypy --strict` says so, correctly. A plain method that raises fails at the
+        call, which is one frame earlier and reads better in a traceback.
+        """
+        raise EngineError(
+            "ClickHouse has no multi-statement transactions, so this adapter will not pretend to "
+            "start one. If these writes have to commit together, declare it: `atomic_with` on the "
+            "entities makes them one colocation group, one group is one engine, and the planner is "
+            "then not permitted to put them here. A silent no-op context manager would let the "
+            "writes proceed and let you believe they were atomic."
+        )