smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/engines/_operator.py

@@ -0,0 +1,394 @@
+"""Native operations for the local cutover executor. Credentials and all rows stay local."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from ..errors import EngineError, MigrationRefused
+from ..schema import QUOTE
+from ._storage import STORAGE_COLUMNS
+
+
+@dataclass(frozen=True)
+class TableIdentity:
+    dialect: str
+    server: str
+    database: str
+    object: str
+    namespace: str
+    name: str
+
+    @property
+    def physical_key(self) -> tuple[str, str, str, str]:
+        return self.dialect, self.server, self.database, self.object
+
+    def as_record(self) -> dict[str, str]:
+        return {
+            "dialect": self.dialect,
+            "server": self.server,
+            "database": self.database,
+            "object": self.object,
+            "namespace": self.namespace,
+            "name": self.name,
+        }
+
+
+class NativeOperator:
+    """A dedicated stock adapter plus dedicated read-only probes for declared runtime logins."""
+
+    def __init__(self, engine: Any, runtime: Sequence[Any]) -> None:
+        from .clickhouse import ClickHouseEngine
+        from .postgres import PostgresEngine
+
+        if not isinstance(engine, (PostgresEngine, ClickHouseEngine)) or not runtime:
+            raise MigrationRefused(
+                "local cutover requires stock operator adapters and runtime probes"
+            )
+        if any(
+            type(probe) is not type(engine) or probe is engine or probe._cx is engine._cx
+            for probe in runtime
+        ):
+            raise MigrationRefused(
+                "runtime probes must be separate connections of the same dialect"
+            )
+        self.engine = engine
+        self.runtime = tuple(runtime)
+        self.dialect = engine.dialect
+        self.quote = QUOTE[self.dialect]
+        self.principals: tuple[str, ...] = ()
+        self.principal_ids: dict[str, str] = {}
+        if self.dialect == "postgres" and int(engine._cx.info.transaction_status) != 0:
+            raise MigrationRefused("operator connection must not have an open transaction")
+        if self.dialect == "clickhouse":
+            # Every acknowledged repair/metadata INSERT must have reached the engine.
+            engine._cx.set_client_setting("async_insert", 0)
+            engine._cx.set_client_setting("wait_for_async_insert", 1)
+
+    def rows(self, sql: str, params: Any = None, *, engine: Any = None) -> list[Any]:
+        adapter = self.engine if engine is None else engine
+        try:
+            if self.dialect == "postgres":
+                with adapter._cx.cursor() as cursor:
+                    cursor.execute(sql, params)
+                    return list(cursor.fetchall()) if cursor.description else []
+            return list(adapter._cx.query(sql, parameters=params).result_rows)
+        except Exception as exc:
+            raise EngineError(f"local operator metadata query failed: {exc}") from exc
+
+    def command(self, sql: str) -> None:
+        if self.dialect == "postgres":
+            self.rows(sql)
+        else:
+            try:
+                self.engine._cx.command(sql)
+            except Exception as exc:
+                raise EngineError(
+                    f"local operator command has an uncertain outcome: {exc}"
+                ) from exc
+
+    def endpoint(self, engine: Any = None) -> tuple[str, str, str]:
+        if self.dialect == "postgres":
+            rows = self.rows(
+                "SELECT c.system_identifier::text, d.oid::text, current_database() "
+                "FROM pg_control_system() c CROSS JOIN pg_database d "
+                "WHERE d.datname=current_database()",
+                engine=engine,
+            )
+        else:
+            rows = self.rows(
+                "SELECT toString(serverUUID()), toString(uuid), name "
+                "FROM system.databases WHERE name=currentDatabase()",
+                engine=engine,
+            )
+        if len(rows) != 1 or any(not str(value) for value in rows[0]):
+            raise MigrationRefused("native endpoint identity is unavailable")
+        return tuple(str(value) for value in rows[0])  # type: ignore[return-value]
+
+    def identity(self, table: str, *, engine: Any = None) -> TableIdentity:
+        server, database, database_name = self.endpoint(engine)
+        if self.dialect == "postgres":
+            rows = self.rows(
+                "SELECT c.oid::text,n.nspname,c.relname,c.relkind,c.relpersistence,"
+                "c.relrowsecurity,c.relforcerowsecurity,"
+                "EXISTS(SELECT 1 FROM pg_inherits i WHERE i.inhrelid=c.oid OR i.inhparent=c.oid),"
+                "EXISTS(SELECT 1 FROM pg_trigger t WHERE t.tgrelid=c.oid AND NOT t.tgisinternal),"
+                "EXISTS(SELECT 1 FROM pg_rewrite r WHERE r.ev_class=c.oid) "
+                "FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace "
+                "WHERE c.oid=to_regclass(%s)",
+                [self.quote(table)],
+                engine=engine,
+            )
+            if len(rows) != 1:
+                raise MigrationRefused("cutover table is missing or invisible")
+            oid, namespace, actual, kind, persistence, *unsupported = rows[0]
+            if kind != "r" or persistence != "p" or any(unsupported):
+                raise MigrationRefused(
+                    "cutover requires persistent ordinary tables "
+                    "without RLS, triggers, rules or inheritance"
+                )
+        else:
+            rows = self.rows(
+                "SELECT toString(uuid),database,name,engine,length(dependencies_table) "
+                "FROM system.tables WHERE database=currentDatabase() AND name={table:String}",
+                {"table": table},
+                engine=engine,
+            )
+            if len(rows) != 1:
+                raise MigrationRefused("cutover table is missing or invisible")
+            oid, namespace, actual, kind, dependencies = rows[0]
+            if kind not in ("MergeTree", "ReplacingMergeTree") or dependencies:
+                raise MigrationRefused(
+                    "cutover requires local MergeTree tables without dependent views"
+                )
+            if self.rows(
+                "SELECT count() FROM system.row_policies WHERE database={database:String} "
+                "AND table={table:String}",
+                {"database": database_name, "table": table},
+            )[0][0]:
+                raise MigrationRefused("cutover does not transfer row policies")
+        return TableIdentity(self.dialect, server, database, str(oid), str(namespace), str(actual))
+
+    def qualify(
+        self, tables: Sequence[str], allowed_tables: Sequence[str], *, required_access: bool = True
+    ) -> tuple[str, ...]:
+        """Require direct restricted logins whose table grants can be removed independently."""
+        endpoint = self.endpoint()
+        principals: list[str] = []
+        for probe in self.runtime:
+            if self.endpoint(probe) != endpoint:
+                raise MigrationRefused(
+                    "runtime and operator connections name different native databases"
+                )
+            if self.dialect == "postgres":
+                rows = self.rows(
+                    "SELECT current_user,session_user,r.oid::text,r.rolsuper,r.rolcreaterole,"
+                    "r.rolcreatedb,r.rolreplication,r.rolbypassrls,"
+                    "EXISTS(SELECT 1 FROM pg_auth_members m WHERE m.member=r.oid) "
+                    "FROM pg_roles r WHERE r.rolname=current_user",
+                    engine=probe,
+                )
+                user, session_user, oid, *powers = rows[0]
+                if user != session_user or any(powers):
+                    raise MigrationRefused(
+                        "cutover runtime must be an unprivileged direct login "
+                        "without role memberships"
+                    )
+                for table in tables:
+                    if (
+                        self.identity(table, engine=probe).physical_key
+                        != self.identity(table).physical_key
+                    ):
+                        raise MigrationRefused("runtime resolves a different physical table")
+                    rights = self.rows(
+                        "SELECT c.relowner=%s::oid,has_schema_privilege(%s,n.oid,'CREATE'),"
+                        "has_table_privilege(%s,c.oid,'SELECT'),has_table_privilege(%s,c.oid,'INSERT'),"
+                        "has_table_privilege(%s,c.oid,'UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER') "
+                        "FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace "
+                        "WHERE c.oid=to_regclass(%s)",
+                        [oid, user, user, user, user, self.quote(table)],
+                    )[0]
+                    if (
+                        rights[0]
+                        or rights[1]
+                        or rights[4]
+                        or (required_access and (not rights[2] or not rights[3]))
+                    ):
+                        raise MigrationRefused(
+                            "runtime needs only SELECT/INSERT table rights "
+                            "and no ownership or schema CREATE"
+                        )
+                    shared = self.rows(
+                        "SELECT count(*) FROM pg_class c, LATERAL aclexplode(c.relacl) a "
+                        "WHERE c.oid=to_regclass(%s) AND (a.grantee=0 OR "
+                        "(a.grantee=%s::oid AND a.is_grantable))",
+                        [self.quote(table), oid],
+                    )[0][0]
+                    columns = self.rows(
+                        "SELECT count(*) FROM pg_attribute c, LATERAL aclexplode(c.attacl) a "
+                        "WHERE c.attrelid=to_regclass(%s) AND a.grantee IN (0,%s::oid)",
+                        [self.quote(table), oid],
+                    )[0][0]
+                    if shared or columns:
+                        raise MigrationRefused(
+                            "PUBLIC, grant-option or column grants "
+                            "prevent isolated runtime revocation"
+                        )
+            else:
+                user = str(self.rows("SELECT currentUser()", engine=probe)[0][0])
+                if self.rows(
+                    "SELECT count() FROM system.role_grants WHERE user_name={user:String}",
+                    {"user": user},
+                )[0][0]:
+                    raise MigrationRefused(
+                        "cutover runtime must use direct grants without role memberships"
+                    )
+                rows = self.rows(
+                    "SELECT storage,toString(id) FROM system.users WHERE name={user:String}",
+                    {"user": user},
+                )
+                if len(rows) != 1 or rows[0][0] != "local_directory":
+                    raise MigrationRefused("cutover runtime must be a local SQL-managed user")
+                oid = str(rows[0][1])
+                grants = self.rows(
+                    "SELECT access_type,database,table,column,is_partial_revoke,grant_option "
+                    "FROM system.grants WHERE user_name={user:String}",
+                    {"user": user},
+                )
+                for access, database, table, column, partial, option in grants:
+                    ordinary = (
+                        database == endpoint[2]
+                        and table in allowed_tables
+                        and access in ("SELECT", "INSERT")
+                    )
+                    # Two whole system tables: settings, which the Python driver reads on connect,
+                    # and data_skipping_indices, the optional grant docs/runtime-roles.md offers
+                    # for verifying declared indexes - it shows a login its own tables' indexes
+                    # only (measured on 24.8).
+                    settings = (
+                        database == "system"
+                        and table in ("settings", "data_skipping_indices")
+                        and access == "SELECT"
+                    )
+                    # The one column grant admitted: what a storage measurement reads of
+                    # `system.parts`, which shows a login the parts of its own tables only
+                    # (measured) and says nothing about a migrated table's access.
+                    storage = (
+                        database == "system"
+                        and table == "parts"
+                        and access == "SELECT"
+                        and column in STORAGE_COLUMNS
+                    )
+                    whole = (ordinary or settings) and column is None
+                    if not (whole or storage) or partial or option:
+                        raise MigrationRefused(
+                            "runtime grants must be direct SELECT/INSERT on declared tables, "
+                            "plus the storage columns of system.parts"
+                        )
+                for table in tables:
+                    if required_access and (
+                        self.identity(table, engine=probe).physical_key
+                        != self.identity(table).physical_key
+                    ):
+                        raise MigrationRefused("runtime resolves a different physical table")
+                    rights = {(str(row[0]), row[1], row[2]) for row in grants}
+                    if required_access and any(
+                        (kind, endpoint[2], table) not in rights for kind in ("SELECT", "INSERT")
+                    ):
+                        raise MigrationRefused("runtime is missing direct SELECT/INSERT grants")
+            principals.append(str(user))
+            self.principal_ids[str(user)] = str(oid)
+        if len(set(principals)) != len(principals):
+            raise MigrationRefused("runtime probes repeat the same login")
+        self._qualify_readers(tables, principals)
+        self.principals = tuple(sorted(principals))
+        return self.principals
+
+    def _qualify_readers(self, tables: Sequence[str], principals: Sequence[str]) -> None:
+        """A missing runtime principal would keep reading the retired authority after cutover."""
+        if self.dialect == "postgres":
+            operator = str(self.rows("SELECT current_user")[0][0])
+            allowed = {*principals, operator}
+            for table in tables:
+                grants = self.rows(
+                    "SELECT pg_get_userbyid(a.grantee) FROM pg_class c, "
+                    "LATERAL aclexplode(c.relacl) a WHERE c.oid=to_regclass(%s) "
+                    "AND a.grantee <> c.relowner UNION "
+                    "SELECT pg_get_userbyid(a.grantee) FROM pg_attribute p "
+                    "JOIN pg_class c ON c.oid=p.attrelid, LATERAL aclexplode(p.attacl) a "
+                    "WHERE c.oid=to_regclass(%s) AND a.grantee <> c.relowner",
+                    [self.quote(table), self.quote(table)],
+                )
+                if any(str(row[0]) not in allowed for row in grants):
+                    raise MigrationRefused("cutover table has an undeclared runtime grantee")
+        else:
+            operator = str(self.rows("SELECT currentUser()")[0][0])
+            database = self.endpoint()[2]
+            allowed = {*principals, operator}
+            for table in tables:
+                grants = self.rows(
+                    "SELECT user_name,role_name FROM system.grants "
+                    "WHERE (database IS NULL OR database='' OR database={database:String}) "
+                    "AND (table IS NULL OR table='' OR table={table:String})",
+                    {"database": database, "table": table},
+                )
+                if any(role is not None or user not in allowed for user, role in grants):
+                    raise MigrationRefused("cutover table has an undeclared runtime grantee")
+
+    def access(self, tables: Sequence[TableIdentity], *, enabled: bool) -> None:
+        if not self.principals:
+            raise MigrationRefused("runtime principals were not qualified")
+        for table in tables:
+            if self.identity(table.name).physical_key != table.physical_key:
+                raise MigrationRefused("physical table identity changed before access transition")
+            qualified = self.quote(table.namespace) + "." + self.quote(table.name)
+            for name in self.principals:
+                action, direction = ("GRANT", "TO") if enabled else ("REVOKE", "FROM")
+                self.command(
+                    f"{action} SELECT, INSERT ON {qualified} {direction} {self.quote(name)}"
+                )
+        for table in tables:
+            qualified = self.quote(table.namespace) + "." + self.quote(table.name)
+            for name in self.principals:
+                if self.dialect == "postgres":
+                    rights = self.rows(
+                        "SELECT has_table_privilege(%s,to_regclass(%s),'SELECT'),"
+                        "has_table_privilege(%s,to_regclass(%s),'INSERT')",
+                        [name, qualified, name, qualified],
+                    )[0]
+                else:
+                    memberships = self.rows(
+                        "SELECT count() FROM system.role_grants WHERE user_name={user:String}",
+                        {"user": name},
+                    )[0][0]
+                    if memberships:
+                        raise MigrationRefused(
+                            "runtime memberships changed during access transition"
+                        )
+                    grants = self.rows(
+                        "SELECT access_type FROM system.grants WHERE user_name={user:String} "
+                        "AND database={database:String} AND table={table:String} "
+                        "AND column IS NULL AND is_partial_revoke=0",
+                        {"user": name, "database": table.namespace, "table": table.name},
+                    )
+                    present = {str(row[0]) for row in grants}
+                    rights = [kind in present for kind in ("SELECT", "INSERT")]
+                if list(rights) != [enabled, enabled]:
+                    raise MigrationRefused(
+                        "runtime SELECT/INSERT grants did not reach the expected state"
+                    )
+        for probe in self.runtime:
+            for table in tables:
+                query = f"SELECT 1 FROM {self.quote(table.name)} LIMIT 0"
+                try:
+                    if self.dialect == "postgres":
+                        probe._cx.execute(query)
+                    else:
+                        probe._cx.query(query)
+                except Exception as exc:
+                    denied = (
+                        getattr(exc, "sqlstate", None) == "42501"
+                        if self.dialect == "postgres"
+                        else getattr(exc, "code", None) == 497
+                    )
+                    if enabled or not denied:
+                        raise EngineError(
+                            "runtime access probe did not establish the expected permissions"
+                        ) from exc
+                else:
+                    if not enabled:
+                        raise MigrationRefused(
+                            "runtime still reads a table after its grants were revoked"
+                        )
+
+    def truncate(self, tables: Sequence[TableIdentity]) -> None:
+        for table in tables:
+            if self.identity(table.name).physical_key != table.physical_key:
+                raise MigrationRefused("physical target identity changed before repair")
+        names = [self.quote(table.namespace) + "." + self.quote(table.name) for table in tables]
+        if self.dialect == "postgres":
+            self.command("TRUNCATE TABLE " + ", ".join(names))
+        else:
+            for name in names:
+                self.command("TRUNCATE TABLE " + name + " SYNC")

sde/engines/_staging.py

@@ -0,0 +1,222 @@
+"""Create and recognize stage-owned objects through dedicated native operator connections."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from ..errors import EngineError, MigrationRefused
+from ..physical import POSTGRES_METHODS, index_method
+from ..placement import PhysicalLayout
+from ..schema import schema_statements
+from ._operator import NativeOperator, TableIdentity
+
+
+def creation_marker(project_id: str, stage_id: str, table: str) -> str:
+    suffix = hashlib.sha256(table.encode("utf-8")).hexdigest()[:16]
+    return f"sde.stage1.{project_id}.{stage_id}.{suffix}"
+
+
+class NativeStaging:
+    def __init__(self, native: NativeOperator) -> None:
+        self.native = native
+        self.engine = native.engine
+        self.dialect = native.dialect
+        self.quote = native.quote
+
+    def marker(self, table: str) -> tuple[str, str | None] | None:
+        if self.dialect == "postgres":
+            rows = self.native.rows(
+                "SELECT c.oid::text,obj_description(c.oid,'pg_class') FROM pg_class c "
+                "WHERE c.oid=to_regclass(%s)",
+                [self.quote(table)],
+            )
+        else:
+            rows = self.native.rows(
+                "SELECT toString(uuid),comment FROM system.tables "
+                "WHERE database=currentDatabase() AND name={table:String}",
+                {"table": table},
+            )
+        if not rows:
+            return None
+        if len(rows) != 1:
+            raise MigrationRefused("staging table name has an ambiguous native identity")
+        return str(rows[0][0]), None if rows[0][1] is None else str(rows[0][1])
+
+    def preflight(self, layout: PhysicalLayout, keys: Mapping[str, Sequence[str]]) -> None:
+        schema_statements(layout, keys=keys, dialect=self.dialect)
+        for table in layout.tables.values():
+            if self.marker(table) is not None:
+                raise MigrationRefused("a fresh staging name already exists in the target")
+        if self.dialect == "postgres":
+            identifiers = [
+                *layout.tables.values(),
+                *(name for columns in layout.columns.values() for name in columns),
+                *(str(index["name"]) for index in layout.indexes),
+            ]
+            if any(len(name.encode("utf-8")) > 63 for name in identifiers):
+                raise MigrationRefused("staging identifiers cannot be truncated by PostgreSQL")
+            for index in layout.indexes:
+                found = self.native.rows("SELECT to_regclass(%s)", [self.quote(str(index["name"]))])
+                if found[0][0] is not None:
+                    raise MigrationRefused(
+                        "a staging index name already exists in the target namespace"
+                    )
+
+    def create_table(
+        self, *, entity: str, layout: PhysicalLayout, keys: Mapping[str, Sequence[str]], marker: str
+    ) -> TableIdentity:
+        if re.fullmatch(r"sde\.stage1\.[0-9a-f]{32}\.[0-9a-f]{32}\.[0-9a-f]{16}", marker) is None:
+            raise MigrationRefused("staging creation marker is not valid protocol metadata")
+        table = layout.tables[entity]
+        present = self.marker(table)
+        if present is not None:
+            if present[1] != marker:
+                raise MigrationRefused("staging refuses a table without its exact creation marker")
+            return self.native.identity(table)
+        # The whole physical design of this one table: key order, partition and - in ClickHouse,
+        # where `CREATE TABLE` is the only moment an index can be declared without a mutation over
+        # every part - its data-skipping indexes. An earlier version passed the partition alone,
+        # so a staged copy would have silently lost the key order and the indexes its signed map
+        # authorized. PostgreSQL indexes follow separately in `create_indexes`.
+        single = PhysicalLayout(
+            tables={entity: table},
+            columns={entity: layout.columns[entity]},
+            partition_by={entity: layout.partition_by[entity]}
+            if entity in layout.partition_by
+            else {},
+            key_order={entity: layout.key_order[entity]} if entity in layout.key_order else {},
+            indexes=tuple(
+                index
+                for index in layout.indexes
+                if self.dialect == "clickhouse" and str(index["entity"]) == entity
+            ),
+        )
+        statements = schema_statements(single, keys=keys, dialect=self.dialect)
+        if len(statements) != 1 or not statements[0].startswith("CREATE TABLE IF NOT EXISTS "):
+            raise MigrationRefused("staging requires the SDK's ordinary table creation statement")
+        statement = statements[0].replace("CREATE TABLE IF NOT EXISTS ", "CREATE TABLE ", 1)
+        try:
+            if self.dialect == "postgres":
+                from psycopg import sql
+
+                with self.engine._cx.transaction():
+                    self.engine._cx.execute(statement)
+                    self.engine._cx.execute(
+                        sql.SQL("COMMENT ON TABLE {} IS {}").format(
+                            sql.Identifier(table), sql.Literal(marker)
+                        )
+                    )
+            else:
+                # The marker is fixed ASCII protocol metadata, never a credential or a row value.
+                self.engine._cx.command(statement + " COMMENT '" + marker + "'")
+        except Exception as exc:
+            raise EngineError(
+                "staging table creation has an uncertain outcome; resume its intent"
+            ) from exc
+        found = self.marker(table)
+        if found is None or found[1] != marker:
+            raise MigrationRefused("staging creation marker was not established with the table")
+        return self.native.identity(table)
+
+    def owned(
+        self, table: str, marker: str, identity: TableIdentity | None
+    ) -> TableIdentity | None:
+        """This staging's own table under ``table``, or ``None`` when it is absent or not ours.
+
+        Ours means the exact creation marker and, once the identity was recorded, that native
+        object. Anything else under the name - another marker, none, another object - belongs to
+        somebody else and is left alone by an abandonment.
+        """
+        present = self.marker(table)
+        if present is None or present[1] != marker:
+            return None
+        if identity is not None:
+            return identity if present[0] == identity.object else None
+        return self.native.identity(table)
+
+    def drop_owned(self, table: TableIdentity, marker: str) -> None:
+        """Drop this staging's own table; its indexes and generation constraints go with it."""
+        present = self.marker(table.name)
+        if present is None:
+            return  # dropped before a crash; nothing is left to remove
+        if present[1] != marker or present[0] != table.object:
+            raise MigrationRefused("the table to abandon is no longer this staging's own")
+        # SYNC: in an Atomic database a dropped table otherwise lingers until the server removes
+        # it, and its name and data with it.
+        suffix = " SYNC" if self.dialect == "clickhouse" else ""
+        self.native.command(f"DROP TABLE {self.quote(table.name)}{suffix}")
+        if self.marker(table.name) is not None:
+            raise MigrationRefused("an abandoned staging table is still in the catalogue")
+
+    def revoke_runtime(self, tables: Sequence[TableIdentity], principals: Sequence[str]) -> None:
+        """Take back the runtime grants on dropped tables; ClickHouse keeps them after DROP TABLE.
+
+        Measured on 24.8.14.39: ``system.grants`` still lists a table's grants once the table is
+        gone, and ``REVOKE`` on the dropped table is accepted and removes them. PostgreSQL removes a
+        table's privileges with the table, so there is nothing to take back there.
+        """
+        if self.dialect != "clickhouse":
+            return
+        for table in tables:
+            qualified = self.quote(table.namespace) + "." + self.quote(table.name)
+            for name in principals:
+                self.native.command(f"REVOKE SELECT, INSERT ON {qualified} FROM {self.quote(name)}")
+        for table in tables:
+            for name in principals:
+                rows = self.native.rows(
+                    "SELECT count() FROM system.grants WHERE user_name={user:String} "
+                    "AND database={database:String} AND table={table:String}",
+                    {"user": name, "database": table.namespace, "table": table.name},
+                )
+                if rows[0][0]:
+                    raise MigrationRefused("a runtime grant on an abandoned staging table remains")
+
+    def create_indexes(self, layout: PhysicalLayout) -> None:
+        if self.dialect != "postgres":
+            # ClickHouse data-skipping indexes were declared inside `CREATE TABLE`; qualification
+            # reads them back from `system.data_skipping_indices` before the stage is accepted.
+            return
+        for index in layout.indexes:
+            name, entity = str(index["name"]), str(index["entity"])
+            declared_method = index_method(index)
+            if declared_method not in POSTGRES_METHODS:
+                raise MigrationRefused(f"PostgreSQL cannot create a {declared_method} index")
+            table = self.native.identity(layout.tables[entity])
+            columns = [str(column) for column in index["columns"]]
+            rows = self._index(name)
+            if not rows:
+                column_sql = ", ".join(self.quote(column) for column in columns)
+                using = "" if declared_method == "btree" else f"USING {declared_method} "
+                self.native.command(
+                    f"CREATE INDEX {self.quote(name)} ON {self.quote(table.name)} "
+                    f"{using}({column_sql})"
+                )
+                rows = self._index(name)
+            if len(rows) != 1:
+                raise MigrationRefused("staging index identity could not be established")
+            target, unique, valid, ready, simple, method, actual, options = rows[0]
+            if (
+                str(target) != table.object
+                or unique
+                or not valid
+                or not ready
+                or not simple
+                or method != declared_method
+                or list(actual) != columns
+                or any(options)
+            ):
+                raise MigrationRefused("staging index does not match its owned table and columns")
+
+    def _index(self, name: str) -> list[Any]:
+        return self.native.rows(
+            "SELECT i.indrelid::text,i.indisunique,i.indisvalid,i.indisready,"
+            "i.indpred IS NULL AND i.indexprs IS NULL AND i.indnkeyatts=i.indnatts,a.amname,"
+            "ARRAY(SELECT p.attname FROM unnest(i.indkey) WITH ORDINALITY k(num,pos) "
+            "JOIN pg_attribute p ON p.attrelid=i.indrelid AND p.attnum=k.num ORDER BY k.pos),"
+            "i.indoption::smallint[] FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid "
+            "JOIN pg_am a ON a.oid=c.relam WHERE c.oid=to_regclass(%s)",
+            [self.quote(name)],
+        )

sde/engines/_storage.py

@@ -0,0 +1,22 @@
+"""The ``system.parts`` columns a ClickHouse storage measurement reads - in a module of its own.
+
+Shared by the adapter that reads them, the operator's runtime qualification that admits exactly
+this grant, and the starter that gives it; kept apart from the adapter so that importing the
+qualification does not import a driver.
+"""
+
+from __future__ import annotations
+
+STORAGE_COLUMNS = (
+    "database",
+    "table",
+    "active",
+    "bytes_on_disk",
+    "secondary_indices_compressed_bytes",
+    "secondary_indices_marks_bytes",
+)
+"""The columns of ``system.parts`` a storage measurement reads, and all a runtime login is granted.
+
+ClickHouse 24.8 refuses ``system.parts`` to a login with only table grants; a column grant on these
+names lets it read the parts of the tables it may use and no others (measured). The operator's
+runtime qualification admits exactly this grant, and the starter gives it."""