smart-data-engine-sdk 0.1.0rc1__tar.gz → 0.2.0__tar.gz

{smart_data_engine_sdk-0.1.0rc1 → smart_data_engine_sdk-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: smart-data-engine-sdk
-Version: 0.1.0rc1
+Version: 0.2.0
 Summary: Smart Data Engine client library: declare a data model, we place it and move it
 Project-URL: Homepage, https://github.com/Smart-Data-Engines/smart-data-engine-sdk
 Project-URL: Repository, https://github.com/Smart-Data-Engines/smart-data-engine-sdk

{smart_data_engine_sdk-0.1.0rc1 → smart_data_engine_sdk-0.2.0}/src/sde/__init__.py

@@ -51,6 +51,7 @@ from .errors import (
     SdeError,
 )
 from .explain import Cost, Explains, PlanFinding, QueryPlan, QueryPlanRefused, explain
+from .facts import FACTS_VERSION, engine_facts
 from .frozen_verification import FrozenVerifyReport, verify_frozen
 from .groups import Group, colocation_groups, group_of
 from .hashing import NameMap, hash_identifiers, load_or_create_salt
@@ -69,6 +70,7 @@ from .layout import (
     DIALECTS,
     FIXED_SCHEMA,
     ORDERBOOK_KEY,
+    ORDERBOOK_NULLABLE,
     ORDERBOOK_SHAPE,
     ORDERBOOK_TABLE,
     DerivedLayout,
@@ -77,6 +79,7 @@ from .layout import (
     denormalized_layout,
     fixed_schema_mismatch,
     group_columns,
+    group_nullable,
     snake_case,
     stored_types,
 )
@@ -156,7 +159,7 @@ from .watermark import (
 from .write_fence import EPOCH_COLUMN as WRITE_EPOCH_COLUMN
 from .write_fence import FenceState, WriteFence
 
-__version__ = "0.1.0rc1"
+__version__ = "0.2.0"
 
 __all__ = [
     "ALSO_WRITE_SINCE",
@@ -167,6 +170,7 @@ __all__ = [
     "CUTOVER_RELAYOUT_PROTOCOL",
     "DIALECTS",
     "DIALECT_PRECISION",
+    "FACTS_VERSION",
     "FIXED_SCHEMA",
     "INDEX_CHANGE_PROTOCOL",
     "INDEX_PROTOCOL",
@@ -177,6 +181,7 @@ __all__ = [
     "MAX_PAGE_ROWS",
     "MEASURED_FIELDS",
     "ORDERBOOK_KEY",
+    "ORDERBOOK_NULLABLE",
     "ORDERBOOK_SHAPE",
     "ORDERBOOK_TABLE",
     "PHYSICAL_DESIGN_SINCE",
@@ -277,11 +282,13 @@ __all__ = [
     "denormalized_layout",
     "digest16",
     "enforce_forward_only",
+    "engine_facts",
     "entity",
     "enumerate_shapes",
     "explain",
     "fixed_schema_mismatch",
     "group_columns",
+    "group_nullable",
     "group_of",
     "has_time_dimension",
     "hash_identifiers",

{smart_data_engine_sdk-0.1.0rc1 → smart_data_engine_sdk-0.2.0}/src/sde/cutover.py

@@ -191,6 +191,8 @@ def _load(
         raise MigrationRefused("cutover verification must require the signed before map")
     request.check_session(before, project_id=project_id, group=group)
     spot = before.placement_of(group)
+    # A group without a write generation (contract 6) never gets this far: the map refuses a copy
+    # in such a group, and a cutover needs one.
     if len(spot.derived) != 1 or spot.also_write != spot.derived:
         raise MigrationRefused("cutover requires exactly one derived copy maintained by fan-out")
     source, target = spot.source, spot.derived[0]

{smart_data_engine_sdk-0.1.0rc1 → smart_data_engine_sdk-0.2.0}/src/sde/engines/_index_build.py

@@ -21,6 +21,7 @@ from typing import Any, Literal
 
 from .._operator_deadline import DeadlineInterrupt
 from ..errors import MigrationRefused
+from ..layout import POSTGRES_TYPES
 from ..physical import CLICKHOUSE_METHODS, POSTGRES_METHODS, index_method
 from ..schema import _skip_index_type, clickhouse_index_clause, postgres_index_target
 from ._operator import NativeOperator, TableIdentity
@@ -29,8 +30,8 @@ Status = Literal["absent", "unfinished", "ready", "foreign"]
 
 _NAME = re.compile(r"sde_i_[0-9a-f]{32}_[0-9]{6}")
 """The only names an in-place build creates. Plain identifiers, so ClickHouse records a mutation
-over one as ``MATERIALIZE INDEX <name>`` with the name unquoted - measured, and relied on to find a
-materialization again instead of starting a second one."""
+over one with the name unquoted - measured, and relied on to find a materialization again instead
+of starting a second one (:meth:`NativeIndexBuild._ch_mutations`)."""
 
 POLL_SECONDS = 0.2
 
@@ -66,13 +67,45 @@ class NativeIndexBuild:
     def status(self, table: TableIdentity, index: Mapping[str, Any]) -> Status:
         return self.inspect(table, index)[0]
 
-    def build(self, table: TableIdentity, index: Mapping[str, Any]) -> None:
-        """Bring the index to ``ready`` without pausing the table's writers."""
+    def settle(self, table: TableIdentity, index: Mapping[str, Any]) -> None:
+        """Wait until no statement still running in the server names this build's index.
+
+        On PostgreSQL a deadline or a killed operator ends the client, not its statement: the
+        server notices a vanished client only when it next talks to it, which for a concurrent
+        build is once the index is valid (measured, PostgreSQL 15.19). The build also releases the
+        table's session lock before its last transaction commits, so a drop waiting for that lock
+        started in the window and failed with "tuple concurrently updated". Read after the wait,
+        the catalogue says what that statement finished, which is kept, or left, which is dropped.
+
+        pg_stat_activity shows a statement's text to its own login, which is the one an earlier
+        run of this operator used. The wait is bounded by the operator's deadline. ClickHouse
+        needs none: its materialization is found again in ``system.mutations`` and waited for.
+        """
+        name = self.quote(self._check(table, index))
+        if self.dialect != "postgres":
+            return
+        while self.native.rows(
+            "SELECT 1 FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND state = 'active' "
+            "AND strpos(query, %s) > 0",
+            [name],
+        ):
+            time.sleep(POLL_SECONDS)
+
+    def build(
+        self, table: TableIdentity, index: Mapping[str, Any], columns: Mapping[str, str]
+    ) -> None:
+        """Bring the index to ``ready`` without pausing the table's writers.
+
+        ``columns`` is the entity's layout columns, from the map being built: PostgreSQL renders a
+        text column with the collation the library's reads use, and builds again an index of ours
+        that an earlier library left without it.
+        """
+        self.settle(table, index)
         status, reason = self.inspect(table, index)
         if status == "foreign":
             raise MigrationRefused(reason)
         if self.dialect == "postgres":
-            self._pg_build(table, index, status)
+            self._pg_build(table, index, status, columns)
         else:
             self._ch_build(table, index, status)
         if self.status(table, index) != "ready":
@@ -105,7 +138,9 @@ class NativeIndexBuild:
         if self.dialect == "postgres":
             # IF EXISTS only for an index that goes while this drop waits for its lock: an earlier
             # drop whose client the budget closed runs on in the server until the transaction it
-            # waits for ends. What is left afterwards is read back below, as always.
+            # waits for ends. What is left afterwards is read back below, as always. No settle
+            # here, unlike a build: two drops of one index are serialized by the exclusive lock
+            # the first holds on it until it commits, so this one never meets a half-done drop.
             self.native.command(f"DROP INDEX CONCURRENTLY IF EXISTS {self.quote(name)}")
         else:
             for mutation_id, is_done, _failure in self._ch_mutations(table, name):
@@ -127,7 +162,8 @@ class NativeIndexBuild:
 
         A foreign object under the name means ours is not there: it is left as it is.
         """
-        status, _ = self.inspect(table, index)  # checks the name and the table first
+        self.settle(table, index)  # checks the name and the table first
+        status, _ = self.inspect(table, index)
         if status == "foreign":
             return
         name = str(index["name"])
@@ -195,14 +231,45 @@ class NativeIndexBuild:
             )
         return ("ready" if usable else "unfinished"), ""
 
-    def _pg_build(self, table: TableIdentity, index: Mapping[str, Any], status: Status) -> None:
-        if status == "ready":
+    def _pg_build(
+        self,
+        table: TableIdentity,
+        index: Mapping[str, Any],
+        status: Status,
+        columns: Mapping[str, str],
+    ) -> None:
+        if status == "ready" and not self._pg_text_collation_stale(index, columns):
             return
-        if status == "unfinished":
-            # Our own leftover of an interrupted build: never valid again by itself, and kept by
-            # IF NOT EXISTS. Removed without blocking writers, then built again.
+        if status != "absent":
+            # Our own index, in one of two states that are no use. Unfinished: the leftover of an
+            # interrupted build, never valid again by itself and kept by IF NOT EXISTS. Ready with a
+            # text column outside the reads' collation: built for this map by a library from
+            # before 2 October 2026 and resumed by this one, valid and unusable by a single read.
+            # Either way removed without blocking writers, then built again.
             self.native.command(f"DROP INDEX CONCURRENTLY {self.quote(str(index['name']))}")
-        self.native.command(f"CREATE INDEX CONCURRENTLY {postgres_index_target(index, table.name)}")
+        self.native.command(
+            f"CREATE INDEX CONCURRENTLY {postgres_index_target(index, table.name, columns)}"
+        )
+
+    def _pg_text_collation_stale(
+        self, index: Mapping[str, Any], columns: Mapping[str, str]
+    ) -> bool:
+        """Whether a text column of this ready index has a collation other than ``C``.
+
+        A ready index is plain and of exactly the declared columns (:meth:`_pg_status`), so its
+        collations line up with them one for one.
+        """
+        rows = self.native.rows(
+            "SELECT ARRAY(SELECT coalesce(co.collname, '') FROM unnest(i.indcollation) "
+            "WITH ORDINALITY k(oid, pos) LEFT JOIN pg_collation co ON co.oid = k.oid "
+            "ORDER BY k.pos) FROM pg_index i WHERE i.indexrelid = to_regclass(%s)",
+            [self.quote(str(index["name"]))],
+        )
+        collations = [str(collation) for collation in rows[0][0]] if rows else []
+        return any(
+            columns.get(str(column)) == POSTGRES_TYPES["string"] and collation != "C"
+            for column, collation in zip(index["columns"], collations, strict=False)
+        )
 
     # --- ClickHouse --------------------------------------------------------------------------
 
@@ -220,12 +287,17 @@ class NativeIndexBuild:
 
     def _ch_mutations(self, table: TableIdentity, name: str) -> list[tuple[str, bool, str]]:
         # KILL MUTATION removes the mutation from this table - measured - so what is listed here
-        # is either done or still to be done.
+        # is either done or still to be done. The command is recorded as the server formats it:
+        # `MATERIALIZE INDEX <name>` on 24.8, `(MATERIALIZE INDEX <name>)` on 26.5 (measured).
+        # Matching only the first, a build on 26.5 never saw its materialization: it waited out
+        # its budget, and every resume started another one.
+        command = f"MATERIALIZE INDEX {name}"
         rows = self.native.rows(
             "SELECT mutation_id, is_done, latest_fail_reason FROM system.mutations "
             "WHERE database = currentDatabase() AND table = {table:String} "
-            "AND command = {command:String} AND NOT is_killed ORDER BY create_time",
-            {"table": table.name, "command": f"MATERIALIZE INDEX {name}"},
+            "AND command IN ({command:String}, {parenthesised:String}) AND NOT is_killed "
+            "ORDER BY create_time",
+            {"table": table.name, "command": command, "parenthesised": f"({command})"},
         )
         return [(str(mutation_id), bool(done), str(failure)) for mutation_id, done, failure in rows]
 

{smart_data_engine_sdk-0.1.0rc1 → smart_data_engine_sdk-0.2.0}/src/sde/engines/_operator.py

@@ -42,6 +42,14 @@ class NativeOperator:
         from .clickhouse import ClickHouseEngine
         from .postgres import PostgresEngine
 
+        if not callable(getattr(engine, "write_fence", None)):
+            # The orderbook engine. Its groups carry no write generation (map contract 6), so no
+            # staging, cutover or index build reaches them and there is nothing to bind it for.
+            raise MigrationRefused(
+                f"a local operator acts only on groups whose writes carry a generation, and "
+                f"{type(engine).__name__} cannot fence writes: leave its engine out of the "
+                f"operator bindings"
+            )
         if not isinstance(engine, (PostgresEngine, ClickHouseEngine)) or not runtime:
             raise MigrationRefused(
                 "local cutover requires stock operator adapters and runtime probes"
@@ -151,9 +159,20 @@ class NativeOperator:
         return TableIdentity(self.dialect, server, database, str(oid), str(namespace), str(actual))
 
     def qualify(
-        self, tables: Sequence[str], allowed_tables: Sequence[str], *, required_access: bool = True
+        self,
+        tables: Sequence[str],
+        allowed_tables: Sequence[str],
+        *,
+        required_access: bool = True,
+        readers: bool = True,
     ) -> tuple[str, ...]:
-        """Require direct restricted logins whose table grants can be removed independently."""
+        """Require direct restricted logins whose table grants can be removed independently.
+
+        ``readers`` also requires every other grantee of ``tables`` to be the operator or an
+        administrator (:meth:`_qualify_readers`), which is what a cutover stands on. An in-place
+        index build passes ``False``: it moves no authority, so a principal the operator does
+        not know reads and writes the same rows before the build and after it.
+        """
         endpoint = self.endpoint()
         principals: list[str] = []
         for probe in self.runtime:
@@ -281,12 +300,23 @@ class NativeOperator:
             self.principal_ids[str(user)] = str(oid)
         if len(set(principals)) != len(principals):
             raise MigrationRefused("runtime probes repeat the same login")
-        self._qualify_readers(tables, principals)
+        if readers:
+            self._qualify_readers(tables, principals)
         self.principals = tuple(sorted(principals))
         return self.principals
 
     def _qualify_readers(self, tables: Sequence[str], principals: Sequence[str]) -> None:
-        """A missing runtime principal would keep reading the retired authority after cutover."""
+        """Every grantee of a table must be the operator, a runtime login or an administrator.
+
+        A principal the operator does not know gets no barrier and no access transition, so after
+        a cutover it keeps reading the retired source, or writing to it, and those writes are lost
+        in silence. An administrator is trusted, because this protocol does not revoke
+        administrative powers (docs/local-cutover.md): refusing one protects nothing. On
+        PostgreSQL that is a superuser, who appears in no ACL. On ClickHouse it is a user with a
+        direct global ACCESS MANAGEMENT, who can grant itself anything (:meth:`_administrators`).
+        A role is never an administrator here, and any other grant covering the table refuses,
+        whatever its kind, global ones included.
+        """
         if self.dialect == "postgres":
             operator = str(self.rows("SELECT current_user")[0][0])
             allowed = {*principals, operator}
@@ -300,21 +330,104 @@ class NativeOperator:
                     "WHERE c.oid=to_regclass(%s) AND a.grantee <> c.relowner",
                     [self.quote(table), self.quote(table)],
                 )
-                if any(str(row[0]) not in allowed for row in grants):
-                    raise MigrationRefused("cutover table has an undeclared runtime grantee")
+                outsiders = sorted(str(row[0]) for row in grants if str(row[0]) not in allowed)
+                if outsiders:
+                    raise MigrationRefused(
+                        f"cutover table {table} has an undeclared runtime grantee: "
+                        f"{', '.join(outsiders)} holds a grant on it and is neither this "
+                        f"operator, a runtime login nor a superuser; declare it as a runtime "
+                        f"login or revoke its grant for the operation"
+                    )
         else:
             operator = str(self.rows("SELECT currentUser()")[0][0])
             database = self.endpoint()[2]
-            allowed = {*principals, operator}
+            allowed = {*principals, operator, *self._administrators()}
             for table in tables:
                 grants = self.rows(
-                    "SELECT user_name,role_name FROM system.grants "
+                    "SELECT user_name,role_name,access_type,database,table FROM system.grants "
                     "WHERE (database IS NULL OR database='' OR database={database:String}) "
                     "AND (table IS NULL OR table='' OR table={table:String})",
                     {"database": database, "table": table},
                 )
-                if any(role is not None or user not in allowed for user, role in grants):
-                    raise MigrationRefused("cutover table has an undeclared runtime grantee")
+                held: dict[str, set[str]] = {}
+                for user, role, access, granted_db, granted_table in grants:
+                    if role is not None or user not in allowed:
+                        who = f"role {role}" if role is not None else f"user {user}"
+                        level = f"{granted_db or '*'}.{granted_table or '*'}"
+                        held.setdefault(who, set()).add(f"{access} ON {level}")
+                if held:
+                    named = "; ".join(
+                        who + " (" + ", ".join(sorted(rights)[:3])
+                        + (", ..." if len(rights) > 3 else "") + ")"
+                        for who, rights in sorted(held.items())
+                    )
+                    raise MigrationRefused(
+                        f"cutover table {table} has an undeclared runtime grantee: {named}. A "
+                        f"grantee must be this operator, a runtime login or a user with a direct "
+                        f"global ACCESS MANAGEMENT; declare it as a runtime login or narrow its "
+                        f"grants away from this database for the operation"
+                    )
+
+    def _administrators(self) -> set[str]:
+        """The ClickHouse users holding the whole of ACCESS MANAGEMENT, directly and globally.
+
+        The group is the server's own, read from ``system.privileges``, and so is the way a grant
+        of it is listed (measured): on 24.8 as one row, ``ACCESS MANAGEMENT`` with no database or
+        table; from 26.5 a row per member, because seven of them are now granted per user name
+        (``ON *`` rather than ``ON *.*``) and the group is never shown whole; 26.9 adds a member,
+        ``CREATE TOKEN``. Read only as the row of 24.8, no administrator was found on a current
+        server. So a member counts when the user holds it, a group above it (``ALL`` too), or
+        every privilege below it; and a partial revoke of anything in the group, or above it, at
+        any level, leaves no administrator.
+        """
+        parents: dict[str, str | None] = {
+            str(privilege): None if group is None else str(group)
+            for privilege, group in self.rows(
+                "SELECT privilege, parent_group FROM system.privileges"
+            )
+        }
+        children: dict[str, list[str]] = {}
+        for privilege, parent in parents.items():
+            if parent is not None:
+                children.setdefault(parent, []).append(privilege)
+        group = "ACCESS MANAGEMENT"
+        within = {group}
+        pending = [group]
+        while pending:
+            for child in children.get(pending.pop(), []):
+                within.add(child)
+                pending.append(child)
+        above = parents.get(group)
+        while above is not None:
+            within.add(above)
+            above = parents.get(above)
+
+        held: dict[str, set[str]] = {}
+        revoked: set[str] = set()
+        for user, access, partial, database, table, column in self.rows(
+            "SELECT user_name, access_type, is_partial_revoke, database, table, column "
+            "FROM system.grants WHERE user_name IS NOT NULL"
+        ):
+            if partial:
+                if str(access) in within:
+                    revoked.add(str(user))
+            elif database is None and table is None and column is None:
+                held.setdefault(str(user), set()).add(str(access))
+
+        def covered(rights: set[str], privilege: str) -> bool:
+            node: str | None = privilege
+            while node is not None:
+                if node in rights:
+                    return True
+                node = parents.get(node)
+            below = children.get(privilege, [])
+            return bool(below) and all(covered(rights, child) for child in below)
+
+        return {
+            user
+            for user, rights in held.items()
+            if user not in revoked and covered(rights, group)
+        }
 
     def access(self, tables: Sequence[TableIdentity], *, enabled: bool) -> None:
         if not self.principals:

{smart_data_engine_sdk-0.1.0rc1 → smart_data_engine_sdk-0.2.0}/src/sde/engines/_staging.py

@@ -10,7 +10,7 @@ from typing import Any
 from ..errors import EngineError, MigrationRefused
 from ..physical import POSTGRES_METHODS, index_method
 from ..placement import PhysicalLayout
-from ..schema import schema_statements
+from ..schema import postgres_index_target, schema_statements
 from ._operator import NativeOperator, TableIdentity
 
 
@@ -188,11 +188,11 @@ class NativeStaging:
             columns = [str(column) for column in index["columns"]]
             rows = self._index(name)
             if not rows:
-                column_sql = ", ".join(self.quote(column) for column in columns)
-                using = "" if declared_method == "btree" else f"USING {declared_method} "
+                # The same rendering as a new table's indexes and an in-place build's: one
+                # function, so a staged copy's index has the collation the reads use.
                 self.native.command(
-                    f"CREATE INDEX {self.quote(name)} ON {self.quote(table.name)} "
-                    f"{using}({column_sql})"
+                    "CREATE INDEX "
+                    + postgres_index_target(index, table.name, layout.columns.get(entity, {}))
                 )
                 rows = self._index(name)
             if len(rows) != 1: