mainsequence-metatable 0.1.31.dev58__tar.gz → 0.1.31.dev59__tar.gz

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence-metatable
-Version: 0.1.31.dev58
+Version: 0.1.31.dev59
 Summary: Main Sequence MetaTables Python client and standalone API
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License-Expression: MIT

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/docs/adr/api/0013-application-owned-migrations.md

@@ -124,6 +124,51 @@ HTTP request/response contract is unchanged. Client inventory improvements requi
 a package update and application rebuild. API batching benefits existing clients
 after the shared API is deployed, with larger gains for clients sending batches.
 
+### Conditional catalog reconciliation (2026-10-10 amendment)
+
+Reconciliation means making the catalog describe the freshly observed physical
+table while preserving declared semantic metadata. It is additional MetaTables
+behavior around Alembic, not an instruction to reapply migration scripts or rebuild
+identical catalog rows. An unchanged Alembic revision does not prove absence of
+physical drift or completion of an earlier catalog finalization.
+
+```mermaid
+flowchart TD
+    A[Run pending Alembic revisions] --> B[Fresh admission and physical inspection]
+    B --> C[Reconcile physical facts with existing semantic metadata]
+    C --> D{Do stored projections match?}
+    D -->|Yes| E[Keep existing projection rows and UUIDs]
+    D -->|No| F[Replace damaged or changed projections]
+    E --> G[Update only changed contract and lifecycle fields]
+    F --> G
+    G --> H[Fresh security-policy checks and commit]
+```
+
+- Finalization retains the catalog lock, per-table Writer/provider/source checks,
+  fresh physical inspection, owner reachability, foreign-key target validation,
+  missing-table/removal handling, and Timescale hypertable checks. No physical
+  snapshot or authorization result is cached across requests.
+- The API reads existing column, index and foreign-key projections in three batch
+  queries. Comparison includes their scalar fields, full contract fragments and
+  resolved FK target UIDs, and detects missing, extra or damaged rows. Projection
+  row UUIDs and reflection ordering of independent objects are not differences;
+  column ordinals and the order of columns inside keys/indexes remain meaningful.
+- Matching projections are kept, even if only the recorded Alembic revision or
+  lifecycle metadata changes. Changed projections are replaced inside the existing
+  per-table savepoint. Equal-valued contract, snapshot and lifecycle fields are not
+  assigned, avoiding unnecessary ORM dirtiness and metadata/search writes.
+- Hosted/local runtime security-policy checks are explicitly retained at commit
+  even when table metadata is unchanged. Existing policy reconciliation repairs
+  policy differences; equal policies and their JSON permission manifest do not
+  require metadata writes. A failed physical check or conversion still fails
+  rather than being hidden by equality.
+- A healthy unchanged finalization issues no table/projection metadata DML. A
+  matching contract with damaged projections still repairs them. A previously
+  committed migration with failed finalization can retry at the same revision;
+  physical drift is freshly observed and catalog reconciliation completes without
+  replaying already-applied DDL. The response contract and client calls stay the
+  same; no consuming-application opt-in is required.
+
 ## MetaTables system migrations
 
 Hosted system upgrades also run as a deployment prerequisite Job
@@ -167,3 +212,14 @@ fresh reflection after DDL, unchanged-revision reconciliation, authorization bef
 inspection, independent failures and dialect fallbacks. PostgreSQL query grouping is
 checked with a recording driver; SQLite executes real local DDL. Hosted performance
 and live MySQL/SQL Server/PostgreSQL verification remain pending.
+
+The conditional-reconciliation amendment adds SQL-recording checks for zero
+metadata writes and stable projection UUIDs on unchanged finalization, damaged
+scalar/fragment and missing/extra projection repair, revision-only updates,
+semantic changes at the same revision, drift plus failed-finalization retry, and
+fresh permission/missing/inaccessible/owner checks, including self-referencing key
+projection repair. A real local-runtime check verifies zero catalog DML on an
+unchanged upgrade while still refreshing physical security policies, and detects
+a newly added trigger that changes write safety without changing the contract.
+Existing local SQLite application-provider tests verify actual physical drift and recovery without
+reapplying DDL. These are focused non-container checks, not new required CI jobs.

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mainsequence-metatable"
-version = "0.1.31.dev58"
+version = "0.1.31.dev59"
 description = "Main Sequence MetaTables Python client and standalone API"
 readme = "README.md"
 requires-python = ">=3.13"

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/src/mainsequence_metatable.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence-metatable
-Version: 0.1.31.dev58
+Version: 0.1.31.dev59
 Summary: Main Sequence MetaTables Python client and standalone API
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License-Expression: MIT

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/src/mainsequence_metatable.egg-info/SOURCES.txt

@@ -517,6 +517,7 @@ tests/e2e/test_local_actor_client_api.py
 tests/e2e/test_local_cli_workflow.py
 tests/e2e/test_local_client_selection.py
 tests/e2e/test_managed_finalize_client_api.py
+tests/e2e/test_managed_finalize_unchanged.py
 tests/e2e/test_namespaces_admin_http.py
 tests/e2e/test_public_imports.py
 tests/e2e/test_remaining_client_routes.py

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/src/metatables/api/app/routes/managed_finalize.py

@@ -17,6 +17,7 @@ from sqlalchemy.orm import Session
 from metatables.api.app.auth.boundary import trusted_actor_facts
 from metatables.api.app.auth.local_actor import catalog_session
 from metatables.api.app.routes.table_introspection import data_source_provider
+from metatables.api.backend.backends.access import check_table_security_on_commit
 from metatables.api.backend.backends.operations import (
     PhysicalConnectionUnavailable,
     PhysicalIntrospectionError,
@@ -45,8 +46,11 @@ from metatables.api.backend.persistence.models import (
     TimeIndexTable,
 )
 from metatables.api.backend.persistence.projections import (
+    contract_projection_matches,
+    load_contract_projections,
     materialize_contract_projection,
     resolved_foreign_key_targets,
+    same_contract_state,
 )
 
 logger = logging.getLogger(__name__)
@@ -194,6 +198,7 @@ def finalize_managed(
         finally:
             logger.info("Managed finalization: physical inspection of %s tables took %.3fs",
                         len(eligible), perf_counter() - inspection_started)
+    projections = load_contract_projections(session, [uid for uid, *_ in eligible])
     for uid, table, previous, specialization in eligible:
         try:
             snapshot = snapshots[(table.physical_schema, table.physical_table_name)]
@@ -232,13 +237,24 @@ def finalize_managed(
                 )
                 targets = resolved_foreign_key_targets(
                     session, data_source_uid=table.data_source_uid, contract=contract,
+                    source_table_uid=uid,
                 )
-                session.execute(delete(MetaTableForeignKey).where(MetaTableForeignKey.source_table_uid == uid))
-                session.execute(delete(MetaTableIndex).where(MetaTableIndex.meta_table_uid == uid))
-                session.execute(delete(MetaTableColumn).where(MetaTableColumn.meta_table_uid == uid))
-                session.flush()
-                table.table_contract = contract
-                table.contract_version = contract["version"]
+                replace_projection = not contract_projection_matches(projections[uid], contract, targets)
+                if replace_projection:
+                    session.execute(delete(MetaTableForeignKey).where(MetaTableForeignKey.source_table_uid == uid))
+                    session.execute(delete(MetaTableIndex).where(MetaTableIndex.meta_table_uid == uid))
+                    session.execute(delete(MetaTableColumn).where(MetaTableColumn.meta_table_uid == uid))
+                    session.flush()
+                updates = {
+                    "contract_version": contract["version"],
+                    "physical_schema": contract["physical"]["schema"] or table.physical_schema,
+                    "provisioning_status": "active",
+                    "protect_from_deletion": True,
+                }
+                if not same_contract_state(table.table_contract, contract):
+                    updates["table_contract"] = contract
+                if not same_contract_state(table.introspection_snapshot, snapshot):
+                    updates["introspection_snapshot"] = snapshot
                 authoring = contract.get("authoring") or {}
                 has_time_profile = (
                     contract.get("table_kind") == "time_indexed"
@@ -254,19 +270,22 @@ def finalize_managed(
                         cadence=spec.cadence,
                     )
                     session.add(specialization)
-                    table.kind = "time_index"
-                table.physical_schema = contract["physical"]["schema"] or table.physical_schema
-                convert_time_index_table(source, table, specialization, convert=make_hypertable)
-                table.introspection_snapshot = snapshot
-                table.provisioning_status = "active"
-                table.protect_from_deletion = True
+                    updates["kind"] = "time_index"
                 if "alembic_revision" in payload.model_fields_set:
-                    table.alembic_revision = payload.alembic_revision
+                    updates["alembic_revision"] = payload.alembic_revision
                 if version_root is not None and uid != version_root:
-                    table.alembic_version_meta_table_uid = version_root
-                materialize_contract_projection(
-                    session, table=table, contract=contract, foreign_key_targets=targets
-                )
+                    updates["alembic_version_meta_table_uid"] = version_root
+                # Do not dirty equal-valued rows: pre-flush hooks see assignments
+                # before SQLAlchemy decides whether they need an UPDATE.
+                for field, value in updates.items():
+                    if getattr(table, field) != value:
+                        setattr(table, field, value)
+                convert_time_index_table(source, table, specialization, convert=make_hypertable)
+                if replace_projection:
+                    materialize_contract_projection(
+                        session, table=table, contract=contract, foreign_key_targets=targets
+                    )
+            check_table_security_on_commit(session, table)
             rows.append(_row(table, uid, previous=previous, specialization=specialization,
                              exists=True, finalized=previous != "active"))
         except (CatalogInvariantError, ContractValidationError, IntegrityError) as exc:

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/src/metatables/api/backend/backends/access.py

@@ -5,6 +5,7 @@ runtime DataSource's resolved platform Secret, never persisted in the catalog.
 """
 
 import hmac
+import json
 from dataclasses import dataclass, field
 from hashlib import sha256
 
@@ -328,7 +329,12 @@ class DatabaseAccess:
             self.refresh_policies(session, tables=self.policy_scope(session, tables))
         desired = self.plan(session)
         self.apply(session, desired, state.manifest, full=inspect_objects)
-        state.manifest, state.pending = desired, False
+        # JSON loads arrays as lists, while plan() builds tuples. Compare the
+        # persisted representation so equal grants do not rewrite security state.
+        if json.dumps(state.manifest, sort_keys=True) != json.dumps(desired, sort_keys=True):
+            state.manifest = desired
+        if state.pending:
+            state.pending = False
         session.flush()
 
     def reconcile_change(self, session, *, tables, full=False):
@@ -439,6 +445,14 @@ def install_session_security(session, resolver):
     session.info["sql_security_resolver"] = resolver
 
 
+def check_table_security_on_commit(session, table):
+    """Finalization checks physical policies even when metadata needs no writes."""
+    resolver = session.info.get("sql_security_resolver")
+    if resolver is not None and table.data_source_uid == resolver().source.uid:
+        session.info.setdefault("sql_security_tables", []).append(table)
+        session.info["sql_security_changed"] = True
+
+
 def _changed(session, *_):
     resolver = session.info.get("sql_security_resolver")
     relevant_types = (MetaTable, MetaTableGrant, Namespace, NamespaceGrant, DataSource)

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/src/metatables/api/backend/persistence/projections.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import json
+from collections import Counter
 from uuid import UUID
 
 from sqlalchemy import select
@@ -15,6 +17,69 @@ from metatables.api.backend.persistence.models import (
     MetaTableIndex,
 )
 
+# Compare persisted scalar fields as well as fragments: either can be damaged
+# independently. Row UUIDs and retrieval order are not contract state.
+_PROJECTION_FIELDS = {
+    "columns": (MetaTableColumn, "meta_table_uid", (
+        "name", "logical_name", "data_type", "backend_type", "nullable",
+        "primary_key", "unique", "ordinal_position", "description", "label",
+    )),
+    "indexes": (MetaTableIndex, "meta_table_uid", (
+        "name", "columns", "unique", "method", "expression",
+    )),
+    "foreign_keys": (MetaTableForeignKey, "source_table_uid", (
+        "name", "source_columns", "target_columns", "on_delete",
+    )),
+}
+
+
+def load_contract_projections(session: Session, table_uids: list[UUID]) -> dict:
+    """Read one batch under the caller's catalog lock; never cache across requests."""
+    projections = {uid: {kind: [] for kind in _PROJECTION_FIELDS} for uid in table_uids}
+    if not table_uids:
+        return projections
+    for kind, (model, owner, fields) in _PROJECTION_FIELDS.items():
+        for row in session.scalars(select(model).where(getattr(model, owner).in_(table_uids))):
+            values = {field: getattr(row, field) for field in fields}
+            values["contract_fragment"] = row.contract_fragment
+            if kind == "foreign_keys":
+                values["target_table_uid"] = str(row.target_table_uid)
+            projections[getattr(row, owner)][kind].append(values)
+    return projections
+
+
+def contract_projection_matches(projection: dict, contract: dict, targets: list[UUID]) -> bool:
+    """Require exact projection contents, including duplicate/extra/missing rows."""
+    def encode(value):
+        return json.dumps(value, sort_keys=True, separators=(",", ":"))
+
+    for kind, (_, _, fields) in _PROJECTION_FIELDS.items():
+        expected = []
+        for position, fragment in enumerate(contract[kind]):
+            values = {field: fragment[field] for field in fields}
+            values["contract_fragment"] = fragment
+            if kind == "foreign_keys":
+                values["target_table_uid"] = str(targets[position])
+            expected.append(values)
+        if Counter(map(encode, projection[kind])) != Counter(map(encode, expected)):
+            return False
+    return True
+
+
+def same_contract_state(previous: dict, current: dict) -> bool:
+    """Ignore reflection ordering of object lists, retaining order inside each object."""
+    if not isinstance(previous, dict) or not isinstance(current, dict):
+        return False
+
+    def canonical(value):
+        return {
+            **value,
+            **{key: sorted(value[key], key=lambda item: json.dumps(item, sort_keys=True))
+               for key in ("columns", "indexes", "foreign_keys", "constraints")
+               if isinstance(value.get(key), list)},
+        }
+    return canonical(previous) == canonical(current)
+
 
 def _target_for_foreign_key(
     session: Session,
@@ -40,15 +105,24 @@ def _target_for_foreign_key(
     return target
 
 
-def _check_target_columns(session: Session, *, foreign_key: dict, target: MetaTable) -> None:
-    columns = list(
-        session.scalars(
+def _check_target_columns(
+    session: Session, *, foreign_key: dict, target: MetaTable, contract: dict | None = None
+) -> None:
+    if contract is None:
+        columns = [{"name": row.name, "primary_key": row.primary_key, "unique": row.unique}
+                   for row in session.scalars(
             select(MetaTableColumn)
             .where(MetaTableColumn.meta_table_uid == target.uid)
             .order_by(MetaTableColumn.ordinal_position)
-        )
-    )
-    target_names = {column.name for column in columns}
+        )]
+        indexes = [index.columns for index in session.scalars(select(MetaTableIndex).where(
+            MetaTableIndex.meta_table_uid == target.uid, MetaTableIndex.unique.__eq__(True)))]
+    else:
+        # A self-reference must validate against its freshly observed contract,
+        # not a damaged projection that this finalization is about to repair.
+        columns = sorted(contract["columns"], key=lambda column: column["ordinal_position"])
+        indexes = [index["columns"] for index in contract["indexes"] if index["unique"]]
+    target_names = {column["name"] for column in columns}
     wanted = foreign_key["target_columns"]
     missing = [name for name in wanted if name not in target_names]
     if missing:
@@ -58,19 +132,11 @@ def _check_target_columns(session: Session, *, foreign_key: dict, target: MetaTa
             field="table_contract.foreign_keys",
         )
     single_unique = len(wanted) == 1 and any(
-        column.name == wanted[0] and (column.primary_key or column.unique) for column in columns
+        column["name"] == wanted[0] and (column["primary_key"] or column["unique"]) for column in columns
     )
-    primary_key_names = [column.name for column in columns if column.primary_key]
+    primary_key_names = [column["name"] for column in columns if column["primary_key"]]
     primary_key_set = bool(primary_key_names) and primary_key_names == wanted
-    unique_index = any(
-        list(index.columns or []) == wanted
-        for index in session.scalars(
-            select(MetaTableIndex).where(
-                MetaTableIndex.meta_table_uid == target.uid,
-                MetaTableIndex.unique.__eq__(True),
-            )
-        )
-    )
+    unique_index = any(list(index or []) == wanted for index in indexes)
     if not (single_unique or primary_key_set or unique_index):
         raise ContractValidationError(
             "Foreign key target_columns must reference a primary-key or unique MetaTable column set.",
@@ -84,6 +150,7 @@ def resolved_foreign_key_targets(
     *,
     data_source_uid: UUID,
     contract: dict,
+    source_table_uid: UUID | None = None,
 ) -> list[UUID]:
     """Preflight all targets before registering the new source table."""
     targets = []
@@ -93,7 +160,8 @@ def resolved_foreign_key_targets(
             source_data_source_uid=data_source_uid,
             foreign_key=foreign_key,
         )
-        _check_target_columns(session, foreign_key=foreign_key, target=target)
+        _check_target_columns(session, foreign_key=foreign_key, target=target,
+                              contract=contract if target.uid == source_table_uid else None)
         targets.append(target.uid)
     return targets
 

{mainsequence_metatable-0.1.31.dev58 → mainsequence_metatable-0.1.31.dev59}/tests/e2e/test_application_owned_migrations.py

@@ -127,6 +127,54 @@ def test_two_client_only_providers_evolve_downgrade_and_preserve_system_history(
         engine.dispose()
 
 
+@pytest.mark.parametrize("api_runtime", ["sqlite"], indirect=True)
+def test_unchanged_upgrade_has_no_metadata_writes_and_refreshes_security(
+    api_runtime, providers, monkeypatch
+):
+    from sqlalchemy import event
+    from sqlalchemy.orm import Session
+
+    from metatables import MetaTable
+    from metatables.api.backend.backends.sqlite_access import SQLiteAccess
+    from metatables.api.backend.persistence.models import SQLTablePolicy
+
+    migration = providers[0]
+    author(migration)
+    run_migration(migration)
+    uid = UUID(MetaTable.get(identifier="client_ledger.item").uid)
+    engine = api_runtime.app.state.runtime.bootstrap.engine
+    statements, validated = [], []
+    validate = SQLiteAccess.validate_table
+
+    def fresh_validate(access, connection, table):
+        validated.append(table.uid)
+        return validate(access, connection, table)
+
+    def record(_connection, _cursor, sql, _parameters, _context, _executemany):
+        if sql.lstrip().split()[0].upper() in {"INSERT", "UPDATE", "DELETE"}:
+            statements.append(sql)
+
+    monkeypatch.setattr(SQLiteAccess, "validate_table", fresh_validate)
+    event.listen(engine, "before_cursor_execute", record)
+    try:
+        result = run_migration(migration)
+        assert result["ok"] and not result["migrated"]
+        assert uid in validated
+        assert statements == []
+        # A trigger changes write safety without changing the table contract.
+        with engine.begin() as connection:
+            connection.exec_driver_sql("CREATE TRIGGER observed_trigger AFTER INSERT ON client_ledger_item "
+                                       "BEGIN SELECT 1; END")
+        run_migration(migration)
+        with Session(engine) as session:
+            assert session.get(SQLTablePolicy, uid).writable is False
+        assert not any("meta_table " in sql.lower() or "meta_table_column" in sql.lower()
+                       or "meta_table_index" in sql.lower() or "meta_table_foreign_key" in sql.lower()
+                       for sql in statements)
+    finally:
+        event.remove(engine, "before_cursor_execute", record)
+
+
 @pytest.mark.parametrize("api_runtime", ["sqlite"], indirect=True)
 def test_failed_finalization_can_be_retried_without_reapplying_ddl(
     api_runtime, providers, monkeypatch