mainsequence-metatable 0.1.31.dev60__tar.gz → 0.1.32.dev61__tar.gz

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence-metatable
-Version: 0.1.31.dev60
+Version: 0.1.32.dev61
 Summary: Main Sequence MetaTables Python client and standalone API
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License-Expression: MIT

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/docs/adr/api/0013-application-owned-migrations.md

@@ -169,6 +169,39 @@ flowchart TD
   replaying already-applied DDL. The response contract and client calls stay the
   same; no consuming-application opt-in is required.
 
+### Authored metadata and SQLite policy reflection (2026-10-10 amendment)
+
+The public migration runner sends current authored table descriptions, labels and
+column descriptions, labels and logical names in each finalization batch, including
+when Alembic is already at the requested revision. Reusing an existing binding at
+reservation time must not suppress metadata refresh ([#57](https://github.com/mainsequence-projects/MetaTables/issues/57)).
+The runner compares authored values with the fresh reservation lookup and sends
+only changed, non-empty metadata; it never caches equality across deployments.
+
+The optional `authored_metadata` map is keyed only by UIDs in that request. The API
+applies it after Writer/provider/source admission and fresh physical inspection,
+inside the table's savepoint. The physical snapshot remains authoritative for
+types, nullability, indexes, keys and binding identity. As with registration,
+empty/absent metadata preserves existing descriptions and labels. Equal metadata
+does not dirty rows or replace label links; the next unchanged upgrade still
+performs no catalog DML. Failed reconciliation rolls back metadata and can retry.
+Older clients may omit the map; clients that send it require an API supporting it.
+
+Security dependency reflection uses the backend's physical schema normalization.
+SQLite maps logical provider schemas and physical `main` cascade targets to the
+same physical namespace, rather than querying `PRAGMA public.foreign_key_list`.
+This keeps per-batch checks scoped when a downgrade has dropped tables belonging
+to later batches ([#58](https://github.com/mainsequence-projects/MetaTables/issues/58)).
+The conservative full policy sweep on a genuine reflection failure remains, and
+unexpected missing tables still fail without losing their catalog bindings. Only
+explicitly removed, freshly confirmed absent tables may lose bindings.
+
+Regression coverage runs the public client against a real disposable SQLite API:
+metadata changes at an unchanged revision, zero-write reruns, a provider larger
+than a finalization batch with later-batch drops, and a subsequent normal upgrade.
+Route coverage additionally checks physical-shape rejection, Writer admission,
+unrequested metadata UIDs, and failed metadata finalization/retry.
+
 ## MetaTables system migrations
 
 Hosted system upgrades also run as a deployment prerequisite Job

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/docs/api/openapi.json

@@ -18,6 +18,85 @@
         "title": "Assignments",
         "type": "object"
       },
+      "AuthoredColumnMetadata": {
+        "additionalProperties": false,
+        "properties": {
+          "description": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Description"
+          },
+          "label": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Label"
+          },
+          "logical_name": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Logical Name"
+          },
+          "name": {
+            "title": "Name",
+            "type": "string"
+          }
+        },
+        "required": [
+          "name"
+        ],
+        "title": "AuthoredColumnMetadata",
+        "type": "object"
+      },
+      "AuthoredTableMetadata": {
+        "additionalProperties": false,
+        "properties": {
+          "columns": {
+            "items": {
+              "$ref": "#/components/schemas/AuthoredColumnMetadata"
+            },
+            "title": "Columns",
+            "type": "array"
+          },
+          "description": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Description"
+          },
+          "labels": {
+            "items": {
+              "type": "string"
+            },
+            "title": "Labels",
+            "type": "array"
+          }
+        },
+        "title": "AuthoredTableMetadata",
+        "type": "object"
+      },
       "BatchDetailsPatch": {
         "properties": {
           "update_map": {
@@ -622,6 +701,23 @@
             ],
             "title": "Alembic Version Meta Table Uid"
           },
+          "authored_metadata": {
+            "anyOf": [
+              {
+                "additionalProperties": {
+                  "$ref": "#/components/schemas/AuthoredTableMetadata"
+                },
+                "propertyNames": {
+                  "format": "uuid"
+                },
+                "type": "object"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Authored Metadata"
+          },
           "code_repository_context": {
             "anyOf": [
               {

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/docs/client/deploy-application-migrations.md

@@ -37,6 +37,12 @@ scheduling and workflow coordination.
 An already-current database still reconciles the provider's catalog contracts.
 Checks use fresh table inventories and batched physical reflection; an unchanged
 revision does not skip missing-table checks, permissions or metadata refresh.
+Each finalization batch carries current authored descriptions, labels and column
+descriptions/labels/logical names. Non-empty values refresh existing bindings
+without a new revision; empty or absent values preserve catalog metadata, as in
+registration. Physical shape always comes from fresh database inspection. The
+next equal-valued upgrade skips catalog writes. This metadata payload requires
+the corresponding API update as well as the updated client.
 PostgreSQL batches metadata queries by schema; other dialects use SQLAlchemy's
 bulk reflection and its supported fallbacks. Caches do not survive the request.
 Client improvements require updating the application's installed MetaTables package

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mainsequence-metatable"
-version = "0.1.31.dev60"
+version = "0.1.32.dev61"
 description = "Main Sequence MetaTables Python client and standalone API"
 readme = "README.md"
 requires-python = ">=3.13"

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/src/mainsequence_metatable.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence-metatable
-Version: 0.1.31.dev60
+Version: 0.1.32.dev61
 Summary: Main Sequence MetaTables Python client and standalone API
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License-Expression: MIT

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/src/mainsequence_metatable.egg-info/SOURCES.txt

@@ -519,6 +519,7 @@ tests/e2e/test_local_cli_workflow.py
 tests/e2e/test_local_client_selection.py
 tests/e2e/test_managed_finalize_client_api.py
 tests/e2e/test_managed_finalize_unchanged.py
+tests/e2e/test_migration_metadata_and_downgrade.py
 tests/e2e/test_namespaces_admin_http.py
 tests/e2e/test_public_imports.py
 tests/e2e/test_remaining_client_routes.py

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/src/metatables/api/app/routes/managed_finalize.py

@@ -9,7 +9,7 @@ from typing import Any
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, HTTPException, Response
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 from sqlalchemy import delete
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
 from sqlalchemy.orm import Session
@@ -31,12 +31,14 @@ from metatables.api.backend.backends.timescale import (
 from metatables.api.backend.contracts.table_contracts import ContractValidationError
 from metatables.api.backend.domain.catalog_rules import CatalogInvariantError
 from metatables.api.backend.domain.data_source_rules import DataSourceAdmissionError
+from metatables.api.backend.domain.label_rules import LabelInvariantError, normalize_label_name
 from metatables.api.backend.domain.permission_rules import TrustedActorFacts
 from metatables.api.backend.integrations.data_source_provider import DataSourceProvider
 from metatables.api.backend.integrations.runtime_data_sources import DataSourceIntegrationError
 from metatables.api.backend.operations.finalize_managed import contract_from_physical_snapshot
 from metatables.api.backend.operations.register_catalog import time_index_spec_from_contract
 from metatables.api.backend.persistence.grants import GrantRepository
+from metatables.api.backend.persistence.labels import LabelRepository
 from metatables.api.backend.persistence.locking import lock_catalog
 from metatables.api.backend.persistence.models import (
     MetaTable,
@@ -58,6 +60,28 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/meta-tables", tags=["MetaTables"])
 
 
+class AuthoredColumnMetadata(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    description: str | None = None
+    label: str | None = None
+    logical_name: str | None = None
+
+
+class AuthoredTableMetadata(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    description: str | None = None
+    labels: list[str] = Field(default_factory=list)
+    columns: list[AuthoredColumnMetadata] = Field(default_factory=list)
+
+    @field_validator("labels")
+    @classmethod
+    def valid_labels(cls, labels):
+        return [normalize_label_name(label) for label in labels if label.strip()]
+
+
 class FinalizeManagedRequest(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
@@ -71,6 +95,7 @@ class FinalizeManagedRequest(BaseModel):
     # these may lose their binding when absent; absence alone is never removal.
     removed_meta_table_uids: list[UUID] = Field(default_factory=list, max_length=200)
     code_repository_context: dict[str, Any] | None = None
+    authored_metadata: dict[UUID, AuthoredTableMetadata] | None = None
 
 
 def physical_hypertable_converter() -> Callable[..., None]:
@@ -134,6 +159,8 @@ def finalize_managed(
     """Reconcile each requested row and return a per-table result envelope."""
     started = perf_counter()
     requested = list(dict.fromkeys(payload.meta_table_uids))
+    if set(payload.authored_metadata or {}) - set(requested):
+        raise HTTPException(status_code=422, detail="Authored metadata must name requested MetaTables.")
     removed = set(payload.removed_meta_table_uids)
     lock_catalog(session)
     guards = {
@@ -232,8 +259,10 @@ def finalize_managed(
             continue
         try:
             with session.begin_nested():
+                authored = (payload.authored_metadata or {}).get(uid)
                 contract = contract_from_physical_snapshot(
-                    session=session, table=table, snapshot=snapshot
+                    session=session, table=table, snapshot=snapshot,
+                    authored_metadata=authored.model_dump() if authored else None,
                 )
                 targets = resolved_foreign_key_targets(
                     session, data_source_uid=table.data_source_uid, contract=contract,
@@ -251,6 +280,14 @@ def finalize_managed(
                     "provisioning_status": "active",
                     "protect_from_deletion": True,
                 }
+                if authored is not None:
+                    if authored.description and authored.description.strip():
+                        updates["description"] = authored.description
+                    if authored.labels:
+                        LabelRepository(session).mutate_target(
+                            kind="meta_table", target_uid=uid, mode="replace",
+                            names=authored.labels, created_by_user_uid=actor.user_uid,
+                        )
                 if not same_contract_state(table.table_contract, contract):
                     updates["table_contract"] = contract
                 if not same_contract_state(table.introspection_snapshot, snapshot):
@@ -288,7 +325,7 @@ def finalize_managed(
             check_table_security_on_commit(session, table)
             rows.append(_row(table, uid, previous=previous, specialization=specialization,
                              exists=True, finalized=previous != "active"))
-        except (CatalogInvariantError, ContractValidationError, IntegrityError) as exc:
+        except (CatalogInvariantError, ContractValidationError, LabelInvariantError, IntegrityError) as exc:
             logger.warning("Physical table contract could not be reconciled", exc_info=True)
             rows.append(_row(
                 table, uid, previous=previous, specialization=specialization, exists=True,

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/src/metatables/api/backend/backends/access.py

@@ -217,7 +217,11 @@ class DatabaseAccess:
         }
 
         def key(schema, name):
-            return self.backend.catalog.registration_schema(self.source, schema), name
+            # SQLite stores logical provider schemas such as public, but its
+            # physical relations and cascade targets all live in main.
+            return self.backend.catalog.schema(
+                self.backend.catalog.registration_schema(self.source, schema)
+            ), name
 
         by_name = {key(t.physical_schema, t.physical_table_name): uid for uid, t in tables.items()}
         names = {key(t.physical_schema, t.physical_table_name) for uid, t in tables.items() if uid in changed}
@@ -235,7 +239,8 @@ class DatabaseAccess:
                 continue
             try:
                 references = inspector.get_foreign_keys(
-                    table.physical_table_name, schema=table.physical_schema
+                    table.physical_table_name,
+                    schema=self.backend.catalog.schema(table.physical_schema),
                 )
             except Exception:
                 return None

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/src/metatables/api/backend/operations/finalize_managed.py

@@ -15,7 +15,8 @@ from metatables.api.backend.persistence.models import MetaTable
 
 
 def contract_from_physical_snapshot(
-    *, session: Session, table: MetaTable, snapshot: Mapping[str, Any]
+    *, session: Session, table: MetaTable, snapshot: Mapping[str, Any],
+    authored_metadata: Mapping[str, Any] | None = None,
 ) -> dict[str, Any]:
     """Use physical column, index, and FK facts; preserve declared semantic metadata."""
     schema = str(snapshot.get("schema_name") or table.physical_schema or "").strip()
@@ -38,12 +39,18 @@ def contract_from_physical_snapshot(
         item.get("name"): item for item in existing.get("columns") or ()
         if isinstance(item, Mapping)
     }
+    authored_columns = {column["name"]: column for column in
+                        (authored_metadata or {}).get("columns") or ()}
     columns = []
     for index, item in enumerate(snapshot.get("columns") or ()):
         if not isinstance(item, Mapping) or not item.get("name"):
             continue
         column_name = str(item["name"])
         prior = prior_columns.get(column_name) or {}
+        # Match registration semantics: absent/empty metadata preserves the
+        # catalog's descriptions. Authored shape never overrides fresh DDL.
+        prior = {**prior, **{key: value for key, value in authored_columns.get(column_name, {}).items()
+                            if key in {"description", "label", "logical_name"} and value}}
         columns.append({
             "name": column_name,
             "logical_name": prior.get("logical_name"),

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/src/metatables/migrations/__init__.py

@@ -622,6 +622,26 @@ class AlembicMetaTableMigration:
             dict.fromkeys([str(registry_uid), *prepared.meta_table_uids])
         )
         removed = set(prepared.removed_meta_table_uids)
+        # Existing bindings bypass reservation. Carry their current descriptive
+        # metadata to finalization without trusting models for physical shape.
+        authored_metadata = {}
+        for model in self.metatable_models:
+            uid = model.get_meta_table_uid()
+            if uid in (None, "") or str(uid) in removed:
+                continue
+            request = model.build_registration_request(data_source_uid=prepared.data_source_uid)
+            contract = _collection_create_table_contract_payload(request.table_contract)
+            metadata = {
+                "description": request.description,
+                "labels": list(request.labels or []),
+                "columns": [
+                    {key: column.get(key) for key in ("name", "description", "label", "logical_name")}
+                    for column in contract.get("columns") or []
+                ],
+            }
+            changes = _changed_authored_metadata(_bound_meta_table_for_model(model), metadata)
+            if changes:
+                authored_metadata[str(uid)] = changes
         response = _finalize_in_batches(
             finalize_meta_table_uids,
             lambda batch: MetaTable.finalize_managed(
@@ -633,6 +653,8 @@ class AlembicMetaTableMigration:
                     alembic_version_meta_table_uid=str(registry_uid),
                     alembic_revision=alembic_revision,
                     removed_meta_table_uids=[uid for uid in batch if uid in removed] or None,
+                    authored_metadata={uid: authored_metadata[uid] for uid in batch
+                                       if uid in authored_metadata} or None,
                 ),
                 timeout=timeout,
                 on_status=on_metatable_finalize_status,
@@ -1062,6 +1084,33 @@ def resolve_alembic_revision_metadata(
     return str(resolved.revision), _normalize_down_revision(resolved.down_revision)
 
 
+def _changed_authored_metadata(table: MetaTable | None, metadata: dict[str, Any]) -> dict[str, Any]:
+    """Compare with the fresh reservation lookup, never a cached prior deployment.
+
+    Omit equal or empty values so unchanged upgrades need no extra metadata/label
+    reconciliation. Finalization still inspects physical shape and projections.
+    """
+    if table is None:
+        return metadata
+    changes = {}
+    description = metadata.get("description")
+    if description and description.strip() and description != table.description:
+        changes["description"] = description
+    labels = [" ".join(label.split()) for label in metadata.get("labels") or [] if label.strip()]
+    if labels and set(labels) != set(table.labels):
+        changes["labels"] = labels
+    current = {column["name"]: column for column in table.table_contract.get("columns") or []}
+    columns = []
+    for column in metadata.get("columns") or []:
+        fields = {key: value for key, value in column.items()
+                  if key != "name" and value and current.get(column["name"], {}).get(key) != value}
+        if fields:
+            columns.append({"name": column["name"], **fields})
+    if columns:
+        changes["columns"] = columns
+    return changes
+
+
 def _collection_create_row_from_registration_request(
     request: Any,
     *,

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/src/metatables/models.py

@@ -934,6 +934,10 @@ class MetaTableMigrationConnection(BasePydanticModel):
 
 
 class ManagedMetaTableFinalizeRequest(BasePydanticModel):
+    authored_metadata: dict[str, dict[str, Any]] | None = Field(
+        None,
+        description="Current authored descriptions, labels and column metadata, keyed by requested UID.",
+    )
     meta_table_uids: list[str] = Field(
         ...,
         min_length=1,

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/tests/e2e/test_application_owned_migrations.py

@@ -242,9 +242,8 @@ def test_batched_finalization_sends_every_batch_and_merges_the_results(monkeypat
 def test_current_revision_still_refreshes_physical_shape_and_rejects_missing_tables(
     api_runtime, providers, caplog
 ):
-    from mainsequence.client.exceptions import BadRequestError
-
     from metatables import MetaTable
+    from metatables.migrations import AlembicProviderPhysicalStateError
     from metatables.runtime_context import get_runtime_context
 
     migration = providers[0]
@@ -265,9 +264,9 @@ def test_current_revision_still_refreshes_physical_shape_and_rejects_missing_tab
             assert f"{phase} took" in caplog.text
         with engine.begin() as connection:
             connection.exec_driver_sql("DROP TABLE client_ledger_item")
-        # SQLite's existing permission reconciliation rejects an active binding
-        # whose table is gone, before finalization can commit its result envelope.
-        with pytest.raises(BadRequestError, match="ordinary_table_required"):
+        # Scoped policy reflection lets the route return the precise missing-table
+        # failure without deleting the unexpectedly missing binding.
+        with pytest.raises(AlembicProviderPhysicalStateError, match="physical_table_missing"):
             run_migration(migration)
         assert MetaTable.get(identifier="client_ledger.item").uid == item.uid
     finally:

{mainsequence_metatable-0.1.31.dev60 → mainsequence_metatable-0.1.32.dev61}/tests/e2e/test_managed_finalize_unchanged.py

@@ -254,3 +254,68 @@ def test_unchanged_state_still_runs_fresh_checks(finalized, failure):
     assert finalized.statements == []
     if failure == "permission":
         assert [name for _, name in finalized.inspections[-1]] == ["parents"]
+
+
+def test_authored_metadata_changes_only_semantics_and_converges(finalized):
+    uid = finalized.uids[1]
+    finalized.payload["authored_metadata"] = {str(uid): {
+        "description": "Changed authored table meaning",
+        "labels": ["authored"],
+        "columns": [{"name": "value", "description": "Changed authored column meaning",
+                     "label": "Value", "logical_name": "semantic_value"}],
+    }}
+    response = finalized.finalize()
+    assert response.status_code == 200, response.text
+    with Session(finalized.engine) as session:
+        table = session.get(MetaTable, uid)
+        assert table.description == "Changed authored table meaning"
+        column = session.scalar(select(MetaTableColumn).where(
+            MetaTableColumn.meta_table_uid == uid, MetaTableColumn.name == "value"))
+        assert column.description == "Changed authored column meaning"
+        assert column.data_type == "integer" and column.nullable
+    before = projection_uids(finalized.engine)
+    assert finalized.finalize().status_code == 200
+    assert finalized.statements == []
+    assert projection_uids(finalized.engine) == before
+
+
+@pytest.mark.parametrize("invalid", ["shape", "unrequested", "denied"])
+def test_authored_metadata_cannot_bypass_shape_or_writer_admission(finalized, invalid):
+    uid = finalized.uids[1]
+    metadata = {"description": "Must not be applied"}
+    if invalid == "shape":
+        metadata["columns"] = [{"name": "value", "data_type": "text", "nullable": False}]
+    if invalid == "unrequested":
+        uid = uuid4()
+    if invalid == "denied":
+        with Session(finalized.engine) as session:
+            GrantRepository(session).revoke_table_grant(
+                table_uid=uid, principal_kind="user", principal_uid=finalized.user)
+            session.commit()
+    finalized.payload["authored_metadata"] = {str(uid): metadata}
+    response = finalized.finalize()
+    assert response.status_code == (409 if invalid == "denied" else 422)
+    assert finalized.statements == []
+    with Session(finalized.engine) as session:
+        assert session.get(MetaTable, finalized.uids[1]).description != "Must not be applied"
+
+
+def test_failed_authored_metadata_finalization_rolls_back_and_retries(finalized, monkeypatch):
+    uid = finalized.uids[1]
+    finalized.payload["authored_metadata"] = {str(uid): {
+        "description": "Retry table description", "labels": ["retry"],
+        "columns": [{"name": "value", "description": "Retry column description"}],
+    }}
+    with Session(finalized.engine) as session:
+        before = session.get(MetaTable, uid).description
+    with monkeypatch.context() as failure:
+        failure.setattr(managed_finalize, "materialize_contract_projection",
+                       lambda *a, **kw: (_ for _ in ()).throw(IntegrityError("insert", {}, Exception())))
+        assert finalized.finalize().status_code == 409
+    with Session(finalized.engine) as session:
+        assert session.get(MetaTable, uid).description == before
+    assert finalized.finalize().status_code == 200
+    with Session(finalized.engine) as session:
+        assert session.get(MetaTable, uid).description == "Retry table description"
+    assert finalized.finalize().status_code == 200
+    assert finalized.statements == []