mainsequence-metatable 0.1.0__tar.gz → 0.1.2.dev3__tar.gz

{mainsequence_metatable-0.1.0/mainsequence_metatable.egg-info → mainsequence_metatable-0.1.2.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence-metatable
-Version: 0.1.0
+Version: 0.1.2.dev3
 Summary: Main Sequence MetaTables Python client and standalone API
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License-Expression: MIT
@@ -32,7 +32,7 @@ Requires-Dist: pyodbc<6,>=5
 Requires-Dist: pydantic<3,>=2.11
 Requires-Dist: pyyaml
 Requires-Dist: requests>=2.33.0
-Requires-Dist: sqlalchemy<3,>=2
+Requires-Dist: sqlalchemy<3,>=2.1
 Requires-Dist: psycopg[binary]<4,>=3.2
 Requires-Dist: structlog
 Requires-Dist: tqdm<5,>=4.66

{mainsequence_metatable-0.1.0 → mainsequence_metatable-0.1.2.dev3}/api/app/bootstrap.py

@@ -33,7 +33,12 @@ from api.backend.persistence.database import (
     migration_config,
     upgrade_catalog,
 )
-from api.backend.persistence.models import Base, DataSource, MetaTable, PhysicalOperation
+from api.backend.persistence.models import (
+    Base,
+    DataSource,
+    MetaTable,
+    PhysicalOperation,
+)
 from metatables.runtime_contract import RuntimeBootstrapDescriptor
 
 VERSION_TABLE = "metatables_catalog_version"
@@ -77,7 +82,8 @@ class RuntimeBootstrap:
         self.uid = None
         self.selected_source_uid = None
         self.saved_path = (config.configuration_path or Path.cwd() / "configuration.yaml").resolve().parent / ".local/runtime-data-sources.json"
-        self.selection_key = str(self.workspace.uid) if self.workspace else "hosted"
+        self.selection_key = "local" if self.workspace else "hosted"
+        self.save_local_selection = False
 
     def descriptor(self, user_uid, *, is_admin=False):
         can_configure = is_admin
@@ -136,8 +142,20 @@ class RuntimeBootstrap:
                     if self.selection_key == "hosted":
                         self.selected_source_uid = saved.get("hosted_source_uid")
                     return saved[self.selection_key]
+                if self.workspace:
+                    # Adopt an unambiguous old file intact, never copy catalog rows.
+                    legacy = {value["configuration"]["path"]: value for value in saved.values()
+                              if isinstance(value, dict) and value.get("class_type") == "sqlite"
+                              and isinstance(value.get("configuration", {}).get("path"), str)}
+                    if len(legacy) == 1:
+                        self.save_local_selection = True
+                        return next(iter(legacy.values()))
+                    if legacy:
+                        self.error = "Multiple legacy local databases are saved. Choose the existing local runtime file in Settings; no database has been replaced."
+                        return None
             except (OSError, ValueError):
                 self.error = "Saved runtime selection cannot be read. Configure the DataSource again."
+                return None
         if self.workspace:
             return dict(display_name=self.workspace.data_source.display_name, class_type="sqlite",
                         configuration={"path": str(self.workspace.tables_path)})
@@ -164,7 +182,15 @@ class RuntimeBootstrap:
             path = path.resolve()
             marker = _read_marker(path.with_name(path.name + ".metatables-scope.json"))
             if marker and marker != {"workspace_uid": str(self.workspace.uid), "purpose": "runtime"}:
-                raise BootstrapError("This file belongs to another workspace or the retired two-file layout.")
+                # Preserve the selected legacy file's runtime/source IDs and log
+                # ownership. Git branches no longer define storage identity.
+                saved = json.loads(self.saved_path.read_text()) if self.saved_path.exists() else {}
+                known = saved.get("local") or saved.get(marker.get("workspace_uid"))
+                if (marker.get("purpose") != "runtime" or not isinstance(known, dict)
+                        or known.get("class_type") != "sqlite"
+                        or known.get("configuration", {}).get("path") != str(path)):
+                    raise BootstrapError("This file belongs to another workspace or the retired two-file layout.")
+                self.workspace = replace(self.workspace, uid=UUID(marker["workspace_uid"]))
             self.workspace = replace(self.workspace, database_path=path)
             values["configuration"] = {"path": str(path)}
         else:
@@ -298,21 +324,6 @@ class RuntimeBootstrap:
                         raise BootstrapError("The existing runtime registration cannot be replaced by this configuration or caller.")
                     from api.backend.backends.registry import backend_for
                     session.flush()
-                    if self.workspace:
-                        # Existing local candidate metadata is materialized into
-                        # the catalog, retaining UIDs. Credentials require the
-                        # explicit import or password replacement procedure.
-                        from api.app.source_registrations import read_registry
-                        saved = read_registry(self.saved_path.with_name("runtime-source-candidates.json"))
-                        for candidate in saved.values():
-                            if candidate.get("class_type") not in {"postgresql", "timescale_db", "mysql", "mssql"}:
-                                continue
-                            candidate_uid = UUID(candidate["uid"])
-                            if session.get(DataSource, candidate_uid) is None:
-                                session.add(DataSource(uid=candidate_uid, created_by_user_uid=user_uid,
-                                    display_name=candidate["display_name"], class_type=candidate["class_type"],
-                                    configuration=candidate["configuration"], storage_access_mode=candidate["storage_access_mode"],
-                                    status=candidate["status"], is_default=False, is_runtime=False))
                     backend_for(replace(self.backend.source, uid=row.uid)).access.initialize(session)
                     credentials = bind_store(self.credential_store, session, user_uid)
                     if hasattr(credentials, "initialize"):
@@ -341,6 +352,9 @@ class RuntimeBootstrap:
         self.saved_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
         # Public configuration only: password and TLS material remain Secret references.
         saved = json.loads(self.saved_path.read_text()) if self.saved_path.exists() else {}
+        if self.workspace:
+            saved = {key: value for key, value in saved.items()
+                     if not (isinstance(value, dict) and value.get("class_type") == "sqlite")}
         saved[self.selection_key] = self.candidate
         if self.selection_key == "hosted":
             if self.selected_source_uid:
@@ -413,9 +427,10 @@ class RuntimeBootstrap:
         if self.workspace:
             _bind_file(Path(self.candidate["configuration"]["path"]),
                        workspace_uid=self.workspace.uid, purpose="runtime")
-        if persist:
+        if persist or self.save_local_selection:
             try:
                 self.persist_selection()
+                self.save_local_selection = False
             except Exception:
                 raise BootstrapError("Unable to save runtime selection. The database is initialized but was not activated.") from None
         binding = RuntimeStorageBinding(catalog_url=str(self.engine.url), workspace=self.workspace,

{mainsequence_metatable-0.1.0 → mainsequence_metatable-0.1.2.dev3}/api/app/main.py

@@ -25,6 +25,7 @@ from api.app.routes.table_crud import router as table_crud_router
 from api.app.routes.table_delete import router as table_delete_router
 from api.app.routes.table_filter import router as table_filter_router
 from api.app.routes.table_import import router as table_import_router
+from api.app.routes.table_import import source_router as source_relations_router
 from api.app.routes.table_introspection import router as table_introspection_router
 from api.app.routes.table_patch import router as table_patch_router
 from api.app.routes.table_registration import router as table_registration_router
@@ -180,6 +181,7 @@ def create_app(config: RuntimeConfig | None = None) -> FastAPI:
     service.include_router(migration_connection_router)
     service.include_router(application_migrations_router)
     service.include_router(table_import_router)
+    service.include_router(source_relations_router)
     service.include_router(table_registration_router)
     service.include_router(table_cascade_router)
     for inherited in (contract_validation_router, column_search_router, schema_graph_router,

{mainsequence_metatable-0.1.0 → mainsequence_metatable-0.1.2.dev3}/api/app/routes/table_import.py

@@ -2,7 +2,7 @@
 
 from uuid import UUID
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
 
@@ -15,14 +15,21 @@ from api.backend.contracts.relation_import import (
     ImportRelations,
     ImportResult,
     ReadRelation,
+    RelationDiscovery,
     RelationRows,
 )
 from api.backend.domain.data_source_rules import DataSourceAdmissionError
 from api.backend.domain.permission_rules import TrustedActorFacts
+from api.backend.integrations.credential_store import CredentialStoreError
 from api.backend.integrations.runtime_data_sources import DataSourceIntegrationError
-from api.backend.operations.import_relations import import_relations, read_relation
+from api.backend.operations.import_relations import (
+    discover_import_relations,
+    import_relations,
+    read_relation,
+)
 
 router = APIRouter(prefix="/meta-tables", tags=["MetaTables"])
+source_router = APIRouter(prefix="/data-sources", tags=["DataSources"])
 
 
 def invoke(session, operation):
@@ -31,6 +38,9 @@ def invoke(session, operation):
     except RelationError as exc:
         session.rollback()
         raise HTTPException(exc.status, {"code": exc.code}) from None
+    except CredentialStoreError as exc:
+        session.rollback()
+        raise HTTPException(503, {"code": exc.code, "detail": exc.message}) from None
     except (DataSourceIntegrationError, DataSourceAdmissionError, SQLExecutionError) as exc:
         session.rollback()
         raise HTTPException(getattr(exc, "status_code", 409), {"code": exc.code}) from None
@@ -45,6 +55,19 @@ def invoke(session, operation):
         raise HTTPException(422, {"code": "invalid_relation_contract"}) from None
 
 
+@source_router.get("/{source_uid}/relations/", response_model=RelationDiscovery)
+def discover_source_relations(
+    source_uid: UUID,
+    physical_schema: str | None = Query(default=None, max_length=128),
+    session: Session = Depends(catalog_session),
+    actor: TrustedActorFacts = Depends(trusted_actor_facts),
+    sources=Depends(data_source_provider),
+):
+    return invoke(session, lambda: discover_import_relations(
+        session, sources, actor, source_uid, physical_schema
+    ))
+
+
 @router.post("/import-from-data-source/", response_model=ImportResult)
 def import_source(
     payload: ImportRelations,

{mainsequence_metatable-0.1.0 → mainsequence_metatable-0.1.2.dev3}/api/backend/backends/contracts.py

@@ -35,7 +35,8 @@ class OperationContext:
 class TableBackend(Protocol):
     def context_for_catalog(self, session, context): ...
     def discover_relations(self, *, schema=None, context, external=False): ...
-    def introspect_relations(self, *, schema, names, context, external=False): ...
+    def inspect_relations(self, *, schema=None, context, external=False): ...
+    def introspect_relations(self, *, schema, names, context, external=False, check_read=True): ...
     def read_relation(self, *, table, selection, admission, context): ...
 
     def statistics(

{mainsequence_metatable-0.1.0 → mainsequence_metatable-0.1.2.dev3}/api/backend/backends/relation_tables.py

@@ -14,6 +14,7 @@ from time import monotonic
 from uuid import UUID
 
 import sqlalchemy as sa
+from sqlalchemy.engine.reflection import ObjectKind, ObjectScope
 from sqlalchemy.exc import SQLAlchemyError
 
 from api.backend.backends.restricted_sql import json_value
@@ -42,6 +43,12 @@ class RelationContext:
         return value
 
 
+@dataclass
+class RelationSnapshotBatch:
+    snapshots: dict[str, dict]
+    failures: dict[str, RelationError]
+
+
 def protected(schema, name):
     from api.backend.persistence.models import Base
 
@@ -215,29 +222,59 @@ class RelationAdapter:
         inspector = sa.inspect(connection)
         return inspector.has_table("credential_vault", schema=inspector.default_schema_name)
 
-    def discover(self, connection, schema):
+    def discover(self, connection, schema, inspector=None):
         schema = self.physical_schema(schema)
-        inspector = sa.inspect(connection)
+        inspector = inspector if inspector is not None else sa.inspect(connection)
         relations = {name: "table" for name in inspector.get_table_names(schema=schema)}
         relations.update({name: "view" for name in inspector.get_view_names(schema=schema)})
         return {name: kind for name, kind in relations.items() if not protected(schema, name)}
 
-    def inspect(self, connection, schema, name, kind):
-        canonical_schema = schema
-        schema = self.physical_schema(schema)
-        inspector = sa.inspect(connection)
-        columns = inspector.get_columns(name, schema=schema)
-        keys = (
-            inspector.get_pk_constraint(name, schema=schema).get("constrained_columns") or []
-            if kind == "table"
-            else []
-        )
-        indexes = inspector.get_indexes(name, schema=schema) if kind == "table" else []
-        unique = (
-            self.backend.catalog.unique_constraints(inspector, name, schema=schema)
-            if kind == "table"
-            else []
+    def unique_constraints(self, inspector, options):
+        return inspector.get_multi_unique_constraints(**options)
+
+    def inspect_many(self, inspector, schema, names, visible):
+        """Reflect a selection on one Inspector, retaining its dialect metadata cache.
+
+        PostgreSQL and MSSQL use native bulk catalog queries. Dialects that
+        reflect individual definitions (MySQL/SQLite) reuse the same connection
+        and cached definitions across columns, keys and indexes.
+        """
+        if not names:
+            return RelationSnapshotBatch({}, {})
+        physical_schema = self.physical_schema(schema)
+        unreflectable = {}
+        options = dict(
+            schema=physical_schema, filter_names=names, kind=ObjectKind.ANY,
+            scope=ObjectScope.ANY, unreflectable=unreflectable,
         )
+        # Names have already been admitted by discovery. ANY avoids redundant
+        # name discovery in SQLAlchemy's per-object reflection fallback.
+        columns = inspector.get_multi_columns(**options)
+        table_names = [name for name in names if visible[name] == "table"]
+        keys, indexes, unique, foreign_keys = {}, {}, {}, {}
+        if table_names:
+            options = {**options, "filter_names": table_names}
+            keys = inspector.get_multi_pk_constraint(**options)
+            indexes = inspector.get_multi_indexes(**options)
+            unique = self.unique_constraints(inspector, options)
+            foreign_keys = inspector.get_multi_foreign_keys(**options)
+        result = RelationSnapshotBatch({}, {})
+        for name in names:
+            key = (physical_schema, name)
+            if key in unreflectable or not columns.get(key):
+                result.failures[name] = RelationError("relation_metadata_unavailable")
+                continue
+            try:
+                result.snapshots[name] = self.snapshot(
+                    inspector.bind, schema, name, visible[name], columns[key],
+                    keys.get(key, {}).get("constrained_columns") or [],
+                    indexes.get(key, []), unique.get(key, []), foreign_keys.get(key, []),
+                )
+            except RelationError as exc:
+                result.failures[name] = exc
+        return result
+
+    def snapshot(self, connection, schema, name, kind, columns, keys, indexes, unique, foreign_keys):
         by_name = {}
         for index in [*indexes, *[{**item, "unique": True} for item in unique]]:
             names = index.get("column_names") or []
@@ -249,9 +286,8 @@ class RelationAdapter:
             for item in by_name.values()
             if item["unique"] and len(item["columns"]) == 1
         }
-        foreign_keys = inspector.get_foreign_keys(name, schema=schema) if kind == "table" else []
         result = dict(
-            schema_name=canonical_schema or "",
+            schema_name=schema or "",
             table_name=name,
             relation_kind=kind,
             dialect=connection.dialect.name,
@@ -272,7 +308,7 @@ class RelationAdapter:
                 dict(
                     name=f.get("name") or "fk_" + "_".join(f["constrained_columns"]),
                     source_columns=f["constrained_columns"],
-                    target_schema=f.get("referred_schema") or canonical_schema or "",
+                    target_schema=f.get("referred_schema") or schema or "",
                     target_table=f["referred_table"],
                     target_columns=f["referred_columns"],
                     on_delete=(f.get("options", {}).get("ondelete") or "no action").lower(),
@@ -280,13 +316,25 @@ class RelationAdapter:
                 for f in foreign_keys
             ],
         )
-        # A bounded probe tests database read access, including view dependencies.
-        table = sa.table(name, *[sa.column(c["name"]) for c in columns], schema=schema)
-        rows = connection.execute(sa.select(table).limit(1)).mappings().all()
-        for row in rows:
-            wire_value(dict(row))
         return bounded_json(result, 262144, "relation_metadata_limit_exceeded")
 
+    def validate_reads(self, connection, snapshots):
+        # Each zero-row subquery binds every selected column for permission and
+        # view validation. A constant outer projection makes different physical
+        # shapes compatible in one UNION ALL, without reading application rows.
+        statements = []
+        for snapshot in snapshots:
+            table = sa.table(
+                snapshot["table_name"],
+                *[sa.column(c["name"]) for c in snapshot["columns"]],
+                schema=self.physical_schema(snapshot["schema_name"]),
+            )
+            statements.append(
+                sa.select(sa.literal_column("1")).select_from(sa.select(table).limit(0).subquery())
+            )
+        with connection.execute(sa.union_all(*statements)):
+            pass
+
     def relation_identity(self, connection, identity, table):
         return identity
 
@@ -499,6 +547,11 @@ class MySQLRelations(RelationAdapter):
 
 
 class MSSQLRelations(RelationAdapter):
+    def unique_constraints(self, inspector, options):
+        # SQL Server's reflected indexes already include unique constraints.
+        # Its unique-constraint Inspector method is not implemented.
+        return {}
+
     def database_identity(self, connection):
         return (
             "mssql",
@@ -550,6 +603,77 @@ class MSSQLRelations(RelationAdapter):
                 pending.append((target_schema, target))
 
 
+class RelationInspection:
+    """One read connection and one discovery result for a selection and FK closure."""
+
+    def __init__(self, adapter, connection, schema, context, external):
+        self.adapter, self.connection = adapter, connection
+        self.schema, self.context, self.external = schema, context, external
+        self.inspector = sa.inspect(connection)
+        self.visible = adapter.discover(connection, schema, self.inspector)
+
+    def inspect(self, names, *, check_read=True):
+        result = RelationSnapshotBatch({}, {})
+        selected = []
+        for name in dict.fromkeys(names):
+            if name not in self.visible:
+                result.failures[name] = RelationError("relation_not_found", 404)
+            else:
+                selected.append(name)
+        self.context.remaining()
+        reflected = self.adapter.inspect_many(
+            self.inspector, self.schema, selected, self.visible
+        )
+        result.failures.update(reflected.failures)
+        for name, snapshot in reflected.snapshots.items():
+            self.context.remaining()
+            try:
+                if not self.external and self.visible[name] == "view":
+                    self.adapter.validate_runtime_view(self.connection, self.schema, name)
+            except RelationError as exc:
+                if exc.status == 504:
+                    raise
+                result.failures[name] = exc
+            except SQLAlchemyError:
+                self.context.remaining()
+                if self.connection.invalidated:
+                    raise RelationError("relation_access_failed", 503) from None
+                self.reset_transaction()
+                result.failures[name] = RelationError("relation_access_failed", 503)
+            else:
+                result.snapshots[name] = snapshot
+        if check_read and result.snapshots:
+            failures = self.check_reads(list(result.snapshots.values()))
+            result.failures.update(failures)
+            for name in failures:
+                del result.snapshots[name]
+        self.context.remaining()
+        return result
+
+    def reset_transaction(self):
+        # A denied SELECT may abort a PostgreSQL transaction. Restore the
+        # read-only settings before checking the remaining relations.
+        self.connection.rollback()
+        self.adapter.configure(self.connection, self.context)
+
+    def check_reads(self, snapshots):
+        self.context.remaining()
+        try:
+            self.adapter.validate_reads(self.connection, snapshots)
+        except SQLAlchemyError:
+            self.context.remaining()
+            if self.connection.invalidated:
+                raise RelationError("relation_access_failed", 503) from None
+            self.reset_transaction()
+            if len(snapshots) == 1:
+                return {snapshots[0]["table_name"]: RelationError("relation_access_failed", 503)}
+            # Isolate denied/broken relations only on failure. The normal path
+            # checks the entire batch with one round trip.
+            middle = len(snapshots) // 2
+            return self.check_reads(snapshots[:middle]) | self.check_reads(snapshots[middle:])
+        return {}
+
+
 class RelationTables:
     @property
     def relations(self):
@@ -568,29 +692,24 @@ class RelationTables:
         return replace(context, runtime_database_identity=adapter.database_identity(connection))
 
     def discover_relations(self, *, schema=None, context, external=False):
-        adapter = self.relations
-        schema = adapter.schema(schema)
-        with adapter.connection(context) as connection:
-            if external:
-                adapter.require_external(connection, context)
-            return schema or "", adapter.discover(connection, schema)
+        with self.inspect_relations(schema=schema, context=context, external=external) as reader:
+            return reader.schema or "", reader.visible
 
-    def introspect_relations(self, *, schema, names, context, external=False):
+    @contextmanager
+    def inspect_relations(self, *, schema=None, context, external=False):
         adapter = self.relations
         schema = adapter.schema(schema)
         with adapter.connection(context) as connection:
             if external:
                 adapter.require_external(connection, context)
-            visible = adapter.discover(connection, schema)
-            results = {}
-            for name in names:
-                context.remaining()
-                if name not in visible:
-                    raise RelationError("relation_not_found", 404)
-                if not external and visible[name] == "view":
-                    adapter.validate_runtime_view(connection, schema, name)
-                results[name] = adapter.inspect(connection, schema, name, visible[name])
-            return results
+            yield RelationInspection(adapter, connection, schema, context, external)
+
+    def introspect_relations(self, *, schema, names, context, external=False, check_read=True):
+        with self.inspect_relations(schema=schema, context=context, external=external) as reader:
+            batch = reader.inspect(names, check_read=check_read)
+            if batch.failures:
+                raise next(iter(batch.failures.values()))
+            return batch.snapshots
 
     def read_relation(self, *, table, selection, admission, context):
         adapter = self.relations

{mainsequence_metatable-0.1.0 → mainsequence_metatable-0.1.2.dev3}/api/backend/contracts/relation_import.py

@@ -6,6 +6,20 @@ from uuid import UUID
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
+class DiscoveredRelation(BaseModel):
+    name: str
+    relation_kind: Literal["table", "view"]
+    meta_table_uid: UUID | None
+    importable: bool
+    blocked_reason: str | None = None
+
+
+class RelationDiscovery(BaseModel):
+    data_source_uid: UUID
+    physical_schema: str
+    relations: list[DiscoveredRelation]
+
+
 class ImportRelations(BaseModel):
     model_config = ConfigDict(extra="forbid")
     data_source_uid: UUID

{mainsequence_metatable-0.1.0 → mainsequence_metatable-0.1.2.dev3}/api/backend/integrations/local_workspace.py

@@ -1,4 +1,4 @@
-"""API-owned local storage binding using SDK identity and frozen Git facts."""
+"""One API-owned local runtime per checkout, independent of Git branches."""
 
 from __future__ import annotations
 
@@ -47,7 +47,7 @@ class LocalWorkspace:
             status="AVAILABLE",
             storage_access_mode='read_write',
             connection=SQLiteConnectionSettings(path=self.tables_path),
-            display_name=f"Local MetaTables ({self.source.repository_branch})",
+            display_name="Local MetaTables",
         )
 
 
@@ -72,7 +72,7 @@ def _file_path(value: str, *, directory: Path) -> Path:
 
 
 def _bind_file(path: Path, *, workspace_uid: UUID, purpose: str) -> None:
-    """An explicit file override cannot silently join another branch's store."""
+    """Bind a runtime file without changing its persistent storage identity."""
     path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
     marker = path.with_name(path.name + ".metatables-scope.json")
     payload = {"workspace_uid": str(workspace_uid), "purpose": purpose}
@@ -107,7 +107,7 @@ def _read_marker(path: Path):
 
 
 def bind_workspace_files(workspace: LocalWorkspace) -> None:
-    """Explicit initialization binds the single runtime file to its Git workspace."""
+    """Explicit initialization binds the single local runtime file."""
     _bind_file(workspace.tables_path, workspace_uid=workspace.uid, purpose="runtime")
 
 
@@ -161,7 +161,6 @@ def select_local_workspace(
         [
             str(source.repository_root.resolve()),
             source.canonical_repository_identity,
-            source.repository_branch,
         ],
         separators=(",", ":"),
     )