mainsequence 4.2.16__tar.gz → 4.2.25__tar.gz

{mainsequence-4.2.16/mainsequence.egg-info → mainsequence-4.2.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence
-Version: 4.2.16
+Version: 4.2.25
 Summary: Main Sequence SDK 
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License: MainSequence GmbH SDK License Agreement
@@ -68,7 +68,7 @@ Requires-Dist: pydantic
 Requires-Dist: pytz
 Requires-Dist: pyyaml
 Requires-Dist: requests
-Requires-Dist: sqlalchemy
+Requires-Dist: sqlalchemy<3,>=2
 Requires-Dist: structlog
 Requires-Dist: tqdm
 Requires-Dist: typer

{mainsequence-4.2.16 → mainsequence-4.2.25}/agent_scaffold/skills/data_publishing/data_nodes/SKILL.md

@@ -148,7 +148,7 @@ class Base(DeclarativeBase):
 
 class PricesTable(PlatformTimeIndexMetaData, Base):
     __metatable_namespace__ = "<domain_namespace>"
-    __metatable_identifier__ = "<table_identifier>"
+    __metatable_identifier__ = "<project_name>.<table_identifier>"
     __metatable_extra_hash_components__ = {"storage_name": "<stable_storage_name>"}
     __metatable_description__ = (
         "Daily close prices keyed by asset unique identifier for portfolio and "
@@ -387,9 +387,9 @@ DataNode storage table needs a platform-managed FK, use
 `ForeignKey(Target.__table__.c.uid)`, table fullnames, or explicit target UID
 maps in DataNode examples.
 
-Do not ask users to name these foreign keys. `MetaTableForeignKey(...)` derives
-a stable contract name when `name` is omitted; `name=...` is only for deliberate
-overrides.
+Do not ask users to name these foreign keys. Platform-managed
+`MetaTableForeignKey(...)` contracts store logical relationships only. Alembic,
+SQLAlchemy, and the database own physical FK constraint names.
 
 Registration of the storage class follows the MetaTable migration lifecycle.
 Migration tooling recursively resolves/registers unresolved FK target model
@@ -402,6 +402,9 @@ Do not add DataNode configuration fields just to mutate storage metadata.
 
 Production-quality table identifiers, descriptions, labels, column docs, and
 foreign-key metadata belong to the storage class/MetaTable registration path.
+Prefix explicit table identifiers, explicit physical table names, and Alembic
+version table names with the project or package name rather than using bare
+names that can collide across projects.
 
 Do not put schema or published table metadata on the DataNode configuration.
 

{mainsequence-4.2.16 → mainsequence-4.2.25}/agent_scaffold/skills/data_publishing/meta_tables/SKILL.md

@@ -152,12 +152,17 @@ table. Do not use it for labels, descriptions, runtime options, test isolation,
 backend UIDs, data-source UIDs, or updater scope. Use `hash_namespace` for test
 or experiment isolation.
 
+Prefix explicit table identifiers, explicit physical table names, and Alembic
+version table names with the project or package name. Bare names such as
+`Account`, `Asset`, or `alembic_version` can collide across projects sharing an
+organization or database schema.
+
 Register through the class API:
 
 ```python
 class Account(PlatformManagedMetaTable, Base):
     __metatable_namespace__ = "sdk-examples"
-    __metatable_identifier__ = "Account"
+    __metatable_identifier__ = "sdk_examples.Account"
     __metatable_extra_hash_components__ = {"storage_name": "account"}
     __metatable_description__ = (
         "Customer account master records used to scope balances, holdings, and "
@@ -222,10 +227,9 @@ tooling resolves/registers unresolved target model classes, stores each
 returned `MetaTable` in a local process registry keyed by `storage_hash`, and
 uses the target `MetaTable.uid` in the child FK contract.
 
-Do not require users to provide foreign-key names. `MetaTableForeignKey(...)`
-accepts `name=...` only as an override; when omitted, the SDK derives a stable
-PostgreSQL-safe contract name from the child table and source column after the
-column is attached to the SQLAlchemy table.
+Do not require users to provide foreign-key names. Platform-managed
+`MetaTableForeignKey(...)` contracts store logical relationships only. Alembic,
+SQLAlchemy, and the database own physical FK constraint names.
 
 Use this pattern:
 
@@ -300,6 +304,9 @@ global identity. If it does not, the SDK derives the identifier from
 `[project].name` in `pyproject.toml` plus
 `<model.__module__>.<model.__qualname__>`. Pin an explicit identifier when a
 class is renamed or moved but must keep the same platform identity.
+When declaring an explicit identifier, explicit physical table name, or Alembic
+version table name, prefix it with the project or package name rather than using
+a bare table name.
 
 Do not ask users to construct backend migration payloads, call low-level
 migration request models, or use SDK helper functions directly. The backend

{mainsequence-4.2.16 → mainsequence-4.2.25}/mainsequence/cli/migrations.py

@@ -7,6 +7,7 @@ import re
 import sys
 from collections.abc import Mapping, Sequence
 from contextlib import contextmanager
+from types import SimpleNamespace
 from typing import Any
 
 import click
@@ -19,6 +20,7 @@ from mainsequence.client.metatables import (
 from mainsequence.meta_tables.migrations import (
     AlembicMetaTableMigration,
     alembic_config_for_provider,
+    apply_mainsequence_migration_role,
     load_alembic_metatable_migration_provider,
 )
 
@@ -70,9 +72,7 @@ def _forward_alembic_logging():
     previous_state = [(logger, logger.level, logger.propagate) for logger in loggers]
     handler = _AlembicLogHandler()
     handler.setLevel(logging.DEBUG)
-    handler.setFormatter(
-        logging.Formatter("[alembic] %(levelname)s %(name)s: %(message)s")
-    )
+    handler.setFormatter(logging.Formatter("[alembic] %(levelname)s %(name)s: %(message)s"))
     root_logger = logging.getLogger("alembic")
     root_logger.addHandler(handler)
     for logger in loggers:
@@ -120,6 +120,72 @@ def _load_alembic_command(command_name: str) -> Any:
     return command
 
 
+def _emit_alembic_script_context(
+    config: Any,
+    *,
+    target_revision: str | None = None,
+) -> None:
+    script_location = config.get_main_option("script_location")
+    version_table = config.get_main_option("version_table")
+    version_table_schema = config.get_main_option("version_table_schema")
+    version_table_label = (
+        f"{version_table_schema}.{version_table}"
+        if version_table_schema not in (None, "")
+        else version_table
+    )
+    _emit_status(
+        "Alembic script context "
+        f"script_location={script_location} version_table={version_table_label}"
+    )
+    try:
+        from alembic.script import ScriptDirectory
+    except ImportError:
+        _emit_status("Alembic ScriptDirectory is unavailable.")
+        return
+
+    try:
+        script = ScriptDirectory.from_config(config)
+    except Exception as exc:
+        _emit_status(f"Alembic script directory could not be resolved: {exc}")
+        return
+
+    _emit_status(
+        "Alembic script directory resolved "
+        f"path={getattr(script, 'dir', None)} versions={getattr(script, 'versions', None)}"
+    )
+    try:
+        heads = [str(head) for head in script.get_heads()]
+    except Exception as exc:
+        _emit_status(f"Alembic heads could not be resolved: {exc}")
+        heads = []
+    else:
+        _emit_status(f"Alembic heads={','.join(heads) if heads else '<none>'}")
+
+    if target_revision in (None, "", "base"):
+        return
+    if target_revision in ("head", "heads"):
+        for head in heads:
+            _emit_revision_path(script, head)
+        return
+    _emit_revision_path(script, target_revision)
+
+
+def _emit_revision_path(script: Any, revision: str) -> None:
+    try:
+        resolved = script.get_revision(revision)
+    except Exception as exc:
+        _emit_status(f"Alembic revision {revision} could not be resolved: {exc}")
+        return
+    if resolved is None:
+        _emit_status(f"Alembic revision {revision} was not found.")
+        return
+    _emit_status(
+        "Alembic revision resolved "
+        f"revision={resolved.revision} path={getattr(resolved, 'path', None)} "
+        f"down_revision={resolved.down_revision}"
+    )
+
+
 def _jsonable(value: Any) -> Any:
     if hasattr(value, "model_dump"):
         return value.model_dump(mode="json")
@@ -162,7 +228,11 @@ def _contract_physical_table_name(item: Any) -> Any:
     contract = _item_value(item, "table_contract")
     if contract is None:
         return None
-    physical = contract.get("physical") if isinstance(contract, Mapping) else getattr(contract, "physical", None)
+    physical = (
+        contract.get("physical")
+        if isinstance(contract, Mapping)
+        else getattr(contract, "physical", None)
+    )
     if physical is None:
         return None
     if isinstance(physical, Mapping):
@@ -170,6 +240,32 @@ def _contract_physical_table_name(item: Any) -> Any:
     return getattr(physical, "table_name", None)
 
 
+def _contract_physical_schema(item: Any) -> Any:
+    contract = _item_value(item, "table_contract")
+    if contract is None:
+        return None
+    physical = (
+        contract.get("physical")
+        if isinstance(contract, Mapping)
+        else getattr(contract, "physical", None)
+    )
+    if physical is None:
+        return None
+    if isinstance(physical, Mapping):
+        return (
+            physical.get("schema")
+            or physical.get("schema_")
+            or physical.get("table_schema")
+            or physical.get("schema_name")
+        )
+    return (
+        getattr(physical, "schema", None)
+        or getattr(physical, "schema_", None)
+        or getattr(physical, "table_schema", None)
+        or getattr(physical, "schema_name", None)
+    )
+
+
 def _meta_table_uid(item: Any) -> str | None:
     if item is None:
         return None
@@ -191,9 +287,7 @@ def _include_alembic_registry_in_scope(
     registry_uid = _meta_table_uid(registry_meta_table)
     if registry_uid in (None, ""):
         return
-    prepared.meta_table_uids = list(
-        dict.fromkeys([registry_uid, *list(prepared.meta_table_uids)])
-    )
+    prepared.meta_table_uids = list(dict.fromkeys([registry_uid, *list(prepared.meta_table_uids)]))
 
 
 def _metatable_message(
@@ -210,13 +304,15 @@ def _metatable_message(
         or model_name
     )
     uid = _item_value(item, "meta_table_uid") or _item_value(item, "uid")
-    physical_table_name = (
-        _item_value(item, "physical_table_name")
-        or _contract_physical_table_name(item)
+    physical_table_name = _item_value(item, "physical_table_name") or _contract_physical_table_name(
+        item
     )
     provisioning_status = _item_value(item, "provisioning_status")
     created = _item_value(item, "created")
     matched_by = _item_value(item, "matched_by")
+    finalized = _item_value(item, "finalized")
+    physical_table_exists = _item_value(item, "physical_table_exists")
+    error = _item_value(item, "error")
 
     parts = [
         f"POST {endpoint}",
@@ -234,6 +330,12 @@ def _metatable_message(
         parts.append(f"created={created}")
     if matched_by not in (None, ""):
         parts.append(f"matched_by={matched_by}")
+    if finalized is not None:
+        parts.append(f"finalized={finalized}")
+    if physical_table_exists is not None:
+        parts.append(f"physical_table_exists={physical_table_exists}")
+    if error not in (None, "", {}):
+        parts.append(f"error={json.dumps(error, sort_keys=True, default=str)}")
     return " ".join(parts)
 
 
@@ -252,18 +354,19 @@ def _emit_metatable_reservation_request(
     models: Sequence[type[Any]],
     tables: Sequence[Any],
 ) -> None:
-    identifiers = []
+    table_names = []
     for model, table in zip(models, tables, strict=True):
-        identifiers.append(
+        table_names.append(
             str(
-                _item_value(table, "identifier")
+                _item_value(table, "physical_table_name")
+                or _item_value(table, "identifier")
                 or getattr(model, "__metatable_identifier__", None)
                 or getattr(model, "__name__", repr(model))
             )
         )
     _emit_status(
         f"Sending POST {RESERVE_MANAGED_ENDPOINT} request for {len(tables)} "
-        f"MetaTables identifiers={','.join(identifiers)}"
+        f"MetaTables table_names={','.join(table_names)}"
     )
 
 
@@ -279,10 +382,12 @@ def _emit_metatable_reservation(model: type[Any], item: Any) -> None:
 
 
 def _emit_metatable_finalization(model: type[Any], item: Any) -> None:
+    finalized = _item_value(item, "finalized")
+    action = "finalized" if finalized is not False else "finalize-failed"
     _emit_progress(
         _metatable_message(
             endpoint=FINALIZE_MANAGED_ENDPOINT,
-            action="finalized",
+            action=action,
             model=model,
             item=item,
         ),
@@ -295,6 +400,9 @@ def _prepare_alembic_config(
     timeout: float | None,
     ttl_seconds: int,
     alembic_output: _AlembicOutput,
+    stage_existing_schema_management: bool = True,
+    require_existing_contract_match: bool = True,
+    prepare_provider_metatables: bool = True,
 ) -> tuple[Any, Any]:
     _emit_status("Ensuring Alembic registry MetaTable...")
     registry_meta_table = migration.ensure_alembic_registry(
@@ -302,13 +410,23 @@ def _prepare_alembic_config(
         on_metatable_registered=_emit_metatable_registration,
     )
 
-    _emit_status("Preparing platform-managed MetaTable reservations...")
-    prepared = migration.prepare_for_alembic(
-        timeout=timeout,
-        on_metatable_reservation_request=_emit_metatable_reservation_request,
-        on_metatable_reservation_status=_emit_status,
-        on_metatable_reserved=_emit_metatable_reservation,
-    )
+    if prepare_provider_metatables:
+        _emit_status("Preparing platform-managed MetaTable reservations...")
+        prepared = migration.prepare_for_alembic(
+            timeout=timeout,
+            stage_existing_schema_management=stage_existing_schema_management,
+            require_existing_contract_match=require_existing_contract_match,
+            on_metatable_reservation_request=_emit_metatable_reservation_request,
+            on_metatable_reservation_status=_emit_status,
+            on_metatable_reserved=_emit_metatable_reservation,
+        )
+    else:
+        _emit_status("Skipping provider MetaTable reservations for read-only Alembic command.")
+        prepared = SimpleNamespace(
+            data_source_uid=migration._resolve_provider_data_source_uid(),
+            meta_table_uids=[],
+            owner_role_name=None,
+        )
     _include_alembic_registry_in_scope(migration, prepared, registry_meta_table)
     _emit_status(
         "Prepared migration scope "
@@ -343,6 +461,103 @@ def _prepare_alembic_config(
     return prepared, config
 
 
+def _prepared_physical_table_refs(prepared: Any) -> list[tuple[str | None, str]]:
+    refs: list[tuple[str | None, str]] = []
+    for item in getattr(prepared, "reserved_tables", []) or []:
+        table_name = _item_value(item, "physical_table_name") or _contract_physical_table_name(item)
+        if table_name in (None, ""):
+            continue
+        schema = _contract_physical_schema(item)
+        refs.append((str(schema) if schema not in (None, "") else "public", str(table_name)))
+    return list(dict.fromkeys(refs))
+
+
+def _alembic_script_heads(config: Any) -> list[str]:
+    try:
+        from alembic.script import ScriptDirectory
+    except ImportError as exc:
+        raise RuntimeError("Alembic is required for migration commands.") from exc
+    with _forward_alembic_logging():
+        return [str(head) for head in ScriptDirectory.from_config(config).get_heads()]
+
+
+def _assert_autogenerate_baseline_visible(prepared: Any, config: Any) -> None:
+    script_heads = _alembic_script_heads(config)
+    if not script_heads:
+        _emit_status("No existing Alembic heads; allowing initial autogenerate baseline.")
+        return
+
+    table_refs = _prepared_physical_table_refs(prepared)
+    if not table_refs:
+        _emit_status(
+            "No provider physical table names were available for autogenerate "
+            "baseline visibility check."
+        )
+        return
+
+    try:
+        from sqlalchemy import create_engine, inspect
+        from sqlalchemy.pool import NullPool
+    except ImportError as exc:
+        raise RuntimeError("SQLAlchemy is required for Alembic autogenerate preflight.") from exc
+
+    sqlalchemy_url = getattr(config, "attributes", {}).get("mainsequence_migration_sqlalchemy_url")
+    if sqlalchemy_url in (None, ""):
+        raise RuntimeError(
+            "Alembic autogenerate preflight cannot inspect the scoped database because "
+            "the migration SQLAlchemy URL was not attached to the Alembic config."
+        )
+
+    _emit_status(
+        "Checking Alembic autogenerate baseline visibility "
+        f"script_heads={','.join(script_heads)} table_count={len(table_refs)}..."
+    )
+    visible: list[tuple[str | None, str]] = []
+    missing: list[tuple[str | None, str]] = []
+    engine = create_engine(str(sqlalchemy_url), poolclass=NullPool)
+    try:
+        with engine.connect() as connection:
+            apply_mainsequence_migration_role(connection, config)
+            inspector = inspect(connection)
+            for schema, table_name in table_refs:
+                if inspector.has_table(table_name, schema=schema):
+                    visible.append((schema, table_name))
+                else:
+                    missing.append((schema, table_name))
+    finally:
+        engine.dispose()
+
+    if visible:
+        _emit_status(
+            "Alembic autogenerate baseline is visible "
+            f"existing_tables={len(visible)} missing_tables={len(missing)}."
+        )
+        if missing:
+            sample = ",".join(_format_table_ref(ref) for ref in missing[:5])
+            _emit_status(
+                "Some provider physical tables were not visible during preflight; "
+                f"Alembic may treat them as new if they are not intentional additions. "
+                f"sample={sample}"
+            )
+        return
+
+    sample = ",".join(_format_table_ref(ref) for ref in table_refs[:5])
+    raise RuntimeError(
+        "Refusing to autogenerate a migration because this provider already has "
+        f"Alembic head(s) {','.join(script_heads)}, but the scoped migration "
+        "connection cannot see any provider physical tables. Alembic would emit a "
+        "duplicate initial create-all migration instead of a schema diff. Apply the "
+        "existing baseline with `mainsequence migrations upgrade ... head`, or fix "
+        "the scoped migration connection/table visibility before running revision "
+        f"again. checked_sample={sample}"
+    )
+
+
+def _format_table_ref(ref: tuple[str | None, str]) -> str:
+    schema, table_name = ref
+    return f"{schema}.{table_name}" if schema not in (None, "") else table_name
+
+
 def _next_sequential_revision_id(
     migration: AlembicMetaTableMigration,
     *,
@@ -411,7 +626,11 @@ def current(
         timeout=timeout,
         ttl_seconds=ttl_seconds,
         alembic_output=alembic_output,
+        stage_existing_schema_management=False,
+        require_existing_contract_match=False,
+        prepare_provider_metatables=False,
     )
+    _emit_alembic_script_context(config)
     _emit_status("Starting Alembic current now...")
     with _forward_alembic_logging():
         command.current(config, verbose=verbose)
@@ -462,7 +681,11 @@ def revision(
         timeout=timeout,
         ttl_seconds=ttl_seconds,
         alembic_output=alembic_output,
+        stage_existing_schema_management=False,
     )
+    _emit_alembic_script_context(config, target_revision=head)
+    if autogenerate:
+        _assert_autogenerate_baseline_visible(prepared, config)
     _emit_status(f"Starting Alembic revision now rev_id={resolved_rev_id}...")
     with _forward_alembic_logging():
         script = command.revision(
@@ -508,6 +731,7 @@ def upgrade(
         ttl_seconds=ttl_seconds,
         alembic_output=alembic_output,
     )
+    _emit_alembic_script_context(config, target_revision=target_revision)
     _emit_status(f"Starting Alembic upgrade now target={target_revision}...")
     with _forward_alembic_logging():
         command.upgrade(config, target_revision)
@@ -559,6 +783,7 @@ def downgrade(
         ttl_seconds=ttl_seconds,
         alembic_output=alembic_output,
     )
+    _emit_alembic_script_context(config, target_revision=target_revision)
     _emit_status(f"Starting Alembic downgrade now target={target_revision}...")
     with _forward_alembic_logging():
         command.downgrade(config, target_revision)

{mainsequence-4.2.16 → mainsequence-4.2.25}/mainsequence/client/metatables/core.py

@@ -610,8 +610,9 @@ class ManagedMetaTableReservationTable(MetaTableRequestFields):
     table_contract: MetaTableContract | dict[str, Any] = Field(
         ...,
         description=(
-            "Relational table contract used to reserve physical, index, and "
-            "foreign-key names before Alembic renders SQL."
+            "Relational table contract used to reserve the physical table name "
+            "before Alembic renders SQL. Index and foreign-key names remain "
+            "client-authored or database-authored DDL metadata."
         ),
     )
     time_index_name: str | None = Field(
@@ -667,8 +668,9 @@ class ManagedMetaTableReservationItem(BasePydanticModel):
     table_contract: dict[str, Any] = Field(
         ...,
         description=(
-            "Backend-normalized contract containing resolved physical table, "
-            "index, and foreign-key names."
+            "Backend-normalized contract containing the resolved physical table "
+            "name. Index and foreign-key names are preserved only when authored "
+            "or observed; TS Manager does not generate them."
         ),
     )
     schema_management: dict[str, Any] | None = Field(
@@ -1415,6 +1417,28 @@ class MetaTable(BasePydanticModel, LabelableObjectMixin, ShareableObjectMixin, B
             raise_for_response(response, payload=request_payload)
         return response.json()
 
+    @classmethod
+    def filter_by_body(
+        cls,
+        *,
+        timeout: int | float | tuple[float, float] | None = None,
+        **filters: Any,
+    ) -> list[MetaTable]:
+        response_json = cls._post_action(
+            "filter",
+            filters,
+            timeout=timeout,
+            expected_statuses=(200,),
+        )
+        if isinstance(response_json, dict) and isinstance(
+            response_json.get("results"),
+            list,
+        ):
+            return [cls(**item) for item in response_json["results"]]
+        if isinstance(response_json, list):
+            return [cls(**item) for item in response_json]
+        raise TypeError("MetaTable.filter_by_body expected a list or paginated response.")
+
     def _post_detail_action(
         self,
         action_name: str,