mainsequence 4.2.25__tar.gz → 4.2.37__tar.gz

{mainsequence-4.2.25/mainsequence.egg-info → mainsequence-4.2.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence
-Version: 4.2.25
+Version: 4.2.37
 Summary: Main Sequence SDK 
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License: MainSequence GmbH SDK License Agreement

{mainsequence-4.2.25 → mainsequence-4.2.37}/agent_scaffold/skills/data_publishing/data_nodes/SKILL.md

@@ -139,7 +139,10 @@ import datetime
 from sqlalchemy import DateTime, Float, MetaData
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
-from mainsequence.meta_tables import PlatformTimeIndexMetaData
+from mainsequence.meta_tables import PlatformTimeIndexMetaData, schema_table_name
+
+PROJECT_NAME = "<project_name>"
+PRICES_TABLE_NAME = schema_table_name(PROJECT_NAME, "prices")
 
 
 class Base(DeclarativeBase):
@@ -147,6 +150,7 @@ class Base(DeclarativeBase):
 
 
 class PricesTable(PlatformTimeIndexMetaData, Base):
+    __tablename__ = PRICES_TABLE_NAME
     __metatable_namespace__ = "<domain_namespace>"
     __metatable_identifier__ = "<project_name>.<table_identifier>"
     __metatable_extra_hash_components__ = {"storage_name": "<stable_storage_name>"}
@@ -378,23 +382,24 @@ changing it changes the dependency graph and update identity.
 
 Do not construct dependency graphs dynamically inside `update()`.
 
-### 8. Foreign Keys Belong To The Storage Contract
+### 8. Foreign Keys Belong To SQLAlchemy And Alembic
 
 For new code, model foreign keys on the `PlatformTimeIndexMetaData` storage
-class, or route the storage-contract work to the MetaTable skill. When a
-DataNode storage table needs a platform-managed FK, use
-`MetaTableForeignKey(TargetModel, column=...)` on the storage class. Do not use
-`ForeignKey(Target.__table__.c.uid)`, table fullnames, or explicit target UID
-maps in DataNode examples.
+class, or route the storage work to the MetaTable skill. When a DataNode storage
+table needs a platform-managed FK, use normal SQLAlchemy `ForeignKey(...)` /
+`ForeignKeyConstraint(...)` metadata on the storage class. Prefer
+project-prefixed SQLAlchemy table names for explicit FK string targets so
+projects sharing one schema do not collide. Generate those names with
+`schema_table_name(project_or_app, concept)` from `mainsequence.meta_tables`.
 
-Do not ask users to name these foreign keys. Platform-managed
-`MetaTableForeignKey(...)` contracts store logical relationships only. Alembic,
-SQLAlchemy, and the database own physical FK constraint names.
+Do not ask users to put FK target `MetaTable.uid` values into DataNode config or
+MetaTable registration contracts. Alembic, SQLAlchemy, and the database own FK
+DDL and physical FK constraint names.
 
 Registration of the storage class follows the MetaTable migration lifecycle.
-Migration tooling recursively resolves/registers unresolved FK target model
-classes, uses the local process registry keyed by `storage_hash`, and writes the
-target `MetaTable.uid` into the FK contract.
+The migration provider reserves the MetaTable rows, Alembic renders/applies FK
+DDL from SQLAlchemy metadata, and catalog finalization refreshes the MetaTable
+binding after upgrade.
 
 Do not add DataNode configuration fields just to mutate storage metadata.
 
@@ -404,7 +409,8 @@ Production-quality table identifiers, descriptions, labels, column docs, and
 foreign-key metadata belong to the storage class/MetaTable registration path.
 Prefix explicit table identifiers, explicit physical table names, and Alembic
 version table names with the project or package name rather than using bare
-names that can collide across projects.
+names that can collide across projects. Use `schema_table_name(...)` for
+authored SQLAlchemy table names, including DataNode storage tables.
 
 Do not put schema or published table metadata on the DataNode configuration.
 

{mainsequence-4.2.25 → mainsequence-4.2.37}/agent_scaffold/skills/data_publishing/meta_tables/SKILL.md

@@ -17,7 +17,7 @@ This skill is for schema-driven application tables registered through TS Manager
 - choose `platform_managed` or `external_registered` management mode
 - register platform-managed tables through the model class API
 - build registration requests from resolved SQLAlchemy metadata when inspection is useful
-- define indexes and foreign keys in the table contract
+- define indexes and foreign keys in SQLAlchemy metadata for Alembic-owned DDL
 - design governed compiled SQL read and write operations
 - design provider-based Alembic contract evolution for MetaTables
 - run the documented `mainsequence migrations ...` lifecycle for Alembic-backed MetaTable changes
@@ -122,11 +122,15 @@ Keep the application table model as the authoring source for the neutral table c
 
 Do not hand-build contract fragments when the SQLAlchemy helper can derive them.
 
-### 2. Use storage-hash physical names for backend-managed tables
+### 2. Use explicit project-prefixed table names
 
 For `platform_managed`, inherit from `PlatformManagedMetaTable`.
 
-The mixin derives the SQLAlchemy physical table name from storage-relevant configuration and table shape. Do not hand-write `__tablename__` for normal backend-managed tables.
+Declare an explicit project-prefixed SQLAlchemy `__tablename__`. Use
+`schema_table_name(project_or_app, concept)` from `mainsequence.meta_tables` to
+generate that name. The mixin derives only the logical `storage_hash` from
+storage-relevant configuration and table shape; it must not use that hash as the
+SQLAlchemy table name.
 
 When a platform-managed table must support in-place contract migrations from its
 first version, use Alembic. Keep the SDK model as a normal
@@ -155,12 +159,21 @@ or experiment isolation.
 Prefix explicit table identifiers, explicit physical table names, and Alembic
 version table names with the project or package name. Bare names such as
 `Account`, `Asset`, or `alembic_version` can collide across projects sharing an
-organization or database schema.
+organization or database schema. Prefer `schema_table_name(...)` over
+hand-built f-strings so project/app prefixes, bounded length, and separators are
+consistent.
 
 Register through the class API:
 
 ```python
+from mainsequence.meta_tables import PlatformManagedMetaTable, schema_table_name
+
+PROJECT_NAME = "sdk_examples"
+ACCOUNT_TABLE_NAME = schema_table_name(PROJECT_NAME, "account")
+
+
 class Account(PlatformManagedMetaTable, Base):
+    __tablename__ = ACCOUNT_TABLE_NAME
     __metatable_namespace__ = "sdk-examples"
     __metatable_identifier__ = "sdk_examples.Account"
     __metatable_extra_hash_components__ = {"storage_name": "account"}
@@ -215,28 +228,25 @@ Do not add generic labels such as `"meta-table"` or `"platform-managed"` to exam
 Do not add a `MAINSEQUENCE_META_TABLE_REGISTER` toggle in platform-managed
 examples. Platform-managed examples should be migration-first.
 
-### 3. Register parent tables before child tables
-
-Foreign-key contracts reference the target `MetaTable` UID.
+### 3. Keep foreign keys in SQLAlchemy/Alembic metadata
 
-For `PlatformManagedMetaTable`, define foreign keys with
-`MetaTableForeignKey(TargetModel, column=...)`. Do not write raw SQLAlchemy
-table fullnames, `Parent.__table__.c.<column>` targets, or explicit target UID
-maps in the platform-managed path. Migration is the lifecycle path. Migration
-tooling resolves/registers unresolved target model classes, stores each
-returned `MetaTable` in a local process registry keyed by `storage_hash`, and
-uses the target `MetaTable.uid` in the child FK contract.
+For `PlatformManagedMetaTable`, define foreign keys with normal SQLAlchemy
+`ForeignKey(...)` / `ForeignKeyConstraint(...)` metadata. Do not write explicit
+target `MetaTable.uid` maps in the platform-managed path. Migration is the
+lifecycle path: the SDK reserves provider MetaTable rows, Alembic renders and
+applies FK/index DDL from SQLAlchemy metadata, and finalization refreshes the
+catalog after upgrade.
 
-Do not require users to provide foreign-key names. Platform-managed
-`MetaTableForeignKey(...)` contracts store logical relationships only. Alembic,
-SQLAlchemy, and the database own physical FK constraint names.
+Prefer project-prefixed SQLAlchemy table names for explicit FK string targets.
+Alembic, SQLAlchemy, and the database own physical FK/index names unless the
+project explicitly names them in SQLAlchemy.
 
 Use this pattern:
 
 ```python
 account_uid: Mapped[uuid.UUID] = mapped_column(
     Uuid,
-    MetaTableForeignKey(Account, column="uid", ondelete="RESTRICT"),
+    ForeignKey("public.sdk_examples__account.uid", ondelete="RESTRICT"),
     nullable=False,
 )
 ```
@@ -253,21 +263,19 @@ migration = AlembicMetaTableMigration(
 )
 ```
 
-Migration tooling registers `Account` first if `Asset` depends on it and it has
-not already been bound in the current process. The local registry prevents
-duplicate backend registration attempts for the same `storage_hash` and raises
-a clear error for recursive registration cycles.
+Migration tooling reserves provider MetaTable rows in the provider-declared
+model order. Include related parent and child models in the same selected
+provider when Alembic must create or evolve their FK DDL.
 
-For `external_registered`, there is no platform-managed parent lookup. Register
-the parent first, then build the child registration request with
-`target_meta_tables={Account: account_meta_table}`:
+For `external_registered`, register each external table contract directly. Do
+not encode FK target MetaTable UIDs in the child registration request; FKs are
+database DDL/introspection metadata, not SDK registration contract fields:
 
 ```python
 account_meta_table = MetaTable.register(account_request)
 asset_request = external_registered_registration_request_from_sqlalchemy_model(
     Asset,
     data_source_uid=data_source_uid,
-    target_meta_tables={Account: account_meta_table},
 )
 asset_meta_table = MetaTable.register(asset_request)
 ```
@@ -292,21 +300,19 @@ For contract evolution, define or update one selected
 - set `package`, `migration_namespace`, `script_location`, and `target_metadata`
 - set `alembic_registry` to an `AlembicVersionMetaTable` subclass
 - list the post-apply catalog scope in `metatable_models`
-- generate, render, dry-run, apply, and refresh catalog bindings
+- generate revisions, apply migrations, and refresh catalog bindings
   through `mainsequence migrations ...` commands
 
 `alembic_version_meta_table_uid` is the UID of the catalog binding for Alembic's
 version table. It is not the UID of the table being migrated.
 
-Application MetaTable catalog sync resolves existing rows by exact
-`identifier`. If a model declares `__metatable_identifier__`, that value is the
-global identity. If it does not, the SDK derives the identifier from
-`[project].name` in `pyproject.toml` plus
-`<model.__module__>.<model.__qualname__>`. Pin an explicit identifier when a
-class is renamed or moved but must keep the same platform identity.
+Application MetaTable catalog sync resolves existing rows by the authored
+SQLAlchemy table name used by the provider model. Keep that table name stable
+when a class is renamed or moved but must keep the same platform identity.
 When declaring an explicit identifier, explicit physical table name, or Alembic
 version table name, prefix it with the project or package name rather than using
-a bare table name.
+a bare table name. Use `schema_table_name(project_or_app, concept)` for the
+physical table and Alembic version table names.
 
 Do not ask users to construct backend migration payloads, call low-level
 migration request models, or use SDK helper functions directly. The backend
@@ -359,9 +365,8 @@ When reviewing an existing MetaTable workflow, look for:
 - backend-managed examples that use namespace environment variables instead of a plain `sdk-examples` namespace
 - duplicate schema sources outside SQLAlchemy table metadata
 - external tables registered with unstable physical names
-- platform-managed examples that manually sequence parent registration instead
-  of relying on `MetaTableForeignKey(...)` recursive registration
-- external child registrations that do not map foreign-key targets to registered parent `MetaTable.uid` values
+- platform-managed examples that try to sequence parent registration through FK metadata
+- external child registrations that try to encode FK target MetaTable UIDs in registration contracts
 - contract changes attempted through normal registration instead of an Alembic migration
 - migration work that asks users to define backend payloads, artifact rows, or SDK request objects
 - compiled SQL operations without complete table scope
@@ -376,9 +381,9 @@ Do not claim success until you have checked:
 - the table has an intention-rich `__metatable_description__`
 - every mapped column has an intention-rich `info.description`
 - indexes are intentional
-- foreign keys resolve to the correct dependency targets
+- foreign keys are present in SQLAlchemy metadata for Alembic when required
 - management mode is correct
-- backend-managed physical names match the storage hash
+- authored physical names are explicit, project-prefixed SQLAlchemy table names
 - registration returns a `MetaTable.uid`
 - compiled SQL operations declare table scope
 - migrations use Alembic-rendered SQL
@@ -391,9 +396,8 @@ Do not claim success until you have checked:
 For related tables, also check:
 
 - aliases are readable
-- platform-managed child registration recursively resolves parent
-  `MetaTableForeignKey(...)` targets
-- external child registration requests map FK targets to the registered parent UIDs
+- platform-managed child tables and parent tables are included in the selected migration provider
+- FK target strings use stable project-prefixed SQLAlchemy table names where explicit strings are authored
 - query results still match the expected response contract
 
 ## This Skill Must Stop And Escalate When

{mainsequence-4.2.25 → mainsequence-4.2.37}/mainsequence/cli/cli.py

@@ -7038,9 +7038,6 @@ def _meta_table_detail_impl(meta_table_uid: str, timeout: int | None) -> None:
             ("Contract Version", str(meta_table.get("contract_version") or "-")),
             ("Table Contract", _format_json_value(meta_table.get("table_contract"))),
             ("Columns", _format_json_value(meta_table.get("columns"))),
-            ("Indexes", _format_json_value(meta_table.get("indexes_meta"))),
-            ("Foreign Keys", _format_json_value(meta_table.get("foreign_keys"))),
-            ("Incoming FKs", _format_json_value(meta_table.get("incoming_fks"))),
             ("Introspection", _format_json_value(meta_table.get("introspection_snapshot"))),
         ],
     )

{mainsequence-4.2.25 → mainsequence-4.2.37}/mainsequence/cli/migrations.py

@@ -16,6 +16,7 @@ import typer
 from mainsequence.client.metatables import (
     DynamicTableDataSource,
     DynamicTableDataSourceMigrationConnectionRequest,
+    TimeIndexMetaData,
 )
 from mainsequence.meta_tables.migrations import (
     AlembicMetaTableMigration,
@@ -26,7 +27,8 @@ from mainsequence.meta_tables.migrations import (
 
 migrations = typer.Typer(help="Alembic-owned MetaTable migration commands")
 REGISTER_ENDPOINT = "/orm/api/ts_manager/meta_table/register/"
-RESERVE_MANAGED_ENDPOINT = "/orm/api/ts_manager/meta_table/reserve-managed/"
+METATABLE_COLLECTION_ENDPOINT = "/orm/api/ts_manager/meta_table/"
+DYNAMIC_TABLE_COLLECTION_ENDPOINT = "/orm/api/ts_manager/dynamic_table/"
 FINALIZE_MANAGED_ENDPOINT = "/orm/api/ts_manager/meta_table/finalize-managed/"
 ALEMBIC_PROVIDER_RESET_ENDPOINT = "/orm/api/ts_manager/meta_table/alembic-provider-reset/"
 
@@ -339,6 +341,20 @@ def _metatable_message(
     return " ".join(parts)
 
 
+def _collection_create_endpoint_for_items(items: Sequence[Any]) -> str:
+    if items and all(_item_value(item, "time_index_name") not in (None, "") for item in items):
+        return DYNAMIC_TABLE_COLLECTION_ENDPOINT
+    return METATABLE_COLLECTION_ENDPOINT
+
+
+def _collection_create_endpoint_for_item(item: Any) -> str:
+    if isinstance(item, TimeIndexMetaData):
+        return DYNAMIC_TABLE_COLLECTION_ENDPOINT
+    if _item_value(item, "time_index_name") not in (None, ""):
+        return DYNAMIC_TABLE_COLLECTION_ENDPOINT
+    return METATABLE_COLLECTION_ENDPOINT
+
+
 def _emit_metatable_registration(model: type[Any], item: Any) -> None:
     _emit_progress(
         _metatable_message(
@@ -354,6 +370,7 @@ def _emit_metatable_reservation_request(
     models: Sequence[type[Any]],
     tables: Sequence[Any],
 ) -> None:
+    endpoint = _collection_create_endpoint_for_items(tables)
     table_names = []
     for model, table in zip(models, tables, strict=True):
         table_names.append(
@@ -365,7 +382,7 @@ def _emit_metatable_reservation_request(
             )
         )
     _emit_status(
-        f"Sending POST {RESERVE_MANAGED_ENDPOINT} request for {len(tables)} "
+        f"Sending POST {endpoint} collection-create request for {len(tables)} "
         f"MetaTables table_names={','.join(table_names)}"
     )
 
@@ -373,7 +390,7 @@ def _emit_metatable_reservation_request(
 def _emit_metatable_reservation(model: type[Any], item: Any) -> None:
     _emit_progress(
         _metatable_message(
-            endpoint=RESERVE_MANAGED_ENDPOINT,
+            endpoint=_collection_create_endpoint_for_item(item),
             action="reserved",
             model=model,
             item=item,
@@ -400,8 +417,6 @@ def _prepare_alembic_config(
     timeout: float | None,
     ttl_seconds: int,
     alembic_output: _AlembicOutput,
-    stage_existing_schema_management: bool = True,
-    require_existing_contract_match: bool = True,
     prepare_provider_metatables: bool = True,
 ) -> tuple[Any, Any]:
     _emit_status("Ensuring Alembic registry MetaTable...")
@@ -414,8 +429,6 @@ def _prepare_alembic_config(
         _emit_status("Preparing platform-managed MetaTable reservations...")
         prepared = migration.prepare_for_alembic(
             timeout=timeout,
-            stage_existing_schema_management=stage_existing_schema_management,
-            require_existing_contract_match=require_existing_contract_match,
             on_metatable_reservation_request=_emit_metatable_reservation_request,
             on_metatable_reservation_status=_emit_status,
             on_metatable_reserved=_emit_metatable_reservation,
@@ -435,20 +448,16 @@ def _prepare_alembic_config(
     )
     _emit_status(f"Loading DynamicTableDataSource uid={prepared.data_source_uid}...")
     data_source = DynamicTableDataSource.get_by_uid(prepared.data_source_uid)
-    _emit_status(
-        "Requesting scoped migration connection "
-        f"meta_table_count={len(prepared.meta_table_uids)} ttl_seconds={ttl_seconds}..."
-    )
+    _emit_status(f"Requesting migration connection ttl_seconds={ttl_seconds}...")
     connection = data_source.issue_migration_connection(
         DynamicTableDataSourceMigrationConnectionRequest(
             package=migration.package,
             migration_namespace=migration.migration_namespace,
-            meta_table_uids=prepared.meta_table_uids,
             ttl_seconds=ttl_seconds,
         ),
         timeout=timeout,
     )
-    _emit_status("Scoped migration connection acquired.")
+    _emit_status("Migration connection acquired.")
     _emit_status("Building Alembic config...")
     config = alembic_config_for_provider(
         migration,
@@ -461,6 +470,44 @@ def _prepare_alembic_config(
     return prepared, config
 
 
+def _build_revision_alembic_config(
+    migration: AlembicMetaTableMigration,
+    *,
+    sqlalchemy_url: str | None,
+    requires_database: bool,
+    timeout: float | None,
+    ttl_seconds: int,
+    alembic_output: _AlembicOutput,
+) -> Any:
+    _emit_status("Building local Alembic config for revision...")
+    owner_role_name = None
+    if requires_database and sqlalchemy_url in (None, ""):
+        data_source_uid = migration._resolve_provider_data_source_uid()
+        _emit_status(f"Loading DynamicTableDataSource uid={data_source_uid} for revision...")
+        data_source = DynamicTableDataSource.get_by_uid(data_source_uid)
+        _emit_status(f"Requesting revision migration connection ttl_seconds={ttl_seconds}...")
+        connection = data_source.issue_migration_connection(
+            DynamicTableDataSourceMigrationConnectionRequest(
+                package=migration.package,
+                migration_namespace=migration.migration_namespace,
+                ttl_seconds=ttl_seconds,
+            ),
+            timeout=timeout,
+        )
+        _emit_status("Revision migration connection acquired.")
+        sqlalchemy_url = connection.uri
+        owner_role_name = connection.owner_role_name
+    config = alembic_config_for_provider(
+        migration,
+        sqlalchemy_url=sqlalchemy_url or "postgresql://",
+        owner_role_name=owner_role_name,
+        stdout=alembic_output,
+        output_buffer=alembic_output,
+    )
+    _emit_status("Local Alembic config built.")
+    return config
+
+
 def _prepared_physical_table_refs(prepared: Any) -> list[tuple[str | None, str]]:
     refs: list[tuple[str | None, str]] = []
     for item in getattr(prepared, "reserved_tables", []) or []:
@@ -544,11 +591,11 @@ def _assert_autogenerate_baseline_visible(prepared: Any, config: Any) -> None:
     sample = ",".join(_format_table_ref(ref) for ref in table_refs[:5])
     raise RuntimeError(
         "Refusing to autogenerate a migration because this provider already has "
-        f"Alembic head(s) {','.join(script_heads)}, but the scoped migration "
+        f"Alembic head(s) {','.join(script_heads)}, but the migration "
         "connection cannot see any provider physical tables. Alembic would emit a "
         "duplicate initial create-all migration instead of a schema diff. Apply the "
         "existing baseline with `mainsequence migrations upgrade ... head`, or fix "
-        "the scoped migration connection/table visibility before running revision "
+        "the migration connection/table visibility before running revision "
         f"again. checked_sample={sample}"
     )
 
@@ -626,8 +673,6 @@ def current(
         timeout=timeout,
         ttl_seconds=ttl_seconds,
         alembic_output=alembic_output,
-        stage_existing_schema_management=False,
-        require_existing_contract_match=False,
         prepare_provider_metatables=False,
     )
     _emit_alembic_script_context(config)
@@ -653,7 +698,7 @@ def revision(
     autogenerate: bool = typer.Option(
         True,
         "--autogenerate/--no-autogenerate",
-        help="Use Alembic autogenerate against the reserved MetaTable metadata.",
+        help="Use Alembic autogenerate against the local database URL and provider metadata.",
     ),
     provider: str | None = typer.Option(
         None,
@@ -662,8 +707,22 @@ def revision(
     ),
     rev_id: str | None = typer.Option(None, "--rev-id", help="Explicit Alembic revision id."),
     head: str = typer.Option("head", "--head", help="Alembic head to base the revision on."),
-    timeout: float | None = typer.Option(None, "--timeout"),
-    ttl_seconds: int = typer.Option(900, "--ttl-seconds", min=1),
+    timeout: float | None = typer.Option(
+        None,
+        "--timeout",
+        help="Request timeout for the revision migration connection when --sqlalchemy-url is omitted.",
+    ),
+    ttl_seconds: int = typer.Option(
+        900,
+        "--ttl-seconds",
+        min=1,
+        help="TTL for the revision migration connection when --sqlalchemy-url is omitted.",
+    ),
+    sqlalchemy_url: str | None = typer.Option(
+        None,
+        "--sqlalchemy-url",
+        help="Local database URL for Alembic autogenerate. Required when --autogenerate is used.",
+    ),
     json_output: bool = typer.Option(False, "--json", help="Emit JSON."),
 ) -> None:
     """Create a normal Alembic revision for the selected provider."""
@@ -676,16 +735,15 @@ def revision(
         migration,
         alembic_output=alembic_output,
     )
-    prepared, config = _prepare_alembic_config(
+    config = _build_revision_alembic_config(
         migration,
+        sqlalchemy_url=sqlalchemy_url,
+        requires_database=autogenerate,
         timeout=timeout,
         ttl_seconds=ttl_seconds,
         alembic_output=alembic_output,
-        stage_existing_schema_management=False,
     )
     _emit_alembic_script_context(config, target_revision=head)
-    if autogenerate:
-        _assert_autogenerate_baseline_visible(prepared, config)
     _emit_status(f"Starting Alembic revision now rev_id={resolved_rev_id}...")
     with _forward_alembic_logging():
         script = command.revision(
@@ -702,7 +760,6 @@ def revision(
             "path": getattr(script, "path", None),
             "package": migration.package,
             "migration_namespace": migration.migration_namespace,
-            "meta_table_uids": prepared.meta_table_uids,
         },
         json_output=json_output,
     )