mainsequence 4.2.1__tar.gz → 4.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. {mainsequence-4.2.1/mainsequence.egg-info → mainsequence-4.2.4}/PKG-INFO +1 -1
  2. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/data_publishing/data_nodes/SKILL.md +1 -1
  3. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/data_publishing/meta_tables/SKILL.md +26 -7
  4. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/cli.py +1 -2
  5. mainsequence-4.2.4/mainsequence/cli/migrations.py +282 -0
  6. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/metatables/__init__.py +0 -9
  7. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/metatables/core.py +146 -1
  8. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/__init__.py +7 -10
  9. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/build_operations.py +1 -1
  10. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/persist_managers.py +8 -3
  11. mainsequence-4.2.4/mainsequence/meta_tables/migrations.py +975 -0
  12. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/sqlalchemy_contracts.py +83 -18
  13. {mainsequence-4.2.1 → mainsequence-4.2.4/mainsequence.egg-info}/PKG-INFO +1 -1
  14. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence.egg-info/SOURCES.txt +0 -1
  15. {mainsequence-4.2.1 → mainsequence-4.2.4}/pyproject.toml +1 -1
  16. mainsequence-4.2.4/tests/test_cli_migrations.py +162 -0
  17. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_meta_table_migrations.py +158 -313
  18. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_meta_tables_client_models.py +108 -105
  19. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_meta_tables_sqlalchemy_contracts.py +73 -11
  20. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_run_configuration.py +6 -2
  21. mainsequence-4.2.1/mainsequence/cli/migrations.py +0 -421
  22. mainsequence-4.2.1/mainsequence/client/metatables/migrations.py +0 -149
  23. mainsequence-4.2.1/mainsequence/meta_tables/migrations.py +0 -682
  24. mainsequence-4.2.1/tests/test_cli_migrations.py +0 -644
  25. {mainsequence-4.2.1 → mainsequence-4.2.4}/LICENSE +0 -0
  26. {mainsequence-4.2.1 → mainsequence-4.2.4}/README.md +0 -0
  27. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/AGENTS.md +0 -0
  28. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/a2a_communication/SKILL.md +0 -0
  29. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/application_surfaces/api_surfaces/SKILL.md +0 -0
  30. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/command_center/adapter_from_api/SKILL.md +0 -0
  31. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/command_center/api_mock_prototyping/SKILL.md +0 -0
  32. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/command_center/app_components/SKILL.md +0 -0
  33. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/command_center/connections/SKILL.md +0 -0
  34. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/command_center/workspace_analysis/SKILL.md +0 -0
  35. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/command_center/workspace_builder/SKILL.md +0 -0
  36. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/command_center/workspace_design/SKILL.md +0 -0
  37. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/dashboards/streamlit/SKILL.md +0 -0
  38. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/data_access/exploration/SKILL.md +0 -0
  39. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/maintenance/bug_auditor/SKILL.md +0 -0
  40. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/ms-markets/SKILL.md +0 -0
  41. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/platform_operations/access_control_and_sharing/SKILL.md +0 -0
  42. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/platform_operations/orchestration_and_releases/SKILL.md +0 -0
  43. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/project_builder/SKILL.md +0 -0
  44. {mainsequence-4.2.1 → mainsequence-4.2.4}/agent_scaffold/skills/project_to_agent/SKILL.md +0 -0
  45. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/__init__.py +0 -0
  46. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/__main__.py +0 -0
  47. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/bootstrap.py +0 -0
  48. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/__init__.py +0 -0
  49. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/api.py +0 -0
  50. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/browser_auth.py +0 -0
  51. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/config.py +0 -0
  52. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/docker_utils.py +0 -0
  53. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/doctor.py +0 -0
  54. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/local_ops.py +0 -0
  55. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/model_filters.py +0 -0
  56. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/project_status.py +0 -0
  57. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/pydantic_cli.py +0 -0
  58. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/sdk_utils.py +0 -0
  59. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/ssh_utils.py +0 -0
  60. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/cli/ui.py +0 -0
  61. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/__init__.py +0 -0
  62. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/agent_runtime_models.py +0 -0
  63. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/base.py +0 -0
  64. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/client.py +0 -0
  65. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/command_center/__init__.py +0 -0
  66. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/command_center/app_component.py +0 -0
  67. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/command_center/connections.py +0 -0
  68. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/command_center/data_models.py +0 -0
  69. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/command_center/workspace.py +0 -0
  70. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/command_center/workspace_snapshot.py +0 -0
  71. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/compute_validation.py +0 -0
  72. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/data_sources_interfaces/__init__.py +0 -0
  73. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/data_sources_interfaces/duckdb.py +0 -0
  74. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/data_sources_interfaces/local_paths.py +0 -0
  75. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/data_sources_interfaces/sqlite.py +0 -0
  76. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/dtype_codec.py +0 -0
  77. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/exceptions.py +0 -0
  78. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/fastapi/__init__.py +0 -0
  79. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/fastapi/auth.py +0 -0
  80. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/models_foundry.py +0 -0
  81. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/models_helpers.py +0 -0
  82. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/models_user.py +0 -0
  83. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/client/utils.py +0 -0
  84. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/defaults.py +0 -0
  85. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/instrumentation/__init__.py +0 -0
  86. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/instrumentation/utils.py +0 -0
  87. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/logconf.py +0 -0
  88. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/__main__.py +0 -0
  89. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/compiled_sql/__init__.py +0 -0
  90. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/compiled_sql/v1.py +0 -0
  91. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/__init__.py +0 -0
  92. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/data_nodes.py +0 -0
  93. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/models.py +0 -0
  94. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/namespacing.py +0 -0
  95. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/run_operations.py +0 -0
  96. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/data_nodes/utils.py +0 -0
  97. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/future_registry.py +0 -0
  98. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/hashing.py +0 -0
  99. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/meta_tables/pydantic_metadata.py +0 -0
  100. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence/runtime_flags.py +0 -0
  101. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence.egg-info/dependency_links.txt +0 -0
  102. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence.egg-info/entry_points.txt +0 -0
  103. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence.egg-info/requires.txt +0 -0
  104. {mainsequence-4.2.1 → mainsequence-4.2.4}/mainsequence.egg-info/top_level.txt +0 -0
  105. {mainsequence-4.2.1 → mainsequence-4.2.4}/setup.cfg +0 -0
  106. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_auth_precedence.py +0 -0
  107. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_build_operations_hashing.py +0 -0
  108. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_cli.py +0 -0
  109. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_cli_browser_auth.py +0 -0
  110. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_client.py +0 -0
  111. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_command_center_app_component_models.py +0 -0
  112. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_command_center_data_models.py +0 -0
  113. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_command_center_models.py +0 -0
  114. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_data_access_mixin_dimension_audit.py +0 -0
  115. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_data_node_storage_dimension_queries.py +0 -0
  116. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_data_node_update_flow.py +0 -0
  117. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_dependency_extras.py +0 -0
  118. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_duckdb_interface_dimensions.py +0 -0
  119. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_filter_normalization.py +0 -0
  120. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_logconf.py +0 -0
  121. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_models_user_request_bound_auth.py +0 -0
  122. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_pod_project_resolution.py +0 -0
  123. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_project_batch_jobs_from_file.py +0 -0
  124. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_secret_client_model.py +0 -0
  125. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_source_table_configuration.py +0 -0
  126. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_sqlite_interface_dimensions.py +0 -0
  127. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_update_runner_uid_runtime.py +0 -0
  128. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_update_statistics.py +0 -0
  129. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_update_uid_guards.py +0 -0
  130. {mainsequence-4.2.1 → mainsequence-4.2.4}/tests/test_workspace_snapshot.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mainsequence
3
- Version: 4.2.1
3
+ Version: 4.2.4
4
4
  Summary: Main Sequence SDK
5
5
  Author-email: Main Sequence GmbH <dev@main-sequence.io>
6
6
  License: MainSequence GmbH SDK License Agreement
@@ -184,7 +184,7 @@ class PricesTable(PlatformTimeIndexMetaData, Base):
184
184
 
185
185
  Storage registration is migration-first. Add the storage model to the
186
186
  MetaTable migration provider and run `mainsequence migrations upgrade --provider
187
- ... --to head`. Do not call `PricesTable.register()` directly and do not rely on
187
+ ... head`. Do not call `PricesTable.register()` directly and do not rely on
188
188
  DataNode construction to register storage tables.
189
189
 
190
190
  `PlatformTimeIndexMetaData.register()` remains SDK plumbing for the migration
@@ -95,6 +95,27 @@ For every non-trivial task, decide:
95
95
 
96
96
  ## Build Rules
97
97
 
98
+ ### 0. Creation And Deletion Are SDK-Managed
99
+
100
+ Do not build custom migrations for creating or deleting MetaTables.
101
+
102
+ MetaTable creation and deletion are handled through `mainsequence-sdk` and the
103
+ Main Sequence CLI. Assistants should define the SQLAlchemy MetaTable model,
104
+ select the SDK migration provider when schema evolution is needed, and use the
105
+ documented `mainsequence migrations ...` commands.
106
+
107
+ Do not hand-author create/delete migration files, backend migration payloads,
108
+ manifest rows, registry rows, or low-level SDK migration requests for MetaTable
109
+ creation or deletion.
110
+
111
+ The only migration workflow to recommend is the Main Sequence CLI lifecycle:
112
+
113
+ ```bash
114
+ mainsequence migrations current --provider mainsequence_migrations:migration
115
+ mainsequence migrations revision --provider mainsequence_migrations:migration
116
+ mainsequence migrations upgrade --provider mainsequence_migrations:migration head
117
+ ```
118
+
98
119
  ### 1. SQLAlchemy metadata is the authoring source
99
120
 
100
121
  Keep the application table model as the authoring source for the neutral table contract.
@@ -166,8 +187,8 @@ class Account(PlatformManagedMetaTable, Base):
166
187
  Registration metadata belongs on the class. Do not call `Account.register()`
167
188
  directly for platform-managed models. Add platform-managed models to the
168
189
  selected `AlembicMetaTableMigration.metatable_models` list and let
169
- `mainsequence migrations upgrade --provider ... --to head` resolve/register and
170
- bind them.
190
+ `mainsequence migrations upgrade --provider ... head` reserve, migrate, refresh,
191
+ and bind them.
171
192
 
172
193
  For platform-managed migration registration, the data source is resolved from
173
194
  the active Main Sequence project/session, the same way DataNode does. Do not
@@ -287,13 +308,11 @@ request shape is reference material in the tutorial; the user-facing path is:
287
308
  ```bash
288
309
  mainsequence migrations current --provider mainsequence_migrations:migration
289
310
  mainsequence migrations revision --provider mainsequence_migrations:migration
290
- mainsequence migrations render --provider mainsequence_migrations:migration --to head
291
- mainsequence migrations upgrade --provider mainsequence_migrations:migration --to head --dry-run
292
- mainsequence migrations upgrade --provider mainsequence_migrations:migration --to head
311
+ mainsequence migrations upgrade --provider mainsequence_migrations:migration head
293
312
  ```
294
313
 
295
- `current` and `upgrade` automatically register the provider's
296
- `AlembicVersionMetaTable` binding when backend migration state is needed.
314
+ All migration commands prepare the provider, reserve provider-scoped
315
+ platform-managed MetaTables, bind backend names, and call Alembic directly.
297
316
  `revision` accepts optional `-m/--message`; if omitted, the CLI uses
298
317
  `migration`. `revision --autogenerate` is optional and requires an explicit
299
318
  `--sqlalchemy-url` for the baseline database.
@@ -337,7 +337,6 @@ connection = typer.Typer(help="Connection commands")
337
337
  organization = typer.Typer(help="Organization commands")
338
338
  organization_teams_group = typer.Typer(help="Organization team commands")
339
339
  meta_table_group = typer.Typer(help="MetaTable table-storage commands")
340
- migrations_root_group = migrations_group
341
340
  data_node_storage_group = typer.Typer(help="DataNode update/read-helper commands")
342
341
  project = typer.Typer(help="Project commands (remote + local operations)")
343
342
  project_list_group = typer.Typer(help="List-related project commands")
@@ -377,7 +376,6 @@ app.add_typer(organization, name="organization")
377
376
  app.add_typer(skills, name="skills")
378
377
  app.add_typer(meta_table_group, name="meta-table")
379
378
  app.add_typer(meta_table_group, name="meta_table")
380
- app.add_typer(migrations_root_group, name="migrations")
381
379
  app.add_typer(data_node_storage_group, name="data-node")
382
380
  app.add_typer(data_node_storage_group, name="data_node")
383
381
  app.add_typer(data_node_storage_group, name="data-node-storage", hidden=True)
@@ -391,6 +389,7 @@ project.add_typer(project_jobs_group, name="jobs")
391
389
  project_jobs_group.add_typer(project_job_runs_group, name="runs")
392
390
  app.add_typer(settings, name="settings")
393
391
  app.add_typer(sdk, name="sdk")
392
+ app.add_typer(migrations_group, name="migrations")
394
393
 
395
394
 
396
395
  @app.callback()
@@ -0,0 +1,282 @@
1
+ from __future__ import annotations
2
+
3
+ import dataclasses
4
+ import json
5
+ import re
6
+ from typing import Any
7
+
8
+ import click
9
+ import typer
10
+
11
+ from mainsequence.client.metatables import (
12
+ DynamicTableDataSource,
13
+ DynamicTableDataSourceMigrationConnectionRequest,
14
+ )
15
+ from mainsequence.meta_tables.migrations import (
16
+ AlembicMetaTableMigration,
17
+ alembic_config_for_provider,
18
+ load_alembic_metatable_migration_provider,
19
+ )
20
+
21
+ migrations = typer.Typer(help="Alembic-owned MetaTable migration commands")
22
+
23
+
24
+ def _load_migration(provider: str | None) -> AlembicMetaTableMigration:
25
+ try:
26
+ return load_alembic_metatable_migration_provider(provider)
27
+ except Exception as exc:
28
+ raise typer.BadParameter(str(exc), param_hint="--provider") from exc
29
+
30
+
31
+ def _jsonable(value: Any) -> Any:
32
+ if hasattr(value, "model_dump"):
33
+ return value.model_dump(mode="json")
34
+ if dataclasses.is_dataclass(value):
35
+ return dataclasses.asdict(value)
36
+ if isinstance(value, dict):
37
+ return {str(key): _jsonable(item) for key, item in value.items()}
38
+ if isinstance(value, (list, tuple, set)):
39
+ return [_jsonable(item) for item in value]
40
+ return value
41
+
42
+
43
+ def _emit(payload: Any, *, json_output: bool = False) -> None:
44
+ if json_output or _json_output_enabled():
45
+ typer.echo(json.dumps(_jsonable(payload), indent=2, ensure_ascii=False))
46
+ return
47
+ if isinstance(payload, str):
48
+ typer.echo(payload)
49
+ return
50
+ for key, value in _jsonable(payload).items():
51
+ typer.echo(f"{key}: {value}")
52
+
53
+
54
+ def _json_output_enabled() -> bool:
55
+ ctx = click.get_current_context(silent=True)
56
+ if ctx is None:
57
+ return False
58
+ root = ctx.find_root()
59
+ obj = getattr(root, "obj", None) or {}
60
+ return bool(obj.get("json_output"))
61
+
62
+
63
+ def _prepare_alembic_config(
64
+ migration: AlembicMetaTableMigration,
65
+ *,
66
+ timeout: float | None,
67
+ ttl_seconds: int,
68
+ ) -> tuple[Any, Any]:
69
+ migration.ensure_alembic_registry(timeout=timeout)
70
+ prepared = migration.prepare_for_alembic(timeout=timeout)
71
+ data_source = DynamicTableDataSource.get_by_uid(prepared.data_source_uid)
72
+ connection = data_source.issue_migration_connection(
73
+ DynamicTableDataSourceMigrationConnectionRequest(
74
+ package=migration.package,
75
+ migration_namespace=migration.migration_namespace,
76
+ meta_table_uids=prepared.meta_table_uids,
77
+ ttl_seconds=ttl_seconds,
78
+ ),
79
+ timeout=timeout,
80
+ )
81
+ config = alembic_config_for_provider(
82
+ migration,
83
+ sqlalchemy_url=connection.uri,
84
+ owner_role_name=connection.owner_role_name or prepared.owner_role_name,
85
+ )
86
+ return prepared, config
87
+
88
+
89
+ def _next_sequential_revision_id(migration: AlembicMetaTableMigration) -> str:
90
+ try:
91
+ from alembic.script import ScriptDirectory
92
+ except ImportError as exc:
93
+ raise RuntimeError("Alembic is required for revision generation.") from exc
94
+
95
+ script = ScriptDirectory.from_config(
96
+ alembic_config_for_provider(migration, sqlalchemy_url="postgresql://")
97
+ )
98
+ heads = list(script.get_heads())
99
+ if len(heads) > 1:
100
+ raise typer.BadParameter(
101
+ "Sequential revision IDs require a single Alembic head. Pass --rev-id "
102
+ "explicitly for branched histories.",
103
+ param_hint="--rev-id",
104
+ )
105
+ if heads and not re.fullmatch(r"\d{4,}", str(heads[0])):
106
+ raise typer.BadParameter(
107
+ "Sequential revision IDs require the current Alembic head to be numeric. "
108
+ "Pass --rev-id explicitly for non-numeric histories.",
109
+ param_hint="--rev-id",
110
+ )
111
+
112
+ numeric_revisions: list[int] = []
113
+ for revision in script.walk_revisions():
114
+ revision_id = str(revision.revision)
115
+ if re.fullmatch(r"\d{4,}", revision_id):
116
+ numeric_revisions.append(int(revision_id))
117
+ return f"{max(numeric_revisions, default=0) + 1:04d}"
118
+
119
+
120
+ @migrations.command("current")
121
+ def current(
122
+ provider: str | None = typer.Option(
123
+ None,
124
+ "--provider",
125
+ help="Migration provider reference, for example msm.migrations:migration.",
126
+ ),
127
+ verbose: bool = typer.Option(False, "--verbose", "-v"),
128
+ timeout: float | None = typer.Option(None, "--timeout"),
129
+ ttl_seconds: int = typer.Option(900, "--ttl-seconds", min=1),
130
+ ) -> None:
131
+ """Read current Alembic revision through a scoped migration credential."""
132
+
133
+ try:
134
+ from alembic import command
135
+ except ImportError as exc:
136
+ raise typer.BadParameter("Alembic is required for migration commands.") from exc
137
+
138
+ migration = _load_migration(provider)
139
+ _, config = _prepare_alembic_config(
140
+ migration,
141
+ timeout=timeout,
142
+ ttl_seconds=ttl_seconds,
143
+ )
144
+ command.current(config, verbose=verbose)
145
+
146
+
147
+ @migrations.command("revision")
148
+ def revision(
149
+ message: str | None = typer.Option(
150
+ None,
151
+ "--message",
152
+ "-m",
153
+ help="Alembic revision message. Defaults to 'migration'.",
154
+ ),
155
+ autogenerate: bool = typer.Option(
156
+ True,
157
+ "--autogenerate/--no-autogenerate",
158
+ help="Use Alembic autogenerate against the reserved MetaTable metadata.",
159
+ ),
160
+ provider: str | None = typer.Option(
161
+ None,
162
+ "--provider",
163
+ help="Migration provider reference, for example msm.migrations:migration.",
164
+ ),
165
+ rev_id: str | None = typer.Option(None, "--rev-id", help="Explicit Alembic revision id."),
166
+ head: str = typer.Option("head", "--head", help="Alembic head to base the revision on."),
167
+ timeout: float | None = typer.Option(None, "--timeout"),
168
+ ttl_seconds: int = typer.Option(900, "--ttl-seconds", min=1),
169
+ json_output: bool = typer.Option(False, "--json", help="Emit JSON."),
170
+ ) -> None:
171
+ """Create a normal Alembic revision for the selected provider."""
172
+
173
+ try:
174
+ from alembic import command
175
+ except ImportError as exc:
176
+ raise typer.BadParameter("Alembic is required for revision generation.") from exc
177
+
178
+ migration = _load_migration(provider)
179
+ resolved_message = (message or "").strip() or "migration"
180
+ resolved_rev_id = rev_id or _next_sequential_revision_id(migration)
181
+ prepared, config = _prepare_alembic_config(
182
+ migration,
183
+ timeout=timeout,
184
+ ttl_seconds=ttl_seconds,
185
+ )
186
+ script = command.revision(
187
+ config,
188
+ message=resolved_message,
189
+ autogenerate=autogenerate,
190
+ rev_id=resolved_rev_id,
191
+ head=head,
192
+ )
193
+ _emit(
194
+ {
195
+ "revision": getattr(script, "revision", None),
196
+ "path": getattr(script, "path", None),
197
+ "package": migration.package,
198
+ "migration_namespace": migration.migration_namespace,
199
+ "meta_table_uids": prepared.meta_table_uids,
200
+ },
201
+ json_output=json_output,
202
+ )
203
+
204
+
205
+ @migrations.command("upgrade")
206
+ def upgrade(
207
+ target_revision: str = typer.Argument("head", help="Target Alembic revision."),
208
+ provider: str | None = typer.Option(
209
+ None,
210
+ "--provider",
211
+ help="Migration provider reference, for example msm.migrations:migration.",
212
+ ),
213
+ timeout: float | None = typer.Option(None, "--timeout"),
214
+ ttl_seconds: int = typer.Option(900, "--ttl-seconds", min=1),
215
+ json_output: bool = typer.Option(False, "--json", help="Emit JSON."),
216
+ ) -> None:
217
+ """Run Alembic upgrade directly and refresh MetaTable catalog rows."""
218
+
219
+ try:
220
+ from alembic import command
221
+ except ImportError as exc:
222
+ raise typer.BadParameter("Alembic is required for migration commands.") from exc
223
+
224
+ migration = _load_migration(provider)
225
+ prepared, config = _prepare_alembic_config(
226
+ migration,
227
+ timeout=timeout,
228
+ ttl_seconds=ttl_seconds,
229
+ )
230
+ command.upgrade(config, target_revision)
231
+ registered = migration.refresh_metatable_catalog(timeout=timeout)
232
+ _emit(
233
+ {
234
+ "ok": True,
235
+ "revision": target_revision,
236
+ "package": migration.package,
237
+ "migration_namespace": migration.migration_namespace,
238
+ "meta_table_uids": prepared.meta_table_uids,
239
+ "registered_count": len(registered),
240
+ },
241
+ json_output=json_output,
242
+ )
243
+
244
+
245
+ @migrations.command("downgrade")
246
+ def downgrade(
247
+ target_revision: str = typer.Argument(..., help="Target Alembic downgrade revision."),
248
+ provider: str | None = typer.Option(
249
+ None,
250
+ "--provider",
251
+ help="Migration provider reference, for example msm.migrations:migration.",
252
+ ),
253
+ timeout: float | None = typer.Option(None, "--timeout"),
254
+ ttl_seconds: int = typer.Option(900, "--ttl-seconds", min=1),
255
+ json_output: bool = typer.Option(False, "--json", help="Emit JSON."),
256
+ ) -> None:
257
+ """Run Alembic downgrade directly and refresh MetaTable catalog rows."""
258
+
259
+ try:
260
+ from alembic import command
261
+ except ImportError as exc:
262
+ raise typer.BadParameter("Alembic is required for migration commands.") from exc
263
+
264
+ migration = _load_migration(provider)
265
+ prepared, config = _prepare_alembic_config(
266
+ migration,
267
+ timeout=timeout,
268
+ ttl_seconds=ttl_seconds,
269
+ )
270
+ command.downgrade(config, target_revision)
271
+ registered = migration.refresh_metatable_catalog(timeout=timeout)
272
+ _emit(
273
+ {
274
+ "ok": True,
275
+ "revision": target_revision,
276
+ "package": migration.package,
277
+ "migration_namespace": migration.migration_namespace,
278
+ "meta_table_uids": prepared.meta_table_uids,
279
+ "registered_count": len(registered),
280
+ },
281
+ json_output=json_output,
282
+ )
@@ -4,14 +4,9 @@ import sys
4
4
  import types
5
5
 
6
6
  from . import core as core
7
- from . import migrations as migrations
8
7
  from .core import * # noqa: F403
9
- from .migrations import * # noqa: F403
10
-
11
- migrations._bind_meta_table_migration_methods(core.MetaTable)
12
8
 
13
9
  __all__ = [
14
- *migrations.__all__,
15
10
  *core.__all__,
16
11
  ]
17
12
 
@@ -19,8 +14,6 @@ __all__ = [
19
14
  def __getattr__(name: str):
20
15
  if hasattr(core, name):
21
16
  return getattr(core, name)
22
- if hasattr(migrations, name):
23
- return getattr(migrations, name)
24
17
  raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
25
18
 
26
19
 
@@ -28,8 +21,6 @@ class _MetaTablesModule(types.ModuleType):
28
21
  def __setattr__(self, name: str, value):
29
22
  if hasattr(core, name):
30
23
  setattr(core, name, value)
31
- if hasattr(migrations, name):
32
- setattr(migrations, name, value)
33
24
  super().__setattr__(name, value)
34
25
 
35
26
 
@@ -286,7 +286,7 @@ class MetaTableColumnContract(BasePydanticModel):
286
286
 
287
287
 
288
288
  class MetaTableIndexContract(BasePydanticModel):
289
- name: str
289
+ name: str | None = None
290
290
  columns: list[str] = Field(default_factory=list)
291
291
  unique: bool = False
292
292
  method: str | None = None
@@ -487,6 +487,69 @@ class MetaTableValidateContractRequest(BasePydanticModel):
487
487
  return self
488
488
 
489
489
 
490
+ class DynamicTableDataSourceMigrationConnectionRequest(BasePydanticModel):
491
+ purpose: Literal["schema_migration"] = "schema_migration"
492
+ package: str = ""
493
+ migration_namespace: str = ""
494
+ meta_table_uids: list[str]
495
+ ttl_seconds: int = Field(default=900, ge=1)
496
+
497
+
498
+ class DynamicTableDataSourceMigrationConnection(BasePydanticModel):
499
+ ok: bool
500
+ data_source_uid: str
501
+ dialect: str
502
+ credential_kind: str
503
+ role_name: str
504
+ owner_role_name: str | None = None
505
+ expires_at: datetime.datetime
506
+ uri: str
507
+
508
+ model_config = ConfigDict(extra="allow")
509
+
510
+
511
+ class ManagedMetaTableReservationTable(BasePydanticModel):
512
+ identifier: str
513
+ namespace: str | None = None
514
+ data_source_uid: str | None = None
515
+ management_mode: Literal["platform_managed"] = "platform_managed"
516
+ storage_hash: str | None = None
517
+ physical_table_name: str | None = None
518
+ description: str | None = None
519
+ labels: list[str] = Field(default_factory=list)
520
+ protect_from_deletion: bool = False
521
+ open_for_everyone: bool = False
522
+ table_contract: MetaTableContract | dict[str, Any]
523
+ time_index_name: str | None = None
524
+ partition_strategy: str | None = None
525
+
526
+
527
+ class ManagedMetaTableReservationRequest(BasePydanticModel):
528
+ version: Literal["managed-metatable-reservation.v1"] = "managed-metatable-reservation.v1"
529
+ data_source_uid: str | None = None
530
+ tables: list[ManagedMetaTableReservationTable]
531
+
532
+
533
+ class ManagedMetaTableReservationItem(BasePydanticModel):
534
+ identifier: str
535
+ namespace: str | None = None
536
+ meta_table_uid: str
537
+ data_source_uid: str
538
+ management_mode: str
539
+ storage_hash: str
540
+ physical_table_name: str
541
+ table_contract: dict[str, Any]
542
+ reservation_status: str
543
+ existing: bool
544
+
545
+ model_config = ConfigDict(extra="allow")
546
+
547
+
548
+ class ManagedMetaTableReservationResponse(BasePydanticModel):
549
+ version: Literal["managed-metatable-reservation.v1"] = "managed-metatable-reservation.v1"
550
+ tables: list[ManagedMetaTableReservationItem]
551
+
552
+
490
553
  class DataSource(BasePydanticModel, BaseObjectOrm):
491
554
  uid: str | None = Field(
492
555
  None,
@@ -736,6 +799,44 @@ class DynamicTableDataSource(BasePydanticModel, BaseObjectOrm):
736
799
  dump["related_resource"] = self.related_resource.model_dump()
737
800
  return json.dumps(dump, **json_dumps_kwargs)
738
801
 
802
+ def _public_uid(self) -> str:
803
+ if self.uid in (None, ""):
804
+ raise ValueError("DynamicTableDataSource must have a uid before calling this endpoint.")
805
+ return str(self.uid)
806
+
807
+ def issue_migration_connection(
808
+ self,
809
+ request: DynamicTableDataSourceMigrationConnectionRequest | Mapping[str, Any] | None = None,
810
+ *,
811
+ timeout: int | float | tuple[float, float] | None = None,
812
+ **kwargs: Any,
813
+ ) -> DynamicTableDataSourceMigrationConnection:
814
+ if request is not None and kwargs:
815
+ raise ValueError("Pass either request or keyword fields, not both.")
816
+ payload = (
817
+ request
818
+ if request is not None
819
+ else DynamicTableDataSourceMigrationConnectionRequest(**kwargs)
820
+ )
821
+ if isinstance(payload, Mapping):
822
+ payload = DynamicTableDataSourceMigrationConnectionRequest(**payload)
823
+
824
+ url = (
825
+ f"{type(self).get_object_url().rstrip('/')}/{self._public_uid()}/migration-connection/"
826
+ )
827
+ request_payload = {"json": _payload_json(payload)}
828
+ response = make_request(
829
+ s=type(self).build_session(),
830
+ loaders=type(self).LOADERS,
831
+ r_type="POST",
832
+ url=url,
833
+ payload=request_payload,
834
+ time_out=timeout,
835
+ )
836
+ if response.status_code != 200:
837
+ raise_for_response(response, payload=request_payload)
838
+ return DynamicTableDataSourceMigrationConnection(**response.json())
839
+
739
840
  @classmethod
740
841
  def get_or_create_duck_db(cls, time_out=None, *args, **kwargs):
741
842
  url = cls.get_object_url() + "/get_or_create_duck_db/"
@@ -1068,6 +1169,25 @@ class MetaTable(BasePydanticModel, LabelableObjectMixin, ShareableObjectMixin, B
1068
1169
  )
1069
1170
  return cls(**response_json)
1070
1171
 
1172
+ @classmethod
1173
+ def reserve_managed(
1174
+ cls,
1175
+ request: ManagedMetaTableReservationRequest | Mapping[str, Any] | None = None,
1176
+ *,
1177
+ timeout: int | float | tuple[float, float] | None = None,
1178
+ **kwargs: Any,
1179
+ ) -> ManagedMetaTableReservationResponse:
1180
+ if request is not None and kwargs:
1181
+ raise ValueError("Pass either request or keyword fields, not both.")
1182
+ payload = request if request is not None else ManagedMetaTableReservationRequest(**kwargs)
1183
+ response_json = cls._post_action(
1184
+ "reserve-managed",
1185
+ payload,
1186
+ timeout=timeout,
1187
+ expected_statuses=(200, 201),
1188
+ )
1189
+ return ManagedMetaTableReservationResponse(**response_json)
1190
+
1071
1191
  @classmethod
1072
1192
  def validate_contract(
1073
1193
  cls,
@@ -1102,6 +1222,25 @@ class MetaTable(BasePydanticModel, LabelableObjectMixin, ShareableObjectMixin, B
1102
1222
  *,
1103
1223
  timeout: int | float | tuple[float, float] | None = None,
1104
1224
  ) -> dict[str, Any]:
1225
+ """
1226
+ Refresh this MetaTable's physical database shape snapshot.
1227
+
1228
+ This calls the backend ``POST /meta_table/<uid>/introspect/`` action.
1229
+ The backend reads the real physical table through the MetaTable's data
1230
+ source, reflects columns, indexes, and constraints, stores that data on
1231
+ ``MetaTable.introspection_snapshot``, and returns the full response.
1232
+
1233
+ This method is intended for admin, debugging, and reconciliation
1234
+ workflows. Use it when a client needs to inspect what the database
1235
+ currently has, diagnose catalog/physical drift, or refresh metadata
1236
+ after an out-of-band DDL change. It is not required for normal reads,
1237
+ writes, registration, or migration-first application startup.
1238
+
1239
+ Returns:
1240
+ Backend response containing ``ok``, ``meta_table_uid``, and
1241
+ ``introspection_snapshot``. When the snapshot is an object, this
1242
+ instance's ``introspection_snapshot`` attribute is updated in place.
1243
+ """
1105
1244
  response_json = self._post_detail_action("introspect", timeout=timeout)
1106
1245
  snapshot = response_json.get("introspection_snapshot")
1107
1246
  if isinstance(snapshot, dict):
@@ -4272,12 +4411,18 @@ __all__ = [
4272
4411
  "DataNodeUpdateDetails",
4273
4412
  "DataSource",
4274
4413
  "DynamicTableDataSource",
4414
+ "DynamicTableDataSourceMigrationConnection",
4415
+ "DynamicTableDataSourceMigrationConnectionRequest",
4275
4416
  "DUCK_DB",
4276
4417
  "HistoricalUpdateRecord",
4277
4418
  "LastUpdateIndexTimePayload",
4278
4419
  "LastUpdateMultiIndexStatsPayload",
4279
4420
  "LOCAL_DATA_SOURCE_CLASS_TYPES",
4280
4421
  "LocalTimeSeriesHistoricalUpdate",
4422
+ "ManagedMetaTableReservationItem",
4423
+ "ManagedMetaTableReservationRequest",
4424
+ "ManagedMetaTableReservationResponse",
4425
+ "ManagedMetaTableReservationTable",
4281
4426
  "MetaTable",
4282
4427
  "MetaTableColumnContract",
4283
4428
  "MetaTableColumnPayload",
@@ -14,10 +14,6 @@ _LAZY_IMPORTS = {
14
14
  "AlembicVersionMetaTable": (".migrations", "AlembicVersionMetaTable"),
15
15
  "PlatformManagedMetaTable": (".sqlalchemy_contracts", "PlatformManagedMetaTable"),
16
16
  "PlatformTimeIndexMetaData": (".sqlalchemy_contracts", "PlatformTimeIndexMetaData"),
17
- "PackagedAlembicMigrationArtifact": (
18
- ".migrations",
19
- "PackagedAlembicMigrationArtifact",
20
- ),
21
17
  "POSTGRES_IDENTIFIER_MAX_LENGTH": (".hashing", "POSTGRES_IDENTIFIER_MAX_LENGTH"),
22
18
  "BaseConfiguration": (".data_nodes", "BaseConfiguration"),
23
19
  "APIDataNode": (".data_nodes", "APIDataNode"),
@@ -77,18 +73,19 @@ _LAZY_IMPORTS = {
77
73
  ".sqlalchemy_contracts",
78
74
  "resolve_metatable_identifier",
79
75
  ),
80
- "render_packaged_alembic_migration": (
81
- ".migrations",
82
- "render_packaged_alembic_migration",
83
- ),
84
- "render_packaged_alembic_migration_for_provider": (
76
+ "alembic_config_for_provider": (".migrations", "alembic_config_for_provider"),
77
+ "apply_mainsequence_migration_role": (
85
78
  ".migrations",
86
- "render_packaged_alembic_migration_for_provider",
79
+ "apply_mainsequence_migration_role",
87
80
  ),
88
81
  "load_alembic_metatable_migration_provider": (
89
82
  ".migrations",
90
83
  "load_alembic_metatable_migration_provider",
91
84
  ),
85
+ "PreparedAlembicMetaTableMigration": (
86
+ ".migrations",
87
+ "PreparedAlembicMetaTableMigration",
88
+ ),
92
89
  "resolve_alembic_revision_metadata": (
93
90
  ".migrations",
94
91
  "resolve_alembic_revision_metadata",
@@ -97,7 +97,7 @@ def _(value: type[Any]) -> Any:
97
97
  if uid in (None, ""):
98
98
  raise ValueError(
99
99
  "PlatformTimeIndexMetaData config value is not registered. Run "
100
- "`mainsequence migrations upgrade --provider <provider> --to head` "
100
+ "`mainsequence migrations upgrade --provider <provider> head` "
101
101
  "before using it in DataNode configuration."
102
102
  )
103
103
 
@@ -99,14 +99,19 @@ def ensure_registered_storage_table(
99
99
  if storage_table.get_time_index_metadata() is None:
100
100
  raise ValueError(
101
101
  f"{context} storage_table is not registered. Run "
102
- "`mainsequence migrations upgrade --provider <provider> --to head` "
102
+ "`mainsequence migrations upgrade --provider <provider> head` "
103
103
  "before using this DataNode storage table."
104
104
  )
105
105
 
106
106
  storage_metadata = storage_table.get_time_index_metadata()
107
107
  if storage_metadata is None:
108
- raise ValueError(
109
- f"{context} storage_table is missing TimeIndexMetaData metadata."
108
+ raise ValueError(f"{context} storage_table is missing TimeIndexMetaData metadata.")
109
+ from mainsequence.client.metatables import TimeIndexMetaData
110
+
111
+ if not isinstance(storage_metadata, TimeIndexMetaData):
112
+ raise TypeError(
113
+ f"{context} storage_table must bind TimeIndexMetaData metadata; "
114
+ f"got {type(storage_metadata).__name__}."
110
115
  )
111
116
  if storage_table.get_meta_table_uid() in (None, ""):
112
117
  raise ValueError(f"{context} storage_table must provide a MetaTable UID.")