mainsequence-metatable 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. api/__init__.py +3 -0
  2. api/app/__init__.py +0 -0
  3. api/app/auth/__init__.py +1 -0
  4. api/app/auth/boundary.py +104 -0
  5. api/app/auth/caller_assertions.py +39 -0
  6. api/app/auth/local_actor.py +55 -0
  7. api/app/bootstrap.py +607 -0
  8. api/app/configuration.py +37 -0
  9. api/app/credential_admin.py +107 -0
  10. api/app/development.py +175 -0
  11. api/app/development_client.py +59 -0
  12. api/app/main.py +201 -0
  13. api/app/platform_cache.py +61 -0
  14. api/app/routes/__init__.py +0 -0
  15. api/app/routes/application_migrations.py +170 -0
  16. api/app/routes/column_search.py +64 -0
  17. api/app/routes/contract_validation.py +84 -0
  18. api/app/routes/data_sources.py +428 -0
  19. api/app/routes/managed_finalize.py +244 -0
  20. api/app/routes/migration_connection.py +41 -0
  21. api/app/routes/namespaces.py +119 -0
  22. api/app/routes/run_logs.py +78 -0
  23. api/app/routes/runtime_context.py +122 -0
  24. api/app/routes/schema_graph.py +37 -0
  25. api/app/routes/security.py +267 -0
  26. api/app/routes/table_cascade.py +155 -0
  27. api/app/routes/table_crud.py +45 -0
  28. api/app/routes/table_delete.py +161 -0
  29. api/app/routes/table_filter.py +38 -0
  30. api/app/routes/table_import.py +66 -0
  31. api/app/routes/table_introspection.py +85 -0
  32. api/app/routes/table_patch.py +155 -0
  33. api/app/routes/table_registration.py +216 -0
  34. api/app/routes/table_response.py +29 -0
  35. api/app/routes/table_search_document.py +54 -0
  36. api/app/routes/table_sql.py +167 -0
  37. api/app/routes/table_update_pipeline.py +31 -0
  38. api/app/routes/time_index_delete.py +136 -0
  39. api/app/routes/time_index_reads.py +183 -0
  40. api/app/routes/time_index_stats.py +51 -0
  41. api/app/routes/update_batch.py +173 -0
  42. api/app/routes/update_chunk_insert.py +147 -0
  43. api/app/routes/update_constants.py +19 -0
  44. api/app/routes/update_crud.py +280 -0
  45. api/app/routes/update_details.py +141 -0
  46. api/app/routes/update_graph.py +120 -0
  47. api/app/routes/update_index_statistics.py +107 -0
  48. api/app/routes/update_registration.py +92 -0
  49. api/app/routes/update_runs.py +303 -0
  50. api/app/routes/update_statistics.py +77 -0
  51. api/app/runtime.py +317 -0
  52. api/app/runtime_control.py +154 -0
  53. api/app/source_passwords.py +43 -0
  54. api/app/source_registrations.py +216 -0
  55. api/backend/__init__.py +1 -0
  56. api/backend/backends/__init__.py +1 -0
  57. api/backend/backends/access.py +363 -0
  58. api/backend/backends/adapters/__init__.py +1 -0
  59. api/backend/backends/adapters/maintenance/__init__.py +6 -0
  60. api/backend/backends/adapters/maintenance/postgresql.py +2788 -0
  61. api/backend/backends/adapters/maintenance/runtime.py +256 -0
  62. api/backend/backends/adapters/maintenance/temporal.py +333 -0
  63. api/backend/backends/adapters/maintenance/timescaledb.py +657 -0
  64. api/backend/backends/adapters/maintenance/validation.py +10 -0
  65. api/backend/backends/adapters/postgresql_cascade_delete.py +39 -0
  66. api/backend/backends/adapters/postgresql_connection.py +103 -0
  67. api/backend/backends/adapters/postgresql_introspection.py +128 -0
  68. api/backend/backends/adapters/postgresql_table_delete.py +42 -0
  69. api/backend/backends/adapters/postgresql_tables.py +245 -0
  70. api/backend/backends/adapters/postgresql_time_index.py +141 -0
  71. api/backend/backends/adapters/postgresql_time_index_delete.py +140 -0
  72. api/backend/backends/adapters/postgresql_time_index_queries.py +281 -0
  73. api/backend/backends/adapters/postgresql_time_index_stats.py +89 -0
  74. api/backend/backends/adapters/postgresql_time_index_write.py +185 -0
  75. api/backend/backends/adapters/source_validation.py +80 -0
  76. api/backend/backends/adapters/sqlite_connection.py +324 -0
  77. api/backend/backends/adapters/sqlite_introspection.py +135 -0
  78. api/backend/backends/adapters/table_cursor.py +14 -0
  79. api/backend/backends/catalog.py +288 -0
  80. api/backend/backends/catalog_ddl.py +126 -0
  81. api/backend/backends/catalog_types.py +32 -0
  82. api/backend/backends/contracts.py +322 -0
  83. api/backend/backends/hosted.py +495 -0
  84. api/backend/backends/hosted_access.py +378 -0
  85. api/backend/backends/native.py +105 -0
  86. api/backend/backends/operations.py +247 -0
  87. api/backend/backends/postgresql_access.py +302 -0
  88. api/backend/backends/registry.py +114 -0
  89. api/backend/backends/relation_tables.py +676 -0
  90. api/backend/backends/relational.py +804 -0
  91. api/backend/backends/restricted_sql.py +181 -0
  92. api/backend/backends/sqlite_access.py +129 -0
  93. api/backend/contracts/__init__.py +1 -0
  94. api/backend/contracts/relation_import.py +82 -0
  95. api/backend/contracts/storage_layouts.py +287 -0
  96. api/backend/contracts/table_contracts.py +361 -0
  97. api/backend/contracts/values.py +77 -0
  98. api/backend/domain/__init__.py +1 -0
  99. api/backend/domain/catalog_rules.py +161 -0
  100. api/backend/domain/data_source_rules.py +32 -0
  101. api/backend/domain/index_statistics.py +121 -0
  102. api/backend/domain/label_rules.py +32 -0
  103. api/backend/domain/permission_rules.py +18 -0
  104. api/backend/domain/update_constants.py +32 -0
  105. api/backend/integrations/__init__.py +1 -0
  106. api/backend/integrations/catalog_data_sources.py +166 -0
  107. api/backend/integrations/credential_keys.py +172 -0
  108. api/backend/integrations/credential_store.py +115 -0
  109. api/backend/integrations/data_source_provider.py +46 -0
  110. api/backend/integrations/local_credentials.py +192 -0
  111. api/backend/integrations/local_workspace.py +178 -0
  112. api/backend/integrations/platform_principals.py +32 -0
  113. api/backend/integrations/run_logs.py +271 -0
  114. api/backend/integrations/runtime_binding.py +65 -0
  115. api/backend/integrations/runtime_data_sources.py +81 -0
  116. api/backend/integrations/source_validation.py +6 -0
  117. api/backend/migrations/__init__.py +1 -0
  118. api/backend/migrations/env.py +55 -0
  119. api/backend/migrations/script.py.mako +25 -0
  120. api/backend/migrations/versions/0001_initial.py +522 -0
  121. api/backend/migrations/versions/0002_security_model.py +73 -0
  122. api/backend/migrations/versions/0003_run_log_capture.py +17 -0
  123. api/backend/migrations/versions/0004_historical_run_graphs.py +33 -0
  124. api/backend/migrations/versions/0005_database_access.py +40 -0
  125. api/backend/migrations/versions/0006_local_credentials.py +36 -0
  126. api/backend/migrations/versions/0007_external_relation_names.py +34 -0
  127. api/backend/migrations/versions/__init__.py +1 -0
  128. api/backend/operations/__init__.py +0 -0
  129. api/backend/operations/bootstrap_migrations.py +275 -0
  130. api/backend/operations/catalog_schema.py +56 -0
  131. api/backend/operations/create_table.py +170 -0
  132. api/backend/operations/finalize_managed.py +108 -0
  133. api/backend/operations/import_relations.py +407 -0
  134. api/backend/operations/introspect_table.py +88 -0
  135. api/backend/operations/read_time_index.py +75 -0
  136. api/backend/operations/register_catalog.py +211 -0
  137. api/backend/operations/search_documents.py +65 -0
  138. api/backend/persistence/__init__.py +1 -0
  139. api/backend/persistence/cascade.py +40 -0
  140. api/backend/persistence/catalog.py +176 -0
  141. api/backend/persistence/database.py +59 -0
  142. api/backend/persistence/grants.py +146 -0
  143. api/backend/persistence/labels.py +171 -0
  144. api/backend/persistence/locking.py +8 -0
  145. api/backend/persistence/models.py +593 -0
  146. api/backend/persistence/physical_operations.py +150 -0
  147. api/backend/persistence/projections.py +146 -0
  148. api/backend/persistence/run_graphs.py +287 -0
  149. api/backend/persistence/schema_graph.py +136 -0
  150. api/backend/persistence/sqlite_types.py +25 -0
  151. api/backend/persistence/table_body_filter.py +297 -0
  152. api/backend/persistence/table_queries.py +92 -0
  153. api/backend/persistence/table_response.py +209 -0
  154. api/backend/persistence/update_graph.py +106 -0
  155. api/backend/persistence/update_pipeline.py +139 -0
  156. api/backend/persistence/updates.py +314 -0
  157. api/backend/presentation/data_source_summary.py +99 -0
  158. api/metatables/__init__.py +1 -0
  159. api/metatables/configuration.yaml +3 -0
  160. api/metatables/main.py +8 -0
  161. examples/README.md +26 -0
  162. examples/__init__.py +1 -0
  163. examples/data_sources/README.md +25 -0
  164. examples/data_sources/mssql.json +16 -0
  165. examples/data_sources/mysql.json +16 -0
  166. examples/data_sources/postgresql.json +15 -0
  167. examples/data_sources/register.py +23 -0
  168. examples/external_table.py +20 -0
  169. examples/query.py +19 -0
  170. examples/quickstart.py +17 -0
  171. examples/reader.py +12 -0
  172. examples/scripts/__init__.py +1 -0
  173. examples/scripts/setup_metatables.py +42 -0
  174. examples/tables.py +49 -0
  175. examples/tutorial/README.md +80 -0
  176. examples/tutorial/__init__.py +1 -0
  177. examples/tutorial/__main__.py +76 -0
  178. examples/tutorial/data/README.md +11 -0
  179. examples/tutorial/data/yahoo_spy_qqq_iwm_2026-06-15_2026-07-17.csv +70 -0
  180. examples/tutorial/frames.py +48 -0
  181. examples/tutorial/migrations/__init__.py +29 -0
  182. examples/tutorial/migrations/env.py +6 -0
  183. examples/tutorial/migrations/registry.py +9 -0
  184. examples/tutorial/migrations/script.py.mako +28 -0
  185. examples/tutorial/migrations/versions/__init__.py +0 -0
  186. examples/tutorial/migrations/versions/metatables_market_tutorial/0001_create_market_tutorial_tables.py +81 -0
  187. examples/tutorial/migrations/versions/metatables_market_tutorial/__init__.py +0 -0
  188. examples/tutorial/operations.py +67 -0
  189. examples/tutorial/tables.py +114 -0
  190. examples/tutorial/updaters.py +95 -0
  191. examples/updater.py +42 -0
  192. mainsequence_metatable-0.1.0.dist-info/METADATA +137 -0
  193. mainsequence_metatable-0.1.0.dist-info/RECORD +433 -0
  194. mainsequence_metatable-0.1.0.dist-info/WHEEL +5 -0
  195. mainsequence_metatable-0.1.0.dist-info/entry_points.txt +2 -0
  196. mainsequence_metatable-0.1.0.dist-info/licenses/LICENSE +21 -0
  197. mainsequence_metatable-0.1.0.dist-info/top_level.txt +3 -0
  198. metatables/__init__.py +202 -0
  199. metatables/_api_workflow.yaml +10 -0
  200. metatables/_client_skills/metatables-meta-tables/SKILL.md +106 -0
  201. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0001-unified-api-storage-and-local-sqlite.md +634 -0
  202. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0002-application-administration-and-table-ownership.md +366 -0
  203. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0004-approved-application-migrations.md +57 -0
  204. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0005-run-logs-and-local-file-capture.md +237 -0
  205. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0006-historical-run-graphs.md +288 -0
  206. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0007-database-enforced-table-access.md +275 -0
  207. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0007-database-roles-and-privileges.svg +120 -0
  208. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0007-query-enforcement-path.svg +76 -0
  209. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0008-mysql-mssql-table-workflows.md +418 -0
  210. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0009-shared-credential-store.md +189 -0
  211. metatables/_client_skills/metatables-meta-tables/references/docs/adr/api/0010-import-external-tables-and-views.md +567 -0
  212. metatables/_client_skills/metatables-meta-tables/references/docs/adr/client/0003-api-endpoint-resolution.md +236 -0
  213. metatables/_client_skills/metatables-meta-tables/references/docs/adr/index.md +33 -0
  214. metatables/_client_skills/metatables-meta-tables/references/docs/api/data-sources.md +197 -0
  215. metatables/_client_skills/metatables-meta-tables/references/docs/client/build-an-updater.md +54 -0
  216. metatables/_client_skills/metatables-meta-tables/references/docs/client/define-and-migrate-tables.md +80 -0
  217. metatables/_client_skills/metatables-meta-tables/references/docs/client/installation-and-connection.md +127 -0
  218. metatables/_client_skills/metatables-meta-tables/references/docs/client/query-and-mutate.md +78 -0
  219. metatables/_client_skills/metatables-meta-tables/references/docs/client/register-existing-tables.md +93 -0
  220. metatables/_client_skills/metatables-meta-tables/references/docs/client/tutorial.md +185 -0
  221. metatables/_client_skills/metatables-meta-tables/references/docs/concepts/architecture.md +90 -0
  222. metatables/_client_skills/metatables-meta-tables/references/docs/concepts/data-sources.md +162 -0
  223. metatables/_client_skills/metatables-meta-tables/references/docs/concepts/identity-and-scope.md +53 -0
  224. metatables/_client_skills/metatables-meta-tables/references/docs/concepts/permissions-namespaces-labels.md +37 -0
  225. metatables/_client_skills/metatables-meta-tables/references/docs/concepts/runs-and-execution-graphs.md +115 -0
  226. metatables/_client_skills/metatables-meta-tables/references/docs/concepts/table-contracts-and-lifecycle.md +58 -0
  227. metatables/_client_skills/metatables-meta-tables/references/docs/concepts/time-index-tables-and-updates.md +60 -0
  228. metatables/_client_skills/metatables-meta-tables/references/docs/contributing/database-backends.md +77 -0
  229. metatables/_client_skills/metatables-meta-tables/references/docs/contributing/releasing.md +148 -0
  230. metatables/_client_skills/metatables-meta-tables/references/docs/contributing/testing.md +115 -0
  231. metatables/_client_skills/metatables-meta-tables/references/docs/examples/data_sources/README.md +27 -0
  232. metatables/_client_skills/metatables-meta-tables/references/docs/examples/data_sources/mssql.json +16 -0
  233. metatables/_client_skills/metatables-meta-tables/references/docs/examples/data_sources/mysql.json +16 -0
  234. metatables/_client_skills/metatables-meta-tables/references/docs/examples/data_sources/postgresql.json +15 -0
  235. metatables/_client_skills/metatables-meta-tables/references/docs/examples/external_table.py +20 -0
  236. metatables/_client_skills/metatables-meta-tables/references/docs/examples/query.py +19 -0
  237. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tables.py +49 -0
  238. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/README.md +82 -0
  239. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/data/README.md +13 -0
  240. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/data/yahoo_spy_qqq_iwm_2026-06-15_2026-07-17.csv +70 -0
  241. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/frames.py +48 -0
  242. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/migrations/__init__.py +29 -0
  243. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/migrations/registry.py +9 -0
  244. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/migrations/versions/metatables_market_tutorial/0001_create_market_tutorial_tables.py +81 -0
  245. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/operations.py +67 -0
  246. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/tables.py +114 -0
  247. metatables/_client_skills/metatables-meta-tables/references/docs/examples/tutorial/updaters.py +95 -0
  248. metatables/_client_skills/metatables-meta-tables/references/docs/examples/updater.py +42 -0
  249. metatables/_client_skills/metatables-meta-tables/references/docs/operations/catalog-migrations.md +146 -0
  250. metatables/_client_skills/metatables-meta-tables/references/docs/operations/configuration.md +58 -0
  251. metatables/_client_skills/metatables-meta-tables/references/docs/operations/local-runtime.md +152 -0
  252. metatables/_client_skills/metatables-meta-tables/references/docs/operations/recovery-and-observability.md +60 -0
  253. metatables/_client_skills/metatables-meta-tables/references/docs/operations/run-history.md +128 -0
  254. metatables/_client_skills/metatables-meta-tables/references/docs/operations/run-logs.md +128 -0
  255. metatables/_client_skills/metatables-meta-tables/references/docs/reference/capabilities.md +298 -0
  256. metatables/_client_skills/metatables-meta-tables/references/docs/security/index.md +172 -0
  257. metatables/_client_skills/metatables-meta-tables/references/docs/security/ownership-and-sharing.md +145 -0
  258. metatables/_client_skills/metatables-meta-tables/references/docs/security/permissions.md +105 -0
  259. metatables/_client_skills/metatables-meta-tables/references/examples/external_table.py +20 -0
  260. metatables/_client_skills/metatables-meta-tables/references/examples/query.py +19 -0
  261. metatables/_client_skills/metatables-meta-tables/references/examples/tables.py +49 -0
  262. metatables/_client_skills/metatables-migrations/SKILL.md +132 -0
  263. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0001-unified-api-storage-and-local-sqlite.md +634 -0
  264. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0002-application-administration-and-table-ownership.md +366 -0
  265. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0004-approved-application-migrations.md +57 -0
  266. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0005-run-logs-and-local-file-capture.md +237 -0
  267. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0006-historical-run-graphs.md +288 -0
  268. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0007-database-enforced-table-access.md +275 -0
  269. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0007-database-roles-and-privileges.svg +120 -0
  270. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0007-query-enforcement-path.svg +76 -0
  271. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0008-mysql-mssql-table-workflows.md +418 -0
  272. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0009-shared-credential-store.md +189 -0
  273. metatables/_client_skills/metatables-migrations/references/docs/adr/api/0010-import-external-tables-and-views.md +567 -0
  274. metatables/_client_skills/metatables-migrations/references/docs/adr/client/0003-api-endpoint-resolution.md +236 -0
  275. metatables/_client_skills/metatables-migrations/references/docs/adr/index.md +33 -0
  276. metatables/_client_skills/metatables-migrations/references/docs/api/data-sources.md +197 -0
  277. metatables/_client_skills/metatables-migrations/references/docs/api/tables-and-migrations.md +142 -0
  278. metatables/_client_skills/metatables-migrations/references/docs/client/build-an-updater.md +54 -0
  279. metatables/_client_skills/metatables-migrations/references/docs/client/define-and-migrate-tables.md +80 -0
  280. metatables/_client_skills/metatables-migrations/references/docs/client/installation-and-connection.md +127 -0
  281. metatables/_client_skills/metatables-migrations/references/docs/client/query-and-mutate.md +78 -0
  282. metatables/_client_skills/metatables-migrations/references/docs/client/register-existing-tables.md +93 -0
  283. metatables/_client_skills/metatables-migrations/references/docs/client/tutorial.md +185 -0
  284. metatables/_client_skills/metatables-migrations/references/docs/concepts/architecture.md +90 -0
  285. metatables/_client_skills/metatables-migrations/references/docs/concepts/data-sources.md +162 -0
  286. metatables/_client_skills/metatables-migrations/references/docs/concepts/identity-and-scope.md +53 -0
  287. metatables/_client_skills/metatables-migrations/references/docs/concepts/permissions-namespaces-labels.md +37 -0
  288. metatables/_client_skills/metatables-migrations/references/docs/concepts/runs-and-execution-graphs.md +115 -0
  289. metatables/_client_skills/metatables-migrations/references/docs/concepts/table-contracts-and-lifecycle.md +58 -0
  290. metatables/_client_skills/metatables-migrations/references/docs/concepts/time-index-tables-and-updates.md +60 -0
  291. metatables/_client_skills/metatables-migrations/references/docs/contributing/database-backends.md +77 -0
  292. metatables/_client_skills/metatables-migrations/references/docs/contributing/releasing.md +148 -0
  293. metatables/_client_skills/metatables-migrations/references/docs/contributing/testing.md +115 -0
  294. metatables/_client_skills/metatables-migrations/references/docs/examples/data_sources/README.md +27 -0
  295. metatables/_client_skills/metatables-migrations/references/docs/examples/data_sources/mssql.json +16 -0
  296. metatables/_client_skills/metatables-migrations/references/docs/examples/data_sources/mysql.json +16 -0
  297. metatables/_client_skills/metatables-migrations/references/docs/examples/data_sources/postgresql.json +15 -0
  298. metatables/_client_skills/metatables-migrations/references/docs/examples/external_table.py +20 -0
  299. metatables/_client_skills/metatables-migrations/references/docs/examples/query.py +19 -0
  300. metatables/_client_skills/metatables-migrations/references/docs/examples/tables.py +49 -0
  301. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/README.md +82 -0
  302. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/data/README.md +13 -0
  303. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/data/yahoo_spy_qqq_iwm_2026-06-15_2026-07-17.csv +70 -0
  304. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/frames.py +48 -0
  305. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/migrations/__init__.py +29 -0
  306. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/migrations/registry.py +9 -0
  307. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/migrations/versions/metatables_market_tutorial/0001_create_market_tutorial_tables.py +81 -0
  308. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/operations.py +67 -0
  309. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/tables.py +114 -0
  310. metatables/_client_skills/metatables-migrations/references/docs/examples/tutorial/updaters.py +95 -0
  311. metatables/_client_skills/metatables-migrations/references/docs/examples/updater.py +42 -0
  312. metatables/_client_skills/metatables-migrations/references/docs/operations/catalog-migrations.md +146 -0
  313. metatables/_client_skills/metatables-migrations/references/docs/operations/configuration.md +58 -0
  314. metatables/_client_skills/metatables-migrations/references/docs/operations/local-runtime.md +152 -0
  315. metatables/_client_skills/metatables-migrations/references/docs/operations/recovery-and-observability.md +60 -0
  316. metatables/_client_skills/metatables-migrations/references/docs/operations/run-history.md +128 -0
  317. metatables/_client_skills/metatables-migrations/references/docs/operations/run-logs.md +128 -0
  318. metatables/_client_skills/metatables-migrations/references/docs/reference/capabilities.md +298 -0
  319. metatables/_client_skills/metatables-migrations/references/docs/security/index.md +172 -0
  320. metatables/_client_skills/metatables-migrations/references/docs/security/ownership-and-sharing.md +145 -0
  321. metatables/_client_skills/metatables-migrations/references/docs/security/permissions.md +105 -0
  322. metatables/_client_skills/metatables-migrations/references/examples/scripts/setup_metatables.py +42 -0
  323. metatables/_client_skills/metatables-migrations/references/examples/tables.py +49 -0
  324. metatables/_client_skills/metatables-time-index-table-updates/SKILL.md +103 -0
  325. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0001-unified-api-storage-and-local-sqlite.md +634 -0
  326. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0002-application-administration-and-table-ownership.md +366 -0
  327. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0004-approved-application-migrations.md +57 -0
  328. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0005-run-logs-and-local-file-capture.md +237 -0
  329. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0006-historical-run-graphs.md +288 -0
  330. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0007-database-enforced-table-access.md +275 -0
  331. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0007-database-roles-and-privileges.svg +120 -0
  332. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0007-query-enforcement-path.svg +76 -0
  333. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0008-mysql-mssql-table-workflows.md +418 -0
  334. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0009-shared-credential-store.md +189 -0
  335. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/api/0010-import-external-tables-and-views.md +567 -0
  336. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/client/0003-api-endpoint-resolution.md +236 -0
  337. metatables/_client_skills/metatables-time-index-table-updates/references/docs/adr/index.md +33 -0
  338. metatables/_client_skills/metatables-time-index-table-updates/references/docs/api/authentication-and-errors.md +71 -0
  339. metatables/_client_skills/metatables-time-index-table-updates/references/docs/api/data-sources.md +197 -0
  340. metatables/_client_skills/metatables-time-index-table-updates/references/docs/api/openapi.json +9302 -0
  341. metatables/_client_skills/metatables-time-index-table-updates/references/docs/api/reference.md +115 -0
  342. metatables/_client_skills/metatables-time-index-table-updates/references/docs/api/time-index-and-updates.md +98 -0
  343. metatables/_client_skills/metatables-time-index-table-updates/references/docs/client/build-an-updater.md +54 -0
  344. metatables/_client_skills/metatables-time-index-table-updates/references/docs/client/define-and-migrate-tables.md +80 -0
  345. metatables/_client_skills/metatables-time-index-table-updates/references/docs/client/installation-and-connection.md +127 -0
  346. metatables/_client_skills/metatables-time-index-table-updates/references/docs/client/query-and-mutate.md +78 -0
  347. metatables/_client_skills/metatables-time-index-table-updates/references/docs/client/read-existing-time-index-tables.md +32 -0
  348. metatables/_client_skills/metatables-time-index-table-updates/references/docs/client/tutorial.md +185 -0
  349. metatables/_client_skills/metatables-time-index-table-updates/references/docs/concepts/architecture.md +90 -0
  350. metatables/_client_skills/metatables-time-index-table-updates/references/docs/concepts/data-sources.md +162 -0
  351. metatables/_client_skills/metatables-time-index-table-updates/references/docs/concepts/permissions-namespaces-labels.md +37 -0
  352. metatables/_client_skills/metatables-time-index-table-updates/references/docs/concepts/runs-and-execution-graphs.md +115 -0
  353. metatables/_client_skills/metatables-time-index-table-updates/references/docs/concepts/table-contracts-and-lifecycle.md +58 -0
  354. metatables/_client_skills/metatables-time-index-table-updates/references/docs/concepts/time-index-tables-and-updates.md +60 -0
  355. metatables/_client_skills/metatables-time-index-table-updates/references/docs/contributing/database-backends.md +77 -0
  356. metatables/_client_skills/metatables-time-index-table-updates/references/docs/contributing/releasing.md +148 -0
  357. metatables/_client_skills/metatables-time-index-table-updates/references/docs/contributing/testing.md +115 -0
  358. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/data_sources/README.md +27 -0
  359. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/data_sources/mssql.json +16 -0
  360. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/data_sources/mysql.json +16 -0
  361. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/data_sources/postgresql.json +15 -0
  362. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/query.py +19 -0
  363. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/reader.py +12 -0
  364. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tables.py +49 -0
  365. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/README.md +82 -0
  366. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/data/README.md +13 -0
  367. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/data/yahoo_spy_qqq_iwm_2026-06-15_2026-07-17.csv +70 -0
  368. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/frames.py +48 -0
  369. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/migrations/__init__.py +29 -0
  370. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/migrations/registry.py +9 -0
  371. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/migrations/versions/metatables_market_tutorial/0001_create_market_tutorial_tables.py +81 -0
  372. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/operations.py +67 -0
  373. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/tables.py +114 -0
  374. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/tutorial/updaters.py +95 -0
  375. metatables/_client_skills/metatables-time-index-table-updates/references/docs/examples/updater.py +42 -0
  376. metatables/_client_skills/metatables-time-index-table-updates/references/docs/operations/catalog-migrations.md +146 -0
  377. metatables/_client_skills/metatables-time-index-table-updates/references/docs/operations/configuration.md +58 -0
  378. metatables/_client_skills/metatables-time-index-table-updates/references/docs/operations/local-runtime.md +152 -0
  379. metatables/_client_skills/metatables-time-index-table-updates/references/docs/operations/recovery-and-observability.md +60 -0
  380. metatables/_client_skills/metatables-time-index-table-updates/references/docs/operations/run-history.md +128 -0
  381. metatables/_client_skills/metatables-time-index-table-updates/references/docs/operations/run-logs.md +128 -0
  382. metatables/_client_skills/metatables-time-index-table-updates/references/docs/reference/capabilities.md +298 -0
  383. metatables/_client_skills/metatables-time-index-table-updates/references/docs/security/index.md +172 -0
  384. metatables/_client_skills/metatables-time-index-table-updates/references/docs/security/ownership-and-sharing.md +145 -0
  385. metatables/_client_skills/metatables-time-index-table-updates/references/docs/security/permissions.md +105 -0
  386. metatables/_client_skills/metatables-time-index-table-updates/references/examples/reader.py +12 -0
  387. metatables/_client_skills/metatables-time-index-table-updates/references/examples/tables.py +49 -0
  388. metatables/_client_skills/metatables-time-index-table-updates/references/examples/updater.py +42 -0
  389. metatables/_skill_bundle.py +49 -0
  390. metatables/cli/__init__.py +1 -0
  391. metatables/cli/app.py +444 -0
  392. metatables/cli/filters.py +144 -0
  393. metatables/cli/migrations.py +906 -0
  394. metatables/cli/skills.py +90 -0
  395. metatables/compiled_sql/__init__.py +5 -0
  396. metatables/compiled_sql/mssql.py +35 -0
  397. metatables/compiled_sql/v1.py +219 -0
  398. metatables/dtype_codec.py +460 -0
  399. metatables/endpoint.py +243 -0
  400. metatables/future_registry.py +28 -0
  401. metatables/hashing.py +173 -0
  402. metatables/instrumentation.py +5 -0
  403. metatables/local_run_logs.py +280 -0
  404. metatables/log_contract.py +112 -0
  405. metatables/migrations/__init__.py +1670 -0
  406. metatables/migrations/alembic.py +167 -0
  407. metatables/migrations/env.py +141 -0
  408. metatables/migrations/provider.py +25 -0
  409. metatables/migrations/registry.py +75 -0
  410. metatables/migrations/scaffold.py +221 -0
  411. metatables/migrations/templates/__init__.py +1 -0
  412. metatables/migrations/templates/env.py.mako +8 -0
  413. metatables/migrations/templates/script.py.mako +28 -0
  414. metatables/migrations/tls.py +23 -0
  415. metatables/models.py +4804 -0
  416. metatables/pydantic_metadata.py +99 -0
  417. metatables/run_logging.py +172 -0
  418. metatables/runtime_context.py +65 -0
  419. metatables/runtime_contract.py +61 -0
  420. metatables/schema_names.py +304 -0
  421. metatables/sdk_compat.py +34 -0
  422. metatables/sqlalchemy_contracts.py +1771 -0
  423. metatables/time_index_table_refs.py +290 -0
  424. metatables/transport.py +73 -0
  425. metatables/update_configuration_contract.py +79 -0
  426. metatables/updaters/__init__.py +27 -0
  427. metatables/updaters/configuration.py +546 -0
  428. metatables/updaters/dependencies.py +9 -0
  429. metatables/updaters/managers.py +813 -0
  430. metatables/updaters/namespacing.py +52 -0
  431. metatables/updaters/runner.py +919 -0
  432. metatables/updaters/updaters.py +792 -0
  433. metatables/updaters/utils.py +23 -0
api/app/bootstrap.py ADDED
@@ -0,0 +1,607 @@
1
+ """Explicit, mode-independent initialization of the selected runtime database."""
2
+ import json
3
+ import os
4
+ from contextlib import ExitStack, contextmanager
5
+ from dataclasses import replace
6
+ from pathlib import Path
7
+ from threading import Lock
8
+ from types import SimpleNamespace
9
+ from typing import Literal
10
+ from uuid import UUID, uuid4
11
+
12
+ from alembic.migration import MigrationContext
13
+ from alembic.script import ScriptDirectory
14
+ from fastapi import APIRouter, Depends, HTTPException, Request
15
+ from pydantic import BaseModel, ConfigDict, Field
16
+ from sqlalchemy import inspect, select
17
+ from sqlalchemy.engine import URL
18
+ from sqlalchemy.orm import sessionmaker
19
+
20
+ from api.app.auth.boundary import AdmittedCaller, require_admin
21
+ from api.backend.backends.contracts import CatalogConfigurationError
22
+ from api.backend.integrations.catalog_data_sources import parse_configuration, resolve_source
23
+ from api.backend.integrations.credential_store import (
24
+ CredentialStoreError,
25
+ SDKCredentialStore,
26
+ UnavailableCredentialStore,
27
+ bind_store,
28
+ )
29
+ from api.backend.integrations.local_workspace import _bind_file, _read_marker
30
+ from api.backend.integrations.runtime_binding import RuntimeStorageBinding
31
+ from api.backend.persistence.database import (
32
+ create_catalog_engine,
33
+ migration_config,
34
+ upgrade_catalog,
35
+ )
36
+ from api.backend.persistence.models import Base, DataSource, MetaTable, PhysicalOperation
37
+ from metatables.runtime_contract import RuntimeBootstrapDescriptor
38
+
39
+ VERSION_TABLE = "metatables_catalog_version"
40
+
41
+
42
+ class RuntimeSourceInput(BaseModel):
43
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
44
+ display_name: str = Field(min_length=1, max_length=255)
45
+ class_type: str
46
+ configuration: dict
47
+
48
+
49
+ class BootstrapError(RuntimeError):
50
+ pass
51
+
52
+
53
+ class DestroyLocalInput(BaseModel):
54
+ model_config = ConfigDict(extra="forbid")
55
+ path: str
56
+ confirmation: Literal["DESTROY"]
57
+
58
+
59
+ class RuntimeBootstrap:
60
+ def __init__(self, config, identity):
61
+ self.config, self.identity = config, identity
62
+ self.credential_store = UnavailableCredentialStore() if config.local_mode else SDKCredentialStore()
63
+ self.candidate = None
64
+ self.owner = None
65
+ self.reader = None
66
+ self.engine = None
67
+ self.backend = None
68
+ self.factory = None
69
+ self.resources = ExitStack()
70
+ self.lock = Lock()
71
+ self.status = "unconfigured"
72
+ self.error = None
73
+ self.current = []
74
+ self.expected = list(ScriptDirectory.from_config(migration_config()).get_heads())
75
+ self.workspace = identity.workspace if identity else None
76
+ self.active = False
77
+ self.uid = None
78
+ self.selected_source_uid = None
79
+ self.saved_path = (config.configuration_path or Path.cwd() / "configuration.yaml").resolve().parent / ".local/runtime-data-sources.json"
80
+ self.selection_key = str(self.workspace.uid) if self.workspace else "hosted"
81
+
82
+ def descriptor(self, user_uid, *, is_admin=False):
83
+ can_configure = is_admin
84
+ return dict(status=self.status, error=self.error, candidate=self.candidate if can_configure else None,
85
+ can_configure=can_configure, selected_source_uid=str(self.uid) if self.uid else self.selected_source_uid,
86
+ **self.migration_descriptor(),
87
+ active=self.active)
88
+
89
+ def migration_descriptor(self):
90
+ """Compare live Alembic heads with the files without applying migrations."""
91
+ result = dict(current_revisions=[], required_revisions=[], pending_revisions=[],
92
+ migration_status="unavailable", migration_error=None)
93
+ try:
94
+ script = ScriptDirectory.from_config(migration_config())
95
+ result["required_revisions"] = sorted(script.get_heads())
96
+ if not result["required_revisions"]:
97
+ raise ValueError("Migration files have no head revision")
98
+ except Exception:
99
+ result["migration_error"] = "Cannot read the API's migration files. Check the migration directory and refresh."
100
+ return result
101
+ if not self.candidate:
102
+ result["migration_status"] = "unconfigured"
103
+ return result
104
+ if self.status == "migrating":
105
+ result.update(current_revisions=self.current, migration_status="migrating")
106
+ return result
107
+ try:
108
+ engine = self.reader if self.reader is not None else self.open_engine()
109
+ if engine is not None:
110
+ with engine.connect() as connection:
111
+ result["current_revisions"] = sorted(MigrationContext.configure(
112
+ connection, opts={"version_table": VERSION_TABLE}).get_current_heads())
113
+ except Exception:
114
+ result["migration_error"] = "Cannot read the database's applied migrations. Check its connection and refresh."
115
+ return result
116
+ try:
117
+ current = result["current_revisions"]
118
+ known = {revision.revision for revision in script.walk_revisions()}
119
+ if not set(current).issubset(known):
120
+ raise ValueError("Unknown database revision")
121
+ pending = list(script.iterate_revisions("heads", tuple(current)))
122
+ result["pending_revisions"] = [revision.revision for revision in reversed(pending)]
123
+ if not pending and set(current) != set(result["required_revisions"]):
124
+ raise ValueError("Database heads do not match the migration files")
125
+ result["migration_status"] = "pending" if pending else "up_to_date"
126
+ except Exception:
127
+ result.update(migration_status="incompatible",
128
+ migration_error="The database revision is not compatible with these migration files. Review the migration history before applying changes.")
129
+ return result
130
+
131
+ def initial_candidate(self):
132
+ if self.saved_path.exists():
133
+ try:
134
+ saved = json.loads(self.saved_path.read_text())
135
+ if self.selection_key in saved:
136
+ if self.selection_key == "hosted":
137
+ self.selected_source_uid = saved.get("hosted_source_uid")
138
+ return saved[self.selection_key]
139
+ except (OSError, ValueError):
140
+ self.error = "Saved runtime selection cannot be read. Configure the DataSource again."
141
+ if self.workspace:
142
+ return dict(display_name=self.workspace.data_source.display_name, class_type="sqlite",
143
+ configuration={"path": str(self.workspace.tables_path)})
144
+ return None
145
+
146
+ def close(self):
147
+ if self.engine is not None:
148
+ self.engine.dispose()
149
+ if self.reader is not None:
150
+ self.reader.dispose()
151
+ self.reader = None
152
+ self.resources.close()
153
+
154
+ def configure(self, body, owner):
155
+ # Validate public configuration before replacing the current candidate.
156
+ values = RuntimeSourceInput.model_validate(body).model_dump(mode="json")
157
+ kind = values["class_type"]
158
+ if self.config.local_mode:
159
+ if kind != "sqlite" or set(values["configuration"]) != {"path"}:
160
+ raise BootstrapError("Local runtime requires one SQLite file.")
161
+ path = Path(values["configuration"]["path"]).expanduser()
162
+ if not path.is_absolute() or path.name == ":memory:" or path.is_dir():
163
+ raise BootstrapError("Choose an absolute SQLite file path.")
164
+ path = path.resolve()
165
+ marker = _read_marker(path.with_name(path.name + ".metatables-scope.json"))
166
+ if marker and marker != {"workspace_uid": str(self.workspace.uid), "purpose": "runtime"}:
167
+ raise BootstrapError("This file belongs to another workspace or the retired two-file layout.")
168
+ self.workspace = replace(self.workspace, database_path=path)
169
+ values["configuration"] = {"path": str(path)}
170
+ else:
171
+ from api.backend.backends.registry import engine_descriptor
172
+ descriptor = engine_descriptor(kind)
173
+ if kind == 'sqlite':
174
+ raise BootstrapError('This engine does not support hosted runtime workflows.')
175
+ parsed = parse_configuration(kind, values['configuration'])
176
+ descriptor.catalog.validate_configuration(parsed)
177
+ values["configuration"] = parsed.model_dump(mode="json")
178
+ self.close()
179
+ self.resources = ExitStack()
180
+ self.engine = self.factory = self.backend = None
181
+ self.candidate, self.owner = values, owner
182
+ self.active, self.uid = False, None
183
+ self.status, self.error, self.current = "unconfigured", None, []
184
+ self.inspect()
185
+
186
+ def open_engine(self, *, create=False):
187
+ from api.backend.backends.registry import backend_for_engine
188
+ if self.backend is None:
189
+ self.backend = backend_for_engine(self.candidate, self.credential_store, workspace=self.workspace)
190
+ if self.engine is not None:
191
+ return self.engine
192
+ self.engine = self.backend.catalog.open_engine(self.backend.source, self.resources,
193
+ create=create, workspace=self.workspace)
194
+ if self.engine is not None:
195
+ self.engine.update_execution_options(metatables_binding={key: self.candidate[key] for key in ("class_type", "configuration")})
196
+ self.factory = sessionmaker(self.engine, expire_on_commit=False)
197
+ return self.engine
198
+
199
+ def inspect(self):
200
+ self.error = None
201
+ try:
202
+ script = ScriptDirectory.from_config(migration_config())
203
+ self.expected = list(script.get_heads())
204
+ self.current = []
205
+ engine = self.open_engine()
206
+ if engine is None:
207
+ self.status = "migration_required"
208
+ return
209
+ with engine.connect() as connection:
210
+ catalog = inspect(connection)
211
+ tables = set(catalog.get_table_names())
212
+ self.current = list(MigrationContext.configure(connection, opts={"version_table": VERSION_TABLE}).get_current_heads())
213
+ from api.backend.operations.bootstrap_migrations import validate_progress
214
+ progress = validate_progress(connection, engine)
215
+ if VERSION_TABLE not in tables and tables.intersection(Base.metadata.tables) and progress is None:
216
+ self.status = "incompatible"
217
+ self.error = "Existing system tables have no current MetaTables migration history. Select a fresh development database in Settings."
218
+ return
219
+ try:
220
+ for revision in self.current:
221
+ if script.get_revision(revision) is None:
222
+ raise ValueError("Unknown revision")
223
+ except Exception:
224
+ self.status, self.error = "incompatible", "The database has an unsupported MetaTables migration revision. Earlier development databases require recreation."
225
+ return
226
+ if set(self.current) != set(self.expected):
227
+ self.status = "migration_required"
228
+ return
229
+ from api.backend.operations.catalog_schema import validate_system_schema
230
+ validate_system_schema(connection, self.backend.catalog)
231
+ with self.factory() as session:
232
+ row = session.scalar(select(DataSource).where(DataSource.is_runtime.__eq__(True)))
233
+ if row is None:
234
+ self.status = "registration_required"
235
+ elif not self.matches(row):
236
+ self.status = "incompatible"
237
+ self.error = "The stored runtime DataSource does not match this connection."
238
+ else:
239
+ binding = RuntimeStorageBinding(str(engine.url), self.workspace, row.uid, self.candidate["configuration"])
240
+ for registered in session.scalars(select(DataSource)):
241
+ binding.validate_source(registered)
242
+ for model in (MetaTable, PhysicalOperation):
243
+ query = select(model.uid).where(model.data_source_uid != row.uid)
244
+ if model is MetaTable:
245
+ query = query.where(MetaTable.management_mode != "external_registered")
246
+ if session.scalar(query.limit(1)):
247
+ raise BootstrapError("Managed runtime catalog references another DataSource.")
248
+ if session.scalar(select(MetaTable.uid).where(~MetaTable.data_source_uid.in_(select(DataSource.uid))).limit(1)):
249
+ raise BootstrapError("MetaTable references an unregistered DataSource.")
250
+ self.uid = row.uid
251
+ self.status = "ready"
252
+ except (BootstrapError, CatalogConfigurationError) as exc:
253
+ self.status, self.error = "incompatible", str(exc)
254
+ except CredentialStoreError as exc:
255
+ self.status, self.error = "unavailable", exc.message
256
+ except Exception:
257
+ self.status = "unavailable"
258
+ self.error = "Cannot verify the selected database or its Alembic history. Check configuration, Secret access and the API's migration version."
259
+
260
+ def matches(self, row):
261
+ return (row.class_type == self.candidate["class_type"]
262
+ and row.configuration == self.candidate["configuration"]
263
+ and row.is_default and row.storage_access_mode == "read_write" and row.status == "AVAILABLE")
264
+
265
+ @contextmanager
266
+ def migration_lock(self):
267
+ engine = self.open_engine(create=True)
268
+ with self.backend.catalog.bootstrap_lock(engine):
269
+ yield
270
+
271
+ def migrate(self, user_uid):
272
+ if not self.candidate:
273
+ raise BootstrapError("Configure a runtime DataSource first.")
274
+ try:
275
+ # Check before creating a file or scope marker: an earlier local
276
+ # store must remain untouched even when this target is absent.
277
+ self.inspect()
278
+ if self.status not in {"migration_required", "registration_required", "ready"}:
279
+ raise BootstrapError(self.error or "The selected database cannot be migrated.")
280
+ with self.migration_lock():
281
+ self.inspect()
282
+ if self.status not in {"migration_required", "registration_required", "ready"}:
283
+ raise BootstrapError(self.error or "The selected database cannot be migrated.")
284
+ self.status = "migrating"
285
+ upgrade_catalog(self.engine)
286
+ # The registry is used only after Alembic created/upgraded it.
287
+ with self.factory() as session:
288
+ row = session.scalar(select(DataSource).where(DataSource.is_runtime.__eq__(True)))
289
+ if row is None:
290
+ if session.scalar(select(DataSource.uid).where(DataSource.is_default.__eq__(True))):
291
+ raise BootstrapError("An existing default requires explicit reconciliation before runtime registration.")
292
+ row = DataSource(uid=self.workspace.data_source.uid if self.workspace else
293
+ UUID(self.selected_source_uid) if self.selected_source_uid else uuid4(),
294
+ created_by_user_uid=user_uid, **self.candidate, is_runtime=True,
295
+ is_default=True, status="AVAILABLE", storage_access_mode="read_write")
296
+ session.add(row)
297
+ elif not self.matches(row):
298
+ raise BootstrapError("The existing runtime registration cannot be replaced by this configuration or caller.")
299
+ from api.backend.backends.registry import backend_for
300
+ session.flush()
301
+ if self.workspace:
302
+ # Existing local candidate metadata is materialized into
303
+ # the catalog, retaining UIDs. Credentials require the
304
+ # explicit import or password replacement procedure.
305
+ from api.app.source_registrations import read_registry
306
+ saved = read_registry(self.saved_path.with_name("runtime-source-candidates.json"))
307
+ for candidate in saved.values():
308
+ if candidate.get("class_type") not in {"postgresql", "timescale_db", "mysql", "mssql"}:
309
+ continue
310
+ candidate_uid = UUID(candidate["uid"])
311
+ if session.get(DataSource, candidate_uid) is None:
312
+ session.add(DataSource(uid=candidate_uid, created_by_user_uid=user_uid,
313
+ display_name=candidate["display_name"], class_type=candidate["class_type"],
314
+ configuration=candidate["configuration"], storage_access_mode=candidate["storage_access_mode"],
315
+ status=candidate["status"], is_default=False, is_runtime=False))
316
+ backend_for(replace(self.backend.source, uid=row.uid)).access.initialize(session)
317
+ credentials = bind_store(self.credential_store, session, user_uid)
318
+ if hasattr(credentials, "initialize"):
319
+ credentials.initialize()
320
+ session.commit()
321
+ self.inspect()
322
+ if self.status != "ready":
323
+ raise BootstrapError(self.error or "Migration finished but the runtime is not ready.")
324
+ except BootstrapError:
325
+ self.inspect()
326
+ raise
327
+ except Exception as exc:
328
+ self.inspect()
329
+ if isinstance(exc, CredentialStoreError):
330
+ self.error = exc.message
331
+ raise BootstrapError(exc.message) from None
332
+ from api.backend.backends.contracts import SQLExecutionError
333
+ if isinstance(exc, SQLExecutionError):
334
+ self.error = f"Database access setup failed ({exc.code}). Check the DataSource password Secret and database permissions, then retry setup."
335
+ raise BootstrapError(self.error) from None
336
+ self.error = "MetaTables migration or registration failed. The runtime was not activated; inspect the database before retrying."
337
+ raise BootstrapError(self.error) from None
338
+
339
+ def persist_selection(self):
340
+ from api.app.runtime_control import write_state
341
+ self.saved_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
342
+ # Public configuration only: password and TLS material remain Secret references.
343
+ saved = json.loads(self.saved_path.read_text()) if self.saved_path.exists() else {}
344
+ saved[self.selection_key] = self.candidate
345
+ if self.selection_key == "hosted":
346
+ if self.selected_source_uid:
347
+ saved["hosted_source_uid"] = self.selected_source_uid
348
+ else:
349
+ saved.pop("hosted_source_uid", None)
350
+ write_state(self.saved_path, saved)
351
+
352
+ def destroy_local(self, request, path, user_uid):
353
+ import fcntl
354
+
355
+ from api.app.runtime_control import write_state
356
+ from api.backend.integrations.local_workspace import local_store_files
357
+
358
+ if not self.config.local_mode or self.workspace is None:
359
+ raise BootstrapError("Database destruction is available only in Local mode.")
360
+ if not self.candidate or path != self.candidate["configuration"]["path"]:
361
+ raise BootstrapError("The selected local database changed. Reload Settings before destruction.")
362
+ try:
363
+ with ExitStack() as stack:
364
+ target = self.workspace.database_path
365
+ if target.parent.exists():
366
+ descriptor = os.open(str(target) + ".bootstrap.lock",
367
+ os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
368
+ stream = stack.enter_context(os.fdopen(descriptor, "a"))
369
+ try:
370
+ fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
371
+ except BlockingIOError:
372
+ raise BootstrapError("Another local initialization is in progress. Retry when it finishes.") from None
373
+ files = local_store_files(self.workspace)
374
+ saved = json.loads(self.saved_path.read_text()) if self.saved_path.exists() else {}
375
+ if self.selection_key in saved:
376
+ del saved[self.selection_key]
377
+ write_state(self.saved_path, saved)
378
+ # Admission is drained before closing pools or deleting files.
379
+ self.close()
380
+ self.resources = ExitStack()
381
+ self.engine = self.factory = None
382
+ self.active, self.uid, self.owner = False, None, user_uid
383
+ self.current, self.error = [], None
384
+ self.status = "migration_required"
385
+ request.app.state.runtime = replace(request.app.state.runtime,
386
+ data_sources=None, binding=RuntimeStorageBinding())
387
+ request.app.state.runtime_control.instance_id = str(uuid4())
388
+ from metatables.local_run_logs import destroy_store
389
+ destroy_store(target, str(self.workspace.uid))
390
+ for file in files:
391
+ file.unlink(missing_ok=True)
392
+ self.inspect()
393
+ except BootstrapError:
394
+ raise
395
+ except RuntimeError as exc:
396
+ raise BootstrapError(str(exc)) from None
397
+ except Exception:
398
+ if not self.active:
399
+ self.status = "unavailable"
400
+ self.error = "Unable to finish local database destruction. Check file access and retry."
401
+ raise BootstrapError("Unable to finish local database destruction. Check file access and retry.") from None
402
+
403
+ def activate(self, request, user_uid=None, *, persist=True):
404
+ self.inspect()
405
+ if self.status != "ready":
406
+ raise BootstrapError(self.error or "Run MetaTables migrations before selecting this DataSource.")
407
+ with self.factory() as session:
408
+ row = session.get(DataSource, self.uid)
409
+ self.owner = row.created_by_user_uid
410
+ if not self.workspace:
411
+ from api.app.source_registrations import reconcile_runtime
412
+ reconcile_runtime(self, row, persist=persist)
413
+ if self.workspace:
414
+ _bind_file(Path(self.candidate["configuration"]["path"]),
415
+ workspace_uid=self.workspace.uid, purpose="runtime")
416
+ if persist:
417
+ try:
418
+ self.persist_selection()
419
+ except Exception:
420
+ raise BootstrapError("Unable to save runtime selection. The database is initialized but was not activated.") from None
421
+ binding = RuntimeStorageBinding(catalog_url=str(self.engine.url), workspace=self.workspace,
422
+ source_uid=self.uid, source_configuration=self.candidate["configuration"])
423
+ from sqlalchemy import create_engine
424
+
425
+ from api.backend.integrations.catalog_data_sources import CatalogDataSourceProvider
426
+ if self.reader is not None:
427
+ self.reader.dispose()
428
+ self.reader = self.backend.catalog.open_reader(self.backend.source, self.resources)
429
+ state = request.app.state.runtime
430
+ identity = replace(state.identity, workspace=self.workspace) if self.workspace else state.identity
431
+ request.app.state.runtime = replace(state, identity=identity, binding=binding,
432
+ data_sources=CatalogDataSourceProvider(sessionmaker(self.reader, expire_on_commit=False), self.credential_store, binding=binding),
433
+ credential_store=self.credential_store)
434
+ self.active = True
435
+ request.app.state.runtime_control.instance_id = str(uuid4())
436
+
437
+
438
+ router = APIRouter(prefix="/runtime-bootstrap", tags=["Runtime bootstrap"])
439
+
440
+
441
+ def manager(request):
442
+ value = request.app.state.runtime.bootstrap
443
+ if value is None:
444
+ raise HTTPException(503, "Runtime bootstrap is unavailable.")
445
+ return value
446
+
447
+
448
+ @contextmanager
449
+ def mutation(request, caller):
450
+ value = manager(request)
451
+ if not value.lock.acquire(blocking=False):
452
+ raise HTTPException(409, "Runtime configuration is already in progress.")
453
+ try:
454
+ control = request.app.state.runtime_control
455
+ control.begin()
456
+ try:
457
+ if value.active:
458
+ from api.app.runtime_control import unfinished_work
459
+ with value.factory() as session:
460
+ if unfinished_work(session):
461
+ raise HTTPException(409, "Finish updates, migrations and physical operations before reconfiguring the runtime.")
462
+ yield value
463
+ finally:
464
+ control.cancel()
465
+ except BootstrapError as exc:
466
+ raise HTTPException(409, str(exc)) from None
467
+ finally:
468
+ value.lock.release()
469
+
470
+
471
+ class RuntimeSourceSelection(BaseModel):
472
+ model_config = ConfigDict(extra="forbid")
473
+ source_uid: UUID
474
+
475
+
476
+ def registered_hosted_source(request, uid):
477
+ from api.app.source_registrations import get_row, registry_path, remember
478
+ state = request.app.state.runtime
479
+ row = None
480
+ if state.bootstrap.active:
481
+ with state.bootstrap.factory() as session:
482
+ registered = session.get(DataSource, uid)
483
+ if registered is not None:
484
+ if registered.class_type == "sqlite":
485
+ raise HTTPException(409, "Select an enabled hosted database DataSource.")
486
+ remember(registered, request)
487
+ row = dict(display_name=registered.display_name, class_type=registered.class_type,
488
+ configuration=registered.configuration, storage_access_mode=registered.storage_access_mode)
489
+ if row is None:
490
+ row = get_row(registry_path(request), uid)
491
+ from api.backend.backends.registry import engine_descriptor
492
+ engine_descriptor(row["class_type"])
493
+ if row["class_type"] == "sqlite" or row["storage_access_mode"] != "read_write":
494
+ raise HTTPException(409, "Select an enabled hosted database DataSource.")
495
+ return {key: row[key] for key in ("display_name", "class_type", "configuration")}
496
+
497
+
498
+ def persist_registered_selection(candidate):
499
+ if candidate.uid:
500
+ from api.app.source_registrations import reconcile_runtime
501
+ with candidate.factory() as session:
502
+ reconcile_runtime(candidate, session.get(DataSource, candidate.uid))
503
+ else:
504
+ candidate.persist_selection()
505
+
506
+
507
+ @router.post("/hosted/select/", response_model=RuntimeBootstrapDescriptor)
508
+ def select_hosted_before_switch(body: RuntimeSourceSelection, request: Request,
509
+ caller: AdmittedCaller = Depends(require_admin)):
510
+ """Select a registered hosted DataSource without changing the active local runtime."""
511
+ state = request.app.state.runtime
512
+ current = state.hosted_bootstrap
513
+ if not state.config.local_mode or current is None:
514
+ raise HTTPException(403, "Hosted DataSource preparation is available only in Local mode.")
515
+ if not current.lock.acquire(blocking=False):
516
+ raise HTTPException(409, "Hosted DataSource configuration is already in progress.")
517
+ try:
518
+ if request.app.state.runtime.hosted_bootstrap is not current:
519
+ raise HTTPException(409, "Hosted DataSource configuration changed. Refresh Settings.")
520
+ source = registered_hosted_source(request, body.source_uid)
521
+ candidate = RuntimeBootstrap(replace(state.config, local_mode=False),
522
+ replace(state.identity, workspace=None))
523
+ try:
524
+ candidate.selected_source_uid = str(body.source_uid)
525
+ candidate.configure(source, caller.user_uid)
526
+ if candidate.status == "unavailable":
527
+ raise BootstrapError(candidate.error or "Cannot verify the hosted DataSource connection.")
528
+ persist_registered_selection(candidate)
529
+ request.app.state.runtime = replace(request.app.state.runtime, hosted_bootstrap=candidate)
530
+ current.close()
531
+ return candidate.descriptor(caller.user_uid, is_admin=caller.is_admin)
532
+ except BootstrapError as exc:
533
+ candidate.close()
534
+ raise HTTPException(409, str(exc)) from None
535
+ except Exception:
536
+ candidate.close()
537
+ raise HTTPException(503, "Unable to save the hosted DataSource configuration.") from None
538
+ finally:
539
+ current.lock.release()
540
+
541
+
542
+ @router.post("/select/", response_model=RuntimeBootstrapDescriptor)
543
+ def select_hosted_runtime_source(body: RuntimeSourceSelection, request: Request,
544
+ caller: AdmittedCaller = Depends(require_admin)):
545
+ if request.app.state.runtime.config.local_mode:
546
+ raise HTTPException(409, "Use hosted selection while Local mode is active.")
547
+ source = registered_hosted_source(request, body.source_uid)
548
+ with mutation(request, caller) as current:
549
+ state = request.app.state.runtime
550
+ candidate = RuntimeBootstrap(state.config, state.identity)
551
+ try:
552
+ candidate.selected_source_uid = str(body.source_uid)
553
+ candidate.configure(source, caller.user_uid)
554
+ if candidate.status == "unavailable":
555
+ raise BootstrapError(candidate.error or "Cannot verify the DataSource connection.")
556
+ persist_registered_selection(candidate)
557
+ request.app.state.runtime = replace(state, bootstrap=candidate,
558
+ data_sources=None, binding=RuntimeStorageBinding())
559
+ request.app.state.runtime_control.instance_id = str(uuid4())
560
+ current.close()
561
+ return candidate.descriptor(caller.user_uid, is_admin=caller.is_admin)
562
+ except BootstrapError as exc:
563
+ candidate.close()
564
+ raise HTTPException(409, str(exc)) from None
565
+ except Exception:
566
+ candidate.close()
567
+ raise HTTPException(503, "Unable to select the runtime DataSource.") from None
568
+
569
+
570
+ @router.post("/configure/", response_model=RuntimeBootstrapDescriptor)
571
+ def configure(body: RuntimeSourceInput, request: Request, caller: AdmittedCaller = Depends(require_admin)):
572
+ with mutation(request, caller) as value:
573
+ try:
574
+ value.configure(body, caller.user_uid)
575
+ except BootstrapError:
576
+ raise
577
+ except Exception:
578
+ raise BootstrapError("Invalid runtime DataSource configuration.") from None
579
+ request.app.state.runtime = replace(request.app.state.runtime, data_sources=None, binding=RuntimeStorageBinding())
580
+ request.app.state.runtime_control.instance_id = str(uuid4())
581
+ return value.descriptor(caller.user_uid, is_admin=caller.is_admin)
582
+
583
+
584
+ @router.post("/migrate/", response_model=RuntimeBootstrapDescriptor)
585
+ def migrate(request: Request, caller: AdmittedCaller = Depends(require_admin)):
586
+ with mutation(request, caller) as value:
587
+ value.active = False
588
+ request.app.state.runtime = replace(request.app.state.runtime, data_sources=None, binding=RuntimeStorageBinding())
589
+ value.migrate(caller.user_uid)
590
+ value.activate(request, caller.user_uid)
591
+ return value.descriptor(caller.user_uid, is_admin=caller.is_admin)
592
+
593
+
594
+ @router.post("/activate/", response_model=RuntimeBootstrapDescriptor)
595
+ def activate(request: Request, caller: AdmittedCaller = Depends(require_admin)):
596
+ with mutation(request, caller) as value:
597
+ value.activate(request, caller.user_uid)
598
+ return value.descriptor(caller.user_uid, is_admin=caller.is_admin)
599
+
600
+
601
+ @router.post("/destroy-local/", response_model=RuntimeBootstrapDescriptor)
602
+ def destroy_local(body: DestroyLocalInput, request: Request, caller: AdmittedCaller = Depends(require_admin)):
603
+ if not request.app.state.runtime.config.local_mode:
604
+ raise HTTPException(403, "Database destruction is available only in Local mode.")
605
+ with mutation(request, caller) as value:
606
+ value.destroy_local(request, body.path, caller.user_uid)
607
+ return value.descriptor(caller.user_uid, is_admin=caller.is_admin)
@@ -0,0 +1,37 @@
1
+ """Deployment capabilities are code-owned configuration, never environment switches."""
2
+
3
+ from pathlib import Path
4
+ from typing import Literal
5
+
6
+ import yaml
7
+ from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr
8
+
9
+
10
+ class LocalCredentialConfiguration(BaseModel):
11
+ model_config = ConfigDict(extra="forbid", frozen=True)
12
+ provider: Literal["native", "file"] = "native"
13
+ key_directory: Path | None = None
14
+
15
+
16
+ class DeploymentConfiguration(BaseModel):
17
+ model_config = ConfigDict(extra="forbid", frozen=True)
18
+ local_mode_available: StrictBool = False
19
+ application_migration_providers: dict[StrictStr, StrictStr] = Field(default_factory=dict)
20
+ local_credentials: LocalCredentialConfiguration = Field(default_factory=LocalCredentialConfiguration)
21
+
22
+ @classmethod
23
+ def load(cls, path: Path | None = None):
24
+ selected = path or Path.cwd() / "configuration.yaml"
25
+ if not selected.exists():
26
+ if path is not None:
27
+ raise RuntimeError(f"Configuration file does not exist: {selected}")
28
+ return cls()
29
+ try:
30
+ values = yaml.safe_load(selected.read_text(encoding="utf-8"))
31
+ return cls.model_validate(values)
32
+ except (OSError, ValueError, yaml.YAMLError) as exc:
33
+ raise RuntimeError(f"Invalid deployment configuration: {selected}") from exc
34
+
35
+ def require_development(self):
36
+ if not self.local_mode_available:
37
+ raise RuntimeError("Local development is disabled by configuration.yaml.")