mainsequence 4.3.14__tar.gz → 4.3.16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. {mainsequence-4.3.14/mainsequence.egg-info → mainsequence-4.3.16}/PKG-INFO +1 -1
  2. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/metatables/core.py +17 -0
  3. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/data_nodes.py +24 -3
  4. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/persist_managers.py +140 -18
  5. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/run_operations.py +174 -41
  6. {mainsequence-4.3.14 → mainsequence-4.3.16/mainsequence.egg-info}/PKG-INFO +1 -1
  7. {mainsequence-4.3.14 → mainsequence-4.3.16}/pyproject.toml +1 -1
  8. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_meta_tables_sqlalchemy_contracts.py +44 -2
  9. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_update_runner_uid_runtime.py +205 -1
  10. {mainsequence-4.3.14 → mainsequence-4.3.16}/LICENSE +0 -0
  11. {mainsequence-4.3.14 → mainsequence-4.3.16}/README.md +0 -0
  12. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/AGENTS.md +0 -0
  13. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/a2a_communication/SKILL.md +0 -0
  14. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/application_surfaces/api_surfaces/SKILL.md +0 -0
  15. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/command_center/adapter_from_api/SKILL.md +0 -0
  16. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/command_center/api_mock_prototyping/SKILL.md +0 -0
  17. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/command_center/app_components/SKILL.md +0 -0
  18. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/command_center/connections/SKILL.md +0 -0
  19. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/command_center/workspace_analysis/SKILL.md +0 -0
  20. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/command_center/workspace_builder/SKILL.md +0 -0
  21. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/command_center/workspace_design/SKILL.md +0 -0
  22. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/dashboards/streamlit/SKILL.md +0 -0
  23. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/data_access/exploration/SKILL.md +0 -0
  24. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/data_publishing/data_nodes/SKILL.md +0 -0
  25. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/data_publishing/meta_table_migrations/SKILL.md +0 -0
  26. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/data_publishing/meta_tables/SKILL.md +0 -0
  27. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/maintenance/bug_auditor/SKILL.md +0 -0
  28. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/ms-markets/SKILL.md +0 -0
  29. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/platform_operations/access_control_and_sharing/SKILL.md +0 -0
  30. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/platform_operations/orchestration_and_releases/SKILL.md +0 -0
  31. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/project_builder/SKILL.md +0 -0
  32. {mainsequence-4.3.14 → mainsequence-4.3.16}/agent_scaffold/skills/project_to_agent/SKILL.md +0 -0
  33. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/__init__.py +0 -0
  34. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/__main__.py +0 -0
  35. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/bootstrap.py +0 -0
  36. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/__init__.py +0 -0
  37. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/api.py +0 -0
  38. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/browser_auth.py +0 -0
  39. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/cli.py +0 -0
  40. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/config.py +0 -0
  41. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/docker_utils.py +0 -0
  42. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/doctor.py +0 -0
  43. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/local_ops.py +0 -0
  44. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/migrations.py +0 -0
  45. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/model_filters.py +0 -0
  46. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/project_status.py +0 -0
  47. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/pydantic_cli.py +0 -0
  48. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/sdk_utils.py +0 -0
  49. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/ssh_utils.py +0 -0
  50. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/cli/ui.py +0 -0
  51. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/__init__.py +0 -0
  52. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/agent_runtime_models.py +0 -0
  53. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/base.py +0 -0
  54. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/client.py +0 -0
  55. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/command_center/__init__.py +0 -0
  56. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/command_center/app_component.py +0 -0
  57. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/command_center/connections.py +0 -0
  58. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/command_center/data_models.py +0 -0
  59. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/command_center/workspace.py +0 -0
  60. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/command_center/workspace_snapshot.py +0 -0
  61. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/compute_validation.py +0 -0
  62. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/data_sources_interfaces/__init__.py +0 -0
  63. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/data_sources_interfaces/duckdb.py +0 -0
  64. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/data_sources_interfaces/local_paths.py +0 -0
  65. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/data_sources_interfaces/sqlite.py +0 -0
  66. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/dtype_codec.py +0 -0
  67. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/exceptions.py +0 -0
  68. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/fastapi/__init__.py +0 -0
  69. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/fastapi/auth.py +0 -0
  70. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/metatables/__init__.py +0 -0
  71. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/models_foundry.py +0 -0
  72. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/models_helpers.py +0 -0
  73. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/models_user.py +0 -0
  74. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/client/utils.py +0 -0
  75. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/defaults.py +0 -0
  76. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/instrumentation/__init__.py +0 -0
  77. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/instrumentation/utils.py +0 -0
  78. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/logconf.py +0 -0
  79. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/__init__.py +0 -0
  80. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/__main__.py +0 -0
  81. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/compiled_sql/__init__.py +0 -0
  82. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/compiled_sql/v1.py +0 -0
  83. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/__init__.py +0 -0
  84. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/build_operations.py +0 -0
  85. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/models.py +0 -0
  86. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/namespacing.py +0 -0
  87. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/data_nodes/utils.py +0 -0
  88. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/future_registry.py +0 -0
  89. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/hashing.py +0 -0
  90. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/__init__.py +0 -0
  91. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/alembic.py +0 -0
  92. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/env.py +0 -0
  93. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/provider.py +0 -0
  94. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/registry.py +0 -0
  95. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/scaffold.py +0 -0
  96. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/templates/__init__.py +0 -0
  97. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/templates/env.py.mako +0 -0
  98. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/migrations/templates/script.py.mako +0 -0
  99. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/pydantic_metadata.py +0 -0
  100. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/schema_names.py +0 -0
  101. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/meta_tables/sqlalchemy_contracts.py +0 -0
  102. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence/runtime_flags.py +0 -0
  103. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence.egg-info/SOURCES.txt +0 -0
  104. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence.egg-info/dependency_links.txt +0 -0
  105. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence.egg-info/entry_points.txt +0 -0
  106. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence.egg-info/requires.txt +0 -0
  107. {mainsequence-4.3.14 → mainsequence-4.3.16}/mainsequence.egg-info/top_level.txt +0 -0
  108. {mainsequence-4.3.14 → mainsequence-4.3.16}/setup.cfg +0 -0
  109. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_auth_precedence.py +0 -0
  110. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_build_operations_hashing.py +0 -0
  111. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_cli.py +0 -0
  112. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_cli_browser_auth.py +0 -0
  113. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_cli_migrations.py +0 -0
  114. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_client.py +0 -0
  115. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_command_center_app_component_models.py +0 -0
  116. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_command_center_data_models.py +0 -0
  117. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_command_center_models.py +0 -0
  118. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_data_access_mixin_dimension_audit.py +0 -0
  119. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_data_node_storage_dimension_queries.py +0 -0
  120. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_data_node_update_flow.py +0 -0
  121. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_dependency_extras.py +0 -0
  122. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_duckdb_interface_dimensions.py +0 -0
  123. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_filter_normalization.py +0 -0
  124. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_instrumentation.py +0 -0
  125. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_logconf.py +0 -0
  126. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_meta_table_migrations.py +0 -0
  127. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_meta_tables_client_models.py +0 -0
  128. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_models_user_request_bound_auth.py +0 -0
  129. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_pod_project_resolution.py +0 -0
  130. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_project_batch_jobs_from_file.py +0 -0
  131. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_run_configuration.py +0 -0
  132. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_schema_names.py +0 -0
  133. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_secret_client_model.py +0 -0
  134. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_source_table_configuration.py +0 -0
  135. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_sqlite_interface_dimensions.py +0 -0
  136. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_update_statistics.py +0 -0
  137. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_update_uid_guards.py +0 -0
  138. {mainsequence-4.3.14 → mainsequence-4.3.16}/tests/test_workspace_snapshot.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mainsequence
3
- Version: 4.3.14
3
+ Version: 4.3.16
4
4
  Summary: Main Sequence SDK
5
5
  Author-email: Main Sequence GmbH <dev@main-sequence.io>
6
6
  License: MainSequence GmbH SDK License Agreement
@@ -2375,6 +2375,23 @@ class DataNodeUpdate(TableUpdateNode, BaseObjectOrm):
2375
2375
 
2376
2376
  return depth_df
2377
2377
 
2378
+ def clear_dependencies(self, timeout=None) -> dict[str, Any] | None:
2379
+ url = self.get_object_url() + f"/{self._public_uid()}/clear-dependencies/"
2380
+ payload = {"json": {}}
2381
+ r = make_request(
2382
+ s=self.build_session(),
2383
+ loaders=self.LOADERS,
2384
+ r_type="POST",
2385
+ url=url,
2386
+ payload=payload,
2387
+ time_out=timeout,
2388
+ )
2389
+ if r.status_code not in (200, 204):
2390
+ raise_for_response(r, payload=payload)
2391
+ if not r.content:
2392
+ return None
2393
+ return r.json()
2394
+
2378
2395
  @classmethod
2379
2396
  def get_upstream_nodes(cls, storage_hash, data_source_uid, timeout=None):
2380
2397
  s = cls.build_session()
@@ -807,12 +807,29 @@ class DataNode(DataAccessMixin, ABC):
807
807
  local_configuration=self.local_initial_configuration,
808
808
  )
809
809
 
810
- def set_relation_tree(self):
810
+ def set_relation_tree(
811
+ self,
812
+ *,
813
+ force_rebuild: bool = False,
814
+ _visited_update_uids: set[str] | None = None,
815
+ ):
811
816
  """Sets the node relationships in the backend by calling the dependencies() method."""
812
817
 
813
818
  if self.local_persist_manager.data_node_update is None:
814
819
  self.verify_and_build_remote_objects() #
815
- if self.local_persist_manager.is_local_relation_tree_set():
820
+ data_node_update_uid = str(getattr(self.local_persist_manager.data_node_update, "uid", ""))
821
+ if _visited_update_uids is None:
822
+ _visited_update_uids = set()
823
+ if data_node_update_uid:
824
+ if data_node_update_uid in _visited_update_uids:
825
+ return
826
+ _visited_update_uids.add(data_node_update_uid)
827
+
828
+ if force_rebuild:
829
+ self.local_persist_manager.clear_dependencies()
830
+ self.depth_df = pd.DataFrame()
831
+ self.dependencies_df = None
832
+ elif self.local_persist_manager.is_local_relation_tree_set():
816
833
  return
817
834
  declared_dependencies = self.dependencies() or {}
818
835
 
@@ -827,7 +844,11 @@ class DataNode(DataAccessMixin, ABC):
827
844
  self.local_persist_manager.depends_on_connect(dependency_ts, is_api=is_api)
828
845
 
829
846
  # Recursively set the relation tree for the dependency
830
- dependency_ts.set_relation_tree()
847
+ if not is_api:
848
+ dependency_ts.set_relation_tree(
849
+ force_rebuild=force_rebuild,
850
+ _visited_update_uids=_visited_update_uids,
851
+ )
831
852
 
832
853
  self.local_persist_manager.set_ogm_dependencies_linked()
833
854
 
@@ -4,6 +4,7 @@ import hashlib
4
4
  import inspect
5
5
  import threading
6
6
  from concurrent.futures import Future
7
+ from dataclasses import dataclass
7
8
  from typing import Any, ClassVar
8
9
 
9
10
  import pandas as pd
@@ -29,6 +30,15 @@ from mainsequence.meta_tables import PlatformTimeIndexMetaTable
29
30
 
30
31
  from .. import future_registry
31
32
 
33
+ _STORAGE_TABLE_LOOKUP_LIMIT = 20
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class _StorageTableLookupResult:
38
+ table_name: str | None
39
+ filters: dict[str, Any]
40
+ matches: list[TimeIndexMetaTable]
41
+
32
42
 
33
43
  def get_data_node_source_code(DataNodeClass: type[Any]) -> str:
34
44
  """
@@ -96,17 +106,18 @@ def ensure_registered_storage_table(
96
106
  f"model class; got {type(storage_table).__name__}."
97
107
  )
98
108
 
109
+ lookup_result: _StorageTableLookupResult | None = None
99
110
  if storage_table.get_time_index_meta_table() is None:
100
- _bind_registered_storage_table(storage_table)
111
+ lookup_result = _bind_registered_storage_table(storage_table)
101
112
 
102
113
  storage_metadata = storage_table.get_time_index_meta_table()
103
114
  if storage_metadata is None:
104
115
  raise ValueError(
105
- f"{context} storage_table class is not bound to backend "
106
- "TimeIndexMetaTable "
107
- "metadata in this Python process. The backend table may already exist; "
108
- "the SDK could not resolve a unique TimeIndexMetaTable row for "
109
- f"{_storage_table_lookup_label(storage_table)}."
116
+ _unbound_storage_table_message(
117
+ storage_table,
118
+ context=context,
119
+ lookup_result=lookup_result,
120
+ )
110
121
  )
111
122
  if not isinstance(storage_metadata, TimeIndexMetaTable):
112
123
  raise TypeError(
@@ -120,25 +131,34 @@ def ensure_registered_storage_table(
120
131
  return storage_table
121
132
 
122
133
 
123
- def _bind_registered_storage_table(storage_table: type[PlatformTimeIndexMetaTable]) -> None:
124
- matches = _registered_storage_table_matches(storage_table)
125
- if len(matches) == 1:
126
- storage_table._bind_meta_table(matches[0])
134
+ def _bind_registered_storage_table(
135
+ storage_table: type[PlatformTimeIndexMetaTable],
136
+ ) -> _StorageTableLookupResult:
137
+ lookup_result = _registered_storage_table_lookup(storage_table)
138
+ if len(lookup_result.matches) == 1:
139
+ storage_table._bind_meta_table(lookup_result.matches[0])
140
+ return lookup_result
127
141
 
128
142
 
129
- def _registered_storage_table_matches(
143
+ def _registered_storage_table_lookup(
130
144
  storage_table: type[PlatformTimeIndexMetaTable],
131
- ) -> list[TimeIndexMetaTable]:
145
+ ) -> _StorageTableLookupResult:
132
146
  table_name = _storage_table_physical_table_name(storage_table)
133
147
  if table_name:
148
+ filters = {
149
+ "physical_table_name__in": [table_name],
150
+ "limit": _STORAGE_TABLE_LOOKUP_LIMIT,
151
+ }
134
152
  matches = TimeIndexMetaTable.filter_by_body(
135
- physical_table_name__in=[table_name],
136
- limit=1,
153
+ **filters,
154
+ )
155
+ return _StorageTableLookupResult(
156
+ table_name=table_name,
157
+ filters=filters,
158
+ matches=matches,
137
159
  )
138
- if matches:
139
- return matches
140
160
 
141
- return []
161
+ return _StorageTableLookupResult(table_name=None, filters={}, matches=[])
142
162
 
143
163
 
144
164
  def _storage_table_physical_table_name(
@@ -159,6 +179,100 @@ def _storage_table_lookup_label(storage_table: type[PlatformTimeIndexMetaTable])
159
179
  return f"{storage_table.__name__}(table={table_name})"
160
180
 
161
181
 
182
+ def _unbound_storage_table_message(
183
+ storage_table: type[PlatformTimeIndexMetaTable],
184
+ *,
185
+ context: str,
186
+ lookup_result: _StorageTableLookupResult | None,
187
+ ) -> str:
188
+ label = _storage_table_lookup_label(storage_table)
189
+ identity = _storage_table_identity_summary(storage_table)
190
+ message = (
191
+ f"{context} storage_table class is not bound to backend TimeIndexMetaTable "
192
+ f"metadata in this Python process for {label}. {identity} "
193
+ "Expected exactly one backend TimeIndexMetaTable catalog row before "
194
+ "constructing a DataNode."
195
+ )
196
+ if lookup_result is None:
197
+ return (
198
+ f"{message} The SDK did not run a backend lookup because the model "
199
+ "already appeared to be bound before validation failed. Check the "
200
+ "bound MetaTable UID and data-source UID on the storage model."
201
+ )
202
+ if lookup_result.table_name is None:
203
+ return (
204
+ f"{message} The SDK could not determine a physical table name from "
205
+ "the storage model, so no backend catalog lookup was possible."
206
+ )
207
+
208
+ filters = _format_lookup_filters(lookup_result.filters)
209
+ match_count = len(lookup_result.matches)
210
+ if match_count == 0:
211
+ return (
212
+ f"{message} Lookup used TimeIndexMetaTable.filter_by_body({filters}) "
213
+ "and found no backend TimeIndexMetaTable catalog row. This usually "
214
+ "means the exact storage model/table has not been reserved and "
215
+ "finalized by its migration provider. If the SQL table already exists "
216
+ "without a TimeIndexMetaTable catalog row, it is still unusable by "
217
+ "DataNodes until the migration finalization step creates that row. "
218
+ "Add this exact storage model to the relevant migration provider "
219
+ "or dynamic scoped provider and run the provider upgrade before "
220
+ "constructing the DataNode."
221
+ )
222
+
223
+ candidates = "; ".join(
224
+ _time_index_meta_table_candidate_summary(match)
225
+ for match in lookup_result.matches[:_STORAGE_TABLE_LOOKUP_LIMIT]
226
+ )
227
+ qualifier = "at least " if match_count >= _STORAGE_TABLE_LOOKUP_LIMIT else ""
228
+ return (
229
+ f"{message} Lookup used TimeIndexMetaTable.filter_by_body({filters}) "
230
+ f"and found {qualifier}{match_count} matching backend TimeIndexMetaTable "
231
+ "catalog rows, so the SDK refused to guess. Matching rows: "
232
+ f"{candidates}. Remove or repair the duplicate catalog state so exactly "
233
+ "one row owns this physical table."
234
+ )
235
+
236
+
237
+ def _storage_table_identity_summary(storage_table: type[PlatformTimeIndexMetaTable]) -> str:
238
+ parts = []
239
+ identifier = getattr(storage_table, "__metatable_identifier__", None)
240
+ if identifier not in (None, ""):
241
+ parts.append(f"identifier={identifier!r}")
242
+ try:
243
+ storage_hash = storage_table.get_storage_hash()
244
+ except Exception:
245
+ storage_hash = None
246
+ if storage_hash not in (None, ""):
247
+ parts.append(f"storage_hash={storage_hash!r}")
248
+ data_source_uid = storage_table.get_data_source_uid()
249
+ if data_source_uid not in (None, ""):
250
+ parts.append(f"model_data_source_uid={data_source_uid!r}")
251
+ if not parts:
252
+ return "Storage identity is incomplete."
253
+ return "Storage identity: " + ", ".join(parts) + "."
254
+
255
+
256
+ def _format_lookup_filters(filters: dict[str, Any]) -> str:
257
+ return ", ".join(f"{key}={value!r}" for key, value in filters.items())
258
+
259
+
260
+ def _time_index_meta_table_candidate_summary(meta_table: TimeIndexMetaTable) -> str:
261
+ parts = []
262
+ for attr in (
263
+ "uid",
264
+ "data_source_uid",
265
+ "identifier",
266
+ "physical_table_name",
267
+ "storage_hash",
268
+ "provisioning_status",
269
+ ):
270
+ value = getattr(meta_table, attr, None)
271
+ if value not in (None, ""):
272
+ parts.append(f"{attr}={value}")
273
+ return "{" + ", ".join(parts) + "}" if parts else "{unidentified TimeIndexMetaTable}"
274
+
275
+
162
276
  class BasePersistManager:
163
277
  UPDATE_CLASS: ClassVar[type[Any] | None] = None
164
278
  UPDATE_DETAILS_CLASS: ClassVar[type[Any] | None] = None
@@ -336,8 +450,16 @@ class BasePersistManager:
336
450
  def get_all_dependencies_update_priority(self) -> pd.DataFrame:
337
451
  return self.data_node_update.get_all_dependencies_update_priority()
338
452
 
453
+ def clear_dependencies(self) -> Any:
454
+ result = self.data_node_update.clear_dependencies()
455
+ self.set_data_node_update_lazy(force_registry=True, include_relations_detail=True)
456
+ return result
457
+
458
+ def set_ogm_dependencies_unlinked(self) -> None:
459
+ self.set_data_node_update(self.data_node_update.patch(ogm_dependencies_linked=False))
460
+
339
461
  def set_ogm_dependencies_linked(self) -> None:
340
- self.data_node_update.patch(ogm_dependencies_linked=True)
462
+ self.set_data_node_update(self.data_node_update.patch(ogm_dependencies_linked=True))
341
463
 
342
464
  @property
343
465
  def update_details(self) -> Any | None:
@@ -307,6 +307,9 @@ class UpdateRunner:
307
307
  if self.ts.dependencies_df is None:
308
308
  self.ts.set_dependencies_df()
309
309
 
310
+ if self.update_tree:
311
+ self._ensure_dependency_tree_matches_current_declarations()
312
+
310
313
  # 2. Connect the dependency tree to the scheduler if it hasn't been already.
311
314
  if not self.ts._scheduler_tree_connected and self.update_tree:
312
315
  self.logger.debug("Connecting dependency tree to scheduler...")
@@ -555,53 +558,15 @@ class UpdateRunner:
555
558
  Backend dependency metadata is ordering/state only; it is not used to
556
559
  cold-rebuild executable DataNode instances.
557
560
  """
558
- # 1. Ensure the dependency graph is built in the backend
559
- declared_dependencies = self.ts.dependencies() or {}
560
- deps_uids = [
561
- (
562
- _require_uid(d.data_node_update, "DataNodeUpdate")
563
- if (not d.is_api and d.data_node_update is not None)
564
- else None
565
- )
566
- for d in declared_dependencies.values()
567
- ]
568
-
569
- # 2. Get the list of dependencies to update
570
- dependencies_df = self.ts.dependencies_df
571
- if (
572
- dependencies_df is not None
573
- and not dependencies_df.empty
574
- and "update_node_uid" not in dependencies_df.columns
575
- ):
576
- raise ValueError("Dependency dataframe must include 'update_node_uid'.")
577
- dependency_uids_in_tree = (
578
- dependencies_df["update_node_uid"].astype(str).to_list()
579
- if dependencies_df is not None and not dependencies_df.empty
580
- else []
561
+ _, dependencies_df, update_map = (
562
+ self._ensure_dependency_tree_matches_current_declarations()
581
563
  )
582
564
 
583
- if any([a is None for a in deps_uids]) or any(
584
- [d not in dependency_uids_in_tree for d in deps_uids]
585
- ):
586
- # Datanode not update set
587
- self.ts.local_persist_manager.data_node_update.patch(ogm_dependencies_linked=False)
588
-
589
- if not self.ts.local_persist_manager.data_node_update.ogm_dependencies_linked:
590
- self.logger.info("Dependency tree not set. Building now...")
591
- start_time = time.time()
592
- self.ts.set_relation_tree()
593
- self.logger.debug(f"Tree build took {time.time() - start_time:.2f}s.")
594
- self.ts.set_dependencies_df()
595
- dependencies_df = self.ts.dependencies_df
596
-
597
565
  if dependencies_df.empty:
598
566
  self.logger.debug("No dependencies to update.")
599
567
  return
600
568
 
601
- # 3. Build the executable dependency map from source declarations.
602
- update_map = self._get_update_map(declared_dependencies, logger=self.logger)
603
-
604
- # 4. Delegate to the appropriate execution method
569
+ # Delegate to the appropriate execution method.
605
570
  self.logger.debug(f"Starting update for {len(dependencies_df)} dependencies...")
606
571
 
607
572
  if self.debug_mode:
@@ -618,6 +583,174 @@ class UpdateRunner:
618
583
 
619
584
  self.logger.debug(f"Dependency tree evaluation complete for {self.ts}.")
620
585
 
586
+ def _ensure_dependency_tree_matches_current_declarations(
587
+ self,
588
+ ) -> tuple[dict[str, DataNode], pd.DataFrame, dict[str, dict[str, Any]]]:
589
+ declared_dependencies = self.ts.dependencies() or {}
590
+
591
+ if self._has_uninitialized_non_api_dependencies(declared_dependencies):
592
+ self.ts.local_persist_manager.set_ogm_dependencies_unlinked()
593
+ dependencies_df = self._rebuild_dependency_tree_from_current_declarations(
594
+ force_rebuild=True,
595
+ reason="Dependency declarations include uninitialized update nodes.",
596
+ )
597
+ declared_dependencies = self.ts.dependencies() or {}
598
+ elif not self.ts.local_persist_manager.data_node_update.ogm_dependencies_linked:
599
+ dependencies_df = self._rebuild_dependency_tree_from_current_declarations(
600
+ force_rebuild=False,
601
+ reason="Dependency tree not set. Building now.",
602
+ )
603
+ declared_dependencies = self.ts.dependencies() or {}
604
+ else:
605
+ dependencies_df = self._dependencies_df_or_empty(self.ts.dependencies_df)
606
+
607
+ update_map = self._get_update_map(declared_dependencies, logger=self.logger)
608
+ mismatch = self._dependency_tree_mismatch(
609
+ dependencies_df=dependencies_df,
610
+ update_map=update_map,
611
+ )
612
+ if self._mismatch_is_empty(mismatch):
613
+ return declared_dependencies, dependencies_df, update_map
614
+
615
+ self._log_dependency_tree_rebuild(mismatch)
616
+ dependencies_df = self._rebuild_dependency_tree_from_current_declarations(
617
+ force_rebuild=True,
618
+ reason="Backend dependency tree drift detected.",
619
+ )
620
+ declared_dependencies = self.ts.dependencies() or {}
621
+ update_map = self._get_update_map(declared_dependencies, logger=self.logger)
622
+ mismatch = self._dependency_tree_mismatch(
623
+ dependencies_df=dependencies_df,
624
+ update_map=update_map,
625
+ )
626
+ if not self._mismatch_is_empty(mismatch):
627
+ raise DependencyUpdateError(
628
+ self._dependency_tree_mismatch_message(
629
+ mismatch,
630
+ after_automatic_refresh=True,
631
+ )
632
+ )
633
+
634
+ return declared_dependencies, dependencies_df, update_map
635
+
636
+ @staticmethod
637
+ def _has_uninitialized_non_api_dependencies(
638
+ declared_dependencies: dict[str, DataNode],
639
+ ) -> bool:
640
+ return any(
641
+ not dependency_ts.is_api
642
+ and getattr(dependency_ts, "data_node_update", None) is None
643
+ for dependency_ts in declared_dependencies.values()
644
+ )
645
+
646
+ def _rebuild_dependency_tree_from_current_declarations(
647
+ self,
648
+ *,
649
+ force_rebuild: bool,
650
+ reason: str,
651
+ ) -> pd.DataFrame:
652
+ self.logger.info(reason)
653
+ start_time = time.time()
654
+ self.ts.set_relation_tree(force_rebuild=force_rebuild)
655
+ self.logger.debug(f"Tree build took {time.time() - start_time:.2f}s.")
656
+ self.ts.set_dependencies_df()
657
+ return self._dependencies_df_or_empty(self.ts.dependencies_df)
658
+
659
+ @staticmethod
660
+ def _dependencies_df_or_empty(dependencies_df: pd.DataFrame | None) -> pd.DataFrame:
661
+ if dependencies_df is None:
662
+ return pd.DataFrame()
663
+ if not dependencies_df.empty and "update_node_uid" not in dependencies_df.columns:
664
+ raise ValueError("Dependency dataframe must include 'update_node_uid'.")
665
+ return dependencies_df
666
+
667
+ @staticmethod
668
+ def _dependency_tree_mismatch(
669
+ *,
670
+ dependencies_df: pd.DataFrame,
671
+ update_map: dict[str, dict[str, Any]],
672
+ ) -> dict[str, list[str]]:
673
+ if not dependencies_df.empty and "update_node_uid" not in dependencies_df.columns:
674
+ raise ValueError("Dependency dataframe must include 'update_node_uid'.")
675
+ backend_uids = (
676
+ set(dependencies_df["update_node_uid"].astype(str).to_list())
677
+ if not dependencies_df.empty
678
+ else set()
679
+ )
680
+ declared_uids = set(update_map)
681
+ return {
682
+ "stale_backend_uids": sorted(backend_uids - declared_uids),
683
+ "missing_backend_uids": sorted(declared_uids - backend_uids),
684
+ }
685
+
686
+ @staticmethod
687
+ def _mismatch_is_empty(mismatch: dict[str, list[str]]) -> bool:
688
+ return not mismatch["stale_backend_uids"] and not mismatch["missing_backend_uids"]
689
+
690
+ def _log_dependency_tree_rebuild(self, mismatch: dict[str, list[str]]) -> None:
691
+ message = self._dependency_tree_mismatch_message(
692
+ mismatch,
693
+ after_automatic_refresh=False,
694
+ )
695
+ warning = getattr(self.logger, "warning", None)
696
+ if callable(warning):
697
+ warning(message)
698
+ else:
699
+ self.logger.info(message)
700
+
701
+ @staticmethod
702
+ def _dependency_tree_mismatch_message(
703
+ mismatch: dict[str, list[str]],
704
+ *,
705
+ after_automatic_refresh: bool,
706
+ ) -> str:
707
+ stale_backend_uids = mismatch["stale_backend_uids"]
708
+ missing_backend_uids = mismatch["missing_backend_uids"]
709
+ message_parts = [
710
+ "Backend dependency tree is out of sync with the current "
711
+ "DataNode.dependencies() graph."
712
+ ]
713
+ if stale_backend_uids:
714
+ message_parts.append(
715
+ "Backend contains stale dependency update UIDs that current Python "
716
+ f"code does not declare: {stale_backend_uids!r}."
717
+ )
718
+ if missing_backend_uids:
719
+ message_parts.append(
720
+ "Backend is missing dependency update UIDs declared by current "
721
+ f"Python code: {missing_backend_uids!r}."
722
+ )
723
+ if after_automatic_refresh:
724
+ message_parts.append(
725
+ "The SDK already cleared and rebuilt the dependency tree once, but "
726
+ "the backend response still does not match the current graph."
727
+ )
728
+ else:
729
+ message_parts.append(
730
+ "The SDK will clear and rebuild backend dependency edges from the "
731
+ "current declarations before executing dependency updates."
732
+ )
733
+ return " ".join(message_parts)
734
+
735
+ @staticmethod
736
+ def _validate_dependency_tree_matches_update_map(
737
+ *,
738
+ dependencies_df: pd.DataFrame,
739
+ update_map: dict[str, dict[str, Any]],
740
+ ) -> None:
741
+ mismatch = UpdateRunner._dependency_tree_mismatch(
742
+ dependencies_df=dependencies_df,
743
+ update_map=update_map,
744
+ )
745
+ if UpdateRunner._mismatch_is_empty(mismatch):
746
+ return
747
+ raise DependencyUpdateError(
748
+ UpdateRunner._dependency_tree_mismatch_message(
749
+ mismatch,
750
+ after_automatic_refresh=True,
751
+ )
752
+ )
753
+
621
754
  def _get_update_map(
622
755
  self,
623
756
  declared_dependencies: dict[str, DataNode],
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mainsequence
3
- Version: 4.3.14
3
+ Version: 4.3.16
4
4
  Summary: Main Sequence SDK
5
5
  Author-email: Main Sequence GmbH <dev@main-sequence.io>
6
6
  License: MainSequence GmbH SDK License Agreement
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "mainsequence"
7
- version = "4.3.14"
7
+ version = "4.3.16"
8
8
  description = "Main Sequence SDK "
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -1566,8 +1566,12 @@ def test_ensure_registered_storage_table_rejects_unbound_storage(monkeypatch):
1566
1566
  classmethod(lambda cls, **filters: []),
1567
1567
  )
1568
1568
 
1569
- with pytest.raises(ValueError, match="not bound to backend TimeIndexMetaTable"):
1569
+ with pytest.raises(ValueError) as exc_info:
1570
1570
  ensure_registered_storage_table(AssetSnapshots, context="DataNode")
1571
+ message = str(exc_info.value)
1572
+ assert "not bound to backend TimeIndexMetaTable" in message
1573
+ assert "found no backend TimeIndexMetaTable catalog row" in message
1574
+ assert "example_assets__asset_snapshots" in message
1571
1575
 
1572
1576
 
1573
1577
  def test_ensure_registered_storage_table_binds_existing_time_index_meta_table(monkeypatch):
@@ -1607,10 +1611,48 @@ def test_ensure_registered_storage_table_binds_existing_time_index_meta_table(mo
1607
1611
  assert AssetSnapshots.get_meta_table_uid() == "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
1608
1612
  assert captured == {
1609
1613
  "physical_table_name__in": ["example_assets__asset_snapshots"],
1610
- "limit": 1,
1614
+ "limit": 20,
1611
1615
  }
1612
1616
 
1613
1617
 
1618
+ def test_ensure_registered_storage_table_reports_duplicate_matches(monkeypatch):
1619
+ columns = [
1620
+ FakeColumn("time_index", DateTime(timezone=True), nullable=False),
1621
+ FakeColumn("asset_uid", Uuid(), nullable=False),
1622
+ ]
1623
+ table = FakeTable("example_assets__asset_snapshots", columns=columns)
1624
+ AssetSnapshots = _time_index_model_class(
1625
+ "AssetSnapshots",
1626
+ table,
1627
+ index_names=["time_index", "asset_uid"],
1628
+ )
1629
+ first = TimeIndexMetaTable.model_construct(
1630
+ uid="aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
1631
+ data_source_uid="dddddddd-dddd-4ddd-8ddd-dddddddddddd",
1632
+ storage_hash="storage-hash",
1633
+ physical_table_name="example_assets__asset_snapshots",
1634
+ )
1635
+ second = TimeIndexMetaTable.model_construct(
1636
+ uid="bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
1637
+ data_source_uid="eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee",
1638
+ storage_hash="storage-hash",
1639
+ physical_table_name="example_assets__asset_snapshots",
1640
+ )
1641
+
1642
+ monkeypatch.setattr(
1643
+ TimeIndexMetaTable,
1644
+ "filter_by_body",
1645
+ classmethod(lambda cls, **filters: [first, second]),
1646
+ )
1647
+
1648
+ with pytest.raises(ValueError) as exc_info:
1649
+ ensure_registered_storage_table(AssetSnapshots, context="DataNode")
1650
+ message = str(exc_info.value)
1651
+ assert "found 2 matching backend TimeIndexMetaTable catalog rows" in message
1652
+ assert "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" in message
1653
+ assert "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" in message
1654
+
1655
+
1614
1656
  def test_time_index_storage_name_hash_component_separates_identical_table_shapes():
1615
1657
  columns_a = [
1616
1658
  FakeColumn("time_index", DateTime(timezone=True), nullable=False),