mainsequence 4.1.1__tar.gz → 4.1.3__tar.gz

{mainsequence-4.1.1 → mainsequence-4.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence
-Version: 4.1.1
+Version: 4.1.3
 Summary: Main Sequence SDK 
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License: MainSequence GmbH SDK License Agreement

{mainsequence-4.1.1 → mainsequence-4.1.3}/agent_scaffold/skills/data_publishing/data_nodes/SKILL.md

@@ -161,6 +161,16 @@ The DataNode constructor should accept:
 - a registered `storage_table: type[PlatformTimeIndexMetaData]`
 - optional `hash_namespace`
 
+The constructor `storage_table` is the output storage contract. Keep it out of
+`DataNodeConfiguration`.
+
+If the DataNode needs to select another DataNode's storage table as a
+dependency, put that dependency storage reference in the config as
+`type[PlatformTimeIndexMetaData]`. Do not add an extra constructor argument for
+dependency storage tables. Config values of this type are hashed by the bound
+`TimeIndexMetaData.uid` from `StorageClass.__time_index_metadata__`, so the class
+must be registered or bound before DataNode construction.
+
 Do not accept `test_node`. It has been removed. Use explicit
 `hash_namespace(...)` or `hash_namespace="..."`.
 
@@ -287,6 +297,9 @@ Time index must be datetime64[ns, UTC]
 
 Dependencies belong in constructor setup and `dependencies()`.
 
+Dependency storage-table selection belongs in `DataNodeConfiguration`, because
+changing it changes the dependency graph and update identity.
+
 Do not construct dependency graphs dynamically inside `update()`.
 
 ### 8. Asset-Scoped Updates Must Be Explicit
@@ -318,7 +331,8 @@ surface for new DataNode work.
 
 When reviewing an existing DataNode, look for:
 
-- storage contract hidden in `DataNodeConfiguration`
+- output storage contract hidden in `DataNodeConfiguration`
+- dependency storage table passed as an ad hoc constructor argument
 - old `RecordDefinition` or `DataNodeMetaData` schema patterns
 - `update_only`, `runtime_only`, or `ignore_from_storage_hash`
 - `test_node=True`
@@ -339,6 +353,7 @@ Do not claim success until you have checked:
 - the relevant docs were read first
 - storage is a registered or bound `PlatformTimeIndexMetaData` class
 - the DataNode constructor requires `storage_table`
+- dependency storage-table references live in config and are registered or bound
 - config fields are updater-scoped by default
 - no removed hash metadata markers remain
 - no `test_node` usage remains

{mainsequence-4.1.1 → mainsequence-4.1.3}/mainsequence/client/dtype_codec.py

@@ -163,9 +163,7 @@ def record_definitions_to_column_dtypes_map(
         )
 
     if duplicate_columns:
-        raise ValueError(
-            f"Duplicate DataNode record column names: {sorted(duplicate_columns)}"
-        )
+        raise ValueError(f"Duplicate DataNode record column names: {sorted(duplicate_columns)}")
 
     return column_dtypes_map
 
@@ -247,10 +245,14 @@ def token_to_pandas_series(
     is_time_index: bool = False,
     nullable: bool = True,
 ) -> pd.Series:
-    normalized = TIMESTAMP_TZ if is_time_index else normalize_dtype_token(
-        token,
-        remote=False,
-        allow_naive_datetime=True,
+    normalized = (
+        TIMESTAMP_TZ
+        if is_time_index
+        else normalize_dtype_token(
+            token,
+            remote=False,
+            allow_naive_datetime=True,
+        )
     )
     if normalized == TIMESTAMP_TZ:
         return pd.to_datetime(series, errors="coerce", utc=True)
@@ -319,17 +321,15 @@ def _serialize_timestamp_tz_value(value: Any) -> str | None:
         raise ValueError(
             f"Remote datetime value {value!r} must include an explicit timezone offset or Z."
         )
-    if isinstance(value, (pd.Timestamp, datetime.datetime)) and not _timestamp_is_timezone_aware(value):
-        raise ValueError(
-            f"Remote datetime value {value!r} must be timezone-aware."
-        )
+    if isinstance(value, (pd.Timestamp, datetime.datetime)) and not _timestamp_is_timezone_aware(
+        value
+    ):
+        raise ValueError(f"Remote datetime value {value!r} must be timezone-aware.")
     timestamp = pd.to_datetime(value, errors="raise", utc=True)
     if not isinstance(timestamp, pd.Timestamp):
         timestamp = pd.Timestamp(timestamp).tz_convert("UTC")
     if timestamp.tzinfo is None:
-        raise ValueError(
-            f"Remote datetime value {value!r} must be timezone-aware."
-        )
+        raise ValueError(f"Remote datetime value {value!r} must be timezone-aware.")
     timestamp = timestamp.tz_convert("UTC")
     return timestamp.isoformat().replace("+00:00", "Z")
 
@@ -375,11 +375,8 @@ def prepare_dataframe_for_remote_write(
         if column_name not in prepared.columns:
             continue
         normalized = normalize_dtype_token(token, remote=True)
-        if normalized not in {DATE, TIMESTAMP_TZ}:
-            continue
         prepared[column_name] = [
-            serialize_remote_value(value, normalized)
-            for value in prepared[column_name].tolist()
+            serialize_remote_value(value, normalized) for value in prepared[column_name].tolist()
         ]
     return prepared
 
@@ -459,7 +456,11 @@ def sqlalchemy_type_to_token(column_type: Any, *, remote: bool = True) -> str:
                     "by the remote TS Manager contract."
                 )
             return LOCAL_DATETIME_NAIVE
-        if timezone is True or "with time zone" in normalized_backend or "timestamptz" in normalized_backend:
+        if (
+            timezone is True
+            or "with time zone" in normalized_backend
+            or "timestamptz" in normalized_backend
+        ):
             return TIMESTAMP_TZ
         if remote:
             raise ValueError(

{mainsequence-4.1.1 → mainsequence-4.1.3}/mainsequence/client/models_metatables.py

@@ -1244,9 +1244,6 @@ class MetaTable(BasePydanticModel, LabelableObjectMixin, ShareableObjectMixin, B
         )
 
 
-LOGICAL_COLUMN_DTYPES_ATTR = "mainsequence_column_dtypes_map"
-
-
 def _warn_legacy_compat(message: str, *, stacklevel: int = 3) -> None:
     warnings.warn(
         f"Deprecated TDAG compatibility path: {message}",
@@ -2143,7 +2140,6 @@ class DataNodeUpdate(TableUpdateNode, BaseObjectOrm):
         remote_dtypes: bool = True,
         allow_naive_datetime: bool = False,
     ):
-        logical_column_dtypes_map = data_frame.attrs.get(LOGICAL_COLUMN_DTYPES_ATTR)
         record_column_dtypes_map = record_definitions_to_column_dtypes_map(
             records,
             remote=remote_dtypes,
@@ -2178,40 +2174,6 @@ class DataNodeUpdate(TableUpdateNode, BaseObjectOrm):
                 "DataNode records declare columns not present in the DataFrame: "
                 f"{missing_record_columns}"
             )
-        if logical_column_dtypes_map is not None:
-            logical_column_dtypes_map = normalize_column_dtypes_map(
-                logical_column_dtypes_map,
-                remote=remote_dtypes,
-                allow_naive_datetime=allow_naive_datetime,
-            )
-            missing_logical_columns = [
-                column_name
-                for column_name in logical_column_dtypes_map
-                if column_name not in data_frame.columns
-            ]
-            if missing_logical_columns:
-                raise ValueError(
-                    "Logical column dtype contract contains columns not present "
-                    f"in the DataFrame: {missing_logical_columns}"
-                )
-            conflicting_declared_dtypes = {
-                column_name: {
-                    "record_dtype": record_column_dtypes_map[column_name],
-                    "logical_dtype": logical_column_dtypes_map[column_name],
-                }
-                for column_name in logical_column_dtypes_map
-                if (
-                    column_name in record_column_dtypes_map
-                    and logical_column_dtypes_map[column_name]
-                    != record_column_dtypes_map[column_name]
-                )
-            }
-            if conflicting_declared_dtypes:
-                raise ValueError(
-                    "Logical column dtype contract conflicts with DataNode records: "
-                    f"{conflicting_declared_dtypes}"
-                )
-            column_dtypes_map.update(logical_column_dtypes_map)
         column_dtypes_map.update(record_column_dtypes_map)
 
         data_frame = data_frame.replace({np.nan: None})
@@ -3970,10 +3932,70 @@ class UpdateStatistics(BaseUpdateStatistics):
 def _assign_nested_coordinate(root: dict[Any, Any], keys: list[Any], value: Any) -> None:
     if not keys:
         return
+    normalized_keys = [_normalize_update_stat_key(key) for key in keys]
     sub = root
-    for key in keys[:-1]:
-        sub = sub.setdefault(key, {})
-    sub[keys[-1]] = value
+    for key in normalized_keys[:-1]:
+        existing = sub.setdefault(key, {})
+        if not isinstance(existing, dict):
+            raise ValueError("Update statistics coordinate keys collide after JSON normalization.")
+        sub = existing
+    final_key = normalized_keys[-1]
+    if final_key in sub:
+        raise ValueError("Update statistics coordinate keys collide after JSON normalization.")
+    sub[final_key] = value
+
+
+def _normalize_update_stat_key(key: Any) -> str:
+    if isinstance(key, str):
+        return key
+    if isinstance(key, np.generic):
+        key = key.item()
+    if isinstance(key, datetime.datetime):
+        value = serialize_to_json({"_": key})["_"]
+        return str(value)
+    if isinstance(key, datetime.date):
+        return key.isoformat()
+    if key is None:
+        return "null"
+    if isinstance(key, bool):
+        return "true" if key else "false"
+    if isinstance(key, int | float):
+        try:
+            return json.dumps(key, allow_nan=False)
+        except (TypeError, ValueError):
+            return str(key)
+
+    normalized = serialize_to_json({key: None})
+    normalized_key = next(iter(normalized.keys()))
+    if normalized_key is None:
+        return "null"
+    if isinstance(normalized_key, bool):
+        return "true" if normalized_key else "false"
+    if isinstance(normalized_key, int | float):
+        try:
+            return json.dumps(normalized_key, allow_nan=False)
+        except (TypeError, ValueError):
+            return str(normalized_key)
+    return str(normalized_key)
+
+
+def _normalize_update_stat_mapping_keys(value: Any) -> Any:
+    if isinstance(value, Mapping):
+        normalized: dict[str, Any] = {}
+        for key, item in value.items():
+            normalized_key = _normalize_update_stat_key(key)
+            if normalized_key in normalized:
+                raise ValueError(
+                    "Update statistics coordinate keys collide after JSON normalization: "
+                    f"{normalized_key!r}."
+                )
+            normalized[normalized_key] = _normalize_update_stat_mapping_keys(item)
+        return normalized
+    if isinstance(value, list):
+        return [_normalize_update_stat_mapping_keys(item) for item in value]
+    if isinstance(value, tuple):
+        return [_normalize_update_stat_mapping_keys(item) for item in value]
+    return value
 
 
 def get_index_progress_chunk_stats(chunk_df, time_index_name, index_names):
@@ -4081,6 +4103,11 @@ class LastUpdateMultiIndexStatsPayload(BaseModel):
     index_progress: dict[str, Any] = Field(default_factory=dict)
     index_min: dict[str, Any] = Field(default_factory=dict)
 
+    @model_validator(mode="before")
+    @classmethod
+    def _normalize_mapping_keys(cls, value: Any) -> Any:
+        return _normalize_update_stat_mapping_keys(value)
+
     def to_payload(self) -> dict[str, Any]:
         return {
             "_GLOBAL_": self.global_stats,
@@ -4099,6 +4126,11 @@ class LastUpdateIndexTimePayload(BaseModel):
     multi_index_stats: LastUpdateMultiIndexStatsPayload | None = None
     multi_index_column_stats: dict[str, Any] | None = Field(default_factory=dict)
 
+    @model_validator(mode="before")
+    @classmethod
+    def _normalize_mapping_keys(cls, value: Any) -> Any:
+        return _normalize_update_stat_mapping_keys(value)
+
     @model_validator(mode="after")
     def _validate_shape(self):
         top_level_progress_keys = [
@@ -4164,7 +4196,8 @@ def build_last_update_index_time_payload(
             }
         )
 
-    return LastUpdateIndexTimePayload.model_validate(raw_payload).to_nested_payload()
+    normalized_payload = _normalize_update_stat_mapping_keys(raw_payload)
+    return LastUpdateIndexTimePayload.model_validate(normalized_payload).to_nested_payload()
 
 
 class HistoricalUpdateRecord:

{mainsequence-4.1.1 → mainsequence-4.1.3}/mainsequence/client/utils.py

@@ -9,11 +9,11 @@ import socket
 import subprocess
 import threading
 import time
-import uuid
 from dataclasses import dataclass, field
 from decimal import Decimal
 from enum import Enum
 from typing import TypedDict
+from uuid import UUID, getnode
 
 import psutil
 import pytz
@@ -761,6 +761,9 @@ def serialize_to_json(kwargs):
         if isinstance(v, Decimal):
             return str(v)
 
+        if isinstance(v, UUID):
+            return str(v)
+
         if isinstance(v, datetime.datetime):
             dt = v
             if dt.tzinfo is None:
@@ -776,13 +779,19 @@ def serialize_to_json(kwargs):
                 return v.model_dump()
 
         if isinstance(v, dict):
-            return {k: to_jsonable(x) for k, x in v.items()}
+            return {to_json_key(k): to_jsonable(x) for k, x in v.items()}
         if isinstance(v, (list, tuple)):
             return [to_jsonable(x) for x in v]
 
         return v
 
-    return {k: to_jsonable(v) for k, v in kwargs.items()}
+    def to_json_key(value):
+        key = to_jsonable(value)
+        if key is None or isinstance(key, str | int | float | bool):
+            return key
+        return str(key)
+
+    return {to_json_key(k): to_jsonable(v) for k, v in kwargs.items()}
 
 
 def _linux_machine_id() -> str | None:
@@ -839,7 +848,7 @@ def bios_uuid() -> str:
         return mid
 
     # Tier 4 – MAC address (uuid.getnode). Always available.
-    return f"{uuid.getnode():012x}"
+    return f"{getnode():012x}"
 
 
 def _install_retry_adapters_in_place(

{mainsequence-4.1.1 → mainsequence-4.1.3}/mainsequence/meta_tables/data_nodes/build_operations.py

@@ -13,22 +13,19 @@ from enum import Enum
 from functools import singledispatch
 from types import SimpleNamespace
 from typing import TYPE_CHECKING, Any
+from uuid import UUID
 
 from pydantic import BaseModel
 
 from mainsequence.client import BaseObjectOrm
 from mainsequence.client.models_helpers import get_model_class
 from mainsequence.client.models_metatables import _resolve_local_pod_project
-from mainsequence.instrumentation import tracer, tracer_instrumentator
 from mainsequence.meta_tables.pydantic_metadata import (
     is_serialized_pydantic_model,
     serialize_pydantic_model,
     strip_pydantic_hash_exclusions,
 )
 
-from .namespacing import disable_hash_namespace
-from .persist_managers import PersistManager
-
 if TYPE_CHECKING:
     from .data_nodes import APIDataNode, DataNode
 
@@ -70,11 +67,56 @@ def _serialize_api_timeserie(value: APIDataNode) -> dict[str, Any]:
     }
 
 
+def _import_qualified_name(module_name: str, qualname: str) -> Any:
+    module = importlib.import_module(module_name)
+    value: Any = module
+    for part in qualname.split("."):
+        value = getattr(value, part)
+    return value
+
+
+def _is_platform_time_index_metadata_class(value: Any) -> bool:
+    try:
+        from mainsequence.meta_tables.sqlalchemy_contracts import PlatformTimeIndexMetaData
+    except ImportError:
+        return False
+
+    try:
+        return isinstance(value, type) and issubclass(value, PlatformTimeIndexMetaData)
+    except TypeError:
+        return False
+
+
+@serialize_argument.register(type)
+def _(value: type[Any]) -> Any:
+    if not _is_platform_time_index_metadata_class(value):
+        return value
+
+    time_index_metadata = value.get_time_index_metadata()
+    uid = getattr(time_index_metadata, "uid", None)
+    if uid in (None, ""):
+        raise ValueError(
+            "PlatformTimeIndexMetaData config values must be registered or bound "
+            "before they can be hashed."
+        )
+    return {
+        "__type__": "platform_time_index_metadata",
+        "uid": str(uid),
+        "module": value.__module__,
+        "qualname": value.__qualname__,
+    }
+
+
 @serialize_argument.register(datetime.datetime)
 def _(value: datetime.datetime) -> str:
     return value.isoformat()
 
 
+@serialize_argument.register(UUID)
+def _(value: UUID) -> str:
+    return str(value)
+
+
 @serialize_argument.register(BaseModel)
 def _(value: BaseModel) -> dict[str, Any]:
     """Serialization logic for any Pydantic BaseModel."""
@@ -180,6 +222,8 @@ def parse_dictionary_before_hashing(dictionary: dict[str, Any]) -> dict[str, Any
                 # The value["items"] are already serialized dicts
 
                 local_ts_dict_to_hash[key] = [v["unique_identifier"] for v in value["items"]]
+            elif value.get("__type__") == "platform_time_index_metadata":
+                local_ts_dict_to_hash[key] = value["uid"]
             else:
                 # recursively apply hash signature
                 local_ts_dict_to_hash[key] = parse_dictionary_before_hashing(value)
@@ -327,6 +371,8 @@ class ConfigRebuilder(BaseRebuilder):
         return build_model(value)
 
     def _handle_complex_type(self, value: dict, **kwargs) -> Any:
+        if value.get("__type__") == "platform_time_index_metadata":
+            return _import_qualified_name(value["module"], value["qualname"])
         # Special case for ORM lists within the generic complex type handler
         if value.get("__type__") == "orm_model_list":
             return [build_model(item) for item in value["items"]]
@@ -434,97 +480,3 @@ def create_config(
         remote_initial_configuration=remote_config,
         build_configuration_json_schema=build_configuration_json_schema,
     )
-
-
-def rebuild_and_set_from_update_hash(
-    update_hash: str,
-    data_source_uid: str,
-    set_dependencies_df: bool = False,
-    graph_depth_limit: int = 1,
-) -> DataNode:
-    """
-    Rebuilds a DataNode from stored configuration and attaches runtime sessions.
-
-    Args:
-        update_hash: The local hash ID of the DataNode.
-        data_source_uid: The data source UID.
-        set_dependencies_df: Whether to set the dependencies DataFrame.
-        graph_depth_limit: The depth limit for graph traversal.
-
-    Returns:
-        The rebuilt DataNode object.
-    """
-    ts = rebuild_from_configuration(
-        update_hash=update_hash,
-        data_source=data_source_uid,
-    )
-    if set_dependencies_df:
-        ts.set_relation_tree()
-    ts._set_state_with_sessions(
-        graph_depth=0,
-        graph_depth_limit=graph_depth_limit,
-        include_client_objects=False,
-    )
-    ts.logger.debug(f"ts {update_hash} rebuilt from configuration")
-    return ts
-
-
-@tracer.start_as_current_span("TS: Rebuild From Configuration")
-def rebuild_from_configuration(update_hash: str, data_source: str | object) -> DataNode:
-    """
-    Rebuilds a DataNode instance from its configuration.
-
-    Args:
-        update_hash: The local hash ID of the DataNode.
-        data_source: The data source UID or object.
-
-    Returns:
-        The rebuilt DataNode instance.
-    """
-    import importlib
-
-    tracer_instrumentator.append_attribute_to_current_span("update_hash", update_hash)
-
-    data_source_uid = getattr(data_source, "uid", data_source)
-    data_node_update = PersistManager.UPDATE_CLASS.get_or_none(
-        update_hash=update_hash,
-        remote_table__data_source__uid=str(data_source_uid),
-        include_relations_detail=True,
-    )
-    if data_node_update is None:
-        raise ValueError(
-            f"DataNodeUpdate {update_hash!r} with data source {data_source_uid!r} was not found."
-        )
-    storage_table = data_node_update.data_node_storage
-    persist_manager = PersistManager.get_from_storage_table(
-        update_hash=update_hash,
-        storage_table=storage_table,
-        data_node_update=data_node_update,
-    )
-    try:
-        time_serie_config = persist_manager.local_build_configuration
-    except Exception as e:
-        raise e
-
-    try:
-        mod = importlib.import_module(time_serie_config["time_series_class_import_path"]["module"])
-        TimeSerieClass = getattr(
-            mod, time_serie_config["time_series_class_import_path"]["qualname"]
-        )
-    except Exception as e:
-        raise e
-
-    time_serie_class_name = time_serie_config["time_series_class_import_path"]["qualname"]
-
-    time_serie_config.pop("time_series_class_import_path")
-    time_serie_config = DeserializerManager().rebuild_serialized_config(
-        time_serie_config, time_serie_class_name=time_serie_class_name
-    )
-
-    # IMPORTANT:
-    # When rebuilding from stored config, ignore any ambient test namespace.
-    # If the stored config includes 'hash_namespace' (test tables), it will still be passed explicitly.
-    with disable_hash_namespace():
-        re_build_ts = TimeSerieClass(**time_serie_config)
-
-    return re_build_ts

{mainsequence-4.1.1 → mainsequence-4.1.3}/mainsequence/meta_tables/data_nodes/persist_managers.py

@@ -438,11 +438,22 @@ class BasePersistManager:
                 data=temp_df,
                 data_source=self.data_source,
                 overwrite=overwrite,
+                source_table_schema=self._source_table_schema(),
             )
 
             persisted = True
         return persisted
 
+    def _source_table_schema(self) -> dict[str, Any]:
+        time_index_name, index_names, column_dtypes_map = (
+            self.storage_metadata._require_time_indexed_table_contract()
+        )
+        return {
+            "time_index_name": time_index_name,
+            "index_names": list(index_names),
+            "column_dtypes_map": dict(column_dtypes_map),
+        }
+
     def get_update_statistics_for_table(self) -> UpdateStatistics:
         return self.storage_metadata.get_data_updates()