mainsequence 4.1.7__tar.gz → 4.1.9__tar.gz

{mainsequence-4.1.7 → mainsequence-4.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mainsequence
-Version: 4.1.7
+Version: 4.1.9
 Summary: Main Sequence SDK 
 Author-email: Main Sequence GmbH <dev@main-sequence.io>
 License: MainSequence GmbH SDK License Agreement

{mainsequence-4.1.7 → mainsequence-4.1.9}/agent_scaffold/skills/data_publishing/data_nodes/SKILL.md

@@ -25,7 +25,7 @@ Canonical workflow:
 - modify an existing `DataNode` update process
 - review whether a DataNode change affects update identity or table contract
 - define or refactor `DataNodeConfiguration`
-- classify config fields into update identity and hash-excluded descriptive metadata
+- classify values as hashed config fields or non-config class/runtime values
 - implement or review:
   - `dependencies()`
   - `update()`
@@ -112,6 +112,16 @@ Every storage class must include `__metatable_description__`. The description
 should explain the table's intention, row grain, and downstream use, not only
 list columns or schema mechanics.
 
+Use `__metatable_extra_hash_components__` on storage classes when distinct
+DataNode storage tables could otherwise have the same storage-relevant shape.
+For example, two one-index daily tables with one float column need a stable
+component such as `{"storage_name": "daily_random_number"}` versus
+`{"storage_name": "daily_random_addition"}`.
+
+This is storage identity. Changing it creates a different storage table. Do not
+use it for labels, descriptions, runtime options, test isolation, backend UIDs,
+data-source UIDs, or updater scope.
+
 Do not put those concerns in `DataNodeConfiguration`.
 
 Minimal pattern:
@@ -132,6 +142,7 @@ class Base(DeclarativeBase):
 class PricesTable(PlatformTimeIndexMetaData, Base):
     __metatable_namespace__ = "<domain_namespace>"
     __metatable_identifier__ = "<table_identifier>"
+    __metatable_extra_hash_components__ = {"storage_name": "<stable_storage_name>"}
     __metatable_description__ = (
         "Daily close prices keyed by asset unique identifier for portfolio and "
         "risk analytics."
@@ -183,12 +194,21 @@ Do not accept `test_node`. It has been removed. Use explicit
 Pattern:
 
 ```python
+from typing import ClassVar
+
+from pydantic import Field
+
 from mainsequence.meta_tables import DataNode, DataNodeConfiguration
 from mainsequence.meta_tables import PlatformTimeIndexMetaData
 
 
 class PricesConfig(DataNodeConfiguration):
-    shard_id: str
+    shard_id: str = Field(
+        ...,
+        description="Stable updater partition for this price job.",
+        examples=["us_equities_daily"],
+    )
+    reference_dimension: ClassVar[str] = "unique_identifier"
 
 
 class PricesUpdate(DataNode):
@@ -216,23 +236,45 @@ class PricesUpdate(DataNode):
 ### 3. Configuration Is Update-Scoped By Default
 
 Every `DataNodeConfiguration` field participates in `update_hash` by default.
+Declare values that change output values, dependencies, source choice, or
+updater scope as normal config fields.
 
-Do not use:
+Every config field must be declared with `Field(...)`. Include a clear
+`description` and add `examples=[...]` whenever a realistic example helps. The
+description must explain what the value means for the update process, not repeat
+the Python type.
 
-- `json_schema_extra={"update_only": True}`
-- `json_schema_extra={"runtime_only": True}`
-- `json_schema_extra={"ignore_from_storage_hash": True}`
-- `_ARGS_IGNORE_IN_STORAGE_HASH`
+Values that must not affect `update_hash` should not be Pydantic config fields.
+Use `ClassVar[...]` for class-level invariants and implementation constants, or
+keep runtime controls in environment/runtime configuration outside the
+DataNode config.
 
-Those are removed. The only supported opt-out is:
+If a value genuinely must remain a Pydantic field while not affecting
+`update_hash`, the only supported field-level opt-out is
+`json_schema_extra={"hash_excluded": True}`:
 
 ```python
-Field(..., json_schema_extra={"hash_excluded": True})
+from pydantic import Field
+
+from mainsequence.meta_tables import DataNodeConfiguration
+
+
+class PricesConfig(DataNodeConfiguration):
+    shard_id: str = Field(
+        ...,
+        description="Stable updater partition for this price job.",
+        examples=["us_equities_daily"],
+    )
+    display_label: str | None = Field(
+        default=None,
+        description="Optional human-facing label for UI display only.",
+        examples=["US equities daily prices"],
+        json_schema_extra={"hash_excluded": True},
+    )
 ```
 
-Use `hash_excluded` only for descriptive metadata that must not affect update
-identity. If a field changes output values, dependencies, source choice, or
-updater scope, it must remain a normal config field.
+Do not invent other metadata-marker exceptions. Scope, dependency, source, and
+output-affecting fields must remain hashed config fields.
 
 ### 4. `hash_namespace` Is Isolation Only
 
@@ -311,7 +353,16 @@ Do not construct dependency graphs dynamically inside `update()`.
 ### 8. Foreign Keys Belong To The Storage Contract
 
 For new code, model foreign keys on the `PlatformTimeIndexMetaData` storage
-class or route the storage-contract work to the MetaTable skill.
+class or route the storage-contract work to the MetaTable skill. When a
+DataNode storage table needs a platform-managed FK, use
+`MetaTableForeignKey(TargetModel, column=...)` on the storage class. Do not use
+`ForeignKey(Target.__table__.c.uid)`, table fullnames, or explicit target UID
+maps in DataNode examples.
+
+Registration of the storage class follows the MetaTable lifecycle:
+`register()` recursively registers unresolved FK target model classes, uses the
+local process registry keyed by `storage_hash`, and writes the target
+`MetaTable.uid` into the FK contract.
 
 Do not add DataNode configuration fields just to mutate storage metadata.
 
@@ -330,11 +381,10 @@ When reviewing an existing DataNode, look for:
 - missing `__metatable_description__` on the storage table
 - dependency storage table passed as an ad hoc constructor argument
 - schema or published table metadata hidden in DataNode configuration
-- `update_only`, `runtime_only`, or `ignore_from_storage_hash`
 - `test_node=True`
 - missing explicit `storage_table`
 - accidental storage registration inside the DataNode
-- wrong meaning/scope/hash-excluded split
+- wrong split between hashed config fields and non-config class/runtime values
 - misuse of `hash_namespace`
 - non-incremental `update()` behavior
 - hidden dependency creation inside `update()`

{mainsequence-4.1.7 → mainsequence-4.1.9}/agent_scaffold/skills/data_publishing/meta_tables/SKILL.md

@@ -103,12 +103,23 @@ Always declare `__metatable_description__` on the model. The description must
 explain the table's business intention, row grain, and expected use, not only
 the schema. Column-level descriptions stay in `mapped_column(info={...})`.
 
+Use `__metatable_extra_hash_components__` when two backend-managed tables could
+otherwise produce the same storage hash because their storage-relevant shape is
+identical or intentionally generic. The value must be stable and deterministic,
+usually a small mapping such as `{"storage_name": "account_holdings"}`.
+
+This attribute is part of storage identity. Changing it defines a different
+table. Do not use it for labels, descriptions, runtime options, test isolation,
+backend UIDs, data-source UIDs, or updater scope. Use `hash_namespace` for test
+or experiment isolation.
+
 Register through the class API:
 
 ```python
 class Account(PlatformManagedMetaTable, Base):
     __metatable_namespace__ = "sdk-examples"
     __metatable_identifier__ = "Account"
+    __metatable_extra_hash_components__ = {"storage_name": "account"}
     __metatable_description__ = (
         "Customer account master records used to scope balances, holdings, and "
         "account-level limits."
@@ -141,29 +152,48 @@ Do not add a `MAINSEQUENCE_META_TABLE_REGISTER` toggle in registration examples.
 
 Foreign-key contracts reference the target `MetaTable` UID.
 
-For `PlatformManagedMetaTable`, register parent tables first and then register child tables normally. The SDK inspects SQLAlchemy foreign-key constraints and resolves each target `MetaTable` by looking up the already registered table in the same data source, schema, and physical table name.
+For `PlatformManagedMetaTable`, define foreign keys with
+`MetaTableForeignKey(TargetModel, column=...)`. Do not write raw SQLAlchemy
+table fullnames, `Parent.__table__.c.<column>` targets, or explicit target UID
+maps in the platform-managed path. Registration is the lifecycle path:
+`register()` recursively registers unresolved target model classes, stores each
+returned `MetaTable` in a local process registry keyed by `storage_hash`, and
+uses the target `MetaTable.uid` in the child FK contract.
+
+Use this pattern:
+
+```python
+account_uid: Mapped[uuid.UUID] = mapped_column(
+    Uuid,
+    MetaTableForeignKey(Account, column="uid", ondelete="RESTRICT"),
+    nullable=False,
+)
+```
+
+Every participating table must include `__metatable_description__` describing
+both the schema and the table's intention.
 
 Example registration order:
 
 ```python
-account_meta_table = Account.register(...)
 asset_meta_table = Asset.register(...)
 ```
 
-The child registration will fail if the parent table has not already been registered, because there is no target `MetaTable.uid` for the backend FK contract.
-
-Do not pass `target_meta_tables` or `target_meta_table_uid_by_fullname` in the normal platform-managed path. Use explicit FK target mappings only for edge cases where automatic lookup is ambiguous or impossible.
+The child registration registers `Account` first if it has not already been
+registered in the current process. The local registry prevents duplicate backend
+registration attempts for the same `storage_hash` and raises a clear error for
+recursive registration cycles.
 
-For `external_registered`, there is no platform-managed parent lookup through the model class. Register the parent first, then build the child registration request with the parent UID mapped by target table fullname:
+For `external_registered`, there is no platform-managed parent lookup. Register
+the parent first, then build the child registration request with
+`target_meta_tables={Account: account_meta_table}`:
 
 ```python
 account_meta_table = MetaTable.register(account_request)
 asset_request = external_registered_registration_request_from_sqlalchemy_model(
     Asset,
     data_source_uid=data_source_uid,
-    target_meta_table_uid_by_fullname={
-        Account.__table__.fullname: account_meta_table.uid,
-    },
+    target_meta_tables={Account: account_meta_table},
 )
 asset_meta_table = MetaTable.register(asset_request)
 ```

{mainsequence-4.1.7 → mainsequence-4.1.9}/mainsequence/meta_tables/__init__.py

@@ -9,6 +9,7 @@ _LAZY_IMPORTS = {
         ".sqlalchemy_contracts",
         "DEFAULT_PLATFORM_MANAGED_PROVISIONING",
     ),
+    "MetaTableForeignKey": (".sqlalchemy_contracts", "MetaTableForeignKey"),
     "PlatformManagedMetaTable": (".sqlalchemy_contracts", "PlatformManagedMetaTable"),
     "PlatformTimeIndexMetaData": (".sqlalchemy_contracts", "PlatformTimeIndexMetaData"),
     "POSTGRES_IDENTIFIER_MAX_LENGTH": (".hashing", "POSTGRES_IDENTIFIER_MAX_LENGTH"),

{mainsequence-4.1.7 → mainsequence-4.1.9}/mainsequence/meta_tables/data_nodes/data_nodes.py

@@ -420,13 +420,6 @@ class DataNode(DataAccessMixin, ABC):
         """
         super().__init_subclass__(**kwargs)
 
-        if "_ARGS_IGNORE_IN_STORAGE_HASH" in cls.__dict__:
-            raise TypeError(
-                f"{cls.__name__} uses removed class attribute _ARGS_IGNORE_IN_STORAGE_HASH; "
-                "move those fields into DataNodeConfiguration. Configuration fields "
-                "participate in update hashing by default."
-            )
-
         # Get the original __init__ from the new subclass
         original_init = cls.__init__
 

{mainsequence-4.1.7 → mainsequence-4.1.9}/mainsequence/meta_tables/pydantic_metadata.py

@@ -20,24 +20,6 @@ def serialize_pydantic_model(
 
     for field_name, field_info in value.__class__.model_fields.items():
         extra = field_info.json_schema_extra or {}
-        if "ignore_from_storage_hash" in extra:
-            raise ValueError(
-                f"{value.__class__.__name__}.{field_name} uses removed metadata "
-                "'ignore_from_storage_hash'. All configuration fields participate in "
-                'update hashing by default; use json_schema_extra={"hash_excluded": True} '
-                "only for fields that should not affect update identity."
-            )
-        if "update_only" in extra:
-            raise ValueError(
-                f"{value.__class__.__name__}.{field_name} uses removed metadata "
-                "'update_only'. All configuration fields are update-scoped by default."
-            )
-        if "runtime_only" in extra:
-            raise ValueError(
-                f"{value.__class__.__name__}.{field_name} uses removed metadata "
-                "'runtime_only'; use json_schema_extra={\"hash_excluded\": True} instead."
-            )
-
         is_hash_excluded = extra.get("hash_excluded", False)
         if not isinstance(is_hash_excluded, bool):
             raise ValueError(