moose-lib 0.4.238__py3-none-any.whl → 0.4.240__py3-none-any.whl

moose_lib/data_models.py

@@ -280,3 +280,16 @@ class StringToEnumMixin:
             return cls(value)  # fallback to default enum validation
 
         return core_schema.with_info_before_validator_function(validate, core_schema.enum_schema(cls, list(cls)))
+
+
+def is_array_nested_type(data_type: DataType) -> bool:
+    """Type guard to check if a data type is Array(Nested(...))."""
+    return (
+        isinstance(data_type, ArrayType) and
+        isinstance(data_type.element_type, Nested)
+    )
+
+
+def is_nested_type(data_type: DataType) -> bool:
+    """Type guard to check if a data type is Nested (not Array)."""
+    return isinstance(data_type, Nested)

moose_lib/dmv2/olap_table.py

@@ -15,6 +15,7 @@ from moose_lib import ClickHouseEngines
 from ..config.runtime import RuntimeClickHouseConfig
 from .types import TypedMooseResource, T
 from ._registry import _tables
+from ..data_models import Column, is_array_nested_type, is_nested_type, _to_columns
 
 @dataclass
 class InsertOptions:
@@ -458,9 +459,11 @@ class OlapTable(TypedMooseResource, Generic[T]):
         dict_data = []
         for record in validated_data:
             if hasattr(record, 'model_dump'):
-                dict_data.append(record.model_dump())
+                record_dict = record.model_dump()
             else:
-                dict_data.append(record)
+                record_dict = record
+            preprocessed_record = self._map_to_clickhouse_record(record_dict)
+            dict_data.append(preprocessed_record)
         if not dict_data:
             return table_name, b"", settings
         json_lines = self._to_json_each_row(dict_data)
@@ -527,9 +530,11 @@ class OlapTable(TypedMooseResource, Generic[T]):
         records_dict = []
         for record in records:
             if hasattr(record, 'model_dump'):
-                records_dict.append(record.model_dump())
+                record_dict = record.model_dump()
             else:
-                records_dict.append(record)
+                record_dict = record
+            preprocessed_record = self._map_to_clickhouse_record(record_dict)
+            records_dict.append(preprocessed_record)
 
         RETRY_BATCH_SIZE = 10
         for i in range(0, len(records_dict), RETRY_BATCH_SIZE):
@@ -762,4 +767,48 @@ class OlapTable(TypedMooseResource, Generic[T]):
                 should_validate,
                 strategy,
                 options
-            )
+            )
+
+    def _map_to_clickhouse_record(self, record: dict, columns: Optional[List[Column]] = None) -> dict:
+        """
+        Recursively transforms a record to match ClickHouse's JSONEachRow requirements.
+
+        - For every Array(Nested(...)) field at any depth, each item is wrapped in its own array and recursively processed.
+        - For every Nested struct (not array), it recurses into the struct.
+        - This ensures compatibility with kafka_clickhouse_sync
+
+        Args:
+            record: The input record to transform (may be deeply nested)
+            columns: The schema columns for this level (defaults to model columns at the top level)
+
+        Returns:
+            The transformed record, ready for ClickHouse JSONEachRow insertion
+        """
+        if columns is None:
+            columns = _to_columns(self._t)
+
+        result = record.copy()
+
+        for col in columns:
+            if col.name not in record:
+                continue
+
+            value = record[col.name]
+            data_type = col.data_type
+
+            if is_array_nested_type(data_type):
+                # For Array(Nested(...)), wrap each item in its own array and recurse
+                if (isinstance(value, list) and
+                    (len(value) == 0 or isinstance(value[0], dict))):
+                    nested_columns = data_type.element_type.columns
+                    result[col.name] = [
+                        [self._map_to_clickhouse_record(item, nested_columns)]
+                        for item in value
+                    ]
+            elif is_nested_type(data_type):
+                # For Nested struct (not array), recurse into it
+                if value and isinstance(value, dict):
+                    result[col.name] = self._map_to_clickhouse_record(value, data_type.columns)
+            # All other types: leave as is
+
+        return result

{moose_lib-0.4.238.dist-info → moose_lib-0.4.240.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.4.238
+Version: 0.4.240
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

{moose_lib-0.4.238.dist-info → moose_lib-0.4.240.dist-info}/RECORD

@@ -1,7 +1,7 @@
 moose_lib/__init__.py,sha256=0MpzYNnjpqcBaXjR5CBr1b0M5YjXXjj9y1RKyNeOJQ8,183
 moose_lib/blocks.py,sha256=_wdvC2NC_Y3MMEnB71WTgWbeQ--zPNHk19xjToJW0C0,3185
 moose_lib/commons.py,sha256=BV5X78MuOWHiZV9bsWSN69JIvzTNWUi-gnuMiAtaO8A,2489
-moose_lib/data_models.py,sha256=8zYPNkOYi0o5Ut2llXwqdIqbXcg_jdv6M_nCEzJ2Onw,9512
+moose_lib/data_models.py,sha256=4eoo68DXQBCy6XS0p_UPoxT9jc42LuKisuFxcDfMunM,9912
 moose_lib/dmv2-serializer.py,sha256=CL_Pvvg8tJOT8Qk6hywDNzY8MYGhMVdTOw8arZi3jng,49
 moose_lib/internal.py,sha256=ezqTTWS3T6nAPAxcjMPGYs-6ZwZwxOTZVuVFHiSkEmw,14269
 moose_lib/main.py,sha256=In-u7yA1FsLDeP_2bhIgBtHY_BkXaZqDwf7BxwyC21c,8471
@@ -18,7 +18,7 @@ moose_lib/dmv2/consumption.py,sha256=vt1APul1O21yPHaB0MRjF_9ohz5MJs5QIpDCnzFwyJA
 moose_lib/dmv2/ingest_api.py,sha256=Snek9NGwaJl_BuImSWGtQq91m9D3AJ4qBoGiKZ-9yTQ,2323
 moose_lib/dmv2/ingest_pipeline.py,sha256=Y1gsvHZjlW07gMapLnBRJEsoAPv7ThvLABoLmVV7BHE,6714
 moose_lib/dmv2/materialized_view.py,sha256=kcx-sJFTM-cH3Uc1GoldgFGodjoz0AegAQEMmohdS38,3826
-moose_lib/dmv2/olap_table.py,sha256=D3qpRGMnYF0gu5FRW8E5oDBqdWMCWLRv7fWv81DURsk,28378
+moose_lib/dmv2/olap_table.py,sha256=U-ZdWzLKbxq1ANcG37rI3xJtouW3VM_NFGCWOcuwEs8,30567
 moose_lib/dmv2/registry.py,sha256=AaGS6Xy0vKz-wHLPgRVxfKfSwW5KksMePjZ8N7-2OKU,2054
 moose_lib/dmv2/sql_resource.py,sha256=kUZoGqxhZMHMthtBZGYJBxTFjXkspXiWLXhJRYXgGUM,1864
 moose_lib/dmv2/stream.py,sha256=H5nzqVHIXulFNMNaGZUQnhGjNx7fIg0X95kxAO_qlls,10600
@@ -31,7 +31,7 @@ tests/__init__.py,sha256=0Gh4yzPkkC3TzBGKhenpMIxJcRhyrrCfxLSfpTZnPMQ,53
 tests/conftest.py,sha256=ZVJNbnr4DwbcqkTmePW6U01zAzE6QD0kNAEZjPG1f4s,169
 tests/test_moose.py,sha256=mBsx_OYWmL8ppDzL_7Bd7xR6qf_i3-pCIO3wm2iQNaA,2136
 tests/test_redis_client.py,sha256=d9_MLYsJ4ecVil_jPB2gW3Q5aWnavxmmjZg2uYI3LVo,3256
-moose_lib-0.4.238.dist-info/METADATA,sha256=zqv08nRKe6ynI1OUT1BiueMWrzaYk6MUmhTTS-bOhKw,729
-moose_lib-0.4.238.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-moose_lib-0.4.238.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
-moose_lib-0.4.238.dist-info/RECORD,,
+moose_lib-0.4.240.dist-info/METADATA,sha256=RjxA65DeF3XIFJU-BEUkhGOuB_YkYMjOIY_Xdvni3go,729
+moose_lib-0.4.240.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+moose_lib-0.4.240.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
+moose_lib-0.4.240.dist-info/RECORD,,