moose-lib 0.6.90__py3-none-any.whl → 0.6.283__py3-none-any.whl

moose_lib/dmv2/olap_table.py

@@ -4,6 +4,7 @@ OLAP table definitions for Moose Data Model v2 (dmv2).
 This module provides classes for defining and configuring OLAP tables,
 particularly for ClickHouse.
 """
+
 import json
 import warnings
 from clickhouse_connect import get_client
@@ -11,7 +12,17 @@ from clickhouse_connect.driver.client import Client
 from clickhouse_connect.driver.exceptions import ClickHouseError
 from dataclasses import dataclass
 from pydantic import BaseModel
-from typing import List, Optional, Any, Literal, Union, Tuple, TypeVar, Generic, Iterator
+from typing import (
+    List,
+    Optional,
+    Any,
+    Literal,
+    Union,
+    Tuple,
+    TypeVar,
+    Generic,
+    Iterator,
+)
 from ..blocks import ClickHouseEngines, EngineConfig
 from ..commons import Logger
 from ..config.runtime import RuntimeClickHouseConfig
@@ -20,6 +31,8 @@ from .types import TypedMooseResource, T, Cols
 from ._registry import _tables
 from ..data_models import Column, is_array_nested_type, is_nested_type, _to_columns
 from .life_cycle import LifeCycle
+from ._source_capture import get_source_file_from_stack
+
 
 @dataclass
 class InsertOptions:
@@ -32,12 +45,14 @@ class InsertOptions:
         validate: Whether to validate data against schema before insertion.
         skip_validation_on_retry: Whether to skip validation for individual records during retries.
     """
+
     allow_errors: Optional[int] = None
     allow_errors_ratio: Optional[float] = None
     strategy: Literal["fail-fast", "discard", "isolate"] = "fail-fast"
     validate: bool = True
     skip_validation_on_retry: bool = False
 
+
 @dataclass
 class FailedRecord(Generic[T]):
     """Represents a failed record during insertion with error details.
@@ -47,10 +62,12 @@ class FailedRecord(Generic[T]):
         error: The error message describing why the insertion failed.
         index: Optional index of this record in the original batch.
     """
+
     record: T
     error: str
     index: Optional[int] = None
 
+
 @dataclass
 class ValidationError:
     """Validation error for a record with detailed error information.
@@ -61,11 +78,13 @@ class ValidationError:
         index: Optional index of this record in the original batch.
         path: Optional path to the field that failed validation.
     """
+
     record: Any
     error: str
     index: Optional[int] = None
     path: Optional[str] = None
 
+
 @dataclass
 class ValidationResult(Generic[T]):
     """Result of data validation with success/failure breakdown.
@@ -75,10 +94,12 @@ class ValidationResult(Generic[T]):
         invalid: Records that failed validation with detailed error information.
         total: Total number of records processed.
     """
+
     valid: List[T]
     invalid: List[ValidationError]
     total: int
 
+
 @dataclass
 class InsertResult(Generic[T]):
     """Result of an insert operation with detailed success/failure information.
@@ -89,20 +110,32 @@ class InsertResult(Generic[T]):
         total: Total number of records processed.
         failed_records: Detailed information about failed records (if record isolation was used).
     """
+
     successful: int
     failed: int
     total: int
     failed_records: Optional[List[FailedRecord[T]]] = None
 
+
 class OlapConfig(BaseModel):
     model_config = {"extra": "forbid"}  # Reject unknown fields for a clean API
-    
+
     """Configuration for OLAP tables (e.g., ClickHouse tables).
 
     Attributes:
         order_by_fields: List of column names to use for the ORDER BY clause.
                        Crucial for `ReplacingMergeTree` and performance.
+        order_by_expression: An arbitrary ClickHouse expression for ORDER BY. Example:
+                             `order_by_expression="(id, name)"` is equivalent to order_by_fields=["id", "name"], or
+                             "tuple()" for no sorting.
         partition_by: Optional PARTITION BY expression (single ClickHouse SQL expression).
+        sample_by_expression: Optional SAMPLE BY expression for data sampling (single ClickHouse SQL expression).
+                              Used to enable efficient approximate query processing with SAMPLE clause.
+        primary_key_expression: Optional PRIMARY KEY expression. When specified, this overrides the primary key
+                               inferred from Key[T] column annotations. This allows for complex primary keys using
+                               functions (e.g., "cityHash64(id)") or different column ordering in primary key vs
+                               schema definition. Note: When this is set, any Key[T] annotations on columns are
+                               ignored for PRIMARY KEY generation.
         engine: The ClickHouse table engine to use. Can be either a ClickHouseEngines enum value
                 (for backward compatibility) or an EngineConfig instance (recommended).
         version: Optional version string for tracking configuration changes.
@@ -110,14 +143,113 @@ class OlapConfig(BaseModel):
         life_cycle: Determines how changes in code will propagate to the resources.
         settings: Optional table-level settings that can be modified with ALTER TABLE MODIFY SETTING.
                   These are alterable settings that can be changed without recreating the table.
+        cluster: Optional cluster name for ON CLUSTER support in ClickHouse.
+                 Use this to enable replicated tables across ClickHouse clusters.
+                 The cluster must be defined in moose.config.toml (dev environment only).
+                 Example: cluster="prod_cluster"
     """
     order_by_fields: list[str] = []
+    order_by_expression: Optional[str] = None
     partition_by: Optional[str] = None
+    sample_by_expression: Optional[str] = None
+    primary_key_expression: Optional[str] = None
     engine: Optional[Union[ClickHouseEngines, EngineConfig]] = None
     version: Optional[str] = None
     metadata: Optional[dict] = None
     life_cycle: Optional[LifeCycle] = None
     settings: Optional[dict[str, str]] = None
+    # Optional table-level TTL expression (without leading 'TTL')
+    ttl: Optional[str] = None
+    # Optional cluster name for ON CLUSTER support in ClickHouse
+    cluster: Optional[str] = None
+
+    # Optional secondary/data-skipping indexes
+    class TableIndex(BaseModel):
+        name: str
+        expression: str
+        type: str
+        arguments: list[str] = []
+        granularity: int = 1
+
+    indexes: list[TableIndex] = []
+    database: Optional[str] = None  # Optional database name for multi-database support
+
+    def model_post_init(self, __context):
+        has_fields = bool(self.order_by_fields)
+        has_expr = (
+            isinstance(self.order_by_expression, str)
+            and len(self.order_by_expression) > 0
+        )
+        if has_fields and has_expr:
+            raise ValueError(
+                "Provide either order_by_fields or order_by_expression, not both."
+            )
+
+        # Validate that non-MergeTree engines don't have unsupported clauses
+        if self.engine:
+            from ..blocks import (
+                S3Engine,
+                S3QueueEngine,
+                BufferEngine,
+                DistributedEngine,
+                IcebergS3Engine,
+                KafkaEngine,
+            )
+
+            # S3QueueEngine, BufferEngine, DistributedEngine, KafkaEngine, and IcebergS3Engine don't support ORDER BY
+            # Note: S3Engine DOES support ORDER BY (unlike S3Queue)
+            engines_without_order_by = (
+                S3QueueEngine,
+                BufferEngine,
+                DistributedEngine,
+                KafkaEngine,
+                IcebergS3Engine,
+            )
+            if isinstance(self.engine, engines_without_order_by):
+                engine_name = type(self.engine).__name__
+
+                if has_fields or has_expr:
+                    raise ValueError(
+                        f"{engine_name} does not support ORDER BY clauses. "
+                        f"Remove order_by_fields or order_by_expression from your configuration."
+                    )
+
+            # All non-MergeTree engines don't support SAMPLE BY
+            engines_without_sample_by = (
+                S3Engine,
+                S3QueueEngine,
+                BufferEngine,
+                DistributedEngine,
+                KafkaEngine,
+                IcebergS3Engine,
+            )
+            if isinstance(self.engine, engines_without_sample_by):
+                engine_name = type(self.engine).__name__
+
+                if self.sample_by_expression:
+                    raise ValueError(
+                        f"{engine_name} does not support SAMPLE BY clause. "
+                        f"Remove sample_by_expression from your configuration."
+                    )
+
+            # Only S3QueueEngine, BufferEngine, DistributedEngine, KafkaEngine, and IcebergS3Engine don't support PARTITION BY
+            # S3Engine DOES support PARTITION BY
+            engines_without_partition_by = (
+                S3QueueEngine,
+                BufferEngine,
+                DistributedEngine,
+                KafkaEngine,
+                IcebergS3Engine,
+            )
+            if isinstance(self.engine, engines_without_partition_by):
+                engine_name = type(self.engine).__name__
+
+                if self.partition_by:
+                    raise ValueError(
+                        f"{engine_name} does not support PARTITION BY clause. "
+                        f"Remove partition_by from your configuration."
+                    )
+
 
 class OlapTable(TypedMooseResource, Generic[T]):
     """Represents an OLAP table (e.g., a ClickHouse table) typed with a Pydantic model.
@@ -134,6 +266,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
         model_type (type[T]): The Pydantic model associated with this table.
         kind: The kind of the table (e.g., "OlapTable").
     """
+
     config: OlapConfig
     kind: str = "OlapTable"
     _memoized_client: Optional[Client] = None
@@ -146,15 +279,77 @@ class OlapTable(TypedMooseResource, Generic[T]):
         super().__init__()
         self._set_type(name, self._get_type(kwargs))
         self.config = config
-        self.metadata = config.metadata
+
+        if config.metadata:
+            self.metadata = (
+                config.metadata.copy()
+                if isinstance(config.metadata, dict)
+                else config.metadata
+            )
+        else:
+            self.metadata = {}
+
+        if not isinstance(self.metadata, dict):
+            self.metadata = {}
+        if "source" not in self.metadata:
+            source_file = get_source_file_from_stack()
+            if source_file:
+                self.metadata["source"] = {"file": source_file}
+
         self._column_list = _to_columns(self._t)
+
+        # Create Cols instance for backward compatibility
+        # This works for both BaseModel and MooseModel
         self._cols = Cols(self._column_list)
-        _tables[name] = self
-        
+
+        # NOTE: For MooseModel types, columns are also accessible directly
+        # on the model class (e.g., MyModel.field_name) thanks to the metaclass.
+        # This provides LSP autocomplete without requiring .cols access.
+
+        registry_key = f"{name}_{config.version}" if config.version else name
+        if registry_key in _tables:
+            raise ValueError(
+                f"OlapTable with name {name} and version {config.version or 'unversioned'} already exists"
+            )
+        _tables[registry_key] = self
+
+        # Validate cluster and explicit replication params are not both specified
+        if config.cluster:
+            from moose_lib.blocks import (
+                ReplicatedMergeTreeEngine,
+                ReplicatedReplacingMergeTreeEngine,
+                ReplicatedAggregatingMergeTreeEngine,
+                ReplicatedSummingMergeTreeEngine,
+                ReplicatedCollapsingMergeTreeEngine,
+                ReplicatedVersionedCollapsingMergeTreeEngine,
+            )
+
+            if isinstance(
+                config.engine,
+                (
+                    ReplicatedMergeTreeEngine,
+                    ReplicatedReplacingMergeTreeEngine,
+                    ReplicatedAggregatingMergeTreeEngine,
+                    ReplicatedSummingMergeTreeEngine,
+                    ReplicatedCollapsingMergeTreeEngine,
+                    ReplicatedVersionedCollapsingMergeTreeEngine,
+                ),
+            ):
+                if (
+                    config.engine.keeper_path is not None
+                    or config.engine.replica_name is not None
+                ):
+                    raise ValueError(
+                        f"OlapTable {name}: Cannot specify both 'cluster' and explicit replication params "
+                        f"('keeper_path' or 'replica_name'). "
+                        f"Use 'cluster' for auto-injected params, or use explicit 'keeper_path' and "
+                        f"'replica_name' without 'cluster'."
+                    )
+
         # Check if using legacy enum-based engine configuration
         if config.engine and isinstance(config.engine, ClickHouseEngines):
             logger = Logger(action="OlapTable")
-            
+
             # Special case for S3Queue - more detailed migration message
             if config.engine == ClickHouseEngines.S3Queue:
                 logger.highlight(
@@ -174,13 +369,13 @@ class OlapTable(TypedMooseResource, Generic[T]):
                     f"  New: from moose_lib.blocks import {engine_name}Engine; engine={engine_name}Engine()\n"
                     "The new API provides better type safety and configuration options."
                 )
-            
+
             # Also emit a Python warning for development environments
             warnings.warn(
                 f"Table '{name}' uses deprecated ClickHouseEngines enum. "
                 f"Please migrate to engine configuration classes (e.g., {config.engine.value}Engine).",
                 DeprecationWarning,
-                stacklevel=2
+                stacklevel=2,
             )
 
     @property
@@ -217,10 +412,13 @@ class OlapTable(TypedMooseResource, Generic[T]):
             A 16-character hex hash of the configuration.
         """
         import hashlib
+
+        # Use per-table database if specified, otherwise fall back to global config
+        effective_database = self.config.database or clickhouse_config.database
         config_string = (
             f"{clickhouse_config.host}:{clickhouse_config.port}:"
             f"{clickhouse_config.username}:{clickhouse_config.password}:"
-            f"{clickhouse_config.database}:{clickhouse_config.use_ssl}"
+            f"{effective_database}:{clickhouse_config.use_ssl}"
         )
         return hashlib.sha256(config_string.encode()).hexdigest()[:16]
 
@@ -255,14 +453,16 @@ class OlapTable(TypedMooseResource, Generic[T]):
 
         try:
             # Create new client with standard configuration
-            interface = 'https' if clickhouse_config.use_ssl else 'http'
+            # Use per-table database if specified, otherwise fall back to global config
+            effective_database = self.config.database or clickhouse_config.database
+            interface = "https" if clickhouse_config.use_ssl else "http"
             client = get_client(
                 interface=interface,
                 host=clickhouse_config.host,
                 port=int(clickhouse_config.port),
                 username=clickhouse_config.username,
                 password=clickhouse_config.password,
-                database=clickhouse_config.database,
+                database=effective_database,
             )
 
             # Cache the new client and config hash
@@ -298,7 +498,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
         Returns:
             Tuple of (validated_data, error_message). If validation succeeds,
             validated_data will be the validated record and error_message will be None.
-            If validation fails for any reason, validated_data will be None and error_message 
+            If validation fails for any reason, validated_data will be None and error_message
             will contain the error details.
         """
         try:
@@ -324,23 +524,19 @@ class OlapTable(TypedMooseResource, Generic[T]):
             if validated is not None:
                 valid.append(validated)
             else:
-                invalid.append(ValidationError(
-                    record=record,
-                    error=error or "Validation failed",
-                    index=i,
-                    path="root"
-                ))
-
-        return ValidationResult(
-            valid=valid,
-            invalid=invalid,
-            total=len(data)
-        )
+                invalid.append(
+                    ValidationError(
+                        record=record,
+                        error=error or "Validation failed",
+                        index=i,
+                        path="root",
+                    )
+                )
+
+        return ValidationResult(valid=valid, invalid=invalid, total=len(data))
 
     def _validate_insert_parameters(
-        self,
-        data: Union[List[T], Iterator[T]],
-        options: Optional[InsertOptions]
+        self, data: Union[List[T], Iterator[T]], options: Optional[InsertOptions]
     ) -> Tuple[bool, str, bool]:
         """Validate input parameters and strategy compatibility.
 
@@ -362,7 +558,9 @@ class OlapTable(TypedMooseResource, Generic[T]):
             )
 
         if is_stream and should_validate:
-            print("Warning: Validation is not supported with stream input. Validation will be skipped.")
+            print(
+                "Warning: Validation is not supported with stream input. Validation will be skipped."
+            )
 
         return is_stream, strategy, should_validate
 
@@ -371,7 +569,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
         data: List[T],
         should_validate: bool,
         strategy: str,
-        options: Optional[InsertOptions] = None
+        options: Optional[InsertOptions] = None,
     ) -> Tuple[List[T], List[ValidationError]]:
         """Perform pre-insertion validation for array data.
 
@@ -394,10 +592,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
 
             if validation_errors:
                 self._handle_validation_errors(
-                    validation_errors,
-                    strategy,
-                    data,
-                    options
+                    validation_errors, strategy, data, options
                 )
 
                 if strategy == "discard":
@@ -420,7 +615,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
         validation_errors: List[ValidationError],
         strategy: str,
         data: List[T],
-        options: Optional[InsertOptions]
+        options: Optional[InsertOptions],
     ) -> None:
         """Handle validation errors based on the specified strategy.
 
@@ -436,17 +631,13 @@ class OlapTable(TypedMooseResource, Generic[T]):
                 f"Validation failed for record at index {first_error.index}: {first_error.error}"
             )
         elif strategy == "discard":
-            self._check_validation_thresholds(
-                validation_errors,
-                len(data),
-                options
-            )
+            self._check_validation_thresholds(validation_errors, len(data), options)
 
     def _check_validation_thresholds(
         self,
         validation_errors: List[ValidationError],
         total_records: int,
-        options: Optional[InsertOptions]
+        options: Optional[InsertOptions],
     ) -> None:
         """Check if validation errors exceed configured thresholds.
 
@@ -458,15 +649,21 @@ class OlapTable(TypedMooseResource, Generic[T]):
         validation_failed_count = len(validation_errors)
         validation_failed_ratio = validation_failed_count / total_records
 
-        if (options and options.allow_errors is not None and
-            validation_failed_count > options.allow_errors):
+        if (
+            options
+            and options.allow_errors is not None
+            and validation_failed_count > options.allow_errors
+        ):
             raise ValueError(
                 f"Too many validation failures: {validation_failed_count} > {options.allow_errors}. "
                 f"Errors: {', '.join(e.error for e in validation_errors)}"
             )
 
-        if (options and options.allow_errors_ratio is not None and
-            validation_failed_ratio > options.allow_errors_ratio):
+        if (
+            options
+            and options.allow_errors_ratio is not None
+            and validation_failed_ratio > options.allow_errors_ratio
+        ):
             raise ValueError(
                 f"Validation failure ratio too high: {validation_failed_ratio:.3f} > "
                 f"{options.allow_errors_ratio}. Errors: {', '.join(e.error for e in validation_errors)}"
@@ -477,10 +674,10 @@ class OlapTable(TypedMooseResource, Generic[T]):
 
     def _with_wait_end_settings(self, settings: dict) -> dict:
         """Add wait_end_of_query setting to ensure at least once delivery for INSERT operations.
-        
+
         Args:
             settings: Base settings dictionary
-            
+
         Returns:
             Settings dictionary with wait_end_of_query added
         """
@@ -493,20 +690,28 @@ class OlapTable(TypedMooseResource, Generic[T]):
         validated_data: List[T],
         is_stream: bool,
         strategy: str,
-        options: Optional[InsertOptions]
+        options: Optional[InsertOptions],
     ) -> tuple[str, bytes, dict]:
         """Prepare insert options for JSONEachRow raw SQL insert, returning settings dict."""
         # Base settings for all inserts
         base_settings = {
             "date_time_input_format": "best_effort",
-            "max_insert_block_size": 100000 if is_stream else min(len(validated_data), 100000),
+            "max_insert_block_size": (
+                100000 if is_stream else min(len(validated_data), 100000)
+            ),
             "max_block_size": 65536,
             "async_insert": 1 if len(validated_data) > 1000 else 0,
             "wait_for_async_insert": 1,
         }
         settings = self._with_wait_end_settings(base_settings)
-        if (strategy == "discard" and options and
-            (options.allow_errors is not None or options.allow_errors_ratio is not None)):
+        if (
+            strategy == "discard"
+            and options
+            and (
+                options.allow_errors is not None
+                or options.allow_errors_ratio is not None
+            )
+        ):
             if options.allow_errors is not None:
                 settings["input_format_allow_errors_num"] = options.allow_errors
             if options.allow_errors_ratio is not None:
@@ -519,7 +724,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
             validated_data = [validated_data]
         dict_data = []
         for record in validated_data:
-            if hasattr(record, 'model_dump'):
+            if hasattr(record, "model_dump"):
                 record_dict = record.model_dump()
             else:
                 record_dict = record
@@ -537,7 +742,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
         validation_errors: List[ValidationError],
         is_stream: bool,
         should_validate: bool,
-        strategy: str
+        strategy: str,
     ) -> InsertResult[T]:
         """Create appropriate result based on input type.
 
@@ -553,11 +758,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
             InsertResult with appropriate counts and error information.
         """
         if is_stream:
-            return InsertResult(
-                successful=-1,
-                failed=0,
-                total=-1
-            )
+            return InsertResult(successful=-1, failed=0, total=-1)
 
         inserted_count = len(validated_data)
         total_processed = len(data) if not is_stream else inserted_count
@@ -565,32 +766,30 @@ class OlapTable(TypedMooseResource, Generic[T]):
         result = InsertResult(
             successful=inserted_count,
             failed=len(validation_errors) if should_validate else 0,
-            total=total_processed
+            total=total_processed,
         )
 
-        if (should_validate and validation_errors and strategy == "discard"):
+        if should_validate and validation_errors and strategy == "discard":
             result.failed_records = [
                 FailedRecord(
                     record=ve.record,
                     error=f"Validation error: {ve.error}",
-                    index=ve.index
-                ) for ve in validation_errors
+                    index=ve.index,
+                )
+                for ve in validation_errors
             ]
 
         return result
 
     def _retry_individual_records(
-        self,
-        client: Client,
-        records: List[T],
-        options: InsertOptions
+        self, client: Client, records: List[T], options: InsertOptions
     ) -> InsertResult[T]:
         successful: List[T] = []
         failed: List[FailedRecord[T]] = []
         table_name = quote_identifier(self._generate_table_name())
         records_dict = []
         for record in records:
-            if hasattr(record, 'model_dump'):
+            if hasattr(record, "model_dump"):
                 record_dict = record.model_dump()
             else:
                 record_dict = record
@@ -599,41 +798,42 @@ class OlapTable(TypedMooseResource, Generic[T]):
 
         RETRY_BATCH_SIZE = 10
         for i in range(0, len(records_dict), RETRY_BATCH_SIZE):
-            batch = records_dict[i:i + RETRY_BATCH_SIZE]
+            batch = records_dict[i : i + RETRY_BATCH_SIZE]
             try:
                 sql = f"INSERT INTO {table_name} FORMAT JSONEachRow"
                 base_settings = {
                     "date_time_input_format": "best_effort",
                     "max_insert_block_size": RETRY_BATCH_SIZE,
                     "max_block_size": RETRY_BATCH_SIZE,
-                    "async_insert": 0
+                    "async_insert": 0,
                 }
                 settings = self._with_wait_end_settings(base_settings)
                 json_lines = self._to_json_each_row(batch)
                 client.command(sql, data=json_lines, settings=settings)
-                successful.extend(records[i:i + RETRY_BATCH_SIZE])
+                successful.extend(records[i : i + RETRY_BATCH_SIZE])
             except ClickHouseError as batch_error:
                 for j, record_dict in enumerate(batch):
                     try:
                         sql = f"INSERT INTO {table_name} FORMAT JSONEachRow"
-                        individual_settings = self._with_wait_end_settings({
-                            "date_time_input_format": "best_effort",
-                            "async_insert": 0
-                        })
+                        individual_settings = self._with_wait_end_settings(
+                            {"date_time_input_format": "best_effort", "async_insert": 0}
+                        )
                         json_line = self._to_json_each_row([record_dict])
-                        client.command(sql, data=json_line, settings=individual_settings)
+                        client.command(
+                            sql, data=json_line, settings=individual_settings
+                        )
                         successful.append(records[i + j])
                     except ClickHouseError as error:
-                        failed.append(FailedRecord(
-                            record=records[i + j],
-                            error=str(error),
-                            index=i + j
-                        ))
+                        failed.append(
+                            FailedRecord(
+                                record=records[i + j], error=str(error), index=i + j
+                            )
+                        )
         return InsertResult(
             successful=len(successful),
             failed=len(failed),
             total=len(records),
-            failed_records=failed if failed else None
+            failed_records=failed if failed else None,
         )
 
     def _insert_array_data(
@@ -643,7 +843,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
         data: List[T],
         should_validate: bool,
         strategy: str,
-        options: Optional[InsertOptions]
+        options: Optional[InsertOptions],
     ) -> InsertResult[T]:
         """Insert array data into the table with validation and error handling.
 
@@ -659,19 +859,11 @@ class OlapTable(TypedMooseResource, Generic[T]):
             InsertResult with detailed success/failure information.
         """
         validated_data, validation_errors = self._perform_pre_insertion_validation(
-            data,
-            should_validate,
-            strategy,
-            options
+            data, should_validate, strategy, options
         )
         try:
             table_name, json_lines, settings = self._prepare_insert_options(
-                table_name,
-                data,
-                validated_data,
-                False,
-                strategy,
-                options
+                table_name, data, validated_data, False, strategy, options
             )
             sql = f"INSERT INTO {table_name} FORMAT JSONEachRow"
             client.command(sql, data=json_lines, settings=settings)
@@ -681,7 +873,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
                 validation_errors,
                 False,
                 should_validate,
-                strategy
+                strategy,
             )
         except ClickHouseError as e:
             if strategy == "fail-fast":
@@ -692,7 +884,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
                 return self._retry_individual_records(
                     client,
                     validated_data if not options.skip_validation_on_retry else data,
-                    options
+                    options,
                 )
 
     def _insert_stream(
@@ -701,7 +893,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
         table_name: str,
         data: Iterator[T],
         strategy: str,
-        options: Optional[InsertOptions]
+        options: Optional[InsertOptions],
     ) -> InsertResult[T]:
         """Insert data from an iterator into the table.
 
@@ -719,17 +911,12 @@ class OlapTable(TypedMooseResource, Generic[T]):
             total_inserted = 0
 
             _, _, settings = self._prepare_insert_options(
-                table_name,
-                data,
-                [],
-                True,
-                strategy,
-                options
+                table_name, data, [], True, strategy, options
             )
 
             for record in data:
                 # Convert record to dict using model_dump if available
-                if hasattr(record, 'model_dump'):
+                if hasattr(record, "model_dump"):
                     batch.append(record.model_dump())
                 else:
                     batch.append(record)
@@ -754,9 +941,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
                 total_inserted += len(batch)
 
             return InsertResult(
-                successful=total_inserted,
-                failed=0,
-                total=total_inserted
+                successful=total_inserted, failed=0, total=total_inserted
             )
         except ClickHouseError as e:
             if strategy == "fail-fast":
@@ -764,9 +949,7 @@ class OlapTable(TypedMooseResource, Generic[T]):
             raise ValueError(f"Too many errors during stream insert: {e}")
 
     def insert(
-        self,
-        data: Union[List[T], Iterator[T]],
-        options: Optional[InsertOptions] = None
+        self, data: Union[List[T], Iterator[T]], options: Optional[InsertOptions] = None
     ) -> InsertResult[T]:
         """Insert data into the table with validation and error handling.
 
@@ -818,7 +1001,9 @@ class OlapTable(TypedMooseResource, Generic[T]):
             ```
         """
         options = options or InsertOptions()
-        is_stream, strategy, should_validate = self._validate_insert_parameters(data, options)
+        is_stream, strategy, should_validate = self._validate_insert_parameters(
+            data, options
+        )
         if (is_stream and not data) or (not is_stream and not data):
             return InsertResult(successful=0, failed=0, total=0)
 
@@ -829,15 +1014,12 @@ class OlapTable(TypedMooseResource, Generic[T]):
             return self._insert_stream(client, table_name, data, strategy, options)
         else:
             return self._insert_array_data(
-                client,
-                table_name,
-                data,
-                should_validate,
-                strategy,
-                options
+                client, table_name, data, should_validate, strategy, options
             )
 
-    def _map_to_clickhouse_record(self, record: dict, columns: Optional[List[Column]] = None) -> dict:
+    def _map_to_clickhouse_record(
+        self, record: dict, columns: Optional[List[Column]] = None
+    ) -> dict:
         """
         Recursively transforms a record to match ClickHouse's JSONEachRow requirements.
 
@@ -866,8 +1048,9 @@ class OlapTable(TypedMooseResource, Generic[T]):
 
             if is_array_nested_type(data_type):
                 # For Array(Nested(...)), wrap each item in its own array and recurse
-                if (isinstance(value, list) and
-                    (len(value) == 0 or isinstance(value[0], dict))):
+                if isinstance(value, list) and (
+                    len(value) == 0 or isinstance(value[0], dict)
+                ):
                     nested_columns = data_type.element_type.columns
                     result[col.name] = [
                         [self._map_to_clickhouse_record(item, nested_columns)]
@@ -876,7 +1059,9 @@ class OlapTable(TypedMooseResource, Generic[T]):
             elif is_nested_type(data_type):
                 # For Nested struct (not array), recurse into it
                 if value and isinstance(value, dict):
-                    result[col.name] = self._map_to_clickhouse_record(value, data_type.columns)
+                    result[col.name] = self._map_to_clickhouse_record(
+                        value, data_type.columns
+                    )
             # All other types: leave as is
 
-        return result
+        return result