moose-lib 0.6.43__py3-none-any.whl → 0.6.44__py3-none-any.whl

moose_lib/blocks.py

@@ -1,6 +1,8 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
-from typing import Dict, Optional
+from typing import Dict, List, Optional, Any, Union
+from abc import ABC
+import warnings
 
 
 class ClickHouseEngines(Enum):
@@ -11,6 +13,128 @@ class ClickHouseEngines(Enum):
     CollapsingMergeTree = "CollapsingMergeTree"
     VersionedCollapsingMergeTree = "VersionedCollapsingMergeTree"
     GraphiteMergeTree = "GraphiteMergeTree"
+    S3Queue = "S3Queue"
+
+# ==========================
+# New Engine Configuration Classes
+# ==========================
+
+@dataclass
+class EngineConfig(ABC):
+    """Base class for engine configurations"""
+    pass
+
+@dataclass
+class MergeTreeEngine(EngineConfig):
+    """Configuration for MergeTree engine"""
+    pass
+
+@dataclass  
+class ReplacingMergeTreeEngine(EngineConfig):
+    """Configuration for ReplacingMergeTree engine (with deduplication)"""
+    pass
+
+@dataclass
+class AggregatingMergeTreeEngine(EngineConfig):
+    """Configuration for AggregatingMergeTree engine"""
+    pass
+
+@dataclass
+class SummingMergeTreeEngine(EngineConfig):
+    """Configuration for SummingMergeTree engine"""
+    pass
+
+@dataclass
+class S3QueueEngine(EngineConfig):
+    """Configuration for S3Queue engine - only non-alterable constructor parameters.
+    
+    S3Queue-specific settings like 'mode', 'keeper_path', etc. should be specified
+    in the settings field of OlapConfig, not here.
+    """
+    
+    # Required fields
+    s3_path: str  # S3 bucket path with wildcards (e.g., 's3://bucket/prefix/*.json')
+    format: str   # Data format (e.g., 'JSONEachRow', 'CSV', 'Parquet')
+    
+    # Optional AWS credentials
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    
+    # Optional configuration
+    compression: Optional[str] = None  # e.g., 'gzip', 'zstd'
+    headers: Optional[Dict[str, str]] = None
+    
+    def __post_init__(self):
+        """Validate required fields"""
+        if not self.s3_path:
+            raise ValueError("S3Queue engine requires 's3_path'")
+        if not self.format:
+            raise ValueError("S3Queue engine requires 'format'")
+
+# ==========================
+# New Table Configuration (Recommended API)
+# ==========================
+
+@dataclass
+class TableConfig:
+    """Modern table configuration with engine-specific settings"""
+    
+    # Engine configuration (required in new API)
+    engine: EngineConfig
+    
+    # Common settings
+    name: str
+    columns: Dict[str, str]
+    order_by: Optional[str] = None
+    
+    @classmethod
+    def with_s3_queue(cls,
+                      name: str,
+                      columns: Dict[str, str],
+                      s3_path: str,
+                      format: str,
+                      order_by: Optional[str] = None,
+                      **kwargs) -> 'TableConfig':
+        """Create a table with S3Queue engine"""
+        return cls(
+            name=name,
+            columns=columns,
+            engine=S3QueueEngine(s3_path=s3_path, format=format, **kwargs),
+            order_by=order_by
+        )
+    
+    @classmethod
+    def with_merge_tree(cls,
+                        name: str,
+                        columns: Dict[str, str],
+                        order_by: Optional[str] = None,
+                        deduplicate: bool = False,
+                        **kwargs) -> 'TableConfig':
+        """Create a table with MergeTree or ReplacingMergeTree engine"""
+        engine = ReplacingMergeTreeEngine() if deduplicate else MergeTreeEngine()
+        return cls(
+            name=name,
+            columns=columns,
+            engine=engine,
+            order_by=order_by
+        )
+
+# ==========================
+# Legacy API Support (Deprecated)
+# ==========================
+
+@dataclass
+class S3QueueEngineConfig:
+    """Legacy S3Queue configuration (deprecated - use S3QueueEngine instead)"""
+    path: str  # S3 path pattern (e.g., 's3://bucket/data/*.json')
+    format: str  # Data format (e.g., 'JSONEachRow', 'CSV', etc.)
+    # Optional S3 access credentials - can be NOSIGN for public buckets
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    # Optional compression
+    compression: Optional[str] = None
+    # Optional headers
+    headers: Optional[Dict[str, str]] = None
 
 @dataclass
 class TableCreateOptions:
@@ -18,6 +142,82 @@ class TableCreateOptions:
     columns: Dict[str, str]
     engine: Optional[ClickHouseEngines] = ClickHouseEngines.MergeTree
     order_by: Optional[str] = None
+    s3_queue_engine_config: Optional[S3QueueEngineConfig] = None  # Required when engine is S3Queue
+
+    def __post_init__(self):
+        """Validate S3Queue configuration"""
+        if self.engine == ClickHouseEngines.S3Queue and self.s3_queue_engine_config is None:
+            raise ValueError(
+                "s3_queue_engine_config is required when using ClickHouseEngines.S3Queue engine. "
+                "Please provide s3_queue_engine_config with path, format, and optional settings."
+            )
+
+# ==========================
+# Backward Compatibility Layer
+# ==========================
+
+def is_new_config(config: Any) -> bool:
+    """Check if configuration uses new API"""
+    if isinstance(config, TableConfig):
+        return True
+    if hasattr(config, 'engine') and isinstance(getattr(config, 'engine'), EngineConfig):
+        return True
+    return False
+
+def migrate_legacy_config(legacy: TableCreateOptions) -> TableConfig:
+    """Convert legacy configuration to new format"""
+    
+    # Show deprecation warning
+    warnings.warn(
+        "Using deprecated TableCreateOptions. Please migrate to TableConfig:\n"
+        "- For S3Queue: Use TableConfig.with_s3_queue()\n"
+        "- For deduplication: Use engine=ReplacingMergeTreeEngine()\n"
+        "See documentation for examples.",
+        DeprecationWarning,
+        stacklevel=2
+    )
+    
+    # Handle S3Queue with separate config
+    if legacy.engine == ClickHouseEngines.S3Queue and legacy.s3_queue_engine_config:
+        s3_config = legacy.s3_queue_engine_config
+        return TableConfig(
+            name=legacy.name,
+            columns=legacy.columns,
+            engine=S3QueueEngine(
+                s3_path=s3_config.path,
+                format=s3_config.format,
+                aws_access_key_id=s3_config.aws_access_key_id,
+                aws_secret_access_key=s3_config.aws_secret_access_key,
+                compression=s3_config.compression,
+                headers=s3_config.headers
+            ),
+            order_by=legacy.order_by
+        )
+    
+    # Map legacy engine enum to new engine classes
+    engine_map = {
+        ClickHouseEngines.MergeTree: MergeTreeEngine(),
+        ClickHouseEngines.ReplacingMergeTree: ReplacingMergeTreeEngine(),
+        ClickHouseEngines.AggregatingMergeTree: AggregatingMergeTreeEngine(),
+        ClickHouseEngines.SummingMergeTree: SummingMergeTreeEngine(),
+    }
+    
+    engine = engine_map.get(legacy.engine) if legacy.engine else MergeTreeEngine()
+    if engine is None:
+        engine = MergeTreeEngine()
+    
+    return TableConfig(
+        name=legacy.name,
+        columns=legacy.columns,
+        engine=engine,
+        order_by=legacy.order_by
+    )
+
+def normalize_config(config: Union[TableConfig, TableCreateOptions]) -> TableConfig:
+    """Normalize any configuration format to new API"""
+    if is_new_config(config):
+        return config  # type: ignore
+    return migrate_legacy_config(config)  # type: ignore
 
 @dataclass
 class AggregationCreateOptions:
@@ -93,7 +293,7 @@ def create_table(options: TableCreateOptions) -> str:
     """
     column_definitions = ",\n".join([f"{name} {type}" for name, type in options.columns.items()])
     order_by_clause = f"ORDER BY {options.order_by}" if options.order_by else ""
-    engine = options.engine.value
+    engine = options.engine.value if options.engine else "MergeTree"
 
     return f"""
     CREATE TABLE IF NOT EXISTS {options.name} 

moose_lib/dmv2/materialized_view.py

@@ -96,7 +96,6 @@ class MaterializedView(SqlResource, BaseTypedResource, Generic[T]):
 
         setup = [
             f"CREATE MATERIALIZED VIEW IF NOT EXISTS {quote_identifier(options.materialized_view_name)} TO {quote_identifier(target_table.name)} AS {options.select_statement}",
-            f"INSERT INTO {quote_identifier(target_table.name)} {options.select_statement}"
         ]
         teardown = [f"DROP VIEW IF EXISTS {quote_identifier(options.materialized_view_name)}"]
 

moose_lib/dmv2/olap_table.py

@@ -5,6 +5,7 @@ This module provides classes for defining and configuring OLAP tables,
 particularly for ClickHouse.
 """
 import json
+import warnings
 from clickhouse_connect import get_client
 from clickhouse_connect.driver.client import Client
 from clickhouse_connect.driver.exceptions import ClickHouseError
@@ -12,6 +13,8 @@ from dataclasses import dataclass
 from pydantic import BaseModel
 from typing import List, Optional, Any, Literal, Union, Tuple, TypeVar, Generic, Iterator
 from moose_lib import ClickHouseEngines
+from ..blocks import EngineConfig  # Add import for EngineConfig
+from ..commons import Logger  # Add import for Moose logging
 from ..config.runtime import RuntimeClickHouseConfig
 from ..utilities.sql import quote_identifier
 from .types import TypedMooseResource, T
@@ -93,21 +96,27 @@ class InsertResult(Generic[T]):
     failed_records: Optional[List[FailedRecord[T]]] = None
 
 class OlapConfig(BaseModel):
+    model_config = {"extra": "forbid"}  # Reject unknown fields for a clean API
+    
     """Configuration for OLAP tables (e.g., ClickHouse tables).
 
     Attributes:
         order_by_fields: List of column names to use for the ORDER BY clause.
                        Crucial for `ReplacingMergeTree` and performance.
-        engine: The ClickHouse table engine to use (e.g., MergeTree, ReplacingMergeTree).
+        engine: The ClickHouse table engine to use. Can be either a ClickHouseEngines enum value
+                (for backward compatibility) or an EngineConfig instance (recommended).
         version: Optional version string for tracking configuration changes.
         metadata: Optional metadata for the table.
         life_cycle: Determines how changes in code will propagate to the resources.
+        settings: Optional table-level settings that can be modified with ALTER TABLE MODIFY SETTING.
+                  These are alterable settings that can be changed without recreating the table.
     """
     order_by_fields: list[str] = []
-    engine: Optional[ClickHouseEngines] = None
+    engine: Optional[Union[ClickHouseEngines, EngineConfig]] = None
     version: Optional[str] = None
     metadata: Optional[dict] = None
     life_cycle: Optional[LifeCycle] = None
+    settings: Optional[dict[str, str]] = None
 
 class OlapTable(TypedMooseResource, Generic[T]):
     """Represents an OLAP table (e.g., a ClickHouse table) typed with a Pydantic model.
@@ -136,6 +145,38 @@ class OlapTable(TypedMooseResource, Generic[T]):
         self.config = config
         self.metadata = config.metadata
         _tables[name] = self
+        
+        # Check if using legacy enum-based engine configuration
+        if config.engine and isinstance(config.engine, ClickHouseEngines):
+            logger = Logger(action="OlapTable")
+            
+            # Special case for S3Queue - more detailed migration message
+            if config.engine == ClickHouseEngines.S3Queue:
+                logger.highlight(
+                    f"Table '{name}' uses legacy S3Queue configuration. "
+                    "Please migrate to the new API:\n"
+                    "  Old: engine=ClickHouseEngines.S3Queue\n"
+                    "  New: engine=S3QueueEngine(s3_path='...', format='...')\n"
+                    "See documentation for complete S3Queue configuration options."
+                )
+            else:
+                # Generic message for other engines
+                engine_name = config.engine.value
+                logger.highlight(
+                    f"Table '{name}' uses legacy engine configuration. "
+                    f"Consider migrating to the new API:\n"
+                    f"  Old: engine=ClickHouseEngines.{engine_name}\n"
+                    f"  New: from moose_lib.blocks import {engine_name}Engine; engine={engine_name}Engine()\n"
+                    "The new API provides better type safety and configuration options."
+                )
+            
+            # Also emit a Python warning for development environments
+            warnings.warn(
+                f"Table '{name}' uses deprecated ClickHouseEngines enum. "
+                f"Please migrate to engine configuration classes (e.g., {config.engine.value}Engine).",
+                DeprecationWarning,
+                stacklevel=2
+            )
 
     def _generate_table_name(self) -> str:
         """Generate the versioned table name following Moose's naming convention.

moose_lib/internal.py

@@ -7,10 +7,11 @@ to convert the user-defined resources (from `dmv2.py`) into a serializable
 JSON format expected by the Moose infrastructure management system.
 """
 from importlib import import_module
-from typing import Literal, Optional, List, Any
+from typing import Literal, Optional, List, Any, Dict, Union, TYPE_CHECKING
 from pydantic import BaseModel, ConfigDict, AliasGenerator, Field
 import json
 from .data_models import Column, _to_columns
+from .blocks import EngineConfig, ClickHouseEngines
 from moose_lib.dmv2 import (
     get_tables,
     get_streams,
@@ -53,6 +54,46 @@ class Consumer(BaseModel):
     """
     version: Optional[str] = None
 
+class BaseEngineConfigDict(BaseModel):
+    """Base engine configuration for all ClickHouse table engines."""
+    model_config = model_config
+    engine: str
+
+class MergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for MergeTree engine."""
+    engine: Literal["MergeTree"] = "MergeTree"
+
+class ReplacingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for ReplacingMergeTree engine."""
+    engine: Literal["ReplacingMergeTree"] = "ReplacingMergeTree"
+
+class AggregatingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for AggregatingMergeTree engine."""
+    engine: Literal["AggregatingMergeTree"] = "AggregatingMergeTree"
+
+class SummingMergeTreeConfigDict(BaseEngineConfigDict):
+    """Configuration for SummingMergeTree engine."""
+    engine: Literal["SummingMergeTree"] = "SummingMergeTree"
+
+class S3QueueConfigDict(BaseEngineConfigDict):
+    """Configuration for S3Queue engine with all specific fields."""
+    engine: Literal["S3Queue"] = "S3Queue"
+    s3_path: str
+    format: str
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    compression: Optional[str] = None
+    headers: Optional[Dict[str, str]] = None
+
+# Discriminated union of all engine configurations
+EngineConfigDict = Union[
+    MergeTreeConfigDict,
+    ReplacingMergeTreeConfigDict,
+    AggregatingMergeTreeConfigDict,
+    SummingMergeTreeConfigDict,
+    S3QueueConfigDict
+]
+
 class TableConfig(BaseModel):
     """Internal representation of an OLAP table configuration for serialization.
 
@@ -60,20 +101,22 @@ class TableConfig(BaseModel):
         name: Name of the table.
         columns: List of columns with their types and attributes.
         order_by: List of columns used for the ORDER BY clause.
-        engine: The name of the ClickHouse engine used.
+        engine_config: Engine configuration with type-safe, engine-specific parameters.
         version: Optional version string of the table configuration.
         metadata: Optional metadata for the table.
         life_cycle: Lifecycle management setting for the table.
+        table_settings: Optional table-level settings that can be modified with ALTER TABLE MODIFY SETTING.
     """
     model_config = model_config
 
     name: str
     columns: List[Column]
     order_by: List[str]
-    engine: Optional[str]
+    engine_config: Optional[EngineConfigDict] = Field(None, discriminator='engine')
     version: Optional[str] = None
     metadata: Optional[dict] = None
     life_cycle: Optional[str] = None
+    table_settings: Optional[Dict[str, str]] = None
 
 class TopicConfig(BaseModel):
     """Internal representation of a stream/topic configuration for serialization.
@@ -249,6 +292,86 @@ def _map_sql_resource_ref(r: Any) -> InfrastructureSignatureJson:
         raise TypeError(f"Object {r} lacks a 'kind' attribute for dependency mapping.")
 
 
+def _convert_engine_to_config_dict(engine: Union[ClickHouseEngines, EngineConfig], table: OlapTable) -> EngineConfigDict:
+    """Convert engine enum or EngineConfig instance to new engine config format.
+    
+    Args:
+        engine: Either a ClickHouseEngines enum value or an EngineConfig instance
+        table: The OlapTable instance with configuration
+        
+    Returns:
+        EngineConfigDict with engine-specific configuration
+    """
+    from moose_lib import ClickHouseEngines
+    from moose_lib.blocks import (
+        EngineConfig, S3QueueEngine, MergeTreeEngine, 
+        ReplacingMergeTreeEngine, AggregatingMergeTreeEngine, 
+        SummingMergeTreeEngine
+    )
+    from moose_lib.commons import Logger
+    
+    # Check if engine is an EngineConfig instance (new API)
+    if isinstance(engine, EngineConfig):
+        if isinstance(engine, S3QueueEngine):
+            return S3QueueConfigDict(
+                s3_path=engine.s3_path,
+                format=engine.format,
+                aws_access_key_id=engine.aws_access_key_id,
+                aws_secret_access_key=engine.aws_secret_access_key,
+                compression=engine.compression,
+                headers=engine.headers
+            )
+        elif isinstance(engine, ReplacingMergeTreeEngine):
+            return ReplacingMergeTreeConfigDict()
+        elif isinstance(engine, AggregatingMergeTreeEngine):
+            return AggregatingMergeTreeConfigDict()
+        elif isinstance(engine, SummingMergeTreeEngine):
+            return SummingMergeTreeConfigDict()
+        elif isinstance(engine, MergeTreeEngine):
+            return MergeTreeConfigDict()
+        else:
+            # Fallback for any other EngineConfig subclass - use base class
+            return BaseEngineConfigDict(engine=engine.__class__.__name__.replace("Engine", ""))
+    
+    # Handle legacy enum-based engine configuration
+    if isinstance(engine, ClickHouseEngines):
+        engine_name = engine.value
+    else:
+        engine_name = str(engine)
+    
+    # For S3Queue with legacy configuration, check for s3_queue_engine_config
+    if engine_name == "S3Queue" and hasattr(table.config, 's3_queue_engine_config'):
+        s3_config = table.config.s3_queue_engine_config
+        if s3_config:
+            logger = Logger(action="S3QueueConfig")
+            logger.highlight(
+                "Using deprecated s3_queue_engine_config. Please migrate to:\n"
+                "  engine=S3QueueEngine(s3_path='...', format='...', ...)"
+            )
+            return S3QueueConfigDict(
+                s3_path=s3_config.path,
+                format=s3_config.format,
+                aws_access_key_id=s3_config.aws_access_key_id,
+                aws_secret_access_key=s3_config.aws_secret_access_key,
+                compression=s3_config.compression,
+                headers=s3_config.headers
+            )
+    
+    # Map engine names to specific config classes
+    engine_map = {
+        "MergeTree": MergeTreeConfigDict,
+        "ReplacingMergeTree": ReplacingMergeTreeConfigDict,
+        "AggregatingMergeTree": AggregatingMergeTreeConfigDict,
+        "SummingMergeTree": SummingMergeTreeConfigDict,
+    }
+    
+    config_class = engine_map.get(engine_name)
+    if config_class:
+        return config_class()
+    
+    # Fallback for unknown engines
+    return BaseEngineConfigDict(engine=engine_name)
+
 def to_infra_map() -> dict:
     """Converts the registered `dmv2` resources into the serializable `InfrastructureMap` format.
 
@@ -268,15 +391,29 @@ def to_infra_map() -> dict:
     workflows = {}
 
     for name, table in get_tables().items():
-        engine = table.config.engine
+        # Convert engine configuration to new format
+        engine_config = None
+        if table.config.engine:
+            engine_config = _convert_engine_to_config_dict(table.config.engine, table)
+        
+        # Get table settings, applying defaults for S3Queue
+        table_settings = table.config.settings.copy() if table.config.settings else {}
+        
+        # Apply default settings for S3Queue if not already specified
+        if engine_config and engine_config.engine == "S3Queue":
+            # Set default mode to 'unordered' if not specified
+            if "mode" not in table_settings:
+                table_settings["mode"] = "unordered"
+        
         tables[name] = TableConfig(
             name=name,
             columns=_to_columns(table._t),
             order_by=table.config.order_by_fields,
-            engine=None if engine is None else engine.value,
+            engine_config=engine_config,
             version=table.config.version,
             metadata=getattr(table, "metadata", None),
             life_cycle=table.config.life_cycle.value if table.config.life_cycle else None,
+            table_settings=table_settings if table_settings else None,  # Map 'settings' to 'table_settings' for internal use
         )
 
     for name, stream in get_streams().items():
@@ -321,10 +458,10 @@ def to_infra_map() -> dict:
                 name=api.config.destination.name
             ),
             metadata=getattr(api, "metadata", None),
-            dead_letter_queue=api.config.dead_letter_queue.name,
             json_schema=api._t.model_json_schema(
                 ref_template='#/components/schemas/{model}'
             ),
+            dead_letter_queue=api.config.dead_letter_queue.name if api.config.dead_letter_queue else None
         )
 
     for name, api in get_apis().items():

{moose_lib-0.6.43.dist-info → moose_lib-0.6.44.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.43
+Version: 0.6.44
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

{moose_lib-0.6.43.dist-info → moose_lib-0.6.44.dist-info}/RECORD

@@ -1,9 +1,9 @@
 moose_lib/__init__.py,sha256=LiUdVqmtASPqbMF53dXVZzCdU1jtFVx62_tiSbW65a0,162
-moose_lib/blocks.py,sha256=_wdvC2NC_Y3MMEnB71WTgWbeQ--zPNHk19xjToJW0C0,3185
+moose_lib/blocks.py,sha256=5QNgaTPC4hfWrIxY5lG4FeNoFP2-JJuFmWjbAVVxSlk,10091
 moose_lib/commons.py,sha256=FUpRv8D3-LeGcjhcqtDyiimz5izwpCq53h50ydxC_uA,3711
 moose_lib/data_models.py,sha256=PgoFXTzf4C66FlKUowMT4VOrwSRR9_bk36P43ub4LzU,10274
 moose_lib/dmv2-serializer.py,sha256=CL_Pvvg8tJOT8Qk6hywDNzY8MYGhMVdTOw8arZi3jng,49
-moose_lib/internal.py,sha256=Mi6Zng5-bsIKbtV9qhoxUV6J9U-mZw0Q0WggiMJ7cXQ,14600
+moose_lib/internal.py,sha256=8LZXIOpaDUviZul3qjtBIlLVCyz7uAmZadrqJBndJzo,20505
 moose_lib/main.py,sha256=XcVX_sTnt5QbrPXKNLCKZGCvpFpE8oiqSG2S1So9ztI,16713
 moose_lib/query_param.py,sha256=kxcR09BMIsEg4o2qetjKrVu1YFRaLfMEzwzyGsKUpvA,6474
 moose_lib/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -17,8 +17,8 @@ moose_lib/dmv2/consumption.py,sha256=f43XPgJn1JLPPEpYbJ1FQ8Efi92YJH7DWa1wb5TY38g
 moose_lib/dmv2/ingest_api.py,sha256=Snek9NGwaJl_BuImSWGtQq91m9D3AJ4qBoGiKZ-9yTQ,2323
 moose_lib/dmv2/ingest_pipeline.py,sha256=2tDM4gCjQMdDRDjvymI5Z1DlkXULx9nY4nG9M3YXpjk,7033
 moose_lib/dmv2/life_cycle.py,sha256=wl0k6yzwU1MJ_fO_UkN29buoY5G6ChYZvfwigP9fVfM,1254
-moose_lib/dmv2/materialized_view.py,sha256=sNwYwNlNNoaYdmlNHBNuJXp7uc4CThxtYkYPNUaeQpw,4968
-moose_lib/dmv2/olap_table.py,sha256=WT2ZQ-G4xwLOfpKSG1Duk-DHlFFTtQQzGiMG8hc-uWg,31525
+moose_lib/dmv2/materialized_view.py,sha256=3JbNLC26p7xOQK0DpgkreWKx7k5V3x3bz9a7kjDqKuU,4876
+moose_lib/dmv2/olap_table.py,sha256=h4aF9hhFy8g0CHiQ_GwtnYqzagi1e4mGei406l8Z8tg,33808
 moose_lib/dmv2/registry.py,sha256=janjYszIXDnMfuwkNhoiBmMLar3umQ3Npw9Ms-u6p-8,2373
 moose_lib/dmv2/sql_resource.py,sha256=kUZoGqxhZMHMthtBZGYJBxTFjXkspXiWLXhJRYXgGUM,1864
 moose_lib/dmv2/stream.py,sha256=jiUWBsjFalLLP63mikOxyHRdieiDAlzf9lXfLye-Wjc,10761
@@ -33,7 +33,8 @@ tests/__init__.py,sha256=0Gh4yzPkkC3TzBGKhenpMIxJcRhyrrCfxLSfpTZnPMQ,53
 tests/conftest.py,sha256=ZVJNbnr4DwbcqkTmePW6U01zAzE6QD0kNAEZjPG1f4s,169
 tests/test_moose.py,sha256=mBsx_OYWmL8ppDzL_7Bd7xR6qf_i3-pCIO3wm2iQNaA,2136
 tests/test_redis_client.py,sha256=d9_MLYsJ4ecVil_jPB2gW3Q5aWnavxmmjZg2uYI3LVo,3256
-moose_lib-0.6.43.dist-info/METADATA,sha256=wadzNkPUXbeVD4YhUJYjR-K3qYge6r6AgScTd4N8EEw,730
-moose_lib-0.6.43.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-moose_lib-0.6.43.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
-moose_lib-0.6.43.dist-info/RECORD,,
+tests/test_s3queue_config.py,sha256=F05cnD61S2wBKPabcpEJxf55-DJGF4nLqwBb6aFbprc,9741
+moose_lib-0.6.44.dist-info/METADATA,sha256=WED0zZnLw2jk6NR90k1PfiR-6Auqmuf3rrHGdOMJpCk,730
+moose_lib-0.6.44.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+moose_lib-0.6.44.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
+moose_lib-0.6.44.dist-info/RECORD,,

tests/test_s3queue_config.py

@@ -0,0 +1,311 @@
+"""Tests for S3Queue engine configuration with the new type hints."""
+
+import pytest
+from pydantic import BaseModel
+from datetime import datetime
+import warnings
+
+from moose_lib import OlapTable, OlapConfig, ClickHouseEngines
+from moose_lib.blocks import S3QueueEngine, MergeTreeEngine, ReplacingMergeTreeEngine
+from moose_lib.internal import (
+    _convert_engine_to_config_dict, 
+    EngineConfigDict,
+    S3QueueConfigDict,
+    MergeTreeConfigDict,
+    ReplacingMergeTreeConfigDict
+)
+
+
+class SampleEvent(BaseModel):
+    """Sample model for S3Queue table tests."""
+    id: str
+    timestamp: datetime
+    message: str
+
+
+def test_olap_config_accepts_enum():
+    """Test that OlapConfig accepts ClickHouseEngines enum values."""
+    config = OlapConfig(
+        engine=ClickHouseEngines.MergeTree,
+        order_by_fields=["id"]
+    )
+    assert config.engine == ClickHouseEngines.MergeTree
+
+
+def test_olap_config_accepts_engine_config():
+    """Test that OlapConfig accepts EngineConfig instances."""
+    s3_engine = S3QueueEngine(
+        s3_path="s3://bucket/data/*.json",
+        format="JSONEachRow",
+        aws_access_key_id="AKIA123",
+        aws_secret_access_key="secret123"
+    )
+    config = OlapConfig(
+        engine=s3_engine,
+        order_by_fields=["timestamp"]
+    )
+    assert isinstance(config.engine, S3QueueEngine)
+    assert config.engine.s3_path == "s3://bucket/data/*.json"
+
+
+def test_olap_table_with_s3queue_engine():
+    """Test creating OlapTable with S3QueueEngine."""
+    table = OlapTable[SampleEvent](
+        "TestS3Table",
+        OlapConfig(
+            engine=S3QueueEngine(
+                s3_path="s3://test-bucket/logs/*.json",
+                format="JSONEachRow",
+                compression="gzip"
+            ),
+            order_by_fields=["timestamp", "id"],
+            settings={
+                "s3queue_mode": "unordered",
+                "s3queue_keeper_path": "/clickhouse/s3queue/test"
+            }
+        )
+    )
+    
+    assert table.name == "TestS3Table"
+    assert isinstance(table.config.engine, S3QueueEngine)
+    assert table.config.engine.s3_path == "s3://test-bucket/logs/*.json"
+    assert table.config.settings["s3queue_mode"] == "unordered"
+
+
+def test_olap_table_with_mergetree_engines():
+    """Test creating OlapTable with various MergeTree engine configs."""
+    # Test with MergeTreeEngine
+    table1 = OlapTable[SampleEvent](
+        "MergeTreeTable",
+        OlapConfig(
+            engine=MergeTreeEngine(),
+            order_by_fields=["id"]
+        )
+    )
+    assert isinstance(table1.config.engine, MergeTreeEngine)
+    
+    # Test with ReplacingMergeTreeEngine
+    table2 = OlapTable[SampleEvent](
+        "ReplacingTable",
+        OlapConfig(
+            engine=ReplacingMergeTreeEngine(),
+            order_by_fields=["id"]
+        )
+    )
+    assert isinstance(table2.config.engine, ReplacingMergeTreeEngine)
+
+
+def test_engine_conversion_to_dict():
+    """Test conversion of engine configs to EngineConfigDict."""
+    # Create a mock table with S3QueueEngine
+    table = OlapTable[SampleEvent](
+        "TestTable",
+        OlapConfig(
+            engine=S3QueueEngine(
+                s3_path="s3://bucket/data/*.parquet",
+                format="Parquet",
+                aws_access_key_id="AKIA456",
+                aws_secret_access_key="secret456",
+                compression="zstd",
+                headers={"X-Custom": "value"}
+            )
+        )
+    )
+    
+    # Convert engine to dict
+    engine_dict = _convert_engine_to_config_dict(table.config.engine, table)
+    
+    assert engine_dict.engine == "S3Queue"
+    assert engine_dict.s3_path == "s3://bucket/data/*.parquet"
+    assert engine_dict.format == "Parquet"
+    assert engine_dict.aws_access_key_id == "AKIA456"
+    assert engine_dict.aws_secret_access_key == "secret456"
+    assert engine_dict.compression == "zstd"
+    assert engine_dict.headers == {"X-Custom": "value"}
+
+
+def test_engine_conversion_with_enum():
+    """Test conversion of enum engines to EngineConfigDict."""
+    # Create a mock table with enum engine
+    table = OlapTable[SampleEvent](
+        "TestTable",
+        OlapConfig(
+            engine=ClickHouseEngines.ReplacingMergeTree
+        )
+    )
+    
+    # Convert engine to dict
+    engine_dict = _convert_engine_to_config_dict(table.config.engine, table)
+    
+    assert engine_dict.engine == "ReplacingMergeTree"
+
+
+def test_backward_compatibility():
+    """Test that both old and new APIs work together."""
+    # Old API with enum
+    old_table = OlapTable[SampleEvent](
+        "OldTable",
+        OlapConfig(
+            engine=ClickHouseEngines.MergeTree,
+            order_by_fields=["id"]
+        )
+    )
+    
+    # New API with EngineConfig
+    new_table = OlapTable[SampleEvent](
+        "NewTable", 
+        OlapConfig(
+            engine=MergeTreeEngine(),
+            order_by_fields=["id"]
+        )
+    )
+    
+    # Both should work
+    assert old_table.config.engine == ClickHouseEngines.MergeTree
+    assert isinstance(new_table.config.engine, MergeTreeEngine)
+
+
+def test_deprecation_warning_for_enum():
+    """Test that using enum engine triggers deprecation warning."""
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        
+        table = OlapTable[SampleEvent](
+            "LegacyTable",
+            OlapConfig(
+                engine=ClickHouseEngines.S3Queue,
+                order_by_fields=["id"]
+            )
+        )
+        
+        # Check that a deprecation warning was issued
+        assert len(w) == 1
+        assert issubclass(w[0].category, DeprecationWarning)
+        assert "deprecated ClickHouseEngines enum" in str(w[0].message)
+        assert "S3QueueEngine" in str(w[0].message)
+
+
+def test_s3queue_with_all_options():
+    """Test S3QueueEngine with all configuration options."""
+    engine = S3QueueEngine(
+        s3_path="s3://my-bucket/path/to/data/*.json",
+        format="JSONEachRow",
+        aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
+        aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        compression="gzip",
+        headers={
+            "X-Custom-Header": "value1",
+            "Authorization": "Bearer token"
+        }
+    )
+    
+    table = OlapTable[SampleEvent](
+        "FullConfigTable",
+        OlapConfig(
+            engine=engine,
+            order_by_fields=["timestamp"],
+            settings={
+                "s3queue_mode": "ordered",
+                "s3queue_keeper_path": "/clickhouse/s3queue/full",
+                "s3queue_loading_retries": "5",
+                "s3queue_processing_threads_num": "8"
+            }
+        )
+    )
+    
+    assert table.config.engine.s3_path == "s3://my-bucket/path/to/data/*.json"
+    assert table.config.engine.format == "JSONEachRow"
+    assert table.config.engine.compression == "gzip"
+    assert table.config.engine.headers["X-Custom-Header"] == "value1"
+
+
+def test_s3queue_public_bucket():
+    """Test S3QueueEngine for public bucket (no credentials)."""
+    engine = S3QueueEngine(
+        s3_path="s3://public-bucket/open-data/*.parquet",
+        format="Parquet"
+        # No AWS credentials needed for public buckets
+    )
+    
+    table = OlapTable[SampleEvent](
+        "PublicBucketTable",
+        OlapConfig(
+            engine=engine,
+            order_by_fields=["id"]
+        )
+    )
+    
+    assert table.config.engine.aws_access_key_id is None
+    assert table.config.engine.aws_secret_access_key is None
+
+
+def test_migration_from_legacy_to_new():
+    """Test migration path from legacy to new API."""
+    # Legacy approach
+    legacy_config = OlapConfig(
+        engine=ClickHouseEngines.S3Queue,
+        order_by_fields=["timestamp"]
+    )
+    
+    # New approach - equivalent configuration
+    new_config = OlapConfig(
+        engine=S3QueueEngine(
+            s3_path="s3://bucket/data/*.json",
+            format="JSONEachRow"
+        ),
+        order_by_fields=["timestamp"]
+    )
+    
+    # Both should have the same order_by_fields
+    assert legacy_config.order_by_fields == new_config.order_by_fields
+    
+    # Engine types should be different
+    assert isinstance(legacy_config.engine, ClickHouseEngines)
+    assert isinstance(new_config.engine, S3QueueEngine)
+
+
+def test_engine_config_validation():
+    """Test that S3QueueEngine validates required fields."""
+    # Test missing required fields
+    with pytest.raises(ValueError, match="S3Queue engine requires 's3_path'"):
+        S3QueueEngine(
+            s3_path="",  # Empty path should fail
+            format="JSONEachRow"
+        )
+    
+    with pytest.raises(ValueError, match="S3Queue engine requires 'format'"):
+        S3QueueEngine(
+            s3_path="s3://bucket/data/*.json",
+            format=""  # Empty format should fail
+        )
+
+
+def test_multiple_engine_types():
+    """Test that different engine types can be used in the same application."""
+    tables = []
+    
+    # Create tables with different engine types
+    tables.append(OlapTable[SampleEvent](
+        "MergeTreeTable",
+        OlapConfig(engine=MergeTreeEngine())
+    ))
+    
+    tables.append(OlapTable[SampleEvent](
+        "ReplacingTreeTable",
+        OlapConfig(engine=ReplacingMergeTreeEngine())
+    ))
+    
+    tables.append(OlapTable[SampleEvent](
+        "S3QueueTable",
+        OlapConfig(
+            engine=S3QueueEngine(
+                s3_path="s3://bucket/*.json",
+                format="JSONEachRow"
+            )
+        )
+    ))
+    
+    # Verify all tables were created with correct engine types
+    assert isinstance(tables[0].config.engine, MergeTreeEngine)
+    assert isinstance(tables[1].config.engine, ReplacingMergeTreeEngine)
+    assert isinstance(tables[2].config.engine, S3QueueEngine)