moose-lib 0.6.161__tar.gz → 0.6.162__tar.gz

{moose_lib-0.6.161 → moose_lib-0.6.162}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.161
+Version: 0.6.162
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

{moose_lib-0.6.161 → moose_lib-0.6.162}/moose_lib/blocks.py

@@ -14,6 +14,9 @@ class ClickHouseEngines(Enum):
     VersionedCollapsingMergeTree = "VersionedCollapsingMergeTree"
     GraphiteMergeTree = "GraphiteMergeTree"
     S3Queue = "S3Queue"
+    S3 = "S3"
+    Buffer = "Buffer"
+    Distributed = "Distributed"
     ReplicatedMergeTree = "ReplicatedMergeTree"
     ReplicatedReplacingMergeTree = "ReplicatedReplacingMergeTree"
     ReplicatedAggregatingMergeTree = "ReplicatedAggregatingMergeTree"
@@ -177,6 +180,110 @@ class S3QueueEngine(EngineConfig):
         if not self.format:
             raise ValueError("S3Queue engine requires 'format'")
 
+@dataclass
+class S3Engine(EngineConfig):
+    """Configuration for S3 engine - direct read/write from S3 storage.
+    
+    Args:
+        path: S3 path to the data file(s) (e.g., 's3://bucket/path/file.json')
+        format: Data format (e.g., 'JSONEachRow', 'CSV', 'Parquet')
+        aws_access_key_id: AWS access key ID (optional, omit for public buckets)
+        aws_secret_access_key: AWS secret access key (optional, omit for public buckets)
+        compression: Compression type (e.g., 'gzip', 'zstd', 'auto')
+        partition_strategy: Optional partition strategy
+        partition_columns_in_data_file: Optional partition columns in data file
+    """
+    
+    # Required fields
+    path: str
+    format: str
+    
+    # Optional fields
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    compression: Optional[str] = None
+    partition_strategy: Optional[str] = None
+    partition_columns_in_data_file: Optional[str] = None
+    
+    def __post_init__(self):
+        """Validate required fields"""
+        if not self.path:
+            raise ValueError("S3 engine requires 'path'")
+        if not self.format:
+            raise ValueError("S3 engine requires 'format'")
+
+@dataclass
+class BufferEngine(EngineConfig):
+    """Configuration for Buffer engine - in-memory buffer that flushes to a destination table.
+    
+    Args:
+        target_database: Target database name for the destination table
+        target_table: Target table name where data will be flushed
+        num_layers: Number of buffer layers (typically 16)
+        min_time: Minimum time in seconds before flushing
+        max_time: Maximum time in seconds before flushing
+        min_rows: Minimum number of rows before flushing
+        max_rows: Maximum number of rows before flushing
+        min_bytes: Minimum bytes before flushing
+        max_bytes: Maximum bytes before flushing
+        flush_time: Optional flush time in seconds
+        flush_rows: Optional flush number of rows
+        flush_bytes: Optional flush number of bytes
+    """
+    
+    # Required fields
+    target_database: str
+    target_table: str
+    num_layers: int
+    min_time: int
+    max_time: int
+    min_rows: int
+    max_rows: int
+    min_bytes: int
+    max_bytes: int
+    
+    # Optional fields
+    flush_time: Optional[int] = None
+    flush_rows: Optional[int] = None
+    flush_bytes: Optional[int] = None
+    
+    def __post_init__(self):
+        """Validate required fields"""
+        if not self.target_database:
+            raise ValueError("Buffer engine requires 'target_database'")
+        if not self.target_table:
+            raise ValueError("Buffer engine requires 'target_table'")
+
+@dataclass
+class DistributedEngine(EngineConfig):
+    """Configuration for Distributed engine - distributed table across a cluster.
+    
+    Args:
+        cluster: Cluster name from the ClickHouse configuration
+        target_database: Database name on the cluster
+        target_table: Table name on the cluster
+        sharding_key: Optional sharding key expression for data distribution
+        policy_name: Optional policy name for data distribution
+    """
+    
+    # Required fields
+    cluster: str
+    target_database: str
+    target_table: str
+    
+    # Optional fields
+    sharding_key: Optional[str] = None
+    policy_name: Optional[str] = None
+    
+    def __post_init__(self):
+        """Validate required fields"""
+        if not self.cluster:
+            raise ValueError("Distributed engine requires 'cluster'")
+        if not self.target_database:
+            raise ValueError("Distributed engine requires 'target_database'")
+        if not self.target_table:
+            raise ValueError("Distributed engine requires 'target_table'")
+
 # ==========================
 # New Table Configuration (Recommended API)
 # ==========================

{moose_lib-0.6.161 → moose_lib-0.6.162}/moose_lib/dmv2/olap_table.py

@@ -151,6 +151,39 @@ class OlapConfig(BaseModel):
         if has_fields and has_expr:
             raise ValueError("Provide either order_by_fields or order_by_expression, not both.")
 
+        # Validate that non-MergeTree engines don't have unsupported clauses
+        if self.engine:
+            from ..blocks import S3Engine, S3QueueEngine, BufferEngine, DistributedEngine
+
+            # All non-MergeTree engines don't support ORDER BY
+            non_mergetree_engines = (S3Engine, S3QueueEngine, BufferEngine, DistributedEngine)
+            if isinstance(self.engine, non_mergetree_engines):
+                engine_name = type(self.engine).__name__
+
+                if has_fields or has_expr:
+                    raise ValueError(
+                        f"{engine_name} does not support ORDER BY clauses. "
+                        f"Remove order_by_fields or order_by_expression from your configuration."
+                    )
+
+                if self.sample_by_expression:
+                    raise ValueError(
+                        f"{engine_name} does not support SAMPLE BY clause. "
+                        f"Remove sample_by_expression from your configuration."
+                    )
+
+            # Only S3QueueEngine, BufferEngine, and DistributedEngine don't support PARTITION BY
+            # S3Engine DOES support PARTITION BY
+            engines_without_partition_by = (S3QueueEngine, BufferEngine, DistributedEngine)
+            if isinstance(self.engine, engines_without_partition_by):
+                engine_name = type(self.engine).__name__
+
+                if self.partition_by:
+                    raise ValueError(
+                        f"{engine_name} does not support PARTITION BY clause. "
+                        f"Remove partition_by from your configuration."
+                    )
+
 
 class OlapTable(TypedMooseResource, Generic[T]):
     """Represents an OLAP table (e.g., a ClickHouse table) typed with a Pydantic model.

{moose_lib-0.6.161 → moose_lib-0.6.162}/moose_lib/internal.py

@@ -128,6 +128,45 @@ class S3QueueConfigDict(BaseEngineConfigDict):
     headers: Optional[Dict[str, str]] = None
 
 
+class S3ConfigDict(BaseEngineConfigDict):
+    """Configuration for S3 engine."""
+    engine: Literal["S3"] = "S3"
+    path: str
+    format: str
+    aws_access_key_id: Optional[str] = None
+    aws_secret_access_key: Optional[str] = None
+    compression: Optional[str] = None
+    partition_strategy: Optional[str] = None
+    partition_columns_in_data_file: Optional[str] = None
+
+
+class BufferConfigDict(BaseEngineConfigDict):
+    """Configuration for Buffer engine."""
+    engine: Literal["Buffer"] = "Buffer"
+    target_database: str
+    target_table: str
+    num_layers: int
+    min_time: int
+    max_time: int
+    min_rows: int
+    max_rows: int
+    min_bytes: int
+    max_bytes: int
+    flush_time: Optional[int] = None
+    flush_rows: Optional[int] = None
+    flush_bytes: Optional[int] = None
+
+
+class DistributedConfigDict(BaseEngineConfigDict):
+    """Configuration for Distributed engine."""
+    engine: Literal["Distributed"] = "Distributed"
+    cluster: str
+    target_database: str
+    target_table: str
+    sharding_key: Optional[str] = None
+    policy_name: Optional[str] = None
+
+
 # Discriminated union of all engine configurations
 EngineConfigDict = Union[
     MergeTreeConfigDict,
@@ -138,7 +177,10 @@ EngineConfigDict = Union[
     ReplicatedReplacingMergeTreeConfigDict,
     ReplicatedAggregatingMergeTreeConfigDict,
     ReplicatedSummingMergeTreeConfigDict,
-    S3QueueConfigDict
+    S3QueueConfigDict,
+    S3ConfigDict,
+    BufferConfigDict,
+    DistributedConfigDict
 ]
 
 
@@ -464,7 +506,7 @@ def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineCon
     Returns:
         EngineConfigDict with engine-specific configuration
     """
-    from moose_lib.blocks import S3QueueEngine
+    from moose_lib.blocks import S3QueueEngine, S3Engine, BufferEngine, DistributedEngine
 
     # Try S3Queue first
     if isinstance(engine, S3QueueEngine):
@@ -476,6 +518,45 @@ def _convert_engine_instance_to_config_dict(engine: "EngineConfig") -> EngineCon
             compression=engine.compression,
             headers=engine.headers
         )
+    
+    # Try S3
+    if isinstance(engine, S3Engine):
+        return S3ConfigDict(
+            path=engine.path,
+            format=engine.format,
+            aws_access_key_id=engine.aws_access_key_id,
+            aws_secret_access_key=engine.aws_secret_access_key,
+            compression=engine.compression,
+            partition_strategy=engine.partition_strategy,
+            partition_columns_in_data_file=engine.partition_columns_in_data_file
+        )
+    
+    # Try Buffer
+    if isinstance(engine, BufferEngine):
+        return BufferConfigDict(
+            target_database=engine.target_database,
+            target_table=engine.target_table,
+            num_layers=engine.num_layers,
+            min_time=engine.min_time,
+            max_time=engine.max_time,
+            min_rows=engine.min_rows,
+            max_rows=engine.max_rows,
+            min_bytes=engine.min_bytes,
+            max_bytes=engine.max_bytes,
+            flush_time=engine.flush_time,
+            flush_rows=engine.flush_rows,
+            flush_bytes=engine.flush_bytes
+        )
+    
+    # Try Distributed
+    if isinstance(engine, DistributedEngine):
+        return DistributedConfigDict(
+            cluster=engine.cluster,
+            target_database=engine.target_database,
+            target_table=engine.target_table,
+            sharding_key=engine.sharding_key,
+            policy_name=engine.policy_name
+        )
 
     # Try basic engines
     basic_config = _convert_basic_engine_instance(engine)

{moose_lib-0.6.161 → moose_lib-0.6.162}/moose_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.6.161
+Version: 0.6.162
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

{moose_lib-0.6.161 → moose_lib-0.6.162}/tests/test_s3queue_config.py

@@ -8,7 +8,7 @@ import warnings
 from moose_lib import OlapTable, OlapConfig, ClickHouseEngines
 from moose_lib.blocks import S3QueueEngine, MergeTreeEngine, ReplacingMergeTreeEngine
 from moose_lib.internal import (
-    _convert_engine_to_config_dict, 
+    _convert_engine_to_config_dict,
     EngineConfigDict,
     S3QueueConfigDict,
     MergeTreeConfigDict,
@@ -41,8 +41,8 @@ def test_olap_config_accepts_engine_config():
         aws_secret_access_key="secret123"
     )
     config = OlapConfig(
-        engine=s3_engine,
-        order_by_fields=["timestamp"]
+        engine=s3_engine
+        # Note: S3QueueEngine does not support order_by_fields
     )
     assert isinstance(config.engine, S3QueueEngine)
     assert config.engine.s3_path == "s3://bucket/data/*.json"
@@ -58,14 +58,14 @@ def test_olap_table_with_s3queue_engine():
                 format="JSONEachRow",
                 compression="gzip"
             ),
-            order_by_fields=["timestamp", "id"],
+            # Note: S3QueueEngine does not support order_by_fields
             settings={
                 "s3queue_mode": "unordered",
                 "s3queue_keeper_path": "/clickhouse/s3queue/test"
             }
         )
     )
-    
+
     assert table.name == "TestS3Table"
     assert isinstance(table.config.engine, S3QueueEngine)
     assert table.config.engine.s3_path == "s3://test-bucket/logs/*.json"
@@ -83,7 +83,7 @@ def test_olap_table_with_mergetree_engines():
         )
     )
     assert isinstance(table1.config.engine, MergeTreeEngine)
-    
+
     # Test with ReplacingMergeTreeEngine
     table2 = OlapTable[SampleEvent](
         "ReplacingTable",
@@ -111,10 +111,10 @@ def test_engine_conversion_to_dict():
             )
         )
     )
-    
+
     # Convert engine to dict
     engine_dict = _convert_engine_to_config_dict(table.config.engine, table)
-    
+
     assert engine_dict.engine == "S3Queue"
     assert engine_dict.s3_path == "s3://bucket/data/*.parquet"
     assert engine_dict.format == "Parquet"
@@ -133,10 +133,10 @@ def test_engine_conversion_with_enum():
             engine=ClickHouseEngines.ReplacingMergeTree
         )
     )
-    
+
     # Convert engine to dict
     engine_dict = _convert_engine_to_config_dict(table.config.engine, table)
-    
+
     assert engine_dict.engine == "ReplacingMergeTree"
 
 
@@ -150,16 +150,16 @@ def test_backward_compatibility():
             order_by_fields=["id"]
         )
     )
-    
+
     # New API with EngineConfig
     new_table = OlapTable[SampleEvent](
-        "NewTable", 
+        "NewTable",
         OlapConfig(
             engine=MergeTreeEngine(),
             order_by_fields=["id"]
         )
     )
-    
+
     # Both should work
     assert old_table.config.engine == ClickHouseEngines.MergeTree
     assert isinstance(new_table.config.engine, MergeTreeEngine)
@@ -169,7 +169,7 @@ def test_deprecation_warning_for_enum():
     """Test that using enum engine triggers deprecation warning."""
     with warnings.catch_warnings(record=True) as w:
         warnings.simplefilter("always")
-        
+
         table = OlapTable[SampleEvent](
             "LegacyTable",
             OlapConfig(
@@ -177,7 +177,7 @@ def test_deprecation_warning_for_enum():
                 order_by_fields=["id"]
             )
         )
-        
+
         # Check that a deprecation warning was issued
         assert len(w) == 1
         assert issubclass(w[0].category, DeprecationWarning)
@@ -198,12 +198,12 @@ def test_s3queue_with_all_options():
             "Authorization": "Bearer token"
         }
     )
-    
+
     table = OlapTable[SampleEvent](
         "FullConfigTable",
         OlapConfig(
             engine=engine,
-            order_by_fields=["timestamp"],
+            # Note: S3QueueEngine does not support order_by_fields
             settings={
                 "s3queue_mode": "ordered",
                 "s3queue_keeper_path": "/clickhouse/s3queue/full",
@@ -212,7 +212,7 @@ def test_s3queue_with_all_options():
             }
         )
     )
-    
+
     assert table.config.engine.s3_path == "s3://my-bucket/path/to/data/*.json"
     assert table.config.engine.format == "JSONEachRow"
     assert table.config.engine.compression == "gzip"
@@ -226,42 +226,49 @@ def test_s3queue_public_bucket():
         format="Parquet"
         # No AWS credentials needed for public buckets
     )
-    
+
     table = OlapTable[SampleEvent](
         "PublicBucketTable",
         OlapConfig(
-            engine=engine,
-            order_by_fields=["id"]
+            engine=engine
+            # Note: S3QueueEngine does not support order_by_fields
         )
     )
-    
+
     assert table.config.engine.aws_access_key_id is None
     assert table.config.engine.aws_secret_access_key is None
 
 
 def test_migration_from_legacy_to_new():
     """Test migration path from legacy to new API."""
-    # Legacy approach
+    # Legacy approach (with MergeTree, which supports order_by_fields)
     legacy_config = OlapConfig(
-        engine=ClickHouseEngines.S3Queue,
+        engine=ClickHouseEngines.MergeTree,
         order_by_fields=["timestamp"]
     )
-    
-    # New approach - equivalent configuration
+
+    # New approach - equivalent configuration for MergeTree
     new_config = OlapConfig(
-        engine=S3QueueEngine(
-            s3_path="s3://bucket/data/*.json",
-            format="JSONEachRow"
-        ),
+        engine=MergeTreeEngine(),
         order_by_fields=["timestamp"]
     )
-    
+
     # Both should have the same order_by_fields
     assert legacy_config.order_by_fields == new_config.order_by_fields
-    
+
     # Engine types should be different
     assert isinstance(legacy_config.engine, ClickHouseEngines)
-    assert isinstance(new_config.engine, S3QueueEngine)
+    assert isinstance(new_config.engine, MergeTreeEngine)
+
+    # For S3Queue, the new API correctly prevents unsupported clauses
+    s3queue_config = OlapConfig(
+        engine=S3QueueEngine(
+            s3_path="s3://bucket/data/*.json",
+            format="JSONEachRow"
+        )
+        # Note: order_by_fields is not supported for S3QueueEngine
+    )
+    assert isinstance(s3queue_config.engine, S3QueueEngine)
 
 
 def test_engine_config_validation():
@@ -272,7 +279,7 @@ def test_engine_config_validation():
             s3_path="",  # Empty path should fail
             format="JSONEachRow"
         )
-    
+
     with pytest.raises(ValueError, match="S3Queue engine requires 'format'"):
         S3QueueEngine(
             s3_path="s3://bucket/data/*.json",
@@ -280,21 +287,143 @@ def test_engine_config_validation():
         )
 
 
+def test_non_mergetree_engines_reject_unsupported_clauses():
+    """Test that non-MergeTree engines reject unsupported ORDER BY and SAMPLE BY clauses."""
+    from moose_lib.blocks import S3Engine, S3QueueEngine, BufferEngine, DistributedEngine
+
+    # Test S3Engine rejects ORDER BY
+    with pytest.raises(ValueError, match="S3Engine does not support ORDER BY clauses"):
+        OlapConfig(
+            engine=S3Engine(path="s3://bucket/file.json", format="JSONEachRow"),
+            order_by_fields=["id"]
+        )
+
+    with pytest.raises(ValueError, match="S3Engine does not support ORDER BY clauses"):
+        OlapConfig(
+            engine=S3Engine(path="s3://bucket/file.json", format="JSONEachRow"),
+            order_by_expression="(id, name)"
+        )
+
+    # Test S3Engine rejects SAMPLE BY
+    with pytest.raises(ValueError, match="S3Engine does not support SAMPLE BY clause"):
+        OlapConfig(
+            engine=S3Engine(path="s3://bucket/file.json", format="JSONEachRow"),
+            sample_by_expression="cityHash64(id)"
+        )
+
+    # Test S3Engine DOES support PARTITION BY (should not raise)
+    config_s3_with_partition = OlapConfig(
+        engine=S3Engine(path="s3://bucket/file.json", format="JSONEachRow"),
+        partition_by="toYYYYMM(timestamp)"
+    )
+    assert config_s3_with_partition.partition_by == "toYYYYMM(timestamp)"
+
+    # Test S3QueueEngine rejects ORDER BY
+    with pytest.raises(ValueError, match="S3QueueEngine does not support ORDER BY clauses"):
+        OlapConfig(
+            engine=S3QueueEngine(s3_path="s3://bucket/*.json", format="JSONEachRow"),
+            order_by_fields=["id"]
+        )
+
+    # Test S3QueueEngine rejects PARTITION BY (unlike S3Engine)
+    with pytest.raises(ValueError, match="S3QueueEngine does not support PARTITION BY clause"):
+        OlapConfig(
+            engine=S3QueueEngine(s3_path="s3://bucket/*.json", format="JSONEachRow"),
+            partition_by="toYYYYMM(timestamp)"
+        )
+
+    # Test BufferEngine rejects ORDER BY
+    with pytest.raises(ValueError, match="BufferEngine does not support ORDER BY clauses"):
+        OlapConfig(
+            engine=BufferEngine(
+                target_database="default",
+                target_table="dest",
+                num_layers=16,
+                min_time=10,
+                max_time=100,
+                min_rows=10000,
+                max_rows=100000,
+                min_bytes=10000000,
+                max_bytes=100000000
+            ),
+            order_by_fields=["id"]
+        )
+
+    # Test BufferEngine rejects PARTITION BY
+    with pytest.raises(ValueError, match="BufferEngine does not support PARTITION BY clause"):
+        OlapConfig(
+            engine=BufferEngine(
+                target_database="default",
+                target_table="dest",
+                num_layers=16,
+                min_time=10,
+                max_time=100,
+                min_rows=10000,
+                max_rows=100000,
+                min_bytes=10000000,
+                max_bytes=100000000
+            ),
+            partition_by="date"
+        )
+
+    # Test DistributedEngine rejects PARTITION BY
+    with pytest.raises(ValueError, match="DistributedEngine does not support PARTITION BY clause"):
+        OlapConfig(
+            engine=DistributedEngine(
+                cluster="my_cluster",
+                target_database="default",
+                target_table="local_table"
+            ),
+            partition_by="date"
+        )
+
+    # Verify that S3Engine works without unsupported clauses
+    config = OlapConfig(
+        engine=S3Engine(path="s3://bucket/file.json", format="JSONEachRow")
+    )
+    assert isinstance(config.engine, S3Engine)
+
+
+def test_mergetree_engines_still_accept_clauses():
+    """Test that MergeTree engines still accept ORDER BY, PARTITION BY, and SAMPLE BY clauses."""
+    from moose_lib.blocks import MergeTreeEngine, ReplacingMergeTreeEngine
+
+    # MergeTree should accept all clauses
+    config1 = OlapConfig(
+        engine=MergeTreeEngine(),
+        order_by_fields=["id", "timestamp"],
+        partition_by="toYYYYMM(timestamp)",
+        sample_by_expression="cityHash64(id)"
+    )
+    assert config1.order_by_fields == ["id", "timestamp"]
+    assert config1.partition_by == "toYYYYMM(timestamp)"
+    assert config1.sample_by_expression == "cityHash64(id)"
+
+    # ReplacingMergeTree should also accept these clauses
+    config2 = OlapConfig(
+        engine=ReplacingMergeTreeEngine(ver="updated_at"),
+        order_by_expression="(id, name)",
+        partition_by="date"
+    )
+    assert config2.order_by_expression == "(id, name)"
+    assert config2.partition_by == "date"
+
+
 def test_multiple_engine_types():
     """Test that different engine types can be used in the same application."""
     tables = []
-    
+
     # Create tables with different engine types
     tables.append(OlapTable[SampleEvent](
         "MergeTreeTable",
         OlapConfig(engine=MergeTreeEngine())
     ))
-    
+
     tables.append(OlapTable[SampleEvent](
         "ReplacingTreeTable",
         OlapConfig(engine=ReplacingMergeTreeEngine())
     ))
-    
+
     tables.append(OlapTable[SampleEvent](
         "S3QueueTable",
         OlapConfig(
@@ -304,7 +433,7 @@ def test_multiple_engine_types():
             )
         )
     ))
-    
+
     # Verify all tables were created with correct engine types
     assert isinstance(tables[0].config.engine, MergeTreeEngine)
     assert isinstance(tables[1].config.engine, ReplacingMergeTreeEngine)