duckrun 0.2.16.dev0__py3-none-any.whl → 0.2.16.dev2__py3-none-any.whl

duckrun/runner.py

@@ -7,45 +7,7 @@ import importlib.util
 from typing import List, Tuple, Dict, Optional, Callable, Any
 from string import Template
 from deltalake import DeltaTable, write_deltalake
-# Row Group configuration for optimal Delta Lake performance
-RG = 8_000_000
-
-
-def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=None):
-    """
-    Build arguments for write_deltalake based on requirements:
-    - If schema_mode='merge': use rust engine (no row group params)
-    - Otherwise: use pyarrow engine with row group optimization (if supported)
-    """
-    args = {
-        'table_or_uri': path,
-        'data': df,
-        'mode': mode
-    }
-    
-    # Add partition_by if specified
-    if partition_by:
-        args['partition_by'] = partition_by
-    
-    # Engine selection based on schema_mode
-    if schema_mode == 'merge':
-        # Use rust engine for schema merging (no row group params supported)
-        args['schema_mode'] = 'merge'
-        args['engine'] = 'rust'
-    else:
-        # Try to use pyarrow engine with row group optimization
-        # Check if row group parameters are supported by inspecting function signature
-        import inspect
-        sig = inspect.signature(write_deltalake)
-        
-        if 'max_rows_per_file' in sig.parameters:
-            # Older deltalake version - use row group optimization
-            args['max_rows_per_file'] = RG
-            args['max_rows_per_group'] = RG
-            args['min_rows_per_group'] = RG
-        # For newer versions, just use default parameters
-    
-    return args
+from .writer import _build_write_deltalake_args
 
 
 def run(duckrun_instance, pipeline: List[Tuple]) -> bool:

duckrun/writer.py

@@ -3,6 +3,20 @@ Delta Lake writer functionality for duckrun - Spark-style write API
 """
 from deltalake import DeltaTable, write_deltalake, __version__ as deltalake_version
 
+# Try to import WriterProperties for Rust engine (available in 0.18.2+)
+try:
+    from deltalake.writer import WriterProperties
+    _HAS_WRITER_PROPERTIES = True
+except ImportError:
+    _HAS_WRITER_PROPERTIES = False
+
+# Try to import PyArrow dataset for old PyArrow engine
+try:
+    import pyarrow.dataset as ds
+    _HAS_PYARROW_DATASET = True
+except ImportError:
+    _HAS_PYARROW_DATASET = False
+
 
 # Row Group configuration for optimal Delta Lake performance
 RG = 8_000_000
@@ -23,12 +37,14 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
     - Has max_rows_per_file/max_rows_per_group/min_rows_per_group for optimization
     - When mergeSchema=True: must set schema_mode='merge' + engine='rust', NO row group params
     - When mergeSchema=False: use row group params, DON'T set engine (pyarrow is default)
+    - COMPRESSION: Defaults to ZSTD via writer_properties (rust) or file_options (pyarrow)
     
     deltalake 0.20+:
     - Does NOT have 'engine' parameter (everything is rust, pyarrow deprecated)
     - Does NOT have max_rows_per_file (row group optimization removed)
     - When mergeSchema=True: must set schema_mode='merge'
     - When mergeSchema=False: just write normally (no special params)
+    - COMPRESSION: Defaults to ZSTD via writer_properties (rust only)
     
     Uses version detection for simpler logic.
     """
@@ -50,7 +66,13 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
             # deltalake 0.18.2-0.19.x: must also set engine='rust' for schema merging
             # Do NOT use row group params (they conflict with rust engine)
             args['engine'] = 'rust'
-        # For version 0.20+: just schema_mode='merge' is enough, rust is default
+            # Set ZSTD compression for Rust engine
+            if _HAS_WRITER_PROPERTIES:
+                args['writer_properties'] = WriterProperties(compression='ZSTD')
+        else:
+            # Version 0.20+: rust is default, just add compression
+            if _HAS_WRITER_PROPERTIES:
+                args['writer_properties'] = WriterProperties(compression='ZSTD')
     else:
         # Normal write mode (no schema merging)
         if _IS_OLD_DELTALAKE:
@@ -59,7 +81,14 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
             args['max_rows_per_file'] = RG
             args['max_rows_per_group'] = RG
             args['min_rows_per_group'] = RG
-        # For version 0.20+: no optimization available (rust by default, no row group params supported)
+            # Set ZSTD compression for PyArrow engine
+            if _HAS_PYARROW_DATASET:
+                args['file_options'] = ds.ParquetFileFormat().make_write_options(compression='ZSTD')
+        else:
+            # Version 0.20+: no optimization available (rust by default, no row group params supported)
+            # Set ZSTD compression for Rust engine
+            if _HAS_WRITER_PROPERTIES:
+                args['writer_properties'] = WriterProperties(compression='ZSTD')
     
     return args
 
@@ -135,14 +164,14 @@ class DeltaWriter:
         # Prepare info message based on version and settings
         if self._schema_mode == 'merge':
             if _IS_OLD_DELTALAKE:
-                engine_info = " (engine=rust, schema_mode=merge)"
+                engine_info = " (engine=rust, schema_mode=merge, compression=ZSTD)"
             else:
-                engine_info = " (schema_mode=merge, rust by default)"
+                engine_info = " (schema_mode=merge, rust by default, compression=ZSTD)"
         else:
             if _IS_OLD_DELTALAKE:
-                engine_info = " (engine=pyarrow, optimized row groups)"
+                engine_info = " (engine=pyarrow, optimized row groups, compression=ZSTD)"
             else:
-                engine_info = " (engine=rust by default)"
+                engine_info = " (engine=rust by default, compression=ZSTD)"
         
         partition_info = f" partitioned by {self._partition_by}" if self._partition_by else ""
         print(f"Writing to Delta table: {schema}.{table} (mode={self._mode}){engine_info}{partition_info}")

{duckrun-0.2.16.dev0.dist-info → duckrun-0.2.16.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.16.dev0
+Version: 0.2.16.dev2
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT

{duckrun-0.2.16.dev0.dist-info → duckrun-0.2.16.dev2.dist-info}/RECORD

@@ -3,12 +3,12 @@ duckrun/auth.py,sha256=EMaf-L2zeNOjbHOT97xYxfZNfWo4WrwrU1h3vBQTgEc,9624
 duckrun/core.py,sha256=c98sASAWlq0DDIR9gYbj5ZaKOa6MoO8Z09qhRhG4JWI,67097
 duckrun/files.py,sha256=Fvdjg3DyHJzIVzKo8M_j-eGz4zU61lOB38Y_onbQJkI,10137
 duckrun/lakehouse.py,sha256=j--Z3zo8AOWt1GF9VzRosmmTAy6ey2D0LVubti58twU,14109
-duckrun/runner.py,sha256=JnRJoQ_Db__iXlhjTohplXR83NUJxItgyaa7AzrDxwE,14833
+duckrun/runner.py,sha256=NGVyerJA44UP2umRdndfL0fuFM_gdOZmuJUz-PLOFf0,13461
 duckrun/semantic_model.py,sha256=obzlN2-dbEW3JmDop-vrZGGGLi9u3ThhTbgtDjou7uY,29509
 duckrun/stats.py,sha256=EqrCN1xwGo5nZgwezBvb6RepXT6b8H7xgK0yJJGFLfE,15155
-duckrun/writer.py,sha256=svUuPCYOhrz299NgnpTKhARKjfej0PxnoND2iPDSypk,8098
-duckrun-0.2.16.dev0.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
-duckrun-0.2.16.dev0.dist-info/METADATA,sha256=CembRLJLoYfx6NS_kmtRDVuORH-E32EYPrq7kQ2yHmY,20771
-duckrun-0.2.16.dev0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-duckrun-0.2.16.dev0.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
-duckrun-0.2.16.dev0.dist-info/RECORD,,
+duckrun/writer.py,sha256=wIsU77DSj4J7d9_bIhvk6AbC51uUrLW0e6pcSPQOY1c,9424
+duckrun-0.2.16.dev2.dist-info/licenses/LICENSE,sha256=-DeQQwdbCbkB4507ZF3QbocysB-EIjDtaLexvqRkGZc,1083
+duckrun-0.2.16.dev2.dist-info/METADATA,sha256=zx6oKCdIOOZuIF4yhK2u8UjgNh16KG03m2dmXGMf90c,20771
+duckrun-0.2.16.dev2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+duckrun-0.2.16.dev2.dist-info/top_level.txt,sha256=BknMEwebbUHrVAp3SC92ps8MPhK7XSYsaogTvi_DmEU,8
+duckrun-0.2.16.dev2.dist-info/RECORD,,