duckrun 0.2.15__tar.gz → 0.2.16.dev1__tar.gz

{duckrun-0.2.15 → duckrun-0.2.16.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.15
+Version: 0.2.16.dev1
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT

{duckrun-0.2.15 → duckrun-0.2.16.dev1}/duckrun/stats.py

@@ -193,22 +193,26 @@ def get_stats(duckrun_instance, source: str):
                         0 as num_row_groups,
                         0 as size,
                         {vorder} as vorder,
+                        '' as compression,
                         '{timestamp}' as timestamp
                     WHERE false
                 ''')
             else:
-                # Get parquet metadata and create temp table
+                # Get parquet metadata and create temp table with compression info
                 con.execute(f'''
                     CREATE OR REPLACE TEMP TABLE tbl_{idx} AS
                     SELECT 
                         '{tbl}' as tbl,
-                        file_name,
-                        num_rows,
-                        num_row_groups,
+                        fm.file_name,
+                        fm.num_rows,
+                        fm.num_row_groups,
                         CEIL({total_size}/(1024*1024)) as size,
                         {vorder} as vorder,
+                        COALESCE(STRING_AGG(DISTINCT pm.compression, ', ' ORDER BY pm.compression), 'UNCOMPRESSED') as compression,
                         '{timestamp}' as timestamp
-                    FROM parquet_file_metadata({delta})
+                    FROM parquet_file_metadata({delta}) fm
+                    LEFT JOIN parquet_metadata({delta}) pm ON fm.file_name = pm.file_name
+                    GROUP BY fm.file_name, fm.num_rows, fm.num_row_groups
                 ''')
             
         except Exception as e:
@@ -239,6 +243,7 @@ def get_stats(duckrun_instance, source: str):
                             0 as num_row_groups,
                             0 as size,
                             false as vorder,
+                            '' as compression,
                             '{timestamp}' as timestamp
                         WHERE false
                     ''')
@@ -255,18 +260,21 @@ def get_stats(duckrun_instance, source: str):
                             filename = full_path
                         filenames.append(table_path + "/" + filename)
                     
-                    # Use parquet_file_metadata to get actual parquet stats
+                    # Use parquet_file_metadata to get actual parquet stats with compression
                     con.execute(f'''
                         CREATE OR REPLACE TEMP TABLE tbl_{idx} AS
                         SELECT 
                             '{tbl}' as tbl,
-                            file_name,
-                            num_rows,
-                            num_row_groups,
+                            fm.file_name,
+                            fm.num_rows,
+                            fm.num_row_groups,
                             0 as size,
                             false as vorder,
+                            COALESCE(STRING_AGG(DISTINCT pm.compression, ', ' ORDER BY pm.compression), 'UNCOMPRESSED') as compression,
                             '{timestamp}' as timestamp
-                        FROM parquet_file_metadata({filenames})
+                        FROM parquet_file_metadata({filenames}) fm
+                        LEFT JOIN parquet_metadata({filenames}) pm ON fm.file_name = pm.file_name
+                        GROUP BY fm.file_name, fm.num_rows, fm.num_row_groups
                     ''')
                 
                 print(f"   ✓ Successfully processed '{tbl}' using DuckDB fallback with parquet metadata")
@@ -284,7 +292,7 @@ def get_stats(duckrun_instance, source: str):
         print("⚠️  No tables could be processed successfully")
         import pandas as pd
         return pd.DataFrame(columns=['tbl', 'total_rows', 'num_files', 'num_row_group', 
-                                     'average_row_group', 'file_size_MB', 'vorder', 'timestamp'])
+                                     'average_row_group', 'file_size_MB', 'vorder', 'compression', 'timestamp'])
     
     # Union all successfully processed temp tables
     union_parts = [f'SELECT * FROM tbl_{i}' for i in successful_tables]
@@ -300,6 +308,7 @@ def get_stats(duckrun_instance, source: str):
             CAST(CEIL(SUM(num_rows)::DOUBLE / NULLIF(SUM(num_row_groups), 0)) AS INTEGER) as average_row_group,
             MIN(size) as file_size_MB,
             ANY_VALUE(vorder) as vorder,
+            STRING_AGG(DISTINCT compression, ', ' ORDER BY compression) as compression,
             ANY_VALUE(timestamp) as timestamp
         FROM ({union_query})
         WHERE tbl IS NOT NULL

{duckrun-0.2.15 → duckrun-0.2.16.dev1}/duckrun/writer.py

@@ -3,6 +3,20 @@ Delta Lake writer functionality for duckrun - Spark-style write API
 """
 from deltalake import DeltaTable, write_deltalake, __version__ as deltalake_version
 
+# Try to import WriterProperties for Rust engine (available in 0.18.2+)
+try:
+    from deltalake.writer import WriterProperties
+    _HAS_WRITER_PROPERTIES = True
+except ImportError:
+    _HAS_WRITER_PROPERTIES = False
+
+# Try to import PyArrow dataset for old PyArrow engine
+try:
+    import pyarrow.dataset as ds
+    _HAS_PYARROW_DATASET = True
+except ImportError:
+    _HAS_PYARROW_DATASET = False
+
 
 # Row Group configuration for optimal Delta Lake performance
 RG = 8_000_000
@@ -23,12 +37,14 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
     - Has max_rows_per_file/max_rows_per_group/min_rows_per_group for optimization
     - When mergeSchema=True: must set schema_mode='merge' + engine='rust', NO row group params
     - When mergeSchema=False: use row group params, DON'T set engine (pyarrow is default)
+    - COMPRESSION: Defaults to ZSTD via writer_properties (rust) or file_options (pyarrow)
     
     deltalake 0.20+:
     - Does NOT have 'engine' parameter (everything is rust, pyarrow deprecated)
     - Does NOT have max_rows_per_file (row group optimization removed)
     - When mergeSchema=True: must set schema_mode='merge'
     - When mergeSchema=False: just write normally (no special params)
+    - COMPRESSION: Defaults to ZSTD via writer_properties (rust only)
     
     Uses version detection for simpler logic.
     """
@@ -50,7 +66,13 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
             # deltalake 0.18.2-0.19.x: must also set engine='rust' for schema merging
             # Do NOT use row group params (they conflict with rust engine)
             args['engine'] = 'rust'
-        # For version 0.20+: just schema_mode='merge' is enough, rust is default
+            # Set ZSTD compression for Rust engine
+            if _HAS_WRITER_PROPERTIES:
+                args['writer_properties'] = WriterProperties(compression='ZSTD')
+        else:
+            # Version 0.20+: rust is default, just add compression
+            if _HAS_WRITER_PROPERTIES:
+                args['writer_properties'] = WriterProperties(compression='ZSTD')
     else:
         # Normal write mode (no schema merging)
         if _IS_OLD_DELTALAKE:
@@ -59,7 +81,14 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
             args['max_rows_per_file'] = RG
             args['max_rows_per_group'] = RG
             args['min_rows_per_group'] = RG
-        # For version 0.20+: no optimization available (rust by default, no row group params supported)
+            # Set ZSTD compression for PyArrow engine
+            if _HAS_PYARROW_DATASET:
+                args['file_options'] = ds.ParquetFileFormat().make_write_options(compression='ZSTD')
+        else:
+            # Version 0.20+: no optimization available (rust by default, no row group params supported)
+            # Set ZSTD compression for Rust engine
+            if _HAS_WRITER_PROPERTIES:
+                args['writer_properties'] = WriterProperties(compression='ZSTD')
     
     return args
 
@@ -135,14 +164,14 @@ class DeltaWriter:
         # Prepare info message based on version and settings
         if self._schema_mode == 'merge':
             if _IS_OLD_DELTALAKE:
-                engine_info = " (engine=rust, schema_mode=merge)"
+                engine_info = " (engine=rust, schema_mode=merge, compression=ZSTD)"
             else:
-                engine_info = " (schema_mode=merge, rust by default)"
+                engine_info = " (schema_mode=merge, rust by default, compression=ZSTD)"
         else:
             if _IS_OLD_DELTALAKE:
-                engine_info = " (engine=pyarrow, optimized row groups)"
+                engine_info = " (engine=pyarrow, optimized row groups, compression=ZSTD)"
             else:
-                engine_info = " (engine=rust by default)"
+                engine_info = " (engine=rust by default, compression=ZSTD)"
         
         partition_info = f" partitioned by {self._partition_by}" if self._partition_by else ""
         print(f"Writing to Delta table: {schema}.{table} (mode={self._mode}){engine_info}{partition_info}")

{duckrun-0.2.15 → duckrun-0.2.16.dev1}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.15
+Version: 0.2.16.dev1
 Summary: Lakehouse task runner powered by DuckDB for Microsoft Fabric
 Author: mim
 License: MIT

{duckrun-0.2.15 → duckrun-0.2.16.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.2.15"
+version = "0.2.16.dev1"
 description = "Lakehouse task runner powered by DuckDB for Microsoft Fabric"
 readme = "README.md"
 license = {text = "MIT"}