duckrun 0.2.19.dev5__tar.gz → 0.2.19.dev7__tar.gz

{duckrun-0.2.19.dev5 → duckrun-0.2.19.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.19.dev5
+Version: 0.2.19.dev7
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.19.dev5 → duckrun-0.2.19.dev7}/duckrun/core.py

@@ -1425,7 +1425,8 @@ class Duckrun(WorkspaceOperationsMixin):
             max_cardinality_pct=max_cardinality_pct,
             max_ordering_depth=max_ordering_depth,
             schema_name=schema_name,
-            table_display_name=tbl
+            table_display_name=tbl,
+            duckrun_instance=self  # Pass duckrun instance for detailed parquet stats
         )
 
     def close(self):

{duckrun-0.2.19.dev5 → duckrun-0.2.19.dev7}/duckrun/rle.py

@@ -1,6 +1,7 @@
 import itertools
 from typing import List, Dict, Tuple, Optional
 import pandas as pd
+from .stats import get_stats
 
 def analyze_parquet_row_groups(con, parquet_path: str) -> pd.DataFrame:
     """
@@ -392,7 +393,8 @@ def test_column_orderings_smart(con, delta_path: str, table_name: str = None, li
                                 max_cardinality_pct: float = 0.01,
                                 max_ordering_depth: int = 3,
                                 schema_name: str = None,
-                                table_display_name: str = None) -> pd.DataFrame:
+                                table_display_name: str = None,
+                                duckrun_instance = None) -> pd.DataFrame:
     """
     Test column orderings for RLE optimization.
     
@@ -412,6 +414,7 @@ def test_column_orderings_smart(con, delta_path: str, table_name: str = None, li
         max_ordering_depth: Maximum depth for greedy incremental search in "advanced" mode (default: 3)
         schema_name: Optional schema name to include in results (default: None)
         table_display_name: Optional table name to include in results (default: None)
+        duckrun_instance: Optional Duckrun instance to fetch detailed parquet stats (default: None)
     
     Returns:
         DataFrame with columns: schema, table, sort_order, columns_used, total_rle_all, and individual column RLE counts
@@ -472,6 +475,35 @@ def test_column_orderings_smart(con, delta_path: str, table_name: str = None, li
         print(f"✓ Analysis complete!")
         print(f"{'='*60}")
         
+        # Get detailed parquet stats if duckrun_instance is provided
+        parquet_stats = None
+        vorder_status = False
+        table_size_mb = None
+        if duckrun_instance and table_display_name:
+            print("\nFetching detailed parquet metadata...")
+            try:
+                # For single-schema connections, just use the table name
+                # For multi-schema connections, use schema.table format
+                if hasattr(duckrun_instance, 'scan_all_schemas') and duckrun_instance.scan_all_schemas and schema_name:
+                    source_param = f"{schema_name}.{table_display_name}"
+                else:
+                    source_param = table_display_name
+                
+                parquet_stats = get_stats(duckrun_instance, source=source_param, detailed=True)
+                print(f"✓ Retrieved parquet metadata for {len(parquet_stats)} row groups/columns")
+                
+                # Get vorder status from the stats if available
+                if 'vorder' in parquet_stats.columns:
+                    vorder_status = parquet_stats['vorder'].iloc[0] if len(parquet_stats) > 0 else False
+                
+                # Calculate total table size from compressed sizes
+                if 'total_compressed_size' in parquet_stats.columns:
+                    total_bytes = parquet_stats['total_compressed_size'].sum()
+                    table_size_mb = round(total_bytes / (1024 * 1024), 2) if total_bytes else None
+            except Exception as e:
+                print(f"⚠️  Could not fetch parquet stats: {e}")
+                parquet_stats = None
+        
         # Transform to long format
         long_format_results = []
         
@@ -489,6 +521,45 @@ def test_column_orderings_smart(con, delta_path: str, table_name: str = None, li
             # Get total rows and NDV from card_stats if available
             total_rows = card_stats[data_columns[0]]['total_rows'] if card_stats and data_columns else None
             
+            # Aggregate parquet stats per column if available
+            parquet_by_column = {}
+            avg_row_group_size = None  # Calculate once for the table
+            
+            if parquet_stats is not None and not parquet_stats.empty:
+                # Calculate average row group size once (same for all columns)
+                if 'row_group_num_rows' in parquet_stats.columns:
+                    # Get unique row groups to avoid counting duplicates (one entry per column per row group)
+                    unique_rg_sizes = parquet_stats.drop_duplicates(subset=['row_group_id'])['row_group_num_rows']
+                    avg_row_group_size = int(unique_rg_sizes.mean())
+                
+                # Determine column name field - can be 'name' or 'path_in_schema'
+                col_name_field = 'path_in_schema' if 'path_in_schema' in parquet_stats.columns else 'name'
+                
+                # Group by column name and aggregate
+                for col_name in data_columns:
+                    col_stats = parquet_stats[parquet_stats[col_name_field] == col_name] if col_name_field in parquet_stats.columns else pd.DataFrame()
+                    
+                    if not col_stats.empty:
+                        # Aggregate stats across all row groups for this column
+                        total_compressed_bytes = col_stats['total_compressed_size'].sum() if 'total_compressed_size' in col_stats.columns else None
+                        total_compressed_mb = round(total_compressed_bytes / (1024 * 1024), 2) if total_compressed_bytes else None
+                        # Handle both 'encodings' (multiple) and 'encoding' (single) column names
+                        if 'encodings' in col_stats.columns:
+                            encodings = col_stats['encodings'].unique().tolist()
+                        elif 'encoding' in col_stats.columns:
+                            encodings = col_stats['encoding'].unique().tolist()
+                        else:
+                            encodings = []
+                        compressions = col_stats['compression'].unique().tolist() if 'compression' in col_stats.columns else []
+                        num_row_groups = col_stats['row_group_id'].nunique() if 'row_group_id' in col_stats.columns else len(col_stats)
+                        
+                        parquet_by_column[col_name] = {
+                            'encoding': ', '.join([str(e) for e in encodings if e is not None]),
+                            'compression': ', '.join([str(c) for c in compressions if c is not None]),
+                            'total_compressed_size_mb': total_compressed_mb,
+                            'num_row_groups': num_row_groups
+                        }
+            
             # Create one row per data column
             for col in data_columns:
                 rle_value = row[col]
@@ -496,16 +567,25 @@ def test_column_orderings_smart(con, delta_path: str, table_name: str = None, li
                 # Get NDV from card_stats
                 ndv_value = card_stats[col]['distinct_values'] if card_stats and col in card_stats else None
                 
+                # Get parquet stats for this column
+                col_parquet = parquet_by_column.get(col, {})
+                
                 long_format_results.append({
                     'schema': schema_val,
                     'table': table_val,
                     'sort_type': sort_order,
-                    'column': col,
+                    'name': col,
                     'order': None,
                     'RLE': rle_value,
                     'NDV': ndv_value,
                     'total_rows': total_rows,
                     'total_RLE': total_rle_all_val,
+                    'encoding': col_parquet.get('encoding', ''),
+                    'compression': col_parquet.get('compression', ''),
+                    'column_size_mb': col_parquet.get('total_compressed_size_mb', None),
+                    'avg_row_group_size': avg_row_group_size,
+                    'table_size_mb': table_size_mb,
+                    'vorder': vorder_status,
                     'comments': ''
                 })
         
@@ -829,7 +909,7 @@ def test_column_orderings_smart(con, delta_path: str, table_name: str = None, li
                 'schema': schema_val,
                 'table': table_val,
                 'sort_type': sort_order,
-                'column': col,
+                'name': col,
                 'order': order_position,
                 'RLE': rle_value,
                 'NDV': ndv_value,

{duckrun-0.2.19.dev5 → duckrun-0.2.19.dev7}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.19.dev5
+Version: 0.2.19.dev7
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.19.dev5 → duckrun-0.2.19.dev7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.2.19.dev5"
+version = "0.2.19.dev7"
 description = "Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)"
 readme = "README.md"
 license = {text = "MIT"}

{duckrun-0.2.19.dev5 → duckrun-0.2.19.dev7}/tests/test_rle.py

@@ -7,10 +7,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import duckrun
 
 # Analyze multiple schemas/tables
-conn = duckrun.connect("tmp/data.lakehouse/deltars_sorted")
+conn = duckrun.connect("tmp/data.lakehouse/spark_vorder")
 
 # Analyze tables - now returns long format automatically
-result = conn.rle("calendar",'auto')
+result = conn.rle("summary")
 print(result)
 conn.close()