duckrun 0.2.22.dev0__tar.gz → 0.2.22.dev2__tar.gz

{duckrun-0.2.22.dev0 → duckrun-0.2.22.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.22.dev0
+Version: 0.2.22.dev2
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.22.dev0 → duckrun-0.2.22.dev2}/duckrun/__init__.py

@@ -2,11 +2,10 @@
 
 from duckrun.core import Duckrun
 from duckrun.notebook import import_notebook_from_web, import_notebook
-from duckrun import rle
 
-__version__ = "0.2.18"
+__version__ = "0.2.22.dev2"
 
 # Expose unified connect method at module level
 connect = Duckrun.connect
 
-__all__ = ["Duckrun", "connect", "import_notebook_from_web", "import_notebook", "rle"]
+__all__ = ["Duckrun", "connect", "import_notebook_from_web", "import_notebook"]

{duckrun-0.2.22.dev0 → duckrun-0.2.22.dev2}/duckrun/core.py

@@ -7,7 +7,7 @@ import time
 from typing import List, Tuple, Union, Optional, Callable, Dict, Any
 from string import Template
 from datetime import datetime
-from .stats import get_stats as _get_stats
+from .stats import get_stats as _get_stats, get_rle as _get_rle
 from .runner import run as _run
 from .files import copy as _copy, download as _download
 from .writer import QueryResult
@@ -1181,6 +1181,39 @@ class Duckrun(WorkspaceOperationsMixin):
         """
         return _get_stats(self, source, detailed)
 
+    def get_rle(self, source: str = None):
+        """
+        Get RLE (Run-Length Encoding) statistics for Delta Lake tables.
+        Measures compression potential by counting consecutive identical values.
+        
+        Args:
+            source: Optional. Can be one of:
+                   - None: Use all tables in the connection's schema (default)
+                   - Table name: 'table_name' (uses current schema)
+                   - Schema.table: 'schema.table_name' (specific table in schema)
+                   - Schema only: 'schema' (all tables in schema)
+                   - Wildcard patterns: '*.summary' or 'schema.*'
+        
+        Returns:
+            DataFrame with columns: schema_name, table_name, total_rle_runs
+            
+        Examples:
+            con = duckrun.connect("tmp/data.lakehouse/aemo")
+            
+            # All tables in current schema
+            rle = con.get_rle()
+            
+            # Single table in current schema
+            rle = con.get_rle('price')
+            
+            # Specific table in different schema
+            rle = con.get_rle('deltars.summary')
+            
+            # All tables matching pattern
+            rle = con.get_rle('*.summary')
+        """
+        return _get_rle(self, source)
+
     def list_lakehouses(self) -> List[str]:
         """
         List all lakehouses in the current workspace.

{duckrun-0.2.22.dev0 → duckrun-0.2.22.dev2}/duckrun/stats.py

@@ -454,3 +454,195 @@ def get_stats(duckrun_instance, source: str = None, detailed = False):
     return final_result
 
 
+def get_rle(duckrun_instance, source: str = None) -> 'pd.DataFrame':
+    """
+    Get RLE statistics for tables at the column level.
+    
+    Args:
+        duckrun_instance: Duckrun connection (from duckrun.connect())
+        source: Optional. Can be one of:
+               - None: Use all tables in the connection's schema (default)
+               - Table name: 'table_name' (uses main schema in DuckDB)
+               - Schema.table: 'schema.table_name' (specific table in schema)
+               - Schema only: 'schema' (all tables in schema)
+               - Wildcard pattern: '*.summary' (matches tables across all schemas)
+    
+    Returns:
+        DataFrame with columns:
+        - schema_name: Schema name
+        - table_name: Table name
+        - column_name: Column name
+        - total_rows: Total number of rows
+        - rle_runs: RLE runs for this column in natural order
+        - ndv: Number of distinct values
+        - total_rle_runs: Sum of RLE runs across all columns (same for all rows of a table)
+    """
+    import fnmatch
+    import pandas as pd
+    
+    con = duckrun_instance.con  # Get underlying DuckDB connection
+    
+    # Determine which tables to process
+    tables_to_process = []  # List of (schema, table) tuples
+    
+    if source is None:
+        # Get all tables in the connection's schema
+        schema_name = duckrun_instance.schema if hasattr(duckrun_instance, 'schema') else 'main'
+        try:
+            if schema_name == 'main':
+                query = "SHOW TABLES"
+                result = con.execute(query).fetchall()
+                if result:
+                    tables = [row[0] for row in result if not row[0].startswith('tbl_')]
+                    tables_to_process = [(schema_name, tbl) for tbl in tables]
+            else:
+                query = f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{schema_name}'"
+                result = con.execute(query).fetchall()
+                if result:
+                    tables = [row[0] for row in result if not row[0].startswith('tbl_')]
+                    tables_to_process = [(schema_name, tbl) for tbl in tables]
+        except:
+            pass
+    
+    elif '.' in source:
+        parts = source.split('.', 1)
+        schema_pattern, table_pattern = parts[0], parts[1]
+        
+        # Check if patterns contain wildcards
+        if '*' in schema_pattern or '*' in table_pattern:
+            # Wildcard matching
+            query = """
+                SELECT table_schema, table_name 
+                FROM information_schema.tables 
+                WHERE table_schema NOT LIKE 'pg_%' 
+                AND table_schema != 'information_schema'
+                AND table_name NOT LIKE 'tbl_%'
+            """
+            result = con.execute(query).fetchall()
+            for schema, table in result:
+                if fnmatch.fnmatch(schema, schema_pattern) and fnmatch.fnmatch(table, table_pattern):
+                    tables_to_process.append((schema, table))
+        else:
+            # Exact schema.table
+            tables_to_process = [(schema_pattern, table_pattern)]
+    
+    elif '*' in source:
+        # Wildcard pattern for table names across all schemas
+        query = """
+            SELECT table_schema, table_name 
+            FROM information_schema.tables 
+            WHERE table_schema NOT LIKE 'pg_%' 
+            AND table_schema != 'information_schema'
+            AND table_name NOT LIKE 'tbl_%'
+        """
+        result = con.execute(query).fetchall()
+        for schema, table in result:
+            if fnmatch.fnmatch(table, source):
+                tables_to_process.append((schema, table))
+    
+    else:
+        # Check if it's a schema name or table name
+        try:
+            # Try as schema first
+            schema_query = f"SELECT 1 FROM information_schema.schemata WHERE schema_name = '{source}' LIMIT 1"
+            schema_exists = con.execute(schema_query).fetchone()
+            
+            if schema_exists:
+                # It's a schema - get all tables
+                tables_query = f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{source}'"
+                result = con.execute(tables_query).fetchall()
+                if result:
+                    tables = [row[0] for row in result if not row[0].startswith('tbl_')]
+                    tables_to_process = [(source, tbl) for tbl in tables]
+            else:
+                # It's a table name in default schema
+                schema_name = duckrun_instance.schema if hasattr(duckrun_instance, 'schema') else 'main'
+                tables_to_process = [(schema_name, source)]
+        except:
+            # Assume it's a table name
+            schema_name = duckrun_instance.schema if hasattr(duckrun_instance, 'schema') else 'main'
+            tables_to_process = [(schema_name, source)]
+    
+    if not tables_to_process:
+        print("No tables found matching the criteria")
+        return pd.DataFrame(columns=['schema_name', 'table_name', 'column_name', 'total_rows', 
+                                     'rle_runs', 'ndv', 'total_rle_runs'])
+    
+    print(f"Processing {len(tables_to_process)} table(s)...")
+    
+    # Process each table
+    results = []
+    for schema, table in tables_to_process:
+        table_path = f"{duckrun_instance.table_base_url}{schema}/{table}"
+        
+        print(f"\nCalculating RLE runs for {schema}.{table}...")
+        
+        # Get column names and row count
+        try:
+            schema_info = con.sql(f"""
+                SELECT column_name
+                FROM (DESCRIBE SELECT * FROM delta_scan('{table_path}'))
+            """).df()
+            
+            # Get total row count
+            total_rows = con.sql(f"SELECT COUNT(*) FROM delta_scan('{table_path}')").fetchone()[0]
+            
+            if schema_info.empty:
+                continue
+            
+            # Track total RLE runs for this table
+            table_total_rle = 0
+            table_results = []
+            
+            for _, row in schema_info.iterrows():
+                col_name = row['column_name']
+                
+                # Calculate RLE runs in natural (physical) order using delta_scan
+                rle_query = f"""
+                WITH numbered AS (
+                    SELECT 
+                        filename,
+                        file_row_number,
+                        {col_name},
+                        LAG({col_name}) OVER (ORDER BY filename, file_row_number) as prev_value
+                    FROM delta_scan('{table_path}', file_row_number=1, filename=1)
+                )
+                SELECT COUNT(*) as runs
+                FROM numbered
+                WHERE prev_value IS NULL OR {col_name} != prev_value OR {col_name} IS NULL OR prev_value IS NULL
+                """
+                
+                try:
+                    runs = con.sql(rle_query).fetchone()[0]
+                    
+                    # Also calculate NDV for this column
+                    ndv_query = f"SELECT COUNT(DISTINCT {col_name}) FROM delta_scan('{table_path}')"
+                    ndv = con.sql(ndv_query).fetchone()[0]
+                    
+                    table_total_rle += runs
+                    
+                    print(f"  {col_name}: {runs:,} runs, ndv={ndv:,}")
+                    
+                    table_results.append({
+                        'schema_name': schema,
+                        'table_name': table,
+                        'column_name': col_name,
+                        'total_rows': total_rows,
+                        'rle_runs': runs,
+                        'ndv': ndv
+                    })
+                except Exception as e:
+                    print(f"  Warning: Could not calculate RLE runs for {col_name}: {e}")
+            
+            # Add total_rle_runs to all rows for this table
+            for result in table_results:
+                result['total_rle_runs'] = table_total_rle
+                results.append(result)
+            
+            print(f"  Total RLE runs for table: {table_total_rle:,}")
+        
+        except Exception as e:
+            print(f"  Error processing table: {e}")
+    
+    return pd.DataFrame(results)
+

{duckrun-0.2.22.dev0 → duckrun-0.2.22.dev2}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.22.dev0
+Version: 0.2.22.dev2
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.22.dev0 → duckrun-0.2.22.dev2}/duckrun.egg-info/SOURCES.txt

@@ -8,7 +8,6 @@ duckrun/ducklake_metadata.py
 duckrun/files.py
 duckrun/lakehouse.py
 duckrun/notebook.py
-duckrun/rle.py
 duckrun/runner.py
 duckrun/semantic_model.py
 duckrun/stats.py
@@ -19,11 +18,13 @@ duckrun.egg-info/dependency_links.txt
 duckrun.egg-info/requires.txt
 duckrun.egg-info/top_level.txt
 tests/test_checkpoint_format.py
+tests/test_consecutive_values.py
 tests/test_deploy_fresh.py
 tests/test_ducklake_export.py
 tests/test_filename.py
 tests/test_register.py
 tests/test_rle.py
 tests/test_rle_analysis.py
+tests/test_rle_summary.py
 tests/test_writer_dictionary.py
 tests/test_writer_integration.py

{duckrun-0.2.22.dev0 → duckrun-0.2.22.dev2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.2.22.dev0"
+version = "0.2.22.dev2"
 description = "Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)"
 readme = "README.md"
 license = {text = "MIT"}

duckrun-0.2.22.dev2/tests/test_consecutive_values.py

@@ -0,0 +1,115 @@
+"""
+Test: Analyze Consecutive Values in Delta Table
+
+This test connects to tmp/data.lakehouse and analyzes the deltars.summary table
+for consecutive runs in the 'time' column.
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import duckrun
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import duckrun
+from duckrun.rle import analyze_consecutive_values
+
+
+def test_consecutive_values():
+    """Test consecutive value analysis on deltars.summary table"""
+    
+    lakehouse_path = "tmp/data.lakehouse"
+    
+    print("=" * 80)
+    print("CONSECUTIVE VALUES ANALYSIS TEST")
+    print("=" * 80)
+    print(f"Lakehouse: {lakehouse_path}")
+    print(f"Table: deltars.summary")
+    print(f"Column: time")
+    
+    try:
+        # Connect to lakehouse
+        print("\nConnecting to lakehouse...")
+        con = duckrun.connect(lakehouse_path)
+        
+        # Analyze consecutive values in the 'time' column
+        print("\n" + "=" * 80)
+        print("ANALYZING CONSECUTIVE VALUES")
+        print("=" * 80)
+        
+        df = analyze_consecutive_values(
+            duckrun_con=con,
+            table_name='summary',
+            column_name='time',
+            min_consecutive=3,
+            schema_name='deltars'
+        )
+        
+        # Display results
+        if not df.empty:
+            print("\n" + "=" * 80)
+            print("RESULTS")
+            print("=" * 80)
+            print(f"\nFound {len(df)} consecutive sequences")
+            print("\nTop 20 longest sequences:")
+            print(df.head(20).to_string(index=False))
+            
+            # Statistics
+            print("\n" + "=" * 80)
+            print("STATISTICS")
+            print("=" * 80)
+            total_in_sequences = df['consecutive_count'].sum()
+            longest = df['consecutive_count'].max()
+            shortest = df['consecutive_count'].min()
+            avg = df['consecutive_count'].mean()
+            
+            print(f"Total values in sequences: {total_in_sequences:,}")
+            print(f"Longest sequence: {longest:,}")
+            print(f"Shortest sequence: {shortest:,}")
+            print(f"Average sequence length: {avg:.2f}")
+            
+            # File distribution
+            print("\n" + "=" * 80)
+            print("FILE DISTRIBUTION")
+            print("=" * 80)
+            file_counts = df.groupby('filename').agg({
+                'consecutive_count': ['count', 'sum', 'max']
+            }).reset_index()
+            file_counts.columns = ['filename', 'num_sequences', 'total_values', 'max_sequence']
+            
+            # Extract just filename from path
+            file_counts['filename'] = file_counts['filename'].apply(
+                lambda x: x.split('/')[-1] if '/' in str(x) else x
+            )
+            
+            print(f"\nSequences across {len(file_counts)} files:")
+            print(file_counts.to_string(index=False))
+            
+            print("\n✅ Test completed successfully!")
+            return True
+        else:
+            print("\n⚠ No consecutive sequences found")
+            return True
+        
+    except Exception as e:
+        print(f"\n❌ Error during analysis: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+if __name__ == "__main__":
+    print("\n" + "=" * 80)
+    print("STARTING CONSECUTIVE VALUES TEST")
+    print("=" * 80)
+    
+    success = test_consecutive_values()
+    
+    print("\n" + "=" * 80)
+    if success:
+        print("✅ TEST PASSED")
+    else:
+        print("❌ TEST FAILED")
+    print("=" * 80)
+    
+    sys.exit(0 if success else 1)

duckrun-0.2.22.dev2/tests/test_rle_summary.py

@@ -0,0 +1,22 @@
+"""
+Test: Get RLE statistics for deltars.summary table
+"""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path to import duckrun
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import duckrun
+
+# Connect to lakehouse
+con = duckrun.connect("tmp/data.lakehouse")
+
+# Get RLE stats for deltars.summary
+rle_df = con.get_rle('*.summary')
+
+print("\n" + "=" * 80)
+print("RESULTS")
+print("=" * 80)
+print(rle_df.to_string(index=False))

duckrun-0.2.22.dev0/duckrun/rle.py

@@ -1,362 +0,0 @@
-from typing import List, Dict, Tuple, Optional
-import pandas as pd
-
-
-def get_table_stats(duckrun_con, table_name: str, 
-                   top_n_values: int = 10) -> pd.DataFrame:
-    """
-    Get comprehensive table statistics including NDV and value frequency analysis.
-    
-    The theory: If a value appears frequently (high repetition), it may provide better RLE compression
-    even if the column has higher NDV. This function helps identify such patterns.
-    
-    Args:
-        duckrun_con: Duckrun connection (from duckrun.connect())
-        table_name: Name of the table to analyze
-        top_n_values: Number of top frequent values to show per column (default: 10)
-    
-    Returns:
-        DataFrame with columns:
-        - column_name: Name of the column
-        - data_type: Data type of the column
-        - total_rows: Total number of rows
-        - null_count: Number of NULL values
-        - null_pct: Percentage of NULL values
-        - ndv: Number of distinct values (exact)
-        - cardinality_ratio: NDV / total_rows (lower = better for RLE)
-        - top_value: Most frequent value
-        - top_value_count: Count of most frequent value
-        - top_value_pct: Percentage of most frequent value
-        - top_n_coverage: Percentage covered by top N values
-        - repetition_score: Custom score indicating RLE potential (higher = better)
-    """
-    con = duckrun_con.con  # Get underlying DuckDB connection
-    from_clause = table_name
-    
-    # Get column names and types
-    schema_info = con.sql(f"""
-        SELECT column_name, column_type
-        FROM (DESCRIBE SELECT * FROM {from_clause})
-    """).df()
-    
-    if schema_info.empty:
-        return pd.DataFrame()
-    
-    # Get total row count once
-    total_rows = con.sql(f"SELECT COUNT(*) FROM {from_clause}").fetchone()[0]
-    print(f"Analyzing {len(schema_info)} columns across {total_rows:,} rows...")
-    
-    results = []
-    
-    for idx, row in schema_info.iterrows():
-        col_name = row['column_name']
-        col_type = row['column_type']
-        
-        print(f"  [{idx+1}/{len(schema_info)}] Analyzing column: {col_name}")
-        
-        # Get basic stats in one query
-        stats_query = f"""
-        SELECT 
-            COUNT(*) as total,
-            COUNT({col_name}) as non_null,
-            COUNT(DISTINCT {col_name}) as ndv
-        FROM {from_clause}
-        """
-        
-        stats = con.sql(stats_query).fetchone()
-        total = stats[0]
-        non_null = stats[1]
-        ndv = stats[2]
-        null_count = total - non_null
-        null_pct = (null_count / total * 100) if total > 0 else 0
-        cardinality_ratio = (ndv / total) if total > 0 else 0
-        
-        # Get top N values with their frequencies
-        top_values_query = f"""
-        SELECT 
-            {col_name} as value,
-            COUNT(*) as count,
-            COUNT(*) * 100.0 / {total} as percentage
-        FROM {from_clause}
-        WHERE {col_name} IS NOT NULL
-        GROUP BY {col_name}
-        ORDER BY count DESC
-        LIMIT {top_n_values}
-        """
-        
-        top_values = con.sql(top_values_query).df()
-        
-        # Extract top value info
-        if not top_values.empty:
-            top_value = top_values.iloc[0]['value']
-            top_value_count = top_values.iloc[0]['count']
-            top_value_pct = top_values.iloc[0]['percentage']
-            top_n_coverage = top_values['percentage'].sum()
-        else:
-            top_value = None
-            top_value_count = 0
-            top_value_pct = 0
-            top_n_coverage = 0
-        
-        # Calculate repetition score: higher means better for RLE
-        # Score considers:
-        # 1. How much the top value covers (higher = better)
-        # 2. How much top N values cover (higher = better)  
-        # 3. Inverse of cardinality ratio (lower cardinality = better)
-        repetition_score = (top_value_pct * 2 + top_n_coverage) / 3 / (cardinality_ratio + 0.01)
-        
-        results.append({
-            'column_name': col_name,
-            'data_type': col_type,
-            'total_rows': total_rows,
-            'null_count': null_count,
-            'null_pct': round(null_pct, 2),
-            'ndv': ndv,
-            'cardinality_ratio': round(cardinality_ratio, 4),
-            'top_value': top_value,
-            'top_value_count': top_value_count,
-            'top_value_pct': round(top_value_pct, 2),
-            'top_n_coverage': round(top_n_coverage, 2),
-            'repetition_score': round(repetition_score, 2)
-        })
-    
-    df = pd.DataFrame(results)
-    
-    # Sort by repetition score (best RLE candidates first)
-    df = df.sort_values('repetition_score', ascending=False).reset_index(drop=True)
-    
-    print(f"\n✓ Analysis complete!")
-    print(f"\nTop columns by repetition score (best RLE candidates):")
-    for idx, row in df.head(5).iterrows():
-        print(f"  {idx+1}. {row['column_name']}: score={row['repetition_score']}, "
-              f"top_value_pct={row['top_value_pct']}%, ndv={row['ndv']:,}")
-    
-    return df
-
-
-def get_value_frequency_details(duckrun_con, table_name: str, column_name: str, 
-                                limit: int = 20) -> pd.DataFrame:
-    """
-    Get detailed value frequency distribution for a specific column.
-    
-    Shows the most frequent values and their counts/percentages.
-    Useful for understanding repetition patterns that drive RLE compression.
-    
-    Args:
-        duckrun_con: Duckrun connection (from duckrun.connect())
-        table_name: Name of the table to analyze
-        column_name: Name of the column to analyze
-        limit: Maximum number of values to return (default: 20)
-    
-    Returns:
-        DataFrame with columns:
-        - value: The distinct value
-        - count: Number of occurrences
-        - percentage: Percentage of total rows
-        - cumulative_pct: Cumulative percentage
-    """
-    con = duckrun_con.con  # Get underlying DuckDB connection
-    from_clause = table_name
-    
-    # Get total row count
-    total_rows = con.sql(f"SELECT COUNT(*) FROM {from_clause}").fetchone()[0]
-    
-    # Get value frequencies
-    query = f"""
-    WITH value_counts AS (
-        SELECT 
-            {column_name} as value,
-            COUNT(*) as count,
-            COUNT(*) * 100.0 / {total_rows} as percentage
-        FROM {from_clause}
-        WHERE {column_name} IS NOT NULL
-        GROUP BY {column_name}
-        ORDER BY count DESC
-        LIMIT {limit}
-    )
-    SELECT 
-        value,
-        count,
-        percentage,
-        SUM(percentage) OVER (ORDER BY count DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) as cumulative_pct
-    FROM value_counts
-    ORDER BY count DESC
-    """
-    
-    df = con.sql(query).df()
-    
-    # Round percentages
-    if not df.empty:
-        df['percentage'] = df['percentage'].round(2)
-        df['cumulative_pct'] = df['cumulative_pct'].round(2)
-    
-    return df
-
-
-def find_optimal_sort_order(duckrun_con, table_name: str, 
-                           max_combinations: int = 10) -> pd.DataFrame:
-    """
-    Determine optimal sort order using V-Order-like logic: pure compression testing.
-    
-    This mimics how VertiPaq/V-Order actually works:
-    1. Calculate cardinality for each column
-    2. Test different sort orderings
-    3. Measure actual RLE run counts for each ordering
-    4. Pick the ordering with best overall compression (fewest total runs)
-    
-    NO semantic understanding, NO query pattern assumptions.
-    Pure mechanical testing of compression effectiveness.
-    
-    Args:
-        duckrun_con: Duckrun connection (from duckrun.connect())
-        table_name: Name of the table to analyze
-        max_combinations: Maximum sort orderings to test (default: 10)
-    
-    Returns:
-        DataFrame with tested orderings ranked by compression effectiveness
-    """
-    from itertools import permutations
-    
-    con = duckrun_con.con  # Get underlying DuckDB connection
-    from_clause = table_name
-    
-    # Get column names and cardinalities
-    print("Step 1: Analyzing column cardinalities...")
-    schema_info = con.sql(f"""
-        SELECT column_name, column_type
-        FROM (DESCRIBE SELECT * FROM {from_clause})
-    """).df()
-    
-    total_rows = con.sql(f"SELECT COUNT(*) FROM {from_clause}").fetchone()[0]
-    
-    # Calculate NDV for each column
-    cardinality_map = {}
-    for _, row in schema_info.iterrows():
-        col = row['column_name']
-        ndv = con.sql(f"SELECT COUNT(DISTINCT {col}) FROM {from_clause}").fetchone()[0]
-        cardinality_ratio = ndv / total_rows
-        cardinality_map[col] = {'ndv': ndv, 'ratio': cardinality_ratio}
-        print(f"  {col}: {ndv:,} distinct ({cardinality_ratio*100:.4f}%)")
-    
-    # Filter to low-cardinality columns only (< 1% cardinality)
-    # High cardinality columns won't benefit from reordering
-    low_card_cols = [col for col, stats in cardinality_map.items() 
-                     if stats['ratio'] < 0.01]
-    
-    print(f"\nStep 2: Testing sort orderings for {len(low_card_cols)} low-cardinality columns...")
-    print(f"Columns to test: {', '.join(low_card_cols)}")
-    
-    if len(low_card_cols) < 2:
-        print("Not enough columns to test different orderings!")
-        return pd.DataFrame()
-    
-    # Generate candidate orderings
-    # Start with cardinality-based orderings
-    sorted_by_card = sorted(low_card_cols, key=lambda c: cardinality_map[c]['ndv'])
-    
-    test_orderings = [
-        sorted_by_card,  # Lowest cardinality first
-        sorted_by_card[::-1],  # Highest cardinality first
-    ]
-    
-    # Add some permutations of top 3 columns
-    if len(low_card_cols) >= 3:
-        for perm in permutations(sorted_by_card[:3]):
-            if list(perm) not in test_orderings:
-                test_orderings.append(list(perm))
-            if len(test_orderings) >= max_combinations:
-                break
-    
-    # Test each ordering by calculating actual RLE runs
-    print(f"\nStep 3: Testing {len(test_orderings)} different orderings...")
-    results = []
-    
-    for idx, ordering in enumerate(test_orderings, 1):
-        print(f"\n[{idx}/{len(test_orderings)}] Testing: {' → '.join(ordering)}")
-        
-        # Calculate RLE runs for each column with this ordering
-        # We'll sort the data by the ordering and count runs
-        order_clause = ', '.join(ordering)
-        
-        column_rle = {}
-        for col in schema_info['column_name']:
-            # Count runs: a new run starts when value changes
-            rle_query = f"""
-            WITH sorted_data AS (
-                SELECT 
-                    {col},
-                    ROW_NUMBER() OVER (ORDER BY {order_clause}) as rn
-                FROM {from_clause}
-            ),
-            with_prev AS (
-                SELECT 
-                    {col},
-                    LAG({col}) OVER (ORDER BY rn) as prev_val
-                FROM sorted_data
-            )
-            SELECT COUNT(*) as runs
-            FROM with_prev
-            WHERE prev_val IS NULL OR {col} != prev_val OR {col} IS NULL OR prev_val IS NULL
-            """
-            
-            runs = con.sql(rle_query).fetchone()[0]
-            column_rle[col] = runs
-            print(f"    {col}: {runs:,} runs")
-        
-        total_runs = sum(column_rle.values())
-        print(f"    TOTAL: {total_runs:,} runs")
-        
-        results.append({
-            'sort_order': ' → '.join(ordering),
-            'total_runs': total_runs,
-            'compression_score': total_rows / total_runs,  # Higher = better compression
-            **column_rle
-        })
-    
-    # Create results DataFrame
-    df = pd.DataFrame(results)
-    df = df.sort_values('total_runs').reset_index(drop=True)
-    
-    print("\n" + "=" * 80)
-    print("RESULTS: Best to Worst Compression")
-    print("=" * 80)
-    
-    for idx, row in df.iterrows():
-        print(f"\n{idx + 1}. {row['sort_order']}")
-        print(f"   Total runs: {row['total_runs']:,}")
-        print(f"   Compression score: {row['compression_score']:.2f}x")
-        if idx == 0:
-            print("   ⭐ BEST COMPRESSION")
-    
-    print("\n" + "=" * 80)
-    print("CONCLUSION")
-    print("=" * 80)
-    best = df.iloc[0]
-    print(f"\nOptimal sort order: {best['sort_order']}")
-    print(f"This ordering achieves the fewest total RLE runs ({best['total_runs']:,})")
-    print(f"\nThis is how V-Order actually works:")
-    print("✓ No query pattern assumptions")
-    print("✓ No semantic understanding") 
-    print("✓ Pure compression effectiveness testing")
-    print("✓ Mechanical optimization based on data patterns")
-    
-    return df
-
-
-# Example usage:
-#
-# import duckrun
-#
-# con = duckrun.connect('workspace/lakehouse.lakehouse')
-#
-# # Get RLE statistics:
-# stats_df = con.get_rle_stats('my_table', top_n_values=10)
-# print(stats_df)
-#
-# # Detailed frequency distribution for a specific column:
-# freq_df = con.get_value_frequency('my_table', 'status_column', limit=20)
-# print(freq_df)
-#
-# # Find optimal sort order (V-Order simulation):
-# optimal_df = con.find_optimal_sort_order('my_table', max_combinations=10)
-# print(optimal_df)