duckrun 0.2.18.dev5__tar.gz → 0.2.19__tar.gz

{duckrun-0.2.18.dev5 → duckrun-0.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.18.dev5
+Version: 0.2.19
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.18.dev5 → duckrun-0.2.19}/duckrun/__init__.py

@@ -2,10 +2,11 @@
 
 from duckrun.core import Duckrun
 from duckrun.notebook import import_notebook_from_web, import_notebook
+from duckrun import rle
 
-__version__ = "0.2.18.dev2"
+__version__ = "0.2.18"
 
 # Expose unified connect method at module level
 connect = Duckrun.connect
 
-__all__ = ["Duckrun", "connect", "import_notebook_from_web", "import_notebook"]
+__all__ = ["Duckrun", "connect", "import_notebook_from_web", "import_notebook", "rle"]

{duckrun-0.2.18.dev5 → duckrun-0.2.19}/duckrun/core.py

@@ -1035,7 +1035,22 @@ class Duckrun(WorkspaceOperationsMixin):
         """Get underlying DuckDB connection"""
         return self.con
 
-    def get_stats(self, source: str = None):
+    def register(self, name: str, df):
+        """
+        Register a pandas DataFrame as a virtual table in DuckDB.
+        
+        Args:
+            name: Name for the virtual table
+            df: pandas DataFrame to register
+            
+        Example:
+            con = duckrun.connect("workspace/lakehouse.lakehouse")
+            con.register("tb", df)
+            con.sql("SELECT * FROM tb").show()
+        """
+        self.con.register(name, df)
+
+    def get_stats(self, source: str = None, detailed = False):
         """
         Get comprehensive statistics for Delta Lake tables.
         
@@ -1045,27 +1060,34 @@ class Duckrun(WorkspaceOperationsMixin):
                    - Table name: 'table_name' (uses current schema)
                    - Schema.table: 'schema.table_name' (specific table in schema)
                    - Schema only: 'schema' (all tables in schema)
+            detailed: Optional. Controls the level of detail in statistics:
+                     - False (default): Aggregated table-level stats
+                     - True: Row group level statistics with compression details
         
         Returns:
-            Arrow table with statistics including total rows, file count, row groups, 
-            average row group size, file sizes, VORDER status, and timestamp
+            DataFrame with statistics based on detailed parameter:
+            - If detailed=False: Aggregated table-level summary
+            - If detailed=True: Granular file and row group level stats
         
         Examples:
             con = duckrun.connect("tmp/data.lakehouse/aemo")
             
-            # All tables in current schema (aemo)
+            # All tables in current schema (aemo) - aggregated
             stats = con.get_stats()
             
-            # Single table in current schema
+            # Single table in current schema - aggregated
             stats = con.get_stats('price')
             
+            # Single table with detailed row group statistics
+            stats_detailed = con.get_stats('price', detailed=True)
+            
             # Specific table in different schema
             stats = con.get_stats('aemo.price')
             
             # All tables in a schema
             stats = con.get_stats('aemo')
         """
-        return _get_stats(self, source)
+        return _get_stats(self, source, detailed)
 
     def list_lakehouses(self) -> List[str]:
         """
@@ -1237,8 +1259,194 @@ class Duckrun(WorkspaceOperationsMixin):
             refresh=refresh
         )
 
+    def rle(self, table_name: str = None, mode = "natural",
+            min_distinct_threshold: int = 2, max_cardinality_pct: float = 0.01,
+            max_ordering_depth: int = 3, limit: int = None):
+        """
+        Analyze RLE (Run-Length Encoding) compression potential for Delta Lake tables.
+        
+        Args:
+            table_name: Name of the table to analyze. Can be:
+                - 'table_name' (uses current schema)
+                - 'schema.table_name' (specific schema)
+            mode: Analysis mode or column ordering:
+                - "natural": Calculate RLE for natural order only (fastest)
+                - "auto": Natural order + cardinality-based ordering (recommended)
+                - "advanced": Natural + cardinality + greedy incremental search (most thorough)
+                - List[str]: Specific column ordering to test, e.g., ['date', 'duid']
+            min_distinct_threshold: Exclude columns with fewer distinct values (default: 2)
+            max_cardinality_pct: Exclude columns with cardinality above this % (default: 0.01 = 1%)
+            max_ordering_depth: Maximum depth for greedy search in "advanced" mode (default: 3)
+            limit: Optional row limit for testing/development (default: None, analyzes all rows)
+        
+        Returns:
+            DataFrame with RLE analysis results
+        
+        Examples:
+            # Natural order only (baseline)
+            con = duckrun.connect("workspace/lakehouse.lakehouse/schema")
+            con.rle("mytable")  # same as con.rle("mytable", "natural")
+            
+            # Auto optimization (natural + cardinality-based)
+            con.rle("mytable", "auto")
+            
+            # Advanced optimization (greedy incremental search)
+            con.rle("mytable", "advanced")
+            
+            # Test specific column ordering
+            con.rle("mytable", ["date", "duid"])
+            con.rle("mytable", ["cutoff", "time", "DUID", "date"])
+            
+            # Advanced with custom depth
+            con.rle("mytable", "advanced", max_ordering_depth=4)
+            
+            # Analyze table from different schema
+            con.rle("otherschema.mytable", "auto")
+            
+            # Custom thresholds for small tables
+            con.rle("mytable", "auto", max_cardinality_pct=0.05)
+            
+            # Limit rows for testing
+            con.rle("mytable", "auto", limit=10000)
+        """
+        from .rle import (
+            calculate_cardinality_ratio,
+            test_column_orderings_smart,
+            calculate_rle_for_columns
+        )
+        from deltalake import DeltaTable
+        
+        # Parse table name and construct path
+        if table_name is None:
+            if mode != "summary":
+                print("⚠️  Table name is required for 'smart' and 'full' modes")
+                return None
+            # TODO: Implement all-tables summary
+            print("⚠️  All-tables summary not yet implemented. Please specify a table name.")
+            return None
+        
+        # Parse schema.table or just table
+        if '.' in table_name:
+            schema_name, tbl = table_name.split('.', 1)
+        else:
+            schema_name = self.schema
+            tbl = table_name
+        
+        # Construct the full table path using the same logic as get_stats
+        table_path = f"{self.table_base_url}{schema_name}/{tbl}"
+        
+        # Verify table exists and is not empty
+        print(f"📊 Analyzing table: {schema_name}.{tbl}")
+        
+        try:
+            dt = DeltaTable(table_path)
+            delta_files = dt.files()
+            
+            if not delta_files:
+                print("⚠️  Table is empty (no files)")
+                return None
+            
+        except Exception as e:
+            print(f"❌ Error accessing Delta table: {e}")
+            return None
+        
+        # Check if mode is a list of columns (custom ordering)
+        if isinstance(mode, list):
+            # User wants to test a specific column ordering
+            print(f"Testing custom column ordering: {', '.join(mode)}")
+            
+            # Calculate cardinality for NDV values
+            card_stats = calculate_cardinality_ratio(self.con, table_name if table_name else f"delta_scan('{table_path}')", is_parquet=False)
+            
+            # Calculate RLE for the specified ordering
+            rle_counts = calculate_rle_for_columns(self.con, table_path, mode, limit)
+            
+            total_rle_all = sum(rle_counts.values())
+            
+            print(f"\nResults:")
+            print(f"  Custom ordering: [{', '.join(mode)}]")
+            print(f"  Total RLE (all columns): {total_rle_all:,} runs")
+            
+            # Return as DataFrame for consistency
+            import pandas as pd
+            results = [{
+                'schema': schema_name,
+                'table': tbl,
+                'sort_order': 'custom',
+                'columns_used': ', '.join(mode),
+                'total_rle_all': total_rle_all,
+                **rle_counts
+            }]
+            
+            df = pd.DataFrame(results)
+            
+            # Transform to long format
+            long_format_results = []
+            
+            for _, row in df.iterrows():
+                schema_val = row['schema']
+                table_val = row['table']
+                sort_order = row['sort_order']
+                columns_used = row['columns_used']
+                total_rle_all_val = row['total_rle_all']
+                
+                # Get all column names except metadata columns
+                metadata_cols = ['schema', 'table', 'sort_order', 'columns_used', 'total_rle_all']
+                data_columns = [col for col in df.columns if col not in metadata_cols]
+                
+                # Get total rows from card_stats if available
+                total_rows = card_stats[data_columns[0]]['total_rows'] if card_stats and data_columns else None
+                
+                # Parse the columns_used to get ordering
+                sort_columns_list = [c.strip() for c in columns_used.split(',')]
+                
+                # Create one row per data column
+                for col in data_columns:
+                    rle_value = row[col]
+                    
+                    # Get NDV from card_stats
+                    ndv_value = card_stats[col]['distinct_values'] if card_stats and col in card_stats else None
+                    
+                    # Determine if column was included in the sort and its position
+                    is_in_sort = col in sort_columns_list
+                    order_position = sort_columns_list.index(col) + 1 if is_in_sort else None
+                    comment = '' if is_in_sort else 'not included in the sort'
+                    
+                    long_format_results.append({
+                        'schema': schema_val,
+                        'table': table_val,
+                        'sort_type': sort_order,
+                        'column': col,
+                        'order': order_position,
+                        'RLE': rle_value,
+                        'NDV': ndv_value,
+                        'total_rows': total_rows,
+                        'total_RLE': total_rle_all_val,
+                        'comments': comment
+                    })
+            
+            long_df = pd.DataFrame(long_format_results)
+            
+            return long_df
+        
+        # All modes now use test_column_orderings_smart with the mode parameter
+        return test_column_orderings_smart(
+            self.con,
+            table_path,
+            table_name=table_name,  # Pass table name for cardinality calculation on full dataset
+            mode=mode,
+            limit=limit,
+            min_distinct_threshold=min_distinct_threshold,
+            max_cardinality_pct=max_cardinality_pct,
+            max_ordering_depth=max_ordering_depth,
+            schema_name=schema_name,
+            table_display_name=tbl,
+            duckrun_instance=self  # Pass duckrun instance for detailed parquet stats
+        )
+
     def close(self):
         """Close DuckDB connection"""
+
         if self.con:
             self.con.close()
             print("Connection closed")

{duckrun-0.2.18.dev5 → duckrun-0.2.19}/duckrun/notebook.py

@@ -160,6 +160,7 @@ def import_notebook_from_web(
             update_url = f"{base_url}/workspaces/{workspace_id}/notebooks/{notebook_id}/updateDefinition"
             payload = {
                 "definition": {
+                    "format": "ipynb",
                     "parts": [
                         {
                             "path": "notebook-content.py",
@@ -192,6 +193,7 @@ def import_notebook_from_web(
             payload = {
                 "displayName": notebook_name,
                 "definition": {
+                    "format": "ipynb",
                     "parts": [
                         {
                             "path": "notebook-content.py",