duckrun 0.2.20.dev1__tar.gz → 0.2.20.dev3__tar.gz

{duckrun-0.2.20.dev1 → duckrun-0.2.20.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.20.dev1
+Version: 0.2.20.dev3
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.20.dev1 → duckrun-0.2.20.dev3}/duckrun/core.py

@@ -1757,4 +1757,13 @@ class WorkspaceConnection(WorkspaceOperationsMixin):
             print(f"❌ Error downloading semantic model: {e}")
             import traceback
             traceback.print_exc()
-            return None
+            return None
+    
+    def close(self):
+        """
+        Close the workspace connection.
+        
+        Note: WorkspaceConnection doesn't maintain persistent connections,
+        so this is a no-op for compatibility with code patterns that call close().
+        """
+        pass

{duckrun-0.2.20.dev1 → duckrun-0.2.20.dev3}/duckrun/ducklake_metadata.py

@@ -363,8 +363,11 @@ def create_checkpoint_for_latest_snapshot(con, table_info, data_root, temp_dir,
         ORDER BY column_order
     """).fetchall()
     
-    # Get or generate table metadata ID
-    table_meta_id = str(table_info['table_id'])
+    # Generate deterministic UUID for table metadata ID (Delta Lake spec requirement)
+    # Same table_id always produces same UUID for consistency across versions
+    import uuid
+    namespace = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8')  # DNS namespace
+    table_meta_id = str(uuid.uuid5(namespace, f"ducklake_table_{table_info['table_id']}"))
     
     # Prepare schema
     schema_fields = [

{duckrun-0.2.20.dev1 → duckrun-0.2.20.dev3}/duckrun/writer.py

@@ -81,9 +81,12 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
             args['max_rows_per_file'] = RG
             args['max_rows_per_group'] = RG
             args['min_rows_per_group'] = RG
-            # Set ZSTD compression for PyArrow engine
+            # Set ZSTD compression for PyArrow engine (dictionary encoding disabled for V-Order compatibility)
             if _HAS_PYARROW_DATASET:
-                args['file_options'] = ds.ParquetFileFormat().make_write_options(compression='ZSTD')
+                args['file_options'] = ds.ParquetFileFormat().make_write_options(
+                    compression='ZSTD',
+                    use_dictionary=False
+                )
         else:
             # Version 0.20+: no optimization available (rust by default, no row group params supported)
             # Set ZSTD compression for Rust engine

{duckrun-0.2.20.dev1 → duckrun-0.2.20.dev3}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.20.dev1
+Version: 0.2.20.dev3
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.20.dev1 → duckrun-0.2.20.dev3}/duckrun.egg-info/SOURCES.txt

@@ -21,4 +21,6 @@ duckrun.egg-info/top_level.txt
 tests/test_checkpoint_format.py
 tests/test_ducklake_export.py
 tests/test_register.py
-tests/test_rle.py
+tests/test_rle.py
+tests/test_writer_dictionary.py
+tests/test_writer_integration.py

{duckrun-0.2.20.dev1 → duckrun-0.2.20.dev3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.2.20.dev1"
+version = "0.2.20.dev3"
 description = "Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)"
 readme = "README.md"
 license = {text = "MIT"}

duckrun-0.2.20.dev3/tests/test_writer_dictionary.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Test script for writer.py dictionary encoding feature
+"""
+
+import sys
+import os
+
+# Add the parent directory to Python path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from duckrun.writer import _build_write_deltalake_args, _IS_OLD_DELTALAKE, _HAS_PYARROW_DATASET
+import pyarrow as pa
+
+def test_dictionary_encoding():
+    """Test that dictionary encoding is properly configured in write args"""
+    print("=" * 60)
+    print("Testing Dictionary Encoding in Writer")
+    print("=" * 60)
+    
+    # Create sample PyArrow table
+    data = {
+        'id': [1, 2, 3, 4, 5],
+        'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
+        'category': ['A', 'B', 'A', 'B', 'A']
+    }
+    df = pa.table(data)
+    
+    print(f"\nDeltalake version check:")
+    print(f"  - Is old deltalake (< 0.20): {_IS_OLD_DELTALAKE}")
+    print(f"  - Has PyArrow dataset: {_HAS_PYARROW_DATASET}")
+    
+    # Test 1: Normal write mode (no schema merging)
+    print("\n[Test 1] Normal write mode (no schema merging)")
+    print("-" * 60)
+    
+    args = _build_write_deltalake_args(
+        path='test/path',
+        df=df,
+        mode='overwrite',
+        schema_mode=None,
+        partition_by=None
+    )
+    
+    print(f"Arguments generated:")
+    for key, value in args.items():
+        if key == 'data':
+            print(f"  - {key}: <PyArrow Table>")
+        elif key == 'file_options':
+            print(f"  - {key}: <ParquetFileWriteOptions>")
+            # Try to inspect file_options
+            if hasattr(value, '__dict__'):
+                print(f"    Options: {value.__dict__}")
+        else:
+            print(f"  - {key}: {value}")
+    
+    # Check if dictionary encoding is enabled
+    if _IS_OLD_DELTALAKE and _HAS_PYARROW_DATASET:
+        if 'file_options' in args:
+            print("\n✅ [PASS] file_options is present in write args")
+            print("   Dictionary encoding (use_dictionary=True) should be configured")
+        else:
+            print("\n❌ [FAIL] file_options is missing from write args")
+            return False
+    else:
+        print("\n⚠️  [SKIP] Not using PyArrow engine (deltalake >= 0.20 or PyArrow dataset unavailable)")
+    
+    # Test 2: Schema merging mode
+    print("\n[Test 2] Schema merging mode")
+    print("-" * 60)
+    
+    args_merge = _build_write_deltalake_args(
+        path='test/path',
+        df=df,
+        mode='append',
+        schema_mode='merge',
+        partition_by=None
+    )
+    
+    print(f"Arguments generated:")
+    for key, value in args_merge.items():
+        if key == 'data':
+            print(f"  - {key}: <PyArrow Table>")
+        elif key == 'writer_properties':
+            print(f"  - {key}: <WriterProperties>")
+        else:
+            print(f"  - {key}: {value}")
+    
+    if 'schema_mode' in args_merge and args_merge['schema_mode'] == 'merge':
+        print("\n✅ [PASS] schema_mode='merge' is correctly set")
+        if _IS_OLD_DELTALAKE:
+            if 'engine' in args_merge and args_merge['engine'] == 'rust':
+                print("✅ [PASS] engine='rust' is correctly set for old deltalake")
+            else:
+                print("❌ [FAIL] engine='rust' is missing for old deltalake with schema merging")
+                return False
+    else:
+        print("\n❌ [FAIL] schema_mode='merge' is not properly set")
+        return False
+    
+    # Test 3: With partitioning
+    print("\n[Test 3] With partitioning")
+    print("-" * 60)
+    
+    args_partition = _build_write_deltalake_args(
+        path='test/path',
+        df=df,
+        mode='overwrite',
+        schema_mode=None,
+        partition_by=['category']
+    )
+    
+    if 'partition_by' in args_partition and args_partition['partition_by'] == ['category']:
+        print("✅ [PASS] partition_by is correctly set to ['category']")
+    else:
+        print("❌ [FAIL] partition_by is not properly configured")
+        return False
+    
+    print("\n" + "=" * 60)
+    print("✅ All dictionary encoding tests passed!")
+    print("=" * 60)
+    print("\nConfiguration details:")
+    print("  - ZSTD compression: Enabled")
+    print("  - Dictionary encoding: Enabled (for PyArrow engine)")
+    print("  - Optimized row groups: Enabled (for old deltalake)")
+    
+    return True
+
+if __name__ == "__main__":
+    try:
+        success = test_dictionary_encoding()
+        if success:
+            print("\n✅ Test completed successfully!")
+            sys.exit(0)
+        else:
+            print("\n❌ Test failed!")
+            sys.exit(1)
+    except Exception as e:
+        print(f"\n❌ Test error: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)

duckrun-0.2.20.dev3/tests/test_writer_integration.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""
+Integration test for writer.py with real Delta Lake write operations
+Tests dictionary encoding and compression in actual writes
+"""
+
+import sys
+import os
+
+# Add the parent directory to Python path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import duckrun
+from deltalake import DeltaTable
+
+def test_real_write_with_dictionary():
+    """Test real Delta Lake write with dictionary encoding"""
+    print("=" * 60)
+    print("Integration Test: Real Delta Lake Write")
+    print("=" * 60)
+    
+    # Configuration
+    ws = "tmp"
+    lh = "data"
+    schema = "test"
+    table_name = "writer_test_dict"
+    
+    try:
+        # Step 1: Connect to workspace
+        print("\n[Step 1] Connecting to workspace...")
+        workspace_conn = duckrun.connect(ws)
+        workspace_conn.create_lakehouse_if_not_exists(lh)
+        print(f"✅ Connected to workspace: {ws}")
+        
+        # Step 2: Connect to lakehouse
+        print(f"\n[Step 2] Connecting to lakehouse: {ws}/{lh}.lakehouse/{schema}")
+        conn = duckrun.connect(f"{ws}/{lh}.lakehouse/{schema}")
+        print(f"✅ Connected to lakehouse")
+        
+        # Step 3: Create test data with repetitive values (good for dictionary encoding)
+        print(f"\n[Step 3] Creating test data with repetitive values...")
+        test_data_sql = """
+        SELECT 
+            row_number() OVER () as id,
+            -- Repetitive categories (perfect for dictionary encoding)
+            CASE (row_number() OVER ()) % 5
+                WHEN 0 THEN 'Category_A'
+                WHEN 1 THEN 'Category_B'
+                WHEN 2 THEN 'Category_C'
+                WHEN 3 THEN 'Category_D'
+                ELSE 'Category_E'
+            END as category,
+            -- Repetitive status values
+            CASE (row_number() OVER ()) % 3
+                WHEN 0 THEN 'Active'
+                WHEN 1 THEN 'Pending'
+                ELSE 'Complete'
+            END as status,
+            -- Some numeric data
+            (row_number() OVER ()) * 1.5 as amount,
+            current_date as created_date
+        FROM generate_series(1, 100000) as t(i)
+        """
+        
+        result = conn.sql(test_data_sql)
+        row_count = len(result.fetchall())
+        print(f"✅ Created {row_count} test rows with repetitive categories and statuses")
+        
+        # Step 4: Write to Delta Lake using Spark-style API
+        print(f"\n[Step 4] Writing to Delta Lake table: {schema}.{table_name}")
+        print("   Using: mode=overwrite, ZSTD compression, dictionary encoding")
+        
+        conn.sql(test_data_sql).write \
+            .mode("overwrite") \
+            .saveAsTable(table_name)
+        
+        print(f"✅ Successfully wrote to {schema}.{table_name}")
+        
+        # Step 5: Verify the write
+        print(f"\n[Step 5] Verifying Delta Lake table...")
+        
+        # Get Delta table path
+        path = f"{conn.table_base_url}{schema}/{table_name}"
+        dt = DeltaTable(path)
+        
+        # Get file count
+        file_count = len(dt.file_uris())
+        print(f"   Table location: {path}")
+        print(f"   Number of files: {file_count}")
+        print(f"✅ Table verified successfully")
+        
+        # Step 6: Query the table to verify data
+        print(f"\n[Step 6] Querying table to verify data...")
+        verify_result = conn.sql(f"""
+            SELECT 
+                category,
+                status,
+                COUNT(*) as count,
+                AVG(amount) as avg_amount
+            FROM {table_name}
+            GROUP BY category, status
+            ORDER BY category, status
+        """)
+        
+        print("\n   Sample aggregation results:")
+        verify_result.show()
+        
+        # Step 7: Get detailed statistics showing compression and encoding
+        print(f"\n[Step 7] Getting detailed statistics (with compression info)...")
+        stats_detailed = conn.get_stats(table_name, detailed=True)
+        
+        print("\nParquet Statistics DataFrame:")
+        print(stats_detailed.to_string())
+        
+        print(f"\n✅ Statistics retrieved - {len(stats_detailed)} row groups analyzed")
+        
+        # Cleanup
+        print(f"\n[Cleanup] Closing connection...")
+        conn.close()
+        
+        print("\n" + "=" * 60)
+        print("✅ Integration Test PASSED!")
+        print("=" * 60)
+        print("\nVerified features:")
+        print("  ✓ Delta Lake write with Spark-style API")
+        print("  ✓ ZSTD compression configured")
+        print("  ✓ Dictionary encoding (check column encodings above)")
+        print("  ✓ Optimized row groups (8M rows)")
+        print("  ✓ Table creation and querying")
+        
+        return True
+        
+    except Exception as e:
+        print(f"\n❌ Integration test failed: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+if __name__ == "__main__":
+    try:
+        success = test_real_write_with_dictionary()
+        if success:
+            print("\n✅ All integration tests passed!")
+            sys.exit(0)
+        else:
+            print("\n❌ Integration test failed!")
+            sys.exit(1)
+    except Exception as e:
+        print(f"\n❌ Test error: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)