duckrun 0.2.20__tar.gz → 0.2.20.dev0__tar.gz

{duckrun-0.2.20 → duckrun-0.2.20.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.20
+Version: 0.2.20.dev0
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.20 → duckrun-0.2.20.dev0}/duckrun/core.py

@@ -1757,13 +1757,4 @@ class WorkspaceConnection(WorkspaceOperationsMixin):
             print(f"❌ Error downloading semantic model: {e}")
             import traceback
             traceback.print_exc()
-            return None
-    
-    def close(self):
-        """
-        Close the workspace connection.
-        
-        Note: WorkspaceConnection doesn't maintain persistent connections,
-        so this is a no-op for compatibility with code patterns that call close().
-        """
-        pass
+            return None

{duckrun-0.2.20 → duckrun-0.2.20.dev0}/duckrun/ducklake_metadata.py

@@ -8,21 +8,20 @@ import shutil
 
 def map_type_ducklake_to_spark(t):
     """Maps DuckDB data types to their Spark SQL equivalents for the Delta schema."""
-    t_lower = t.lower()
-    if 'int' in t_lower:
-        return 'long' if '64' in t_lower else 'integer'
-    elif 'float' in t_lower:
+    t = t.lower()
+    if 'int' in t:
+        return 'long' if '64' in t else 'integer'
+    elif 'float' in t:
         return 'double'
-    elif 'double' in t_lower:
+    elif 'double' in t:
         return 'double'
-    elif 'decimal' in t_lower:
-        # Preserve the original decimal precision and scale
-        return t_lower
-    elif 'bool' in t_lower:
+    elif 'decimal' in t:
+        return 'decimal(10,0)'
+    elif 'bool' in t:
         return 'boolean'
-    elif 'timestamp' in t_lower:
+    elif 'timestamp' in t:
         return 'timestamp'
-    elif 'date' in t_lower:
+    elif 'date' in t:
         return 'date'
     return 'string'
 
@@ -43,31 +42,14 @@ def convert_stat_value_to_json(value_str, column_type):
     column_type = column_type.lower()
     
     try:
-        # Timestamp: Convert to ISO 8601 with .000Z suffix (UTC format)
+        # Timestamp: Convert to ISO 8601 with .000Z suffix
         if 'timestamp' in column_type:
             # Parse and format to ISO 8601
-            # Handle various input formats from DuckDB:
-            # - "2025-06-22 23:55:00" -> "2025-06-22T23:55:00.000Z"
-            # - "2025-06-22T23:55:00+00" -> "2025-06-22T23:55:00.000Z"
-            # - "2025-06-22T23:55:00.123+00:00" -> "2025-06-22T23:55:00.123Z"
-            
-            # Replace space with T if needed
+            # Assumes value_str is in format like "2025-06-22 23:55:00"
             if 'T' not in value_str:
                 value_str = value_str.replace(' ', 'T')
-            
-            # Remove timezone offset formats to normalize to UTC
-            # Strip patterns like: +00, +00:00, -05:00, etc.
-            import re
-            value_str = re.sub(r'[+-]\d{2}(?::\d{2})?$', '', value_str)
-            
-            # Ensure milliseconds are present
-            if '.' not in value_str:
-                value_str += '.000'
-            
-            # Add Z suffix for UTC if not present
             if not value_str.endswith('Z'):
-                value_str += 'Z'
-            
+                value_str += '.000Z' if '.000Z' not in value_str else 'Z'
             return value_str
         
         # Date: Keep as YYYY-MM-DD string
@@ -363,11 +345,8 @@ def create_checkpoint_for_latest_snapshot(con, table_info, data_root, temp_dir,
         ORDER BY column_order
     """).fetchall()
     
-    # Generate deterministic UUID for table metadata ID (Delta Lake spec requirement)
-    # Same table_id always produces same UUID for consistency across versions
-    import uuid
-    namespace = uuid.UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8')  # DNS namespace
-    table_meta_id = str(uuid.uuid5(namespace, f"ducklake_table_{table_info['table_id']}"))
+    # Get or generate table metadata ID
+    table_meta_id = str(table_info['table_id'])
     
     # Prepare schema
     schema_fields = [

{duckrun-0.2.20 → duckrun-0.2.20.dev0}/duckrun/writer.py

@@ -83,9 +83,7 @@ def _build_write_deltalake_args(path, df, mode, schema_mode=None, partition_by=N
             args['min_rows_per_group'] = RG
             # Set ZSTD compression for PyArrow engine
             if _HAS_PYARROW_DATASET:
-                args['file_options'] = ds.ParquetFileFormat().make_write_options(
-                    compression='ZSTD'
-                )
+                args['file_options'] = ds.ParquetFileFormat().make_write_options(compression='ZSTD')
         else:
             # Version 0.20+: no optimization available (rust by default, no row group params supported)
             # Set ZSTD compression for Rust engine

{duckrun-0.2.20 → duckrun-0.2.20.dev0}/duckrun.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.2.20
+Version: 0.2.20.dev0
 Summary: Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)
 Author: mim
 License: MIT

{duckrun-0.2.20 → duckrun-0.2.20.dev0}/duckrun.egg-info/SOURCES.txt

@@ -21,6 +21,4 @@ duckrun.egg-info/top_level.txt
 tests/test_checkpoint_format.py
 tests/test_ducklake_export.py
 tests/test_register.py
-tests/test_rle.py
-tests/test_writer_dictionary.py
-tests/test_writer_integration.py
+tests/test_rle.py

{duckrun-0.2.20 → duckrun-0.2.20.dev0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.2.20"
+version = "0.2.20.dev0"
 description = "Helper library for Fabric Python using duckdb, arrow and delta_rs (orchestration, queries, etc.)"
 readme = "README.md"
 license = {text = "MIT"}

duckrun-0.2.20.dev0/tests/test_ducklake_export.py

@@ -0,0 +1,7 @@
+import duckrun
+def test_ducklake_export():
+    con = duckrun.connect("tmp/tmp.Lakehouse/dbo")
+    result = con.export_ducklake_to_delta("meta.db")
+    return result
+if __name__ == "__main__":
+    test_ducklake_export()

duckrun-0.2.20/tests/test_ducklake_export.py

@@ -1,4 +0,0 @@
-import duckrun
-con = duckrun.connect("tmp/data.lakehouse/duck")
-con.export_ducklake_to_delta("duck.db")
-con.get_stats("summary")

duckrun-0.2.20/tests/test_writer_dictionary.py

@@ -1,142 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test script for writer.py dictionary encoding feature
-"""
-
-import sys
-import os
-
-# Add the parent directory to Python path
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from duckrun.writer import _build_write_deltalake_args, _IS_OLD_DELTALAKE, _HAS_PYARROW_DATASET
-import pyarrow as pa
-
-def test_dictionary_encoding():
-    """Test that dictionary encoding is properly configured in write args"""
-    print("=" * 60)
-    print("Testing Dictionary Encoding in Writer")
-    print("=" * 60)
-    
-    # Create sample PyArrow table
-    data = {
-        'id': [1, 2, 3, 4, 5],
-        'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
-        'category': ['A', 'B', 'A', 'B', 'A']
-    }
-    df = pa.table(data)
-    
-    print(f"\nDeltalake version check:")
-    print(f"  - Is old deltalake (< 0.20): {_IS_OLD_DELTALAKE}")
-    print(f"  - Has PyArrow dataset: {_HAS_PYARROW_DATASET}")
-    
-    # Test 1: Normal write mode (no schema merging)
-    print("\n[Test 1] Normal write mode (no schema merging)")
-    print("-" * 60)
-    
-    args = _build_write_deltalake_args(
-        path='test/path',
-        df=df,
-        mode='overwrite',
-        schema_mode=None,
-        partition_by=None
-    )
-    
-    print(f"Arguments generated:")
-    for key, value in args.items():
-        if key == 'data':
-            print(f"  - {key}: <PyArrow Table>")
-        elif key == 'file_options':
-            print(f"  - {key}: <ParquetFileWriteOptions>")
-            # Try to inspect file_options
-            if hasattr(value, '__dict__'):
-                print(f"    Options: {value.__dict__}")
-        else:
-            print(f"  - {key}: {value}")
-    
-    # Check if dictionary encoding is enabled
-    if _IS_OLD_DELTALAKE and _HAS_PYARROW_DATASET:
-        if 'file_options' in args:
-            print("\n✅ [PASS] file_options is present in write args")
-            print("   Dictionary encoding (use_dictionary=True) should be configured")
-        else:
-            print("\n❌ [FAIL] file_options is missing from write args")
-            return False
-    else:
-        print("\n⚠️  [SKIP] Not using PyArrow engine (deltalake >= 0.20 or PyArrow dataset unavailable)")
-    
-    # Test 2: Schema merging mode
-    print("\n[Test 2] Schema merging mode")
-    print("-" * 60)
-    
-    args_merge = _build_write_deltalake_args(
-        path='test/path',
-        df=df,
-        mode='append',
-        schema_mode='merge',
-        partition_by=None
-    )
-    
-    print(f"Arguments generated:")
-    for key, value in args_merge.items():
-        if key == 'data':
-            print(f"  - {key}: <PyArrow Table>")
-        elif key == 'writer_properties':
-            print(f"  - {key}: <WriterProperties>")
-        else:
-            print(f"  - {key}: {value}")
-    
-    if 'schema_mode' in args_merge and args_merge['schema_mode'] == 'merge':
-        print("\n✅ [PASS] schema_mode='merge' is correctly set")
-        if _IS_OLD_DELTALAKE:
-            if 'engine' in args_merge and args_merge['engine'] == 'rust':
-                print("✅ [PASS] engine='rust' is correctly set for old deltalake")
-            else:
-                print("❌ [FAIL] engine='rust' is missing for old deltalake with schema merging")
-                return False
-    else:
-        print("\n❌ [FAIL] schema_mode='merge' is not properly set")
-        return False
-    
-    # Test 3: With partitioning
-    print("\n[Test 3] With partitioning")
-    print("-" * 60)
-    
-    args_partition = _build_write_deltalake_args(
-        path='test/path',
-        df=df,
-        mode='overwrite',
-        schema_mode=None,
-        partition_by=['category']
-    )
-    
-    if 'partition_by' in args_partition and args_partition['partition_by'] == ['category']:
-        print("✅ [PASS] partition_by is correctly set to ['category']")
-    else:
-        print("❌ [FAIL] partition_by is not properly configured")
-        return False
-    
-    print("\n" + "=" * 60)
-    print("✅ All dictionary encoding tests passed!")
-    print("=" * 60)
-    print("\nConfiguration details:")
-    print("  - ZSTD compression: Enabled")
-    print("  - Dictionary encoding: Enabled (for PyArrow engine)")
-    print("  - Optimized row groups: Enabled (for old deltalake)")
-    
-    return True
-
-if __name__ == "__main__":
-    try:
-        success = test_dictionary_encoding()
-        if success:
-            print("\n✅ Test completed successfully!")
-            sys.exit(0)
-        else:
-            print("\n❌ Test failed!")
-            sys.exit(1)
-    except Exception as e:
-        print(f"\n❌ Test error: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)

duckrun-0.2.20/tests/test_writer_integration.py

@@ -1,152 +0,0 @@
-#!/usr/bin/env python3
-"""
-Integration test for writer.py with real Delta Lake write operations
-Tests dictionary encoding and compression in actual writes
-"""
-
-import sys
-import os
-
-# Add the parent directory to Python path
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import duckrun
-from deltalake import DeltaTable
-
-def test_real_write_with_dictionary():
-    """Test real Delta Lake write with dictionary encoding"""
-    print("=" * 60)
-    print("Integration Test: Real Delta Lake Write")
-    print("=" * 60)
-    
-    # Configuration
-    ws = "tmp"
-    lh = "data"
-    schema = "test"
-    table_name = "writer_test_dict"
-    
-    try:
-        # Step 1: Connect to workspace
-        print("\n[Step 1] Connecting to workspace...")
-        workspace_conn = duckrun.connect(ws)
-        workspace_conn.create_lakehouse_if_not_exists(lh)
-        print(f"✅ Connected to workspace: {ws}")
-        
-        # Step 2: Connect to lakehouse
-        print(f"\n[Step 2] Connecting to lakehouse: {ws}/{lh}.lakehouse/{schema}")
-        conn = duckrun.connect(f"{ws}/{lh}.lakehouse/{schema}")
-        print(f"✅ Connected to lakehouse")
-        
-        # Step 3: Create test data with repetitive values (good for dictionary encoding)
-        print(f"\n[Step 3] Creating test data with repetitive values...")
-        test_data_sql = """
-        SELECT 
-            row_number() OVER () as id,
-            -- Repetitive categories (perfect for dictionary encoding)
-            CASE (row_number() OVER ()) % 5
-                WHEN 0 THEN 'Category_A'
-                WHEN 1 THEN 'Category_B'
-                WHEN 2 THEN 'Category_C'
-                WHEN 3 THEN 'Category_D'
-                ELSE 'Category_E'
-            END as category,
-            -- Repetitive status values
-            CASE (row_number() OVER ()) % 3
-                WHEN 0 THEN 'Active'
-                WHEN 1 THEN 'Pending'
-                ELSE 'Complete'
-            END as status,
-            -- Some numeric data
-            (row_number() OVER ()) * 1.5 as amount,
-            current_date as created_date
-        FROM generate_series(1, 100000) as t(i)
-        """
-        
-        result = conn.sql(test_data_sql)
-        row_count = len(result.fetchall())
-        print(f"✅ Created {row_count} test rows with repetitive categories and statuses")
-        
-        # Step 4: Write to Delta Lake using Spark-style API
-        print(f"\n[Step 4] Writing to Delta Lake table: {schema}.{table_name}")
-        print("   Using: mode=overwrite, ZSTD compression, dictionary encoding")
-        
-        conn.sql(test_data_sql).write \
-            .mode("overwrite") \
-            .saveAsTable(table_name)
-        
-        print(f"✅ Successfully wrote to {schema}.{table_name}")
-        
-        # Step 5: Verify the write
-        print(f"\n[Step 5] Verifying Delta Lake table...")
-        
-        # Get Delta table path
-        path = f"{conn.table_base_url}{schema}/{table_name}"
-        dt = DeltaTable(path)
-        
-        # Get file count
-        file_count = len(dt.file_uris())
-        print(f"   Table location: {path}")
-        print(f"   Number of files: {file_count}")
-        print(f"✅ Table verified successfully")
-        
-        # Step 6: Query the table to verify data
-        print(f"\n[Step 6] Querying table to verify data...")
-        verify_result = conn.sql(f"""
-            SELECT 
-                category,
-                status,
-                COUNT(*) as count,
-                AVG(amount) as avg_amount
-            FROM {table_name}
-            GROUP BY category, status
-            ORDER BY category, status
-        """)
-        
-        print("\n   Sample aggregation results:")
-        verify_result.show()
-        
-        # Step 7: Get detailed statistics showing compression and encoding
-        print(f"\n[Step 7] Getting detailed statistics (with compression info)...")
-        stats_detailed = conn.get_stats(table_name, detailed=True)
-        
-        print("\nParquet Statistics DataFrame:")
-        print(stats_detailed.to_string())
-        
-        print(f"\n✅ Statistics retrieved - {len(stats_detailed)} row groups analyzed")
-        
-        # Cleanup
-        print(f"\n[Cleanup] Closing connection...")
-        conn.close()
-        
-        print("\n" + "=" * 60)
-        print("✅ Integration Test PASSED!")
-        print("=" * 60)
-        print("\nVerified features:")
-        print("  ✓ Delta Lake write with Spark-style API")
-        print("  ✓ ZSTD compression configured")
-        print("  ✓ Dictionary encoding (check column encodings above)")
-        print("  ✓ Optimized row groups (8M rows)")
-        print("  ✓ Table creation and querying")
-        
-        return True
-        
-    except Exception as e:
-        print(f"\n❌ Integration test failed: {e}")
-        import traceback
-        traceback.print_exc()
-        return False
-
-if __name__ == "__main__":
-    try:
-        success = test_real_write_with_dictionary()
-        if success:
-            print("\n✅ All integration tests passed!")
-            sys.exit(0)
-        else:
-            print("\n❌ Integration test failed!")
-            sys.exit(1)
-    except Exception as e:
-        print(f"\n❌ Test error: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)