ducklake-delta-exporter 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

ducklake_delta_exporter/__init__.py

@@ -1,12 +1,7 @@
 # File: ducklake_delta_exporter.py
-import os
 import json
-import uuid
 import time
 import duckdb
-import pyarrow as pa
-import pyarrow.parquet as pq
-from datetime import datetime
 
 def map_type_ducklake_to_spark(t):
     """Maps DuckDB data types to their Spark SQL equivalents for the Delta schema."""
@@ -31,148 +26,265 @@ def create_spark_schema_string(fields):
     """Creates a JSON string for the Spark schema from a list of fields."""
     return json.dumps({"type": "struct", "fields": fields})
 
-def get_spark_checkpoint_schema():
-    """Returns the PyArrow schema for a Delta Lake checkpoint file."""
-    return pa.schema([
-        pa.field("protocol", pa.struct([
-            pa.field("minReaderVersion", pa.int32()),   # Made nullable
-            pa.field("minWriterVersion", pa.int32())    # Made nullable
-        ]), nullable=True),
-        pa.field("metaData", pa.struct([
-            pa.field("id", pa.string()),
-            pa.field("name", pa.string()),
-            pa.field("description", pa.string()),
-            pa.field("format", pa.struct([
-                pa.field("provider", pa.string()),
-                pa.field("options", pa.map_(pa.string(), pa.string()))
-            ])),
-            pa.field("schemaString", pa.string()),
-            pa.field("partitionColumns", pa.list_(pa.string())),
-            pa.field("createdTime", pa.int64()),
-            pa.field("configuration", pa.map_(pa.string(), pa.string()))
-        ]), nullable=True),
-        pa.field("add", pa.struct([
-            pa.field("path", pa.string()),
-            pa.field("partitionValues", pa.map_(pa.string(), pa.string())),
-            pa.field("size", pa.int64()),
-            pa.field("modificationTime", pa.int64()),
-            pa.field("dataChange", pa.bool_()),
-            pa.field("stats", pa.string(), nullable=True),
-            pa.field("tags", pa.map_(pa.string(), pa.string()), nullable=True)
-            # Removed deletionVector, baseRowId, defaultRowCommitVersion, clusteringProvider
-        ]), nullable=True),
-        pa.field("remove", pa.struct([
-            pa.field("path", pa.string()),
-            pa.field("deletionTimestamp", pa.int64()),
-            pa.field("dataChange", pa.bool_())
-        ]), nullable=True),
-        pa.field("commitInfo", pa.struct([
-            pa.field("timestamp", pa.timestamp('ms'), False),  # Changed from pa.int64() to pa.timestamp('ms')
-            pa.field("operation", pa.string()),
-            pa.field("operationParameters", pa.map_(pa.string(), pa.string())),
-            pa.field("isBlindAppend", pa.bool_(), nullable=True),
-            pa.field("engineInfo", pa.string(), nullable=True),
-            pa.field("clientVersion", pa.string(), nullable=True)
-        ]), nullable=True)
-    ])
-
-def get_latest_delta_version_info(delta_log_path, con, table_id):
+def get_latest_ducklake_snapshot(con, table_id):
+    """
+    Get the latest DuckLake snapshot ID for a table.
     """
-    Determines the latest Delta version exported and reconstructs the set of files
-    that were part of that Delta version, based on the embedded DuckLake snapshot ID.
-    Also retrieves the consistent metaData.id if available from version 0.
+    latest_snapshot  = con.execute(f""" SELECT MAX(begin_snapshot) as latest_snapshot FROM ducklake_data_file  WHERE table_id = {table_id} """).fetchone()[0]
+    return latest_snapshot
 
-    Returns (latest_delta_version, set_of_files_in_that_version, latest_ducklake_snapshot_id_in_delta, meta_id_from_delta_log).
+def get_latest_delta_checkpoint(con, table_id):
+    """
+    check how many times a table has being modified.
     """
-    last_delta_version_idx = -1
-    last_exported_ducklake_snapshot_id = None
-    files_in_last_delta_version = set()
-    meta_id_from_delta_log = None # This should be consistent for the table
+    delta_checkpoint = con.execute(f""" SELECT count(snapshot_id) FROM ducklake_snapshot_changes
+                                   where changes_made like '%:{table_id}' or changes_made like '%:{table_id},%' """).fetchone()[0]
+    print(table_id)
+    print(delta_checkpoint)
+    return delta_checkpoint
 
-    # Collect all files ending with .json
-    log_files = [f for f in os.listdir(delta_log_path) if f.endswith('.json')]
+def get_file_modification_time(dummy_time):
+    """
+    Return a dummy modification time for parquet files.
+    This avoids the latency of actually reading file metadata.
+    
+    Args:
+        dummy_time: Timestamp in milliseconds to use as modification time
     
-    if not log_files:
-        return last_delta_version_idx, files_in_last_delta_version, last_exported_ducklake_snapshot_id, meta_id_from_delta_log
+    Returns:
+        Modification time in milliseconds
+    """
+    return dummy_time
 
-    try:
-        # Collect valid version numbers from file names
-        found_versions = []
-        for f_name in log_files:
-            base_name = f_name.split('.')[0]
-            # Check if filename starts with '0000' and consists entirely of digits
-            if base_name.startswith('0000') and base_name.isdigit():
-                found_versions.append(int(base_name))
+def create_dummy_json_log(table_root, delta_version, table_info, schema_fields, now):
+    """
+    Create a minimal JSON log file for Spark compatibility using DuckDB.
+    """
+    json_log_file = table_root + f"_delta_log/{delta_version:020d}.json"
+    
+    # Create JSON log entries using DuckDB
+    duckdb.execute("DROP TABLE IF EXISTS json_log_table")
+    
+    # Protocol entry
+    protocol_json = json.dumps({
+        "protocol": {
+            "minReaderVersion": 1,
+            "minWriterVersion": 2
+        }
+    })
+    
+    # Metadata entry
+    metadata_json = json.dumps({
+        "metaData": {
+            "id": str(table_info['table_id']),
+            "name": table_info['table_name'],
+            "description": None,
+            "format": {
+                "provider": "parquet",
+                "options": {}
+            },
+            "schemaString": create_spark_schema_string(schema_fields),
+            "partitionColumns": [],
+            "createdTime": now,
+            "configuration": {
+                "delta.logRetentionDuration": "interval 1 hour"
+            }
+        }
+    })
+    
+    # Commit info entry
+    commitinfo_json = json.dumps({
+        "commitInfo": {
+            "timestamp": now,
+            "operation": "CONVERT",
+            "operationParameters": {
+                "convertedFrom": "DuckLake"
+            },
+            "isBlindAppend": True,
+            "engineInfo": "DuckLake-Delta-Exporter",
+            "clientVersion": "1.0.0"
+        }
+    })
+    
+    # Create table with JSON entries
+    duckdb.execute("""
+        CREATE TABLE json_log_table AS
+        SELECT ? AS json_line
+        UNION ALL
+        SELECT ? AS json_line
+        UNION ALL
+        SELECT ? AS json_line
+    """, [protocol_json, metadata_json, commitinfo_json])
+    
+    # Write JSON log file using DuckDB
+    duckdb.execute(f"COPY (SELECT json_line FROM json_log_table) TO '{json_log_file}' (FORMAT CSV, HEADER false, QUOTE '')")
+    
+    # Clean up
+    duckdb.execute("DROP TABLE IF EXISTS json_log_table")
+    
+    return json_log_file
 
-        if not found_versions:
-            # No valid versioned log files found with the '0000' prefix
-            return last_delta_version_idx, files_in_last_delta_version, last_exported_ducklake_snapshot_id, meta_id_from_delta_log
+def build_file_path(table_root, relative_path):
+    """
+    Build full file path from table root and relative path.
+    Works with both local paths and S3 URLs.
+    """
+    table_root = table_root.rstrip('/')
+    relative_path = relative_path.lstrip('/')
+    return f"{table_root}/{relative_path}"
 
-        # Get the highest version index
-        last_delta_version_idx = max(found_versions)
-        last_log_file = os.path.join(delta_log_path, f"{last_delta_version_idx:020d}.json")
+def create_checkpoint_for_latest_snapshot(con, table_info, data_root):
+    """
+    Create a Delta checkpoint file for the latest DuckLake snapshot.
+    """
+    table_root = data_root.rstrip('/') + '/' + table_info['schema_path'] + table_info['table_path']
+    
+    # Get the latest snapshot
+    latest_snapshot = get_latest_ducklake_snapshot(con, table_info['table_id'])
+    if latest_snapshot is None:
+        print(f"⚠️ {table_info['schema_name']}.{table_info['table_name']}: No snapshots found")
+        return False
+    delta_version   = get_latest_delta_checkpoint(con, table_info['table_id'])
+    checkpoint_file = table_root + f"_delta_log/{delta_version:020d}.checkpoint.parquet"
+    json_log_file   = table_root + f"_delta_log/{delta_version:020d}.json"
+    
+    try:
+        con.execute(f"SELECT protocol FROM '{checkpoint_file}' limit 0 ")
+        print(f"⚠️ {table_info['schema_name']}.{table_info['table_name']}: Checkpoint file already exists: {checkpoint_file}")
+    except:
+    
+        now = int(time.time() * 1000)
         
-        # Attempt to read the last log file for commitInfo and metaData (if present)
-        with open(last_log_file, 'r') as f:
-            for line in f:
-                try:
-                    action = json.loads(line)
-                    if 'commitInfo' in action:
-                        commit_info = action['commitInfo']
-                        if 'operationParameters' in commit_info and 'duckLakeSnapshotId' in commit_info['operationParameters']:
-                            last_exported_ducklake_snapshot_id = int(commit_info['operationParameters']['duckLakeSnapshotId'])
-                    if 'metaData' in action:
-                        meta_id_from_delta_log = action['metaData'].get('id')
-                except json.JSONDecodeError as e:
-                    print(f"ERROR: Failed to parse JSON line in {last_log_file}: {line.strip()}. Error: {e}")
-                except Exception as e:
-                    print(f"ERROR: Unexpected error processing line in {last_log_file}: {e}")
+        # Get all files for the latest snapshot
+        file_rows = con.execute(f"""
+            SELECT path, file_size_bytes FROM ducklake_data_file
+            WHERE table_id = {table_info['table_id']}
+            AND begin_snapshot <= {latest_snapshot} 
+            AND (end_snapshot IS NULL OR end_snapshot > {latest_snapshot})
+        """).fetchall()
         
-        # If metaData.id was not found in the latest log file, try to get it from version 0
-        if meta_id_from_delta_log is None:
-            v0_log_file = os.path.join(delta_log_path, "00000000000000000000.json")
-            if os.path.exists(v0_log_file):
-                with open(v0_log_file, 'r') as v0f:
-                    for v0_line in v0f:
-                        try:
-                            v0_action = json.loads(v0_line)
-                            if 'metaData' in v0_action:
-                                meta_id_from_delta_log = v0_action['metaData'].get('id')
-                                break
-                        except json.JSONDecodeError:
-                            pass # Ignore parsing errors for v0 metadata, just try next line
-
-        # If a valid last_exported_ducklake_snapshot_id was found, reconstruct the files
-        if last_exported_ducklake_snapshot_id is not None:
-            file_rows = con.execute(f"""
-                SELECT path FROM ducklake_data_file
-                WHERE table_id = {table_id}
-                  AND begin_snapshot <= {last_exported_ducklake_snapshot_id} AND (end_snapshot IS NULL OR end_snapshot > {last_exported_ducklake_snapshot_id})
-            """).fetchall()
-            files_in_last_delta_version = {path.lstrip('/') for path, in file_rows}
-        else:
-            print(f"WARNING: 'duckLakeSnapshotId' not found or parsed from latest log ({last_log_file}). Cannot reconstruct previous Delta table state accurately for diffing.")
-
-    except Exception as e:
-        print(f"ERROR: Unhandled exception in get_latest_delta_version_info for {delta_log_path}. Resetting state. Error: {e}")
-        last_delta_version_idx = -1 # Reset to -1 if there's an issue parsing or finding files
-
-    return last_delta_version_idx, files_in_last_delta_version, last_exported_ducklake_snapshot_id, meta_id_from_delta_log
-
+        # Get schema for the latest snapshot
+        columns = con.execute(f"""
+            SELECT column_name, column_type FROM ducklake_column
+            WHERE table_id = {table_info['table_id']}
+            AND begin_snapshot <= {latest_snapshot} 
+            AND (end_snapshot IS NULL OR end_snapshot > {latest_snapshot})
+            ORDER BY column_order
+        """).fetchall()
+        
+        # Get or generate table metadata ID
+        table_meta_id = str(table_info['table_id'])
+        
+        # Prepare schema
+        schema_fields = [
+            {"name": name, "type": map_type_ducklake_to_spark(typ), "nullable": True, "metadata": {}} 
+            for name, typ in columns
+        ]
+        
+        # Create checkpoint data using DuckDB directly
+        checkpoint_data = []
+        
+        # Create checkpoint data directly in DuckDB using proper data types
+        duckdb.execute("DROP TABLE IF EXISTS checkpoint_table")
+        
+        # Create the checkpoint table with proper nested structure
+        duckdb.execute("""
+            CREATE TABLE checkpoint_table AS
+            WITH checkpoint_data AS (
+                -- Protocol record
+                SELECT 
+                    {'minReaderVersion': 1, 'minWriterVersion': 2}::STRUCT(minReaderVersion INTEGER, minWriterVersion INTEGER) AS protocol,
+                    NULL::STRUCT(id VARCHAR, name VARCHAR, description VARCHAR, format STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)), schemaString VARCHAR, partitionColumns VARCHAR[], createdTime BIGINT, configuration MAP(VARCHAR, VARCHAR)) AS metaData,
+                    NULL::STRUCT(path VARCHAR, partitionValues MAP(VARCHAR, VARCHAR), size BIGINT, modificationTime BIGINT, dataChange BOOLEAN, stats VARCHAR, tags MAP(VARCHAR, VARCHAR)) AS add,
+                    NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
+                    NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isBlindAppend BOOLEAN, engineInfo VARCHAR, clientVersion VARCHAR) AS commitInfo
+                
+                UNION ALL
+                
+                -- Metadata record
+                SELECT 
+                    NULL::STRUCT(minReaderVersion INTEGER, minWriterVersion INTEGER) AS protocol,
+                    {
+                        'id': ?, 
+                        'name': ?, 
+                        'description': NULL, 
+                        'format': {'provider': 'parquet', 'options': MAP{}}::STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)),
+                        'schemaString': ?, 
+                        'partitionColumns': []::VARCHAR[], 
+                        'createdTime': ?, 
+                        'configuration': MAP{'delta.logRetentionDuration': 'interval 1 hour'}
+                    }::STRUCT(id VARCHAR, name VARCHAR, description VARCHAR, format STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)), schemaString VARCHAR, partitionColumns VARCHAR[], createdTime BIGINT, configuration MAP(VARCHAR, VARCHAR)) AS metaData,
+                    NULL::STRUCT(path VARCHAR, partitionValues MAP(VARCHAR, VARCHAR), size BIGINT, modificationTime BIGINT, dataChange BOOLEAN, stats VARCHAR, tags MAP(VARCHAR, VARCHAR)) AS add,
+                    NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
+                    NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isBlindAppend BOOLEAN, engineInfo VARCHAR, clientVersion VARCHAR) AS commitInfo
+            )
+            SELECT * FROM checkpoint_data
+        """, [table_meta_id, table_info['table_name'], create_spark_schema_string(schema_fields), now])
+        
+        # Add file records
+        for path, size in file_rows:
+            rel_path = path.lstrip('/')
+            full_path = build_file_path(table_root, rel_path)
+            mod_time = get_file_modification_time(now)
+            
+            duckdb.execute("""
+                INSERT INTO checkpoint_table
+                SELECT 
+                    NULL::STRUCT(minReaderVersion INTEGER, minWriterVersion INTEGER) AS protocol,
+                    NULL::STRUCT(id VARCHAR, name VARCHAR, description VARCHAR, format STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)), schemaString VARCHAR, partitionColumns VARCHAR[], createdTime BIGINT, configuration MAP(VARCHAR, VARCHAR)) AS metaData,
+                    {
+                        'path': ?, 
+                        'partitionValues': MAP{}::MAP(VARCHAR, VARCHAR), 
+                        'size': ?, 
+                        'modificationTime': ?, 
+                        'dataChange': true, 
+                        'stats': ?, 
+                        'tags': NULL::MAP(VARCHAR, VARCHAR)
+                    }::STRUCT(path VARCHAR, partitionValues MAP(VARCHAR, VARCHAR), size BIGINT, modificationTime BIGINT, dataChange BOOLEAN, stats VARCHAR, tags MAP(VARCHAR, VARCHAR)) AS add,
+                    NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
+                    NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isBlindAppend BOOLEAN, engineInfo VARCHAR, clientVersion VARCHAR) AS commitInfo
+            """, [rel_path, size, mod_time, json.dumps({"numRecords": None})])
+        
+        # Create the _delta_log directory if it doesn't exist
+        duckdb.execute(f"COPY (SELECT 43) TO '{table_root}_delta_log' (FORMAT PARQUET, PER_THREAD_OUTPUT, OVERWRITE_OR_IGNORE)")
+            
+        # Write the checkpoint file
+        duckdb.execute(f"COPY (SELECT * FROM checkpoint_table) TO '{checkpoint_file}' (FORMAT PARQUET)")
+        
+        # Create dummy JSON log file for Spark compatibility
+        create_dummy_json_log(table_root, delta_version, table_info, schema_fields, now)
+            
+        # Write the _last_checkpoint file
+        total_records = 2 + len(file_rows)  # protocol + metadata + file records
+        duckdb.execute(f"""
+            COPY (SELECT {delta_version} AS version, {total_records} AS size) 
+            TO '{table_root}_delta_log/_last_checkpoint' (FORMAT JSON, ARRAY false)
+        """)
+            
+        print(f"✅ Exported DuckLake snapshot {latest_snapshot} as Delta checkpoint v{delta_version}")
+        print(f"✅ Created JSON log file: {json_log_file}")
+        
+        # Clean up temporary tables
+        duckdb.execute("DROP TABLE IF EXISTS checkpoint_table")
+    
+    return True, delta_version, latest_snapshot
 
-def generate_latest_delta_log(db_path: str, data_root: str='/lakehouse/default/Tables', checkpoint_interval: int = 1):
+def generate_latest_delta_log(db_path: str, data_root: str = None):
     """
-    Generates a Delta Lake transaction log for the LATEST state of each table in a DuckLake database.
-    This creates incremental updates to Delta, not a full history.
+    Export the latest DuckLake snapshot for each table as a Delta checkpoint file.
+    Creates both checkpoint files and minimal JSON log files for Spark compatibility.
     
     Args:
         db_path (str): The path to the DuckLake database file.
         data_root (str): The root directory for the lakehouse data.
-        checkpoint_interval (int): The interval at which to create checkpoint files.
     """
     con = duckdb.connect(db_path, read_only=True)
-
-    tables = con.sql("""
+    
+    if data_root is None:
+        data_root = con.sql("SELECT value FROM ducklake_metadata WHERE key = 'data_path'").fetchone()[0]
+    
+    # Get all active tables
+    tables = con.execute("""
         SELECT 
             t.table_id, 
             t.table_name, 
@@ -182,280 +294,33 @@ def generate_latest_delta_log(db_path: str, data_root: str='/lakehouse/default/T
         FROM ducklake_table t
         JOIN ducklake_schema s USING(schema_id)
         WHERE t.end_snapshot IS NULL
-    """).df()
-
-    for row in tables.itertuples():
-        table_key = f"{row.schema_name}.{row.table_name}"
-        table_root = os.path.join(data_root, row.schema_path, row.table_path)
-        delta_log_path = os.path.join(table_root, "_delta_log")
-        os.makedirs(delta_log_path, exist_ok=True)
-        
-        # 1. Get the LATEST DuckLake snapshot for this table
-        latest_ducklake_snapshot_raw = con.execute(f"""
-            SELECT MAX(begin_snapshot) FROM ducklake_data_file
-            WHERE table_id = {row.table_id}
-        """).fetchone()
-        
-        if not latest_ducklake_snapshot_raw or latest_ducklake_snapshot_raw[0] is None:
-            print(f"⚠️ {table_key}: No data files found in DuckLake, skipping Delta log generation.")
-            continue
+    """).fetchall()
+    
+    total_tables = len(tables)
+    successful_exports = 0
+    
+    for table_row in tables:
+        table_info = {
+            'table_id': table_row[0],
+            'table_name': table_row[1],
+            'schema_name': table_row[2],
+            'table_path': table_row[3],
+            'schema_path': table_row[4]
+        }
         
-        latest_ducklake_snapshot_id = latest_ducklake_snapshot_raw[0]
-
-        # 2. Determine the current state of the Delta table and next Delta version
-        last_delta_version_idx, previously_exported_files, last_exported_ducklake_snapshot_id, existing_meta_id = \
-            get_latest_delta_version_info(delta_log_path, con, row.table_id)
+        table_key = f"{table_info['schema_name']}.{table_info['table_name']}"
+        print(f"Processing {table_key}...")
         
-        next_delta_version = last_delta_version_idx + 1
-
-        # Check if the Delta table is already up-to-date with the latest DuckLake snapshot
-        if last_exported_ducklake_snapshot_id == latest_ducklake_snapshot_id:
-            print(f"✅ {table_key}: Delta table already at latest DuckLake snapshot {latest_ducklake_snapshot_id} (Delta version {last_delta_version_idx}), skipping export.")
-            continue # Nothing to do, skip to next table
-
         try:
-            now = int(time.time() * 1000)
-            now_timestamp = datetime.fromtimestamp(now / 1000)  # Convert to datetime for checkpoint
-            log_file = os.path.join(delta_log_path, f"{next_delta_version:020d}.json")
-            checkpoint_file = os.path.join(delta_log_path, f"{next_delta_version:020d}.checkpoint.parquet")
-
-            # Fetch all current files associated with the LATEST DuckLake snapshot
-            file_rows_for_current_version = con.execute(f"""
-                SELECT path, file_size_bytes FROM ducklake_data_file
-                WHERE table_id = {row.table_id}
-                  AND begin_snapshot <= {latest_ducklake_snapshot_id} AND (end_snapshot IS NULL OR end_snapshot > {latest_ducklake_snapshot_id})
-            """).fetchall()
-
-            current_files_map = {}
-            for path, size in file_rows_for_current_version:
-                rel_path = path.lstrip('/')
-                full_path = os.path.join(table_root, rel_path)
-                mod_time = int(os.path.getmtime(full_path) * 1000) if os.path.exists(full_path) else now
-                current_files_map[rel_path] = {
-                    "path": rel_path, "size": size, "modification_time": mod_time,
-                    "stats": json.dumps({"numRecords": None}) # Stats would require reading files
-                }
-            current_file_paths = set(current_files_map.keys())
-
-            added_files_data = []
-            removed_files_paths = []
-
-            # Calculate the diff between the previous Delta state and the current latest DuckLake snapshot
-            added_file_paths = current_file_paths - previously_exported_files
-            removed_file_paths_set = previously_exported_files - current_file_paths
+            result = create_checkpoint_for_latest_snapshot(con, table_info, data_root)
             
-            added_files_data = [current_files_map[p] for p in added_file_paths]
-            # removed_files_paths only need the path, not full dict
-            removed_files_paths = list(removed_file_paths_set)
-
-            # If no changes and not the initial version 0, skip writing a log file
-            # Version 0 should always be written if it's the first export, even if empty (e.g., empty table)
-            if not added_files_data and not removed_files_paths and next_delta_version > 0:
-                print(f" {table_key}: No *detectable* changes between previous Delta state and latest DuckLake snapshot {latest_ducklake_snapshot_id}. Skipping new Delta log for version {next_delta_version}.")
-                continue # Skip to the next table
-
-            # Get schema for metadata (always from the latest DuckLake snapshot)
-            columns = con.execute(f"""
-                SELECT column_name, column_type FROM ducklake_column
-                WHERE table_id = {row.table_id}
-                  AND begin_snapshot <= {latest_ducklake_snapshot_id} AND (end_snapshot IS NULL OR end_snapshot > {latest_ducklake_snapshot_id})
-                ORDER BY column_order
-            """).fetchall()
-
-            with open(log_file, 'w') as f:
-                # Protocol always comes first
-                f.write(json.dumps({"protocol": {"minReaderVersion": 1, "minWriterVersion": 2}}) + "\n")
-
-                # Determine the table_meta_id
-                table_meta_id = existing_meta_id if existing_meta_id else str(uuid.uuid4())
+            if result:
+                successful_exports += 1
+            else:
+                print(f"⚠️ {table_key}: No data to export")
                 
-                # Metadata always comes second
-                schema_fields = [{"name": name, "type": map_type_ducklake_to_spark(typ), "nullable": True, "metadata": {}} for name, typ in columns]
-                
-                # Configuration, including logRetentionDuration
-                table_configuration = {"delta.logRetentionDuration": "interval 1 hour"}
-
-                f.write(json.dumps({
-                    "metaData": {
-                        "id": table_meta_id, 
-                        "name": row.table_name if row.table_name else None,
-                        "description": None,
-                        "format": {"provider": "parquet", "options": {}},
-                        "schemaString": create_spark_schema_string(schema_fields),
-                        "partitionColumns": [],
-                        "createdTime": now, 
-                        "configuration": table_configuration
-                    }
-                }) + "\n")
-
-                # Write remove actions
-                for path in removed_files_paths:
-                    f.write(json.dumps({"remove": {"path": path, "deletionTimestamp": now, "dataChange": True}}) + "\n")
-                
-                # Write add actions, excluding the explicitly removed fields
-                for af in added_files_data:
-                    f.write(json.dumps({
-                        "add": {
-                            "path": af["path"],
-                            "partitionValues": {},
-                            "size": af["size"],
-                            "modificationTime": af["modification_time"],
-                            "dataChange": True,
-                            "stats": af["stats"],
-                            "tags": None # Set to null as per example
-                            # Removed deletionVector, baseRowId, defaultRowCommitVersion, clusteringProvider
-                        }
-                    }) + "\n")
-                
-                # Prepare operationParameters for commitInfo based on Delta version
-                commit_operation_parameters = {
-                    "mode": "Overwrite",
-                    "partitionBy": "[]",
-                    "duckLakeSnapshotId": str(latest_ducklake_snapshot_id)
-                }
-                commit_operation = "WRITE"
-
-                if next_delta_version == 0:
-                    # For v0, emulate the 'CREATE TABLE' operation parameters as per example
-                    commit_operation = "CREATE TABLE"
-                    commit_operation_parameters = {
-                        "mode": "ErrorIfExists",
-                        "location": f"{data_root}/{row.schema_path}/{row.table_path}", # Construct location based on data_root
-                        "protocol": json.dumps({"minReaderVersion": 1, "minWriterVersion": 2}),
-                        "metadata": json.dumps({ # Stringify metadata object
-                            "configuration": table_configuration,
-                            "createdTime": now,
-                            "description": None,
-                            "format": {"options": {}, "provider": "parquet"},
-                            "id": table_meta_id,
-                            "name": row.table_name if row.table_name else None,
-                            "partitionColumns": [],
-                            "schemaString": create_spark_schema_string(schema_fields)
-                        })
-                    }
-
-                # Write CommitInfo
-                f.write(json.dumps({
-                    "commitInfo": {
-                        "timestamp": now,
-                        "operation": commit_operation,
-                        "operationParameters": commit_operation_parameters,
-                        "isBlindAppend": not removed_files_paths,
-                        "engineInfo": "DuckLake-Delta-Export-Latest",
-                        "clientVersion": "delta-rs.0.18.1" if next_delta_version == 0 else "DuckLake-Delta-Python" # Use example clientVersion for v0
-                    }
-                }) + "\n")
-
-            print(f"✅ {table_key}: Delta log written v{next_delta_version} (DuckLake snapshot: {latest_ducklake_snapshot_id})")
-            
-            # --- CHECKPOINT LOGIC ---
-            # Create checkpoint if it's a checkpoint version and doesn't already exist
-            if next_delta_version > 0 and next_delta_version % checkpoint_interval == 0 and not os.path.exists(checkpoint_file):
-                # Fixed checkpoint creation with proper protocol handling
-                checkpoint_records = []
-                
-                # First record: protocol only
-                checkpoint_records.append({
-                    "protocol": {"minReaderVersion": 1, "minWriterVersion": 2}, 
-                    "metaData": None, 
-                    "add": None, 
-                    "remove": None, 
-                    "commitInfo": None
-                })
-                
-                # Second record: metadata only
-                checkpoint_meta_id = existing_meta_id if existing_meta_id else str(uuid.uuid4())
-                checkpoint_records.append({
-                    "protocol": None, 
-                    "commitInfo": None, 
-                    "remove": None, 
-                    "add": None,
-                    "metaData": {
-                        "id": checkpoint_meta_id, 
-                        "name": row.table_name if row.table_name else None,
-                        "description": None,
-                        "format": {"provider": "parquet", "options": {}},
-                        "schemaString": create_spark_schema_string(schema_fields),
-                        "partitionColumns": [], 
-                        "createdTime": now, 
-                        "configuration": {"delta.logRetentionDuration": "interval 1 hour"}
-                    },
-                })
-
-                # Add all current files from the latest DuckLake snapshot to the checkpoint
-                for af_path in current_file_paths:
-                    af = current_files_map[af_path]
-                    checkpoint_records.append({
-                        "protocol": None, 
-                        "metaData": None, 
-                        "remove": None, 
-                        "commitInfo": None,
-                        "add": {
-                            "path": af["path"],
-                            "partitionValues": {},
-                            "size": af["size"],
-                            "modificationTime": af["modification_time"],
-                            "dataChange": True,
-                            "stats": af["stats"],
-                            "tags": None # Set to null as per example
-                            # Removed deletionVector, baseRowId, defaultRowCommitVersion, clusteringProvider
-                        },
-                    })
-                
-                # Create PyArrow table with proper handling of None values
-                table_data = {
-                    'protocol': [record.get("protocol") for record in checkpoint_records],
-                    'metaData': [record.get("metaData") for record in checkpoint_records],
-                    'add': [record.get("add") for record in checkpoint_records],
-                    'remove': [record.get("remove") for record in checkpoint_records],
-                    'commitInfo': [record.get("commitInfo") for record in checkpoint_records]
-                }
-                
-                # Create table directly with target schema to avoid casting issues
-                target_schema = get_spark_checkpoint_schema()
-                table = pa.table(table_data, schema=target_schema)
-                pq.write_table(table, checkpoint_file, compression='snappy')
-                
-                with open(os.path.join(delta_log_path, "_last_checkpoint"), 'w') as f:
-                    json.dump({"version": next_delta_version, "size": len(checkpoint_records)}, f)
-                
-                print(f"📸 {table_key}: Checkpoint created at Delta version {next_delta_version} (DuckLake snapshot: {latest_ducklake_snapshot_id})")
-
-                # --- Cleanup old JSON log files and Checkpoint files ---
-                print(f"🧹 {table_key}: Cleaning up old log and checkpoint files before Delta version {next_delta_version}...")
-                for f_name in os.listdir(delta_log_path):
-                    base_name = f_name.split('.')[0]
-                    # Check for versioned JSON log files
-                    if f_name.endswith('.json') and base_name.startswith('0000') and base_name.isdigit():
-                        log_version = int(base_name)
-                        if log_version < next_delta_version:
-                            file_to_delete = os.path.join(delta_log_path, f_name)
-                            try:
-                                os.remove(file_to_delete)
-                                print(f"  Deleted JSON log: {f_name}")
-                            except OSError as e:
-                                print(f"  Error deleting JSON log {f_name}: {e}")
-                    # Check for versioned Parquet checkpoint files
-                    elif f_name.endswith('.checkpoint.parquet'):
-                        checkpoint_base_name = f_name.split('.checkpoint.parquet')[0]
-                        if checkpoint_base_name.startswith('0000') and checkpoint_base_name.isdigit():
-                            checkpoint_version = int(checkpoint_base_name)
-                            if checkpoint_version < next_delta_version:
-                                file_to_delete = os.path.join(delta_log_path, f_name)
-                                try:
-                                    os.remove(file_to_delete)
-                                    print(f"  Deleted checkpoint: {f_name}")
-                                except OSError as e:
-                                    print(f"  Error deleting checkpoint {f_name}: {e}")
-                print(f"🧹 {table_key}: Cleanup complete.")
-
-            elif next_delta_version > 0 and next_delta_version % checkpoint_interval == 0 and os.path.exists(checkpoint_file):
-                print(f"⏩ {table_key}: Checkpoint for Delta version {next_delta_version} (DuckLake snapshot: {latest_ducklake_snapshot_id}) already exists, skipping generation.")
-
         except Exception as e:
-            print(f"❌ Failed processing {table_key} for Delta version {next_delta_version} (DuckLake snapshot: {latest_ducklake_snapshot_id}): {e}")
-            # This should ideally rollback the written log file if it partially succeeded,
-            # but for this script, we just log and continue to next table.
-
+            print(f"❌ {table_key}: Failed to export checkpoint - {e}")
+    
     con.close()
-    print("Delta export finished.")
+    print(f"\n🎉 Export completed! {successful_exports}/{total_tables} tables exported successfully.")

ducklake_delta_exporter-0.1.3.dist-info/METADATA

@@ -0,0 +1,123 @@
+Metadata-Version: 2.4
+Name: ducklake-delta-exporter
+Version: 0.1.3
+Summary: A utility to export DuckLake database metadata to Delta Lake transaction logs.
+Home-page: https://github.com/djouallah/ducklake_delta_exporter
+Author: mim
+Author-email: your.email@example.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Development Status :: 3 - Alpha
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: duckdb
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# DuckLake Delta Exporter
+
+A Python package for exporting DuckLake snapshots as Delta Lake checkpoint files, enabling compatibility with Delta Lake readers, support local path, s3 and gcs, for onelake use mounted storage as azure storage is not supported
+
+this is just a fun project
+
+## Repository
+
+https://github.com/djouallah/ducklake_delta_exporter
+
+## Installation
+
+```bash
+pip install ducklake-delta-exporter
+```
+
+## Usage
+
+```python
+from ducklake_delta_exporter import generate_latest_delta_log
+
+# Export all tables from a DuckLake database
+generate_latest_delta_log("/path/to/ducklake.db")
+
+# Specify a custom data root directory
+generate_latest_delta_log("/path/to/ducklake.db", data_root="/custom/data/path")
+```
+
+## What it does
+
+This package converts DuckLake table snapshots into Delta Lake format by:
+
+1. **Reading DuckLake metadata** - Extracts table schemas, file paths, and snapshot information
+2. **Creating Delta checkpoint files** - Generates `.checkpoint.parquet` files with Delta Lake metadata
+3. **Writing JSON transaction logs** - Creates minimal `.json` log files for Spark compatibility
+4. **Mapping data types** - Converts DuckDB types to Spark SQL equivalents
+
+## Features
+
+- ✅ **Spark Compatible** - Generated Delta files can be read by Spark and other Delta Lake tools
+- ✅ **Type Mapping** - Automatic conversion between DuckDB and Spark data types
+- ✅ **Batch Processing** - Exports all tables in a DuckLake database
+- ✅ **Error Handling** - Graceful handling of missing snapshots and other issues
+- ✅ **Progress Reporting** - Clear feedback on export progress and results
+
+## Requirements
+
+- Python 3.8+
+- DuckDB
+
+## File Structure
+
+After running the exporter, your Delta tables will have the following structure:
+
+```
+your_table/
+├── data_file_1.parquet
+├── data_file_2.parquet
+└── _delta_log/
+    ├── 00000000000000000000.json
+    ├── 00000000000000000000.checkpoint.parquet
+    └── _last_checkpoint
+```
+
+## Type Mapping
+
+The exporter automatically maps DuckDB types to Spark SQL types:
+
+| DuckDB Type | Spark Type |
+|-------------|------------|
+| INTEGER     | integer    |
+| BIGINT      | long       |
+| FLOAT       | double     |
+| DOUBLE      | double     |
+| DECIMAL     | decimal(10,0) |
+| BOOLEAN     | boolean    |
+| TIMESTAMP   | timestamp  |
+| DATE        | date       |
+| VARCHAR     | string     |
+| Others      | string     |
+
+## Error Handling
+
+The exporter handles various error conditions:
+
+- **Missing snapshots** - Skips tables with no data
+- **Existing checkpoints** - Avoids overwriting existing files
+- **Schema changes** - Uses the latest schema for each table
+- **File system errors** - Reports and continues with other tables
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

ducklake_delta_exporter-0.1.3.dist-info/RECORD

@@ -0,0 +1,5 @@
+ducklake_delta_exporter/__init__.py,sha256=oTLyKljSO2MlSJD9sRtg3wcn2nhBGpPMfwEOuKw360Q,14430
+ducklake_delta_exporter-0.1.3.dist-info/METADATA,sha256=6iaajQHsbymIXvkNnz9hgkXRRy9V2luvnl2m5qcO4Os,3855
+ducklake_delta_exporter-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ducklake_delta_exporter-0.1.3.dist-info/top_level.txt,sha256=cGISjIUrP9eP3UexjiCEWnWy8N5woIBV2QVF21OgdtQ,24
+ducklake_delta_exporter-0.1.3.dist-info/RECORD,,

ducklake_delta_exporter-0.1.1.dist-info/METADATA

@@ -1,71 +0,0 @@
-Metadata-Version: 2.4
-Name: ducklake-delta-exporter
-Version: 0.1.1
-Summary: A utility to export DuckLake database metadata to Delta Lake transaction logs.
-Home-page: https://github.com/djouallah/ducklake_delta_exporter
-Author: mim
-Author-email: your.email@example.com
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Intended Audience :: Developers
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: Development Status :: 3 - Alpha
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-Requires-Dist: duckdb
-Requires-Dist: pyarrow
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
-
-# 🦆 DuckLake Delta Exporter
-
-A Python utility to **bridge the gap between DuckLake and Delta Lake** by generating Delta-compatible transaction logs directly from DuckLake metadata.
-
-This isn’t your typical general-purpose library. It’s mostly battle-tested with **OneLake mounted storage**, and while it *should* work with local filesystems, there’s **no support for S3, GCS, or ABFSS** .
-
-It doesn’t use the `deltalake` Python package either. The metadata is handcrafted from scratch — because why not reinvent the wheel for fun and learning?
-
-**Goal?**  
-Mostly to annoy DuckDB developers into finally shipping a proper Delta Lake metadata exporter  😎
-
-🔗 [Source code on GitHub](https://github.com/djouallah/ducklake_delta_exporter)
-
----
-
-## ✨ Features
-
-- **DuckLake → Delta Sync**  
-  Generates Delta Lake `_delta_log/*.json` transaction files and Parquet checkpoints from the latest DuckLake state.
-
-- **Schema Mapping**  
-  Converts DuckDB types to their Spark SQL equivalents so Delta can understand them without throwing a tantrum.
-
-- **Change Detection**  
-  Detects file-level additions/removals since the last export — keeps things incremental and tidy.
-
-- **Checkpointing**  
-  Automatically writes Delta checkpoints every N versions (configurable), so readers don’t have to replay the entire log from scratch.
-
----
-
-## ⚙️ Installation & Usage
-
-Install via pip:
-
-```bash
-pip install ducklake-delta-exporter
-```
-
-```
-from ducklake_delta_exporter import generate_latest_delta_log
-
-generate_latest_delta_log('/lakehouse/default/Files/meta.db')
-```

ducklake_delta_exporter-0.1.1.dist-info/RECORD

@@ -1,5 +0,0 @@
-ducklake_delta_exporter/__init__.py,sha256=dCfDXzdj-LOi8FXGPuA_bfD4AQgmblMFF-x0YJR6SXA,24797
-ducklake_delta_exporter-0.1.1.dist-info/METADATA,sha256=Vmk_LrjMZIM41ZnueTUnEZLTY1TFoAdOgMZQWjP4Ouk,2539
-ducklake_delta_exporter-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ducklake_delta_exporter-0.1.1.dist-info/top_level.txt,sha256=cGISjIUrP9eP3UexjiCEWnWy8N5woIBV2QVF21OgdtQ,24
-ducklake_delta_exporter-0.1.1.dist-info/RECORD,,