ducklake-delta-exporter 0.1.3__tar.gz → 0.2.0__tar.gz

{ducklake_delta_exporter-0.1.3 → ducklake_delta_exporter-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ducklake-delta-exporter
-Version: 0.1.3
+Version: 0.2.0
 Summary: A utility to export DuckLake database metadata to Delta Lake transaction logs.
 Home-page: https://github.com/djouallah/ducklake_delta_exporter
 Author: mim
@@ -13,7 +13,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Development Status :: 3 - Alpha
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: duckdb
+Requires-Dist: duckdb>=1.4.4
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -28,7 +28,7 @@ Dynamic: summary
 
 A Python package for exporting DuckLake snapshots as Delta Lake checkpoint files, enabling compatibility with Delta Lake readers, support local path, s3 and gcs, for onelake use mounted storage as azure storage is not supported
 
-this is just a fun project
+this is just a fun project, please vote for a proper support in duckdb https://github.com/duckdb/duckdb-delta/issues/218
 
 ## Repository
 

{ducklake_delta_exporter-0.1.3 → ducklake_delta_exporter-0.2.0}/README.md

@@ -2,7 +2,7 @@
 
 A Python package for exporting DuckLake snapshots as Delta Lake checkpoint files, enabling compatibility with Delta Lake readers, support local path, s3 and gcs, for onelake use mounted storage as azure storage is not supported
 
-this is just a fun project
+this is just a fun project, please vote for a proper support in duckdb https://github.com/duckdb/duckdb-delta/issues/218
 
 ## Repository
 

ducklake_delta_exporter-0.2.0/ducklake_delta_exporter/__init__.py

@@ -0,0 +1,486 @@
+# File: ducklake_delta_exporter.py
+import duckdb
+
+
+def generate_latest_delta_log(db_path: str):
+    """
+    Export the latest DuckLake snapshot for each table as Delta checkpoint files.
+    Uses DuckDB 1.4.4+ native support for writing to abfss://, s3://, etc.
+
+    Args:
+        db_path (str): The path to the DuckLake database file (or connection string).
+    """
+    # For remote paths (abfss://, s3://, etc.), use in-memory connection with ATTACH
+    is_remote = any(db_path.startswith(prefix) for prefix in ['abfss://', 's3://', 'gs://', 'az://', 'http://', 'https://'])
+
+    if is_remote:
+        con = duckdb.connect()
+        # Load required extensions for cloud storage
+        if db_path.startswith('abfss://') or db_path.startswith('az://'):
+            con.execute("LOAD azure")
+            # Load persistent secrets
+            con.execute("SELECT * FROM duckdb_secrets()")
+        elif db_path.startswith('s3://'):
+            con.execute("LOAD httpfs")
+        con.execute(f"ATTACH '{db_path}' AS ducklake_db (READ_ONLY)")
+        con.execute("USE ducklake_db")
+    else:
+        con = duckdb.connect(db_path, read_only=True)
+
+    # Build export summary - identify which tables have data
+    con.execute("""
+        CREATE OR REPLACE TEMP TABLE export_summary AS
+        WITH
+        data_root_config AS (
+            SELECT value AS data_root FROM ducklake_metadata WHERE key = 'data_path'
+        ),
+        active_tables AS (
+            SELECT
+                t.table_id,
+                t.table_name,
+                s.schema_name,
+                t.path AS table_path,
+                s.path AS schema_path,
+                rtrim((SELECT data_root FROM data_root_config), '/') || '/' ||
+                    CASE
+                        WHEN trim(s.path, '/') != '' THEN trim(s.path, '/') || '/'
+                        ELSE ''
+                    END ||
+                    trim(t.path, '/') AS table_root
+            FROM ducklake_table t
+            JOIN ducklake_schema s USING(schema_id)
+            WHERE t.end_snapshot IS NULL
+        ),
+        current_snapshot AS (
+            SELECT MAX(snapshot_id) AS snapshot_id FROM ducklake_snapshot
+        ),
+        table_last_modified AS (
+            SELECT
+                t.*,
+                COALESCE(
+                    (SELECT MAX(sc.snapshot_id)
+                     FROM ducklake_snapshot_changes sc
+                     WHERE regexp_matches(sc.changes_made, '[:,]' || t.table_id || '([^0-9]|$)')
+                    ),
+                    (SELECT cs.snapshot_id
+                     FROM current_snapshot cs
+                     WHERE EXISTS (
+                         SELECT 1 FROM ducklake_data_file df
+                         WHERE df.table_id = t.table_id
+                           AND df.end_snapshot IS NULL
+                     )
+                    )
+                ) AS last_modified_snapshot,
+                (SELECT COUNT(*) FROM ducklake_data_file df
+                 WHERE df.table_id = t.table_id
+                   AND df.end_snapshot IS NULL
+                ) AS file_count
+            FROM active_tables t
+        )
+        SELECT
+            table_id,
+            schema_name,
+            table_name,
+            table_root,
+            CASE
+                WHEN file_count = 0 THEN 'no_data_files'
+                WHEN last_modified_snapshot IS NULL THEN 'no_changes'
+                ELSE 'needs_export'
+            END AS status,
+            last_modified_snapshot AS snapshot_id,
+            file_count
+        FROM table_last_modified
+    """)
+
+    # Get tables that need export
+    tables_to_export = con.execute("""
+        SELECT table_id, schema_name, table_name, table_root, snapshot_id, file_count
+        FROM export_summary
+        WHERE status = 'needs_export'
+    """).fetchall()
+
+    # Show summary
+    summary = con.execute("""
+        SELECT status, COUNT(*) as cnt FROM export_summary GROUP BY status
+    """).fetchall()
+
+    for status, cnt in summary:
+        print(f"  {status}: {cnt} tables")
+
+    if not tables_to_export:
+        print("\n✅ No tables need export.")
+        con.close()
+        return
+
+    print(f"\n📦 Exporting {len(tables_to_export)} tables...")
+
+    # Process each table
+    for table_id, schema_name, table_name, table_root, snapshot_id, file_count in tables_to_export:
+        table_key = f"{schema_name}.{table_name}"
+
+        # Check if checkpoint already exists for this snapshot
+        checkpoint_path = f"{table_root}/_delta_log/{snapshot_id:020d}.checkpoint.parquet"
+        try:
+            con.execute(f"SELECT 1 FROM '{checkpoint_path}' LIMIT 1")
+            print(f"  ⏭️  {table_key}: snapshot {snapshot_id} already exported")
+            continue
+        except Exception:
+            pass  # File doesn't exist, proceed with export
+
+        print(f"\n  Processing {table_key}...")
+
+        try:
+            # Build checkpoint parquet data for this table
+            con.execute("""
+                CREATE OR REPLACE TEMP TABLE temp_checkpoint_parquet AS
+                WITH
+                table_schemas AS (
+                    SELECT
+                        ? AS table_id,
+                        ? AS table_name,
+                        ? AS snapshot_id,
+                        ? AS table_root,
+                        list({
+                            'name': c.column_name,
+                            'type':
+                                CASE
+                                    WHEN contains(lower(c.column_type), 'int') AND contains(c.column_type, '64') THEN 'long'
+                                    WHEN contains(lower(c.column_type), 'int') THEN 'integer'
+                                    WHEN contains(lower(c.column_type), 'float') THEN 'double'
+                                    WHEN contains(lower(c.column_type), 'double') THEN 'double'
+                                    WHEN contains(lower(c.column_type), 'bool') THEN 'boolean'
+                                    WHEN contains(lower(c.column_type), 'timestamp') THEN 'timestamp'
+                                    WHEN contains(lower(c.column_type), 'date') THEN 'date'
+                                    WHEN contains(lower(c.column_type), 'decimal') THEN lower(c.column_type)
+                                    ELSE 'string'
+                                END,
+                            'nullable': true,
+                            'metadata': MAP{}::MAP(VARCHAR, VARCHAR)
+                        }::STRUCT(name VARCHAR, type VARCHAR, nullable BOOLEAN, metadata MAP(VARCHAR, VARCHAR)) ORDER BY c.column_order) AS schema_fields
+                    FROM ducklake_column c
+                    WHERE c.table_id = ?
+                      AND c.end_snapshot IS NULL
+                ),
+                file_column_stats_agg AS (
+                    SELECT
+                        df.data_file_id,
+                        c.column_name,
+                        ANY_VALUE(c.column_type) AS column_type,
+                        MAX(fcs.value_count) AS value_count,
+                        MIN(fcs.min_value) AS min_value,
+                        MAX(fcs.max_value) AS max_value,
+                        MAX(fcs.null_count) AS null_count
+                    FROM ducklake_data_file df
+                    LEFT JOIN ducklake_file_column_stats fcs ON df.data_file_id = fcs.data_file_id
+                    LEFT JOIN ducklake_column c ON fcs.column_id = c.column_id
+                    WHERE df.table_id = ?
+                      AND df.end_snapshot IS NULL
+                      AND c.column_id IS NOT NULL
+                      AND c.end_snapshot IS NULL
+                    GROUP BY df.data_file_id, c.column_name
+                ),
+                file_column_stats_transformed AS (
+                    SELECT
+                        fca.data_file_id,
+                        fca.column_name,
+                        fca.column_type,
+                        fca.value_count,
+                        fca.null_count,
+                        CASE
+                            WHEN fca.min_value IS NULL THEN NULL
+                            WHEN contains(lower(fca.column_type), 'timestamp') THEN
+                                regexp_replace(
+                                    regexp_replace(replace(fca.min_value, ' ', 'T'), '[+-]\\d{2}(?::\\d{2})?$', ''),
+                                    '^([^.]+)$', '\\1.000'
+                                ) || 'Z'
+                            WHEN contains(lower(fca.column_type), 'date') THEN fca.min_value
+                            WHEN contains(lower(fca.column_type), 'bool') THEN CAST(lower(fca.min_value) IN ('true', 't', '1', 'yes') AS VARCHAR)
+                            WHEN contains(lower(fca.column_type), 'int') OR contains(lower(fca.column_type), 'float')
+                                 OR contains(lower(fca.column_type), 'double') OR contains(lower(fca.column_type), 'decimal') THEN
+                                CASE WHEN contains(fca.min_value, '.') OR contains(lower(fca.min_value), 'e')
+                                     THEN CAST(TRY_CAST(fca.min_value AS DOUBLE) AS VARCHAR)
+                                     ELSE CAST(TRY_CAST(fca.min_value AS BIGINT) AS VARCHAR)
+                                END
+                            ELSE fca.min_value
+                        END AS transformed_min,
+                        CASE
+                            WHEN fca.max_value IS NULL THEN NULL
+                            WHEN contains(lower(fca.column_type), 'timestamp') THEN
+                                regexp_replace(
+                                    regexp_replace(replace(fca.max_value, ' ', 'T'), '[+-]\\d{2}(?::\\d{2})?$', ''),
+                                    '^([^.]+)$', '\\1.000'
+                                ) || 'Z'
+                            WHEN contains(lower(fca.column_type), 'date') THEN fca.max_value
+                            WHEN contains(lower(fca.column_type), 'bool') THEN CAST(lower(fca.max_value) IN ('true', 't', '1', 'yes') AS VARCHAR)
+                            WHEN contains(lower(fca.column_type), 'int') OR contains(lower(fca.column_type), 'float')
+                                 OR contains(lower(fca.column_type), 'double') OR contains(lower(fca.column_type), 'decimal') THEN
+                                CASE WHEN contains(fca.max_value, '.') OR contains(lower(fca.max_value), 'e')
+                                     THEN CAST(TRY_CAST(fca.max_value AS DOUBLE) AS VARCHAR)
+                                     ELSE CAST(TRY_CAST(fca.max_value AS BIGINT) AS VARCHAR)
+                                END
+                            ELSE fca.max_value
+                        END AS transformed_max
+                    FROM file_column_stats_agg fca
+                ),
+                file_metadata AS (
+                    SELECT
+                        ts.table_id,
+                        ts.table_name,
+                        ts.snapshot_id,
+                        ts.table_root,
+                        ts.schema_fields,
+                        df.data_file_id,
+                        df.path AS file_path,
+                        df.file_size_bytes,
+                        COALESCE(MAX(fct.value_count), 0) AS num_records,
+                        COALESCE(map_from_entries(list({
+                            'key': fct.column_name,
+                            'value': fct.transformed_min
+                        } ORDER BY fct.column_name) FILTER (WHERE fct.column_name IS NOT NULL AND fct.transformed_min IS NOT NULL)), MAP{}::MAP(VARCHAR, VARCHAR)) AS min_values,
+                        COALESCE(map_from_entries(list({
+                            'key': fct.column_name,
+                            'value': fct.transformed_max
+                        } ORDER BY fct.column_name) FILTER (WHERE fct.column_name IS NOT NULL AND fct.transformed_max IS NOT NULL)), MAP{}::MAP(VARCHAR, VARCHAR)) AS max_values,
+                        COALESCE(map_from_entries(list({
+                            'key': fct.column_name,
+                            'value': fct.null_count
+                        } ORDER BY fct.column_name) FILTER (WHERE fct.column_name IS NOT NULL AND fct.null_count IS NOT NULL)), MAP{}::MAP(VARCHAR, BIGINT)) AS null_count
+                    FROM table_schemas ts
+                    JOIN ducklake_data_file df ON df.table_id = ts.table_id
+                    LEFT JOIN file_column_stats_transformed fct ON df.data_file_id = fct.data_file_id
+                    WHERE df.end_snapshot IS NULL
+                    GROUP BY ts.table_id, ts.table_name, ts.snapshot_id,
+                             ts.table_root, ts.schema_fields, df.data_file_id, df.path, df.file_size_bytes
+                ),
+                table_aggregates AS (
+                    SELECT
+                        table_id,
+                        table_name,
+                        snapshot_id,
+                        table_root,
+                        schema_fields,
+                        COUNT(*) AS num_files,
+                        SUM(num_records) AS total_rows,
+                        SUM(file_size_bytes) AS total_bytes,
+                        list({
+                            'path': ltrim(file_path, '/'),
+                            'partitionValues': MAP{}::MAP(VARCHAR, VARCHAR),
+                            'size': file_size_bytes,
+                            'modificationTime': epoch_ms(now()),
+                            'dataChange': true,
+                            'stats': COALESCE(to_json({
+                                'numRecords': COALESCE(num_records, 0),
+                                'minValues': COALESCE(min_values, MAP{}::MAP(VARCHAR, VARCHAR)),
+                                'maxValues': COALESCE(max_values, MAP{}::MAP(VARCHAR, VARCHAR)),
+                                'nullCount': COALESCE(null_count, MAP{}::MAP(VARCHAR, BIGINT))
+                            }), '{"numRecords":0}'),
+                            'tags': MAP{}::MAP(VARCHAR, VARCHAR)
+                        }::STRUCT(
+                            path VARCHAR,
+                            partitionValues MAP(VARCHAR, VARCHAR),
+                            size BIGINT,
+                            modificationTime BIGINT,
+                            dataChange BOOLEAN,
+                            stats VARCHAR,
+                            tags MAP(VARCHAR, VARCHAR)
+                        )) AS add_entries
+                    FROM file_metadata
+                    GROUP BY table_id, table_name, snapshot_id, table_root, schema_fields
+                ),
+                checkpoint_data AS (
+                    SELECT
+                        ta.*,
+                        epoch_ms(now()) AS now_ms,
+                        uuid()::VARCHAR AS txn_id,
+                        (substring(md5(ta.table_id::VARCHAR || '-metadata'), 1, 8) || '-' ||
+                         substring(md5(ta.table_id::VARCHAR || '-metadata'), 9, 4) || '-' ||
+                         substring(md5(ta.table_id::VARCHAR || '-metadata'), 13, 4) || '-' ||
+                         substring(md5(ta.table_id::VARCHAR || '-metadata'), 17, 4) || '-' ||
+                         substring(md5(ta.table_id::VARCHAR || '-metadata'), 21, 12)) AS meta_id,
+                        to_json({'type': 'struct', 'fields': ta.schema_fields}) AS schema_string
+                    FROM table_aggregates ta
+                ),
+                checkpoint_parquet_data AS (
+                    SELECT
+                        cd.table_id,
+                        cd.table_name,
+                        cd.snapshot_id,
+                        cd.table_root,
+                        cd.meta_id,
+                        cd.now_ms,
+                        cd.txn_id,
+                        cd.schema_string,
+                        cd.num_files,
+                        cd.total_rows,
+                        cd.total_bytes,
+                        {'minReaderVersion': 1, 'minWriterVersion': 2} AS protocol,
+                        NULL AS metaData,
+                        NULL AS add,
+                        NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
+                        NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isolationLevel VARCHAR, isBlindAppend BOOLEAN, operationMetrics MAP(VARCHAR, VARCHAR), engineInfo VARCHAR, txnId VARCHAR) AS commitInfo,
+                        1 AS row_order
+                    FROM checkpoint_data cd
+                    UNION ALL
+                    SELECT
+                        cd.table_id,
+                        cd.table_name,
+                        cd.snapshot_id,
+                        cd.table_root,
+                        cd.meta_id,
+                        cd.now_ms,
+                        cd.txn_id,
+                        cd.schema_string,
+                        cd.num_files,
+                        cd.total_rows,
+                        cd.total_bytes,
+                        NULL AS protocol,
+                        {
+                            'id': cd.meta_id,
+                            'name': cd.table_name,
+                            'format': {'provider': 'parquet', 'options': MAP{}::MAP(VARCHAR, VARCHAR)}::STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)),
+                            'schemaString': cd.schema_string,
+                            'partitionColumns': []::VARCHAR[],
+                            'createdTime': cd.now_ms,
+                            'configuration': MAP{}::MAP(VARCHAR, VARCHAR)
+                        }::STRUCT(id VARCHAR, name VARCHAR, format STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)), schemaString VARCHAR, partitionColumns VARCHAR[], createdTime BIGINT, configuration MAP(VARCHAR, VARCHAR)) AS metaData,
+                        NULL AS add,
+                        NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
+                        NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isolationLevel VARCHAR, isBlindAppend BOOLEAN, operationMetrics MAP(VARCHAR, VARCHAR), engineInfo VARCHAR, txnId VARCHAR) AS commitInfo,
+                        2 AS row_order
+                    FROM checkpoint_data cd
+                    UNION ALL
+                    SELECT
+                        cd.table_id,
+                        cd.table_name,
+                        cd.snapshot_id,
+                        cd.table_root,
+                        cd.meta_id,
+                        cd.now_ms,
+                        cd.txn_id,
+                        cd.schema_string,
+                        cd.num_files,
+                        cd.total_rows,
+                        cd.total_bytes,
+                        NULL AS protocol,
+                        NULL AS metaData,
+                        unnest(cd.add_entries) AS add,
+                        NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
+                        NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isolationLevel VARCHAR, isBlindAppend BOOLEAN, operationMetrics MAP(VARCHAR, VARCHAR), engineInfo VARCHAR, txnId VARCHAR) AS commitInfo,
+                        3 AS row_order
+                    FROM checkpoint_data cd
+                )
+                SELECT * FROM checkpoint_parquet_data
+            """, [table_id, table_name, snapshot_id, table_root, table_id, table_id])
+
+            # Build JSON log content
+            con.execute("""
+                CREATE OR REPLACE TEMP TABLE temp_checkpoint_json AS
+                SELECT DISTINCT
+                    p.table_id,
+                    p.table_root,
+                    p.snapshot_id,
+                    p.num_files,
+                    to_json({
+                        'commitInfo': {
+                            'timestamp': p.now_ms,
+                            'operation': 'CONVERT',
+                            'operationParameters': {
+                                'convertedFrom': 'DuckLake',
+                                'duckLakeSnapshotId': p.snapshot_id::VARCHAR,
+                                'partitionBy': '[]'
+                            },
+                            'isolationLevel': 'Serializable',
+                            'isBlindAppend': false,
+                            'operationMetrics': {
+                                'numFiles': p.num_files::VARCHAR,
+                                'numOutputRows': p.total_rows::VARCHAR,
+                                'numOutputBytes': p.total_bytes::VARCHAR
+                            },
+                            'engineInfo': 'DuckLake-Delta-Exporter/1.0.0',
+                            'txnId': p.txn_id
+                        }
+                    }) || chr(10) ||
+                    to_json({
+                        'metaData': {
+                            'id': p.meta_id,
+                            'name': p.table_name,
+                            'format': {'provider': 'parquet', 'options': MAP{}},
+                            'schemaString': p.schema_string::VARCHAR,
+                            'partitionColumns': [],
+                            'createdTime': p.now_ms,
+                            'configuration': MAP{}
+                        }
+                    }) || chr(10) ||
+                    to_json({
+                        'protocol': {'minReaderVersion': 1, 'minWriterVersion': 2}
+                    }) AS content
+                FROM temp_checkpoint_parquet p
+                WHERE p.row_order = 1
+            """)
+
+            # Build last checkpoint content
+            con.execute("""
+                CREATE OR REPLACE TEMP TABLE temp_last_checkpoint AS
+                SELECT
+                    table_id,
+                    table_root,
+                    snapshot_id,
+                    '{"version":' || snapshot_id || ',"size":' || (2 + num_files) || '}' AS content
+                FROM temp_checkpoint_parquet
+                WHERE row_order = 1
+            """)
+
+            # Get file paths
+            paths = con.execute("""
+                SELECT
+                    table_root || '/_delta_log/' || lpad(snapshot_id::VARCHAR, 20, '0') || '.checkpoint.parquet' AS checkpoint_file,
+                    table_root || '/_delta_log/' || lpad(snapshot_id::VARCHAR, 20, '0') || '.json' AS json_file,
+                    table_root || '/_delta_log/_last_checkpoint' AS last_checkpoint_file,
+                    table_root || '/_delta_log' AS delta_log_path
+                FROM temp_checkpoint_parquet
+                WHERE row_order = 1
+                LIMIT 1
+            """).fetchone()
+
+            checkpoint_file, json_file, last_checkpoint_file, delta_log_path = paths
+
+            # Create delta_log directory for local paths
+            if not any(table_root.startswith(prefix) for prefix in ['abfss://', 's3://', 'gs://', 'az://', 'http://', 'https://']):
+                con.execute(f"""
+                    COPY (SELECT 1 AS id, 1 AS ".duckdb_init")
+                    TO '{delta_log_path}'
+                    (FORMAT CSV, PARTITION_BY (".duckdb_init"), OVERWRITE_OR_IGNORE)
+                """)
+
+            # Write checkpoint parquet
+            con.execute(f"""
+                COPY (SELECT protocol, metaData, add, remove, commitInfo
+                      FROM temp_checkpoint_parquet ORDER BY row_order)
+                TO '{checkpoint_file}' (FORMAT PARQUET)
+            """)
+
+            # Write JSON log
+            con.execute(f"""
+                COPY (SELECT content FROM temp_checkpoint_json)
+                TO '{json_file}' (FORMAT CSV, HEADER false, QUOTE '')
+            """)
+
+            # Write last checkpoint
+            con.execute(f"""
+                COPY (SELECT content FROM temp_last_checkpoint)
+                TO '{last_checkpoint_file}' (FORMAT CSV, HEADER false, QUOTE '')
+            """)
+
+            print(f"  ✅ {table_key}: exported snapshot {snapshot_id} ({file_count} files)")
+
+        except Exception as e:
+            print(f"  ❌ {table_key}: {e}")
+
+    # Cleanup temp tables
+    con.execute("DROP TABLE IF EXISTS export_summary")
+    con.execute("DROP TABLE IF EXISTS temp_checkpoint_parquet")
+    con.execute("DROP TABLE IF EXISTS temp_checkpoint_json")
+    con.execute("DROP TABLE IF EXISTS temp_last_checkpoint")
+
+    con.close()
+    print("\n🎉 Export completed!")

{ducklake_delta_exporter-0.1.3 → ducklake_delta_exporter-0.2.0}/ducklake_delta_exporter.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ducklake-delta-exporter
-Version: 0.1.3
+Version: 0.2.0
 Summary: A utility to export DuckLake database metadata to Delta Lake transaction logs.
 Home-page: https://github.com/djouallah/ducklake_delta_exporter
 Author: mim
@@ -13,7 +13,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Development Status :: 3 - Alpha
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: duckdb
+Requires-Dist: duckdb>=1.4.4
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -28,7 +28,7 @@ Dynamic: summary
 
 A Python package for exporting DuckLake snapshots as Delta Lake checkpoint files, enabling compatibility with Delta Lake readers, support local path, s3 and gcs, for onelake use mounted storage as azure storage is not supported
 
-this is just a fun project
+this is just a fun project, please vote for a proper support in duckdb https://github.com/duckdb/duckdb-delta/issues/218
 
 ## Repository
 

ducklake_delta_exporter-0.2.0/ducklake_delta_exporter.egg-info/requires.txt

@@ -0,0 +1 @@
+duckdb>=1.4.4

{ducklake_delta_exporter-0.1.3 → ducklake_delta_exporter-0.2.0}/setup.py

@@ -3,9 +3,9 @@ from setuptools import setup, find_packages
 
 setup(
     name='ducklake-delta-exporter',
-    version='0.1.3',
+    version='0.2.0',
     packages=find_packages(),
-    install_requires=['duckdb'],
+    install_requires=['duckdb>=1.4.4'],
     author='mim',
     author_email='your.email@example.com',
     description='A utility to export DuckLake database metadata to Delta Lake transaction logs.',

ducklake_delta_exporter-0.1.3/ducklake_delta_exporter/__init__.py

@@ -1,326 +0,0 @@
-# File: ducklake_delta_exporter.py
-import json
-import time
-import duckdb
-
-def map_type_ducklake_to_spark(t):
-    """Maps DuckDB data types to their Spark SQL equivalents for the Delta schema."""
-    t = t.lower()
-    if 'int' in t:
-        return 'long' if '64' in t else 'integer'
-    elif 'float' in t:
-        return 'double'
-    elif 'double' in t:
-        return 'double'
-    elif 'decimal' in t:
-        return 'decimal(10,0)'
-    elif 'bool' in t:
-        return 'boolean'
-    elif 'timestamp' in t:
-        return 'timestamp'
-    elif 'date' in t:
-        return 'date'
-    return 'string'
-
-def create_spark_schema_string(fields):
-    """Creates a JSON string for the Spark schema from a list of fields."""
-    return json.dumps({"type": "struct", "fields": fields})
-
-def get_latest_ducklake_snapshot(con, table_id):
-    """
-    Get the latest DuckLake snapshot ID for a table.
-    """
-    latest_snapshot  = con.execute(f""" SELECT MAX(begin_snapshot) as latest_snapshot FROM ducklake_data_file  WHERE table_id = {table_id} """).fetchone()[0]
-    return latest_snapshot
-
-def get_latest_delta_checkpoint(con, table_id):
-    """
-    check how many times a table has being modified.
-    """
-    delta_checkpoint = con.execute(f""" SELECT count(snapshot_id) FROM ducklake_snapshot_changes
-                                   where changes_made like '%:{table_id}' or changes_made like '%:{table_id},%' """).fetchone()[0]
-    print(table_id)
-    print(delta_checkpoint)
-    return delta_checkpoint
-
-def get_file_modification_time(dummy_time):
-    """
-    Return a dummy modification time for parquet files.
-    This avoids the latency of actually reading file metadata.
-    
-    Args:
-        dummy_time: Timestamp in milliseconds to use as modification time
-    
-    Returns:
-        Modification time in milliseconds
-    """
-    return dummy_time
-
-def create_dummy_json_log(table_root, delta_version, table_info, schema_fields, now):
-    """
-    Create a minimal JSON log file for Spark compatibility using DuckDB.
-    """
-    json_log_file = table_root + f"_delta_log/{delta_version:020d}.json"
-    
-    # Create JSON log entries using DuckDB
-    duckdb.execute("DROP TABLE IF EXISTS json_log_table")
-    
-    # Protocol entry
-    protocol_json = json.dumps({
-        "protocol": {
-            "minReaderVersion": 1,
-            "minWriterVersion": 2
-        }
-    })
-    
-    # Metadata entry
-    metadata_json = json.dumps({
-        "metaData": {
-            "id": str(table_info['table_id']),
-            "name": table_info['table_name'],
-            "description": None,
-            "format": {
-                "provider": "parquet",
-                "options": {}
-            },
-            "schemaString": create_spark_schema_string(schema_fields),
-            "partitionColumns": [],
-            "createdTime": now,
-            "configuration": {
-                "delta.logRetentionDuration": "interval 1 hour"
-            }
-        }
-    })
-    
-    # Commit info entry
-    commitinfo_json = json.dumps({
-        "commitInfo": {
-            "timestamp": now,
-            "operation": "CONVERT",
-            "operationParameters": {
-                "convertedFrom": "DuckLake"
-            },
-            "isBlindAppend": True,
-            "engineInfo": "DuckLake-Delta-Exporter",
-            "clientVersion": "1.0.0"
-        }
-    })
-    
-    # Create table with JSON entries
-    duckdb.execute("""
-        CREATE TABLE json_log_table AS
-        SELECT ? AS json_line
-        UNION ALL
-        SELECT ? AS json_line
-        UNION ALL
-        SELECT ? AS json_line
-    """, [protocol_json, metadata_json, commitinfo_json])
-    
-    # Write JSON log file using DuckDB
-    duckdb.execute(f"COPY (SELECT json_line FROM json_log_table) TO '{json_log_file}' (FORMAT CSV, HEADER false, QUOTE '')")
-    
-    # Clean up
-    duckdb.execute("DROP TABLE IF EXISTS json_log_table")
-    
-    return json_log_file
-
-def build_file_path(table_root, relative_path):
-    """
-    Build full file path from table root and relative path.
-    Works with both local paths and S3 URLs.
-    """
-    table_root = table_root.rstrip('/')
-    relative_path = relative_path.lstrip('/')
-    return f"{table_root}/{relative_path}"
-
-def create_checkpoint_for_latest_snapshot(con, table_info, data_root):
-    """
-    Create a Delta checkpoint file for the latest DuckLake snapshot.
-    """
-    table_root = data_root.rstrip('/') + '/' + table_info['schema_path'] + table_info['table_path']
-    
-    # Get the latest snapshot
-    latest_snapshot = get_latest_ducklake_snapshot(con, table_info['table_id'])
-    if latest_snapshot is None:
-        print(f"⚠️ {table_info['schema_name']}.{table_info['table_name']}: No snapshots found")
-        return False
-    delta_version   = get_latest_delta_checkpoint(con, table_info['table_id'])
-    checkpoint_file = table_root + f"_delta_log/{delta_version:020d}.checkpoint.parquet"
-    json_log_file   = table_root + f"_delta_log/{delta_version:020d}.json"
-    
-    try:
-        con.execute(f"SELECT protocol FROM '{checkpoint_file}' limit 0 ")
-        print(f"⚠️ {table_info['schema_name']}.{table_info['table_name']}: Checkpoint file already exists: {checkpoint_file}")
-    except:
-    
-        now = int(time.time() * 1000)
-        
-        # Get all files for the latest snapshot
-        file_rows = con.execute(f"""
-            SELECT path, file_size_bytes FROM ducklake_data_file
-            WHERE table_id = {table_info['table_id']}
-            AND begin_snapshot <= {latest_snapshot} 
-            AND (end_snapshot IS NULL OR end_snapshot > {latest_snapshot})
-        """).fetchall()
-        
-        # Get schema for the latest snapshot
-        columns = con.execute(f"""
-            SELECT column_name, column_type FROM ducklake_column
-            WHERE table_id = {table_info['table_id']}
-            AND begin_snapshot <= {latest_snapshot} 
-            AND (end_snapshot IS NULL OR end_snapshot > {latest_snapshot})
-            ORDER BY column_order
-        """).fetchall()
-        
-        # Get or generate table metadata ID
-        table_meta_id = str(table_info['table_id'])
-        
-        # Prepare schema
-        schema_fields = [
-            {"name": name, "type": map_type_ducklake_to_spark(typ), "nullable": True, "metadata": {}} 
-            for name, typ in columns
-        ]
-        
-        # Create checkpoint data using DuckDB directly
-        checkpoint_data = []
-        
-        # Create checkpoint data directly in DuckDB using proper data types
-        duckdb.execute("DROP TABLE IF EXISTS checkpoint_table")
-        
-        # Create the checkpoint table with proper nested structure
-        duckdb.execute("""
-            CREATE TABLE checkpoint_table AS
-            WITH checkpoint_data AS (
-                -- Protocol record
-                SELECT 
-                    {'minReaderVersion': 1, 'minWriterVersion': 2}::STRUCT(minReaderVersion INTEGER, minWriterVersion INTEGER) AS protocol,
-                    NULL::STRUCT(id VARCHAR, name VARCHAR, description VARCHAR, format STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)), schemaString VARCHAR, partitionColumns VARCHAR[], createdTime BIGINT, configuration MAP(VARCHAR, VARCHAR)) AS metaData,
-                    NULL::STRUCT(path VARCHAR, partitionValues MAP(VARCHAR, VARCHAR), size BIGINT, modificationTime BIGINT, dataChange BOOLEAN, stats VARCHAR, tags MAP(VARCHAR, VARCHAR)) AS add,
-                    NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
-                    NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isBlindAppend BOOLEAN, engineInfo VARCHAR, clientVersion VARCHAR) AS commitInfo
-                
-                UNION ALL
-                
-                -- Metadata record
-                SELECT 
-                    NULL::STRUCT(minReaderVersion INTEGER, minWriterVersion INTEGER) AS protocol,
-                    {
-                        'id': ?, 
-                        'name': ?, 
-                        'description': NULL, 
-                        'format': {'provider': 'parquet', 'options': MAP{}}::STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)),
-                        'schemaString': ?, 
-                        'partitionColumns': []::VARCHAR[], 
-                        'createdTime': ?, 
-                        'configuration': MAP{'delta.logRetentionDuration': 'interval 1 hour'}
-                    }::STRUCT(id VARCHAR, name VARCHAR, description VARCHAR, format STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)), schemaString VARCHAR, partitionColumns VARCHAR[], createdTime BIGINT, configuration MAP(VARCHAR, VARCHAR)) AS metaData,
-                    NULL::STRUCT(path VARCHAR, partitionValues MAP(VARCHAR, VARCHAR), size BIGINT, modificationTime BIGINT, dataChange BOOLEAN, stats VARCHAR, tags MAP(VARCHAR, VARCHAR)) AS add,
-                    NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
-                    NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isBlindAppend BOOLEAN, engineInfo VARCHAR, clientVersion VARCHAR) AS commitInfo
-            )
-            SELECT * FROM checkpoint_data
-        """, [table_meta_id, table_info['table_name'], create_spark_schema_string(schema_fields), now])
-        
-        # Add file records
-        for path, size in file_rows:
-            rel_path = path.lstrip('/')
-            full_path = build_file_path(table_root, rel_path)
-            mod_time = get_file_modification_time(now)
-            
-            duckdb.execute("""
-                INSERT INTO checkpoint_table
-                SELECT 
-                    NULL::STRUCT(minReaderVersion INTEGER, minWriterVersion INTEGER) AS protocol,
-                    NULL::STRUCT(id VARCHAR, name VARCHAR, description VARCHAR, format STRUCT(provider VARCHAR, options MAP(VARCHAR, VARCHAR)), schemaString VARCHAR, partitionColumns VARCHAR[], createdTime BIGINT, configuration MAP(VARCHAR, VARCHAR)) AS metaData,
-                    {
-                        'path': ?, 
-                        'partitionValues': MAP{}::MAP(VARCHAR, VARCHAR), 
-                        'size': ?, 
-                        'modificationTime': ?, 
-                        'dataChange': true, 
-                        'stats': ?, 
-                        'tags': NULL::MAP(VARCHAR, VARCHAR)
-                    }::STRUCT(path VARCHAR, partitionValues MAP(VARCHAR, VARCHAR), size BIGINT, modificationTime BIGINT, dataChange BOOLEAN, stats VARCHAR, tags MAP(VARCHAR, VARCHAR)) AS add,
-                    NULL::STRUCT(path VARCHAR, deletionTimestamp BIGINT, dataChange BOOLEAN) AS remove,
-                    NULL::STRUCT(timestamp TIMESTAMP, operation VARCHAR, operationParameters MAP(VARCHAR, VARCHAR), isBlindAppend BOOLEAN, engineInfo VARCHAR, clientVersion VARCHAR) AS commitInfo
-            """, [rel_path, size, mod_time, json.dumps({"numRecords": None})])
-        
-        # Create the _delta_log directory if it doesn't exist
-        duckdb.execute(f"COPY (SELECT 43) TO '{table_root}_delta_log' (FORMAT PARQUET, PER_THREAD_OUTPUT, OVERWRITE_OR_IGNORE)")
-            
-        # Write the checkpoint file
-        duckdb.execute(f"COPY (SELECT * FROM checkpoint_table) TO '{checkpoint_file}' (FORMAT PARQUET)")
-        
-        # Create dummy JSON log file for Spark compatibility
-        create_dummy_json_log(table_root, delta_version, table_info, schema_fields, now)
-            
-        # Write the _last_checkpoint file
-        total_records = 2 + len(file_rows)  # protocol + metadata + file records
-        duckdb.execute(f"""
-            COPY (SELECT {delta_version} AS version, {total_records} AS size) 
-            TO '{table_root}_delta_log/_last_checkpoint' (FORMAT JSON, ARRAY false)
-        """)
-            
-        print(f"✅ Exported DuckLake snapshot {latest_snapshot} as Delta checkpoint v{delta_version}")
-        print(f"✅ Created JSON log file: {json_log_file}")
-        
-        # Clean up temporary tables
-        duckdb.execute("DROP TABLE IF EXISTS checkpoint_table")
-    
-    return True, delta_version, latest_snapshot
-
-def generate_latest_delta_log(db_path: str, data_root: str = None):
-    """
-    Export the latest DuckLake snapshot for each table as a Delta checkpoint file.
-    Creates both checkpoint files and minimal JSON log files for Spark compatibility.
-    
-    Args:
-        db_path (str): The path to the DuckLake database file.
-        data_root (str): The root directory for the lakehouse data.
-    """
-    con = duckdb.connect(db_path, read_only=True)
-    
-    if data_root is None:
-        data_root = con.sql("SELECT value FROM ducklake_metadata WHERE key = 'data_path'").fetchone()[0]
-    
-    # Get all active tables
-    tables = con.execute("""
-        SELECT 
-            t.table_id, 
-            t.table_name, 
-            s.schema_name,
-            t.path as table_path, 
-            s.path as schema_path
-        FROM ducklake_table t
-        JOIN ducklake_schema s USING(schema_id)
-        WHERE t.end_snapshot IS NULL
-    """).fetchall()
-    
-    total_tables = len(tables)
-    successful_exports = 0
-    
-    for table_row in tables:
-        table_info = {
-            'table_id': table_row[0],
-            'table_name': table_row[1],
-            'schema_name': table_row[2],
-            'table_path': table_row[3],
-            'schema_path': table_row[4]
-        }
-        
-        table_key = f"{table_info['schema_name']}.{table_info['table_name']}"
-        print(f"Processing {table_key}...")
-        
-        try:
-            result = create_checkpoint_for_latest_snapshot(con, table_info, data_root)
-            
-            if result:
-                successful_exports += 1
-            else:
-                print(f"⚠️ {table_key}: No data to export")
-                
-        except Exception as e:
-            print(f"❌ {table_key}: Failed to export checkpoint - {e}")
-    
-    con.close()
-    print(f"\n🎉 Export completed! {successful_exports}/{total_tables} tables exported successfully.")

ducklake_delta_exporter-0.1.3/ducklake_delta_exporter.egg-info/requires.txt

@@ -1 +0,0 @@
-duckdb