dlt-iceberg 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

dlt_iceberg/destination_client.py

@@ -34,7 +34,11 @@ from pyiceberg.exceptions import (
 from .schema_converter import convert_dlt_to_iceberg_schema
 from .partition_builder import build_partition_spec
 from .schema_evolution import evolve_schema_if_needed, SchemaEvolutionError
-from .schema_casting import cast_table_safe, CastingError
+from .schema_casting import (
+    cast_table_safe,
+    CastingError,
+    ensure_iceberg_compatible_arrow_data,
+)
 from .error_handling import (
     is_retryable_error,
     log_error_with_context,
@@ -89,6 +93,9 @@ class IcebergRestConfiguration(DestinationClientConfiguration):
     # Schema casting configuration
     strict_casting: bool = False
 
+    # Merge batch size (for upsert operations to avoid memory issues)
+    merge_batch_size: int = 100000
+
 
 class IcebergRestLoadJob(RunnableLoadJob):
     """
@@ -380,7 +387,8 @@ class IcebergRestClient(JobClientBase):
                 # Create table if needed
                 if not table_exists:
                     # Use first file's Arrow table to generate schema
-                    first_arrow_table = file_data[0][2]
+                    # Apply Iceberg compatibility first so schema uses compatible types
+                    first_arrow_table = ensure_iceberg_compatible_arrow_data(file_data[0][2])
                     iceberg_schema = convert_dlt_to_iceberg_schema(
                         table_schema, first_arrow_table
                     )
@@ -401,7 +409,7 @@ class IcebergRestClient(JobClientBase):
                     logger.info(f"Created table {identifier} at {iceberg_table.location()}")
                 else:
                     # Table exists - check if schema evolution is needed
-                    first_arrow_table = file_data[0][2]
+                    first_arrow_table = ensure_iceberg_compatible_arrow_data(file_data[0][2])
                     incoming_schema = convert_dlt_to_iceberg_schema(
                         table_schema, first_arrow_table
                     )
@@ -415,12 +423,15 @@ class IcebergRestClient(JobClientBase):
                         logger.info(f"Schema evolved for table {identifier}")
                         iceberg_table = catalog.load_table(identifier)
 
-                # Combine all Arrow tables and cast to match Iceberg schema
+                # Get expected schema (already has Iceberg-compatible types from creation)
                 expected_schema = schema_to_pyarrow(iceberg_table.schema())
+
+                # Combine all Arrow tables and cast to match Iceberg schema
                 combined_tables = []
 
                 for _, file_path, arrow_table in file_data:
-                    # Cast each table to match Iceberg schema
+                    # Cast to match Iceberg schema
+                    # (compatibility conversions already applied when schema was created)
                     casted_table = cast_table_safe(
                         arrow_table,
                         expected_schema,
@@ -463,15 +474,34 @@ class IcebergRestClient(JobClientBase):
                         iceberg_table.append(combined_table)
                     else:
                         logger.info(f"Merging into table {identifier} on keys {primary_keys}")
-                        upsert_result = iceberg_table.upsert(
-                            df=combined_table,
-                            join_cols=primary_keys,
-                            when_matched_update_all=True,
-                            when_not_matched_insert_all=True,
-                        )
+
+                        # Batch upserts to avoid memory issues on large datasets
+                        batch_size = self.config.merge_batch_size
+                        total_updated = 0
+                        total_inserted = 0
+
+                        for batch_start in range(0, len(combined_table), batch_size):
+                            batch_end = min(batch_start + batch_size, len(combined_table))
+                            batch = combined_table.slice(batch_start, batch_end - batch_start)
+
+                            logger.info(
+                                f"Upserting batch {batch_start//batch_size + 1}: "
+                                f"rows {batch_start} to {batch_end} ({len(batch)} rows)"
+                            )
+
+                            upsert_result = iceberg_table.upsert(
+                                df=batch,
+                                join_cols=primary_keys,
+                                when_matched_update_all=True,
+                                when_not_matched_insert_all=True,
+                            )
+
+                            total_updated += upsert_result.rows_updated
+                            total_inserted += upsert_result.rows_inserted
+
                         logger.info(
-                            f"Upsert completed: {upsert_result.rows_updated} updated, "
-                            f"{upsert_result.rows_inserted} inserted"
+                            f"Upsert completed: {total_updated} updated, "
+                            f"{total_inserted} inserted across {(total_rows + batch_size - 1) // batch_size} batches"
                         )
                 else:
                     raise ValueError(f"Unknown write disposition: {write_disposition}")

dlt_iceberg/schema_casting.py

@@ -6,12 +6,75 @@ and allow users to control casting behavior.
 """
 
 import logging
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Dict, Callable
 import pyarrow as pa
 
 logger = logging.getLogger(__name__)
 
 
+def ensure_iceberg_compatible_arrow_schema(schema: pa.Schema) -> pa.Schema:
+    """
+    Convert Arrow schema to Iceberg-compatible schema.
+
+    Converts types that Iceberg doesn't support:
+    - time32 → time64 (microseconds)
+    - decimal256 → string (Iceberg only supports decimal128)
+    - dictionary → value_type (unwrap dictionary encoding)
+
+    Args:
+        schema: PyArrow schema
+
+    Returns:
+        Iceberg-compatible PyArrow schema
+    """
+    def convert_field(field: pa.Field) -> pa.Field:
+        field_type = field.type
+
+        # time32 → time64(us)
+        if pa.types.is_time32(field_type):
+            return pa.field(field.name, pa.time64("us"), nullable=field.nullable)
+
+        # decimal256 → string (pyarrow doesn't allow downcasting to decimal128)
+        if pa.types.is_decimal256(field_type):
+            logger.warning(
+                f"Converting decimal256 field '{field.name}' to string "
+                f"(Iceberg doesn't support decimal256)"
+            )
+            return pa.field(field.name, pa.string(), nullable=field.nullable)
+
+        # dictionary → value_type (unwrap dictionary encoding)
+        if pa.types.is_dictionary(field_type):
+            return pa.field(field.name, field_type.value_type, nullable=field.nullable)
+
+        # list/struct types - recursively convert nested fields
+        if pa.types.is_list(field_type):
+            value_field = convert_field(pa.field("item", field_type.value_type))
+            return pa.field(field.name, pa.list_(value_field.type), nullable=field.nullable)
+
+        if pa.types.is_struct(field_type):
+            new_fields = [convert_field(f) for f in field_type]
+            return pa.field(field.name, pa.struct(new_fields), nullable=field.nullable)
+
+        return field
+
+    new_fields = [convert_field(field) for field in schema]
+    return pa.schema(new_fields)
+
+
+def ensure_iceberg_compatible_arrow_data(table: pa.Table) -> pa.Table:
+    """
+    Convert Arrow table to Iceberg-compatible schema and cast data.
+
+    Args:
+        table: PyArrow table
+
+    Returns:
+        Table with Iceberg-compatible schema
+    """
+    new_schema = ensure_iceberg_compatible_arrow_schema(table.schema)
+    return table.cast(new_schema)
+
+
 class CastingError(Exception):
     """Raised when a cast would result in data loss in strict mode."""
     pass

{dlt_iceberg-0.1.2.dist-info → dlt_iceberg-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dlt-iceberg
-Version: 0.1.2
+Version: 0.1.4
 Summary: dlt destination for Apache Iceberg with atomic multi-file commits via REST catalogs
 Project-URL: Homepage, https://github.com/sidequery/dlt-iceberg
 Project-URL: Repository, https://github.com/sidequery/dlt-iceberg
@@ -47,9 +47,13 @@ A [dlt](https://dlthub.com/) destination for [Apache Iceberg](https://iceberg.ap
 ## Installation
 
 ```bash
-git clone https://github.com/sidequery/dlt-iceberg.git
-cd dlt-iceberg
-uv sync
+pip install dlt-iceberg
+```
+
+Or with uv:
+
+```bash
+uv add dlt-iceberg
 ```
 
 ## Quick Start
@@ -95,7 +99,38 @@ def generate_users():
 pipeline.run(generate_users())
 ```
 
-## Configuration
+## Configuration Options
+
+All configuration options can be passed to `iceberg_rest()`:
+
+```python
+iceberg_rest(
+    catalog_uri="...",           # Required: REST catalog URI
+    namespace="...",             # Required: Iceberg namespace (database)
+    warehouse="...",             # Optional: Warehouse location
+
+    # Authentication
+    credential="...",            # OAuth2 client credentials
+    oauth2_server_uri="...",     # OAuth2 token endpoint
+    token="...",                 # Bearer token
+
+    # AWS SigV4
+    sigv4_enabled=True,
+    signing_region="us-east-1",
+
+    # S3 configuration
+    s3_endpoint="...",
+    s3_access_key_id="...",
+    s3_secret_access_key="...",
+    s3_region="...",
+
+    # Performance tuning
+    max_retries=5,               # Retry attempts for transient failures
+    retry_backoff_base=2.0,      # Exponential backoff multiplier
+    merge_batch_size=100000,     # Rows per batch for merge operations
+    strict_casting=False,        # Fail on potential data loss
+)
+```
 
 ### Nessie (Docker)
 

{dlt_iceberg-0.1.2.dist-info → dlt_iceberg-0.1.4.dist-info}/RECORD

@@ -1,12 +1,12 @@
 dlt_iceberg/__init__.py,sha256=ONy6E-sGcCvvqia8_fGaYp8da4n4wdjox9W42tmQPK0,780
 dlt_iceberg/destination.py,sha256=F8QJXsQeosOA32Xm1140DL485WQmxbuhiA2QZ6zpVSU,15737
-dlt_iceberg/destination_client.py,sha256=dyJtHHy2Ow0GIFVj17LePC76rKw6MiVJnrS-y28OctQ,22341
+dlt_iceberg/destination_client.py,sha256=l1q8GYvIJ_tBgoQ979IS3VtUQNmg2-hYv80XZkAVFKs,23786
 dlt_iceberg/error_handling.py,sha256=k6Kkldi9BDRsXQ63VEBMMSw1xx2-b1BMjsgRFKI2iB0,7852
 dlt_iceberg/partition_builder.py,sha256=l9YNAh2t6gk2xqsPSOs8ymTDLk9BOEZWVOtVni7ONNU,10081
-dlt_iceberg/schema_casting.py,sha256=Qn4sarRnyJM04lKvKonEjvlvVdizUOGI65J_AmzbEAs,12997
+dlt_iceberg/schema_casting.py,sha256=oSQrnOcCMFcinMS65N8YQ1uzrqnQmN50mCCuQyE3794,15247
 dlt_iceberg/schema_converter.py,sha256=e_eqXQz2cpABOGEAxVwcGbiOdVmv9kaZanRnU83lzXk,5619
 dlt_iceberg/schema_evolution.py,sha256=ieOkCA9ngQdJ5lbZLYQ09deTLZEW8whxDn2arpoH-aM,8326
-dlt_iceberg-0.1.2.dist-info/METADATA,sha256=fR4s357aPlxbCqkVF8l6-tc2qn4fX5DQeDrs3wFszNg,7797
-dlt_iceberg-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dlt_iceberg-0.1.2.dist-info/licenses/LICENSE,sha256=0amGlcH0msYju3WUhlsuUxO4aj3ZODkkIZ0MKOq9fQ4,1066
-dlt_iceberg-0.1.2.dist-info/RECORD,,
+dlt_iceberg-0.1.4.dist-info/METADATA,sha256=2FnlwL3bwN-aItvx0Y1dUQ3iTfjLr9AJUQ0hLrJZql8,8777
+dlt_iceberg-0.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dlt_iceberg-0.1.4.dist-info/licenses/LICENSE,sha256=0amGlcH0msYju3WUhlsuUxO4aj3ZODkkIZ0MKOq9fQ4,1066
+dlt_iceberg-0.1.4.dist-info/RECORD,,