dlt-iceberg 0.1.1__tar.gz → 0.1.3__tar.gz

dlt_iceberg-0.1.3/PKG-INFO

@@ -0,0 +1,279 @@
+Metadata-Version: 2.4
+Name: dlt-iceberg
+Version: 0.1.3
+Summary: dlt destination for Apache Iceberg with atomic multi-file commits via REST catalogs
+Project-URL: Homepage, https://github.com/sidequery/dlt-iceberg
+Project-URL: Repository, https://github.com/sidequery/dlt-iceberg
+Project-URL: Issues, https://github.com/sidequery/dlt-iceberg/issues
+Author-email: Sidequery <hello@sidequery.com>
+License: MIT
+License-File: LICENSE
+Keywords: data-engineering,data-pipeline,dlt,elt,etl,iceberg
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Requires-Dist: boto3>=1.40.50
+Requires-Dist: dlt>=1.17.1
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: pydantic<2.11
+Requires-Dist: pyiceberg[pyiceberg-core]>=0.10.0
+Requires-Dist: requests>=2.32.5
+Requires-Dist: s3fs>=0.4.2
+Requires-Dist: sqlalchemy>=2.0.44
+Description-Content-Type: text/markdown
+
+# dlt-iceberg
+
+A [dlt](https://dlthub.com/) destination for [Apache Iceberg](https://iceberg.apache.org/) tables using REST catalogs.
+
+## Features
+
+- **Atomic Multi-File Commits**: Multiple parquet files committed as single Iceberg snapshot per table
+- **REST Catalog Support**: Works with Nessie, Polaris, AWS Glue, Unity Catalog
+- **Partitioning**: Full support for Iceberg partition transforms (temporal, bucket, truncate, identity)
+- **Authentication**: OAuth2, Bearer token, AWS SigV4
+- **Write Dispositions**: Append, replace, merge (upsert)
+- **Schema Evolution**: Automatic schema updates when adding columns
+- **Retry Logic**: Exponential backoff for transient failures
+
+## Installation
+
+```bash
+git clone https://github.com/sidequery/dlt-iceberg.git
+cd dlt-iceberg
+uv sync
+```
+
+## Quick Start
+
+See [examples/](examples/) directory for working examples.
+
+### Incremental Load
+
+```python
+import dlt
+from dlt_iceberg import iceberg_rest
+
+@dlt.resource(name="events", write_disposition="append")
+def generate_events():
+    yield {"event_id": 1, "value": 100}
+
+pipeline = dlt.pipeline(
+    pipeline_name="my_pipeline",
+    destination=iceberg_rest(
+        catalog_uri="http://localhost:19120/iceberg/main",
+        namespace="analytics",
+        s3_endpoint="http://localhost:9000",
+        s3_access_key_id="minioadmin",
+        s3_secret_access_key="minioadmin",
+        s3_region="us-east-1",
+    ),
+)
+
+pipeline.run(generate_events())
+```
+
+### Merge/Upsert
+
+```python
+@dlt.resource(
+    name="users",
+    write_disposition="merge",
+    primary_key="user_id"
+)
+def generate_users():
+    yield {"user_id": 1, "name": "Alice", "status": "active"}
+
+pipeline.run(generate_users())
+```
+
+## Configuration
+
+### Nessie (Docker)
+
+```python
+iceberg_rest(
+    catalog_uri="http://localhost:19120/iceberg/main",
+    namespace="my_namespace",
+    s3_endpoint="http://localhost:9000",
+    s3_access_key_id="minioadmin",
+    s3_secret_access_key="minioadmin",
+    s3_region="us-east-1",
+)
+```
+
+Start services: `docker compose up -d`
+
+### AWS Glue
+
+```python
+iceberg_rest(
+    catalog_uri="https://glue.us-east-1.amazonaws.com/iceberg",
+    warehouse="<account-id>:s3tablescatalog/<bucket>",
+    namespace="my_database",
+    sigv4_enabled=True,
+    signing_region="us-east-1",
+)
+```
+
+AWS credentials via environment variables.
+
+### Polaris
+
+```python
+iceberg_rest(
+    catalog_uri="https://polaris.example.com/api/catalog",
+    warehouse="s3://bucket/warehouse",
+    namespace="production",
+    credential="client-id:client-secret",
+    oauth2_server_uri="https://polaris.example.com/api/catalog/v1/oauth/tokens",
+)
+```
+
+### Unity Catalog
+
+```python
+iceberg_rest(
+    catalog_uri="https://<workspace>.cloud.databricks.com/api/2.1/unity-catalog/iceberg-rest",
+    warehouse="<catalog-name>",
+    namespace="<schema-name>",
+    token="<databricks-token>",
+)
+```
+
+## Partitioning
+
+Mark columns for partitioning using dlt column hints:
+
+```python
+@dlt.resource(
+    name="events",
+    columns={
+        "event_date": {
+            "data_type": "date",
+            "partition": True,
+            "partition_transform": "day",  # Optional: year, month, day, hour
+        },
+        "region": {
+            "data_type": "text",
+            "partition": True,  # Uses identity transform for strings
+        },
+        "user_id": {
+            "data_type": "bigint",
+            "partition": True,
+            "partition_transform": "bucket[10]",  # Hash into 10 buckets
+        }
+    }
+)
+def events():
+    ...
+```
+
+### Available Transforms
+
+- **Temporal**: `year`, `month`, `day`, `hour` (for timestamp/date columns)
+- **Identity**: No transformation (default for string/integer)
+- **Bucket**: `bucket[N]` - Hash-based partitioning into N buckets
+- **Truncate**: `truncate[N]` - Truncate strings/integers to N width
+
+### Default Behavior
+
+If `partition_transform` is not specified:
+- Timestamp/date columns default to `month`
+- String/integer columns default to `identity`
+
+## Write Dispositions
+
+### Append
+```python
+write_disposition="append"
+```
+Adds new data without modifying existing rows.
+
+### Replace
+```python
+write_disposition="replace"
+```
+Truncates table and inserts new data.
+
+### Merge
+```python
+write_disposition="merge"
+primary_key="user_id"
+```
+Updates existing rows by primary key, inserts new rows.
+
+## Development
+
+### Run Tests
+
+```bash
+# Start Docker services
+docker compose up -d
+
+# Run all tests
+uv run pytest tests/ -v
+
+# Run only unit tests
+uv run pytest tests/ -v -m "not integration"
+
+# Run only integration tests
+uv run pytest tests/ -v -m integration
+```
+
+### Project Structure
+
+```
+dlt-iceberg/
+├── src/dlt_iceberg/
+│   ├── __init__.py              # Public API
+│   ├── destination_client.py   # Class-based destination (atomic commits)
+│   ├── destination.py           # Function-based destination (legacy)
+│   ├── schema_converter.py     # dlt → Iceberg schema conversion
+│   ├── schema_casting.py        # Arrow table casting
+│   ├── schema_evolution.py     # Schema updates
+│   ├── partition_builder.py    # Partition specs
+│   └── error_handling.py       # Retry logic
+├── tests/
+│   ├── test_destination_rest_catalog.py  # Integration tests (Docker)
+│   ├── test_class_based_atomic.py        # Atomic commit tests
+│   ├── test_merge_disposition.py
+│   ├── test_schema_evolution.py
+│   └── ...
+├── examples/
+│   ├── incremental_load.py     # CSV incremental loading
+│   ├── merge_load.py            # CSV merge/upsert
+│   └── data/                    # Sample CSV files
+└── docker-compose.yml           # Nessie + MinIO for testing
+```
+
+## How It Works
+
+The class-based destination uses dlt's `JobClientBase` interface to accumulate parquet files during a load and commit them atomically in `complete_load()`:
+
+1. dlt extracts data and writes parquet files
+2. Each file is registered in module-level global state
+3. After all files complete, `complete_load()` is called
+4. All files for a table are combined and committed as single Iceberg snapshot
+5. Each table gets one snapshot per load
+
+This ensures atomic commits even though dlt creates multiple client instances.
+
+## License
+
+MIT License - see LICENSE file
+
+## Resources
+
+- [dlt Documentation](https://dlthub.com/docs)
+- [Apache Iceberg](https://iceberg.apache.org/)
+- [PyIceberg](https://py.iceberg.apache.org/)
+- [Iceberg REST Spec](https://iceberg.apache.org/rest-catalog-spec/)

dlt_iceberg-0.1.3/pyproject.toml

@@ -0,0 +1,58 @@
+[project]
+name = "dlt-iceberg"
+version = "0.1.3"
+description = "dlt destination for Apache Iceberg with atomic multi-file commits via REST catalogs"
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "MIT"}
+authors = [
+    {name = "Sidequery", email = "hello@sidequery.com"}
+]
+keywords = ["dlt", "iceberg", "data-engineering", "etl", "elt", "data-pipeline"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "boto3>=1.40.50",
+    "dlt>=1.17.1",
+    "pandas>=2.3.3",
+    "pyarrow>=21.0.0",
+    "pydantic<2.11",
+    "pyiceberg[pyiceberg-core]>=0.10.0",
+    "requests>=2.32.5",
+    "s3fs>=0.4.2",
+    "sqlalchemy>=2.0.44",
+]
+
+[project.urls]
+Homepage = "https://github.com/sidequery/dlt-iceberg"
+Repository = "https://github.com/sidequery/dlt-iceberg"
+Issues = "https://github.com/sidequery/dlt-iceberg/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dlt_iceberg"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.2",
+    "pytest-mock>=3.15.1",
+]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+markers = [
+    "integration: marks tests as integration tests that require external services (docker compose)",
+]

{dlt_iceberg-0.1.1 → dlt_iceberg-0.1.3}/src/dlt_iceberg/destination_client.py

@@ -34,7 +34,11 @@ from pyiceberg.exceptions import (
 from .schema_converter import convert_dlt_to_iceberg_schema
 from .partition_builder import build_partition_spec
 from .schema_evolution import evolve_schema_if_needed, SchemaEvolutionError
-from .schema_casting import cast_table_safe, CastingError
+from .schema_casting import (
+    cast_table_safe,
+    CastingError,
+    ensure_iceberg_compatible_arrow_data,
+)
 from .error_handling import (
     is_retryable_error,
     log_error_with_context,
@@ -89,6 +93,9 @@ class IcebergRestConfiguration(DestinationClientConfiguration):
     # Schema casting configuration
     strict_casting: bool = False
 
+    # Merge batch size (for upsert operations to avoid memory issues)
+    merge_batch_size: int = 100000
+
 
 class IcebergRestLoadJob(RunnableLoadJob):
     """
@@ -380,7 +387,8 @@ class IcebergRestClient(JobClientBase):
                 # Create table if needed
                 if not table_exists:
                     # Use first file's Arrow table to generate schema
-                    first_arrow_table = file_data[0][2]
+                    # Apply Iceberg compatibility first so schema uses compatible types
+                    first_arrow_table = ensure_iceberg_compatible_arrow_data(file_data[0][2])
                     iceberg_schema = convert_dlt_to_iceberg_schema(
                         table_schema, first_arrow_table
                     )
@@ -401,7 +409,7 @@ class IcebergRestClient(JobClientBase):
                     logger.info(f"Created table {identifier} at {iceberg_table.location()}")
                 else:
                     # Table exists - check if schema evolution is needed
-                    first_arrow_table = file_data[0][2]
+                    first_arrow_table = ensure_iceberg_compatible_arrow_data(file_data[0][2])
                     incoming_schema = convert_dlt_to_iceberg_schema(
                         table_schema, first_arrow_table
                     )
@@ -415,12 +423,15 @@ class IcebergRestClient(JobClientBase):
                         logger.info(f"Schema evolved for table {identifier}")
                         iceberg_table = catalog.load_table(identifier)
 
-                # Combine all Arrow tables and cast to match Iceberg schema
+                # Get expected schema (already has Iceberg-compatible types from creation)
                 expected_schema = schema_to_pyarrow(iceberg_table.schema())
+
+                # Combine all Arrow tables and cast to match Iceberg schema
                 combined_tables = []
 
                 for _, file_path, arrow_table in file_data:
-                    # Cast each table to match Iceberg schema
+                    # Cast to match Iceberg schema
+                    # (compatibility conversions already applied when schema was created)
                     casted_table = cast_table_safe(
                         arrow_table,
                         expected_schema,
@@ -463,15 +474,34 @@ class IcebergRestClient(JobClientBase):
                         iceberg_table.append(combined_table)
                     else:
                         logger.info(f"Merging into table {identifier} on keys {primary_keys}")
-                        upsert_result = iceberg_table.upsert(
-                            df=combined_table,
-                            join_cols=primary_keys,
-                            when_matched_update_all=True,
-                            when_not_matched_insert_all=True,
-                        )
+
+                        # Batch upserts to avoid memory issues on large datasets
+                        batch_size = self.config.merge_batch_size
+                        total_updated = 0
+                        total_inserted = 0
+
+                        for batch_start in range(0, len(combined_table), batch_size):
+                            batch_end = min(batch_start + batch_size, len(combined_table))
+                            batch = combined_table.slice(batch_start, batch_end - batch_start)
+
+                            logger.info(
+                                f"Upserting batch {batch_start//batch_size + 1}: "
+                                f"rows {batch_start} to {batch_end} ({len(batch)} rows)"
+                            )
+
+                            upsert_result = iceberg_table.upsert(
+                                df=batch,
+                                join_cols=primary_keys,
+                                when_matched_update_all=True,
+                                when_not_matched_insert_all=True,
+                            )
+
+                            total_updated += upsert_result.rows_updated
+                            total_inserted += upsert_result.rows_inserted
+
                         logger.info(
-                            f"Upsert completed: {upsert_result.rows_updated} updated, "
-                            f"{upsert_result.rows_inserted} inserted"
+                            f"Upsert completed: {total_updated} updated, "
+                            f"{total_inserted} inserted across {(total_rows + batch_size - 1) // batch_size} batches"
                         )
                 else:
                     raise ValueError(f"Unknown write disposition: {write_disposition}")

{dlt_iceberg-0.1.1 → dlt_iceberg-0.1.3}/src/dlt_iceberg/schema_casting.py

@@ -6,12 +6,75 @@ and allow users to control casting behavior.
 """
 
 import logging
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Dict, Callable
 import pyarrow as pa
 
 logger = logging.getLogger(__name__)
 
 
+def ensure_iceberg_compatible_arrow_schema(schema: pa.Schema) -> pa.Schema:
+    """
+    Convert Arrow schema to Iceberg-compatible schema.
+
+    Converts types that Iceberg doesn't support:
+    - time32 → time64 (microseconds)
+    - decimal256 → string (Iceberg only supports decimal128)
+    - dictionary → value_type (unwrap dictionary encoding)
+
+    Args:
+        schema: PyArrow schema
+
+    Returns:
+        Iceberg-compatible PyArrow schema
+    """
+    def convert_field(field: pa.Field) -> pa.Field:
+        field_type = field.type
+
+        # time32 → time64(us)
+        if pa.types.is_time32(field_type):
+            return pa.field(field.name, pa.time64("us"), nullable=field.nullable)
+
+        # decimal256 → string (pyarrow doesn't allow downcasting to decimal128)
+        if pa.types.is_decimal256(field_type):
+            logger.warning(
+                f"Converting decimal256 field '{field.name}' to string "
+                f"(Iceberg doesn't support decimal256)"
+            )
+            return pa.field(field.name, pa.string(), nullable=field.nullable)
+
+        # dictionary → value_type (unwrap dictionary encoding)
+        if pa.types.is_dictionary(field_type):
+            return pa.field(field.name, field_type.value_type, nullable=field.nullable)
+
+        # list/struct types - recursively convert nested fields
+        if pa.types.is_list(field_type):
+            value_field = convert_field(pa.field("item", field_type.value_type))
+            return pa.field(field.name, pa.list_(value_field.type), nullable=field.nullable)
+
+        if pa.types.is_struct(field_type):
+            new_fields = [convert_field(f) for f in field_type]
+            return pa.field(field.name, pa.struct(new_fields), nullable=field.nullable)
+
+        return field
+
+    new_fields = [convert_field(field) for field in schema]
+    return pa.schema(new_fields)
+
+
+def ensure_iceberg_compatible_arrow_data(table: pa.Table) -> pa.Table:
+    """
+    Convert Arrow table to Iceberg-compatible schema and cast data.
+
+    Args:
+        table: PyArrow table
+
+    Returns:
+        Table with Iceberg-compatible schema
+    """
+    new_schema = ensure_iceberg_compatible_arrow_schema(table.schema)
+    return table.cast(new_schema)
+
+
 class CastingError(Exception):
     """Raised when a cast would result in data loss in strict mode."""
     pass