dlt-iceberg 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dlt-iceberg might be problematic. Click here for more details.
- dlt_iceberg/destination_client.py +1 -1
- {dlt_iceberg-0.1.3.dist-info → dlt_iceberg-0.1.5.dist-info}/METADATA +40 -5
- {dlt_iceberg-0.1.3.dist-info → dlt_iceberg-0.1.5.dist-info}/RECORD +5 -5
- {dlt_iceberg-0.1.3.dist-info → dlt_iceberg-0.1.5.dist-info}/WHEEL +0 -0
- {dlt_iceberg-0.1.3.dist-info → dlt_iceberg-0.1.5.dist-info}/licenses/LICENSE +0 -0
|
@@ -94,7 +94,7 @@ class IcebergRestConfiguration(DestinationClientConfiguration):
|
|
|
94
94
|
strict_casting: bool = False
|
|
95
95
|
|
|
96
96
|
# Merge batch size (for upsert operations to avoid memory issues)
|
|
97
|
-
merge_batch_size: int =
|
|
97
|
+
merge_batch_size: int = 500000
|
|
98
98
|
|
|
99
99
|
|
|
100
100
|
class IcebergRestLoadJob(RunnableLoadJob):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dlt-iceberg
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: dlt destination for Apache Iceberg with atomic multi-file commits via REST catalogs
|
|
5
5
|
Project-URL: Homepage, https://github.com/sidequery/dlt-iceberg
|
|
6
6
|
Project-URL: Repository, https://github.com/sidequery/dlt-iceberg
|
|
@@ -47,9 +47,13 @@ A [dlt](https://dlthub.com/) destination for [Apache Iceberg](https://iceberg.ap
|
|
|
47
47
|
## Installation
|
|
48
48
|
|
|
49
49
|
```bash
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
pip install dlt-iceberg
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Or with uv:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
uv add dlt-iceberg
|
|
53
57
|
```
|
|
54
58
|
|
|
55
59
|
## Quick Start
|
|
@@ -95,7 +99,38 @@ def generate_users():
|
|
|
95
99
|
pipeline.run(generate_users())
|
|
96
100
|
```
|
|
97
101
|
|
|
98
|
-
## Configuration
|
|
102
|
+
## Configuration Options
|
|
103
|
+
|
|
104
|
+
All configuration options can be passed to `iceberg_rest()`:
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
iceberg_rest(
|
|
108
|
+
catalog_uri="...", # Required: REST catalog URI
|
|
109
|
+
namespace="...", # Required: Iceberg namespace (database)
|
|
110
|
+
warehouse="...", # Optional: Warehouse location
|
|
111
|
+
|
|
112
|
+
# Authentication
|
|
113
|
+
credential="...", # OAuth2 client credentials
|
|
114
|
+
oauth2_server_uri="...", # OAuth2 token endpoint
|
|
115
|
+
token="...", # Bearer token
|
|
116
|
+
|
|
117
|
+
# AWS SigV4
|
|
118
|
+
sigv4_enabled=True,
|
|
119
|
+
signing_region="us-east-1",
|
|
120
|
+
|
|
121
|
+
# S3 configuration
|
|
122
|
+
s3_endpoint="...",
|
|
123
|
+
s3_access_key_id="...",
|
|
124
|
+
s3_secret_access_key="...",
|
|
125
|
+
s3_region="...",
|
|
126
|
+
|
|
127
|
+
# Performance tuning
|
|
128
|
+
max_retries=5, # Retry attempts for transient failures
|
|
129
|
+
retry_backoff_base=2.0, # Exponential backoff multiplier
|
|
130
|
+
merge_batch_size=500000, # Rows per batch for merge operations
|
|
131
|
+
strict_casting=False, # Fail on potential data loss
|
|
132
|
+
)
|
|
133
|
+
```
|
|
99
134
|
|
|
100
135
|
### Nessie (Docker)
|
|
101
136
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
dlt_iceberg/__init__.py,sha256=ONy6E-sGcCvvqia8_fGaYp8da4n4wdjox9W42tmQPK0,780
|
|
2
2
|
dlt_iceberg/destination.py,sha256=F8QJXsQeosOA32Xm1140DL485WQmxbuhiA2QZ6zpVSU,15737
|
|
3
|
-
dlt_iceberg/destination_client.py,sha256=
|
|
3
|
+
dlt_iceberg/destination_client.py,sha256=5rgtCka1rGtNNSPBG0ozKqt9pithFlFdRnkp4tstC7s,23786
|
|
4
4
|
dlt_iceberg/error_handling.py,sha256=k6Kkldi9BDRsXQ63VEBMMSw1xx2-b1BMjsgRFKI2iB0,7852
|
|
5
5
|
dlt_iceberg/partition_builder.py,sha256=l9YNAh2t6gk2xqsPSOs8ymTDLk9BOEZWVOtVni7ONNU,10081
|
|
6
6
|
dlt_iceberg/schema_casting.py,sha256=oSQrnOcCMFcinMS65N8YQ1uzrqnQmN50mCCuQyE3794,15247
|
|
7
7
|
dlt_iceberg/schema_converter.py,sha256=e_eqXQz2cpABOGEAxVwcGbiOdVmv9kaZanRnU83lzXk,5619
|
|
8
8
|
dlt_iceberg/schema_evolution.py,sha256=ieOkCA9ngQdJ5lbZLYQ09deTLZEW8whxDn2arpoH-aM,8326
|
|
9
|
-
dlt_iceberg-0.1.
|
|
10
|
-
dlt_iceberg-0.1.
|
|
11
|
-
dlt_iceberg-0.1.
|
|
12
|
-
dlt_iceberg-0.1.
|
|
9
|
+
dlt_iceberg-0.1.5.dist-info/METADATA,sha256=gPTkC63ZO94S78E4uaqp4zPNecXaI95H6G6hzzpoBbY,8777
|
|
10
|
+
dlt_iceberg-0.1.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
11
|
+
dlt_iceberg-0.1.5.dist-info/licenses/LICENSE,sha256=0amGlcH0msYju3WUhlsuUxO4aj3ZODkkIZ0MKOq9fQ4,1066
|
|
12
|
+
dlt_iceberg-0.1.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|