dlt-altertable 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/PKG-INFO +24 -12
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/README.md +23 -11
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/pyproject.toml +1 -1
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/pyproject.toml.orig +1 -1
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/job_client.py +8 -1
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/table_schema.py +2 -1
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/LICENSE +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/__init__.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/altertable_adapter.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/api.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/configuration.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/destination.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/exceptions.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/layout.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/merge.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/py.typed +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/sql_client.py +0 -0
- {dlt_altertable-0.2.2 → dlt_altertable-0.2.3}/src/dlt_altertable/verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dlt-altertable
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
|
|
5
5
|
Keywords: dlt,altertable,lakehouse,parquet
|
|
6
6
|
Author: Altertable
|
|
@@ -30,7 +30,13 @@ Description-Content-Type: text/markdown
|
|
|
30
30
|
[](https://www.python.org)
|
|
31
31
|
[](LICENSE)
|
|
32
32
|
|
|
33
|
-
Load
|
|
33
|
+
Load [dlt](https://dlthub.com) sources into the [Altertable](https://altertable.ai) lakehouse,
|
|
34
|
+
then [query them](https://altertable.ai/docs/lakehouse/sql-engine) or read them back from Python.
|
|
35
|
+
|
|
36
|
+
- **Upsert merges**: by primary key, with child tables and hard deletes.
|
|
37
|
+
- **Schema evolution**: new columns appear automatically; descriptions become catalog comments.
|
|
38
|
+
- **Resumable runs**: incremental state survives fresh runners.
|
|
39
|
+
- **Verified loads**: `verify_load` reads the last load back and reports problems.
|
|
34
40
|
|
|
35
41
|
## Install and configure
|
|
36
42
|
|
|
@@ -50,14 +56,21 @@ username = "..."
|
|
|
50
56
|
password = "..."
|
|
51
57
|
```
|
|
52
58
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
59
|
+
| Setting | Fallback variable | Default |
|
|
60
|
+
| --- | --- | --- |
|
|
61
|
+
| `host` | `ALTERTABLE_HOST` | Required, the HTTP API host |
|
|
62
|
+
| `catalog` | `ALTERTABLE_CATALOG` | Required, must exist |
|
|
63
|
+
| `username`, `password` | `ALTERTABLE_USERNAME`, `ALTERTABLE_PASSWORD` | Required |
|
|
64
|
+
| `port` | `ALTERTABLE_PORT` | `443` |
|
|
65
|
+
| `tls` | `ALTERTABLE_TLS` | `true` |
|
|
66
|
+
| `compute_size` | `ALTERTABLE_COMPUTE_SIZE` | `XS` |
|
|
67
|
+
| `allow_destructive_refresh` | | `false` |
|
|
56
68
|
|
|
57
|
-
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.)
|
|
58
|
-
|
|
69
|
+
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.) or
|
|
70
|
+
arguments to `altertable(...)`. The pipeline's `dataset_name` is the schema, defaulting to
|
|
71
|
+
`ALTERTABLE_SCHEMA`. Schemas are created automatically.
|
|
59
72
|
|
|
60
|
-
##
|
|
73
|
+
## Quickstart
|
|
61
74
|
|
|
62
75
|
```python
|
|
63
76
|
import dlt
|
|
@@ -79,6 +92,7 @@ def contacts():
|
|
|
79
92
|
|
|
80
93
|
pipeline = dlt.pipeline(pipeline_name="crm", destination=altertable, dataset_name="crm")
|
|
81
94
|
pipeline.run(contacts())
|
|
95
|
+
print(pipeline.dataset().contacts.arrow())
|
|
82
96
|
```
|
|
83
97
|
|
|
84
98
|
## Loading
|
|
@@ -93,7 +107,7 @@ Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps th
|
|
|
93
107
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
94
108
|
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
95
109
|
|
|
96
|
-
|
|
110
|
+
Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
97
111
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
98
112
|
and lists into child tables.
|
|
99
113
|
|
|
@@ -103,13 +117,11 @@ omitted fields become null and omitted lists are cleared.
|
|
|
103
117
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
104
118
|
|
|
105
119
|
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
106
|
-
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
107
120
|
|
|
108
121
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
109
122
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
110
123
|
|
|
111
|
-
Append and replace commit one file at a time.
|
|
112
|
-
file, then appends the rest. Retried appends can duplicate rows. Do not set
|
|
124
|
+
Append and replace commit one file at a time. Retried appends can duplicate rows. Do not set
|
|
113
125
|
`LOAD__PARALLELISM_STRATEGY=parallel`: replacement requires sequential files per table.
|
|
114
126
|
|
|
115
127
|
## Partitioning
|
|
@@ -5,7 +5,13 @@
|
|
|
5
5
|
[](https://www.python.org)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
|
|
8
|
-
Load
|
|
8
|
+
Load [dlt](https://dlthub.com) sources into the [Altertable](https://altertable.ai) lakehouse,
|
|
9
|
+
then [query them](https://altertable.ai/docs/lakehouse/sql-engine) or read them back from Python.
|
|
10
|
+
|
|
11
|
+
- **Upsert merges**: by primary key, with child tables and hard deletes.
|
|
12
|
+
- **Schema evolution**: new columns appear automatically; descriptions become catalog comments.
|
|
13
|
+
- **Resumable runs**: incremental state survives fresh runners.
|
|
14
|
+
- **Verified loads**: `verify_load` reads the last load back and reports problems.
|
|
9
15
|
|
|
10
16
|
## Install and configure
|
|
11
17
|
|
|
@@ -25,14 +31,21 @@ username = "..."
|
|
|
25
31
|
password = "..."
|
|
26
32
|
```
|
|
27
33
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
34
|
+
| Setting | Fallback variable | Default |
|
|
35
|
+
| --- | --- | --- |
|
|
36
|
+
| `host` | `ALTERTABLE_HOST` | Required, the HTTP API host |
|
|
37
|
+
| `catalog` | `ALTERTABLE_CATALOG` | Required, must exist |
|
|
38
|
+
| `username`, `password` | `ALTERTABLE_USERNAME`, `ALTERTABLE_PASSWORD` | Required |
|
|
39
|
+
| `port` | `ALTERTABLE_PORT` | `443` |
|
|
40
|
+
| `tls` | `ALTERTABLE_TLS` | `true` |
|
|
41
|
+
| `compute_size` | `ALTERTABLE_COMPUTE_SIZE` | `XS` |
|
|
42
|
+
| `allow_destructive_refresh` | | `false` |
|
|
31
43
|
|
|
32
|
-
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.)
|
|
33
|
-
|
|
44
|
+
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.) or
|
|
45
|
+
arguments to `altertable(...)`. The pipeline's `dataset_name` is the schema, defaulting to
|
|
46
|
+
`ALTERTABLE_SCHEMA`. Schemas are created automatically.
|
|
34
47
|
|
|
35
|
-
##
|
|
48
|
+
## Quickstart
|
|
36
49
|
|
|
37
50
|
```python
|
|
38
51
|
import dlt
|
|
@@ -54,6 +67,7 @@ def contacts():
|
|
|
54
67
|
|
|
55
68
|
pipeline = dlt.pipeline(pipeline_name="crm", destination=altertable, dataset_name="crm")
|
|
56
69
|
pipeline.run(contacts())
|
|
70
|
+
print(pipeline.dataset().contacts.arrow())
|
|
57
71
|
```
|
|
58
72
|
|
|
59
73
|
## Loading
|
|
@@ -68,7 +82,7 @@ Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps th
|
|
|
68
82
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
69
83
|
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
70
84
|
|
|
71
|
-
|
|
85
|
+
Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
72
86
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
73
87
|
and lists into child tables.
|
|
74
88
|
|
|
@@ -78,13 +92,11 @@ omitted fields become null and omitted lists are cleared.
|
|
|
78
92
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
79
93
|
|
|
80
94
|
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
81
|
-
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
82
95
|
|
|
83
96
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
84
97
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
85
98
|
|
|
86
|
-
Append and replace commit one file at a time.
|
|
87
|
-
file, then appends the rest. Retried appends can duplicate rows. Do not set
|
|
99
|
+
Append and replace commit one file at a time. Retried appends can duplicate rows. Do not set
|
|
88
100
|
`LOAD__PARALLELISM_STRATEGY=parallel`: replacement requires sequential files per table.
|
|
89
101
|
|
|
90
102
|
## Partitioning
|
|
@@ -217,9 +217,16 @@ class AltertableJobClient(DestinationClient, WithStateSync, WithSqlClient):
|
|
|
217
217
|
only_tables: Iterable[str] | None = None,
|
|
218
218
|
expected_update: TSchemaTables | None = None,
|
|
219
219
|
force: bool = False,
|
|
220
|
+
include_existing_tables: bool = False,
|
|
220
221
|
) -> TSchemaTables | None:
|
|
221
222
|
update = super().update_stored_schema(only_tables or (), expected_update or {}, force)
|
|
222
|
-
|
|
223
|
+
rows = altertable_api.execute_sql(
|
|
224
|
+
self.dataset_config,
|
|
225
|
+
"SELECT 1 FROM "
|
|
226
|
+
f"{qualified_table_name(self.dataset_config, self.schema.version_table_name)} "
|
|
227
|
+
f"WHERE version_hash = {escape_duckdb_literal(self.schema.stored_version_hash)}",
|
|
228
|
+
)
|
|
229
|
+
if not rows:
|
|
223
230
|
schema_insert_sql = self._insert_statement(
|
|
224
231
|
self.schema.version_table_name,
|
|
225
232
|
{
|
|
@@ -95,7 +95,8 @@ def apply_descriptions(
|
|
|
95
95
|
if column_comments is None:
|
|
96
96
|
targets[f"TABLE {table_name}"] = table.get("description")
|
|
97
97
|
statements = [
|
|
98
|
-
|
|
98
|
+
# DuckDB 2.0 keeps E'' escapes in comments verbatim until duckdb/duckdb#29087 ships.
|
|
99
|
+
f"COMMENT ON {target} IS '{description.replace("'", "''")}';"
|
|
99
100
|
for target, description in targets.items()
|
|
100
101
|
if description
|
|
101
102
|
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|