dlt-altertable 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/PKG-INFO +30 -12
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/README.md +29 -11
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/pyproject.toml +1 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/pyproject.toml.orig +1 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/destination.py +2 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/job_client.py +8 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/table_schema.py +32 -3
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/LICENSE +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/__init__.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/altertable_adapter.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/api.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/configuration.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/exceptions.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/layout.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/merge.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/py.typed +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/sql_client.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.3}/src/dlt_altertable/verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dlt-altertable
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
|
|
5
5
|
Keywords: dlt,altertable,lakehouse,parquet
|
|
6
6
|
Author: Altertable
|
|
@@ -30,7 +30,13 @@ Description-Content-Type: text/markdown
|
|
|
30
30
|
[](https://www.python.org)
|
|
31
31
|
[](LICENSE)
|
|
32
32
|
|
|
33
|
-
Load
|
|
33
|
+
Load [dlt](https://dlthub.com) sources into the [Altertable](https://altertable.ai) lakehouse,
|
|
34
|
+
then [query them](https://altertable.ai/docs/lakehouse/sql-engine) or read them back from Python.
|
|
35
|
+
|
|
36
|
+
- **Upsert merges**: by primary key, with child tables and hard deletes.
|
|
37
|
+
- **Schema evolution**: new columns appear automatically; descriptions become catalog comments.
|
|
38
|
+
- **Resumable runs**: incremental state survives fresh runners.
|
|
39
|
+
- **Verified loads**: `verify_load` reads the last load back and reports problems.
|
|
34
40
|
|
|
35
41
|
## Install and configure
|
|
36
42
|
|
|
@@ -50,14 +56,21 @@ username = "..."
|
|
|
50
56
|
password = "..."
|
|
51
57
|
```
|
|
52
58
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
59
|
+
| Setting | Fallback variable | Default |
|
|
60
|
+
| --- | --- | --- |
|
|
61
|
+
| `host` | `ALTERTABLE_HOST` | Required, the HTTP API host |
|
|
62
|
+
| `catalog` | `ALTERTABLE_CATALOG` | Required, must exist |
|
|
63
|
+
| `username`, `password` | `ALTERTABLE_USERNAME`, `ALTERTABLE_PASSWORD` | Required |
|
|
64
|
+
| `port` | `ALTERTABLE_PORT` | `443` |
|
|
65
|
+
| `tls` | `ALTERTABLE_TLS` | `true` |
|
|
66
|
+
| `compute_size` | `ALTERTABLE_COMPUTE_SIZE` | `XS` |
|
|
67
|
+
| `allow_destructive_refresh` | | `false` |
|
|
56
68
|
|
|
57
|
-
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.)
|
|
58
|
-
|
|
69
|
+
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.) or
|
|
70
|
+
arguments to `altertable(...)`. The pipeline's `dataset_name` is the schema, defaulting to
|
|
71
|
+
`ALTERTABLE_SCHEMA`. Schemas are created automatically.
|
|
59
72
|
|
|
60
|
-
##
|
|
73
|
+
## Quickstart
|
|
61
74
|
|
|
62
75
|
```python
|
|
63
76
|
import dlt
|
|
@@ -79,6 +92,7 @@ def contacts():
|
|
|
79
92
|
|
|
80
93
|
pipeline = dlt.pipeline(pipeline_name="crm", destination=altertable, dataset_name="crm")
|
|
81
94
|
pipeline.run(contacts())
|
|
95
|
+
print(pipeline.dataset().contacts.arrow())
|
|
82
96
|
```
|
|
83
97
|
|
|
84
98
|
## Loading
|
|
@@ -93,7 +107,7 @@ Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps th
|
|
|
93
107
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
94
108
|
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
95
109
|
|
|
96
|
-
|
|
110
|
+
Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
97
111
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
98
112
|
and lists into child tables.
|
|
99
113
|
|
|
@@ -103,13 +117,11 @@ omitted fields become null and omitted lists are cleared.
|
|
|
103
117
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
104
118
|
|
|
105
119
|
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
106
|
-
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
107
120
|
|
|
108
121
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
109
122
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
110
123
|
|
|
111
|
-
Append and replace commit one file at a time.
|
|
112
|
-
file, then appends the rest. Retried appends can duplicate rows. Do not set
|
|
124
|
+
Append and replace commit one file at a time. Retried appends can duplicate rows. Do not set
|
|
113
125
|
`LOAD__PARALLELISM_STRATEGY=parallel`: replacement requires sequential files per table.
|
|
114
126
|
|
|
115
127
|
## Partitioning
|
|
@@ -125,6 +137,12 @@ pipeline.run(resource)
|
|
|
125
137
|
|
|
126
138
|
Pass `partition=[]` to reset. Sort hints are not supported yet.
|
|
127
139
|
|
|
140
|
+
## Descriptions
|
|
141
|
+
|
|
142
|
+
Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
|
|
143
|
+
become comments in the Altertable catalog. Column comments track changes. Table comments are
|
|
144
|
+
set on create or replace.
|
|
145
|
+
|
|
128
146
|
## Performance
|
|
129
147
|
|
|
130
148
|
Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
|
|
@@ -5,7 +5,13 @@
|
|
|
5
5
|
[](https://www.python.org)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
|
|
8
|
-
Load
|
|
8
|
+
Load [dlt](https://dlthub.com) sources into the [Altertable](https://altertable.ai) lakehouse,
|
|
9
|
+
then [query them](https://altertable.ai/docs/lakehouse/sql-engine) or read them back from Python.
|
|
10
|
+
|
|
11
|
+
- **Upsert merges**: by primary key, with child tables and hard deletes.
|
|
12
|
+
- **Schema evolution**: new columns appear automatically; descriptions become catalog comments.
|
|
13
|
+
- **Resumable runs**: incremental state survives fresh runners.
|
|
14
|
+
- **Verified loads**: `verify_load` reads the last load back and reports problems.
|
|
9
15
|
|
|
10
16
|
## Install and configure
|
|
11
17
|
|
|
@@ -25,14 +31,21 @@ username = "..."
|
|
|
25
31
|
password = "..."
|
|
26
32
|
```
|
|
27
33
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
34
|
+
| Setting | Fallback variable | Default |
|
|
35
|
+
| --- | --- | --- |
|
|
36
|
+
| `host` | `ALTERTABLE_HOST` | Required, the HTTP API host |
|
|
37
|
+
| `catalog` | `ALTERTABLE_CATALOG` | Required, must exist |
|
|
38
|
+
| `username`, `password` | `ALTERTABLE_USERNAME`, `ALTERTABLE_PASSWORD` | Required |
|
|
39
|
+
| `port` | `ALTERTABLE_PORT` | `443` |
|
|
40
|
+
| `tls` | `ALTERTABLE_TLS` | `true` |
|
|
41
|
+
| `compute_size` | `ALTERTABLE_COMPUTE_SIZE` | `XS` |
|
|
42
|
+
| `allow_destructive_refresh` | | `false` |
|
|
31
43
|
|
|
32
|
-
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.)
|
|
33
|
-
|
|
44
|
+
Settings also accept dlt environment variables (`DESTINATION__ALTERTABLE__HOST`, etc.) or
|
|
45
|
+
arguments to `altertable(...)`. The pipeline's `dataset_name` is the schema, defaulting to
|
|
46
|
+
`ALTERTABLE_SCHEMA`. Schemas are created automatically.
|
|
34
47
|
|
|
35
|
-
##
|
|
48
|
+
## Quickstart
|
|
36
49
|
|
|
37
50
|
```python
|
|
38
51
|
import dlt
|
|
@@ -54,6 +67,7 @@ def contacts():
|
|
|
54
67
|
|
|
55
68
|
pipeline = dlt.pipeline(pipeline_name="crm", destination=altertable, dataset_name="crm")
|
|
56
69
|
pipeline.run(contacts())
|
|
70
|
+
print(pipeline.dataset().contacts.arrow())
|
|
57
71
|
```
|
|
58
72
|
|
|
59
73
|
## Loading
|
|
@@ -68,7 +82,7 @@ Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps th
|
|
|
68
82
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
69
83
|
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
70
84
|
|
|
71
|
-
|
|
85
|
+
Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
72
86
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
73
87
|
and lists into child tables.
|
|
74
88
|
|
|
@@ -78,13 +92,11 @@ omitted fields become null and omitted lists are cleared.
|
|
|
78
92
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
79
93
|
|
|
80
94
|
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
81
|
-
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
82
95
|
|
|
83
96
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
84
97
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
85
98
|
|
|
86
|
-
Append and replace commit one file at a time.
|
|
87
|
-
file, then appends the rest. Retried appends can duplicate rows. Do not set
|
|
99
|
+
Append and replace commit one file at a time. Retried appends can duplicate rows. Do not set
|
|
88
100
|
`LOAD__PARALLELISM_STRATEGY=parallel`: replacement requires sequential files per table.
|
|
89
101
|
|
|
90
102
|
## Partitioning
|
|
@@ -100,6 +112,12 @@ pipeline.run(resource)
|
|
|
100
112
|
|
|
101
113
|
Pass `partition=[]` to reset. Sort hints are not supported yet.
|
|
102
114
|
|
|
115
|
+
## Descriptions
|
|
116
|
+
|
|
117
|
+
Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
|
|
118
|
+
become comments in the Altertable catalog. Column comments track changes. Table comments are
|
|
119
|
+
set on create or replace.
|
|
120
|
+
|
|
103
121
|
## Performance
|
|
104
122
|
|
|
105
123
|
Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
|
|
@@ -18,7 +18,7 @@ from dlt.destinations.impl.destination.factory import destination as CustomDesti
|
|
|
18
18
|
|
|
19
19
|
from dlt_altertable import api as altertable_api
|
|
20
20
|
from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
|
|
21
|
-
from dlt_altertable.table_schema import aligned_parquet, create_or_evolve_table
|
|
21
|
+
from dlt_altertable.table_schema import aligned_parquet, apply_descriptions, create_or_evolve_table
|
|
22
22
|
|
|
23
23
|
if TYPE_CHECKING:
|
|
24
24
|
from dlt_altertable.job_client import AltertableJobClient
|
|
@@ -133,6 +133,7 @@ def _upload(
|
|
|
133
133
|
altertable_api.post_parquet(config, "upload", params, upload_path, action)
|
|
134
134
|
|
|
135
135
|
if ingest_mode == "overwrite":
|
|
136
|
+
apply_descriptions(config, table)
|
|
136
137
|
already_replaced.append(table_name)
|
|
137
138
|
|
|
138
139
|
|
|
@@ -217,9 +217,16 @@ class AltertableJobClient(DestinationClient, WithStateSync, WithSqlClient):
|
|
|
217
217
|
only_tables: Iterable[str] | None = None,
|
|
218
218
|
expected_update: TSchemaTables | None = None,
|
|
219
219
|
force: bool = False,
|
|
220
|
+
include_existing_tables: bool = False,
|
|
220
221
|
) -> TSchemaTables | None:
|
|
221
222
|
update = super().update_stored_schema(only_tables or (), expected_update or {}, force)
|
|
222
|
-
|
|
223
|
+
rows = altertable_api.execute_sql(
|
|
224
|
+
self.dataset_config,
|
|
225
|
+
"SELECT 1 FROM "
|
|
226
|
+
f"{qualified_table_name(self.dataset_config, self.schema.version_table_name)} "
|
|
227
|
+
f"WHERE version_hash = {escape_duckdb_literal(self.schema.stored_version_hash)}",
|
|
228
|
+
)
|
|
229
|
+
if not rows:
|
|
223
230
|
schema_insert_sql = self._insert_statement(
|
|
224
231
|
self.schema.version_table_name,
|
|
225
232
|
{
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import tempfile
|
|
3
|
-
from collections.abc import Iterator
|
|
3
|
+
from collections.abc import Iterator, Mapping
|
|
4
4
|
from contextlib import contextmanager
|
|
5
5
|
from typing import cast
|
|
6
6
|
|
|
@@ -79,6 +79,33 @@ def apply_partitioning(
|
|
|
79
79
|
)
|
|
80
80
|
|
|
81
81
|
|
|
82
|
+
def apply_descriptions(
|
|
83
|
+
config: AltertableClientConfiguration,
|
|
84
|
+
table: TTableSchema | PreparedTableSchema,
|
|
85
|
+
column_comments: Mapping[str, str | None] | None = None,
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Publish dlt descriptions as catalog comments, so people and agents see them.
|
|
88
|
+
Only changed column comments are rewritten. The table comment is set on new tables."""
|
|
89
|
+
table_name = qualified_table_name(config, cast(str, table["name"]))
|
|
90
|
+
targets = {
|
|
91
|
+
f"COLUMN {table_name}.{escape_postgres_identifier(name)}": column.get("description")
|
|
92
|
+
for name, column in table["columns"].items()
|
|
93
|
+
if column.get("description") != (column_comments or {}).get(name)
|
|
94
|
+
}
|
|
95
|
+
if column_comments is None:
|
|
96
|
+
targets[f"TABLE {table_name}"] = table.get("description")
|
|
97
|
+
statements = [
|
|
98
|
+
# DuckDB 2.0 keeps E'' escapes in comments verbatim until duckdb/duckdb#29087 ships.
|
|
99
|
+
f"COMMENT ON {target} IS '{description.replace("'", "''")}';"
|
|
100
|
+
for target, description in targets.items()
|
|
101
|
+
if description
|
|
102
|
+
]
|
|
103
|
+
if statements:
|
|
104
|
+
altertable_api.execute_sql(
|
|
105
|
+
config, "\n".join(["BEGIN TRANSACTION;", *statements, "COMMIT;"])
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
82
109
|
def create_table(
|
|
83
110
|
config: AltertableClientConfiguration,
|
|
84
111
|
table: TTableSchema | PreparedTableSchema,
|
|
@@ -98,6 +125,7 @@ def create_table(
|
|
|
98
125
|
f"CREATE TABLE IF NOT EXISTS {qualified_table_name(config, table_name)} ({columns})",
|
|
99
126
|
)
|
|
100
127
|
apply_partitioning(config, table)
|
|
128
|
+
apply_descriptions(config, table)
|
|
101
129
|
logger.info(
|
|
102
130
|
f"Created table {config.catalog}.{config.dataset_name}.{table_name} "
|
|
103
131
|
f"with {len(table['columns'])} columns"
|
|
@@ -124,12 +152,12 @@ def create_or_evolve_table(
|
|
|
124
152
|
table_name = cast(str, table["name"])
|
|
125
153
|
rows = altertable_api.execute_sql(
|
|
126
154
|
config,
|
|
127
|
-
"SELECT column_name, data_type FROM information_schema.columns "
|
|
155
|
+
"SELECT column_name, data_type, column_comment FROM information_schema.columns "
|
|
128
156
|
f"WHERE table_catalog = {escape_duckdb_literal(config.catalog)} "
|
|
129
157
|
f"AND table_schema = {escape_duckdb_literal(config.dataset_name)} "
|
|
130
158
|
f"AND table_name = {escape_duckdb_literal(table_name)}",
|
|
131
159
|
)
|
|
132
|
-
existing =
|
|
160
|
+
existing = {name: data_type for name, data_type, _ in rows}
|
|
133
161
|
if not existing:
|
|
134
162
|
create_table(config, table, column_types)
|
|
135
163
|
return False
|
|
@@ -150,6 +178,7 @@ def create_or_evolve_table(
|
|
|
150
178
|
f"ALTER COLUMN {escape_postgres_identifier(name)} TYPE DECIMAL(20,0)",
|
|
151
179
|
)
|
|
152
180
|
apply_partitioning(config, table)
|
|
181
|
+
apply_descriptions(config, table, {name: comment for name, _, comment in rows})
|
|
153
182
|
return True
|
|
154
183
|
|
|
155
184
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|