dlt-altertable 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/PKG-INFO +13 -4
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/README.md +11 -2
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/pyproject.toml +2 -2
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/pyproject.toml.orig +2 -2
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/destination.py +2 -1
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/job_client.py +16 -3
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/merge.py +12 -1
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/table_schema.py +31 -3
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/verify.py +9 -7
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/LICENSE +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/__init__.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/altertable_adapter.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/api.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/configuration.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/exceptions.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/layout.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/py.typed +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.2}/src/dlt_altertable/sql_client.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dlt-altertable
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
|
|
5
5
|
Keywords: dlt,altertable,lakehouse,parquet
|
|
6
6
|
Author: Altertable
|
|
@@ -13,7 +13,7 @@ Classifier: Topic :: Database
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
-
Requires-Dist: dlt[parquet]~=1.
|
|
16
|
+
Requires-Dist: dlt[parquet]~=1.31
|
|
17
17
|
Requires-Dist: pyarrow>=15.0.0
|
|
18
18
|
Requires-Dist: requests>=2.26
|
|
19
19
|
Requires-Dist: urllib3>=2
|
|
@@ -34,7 +34,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
|
|
|
34
34
|
|
|
35
35
|
## Install and configure
|
|
36
36
|
|
|
37
|
-
Requires Python 3.12+ and `dlt >= 1.
|
|
37
|
+
Requires Python 3.12+ and `dlt >= 1.31, < 2`.
|
|
38
38
|
|
|
39
39
|
```bash
|
|
40
40
|
uv add dlt-altertable
|
|
@@ -91,7 +91,7 @@ pipeline.run(contacts())
|
|
|
91
91
|
|
|
92
92
|
Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
|
|
93
93
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
94
|
-
`merge_key`, and ascending `dedup_sort` are unsupported.
|
|
94
|
+
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
95
95
|
|
|
96
96
|
New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
97
97
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
@@ -102,6 +102,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
|
|
|
102
102
|
omitted fields become null and omitted lists are cleared.
|
|
103
103
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
104
104
|
|
|
105
|
+
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
106
|
+
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
107
|
+
|
|
105
108
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
106
109
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
107
110
|
|
|
@@ -122,6 +125,12 @@ pipeline.run(resource)
|
|
|
122
125
|
|
|
123
126
|
Pass `partition=[]` to reset. Sort hints are not supported yet.
|
|
124
127
|
|
|
128
|
+
## Descriptions
|
|
129
|
+
|
|
130
|
+
Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
|
|
131
|
+
become comments in the Altertable catalog. Column comments track changes. Table comments are
|
|
132
|
+
set on create or replace.
|
|
133
|
+
|
|
125
134
|
## Performance
|
|
126
135
|
|
|
127
136
|
Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
|
|
@@ -9,7 +9,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
|
|
|
9
9
|
|
|
10
10
|
## Install and configure
|
|
11
11
|
|
|
12
|
-
Requires Python 3.12+ and `dlt >= 1.
|
|
12
|
+
Requires Python 3.12+ and `dlt >= 1.31, < 2`.
|
|
13
13
|
|
|
14
14
|
```bash
|
|
15
15
|
uv add dlt-altertable
|
|
@@ -66,7 +66,7 @@ pipeline.run(contacts())
|
|
|
66
66
|
|
|
67
67
|
Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
|
|
68
68
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
69
|
-
`merge_key`, and ascending `dedup_sort` are unsupported.
|
|
69
|
+
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
70
70
|
|
|
71
71
|
New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
72
72
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
@@ -77,6 +77,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
|
|
|
77
77
|
omitted fields become null and omitted lists are cleared.
|
|
78
78
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
79
79
|
|
|
80
|
+
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
81
|
+
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
82
|
+
|
|
80
83
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
81
84
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
82
85
|
|
|
@@ -97,6 +100,12 @@ pipeline.run(resource)
|
|
|
97
100
|
|
|
98
101
|
Pass `partition=[]` to reset. Sort hints are not supported yet.
|
|
99
102
|
|
|
103
|
+
## Descriptions
|
|
104
|
+
|
|
105
|
+
Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
|
|
106
|
+
become comments in the Altertable catalog. Column comments track changes. Table comments are
|
|
107
|
+
set on create or replace.
|
|
108
|
+
|
|
100
109
|
## Performance
|
|
101
110
|
|
|
102
111
|
Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dlt-altertable"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -21,7 +21,7 @@ classifiers = [
|
|
|
21
21
|
]
|
|
22
22
|
requires-python = ">=3.12"
|
|
23
23
|
dependencies = [
|
|
24
|
-
"dlt[parquet]~=1.
|
|
24
|
+
"dlt[parquet]~=1.31",
|
|
25
25
|
"pyarrow>=15.0.0",
|
|
26
26
|
"requests>=2.26",
|
|
27
27
|
"urllib3>=2",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dlt-altertable"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Altertable", email = "hello@altertable.ai" }]
|
|
@@ -18,7 +18,7 @@ classifiers = [
|
|
|
18
18
|
requires-python = ">=3.12"
|
|
19
19
|
|
|
20
20
|
dependencies = [
|
|
21
|
-
"dlt[parquet]~=1.
|
|
21
|
+
"dlt[parquet]~=1.31",
|
|
22
22
|
"pyarrow>=15.0.0",
|
|
23
23
|
"requests>=2.26",
|
|
24
24
|
"urllib3>=2",
|
|
@@ -18,7 +18,7 @@ from dlt.destinations.impl.destination.factory import destination as CustomDesti
|
|
|
18
18
|
|
|
19
19
|
from dlt_altertable import api as altertable_api
|
|
20
20
|
from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
|
|
21
|
-
from dlt_altertable.table_schema import aligned_parquet, create_or_evolve_table
|
|
21
|
+
from dlt_altertable.table_schema import aligned_parquet, apply_descriptions, create_or_evolve_table
|
|
22
22
|
|
|
23
23
|
if TYPE_CHECKING:
|
|
24
24
|
from dlt_altertable.job_client import AltertableJobClient
|
|
@@ -133,6 +133,7 @@ def _upload(
|
|
|
133
133
|
altertable_api.post_parquet(config, "upload", params, upload_path, action)
|
|
134
134
|
|
|
135
135
|
if ingest_mode == "overwrite":
|
|
136
|
+
apply_descriptions(config, table)
|
|
136
137
|
already_replaced.append(table_name)
|
|
137
138
|
|
|
138
139
|
|
|
@@ -28,7 +28,10 @@ from dlt.common.storages.load_storage import ParsedLoadJobFileName
|
|
|
28
28
|
from dlt.destinations.impl.destination.destination import DestinationClient
|
|
29
29
|
from dlt.destinations.job_impl import DestinationParquetLoadJob
|
|
30
30
|
from dlt.destinations.sql_client import WithSqlClient
|
|
31
|
-
from dlt.destinations.utils import
|
|
31
|
+
from dlt.destinations.utils import (
|
|
32
|
+
verify_schema_merge_disposition,
|
|
33
|
+
verify_unsupported_merge_options,
|
|
34
|
+
)
|
|
32
35
|
|
|
33
36
|
from dlt_altertable import api as altertable_api
|
|
34
37
|
from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
|
|
@@ -95,9 +98,19 @@ class AltertableJobClient(DestinationClient, WithStateSync, WithSqlClient):
|
|
|
95
98
|
for column in table["columns"].values():
|
|
96
99
|
sql_type(column)
|
|
97
100
|
partition_expressions(table)
|
|
98
|
-
|
|
101
|
+
upserted_over_http = [
|
|
102
|
+
table for table in tables if not self._requires_staging(cast(str, table["name"]))
|
|
103
|
+
]
|
|
104
|
+
validation_errors = verify_schema_merge_disposition(
|
|
99
105
|
self.schema, tables, self.capabilities
|
|
100
|
-
)
|
|
106
|
+
) + verify_unsupported_merge_options(
|
|
107
|
+
self.schema,
|
|
108
|
+
upserted_over_http,
|
|
109
|
+
self.capabilities,
|
|
110
|
+
"altertable",
|
|
111
|
+
("source_filter", "skip_unchanged_rows"),
|
|
112
|
+
)
|
|
113
|
+
if validation_errors:
|
|
101
114
|
raise validation_errors[0]
|
|
102
115
|
return tables
|
|
103
116
|
|
|
@@ -41,6 +41,9 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
|
|
|
41
41
|
deleted_cond: str | None,
|
|
42
42
|
insert_only: bool = False,
|
|
43
43
|
not_deleted_cond: str | None = None,
|
|
44
|
+
changed_cond: str | None = None,
|
|
45
|
+
insert_cond: str | None = None,
|
|
46
|
+
source_filter: str | None = None,
|
|
44
47
|
) -> list[str]:
|
|
45
48
|
statements = super().gen_upsert_merge_sql(
|
|
46
49
|
root_table_name,
|
|
@@ -49,12 +52,20 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
|
|
|
49
52
|
root_table_column_names,
|
|
50
53
|
hard_delete_col=None,
|
|
51
54
|
deleted_cond=None,
|
|
55
|
+
insert_only=insert_only,
|
|
56
|
+
not_deleted_cond=not_deleted_cond,
|
|
57
|
+
changed_cond=changed_cond,
|
|
58
|
+
insert_cond=insert_cond,
|
|
59
|
+
source_filter=source_filter,
|
|
52
60
|
)
|
|
53
61
|
if hard_delete_col:
|
|
54
62
|
keys = ", ".join(primary_keys)
|
|
63
|
+
staged_deletes = deleted_cond
|
|
64
|
+
if source_filter:
|
|
65
|
+
staged_deletes = f"({deleted_cond}) AND ({source_filter})"
|
|
55
66
|
statements.append(
|
|
56
67
|
f"DELETE FROM {root_table_name} WHERE ({keys}) IN "
|
|
57
|
-
f"(SELECT {keys} FROM {staging_root_table_name} WHERE {
|
|
68
|
+
f"(SELECT {keys} FROM {staging_root_table_name} WHERE {staged_deletes});"
|
|
58
69
|
)
|
|
59
70
|
return statements
|
|
60
71
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import tempfile
|
|
3
|
-
from collections.abc import Iterator
|
|
3
|
+
from collections.abc import Iterator, Mapping
|
|
4
4
|
from contextlib import contextmanager
|
|
5
5
|
from typing import cast
|
|
6
6
|
|
|
@@ -79,6 +79,32 @@ def apply_partitioning(
|
|
|
79
79
|
)
|
|
80
80
|
|
|
81
81
|
|
|
82
|
+
def apply_descriptions(
|
|
83
|
+
config: AltertableClientConfiguration,
|
|
84
|
+
table: TTableSchema | PreparedTableSchema,
|
|
85
|
+
column_comments: Mapping[str, str | None] | None = None,
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Publish dlt descriptions as catalog comments, so people and agents see them.
|
|
88
|
+
Only changed column comments are rewritten. The table comment is set on new tables."""
|
|
89
|
+
table_name = qualified_table_name(config, cast(str, table["name"]))
|
|
90
|
+
targets = {
|
|
91
|
+
f"COLUMN {table_name}.{escape_postgres_identifier(name)}": column.get("description")
|
|
92
|
+
for name, column in table["columns"].items()
|
|
93
|
+
if column.get("description") != (column_comments or {}).get(name)
|
|
94
|
+
}
|
|
95
|
+
if column_comments is None:
|
|
96
|
+
targets[f"TABLE {table_name}"] = table.get("description")
|
|
97
|
+
statements = [
|
|
98
|
+
f"COMMENT ON {target} IS {escape_duckdb_literal(description)};"
|
|
99
|
+
for target, description in targets.items()
|
|
100
|
+
if description
|
|
101
|
+
]
|
|
102
|
+
if statements:
|
|
103
|
+
altertable_api.execute_sql(
|
|
104
|
+
config, "\n".join(["BEGIN TRANSACTION;", *statements, "COMMIT;"])
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
82
108
|
def create_table(
|
|
83
109
|
config: AltertableClientConfiguration,
|
|
84
110
|
table: TTableSchema | PreparedTableSchema,
|
|
@@ -98,6 +124,7 @@ def create_table(
|
|
|
98
124
|
f"CREATE TABLE IF NOT EXISTS {qualified_table_name(config, table_name)} ({columns})",
|
|
99
125
|
)
|
|
100
126
|
apply_partitioning(config, table)
|
|
127
|
+
apply_descriptions(config, table)
|
|
101
128
|
logger.info(
|
|
102
129
|
f"Created table {config.catalog}.{config.dataset_name}.{table_name} "
|
|
103
130
|
f"with {len(table['columns'])} columns"
|
|
@@ -124,12 +151,12 @@ def create_or_evolve_table(
|
|
|
124
151
|
table_name = cast(str, table["name"])
|
|
125
152
|
rows = altertable_api.execute_sql(
|
|
126
153
|
config,
|
|
127
|
-
"SELECT column_name, data_type FROM information_schema.columns "
|
|
154
|
+
"SELECT column_name, data_type, column_comment FROM information_schema.columns "
|
|
128
155
|
f"WHERE table_catalog = {escape_duckdb_literal(config.catalog)} "
|
|
129
156
|
f"AND table_schema = {escape_duckdb_literal(config.dataset_name)} "
|
|
130
157
|
f"AND table_name = {escape_duckdb_literal(table_name)}",
|
|
131
158
|
)
|
|
132
|
-
existing =
|
|
159
|
+
existing = {name: data_type for name, data_type, _ in rows}
|
|
133
160
|
if not existing:
|
|
134
161
|
create_table(config, table, column_types)
|
|
135
162
|
return False
|
|
@@ -150,6 +177,7 @@ def create_or_evolve_table(
|
|
|
150
177
|
f"ALTER COLUMN {escape_postgres_identifier(name)} TYPE DECIMAL(20,0)",
|
|
151
178
|
)
|
|
152
179
|
apply_partitioning(config, table)
|
|
180
|
+
apply_descriptions(config, table, {name: comment for name, _, comment in rows})
|
|
153
181
|
return True
|
|
154
182
|
|
|
155
183
|
|
|
@@ -106,7 +106,7 @@ def load_context(
|
|
|
106
106
|
) -> tuple[LoadInfo, dict[str, int] | None, AltertableClientConfiguration]:
|
|
107
107
|
trace = pipeline.last_trace
|
|
108
108
|
load_info = trace.last_load_info if trace is not None else None
|
|
109
|
-
if load_info is None or not load_info.loads_ids:
|
|
109
|
+
if trace is None or load_info is None or not load_info.loads_ids:
|
|
110
110
|
raise ValueError(
|
|
111
111
|
"verify_load reads the load step of a pipeline.run() from the pipeline trace, "
|
|
112
112
|
"and found none."
|
|
@@ -142,12 +142,14 @@ def table_contract(
|
|
|
142
142
|
key_columns = primary_key_columns(table) if disposition in (MERGE, REPLACE) else []
|
|
143
143
|
if table.get("parent") and disposition in (MERGE, REPLACE):
|
|
144
144
|
key_columns = get_columns_names_with_prop(table, "row_key")
|
|
145
|
-
|
|
145
|
+
upsert_may_skip_rows = disposition == MERGE and bool(
|
|
146
146
|
get_dedup_sort_tuple(table)
|
|
147
147
|
or has_column_with_prop(table, "hard_delete")
|
|
148
148
|
or table.get("parent")
|
|
149
|
+
or table.get("x-merge-skip-unchanged-rows")
|
|
150
|
+
or table.get("x-merge-source-filter")
|
|
149
151
|
)
|
|
150
|
-
return disposition, key_columns,
|
|
152
|
+
return disposition, key_columns, upsert_may_skip_rows
|
|
151
153
|
|
|
152
154
|
|
|
153
155
|
def table_load_predicate(
|
|
@@ -207,7 +209,7 @@ def count_problems(
|
|
|
207
209
|
key_columns: list[str],
|
|
208
210
|
normalized_rows: int | None,
|
|
209
211
|
lakehouse_counts: list[Any],
|
|
210
|
-
|
|
212
|
+
upsert_may_skip_rows: bool,
|
|
211
213
|
) -> list[str]:
|
|
212
214
|
row_count = lakehouse_counts[0]
|
|
213
215
|
problems: list[str] = []
|
|
@@ -220,7 +222,7 @@ def count_problems(
|
|
|
220
222
|
if normalized_rows is None:
|
|
221
223
|
return problems
|
|
222
224
|
nothing_landed_under_this_load_id = normalized_rows > 0 and row_count == 0
|
|
223
|
-
if disposition == MERGE and nothing_landed_under_this_load_id and not
|
|
225
|
+
if disposition == MERGE and nothing_landed_under_this_load_id and not upsert_may_skip_rows:
|
|
224
226
|
problems.append(
|
|
225
227
|
f"{table_name}: dlt normalized {normalized_rows} rows, none landed with this load id"
|
|
226
228
|
)
|
|
@@ -245,7 +247,7 @@ def verify_table(
|
|
|
245
247
|
if table_name not in landed_table_names:
|
|
246
248
|
return [f"{table_name}: the load includes this table, but it does not exist"]
|
|
247
249
|
|
|
248
|
-
disposition, key_columns,
|
|
250
|
+
disposition, key_columns, upsert_may_skip_rows = contract
|
|
249
251
|
if disposition != REPLACE and load_id_predicate is None:
|
|
250
252
|
return [
|
|
251
253
|
f"{table_name}: this table has no _dlt_load_id column, so this load cannot be "
|
|
@@ -271,7 +273,7 @@ def verify_table(
|
|
|
271
273
|
key_columns,
|
|
272
274
|
normalized_rows,
|
|
273
275
|
lakehouse_counts,
|
|
274
|
-
|
|
276
|
+
upsert_may_skip_rows,
|
|
275
277
|
)
|
|
276
278
|
)
|
|
277
279
|
return problems
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|