dlt-altertable 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/PKG-INFO +7 -4
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/README.md +5 -2
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/pyproject.toml +2 -2
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/pyproject.toml.orig +2 -2
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/job_client.py +16 -3
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/merge.py +12 -1
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/verify.py +9 -7
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/LICENSE +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/__init__.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/altertable_adapter.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/api.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/configuration.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/destination.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/exceptions.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/layout.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/py.typed +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/sql_client.py +0 -0
- {dlt_altertable-0.2.0 → dlt_altertable-0.2.1}/src/dlt_altertable/table_schema.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dlt-altertable
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
|
|
5
5
|
Keywords: dlt,altertable,lakehouse,parquet
|
|
6
6
|
Author: Altertable
|
|
@@ -13,7 +13,7 @@ Classifier: Topic :: Database
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
-
Requires-Dist: dlt[parquet]~=1.
|
|
16
|
+
Requires-Dist: dlt[parquet]~=1.31
|
|
17
17
|
Requires-Dist: pyarrow>=15.0.0
|
|
18
18
|
Requires-Dist: requests>=2.26
|
|
19
19
|
Requires-Dist: urllib3>=2
|
|
@@ -34,7 +34,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
|
|
|
34
34
|
|
|
35
35
|
## Install and configure
|
|
36
36
|
|
|
37
|
-
Requires Python 3.12+ and `dlt >= 1.
|
|
37
|
+
Requires Python 3.12+ and `dlt >= 1.31, < 2`.
|
|
38
38
|
|
|
39
39
|
```bash
|
|
40
40
|
uv add dlt-altertable
|
|
@@ -91,7 +91,7 @@ pipeline.run(contacts())
|
|
|
91
91
|
|
|
92
92
|
Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
|
|
93
93
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
94
|
-
`merge_key`, and ascending `dedup_sort` are unsupported.
|
|
94
|
+
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
95
95
|
|
|
96
96
|
New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
97
97
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
@@ -102,6 +102,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
|
|
|
102
102
|
omitted fields become null and omitted lists are cleared.
|
|
103
103
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
104
104
|
|
|
105
|
+
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
106
|
+
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
107
|
+
|
|
105
108
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
106
109
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
107
110
|
|
|
@@ -9,7 +9,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
|
|
|
9
9
|
|
|
10
10
|
## Install and configure
|
|
11
11
|
|
|
12
|
-
Requires Python 3.12+ and `dlt >= 1.
|
|
12
|
+
Requires Python 3.12+ and `dlt >= 1.31, < 2`.
|
|
13
13
|
|
|
14
14
|
```bash
|
|
15
15
|
uv add dlt-altertable
|
|
@@ -66,7 +66,7 @@ pipeline.run(contacts())
|
|
|
66
66
|
|
|
67
67
|
Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
|
|
68
68
|
`lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
|
|
69
|
-
`merge_key`, and ascending `dedup_sort` are unsupported.
|
|
69
|
+
`cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
|
|
70
70
|
|
|
71
71
|
New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
|
|
72
72
|
`altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
|
|
@@ -77,6 +77,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
|
|
|
77
77
|
omitted fields become null and omitted lists are cleared.
|
|
78
78
|
Flat merges without `hard_delete` preserve columns absent from incoming files.
|
|
79
79
|
|
|
80
|
+
`source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
|
|
81
|
+
Flat merges without `hard_delete` reject these options before writing any tables.
|
|
82
|
+
|
|
80
83
|
Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
|
|
81
84
|
([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
|
|
82
85
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dlt-altertable"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -21,7 +21,7 @@ classifiers = [
|
|
|
21
21
|
]
|
|
22
22
|
requires-python = ">=3.12"
|
|
23
23
|
dependencies = [
|
|
24
|
-
"dlt[parquet]~=1.
|
|
24
|
+
"dlt[parquet]~=1.31",
|
|
25
25
|
"pyarrow>=15.0.0",
|
|
26
26
|
"requests>=2.26",
|
|
27
27
|
"urllib3>=2",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dlt-altertable"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Altertable", email = "hello@altertable.ai" }]
|
|
@@ -18,7 +18,7 @@ classifiers = [
|
|
|
18
18
|
requires-python = ">=3.12"
|
|
19
19
|
|
|
20
20
|
dependencies = [
|
|
21
|
-
"dlt[parquet]~=1.
|
|
21
|
+
"dlt[parquet]~=1.31",
|
|
22
22
|
"pyarrow>=15.0.0",
|
|
23
23
|
"requests>=2.26",
|
|
24
24
|
"urllib3>=2",
|
|
@@ -28,7 +28,10 @@ from dlt.common.storages.load_storage import ParsedLoadJobFileName
|
|
|
28
28
|
from dlt.destinations.impl.destination.destination import DestinationClient
|
|
29
29
|
from dlt.destinations.job_impl import DestinationParquetLoadJob
|
|
30
30
|
from dlt.destinations.sql_client import WithSqlClient
|
|
31
|
-
from dlt.destinations.utils import
|
|
31
|
+
from dlt.destinations.utils import (
|
|
32
|
+
verify_schema_merge_disposition,
|
|
33
|
+
verify_unsupported_merge_options,
|
|
34
|
+
)
|
|
32
35
|
|
|
33
36
|
from dlt_altertable import api as altertable_api
|
|
34
37
|
from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
|
|
@@ -95,9 +98,19 @@ class AltertableJobClient(DestinationClient, WithStateSync, WithSqlClient):
|
|
|
95
98
|
for column in table["columns"].values():
|
|
96
99
|
sql_type(column)
|
|
97
100
|
partition_expressions(table)
|
|
98
|
-
|
|
101
|
+
upserted_over_http = [
|
|
102
|
+
table for table in tables if not self._requires_staging(cast(str, table["name"]))
|
|
103
|
+
]
|
|
104
|
+
validation_errors = verify_schema_merge_disposition(
|
|
99
105
|
self.schema, tables, self.capabilities
|
|
100
|
-
)
|
|
106
|
+
) + verify_unsupported_merge_options(
|
|
107
|
+
self.schema,
|
|
108
|
+
upserted_over_http,
|
|
109
|
+
self.capabilities,
|
|
110
|
+
"altertable",
|
|
111
|
+
("source_filter", "skip_unchanged_rows"),
|
|
112
|
+
)
|
|
113
|
+
if validation_errors:
|
|
101
114
|
raise validation_errors[0]
|
|
102
115
|
return tables
|
|
103
116
|
|
|
@@ -41,6 +41,9 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
|
|
|
41
41
|
deleted_cond: str | None,
|
|
42
42
|
insert_only: bool = False,
|
|
43
43
|
not_deleted_cond: str | None = None,
|
|
44
|
+
changed_cond: str | None = None,
|
|
45
|
+
insert_cond: str | None = None,
|
|
46
|
+
source_filter: str | None = None,
|
|
44
47
|
) -> list[str]:
|
|
45
48
|
statements = super().gen_upsert_merge_sql(
|
|
46
49
|
root_table_name,
|
|
@@ -49,12 +52,20 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
|
|
|
49
52
|
root_table_column_names,
|
|
50
53
|
hard_delete_col=None,
|
|
51
54
|
deleted_cond=None,
|
|
55
|
+
insert_only=insert_only,
|
|
56
|
+
not_deleted_cond=not_deleted_cond,
|
|
57
|
+
changed_cond=changed_cond,
|
|
58
|
+
insert_cond=insert_cond,
|
|
59
|
+
source_filter=source_filter,
|
|
52
60
|
)
|
|
53
61
|
if hard_delete_col:
|
|
54
62
|
keys = ", ".join(primary_keys)
|
|
63
|
+
staged_deletes = deleted_cond
|
|
64
|
+
if source_filter:
|
|
65
|
+
staged_deletes = f"({deleted_cond}) AND ({source_filter})"
|
|
55
66
|
statements.append(
|
|
56
67
|
f"DELETE FROM {root_table_name} WHERE ({keys}) IN "
|
|
57
|
-
f"(SELECT {keys} FROM {staging_root_table_name} WHERE {
|
|
68
|
+
f"(SELECT {keys} FROM {staging_root_table_name} WHERE {staged_deletes});"
|
|
58
69
|
)
|
|
59
70
|
return statements
|
|
60
71
|
|
|
@@ -106,7 +106,7 @@ def load_context(
|
|
|
106
106
|
) -> tuple[LoadInfo, dict[str, int] | None, AltertableClientConfiguration]:
|
|
107
107
|
trace = pipeline.last_trace
|
|
108
108
|
load_info = trace.last_load_info if trace is not None else None
|
|
109
|
-
if load_info is None or not load_info.loads_ids:
|
|
109
|
+
if trace is None or load_info is None or not load_info.loads_ids:
|
|
110
110
|
raise ValueError(
|
|
111
111
|
"verify_load reads the load step of a pipeline.run() from the pipeline trace, "
|
|
112
112
|
"and found none."
|
|
@@ -142,12 +142,14 @@ def table_contract(
|
|
|
142
142
|
key_columns = primary_key_columns(table) if disposition in (MERGE, REPLACE) else []
|
|
143
143
|
if table.get("parent") and disposition in (MERGE, REPLACE):
|
|
144
144
|
key_columns = get_columns_names_with_prop(table, "row_key")
|
|
145
|
-
|
|
145
|
+
upsert_may_skip_rows = disposition == MERGE and bool(
|
|
146
146
|
get_dedup_sort_tuple(table)
|
|
147
147
|
or has_column_with_prop(table, "hard_delete")
|
|
148
148
|
or table.get("parent")
|
|
149
|
+
or table.get("x-merge-skip-unchanged-rows")
|
|
150
|
+
or table.get("x-merge-source-filter")
|
|
149
151
|
)
|
|
150
|
-
return disposition, key_columns,
|
|
152
|
+
return disposition, key_columns, upsert_may_skip_rows
|
|
151
153
|
|
|
152
154
|
|
|
153
155
|
def table_load_predicate(
|
|
@@ -207,7 +209,7 @@ def count_problems(
|
|
|
207
209
|
key_columns: list[str],
|
|
208
210
|
normalized_rows: int | None,
|
|
209
211
|
lakehouse_counts: list[Any],
|
|
210
|
-
|
|
212
|
+
upsert_may_skip_rows: bool,
|
|
211
213
|
) -> list[str]:
|
|
212
214
|
row_count = lakehouse_counts[0]
|
|
213
215
|
problems: list[str] = []
|
|
@@ -220,7 +222,7 @@ def count_problems(
|
|
|
220
222
|
if normalized_rows is None:
|
|
221
223
|
return problems
|
|
222
224
|
nothing_landed_under_this_load_id = normalized_rows > 0 and row_count == 0
|
|
223
|
-
if disposition == MERGE and nothing_landed_under_this_load_id and not
|
|
225
|
+
if disposition == MERGE and nothing_landed_under_this_load_id and not upsert_may_skip_rows:
|
|
224
226
|
problems.append(
|
|
225
227
|
f"{table_name}: dlt normalized {normalized_rows} rows, none landed with this load id"
|
|
226
228
|
)
|
|
@@ -245,7 +247,7 @@ def verify_table(
|
|
|
245
247
|
if table_name not in landed_table_names:
|
|
246
248
|
return [f"{table_name}: the load includes this table, but it does not exist"]
|
|
247
249
|
|
|
248
|
-
disposition, key_columns,
|
|
250
|
+
disposition, key_columns, upsert_may_skip_rows = contract
|
|
249
251
|
if disposition != REPLACE and load_id_predicate is None:
|
|
250
252
|
return [
|
|
251
253
|
f"{table_name}: this table has no _dlt_load_id column, so this load cannot be "
|
|
@@ -271,7 +273,7 @@ def verify_table(
|
|
|
271
273
|
key_columns,
|
|
272
274
|
normalized_rows,
|
|
273
275
|
lakehouse_counts,
|
|
274
|
-
|
|
276
|
+
upsert_may_skip_rows,
|
|
275
277
|
)
|
|
276
278
|
)
|
|
277
279
|
return problems
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|