dlt-altertable 0.2.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dlt-altertable
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
5
5
  Keywords: dlt,altertable,lakehouse,parquet
6
6
  Author: Altertable
@@ -13,7 +13,7 @@ Classifier: Topic :: Database
13
13
  Classifier: Programming Language :: Python :: 3.12
14
14
  Classifier: Programming Language :: Python :: 3.13
15
15
  Classifier: Programming Language :: Python :: 3.14
16
- Requires-Dist: dlt[parquet]~=1.30
16
+ Requires-Dist: dlt[parquet]~=1.31
17
17
  Requires-Dist: pyarrow>=15.0.0
18
18
  Requires-Dist: requests>=2.26
19
19
  Requires-Dist: urllib3>=2
@@ -34,7 +34,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
34
34
 
35
35
  ## Install and configure
36
36
 
37
- Requires Python 3.12+ and `dlt >= 1.30, < 2`.
37
+ Requires Python 3.12+ and `dlt >= 1.31, < 2`.
38
38
 
39
39
  ```bash
40
40
  uv add dlt-altertable
@@ -91,7 +91,7 @@ pipeline.run(contacts())
91
91
 
92
92
  Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
93
93
  `lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
94
- `merge_key`, and ascending `dedup_sort` are unsupported.
94
+ `cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
95
95
 
96
96
  New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
97
97
  `altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
@@ -102,6 +102,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
102
102
  omitted fields become null and omitted lists are cleared.
103
103
  Flat merges without `hard_delete` preserve columns absent from incoming files.
104
104
 
105
+ `source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
106
+ Flat merges without `hard_delete` reject these options before writing any tables.
107
+
105
108
  Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
106
109
  ([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
107
110
 
@@ -9,7 +9,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
9
9
 
10
10
  ## Install and configure
11
11
 
12
- Requires Python 3.12+ and `dlt >= 1.30, < 2`.
12
+ Requires Python 3.12+ and `dlt >= 1.31, < 2`.
13
13
 
14
14
  ```bash
15
15
  uv add dlt-altertable
@@ -66,7 +66,7 @@ pipeline.run(contacts())
66
66
 
67
67
  Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
68
68
  `lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
69
- `merge_key`, and ascending `dedup_sort` are unsupported.
69
+ `cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
70
70
 
71
71
  New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
72
72
  `altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
@@ -77,6 +77,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
77
77
  omitted fields become null and omitted lists are cleared.
78
78
  Flat merges without `hard_delete` preserve columns absent from incoming files.
79
79
 
80
+ `source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
81
+ Flat merges without `hard_delete` reject these options before writing any tables.
82
+
80
83
  Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
81
84
  ([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
82
85
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dlt-altertable"
3
- version = "0.2.0"
3
+ version = "0.2.1"
4
4
  description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -21,7 +21,7 @@ classifiers = [
21
21
  ]
22
22
  requires-python = ">=3.12"
23
23
  dependencies = [
24
- "dlt[parquet]~=1.30",
24
+ "dlt[parquet]~=1.31",
25
25
  "pyarrow>=15.0.0",
26
26
  "requests>=2.26",
27
27
  "urllib3>=2",
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dlt-altertable"
3
- version = "0.2.0"
3
+ version = "0.2.1"
4
4
  description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Altertable", email = "hello@altertable.ai" }]
@@ -18,7 +18,7 @@ classifiers = [
18
18
  requires-python = ">=3.12"
19
19
 
20
20
  dependencies = [
21
- "dlt[parquet]~=1.30",
21
+ "dlt[parquet]~=1.31",
22
22
  "pyarrow>=15.0.0",
23
23
  "requests>=2.26",
24
24
  "urllib3>=2",
@@ -28,7 +28,10 @@ from dlt.common.storages.load_storage import ParsedLoadJobFileName
28
28
  from dlt.destinations.impl.destination.destination import DestinationClient
29
29
  from dlt.destinations.job_impl import DestinationParquetLoadJob
30
30
  from dlt.destinations.sql_client import WithSqlClient
31
- from dlt.destinations.utils import verify_schema_merge_disposition
31
+ from dlt.destinations.utils import (
32
+ verify_schema_merge_disposition,
33
+ verify_unsupported_merge_options,
34
+ )
32
35
 
33
36
  from dlt_altertable import api as altertable_api
34
37
  from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
@@ -95,9 +98,19 @@ class AltertableJobClient(DestinationClient, WithStateSync, WithSqlClient):
95
98
  for column in table["columns"].values():
96
99
  sql_type(column)
97
100
  partition_expressions(table)
98
- if validation_errors := verify_schema_merge_disposition(
101
+ upserted_over_http = [
102
+ table for table in tables if not self._requires_staging(cast(str, table["name"]))
103
+ ]
104
+ validation_errors = verify_schema_merge_disposition(
99
105
  self.schema, tables, self.capabilities
100
- ):
106
+ ) + verify_unsupported_merge_options(
107
+ self.schema,
108
+ upserted_over_http,
109
+ self.capabilities,
110
+ "altertable",
111
+ ("source_filter", "skip_unchanged_rows"),
112
+ )
113
+ if validation_errors:
101
114
  raise validation_errors[0]
102
115
  return tables
103
116
 
@@ -41,6 +41,9 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
41
41
  deleted_cond: str | None,
42
42
  insert_only: bool = False,
43
43
  not_deleted_cond: str | None = None,
44
+ changed_cond: str | None = None,
45
+ insert_cond: str | None = None,
46
+ source_filter: str | None = None,
44
47
  ) -> list[str]:
45
48
  statements = super().gen_upsert_merge_sql(
46
49
  root_table_name,
@@ -49,12 +52,20 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
49
52
  root_table_column_names,
50
53
  hard_delete_col=None,
51
54
  deleted_cond=None,
55
+ insert_only=insert_only,
56
+ not_deleted_cond=not_deleted_cond,
57
+ changed_cond=changed_cond,
58
+ insert_cond=insert_cond,
59
+ source_filter=source_filter,
52
60
  )
53
61
  if hard_delete_col:
54
62
  keys = ", ".join(primary_keys)
63
+ staged_deletes = deleted_cond
64
+ if source_filter:
65
+ staged_deletes = f"({deleted_cond}) AND ({source_filter})"
55
66
  statements.append(
56
67
  f"DELETE FROM {root_table_name} WHERE ({keys}) IN "
57
- f"(SELECT {keys} FROM {staging_root_table_name} WHERE {deleted_cond});"
68
+ f"(SELECT {keys} FROM {staging_root_table_name} WHERE {staged_deletes});"
58
69
  )
59
70
  return statements
60
71
 
@@ -106,7 +106,7 @@ def load_context(
106
106
  ) -> tuple[LoadInfo, dict[str, int] | None, AltertableClientConfiguration]:
107
107
  trace = pipeline.last_trace
108
108
  load_info = trace.last_load_info if trace is not None else None
109
- if load_info is None or not load_info.loads_ids:
109
+ if trace is None or load_info is None or not load_info.loads_ids:
110
110
  raise ValueError(
111
111
  "verify_load reads the load step of a pipeline.run() from the pipeline trace, "
112
112
  "and found none."
@@ -142,12 +142,14 @@ def table_contract(
142
142
  key_columns = primary_key_columns(table) if disposition in (MERGE, REPLACE) else []
143
143
  if table.get("parent") and disposition in (MERGE, REPLACE):
144
144
  key_columns = get_columns_names_with_prop(table, "row_key")
145
- upsert_skips_stale_rows = disposition == MERGE and bool(
145
+ upsert_may_skip_rows = disposition == MERGE and bool(
146
146
  get_dedup_sort_tuple(table)
147
147
  or has_column_with_prop(table, "hard_delete")
148
148
  or table.get("parent")
149
+ or table.get("x-merge-skip-unchanged-rows")
150
+ or table.get("x-merge-source-filter")
149
151
  )
150
- return disposition, key_columns, upsert_skips_stale_rows
152
+ return disposition, key_columns, upsert_may_skip_rows
151
153
 
152
154
 
153
155
  def table_load_predicate(
@@ -207,7 +209,7 @@ def count_problems(
207
209
  key_columns: list[str],
208
210
  normalized_rows: int | None,
209
211
  lakehouse_counts: list[Any],
210
- upsert_skips_stale_rows: bool,
212
+ upsert_may_skip_rows: bool,
211
213
  ) -> list[str]:
212
214
  row_count = lakehouse_counts[0]
213
215
  problems: list[str] = []
@@ -220,7 +222,7 @@ def count_problems(
220
222
  if normalized_rows is None:
221
223
  return problems
222
224
  nothing_landed_under_this_load_id = normalized_rows > 0 and row_count == 0
223
- if disposition == MERGE and nothing_landed_under_this_load_id and not upsert_skips_stale_rows:
225
+ if disposition == MERGE and nothing_landed_under_this_load_id and not upsert_may_skip_rows:
224
226
  problems.append(
225
227
  f"{table_name}: dlt normalized {normalized_rows} rows, none landed with this load id"
226
228
  )
@@ -245,7 +247,7 @@ def verify_table(
245
247
  if table_name not in landed_table_names:
246
248
  return [f"{table_name}: the load includes this table, but it does not exist"]
247
249
 
248
- disposition, key_columns, upsert_skips_stale_rows = contract
250
+ disposition, key_columns, upsert_may_skip_rows = contract
249
251
  if disposition != REPLACE and load_id_predicate is None:
250
252
  return [
251
253
  f"{table_name}: this table has no _dlt_load_id column, so this load cannot be "
@@ -271,7 +273,7 @@ def verify_table(
271
273
  key_columns,
272
274
  normalized_rows,
273
275
  lakehouse_counts,
274
- upsert_skips_stale_rows,
276
+ upsert_may_skip_rows,
275
277
  )
276
278
  )
277
279
  return problems
File without changes