dlt-altertable 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dlt-altertable
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
5
5
  Keywords: dlt,altertable,lakehouse,parquet
6
6
  Author: Altertable
@@ -13,7 +13,7 @@ Classifier: Topic :: Database
13
13
  Classifier: Programming Language :: Python :: 3.12
14
14
  Classifier: Programming Language :: Python :: 3.13
15
15
  Classifier: Programming Language :: Python :: 3.14
16
- Requires-Dist: dlt[parquet]~=1.30
16
+ Requires-Dist: dlt[parquet]~=1.31
17
17
  Requires-Dist: pyarrow>=15.0.0
18
18
  Requires-Dist: requests>=2.26
19
19
  Requires-Dist: urllib3>=2
@@ -34,7 +34,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
34
34
 
35
35
  ## Install and configure
36
36
 
37
- Requires Python 3.12+ and `dlt >= 1.30, < 2`.
37
+ Requires Python 3.12+ and `dlt >= 1.31, < 2`.
38
38
 
39
39
  ```bash
40
40
  uv add dlt-altertable
@@ -91,7 +91,7 @@ pipeline.run(contacts())
91
91
 
92
92
  Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
93
93
  `lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
94
- `merge_key`, and ascending `dedup_sort` are unsupported.
94
+ `cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
95
95
 
96
96
  New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
97
97
  `altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
@@ -102,6 +102,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
102
102
  omitted fields become null and omitted lists are cleared.
103
103
  Flat merges without `hard_delete` preserve columns absent from incoming files.
104
104
 
105
+ `source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
106
+ Flat merges without `hard_delete` reject these options before writing any tables.
107
+
105
108
  Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
106
109
  ([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
107
110
 
@@ -122,6 +125,12 @@ pipeline.run(resource)
122
125
 
123
126
  Pass `partition=[]` to reset. Sort hints are not supported yet.
124
127
 
128
+ ## Descriptions
129
+
130
+ Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
131
+ become comments in the Altertable catalog. Column comments track changes. Table comments are
132
+ set on create or replace.
133
+
125
134
  ## Performance
126
135
 
127
136
  Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
@@ -9,7 +9,7 @@ Load data into [Altertable](https://altertable.ai) with [dlt](https://dlthub.com
9
9
 
10
10
  ## Install and configure
11
11
 
12
- Requires Python 3.12+ and `dlt >= 1.30, < 2`.
12
+ Requires Python 3.12+ and `dlt >= 1.31, < 2`.
13
13
 
14
14
  ```bash
15
15
  uv add dlt-altertable
@@ -66,7 +66,7 @@ pipeline.run(contacts())
66
66
 
67
67
  Merge requires a non-nullable primary key. In the example, `dedup_sort` keeps the highest
68
68
  `lastmodifieddate` per `id`, comparing incoming and stored rows. `delete-insert`, `insert-only`, `scd2`,
69
- `merge_key`, and ascending `dedup_sort` are unsupported.
69
+ `cdc`, `merge_key`, and ascending `dedup_sort` are unsupported.
70
70
 
71
71
  New columns are added automatically. Nested data defaults to JSON strings (`max_table_nesting=0`). Set
72
72
  `altertable(naming_convention="snake_case", max_table_nesting=None)` to flatten objects into columns
@@ -77,6 +77,9 @@ With child tables or `hard_delete`, send complete records when inserting or upda
77
77
  omitted fields become null and omitted lists are cleared.
78
78
  Flat merges without `hard_delete` preserve columns absent from incoming files.
79
79
 
80
+ `source_filter` and `skip_unchanged_rows` require child tables or a `hard_delete` column hint.
81
+ Flat merges without `hard_delete` reject these options before writing any tables.
82
+
80
83
  Use `columns={"deleted": {"hard_delete": True}}` to delete records and their children
81
84
  ([dlt deletion rules](https://dlthub.com/docs/general-usage/merge-loading#delete-records)).
82
85
 
@@ -97,6 +100,12 @@ pipeline.run(resource)
97
100
 
98
101
  Pass `partition=[]` to reset. Sort hints are not supported yet.
99
102
 
103
+ ## Descriptions
104
+
105
+ Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
106
+ become comments in the Altertable catalog. Column comments track changes. Table comments are
107
+ set on create or replace.
108
+
100
109
  ## Performance
101
110
 
102
111
  Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dlt-altertable"
3
- version = "0.2.0"
3
+ version = "0.2.2"
4
4
  description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -21,7 +21,7 @@ classifiers = [
21
21
  ]
22
22
  requires-python = ">=3.12"
23
23
  dependencies = [
24
- "dlt[parquet]~=1.30",
24
+ "dlt[parquet]~=1.31",
25
25
  "pyarrow>=15.0.0",
26
26
  "requests>=2.26",
27
27
  "urllib3>=2",
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dlt-altertable"
3
- version = "0.2.0"
3
+ version = "0.2.2"
4
4
  description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Altertable", email = "hello@altertable.ai" }]
@@ -18,7 +18,7 @@ classifiers = [
18
18
  requires-python = ">=3.12"
19
19
 
20
20
  dependencies = [
21
- "dlt[parquet]~=1.30",
21
+ "dlt[parquet]~=1.31",
22
22
  "pyarrow>=15.0.0",
23
23
  "requests>=2.26",
24
24
  "urllib3>=2",
@@ -18,7 +18,7 @@ from dlt.destinations.impl.destination.factory import destination as CustomDesti
18
18
 
19
19
  from dlt_altertable import api as altertable_api
20
20
  from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
21
- from dlt_altertable.table_schema import aligned_parquet, create_or_evolve_table
21
+ from dlt_altertable.table_schema import aligned_parquet, apply_descriptions, create_or_evolve_table
22
22
 
23
23
  if TYPE_CHECKING:
24
24
  from dlt_altertable.job_client import AltertableJobClient
@@ -133,6 +133,7 @@ def _upload(
133
133
  altertable_api.post_parquet(config, "upload", params, upload_path, action)
134
134
 
135
135
  if ingest_mode == "overwrite":
136
+ apply_descriptions(config, table)
136
137
  already_replaced.append(table_name)
137
138
 
138
139
 
@@ -28,7 +28,10 @@ from dlt.common.storages.load_storage import ParsedLoadJobFileName
28
28
  from dlt.destinations.impl.destination.destination import DestinationClient
29
29
  from dlt.destinations.job_impl import DestinationParquetLoadJob
30
30
  from dlt.destinations.sql_client import WithSqlClient
31
- from dlt.destinations.utils import verify_schema_merge_disposition
31
+ from dlt.destinations.utils import (
32
+ verify_schema_merge_disposition,
33
+ verify_unsupported_merge_options,
34
+ )
32
35
 
33
36
  from dlt_altertable import api as altertable_api
34
37
  from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
@@ -95,9 +98,19 @@ class AltertableJobClient(DestinationClient, WithStateSync, WithSqlClient):
95
98
  for column in table["columns"].values():
96
99
  sql_type(column)
97
100
  partition_expressions(table)
98
- if validation_errors := verify_schema_merge_disposition(
101
+ upserted_over_http = [
102
+ table for table in tables if not self._requires_staging(cast(str, table["name"]))
103
+ ]
104
+ validation_errors = verify_schema_merge_disposition(
99
105
  self.schema, tables, self.capabilities
100
- ):
106
+ ) + verify_unsupported_merge_options(
107
+ self.schema,
108
+ upserted_over_http,
109
+ self.capabilities,
110
+ "altertable",
111
+ ("source_filter", "skip_unchanged_rows"),
112
+ )
113
+ if validation_errors:
101
114
  raise validation_errors[0]
102
115
  return tables
103
116
 
@@ -41,6 +41,9 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
41
41
  deleted_cond: str | None,
42
42
  insert_only: bool = False,
43
43
  not_deleted_cond: str | None = None,
44
+ changed_cond: str | None = None,
45
+ insert_cond: str | None = None,
46
+ source_filter: str | None = None,
44
47
  ) -> list[str]:
45
48
  statements = super().gen_upsert_merge_sql(
46
49
  root_table_name,
@@ -49,12 +52,20 @@ class MergeThenDeleteJob(SqlMergeFollowupJob):
49
52
  root_table_column_names,
50
53
  hard_delete_col=None,
51
54
  deleted_cond=None,
55
+ insert_only=insert_only,
56
+ not_deleted_cond=not_deleted_cond,
57
+ changed_cond=changed_cond,
58
+ insert_cond=insert_cond,
59
+ source_filter=source_filter,
52
60
  )
53
61
  if hard_delete_col:
54
62
  keys = ", ".join(primary_keys)
63
+ staged_deletes = deleted_cond
64
+ if source_filter:
65
+ staged_deletes = f"({deleted_cond}) AND ({source_filter})"
55
66
  statements.append(
56
67
  f"DELETE FROM {root_table_name} WHERE ({keys}) IN "
57
- f"(SELECT {keys} FROM {staging_root_table_name} WHERE {deleted_cond});"
68
+ f"(SELECT {keys} FROM {staging_root_table_name} WHERE {staged_deletes});"
58
69
  )
59
70
  return statements
60
71
 
@@ -1,6 +1,6 @@
1
1
  import os
2
2
  import tempfile
3
- from collections.abc import Iterator
3
+ from collections.abc import Iterator, Mapping
4
4
  from contextlib import contextmanager
5
5
  from typing import cast
6
6
 
@@ -79,6 +79,32 @@ def apply_partitioning(
79
79
  )
80
80
 
81
81
 
82
+ def apply_descriptions(
83
+ config: AltertableClientConfiguration,
84
+ table: TTableSchema | PreparedTableSchema,
85
+ column_comments: Mapping[str, str | None] | None = None,
86
+ ) -> None:
87
+ """Publish dlt descriptions as catalog comments, so people and agents see them.
88
+ Only changed column comments are rewritten. The table comment is set on new tables."""
89
+ table_name = qualified_table_name(config, cast(str, table["name"]))
90
+ targets = {
91
+ f"COLUMN {table_name}.{escape_postgres_identifier(name)}": column.get("description")
92
+ for name, column in table["columns"].items()
93
+ if column.get("description") != (column_comments or {}).get(name)
94
+ }
95
+ if column_comments is None:
96
+ targets[f"TABLE {table_name}"] = table.get("description")
97
+ statements = [
98
+ f"COMMENT ON {target} IS {escape_duckdb_literal(description)};"
99
+ for target, description in targets.items()
100
+ if description
101
+ ]
102
+ if statements:
103
+ altertable_api.execute_sql(
104
+ config, "\n".join(["BEGIN TRANSACTION;", *statements, "COMMIT;"])
105
+ )
106
+
107
+
82
108
  def create_table(
83
109
  config: AltertableClientConfiguration,
84
110
  table: TTableSchema | PreparedTableSchema,
@@ -98,6 +124,7 @@ def create_table(
98
124
  f"CREATE TABLE IF NOT EXISTS {qualified_table_name(config, table_name)} ({columns})",
99
125
  )
100
126
  apply_partitioning(config, table)
127
+ apply_descriptions(config, table)
101
128
  logger.info(
102
129
  f"Created table {config.catalog}.{config.dataset_name}.{table_name} "
103
130
  f"with {len(table['columns'])} columns"
@@ -124,12 +151,12 @@ def create_or_evolve_table(
124
151
  table_name = cast(str, table["name"])
125
152
  rows = altertable_api.execute_sql(
126
153
  config,
127
- "SELECT column_name, data_type FROM information_schema.columns "
154
+ "SELECT column_name, data_type, column_comment FROM information_schema.columns "
128
155
  f"WHERE table_catalog = {escape_duckdb_literal(config.catalog)} "
129
156
  f"AND table_schema = {escape_duckdb_literal(config.dataset_name)} "
130
157
  f"AND table_name = {escape_duckdb_literal(table_name)}",
131
158
  )
132
- existing = dict(rows)
159
+ existing = {name: data_type for name, data_type, _ in rows}
133
160
  if not existing:
134
161
  create_table(config, table, column_types)
135
162
  return False
@@ -150,6 +177,7 @@ def create_or_evolve_table(
150
177
  f"ALTER COLUMN {escape_postgres_identifier(name)} TYPE DECIMAL(20,0)",
151
178
  )
152
179
  apply_partitioning(config, table)
180
+ apply_descriptions(config, table, {name: comment for name, _, comment in rows})
153
181
  return True
154
182
 
155
183
 
@@ -106,7 +106,7 @@ def load_context(
106
106
  ) -> tuple[LoadInfo, dict[str, int] | None, AltertableClientConfiguration]:
107
107
  trace = pipeline.last_trace
108
108
  load_info = trace.last_load_info if trace is not None else None
109
- if load_info is None or not load_info.loads_ids:
109
+ if trace is None or load_info is None or not load_info.loads_ids:
110
110
  raise ValueError(
111
111
  "verify_load reads the load step of a pipeline.run() from the pipeline trace, "
112
112
  "and found none."
@@ -142,12 +142,14 @@ def table_contract(
142
142
  key_columns = primary_key_columns(table) if disposition in (MERGE, REPLACE) else []
143
143
  if table.get("parent") and disposition in (MERGE, REPLACE):
144
144
  key_columns = get_columns_names_with_prop(table, "row_key")
145
- upsert_skips_stale_rows = disposition == MERGE and bool(
145
+ upsert_may_skip_rows = disposition == MERGE and bool(
146
146
  get_dedup_sort_tuple(table)
147
147
  or has_column_with_prop(table, "hard_delete")
148
148
  or table.get("parent")
149
+ or table.get("x-merge-skip-unchanged-rows")
150
+ or table.get("x-merge-source-filter")
149
151
  )
150
- return disposition, key_columns, upsert_skips_stale_rows
152
+ return disposition, key_columns, upsert_may_skip_rows
151
153
 
152
154
 
153
155
  def table_load_predicate(
@@ -207,7 +209,7 @@ def count_problems(
207
209
  key_columns: list[str],
208
210
  normalized_rows: int | None,
209
211
  lakehouse_counts: list[Any],
210
- upsert_skips_stale_rows: bool,
212
+ upsert_may_skip_rows: bool,
211
213
  ) -> list[str]:
212
214
  row_count = lakehouse_counts[0]
213
215
  problems: list[str] = []
@@ -220,7 +222,7 @@ def count_problems(
220
222
  if normalized_rows is None:
221
223
  return problems
222
224
  nothing_landed_under_this_load_id = normalized_rows > 0 and row_count == 0
223
- if disposition == MERGE and nothing_landed_under_this_load_id and not upsert_skips_stale_rows:
225
+ if disposition == MERGE and nothing_landed_under_this_load_id and not upsert_may_skip_rows:
224
226
  problems.append(
225
227
  f"{table_name}: dlt normalized {normalized_rows} rows, none landed with this load id"
226
228
  )
@@ -245,7 +247,7 @@ def verify_table(
245
247
  if table_name not in landed_table_names:
246
248
  return [f"{table_name}: the load includes this table, but it does not exist"]
247
249
 
248
- disposition, key_columns, upsert_skips_stale_rows = contract
250
+ disposition, key_columns, upsert_may_skip_rows = contract
249
251
  if disposition != REPLACE and load_id_predicate is None:
250
252
  return [
251
253
  f"{table_name}: this table has no _dlt_load_id column, so this load cannot be "
@@ -271,7 +273,7 @@ def verify_table(
271
273
  key_columns,
272
274
  normalized_rows,
273
275
  lakehouse_counts,
274
- upsert_skips_stale_rows,
276
+ upsert_may_skip_rows,
275
277
  )
276
278
  )
277
279
  return problems
File without changes