dlt-altertable 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dlt-altertable
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
5
5
  Keywords: dlt,altertable,lakehouse,parquet
6
6
  Author: Altertable
@@ -125,6 +125,12 @@ pipeline.run(resource)
125
125
 
126
126
  Pass `partition=[]` to reset. Sort hints are not supported yet.
127
127
 
128
+ ## Descriptions
129
+
130
+ Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
131
+ become comments in the Altertable catalog. Column comments track changes. Table comments are
132
+ set on create or replace.
133
+
128
134
  ## Performance
129
135
 
130
136
  Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
@@ -100,6 +100,12 @@ pipeline.run(resource)
100
100
 
101
101
  Pass `partition=[]` to reset. Sort hints are not supported yet.
102
102
 
103
+ ## Descriptions
104
+
105
+ Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
106
+ become comments in the Altertable catalog. Column comments track changes. Table comments are
107
+ set on create or replace.
108
+
103
109
  ## Performance
104
110
 
105
111
  Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dlt-altertable"
3
- version = "0.2.1"
3
+ version = "0.2.2"
4
4
  description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dlt-altertable"
3
- version = "0.2.1"
3
+ version = "0.2.2"
4
4
  description = "dlt destination that loads into the Altertable lakehouse through its HTTP API"
5
5
  readme = "README.md"
6
6
  authors = [{ name = "Altertable", email = "hello@altertable.ai" }]
@@ -18,7 +18,7 @@ from dlt.destinations.impl.destination.factory import destination as CustomDesti
18
18
 
19
19
  from dlt_altertable import api as altertable_api
20
20
  from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
21
- from dlt_altertable.table_schema import aligned_parquet, create_or_evolve_table
21
+ from dlt_altertable.table_schema import aligned_parquet, apply_descriptions, create_or_evolve_table
22
22
 
23
23
  if TYPE_CHECKING:
24
24
  from dlt_altertable.job_client import AltertableJobClient
@@ -133,6 +133,7 @@ def _upload(
133
133
  altertable_api.post_parquet(config, "upload", params, upload_path, action)
134
134
 
135
135
  if ingest_mode == "overwrite":
136
+ apply_descriptions(config, table)
136
137
  already_replaced.append(table_name)
137
138
 
138
139
 
@@ -1,6 +1,6 @@
1
1
  import os
2
2
  import tempfile
3
- from collections.abc import Iterator
3
+ from collections.abc import Iterator, Mapping
4
4
  from contextlib import contextmanager
5
5
  from typing import cast
6
6
 
@@ -79,6 +79,32 @@ def apply_partitioning(
79
79
  )
80
80
 
81
81
 
82
+ def apply_descriptions(
83
+ config: AltertableClientConfiguration,
84
+ table: TTableSchema | PreparedTableSchema,
85
+ column_comments: Mapping[str, str | None] | None = None,
86
+ ) -> None:
87
+ """Publish dlt descriptions as catalog comments, so people and agents see them.
88
+ Only changed column comments are rewritten. The table comment is set on new tables."""
89
+ table_name = qualified_table_name(config, cast(str, table["name"]))
90
+ targets = {
91
+ f"COLUMN {table_name}.{escape_postgres_identifier(name)}": column.get("description")
92
+ for name, column in table["columns"].items()
93
+ if column.get("description") != (column_comments or {}).get(name)
94
+ }
95
+ if column_comments is None:
96
+ targets[f"TABLE {table_name}"] = table.get("description")
97
+ statements = [
98
+ f"COMMENT ON {target} IS {escape_duckdb_literal(description)};"
99
+ for target, description in targets.items()
100
+ if description
101
+ ]
102
+ if statements:
103
+ altertable_api.execute_sql(
104
+ config, "\n".join(["BEGIN TRANSACTION;", *statements, "COMMIT;"])
105
+ )
106
+
107
+
82
108
  def create_table(
83
109
  config: AltertableClientConfiguration,
84
110
  table: TTableSchema | PreparedTableSchema,
@@ -98,6 +124,7 @@ def create_table(
98
124
  f"CREATE TABLE IF NOT EXISTS {qualified_table_name(config, table_name)} ({columns})",
99
125
  )
100
126
  apply_partitioning(config, table)
127
+ apply_descriptions(config, table)
101
128
  logger.info(
102
129
  f"Created table {config.catalog}.{config.dataset_name}.{table_name} "
103
130
  f"with {len(table['columns'])} columns"
@@ -124,12 +151,12 @@ def create_or_evolve_table(
124
151
  table_name = cast(str, table["name"])
125
152
  rows = altertable_api.execute_sql(
126
153
  config,
127
- "SELECT column_name, data_type FROM information_schema.columns "
154
+ "SELECT column_name, data_type, column_comment FROM information_schema.columns "
128
155
  f"WHERE table_catalog = {escape_duckdb_literal(config.catalog)} "
129
156
  f"AND table_schema = {escape_duckdb_literal(config.dataset_name)} "
130
157
  f"AND table_name = {escape_duckdb_literal(table_name)}",
131
158
  )
132
- existing = dict(rows)
159
+ existing = {name: data_type for name, data_type, _ in rows}
133
160
  if not existing:
134
161
  create_table(config, table, column_types)
135
162
  return False
@@ -150,6 +177,7 @@ def create_or_evolve_table(
150
177
  f"ALTER COLUMN {escape_postgres_identifier(name)} TYPE DECIMAL(20,0)",
151
178
  )
152
179
  apply_partitioning(config, table)
180
+ apply_descriptions(config, table, {name: comment for name, _, comment in rows})
153
181
  return True
154
182
 
155
183
 
File without changes