dlt-altertable 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/PKG-INFO +7 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/README.md +6 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/pyproject.toml +1 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/pyproject.toml.orig +1 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/destination.py +2 -1
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/table_schema.py +31 -3
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/LICENSE +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/__init__.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/altertable_adapter.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/api.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/configuration.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/exceptions.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/job_client.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/layout.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/merge.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/py.typed +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/sql_client.py +0 -0
- {dlt_altertable-0.2.1 → dlt_altertable-0.2.2}/src/dlt_altertable/verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dlt-altertable
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: dlt destination that loads into the Altertable lakehouse through its HTTP API
|
|
5
5
|
Keywords: dlt,altertable,lakehouse,parquet
|
|
6
6
|
Author: Altertable
|
|
@@ -125,6 +125,12 @@ pipeline.run(resource)
|
|
|
125
125
|
|
|
126
126
|
Pass `partition=[]` to reset. Sort hints are not supported yet.
|
|
127
127
|
|
|
128
|
+
## Descriptions
|
|
129
|
+
|
|
130
|
+
Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
|
|
131
|
+
become comments in the Altertable catalog. Column comments track changes. Table comments are
|
|
132
|
+
set on create or replace.
|
|
133
|
+
|
|
128
134
|
## Performance
|
|
129
135
|
|
|
130
136
|
Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
|
|
@@ -100,6 +100,12 @@ pipeline.run(resource)
|
|
|
100
100
|
|
|
101
101
|
Pass `partition=[]` to reset. Sort hints are not supported yet.
|
|
102
102
|
|
|
103
|
+
## Descriptions
|
|
104
|
+
|
|
105
|
+
Table and column `description` hints, such as `columns={"id": {"description": "Order id"}}`,
|
|
106
|
+
become comments in the Altertable catalog. Column comments track changes. Table comments are
|
|
107
|
+
set on create or replace.
|
|
108
|
+
|
|
103
109
|
## Performance
|
|
104
110
|
|
|
105
111
|
Override defaults with [dlt configuration](https://dlthub.com/docs/reference/performance):
|
|
@@ -18,7 +18,7 @@ from dlt.destinations.impl.destination.factory import destination as CustomDesti
|
|
|
18
18
|
|
|
19
19
|
from dlt_altertable import api as altertable_api
|
|
20
20
|
from dlt_altertable.configuration import DATASET_NAME_HINT, AltertableClientConfiguration
|
|
21
|
-
from dlt_altertable.table_schema import aligned_parquet, create_or_evolve_table
|
|
21
|
+
from dlt_altertable.table_schema import aligned_parquet, apply_descriptions, create_or_evolve_table
|
|
22
22
|
|
|
23
23
|
if TYPE_CHECKING:
|
|
24
24
|
from dlt_altertable.job_client import AltertableJobClient
|
|
@@ -133,6 +133,7 @@ def _upload(
|
|
|
133
133
|
altertable_api.post_parquet(config, "upload", params, upload_path, action)
|
|
134
134
|
|
|
135
135
|
if ingest_mode == "overwrite":
|
|
136
|
+
apply_descriptions(config, table)
|
|
136
137
|
already_replaced.append(table_name)
|
|
137
138
|
|
|
138
139
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import tempfile
|
|
3
|
-
from collections.abc import Iterator
|
|
3
|
+
from collections.abc import Iterator, Mapping
|
|
4
4
|
from contextlib import contextmanager
|
|
5
5
|
from typing import cast
|
|
6
6
|
|
|
@@ -79,6 +79,32 @@ def apply_partitioning(
|
|
|
79
79
|
)
|
|
80
80
|
|
|
81
81
|
|
|
82
|
+
def apply_descriptions(
|
|
83
|
+
config: AltertableClientConfiguration,
|
|
84
|
+
table: TTableSchema | PreparedTableSchema,
|
|
85
|
+
column_comments: Mapping[str, str | None] | None = None,
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Publish dlt descriptions as catalog comments, so people and agents see them.
|
|
88
|
+
Only changed column comments are rewritten. The table comment is set on new tables."""
|
|
89
|
+
table_name = qualified_table_name(config, cast(str, table["name"]))
|
|
90
|
+
targets = {
|
|
91
|
+
f"COLUMN {table_name}.{escape_postgres_identifier(name)}": column.get("description")
|
|
92
|
+
for name, column in table["columns"].items()
|
|
93
|
+
if column.get("description") != (column_comments or {}).get(name)
|
|
94
|
+
}
|
|
95
|
+
if column_comments is None:
|
|
96
|
+
targets[f"TABLE {table_name}"] = table.get("description")
|
|
97
|
+
statements = [
|
|
98
|
+
f"COMMENT ON {target} IS {escape_duckdb_literal(description)};"
|
|
99
|
+
for target, description in targets.items()
|
|
100
|
+
if description
|
|
101
|
+
]
|
|
102
|
+
if statements:
|
|
103
|
+
altertable_api.execute_sql(
|
|
104
|
+
config, "\n".join(["BEGIN TRANSACTION;", *statements, "COMMIT;"])
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
82
108
|
def create_table(
|
|
83
109
|
config: AltertableClientConfiguration,
|
|
84
110
|
table: TTableSchema | PreparedTableSchema,
|
|
@@ -98,6 +124,7 @@ def create_table(
|
|
|
98
124
|
f"CREATE TABLE IF NOT EXISTS {qualified_table_name(config, table_name)} ({columns})",
|
|
99
125
|
)
|
|
100
126
|
apply_partitioning(config, table)
|
|
127
|
+
apply_descriptions(config, table)
|
|
101
128
|
logger.info(
|
|
102
129
|
f"Created table {config.catalog}.{config.dataset_name}.{table_name} "
|
|
103
130
|
f"with {len(table['columns'])} columns"
|
|
@@ -124,12 +151,12 @@ def create_or_evolve_table(
|
|
|
124
151
|
table_name = cast(str, table["name"])
|
|
125
152
|
rows = altertable_api.execute_sql(
|
|
126
153
|
config,
|
|
127
|
-
"SELECT column_name, data_type FROM information_schema.columns "
|
|
154
|
+
"SELECT column_name, data_type, column_comment FROM information_schema.columns "
|
|
128
155
|
f"WHERE table_catalog = {escape_duckdb_literal(config.catalog)} "
|
|
129
156
|
f"AND table_schema = {escape_duckdb_literal(config.dataset_name)} "
|
|
130
157
|
f"AND table_name = {escape_duckdb_literal(table_name)}",
|
|
131
158
|
)
|
|
132
|
-
existing =
|
|
159
|
+
existing = {name: data_type for name, data_type, _ in rows}
|
|
133
160
|
if not existing:
|
|
134
161
|
create_table(config, table, column_types)
|
|
135
162
|
return False
|
|
@@ -150,6 +177,7 @@ def create_or_evolve_table(
|
|
|
150
177
|
f"ALTER COLUMN {escape_postgres_identifier(name)} TYPE DECIMAL(20,0)",
|
|
151
178
|
)
|
|
152
179
|
apply_partitioning(config, table)
|
|
180
|
+
apply_descriptions(config, table, {name: comment for name, _, comment in rows})
|
|
153
181
|
return True
|
|
154
182
|
|
|
155
183
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|