dbt-altertable 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt/adapters/altertable/__init__.py +10 -0
- dbt/adapters/altertable/__version__.py +1 -0
- dbt/adapters/altertable/connections.py +279 -0
- dbt/adapters/altertable/impl.py +10 -0
- dbt/include/altertable/__init__.py +3 -0
- dbt/include/altertable/dbt_project.yml +5 -0
- dbt/include/altertable/macros/adapters.sql +279 -0
- dbt/include/altertable/macros/catalog.sql +45 -0
- dbt/include/altertable/macros/columns.sql +23 -0
- dbt/include/altertable/macros/persist_docs.sql +36 -0
- dbt/include/altertable/macros/seed.sql +55 -0
- dbt/include/altertable/macros/snapshot_helper.sql +40 -0
- dbt/include/altertable/macros/utils/any_value.sql +5 -0
- dbt/include/altertable/macros/utils/dateadd.sql +22 -0
- dbt/include/altertable/macros/utils/datediff.sql +12 -0
- dbt/include/altertable/macros/utils/external_location.sql +8 -0
- dbt/include/altertable/macros/utils/generate_series.sql +5 -0
- dbt/include/altertable/macros/utils/lastday.sql +14 -0
- dbt/include/altertable/macros/utils/listagg.sql +22 -0
- dbt/include/altertable/macros/utils/splitpart.sql +3 -0
- dbt/include/altertable/macros/utils/upstream.sql +42 -0
- dbt_altertable-0.1.0.dist-info/METADATA +85 -0
- dbt_altertable-0.1.0.dist-info/RECORD +26 -0
- dbt_altertable-0.1.0.dist-info/WHEEL +5 -0
- dbt_altertable-0.1.0.dist-info/licenses/LICENSE +21 -0
- dbt_altertable-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from dbt.adapters.altertable.connections import AltertableCredentials
|
|
2
|
+
from dbt.adapters.altertable.impl import AltertableAdapter
|
|
3
|
+
from dbt.adapters.base import AdapterPlugin
|
|
4
|
+
from dbt.include import altertable
|
|
5
|
+
|
|
6
|
+
Plugin = AdapterPlugin(
|
|
7
|
+
adapter=AltertableAdapter,
|
|
8
|
+
credentials=AltertableCredentials,
|
|
9
|
+
include_path=altertable.PACKAGE_PATH,
|
|
10
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = "0.1.0"
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
from contextlib import contextmanager
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
|
|
4
|
+
|
|
5
|
+
import altertable_flightsql
|
|
6
|
+
from dbt_common.exceptions import DbtRuntimeError
|
|
7
|
+
from mashumaro.jsonschema.annotations import Maximum, Minimum
|
|
8
|
+
from typing_extensions import Annotated
|
|
9
|
+
|
|
10
|
+
from dbt.adapters.contracts.connection import (
|
|
11
|
+
AdapterResponse,
|
|
12
|
+
Connection,
|
|
13
|
+
ConnectionState,
|
|
14
|
+
Credentials,
|
|
15
|
+
)
|
|
16
|
+
from dbt.adapters.events.logging import AdapterLogger
|
|
17
|
+
from dbt.adapters.sql.connections import SQLConnectionManager
|
|
18
|
+
|
|
19
|
+
logger = AdapterLogger("Altertable")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AltertableCursor:
|
|
23
|
+
"""
|
|
24
|
+
A PEP 249-compliant cursor wrapper around the altertable_flightsql Client.
|
|
25
|
+
|
|
26
|
+
This cursor provides the standard DB-API 2.0 interface that dbt expects,
|
|
27
|
+
translating calls to the altertable_flightsql client methods.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, client: altertable_flightsql.Client):
|
|
31
|
+
self._client = client
|
|
32
|
+
self._results: Optional[List[Tuple[Any, ...]]] = None
|
|
33
|
+
self._description: Optional[List[Tuple[str, Any, None, None, None, None, None]]] = None
|
|
34
|
+
self._rowcount: int = -1
|
|
35
|
+
self._cursor_position: int = 0
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def description(
|
|
39
|
+
self,
|
|
40
|
+
) -> Optional[List[Tuple[str, Any, None, None, None, None, None]]]:
|
|
41
|
+
"""
|
|
42
|
+
PEP 249: Sequence of 7-item sequences describing result columns.
|
|
43
|
+
|
|
44
|
+
Each sequence contains: (name, type_code, display_size, internal_size,
|
|
45
|
+
precision, scale, null_ok). We only populate name and type_code.
|
|
46
|
+
"""
|
|
47
|
+
return self._description
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def rowcount(self) -> int:
|
|
51
|
+
"""
|
|
52
|
+
PEP 249: Number of rows affected by last execute.
|
|
53
|
+
|
|
54
|
+
Returns -1 for SELECT statements or when count is unknown.
|
|
55
|
+
"""
|
|
56
|
+
return self._rowcount
|
|
57
|
+
|
|
58
|
+
def execute(
|
|
59
|
+
self,
|
|
60
|
+
sql: str,
|
|
61
|
+
bindings: Optional[Union[Sequence[Any], Mapping[str, Any]]] = None,
|
|
62
|
+
) -> "AltertableCursor":
|
|
63
|
+
"""
|
|
64
|
+
Execute a SQL statement.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
sql: SQL statement to execute.
|
|
68
|
+
bindings: Optional parameter bindings. Can be a sequence (positional)
|
|
69
|
+
or a mapping (named parameters).
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
Self for method chaining.
|
|
73
|
+
"""
|
|
74
|
+
# Reset state
|
|
75
|
+
self._results = None
|
|
76
|
+
self._description = None
|
|
77
|
+
self._rowcount = -1
|
|
78
|
+
self._cursor_position = 0
|
|
79
|
+
|
|
80
|
+
print("--------------------------------")
|
|
81
|
+
print(f"SQL: {sql}")
|
|
82
|
+
print(f"Bindings: {bindings}")
|
|
83
|
+
print("--------------------------------")
|
|
84
|
+
if bindings is not None:
|
|
85
|
+
# Use prepared statement for parameterized queries
|
|
86
|
+
with self._client.prepare(sql) as stmt:
|
|
87
|
+
reader = stmt.query(parameters=bindings)
|
|
88
|
+
table = reader.read_all()
|
|
89
|
+
else:
|
|
90
|
+
reader = self._client.query(sql)
|
|
91
|
+
table = reader.read_all()
|
|
92
|
+
|
|
93
|
+
self._process_arrow_table(table)
|
|
94
|
+
return self
|
|
95
|
+
|
|
96
|
+
def _process_arrow_table(self, table) -> None:
|
|
97
|
+
"""Process an Arrow table into cursor results."""
|
|
98
|
+
# Build description from schema
|
|
99
|
+
self._description = [
|
|
100
|
+
(field.name, field.type, None, None, None, None, None) for field in table.schema
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
# Convert Arrow table to list of tuples (row-oriented)
|
|
104
|
+
columns = list(table.to_pydict().values())
|
|
105
|
+
if columns:
|
|
106
|
+
num_rows = len(columns[0])
|
|
107
|
+
self._results = [tuple(col[i] for col in columns) for i in range(num_rows)]
|
|
108
|
+
else:
|
|
109
|
+
self._results = []
|
|
110
|
+
|
|
111
|
+
self._rowcount = len(self._results)
|
|
112
|
+
|
|
113
|
+
def fetchone(self) -> Optional[Tuple[Any, ...]]:
|
|
114
|
+
"""
|
|
115
|
+
Fetch the next row of a query result.
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
A single row as a tuple, or None if no more rows.
|
|
119
|
+
"""
|
|
120
|
+
if self._results is None or self._cursor_position >= len(self._results):
|
|
121
|
+
return None
|
|
122
|
+
row = self._results[self._cursor_position]
|
|
123
|
+
self._cursor_position += 1
|
|
124
|
+
return row
|
|
125
|
+
|
|
126
|
+
def fetchmany(self, size: Optional[int] = None) -> List[Tuple[Any, ...]]:
|
|
127
|
+
"""
|
|
128
|
+
Fetch the next set of rows.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
size: Maximum number of rows to fetch.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
List of rows as tuples.
|
|
135
|
+
"""
|
|
136
|
+
if self._results is None:
|
|
137
|
+
return []
|
|
138
|
+
if size is None:
|
|
139
|
+
size = 1 # Default arraysize per PEP 249
|
|
140
|
+
end = min(self._cursor_position + size, len(self._results))
|
|
141
|
+
rows = self._results[self._cursor_position : end]
|
|
142
|
+
self._cursor_position = end
|
|
143
|
+
return rows
|
|
144
|
+
|
|
145
|
+
def fetchall(self) -> List[Tuple[Any, ...]]:
|
|
146
|
+
"""
|
|
147
|
+
Fetch all remaining rows.
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
List of all remaining rows as tuples.
|
|
151
|
+
"""
|
|
152
|
+
if self._results is None:
|
|
153
|
+
return []
|
|
154
|
+
rows = self._results[self._cursor_position :]
|
|
155
|
+
self._cursor_position = len(self._results)
|
|
156
|
+
return rows
|
|
157
|
+
|
|
158
|
+
def close(self) -> None:
|
|
159
|
+
"""Close the cursor."""
|
|
160
|
+
self._results = None
|
|
161
|
+
self._description = None
|
|
162
|
+
|
|
163
|
+
def __iter__(self):
|
|
164
|
+
"""Allow iteration over results."""
|
|
165
|
+
return self
|
|
166
|
+
|
|
167
|
+
def __next__(self) -> Tuple[Any, ...]:
|
|
168
|
+
row = self.fetchone()
|
|
169
|
+
if row is None:
|
|
170
|
+
raise StopIteration
|
|
171
|
+
return row
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class AltertableConnection:
|
|
175
|
+
"""
|
|
176
|
+
A PEP 249-compliant connection wrapper around the altertable_flightsql Client.
|
|
177
|
+
|
|
178
|
+
This provides the connection interface that dbt expects, with a cursor() method
|
|
179
|
+
that returns AltertableCursor instances.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
def __init__(self, client: altertable_flightsql.Client):
|
|
183
|
+
self._client = client
|
|
184
|
+
|
|
185
|
+
def __enter__(self) -> "AltertableConnection":
|
|
186
|
+
return self
|
|
187
|
+
|
|
188
|
+
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
|
189
|
+
self.close()
|
|
190
|
+
|
|
191
|
+
def cursor(self) -> AltertableCursor:
|
|
192
|
+
"""Create a new cursor for this connection."""
|
|
193
|
+
return AltertableCursor(self._client)
|
|
194
|
+
|
|
195
|
+
def close(self) -> None:
|
|
196
|
+
"""Close the connection."""
|
|
197
|
+
self._client.close()
|
|
198
|
+
|
|
199
|
+
def commit(self) -> None:
|
|
200
|
+
"""Commit current transaction (no-op for now as we use auto-commit)."""
|
|
201
|
+
pass
|
|
202
|
+
|
|
203
|
+
def rollback(self) -> None:
|
|
204
|
+
"""Rollback current transaction (no-op for now as we use auto-commit)."""
|
|
205
|
+
pass
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@dataclass
|
|
209
|
+
class AltertableCredentials(Credentials):
|
|
210
|
+
username: str
|
|
211
|
+
password: str
|
|
212
|
+
host: str = "flight.altertable.ai"
|
|
213
|
+
port: Annotated[int, Minimum(0), Maximum(65535)] = 443
|
|
214
|
+
tls: bool = True
|
|
215
|
+
|
|
216
|
+
@property
|
|
217
|
+
def type(self) -> str:
|
|
218
|
+
return "altertable"
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def unique_field(self) -> str:
|
|
222
|
+
return "host"
|
|
223
|
+
|
|
224
|
+
def _connection_keys(self) -> Tuple[str, ...]:
|
|
225
|
+
return (
|
|
226
|
+
"username",
|
|
227
|
+
"password",
|
|
228
|
+
"database",
|
|
229
|
+
"schema",
|
|
230
|
+
"host",
|
|
231
|
+
"port",
|
|
232
|
+
"tls",
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class AltertableConnectionManager(SQLConnectionManager):
|
|
237
|
+
TYPE = "altertable"
|
|
238
|
+
|
|
239
|
+
@contextmanager
|
|
240
|
+
def exception_handler(self, sql: str):
|
|
241
|
+
try:
|
|
242
|
+
yield
|
|
243
|
+
|
|
244
|
+
except Exception as e:
|
|
245
|
+
logger.error(f"Error executing SQL: {sql}")
|
|
246
|
+
raise DbtRuntimeError(e) from e
|
|
247
|
+
|
|
248
|
+
def cancel(self, connection: Connection) -> None:
|
|
249
|
+
logger.debug(f"Attempting to cancel connection: {connection.name}")
|
|
250
|
+
|
|
251
|
+
@classmethod
|
|
252
|
+
def open(cls, connection: Connection) -> Connection:
|
|
253
|
+
if connection.state == ConnectionState.OPEN:
|
|
254
|
+
return connection
|
|
255
|
+
|
|
256
|
+
def connect():
|
|
257
|
+
client = altertable_flightsql.Client(
|
|
258
|
+
username=connection.credentials.username,
|
|
259
|
+
password=connection.credentials.password,
|
|
260
|
+
catalog=connection.credentials.database,
|
|
261
|
+
schema=connection.credentials.schema,
|
|
262
|
+
host=connection.credentials.host,
|
|
263
|
+
port=connection.credentials.port,
|
|
264
|
+
tls=connection.credentials.tls,
|
|
265
|
+
)
|
|
266
|
+
return AltertableConnection(client)
|
|
267
|
+
|
|
268
|
+
return cls.retry_connection(
|
|
269
|
+
connection,
|
|
270
|
+
connect=connect,
|
|
271
|
+
logger=logger,
|
|
272
|
+
retry_limit=1,
|
|
273
|
+
retry_timeout=lambda attempt: attempt**2,
|
|
274
|
+
retryable_exceptions=[Exception],
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
@classmethod
|
|
278
|
+
def get_response(cls, cursor) -> AdapterResponse:
|
|
279
|
+
return AdapterResponse(_message="OK")
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from dbt.adapters.altertable.connections import AltertableConnectionManager
|
|
2
|
+
from dbt.adapters.sql import SQLAdapter
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class AltertableAdapter(SQLAdapter):
|
|
6
|
+
ConnectionManager = AltertableConnectionManager
|
|
7
|
+
|
|
8
|
+
@classmethod
|
|
9
|
+
def date_function(cls) -> str:
|
|
10
|
+
return "now()"
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
{% macro altertable__create_schema(relation) -%}
|
|
2
|
+
{%- call statement('create_schema') -%}
|
|
3
|
+
{% set sql %}
|
|
4
|
+
select type from duckdb_databases()
|
|
5
|
+
where lower(database_name)='{{ relation.database | lower }}'
|
|
6
|
+
and type='sqlite'
|
|
7
|
+
{% endset %}
|
|
8
|
+
{% set results = run_query(sql) %}
|
|
9
|
+
{% if results|length == 0 %}
|
|
10
|
+
create schema if not exists {{ relation.without_identifier() }}
|
|
11
|
+
{% else %}
|
|
12
|
+
{% if relation.schema!='main' %}
|
|
13
|
+
{{ exceptions.raise_compiler_error(
|
|
14
|
+
"Schema must be 'main' when writing to sqlite "
|
|
15
|
+
~ "instead got " ~ relation.schema
|
|
16
|
+
)}}
|
|
17
|
+
{% endif %}
|
|
18
|
+
{% endif %}
|
|
19
|
+
{%- endcall -%}
|
|
20
|
+
{% endmacro %}
|
|
21
|
+
|
|
22
|
+
{% macro altertable__drop_schema(relation) -%}
|
|
23
|
+
{%- call statement('drop_schema') -%}
|
|
24
|
+
drop schema if exists {{ relation.without_identifier() }} cascade
|
|
25
|
+
{%- endcall -%}
|
|
26
|
+
{% endmacro %}
|
|
27
|
+
|
|
28
|
+
{% macro altertable__list_schemas(database) -%}
|
|
29
|
+
{% set sql %}
|
|
30
|
+
select schema_name
|
|
31
|
+
from system.information_schema.schemata
|
|
32
|
+
{% if database is not none %}
|
|
33
|
+
where lower(catalog_name) = '{{ database | lower | trim('"') }}'
|
|
34
|
+
{% endif %}
|
|
35
|
+
{% endset %}
|
|
36
|
+
{{ return(run_query(sql)) }}
|
|
37
|
+
{% endmacro %}
|
|
38
|
+
|
|
39
|
+
{% macro altertable__check_schema_exists(information_schema, schema) -%}
|
|
40
|
+
{% set sql -%}
|
|
41
|
+
select count(*)
|
|
42
|
+
from system.information_schema.schemata
|
|
43
|
+
where lower(schema_name) = '{{ schema | lower }}'
|
|
44
|
+
and lower(catalog_name) = '{{ information_schema.database | lower }}'
|
|
45
|
+
{%- endset %}
|
|
46
|
+
{{ return(run_query(sql)) }}
|
|
47
|
+
{% endmacro %}
|
|
48
|
+
|
|
49
|
+
{% macro get_column_names() %}
|
|
50
|
+
{# loop through user_provided_columns to get column names #}
|
|
51
|
+
{%- set user_provided_columns = model['columns'] -%}
|
|
52
|
+
(
|
|
53
|
+
{% for i in user_provided_columns %}
|
|
54
|
+
{% set col = user_provided_columns[i] %}
|
|
55
|
+
{{ col['name'] }} {{ "," if not loop.last }}
|
|
56
|
+
{% endfor %}
|
|
57
|
+
)
|
|
58
|
+
{% endmacro %}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
{% macro altertable__create_table_as(temporary, relation, compiled_code, language='sql') -%}
|
|
62
|
+
{%- if language == 'sql' -%}
|
|
63
|
+
{% set contract_config = config.get('contract') %}
|
|
64
|
+
{% if contract_config.enforced %}
|
|
65
|
+
{{ get_assert_columns_equivalent(compiled_code) }}
|
|
66
|
+
{% endif %}
|
|
67
|
+
{%- set sql_header = config.get('sql_header', none) -%}
|
|
68
|
+
|
|
69
|
+
{{ sql_header if sql_header is not none }}
|
|
70
|
+
|
|
71
|
+
create {% if temporary: -%}temporary{%- endif %} table
|
|
72
|
+
{{ relation.include(database=(not temporary), schema=(not temporary)) }}
|
|
73
|
+
{% if contract_config.enforced and not temporary %}
|
|
74
|
+
{#-- DuckDB doesnt support constraints on temp tables --#}
|
|
75
|
+
{{ get_table_columns_and_constraints() }} ;
|
|
76
|
+
insert into {{ relation }} {{ get_column_names() }} (
|
|
77
|
+
{{ get_select_subquery(compiled_code) }}
|
|
78
|
+
);
|
|
79
|
+
{% else %}
|
|
80
|
+
as (
|
|
81
|
+
{{ compiled_code }}
|
|
82
|
+
);
|
|
83
|
+
{% endif %}
|
|
84
|
+
{%- elif language == 'python' -%}
|
|
85
|
+
{{ py_write_table(temporary=temporary, relation=relation, compiled_code=compiled_code) }}
|
|
86
|
+
{%- else -%}
|
|
87
|
+
{% do exceptions.raise_compiler_error("altertable__create_table_as macro didn't get supported language, it got %s" % language) %}
|
|
88
|
+
{%- endif -%}
|
|
89
|
+
{% endmacro %}
|
|
90
|
+
|
|
91
|
+
{% macro py_write_table(temporary, relation, compiled_code) -%}
|
|
92
|
+
{{ compiled_code }}
|
|
93
|
+
|
|
94
|
+
def materialize(df, con):
|
|
95
|
+
try:
|
|
96
|
+
import pyarrow
|
|
97
|
+
pyarrow_available = True
|
|
98
|
+
except ImportError:
|
|
99
|
+
pyarrow_available = False
|
|
100
|
+
finally:
|
|
101
|
+
if pyarrow_available and isinstance(df, pyarrow.Table):
|
|
102
|
+
# https://github.com/duckdb/duckdb/issues/6584
|
|
103
|
+
import pyarrow.dataset
|
|
104
|
+
tmp_name = '__dbt_python_model_df_' + '{{ relation.identifier }}'
|
|
105
|
+
con.register(tmp_name, df)
|
|
106
|
+
con.execute('create table {{ relation }} as select * from ' + tmp_name)
|
|
107
|
+
con.unregister(tmp_name)
|
|
108
|
+
{% endmacro %}
|
|
109
|
+
|
|
110
|
+
{% macro altertable__create_view_as(relation, sql) -%}
|
|
111
|
+
{% set contract_config = config.get('contract') %}
|
|
112
|
+
{% if contract_config.enforced %}
|
|
113
|
+
{{ get_assert_columns_equivalent(sql) }}
|
|
114
|
+
{%- endif %}
|
|
115
|
+
{%- set sql_header = config.get('sql_header', none) -%}
|
|
116
|
+
|
|
117
|
+
{{ sql_header if sql_header is not none }}
|
|
118
|
+
create view {{ relation }} as (
|
|
119
|
+
{{ sql }}
|
|
120
|
+
);
|
|
121
|
+
{% endmacro %}
|
|
122
|
+
|
|
123
|
+
{% macro altertable__get_columns_in_relation(relation) -%}
|
|
124
|
+
{% call statement('get_columns_in_relation', fetch_result=True) %}
|
|
125
|
+
select
|
|
126
|
+
column_name,
|
|
127
|
+
data_type,
|
|
128
|
+
character_maximum_length,
|
|
129
|
+
numeric_precision,
|
|
130
|
+
numeric_scale
|
|
131
|
+
|
|
132
|
+
from system.information_schema.columns
|
|
133
|
+
where table_name = '{{ relation.identifier }}'
|
|
134
|
+
{% if relation.schema %}
|
|
135
|
+
and lower(table_schema) = '{{ relation.schema | lower }}'
|
|
136
|
+
{% endif %}
|
|
137
|
+
{% if relation.database %}
|
|
138
|
+
and lower(table_catalog) = '{{ relation.database | lower }}'
|
|
139
|
+
{% endif %}
|
|
140
|
+
order by ordinal_position
|
|
141
|
+
|
|
142
|
+
{% endcall %}
|
|
143
|
+
{% set table = load_result('get_columns_in_relation').table %}
|
|
144
|
+
{{ return(sql_convert_columns_in_relation(table)) }}
|
|
145
|
+
{% endmacro %}
|
|
146
|
+
|
|
147
|
+
{% macro altertable__list_relations_without_caching(schema_relation) %}
|
|
148
|
+
{% call statement('list_relations_without_caching', fetch_result=True) -%}
|
|
149
|
+
select
|
|
150
|
+
'{{ schema_relation.database }}' as database,
|
|
151
|
+
table_name as name,
|
|
152
|
+
table_schema as schema,
|
|
153
|
+
CASE table_type
|
|
154
|
+
WHEN 'BASE TABLE' THEN 'table'
|
|
155
|
+
WHEN 'VIEW' THEN 'view'
|
|
156
|
+
WHEN 'LOCAL TEMPORARY' THEN 'table'
|
|
157
|
+
END as type
|
|
158
|
+
from system.information_schema.tables
|
|
159
|
+
where lower(table_schema) = '{{ schema_relation.schema | lower }}'
|
|
160
|
+
and lower(table_catalog) = '{{ schema_relation.database | lower }}'
|
|
161
|
+
{% endcall %}
|
|
162
|
+
{{ return(load_result('list_relations_without_caching').table) }}
|
|
163
|
+
{% endmacro %}
|
|
164
|
+
|
|
165
|
+
{% macro altertable__make_temp_relation(base_relation, suffix) %}
|
|
166
|
+
{% set tmp_identifier = base_relation.identifier ~ suffix ~ py_current_timestring() %}
|
|
167
|
+
{% do return(base_relation.incorporate(
|
|
168
|
+
path={
|
|
169
|
+
"identifier": tmp_identifier,
|
|
170
|
+
"schema": none,
|
|
171
|
+
"database": none
|
|
172
|
+
})) -%}
|
|
173
|
+
{% endmacro %}
|
|
174
|
+
|
|
175
|
+
{% macro altertable__rename_relation(from_relation, to_relation) -%}
|
|
176
|
+
{% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}
|
|
177
|
+
{% call statement('rename_relation') -%}
|
|
178
|
+
alter {{ to_relation.type }} {{ from_relation }} rename to {{ target_name }}
|
|
179
|
+
{%- endcall %}
|
|
180
|
+
{% endmacro %}
|
|
181
|
+
|
|
182
|
+
{% macro altertable__current_timestamp() -%}
|
|
183
|
+
now()
|
|
184
|
+
{%- endmacro %}
|
|
185
|
+
|
|
186
|
+
{% macro altertable__snapshot_string_as_time(timestamp) -%}
|
|
187
|
+
{%- set result = "'" ~ timestamp ~ "'::timestamp" -%}
|
|
188
|
+
{{ return(result) }}
|
|
189
|
+
{%- endmacro %}
|
|
190
|
+
|
|
191
|
+
{% macro altertable__snapshot_get_time() -%}
|
|
192
|
+
{{ current_timestamp() }}::timestamp
|
|
193
|
+
{%- endmacro %}
|
|
194
|
+
|
|
195
|
+
{% macro altertable__get_incremental_default_sql(arg_dict) %}
|
|
196
|
+
{% do return(get_incremental_delete_insert_sql(arg_dict)) %}
|
|
197
|
+
{% endmacro %}
|
|
198
|
+
|
|
199
|
+
{% macro location_exists(location) -%}
|
|
200
|
+
{% do return(adapter.location_exists(location)) %}
|
|
201
|
+
{% endmacro %}
|
|
202
|
+
|
|
203
|
+
{% macro write_to_file(relation, location, options) -%}
|
|
204
|
+
{% call statement('write_to_file') -%}
|
|
205
|
+
copy {{ relation }} to '{{ location }}' ({{ options }})
|
|
206
|
+
{%- endcall %}
|
|
207
|
+
{% endmacro %}
|
|
208
|
+
|
|
209
|
+
{% macro store_relation(plugin, relation, location, format, config) -%}
|
|
210
|
+
{%- set column_list = adapter.get_columns_in_relation(relation) -%}
|
|
211
|
+
{% do adapter.store_relation(plugin, relation, column_list, location, format, config) %}
|
|
212
|
+
{% endmacro %}
|
|
213
|
+
|
|
214
|
+
{% macro render_write_options(config) -%}
|
|
215
|
+
{% set options = config.get('options', {}) %}
|
|
216
|
+
{% if options is not mapping %}
|
|
217
|
+
{% do exceptions.raise_compiler_error("The options argument must be a dictionary") %}
|
|
218
|
+
{% endif %}
|
|
219
|
+
|
|
220
|
+
{% for k in options %}
|
|
221
|
+
{% set _ = options.update({k: render(options[k])}) %}
|
|
222
|
+
{% endfor %}
|
|
223
|
+
|
|
224
|
+
{# legacy top-level write options #}
|
|
225
|
+
{% if config.get('format') %}
|
|
226
|
+
{% set _ = options.update({'format': render(config.get('format'))}) %}
|
|
227
|
+
{% endif %}
|
|
228
|
+
{% if config.get('delimiter') %}
|
|
229
|
+
{% set _ = options.update({'delimiter': render(config.get('delimiter'))}) %}
|
|
230
|
+
{% endif %}
|
|
231
|
+
|
|
232
|
+
{% do return(options) %}
|
|
233
|
+
{%- endmacro %}
|
|
234
|
+
|
|
235
|
+
{% macro altertable__apply_grants(relation, grant_config, should_revoke=True) %}
|
|
236
|
+
{#-- If grant_config is {} or None, this is a no-op --#}
|
|
237
|
+
{% if grant_config %}
|
|
238
|
+
{{ adapter.warn_once('Grants for relations are not supported by DuckDB') }}
|
|
239
|
+
{% endif %}
|
|
240
|
+
{% endmacro %}
|
|
241
|
+
|
|
242
|
+
{% macro altertable__get_create_index_sql(relation, index_dict) -%}
|
|
243
|
+
{%- set index_config = adapter.parse_index(index_dict) -%}
|
|
244
|
+
{%- set comma_separated_columns = ", ".join(index_config.columns) -%}
|
|
245
|
+
{%- set index_name = index_config.render(relation) -%}
|
|
246
|
+
|
|
247
|
+
create {% if index_config.unique -%}
|
|
248
|
+
unique
|
|
249
|
+
{%- endif %} index
|
|
250
|
+
"{{ index_name }}"
|
|
251
|
+
on {{ relation }}
|
|
252
|
+
({{ comma_separated_columns }});
|
|
253
|
+
{%- endmacro %}
|
|
254
|
+
|
|
255
|
+
{% macro drop_indexes_on_relation(relation) -%}
|
|
256
|
+
{% call statement('get_indexes_on_relation', fetch_result=True) %}
|
|
257
|
+
SELECT index_name
|
|
258
|
+
FROM duckdb_indexes()
|
|
259
|
+
WHERE schema_name = '{{ relation.schema }}'
|
|
260
|
+
AND table_name = '{{ relation.identifier }}'
|
|
261
|
+
{% endcall %}
|
|
262
|
+
|
|
263
|
+
{% set results = load_result('get_indexes_on_relation').table %}
|
|
264
|
+
{% for row in results %}
|
|
265
|
+
{% set index_name = row[0] %}
|
|
266
|
+
{% call statement('drop_index_' + loop.index|string, auto_begin=false) %}
|
|
267
|
+
DROP INDEX "{{ relation.schema }}"."{{ index_name }}"
|
|
268
|
+
{% endcall %}
|
|
269
|
+
{% endfor %}
|
|
270
|
+
|
|
271
|
+
{#-- Verify indexes were dropped --#}
|
|
272
|
+
{% call statement('verify_indexes_dropped', fetch_result=True) %}
|
|
273
|
+
SELECT COUNT(*) as remaining_indexes
|
|
274
|
+
FROM duckdb_indexes()
|
|
275
|
+
WHERE schema_name = '{{ relation.schema }}'
|
|
276
|
+
AND table_name = '{{ relation.identifier }}'
|
|
277
|
+
{% endcall %}
|
|
278
|
+
{% set verify_results = load_result('verify_indexes_dropped').table %}
|
|
279
|
+
{%- endmacro %}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
|
|
2
|
+
{% macro altertable__get_catalog(information_schema, schemas) -%}
|
|
3
|
+
{%- call statement('catalog', fetch_result=True) -%}
|
|
4
|
+
with relations AS (
|
|
5
|
+
select
|
|
6
|
+
t.table_name
|
|
7
|
+
, t.database_name
|
|
8
|
+
, t.schema_name
|
|
9
|
+
, 'BASE TABLE' as table_type
|
|
10
|
+
, t.comment as table_comment
|
|
11
|
+
from duckdb_tables() t
|
|
12
|
+
WHERE t.database_name = '{{ database }}'
|
|
13
|
+
UNION ALL
|
|
14
|
+
SELECT v.view_name as table_name
|
|
15
|
+
, v.database_name
|
|
16
|
+
, v.schema_name
|
|
17
|
+
, 'VIEW' as table_type
|
|
18
|
+
, v.comment as table_comment
|
|
19
|
+
from duckdb_views() v
|
|
20
|
+
WHERE v.database_name = '{{ database }}'
|
|
21
|
+
)
|
|
22
|
+
select
|
|
23
|
+
'{{ database }}' as table_database,
|
|
24
|
+
r.schema_name as table_schema,
|
|
25
|
+
r.table_name,
|
|
26
|
+
r.table_type,
|
|
27
|
+
r.table_comment,
|
|
28
|
+
c.column_name,
|
|
29
|
+
c.column_index as column_index,
|
|
30
|
+
c.data_type as column_type,
|
|
31
|
+
c.comment as column_comment,
|
|
32
|
+
NULL as table_owner
|
|
33
|
+
FROM relations r JOIN duckdb_columns() c ON r.schema_name = c.schema_name AND r.table_name = c.table_name
|
|
34
|
+
WHERE (
|
|
35
|
+
{%- for schema in schemas -%}
|
|
36
|
+
upper(r.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
|
|
37
|
+
{%- endfor -%}
|
|
38
|
+
)
|
|
39
|
+
ORDER BY
|
|
40
|
+
r.schema_name,
|
|
41
|
+
r.table_name,
|
|
42
|
+
c.column_index
|
|
43
|
+
{%- endcall -%}
|
|
44
|
+
{{ return(load_result('catalog').table) }}
|
|
45
|
+
{%- endmacro %}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{% macro altertable__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
|
|
2
|
+
|
|
3
|
+
{% if add_columns %}
|
|
4
|
+
{% for column in add_columns %}
|
|
5
|
+
{% set sql -%}
|
|
6
|
+
alter {{ relation.type }} {{ relation }} add column
|
|
7
|
+
{{ api.Relation.create(identifier=column.name) }} {{ column.data_type }}
|
|
8
|
+
{%- endset -%}
|
|
9
|
+
{% do run_query(sql) %}
|
|
10
|
+
{% endfor %}
|
|
11
|
+
{% endif %}
|
|
12
|
+
|
|
13
|
+
{% if remove_columns %}
|
|
14
|
+
{% for column in remove_columns %}
|
|
15
|
+
{% set sql -%}
|
|
16
|
+
alter {{ relation.type }} {{ relation }} drop column
|
|
17
|
+
{{ api.Relation.create(identifier=column.name) }}
|
|
18
|
+
{%- endset -%}
|
|
19
|
+
{% do run_query(sql) %}
|
|
20
|
+
{% endfor %}
|
|
21
|
+
{% endif %}
|
|
22
|
+
|
|
23
|
+
{% endmacro %}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
|
|
2
|
+
{#
|
|
3
|
+
The logic in this file is adapted from dbt-postgres, since DuckDB matches
|
|
4
|
+
the Postgres relation/column commenting model as of 0.10.1
|
|
5
|
+
#}
|
|
6
|
+
|
|
7
|
+
{#
|
|
8
|
+
By using dollar-quoting like this, users can embed anything they want into their comments
|
|
9
|
+
(including nested dollar-quoting), as long as they do not use this exact dollar-quoting
|
|
10
|
+
label. It would be nice to just pick a new one but eventually you do have to give up.
|
|
11
|
+
#}
|
|
12
|
+
{% macro duckdb_escape_comment(comment) -%}
|
|
13
|
+
{% if comment is not string %}
|
|
14
|
+
{% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}
|
|
15
|
+
{% endif %}
|
|
16
|
+
{%- set magic = '$dbt_comment_literal_block$' -%}
|
|
17
|
+
{%- if magic in comment -%}
|
|
18
|
+
{%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}
|
|
19
|
+
{%- endif -%}
|
|
20
|
+
{{ magic }}{{ comment }}{{ magic }}
|
|
21
|
+
{%- endmacro %}
|
|
22
|
+
|
|
23
|
+
{% macro altertable__alter_relation_comment(relation, comment) %}
|
|
24
|
+
{% set escaped_comment = duckdb_escape_comment(comment) %}
|
|
25
|
+
comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};
|
|
26
|
+
{% endmacro %}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
{% macro altertable__alter_column_comment(relation, column_dict) %}
|
|
30
|
+
{% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %}
|
|
31
|
+
{% for column_name in column_dict if (column_name in existing_columns) %}
|
|
32
|
+
{% set comment = column_dict[column_name]['description'] %}
|
|
33
|
+
{% set escaped_comment = duckdb_escape_comment(comment) %}
|
|
34
|
+
comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};
|
|
35
|
+
{% endfor %}
|
|
36
|
+
{% endmacro %}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
|
|
2
|
+
{% macro altertable__get_binding_char() %}
|
|
3
|
+
{{ return(adapter.get_binding_char()) }}
|
|
4
|
+
{% endmacro %}
|
|
5
|
+
|
|
6
|
+
{% macro altertable__get_batch_size() %}
|
|
7
|
+
{{ return(10000) }}
|
|
8
|
+
{% endmacro %}
|
|
9
|
+
|
|
10
|
+
{% macro altertable__load_csv_rows(model, agate_table) %}
|
|
11
|
+
{% if config.get('fast', true) %}
|
|
12
|
+
{% set seed_file_path = adapter.get_seed_file_path(model) %}
|
|
13
|
+
{% set delimiter = config.get('delimiter', ',') %}
|
|
14
|
+
{% set sql %}
|
|
15
|
+
COPY {{ this.render() }} FROM '{{ seed_file_path }}' (FORMAT CSV, HEADER TRUE, DELIMITER '{{ delimiter }}')
|
|
16
|
+
{% endset %}
|
|
17
|
+
{% do adapter.add_query(sql, abridge_sql_log=True) %}
|
|
18
|
+
{{ return(sql) }}
|
|
19
|
+
{% endif %}
|
|
20
|
+
|
|
21
|
+
{% set batch_size = get_batch_size() %}
|
|
22
|
+
{% set agate_table = adapter.convert_datetimes_to_strs(agate_table) %}
|
|
23
|
+
{% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
|
|
24
|
+
{% set bindings = [] %}
|
|
25
|
+
|
|
26
|
+
{% set statements = [] %}
|
|
27
|
+
|
|
28
|
+
{% for chunk in agate_table.rows | batch(batch_size) %}
|
|
29
|
+
{% set bindings = [] %}
|
|
30
|
+
|
|
31
|
+
{% for row in chunk %}
|
|
32
|
+
{% do bindings.extend(row) %}
|
|
33
|
+
{% endfor %}
|
|
34
|
+
|
|
35
|
+
{% set sql %}
|
|
36
|
+
insert into {{ this.render() }} ({{ cols_sql }}) values
|
|
37
|
+
{% for row in chunk -%}
|
|
38
|
+
({%- for column in agate_table.column_names -%}
|
|
39
|
+
{{ get_binding_char() }}
|
|
40
|
+
{%- if not loop.last%},{%- endif %}
|
|
41
|
+
{%- endfor -%})
|
|
42
|
+
{%- if not loop.last%},{%- endif %}
|
|
43
|
+
{%- endfor %}
|
|
44
|
+
{% endset %}
|
|
45
|
+
|
|
46
|
+
{% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
|
|
47
|
+
|
|
48
|
+
{% if loop.index0 == 0 %}
|
|
49
|
+
{% do statements.append(sql) %}
|
|
50
|
+
{% endif %}
|
|
51
|
+
{% endfor %}
|
|
52
|
+
|
|
53
|
+
{# Return SQL so we can render it out into the compiled files #}
|
|
54
|
+
{{ return(statements[0]) }}
|
|
55
|
+
{% endmacro %}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{% macro altertable__snapshot_merge_sql(target, source, insert_cols) -%}
|
|
2
|
+
{%- set insert_cols_csv = insert_cols | join(', ') -%}
|
|
3
|
+
|
|
4
|
+
{%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%}
|
|
5
|
+
|
|
6
|
+
update {{ target }} as DBT_INTERNAL_TARGET
|
|
7
|
+
set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}
|
|
8
|
+
from {{ source }} as DBT_INTERNAL_SOURCE
|
|
9
|
+
where DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }}::text = DBT_INTERNAL_TARGET.{{ columns.dbt_scd_id }}::text
|
|
10
|
+
and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)
|
|
11
|
+
{% if config.get("dbt_valid_to_current") %}
|
|
12
|
+
and (DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} is null);
|
|
13
|
+
{% else %}
|
|
14
|
+
and DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} is null;
|
|
15
|
+
{% endif %}
|
|
16
|
+
|
|
17
|
+
insert into {{ target }} ({{ insert_cols_csv }})
|
|
18
|
+
select {% for column in insert_cols -%}
|
|
19
|
+
DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}
|
|
20
|
+
{%- endfor %}
|
|
21
|
+
from {{ source }} as DBT_INTERNAL_SOURCE
|
|
22
|
+
where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;
|
|
23
|
+
|
|
24
|
+
{% endmacro %}
|
|
25
|
+
|
|
26
|
+
{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}
|
|
27
|
+
{% set temp_relation = make_temp_relation(target_relation) %}
|
|
28
|
+
|
|
29
|
+
{% set select = snapshot_staging_table(strategy, sql, target_relation) %}
|
|
30
|
+
|
|
31
|
+
{% call statement('build_snapshot_staging_relation') %}
|
|
32
|
+
{{ create_table_as(False, temp_relation, select) }}
|
|
33
|
+
{% endcall %}
|
|
34
|
+
|
|
35
|
+
{% do return(temp_relation) %}
|
|
36
|
+
{% endmacro %}
|
|
37
|
+
|
|
38
|
+
{% macro altertable__post_snapshot(staging_relation) %}
|
|
39
|
+
{% do return(drop_relation(staging_relation)) %}
|
|
40
|
+
{% endmacro %}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{% macro altertable__dateadd(datepart, interval, from_date_or_timestamp) %}
|
|
2
|
+
|
|
3
|
+
{#
|
|
4
|
+
Support both literal and expression intervals (e.g., column references)
|
|
5
|
+
by multiplying an INTERVAL by the value. This avoids DuckDB parser issues
|
|
6
|
+
with "interval (<expr>) <unit>" and works across versions.
|
|
7
|
+
|
|
8
|
+
Also map unsupported units:
|
|
9
|
+
- quarter => 3 months
|
|
10
|
+
- week => 7 days (DuckDB supports WEEK as a literal, but keep it explicit)
|
|
11
|
+
#}
|
|
12
|
+
|
|
13
|
+
{%- set unit = datepart | lower -%}
|
|
14
|
+
{%- if unit == 'quarter' -%}
|
|
15
|
+
({{ from_date_or_timestamp }} + (cast({{ interval }} as bigint) * 3) * interval 1 month)
|
|
16
|
+
{%- elif unit == 'week' -%}
|
|
17
|
+
({{ from_date_or_timestamp }} + (cast({{ interval }} as bigint) * 7) * interval 1 day)
|
|
18
|
+
{%- else -%}
|
|
19
|
+
({{ from_date_or_timestamp }} + cast({{ interval }} as bigint) * interval 1 {{ unit }})
|
|
20
|
+
{%- endif -%}
|
|
21
|
+
|
|
22
|
+
{% endmacro %}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{% macro altertable__datediff(first_date, second_date, datepart) -%}
|
|
2
|
+
{% if datepart == 'week' %}
|
|
3
|
+
({{ datediff(first_date, second_date, 'day') }} // 7 + case
|
|
4
|
+
when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then
|
|
5
|
+
case when {{first_date}} <= {{second_date}} then 0 else -1 end
|
|
6
|
+
else
|
|
7
|
+
case when {{first_date}} <= {{second_date}} then 1 else 0 end
|
|
8
|
+
end)
|
|
9
|
+
{% else %}
|
|
10
|
+
(date_diff('{{ datepart }}', {{ first_date }}::timestamp, {{ second_date}}::timestamp ))
|
|
11
|
+
{% endif %}
|
|
12
|
+
{%- endmacro %}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
{%- macro external_location(relation, config) -%}
|
|
2
|
+
{%- if config.get('options', {}).get('partition_by') is none -%}
|
|
3
|
+
{%- set format = config.get('format', 'parquet') -%}
|
|
4
|
+
{{- adapter.external_root() }}/{{ relation.identifier }}.{{ format }}
|
|
5
|
+
{%- else -%}
|
|
6
|
+
{{- adapter.external_root() }}/{{ relation.identifier }}
|
|
7
|
+
{%- endif -%}
|
|
8
|
+
{%- endmacro -%}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{% macro altertable__last_day(date, datepart) -%}
|
|
2
|
+
|
|
3
|
+
{%- if datepart == 'quarter' -%}
|
|
4
|
+
-- duckdb dateadd does not support quarter interval.
|
|
5
|
+
cast(
|
|
6
|
+
{{dbt.dateadd('day', '-1',
|
|
7
|
+
dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))
|
|
8
|
+
)}}
|
|
9
|
+
as date)
|
|
10
|
+
{%- else -%}
|
|
11
|
+
{{dbt.default_last_day(date, datepart)}}
|
|
12
|
+
{%- endif -%}
|
|
13
|
+
|
|
14
|
+
{%- endmacro %}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{% macro altertable__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}
|
|
2
|
+
{% if limit_num -%}
|
|
3
|
+
list_aggr(
|
|
4
|
+
(array_agg(
|
|
5
|
+
{{ measure }}
|
|
6
|
+
{% if order_by_clause -%}
|
|
7
|
+
{{ order_by_clause }}
|
|
8
|
+
{%- endif %}
|
|
9
|
+
))[1:{{ limit_num }}],
|
|
10
|
+
'string_agg',
|
|
11
|
+
{{ delimiter_text }}
|
|
12
|
+
)
|
|
13
|
+
{%- else %}
|
|
14
|
+
string_agg(
|
|
15
|
+
{{ measure }},
|
|
16
|
+
{{ delimiter_text }}
|
|
17
|
+
{% if order_by_clause -%}
|
|
18
|
+
{{ order_by_clause }}
|
|
19
|
+
{%- endif %}
|
|
20
|
+
)
|
|
21
|
+
{%- endif %}
|
|
22
|
+
{%- endmacro %}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{%- macro register_upstream_external_models() -%}
|
|
2
|
+
{% if execute %}
|
|
3
|
+
{% set upstream_nodes = {} %}
|
|
4
|
+
{% set upstream_schemas = {} %}
|
|
5
|
+
{% for node in selected_resources %}
|
|
6
|
+
{% if node not in graph['nodes'] %}{% continue %}{% endif %}
|
|
7
|
+
{% for upstream_node in graph['nodes'][node]['depends_on']['nodes'] %}
|
|
8
|
+
{% if upstream_node not in upstream_nodes and upstream_node not in selected_resources %}
|
|
9
|
+
{% do upstream_nodes.update({upstream_node: None}) %}
|
|
10
|
+
{% set upstream = graph['nodes'].get(upstream_node) %}
|
|
11
|
+
{% if upstream
|
|
12
|
+
and upstream.resource_type in ('model', 'seed')
|
|
13
|
+
and upstream.config.materialized=='external'
|
|
14
|
+
%}
|
|
15
|
+
{%- set upstream_rel = api.Relation.create(
|
|
16
|
+
database=upstream['database'],
|
|
17
|
+
schema=upstream['schema'],
|
|
18
|
+
identifier=upstream['alias']
|
|
19
|
+
) -%}
|
|
20
|
+
{%- set location = upstream.config.get('location', external_location(upstream_rel, upstream.config)) -%}
|
|
21
|
+
{%- set rendered_options = render_write_options(upstream.config) -%}
|
|
22
|
+
{%- set upstream_location = adapter.external_read_location(location, rendered_options) -%}
|
|
23
|
+
{% if upstream_rel.schema not in upstream_schemas %}
|
|
24
|
+
{% call statement('main', language='sql') -%}
|
|
25
|
+
create schema if not exists {{ upstream_rel.without_identifier() }}
|
|
26
|
+
{%- endcall %}
|
|
27
|
+
{% do upstream_schemas.update({upstream_rel.schema: None}) %}
|
|
28
|
+
{% endif %}
|
|
29
|
+
{% call statement('main', language='sql') -%}
|
|
30
|
+
create or replace view {{ upstream_rel }} as (
|
|
31
|
+
select * from '{{ upstream_location }}'
|
|
32
|
+
);
|
|
33
|
+
{%- endcall %}
|
|
34
|
+
{%- endif %}
|
|
35
|
+
{% endif %}
|
|
36
|
+
{% endfor %}
|
|
37
|
+
{% endfor %}
|
|
38
|
+
{% if upstream_nodes %}
|
|
39
|
+
{% do adapter.commit() %}
|
|
40
|
+
{% endif %}
|
|
41
|
+
{% endif %}
|
|
42
|
+
{%- endmacro -%}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dbt-altertable
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: DBT adapter for Altertable
|
|
5
|
+
Author-email: Altertable Team <leo@altertable.ai>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/altertable/dbt-altertable
|
|
8
|
+
Project-URL: Repository, https://github.com/altertable/dbt-altertable
|
|
9
|
+
Project-URL: Documentation, https://github.com/altertable/dbt-altertable#readme
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/altertable/dbt-altertable/issues
|
|
11
|
+
Keywords: dbt,altertable,adapter,database,duckdb
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Database
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: dbt-core>=1.5.0
|
|
26
|
+
Requires-Dist: dbt-adapters>=0.1.0
|
|
27
|
+
Requires-Dist: altertable-flightsql>=0.2.1
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: pytest-mock>=3.10.0; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff>=0.14; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# dbt-altertable
|
|
35
|
+
|
|
36
|
+
A dbt adapter for Altertable.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
Install the package using pip:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install dbt-altertable
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Or install from source:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
git clone <repository-url>
|
|
50
|
+
cd dbt-altertable
|
|
51
|
+
pip install .
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Configuration
|
|
55
|
+
|
|
56
|
+
Add the following to your `profiles.yml`:
|
|
57
|
+
|
|
58
|
+
```yaml
|
|
59
|
+
your_profile_name:
|
|
60
|
+
target: dev
|
|
61
|
+
outputs:
|
|
62
|
+
dev:
|
|
63
|
+
type: altertable
|
|
64
|
+
username: your_username
|
|
65
|
+
password: your_password
|
|
66
|
+
database: your_database
|
|
67
|
+
schema: your_schema
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Credentials Fields
|
|
71
|
+
|
|
72
|
+
- **username** (required): Your Altertable username
|
|
73
|
+
- **password** (required): Your Altertable password
|
|
74
|
+
- **database** (required): Target database name
|
|
75
|
+
- **schema** (required): Target schema name
|
|
76
|
+
|
|
77
|
+
## SQL Dialect
|
|
78
|
+
|
|
79
|
+
dbt models should use **DuckDB-compatible SQL syntax**. Altertable uses DuckDB as its query engine, so you can leverage all DuckDB SQL features and functions in your models.
|
|
80
|
+
|
|
81
|
+
For DuckDB SQL reference, see: https://duckdb.org/docs/sql/introduction
|
|
82
|
+
|
|
83
|
+
## Credits
|
|
84
|
+
|
|
85
|
+
This adapter is built upon the excellent work of the [dbt-duckdb](https://github.com/duckdb/dbt-duckdb) project.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
dbt/adapters/altertable/__init__.py,sha256=aiDnzI0isB4uL3Egz_rug60LMBteYPcONs2D3DNg_nE,347
|
|
2
|
+
dbt/adapters/altertable/__version__.py,sha256=aOHawL1zuHMfBWKXqwUkXcW96oXLNCY-CXdHDqkz4g4,18
|
|
3
|
+
dbt/adapters/altertable/connections.py,sha256=y7400jr1BNgT3R1nNBGeMAsImfbuOUIeSk-0cFEN6Qk,8412
|
|
4
|
+
dbt/adapters/altertable/impl.py,sha256=mRNdOKRKNCKmmqFPQWFEb_pZYl-gCq5gdyvAjngE-ps,283
|
|
5
|
+
dbt/include/altertable/__init__.py,sha256=vBGWeG-dHHkimfnX8axBJ4IgAowFw8xADmo6Auzn2xc,52
|
|
6
|
+
dbt/include/altertable/dbt_project.yml,sha256=7Prucdcneyyo1yyvIZo_FCc_U-IrEONnnJw1YK86bso,77
|
|
7
|
+
dbt/include/altertable/macros/adapters.sql,sha256=lz2GuFHEufDU3C3wgG3-IyFg_YnAo49ldLEJn6RFJKI,9747
|
|
8
|
+
dbt/include/altertable/macros/catalog.sql,sha256=KRs9iEWDIANzvGjWOr4YaiOnzAV6BC_CL4W9gAnX9N8,1396
|
|
9
|
+
dbt/include/altertable/macros/columns.sql,sha256=kJKaEJWX4CEoJv-l5aOcywj2ejXXa7QMncyb0SdE-RI,711
|
|
10
|
+
dbt/include/altertable/macros/persist_docs.sql,sha256=DTZrfG-5AUSiEDkBGEC-IUB_Q6aIUgMqr7UBiIxkEBU,1628
|
|
11
|
+
dbt/include/altertable/macros/seed.sql,sha256=bwwSGoXMrWHRHM5d8QZ75QnZgTbvNexJa9kv5p66h68,1861
|
|
12
|
+
dbt/include/altertable/macros/snapshot_helper.sql,sha256=QVezsIkYk62csDLrGdXoEoUEd5nn2RAqUuZ_01rCQfQ,1793
|
|
13
|
+
dbt/include/altertable/macros/utils/any_value.sql,sha256=sRgSBSeUnXyaqOy8ZSDHDHYdCdiY6mjnl5y2_1WH83Q,97
|
|
14
|
+
dbt/include/altertable/macros/utils/dateadd.sql,sha256=m7hSTbZe0Pc_92jxmF8o2Uk6pW4Pj4Qp8JLUV2DCDBg,906
|
|
15
|
+
dbt/include/altertable/macros/utils/datediff.sql,sha256=TTO12Yc-7U0YYlkwM5rdA__Ksf76ihX_gJNxLWGLRGo,622
|
|
16
|
+
dbt/include/altertable/macros/utils/external_location.sql,sha256=mqJ5qQbIb8pVe9EjHYJJUSHjpIY_4_rc8HnFWMigw6Q,357
|
|
17
|
+
dbt/include/altertable/macros/utils/generate_series.sql,sha256=B0ZB6406uW1OZwpqAjf-qySO0CbN7BbNjhdeCqoHuv4,170
|
|
18
|
+
dbt/include/altertable/macros/utils/lastday.sql,sha256=fL3CjkTH-q-U2fsJNeuQ_J0eLAMQeSmU9jJq09f4s8c,382
|
|
19
|
+
dbt/include/altertable/macros/utils/listagg.sql,sha256=_1gnxr4dMhgIpFqMsbOT56_ebcavYXJ7Dud8beGGJwA,572
|
|
20
|
+
dbt/include/altertable/macros/utils/splitpart.sql,sha256=q6cWKfZYmQvE_3rE-zgwFqv_74G7NdZfKWAHWAErik0,171
|
|
21
|
+
dbt/include/altertable/macros/utils/upstream.sql,sha256=rzTve-UPACtc6LNcNMVaJIm8lBttTsAxXo02zTsQq80,1798
|
|
22
|
+
dbt_altertable-0.1.0.dist-info/licenses/LICENSE,sha256=va_NkzryPHPb1cPczNYjG-5tPY7LoHMPB3FouVr1aFs,1072
|
|
23
|
+
dbt_altertable-0.1.0.dist-info/METADATA,sha256=DfxLp0khhdXdKR3FVvRePUQ4Uuv-7Ru7---dJfMDNvY,2488
|
|
24
|
+
dbt_altertable-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
25
|
+
dbt_altertable-0.1.0.dist-info/top_level.txt,sha256=B2YH4he17ajilEWOGCKHbRcEJlCuZKwCcgFcLPntLsE,4
|
|
26
|
+
dbt_altertable-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Altertable Team
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dbt
|