dbt-altertable 0.2.7__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/PKG-INFO +46 -14
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/README.md +44 -13
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/adapters/altertable/__init__.py +1 -0
- dbt_altertable-0.2.9/dbt/adapters/altertable/__version__.py +1 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/adapters/altertable/connections.py +1 -0
- dbt_altertable-0.2.9/dbt/adapters/altertable/impl.py +118 -0
- dbt_altertable-0.2.9/dbt/adapters/altertable/relation.py +11 -0
- dbt_altertable-0.2.9/dbt/include/altertable/logs/query_log.sql +0 -0
- dbt_altertable-0.2.9/dbt/include/altertable/macros/adapters.sql +213 -0
- dbt_altertable-0.2.9/dbt/include/altertable/macros/merge.sql +13 -0
- dbt_altertable-0.2.9/dbt/include/altertable/macros/microbatch.sql +43 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/macros/persist_docs.sql +0 -22
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/macros/seed.sql +0 -4
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/macros/snapshot_helper.sql +10 -5
- dbt_altertable-0.2.9/dbt/include/altertable/macros/unsupported.sql +32 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/pyproject.toml +1 -0
- dbt_altertable-0.2.7/dbt/adapters/altertable/__version__.py +0 -1
- dbt_altertable-0.2.7/dbt/adapters/altertable/impl.py +0 -21
- dbt_altertable-0.2.7/dbt/include/altertable/macros/adapters.sql +0 -343
- dbt_altertable-0.2.7/dbt/include/altertable/macros/columns.sql +0 -23
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/any_value.sql +0 -5
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/dateadd.sql +0 -22
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/datediff.sql +0 -12
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/external_location.sql +0 -8
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/generate_series.sql +0 -5
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/lastday.sql +0 -14
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/listagg.sql +0 -22
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/splitpart.sql +0 -3
- dbt_altertable-0.2.7/dbt/include/altertable/macros/utils/upstream.sql +0 -42
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/.gitignore +0 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/LICENSE +0 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/__init__.py +0 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/adapters/altertable/credentials.py +0 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/adapters/altertable/metricflow.py +0 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/__init__.py +0 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/dbt_project.yml +0 -0
- {dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/macros/catalog.sql +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: dbt-altertable
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: DBT adapter for Altertable
|
|
5
5
|
Project-URL: Homepage, https://github.com/altertable-ai/dbt-altertable
|
|
6
6
|
Project-URL: Repository, https://github.com/altertable-ai/dbt-altertable
|
|
@@ -25,6 +25,7 @@ Requires-Python: >=3.10
|
|
|
25
25
|
Requires-Dist: altertable-flightsql>=0.3.1
|
|
26
26
|
Requires-Dist: dbt-adapters<2.0,>=1.5
|
|
27
27
|
Requires-Dist: dbt-core<2.0,>=1.8
|
|
28
|
+
Requires-Dist: dbt-duckdb<1.12,>=1.11
|
|
28
29
|
Requires-Dist: pyarrow>=14
|
|
29
30
|
Provides-Extra: dev
|
|
30
31
|
Requires-Dist: conventional-pre-commit>=4.4; extra == 'dev'
|
|
@@ -80,28 +81,59 @@ my_project:
|
|
|
80
81
|
password: your_password
|
|
81
82
|
database: your_database
|
|
82
83
|
schema: your_schema
|
|
83
|
-
host: flight.altertable.ai
|
|
84
|
-
port: 443
|
|
85
|
-
tls: true
|
|
84
|
+
host: flight.altertable.ai # optional, this is the default
|
|
85
|
+
port: 443 # optional, this is the default
|
|
86
|
+
tls: true # optional, this is the default
|
|
86
87
|
```
|
|
87
88
|
|
|
88
|
-
| Field
|
|
89
|
-
|
|
|
90
|
-
| `username` | yes
|
|
91
|
-
| `password` | yes
|
|
92
|
-
| `database` | yes
|
|
93
|
-
| `schema`
|
|
94
|
-
| `host`
|
|
95
|
-
| `port`
|
|
96
|
-
| `tls`
|
|
89
|
+
| Field | Required | Default | Description |
|
|
90
|
+
| ---------- | -------- | ---------------------- | ------------------------------------- |
|
|
91
|
+
| `username` | yes | — | Altertable username |
|
|
92
|
+
| `password` | yes | — | Altertable password |
|
|
93
|
+
| `database` | yes | — | Target catalog name |
|
|
94
|
+
| `schema` | yes | — | Target schema name |
|
|
95
|
+
| `host` | no | `flight.altertable.ai` | Flight SQL endpoint host |
|
|
96
|
+
| `port` | no | `443` | Flight SQL endpoint port |
|
|
97
|
+
| `tls` | no | `true` | Use TLS for the Flight SQL connection |
|
|
97
98
|
|
|
98
99
|
## SQL dialect
|
|
99
100
|
|
|
100
101
|
dbt models should use **DuckDB-compatible SQL**. Altertable executes queries via DuckDB, so all DuckDB SQL features and functions are available — see the [DuckDB SQL reference](https://duckdb.org/docs/sql/introduction).
|
|
101
102
|
|
|
103
|
+
## Compatibility with dbt-duckdb
|
|
104
|
+
|
|
105
|
+
The adapter builds on [dbt-duckdb](https://github.com/duckdb/dbt-duckdb): `duckdb__` macros from your project and packages (dbt_utils, dbt_date, ...) are used automatically, so projects written for DuckDB run unchanged.
|
|
106
|
+
|
|
107
|
+
## Incremental strategies
|
|
108
|
+
|
|
109
|
+
| Strategy | Notes |
|
|
110
|
+
| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
|
111
|
+
| `append` | Inserts new rows. |
|
|
112
|
+
| `delete+insert` (default) | Deletes rows matching `unique_key`, then inserts. |
|
|
113
|
+
| `merge` | `MERGE` on `unique_key` with dbt-duckdb's `merge_*` configs; at most one UPDATE or DELETE action, no `merge_returning_columns`. |
|
|
114
|
+
| `microbatch` | Replaces each `event_time` batch; batches run one at a time. |
|
|
115
|
+
|
|
116
|
+
## Partitioning and sort order
|
|
117
|
+
|
|
118
|
+
Table and incremental models accept `partitioned_by` and `sorted_by`, applied when the table is created or fully refreshed:
|
|
119
|
+
|
|
120
|
+
```sql
|
|
121
|
+
{{ config(materialized='table', partitioned_by=['year(event_at)', 'country'], sorted_by=['event_at']) }}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Unsupported on Altertable
|
|
125
|
+
|
|
126
|
+
These fail with an explicit error instead of silently producing a different model:
|
|
127
|
+
|
|
128
|
+
- `indexes` config
|
|
129
|
+
- Contract constraints other than `not_null` (`primary_key`, `unique`, `foreign_key`, `check`)
|
|
130
|
+
- `grants` config
|
|
131
|
+
- `external` and `table_function` materializations
|
|
132
|
+
- Python models
|
|
133
|
+
|
|
102
134
|
## Persisting model and column descriptions
|
|
103
135
|
|
|
104
|
-
When you enable [persist_docs](https://docs.getdbt.com/
|
|
136
|
+
When you enable [persist_docs](https://docs.getdbt.com/reference/resource-configs/persist_docs), dbt writes model and column `description` values to the warehouse using DuckDB’s `COMMENT ON TABLE` / `COMMENT ON COLUMN` syntax (one statement per column so Arrow Flight SQL accepts each round-trip).
|
|
105
137
|
|
|
106
138
|
Enable it in `dbt_project.yml` or on a model:
|
|
107
139
|
|
|
@@ -38,28 +38,59 @@ my_project:
|
|
|
38
38
|
password: your_password
|
|
39
39
|
database: your_database
|
|
40
40
|
schema: your_schema
|
|
41
|
-
host: flight.altertable.ai
|
|
42
|
-
port: 443
|
|
43
|
-
tls: true
|
|
41
|
+
host: flight.altertable.ai # optional, this is the default
|
|
42
|
+
port: 443 # optional, this is the default
|
|
43
|
+
tls: true # optional, this is the default
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
-
| Field
|
|
47
|
-
|
|
|
48
|
-
| `username` | yes
|
|
49
|
-
| `password` | yes
|
|
50
|
-
| `database` | yes
|
|
51
|
-
| `schema`
|
|
52
|
-
| `host`
|
|
53
|
-
| `port`
|
|
54
|
-
| `tls`
|
|
46
|
+
| Field | Required | Default | Description |
|
|
47
|
+
| ---------- | -------- | ---------------------- | ------------------------------------- |
|
|
48
|
+
| `username` | yes | — | Altertable username |
|
|
49
|
+
| `password` | yes | — | Altertable password |
|
|
50
|
+
| `database` | yes | — | Target catalog name |
|
|
51
|
+
| `schema` | yes | — | Target schema name |
|
|
52
|
+
| `host` | no | `flight.altertable.ai` | Flight SQL endpoint host |
|
|
53
|
+
| `port` | no | `443` | Flight SQL endpoint port |
|
|
54
|
+
| `tls` | no | `true` | Use TLS for the Flight SQL connection |
|
|
55
55
|
|
|
56
56
|
## SQL dialect
|
|
57
57
|
|
|
58
58
|
dbt models should use **DuckDB-compatible SQL**. Altertable executes queries via DuckDB, so all DuckDB SQL features and functions are available — see the [DuckDB SQL reference](https://duckdb.org/docs/sql/introduction).
|
|
59
59
|
|
|
60
|
+
## Compatibility with dbt-duckdb
|
|
61
|
+
|
|
62
|
+
The adapter builds on [dbt-duckdb](https://github.com/duckdb/dbt-duckdb): `duckdb__` macros from your project and packages (dbt_utils, dbt_date, ...) are used automatically, so projects written for DuckDB run unchanged.
|
|
63
|
+
|
|
64
|
+
## Incremental strategies
|
|
65
|
+
|
|
66
|
+
| Strategy | Notes |
|
|
67
|
+
| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
|
68
|
+
| `append` | Inserts new rows. |
|
|
69
|
+
| `delete+insert` (default) | Deletes rows matching `unique_key`, then inserts. |
|
|
70
|
+
| `merge` | `MERGE` on `unique_key` with dbt-duckdb's `merge_*` configs; at most one UPDATE or DELETE action, no `merge_returning_columns`. |
|
|
71
|
+
| `microbatch` | Replaces each `event_time` batch; batches run one at a time. |
|
|
72
|
+
|
|
73
|
+
## Partitioning and sort order
|
|
74
|
+
|
|
75
|
+
Table and incremental models accept `partitioned_by` and `sorted_by`, applied when the table is created or fully refreshed:
|
|
76
|
+
|
|
77
|
+
```sql
|
|
78
|
+
{{ config(materialized='table', partitioned_by=['year(event_at)', 'country'], sorted_by=['event_at']) }}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Unsupported on Altertable
|
|
82
|
+
|
|
83
|
+
These fail with an explicit error instead of silently producing a different model:
|
|
84
|
+
|
|
85
|
+
- `indexes` config
|
|
86
|
+
- Contract constraints other than `not_null` (`primary_key`, `unique`, `foreign_key`, `check`)
|
|
87
|
+
- `grants` config
|
|
88
|
+
- `external` and `table_function` materializations
|
|
89
|
+
- Python models
|
|
90
|
+
|
|
60
91
|
## Persisting model and column descriptions
|
|
61
92
|
|
|
62
|
-
When you enable [persist_docs](https://docs.getdbt.com/
|
|
93
|
+
When you enable [persist_docs](https://docs.getdbt.com/reference/resource-configs/persist_docs), dbt writes model and column `description` values to the warehouse using DuckDB’s `COMMENT ON TABLE` / `COMMENT ON COLUMN` syntax (one statement per column so Arrow Flight SQL accepts each round-trip).
|
|
63
94
|
|
|
64
95
|
Enable it in `dbt_project.yml` or on a model:
|
|
65
96
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = "0.2.9" # x-release-please-version
|
|
@@ -255,6 +255,7 @@ class AltertableConnection:
|
|
|
255
255
|
def __init__(self, client: altertable_flightsql.Client) -> None:
|
|
256
256
|
self._client = client
|
|
257
257
|
self._transaction: Transaction | None = None
|
|
258
|
+
self.ducklake_catalog_names: set[str] | None = None
|
|
258
259
|
|
|
259
260
|
def __enter__(self) -> "AltertableConnection":
|
|
260
261
|
return self
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import threading
|
|
3
|
+
from functools import cached_property
|
|
4
|
+
from typing import Any, ClassVar
|
|
5
|
+
|
|
6
|
+
from dbt.adapters.base import available
|
|
7
|
+
from dbt.adapters.base.impl import ConstraintSupport
|
|
8
|
+
from dbt.adapters.duckdb.constants import DUCKLAKE_ALTER_RENAME_FIX_VERSION
|
|
9
|
+
from dbt.adapters.events.logging import AdapterLogger
|
|
10
|
+
from dbt.adapters.sql import SQLAdapter
|
|
11
|
+
from dbt_common.contracts.constraints import (
|
|
12
|
+
ColumnLevelConstraint,
|
|
13
|
+
ConstraintType,
|
|
14
|
+
ModelLevelConstraint,
|
|
15
|
+
)
|
|
16
|
+
from dbt_common.exceptions import CompilationError, DbtDatabaseError
|
|
17
|
+
from packaging.version import Version
|
|
18
|
+
|
|
19
|
+
from dbt.adapters.altertable.connections import AltertableConnectionManager
|
|
20
|
+
from dbt.adapters.altertable.relation import AltertableRelation
|
|
21
|
+
|
|
22
|
+
logger = AdapterLogger("Altertable")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AltertableAdapter(SQLAdapter):
|
|
26
|
+
ConnectionManager = AltertableConnectionManager
|
|
27
|
+
Relation = AltertableRelation
|
|
28
|
+
|
|
29
|
+
CONSTRAINT_SUPPORT = {
|
|
30
|
+
ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
|
|
31
|
+
ConstraintType.not_null: ConstraintSupport.ENFORCED,
|
|
32
|
+
ConstraintType.unique: ConstraintSupport.NOT_SUPPORTED,
|
|
33
|
+
ConstraintType.primary_key: ConstraintSupport.NOT_SUPPORTED,
|
|
34
|
+
ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
_warned_messages: ClassVar[set[str]] = set()
|
|
38
|
+
_warned_lock: ClassVar[threading.Lock] = threading.Lock()
|
|
39
|
+
|
|
40
|
+
def valid_incremental_strategies(self) -> list[str]:
|
|
41
|
+
return ["append", "delete+insert", "merge", "microbatch"]
|
|
42
|
+
|
|
43
|
+
@classmethod
|
|
44
|
+
def date_function(cls) -> str:
|
|
45
|
+
return "now()"
|
|
46
|
+
|
|
47
|
+
@classmethod
|
|
48
|
+
def process_parsed_constraint(
|
|
49
|
+
cls,
|
|
50
|
+
parsed_constraint: ColumnLevelConstraint | ModelLevelConstraint,
|
|
51
|
+
render_func,
|
|
52
|
+
) -> str | None:
|
|
53
|
+
if (
|
|
54
|
+
parsed_constraint.type != ConstraintType.custom
|
|
55
|
+
and cls.CONSTRAINT_SUPPORT[parsed_constraint.type] == ConstraintSupport.NOT_SUPPORTED
|
|
56
|
+
):
|
|
57
|
+
raise CompilationError(
|
|
58
|
+
f"{parsed_constraint.type.value} constraints are not supported on Altertable."
|
|
59
|
+
)
|
|
60
|
+
return super().process_parsed_constraint(parsed_constraint, render_func)
|
|
61
|
+
|
|
62
|
+
@available
|
|
63
|
+
def get_seed_file_path(self, model) -> str:
|
|
64
|
+
return os.path.join(model["root_path"], model["original_file_path"])
|
|
65
|
+
|
|
66
|
+
@available.parse(lambda relation: False)
|
|
67
|
+
def is_ducklake(self, relation: AltertableRelation | None) -> bool:
|
|
68
|
+
if relation is None or not relation.database:
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
connection = self.connections.get_thread_connection().handle
|
|
72
|
+
if connection.ducklake_catalog_names is None:
|
|
73
|
+
_, cursor = self.connections.add_select_query(
|
|
74
|
+
"select lower(database_name) from duckdb_databases() where type = 'ducklake'"
|
|
75
|
+
)
|
|
76
|
+
connection.ducklake_catalog_names = {row[0] for row in cursor.fetchall()}
|
|
77
|
+
return relation.database.lower() in connection.ducklake_catalog_names
|
|
78
|
+
|
|
79
|
+
@available.parse(lambda relation: False)
|
|
80
|
+
def use_ducklake_table_workarounds(self, relation: AltertableRelation | None) -> bool:
|
|
81
|
+
return self.is_ducklake(relation) and self.server_duckdb_version < Version(
|
|
82
|
+
DUCKLAKE_ALTER_RENAME_FIX_VERSION
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
@available
|
|
86
|
+
def is_motherduck(self) -> bool:
|
|
87
|
+
return False
|
|
88
|
+
|
|
89
|
+
@available
|
|
90
|
+
def disable_transactions(self) -> bool:
|
|
91
|
+
return False
|
|
92
|
+
|
|
93
|
+
@available
|
|
94
|
+
def parse_index(self, raw_index: Any) -> None:
|
|
95
|
+
raise CompilationError(
|
|
96
|
+
"Indexes are not supported on Altertable. Remove the `indexes` config from this model."
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
@available
|
|
100
|
+
def warn_once(self, msg: str) -> None:
|
|
101
|
+
with self._warned_lock:
|
|
102
|
+
if msg in self._warned_messages:
|
|
103
|
+
return
|
|
104
|
+
self._warned_messages.add(msg)
|
|
105
|
+
logger.warning(msg)
|
|
106
|
+
|
|
107
|
+
@available
|
|
108
|
+
def batch_id_for_model(self, model: Any) -> str:
|
|
109
|
+
# Required by dbt-duckdb; only MotherDuck uses the batch suffix.
|
|
110
|
+
return ""
|
|
111
|
+
|
|
112
|
+
@cached_property
|
|
113
|
+
def server_duckdb_version(self) -> Version:
|
|
114
|
+
_, cursor = self.connections.add_select_query("select version()")
|
|
115
|
+
row = cursor.fetchone()
|
|
116
|
+
if not row or not row[0]:
|
|
117
|
+
raise DbtDatabaseError("Unable to determine the Altertable DuckDB version")
|
|
118
|
+
return Version(str(row[0]).lstrip("v"))
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from dbt.adapters.base.relation import BaseRelation
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen=True, eq=False, repr=False)
|
|
7
|
+
class AltertableRelation(BaseRelation):
|
|
8
|
+
# DuckDB accepts a subquery in FROM without an alias. The BaseRelation
|
|
9
|
+
# default (True) makes `dbt build --empty` emit `_dbt_limit_subq_<name>`
|
|
10
|
+
# immediately before an alias the model already wrote.
|
|
11
|
+
require_alias: bool = False
|
|
File without changes
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
{% macro altertable__create_schema(relation) -%}
|
|
2
|
+
{%- call statement('create_schema') -%}
|
|
3
|
+
create schema if not exists {{ relation.without_identifier() }}
|
|
4
|
+
{%- endcall -%}
|
|
5
|
+
{% endmacro %}
|
|
6
|
+
|
|
7
|
+
{# DuckLake does not support DROP ... CASCADE:
|
|
8
|
+
https://ducklake.select/docs/stable/duckdb/unsupported_features.html
|
|
9
|
+
https://github.com/duckdb/ducklake/discussions/1057
|
|
10
|
+
Override the leaf drop macros to omit CASCADE so
|
|
11
|
+
dbt drop-emitting paths stay compatible with DuckLake, including
|
|
12
|
+
adapter.drop_relation (which resolves to drop_table/view/...) and direct
|
|
13
|
+
get_drop_sql calls used by relation replacement and cleanup flows. #}
|
|
14
|
+
{% macro altertable__drop_schema(relation) -%}
|
|
15
|
+
{%- call statement('drop_schema') -%}
|
|
16
|
+
drop schema if exists {{ relation.without_identifier() }}
|
|
17
|
+
{%- endcall -%}
|
|
18
|
+
{% endmacro %}
|
|
19
|
+
|
|
20
|
+
{% macro altertable__drop_relation(relation) -%}
|
|
21
|
+
{{ return(default__drop_relation(relation)) }}
|
|
22
|
+
{%- endmacro %}
|
|
23
|
+
|
|
24
|
+
{% macro altertable__drop_table(relation) -%}
|
|
25
|
+
drop table if exists {{ relation.render() }}
|
|
26
|
+
{%- endmacro %}
|
|
27
|
+
|
|
28
|
+
{% macro altertable__drop_view(relation) -%}
|
|
29
|
+
drop view if exists {{ relation.render() }}
|
|
30
|
+
{%- endmacro %}
|
|
31
|
+
|
|
32
|
+
{% macro altertable__drop_materialized_view(relation) -%}
|
|
33
|
+
drop materialized view if exists {{ relation.render() }}
|
|
34
|
+
{%- endmacro %}
|
|
35
|
+
|
|
36
|
+
{% macro altertable__list_schemas(database) -%}
|
|
37
|
+
{% set sql %}
|
|
38
|
+
select schema_name
|
|
39
|
+
from information_schema.schemata
|
|
40
|
+
{% if database is not none %}
|
|
41
|
+
where lower(catalog_name) = '{{ database | lower | trim('"') }}'
|
|
42
|
+
{% endif %}
|
|
43
|
+
{% endset %}
|
|
44
|
+
{{ return(run_query(sql)) }}
|
|
45
|
+
{% endmacro %}
|
|
46
|
+
|
|
47
|
+
{% macro altertable__check_schema_exists(information_schema, schema) -%}
|
|
48
|
+
{% set sql -%}
|
|
49
|
+
select count(*)
|
|
50
|
+
from information_schema.schemata
|
|
51
|
+
where lower(schema_name) = '{{ schema | lower }}'
|
|
52
|
+
and lower(catalog_name) = '{{ information_schema.database | lower }}'
|
|
53
|
+
{%- endset %}
|
|
54
|
+
{{ return(run_query(sql)) }}
|
|
55
|
+
{% endmacro %}
|
|
56
|
+
|
|
57
|
+
{#
|
|
58
|
+
Flight SQL accepts a single statement per round-trip, so when a table needs more
|
|
59
|
+
than a CREATE TABLE AS (enforced contract, partitioning or sort order),
|
|
60
|
+
the preliminary statements run here and only the final INSERT is returned for the
|
|
61
|
+
materialization's main statement.
|
|
62
|
+
#}
|
|
63
|
+
{% macro altertable__create_table_as(temporary, relation, compiled_code, language='sql', partitioned_by=none, sorted_by=none) -%}
|
|
64
|
+
{%- if language != 'sql' -%}
|
|
65
|
+
{% do exceptions.raise_compiler_error(
|
|
66
|
+
"Python models are not supported on Altertable; got language '" ~ language ~ "'"
|
|
67
|
+
) %}
|
|
68
|
+
{%- endif -%}
|
|
69
|
+
|
|
70
|
+
{%- set contract_config = config.get('contract') -%}
|
|
71
|
+
{%- set enforce_contract = contract_config.enforced and not temporary -%}
|
|
72
|
+
{%- if contract_config.enforced -%}
|
|
73
|
+
{{ get_assert_columns_equivalent(compiled_code) }}
|
|
74
|
+
{%- endif -%}
|
|
75
|
+
{%- set sql_header = config.get('sql_header', none) -%}
|
|
76
|
+
{%- set target = relation.include(database=(not temporary), schema=(not temporary)) -%}
|
|
77
|
+
|
|
78
|
+
{%- if not enforce_contract and not partitioned_by and not sorted_by -%}
|
|
79
|
+
{{ sql_header if sql_header is not none }}
|
|
80
|
+
create {% if temporary: -%}temporary{%- endif %} table {{ target }} as (
|
|
81
|
+
{{ compiled_code }}
|
|
82
|
+
);
|
|
83
|
+
{%- else -%}
|
|
84
|
+
{% call statement('create_empty_table', auto_begin=true) -%}
|
|
85
|
+
{{ sql_header if sql_header is not none }}
|
|
86
|
+
{% if enforce_contract -%}
|
|
87
|
+
create table {{ target }} {{ get_table_columns_and_constraints() }}
|
|
88
|
+
{%- else -%}
|
|
89
|
+
create {% if temporary: -%}temporary{%- endif %} table {{ target }} as (
|
|
90
|
+
select * from (
|
|
91
|
+
{{ compiled_code }}
|
|
92
|
+
) as model_subq limit 0
|
|
93
|
+
)
|
|
94
|
+
{%- endif %}
|
|
95
|
+
{%- endcall %}
|
|
96
|
+
{% if partitioned_by %}
|
|
97
|
+
{% call statement('set_partitioned_by', auto_begin=true) -%}
|
|
98
|
+
{{ duckdb__alter_table_set_partitioned_by(relation, partitioned_by) }}
|
|
99
|
+
{%- endcall %}
|
|
100
|
+
{% endif %}
|
|
101
|
+
{% if sorted_by %}
|
|
102
|
+
{% call statement('set_sorted_by', auto_begin=true) -%}
|
|
103
|
+
{{ duckdb__alter_table_set_sorted_by(relation, sorted_by) }}
|
|
104
|
+
{%- endcall %}
|
|
105
|
+
{% endif %}
|
|
106
|
+
{% if enforce_contract -%}
|
|
107
|
+
insert into {{ target }} {{ get_column_names() }} (
|
|
108
|
+
{{ get_select_subquery(compiled_code) }}
|
|
109
|
+
);
|
|
110
|
+
{%- else -%}
|
|
111
|
+
insert into {{ target }} select * from (
|
|
112
|
+
{{ compiled_code }}
|
|
113
|
+
) as model_subq;
|
|
114
|
+
{%- endif %}
|
|
115
|
+
{%- endif -%}
|
|
116
|
+
{% endmacro %}
|
|
117
|
+
|
|
118
|
+
{% macro altertable__get_columns_in_relation(relation) -%}
|
|
119
|
+
{% call statement('get_columns_in_relation', fetch_result=True) %}
|
|
120
|
+
select
|
|
121
|
+
column_name,
|
|
122
|
+
data_type,
|
|
123
|
+
cast(null as integer) as character_maximum_length,
|
|
124
|
+
cast(null as integer) as numeric_precision,
|
|
125
|
+
cast(null as integer) as numeric_scale
|
|
126
|
+
from duckdb_columns()
|
|
127
|
+
where table_name = '{{ relation.identifier }}'
|
|
128
|
+
{% if relation.schema %}
|
|
129
|
+
and lower(schema_name) = lower('{{ relation.schema }}')
|
|
130
|
+
{% endif %}
|
|
131
|
+
{% if relation.database %}
|
|
132
|
+
and lower(database_name) = lower('{{ relation.database }}')
|
|
133
|
+
{% endif %}
|
|
134
|
+
order by column_index
|
|
135
|
+
{% endcall %}
|
|
136
|
+
{% set table = load_result('get_columns_in_relation').table %}
|
|
137
|
+
{{ return(sql_convert_columns_in_relation(table)) }}
|
|
138
|
+
{% endmacro %}
|
|
139
|
+
|
|
140
|
+
{% macro altertable__list_relations_without_caching(schema_relation) %}
|
|
141
|
+
{% call statement('list_relations_without_caching', fetch_result=True) -%}
|
|
142
|
+
select
|
|
143
|
+
'{{ schema_relation.database }}' as database,
|
|
144
|
+
table_name as name,
|
|
145
|
+
schema_name as schema,
|
|
146
|
+
'table' as type
|
|
147
|
+
from duckdb_tables()
|
|
148
|
+
where lower(schema_name) = '{{ schema_relation.schema | lower }}'
|
|
149
|
+
and lower(database_name) = '{{ schema_relation.database | lower }}'
|
|
150
|
+
union all
|
|
151
|
+
select
|
|
152
|
+
'{{ schema_relation.database }}' as database,
|
|
153
|
+
view_name as name,
|
|
154
|
+
schema_name as schema,
|
|
155
|
+
'view' as type
|
|
156
|
+
from duckdb_views()
|
|
157
|
+
where lower(schema_name) = '{{ schema_relation.schema | lower }}'
|
|
158
|
+
and lower(database_name) = '{{ schema_relation.database | lower }}'
|
|
159
|
+
{% endcall %}
|
|
160
|
+
{{ return(load_result('list_relations_without_caching').table) }}
|
|
161
|
+
{% endmacro %}
|
|
162
|
+
|
|
163
|
+
{% macro altertable__make_temp_relation(base_relation, suffix) %}
|
|
164
|
+
{% set tmp_identifier = base_relation.identifier ~ suffix ~ py_current_timestring() %}
|
|
165
|
+
{% do return(base_relation.incorporate(
|
|
166
|
+
path={
|
|
167
|
+
"identifier": tmp_identifier,
|
|
168
|
+
"schema": none,
|
|
169
|
+
"database": none
|
|
170
|
+
})) -%}
|
|
171
|
+
{% endmacro %}
|
|
172
|
+
|
|
173
|
+
{#
|
|
174
|
+
dbt's default__get_delete_insert_merge_sql returns DELETE and INSERT as one
|
|
175
|
+
semicolon-separated string, which Flight SQL rejects because it accepts a single
|
|
176
|
+
statement per round-trip. We execute the DELETE as an auto-begin statement and
|
|
177
|
+
return only the INSERT for the materialization's main statement. Both statements
|
|
178
|
+
stay identical to dbt's and share the materialization transaction.
|
|
179
|
+
#}
|
|
180
|
+
{% macro altertable__get_incremental_delete_insert_sql(arg_dict) %}
|
|
181
|
+
{%- set target = arg_dict["target_relation"] -%}
|
|
182
|
+
{%- set source = arg_dict["temp_relation"] -%}
|
|
183
|
+
{%- set unique_key = arg_dict["unique_key"] -%}
|
|
184
|
+
{%- set incremental_predicates = arg_dict["incremental_predicates"] -%}
|
|
185
|
+
{%- set dest_cols_csv = get_quoted_csv(arg_dict["dest_columns"] | map(attribute="name")) -%}
|
|
186
|
+
|
|
187
|
+
{%- if unique_key -%}
|
|
188
|
+
{%- if unique_key is string -%}
|
|
189
|
+
{%- set unique_key = [unique_key] -%}
|
|
190
|
+
{%- endif -%}
|
|
191
|
+
{%- set unique_key_str = unique_key | join(", ") -%}
|
|
192
|
+
|
|
193
|
+
{%- set delete_sql -%}
|
|
194
|
+
delete from {{ target }} as DBT_INTERNAL_DEST
|
|
195
|
+
where ({{ unique_key_str }}) in (
|
|
196
|
+
select distinct {{ unique_key_str }}
|
|
197
|
+
from {{ source }} as DBT_INTERNAL_SOURCE
|
|
198
|
+
)
|
|
199
|
+
{%- for predicate in incremental_predicates or [] %}
|
|
200
|
+
and {{ predicate }}
|
|
201
|
+
{%- endfor %}
|
|
202
|
+
{%- endset -%}
|
|
203
|
+
{% call statement("delete_incremental_rows", auto_begin=true) -%}
|
|
204
|
+
{{ delete_sql }}
|
|
205
|
+
{%- endcall %}
|
|
206
|
+
{%- endif %}
|
|
207
|
+
|
|
208
|
+
insert into {{ target }} ({{ dest_cols_csv }})
|
|
209
|
+
(
|
|
210
|
+
select {{ dest_cols_csv }}
|
|
211
|
+
from {{ source }}
|
|
212
|
+
)
|
|
213
|
+
{% endmacro %}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{#- Same checks as dbt-duckdb's validate_ducklake_restrictions, worded for Altertable users. -#}
|
|
2
|
+
{%- macro validate_ducklake_restrictions(config, target_relation, errors) -%}
|
|
3
|
+
{%- if config.get('merge_returning_columns') -%}
|
|
4
|
+
{%- do errors.append("Altertable MERGE restrictions: merge_returning_columns is not supported.") -%}
|
|
5
|
+
{%- endif -%}
|
|
6
|
+
|
|
7
|
+
{%- set merge_clauses = config.get('merge_clauses') or {} -%}
|
|
8
|
+
{%- set configured_clauses = merge_clauses.get('when_matched', []) + merge_clauses.get('when_not_matched', []) -%}
|
|
9
|
+
{%- set update_or_delete = configured_clauses | selectattr('action', 'defined') | selectattr('action', 'in', ['update', 'delete']) | list -%}
|
|
10
|
+
{%- if update_or_delete | length > 1 -%}
|
|
11
|
+
{%- do errors.append("Altertable MERGE restrictions: merge_clauses can contain only a single UPDATE or DELETE action across when_matched and when_not_matched. Found " ~ update_or_delete | length ~ " UPDATE/DELETE actions.") -%}
|
|
12
|
+
{%- endif -%}
|
|
13
|
+
{%- endmacro -%}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
{#
|
|
2
|
+
Same batch semantics as duckdb__get_incremental_microbatch_sql, but the DELETE runs
|
|
3
|
+
as its own statement because Flight SQL accepts a single statement per round-trip.
|
|
4
|
+
#}
|
|
5
|
+
{% macro altertable__get_incremental_microbatch_sql(arg_dict) -%}
|
|
6
|
+
{%- set event_time = config.get('event_time') -%}
|
|
7
|
+
{%- if not event_time -%}
|
|
8
|
+
{{ exceptions.raise_compiler_error("microbatch incremental strategy requires an 'event_time' model config") }}
|
|
9
|
+
{%- endif -%}
|
|
10
|
+
{%- if config.get('unique_key') -%}
|
|
11
|
+
{{ exceptions.raise_compiler_error("microbatch incremental strategy does not support 'unique_key'. Microbatch runs delete+insert per batch based on 'event_time'. Remove 'unique_key' or use incremental_strategy='merge'.") }}
|
|
12
|
+
{%- endif -%}
|
|
13
|
+
|
|
14
|
+
{%- set batch_ctx = model.get('batch') -%}
|
|
15
|
+
{%- set batch_start = batch_ctx.get('event_time_start') if batch_ctx else none -%}
|
|
16
|
+
{%- set batch_end = batch_ctx.get('event_time_end') if batch_ctx else none -%}
|
|
17
|
+
{%- if not (batch_start and batch_end) -%}
|
|
18
|
+
{{ exceptions.raise_compiler_error("microbatch incremental strategy requires 'batch.event_time_start' and 'batch.event_time_end' to be set in the context") }}
|
|
19
|
+
{%- endif -%}
|
|
20
|
+
|
|
21
|
+
{%- set target = arg_dict["target_relation"] -%}
|
|
22
|
+
{%- set source = arg_dict["temp_relation"] -%}
|
|
23
|
+
{%- set incremental_predicates = normalize_incremental_predicates(arg_dict.get("incremental_predicates")) -%}
|
|
24
|
+
{%- set dest_cols_csv = get_quoted_csv(arg_dict["dest_columns"] | map(attribute="name")) -%}
|
|
25
|
+
|
|
26
|
+
{%- set batch_predicate -%}
|
|
27
|
+
{{ event_time }} >= '{{ batch_start }}'
|
|
28
|
+
and {{ event_time }} < '{{ batch_end }}'
|
|
29
|
+
{%- endset -%}
|
|
30
|
+
|
|
31
|
+
{% call statement("delete_microbatch_rows", auto_begin=true) -%}
|
|
32
|
+
delete from {{ target }}
|
|
33
|
+
where {{ batch_predicate }}
|
|
34
|
+
{%- for predicate in incremental_predicates %}
|
|
35
|
+
and ({{ predicate }})
|
|
36
|
+
{%- endfor %}
|
|
37
|
+
{%- endcall %}
|
|
38
|
+
|
|
39
|
+
insert into {{ target }} ({{ dest_cols_csv }})
|
|
40
|
+
select {{ dest_cols_csv }}
|
|
41
|
+
from {{ source }}
|
|
42
|
+
where {{ batch_predicate }}
|
|
43
|
+
{%- endmacro %}
|
{dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/macros/persist_docs.sql
RENAMED
|
@@ -1,25 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
{#
|
|
3
|
-
The logic in this file is adapted from dbt-postgres, since DuckDB matches
|
|
4
|
-
the Postgres relation/column commenting model as of 0.10.1
|
|
5
|
-
#}
|
|
6
|
-
|
|
7
|
-
{#
|
|
8
|
-
By using dollar-quoting like this, users can embed anything they want into their comments
|
|
9
|
-
(including nested dollar-quoting), as long as they do not use this exact dollar-quoting
|
|
10
|
-
label. It would be nice to just pick a new one but eventually you do have to give up.
|
|
11
|
-
#}
|
|
12
|
-
{% macro duckdb_escape_comment(comment) -%}
|
|
13
|
-
{% if comment is not string %}
|
|
14
|
-
{% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}
|
|
15
|
-
{% endif %}
|
|
16
|
-
{%- set magic = '$dbt_comment_literal_block$' -%}
|
|
17
|
-
{%- if magic in comment -%}
|
|
18
|
-
{%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}
|
|
19
|
-
{%- endif -%}
|
|
20
|
-
{{ magic }}{{ comment }}{{ magic }}
|
|
21
|
-
{%- endmacro %}
|
|
22
|
-
|
|
23
1
|
{% macro altertable__alter_relation_comment(relation, comment) -%}
|
|
24
2
|
{% set escaped_comment = duckdb_escape_comment(comment) %}
|
|
25
3
|
comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }}
|
|
@@ -3,10 +3,6 @@
|
|
|
3
3
|
{{ return('?') }}
|
|
4
4
|
{% endmacro %}
|
|
5
5
|
|
|
6
|
-
{% macro altertable__get_batch_size() %}
|
|
7
|
-
{{ return(10000) }}
|
|
8
|
-
{% endmacro %}
|
|
9
|
-
|
|
10
6
|
{% macro altertable__load_csv_rows(model, agate_table) %}
|
|
11
7
|
{% if config.get('fast', false) %}
|
|
12
8
|
{% set seed_file_path = adapter.get_seed_file_path(model) %}
|
{dbt_altertable-0.2.7 → dbt_altertable-0.2.9}/dbt/include/altertable/macros/snapshot_helper.sql
RENAMED
|
@@ -1,26 +1,31 @@
|
|
|
1
|
+
{#
|
|
2
|
+
duckdb__snapshot_merge_sql returns UPDATE and INSERT as one semicolon-separated
|
|
3
|
+
string, which Flight SQL rejects. The UPDATE runs here and only the INSERT is
|
|
4
|
+
returned for the snapshot's main statement, in the same transaction.
|
|
5
|
+
#}
|
|
1
6
|
{% macro altertable__snapshot_merge_sql(target, source, insert_cols) -%}
|
|
2
7
|
{%- set insert_cols_csv = insert_cols | join(', ') -%}
|
|
3
|
-
|
|
4
8
|
{%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%}
|
|
5
9
|
|
|
10
|
+
{% call statement('snapshot_close_changed_rows', auto_begin=true) -%}
|
|
6
11
|
update {{ target }} as DBT_INTERNAL_TARGET
|
|
7
12
|
set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}
|
|
8
13
|
from {{ source }} as DBT_INTERNAL_SOURCE
|
|
9
14
|
where DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }}::text = DBT_INTERNAL_TARGET.{{ columns.dbt_scd_id }}::text
|
|
10
15
|
and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)
|
|
11
16
|
{% if config.get("dbt_valid_to_current") %}
|
|
12
|
-
and (DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} is null)
|
|
17
|
+
and (DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} is null)
|
|
13
18
|
{% else %}
|
|
14
|
-
and DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} is null
|
|
19
|
+
and DBT_INTERNAL_TARGET.{{ columns.dbt_valid_to }} is null
|
|
15
20
|
{% endif %}
|
|
21
|
+
{%- endcall %}
|
|
16
22
|
|
|
17
23
|
insert into {{ target }} ({{ insert_cols_csv }})
|
|
18
24
|
select {% for column in insert_cols -%}
|
|
19
25
|
DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}
|
|
20
26
|
{%- endfor %}
|
|
21
27
|
from {{ source }} as DBT_INTERNAL_SOURCE
|
|
22
|
-
where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text
|
|
23
|
-
|
|
28
|
+
where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text
|
|
24
29
|
{% endmacro %}
|
|
25
30
|
|
|
26
31
|
{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{#
|
|
2
|
+
Features Altertable cannot provide. They fail loudly instead of
|
|
3
|
+
silently producing a model that differs from what the project asked for.
|
|
4
|
+
#}
|
|
5
|
+
|
|
6
|
+
{% macro altertable__apply_grants(relation, grant_config, should_revoke=True) %}
|
|
7
|
+
{% if grant_config %}
|
|
8
|
+
{% do exceptions.raise_compiler_error(
|
|
9
|
+
"Grants are not supported on Altertable; remove the `grants` config from " ~ relation
|
|
10
|
+
) %}
|
|
11
|
+
{% endif %}
|
|
12
|
+
{% endmacro %}
|
|
13
|
+
|
|
14
|
+
{% macro altertable__get_create_index_sql(relation, index_dict) -%}
|
|
15
|
+
{% do adapter.parse_index(index_dict) %}
|
|
16
|
+
{%- endmacro %}
|
|
17
|
+
|
|
18
|
+
{#- Called on every table rebuild; Altertable tables never have indexes to drop. -#}
|
|
19
|
+
{% macro drop_indexes_on_relation(relation) -%}
|
|
20
|
+
{%- endmacro %}
|
|
21
|
+
|
|
22
|
+
{% materialization external, adapter='altertable' %}
|
|
23
|
+
{% do exceptions.raise_compiler_error(
|
|
24
|
+
"The 'external' materialization is not supported on Altertable; use 'table' instead"
|
|
25
|
+
) %}
|
|
26
|
+
{% endmaterialization %}
|
|
27
|
+
|
|
28
|
+
{% materialization table_function, adapter='altertable' %}
|
|
29
|
+
{% do exceptions.raise_compiler_error(
|
|
30
|
+
"The 'table_function' materialization is not supported on Altertable; use 'view' or 'table' instead"
|
|
31
|
+
) %}
|
|
32
|
+
{% endmaterialization %}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
version = "0.2.7" # x-release-please-version
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
|
|
3
|
-
from dbt.adapters.base import available
|
|
4
|
-
from dbt.adapters.sql import SQLAdapter
|
|
5
|
-
|
|
6
|
-
from dbt.adapters.altertable.connections import AltertableConnectionManager
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class AltertableAdapter(SQLAdapter):
|
|
10
|
-
ConnectionManager = AltertableConnectionManager
|
|
11
|
-
|
|
12
|
-
def valid_incremental_strategies(self) -> list[str]:
|
|
13
|
-
return ["append", "delete+insert"]
|
|
14
|
-
|
|
15
|
-
@classmethod
|
|
16
|
-
def date_function(cls) -> str:
|
|
17
|
-
return "now()"
|
|
18
|
-
|
|
19
|
-
@available
|
|
20
|
-
def get_seed_file_path(self, model) -> str:
|
|
21
|
-
return os.path.join(model["root_path"], model["original_file_path"])
|
|
@@ -1,343 +0,0 @@
|
|
|
1
|
-
{% macro altertable__create_schema(relation) -%}
|
|
2
|
-
{%- call statement('create_schema') -%}
|
|
3
|
-
{% set sql %}
|
|
4
|
-
select type from duckdb_databases()
|
|
5
|
-
where lower(database_name)='{{ relation.database | lower }}'
|
|
6
|
-
and type='sqlite'
|
|
7
|
-
{% endset %}
|
|
8
|
-
{% set results = run_query(sql) %}
|
|
9
|
-
{% if results|length == 0 %}
|
|
10
|
-
create schema if not exists {{ relation.without_identifier() }}
|
|
11
|
-
{% else %}
|
|
12
|
-
{% if relation.schema!='main' %}
|
|
13
|
-
{{ exceptions.raise_compiler_error(
|
|
14
|
-
"Schema must be 'main' when writing to sqlite "
|
|
15
|
-
~ "instead got " ~ relation.schema
|
|
16
|
-
)}}
|
|
17
|
-
{% endif %}
|
|
18
|
-
{% endif %}
|
|
19
|
-
{%- endcall -%}
|
|
20
|
-
{% endmacro %}
|
|
21
|
-
|
|
22
|
-
{# DuckLake does not support DROP ... CASCADE:
|
|
23
|
-
https://ducklake.select/docs/stable/duckdb/unsupported_features.html
|
|
24
|
-
https://github.com/duckdb/ducklake/discussions/1057
|
|
25
|
-
Override the leaf drop macros to omit CASCADE so
|
|
26
|
-
dbt drop-emitting paths stay compatible with DuckLake, including
|
|
27
|
-
adapter.drop_relation (which resolves to drop_table/view/...) and direct
|
|
28
|
-
get_drop_sql calls used by relation replacement and cleanup flows. #}
|
|
29
|
-
{% macro altertable__drop_schema(relation) -%}
|
|
30
|
-
{%- call statement('drop_schema') -%}
|
|
31
|
-
drop schema if exists {{ relation.without_identifier() }}
|
|
32
|
-
{%- endcall -%}
|
|
33
|
-
{% endmacro %}
|
|
34
|
-
|
|
35
|
-
{% macro altertable__drop_table(relation) -%}
|
|
36
|
-
drop table if exists {{ relation.render() }}
|
|
37
|
-
{%- endmacro %}
|
|
38
|
-
|
|
39
|
-
{% macro altertable__drop_view(relation) -%}
|
|
40
|
-
drop view if exists {{ relation.render() }}
|
|
41
|
-
{%- endmacro %}
|
|
42
|
-
|
|
43
|
-
{% macro altertable__drop_materialized_view(relation) -%}
|
|
44
|
-
drop materialized view if exists {{ relation.render() }}
|
|
45
|
-
{%- endmacro %}
|
|
46
|
-
|
|
47
|
-
{% macro altertable__list_schemas(database) -%}
|
|
48
|
-
{% set sql %}
|
|
49
|
-
select schema_name
|
|
50
|
-
from information_schema.schemata
|
|
51
|
-
{% if database is not none %}
|
|
52
|
-
where lower(catalog_name) = '{{ database | lower | trim('"') }}'
|
|
53
|
-
{% endif %}
|
|
54
|
-
{% endset %}
|
|
55
|
-
{{ return(run_query(sql)) }}
|
|
56
|
-
{% endmacro %}
|
|
57
|
-
|
|
58
|
-
{% macro altertable__check_schema_exists(information_schema, schema) -%}
|
|
59
|
-
{% set sql -%}
|
|
60
|
-
select count(*)
|
|
61
|
-
from information_schema.schemata
|
|
62
|
-
where lower(schema_name) = '{{ schema | lower }}'
|
|
63
|
-
and lower(catalog_name) = '{{ information_schema.database | lower }}'
|
|
64
|
-
{%- endset %}
|
|
65
|
-
{{ return(run_query(sql)) }}
|
|
66
|
-
{% endmacro %}
|
|
67
|
-
|
|
68
|
-
{% macro get_column_names() %}
|
|
69
|
-
{# loop through user_provided_columns to get column names #}
|
|
70
|
-
{%- set user_provided_columns = model['columns'] -%}
|
|
71
|
-
(
|
|
72
|
-
{% for i in user_provided_columns %}
|
|
73
|
-
{% set col = user_provided_columns[i] %}
|
|
74
|
-
{{ col['name'] }} {{ "," if not loop.last }}
|
|
75
|
-
{% endfor %}
|
|
76
|
-
)
|
|
77
|
-
{% endmacro %}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
{% macro altertable__create_table_as(temporary, relation, compiled_code, language='sql') -%}
|
|
81
|
-
{%- if language == 'sql' -%}
|
|
82
|
-
{% set contract_config = config.get('contract') %}
|
|
83
|
-
{% if contract_config.enforced %}
|
|
84
|
-
{{ get_assert_columns_equivalent(compiled_code) }}
|
|
85
|
-
{% endif %}
|
|
86
|
-
{%- set sql_header = config.get('sql_header', none) -%}
|
|
87
|
-
|
|
88
|
-
{{ sql_header if sql_header is not none }}
|
|
89
|
-
|
|
90
|
-
create {% if temporary: -%}temporary{%- endif %} table
|
|
91
|
-
{{ relation.include(database=(not temporary), schema=(not temporary)) }}
|
|
92
|
-
{% if contract_config.enforced and not temporary %}
|
|
93
|
-
{#-- DuckDB doesnt support constraints on temp tables --#}
|
|
94
|
-
{{ get_table_columns_and_constraints() }} ;
|
|
95
|
-
insert into {{ relation }} {{ get_column_names() }} (
|
|
96
|
-
{{ get_select_subquery(compiled_code) }}
|
|
97
|
-
);
|
|
98
|
-
{% else %}
|
|
99
|
-
as (
|
|
100
|
-
{{ compiled_code }}
|
|
101
|
-
);
|
|
102
|
-
{% endif %}
|
|
103
|
-
{%- elif language == 'python' -%}
|
|
104
|
-
{{ py_write_table(temporary=temporary, relation=relation, compiled_code=compiled_code) }}
|
|
105
|
-
{%- else -%}
|
|
106
|
-
{% do exceptions.raise_compiler_error("altertable__create_table_as macro didn't get supported language, it got %s" % language) %}
|
|
107
|
-
{%- endif -%}
|
|
108
|
-
{% endmacro %}
|
|
109
|
-
|
|
110
|
-
{% macro py_write_table(temporary, relation, compiled_code) -%}
|
|
111
|
-
{{ compiled_code }}
|
|
112
|
-
|
|
113
|
-
def materialize(df, con):
|
|
114
|
-
try:
|
|
115
|
-
import pyarrow
|
|
116
|
-
pyarrow_available = True
|
|
117
|
-
except ImportError:
|
|
118
|
-
pyarrow_available = False
|
|
119
|
-
finally:
|
|
120
|
-
if pyarrow_available and isinstance(df, pyarrow.Table):
|
|
121
|
-
# https://github.com/duckdb/duckdb/issues/6584
|
|
122
|
-
import pyarrow.dataset
|
|
123
|
-
tmp_name = '__dbt_python_model_df_' + '{{ relation.identifier }}'
|
|
124
|
-
con.register(tmp_name, df)
|
|
125
|
-
con.execute('create table {{ relation }} as select * from ' + tmp_name)
|
|
126
|
-
con.unregister(tmp_name)
|
|
127
|
-
{% endmacro %}
|
|
128
|
-
|
|
129
|
-
{% macro altertable__create_view_as(relation, sql) -%}
|
|
130
|
-
{% set contract_config = config.get('contract') %}
|
|
131
|
-
{% if contract_config.enforced %}
|
|
132
|
-
{{ get_assert_columns_equivalent(sql) }}
|
|
133
|
-
{%- endif %}
|
|
134
|
-
{%- set sql_header = config.get('sql_header', none) -%}
|
|
135
|
-
|
|
136
|
-
{{ sql_header if sql_header is not none }}
|
|
137
|
-
create view {{ relation }} as (
|
|
138
|
-
{{ sql }}
|
|
139
|
-
);
|
|
140
|
-
{% endmacro %}
|
|
141
|
-
|
|
142
|
-
{% macro altertable__get_columns_in_relation(relation) -%}
|
|
143
|
-
{% call statement('get_columns_in_relation', fetch_result=True) %}
|
|
144
|
-
select
|
|
145
|
-
column_name,
|
|
146
|
-
data_type,
|
|
147
|
-
cast(null as integer) as character_maximum_length,
|
|
148
|
-
cast(null as integer) as numeric_precision,
|
|
149
|
-
cast(null as integer) as numeric_scale
|
|
150
|
-
from duckdb_columns()
|
|
151
|
-
where table_name = '{{ relation.identifier }}'
|
|
152
|
-
{% if relation.schema %}
|
|
153
|
-
and lower(schema_name) = lower('{{ relation.schema }}')
|
|
154
|
-
{% endif %}
|
|
155
|
-
{% if relation.database %}
|
|
156
|
-
and lower(database_name) = lower('{{ relation.database }}')
|
|
157
|
-
{% endif %}
|
|
158
|
-
order by column_index
|
|
159
|
-
{% endcall %}
|
|
160
|
-
{% set table = load_result('get_columns_in_relation').table %}
|
|
161
|
-
{{ return(sql_convert_columns_in_relation(table)) }}
|
|
162
|
-
{% endmacro %}
|
|
163
|
-
|
|
164
|
-
{% macro altertable__list_relations_without_caching(schema_relation) %}
|
|
165
|
-
{% call statement('list_relations_without_caching', fetch_result=True) -%}
|
|
166
|
-
select
|
|
167
|
-
'{{ schema_relation.database }}' as database,
|
|
168
|
-
table_name as name,
|
|
169
|
-
schema_name as schema,
|
|
170
|
-
'table' as type
|
|
171
|
-
from duckdb_tables()
|
|
172
|
-
where lower(schema_name) = '{{ schema_relation.schema | lower }}'
|
|
173
|
-
and lower(database_name) = '{{ schema_relation.database | lower }}'
|
|
174
|
-
union all
|
|
175
|
-
select
|
|
176
|
-
'{{ schema_relation.database }}' as database,
|
|
177
|
-
view_name as name,
|
|
178
|
-
schema_name as schema,
|
|
179
|
-
'view' as type
|
|
180
|
-
from duckdb_views()
|
|
181
|
-
where lower(schema_name) = '{{ schema_relation.schema | lower }}'
|
|
182
|
-
and lower(database_name) = '{{ schema_relation.database | lower }}'
|
|
183
|
-
{% endcall %}
|
|
184
|
-
{{ return(load_result('list_relations_without_caching').table) }}
|
|
185
|
-
{% endmacro %}
|
|
186
|
-
|
|
187
|
-
{% macro altertable__make_temp_relation(base_relation, suffix) %}
|
|
188
|
-
{% set tmp_identifier = base_relation.identifier ~ suffix ~ py_current_timestring() %}
|
|
189
|
-
{% do return(base_relation.incorporate(
|
|
190
|
-
path={
|
|
191
|
-
"identifier": tmp_identifier,
|
|
192
|
-
"schema": none,
|
|
193
|
-
"database": none
|
|
194
|
-
})) -%}
|
|
195
|
-
{% endmacro %}
|
|
196
|
-
|
|
197
|
-
{% macro altertable__rename_relation(from_relation, to_relation) -%}
|
|
198
|
-
{% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}
|
|
199
|
-
{% call statement('rename_relation') -%}
|
|
200
|
-
alter {{ to_relation.type }} {{ from_relation }} rename to {{ target_name }}
|
|
201
|
-
{%- endcall %}
|
|
202
|
-
{% endmacro %}
|
|
203
|
-
|
|
204
|
-
{% macro altertable__current_timestamp() -%}
|
|
205
|
-
now()
|
|
206
|
-
{%- endmacro %}
|
|
207
|
-
|
|
208
|
-
{% macro altertable__snapshot_string_as_time(timestamp) -%}
|
|
209
|
-
{%- set result = "'" ~ timestamp ~ "'::timestamp" -%}
|
|
210
|
-
{{ return(result) }}
|
|
211
|
-
{%- endmacro %}
|
|
212
|
-
|
|
213
|
-
{% macro altertable__snapshot_get_time() -%}
|
|
214
|
-
{{ current_timestamp() }}::timestamp
|
|
215
|
-
{%- endmacro %}
|
|
216
|
-
|
|
217
|
-
{#
|
|
218
|
-
dbt's default__get_delete_insert_merge_sql returns DELETE and INSERT as one
|
|
219
|
-
semicolon-separated string, which Flight SQL rejects because it accepts a single
|
|
220
|
-
statement per round-trip. We execute the DELETE as an auto-begin statement and
|
|
221
|
-
return only the INSERT for the materialization's main statement. Both statements
|
|
222
|
-
stay identical to dbt's and share the materialization transaction.
|
|
223
|
-
#}
|
|
224
|
-
{% macro altertable__get_incremental_delete_insert_sql(arg_dict) %}
|
|
225
|
-
{%- set target = arg_dict["target_relation"] -%}
|
|
226
|
-
{%- set source = arg_dict["temp_relation"] -%}
|
|
227
|
-
{%- set unique_key = arg_dict["unique_key"] -%}
|
|
228
|
-
{%- set incremental_predicates = arg_dict["incremental_predicates"] -%}
|
|
229
|
-
{%- set dest_cols_csv = get_quoted_csv(arg_dict["dest_columns"] | map(attribute="name")) -%}
|
|
230
|
-
|
|
231
|
-
{%- if unique_key -%}
|
|
232
|
-
{%- if unique_key is string -%}
|
|
233
|
-
{%- set unique_key = [unique_key] -%}
|
|
234
|
-
{%- endif -%}
|
|
235
|
-
{%- set unique_key_str = unique_key | join(", ") -%}
|
|
236
|
-
|
|
237
|
-
{%- set delete_sql -%}
|
|
238
|
-
delete from {{ target }} as DBT_INTERNAL_DEST
|
|
239
|
-
where ({{ unique_key_str }}) in (
|
|
240
|
-
select distinct {{ unique_key_str }}
|
|
241
|
-
from {{ source }} as DBT_INTERNAL_SOURCE
|
|
242
|
-
)
|
|
243
|
-
{%- for predicate in incremental_predicates or [] %}
|
|
244
|
-
and {{ predicate }}
|
|
245
|
-
{%- endfor %}
|
|
246
|
-
{%- endset -%}
|
|
247
|
-
{% call statement("delete_incremental_rows", auto_begin=true) -%}
|
|
248
|
-
{{ delete_sql }}
|
|
249
|
-
{%- endcall %}
|
|
250
|
-
{%- endif %}
|
|
251
|
-
|
|
252
|
-
insert into {{ target }} ({{ dest_cols_csv }})
|
|
253
|
-
(
|
|
254
|
-
select {{ dest_cols_csv }}
|
|
255
|
-
from {{ source }}
|
|
256
|
-
)
|
|
257
|
-
{% endmacro %}
|
|
258
|
-
|
|
259
|
-
{% macro altertable__get_incremental_default_sql(arg_dict) %}
|
|
260
|
-
{% do return(get_incremental_delete_insert_sql(arg_dict)) %}
|
|
261
|
-
{% endmacro %}
|
|
262
|
-
|
|
263
|
-
{% macro location_exists(location) -%}
|
|
264
|
-
{% do return(adapter.location_exists(location)) %}
|
|
265
|
-
{% endmacro %}
|
|
266
|
-
|
|
267
|
-
{% macro write_to_file(relation, location, options) -%}
|
|
268
|
-
{% call statement('write_to_file') -%}
|
|
269
|
-
copy {{ relation }} to '{{ location }}' ({{ options }})
|
|
270
|
-
{%- endcall %}
|
|
271
|
-
{% endmacro %}
|
|
272
|
-
|
|
273
|
-
{% macro store_relation(plugin, relation, location, format, config) -%}
|
|
274
|
-
{%- set column_list = adapter.get_columns_in_relation(relation) -%}
|
|
275
|
-
{% do adapter.store_relation(plugin, relation, column_list, location, format, config) %}
|
|
276
|
-
{% endmacro %}
|
|
277
|
-
|
|
278
|
-
{% macro render_write_options(config) -%}
|
|
279
|
-
{% set options = config.get('options', {}) %}
|
|
280
|
-
{% if options is not mapping %}
|
|
281
|
-
{% do exceptions.raise_compiler_error("The options argument must be a dictionary") %}
|
|
282
|
-
{% endif %}
|
|
283
|
-
|
|
284
|
-
{% for k in options %}
|
|
285
|
-
{% set _ = options.update({k: render(options[k])}) %}
|
|
286
|
-
{% endfor %}
|
|
287
|
-
|
|
288
|
-
{# legacy top-level write options #}
|
|
289
|
-
{% if config.get('format') %}
|
|
290
|
-
{% set _ = options.update({'format': render(config.get('format'))}) %}
|
|
291
|
-
{% endif %}
|
|
292
|
-
{% if config.get('delimiter') %}
|
|
293
|
-
{% set _ = options.update({'delimiter': render(config.get('delimiter'))}) %}
|
|
294
|
-
{% endif %}
|
|
295
|
-
|
|
296
|
-
{% do return(options) %}
|
|
297
|
-
{%- endmacro %}
|
|
298
|
-
|
|
299
|
-
{% macro altertable__apply_grants(relation, grant_config, should_revoke=True) %}
|
|
300
|
-
{#-- If grant_config is {} or None, this is a no-op --#}
|
|
301
|
-
{% if grant_config %}
|
|
302
|
-
{{ adapter.warn_once('Grants for relations are not supported by DuckDB') }}
|
|
303
|
-
{% endif %}
|
|
304
|
-
{% endmacro %}
|
|
305
|
-
|
|
306
|
-
{% macro altertable__get_create_index_sql(relation, index_dict) -%}
|
|
307
|
-
{%- set index_config = adapter.parse_index(index_dict) -%}
|
|
308
|
-
{%- set comma_separated_columns = ", ".join(index_config.columns) -%}
|
|
309
|
-
{%- set index_name = index_config.render(relation) -%}
|
|
310
|
-
|
|
311
|
-
create {% if index_config.unique -%}
|
|
312
|
-
unique
|
|
313
|
-
{%- endif %} index
|
|
314
|
-
"{{ index_name }}"
|
|
315
|
-
on {{ relation }}
|
|
316
|
-
({{ comma_separated_columns }});
|
|
317
|
-
{%- endmacro %}
|
|
318
|
-
|
|
319
|
-
{% macro drop_indexes_on_relation(relation) -%}
|
|
320
|
-
{% call statement('get_indexes_on_relation', fetch_result=True) %}
|
|
321
|
-
SELECT index_name
|
|
322
|
-
FROM duckdb_indexes()
|
|
323
|
-
WHERE schema_name = '{{ relation.schema }}'
|
|
324
|
-
AND table_name = '{{ relation.identifier }}'
|
|
325
|
-
{% endcall %}
|
|
326
|
-
|
|
327
|
-
{% set results = load_result('get_indexes_on_relation').table %}
|
|
328
|
-
{% for row in results %}
|
|
329
|
-
{% set index_name = row[0] %}
|
|
330
|
-
{% call statement('drop_index_' + loop.index|string, auto_begin=false) %}
|
|
331
|
-
DROP INDEX "{{ relation.schema }}"."{{ index_name }}"
|
|
332
|
-
{% endcall %}
|
|
333
|
-
{% endfor %}
|
|
334
|
-
|
|
335
|
-
{#-- Verify indexes were dropped --#}
|
|
336
|
-
{% call statement('verify_indexes_dropped', fetch_result=True) %}
|
|
337
|
-
SELECT COUNT(*) as remaining_indexes
|
|
338
|
-
FROM duckdb_indexes()
|
|
339
|
-
WHERE schema_name = '{{ relation.schema }}'
|
|
340
|
-
AND table_name = '{{ relation.identifier }}'
|
|
341
|
-
{% endcall %}
|
|
342
|
-
{% set verify_results = load_result('verify_indexes_dropped').table %}
|
|
343
|
-
{%- endmacro %}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{% macro altertable__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
|
|
2
|
-
|
|
3
|
-
{% if add_columns %}
|
|
4
|
-
{% for column in add_columns %}
|
|
5
|
-
{% set sql -%}
|
|
6
|
-
alter {{ relation.type }} {{ relation }} add column
|
|
7
|
-
{{ api.Relation.create(identifier=column.name) }} {{ column.data_type }}
|
|
8
|
-
{%- endset -%}
|
|
9
|
-
{% do run_query(sql) %}
|
|
10
|
-
{% endfor %}
|
|
11
|
-
{% endif %}
|
|
12
|
-
|
|
13
|
-
{% if remove_columns %}
|
|
14
|
-
{% for column in remove_columns %}
|
|
15
|
-
{% set sql -%}
|
|
16
|
-
alter {{ relation.type }} {{ relation }} drop column
|
|
17
|
-
{{ api.Relation.create(identifier=column.name) }}
|
|
18
|
-
{%- endset -%}
|
|
19
|
-
{% do run_query(sql) %}
|
|
20
|
-
{% endfor %}
|
|
21
|
-
{% endif %}
|
|
22
|
-
|
|
23
|
-
{% endmacro %}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{% macro altertable__dateadd(datepart, interval, from_date_or_timestamp) %}
|
|
2
|
-
|
|
3
|
-
{#
|
|
4
|
-
Support both literal and expression intervals (e.g., column references)
|
|
5
|
-
by multiplying an INTERVAL by the value. This avoids DuckDB parser issues
|
|
6
|
-
with "interval (<expr>) <unit>" and works across versions.
|
|
7
|
-
|
|
8
|
-
Also map unsupported units:
|
|
9
|
-
- quarter => 3 months
|
|
10
|
-
- week => 7 days (DuckDB supports WEEK as a literal, but keep it explicit)
|
|
11
|
-
#}
|
|
12
|
-
|
|
13
|
-
{%- set unit = datepart | lower -%}
|
|
14
|
-
{%- if unit == 'quarter' -%}
|
|
15
|
-
({{ from_date_or_timestamp }} + (cast({{ interval }} as bigint) * 3) * interval 1 month)
|
|
16
|
-
{%- elif unit == 'week' -%}
|
|
17
|
-
({{ from_date_or_timestamp }} + (cast({{ interval }} as bigint) * 7) * interval 1 day)
|
|
18
|
-
{%- else -%}
|
|
19
|
-
({{ from_date_or_timestamp }} + cast({{ interval }} as bigint) * interval 1 {{ unit }})
|
|
20
|
-
{%- endif -%}
|
|
21
|
-
|
|
22
|
-
{% endmacro %}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
{% macro altertable__datediff(first_date, second_date, datepart) -%}
|
|
2
|
-
{% if datepart == 'week' %}
|
|
3
|
-
({{ datediff(first_date, second_date, 'day') }} // 7 + case
|
|
4
|
-
when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then
|
|
5
|
-
case when {{first_date}} <= {{second_date}} then 0 else -1 end
|
|
6
|
-
else
|
|
7
|
-
case when {{first_date}} <= {{second_date}} then 1 else 0 end
|
|
8
|
-
end)
|
|
9
|
-
{% else %}
|
|
10
|
-
(date_diff('{{ datepart }}', {{ first_date }}::timestamp, {{ second_date}}::timestamp ))
|
|
11
|
-
{% endif %}
|
|
12
|
-
{%- endmacro %}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
{%- macro external_location(relation, config) -%}
|
|
2
|
-
{%- if config.get('options', {}).get('partition_by') is none -%}
|
|
3
|
-
{%- set format = config.get('format', 'parquet') -%}
|
|
4
|
-
{{- adapter.external_root() }}/{{ relation.identifier }}.{{ format }}
|
|
5
|
-
{%- else -%}
|
|
6
|
-
{{- adapter.external_root() }}/{{ relation.identifier }}
|
|
7
|
-
{%- endif -%}
|
|
8
|
-
{%- endmacro -%}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
{% macro altertable__last_day(date, datepart) -%}
|
|
2
|
-
|
|
3
|
-
{%- if datepart == 'quarter' -%}
|
|
4
|
-
-- duckdb dateadd does not support quarter interval.
|
|
5
|
-
cast(
|
|
6
|
-
{{dbt.dateadd('day', '-1',
|
|
7
|
-
dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))
|
|
8
|
-
)}}
|
|
9
|
-
as date)
|
|
10
|
-
{%- else -%}
|
|
11
|
-
{{dbt.default_last_day(date, datepart)}}
|
|
12
|
-
{%- endif -%}
|
|
13
|
-
|
|
14
|
-
{%- endmacro %}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{% macro altertable__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}
|
|
2
|
-
{% if limit_num -%}
|
|
3
|
-
list_aggr(
|
|
4
|
-
(array_agg(
|
|
5
|
-
{{ measure }}
|
|
6
|
-
{% if order_by_clause -%}
|
|
7
|
-
{{ order_by_clause }}
|
|
8
|
-
{%- endif %}
|
|
9
|
-
))[1:{{ limit_num }}],
|
|
10
|
-
'string_agg',
|
|
11
|
-
{{ delimiter_text }}
|
|
12
|
-
)
|
|
13
|
-
{%- else %}
|
|
14
|
-
string_agg(
|
|
15
|
-
{{ measure }},
|
|
16
|
-
{{ delimiter_text }}
|
|
17
|
-
{% if order_by_clause -%}
|
|
18
|
-
{{ order_by_clause }}
|
|
19
|
-
{%- endif %}
|
|
20
|
-
)
|
|
21
|
-
{%- endif %}
|
|
22
|
-
{%- endmacro %}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
{%- macro register_upstream_external_models() -%}
|
|
2
|
-
{% if execute %}
|
|
3
|
-
{% set upstream_nodes = {} %}
|
|
4
|
-
{% set upstream_schemas = {} %}
|
|
5
|
-
{% for node in selected_resources %}
|
|
6
|
-
{% if node not in graph['nodes'] %}{% continue %}{% endif %}
|
|
7
|
-
{% for upstream_node in graph['nodes'][node]['depends_on']['nodes'] %}
|
|
8
|
-
{% if upstream_node not in upstream_nodes and upstream_node not in selected_resources %}
|
|
9
|
-
{% do upstream_nodes.update({upstream_node: None}) %}
|
|
10
|
-
{% set upstream = graph['nodes'].get(upstream_node) %}
|
|
11
|
-
{% if upstream
|
|
12
|
-
and upstream.resource_type in ('model', 'seed')
|
|
13
|
-
and upstream.config.materialized=='external'
|
|
14
|
-
%}
|
|
15
|
-
{%- set upstream_rel = api.Relation.create(
|
|
16
|
-
database=upstream['database'],
|
|
17
|
-
schema=upstream['schema'],
|
|
18
|
-
identifier=upstream['alias']
|
|
19
|
-
) -%}
|
|
20
|
-
{%- set location = upstream.config.get('location', external_location(upstream_rel, upstream.config)) -%}
|
|
21
|
-
{%- set rendered_options = render_write_options(upstream.config) -%}
|
|
22
|
-
{%- set upstream_location = adapter.external_read_location(location, rendered_options) -%}
|
|
23
|
-
{% if upstream_rel.schema not in upstream_schemas %}
|
|
24
|
-
{% call statement('main', language='sql') -%}
|
|
25
|
-
create schema if not exists {{ upstream_rel.without_identifier() }}
|
|
26
|
-
{%- endcall %}
|
|
27
|
-
{% do upstream_schemas.update({upstream_rel.schema: None}) %}
|
|
28
|
-
{% endif %}
|
|
29
|
-
{% call statement('main', language='sql') -%}
|
|
30
|
-
create or replace view {{ upstream_rel }} as (
|
|
31
|
-
select * from '{{ upstream_location }}'
|
|
32
|
-
);
|
|
33
|
-
{%- endcall %}
|
|
34
|
-
{%- endif %}
|
|
35
|
-
{% endif %}
|
|
36
|
-
{% endfor %}
|
|
37
|
-
{% endfor %}
|
|
38
|
-
{% if upstream_nodes %}
|
|
39
|
-
{% do adapter.commit() %}
|
|
40
|
-
{% endif %}
|
|
41
|
-
{% endif %}
|
|
42
|
-
{%- endmacro -%}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|