dbt-adapters 1.22.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt/adapters/__about__.py +1 -0
- dbt/adapters/__init__.py +8 -0
- dbt/adapters/base/README.md +13 -0
- dbt/adapters/base/__init__.py +16 -0
- dbt/adapters/base/column.py +173 -0
- dbt/adapters/base/connections.py +429 -0
- dbt/adapters/base/impl.py +2036 -0
- dbt/adapters/base/meta.py +150 -0
- dbt/adapters/base/plugin.py +32 -0
- dbt/adapters/base/query_headers.py +106 -0
- dbt/adapters/base/relation.py +648 -0
- dbt/adapters/cache.py +521 -0
- dbt/adapters/capability.py +63 -0
- dbt/adapters/catalogs/__init__.py +14 -0
- dbt/adapters/catalogs/_client.py +54 -0
- dbt/adapters/catalogs/_constants.py +1 -0
- dbt/adapters/catalogs/_exceptions.py +39 -0
- dbt/adapters/catalogs/_integration.py +113 -0
- dbt/adapters/clients/__init__.py +0 -0
- dbt/adapters/clients/jinja.py +24 -0
- dbt/adapters/contracts/__init__.py +0 -0
- dbt/adapters/contracts/connection.py +229 -0
- dbt/adapters/contracts/macros.py +11 -0
- dbt/adapters/contracts/relation.py +160 -0
- dbt/adapters/events/README.md +51 -0
- dbt/adapters/events/__init__.py +0 -0
- dbt/adapters/events/adapter_types_pb2.py +2 -0
- dbt/adapters/events/base_types.py +36 -0
- dbt/adapters/events/logging.py +83 -0
- dbt/adapters/events/types.py +436 -0
- dbt/adapters/exceptions/__init__.py +40 -0
- dbt/adapters/exceptions/alias.py +24 -0
- dbt/adapters/exceptions/cache.py +68 -0
- dbt/adapters/exceptions/compilation.py +269 -0
- dbt/adapters/exceptions/connection.py +16 -0
- dbt/adapters/exceptions/database.py +51 -0
- dbt/adapters/factory.py +264 -0
- dbt/adapters/protocol.py +150 -0
- dbt/adapters/py.typed +0 -0
- dbt/adapters/record/__init__.py +2 -0
- dbt/adapters/record/base.py +291 -0
- dbt/adapters/record/cursor/cursor.py +69 -0
- dbt/adapters/record/cursor/description.py +37 -0
- dbt/adapters/record/cursor/execute.py +39 -0
- dbt/adapters/record/cursor/fetchall.py +69 -0
- dbt/adapters/record/cursor/fetchmany.py +23 -0
- dbt/adapters/record/cursor/fetchone.py +23 -0
- dbt/adapters/record/cursor/rowcount.py +23 -0
- dbt/adapters/record/handle.py +55 -0
- dbt/adapters/record/serialization.py +115 -0
- dbt/adapters/reference_keys.py +39 -0
- dbt/adapters/relation_configs/README.md +25 -0
- dbt/adapters/relation_configs/__init__.py +12 -0
- dbt/adapters/relation_configs/config_base.py +46 -0
- dbt/adapters/relation_configs/config_change.py +26 -0
- dbt/adapters/relation_configs/config_validation.py +57 -0
- dbt/adapters/sql/__init__.py +2 -0
- dbt/adapters/sql/connections.py +263 -0
- dbt/adapters/sql/impl.py +286 -0
- dbt/adapters/utils.py +69 -0
- dbt/include/__init__.py +3 -0
- dbt/include/global_project/__init__.py +4 -0
- dbt/include/global_project/dbt_project.yml +7 -0
- dbt/include/global_project/docs/overview.md +43 -0
- dbt/include/global_project/macros/adapters/apply_grants.sql +167 -0
- dbt/include/global_project/macros/adapters/columns.sql +144 -0
- dbt/include/global_project/macros/adapters/freshness.sql +32 -0
- dbt/include/global_project/macros/adapters/indexes.sql +41 -0
- dbt/include/global_project/macros/adapters/metadata.sql +105 -0
- dbt/include/global_project/macros/adapters/persist_docs.sql +33 -0
- dbt/include/global_project/macros/adapters/relation.sql +84 -0
- dbt/include/global_project/macros/adapters/schema.sql +20 -0
- dbt/include/global_project/macros/adapters/show.sql +26 -0
- dbt/include/global_project/macros/adapters/timestamps.sql +52 -0
- dbt/include/global_project/macros/adapters/validate_sql.sql +10 -0
- dbt/include/global_project/macros/etc/datetime.sql +62 -0
- dbt/include/global_project/macros/etc/statement.sql +52 -0
- dbt/include/global_project/macros/generic_test_sql/accepted_values.sql +27 -0
- dbt/include/global_project/macros/generic_test_sql/not_null.sql +9 -0
- dbt/include/global_project/macros/generic_test_sql/relationships.sql +23 -0
- dbt/include/global_project/macros/generic_test_sql/unique.sql +12 -0
- dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql +36 -0
- dbt/include/global_project/macros/get_custom_name/get_custom_database.sql +32 -0
- dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql +60 -0
- dbt/include/global_project/macros/materializations/configs.sql +21 -0
- dbt/include/global_project/macros/materializations/functions/aggregate.sql +65 -0
- dbt/include/global_project/macros/materializations/functions/function.sql +20 -0
- dbt/include/global_project/macros/materializations/functions/helpers.sql +20 -0
- dbt/include/global_project/macros/materializations/functions/scalar.sql +69 -0
- dbt/include/global_project/macros/materializations/hooks.sql +35 -0
- dbt/include/global_project/macros/materializations/models/clone/can_clone_table.sql +7 -0
- dbt/include/global_project/macros/materializations/models/clone/clone.sql +67 -0
- dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql +7 -0
- dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql +80 -0
- dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +99 -0
- dbt/include/global_project/macros/materializations/models/incremental/is_incremental.sql +13 -0
- dbt/include/global_project/macros/materializations/models/incremental/merge.sql +120 -0
- dbt/include/global_project/macros/materializations/models/incremental/on_schema_change.sql +159 -0
- dbt/include/global_project/macros/materializations/models/incremental/strategies.sql +92 -0
- dbt/include/global_project/macros/materializations/models/materialized_view.sql +121 -0
- dbt/include/global_project/macros/materializations/models/table.sql +64 -0
- dbt/include/global_project/macros/materializations/models/view.sql +72 -0
- dbt/include/global_project/macros/materializations/seeds/helpers.sql +128 -0
- dbt/include/global_project/macros/materializations/seeds/seed.sql +60 -0
- dbt/include/global_project/macros/materializations/snapshots/helpers.sql +345 -0
- dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +109 -0
- dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +34 -0
- dbt/include/global_project/macros/materializations/snapshots/strategies.sql +184 -0
- dbt/include/global_project/macros/materializations/tests/helpers.sql +44 -0
- dbt/include/global_project/macros/materializations/tests/test.sql +66 -0
- dbt/include/global_project/macros/materializations/tests/unit.sql +40 -0
- dbt/include/global_project/macros/materializations/tests/where_subquery.sql +15 -0
- dbt/include/global_project/macros/python_model/python.sql +114 -0
- dbt/include/global_project/macros/relations/column/columns_spec_ddl.sql +89 -0
- dbt/include/global_project/macros/relations/create.sql +23 -0
- dbt/include/global_project/macros/relations/create_backup.sql +17 -0
- dbt/include/global_project/macros/relations/create_intermediate.sql +17 -0
- dbt/include/global_project/macros/relations/drop.sql +41 -0
- dbt/include/global_project/macros/relations/drop_backup.sql +14 -0
- dbt/include/global_project/macros/relations/materialized_view/alter.sql +55 -0
- dbt/include/global_project/macros/relations/materialized_view/create.sql +10 -0
- dbt/include/global_project/macros/relations/materialized_view/drop.sql +14 -0
- dbt/include/global_project/macros/relations/materialized_view/refresh.sql +9 -0
- dbt/include/global_project/macros/relations/materialized_view/rename.sql +10 -0
- dbt/include/global_project/macros/relations/materialized_view/replace.sql +10 -0
- dbt/include/global_project/macros/relations/rename.sql +35 -0
- dbt/include/global_project/macros/relations/rename_intermediate.sql +14 -0
- dbt/include/global_project/macros/relations/replace.sql +50 -0
- dbt/include/global_project/macros/relations/schema.sql +8 -0
- dbt/include/global_project/macros/relations/table/create.sql +60 -0
- dbt/include/global_project/macros/relations/table/drop.sql +14 -0
- dbt/include/global_project/macros/relations/table/rename.sql +10 -0
- dbt/include/global_project/macros/relations/table/replace.sql +10 -0
- dbt/include/global_project/macros/relations/view/create.sql +27 -0
- dbt/include/global_project/macros/relations/view/drop.sql +14 -0
- dbt/include/global_project/macros/relations/view/rename.sql +10 -0
- dbt/include/global_project/macros/relations/view/replace.sql +66 -0
- dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +107 -0
- dbt/include/global_project/macros/utils/any_value.sql +9 -0
- dbt/include/global_project/macros/utils/array_append.sql +8 -0
- dbt/include/global_project/macros/utils/array_concat.sql +7 -0
- dbt/include/global_project/macros/utils/array_construct.sql +12 -0
- dbt/include/global_project/macros/utils/bool_or.sql +9 -0
- dbt/include/global_project/macros/utils/cast.sql +7 -0
- dbt/include/global_project/macros/utils/cast_bool_to_text.sql +7 -0
- dbt/include/global_project/macros/utils/concat.sql +7 -0
- dbt/include/global_project/macros/utils/data_types.sql +129 -0
- dbt/include/global_project/macros/utils/date.sql +10 -0
- dbt/include/global_project/macros/utils/date_spine.sql +75 -0
- dbt/include/global_project/macros/utils/date_trunc.sql +7 -0
- dbt/include/global_project/macros/utils/dateadd.sql +14 -0
- dbt/include/global_project/macros/utils/datediff.sql +14 -0
- dbt/include/global_project/macros/utils/equals.sql +14 -0
- dbt/include/global_project/macros/utils/escape_single_quotes.sql +8 -0
- dbt/include/global_project/macros/utils/except.sql +9 -0
- dbt/include/global_project/macros/utils/generate_series.sql +53 -0
- dbt/include/global_project/macros/utils/hash.sql +7 -0
- dbt/include/global_project/macros/utils/intersect.sql +9 -0
- dbt/include/global_project/macros/utils/last_day.sql +15 -0
- dbt/include/global_project/macros/utils/length.sql +11 -0
- dbt/include/global_project/macros/utils/listagg.sql +30 -0
- dbt/include/global_project/macros/utils/literal.sql +7 -0
- dbt/include/global_project/macros/utils/position.sql +11 -0
- dbt/include/global_project/macros/utils/replace.sql +14 -0
- dbt/include/global_project/macros/utils/right.sql +12 -0
- dbt/include/global_project/macros/utils/safe_cast.sql +9 -0
- dbt/include/global_project/macros/utils/split_part.sql +26 -0
- dbt/include/global_project/tests/generic/builtin.sql +30 -0
- dbt/include/py.typed +0 -0
- dbt_adapters-1.22.2.dist-info/METADATA +124 -0
- dbt_adapters-1.22.2.dist-info/RECORD +173 -0
- dbt_adapters-1.22.2.dist-info/WHEEL +4 -0
- dbt_adapters-1.22.2.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
{#
|
|
2
|
+
Add new columns to the table if applicable
|
|
3
|
+
#}
|
|
4
|
+
{% macro create_columns(relation, columns) %}
|
|
5
|
+
{{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}
|
|
6
|
+
{% endmacro %}
|
|
7
|
+
|
|
8
|
+
{% macro default__create_columns(relation, columns) %}
|
|
9
|
+
{% for column in columns %}
|
|
10
|
+
{% call statement() %}
|
|
11
|
+
alter table {{ relation.render() }} add column {{ adapter.quote(column.name) }} {{ column.expanded_data_type }};
|
|
12
|
+
{% endcall %}
|
|
13
|
+
{% endfor %}
|
|
14
|
+
{% endmacro %}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
{% macro post_snapshot(staging_relation) %}
|
|
18
|
+
{{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}
|
|
19
|
+
{% endmacro %}
|
|
20
|
+
|
|
21
|
+
{% macro default__post_snapshot(staging_relation) %}
|
|
22
|
+
{# no-op #}
|
|
23
|
+
{% endmacro %}
|
|
24
|
+
|
|
25
|
+
{% macro get_true_sql() %}
|
|
26
|
+
{{ adapter.dispatch('get_true_sql', 'dbt')() }}
|
|
27
|
+
{% endmacro %}
|
|
28
|
+
|
|
29
|
+
{% macro default__get_true_sql() %}
|
|
30
|
+
{{ return('TRUE') }}
|
|
31
|
+
{% endmacro %}
|
|
32
|
+
|
|
33
|
+
{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}
|
|
34
|
+
{{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}
|
|
35
|
+
{% endmacro %}
|
|
36
|
+
|
|
37
|
+
{% macro get_snapshot_table_column_names() %}
|
|
38
|
+
{{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }}
|
|
39
|
+
{% endmacro %}
|
|
40
|
+
|
|
41
|
+
{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}
|
|
42
|
+
{% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}
|
|
43
|
+
{% if strategy.hard_deletes == 'new_record' %}
|
|
44
|
+
{% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}
|
|
45
|
+
{% endif %}
|
|
46
|
+
with snapshot_query as (
|
|
47
|
+
|
|
48
|
+
{{ source_sql }}
|
|
49
|
+
|
|
50
|
+
),
|
|
51
|
+
|
|
52
|
+
snapshotted_data as (
|
|
53
|
+
|
|
54
|
+
select *, {{ unique_key_fields(strategy.unique_key) }}
|
|
55
|
+
from {{ target_relation }}
|
|
56
|
+
where
|
|
57
|
+
{% if config.get('dbt_valid_to_current') %}
|
|
58
|
+
{% set source_unique_key = columns.dbt_valid_to | trim %}
|
|
59
|
+
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
|
|
60
|
+
|
|
61
|
+
{# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}
|
|
62
|
+
( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
|
|
63
|
+
{% else %}
|
|
64
|
+
{{ columns.dbt_valid_to }} is null
|
|
65
|
+
{% endif %}
|
|
66
|
+
|
|
67
|
+
),
|
|
68
|
+
|
|
69
|
+
insertions_source_data as (
|
|
70
|
+
|
|
71
|
+
select *, {{ unique_key_fields(strategy.unique_key) }},
|
|
72
|
+
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
|
|
73
|
+
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
|
|
74
|
+
{{ get_dbt_valid_to_current(strategy, columns) }},
|
|
75
|
+
{{ strategy.scd_id }} as {{ columns.dbt_scd_id }}
|
|
76
|
+
|
|
77
|
+
from snapshot_query
|
|
78
|
+
),
|
|
79
|
+
|
|
80
|
+
updates_source_data as (
|
|
81
|
+
|
|
82
|
+
select *, {{ unique_key_fields(strategy.unique_key) }},
|
|
83
|
+
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
|
|
84
|
+
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
|
|
85
|
+
{{ strategy.updated_at }} as {{ columns.dbt_valid_to }}
|
|
86
|
+
|
|
87
|
+
from snapshot_query
|
|
88
|
+
),
|
|
89
|
+
|
|
90
|
+
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
|
|
91
|
+
|
|
92
|
+
deletes_source_data as (
|
|
93
|
+
|
|
94
|
+
select *, {{ unique_key_fields(strategy.unique_key) }}
|
|
95
|
+
from snapshot_query
|
|
96
|
+
),
|
|
97
|
+
{% endif %}
|
|
98
|
+
|
|
99
|
+
insertions as (
|
|
100
|
+
|
|
101
|
+
select
|
|
102
|
+
'insert' as dbt_change_type,
|
|
103
|
+
source_data.*
|
|
104
|
+
{%- if strategy.hard_deletes == 'new_record' -%}
|
|
105
|
+
,'False' as {{ columns.dbt_is_deleted }}
|
|
106
|
+
{%- endif %}
|
|
107
|
+
|
|
108
|
+
from insertions_source_data as source_data
|
|
109
|
+
left outer join snapshotted_data
|
|
110
|
+
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
|
111
|
+
where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }}
|
|
112
|
+
or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and (
|
|
113
|
+
{{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
),
|
|
119
|
+
|
|
120
|
+
updates as (
|
|
121
|
+
|
|
122
|
+
select
|
|
123
|
+
'update' as dbt_change_type,
|
|
124
|
+
source_data.*,
|
|
125
|
+
snapshotted_data.{{ columns.dbt_scd_id }}
|
|
126
|
+
{%- if strategy.hard_deletes == 'new_record' -%}
|
|
127
|
+
, snapshotted_data.{{ columns.dbt_is_deleted }}
|
|
128
|
+
{%- endif %}
|
|
129
|
+
|
|
130
|
+
from updates_source_data as source_data
|
|
131
|
+
join snapshotted_data
|
|
132
|
+
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
|
133
|
+
where (
|
|
134
|
+
{{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}
|
|
135
|
+
)
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
|
|
139
|
+
,
|
|
140
|
+
deletes as (
|
|
141
|
+
|
|
142
|
+
select
|
|
143
|
+
'delete' as dbt_change_type,
|
|
144
|
+
source_data.*,
|
|
145
|
+
{{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},
|
|
146
|
+
{{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},
|
|
147
|
+
{{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},
|
|
148
|
+
snapshotted_data.{{ columns.dbt_scd_id }}
|
|
149
|
+
{%- if strategy.hard_deletes == 'new_record' -%}
|
|
150
|
+
, snapshotted_data.{{ columns.dbt_is_deleted }}
|
|
151
|
+
{%- endif %}
|
|
152
|
+
from snapshotted_data
|
|
153
|
+
left join deletes_source_data as source_data
|
|
154
|
+
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
|
155
|
+
where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
|
|
156
|
+
|
|
157
|
+
{%- if strategy.hard_deletes == 'new_record' %}
|
|
158
|
+
and not (
|
|
159
|
+
--avoid updating the record's valid_to if the latest entry is marked as deleted
|
|
160
|
+
snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
|
|
161
|
+
and
|
|
162
|
+
{% if config.get('dbt_valid_to_current') -%}
|
|
163
|
+
snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}
|
|
164
|
+
{%- else -%}
|
|
165
|
+
snapshotted_data.{{ columns.dbt_valid_to }} is null
|
|
166
|
+
{%- endif %}
|
|
167
|
+
)
|
|
168
|
+
{%- endif %}
|
|
169
|
+
)
|
|
170
|
+
{%- endif %}
|
|
171
|
+
|
|
172
|
+
{%- if strategy.hard_deletes == 'new_record' %}
|
|
173
|
+
{% set snapshotted_cols = get_list_of_column_names(get_columns_in_relation(target_relation)) %}
|
|
174
|
+
{% set source_sql_cols = get_column_schema_from_query(source_sql) %}
|
|
175
|
+
,
|
|
176
|
+
deletion_records as (
|
|
177
|
+
|
|
178
|
+
select
|
|
179
|
+
'insert' as dbt_change_type,
|
|
180
|
+
{#/*
|
|
181
|
+
If a column has been added to the source it won't yet exist in the
|
|
182
|
+
snapshotted table so we insert a null value as a placeholder for the column.
|
|
183
|
+
*/#}
|
|
184
|
+
{%- for col in source_sql_cols -%}
|
|
185
|
+
{%- if col.name in snapshotted_cols -%}
|
|
186
|
+
snapshotted_data.{{ adapter.quote(col.column) }},
|
|
187
|
+
{%- else -%}
|
|
188
|
+
NULL as {{ adapter.quote(col.column) }},
|
|
189
|
+
{%- endif -%}
|
|
190
|
+
{% endfor -%}
|
|
191
|
+
{%- if strategy.unique_key | is_list -%}
|
|
192
|
+
{%- for key in strategy.unique_key -%}
|
|
193
|
+
snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},
|
|
194
|
+
{% endfor -%}
|
|
195
|
+
{%- else -%}
|
|
196
|
+
snapshotted_data.dbt_unique_key as dbt_unique_key,
|
|
197
|
+
{% endif -%}
|
|
198
|
+
{{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},
|
|
199
|
+
{{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},
|
|
200
|
+
snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},
|
|
201
|
+
{{ new_scd_id }} as {{ columns.dbt_scd_id }},
|
|
202
|
+
'True' as {{ columns.dbt_is_deleted }}
|
|
203
|
+
from snapshotted_data
|
|
204
|
+
left join deletes_source_data as source_data
|
|
205
|
+
on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
|
|
206
|
+
where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
|
|
207
|
+
and not (
|
|
208
|
+
--avoid inserting a new record if the latest one is marked as deleted
|
|
209
|
+
snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
|
|
210
|
+
and
|
|
211
|
+
{% if config.get('dbt_valid_to_current') -%}
|
|
212
|
+
snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}
|
|
213
|
+
{%- else -%}
|
|
214
|
+
snapshotted_data.{{ columns.dbt_valid_to }} is null
|
|
215
|
+
{%- endif %}
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
)
|
|
219
|
+
{%- endif %}
|
|
220
|
+
|
|
221
|
+
select * from insertions
|
|
222
|
+
union all
|
|
223
|
+
select * from updates
|
|
224
|
+
{%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
|
|
225
|
+
union all
|
|
226
|
+
select * from deletes
|
|
227
|
+
{%- endif %}
|
|
228
|
+
{%- if strategy.hard_deletes == 'new_record' %}
|
|
229
|
+
union all
|
|
230
|
+
select * from deletion_records
|
|
231
|
+
{%- endif %}
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
{%- endmacro %}
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
{% macro build_snapshot_table(strategy, sql) -%}
|
|
238
|
+
{{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}
|
|
239
|
+
{% endmacro %}
|
|
240
|
+
|
|
241
|
+
{% macro default__build_snapshot_table(strategy, sql) %}
|
|
242
|
+
{% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}
|
|
243
|
+
|
|
244
|
+
select *,
|
|
245
|
+
{{ strategy.scd_id }} as {{ columns.dbt_scd_id }},
|
|
246
|
+
{{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
|
|
247
|
+
{{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
|
|
248
|
+
{{ get_dbt_valid_to_current(strategy, columns) }}
|
|
249
|
+
{%- if strategy.hard_deletes == 'new_record' -%}
|
|
250
|
+
, 'False' as {{ columns.dbt_is_deleted }}
|
|
251
|
+
{% endif -%}
|
|
252
|
+
from (
|
|
253
|
+
{{ sql }}
|
|
254
|
+
) sbq
|
|
255
|
+
|
|
256
|
+
{% endmacro %}
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}
|
|
260
|
+
{% set temp_relation = make_temp_relation(target_relation) %}
|
|
261
|
+
|
|
262
|
+
{% set select = snapshot_staging_table(strategy, sql, target_relation) %}
|
|
263
|
+
|
|
264
|
+
{% call statement('build_snapshot_staging_relation') %}
|
|
265
|
+
{{ create_table_as(True, temp_relation, select) }}
|
|
266
|
+
{% endcall %}
|
|
267
|
+
|
|
268
|
+
{% do return(temp_relation) %}
|
|
269
|
+
{% endmacro %}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
{% macro get_updated_at_column_data_type(snapshot_sql) %}
|
|
273
|
+
{% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}
|
|
274
|
+
{% set dbt_updated_at_data_type = null %}
|
|
275
|
+
{% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}
|
|
276
|
+
{% set ns.dbt_updated_at_data_type = null -%}
|
|
277
|
+
{% for column in snapshot_sql_column_schema %}
|
|
278
|
+
{% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}
|
|
279
|
+
{% set ns.dbt_updated_at_data_type = column.dtype %}
|
|
280
|
+
{% endif %}
|
|
281
|
+
{% endfor %}
|
|
282
|
+
{{ return(ns.dbt_updated_at_data_type or none) }}
|
|
283
|
+
{% endmacro %}
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
{% macro check_time_data_types(sql) %}
|
|
287
|
+
{% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}
|
|
288
|
+
{% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}
|
|
289
|
+
{% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}
|
|
290
|
+
{% if exceptions.warn_snapshot_timestamp_data_types %}
|
|
291
|
+
{{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}
|
|
292
|
+
{% endif %}
|
|
293
|
+
{% endif %}
|
|
294
|
+
{% endmacro %}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
{% macro get_dbt_valid_to_current(strategy, columns) %}
|
|
298
|
+
{% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %}
|
|
299
|
+
coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})
|
|
300
|
+
as {{ columns.dbt_valid_to }}
|
|
301
|
+
{% endmacro %}
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
{% macro unique_key_fields(unique_key) %}
|
|
305
|
+
{% if unique_key | is_list %}
|
|
306
|
+
{% for key in unique_key %}
|
|
307
|
+
{{ key }} as dbt_unique_key_{{ loop.index }}
|
|
308
|
+
{%- if not loop.last %} , {%- endif %}
|
|
309
|
+
{% endfor %}
|
|
310
|
+
{% else %}
|
|
311
|
+
{{ unique_key }} as dbt_unique_key
|
|
312
|
+
{% endif %}
|
|
313
|
+
{% endmacro %}
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}
|
|
317
|
+
{% if unique_key | is_list %}
|
|
318
|
+
{% for key in unique_key %}
|
|
319
|
+
{% set source_unique_key = (identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
|
|
320
|
+
{% set target_unique_key = (from_identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
|
|
321
|
+
{{ equals(source_unique_key, target_unique_key) }}
|
|
322
|
+
{%- if not loop.last %} and {%- endif %}
|
|
323
|
+
{% endfor %}
|
|
324
|
+
{% else %}
|
|
325
|
+
{{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key
|
|
326
|
+
{% endif %}
|
|
327
|
+
{% endmacro %}
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
{% macro unique_key_is_null(unique_key, identifier) %}
|
|
331
|
+
{% if unique_key | is_list %}
|
|
332
|
+
{{ identifier }}.dbt_unique_key_1 is null
|
|
333
|
+
{% else %}
|
|
334
|
+
{{ identifier }}.dbt_unique_key is null
|
|
335
|
+
{% endif %}
|
|
336
|
+
{% endmacro %}
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
{% macro unique_key_is_not_null(unique_key, identifier) %}
|
|
340
|
+
{% if unique_key | is_list %}
|
|
341
|
+
{{ identifier }}.dbt_unique_key_1 is not null
|
|
342
|
+
{% else %}
|
|
343
|
+
{{ identifier }}.dbt_unique_key is not null
|
|
344
|
+
{% endif %}
|
|
345
|
+
{% endmacro %}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
{% materialization snapshot, default %}
|
|
2
|
+
|
|
3
|
+
{%- set target_table = model.get('alias', model.get('name')) -%}
|
|
4
|
+
|
|
5
|
+
{%- set strategy_name = config.get('strategy') -%}
|
|
6
|
+
{%- set unique_key = config.get('unique_key') %}
|
|
7
|
+
-- grab current tables grants config for comparision later on
|
|
8
|
+
{%- set grant_config = config.get('grants') -%}
|
|
9
|
+
|
|
10
|
+
{% set target_relation_exists, target_relation = get_or_create_relation(
|
|
11
|
+
database=model.database,
|
|
12
|
+
schema=model.schema,
|
|
13
|
+
identifier=target_table,
|
|
14
|
+
type='table') -%}
|
|
15
|
+
|
|
16
|
+
{%- if not target_relation.is_table -%}
|
|
17
|
+
{% do exceptions.relation_wrong_type(target_relation, 'table') %}
|
|
18
|
+
{%- endif -%}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
|
|
22
|
+
|
|
23
|
+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
|
|
24
|
+
|
|
25
|
+
{% set strategy_macro = strategy_dispatch(strategy_name) %}
|
|
26
|
+
{# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}
|
|
27
|
+
{# It was a dictionary of config, instead of the config object from the context #}
|
|
28
|
+
{% set strategy = strategy_macro(model, "snapshotted_data", "source_data", model['config'], target_relation_exists) %}
|
|
29
|
+
|
|
30
|
+
{% if not target_relation_exists %}
|
|
31
|
+
|
|
32
|
+
{% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}
|
|
33
|
+
{% set build_or_select_sql = build_sql %}
|
|
34
|
+
{% set final_sql = create_table_as(False, target_relation, build_sql) %}
|
|
35
|
+
|
|
36
|
+
{% else %}
|
|
37
|
+
|
|
38
|
+
{% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %}
|
|
39
|
+
|
|
40
|
+
{{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}
|
|
41
|
+
|
|
42
|
+
{% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}
|
|
43
|
+
{% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}
|
|
44
|
+
|
|
45
|
+
-- this may no-op if the database does not require column expansion
|
|
46
|
+
{% do adapter.expand_target_column_types(from_relation=staging_table,
|
|
47
|
+
to_relation=target_relation) %}
|
|
48
|
+
|
|
49
|
+
{% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}
|
|
50
|
+
{% if unique_key | is_list %}
|
|
51
|
+
{% for key in strategy.unique_key %}
|
|
52
|
+
{{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}
|
|
53
|
+
{{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}
|
|
54
|
+
{% endfor %}
|
|
55
|
+
{% endif %}
|
|
56
|
+
|
|
57
|
+
{% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)
|
|
58
|
+
| rejectattr('name', 'in', remove_columns)
|
|
59
|
+
| list %}
|
|
60
|
+
|
|
61
|
+
{% do create_columns(target_relation, missing_columns) %}
|
|
62
|
+
|
|
63
|
+
{% set source_columns = adapter.get_columns_in_relation(staging_table)
|
|
64
|
+
| rejectattr('name', 'in', remove_columns)
|
|
65
|
+
| list %}
|
|
66
|
+
|
|
67
|
+
{% set quoted_source_columns = [] %}
|
|
68
|
+
{% for column in source_columns %}
|
|
69
|
+
{% do quoted_source_columns.append(adapter.quote(column.name)) %}
|
|
70
|
+
{% endfor %}
|
|
71
|
+
|
|
72
|
+
{% set final_sql = snapshot_merge_sql(
|
|
73
|
+
target = target_relation,
|
|
74
|
+
source = staging_table,
|
|
75
|
+
insert_cols = quoted_source_columns
|
|
76
|
+
)
|
|
77
|
+
%}
|
|
78
|
+
|
|
79
|
+
{% endif %}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
{{ check_time_data_types(build_or_select_sql) }}
|
|
83
|
+
|
|
84
|
+
{% call statement('main') %}
|
|
85
|
+
{{ final_sql }}
|
|
86
|
+
{% endcall %}
|
|
87
|
+
|
|
88
|
+
{% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}
|
|
89
|
+
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
|
90
|
+
|
|
91
|
+
{% do persist_docs(target_relation, model) %}
|
|
92
|
+
|
|
93
|
+
{% if not target_relation_exists %}
|
|
94
|
+
{% do create_indexes(target_relation) %}
|
|
95
|
+
{% endif %}
|
|
96
|
+
|
|
97
|
+
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
|
98
|
+
|
|
99
|
+
{{ adapter.commit() }}
|
|
100
|
+
|
|
101
|
+
{% if staging_table is defined %}
|
|
102
|
+
{% do post_snapshot(staging_table) %}
|
|
103
|
+
{% endif %}
|
|
104
|
+
|
|
105
|
+
{{ run_hooks(post_hooks, inside_transaction=False) }}
|
|
106
|
+
|
|
107
|
+
{{ return({'relations': [target_relation]}) }}
|
|
108
|
+
|
|
109
|
+
{% endmaterialization %}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
|
|
2
|
+
{% macro snapshot_merge_sql(target, source, insert_cols) -%}
|
|
3
|
+
{{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}
|
|
4
|
+
{%- endmacro %}
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}
|
|
8
|
+
{%- set insert_cols_csv = insert_cols | join(', ') -%}
|
|
9
|
+
|
|
10
|
+
{%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%}
|
|
11
|
+
|
|
12
|
+
merge into {{ target.render() }} as DBT_INTERNAL_DEST
|
|
13
|
+
using {{ source }} as DBT_INTERNAL_SOURCE
|
|
14
|
+
on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}
|
|
15
|
+
|
|
16
|
+
when matched
|
|
17
|
+
{% if config.get("dbt_valid_to_current") %}
|
|
18
|
+
{% set source_unique_key = ("DBT_INTERNAL_DEST." ~ columns.dbt_valid_to) | trim %}
|
|
19
|
+
{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
|
|
20
|
+
and ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null)
|
|
21
|
+
|
|
22
|
+
{% else %}
|
|
23
|
+
and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
|
|
24
|
+
{% endif %}
|
|
25
|
+
and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
|
|
26
|
+
then update
|
|
27
|
+
set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}
|
|
28
|
+
|
|
29
|
+
when not matched
|
|
30
|
+
and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'
|
|
31
|
+
then insert ({{ insert_cols_csv }})
|
|
32
|
+
values ({{ insert_cols_csv }})
|
|
33
|
+
|
|
34
|
+
{% endmacro %}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
{#
|
|
2
|
+
Dispatch strategies by name, optionally qualified to a package
|
|
3
|
+
#}
|
|
4
|
+
{% macro strategy_dispatch(name) -%}
|
|
5
|
+
{% set original_name = name %}
|
|
6
|
+
{% if '.' in name %}
|
|
7
|
+
{% set package_name, name = name.split(".", 1) %}
|
|
8
|
+
{% else %}
|
|
9
|
+
{% set package_name = none %}
|
|
10
|
+
{% endif %}
|
|
11
|
+
|
|
12
|
+
{% if package_name is none %}
|
|
13
|
+
{% set package_context = context %}
|
|
14
|
+
{% elif package_name in context %}
|
|
15
|
+
{% set package_context = context[package_name] %}
|
|
16
|
+
{% else %}
|
|
17
|
+
{% set error_msg %}
|
|
18
|
+
Could not find package '{{package_name}}', called with '{{original_name}}'
|
|
19
|
+
{% endset %}
|
|
20
|
+
{{ exceptions.raise_compiler_error(error_msg | trim) }}
|
|
21
|
+
{% endif %}
|
|
22
|
+
|
|
23
|
+
{%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}
|
|
24
|
+
|
|
25
|
+
{% if search_name not in package_context %}
|
|
26
|
+
{% set error_msg %}
|
|
27
|
+
The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'
|
|
28
|
+
{% endset %}
|
|
29
|
+
{{ exceptions.raise_compiler_error(error_msg | trim) }}
|
|
30
|
+
{% endif %}
|
|
31
|
+
{{ return(package_context[search_name]) }}
|
|
32
|
+
{%- endmacro %}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
{#
|
|
36
|
+
Create SCD Hash SQL fields cross-db
|
|
37
|
+
#}
|
|
38
|
+
{% macro snapshot_hash_arguments(args) -%}
|
|
39
|
+
{{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}
|
|
40
|
+
{%- endmacro %}
|
|
41
|
+
|
|
42
|
+
{% macro default__snapshot_hash_arguments(args) -%}
|
|
43
|
+
md5({%- for arg in args -%}
|
|
44
|
+
coalesce(cast({{ arg }} as varchar ), '')
|
|
45
|
+
{% if not loop.last %} || '|' || {% endif %}
|
|
46
|
+
{%- endfor -%})
|
|
47
|
+
{%- endmacro %}
|
|
48
|
+
|
|
49
|
+
{#
|
|
50
|
+
Core strategy definitions
|
|
51
|
+
#}
|
|
52
|
+
|
|
53
|
+
{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}
|
|
54
|
+
{# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}
|
|
55
|
+
{% set primary_key = config.get('unique_key') %}
|
|
56
|
+
{% set updated_at = config.get('updated_at') %}
|
|
57
|
+
{% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}
|
|
58
|
+
{% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}
|
|
59
|
+
{% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %}
|
|
60
|
+
|
|
61
|
+
{#/*
|
|
62
|
+
The snapshot relation might not have an {{ updated_at }} value if the
|
|
63
|
+
snapshot strategy is changed from `check` to `timestamp`. We
|
|
64
|
+
should use a dbt-created column for the comparison in the snapshot
|
|
65
|
+
table instead of assuming that the user-supplied {{ updated_at }}
|
|
66
|
+
will be present in the historical data.
|
|
67
|
+
|
|
68
|
+
See https://github.com/dbt-labs/dbt-core/issues/2350
|
|
69
|
+
*/ #}
|
|
70
|
+
{% set row_changed_expr -%}
|
|
71
|
+
({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})
|
|
72
|
+
{%- endset %}
|
|
73
|
+
|
|
74
|
+
{% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}
|
|
75
|
+
{% set scd_id_expr = snapshot_hash_arguments(scd_args) %}
|
|
76
|
+
|
|
77
|
+
{% do return({
|
|
78
|
+
"unique_key": primary_key,
|
|
79
|
+
"updated_at": updated_at,
|
|
80
|
+
"row_changed": row_changed_expr,
|
|
81
|
+
"scd_id": scd_id_expr,
|
|
82
|
+
"invalidate_hard_deletes": invalidate_hard_deletes,
|
|
83
|
+
"hard_deletes": hard_deletes
|
|
84
|
+
}) %}
|
|
85
|
+
{% endmacro %}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
{% macro snapshot_string_as_time(timestamp) -%}
|
|
89
|
+
{{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}
|
|
90
|
+
{%- endmacro %}
|
|
91
|
+
|
|
92
|
+
{% macro default__snapshot_string_as_time(timestamp) %}
|
|
93
|
+
{% do exceptions.raise_not_implemented(
|
|
94
|
+
'snapshot_string_as_time macro not implemented for adapter '+adapter.type()
|
|
95
|
+
) %}
|
|
96
|
+
{% endmacro %}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}
|
|
100
|
+
{%- if not target_exists -%}
|
|
101
|
+
{#-- no table yet -> return whatever the query does --#}
|
|
102
|
+
{{ return((false, query_columns)) }}
|
|
103
|
+
{%- endif -%}
|
|
104
|
+
|
|
105
|
+
{#-- handle any schema changes --#}
|
|
106
|
+
{%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}
|
|
107
|
+
|
|
108
|
+
{% if check_cols_config == 'all' %}
|
|
109
|
+
{%- set query_columns = get_columns_in_query(node['compiled_code']) -%}
|
|
110
|
+
|
|
111
|
+
{% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}
|
|
112
|
+
{#-- query for proper casing/quoting, to support comparison below --#}
|
|
113
|
+
{%- set select_check_cols_from_target -%}
|
|
114
|
+
{#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}
|
|
115
|
+
{#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}
|
|
116
|
+
select {{ check_cols_config | join(', ') }} from (
|
|
117
|
+
{{ node['compiled_code'] }}
|
|
118
|
+
) subq
|
|
119
|
+
{%- endset -%}
|
|
120
|
+
{% set query_columns = get_columns_in_query(select_check_cols_from_target) %}
|
|
121
|
+
|
|
122
|
+
{% else %}
|
|
123
|
+
{% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %}
|
|
124
|
+
{% endif %}
|
|
125
|
+
|
|
126
|
+
{%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}
|
|
127
|
+
{%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}
|
|
128
|
+
{%- set ns.column_added = false -%}
|
|
129
|
+
|
|
130
|
+
{%- set intersection = [] -%}
|
|
131
|
+
{%- for col in query_columns -%}
|
|
132
|
+
{%- if col in existing_cols -%}
|
|
133
|
+
{%- do intersection.append(adapter.quote(col)) -%}
|
|
134
|
+
{%- else -%}
|
|
135
|
+
{% set ns.column_added = true %}
|
|
136
|
+
{%- endif -%}
|
|
137
|
+
{%- endfor -%}
|
|
138
|
+
{{ return((ns.column_added, intersection)) }}
|
|
139
|
+
{%- endmacro %}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}
|
|
143
|
+
{# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}
|
|
144
|
+
{% set check_cols_config = config.get('check_cols') %}
|
|
145
|
+
{% set primary_key = config.get('unique_key') %}
|
|
146
|
+
{% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}
|
|
147
|
+
{% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}
|
|
148
|
+
{% set updated_at = config.get('updated_at') or snapshot_get_time() %}
|
|
149
|
+
|
|
150
|
+
{% set column_added = false %}
|
|
151
|
+
|
|
152
|
+
{% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}
|
|
153
|
+
|
|
154
|
+
{%- set row_changed_expr -%}
|
|
155
|
+
(
|
|
156
|
+
{%- if column_added -%}
|
|
157
|
+
{{ get_true_sql() }}
|
|
158
|
+
{%- else -%}
|
|
159
|
+
{%- for col in check_cols -%}
|
|
160
|
+
{{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}
|
|
161
|
+
or
|
|
162
|
+
(
|
|
163
|
+
(({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))
|
|
164
|
+
or
|
|
165
|
+
((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))
|
|
166
|
+
)
|
|
167
|
+
{%- if not loop.last %} or {% endif -%}
|
|
168
|
+
{%- endfor -%}
|
|
169
|
+
{%- endif -%}
|
|
170
|
+
)
|
|
171
|
+
{%- endset %}
|
|
172
|
+
|
|
173
|
+
{% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}
|
|
174
|
+
{% set scd_id_expr = snapshot_hash_arguments(scd_args) %}
|
|
175
|
+
|
|
176
|
+
{% do return({
|
|
177
|
+
"unique_key": primary_key,
|
|
178
|
+
"updated_at": updated_at,
|
|
179
|
+
"row_changed": row_changed_expr,
|
|
180
|
+
"scd_id": scd_id_expr,
|
|
181
|
+
"invalidate_hard_deletes": invalidate_hard_deletes,
|
|
182
|
+
"hard_deletes": hard_deletes
|
|
183
|
+
}) %}
|
|
184
|
+
{% endmacro %}
|