dbt-adapters 1.22.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. dbt/adapters/__about__.py +1 -0
  2. dbt/adapters/__init__.py +8 -0
  3. dbt/adapters/base/README.md +13 -0
  4. dbt/adapters/base/__init__.py +16 -0
  5. dbt/adapters/base/column.py +173 -0
  6. dbt/adapters/base/connections.py +429 -0
  7. dbt/adapters/base/impl.py +2036 -0
  8. dbt/adapters/base/meta.py +150 -0
  9. dbt/adapters/base/plugin.py +32 -0
  10. dbt/adapters/base/query_headers.py +106 -0
  11. dbt/adapters/base/relation.py +648 -0
  12. dbt/adapters/cache.py +521 -0
  13. dbt/adapters/capability.py +63 -0
  14. dbt/adapters/catalogs/__init__.py +14 -0
  15. dbt/adapters/catalogs/_client.py +54 -0
  16. dbt/adapters/catalogs/_constants.py +1 -0
  17. dbt/adapters/catalogs/_exceptions.py +39 -0
  18. dbt/adapters/catalogs/_integration.py +113 -0
  19. dbt/adapters/clients/__init__.py +0 -0
  20. dbt/adapters/clients/jinja.py +24 -0
  21. dbt/adapters/contracts/__init__.py +0 -0
  22. dbt/adapters/contracts/connection.py +229 -0
  23. dbt/adapters/contracts/macros.py +11 -0
  24. dbt/adapters/contracts/relation.py +160 -0
  25. dbt/adapters/events/README.md +51 -0
  26. dbt/adapters/events/__init__.py +0 -0
  27. dbt/adapters/events/adapter_types_pb2.py +2 -0
  28. dbt/adapters/events/base_types.py +36 -0
  29. dbt/adapters/events/logging.py +83 -0
  30. dbt/adapters/events/types.py +436 -0
  31. dbt/adapters/exceptions/__init__.py +40 -0
  32. dbt/adapters/exceptions/alias.py +24 -0
  33. dbt/adapters/exceptions/cache.py +68 -0
  34. dbt/adapters/exceptions/compilation.py +269 -0
  35. dbt/adapters/exceptions/connection.py +16 -0
  36. dbt/adapters/exceptions/database.py +51 -0
  37. dbt/adapters/factory.py +264 -0
  38. dbt/adapters/protocol.py +150 -0
  39. dbt/adapters/py.typed +0 -0
  40. dbt/adapters/record/__init__.py +2 -0
  41. dbt/adapters/record/base.py +291 -0
  42. dbt/adapters/record/cursor/cursor.py +69 -0
  43. dbt/adapters/record/cursor/description.py +37 -0
  44. dbt/adapters/record/cursor/execute.py +39 -0
  45. dbt/adapters/record/cursor/fetchall.py +69 -0
  46. dbt/adapters/record/cursor/fetchmany.py +23 -0
  47. dbt/adapters/record/cursor/fetchone.py +23 -0
  48. dbt/adapters/record/cursor/rowcount.py +23 -0
  49. dbt/adapters/record/handle.py +55 -0
  50. dbt/adapters/record/serialization.py +115 -0
  51. dbt/adapters/reference_keys.py +39 -0
  52. dbt/adapters/relation_configs/README.md +25 -0
  53. dbt/adapters/relation_configs/__init__.py +12 -0
  54. dbt/adapters/relation_configs/config_base.py +46 -0
  55. dbt/adapters/relation_configs/config_change.py +26 -0
  56. dbt/adapters/relation_configs/config_validation.py +57 -0
  57. dbt/adapters/sql/__init__.py +2 -0
  58. dbt/adapters/sql/connections.py +263 -0
  59. dbt/adapters/sql/impl.py +286 -0
  60. dbt/adapters/utils.py +69 -0
  61. dbt/include/__init__.py +3 -0
  62. dbt/include/global_project/__init__.py +4 -0
  63. dbt/include/global_project/dbt_project.yml +7 -0
  64. dbt/include/global_project/docs/overview.md +43 -0
  65. dbt/include/global_project/macros/adapters/apply_grants.sql +167 -0
  66. dbt/include/global_project/macros/adapters/columns.sql +144 -0
  67. dbt/include/global_project/macros/adapters/freshness.sql +32 -0
  68. dbt/include/global_project/macros/adapters/indexes.sql +41 -0
  69. dbt/include/global_project/macros/adapters/metadata.sql +105 -0
  70. dbt/include/global_project/macros/adapters/persist_docs.sql +33 -0
  71. dbt/include/global_project/macros/adapters/relation.sql +84 -0
  72. dbt/include/global_project/macros/adapters/schema.sql +20 -0
  73. dbt/include/global_project/macros/adapters/show.sql +26 -0
  74. dbt/include/global_project/macros/adapters/timestamps.sql +52 -0
  75. dbt/include/global_project/macros/adapters/validate_sql.sql +10 -0
  76. dbt/include/global_project/macros/etc/datetime.sql +62 -0
  77. dbt/include/global_project/macros/etc/statement.sql +52 -0
  78. dbt/include/global_project/macros/generic_test_sql/accepted_values.sql +27 -0
  79. dbt/include/global_project/macros/generic_test_sql/not_null.sql +9 -0
  80. dbt/include/global_project/macros/generic_test_sql/relationships.sql +23 -0
  81. dbt/include/global_project/macros/generic_test_sql/unique.sql +12 -0
  82. dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql +36 -0
  83. dbt/include/global_project/macros/get_custom_name/get_custom_database.sql +32 -0
  84. dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql +60 -0
  85. dbt/include/global_project/macros/materializations/configs.sql +21 -0
  86. dbt/include/global_project/macros/materializations/functions/aggregate.sql +65 -0
  87. dbt/include/global_project/macros/materializations/functions/function.sql +20 -0
  88. dbt/include/global_project/macros/materializations/functions/helpers.sql +20 -0
  89. dbt/include/global_project/macros/materializations/functions/scalar.sql +69 -0
  90. dbt/include/global_project/macros/materializations/hooks.sql +35 -0
  91. dbt/include/global_project/macros/materializations/models/clone/can_clone_table.sql +7 -0
  92. dbt/include/global_project/macros/materializations/models/clone/clone.sql +67 -0
  93. dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql +7 -0
  94. dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql +80 -0
  95. dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +99 -0
  96. dbt/include/global_project/macros/materializations/models/incremental/is_incremental.sql +13 -0
  97. dbt/include/global_project/macros/materializations/models/incremental/merge.sql +120 -0
  98. dbt/include/global_project/macros/materializations/models/incremental/on_schema_change.sql +159 -0
  99. dbt/include/global_project/macros/materializations/models/incremental/strategies.sql +92 -0
  100. dbt/include/global_project/macros/materializations/models/materialized_view.sql +121 -0
  101. dbt/include/global_project/macros/materializations/models/table.sql +64 -0
  102. dbt/include/global_project/macros/materializations/models/view.sql +72 -0
  103. dbt/include/global_project/macros/materializations/seeds/helpers.sql +128 -0
  104. dbt/include/global_project/macros/materializations/seeds/seed.sql +60 -0
  105. dbt/include/global_project/macros/materializations/snapshots/helpers.sql +345 -0
  106. dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +109 -0
  107. dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +34 -0
  108. dbt/include/global_project/macros/materializations/snapshots/strategies.sql +184 -0
  109. dbt/include/global_project/macros/materializations/tests/helpers.sql +44 -0
  110. dbt/include/global_project/macros/materializations/tests/test.sql +66 -0
  111. dbt/include/global_project/macros/materializations/tests/unit.sql +40 -0
  112. dbt/include/global_project/macros/materializations/tests/where_subquery.sql +15 -0
  113. dbt/include/global_project/macros/python_model/python.sql +114 -0
  114. dbt/include/global_project/macros/relations/column/columns_spec_ddl.sql +89 -0
  115. dbt/include/global_project/macros/relations/create.sql +23 -0
  116. dbt/include/global_project/macros/relations/create_backup.sql +17 -0
  117. dbt/include/global_project/macros/relations/create_intermediate.sql +17 -0
  118. dbt/include/global_project/macros/relations/drop.sql +41 -0
  119. dbt/include/global_project/macros/relations/drop_backup.sql +14 -0
  120. dbt/include/global_project/macros/relations/materialized_view/alter.sql +55 -0
  121. dbt/include/global_project/macros/relations/materialized_view/create.sql +10 -0
  122. dbt/include/global_project/macros/relations/materialized_view/drop.sql +14 -0
  123. dbt/include/global_project/macros/relations/materialized_view/refresh.sql +9 -0
  124. dbt/include/global_project/macros/relations/materialized_view/rename.sql +10 -0
  125. dbt/include/global_project/macros/relations/materialized_view/replace.sql +10 -0
  126. dbt/include/global_project/macros/relations/rename.sql +35 -0
  127. dbt/include/global_project/macros/relations/rename_intermediate.sql +14 -0
  128. dbt/include/global_project/macros/relations/replace.sql +50 -0
  129. dbt/include/global_project/macros/relations/schema.sql +8 -0
  130. dbt/include/global_project/macros/relations/table/create.sql +60 -0
  131. dbt/include/global_project/macros/relations/table/drop.sql +14 -0
  132. dbt/include/global_project/macros/relations/table/rename.sql +10 -0
  133. dbt/include/global_project/macros/relations/table/replace.sql +10 -0
  134. dbt/include/global_project/macros/relations/view/create.sql +27 -0
  135. dbt/include/global_project/macros/relations/view/drop.sql +14 -0
  136. dbt/include/global_project/macros/relations/view/rename.sql +10 -0
  137. dbt/include/global_project/macros/relations/view/replace.sql +66 -0
  138. dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +107 -0
  139. dbt/include/global_project/macros/utils/any_value.sql +9 -0
  140. dbt/include/global_project/macros/utils/array_append.sql +8 -0
  141. dbt/include/global_project/macros/utils/array_concat.sql +7 -0
  142. dbt/include/global_project/macros/utils/array_construct.sql +12 -0
  143. dbt/include/global_project/macros/utils/bool_or.sql +9 -0
  144. dbt/include/global_project/macros/utils/cast.sql +7 -0
  145. dbt/include/global_project/macros/utils/cast_bool_to_text.sql +7 -0
  146. dbt/include/global_project/macros/utils/concat.sql +7 -0
  147. dbt/include/global_project/macros/utils/data_types.sql +129 -0
  148. dbt/include/global_project/macros/utils/date.sql +10 -0
  149. dbt/include/global_project/macros/utils/date_spine.sql +75 -0
  150. dbt/include/global_project/macros/utils/date_trunc.sql +7 -0
  151. dbt/include/global_project/macros/utils/dateadd.sql +14 -0
  152. dbt/include/global_project/macros/utils/datediff.sql +14 -0
  153. dbt/include/global_project/macros/utils/equals.sql +14 -0
  154. dbt/include/global_project/macros/utils/escape_single_quotes.sql +8 -0
  155. dbt/include/global_project/macros/utils/except.sql +9 -0
  156. dbt/include/global_project/macros/utils/generate_series.sql +53 -0
  157. dbt/include/global_project/macros/utils/hash.sql +7 -0
  158. dbt/include/global_project/macros/utils/intersect.sql +9 -0
  159. dbt/include/global_project/macros/utils/last_day.sql +15 -0
  160. dbt/include/global_project/macros/utils/length.sql +11 -0
  161. dbt/include/global_project/macros/utils/listagg.sql +30 -0
  162. dbt/include/global_project/macros/utils/literal.sql +7 -0
  163. dbt/include/global_project/macros/utils/position.sql +11 -0
  164. dbt/include/global_project/macros/utils/replace.sql +14 -0
  165. dbt/include/global_project/macros/utils/right.sql +12 -0
  166. dbt/include/global_project/macros/utils/safe_cast.sql +9 -0
  167. dbt/include/global_project/macros/utils/split_part.sql +26 -0
  168. dbt/include/global_project/tests/generic/builtin.sql +30 -0
  169. dbt/include/py.typed +0 -0
  170. dbt_adapters-1.22.2.dist-info/METADATA +124 -0
  171. dbt_adapters-1.22.2.dist-info/RECORD +173 -0
  172. dbt_adapters-1.22.2.dist-info/WHEEL +4 -0
  173. dbt_adapters-1.22.2.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,345 @@
1
+ {#
2
+ Add new columns to the table if applicable
3
+ #}
4
+ {% macro create_columns(relation, columns) %}
5
+ {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}
6
+ {% endmacro %}
7
+
8
+ {% macro default__create_columns(relation, columns) %}
9
+ {% for column in columns %}
10
+ {% call statement() %}
11
+ alter table {{ relation.render() }} add column {{ adapter.quote(column.name) }} {{ column.expanded_data_type }};
12
+ {% endcall %}
13
+ {% endfor %}
14
+ {% endmacro %}
15
+
16
+
17
+ {% macro post_snapshot(staging_relation) %}
18
+ {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}
19
+ {% endmacro %}
20
+
21
+ {% macro default__post_snapshot(staging_relation) %}
22
+ {# no-op #}
23
+ {% endmacro %}
24
+
25
+ {% macro get_true_sql() %}
26
+ {{ adapter.dispatch('get_true_sql', 'dbt')() }}
27
+ {% endmacro %}
28
+
29
+ {% macro default__get_true_sql() %}
30
+ {{ return('TRUE') }}
31
+ {% endmacro %}
32
+
33
+ {% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}
34
+ {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}
35
+ {% endmacro %}
36
+
37
+ {% macro get_snapshot_table_column_names() %}
38
+ {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }}
39
+ {% endmacro %}
40
+
41
+ {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}
42
+ {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}
43
+ {% if strategy.hard_deletes == 'new_record' %}
44
+ {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}
45
+ {% endif %}
46
+ with snapshot_query as (
47
+
48
+ {{ source_sql }}
49
+
50
+ ),
51
+
52
+ snapshotted_data as (
53
+
54
+ select *, {{ unique_key_fields(strategy.unique_key) }}
55
+ from {{ target_relation }}
56
+ where
57
+ {% if config.get('dbt_valid_to_current') %}
58
+ {% set source_unique_key = columns.dbt_valid_to | trim %}
59
+ {% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
60
+
61
+ {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}
62
+ ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )
63
+ {% else %}
64
+ {{ columns.dbt_valid_to }} is null
65
+ {% endif %}
66
+
67
+ ),
68
+
69
+ insertions_source_data as (
70
+
71
+ select *, {{ unique_key_fields(strategy.unique_key) }},
72
+ {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
73
+ {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
74
+ {{ get_dbt_valid_to_current(strategy, columns) }},
75
+ {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}
76
+
77
+ from snapshot_query
78
+ ),
79
+
80
+ updates_source_data as (
81
+
82
+ select *, {{ unique_key_fields(strategy.unique_key) }},
83
+ {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
84
+ {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
85
+ {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}
86
+
87
+ from snapshot_query
88
+ ),
89
+
90
+ {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
91
+
92
+ deletes_source_data as (
93
+
94
+ select *, {{ unique_key_fields(strategy.unique_key) }}
95
+ from snapshot_query
96
+ ),
97
+ {% endif %}
98
+
99
+ insertions as (
100
+
101
+ select
102
+ 'insert' as dbt_change_type,
103
+ source_data.*
104
+ {%- if strategy.hard_deletes == 'new_record' -%}
105
+ ,'False' as {{ columns.dbt_is_deleted }}
106
+ {%- endif %}
107
+
108
+ from insertions_source_data as source_data
109
+ left outer join snapshotted_data
110
+ on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
111
+ where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }}
112
+ or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and (
113
+ {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}
114
+ )
115
+
116
+ )
117
+
118
+ ),
119
+
120
+ updates as (
121
+
122
+ select
123
+ 'update' as dbt_change_type,
124
+ source_data.*,
125
+ snapshotted_data.{{ columns.dbt_scd_id }}
126
+ {%- if strategy.hard_deletes == 'new_record' -%}
127
+ , snapshotted_data.{{ columns.dbt_is_deleted }}
128
+ {%- endif %}
129
+
130
+ from updates_source_data as source_data
131
+ join snapshotted_data
132
+ on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
133
+ where (
134
+ {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}
135
+ )
136
+ )
137
+
138
+ {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
139
+ ,
140
+ deletes as (
141
+
142
+ select
143
+ 'delete' as dbt_change_type,
144
+ source_data.*,
145
+ {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},
146
+ {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},
147
+ {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},
148
+ snapshotted_data.{{ columns.dbt_scd_id }}
149
+ {%- if strategy.hard_deletes == 'new_record' -%}
150
+ , snapshotted_data.{{ columns.dbt_is_deleted }}
151
+ {%- endif %}
152
+ from snapshotted_data
153
+ left join deletes_source_data as source_data
154
+ on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
155
+ where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
156
+
157
+ {%- if strategy.hard_deletes == 'new_record' %}
158
+ and not (
159
+ --avoid updating the record's valid_to if the latest entry is marked as deleted
160
+ snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
161
+ and
162
+ {% if config.get('dbt_valid_to_current') -%}
163
+ snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}
164
+ {%- else -%}
165
+ snapshotted_data.{{ columns.dbt_valid_to }} is null
166
+ {%- endif %}
167
+ )
168
+ {%- endif %}
169
+ )
170
+ {%- endif %}
171
+
172
+ {%- if strategy.hard_deletes == 'new_record' %}
173
+ {% set snapshotted_cols = get_list_of_column_names(get_columns_in_relation(target_relation)) %}
174
+ {% set source_sql_cols = get_column_schema_from_query(source_sql) %}
175
+ ,
176
+ deletion_records as (
177
+
178
+ select
179
+ 'insert' as dbt_change_type,
180
+ {#/*
181
+ If a column has been added to the source it won't yet exist in the
182
+ snapshotted table so we insert a null value as a placeholder for the column.
183
+ */#}
184
+ {%- for col in source_sql_cols -%}
185
+ {%- if col.name in snapshotted_cols -%}
186
+ snapshotted_data.{{ adapter.quote(col.column) }},
187
+ {%- else -%}
188
+ NULL as {{ adapter.quote(col.column) }},
189
+ {%- endif -%}
190
+ {% endfor -%}
191
+ {%- if strategy.unique_key | is_list -%}
192
+ {%- for key in strategy.unique_key -%}
193
+ snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},
194
+ {% endfor -%}
195
+ {%- else -%}
196
+ snapshotted_data.dbt_unique_key as dbt_unique_key,
197
+ {% endif -%}
198
+ {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},
199
+ {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},
200
+ snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},
201
+ {{ new_scd_id }} as {{ columns.dbt_scd_id }},
202
+ 'True' as {{ columns.dbt_is_deleted }}
203
+ from snapshotted_data
204
+ left join deletes_source_data as source_data
205
+ on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }}
206
+ where {{ unique_key_is_null(strategy.unique_key, "source_data") }}
207
+ and not (
208
+ --avoid inserting a new record if the latest one is marked as deleted
209
+ snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'
210
+ and
211
+ {% if config.get('dbt_valid_to_current') -%}
212
+ snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}
213
+ {%- else -%}
214
+ snapshotted_data.{{ columns.dbt_valid_to }} is null
215
+ {%- endif %}
216
+ )
217
+
218
+ )
219
+ {%- endif %}
220
+
221
+ select * from insertions
222
+ union all
223
+ select * from updates
224
+ {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}
225
+ union all
226
+ select * from deletes
227
+ {%- endif %}
228
+ {%- if strategy.hard_deletes == 'new_record' %}
229
+ union all
230
+ select * from deletion_records
231
+ {%- endif %}
232
+
233
+
234
+ {%- endmacro %}
235
+
236
+
237
+ {% macro build_snapshot_table(strategy, sql) -%}
238
+ {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}
239
+ {% endmacro %}
240
+
241
+ {% macro default__build_snapshot_table(strategy, sql) %}
242
+ {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}
243
+
244
+ select *,
245
+ {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},
246
+ {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
247
+ {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
248
+ {{ get_dbt_valid_to_current(strategy, columns) }}
249
+ {%- if strategy.hard_deletes == 'new_record' -%}
250
+ , 'False' as {{ columns.dbt_is_deleted }}
251
+ {% endif -%}
252
+ from (
253
+ {{ sql }}
254
+ ) sbq
255
+
256
+ {% endmacro %}
257
+
258
+
259
+ {% macro build_snapshot_staging_table(strategy, sql, target_relation) %}
260
+ {% set temp_relation = make_temp_relation(target_relation) %}
261
+
262
+ {% set select = snapshot_staging_table(strategy, sql, target_relation) %}
263
+
264
+ {% call statement('build_snapshot_staging_relation') %}
265
+ {{ create_table_as(True, temp_relation, select) }}
266
+ {% endcall %}
267
+
268
+ {% do return(temp_relation) %}
269
+ {% endmacro %}
270
+
271
+
272
+ {% macro get_updated_at_column_data_type(snapshot_sql) %}
273
+ {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}
274
+ {% set dbt_updated_at_data_type = null %}
275
+ {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}
276
+ {% set ns.dbt_updated_at_data_type = null -%}
277
+ {% for column in snapshot_sql_column_schema %}
278
+ {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}
279
+ {% set ns.dbt_updated_at_data_type = column.dtype %}
280
+ {% endif %}
281
+ {% endfor %}
282
+ {{ return(ns.dbt_updated_at_data_type or none) }}
283
+ {% endmacro %}
284
+
285
+
286
+ {% macro check_time_data_types(sql) %}
287
+ {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}
288
+ {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}
289
+ {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}
290
+ {% if exceptions.warn_snapshot_timestamp_data_types %}
291
+ {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}
292
+ {% endif %}
293
+ {% endif %}
294
+ {% endmacro %}
295
+
296
+
297
+ {% macro get_dbt_valid_to_current(strategy, columns) %}
298
+ {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %}
299
+ coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})
300
+ as {{ columns.dbt_valid_to }}
301
+ {% endmacro %}
302
+
303
+
304
+ {% macro unique_key_fields(unique_key) %}
305
+ {% if unique_key | is_list %}
306
+ {% for key in unique_key %}
307
+ {{ key }} as dbt_unique_key_{{ loop.index }}
308
+ {%- if not loop.last %} , {%- endif %}
309
+ {% endfor %}
310
+ {% else %}
311
+ {{ unique_key }} as dbt_unique_key
312
+ {% endif %}
313
+ {% endmacro %}
314
+
315
+
316
+ {% macro unique_key_join_on(unique_key, identifier, from_identifier) %}
317
+ {% if unique_key | is_list %}
318
+ {% for key in unique_key %}
319
+ {% set source_unique_key = (identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
320
+ {% set target_unique_key = (from_identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %}
321
+ {{ equals(source_unique_key, target_unique_key) }}
322
+ {%- if not loop.last %} and {%- endif %}
323
+ {% endfor %}
324
+ {% else %}
325
+ {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key
326
+ {% endif %}
327
+ {% endmacro %}
328
+
329
+
330
+ {% macro unique_key_is_null(unique_key, identifier) %}
331
+ {% if unique_key | is_list %}
332
+ {{ identifier }}.dbt_unique_key_1 is null
333
+ {% else %}
334
+ {{ identifier }}.dbt_unique_key is null
335
+ {% endif %}
336
+ {% endmacro %}
337
+
338
+
339
+ {% macro unique_key_is_not_null(unique_key, identifier) %}
340
+ {% if unique_key | is_list %}
341
+ {{ identifier }}.dbt_unique_key_1 is not null
342
+ {% else %}
343
+ {{ identifier }}.dbt_unique_key is not null
344
+ {% endif %}
345
+ {% endmacro %}
@@ -0,0 +1,109 @@
1
+ {% materialization snapshot, default %}
2
+
3
+ {%- set target_table = model.get('alias', model.get('name')) -%}
4
+
5
+ {%- set strategy_name = config.get('strategy') -%}
6
+ {%- set unique_key = config.get('unique_key') %}
7
+ -- grab current tables grants config for comparision later on
8
+ {%- set grant_config = config.get('grants') -%}
9
+
10
+ {% set target_relation_exists, target_relation = get_or_create_relation(
11
+ database=model.database,
12
+ schema=model.schema,
13
+ identifier=target_table,
14
+ type='table') -%}
15
+
16
+ {%- if not target_relation.is_table -%}
17
+ {% do exceptions.relation_wrong_type(target_relation, 'table') %}
18
+ {%- endif -%}
19
+
20
+
21
+ {{ run_hooks(pre_hooks, inside_transaction=False) }}
22
+
23
+ {{ run_hooks(pre_hooks, inside_transaction=True) }}
24
+
25
+ {% set strategy_macro = strategy_dispatch(strategy_name) %}
26
+ {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}
27
+ {# It was a dictionary of config, instead of the config object from the context #}
28
+ {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", model['config'], target_relation_exists) %}
29
+
30
+ {% if not target_relation_exists %}
31
+
32
+ {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}
33
+ {% set build_or_select_sql = build_sql %}
34
+ {% set final_sql = create_table_as(False, target_relation, build_sql) %}
35
+
36
+ {% else %}
37
+
38
+ {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %}
39
+
40
+ {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}
41
+
42
+ {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}
43
+ {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}
44
+
45
+ -- this may no-op if the database does not require column expansion
46
+ {% do adapter.expand_target_column_types(from_relation=staging_table,
47
+ to_relation=target_relation) %}
48
+
49
+ {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}
50
+ {% if unique_key | is_list %}
51
+ {% for key in strategy.unique_key %}
52
+ {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}
53
+ {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}
54
+ {% endfor %}
55
+ {% endif %}
56
+
57
+ {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)
58
+ | rejectattr('name', 'in', remove_columns)
59
+ | list %}
60
+
61
+ {% do create_columns(target_relation, missing_columns) %}
62
+
63
+ {% set source_columns = adapter.get_columns_in_relation(staging_table)
64
+ | rejectattr('name', 'in', remove_columns)
65
+ | list %}
66
+
67
+ {% set quoted_source_columns = [] %}
68
+ {% for column in source_columns %}
69
+ {% do quoted_source_columns.append(adapter.quote(column.name)) %}
70
+ {% endfor %}
71
+
72
+ {% set final_sql = snapshot_merge_sql(
73
+ target = target_relation,
74
+ source = staging_table,
75
+ insert_cols = quoted_source_columns
76
+ )
77
+ %}
78
+
79
+ {% endif %}
80
+
81
+
82
+ {{ check_time_data_types(build_or_select_sql) }}
83
+
84
+ {% call statement('main') %}
85
+ {{ final_sql }}
86
+ {% endcall %}
87
+
88
+ {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}
89
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
90
+
91
+ {% do persist_docs(target_relation, model) %}
92
+
93
+ {% if not target_relation_exists %}
94
+ {% do create_indexes(target_relation) %}
95
+ {% endif %}
96
+
97
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
98
+
99
+ {{ adapter.commit() }}
100
+
101
+ {% if staging_table is defined %}
102
+ {% do post_snapshot(staging_table) %}
103
+ {% endif %}
104
+
105
+ {{ run_hooks(post_hooks, inside_transaction=False) }}
106
+
107
+ {{ return({'relations': [target_relation]}) }}
108
+
109
+ {% endmaterialization %}
@@ -0,0 +1,34 @@
1
+
2
+ {% macro snapshot_merge_sql(target, source, insert_cols) -%}
3
+ {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}
4
+ {%- endmacro %}
5
+
6
+
7
+ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%}
8
+ {%- set insert_cols_csv = insert_cols | join(', ') -%}
9
+
10
+ {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%}
11
+
12
+ merge into {{ target.render() }} as DBT_INTERNAL_DEST
13
+ using {{ source }} as DBT_INTERNAL_SOURCE
14
+ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}
15
+
16
+ when matched
17
+ {% if config.get("dbt_valid_to_current") %}
18
+ {% set source_unique_key = ("DBT_INTERNAL_DEST." ~ columns.dbt_valid_to) | trim %}
19
+ {% set target_unique_key = config.get('dbt_valid_to_current') | trim %}
20
+ and ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null)
21
+
22
+ {% else %}
23
+ and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
24
+ {% endif %}
25
+ and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
26
+ then update
27
+ set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}
28
+
29
+ when not matched
30
+ and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'
31
+ then insert ({{ insert_cols_csv }})
32
+ values ({{ insert_cols_csv }})
33
+
34
+ {% endmacro %}
@@ -0,0 +1,184 @@
1
+ {#
2
+ Dispatch strategies by name, optionally qualified to a package
3
+ #}
4
+ {% macro strategy_dispatch(name) -%}
5
+ {% set original_name = name %}
6
+ {% if '.' in name %}
7
+ {% set package_name, name = name.split(".", 1) %}
8
+ {% else %}
9
+ {% set package_name = none %}
10
+ {% endif %}
11
+
12
+ {% if package_name is none %}
13
+ {% set package_context = context %}
14
+ {% elif package_name in context %}
15
+ {% set package_context = context[package_name] %}
16
+ {% else %}
17
+ {% set error_msg %}
18
+ Could not find package '{{package_name}}', called with '{{original_name}}'
19
+ {% endset %}
20
+ {{ exceptions.raise_compiler_error(error_msg | trim) }}
21
+ {% endif %}
22
+
23
+ {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}
24
+
25
+ {% if search_name not in package_context %}
26
+ {% set error_msg %}
27
+ The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'
28
+ {% endset %}
29
+ {{ exceptions.raise_compiler_error(error_msg | trim) }}
30
+ {% endif %}
31
+ {{ return(package_context[search_name]) }}
32
+ {%- endmacro %}
33
+
34
+
35
+ {#
36
+ Create SCD Hash SQL fields cross-db
37
+ #}
38
+ {% macro snapshot_hash_arguments(args) -%}
39
+ {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}
40
+ {%- endmacro %}
41
+
42
+ {% macro default__snapshot_hash_arguments(args) -%}
43
+ md5({%- for arg in args -%}
44
+ coalesce(cast({{ arg }} as varchar ), '')
45
+ {% if not loop.last %} || '|' || {% endif %}
46
+ {%- endfor -%})
47
+ {%- endmacro %}
48
+
49
+ {#
50
+ Core strategy definitions
51
+ #}
52
+
53
+ {% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}
54
+ {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}
55
+ {% set primary_key = config.get('unique_key') %}
56
+ {% set updated_at = config.get('updated_at') %}
57
+ {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}
58
+ {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}
59
+ {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %}
60
+
61
+ {#/*
62
+ The snapshot relation might not have an {{ updated_at }} value if the
63
+ snapshot strategy is changed from `check` to `timestamp`. We
64
+ should use a dbt-created column for the comparison in the snapshot
65
+ table instead of assuming that the user-supplied {{ updated_at }}
66
+ will be present in the historical data.
67
+
68
+ See https://github.com/dbt-labs/dbt-core/issues/2350
69
+ */ #}
70
+ {% set row_changed_expr -%}
71
+ ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})
72
+ {%- endset %}
73
+
74
+ {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}
75
+ {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}
76
+
77
+ {% do return({
78
+ "unique_key": primary_key,
79
+ "updated_at": updated_at,
80
+ "row_changed": row_changed_expr,
81
+ "scd_id": scd_id_expr,
82
+ "invalidate_hard_deletes": invalidate_hard_deletes,
83
+ "hard_deletes": hard_deletes
84
+ }) %}
85
+ {% endmacro %}
86
+
87
+
88
+ {% macro snapshot_string_as_time(timestamp) -%}
89
+ {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}
90
+ {%- endmacro %}
91
+
92
+ {% macro default__snapshot_string_as_time(timestamp) %}
93
+ {% do exceptions.raise_not_implemented(
94
+ 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()
95
+ ) %}
96
+ {% endmacro %}
97
+
98
+
99
+ {% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}
100
+ {%- if not target_exists -%}
101
+ {#-- no table yet -> return whatever the query does --#}
102
+ {{ return((false, query_columns)) }}
103
+ {%- endif -%}
104
+
105
+ {#-- handle any schema changes --#}
106
+ {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}
107
+
108
+ {% if check_cols_config == 'all' %}
109
+ {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}
110
+
111
+ {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}
112
+ {#-- query for proper casing/quoting, to support comparison below --#}
113
+ {%- set select_check_cols_from_target -%}
114
+ {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}
115
+ {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}
116
+ select {{ check_cols_config | join(', ') }} from (
117
+ {{ node['compiled_code'] }}
118
+ ) subq
119
+ {%- endset -%}
120
+ {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}
121
+
122
+ {% else %}
123
+ {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %}
124
+ {% endif %}
125
+
126
+ {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}
127
+ {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}
128
+ {%- set ns.column_added = false -%}
129
+
130
+ {%- set intersection = [] -%}
131
+ {%- for col in query_columns -%}
132
+ {%- if col in existing_cols -%}
133
+ {%- do intersection.append(adapter.quote(col)) -%}
134
+ {%- else -%}
135
+ {% set ns.column_added = true %}
136
+ {%- endif -%}
137
+ {%- endfor -%}
138
+ {{ return((ns.column_added, intersection)) }}
139
+ {%- endmacro %}
140
+
141
+
142
+ {% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}
143
+ {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}
144
+ {% set check_cols_config = config.get('check_cols') %}
145
+ {% set primary_key = config.get('unique_key') %}
146
+ {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}
147
+ {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}
148
+ {% set updated_at = config.get('updated_at') or snapshot_get_time() %}
149
+
150
+ {% set column_added = false %}
151
+
152
+ {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}
153
+
154
+ {%- set row_changed_expr -%}
155
+ (
156
+ {%- if column_added -%}
157
+ {{ get_true_sql() }}
158
+ {%- else -%}
159
+ {%- for col in check_cols -%}
160
+ {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}
161
+ or
162
+ (
163
+ (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))
164
+ or
165
+ ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))
166
+ )
167
+ {%- if not loop.last %} or {% endif -%}
168
+ {%- endfor -%}
169
+ {%- endif -%}
170
+ )
171
+ {%- endset %}
172
+
173
+ {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}
174
+ {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}
175
+
176
+ {% do return({
177
+ "unique_key": primary_key,
178
+ "updated_at": updated_at,
179
+ "row_changed": row_changed_expr,
180
+ "scd_id": scd_id_expr,
181
+ "invalidate_hard_deletes": invalidate_hard_deletes,
182
+ "hard_deletes": hard_deletes
183
+ }) %}
184
+ {% endmacro %}