dbt-adapters 1.22.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt/adapters/__about__.py +1 -0
- dbt/adapters/__init__.py +8 -0
- dbt/adapters/base/README.md +13 -0
- dbt/adapters/base/__init__.py +16 -0
- dbt/adapters/base/column.py +173 -0
- dbt/adapters/base/connections.py +429 -0
- dbt/adapters/base/impl.py +2036 -0
- dbt/adapters/base/meta.py +150 -0
- dbt/adapters/base/plugin.py +32 -0
- dbt/adapters/base/query_headers.py +106 -0
- dbt/adapters/base/relation.py +648 -0
- dbt/adapters/cache.py +521 -0
- dbt/adapters/capability.py +63 -0
- dbt/adapters/catalogs/__init__.py +14 -0
- dbt/adapters/catalogs/_client.py +54 -0
- dbt/adapters/catalogs/_constants.py +1 -0
- dbt/adapters/catalogs/_exceptions.py +39 -0
- dbt/adapters/catalogs/_integration.py +113 -0
- dbt/adapters/clients/__init__.py +0 -0
- dbt/adapters/clients/jinja.py +24 -0
- dbt/adapters/contracts/__init__.py +0 -0
- dbt/adapters/contracts/connection.py +229 -0
- dbt/adapters/contracts/macros.py +11 -0
- dbt/adapters/contracts/relation.py +160 -0
- dbt/adapters/events/README.md +51 -0
- dbt/adapters/events/__init__.py +0 -0
- dbt/adapters/events/adapter_types_pb2.py +2 -0
- dbt/adapters/events/base_types.py +36 -0
- dbt/adapters/events/logging.py +83 -0
- dbt/adapters/events/types.py +436 -0
- dbt/adapters/exceptions/__init__.py +40 -0
- dbt/adapters/exceptions/alias.py +24 -0
- dbt/adapters/exceptions/cache.py +68 -0
- dbt/adapters/exceptions/compilation.py +269 -0
- dbt/adapters/exceptions/connection.py +16 -0
- dbt/adapters/exceptions/database.py +51 -0
- dbt/adapters/factory.py +264 -0
- dbt/adapters/protocol.py +150 -0
- dbt/adapters/py.typed +0 -0
- dbt/adapters/record/__init__.py +2 -0
- dbt/adapters/record/base.py +291 -0
- dbt/adapters/record/cursor/cursor.py +69 -0
- dbt/adapters/record/cursor/description.py +37 -0
- dbt/adapters/record/cursor/execute.py +39 -0
- dbt/adapters/record/cursor/fetchall.py +69 -0
- dbt/adapters/record/cursor/fetchmany.py +23 -0
- dbt/adapters/record/cursor/fetchone.py +23 -0
- dbt/adapters/record/cursor/rowcount.py +23 -0
- dbt/adapters/record/handle.py +55 -0
- dbt/adapters/record/serialization.py +115 -0
- dbt/adapters/reference_keys.py +39 -0
- dbt/adapters/relation_configs/README.md +25 -0
- dbt/adapters/relation_configs/__init__.py +12 -0
- dbt/adapters/relation_configs/config_base.py +46 -0
- dbt/adapters/relation_configs/config_change.py +26 -0
- dbt/adapters/relation_configs/config_validation.py +57 -0
- dbt/adapters/sql/__init__.py +2 -0
- dbt/adapters/sql/connections.py +263 -0
- dbt/adapters/sql/impl.py +286 -0
- dbt/adapters/utils.py +69 -0
- dbt/include/__init__.py +3 -0
- dbt/include/global_project/__init__.py +4 -0
- dbt/include/global_project/dbt_project.yml +7 -0
- dbt/include/global_project/docs/overview.md +43 -0
- dbt/include/global_project/macros/adapters/apply_grants.sql +167 -0
- dbt/include/global_project/macros/adapters/columns.sql +144 -0
- dbt/include/global_project/macros/adapters/freshness.sql +32 -0
- dbt/include/global_project/macros/adapters/indexes.sql +41 -0
- dbt/include/global_project/macros/adapters/metadata.sql +105 -0
- dbt/include/global_project/macros/adapters/persist_docs.sql +33 -0
- dbt/include/global_project/macros/adapters/relation.sql +84 -0
- dbt/include/global_project/macros/adapters/schema.sql +20 -0
- dbt/include/global_project/macros/adapters/show.sql +26 -0
- dbt/include/global_project/macros/adapters/timestamps.sql +52 -0
- dbt/include/global_project/macros/adapters/validate_sql.sql +10 -0
- dbt/include/global_project/macros/etc/datetime.sql +62 -0
- dbt/include/global_project/macros/etc/statement.sql +52 -0
- dbt/include/global_project/macros/generic_test_sql/accepted_values.sql +27 -0
- dbt/include/global_project/macros/generic_test_sql/not_null.sql +9 -0
- dbt/include/global_project/macros/generic_test_sql/relationships.sql +23 -0
- dbt/include/global_project/macros/generic_test_sql/unique.sql +12 -0
- dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql +36 -0
- dbt/include/global_project/macros/get_custom_name/get_custom_database.sql +32 -0
- dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql +60 -0
- dbt/include/global_project/macros/materializations/configs.sql +21 -0
- dbt/include/global_project/macros/materializations/functions/aggregate.sql +65 -0
- dbt/include/global_project/macros/materializations/functions/function.sql +20 -0
- dbt/include/global_project/macros/materializations/functions/helpers.sql +20 -0
- dbt/include/global_project/macros/materializations/functions/scalar.sql +69 -0
- dbt/include/global_project/macros/materializations/hooks.sql +35 -0
- dbt/include/global_project/macros/materializations/models/clone/can_clone_table.sql +7 -0
- dbt/include/global_project/macros/materializations/models/clone/clone.sql +67 -0
- dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql +7 -0
- dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql +80 -0
- dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +99 -0
- dbt/include/global_project/macros/materializations/models/incremental/is_incremental.sql +13 -0
- dbt/include/global_project/macros/materializations/models/incremental/merge.sql +120 -0
- dbt/include/global_project/macros/materializations/models/incremental/on_schema_change.sql +159 -0
- dbt/include/global_project/macros/materializations/models/incremental/strategies.sql +92 -0
- dbt/include/global_project/macros/materializations/models/materialized_view.sql +121 -0
- dbt/include/global_project/macros/materializations/models/table.sql +64 -0
- dbt/include/global_project/macros/materializations/models/view.sql +72 -0
- dbt/include/global_project/macros/materializations/seeds/helpers.sql +128 -0
- dbt/include/global_project/macros/materializations/seeds/seed.sql +60 -0
- dbt/include/global_project/macros/materializations/snapshots/helpers.sql +345 -0
- dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +109 -0
- dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +34 -0
- dbt/include/global_project/macros/materializations/snapshots/strategies.sql +184 -0
- dbt/include/global_project/macros/materializations/tests/helpers.sql +44 -0
- dbt/include/global_project/macros/materializations/tests/test.sql +66 -0
- dbt/include/global_project/macros/materializations/tests/unit.sql +40 -0
- dbt/include/global_project/macros/materializations/tests/where_subquery.sql +15 -0
- dbt/include/global_project/macros/python_model/python.sql +114 -0
- dbt/include/global_project/macros/relations/column/columns_spec_ddl.sql +89 -0
- dbt/include/global_project/macros/relations/create.sql +23 -0
- dbt/include/global_project/macros/relations/create_backup.sql +17 -0
- dbt/include/global_project/macros/relations/create_intermediate.sql +17 -0
- dbt/include/global_project/macros/relations/drop.sql +41 -0
- dbt/include/global_project/macros/relations/drop_backup.sql +14 -0
- dbt/include/global_project/macros/relations/materialized_view/alter.sql +55 -0
- dbt/include/global_project/macros/relations/materialized_view/create.sql +10 -0
- dbt/include/global_project/macros/relations/materialized_view/drop.sql +14 -0
- dbt/include/global_project/macros/relations/materialized_view/refresh.sql +9 -0
- dbt/include/global_project/macros/relations/materialized_view/rename.sql +10 -0
- dbt/include/global_project/macros/relations/materialized_view/replace.sql +10 -0
- dbt/include/global_project/macros/relations/rename.sql +35 -0
- dbt/include/global_project/macros/relations/rename_intermediate.sql +14 -0
- dbt/include/global_project/macros/relations/replace.sql +50 -0
- dbt/include/global_project/macros/relations/schema.sql +8 -0
- dbt/include/global_project/macros/relations/table/create.sql +60 -0
- dbt/include/global_project/macros/relations/table/drop.sql +14 -0
- dbt/include/global_project/macros/relations/table/rename.sql +10 -0
- dbt/include/global_project/macros/relations/table/replace.sql +10 -0
- dbt/include/global_project/macros/relations/view/create.sql +27 -0
- dbt/include/global_project/macros/relations/view/drop.sql +14 -0
- dbt/include/global_project/macros/relations/view/rename.sql +10 -0
- dbt/include/global_project/macros/relations/view/replace.sql +66 -0
- dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +107 -0
- dbt/include/global_project/macros/utils/any_value.sql +9 -0
- dbt/include/global_project/macros/utils/array_append.sql +8 -0
- dbt/include/global_project/macros/utils/array_concat.sql +7 -0
- dbt/include/global_project/macros/utils/array_construct.sql +12 -0
- dbt/include/global_project/macros/utils/bool_or.sql +9 -0
- dbt/include/global_project/macros/utils/cast.sql +7 -0
- dbt/include/global_project/macros/utils/cast_bool_to_text.sql +7 -0
- dbt/include/global_project/macros/utils/concat.sql +7 -0
- dbt/include/global_project/macros/utils/data_types.sql +129 -0
- dbt/include/global_project/macros/utils/date.sql +10 -0
- dbt/include/global_project/macros/utils/date_spine.sql +75 -0
- dbt/include/global_project/macros/utils/date_trunc.sql +7 -0
- dbt/include/global_project/macros/utils/dateadd.sql +14 -0
- dbt/include/global_project/macros/utils/datediff.sql +14 -0
- dbt/include/global_project/macros/utils/equals.sql +14 -0
- dbt/include/global_project/macros/utils/escape_single_quotes.sql +8 -0
- dbt/include/global_project/macros/utils/except.sql +9 -0
- dbt/include/global_project/macros/utils/generate_series.sql +53 -0
- dbt/include/global_project/macros/utils/hash.sql +7 -0
- dbt/include/global_project/macros/utils/intersect.sql +9 -0
- dbt/include/global_project/macros/utils/last_day.sql +15 -0
- dbt/include/global_project/macros/utils/length.sql +11 -0
- dbt/include/global_project/macros/utils/listagg.sql +30 -0
- dbt/include/global_project/macros/utils/literal.sql +7 -0
- dbt/include/global_project/macros/utils/position.sql +11 -0
- dbt/include/global_project/macros/utils/replace.sql +14 -0
- dbt/include/global_project/macros/utils/right.sql +12 -0
- dbt/include/global_project/macros/utils/safe_cast.sql +9 -0
- dbt/include/global_project/macros/utils/split_part.sql +26 -0
- dbt/include/global_project/tests/generic/builtin.sql +30 -0
- dbt/include/py.typed +0 -0
- dbt_adapters-1.22.2.dist-info/METADATA +124 -0
- dbt_adapters-1.22.2.dist-info/RECORD +173 -0
- dbt_adapters-1.22.2.dist-info/WHEEL +4 -0
- dbt_adapters-1.22.2.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
{%- materialization clone, default -%}
|
|
2
|
+
|
|
3
|
+
{%- set relations = {'relations': []} -%}
|
|
4
|
+
|
|
5
|
+
{%- if not defer_relation -%}
|
|
6
|
+
-- nothing to do
|
|
7
|
+
{{ log("No relation found in state manifest for " ~ model.unique_id, info=True) }}
|
|
8
|
+
{{ return(relations) }}
|
|
9
|
+
{%- endif -%}
|
|
10
|
+
|
|
11
|
+
{%- set existing_relation = load_cached_relation(this) -%}
|
|
12
|
+
|
|
13
|
+
{%- if existing_relation and not flags.FULL_REFRESH -%}
|
|
14
|
+
-- noop!
|
|
15
|
+
{{ log("Relation " ~ existing_relation ~ " already exists", info=True) }}
|
|
16
|
+
{{ return(relations) }}
|
|
17
|
+
{%- endif -%}
|
|
18
|
+
|
|
19
|
+
{%- set other_existing_relation = load_cached_relation(defer_relation) -%}
|
|
20
|
+
|
|
21
|
+
-- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table
|
|
22
|
+
-- Otherwise, this will be a view
|
|
23
|
+
|
|
24
|
+
{% set can_clone_table = can_clone_table() %}
|
|
25
|
+
|
|
26
|
+
{%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}
|
|
27
|
+
|
|
28
|
+
{%- set target_relation = this.incorporate(type='table') -%}
|
|
29
|
+
{% if existing_relation is not none and not existing_relation.is_table %}
|
|
30
|
+
{{ log("Dropping relation " ~ existing_relation.render() ~ " because it is of type " ~ existing_relation.type) }}
|
|
31
|
+
{{ drop_relation_if_exists(existing_relation) }}
|
|
32
|
+
{% endif %}
|
|
33
|
+
|
|
34
|
+
-- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'
|
|
35
|
+
{% if target_relation.database == defer_relation.database and
|
|
36
|
+
target_relation.schema == defer_relation.schema and
|
|
37
|
+
target_relation.identifier == defer_relation.identifier %}
|
|
38
|
+
{{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation.render()) }}
|
|
39
|
+
{% else %}
|
|
40
|
+
{% call statement('main') %}
|
|
41
|
+
{{ create_or_replace_clone(target_relation, defer_relation) }}
|
|
42
|
+
{% endcall %}
|
|
43
|
+
{% endif %}
|
|
44
|
+
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
|
|
45
|
+
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
|
46
|
+
{% do persist_docs(target_relation, model) %}
|
|
47
|
+
|
|
48
|
+
{{ return({'relations': [target_relation]}) }}
|
|
49
|
+
|
|
50
|
+
{%- else -%}
|
|
51
|
+
|
|
52
|
+
{%- set target_relation = this.incorporate(type='view') -%}
|
|
53
|
+
|
|
54
|
+
-- reuse the view materialization
|
|
55
|
+
-- TODO: support actual dispatch for materialization macros
|
|
56
|
+
-- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799
|
|
57
|
+
{% set search_name = "materialization_view_" ~ adapter.type() %}
|
|
58
|
+
{% if not search_name in context %}
|
|
59
|
+
{% set search_name = "materialization_view_default" %}
|
|
60
|
+
{% endif %}
|
|
61
|
+
{% set materialization_macro = context[search_name] %}
|
|
62
|
+
{% set relations = materialization_macro() %}
|
|
63
|
+
{{ return(relations) }}
|
|
64
|
+
|
|
65
|
+
{%- endif -%}
|
|
66
|
+
|
|
67
|
+
{%- endmaterialization -%}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{% macro create_or_replace_clone(this_relation, defer_relation) %}
|
|
2
|
+
{{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}
|
|
3
|
+
{% endmacro %}
|
|
4
|
+
|
|
5
|
+
{% macro default__create_or_replace_clone(this_relation, defer_relation) %}
|
|
6
|
+
create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}
|
|
7
|
+
{% endmacro %}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/* {#
|
|
2
|
+
Helper macros for internal use with incremental materializations.
|
|
3
|
+
Use with care if calling elsewhere.
|
|
4
|
+
#} */
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
{% macro get_quoted_csv(column_names) %}
|
|
8
|
+
|
|
9
|
+
{% set quoted = [] %}
|
|
10
|
+
{% for col in column_names -%}
|
|
11
|
+
{%- do quoted.append(adapter.quote(col)) -%}
|
|
12
|
+
{%- endfor %}
|
|
13
|
+
|
|
14
|
+
{%- set dest_cols_csv = quoted | join(', ') -%}
|
|
15
|
+
{{ return(dest_cols_csv) }}
|
|
16
|
+
|
|
17
|
+
{% endmacro %}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
{% macro diff_columns(source_columns, target_columns) %}
|
|
21
|
+
|
|
22
|
+
{% set result = [] %}
|
|
23
|
+
{% set source_names = source_columns | map(attribute = 'column') | list %}
|
|
24
|
+
{% set target_names = target_columns | map(attribute = 'column') | list %}
|
|
25
|
+
|
|
26
|
+
{# --check whether the name attribute exists in the target - this does not perform a data type check #}
|
|
27
|
+
{% for sc in source_columns %}
|
|
28
|
+
{% if sc.name not in target_names %}
|
|
29
|
+
{{ result.append(sc) }}
|
|
30
|
+
{% endif %}
|
|
31
|
+
{% endfor %}
|
|
32
|
+
|
|
33
|
+
{{ return(result) }}
|
|
34
|
+
|
|
35
|
+
{% endmacro %}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
{% macro diff_column_data_types(source_columns, target_columns) %}
|
|
39
|
+
|
|
40
|
+
{% set result = [] %}
|
|
41
|
+
{% for sc in source_columns %}
|
|
42
|
+
{% set tc = target_columns | selectattr("name", "equalto", sc.name) | list | first %}
|
|
43
|
+
{% if tc %}
|
|
44
|
+
{% if sc.expanded_data_type != tc.expanded_data_type and not sc.can_expand_to(other_column=tc) %}
|
|
45
|
+
{{ result.append( { 'column_name': tc.name, 'new_type': sc.expanded_data_type } ) }}
|
|
46
|
+
{% endif %}
|
|
47
|
+
{% endif %}
|
|
48
|
+
{% endfor %}
|
|
49
|
+
|
|
50
|
+
{{ return(result) }}
|
|
51
|
+
|
|
52
|
+
{% endmacro %}
|
|
53
|
+
|
|
54
|
+
{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}
|
|
55
|
+
{{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}
|
|
56
|
+
{% endmacro %}
|
|
57
|
+
|
|
58
|
+
{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}
|
|
59
|
+
{%- set default_cols = dest_columns | map(attribute="quoted") | list -%}
|
|
60
|
+
|
|
61
|
+
{%- if merge_update_columns and merge_exclude_columns -%}
|
|
62
|
+
{{ exceptions.raise_compiler_error(
|
|
63
|
+
'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'
|
|
64
|
+
)}}
|
|
65
|
+
{%- elif merge_update_columns -%}
|
|
66
|
+
{%- set update_columns = merge_update_columns -%}
|
|
67
|
+
{%- elif merge_exclude_columns -%}
|
|
68
|
+
{%- set update_columns = [] -%}
|
|
69
|
+
{%- for column in dest_columns -%}
|
|
70
|
+
{% if column.column | lower not in merge_exclude_columns | map("lower") | list %}
|
|
71
|
+
{%- do update_columns.append(column.quoted) -%}
|
|
72
|
+
{% endif %}
|
|
73
|
+
{%- endfor -%}
|
|
74
|
+
{%- else -%}
|
|
75
|
+
{%- set update_columns = default_cols -%}
|
|
76
|
+
{%- endif -%}
|
|
77
|
+
|
|
78
|
+
{{ return(update_columns) }}
|
|
79
|
+
|
|
80
|
+
{% endmacro %}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
|
|
2
|
+
{% materialization incremental, default -%}
|
|
3
|
+
|
|
4
|
+
-- relations
|
|
5
|
+
{%- set existing_relation = load_cached_relation(this) -%}
|
|
6
|
+
{%- set target_relation = this.incorporate(type='table') -%}
|
|
7
|
+
{%- set temp_relation = make_temp_relation(target_relation)-%}
|
|
8
|
+
{%- set intermediate_relation = make_intermediate_relation(target_relation)-%}
|
|
9
|
+
{%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}
|
|
10
|
+
{%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
|
|
11
|
+
|
|
12
|
+
-- configs
|
|
13
|
+
{%- set unique_key = config.get('unique_key') -%}
|
|
14
|
+
{%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}
|
|
15
|
+
{%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
|
|
16
|
+
|
|
17
|
+
-- the temp_ and backup_ relations should not already exist in the database; get_relation
|
|
18
|
+
-- will return None in that case. Otherwise, we get a relation that we can drop
|
|
19
|
+
-- later, before we try to use this name for the current operation. This has to happen before
|
|
20
|
+
-- BEGIN, in a separate transaction
|
|
21
|
+
{%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
|
|
22
|
+
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
|
|
23
|
+
-- grab current tables grants config for comparision later on
|
|
24
|
+
{% set grant_config = config.get('grants') %}
|
|
25
|
+
{{ drop_relation_if_exists(preexisting_intermediate_relation) }}
|
|
26
|
+
{{ drop_relation_if_exists(preexisting_backup_relation) }}
|
|
27
|
+
|
|
28
|
+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
|
|
29
|
+
|
|
30
|
+
-- `BEGIN` happens here:
|
|
31
|
+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
|
|
32
|
+
|
|
33
|
+
{% set to_drop = [] %}
|
|
34
|
+
|
|
35
|
+
{% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
|
|
36
|
+
{% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
|
|
37
|
+
|
|
38
|
+
{% if existing_relation is none %}
|
|
39
|
+
{% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}
|
|
40
|
+
{% set relation_for_indexes = target_relation %}
|
|
41
|
+
{% elif full_refresh_mode %}
|
|
42
|
+
{% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}
|
|
43
|
+
{% set relation_for_indexes = intermediate_relation %}
|
|
44
|
+
{% set need_swap = true %}
|
|
45
|
+
{% else %}
|
|
46
|
+
{% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}
|
|
47
|
+
{% set relation_for_indexes = temp_relation %}
|
|
48
|
+
{% set contract_config = config.get('contract') %}
|
|
49
|
+
{% if not contract_config or not contract_config.enforced %}
|
|
50
|
+
{% do adapter.expand_target_column_types(
|
|
51
|
+
from_relation=temp_relation,
|
|
52
|
+
to_relation=target_relation) %}
|
|
53
|
+
{% endif %}
|
|
54
|
+
{#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}
|
|
55
|
+
{% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}
|
|
56
|
+
{% if not dest_columns %}
|
|
57
|
+
{% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
|
|
58
|
+
{% endif %}
|
|
59
|
+
|
|
60
|
+
{#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
|
|
61
|
+
{% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}
|
|
62
|
+
{% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
|
|
63
|
+
{% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
|
|
64
|
+
|
|
65
|
+
{% endif %}
|
|
66
|
+
|
|
67
|
+
{% call statement("main") %}
|
|
68
|
+
{{ build_sql }}
|
|
69
|
+
{% endcall %}
|
|
70
|
+
|
|
71
|
+
{% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
|
|
72
|
+
{% do create_indexes(relation_for_indexes) %}
|
|
73
|
+
{% endif %}
|
|
74
|
+
|
|
75
|
+
{% if need_swap %}
|
|
76
|
+
{% do adapter.rename_relation(target_relation, backup_relation) %}
|
|
77
|
+
{% do adapter.rename_relation(intermediate_relation, target_relation) %}
|
|
78
|
+
{% do to_drop.append(backup_relation) %}
|
|
79
|
+
{% endif %}
|
|
80
|
+
|
|
81
|
+
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
|
|
82
|
+
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
|
83
|
+
|
|
84
|
+
{% do persist_docs(target_relation, model) %}
|
|
85
|
+
|
|
86
|
+
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
|
87
|
+
|
|
88
|
+
-- `COMMIT` happens here
|
|
89
|
+
{% do adapter.commit() %}
|
|
90
|
+
|
|
91
|
+
{% for rel in to_drop %}
|
|
92
|
+
{% do adapter.drop_relation(rel) %}
|
|
93
|
+
{% endfor %}
|
|
94
|
+
|
|
95
|
+
{{ run_hooks(post_hooks, inside_transaction=False) }}
|
|
96
|
+
|
|
97
|
+
{{ return({'relations': [target_relation]}) }}
|
|
98
|
+
|
|
99
|
+
{%- endmaterialization %}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
|
|
2
|
+
{% macro is_incremental() %}
|
|
3
|
+
{#-- do not run introspective queries in parsing #}
|
|
4
|
+
{% if not execute %}
|
|
5
|
+
{{ return(False) }}
|
|
6
|
+
{% else %}
|
|
7
|
+
{% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
|
|
8
|
+
{{ return(relation is not none
|
|
9
|
+
and relation.type == 'table'
|
|
10
|
+
and model.config.materialized == 'incremental'
|
|
11
|
+
and not should_full_refresh()) }}
|
|
12
|
+
{% endif %}
|
|
13
|
+
{% endmacro %}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}
|
|
2
|
+
-- back compat for old kwarg name
|
|
3
|
+
{% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}
|
|
4
|
+
{{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}
|
|
5
|
+
{%- endmacro %}
|
|
6
|
+
|
|
7
|
+
{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}
|
|
8
|
+
{%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}
|
|
9
|
+
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
|
|
10
|
+
{%- set merge_update_columns = config.get('merge_update_columns') -%}
|
|
11
|
+
{%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}
|
|
12
|
+
{%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}
|
|
13
|
+
{%- set sql_header = config.get('sql_header', none) -%}
|
|
14
|
+
|
|
15
|
+
{% if unique_key %}
|
|
16
|
+
{% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}
|
|
17
|
+
{% for key in unique_key %}
|
|
18
|
+
{% set this_key_match %}
|
|
19
|
+
DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}
|
|
20
|
+
{% endset %}
|
|
21
|
+
{% do predicates.append(this_key_match) %}
|
|
22
|
+
{% endfor %}
|
|
23
|
+
{% else %}
|
|
24
|
+
{% set source_unique_key = ("DBT_INTERNAL_SOURCE." ~ unique_key) | trim %}
|
|
25
|
+
{% set target_unique_key = ("DBT_INTERNAL_DEST." ~ unique_key) | trim %}
|
|
26
|
+
{% set unique_key_match = equals(source_unique_key, target_unique_key) | trim %}
|
|
27
|
+
{% do predicates.append(unique_key_match) %}
|
|
28
|
+
{% endif %}
|
|
29
|
+
{% else %}
|
|
30
|
+
{% do predicates.append('FALSE') %}
|
|
31
|
+
{% endif %}
|
|
32
|
+
|
|
33
|
+
{{ sql_header if sql_header is not none }}
|
|
34
|
+
|
|
35
|
+
merge into {{ target }} as DBT_INTERNAL_DEST
|
|
36
|
+
using {{ source }} as DBT_INTERNAL_SOURCE
|
|
37
|
+
on {{"(" ~ predicates | join(") and (") ~ ")"}}
|
|
38
|
+
|
|
39
|
+
{% if unique_key %}
|
|
40
|
+
when matched then update set
|
|
41
|
+
{% for column_name in update_columns -%}
|
|
42
|
+
{{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}
|
|
43
|
+
{%- if not loop.last %}, {%- endif %}
|
|
44
|
+
{%- endfor %}
|
|
45
|
+
{% endif %}
|
|
46
|
+
|
|
47
|
+
when not matched then insert
|
|
48
|
+
({{ dest_cols_csv }})
|
|
49
|
+
values
|
|
50
|
+
({{ dest_cols_csv }})
|
|
51
|
+
|
|
52
|
+
{% endmacro %}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
|
|
56
|
+
{{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}
|
|
57
|
+
{%- endmacro %}
|
|
58
|
+
|
|
59
|
+
{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
|
|
60
|
+
|
|
61
|
+
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
|
|
62
|
+
|
|
63
|
+
{% if unique_key %}
|
|
64
|
+
{% if unique_key is string %}
|
|
65
|
+
{% set unique_key = [unique_key] %}
|
|
66
|
+
{% endif %}
|
|
67
|
+
|
|
68
|
+
{%- set unique_key_str = unique_key|join(', ') -%}
|
|
69
|
+
|
|
70
|
+
delete from {{ target }} as DBT_INTERNAL_DEST
|
|
71
|
+
where ({{ unique_key_str }}) in (
|
|
72
|
+
select distinct {{ unique_key_str }}
|
|
73
|
+
from {{ source }} as DBT_INTERNAL_SOURCE
|
|
74
|
+
)
|
|
75
|
+
{%- if incremental_predicates %}
|
|
76
|
+
{% for predicate in incremental_predicates %}
|
|
77
|
+
and {{ predicate }}
|
|
78
|
+
{% endfor %}
|
|
79
|
+
{%- endif -%};
|
|
80
|
+
|
|
81
|
+
{% endif %}
|
|
82
|
+
|
|
83
|
+
insert into {{ target }} ({{ dest_cols_csv }})
|
|
84
|
+
(
|
|
85
|
+
select {{ dest_cols_csv }}
|
|
86
|
+
from {{ source }}
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
{%- endmacro %}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}
|
|
93
|
+
{{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}
|
|
94
|
+
{%- endmacro %}
|
|
95
|
+
|
|
96
|
+
{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}
|
|
97
|
+
{#-- The only time include_sql_header is True: --#}
|
|
98
|
+
{#-- BigQuery + insert_overwrite strategy + "static" partitions config --#}
|
|
99
|
+
{#-- We should consider including the sql header at the materialization level instead --#}
|
|
100
|
+
|
|
101
|
+
{%- set predicates = [] if predicates is none else [] + predicates -%}
|
|
102
|
+
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
|
|
103
|
+
{%- set sql_header = config.get('sql_header', none) -%}
|
|
104
|
+
|
|
105
|
+
{{ sql_header if sql_header is not none and include_sql_header }}
|
|
106
|
+
|
|
107
|
+
merge into {{ target }} as DBT_INTERNAL_DEST
|
|
108
|
+
using {{ source }} as DBT_INTERNAL_SOURCE
|
|
109
|
+
on FALSE
|
|
110
|
+
|
|
111
|
+
when not matched by source
|
|
112
|
+
{% if predicates %} and {{ predicates | join(' and ') }} {% endif %}
|
|
113
|
+
then delete
|
|
114
|
+
|
|
115
|
+
when not matched then insert
|
|
116
|
+
({{ dest_cols_csv }})
|
|
117
|
+
values
|
|
118
|
+
({{ dest_cols_csv }})
|
|
119
|
+
|
|
120
|
+
{% endmacro %}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
|
|
2
|
+
|
|
3
|
+
{% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}
|
|
4
|
+
|
|
5
|
+
{% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}
|
|
6
|
+
{% do log(log_message) %}
|
|
7
|
+
|
|
8
|
+
{{ return(default) }}
|
|
9
|
+
|
|
10
|
+
{% else %}
|
|
11
|
+
|
|
12
|
+
{{ return(on_schema_change) }}
|
|
13
|
+
|
|
14
|
+
{% endif %}
|
|
15
|
+
|
|
16
|
+
{% endmacro %}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
{% macro check_for_schema_changes(source_relation, target_relation) %}
|
|
20
|
+
{{ return(adapter.dispatch('check_for_schema_changes', 'dbt')(source_relation, target_relation)) }}
|
|
21
|
+
{% endmacro %}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
{% macro default__check_for_schema_changes(source_relation, target_relation) %}
|
|
25
|
+
|
|
26
|
+
{% set schema_changed = False %}
|
|
27
|
+
|
|
28
|
+
{%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}
|
|
29
|
+
{%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
|
|
30
|
+
{%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
|
|
31
|
+
{%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
|
|
32
|
+
|
|
33
|
+
{% set new_target_types = diff_column_data_types(source_columns, target_columns) %}
|
|
34
|
+
|
|
35
|
+
{% if source_not_in_target != [] %}
|
|
36
|
+
{% set schema_changed = True %}
|
|
37
|
+
{% elif target_not_in_source != [] or new_target_types != [] %}
|
|
38
|
+
{% set schema_changed = True %}
|
|
39
|
+
{% elif new_target_types != [] %}
|
|
40
|
+
{% set schema_changed = True %}
|
|
41
|
+
{% endif %}
|
|
42
|
+
|
|
43
|
+
{% set changes_dict = {
|
|
44
|
+
'schema_changed': schema_changed,
|
|
45
|
+
'source_not_in_target': source_not_in_target,
|
|
46
|
+
'target_not_in_source': target_not_in_source,
|
|
47
|
+
'source_columns': source_columns,
|
|
48
|
+
'target_columns': target_columns,
|
|
49
|
+
'new_target_types': new_target_types
|
|
50
|
+
} %}
|
|
51
|
+
|
|
52
|
+
{% set msg %}
|
|
53
|
+
In {{ target_relation }}:
|
|
54
|
+
Schema changed: {{ schema_changed }}
|
|
55
|
+
Source columns not in target: {{ source_not_in_target }}
|
|
56
|
+
Target columns not in source: {{ target_not_in_source }}
|
|
57
|
+
New column types: {{ new_target_types }}
|
|
58
|
+
{% endset %}
|
|
59
|
+
|
|
60
|
+
{% do log(msg) %}
|
|
61
|
+
|
|
62
|
+
{{ return(changes_dict) }}
|
|
63
|
+
|
|
64
|
+
{% endmacro %}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
|
|
68
|
+
{{ return(adapter.dispatch('sync_column_schemas', 'dbt')(on_schema_change, target_relation, schema_changes_dict)) }}
|
|
69
|
+
{% endmacro %}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
{% macro default__sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
|
|
73
|
+
|
|
74
|
+
{%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}
|
|
75
|
+
{%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}
|
|
76
|
+
{%- set new_target_types = schema_changes_dict['new_target_types'] -%}
|
|
77
|
+
|
|
78
|
+
{%- if on_schema_change == 'append_new_columns'-%}
|
|
79
|
+
{%- if add_to_target_arr | length > 0 -%}
|
|
80
|
+
{%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}
|
|
81
|
+
{%- endif -%}
|
|
82
|
+
|
|
83
|
+
{% elif on_schema_change == 'sync_all_columns' %}
|
|
84
|
+
|
|
85
|
+
{% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}
|
|
86
|
+
{%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}
|
|
87
|
+
{% endif %}
|
|
88
|
+
|
|
89
|
+
{% if new_target_types != [] %}
|
|
90
|
+
{% for ntt in new_target_types %}
|
|
91
|
+
{% set column_name = ntt['column_name'] %}
|
|
92
|
+
{% set new_type = ntt['new_type'] %}
|
|
93
|
+
{% do alter_column_type(target_relation, column_name, new_type) %}
|
|
94
|
+
{% endfor %}
|
|
95
|
+
{% endif %}
|
|
96
|
+
|
|
97
|
+
{% endif %}
|
|
98
|
+
|
|
99
|
+
{% set schema_change_message %}
|
|
100
|
+
In {{ target_relation }}:
|
|
101
|
+
Schema change approach: {{ on_schema_change }}
|
|
102
|
+
Columns added: {{ add_to_target_arr }}
|
|
103
|
+
Columns removed: {{ remove_from_target_arr }}
|
|
104
|
+
Data types changed: {{ new_target_types }}
|
|
105
|
+
{% endset %}
|
|
106
|
+
|
|
107
|
+
{% do log(schema_change_message) %}
|
|
108
|
+
|
|
109
|
+
{% endmacro %}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
|
|
113
|
+
{{ return(adapter.dispatch('process_schema_changes', 'dbt')(on_schema_change, source_relation, target_relation)) }}
|
|
114
|
+
{% endmacro %}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
{% macro default__process_schema_changes(on_schema_change, source_relation, target_relation) %}
|
|
118
|
+
|
|
119
|
+
{% if on_schema_change == 'ignore' %}
|
|
120
|
+
|
|
121
|
+
{{ return({}) }}
|
|
122
|
+
|
|
123
|
+
{% else %}
|
|
124
|
+
|
|
125
|
+
{% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
|
|
126
|
+
|
|
127
|
+
{% if schema_changes_dict['schema_changed'] %}
|
|
128
|
+
|
|
129
|
+
{% if on_schema_change == 'fail' %}
|
|
130
|
+
|
|
131
|
+
{% set fail_msg %}
|
|
132
|
+
The source and target schemas on this incremental model are out of sync!
|
|
133
|
+
They can be reconciled in several ways:
|
|
134
|
+
- set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.
|
|
135
|
+
- Re-run the incremental model with `full_refresh: True` to update the target schema.
|
|
136
|
+
- update the schema manually and re-run the process.
|
|
137
|
+
|
|
138
|
+
Additional troubleshooting context:
|
|
139
|
+
Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}
|
|
140
|
+
Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}
|
|
141
|
+
New column types: {{ schema_changes_dict['new_target_types'] }}
|
|
142
|
+
{% endset %}
|
|
143
|
+
|
|
144
|
+
{% do exceptions.raise_compiler_error(fail_msg) %}
|
|
145
|
+
|
|
146
|
+
{# -- unless we ignore, run the sync operation per the config #}
|
|
147
|
+
{% else %}
|
|
148
|
+
|
|
149
|
+
{% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
|
|
150
|
+
|
|
151
|
+
{% endif %}
|
|
152
|
+
|
|
153
|
+
{% endif %}
|
|
154
|
+
|
|
155
|
+
{{ return(schema_changes_dict['source_columns']) }}
|
|
156
|
+
|
|
157
|
+
{% endif %}
|
|
158
|
+
|
|
159
|
+
{% endmacro %}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
{% macro get_incremental_append_sql(arg_dict) %}
|
|
2
|
+
|
|
3
|
+
{{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}
|
|
4
|
+
|
|
5
|
+
{% endmacro %}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
{% macro default__get_incremental_append_sql(arg_dict) %}
|
|
9
|
+
|
|
10
|
+
{% do return(get_insert_into_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["dest_columns"])) %}
|
|
11
|
+
|
|
12
|
+
{% endmacro %}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
{# snowflake #}
|
|
16
|
+
{% macro get_incremental_delete_insert_sql(arg_dict) %}
|
|
17
|
+
|
|
18
|
+
{{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}
|
|
19
|
+
|
|
20
|
+
{% endmacro %}
|
|
21
|
+
|
|
22
|
+
{% macro default__get_incremental_delete_insert_sql(arg_dict) %}
|
|
23
|
+
|
|
24
|
+
{% do return(get_delete_insert_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["unique_key"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %}
|
|
25
|
+
|
|
26
|
+
{% endmacro %}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
{# snowflake, bigquery, spark #}
|
|
30
|
+
{% macro get_incremental_merge_sql(arg_dict) %}
|
|
31
|
+
|
|
32
|
+
{{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}
|
|
33
|
+
|
|
34
|
+
{% endmacro %}
|
|
35
|
+
|
|
36
|
+
{% macro default__get_incremental_merge_sql(arg_dict) %}
|
|
37
|
+
|
|
38
|
+
{% do return(get_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["unique_key"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %}
|
|
39
|
+
|
|
40
|
+
{% endmacro %}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
{% macro get_incremental_insert_overwrite_sql(arg_dict) %}
|
|
44
|
+
|
|
45
|
+
{{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}
|
|
46
|
+
|
|
47
|
+
{% endmacro %}
|
|
48
|
+
|
|
49
|
+
{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}
|
|
50
|
+
|
|
51
|
+
{% do return(get_insert_overwrite_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %}
|
|
52
|
+
|
|
53
|
+
{% endmacro %}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
{% macro get_incremental_default_sql(arg_dict) %}
|
|
57
|
+
|
|
58
|
+
{{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}
|
|
59
|
+
|
|
60
|
+
{% endmacro %}
|
|
61
|
+
|
|
62
|
+
{% macro default__get_incremental_default_sql(arg_dict) %}
|
|
63
|
+
|
|
64
|
+
{% do return(get_incremental_append_sql(arg_dict)) %}
|
|
65
|
+
|
|
66
|
+
{% endmacro %}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
{% macro get_incremental_microbatch_sql(arg_dict) %}
|
|
70
|
+
|
|
71
|
+
{{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}
|
|
72
|
+
|
|
73
|
+
{% endmacro %}
|
|
74
|
+
|
|
75
|
+
{% macro default__get_incremental_microbatch_sql(arg_dict) %}
|
|
76
|
+
|
|
77
|
+
{{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}
|
|
78
|
+
|
|
79
|
+
{% endmacro %}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}
|
|
83
|
+
|
|
84
|
+
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
|
|
85
|
+
|
|
86
|
+
insert into {{ target_relation }} ({{ dest_cols_csv }})
|
|
87
|
+
(
|
|
88
|
+
select {{ dest_cols_csv }}
|
|
89
|
+
from {{ temp_relation }}
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
{% endmacro %}
|