dbt-firebolt 1.9.3__py3-none-any.whl → 1.9.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt/adapters/firebolt/__init__.py +1 -1
- dbt/include/firebolt/macros/adapters/apply_grants.sql +37 -0
- dbt/include/firebolt/macros/adapters/relation.sql +49 -0
- dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql +141 -0
- dbt/include/firebolt/macros/dbt_external_tables/dropif.sql +6 -0
- dbt/include/firebolt/macros/dbt_external_tables/get_external_build_plan.sql +18 -0
- dbt/include/firebolt/macros/materializations/clone.sql +3 -0
- dbt/include/firebolt/macros/materializations/materialized_view.sql +3 -0
- dbt/include/firebolt/macros/materializations/models/incremental/column_helpers.sql +21 -0
- dbt/include/firebolt/macros/materializations/models/incremental/incremental.sql +131 -0
- dbt/include/firebolt/macros/materializations/models/incremental/is_incremental.sql +14 -0
- dbt/include/firebolt/macros/materializations/models/incremental/merge.sql +38 -0
- dbt/include/firebolt/macros/materializations/models/incremental/on_schema_change.sql +90 -0
- dbt/include/firebolt/macros/materializations/models/incremental/strategies.sql +132 -0
- dbt/include/firebolt/macros/materializations/seed.sql +42 -0
- dbt/include/firebolt/macros/materializations/snapshot_merge.sql +19 -0
- dbt/include/firebolt/macros/materializations/table.sql +40 -0
- dbt/include/firebolt/macros/materializations/test.sql +15 -0
- dbt/include/firebolt/macros/materializations/view.sql +39 -0
- dbt/include/firebolt/macros/relations/materialized_view/alter.sql +11 -0
- dbt/include/firebolt/macros/relations/materialized_view/create.sql +3 -0
- dbt/include/firebolt/macros/relations/materialized_view/describe.sql +3 -0
- dbt/include/firebolt/macros/relations/materialized_view/drop.sql +3 -0
- dbt/include/firebolt/macros/relations/materialized_view/refresh.sql +3 -0
- dbt/include/firebolt/macros/relations/table/create.sql +78 -0
- dbt/include/firebolt/macros/relations/table/drop.sql +3 -0
- dbt/include/firebolt/macros/relations/table/rename.sql +6 -0
- dbt/include/firebolt/macros/relations/table/replace.sql +3 -0
- dbt/include/firebolt/macros/relations/view/create.sql +16 -0
- dbt/include/firebolt/macros/relations/view/drop.sql +3 -0
- dbt/include/firebolt/macros/relations/view/rename.sql +6 -0
- dbt/include/firebolt/macros/relations/view/replace.sql +3 -0
- dbt/include/firebolt/macros/utils/array_append.sql +3 -0
- dbt/include/firebolt/macros/utils/array_concat.sql +3 -0
- dbt/include/firebolt/macros/utils/array_construct.sql +3 -0
- dbt/include/firebolt/macros/utils/bool_or.sql +5 -0
- dbt/include/firebolt/macros/utils/cast_bool_to_text.sql +7 -0
- dbt/include/firebolt/macros/utils/dateadd.sql +9 -0
- dbt/include/firebolt/macros/utils/datediff.sql +9 -0
- dbt/include/firebolt/macros/utils/except.sql +6 -0
- dbt/include/firebolt/macros/utils/intersect.sql +6 -0
- dbt/include/firebolt/macros/utils/listagg.sql +27 -0
- dbt/include/firebolt/macros/utils/position.sql +3 -0
- dbt/include/firebolt/macros/utils/right.sql +12 -0
- dbt/include/firebolt/macros/utils/split_part.sql +14 -0
- dbt/include/firebolt/macros/utils/timestamps.sql +14 -0
- {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/METADATA +1 -1
- dbt_firebolt-1.9.4.dist-info/RECORD +61 -0
- dbt_firebolt-1.9.3.dist-info/RECORD +0 -16
- {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/LICENSE +0 -0
- {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/WHEEL +0 -0
- {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,37 @@
|
|
1
|
+
{% macro firebolt__get_show_grant_sql(relation) %}
|
2
|
+
{# Usually called from apply_grants. Should not be called directly. #}
|
3
|
+
{{ adapter.raise_grant_error() }}
|
4
|
+
{% endmacro %}
|
5
|
+
|
6
|
+
|
7
|
+
{%- macro firebolt__get_grant_sql(relation, privilege, grantee) -%}
|
8
|
+
{# Usually called from apply_grants. Should not be called directly. #}
|
9
|
+
{{ adapter.raise_grant_error() }}
|
10
|
+
{%- endmacro -%}
|
11
|
+
|
12
|
+
|
13
|
+
{%- macro firebolt__get_revoke_sql(relation, privilege, grantee) -%}
|
14
|
+
{# Usually called from apply_grants. Should not be called directly. #}
|
15
|
+
{{ adapter.raise_grant_error() }}
|
16
|
+
{%- endmacro -%}
|
17
|
+
|
18
|
+
|
19
|
+
{% macro firebolt__copy_grants() %}
|
20
|
+
{{ return(True) }}
|
21
|
+
{% endmacro %}
|
22
|
+
|
23
|
+
|
24
|
+
{% macro firebolt__apply_grants(relation, grant_config, should_revoke) %}
|
25
|
+
{% if grant_config %}
|
26
|
+
{{ adapter.raise_grant_error() }}
|
27
|
+
{% endif %}
|
28
|
+
{% endmacro %}
|
29
|
+
|
30
|
+
|
31
|
+
{%- macro firebolt__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}
|
32
|
+
{# Firebolt does not support DCL statements yet #}
|
33
|
+
{% if grant_config %}
|
34
|
+
{{ adapter.raise_grant_error() }}
|
35
|
+
{% endif %}
|
36
|
+
{{ return([]) }}
|
37
|
+
{%- endmacro %}
|
@@ -0,0 +1,49 @@
|
|
1
|
+
{% macro firebolt__rename_relation(source, target) -%}
|
2
|
+
{# Create new relation, do CTAS to swap it, remove original relation.
|
3
|
+
Must check whether relation is view or table, because process is
|
4
|
+
different for the two. #}
|
5
|
+
{% set all_relations = (list_relations_without_caching(source)) %}
|
6
|
+
{% set tables = adapter.filter_table(all_relations, 'type', 'table') %}
|
7
|
+
{% set views = adapter.filter_table(all_relations, 'type', 'view') %}
|
8
|
+
{% if (views.rows | length) > 0 %}
|
9
|
+
{% call statement('view_definition', fetch_result=True) %}
|
10
|
+
|
11
|
+
SELECT view_definition FROM information_schema.views
|
12
|
+
WHERE table_name = '{{ source.identifier }}'
|
13
|
+
{%- endcall %}
|
14
|
+
{% if load_result('view_definition')['data'] %}
|
15
|
+
{# For some reason this ocassionally returns an empty set.
|
16
|
+
In that case do nothing. Don't even ask why I have to jump through
|
17
|
+
the stupid hoops to get the value out of the result in the next line. #}
|
18
|
+
{% set view_ddl = load_result('view_definition')['data'][0][0] %}
|
19
|
+
{% set view_ddl = view_ddl.replace(source.identifier,
|
20
|
+
target.identifier) %}
|
21
|
+
{% call statement('new_view') %}
|
22
|
+
|
23
|
+
DROP VIEW {{ source.identifier }};
|
24
|
+
{{ view_ddl }};
|
25
|
+
{% endcall %}
|
26
|
+
{% endif %}
|
27
|
+
{% elif (tables.rows | length) > 0 %}
|
28
|
+
{# There were no views, so retrieve table. (There must be a table.) #}
|
29
|
+
{% set table_type = adapter.filter_table(all_relations, 'table_type', '') %}
|
30
|
+
{% call statement('tables') %}
|
31
|
+
|
32
|
+
CREATE {{ table_type }} TABLE {{ target.identifier }} AS
|
33
|
+
SELECT * FROM {{ source.identifier }};
|
34
|
+
DROP TABLE {{ source.identifier }};
|
35
|
+
{%- endcall %}
|
36
|
+
{% endif %}
|
37
|
+
{% endmacro %}
|
38
|
+
|
39
|
+
|
40
|
+
{% macro firebolt__drop_relation(relation) -%}
|
41
|
+
{#-
|
42
|
+
Drop relation. Drop both table and view because relation doesn't always
|
43
|
+
have a table_type specified.
|
44
|
+
#}
|
45
|
+
{% call statement('drop') %}
|
46
|
+
|
47
|
+
DROP {{ relation.type | upper }} IF EXISTS {{ relation }} CASCADE;
|
48
|
+
{% endcall %}
|
49
|
+
{% endmacro %}
|
@@ -0,0 +1,141 @@
|
|
1
|
+
{% macro firebolt__create_external_table(source_node) %}
|
2
|
+
{% if source_node.external.strategy == 'copy' %}
|
3
|
+
{{ firebolt__create_with_copy_from(source_node) }}
|
4
|
+
{% else %}
|
5
|
+
{{ firebolt__create_with_external_table(source_node) }}
|
6
|
+
{% endif %}
|
7
|
+
{% endmacro %}
|
8
|
+
|
9
|
+
{% macro firebolt__create_with_external_table(source_node) %}
|
10
|
+
{%- set external = source_node.external -%}
|
11
|
+
{%- if 'partitions' in external -%}
|
12
|
+
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
13
|
+
external.partitions) -%}
|
14
|
+
{%- else -%}
|
15
|
+
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
16
|
+
[]) -%}
|
17
|
+
{%- endif -%}
|
18
|
+
{%- set credentials = external.credentials -%}
|
19
|
+
{# Leaving out "IF NOT EXISTS" because this should only be called by
|
20
|
+
if no DROP IF is necessary. #}
|
21
|
+
CREATE EXTERNAL TABLE {{source(source_node.source_name, source_node.name)}} (
|
22
|
+
{%- for column in columns -%}
|
23
|
+
{{ column }}
|
24
|
+
{{- ',' if not loop.last }}
|
25
|
+
{% endfor -%}
|
26
|
+
)
|
27
|
+
{% if external.url %} URL = '{{external.url}}' {%- endif %}
|
28
|
+
{%- if credentials and credentials.internal_role_arn %}
|
29
|
+
CREDENTIALS = (AWS_ROLE_ARN = '{{credentials.internal_role_arn}}'
|
30
|
+
{%- if credentials.external_role_id %}
|
31
|
+
AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}'
|
32
|
+
{%- endif -%}
|
33
|
+
)
|
34
|
+
{% elif credentials and credentials.aws_key_id %}
|
35
|
+
CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}'
|
36
|
+
AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
|
37
|
+
{%- endif %}
|
38
|
+
{%- if external.object_pattern -%} OBJECT_PATTERN = '{{external.object_pattern}}' {%- endif %}
|
39
|
+
{% if external.object_patterns -%}
|
40
|
+
OBJECT_PATTERN =
|
41
|
+
{%- for obj in external.object_patterns -%}
|
42
|
+
{{ obj }}
|
43
|
+
{{- ',' if not loop.last }}
|
44
|
+
{%- endfor %}
|
45
|
+
{%- endif %}
|
46
|
+
{%- if external.compression -%} COMPRESSION = {{external.compression}} {%- endif %}
|
47
|
+
TYPE = {{ external.type }}
|
48
|
+
{% endmacro %}
|
49
|
+
|
50
|
+
{% macro firebolt__create_with_copy_from(source_node) %}
|
51
|
+
{# COPY FROM is only available in Firebolt 2.0. #}
|
52
|
+
{%- set external = source_node.external -%}
|
53
|
+
{%- set credentials = external.credentials -%}
|
54
|
+
{%- set where_clause = external.where -%}
|
55
|
+
{%- set options = external.options -%}
|
56
|
+
{%- set csv_options = options.csv_options -%}
|
57
|
+
{%- set error_file_credentials = options.error_file_credentials -%}
|
58
|
+
|
59
|
+
{# There are no partitions, but this formats the columns correctly. #}
|
60
|
+
{%- if 'partitions' in external -%}
|
61
|
+
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
62
|
+
external.partitions) -%}
|
63
|
+
{%- else -%}
|
64
|
+
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
65
|
+
[]) -%}
|
66
|
+
{%- endif -%}
|
67
|
+
COPY INTO {{source(source_node.source_name, source_node.name)}}
|
68
|
+
{%- if columns and columns | length > 0 %}
|
69
|
+
(
|
70
|
+
{%- for column in columns -%}
|
71
|
+
{{ column.name }}
|
72
|
+
{%- if column.default %} DEFAULT {{ column.default }}{% endif %}
|
73
|
+
{%- if column.source_column_name %} {{ '$' ~ loop.index0 }}{% endif %}
|
74
|
+
{{- ',' if not loop.last }}
|
75
|
+
{%- endfor -%}
|
76
|
+
)
|
77
|
+
{%- endif %}
|
78
|
+
FROM '{{external.url}}'
|
79
|
+
{%- if options or credentials %}
|
80
|
+
WITH
|
81
|
+
{%- if options.object_pattern %}
|
82
|
+
PATTERN = '{{options.object_pattern}}'
|
83
|
+
{%- endif %}
|
84
|
+
{%- if options.type %}
|
85
|
+
TYPE = {{ options.type }}
|
86
|
+
{%- endif %}
|
87
|
+
{%- if options.auto_create %}
|
88
|
+
AUTO_CREATE = {{ options.auto_create | upper }}
|
89
|
+
{%- endif %}
|
90
|
+
{%- if options.allow_column_mismatch %}
|
91
|
+
ALLOW_COLUMN_MISMATCH = {{ options.allow_column_mismatch | upper }}
|
92
|
+
{%- endif %}
|
93
|
+
{%- if options.error_file %}
|
94
|
+
ERROR_FILE = '{{ options.error_file }}'
|
95
|
+
{%- endif %}
|
96
|
+
{%- if error_file_credentials %}
|
97
|
+
ERROR_FILE_CREDENTIALS = (AWS_KEY_ID = '{{ error_file_credentials.aws_key_id }}' AWS_SECRET_KEY = '{{ error_file_credentials.aws_secret_key }}')
|
98
|
+
{%- endif %}
|
99
|
+
{%- if options.max_errors_per_file %}
|
100
|
+
MAX_ERRORS_PER_FILE = {{ options.max_errors_per_file }}
|
101
|
+
{%- endif %}
|
102
|
+
{%- if csv_options %}
|
103
|
+
{%- if csv_options.header %}
|
104
|
+
HEADER = {{ csv_options.header | upper }}
|
105
|
+
{%- endif %}
|
106
|
+
{%- if csv_options.delimiter %}
|
107
|
+
DELIMITER = '{{ csv_options.delimiter }}'
|
108
|
+
{%- endif %}
|
109
|
+
{%- if csv_options.newline %}
|
110
|
+
NEWLINE = '{{ csv_options.newline }}'
|
111
|
+
{%- endif %}
|
112
|
+
{%- if csv_options.quote %}
|
113
|
+
QUOTE = {{ csv_options.quote }}
|
114
|
+
{%- endif %}
|
115
|
+
{%- if csv_options.escape %}
|
116
|
+
ESCAPE = '{{ csv_options.escape }}'
|
117
|
+
{%- endif %}
|
118
|
+
{%- if csv_options.null_string %}
|
119
|
+
NULL_STRING = '{{ csv_options.null_string }}'
|
120
|
+
{%- endif %}
|
121
|
+
{%- if csv_options.empty_field_as_null %}
|
122
|
+
EMPTY_FIELD_AS_NULL = {{ csv_options.empty_field_as_null | upper }}
|
123
|
+
{%- endif %}
|
124
|
+
{%- if csv_options.skip_blank_lines %}
|
125
|
+
SKIP_BLANK_LINES = {{ csv_options.skip_blank_lines | upper }}
|
126
|
+
{%- endif %}
|
127
|
+
{%- if csv_options.date_format %}
|
128
|
+
DATE_FORMAT = '{{ csv_options.date_format }}'
|
129
|
+
{%- endif %}
|
130
|
+
{%- if csv_options.timestamp_format %}
|
131
|
+
TIMESTAMP_FORMAT = '{{ csv_options.timestamp_format }}'
|
132
|
+
{%- endif %}
|
133
|
+
{%- endif %}
|
134
|
+
{%- if credentials %}
|
135
|
+
CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
|
136
|
+
{%- endif %}
|
137
|
+
{%- endif %}
|
138
|
+
{%- if where_clause %}
|
139
|
+
WHERE {{ where_clause }}
|
140
|
+
{%- endif %}
|
141
|
+
{% endmacro %}
|
@@ -0,0 +1,18 @@
|
|
1
|
+
{% macro firebolt__get_external_build_plan(source_node) %}
|
2
|
+
{% set build_plan = [] %}
|
3
|
+
{% set old_relation = adapter.get_relation(
|
4
|
+
database = source_node.database,
|
5
|
+
schema = source_node.schema,
|
6
|
+
identifier = source_node.identifier
|
7
|
+
) %}
|
8
|
+
{# var(variable, Boolean) defaults to Boolean value if variable isnt set.
|
9
|
+
Firebolt doesn't do refresh because we don't need to—external tables will always
|
10
|
+
pull most recent data from S3, so the default action below is to skip. #}
|
11
|
+
{% if old_relation is none or var('ext_full_refresh', false) %}
|
12
|
+
{% set build_plan = build_plan + [dropif(source_node),
|
13
|
+
create_external_table(source_node)] %}
|
14
|
+
{% else %}
|
15
|
+
{% set build_plan = dbt_external_tables.refresh_external_table(source_node) %}
|
16
|
+
{% endif %}
|
17
|
+
{% do return(build_plan) %}
|
18
|
+
{% endmacro %}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
{% macro intersect_columns(source_columns, target_columns) %}
|
2
|
+
{# Return a List[FireboltColumn] of columns that appear in both source and target.
|
3
|
+
Args:
|
4
|
+
source_columns: List[FireboltColumn]
|
5
|
+
target_columns: List[FireboltColumn]
|
6
|
+
#}
|
7
|
+
{% set result = [] %}
|
8
|
+
{# Note I'm using `column` and not `name` below, as name is a getter
|
9
|
+
and column is the actual field. #}
|
10
|
+
{% set source_names = source_columns | map(attribute = 'column') | list %}
|
11
|
+
{% set target_names = target_columns | map(attribute = 'column') | list %}
|
12
|
+
{# Check whether the name attribute exists in the target - this does
|
13
|
+
not perform a data type check. This is O(m•n), but cardinality
|
14
|
+
of columns is probably low enough that it doesn't matter. #}
|
15
|
+
{% for sc in source_columns %}
|
16
|
+
{% if sc.name in target_names %}
|
17
|
+
{{ result.append(sc) }}
|
18
|
+
{% endif %}
|
19
|
+
{% endfor %}
|
20
|
+
{{ return(result) }}
|
21
|
+
{% endmacro %}
|
@@ -0,0 +1,131 @@
|
|
1
|
+
{% macro dbt_firebolt_get_tmp_relation_type(strategy) %}
|
2
|
+
{#
|
3
|
+
Determine whether to use a temporary view or a table based on
|
4
|
+
specified strategy and config. Users can override the behaviour
|
5
|
+
by setting tmp_replation_type. Default is to use a view (more efficient).
|
6
|
+
|
7
|
+
Arguments:
|
8
|
+
strategy: incremental strategy from config. e.g. "append"
|
9
|
+
#}
|
10
|
+
{%- set tmp_relation_type = config.get('tmp_relation_type') -%}
|
11
|
+
|
12
|
+
{% if tmp_relation_type == "table" %}
|
13
|
+
{{ return("table") }}
|
14
|
+
{% elif tmp_relation_type == "view" %}
|
15
|
+
{{ return("view") }}
|
16
|
+
{% elif strategy in ("default", "append", "insert_overwrite") %}
|
17
|
+
{{ return("view") }}
|
18
|
+
{% else %}
|
19
|
+
{{ return("table") }}
|
20
|
+
{% endif %}
|
21
|
+
{% endmacro %}
|
22
|
+
|
23
|
+
{% materialization incremental, adapter='firebolt' -%}
|
24
|
+
{#
|
25
|
+
Incremental implementation. Actual strategy SQL is determined
|
26
|
+
outside this materialization, in strategies.sql.
|
27
|
+
|
28
|
+
This function works by first creating a view, `new_records`, of type `BaseRelation`,
|
29
|
+
then using a CTE defined in the project model to populate `new_records` with
|
30
|
+
any records that ought to be inserted into the original table (views aren't currently
|
31
|
+
supported), `existing`, using an INSERT. This process is agnostic to the incremental
|
32
|
+
strategy actually used: the specific INSERT is defined in `strategies.sql`.
|
33
|
+
|
34
|
+
Note that all new relations must first be instantiated as BaseRelation objects,
|
35
|
+
and that those objects are used to create and query actual relations in the DB.
|
36
|
+
Care must be taken to correctly define each BaseRelation as either a view or
|
37
|
+
a table, lest subsequent operations on the relations (be the they objects or
|
38
|
+
the DB tables the objects abstract) fail.
|
39
|
+
#}
|
40
|
+
|
41
|
+
{% set unique_key = config.get('unique_key') %}
|
42
|
+
|
43
|
+
{% set grant_config = config.get('grants') %}
|
44
|
+
|
45
|
+
{%- set strategy = config.get('incremental_strategy') -%}
|
46
|
+
{%- if is_incremental() and strategy is none -%}
|
47
|
+
{{ log('Model %s is set to incremental, but no incremental strategy is set. '
|
48
|
+
'Defaulting to append' % this, True) }}
|
49
|
+
{%- set strategy = 'append' -%}
|
50
|
+
{%- endif -%}
|
51
|
+
{# In following lines:
|
52
|
+
|
53
|
+
`target` is just a dbt `BaseRelation` object, not a DB table.
|
54
|
+
|
55
|
+
We're only retrieving `existing` to check for existence; `load_relation()`
|
56
|
+
returns a `BaseRelation` with the dictionary values of any relations that exist
|
57
|
+
in dbt's cache that share the identifier of the passed relation. If none
|
58
|
+
exist, returns None. Note that this does *not* create a table in the DB.
|
59
|
+
|
60
|
+
`target` is a relation with all the same fields as `existing`, but guaranteed
|
61
|
+
to be an actual `BaseRelation` object. #}
|
62
|
+
{%- set target = this.incorporate(type='table') -%}
|
63
|
+
{%- set existing = load_relation(this) -%}
|
64
|
+
{%- set tmp_relation_type = dbt_firebolt_get_tmp_relation_type(strategy) -%}
|
65
|
+
{%- set new_records = make_temp_relation(target) -%}
|
66
|
+
{{ drop_relation_if_exists(new_records) }}
|
67
|
+
{%- set new_records = new_records.incorporate(type=tmp_relation_type) -%}
|
68
|
+
{%- set on_schema_change = incremental_validate_on_schema_change(
|
69
|
+
config.get('on_schema_change'),
|
70
|
+
default='ignore') -%}
|
71
|
+
|
72
|
+
-- `BEGIN` happens here:
|
73
|
+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
|
74
|
+
{%- set partition_by = config.get('partition_by') -%}
|
75
|
+
{# First check whether we want to full refresh for existing view or config reasons. #}
|
76
|
+
{%- if strategy == 'insert_overwrite' and not partition_by -%}
|
77
|
+
{% do exceptions.raise_compiler_error('`insert_overwrite` is specified for model %s, '
|
78
|
+
'but `partition_by` is not. `insert_overwrite` '
|
79
|
+
'requires a partition.' % (target)) %}
|
80
|
+
{%- endif -%}
|
81
|
+
{%- set do_full_refresh = (should_full_refresh()
|
82
|
+
or existing.is_view) %}
|
83
|
+
{% if existing is none %}
|
84
|
+
{%- set build_sql = create_table_as(False, target, sql) -%}
|
85
|
+
{%- elif do_full_refresh -%}
|
86
|
+
{{ drop_relation_if_exists(existing) }}
|
87
|
+
{%- set build_sql = create_table_as(False, target, sql) -%}
|
88
|
+
{%- else -%}
|
89
|
+
{# Actually do the incremental query here. #}
|
90
|
+
{# Instantiate new objects in dbt's internal list. Have to
|
91
|
+
run this query so dbt can query the DB to get the columns in
|
92
|
+
new_records. #}
|
93
|
+
{%- if tmp_relation_type=='view' -%}
|
94
|
+
{%- do run_query(create_view_as(new_records, sql)) -%}
|
95
|
+
{%- else -%}
|
96
|
+
{%- do run_query(create_table_as(True, new_records, sql)) -%}
|
97
|
+
{%- endif -%}
|
98
|
+
{# All errors involving schema changes are dealt with in `process_schema_changes`. #}
|
99
|
+
{%- set dest_columns = process_schema_changes(on_schema_change,
|
100
|
+
new_records,
|
101
|
+
existing) -%}
|
102
|
+
{%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}
|
103
|
+
{%- set build_sql = get_incremental_sql(strategy,
|
104
|
+
new_records,
|
105
|
+
target,
|
106
|
+
unique_key,
|
107
|
+
dest_columns,
|
108
|
+
incremental_predicates) -%}
|
109
|
+
{%- endif -%}
|
110
|
+
{%- call statement("main") -%}
|
111
|
+
{{ build_sql }}
|
112
|
+
{%- endcall -%}
|
113
|
+
|
114
|
+
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
|
115
|
+
{% do apply_grants(target_relation, grant_config, should_revoke) %}
|
116
|
+
|
117
|
+
{# Todo: figure out what persist_docs and create_indexes do. #}
|
118
|
+
{%- do persist_docs(target, model) -%}
|
119
|
+
{%- if existing is none
|
120
|
+
or existing.is_view
|
121
|
+
or should_full_refresh() -%}
|
122
|
+
{%- do create_indexes(target) -%}
|
123
|
+
{%- endif %}
|
124
|
+
|
125
|
+
{{ drop_relation_if_exists(new_records) }}
|
126
|
+
|
127
|
+
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
128
|
+
|
129
|
+
{{ return({'relations': [target]}) }}
|
130
|
+
|
131
|
+
{%- endmaterialization %}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
{% macro is_incremental() %}
|
2
|
+
{#- Return True if model is incremental, otherwise False. -#}
|
3
|
+
{% if not execute %}
|
4
|
+
{{ return(False) }}
|
5
|
+
{% else %}
|
6
|
+
{% set relation = adapter.get_relation(this.database,
|
7
|
+
this.schema,
|
8
|
+
this.table) %}
|
9
|
+
{{ return(relation is not none
|
10
|
+
and relation.type == 'table'
|
11
|
+
and model.config.materialized == 'incremental'
|
12
|
+
and not should_full_refresh()) }}
|
13
|
+
{% endif %}
|
14
|
+
{% endmacro %}
|
@@ -0,0 +1,38 @@
|
|
1
|
+
{% macro firebolt__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
|
2
|
+
|
3
|
+
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
|
4
|
+
|
5
|
+
{% if unique_key %}
|
6
|
+
{% if unique_key is sequence and unique_key is not string %}
|
7
|
+
delete from {{ target }}
|
8
|
+
where
|
9
|
+
({{ get_quoted_csv(unique_key) }}) in (
|
10
|
+
select {{ get_quoted_csv(unique_key) }}
|
11
|
+
from {{ source }}
|
12
|
+
)
|
13
|
+
{% if incremental_predicates %}
|
14
|
+
{% for predicate in incremental_predicates %}
|
15
|
+
and {{ predicate }}
|
16
|
+
{% endfor %}
|
17
|
+
{% endif %};
|
18
|
+
{% else %}
|
19
|
+
delete from {{ target }}
|
20
|
+
where (
|
21
|
+
{{ unique_key }}) in (
|
22
|
+
select ({{ unique_key }})
|
23
|
+
from {{ source }}
|
24
|
+
)
|
25
|
+
{%- if incremental_predicates %}
|
26
|
+
{% for predicate in incremental_predicates %}
|
27
|
+
and {{ predicate }}
|
28
|
+
{% endfor %}
|
29
|
+
{%- endif -%};
|
30
|
+
|
31
|
+
{% endif %}
|
32
|
+
{% endif %}
|
33
|
+
|
34
|
+
insert into {{ target }} ({{ dest_cols_csv }})
|
35
|
+
select {{ dest_cols_csv }}
|
36
|
+
from {{ source }}
|
37
|
+
|
38
|
+
{%- endmacro %}
|
@@ -0,0 +1,90 @@
|
|
1
|
+
{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
|
2
|
+
{#
|
3
|
+
Ensure that the value of `on_schema_change` in dbt_project models is valid.
|
4
|
+
Firebolt doesnt currently support `sync_all_columns`, so this is excluded
|
5
|
+
from valid columns.
|
6
|
+
#}
|
7
|
+
{% if on_schema_change == 'sync_all_columns' %}
|
8
|
+
{% do exceptions.raise_compiler_error(
|
9
|
+
'Firebolt does not support the on_schema_change value "sync_all_columns."') %}
|
10
|
+
{{ return(default) }}
|
11
|
+
{% else %}
|
12
|
+
{{ return(on_schema_change) }}
|
13
|
+
{% endif %}
|
14
|
+
{% endmacro %}
|
15
|
+
|
16
|
+
|
17
|
+
{% macro check_for_schema_changes(source_relation, target_relation) %}
|
18
|
+
{#
|
19
|
+
Return a dict = {
|
20
|
+
'schema_changed': schema_changed,
|
21
|
+
'source_not_in_target': source_not_in_target (List[FireboltColumn]),
|
22
|
+
'target_not_in_source': target_not_in_source (List[FireboltColumn]),
|
23
|
+
'source_columns': source_columns (List[FireboltColumn]),
|
24
|
+
'target_columns': target_columns (List[FireboltColumn]),
|
25
|
+
'new_target_types': new_target_types,
|
26
|
+
'common_columns': common_colums
|
27
|
+
}
|
28
|
+
Args:
|
29
|
+
source_relation: dbt Relation
|
30
|
+
target_relation: dbt Relation
|
31
|
+
#}
|
32
|
+
{%- set schema_changed = False -%}
|
33
|
+
{%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}
|
34
|
+
{%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
|
35
|
+
{%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
|
36
|
+
{%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
|
37
|
+
{%- set new_target_types = diff_column_data_types(source_columns, target_columns) -%}
|
38
|
+
{%- set common_columns = intersect_columns(source_columns, target_columns) -%}
|
39
|
+
{% if source_not_in_target != [] %}
|
40
|
+
{%- set schema_changed = True -%}
|
41
|
+
{% elif target_not_in_source != [] or new_target_types != [] %}
|
42
|
+
{%- set schema_changed = True -%}
|
43
|
+
{% elif new_target_types != [] %}
|
44
|
+
{%- set schema_changed = True -%}
|
45
|
+
{% endif %}
|
46
|
+
{% set changes_dict = {
|
47
|
+
'schema_changed': schema_changed,
|
48
|
+
'source_not_in_target': source_not_in_target,
|
49
|
+
'target_not_in_source': target_not_in_source,
|
50
|
+
'source_columns': source_columns,
|
51
|
+
'target_columns': target_columns,
|
52
|
+
'new_target_types': new_target_types,
|
53
|
+
'common_columns': common_columns
|
54
|
+
} %}
|
55
|
+
{{ return(changes_dict) }}
|
56
|
+
{% endmacro %}
|
57
|
+
|
58
|
+
|
59
|
+
{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
|
60
|
+
{#
|
61
|
+
Check for schema changes. Error out if appropriate, else return the list of columns
|
62
|
+
that will be transferred from source to target, i.e. the intersection of the columns.
|
63
|
+
#}
|
64
|
+
{% if on_schema_change == 'sync_all_columns' %}
|
65
|
+
{% do exceptions.raise_compiler_error(
|
66
|
+
'Firebolt does not allow an `on_schema_change` value of `sync_all_columns`.') %}
|
67
|
+
{% endif %}
|
68
|
+
{% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
|
69
|
+
{% if schema_changes_dict['schema_changed'] %}
|
70
|
+
{% if on_schema_change == 'fail' %}
|
71
|
+
{% do exceptions.raise_compiler_error(
|
72
|
+
'A schema change was detected and `on_schema_change` was set to "fail".') %}
|
73
|
+
{% else %}
|
74
|
+
{% set fail_msg %}
|
75
|
+
Schema changes detected on this incremental!
|
76
|
+
Either revert the change or run the model with the --full-refresh flag
|
77
|
+
on the command line to recreate the model with the new schema definition.
|
78
|
+
Running with the --full-refresh flag drops and recreates the database object.
|
79
|
+
|
80
|
+
Additional troubleshooting context:
|
81
|
+
Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}
|
82
|
+
Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}
|
83
|
+
New column types: {{ schema_changes_dict['new_target_types'] }}
|
84
|
+
{% endset %}
|
85
|
+
|
86
|
+
{% do exceptions.raise_compiler_error(fail_msg) %}
|
87
|
+
{% endif %}
|
88
|
+
{% endif %}
|
89
|
+
{{ return(schema_changes_dict['common_columns']) }}
|
90
|
+
{% endmacro %}
|