dbt-firebolt 1.9.0__py3-none-any.whl → 1.9.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- dbt/adapters/firebolt/__init__.py +1 -1
- dbt/adapters/firebolt/connections.py +4 -2
- dbt/adapters/firebolt/impl.py +2 -3
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/METADATA +2 -2
- dbt_firebolt-1.9.1.dist-info/RECORD +13 -0
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/WHEEL +1 -1
- dbt/include/firebolt/dbt_project.yml +0 -6
- dbt/include/firebolt/macros/adapters/apply_grants.sql +0 -37
- dbt/include/firebolt/macros/adapters/relation.sql +0 -49
- dbt/include/firebolt/macros/adapters.sql +0 -197
- dbt/include/firebolt/macros/catalog.sql +0 -103
- dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql +0 -137
- dbt/include/firebolt/macros/dbt_external_tables/dropif.sql +0 -6
- dbt/include/firebolt/macros/dbt_external_tables/get_external_build_plan.sql +0 -18
- dbt/include/firebolt/macros/materializations/clone.sql +0 -3
- dbt/include/firebolt/macros/materializations/materialized_view.sql +0 -3
- dbt/include/firebolt/macros/materializations/models/incremental/column_helpers.sql +0 -21
- dbt/include/firebolt/macros/materializations/models/incremental/incremental.sql +0 -131
- dbt/include/firebolt/macros/materializations/models/incremental/is_incremental.sql +0 -14
- dbt/include/firebolt/macros/materializations/models/incremental/merge.sql +0 -38
- dbt/include/firebolt/macros/materializations/models/incremental/on_schema_change.sql +0 -90
- dbt/include/firebolt/macros/materializations/models/incremental/strategies.sql +0 -132
- dbt/include/firebolt/macros/materializations/seed.sql +0 -42
- dbt/include/firebolt/macros/materializations/snapshot_merge.sql +0 -19
- dbt/include/firebolt/macros/materializations/table.sql +0 -40
- dbt/include/firebolt/macros/materializations/test.sql +0 -15
- dbt/include/firebolt/macros/materializations/view.sql +0 -39
- dbt/include/firebolt/macros/relations/materialized_view/alter.sql +0 -11
- dbt/include/firebolt/macros/relations/materialized_view/create.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/describe.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/refresh.sql +0 -3
- dbt/include/firebolt/macros/relations/table/create.sql +0 -78
- dbt/include/firebolt/macros/relations/table/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/table/rename.sql +0 -6
- dbt/include/firebolt/macros/relations/table/replace.sql +0 -3
- dbt/include/firebolt/macros/relations/view/create.sql +0 -16
- dbt/include/firebolt/macros/relations/view/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/view/rename.sql +0 -6
- dbt/include/firebolt/macros/relations/view/replace.sql +0 -3
- dbt/include/firebolt/macros/utils/array_append.sql +0 -3
- dbt/include/firebolt/macros/utils/array_concat.sql +0 -3
- dbt/include/firebolt/macros/utils/array_construct.sql +0 -3
- dbt/include/firebolt/macros/utils/bool_or.sql +0 -5
- dbt/include/firebolt/macros/utils/cast_bool_to_text.sql +0 -7
- dbt/include/firebolt/macros/utils/dateadd.sql +0 -9
- dbt/include/firebolt/macros/utils/datediff.sql +0 -9
- dbt/include/firebolt/macros/utils/except.sql +0 -6
- dbt/include/firebolt/macros/utils/intersect.sql +0 -6
- dbt/include/firebolt/macros/utils/listagg.sql +0 -27
- dbt/include/firebolt/macros/utils/position.sql +0 -3
- dbt/include/firebolt/macros/utils/right.sql +0 -12
- dbt/include/firebolt/macros/utils/split_part.sql +0 -14
- dbt/include/firebolt/macros/utils/timestamps.sql +0 -14
- dbt_firebolt-1.9.0.dist-info/RECORD +0 -61
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/LICENSE +0 -0
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/top_level.txt +0 -0
@@ -1,131 +0,0 @@
|
|
1
|
-
{% macro dbt_firebolt_get_tmp_relation_type(strategy) %}
|
2
|
-
{#
|
3
|
-
Determine whether to use a temporary view or a table based on
|
4
|
-
specified strategy and config. Users can override the behaviour
|
5
|
-
by setting tmp_replation_type. Default is to use a view (more efficient).
|
6
|
-
|
7
|
-
Arguments:
|
8
|
-
strategy: incremental strategy from config. e.g. "append"
|
9
|
-
#}
|
10
|
-
{%- set tmp_relation_type = config.get('tmp_relation_type') -%}
|
11
|
-
|
12
|
-
{% if tmp_relation_type == "table" %}
|
13
|
-
{{ return("table") }}
|
14
|
-
{% elif tmp_relation_type == "view" %}
|
15
|
-
{{ return("view") }}
|
16
|
-
{% elif strategy in ("default", "append", "insert_overwrite") %}
|
17
|
-
{{ return("view") }}
|
18
|
-
{% else %}
|
19
|
-
{{ return("table") }}
|
20
|
-
{% endif %}
|
21
|
-
{% endmacro %}
|
22
|
-
|
23
|
-
{% materialization incremental, adapter='firebolt' -%}
|
24
|
-
{#
|
25
|
-
Incremental implementation. Actual strategy SQL is determined
|
26
|
-
outside this materialization, in strategies.sql.
|
27
|
-
|
28
|
-
This function works by first creating a view, `new_records`, of type `BaseRelation`,
|
29
|
-
then using a CTE defined in the project model to populate `new_records` with
|
30
|
-
any records that ought to be inserted into the original table (views aren't currently
|
31
|
-
supported), `existing`, using an INSERT. This process is agnostic to the incremental
|
32
|
-
strategy actually used: the specific INSERT is defined in `strategies.sql`.
|
33
|
-
|
34
|
-
Note that all new relations must first be instantiated as BaseRelation objects,
|
35
|
-
and that those objects are used to create and query actual relations in the DB.
|
36
|
-
Care must be taken to correctly define each BaseRelation as either a view or
|
37
|
-
a table, lest subsequent operations on the relations (be the they objects or
|
38
|
-
the DB tables the objects abstract) fail.
|
39
|
-
#}
|
40
|
-
|
41
|
-
{% set unique_key = config.get('unique_key') %}
|
42
|
-
|
43
|
-
{% set grant_config = config.get('grants') %}
|
44
|
-
|
45
|
-
{%- set strategy = config.get('incremental_strategy') -%}
|
46
|
-
{%- if is_incremental() and strategy is none -%}
|
47
|
-
{{ log('Model %s is set to incremental, but no incremental strategy is set. '
|
48
|
-
'Defaulting to append' % this, True) }}
|
49
|
-
{%- set strategy = 'append' -%}
|
50
|
-
{%- endif -%}
|
51
|
-
{# In following lines:
|
52
|
-
|
53
|
-
`target` is just a dbt `BaseRelation` object, not a DB table.
|
54
|
-
|
55
|
-
We're only retrieving `existing` to check for existence; `load_relation()`
|
56
|
-
returns a `BaseRelation` with the dictionary values of any relations that exist
|
57
|
-
in dbt's cache that share the identifier of the passed relation. If none
|
58
|
-
exist, returns None. Note that this does *not* create a table in the DB.
|
59
|
-
|
60
|
-
`target` is a relation with all the same fields as `existing`, but guaranteed
|
61
|
-
to be an actual `BaseRelation` object. #}
|
62
|
-
{%- set target = this.incorporate(type='table') -%}
|
63
|
-
{%- set existing = load_relation(this) -%}
|
64
|
-
{%- set tmp_relation_type = dbt_firebolt_get_tmp_relation_type(strategy) -%}
|
65
|
-
{%- set new_records = make_temp_relation(target) -%}
|
66
|
-
{{ drop_relation_if_exists(new_records) }}
|
67
|
-
{%- set new_records = new_records.incorporate(type=tmp_relation_type) -%}
|
68
|
-
{%- set on_schema_change = incremental_validate_on_schema_change(
|
69
|
-
config.get('on_schema_change'),
|
70
|
-
default='ignore') -%}
|
71
|
-
|
72
|
-
-- `BEGIN` happens here:
|
73
|
-
{{ run_hooks(pre_hooks, inside_transaction=True) }}
|
74
|
-
{%- set partition_by = config.get('partition_by') -%}
|
75
|
-
{# First check whether we want to full refresh for existing view or config reasons. #}
|
76
|
-
{%- if strategy == 'insert_overwrite' and not partition_by -%}
|
77
|
-
{% do exceptions.raise_compiler_error('`insert_overwrite` is specified for model %s, '
|
78
|
-
'but `partition_by` is not. `insert_overwrite` '
|
79
|
-
'requires a partition.' % (target)) %}
|
80
|
-
{%- endif -%}
|
81
|
-
{%- set do_full_refresh = (should_full_refresh()
|
82
|
-
or existing.is_view) %}
|
83
|
-
{% if existing is none %}
|
84
|
-
{%- set build_sql = create_table_as(False, target, sql) -%}
|
85
|
-
{%- elif do_full_refresh -%}
|
86
|
-
{{ drop_relation_if_exists(existing) }}
|
87
|
-
{%- set build_sql = create_table_as(False, target, sql) -%}
|
88
|
-
{%- else -%}
|
89
|
-
{# Actually do the incremental query here. #}
|
90
|
-
{# Instantiate new objects in dbt's internal list. Have to
|
91
|
-
run this query so dbt can query the DB to get the columns in
|
92
|
-
new_records. #}
|
93
|
-
{%- if tmp_relation_type=='view' -%}
|
94
|
-
{%- do run_query(create_view_as(new_records, sql)) -%}
|
95
|
-
{%- else -%}
|
96
|
-
{%- do run_query(create_table_as(True, new_records, sql)) -%}
|
97
|
-
{%- endif -%}
|
98
|
-
{# All errors involving schema changes are dealt with in `process_schema_changes`. #}
|
99
|
-
{%- set dest_columns = process_schema_changes(on_schema_change,
|
100
|
-
new_records,
|
101
|
-
existing) -%}
|
102
|
-
{%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}
|
103
|
-
{%- set build_sql = get_incremental_sql(strategy,
|
104
|
-
new_records,
|
105
|
-
target,
|
106
|
-
unique_key,
|
107
|
-
dest_columns,
|
108
|
-
incremental_predicates) -%}
|
109
|
-
{%- endif -%}
|
110
|
-
{%- call statement("main") -%}
|
111
|
-
{{ build_sql }}
|
112
|
-
{%- endcall -%}
|
113
|
-
|
114
|
-
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
|
115
|
-
{% do apply_grants(target_relation, grant_config, should_revoke) %}
|
116
|
-
|
117
|
-
{# Todo: figure out what persist_docs and create_indexes do. #}
|
118
|
-
{%- do persist_docs(target, model) -%}
|
119
|
-
{%- if existing is none
|
120
|
-
or existing.is_view
|
121
|
-
or should_full_refresh() -%}
|
122
|
-
{%- do create_indexes(target) -%}
|
123
|
-
{%- endif %}
|
124
|
-
|
125
|
-
{{ drop_relation_if_exists(new_records) }}
|
126
|
-
|
127
|
-
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
128
|
-
|
129
|
-
{{ return({'relations': [target]}) }}
|
130
|
-
|
131
|
-
{%- endmaterialization %}
|
@@ -1,14 +0,0 @@
|
|
1
|
-
{% macro is_incremental() %}
|
2
|
-
{#- Return True if model is incremental, otherwise False. -#}
|
3
|
-
{% if not execute %}
|
4
|
-
{{ return(False) }}
|
5
|
-
{% else %}
|
6
|
-
{% set relation = adapter.get_relation(this.database,
|
7
|
-
this.schema,
|
8
|
-
this.table) %}
|
9
|
-
{{ return(relation is not none
|
10
|
-
and relation.type == 'table'
|
11
|
-
and model.config.materialized == 'incremental'
|
12
|
-
and not should_full_refresh()) }}
|
13
|
-
{% endif %}
|
14
|
-
{% endmacro %}
|
@@ -1,38 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
|
2
|
-
|
3
|
-
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
|
4
|
-
|
5
|
-
{% if unique_key %}
|
6
|
-
{% if unique_key is sequence and unique_key is not string %}
|
7
|
-
delete from {{ target }}
|
8
|
-
where
|
9
|
-
({{ get_quoted_csv(unique_key) }}) in (
|
10
|
-
select {{ get_quoted_csv(unique_key) }}
|
11
|
-
from {{ source }}
|
12
|
-
)
|
13
|
-
{% if incremental_predicates %}
|
14
|
-
{% for predicate in incremental_predicates %}
|
15
|
-
and {{ predicate }}
|
16
|
-
{% endfor %}
|
17
|
-
{% endif %};
|
18
|
-
{% else %}
|
19
|
-
delete from {{ target }}
|
20
|
-
where (
|
21
|
-
{{ unique_key }}) in (
|
22
|
-
select ({{ unique_key }})
|
23
|
-
from {{ source }}
|
24
|
-
)
|
25
|
-
{%- if incremental_predicates %}
|
26
|
-
{% for predicate in incremental_predicates %}
|
27
|
-
and {{ predicate }}
|
28
|
-
{% endfor %}
|
29
|
-
{%- endif -%};
|
30
|
-
|
31
|
-
{% endif %}
|
32
|
-
{% endif %}
|
33
|
-
|
34
|
-
insert into {{ target }} ({{ dest_cols_csv }})
|
35
|
-
select {{ dest_cols_csv }}
|
36
|
-
from {{ source }}
|
37
|
-
|
38
|
-
{%- endmacro %}
|
@@ -1,90 +0,0 @@
|
|
1
|
-
{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
|
2
|
-
{#
|
3
|
-
Ensure that the value of `on_schema_change` in dbt_project models is valid.
|
4
|
-
Firebolt doesnt currently support `sync_all_columns`, so this is excluded
|
5
|
-
from valid columns.
|
6
|
-
#}
|
7
|
-
{% if on_schema_change == 'sync_all_columns' %}
|
8
|
-
{% do exceptions.raise_compiler_error(
|
9
|
-
'Firebolt does not support the on_schema_change value "sync_all_columns."') %}
|
10
|
-
{{ return(default) }}
|
11
|
-
{% else %}
|
12
|
-
{{ return(on_schema_change) }}
|
13
|
-
{% endif %}
|
14
|
-
{% endmacro %}
|
15
|
-
|
16
|
-
|
17
|
-
{% macro check_for_schema_changes(source_relation, target_relation) %}
|
18
|
-
{#
|
19
|
-
Return a dict = {
|
20
|
-
'schema_changed': schema_changed,
|
21
|
-
'source_not_in_target': source_not_in_target (List[FireboltColumn]),
|
22
|
-
'target_not_in_source': target_not_in_source (List[FireboltColumn]),
|
23
|
-
'source_columns': source_columns (List[FireboltColumn]),
|
24
|
-
'target_columns': target_columns (List[FireboltColumn]),
|
25
|
-
'new_target_types': new_target_types,
|
26
|
-
'common_columns': common_colums
|
27
|
-
}
|
28
|
-
Args:
|
29
|
-
source_relation: dbt Relation
|
30
|
-
target_relation: dbt Relation
|
31
|
-
#}
|
32
|
-
{%- set schema_changed = False -%}
|
33
|
-
{%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}
|
34
|
-
{%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
|
35
|
-
{%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
|
36
|
-
{%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
|
37
|
-
{%- set new_target_types = diff_column_data_types(source_columns, target_columns) -%}
|
38
|
-
{%- set common_columns = intersect_columns(source_columns, target_columns) -%}
|
39
|
-
{% if source_not_in_target != [] %}
|
40
|
-
{%- set schema_changed = True -%}
|
41
|
-
{% elif target_not_in_source != [] or new_target_types != [] %}
|
42
|
-
{%- set schema_changed = True -%}
|
43
|
-
{% elif new_target_types != [] %}
|
44
|
-
{%- set schema_changed = True -%}
|
45
|
-
{% endif %}
|
46
|
-
{% set changes_dict = {
|
47
|
-
'schema_changed': schema_changed,
|
48
|
-
'source_not_in_target': source_not_in_target,
|
49
|
-
'target_not_in_source': target_not_in_source,
|
50
|
-
'source_columns': source_columns,
|
51
|
-
'target_columns': target_columns,
|
52
|
-
'new_target_types': new_target_types,
|
53
|
-
'common_columns': common_columns
|
54
|
-
} %}
|
55
|
-
{{ return(changes_dict) }}
|
56
|
-
{% endmacro %}
|
57
|
-
|
58
|
-
|
59
|
-
{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
|
60
|
-
{#
|
61
|
-
Check for schema changes. Error out if appropriate, else return the list of columns
|
62
|
-
that will be transferred from source to target, i.e. the intersection of the columns.
|
63
|
-
#}
|
64
|
-
{% if on_schema_change == 'sync_all_columns' %}
|
65
|
-
{% do exceptions.raise_compiler_error(
|
66
|
-
'Firebolt does not allow an `on_schema_change` value of `sync_all_columns`.') %}
|
67
|
-
{% endif %}
|
68
|
-
{% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
|
69
|
-
{% if schema_changes_dict['schema_changed'] %}
|
70
|
-
{% if on_schema_change == 'fail' %}
|
71
|
-
{% do exceptions.raise_compiler_error(
|
72
|
-
'A schema change was detected and `on_schema_change` was set to "fail".') %}
|
73
|
-
{% else %}
|
74
|
-
{% set fail_msg %}
|
75
|
-
Schema changes detected on this incremental!
|
76
|
-
Either revert the change or run the model with the --full-refresh flag
|
77
|
-
on the command line to recreate the model with the new schema definition.
|
78
|
-
Running with the --full-refresh flag drops and recreates the database object.
|
79
|
-
|
80
|
-
Additional troubleshooting context:
|
81
|
-
Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}
|
82
|
-
Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}
|
83
|
-
New column types: {{ schema_changes_dict['new_target_types'] }}
|
84
|
-
{% endset %}
|
85
|
-
|
86
|
-
{% do exceptions.raise_compiler_error(fail_msg) %}
|
87
|
-
{% endif %}
|
88
|
-
{% endif %}
|
89
|
-
{{ return(schema_changes_dict['common_columns']) }}
|
90
|
-
{% endmacro %}
|
@@ -1,132 +0,0 @@
|
|
1
|
-
{% macro get_incremental_sql(strategy, source, target, unique_key, dest_columns, incremental_predicates) %}
|
2
|
-
{#
|
3
|
-
Retrieve appropriate SQL for whichever incremental strategy is given.
|
4
|
-
Args:
|
5
|
-
strategy: string, which incremental strategy is in to be used.
|
6
|
-
source: string, table from which queried results will be taken.
|
7
|
-
target: string, table into which results will be inserted.
|
8
|
-
unique_key: string, only as a placeholder for future use
|
9
|
-
dest_columns: List[string] of the names of the columns which data will be
|
10
|
-
inserted into.
|
11
|
-
#}
|
12
|
-
{%- if strategy == 'append' -%}
|
13
|
-
{#- Only insert new records into existing table, relying on user to manage
|
14
|
-
merges. -#}
|
15
|
-
{{ get_insert_only_sql(source, target, dest_columns) }}
|
16
|
-
{%- elif strategy == 'insert_overwrite' -%}
|
17
|
-
{#- Insert new data. If any data is duplicate, drop partition containing
|
18
|
-
previous data and insert new. -#}
|
19
|
-
{{ get_insert_overwrite_sql(source, target, dest_columns) }}
|
20
|
-
{%- elif strategy == 'delete+insert' -%}
|
21
|
-
{% do return(get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates)) %}
|
22
|
-
{%- elif strategy is not none -%}
|
23
|
-
{% do exceptions.raise_compiler_error('Model %s has incremental strategy %s '
|
24
|
-
'specified, but that strategy is not '
|
25
|
-
'supported.' % (target, strategy)) %}
|
26
|
-
{% else %}
|
27
|
-
{{ exceptions.raise_compiler_error('No incremental strategy was specified '
|
28
|
-
'for model %s.' % (target)) }}
|
29
|
-
{%- endif -%}
|
30
|
-
{% endmacro %}
|
31
|
-
|
32
|
-
|
33
|
-
{% macro get_insert_only_sql(source, target, dest_columns) -%}
|
34
|
-
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) %}
|
35
|
-
|
36
|
-
INSERT INTO {{ target }} ({{ dest_cols_csv }})
|
37
|
-
SELECT {{ dest_cols_csv }}
|
38
|
-
FROM {{ source }}
|
39
|
-
{%- endmacro %}
|
40
|
-
|
41
|
-
|
42
|
-
{% macro get_insert_overwrite_sql(source, target, dest_columns) %}
|
43
|
-
{#
|
44
|
-
Compile SQL to drop correct partitions in target and insert from source.
|
45
|
-
Args:
|
46
|
-
source: Relation from which queried results will be taken.
|
47
|
-
target: Relation into which results will be inserted.
|
48
|
-
dest_columns: list of the names of the columns which data will be
|
49
|
-
inserted into.
|
50
|
-
#}
|
51
|
-
{%- set partition_cols = config.get('partition_by') -%}
|
52
|
-
{%- set partition_vals = config.get('partitions') -%}
|
53
|
-
{#- Get values of partition columns for each row that will be inserted from
|
54
|
-
source. For each of those rows we'll drop the partition in the target. Use
|
55
|
-
the partition values provided in the confg block. _If_ no partition values
|
56
|
-
are provided, query the DB to find all partition values in the source table
|
57
|
-
and drop them. To match format of SQL query results, we convert each sublist
|
58
|
-
to a string. -#}
|
59
|
-
{%- if partition_vals -%}
|
60
|
-
{# Partition vals are set in config. #}
|
61
|
-
{{ drop_partitions_sql(target, partition_vals, True) }}
|
62
|
-
{%- else -%} {# No partition values were set in config. #}
|
63
|
-
{%- call statement('get_partition_cols', fetch_result=True) %}
|
64
|
-
|
65
|
-
SELECT
|
66
|
-
{% if partition_cols is iterable and partition_cols is not string -%}
|
67
|
-
DISTINCT {{ partition_cols | join(', ') }}
|
68
|
-
{%- else -%}
|
69
|
-
DISTINCT {{ partition_cols }}
|
70
|
-
{%- endif %}
|
71
|
-
FROM {{ source }}
|
72
|
-
{%- endcall -%}
|
73
|
-
{%- set partition_vals = load_result('get_partition_cols').table.rows -%}
|
74
|
-
{{ drop_partitions_sql(target, partition_vals, partition_cols, False) }}
|
75
|
-
{%- endif -%}
|
76
|
-
{%- set dest_columns = adapter.get_columns_in_relation(target) -%}
|
77
|
-
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
|
78
|
-
|
79
|
-
{{ get_insert_only_sql(source, target, dest_columns) }}
|
80
|
-
{% endmacro %}
|
81
|
-
|
82
|
-
|
83
|
-
{% macro drop_partitions_sql(relation,
|
84
|
-
partition_vals,
|
85
|
-
part_col_names,
|
86
|
-
vals_set_in_config) %}
|
87
|
-
{#
|
88
|
-
Write SQL code to drop each partition in `partition_vals`.
|
89
|
-
Args:
|
90
|
-
relation: a relation whose name will be used for `DROP PARTITION` queries.
|
91
|
-
partition_vals: a list of strings, each of which begins with a '['' and
|
92
|
-
ends with a ']', as such: '[val1, val2]', and each of which represents
|
93
|
-
a partition to be dropped.
|
94
|
-
part_col_names: a list of string, the names of the partition columns.
|
95
|
-
vals_set_in_config: a boolean used to determine how to treat `partition_vals`,
|
96
|
-
whether as a list of strings or as a list of Agate rows.
|
97
|
-
#}
|
98
|
-
{%- for vals in partition_vals -%}
|
99
|
-
{%- set vals -%}
|
100
|
-
{%- if vals is iterable and vals is not string -%}
|
101
|
-
{%- if vals_set_in_config -%}
|
102
|
-
{# `vals` is a list of strings. #}
|
103
|
-
{{ vals | string() | trim() | slice(1, -1) }}
|
104
|
-
{%- else -%}
|
105
|
-
{# `vals` is a list of Agate rows. #}
|
106
|
-
{%- if 1 == (vals | length) -%}
|
107
|
-
{#- There's a weird behavior where, if dbt
|
108
|
-
queries for only a single (text?) field `join()` removes the
|
109
|
-
qoutes on the resulting string. So I have to do a little bit
|
110
|
-
of extra work. -#}
|
111
|
-
'{{ vals | join(', ') }}'
|
112
|
-
{%- else -%}
|
113
|
-
'{{ vals | join("', '") }}'
|
114
|
-
{%- endif -%}
|
115
|
-
{%- endif -%}
|
116
|
-
{%- else -%}
|
117
|
-
{{ vals }}
|
118
|
-
{%- endif -%}
|
119
|
-
{%- endset -%}
|
120
|
-
{%- set vals = vals.strip() -%}
|
121
|
-
{%- if vals.startswith("'[") -%}
|
122
|
-
{#- If a single row is returned, but has multiple values. -#}
|
123
|
-
{%- set vals = vals[2:-2] -%}
|
124
|
-
{%- endif -%}
|
125
|
-
{#- At this point, vals is a simple string of values separated by commas. -#}
|
126
|
-
{%- set col_types = adapter.get_columns_in_relation(relation) -%}
|
127
|
-
{%- set vals = adapter.annotate_date_columns_for_partitions(vals,
|
128
|
-
part_col_names,
|
129
|
-
col_types) %}
|
130
|
-
ALTER TABLE {{relation}} DROP PARTITION {{ vals.strip() }};
|
131
|
-
{%- endfor -%}
|
132
|
-
{% endmacro %}
|
@@ -1,42 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_binding_char() %}
|
2
|
-
{# Override the wildcard character in prepared SQL statements. #}
|
3
|
-
{{ return('?') }}
|
4
|
-
{% endmacro %}
|
5
|
-
|
6
|
-
|
7
|
-
{% macro firebolt__create_csv_table(model, agate_table) %}
|
8
|
-
{%- set column_override = model['config'].get('column_types', {}) -%}
|
9
|
-
{%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
|
10
|
-
{% set sql %}
|
11
|
-
|
12
|
-
CREATE DIMENSION TABLE IF NOT EXISTS {{ this.render() }} (
|
13
|
-
{%- for col_name in agate_table.column_names -%}
|
14
|
-
{%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
|
15
|
-
{%- set type = column_override.get(col_name, inferred_type) -%}
|
16
|
-
{%- set column_name = (col_name | string) -%}
|
17
|
-
{{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }}
|
18
|
-
{%- if not loop.last -%}, {%- endif -%}
|
19
|
-
{%- endfor -%}
|
20
|
-
)
|
21
|
-
{% endset %}
|
22
|
-
{% call statement('_') %}
|
23
|
-
{{ sql }}
|
24
|
-
{%- endcall %}
|
25
|
-
{{ return(sql) }}
|
26
|
-
{% endmacro %}
|
27
|
-
|
28
|
-
|
29
|
-
{# TODO: Make sure this is going to run correctly, or delete it. #}
|
30
|
-
{% macro firebolt__reset_csv_table(model,
|
31
|
-
full_refresh,
|
32
|
-
old_relation,
|
33
|
-
agate_table) %}
|
34
|
-
{% set sql = "" %}
|
35
|
-
{% if full_refresh %}
|
36
|
-
{{ adapter.drop_relation(old_relation) }}
|
37
|
-
{% else %}
|
38
|
-
{{ adapter.truncate_relation(old_relation) }}
|
39
|
-
{% endif %}
|
40
|
-
{% set sql = create_csv_table(model, agate_table) %}
|
41
|
-
{{ return(sql) }}
|
42
|
-
{% endmacro %}
|
@@ -1,19 +0,0 @@
|
|
1
|
-
{% macro firebolt__snapshot_merge_sql(target, source, insert_cols) -%}
|
2
|
-
{%- set insert_cols_csv = insert_cols | join(', ') -%}
|
3
|
-
|
4
|
-
UPDATE {{ target.render() }} AS DBT_INTERNAL_DEST
|
5
|
-
SET dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to
|
6
|
-
FROM {{ source }} AS DBT_INTERNAL_SOURCE
|
7
|
-
WHERE DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id
|
8
|
-
AND DBT_INTERNAL_DEST.dbt_valid_to IS NULL
|
9
|
-
AND DBT_INTERNAL_SOURCE.dbt_change_type IN ('update', 'delete');
|
10
|
-
|
11
|
-
INSERT INTO {{ target.render() }} ({{ insert_cols_csv }})
|
12
|
-
SELECT {{ insert_cols_csv }}
|
13
|
-
FROM {{ source }} AS DBT_INTERNAL_SOURCE
|
14
|
-
WHERE DBT_INTERNAL_SOURCE.dbt_scd_id NOT IN (
|
15
|
-
SELECT dbt_scd_id FROM {{ target.render() }}
|
16
|
-
)
|
17
|
-
AND DBT_INTERNAL_SOURCE.dbt_change_type = 'insert';
|
18
|
-
|
19
|
-
{% endmacro %}
|
@@ -1,40 +0,0 @@
|
|
1
|
-
{% materialization table, adapter='firebolt' %}
|
2
|
-
{# Note that a nearly identical materialization appears in table.sql. #}
|
3
|
-
{%- set identifier = model['alias'] -%} {# alias comes from where? #}
|
4
|
-
|
5
|
-
{# Temporary workaround to issue with adapter.get_relation() until
|
6
|
-
the following are resolved:
|
7
|
-
https://github.com/firebolt-db/dbt-firebolt/issues/11
|
8
|
-
https://github.com/dbt-labs/dbt-core/issues/4187
|
9
|
-
the hack is to always drop the view/table if it already exists rather than
|
10
|
-
checking the db first to see if it exists.
|
11
|
-
#}
|
12
|
-
{%- set target_relation = api.Relation.create(database=database,
|
13
|
-
schema=schema,
|
14
|
-
identifier=identifier,
|
15
|
-
type='table') -%}
|
16
|
-
{%- set target_relation_view = api.Relation.create(database=database,
|
17
|
-
schema=schema,
|
18
|
-
identifier=identifier,
|
19
|
-
type='view') -%}
|
20
|
-
|
21
|
-
{%- set grant_config = config.get('grants') -%}
|
22
|
-
|
23
|
-
{{ run_hooks(pre_hooks) }}
|
24
|
-
|
25
|
-
{% do adapter.drop_relation(target_relation) %}
|
26
|
-
{% do adapter.drop_relation(target_relation_view) %}
|
27
|
-
|
28
|
-
-- build model
|
29
|
-
{% call statement('main') -%}
|
30
|
-
{{ create_table_as(False, target_relation, sql) }}
|
31
|
-
{%- endcall %}
|
32
|
-
{% do create_indexes(target_relation) %}
|
33
|
-
{{ run_hooks(post_hooks) }}
|
34
|
-
|
35
|
-
{% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}
|
36
|
-
{% do apply_grants(target_relation, grant_config, should_revoke) %}
|
37
|
-
|
38
|
-
{% do persist_docs(target_relation, model) %}
|
39
|
-
{{ return({'relations': [target_relation]}) }}
|
40
|
-
{% endmaterialization %}
|
@@ -1,15 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) %}
|
2
|
-
|
3
|
-
SELECT
|
4
|
-
COALESCE(CAST({{ fail_calc }} AS INT), 0) AS failures,
|
5
|
-
CASE WHEN {{ fail_calc }} {{ warn_if }}
|
6
|
-
THEN 'true' ELSE 'false'
|
7
|
-
END AS should_warn,
|
8
|
-
CASE WHEN {{ fail_calc }} {{ error_if }}
|
9
|
-
THEN 'true' ELSE 'false'
|
10
|
-
END AS should_error
|
11
|
-
FROM (
|
12
|
-
{{ main_sql }}
|
13
|
-
{{ "LIMIT " ~ limit if limit is not none }}
|
14
|
-
) dbt_internal_test
|
15
|
-
{%- endmacro %}
|
@@ -1,39 +0,0 @@
|
|
1
|
-
{% materialization view, adapter='firebolt' %}
|
2
|
-
{# Note that a nearly identical materialization appears
|
3
|
-
in table.sql #}
|
4
|
-
{%- set identifier = model['alias'] -%} {# alias comes from where? #}
|
5
|
-
|
6
|
-
{# Temporary workaround to issue with adapter.get_relation() until
|
7
|
-
the following are resolved:
|
8
|
-
https://github.com/firebolt-db/dbt-firebolt/issues/11
|
9
|
-
https://github.com/dbt-labs/dbt-core/issues/4187
|
10
|
-
the hack is to always drop the view/table if it already exists rather than
|
11
|
-
checking the db first to see if it exists.
|
12
|
-
#}
|
13
|
-
{%- set target_relation = api.Relation.create(database=database,
|
14
|
-
schema=schema,
|
15
|
-
identifier=identifier,
|
16
|
-
type='view') -%}
|
17
|
-
{%- set target_relation_table = api.Relation.create(database=database,
|
18
|
-
schema=schema,
|
19
|
-
identifier=identifier,
|
20
|
-
type='table') -%}
|
21
|
-
{%- set grant_config = config.get('grants') -%}
|
22
|
-
|
23
|
-
{{ run_hooks(pre_hooks) }}
|
24
|
-
|
25
|
-
{% do adapter.drop_relation(target_relation) %}
|
26
|
-
{% do adapter.drop_relation(target_relation_table) %}
|
27
|
-
|
28
|
-
-- build model
|
29
|
-
{% call statement('main') -%}
|
30
|
-
{{ create_view_as(target_relation, sql) }}
|
31
|
-
{%- endcall %}
|
32
|
-
{{ run_hooks(post_hooks) }}
|
33
|
-
|
34
|
-
{% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}
|
35
|
-
{% do apply_grants(target_relation, grant_config, should_revoke) %}
|
36
|
-
|
37
|
-
{% do persist_docs(target_relation, model) %}
|
38
|
-
{{ return({'relations': [target_relation]}) }}
|
39
|
-
{% endmaterialization %}
|
@@ -1,11 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_alter_materialized_view_as_sql(
|
2
|
-
relation,
|
3
|
-
configuration_changes,
|
4
|
-
sql,
|
5
|
-
existing_relation,
|
6
|
-
backup_relation,
|
7
|
-
intermediate_relation
|
8
|
-
) %}
|
9
|
-
|
10
|
-
{{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
|
11
|
-
{% endmacro %}
|