dbt-firebolt 1.9.3__py3-none-any.whl → 1.9.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. dbt/adapters/firebolt/__init__.py +1 -1
  2. dbt/include/firebolt/macros/adapters/apply_grants.sql +37 -0
  3. dbt/include/firebolt/macros/adapters/relation.sql +49 -0
  4. dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql +141 -0
  5. dbt/include/firebolt/macros/dbt_external_tables/dropif.sql +6 -0
  6. dbt/include/firebolt/macros/dbt_external_tables/get_external_build_plan.sql +18 -0
  7. dbt/include/firebolt/macros/materializations/clone.sql +3 -0
  8. dbt/include/firebolt/macros/materializations/materialized_view.sql +3 -0
  9. dbt/include/firebolt/macros/materializations/models/incremental/column_helpers.sql +21 -0
  10. dbt/include/firebolt/macros/materializations/models/incremental/incremental.sql +131 -0
  11. dbt/include/firebolt/macros/materializations/models/incremental/is_incremental.sql +14 -0
  12. dbt/include/firebolt/macros/materializations/models/incremental/merge.sql +38 -0
  13. dbt/include/firebolt/macros/materializations/models/incremental/on_schema_change.sql +90 -0
  14. dbt/include/firebolt/macros/materializations/models/incremental/strategies.sql +132 -0
  15. dbt/include/firebolt/macros/materializations/seed.sql +42 -0
  16. dbt/include/firebolt/macros/materializations/snapshot_merge.sql +19 -0
  17. dbt/include/firebolt/macros/materializations/table.sql +40 -0
  18. dbt/include/firebolt/macros/materializations/test.sql +15 -0
  19. dbt/include/firebolt/macros/materializations/view.sql +39 -0
  20. dbt/include/firebolt/macros/relations/materialized_view/alter.sql +11 -0
  21. dbt/include/firebolt/macros/relations/materialized_view/create.sql +3 -0
  22. dbt/include/firebolt/macros/relations/materialized_view/describe.sql +3 -0
  23. dbt/include/firebolt/macros/relations/materialized_view/drop.sql +3 -0
  24. dbt/include/firebolt/macros/relations/materialized_view/refresh.sql +3 -0
  25. dbt/include/firebolt/macros/relations/table/create.sql +78 -0
  26. dbt/include/firebolt/macros/relations/table/drop.sql +3 -0
  27. dbt/include/firebolt/macros/relations/table/rename.sql +6 -0
  28. dbt/include/firebolt/macros/relations/table/replace.sql +3 -0
  29. dbt/include/firebolt/macros/relations/view/create.sql +16 -0
  30. dbt/include/firebolt/macros/relations/view/drop.sql +3 -0
  31. dbt/include/firebolt/macros/relations/view/rename.sql +6 -0
  32. dbt/include/firebolt/macros/relations/view/replace.sql +3 -0
  33. dbt/include/firebolt/macros/utils/array_append.sql +3 -0
  34. dbt/include/firebolt/macros/utils/array_concat.sql +3 -0
  35. dbt/include/firebolt/macros/utils/array_construct.sql +3 -0
  36. dbt/include/firebolt/macros/utils/bool_or.sql +5 -0
  37. dbt/include/firebolt/macros/utils/cast_bool_to_text.sql +7 -0
  38. dbt/include/firebolt/macros/utils/dateadd.sql +9 -0
  39. dbt/include/firebolt/macros/utils/datediff.sql +9 -0
  40. dbt/include/firebolt/macros/utils/except.sql +6 -0
  41. dbt/include/firebolt/macros/utils/intersect.sql +6 -0
  42. dbt/include/firebolt/macros/utils/listagg.sql +27 -0
  43. dbt/include/firebolt/macros/utils/position.sql +3 -0
  44. dbt/include/firebolt/macros/utils/right.sql +12 -0
  45. dbt/include/firebolt/macros/utils/split_part.sql +14 -0
  46. dbt/include/firebolt/macros/utils/timestamps.sql +14 -0
  47. {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/METADATA +1 -1
  48. dbt_firebolt-1.9.4.dist-info/RECORD +61 -0
  49. dbt_firebolt-1.9.3.dist-info/RECORD +0 -16
  50. {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/LICENSE +0 -0
  51. {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/WHEEL +0 -0
  52. {dbt_firebolt-1.9.3.dist-info → dbt_firebolt-1.9.4.dist-info}/top_level.txt +0 -0
@@ -7,7 +7,7 @@ from dbt.adapters.firebolt.connections import (
7
7
  from dbt.adapters.firebolt.impl import FireboltAdapter
8
8
  from dbt.include import firebolt
9
9
 
10
- __version__ = "1.9.3"
10
+ __version__ = "1.9.4"
11
11
 
12
12
  Plugin = AdapterPlugin(
13
13
  adapter=FireboltAdapter, # type: ignore
@@ -0,0 +1,37 @@
1
+ {% macro firebolt__get_show_grant_sql(relation) %}
2
+ {# Usually called from apply_grants. Should not be called directly. #}
3
+ {{ adapter.raise_grant_error() }}
4
+ {% endmacro %}
5
+
6
+
7
+ {%- macro firebolt__get_grant_sql(relation, privilege, grantee) -%}
8
+ {# Usually called from apply_grants. Should not be called directly. #}
9
+ {{ adapter.raise_grant_error() }}
10
+ {%- endmacro -%}
11
+
12
+
13
+ {%- macro firebolt__get_revoke_sql(relation, privilege, grantee) -%}
14
+ {# Usually called from apply_grants. Should not be called directly. #}
15
+ {{ adapter.raise_grant_error() }}
16
+ {%- endmacro -%}
17
+
18
+
19
+ {% macro firebolt__copy_grants() %}
20
+ {{ return(True) }}
21
+ {% endmacro %}
22
+
23
+
24
+ {% macro firebolt__apply_grants(relation, grant_config, should_revoke) %}
25
+ {% if grant_config %}
26
+ {{ adapter.raise_grant_error() }}
27
+ {% endif %}
28
+ {% endmacro %}
29
+
30
+
31
+ {%- macro firebolt__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}
32
+ {# Firebolt does not support DCL statements yet #}
33
+ {% if grant_config %}
34
+ {{ adapter.raise_grant_error() }}
35
+ {% endif %}
36
+ {{ return([]) }}
37
+ {%- endmacro %}
@@ -0,0 +1,49 @@
1
+ {% macro firebolt__rename_relation(source, target) -%}
2
+ {# Create new relation, do CTAS to swap it, remove original relation.
3
+ Must check whether relation is view or table, because process is
4
+ different for the two. #}
5
+ {% set all_relations = (list_relations_without_caching(source)) %}
6
+ {% set tables = adapter.filter_table(all_relations, 'type', 'table') %}
7
+ {% set views = adapter.filter_table(all_relations, 'type', 'view') %}
8
+ {% if (views.rows | length) > 0 %}
9
+ {% call statement('view_definition', fetch_result=True) %}
10
+
11
+ SELECT view_definition FROM information_schema.views
12
+ WHERE table_name = '{{ source.identifier }}'
13
+ {%- endcall %}
14
+ {% if load_result('view_definition')['data'] %}
15
+ {# For some reason this ocassionally returns an empty set.
16
+ In that case do nothing. Don't even ask why I have to jump through
17
+ the stupid hoops to get the value out of the result in the next line. #}
18
+ {% set view_ddl = load_result('view_definition')['data'][0][0] %}
19
+ {% set view_ddl = view_ddl.replace(source.identifier,
20
+ target.identifier) %}
21
+ {% call statement('new_view') %}
22
+
23
+ DROP VIEW {{ source.identifier }};
24
+ {{ view_ddl }};
25
+ {% endcall %}
26
+ {% endif %}
27
+ {% elif (tables.rows | length) > 0 %}
28
+ {# There were no views, so retrieve table. (There must be a table.) #}
29
+ {% set table_type = adapter.filter_table(all_relations, 'table_type', '') %}
30
+ {% call statement('tables') %}
31
+
32
+ CREATE {{ table_type }} TABLE {{ target.identifier }} AS
33
+ SELECT * FROM {{ source.identifier }};
34
+ DROP TABLE {{ source.identifier }};
35
+ {%- endcall %}
36
+ {% endif %}
37
+ {% endmacro %}
38
+
39
+
40
+ {% macro firebolt__drop_relation(relation) -%}
41
+ {#-
42
+ Drop relation. Drop both table and view because relation doesn't always
43
+ have a table_type specified.
44
+ #}
45
+ {% call statement('drop') %}
46
+
47
+ DROP {{ relation.type | upper }} IF EXISTS {{ relation }} CASCADE;
48
+ {% endcall %}
49
+ {% endmacro %}
@@ -0,0 +1,141 @@
1
+ {% macro firebolt__create_external_table(source_node) %}
2
+ {% if source_node.external.strategy == 'copy' %}
3
+ {{ firebolt__create_with_copy_from(source_node) }}
4
+ {% else %}
5
+ {{ firebolt__create_with_external_table(source_node) }}
6
+ {% endif %}
7
+ {% endmacro %}
8
+
9
+ {% macro firebolt__create_with_external_table(source_node) %}
10
+ {%- set external = source_node.external -%}
11
+ {%- if 'partitions' in external -%}
12
+ {%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
13
+ external.partitions) -%}
14
+ {%- else -%}
15
+ {%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
16
+ []) -%}
17
+ {%- endif -%}
18
+ {%- set credentials = external.credentials -%}
19
+ {# Leaving out "IF NOT EXISTS" because this should only be called by
20
+ if no DROP IF is necessary. #}
21
+ CREATE EXTERNAL TABLE {{source(source_node.source_name, source_node.name)}} (
22
+ {%- for column in columns -%}
23
+ {{ column }}
24
+ {{- ',' if not loop.last }}
25
+ {% endfor -%}
26
+ )
27
+ {% if external.url %} URL = '{{external.url}}' {%- endif %}
28
+ {%- if credentials and credentials.internal_role_arn %}
29
+ CREDENTIALS = (AWS_ROLE_ARN = '{{credentials.internal_role_arn}}'
30
+ {%- if credentials.external_role_id %}
31
+ AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}'
32
+ {%- endif -%}
33
+ )
34
+ {% elif credentials and credentials.aws_key_id %}
35
+ CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}'
36
+ AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
37
+ {%- endif %}
38
+ {%- if external.object_pattern -%} OBJECT_PATTERN = '{{external.object_pattern}}' {%- endif %}
39
+ {% if external.object_patterns -%}
40
+ OBJECT_PATTERN =
41
+ {%- for obj in external.object_patterns -%}
42
+ {{ obj }}
43
+ {{- ',' if not loop.last }}
44
+ {%- endfor %}
45
+ {%- endif %}
46
+ {%- if external.compression -%} COMPRESSION = {{external.compression}} {%- endif %}
47
+ TYPE = {{ external.type }}
48
+ {% endmacro %}
49
+
50
+ {% macro firebolt__create_with_copy_from(source_node) %}
51
+ {# COPY FROM is only available in Firebolt 2.0. #}
52
+ {%- set external = source_node.external -%}
53
+ {%- set credentials = external.credentials -%}
54
+ {%- set where_clause = external.where -%}
55
+ {%- set options = external.options -%}
56
+ {%- set csv_options = options.csv_options -%}
57
+ {%- set error_file_credentials = options.error_file_credentials -%}
58
+
59
+ {# There are no partitions, but this formats the columns correctly. #}
60
+ {%- if 'partitions' in external -%}
61
+ {%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
62
+ external.partitions) -%}
63
+ {%- else -%}
64
+ {%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
65
+ []) -%}
66
+ {%- endif -%}
67
+ COPY INTO {{source(source_node.source_name, source_node.name)}}
68
+ {%- if columns and columns | length > 0 %}
69
+ (
70
+ {%- for column in columns -%}
71
+ {{ column.name }}
72
+ {%- if column.default %} DEFAULT {{ column.default }}{% endif %}
73
+ {%- if column.source_column_name %} {{ '$' ~ loop.index0 }}{% endif %}
74
+ {{- ',' if not loop.last }}
75
+ {%- endfor -%}
76
+ )
77
+ {%- endif %}
78
+ FROM '{{external.url}}'
79
+ {%- if options or credentials %}
80
+ WITH
81
+ {%- if options.object_pattern %}
82
+ PATTERN = '{{options.object_pattern}}'
83
+ {%- endif %}
84
+ {%- if options.type %}
85
+ TYPE = {{ options.type }}
86
+ {%- endif %}
87
+ {%- if options.auto_create %}
88
+ AUTO_CREATE = {{ options.auto_create | upper }}
89
+ {%- endif %}
90
+ {%- if options.allow_column_mismatch %}
91
+ ALLOW_COLUMN_MISMATCH = {{ options.allow_column_mismatch | upper }}
92
+ {%- endif %}
93
+ {%- if options.error_file %}
94
+ ERROR_FILE = '{{ options.error_file }}'
95
+ {%- endif %}
96
+ {%- if error_file_credentials %}
97
+ ERROR_FILE_CREDENTIALS = (AWS_KEY_ID = '{{ error_file_credentials.aws_key_id }}' AWS_SECRET_KEY = '{{ error_file_credentials.aws_secret_key }}')
98
+ {%- endif %}
99
+ {%- if options.max_errors_per_file %}
100
+ MAX_ERRORS_PER_FILE = {{ options.max_errors_per_file }}
101
+ {%- endif %}
102
+ {%- if csv_options %}
103
+ {%- if csv_options.header %}
104
+ HEADER = {{ csv_options.header | upper }}
105
+ {%- endif %}
106
+ {%- if csv_options.delimiter %}
107
+ DELIMITER = '{{ csv_options.delimiter }}'
108
+ {%- endif %}
109
+ {%- if csv_options.newline %}
110
+ NEWLINE = '{{ csv_options.newline }}'
111
+ {%- endif %}
112
+ {%- if csv_options.quote %}
113
+ QUOTE = {{ csv_options.quote }}
114
+ {%- endif %}
115
+ {%- if csv_options.escape %}
116
+ ESCAPE = '{{ csv_options.escape }}'
117
+ {%- endif %}
118
+ {%- if csv_options.null_string %}
119
+ NULL_STRING = '{{ csv_options.null_string }}'
120
+ {%- endif %}
121
+ {%- if csv_options.empty_field_as_null %}
122
+ EMPTY_FIELD_AS_NULL = {{ csv_options.empty_field_as_null | upper }}
123
+ {%- endif %}
124
+ {%- if csv_options.skip_blank_lines %}
125
+ SKIP_BLANK_LINES = {{ csv_options.skip_blank_lines | upper }}
126
+ {%- endif %}
127
+ {%- if csv_options.date_format %}
128
+ DATE_FORMAT = '{{ csv_options.date_format }}'
129
+ {%- endif %}
130
+ {%- if csv_options.timestamp_format %}
131
+ TIMESTAMP_FORMAT = '{{ csv_options.timestamp_format }}'
132
+ {%- endif %}
133
+ {%- endif %}
134
+ {%- if credentials %}
135
+ CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
136
+ {%- endif %}
137
+ {%- endif %}
138
+ {%- if where_clause %}
139
+ WHERE {{ where_clause }}
140
+ {%- endif %}
141
+ {% endmacro %}
@@ -0,0 +1,6 @@
1
+ {% macro firebolt__dropif(source_node) %}
2
+ {% set ddl %}
3
+ DROP TABLE IF EXISTS {{ source_node['name'] }}
4
+ {% endset %}
5
+ {{return(ddl)}}
6
+ {% endmacro %}
@@ -0,0 +1,18 @@
1
+ {% macro firebolt__get_external_build_plan(source_node) %}
2
+ {% set build_plan = [] %}
3
+ {% set old_relation = adapter.get_relation(
4
+ database = source_node.database,
5
+ schema = source_node.schema,
6
+ identifier = source_node.identifier
7
+ ) %}
8
+ {# var(variable, Boolean) defaults to Boolean value if variable isnt set.
9
+ Firebolt doesn't do refresh because we don't need to—external tables will always
10
+ pull most recent data from S3, so the default action below is to skip. #}
11
+ {% if old_relation is none or var('ext_full_refresh', false) %}
12
+ {% set build_plan = build_plan + [dropif(source_node),
13
+ create_external_table(source_node)] %}
14
+ {% else %}
15
+ {% set build_plan = dbt_external_tables.refresh_external_table(source_node) %}
16
+ {% endif %}
17
+ {% do return(build_plan) %}
18
+ {% endmacro %}
@@ -0,0 +1,3 @@
1
+ {% macro firebolt__can_clone_table() %}
2
+ {{ return(False) }}
3
+ {% endmacro %}
@@ -0,0 +1,3 @@
1
+ {% macro firebolt__get_materialized_view_configuration_changes(existing_relation, new_config) %}
2
+ {{ exceptions.raise_compiler_error("Firebolt does not support materialized views") }}
3
+ {% endmacro %}
@@ -0,0 +1,21 @@
1
+ {% macro intersect_columns(source_columns, target_columns) %}
2
+ {# Return a List[FireboltColumn] of columns that appear in both source and target.
3
+ Args:
4
+ source_columns: List[FireboltColumn]
5
+ target_columns: List[FireboltColumn]
6
+ #}
7
+ {% set result = [] %}
8
+ {# Note I'm using `column` and not `name` below, as name is a getter
9
+ and column is the actual field. #}
10
+ {% set source_names = source_columns | map(attribute = 'column') | list %}
11
+ {% set target_names = target_columns | map(attribute = 'column') | list %}
12
+ {# Check whether the name attribute exists in the target - this does
13
+ not perform a data type check. This is O(m•n), but cardinality
14
+ of columns is probably low enough that it doesn't matter. #}
15
+ {% for sc in source_columns %}
16
+ {% if sc.name in target_names %}
17
+ {{ result.append(sc) }}
18
+ {% endif %}
19
+ {% endfor %}
20
+ {{ return(result) }}
21
+ {% endmacro %}
@@ -0,0 +1,131 @@
1
+ {% macro dbt_firebolt_get_tmp_relation_type(strategy) %}
2
+ {#
3
+ Determine whether to use a temporary view or a table based on
4
+ specified strategy and config. Users can override the behaviour
5
+ by setting tmp_replation_type. Default is to use a view (more efficient).
6
+
7
+ Arguments:
8
+ strategy: incremental strategy from config. e.g. "append"
9
+ #}
10
+ {%- set tmp_relation_type = config.get('tmp_relation_type') -%}
11
+
12
+ {% if tmp_relation_type == "table" %}
13
+ {{ return("table") }}
14
+ {% elif tmp_relation_type == "view" %}
15
+ {{ return("view") }}
16
+ {% elif strategy in ("default", "append", "insert_overwrite") %}
17
+ {{ return("view") }}
18
+ {% else %}
19
+ {{ return("table") }}
20
+ {% endif %}
21
+ {% endmacro %}
22
+
23
+ {% materialization incremental, adapter='firebolt' -%}
24
+ {#
25
+ Incremental implementation. Actual strategy SQL is determined
26
+ outside this materialization, in strategies.sql.
27
+
28
+ This function works by first creating a view, `new_records`, of type `BaseRelation`,
29
+ then using a CTE defined in the project model to populate `new_records` with
30
+ any records that ought to be inserted into the original table (views aren't currently
31
+ supported), `existing`, using an INSERT. This process is agnostic to the incremental
32
+ strategy actually used: the specific INSERT is defined in `strategies.sql`.
33
+
34
+ Note that all new relations must first be instantiated as BaseRelation objects,
35
+ and that those objects are used to create and query actual relations in the DB.
36
+ Care must be taken to correctly define each BaseRelation as either a view or
37
+ a table, lest subsequent operations on the relations (be the they objects or
38
+ the DB tables the objects abstract) fail.
39
+ #}
40
+
41
+ {% set unique_key = config.get('unique_key') %}
42
+
43
+ {% set grant_config = config.get('grants') %}
44
+
45
+ {%- set strategy = config.get('incremental_strategy') -%}
46
+ {%- if is_incremental() and strategy is none -%}
47
+ {{ log('Model %s is set to incremental, but no incremental strategy is set. '
48
+ 'Defaulting to append' % this, True) }}
49
+ {%- set strategy = 'append' -%}
50
+ {%- endif -%}
51
+ {# In following lines:
52
+
53
+ `target` is just a dbt `BaseRelation` object, not a DB table.
54
+
55
+ We're only retrieving `existing` to check for existence; `load_relation()`
56
+ returns a `BaseRelation` with the dictionary values of any relations that exist
57
+ in dbt's cache that share the identifier of the passed relation. If none
58
+ exist, returns None. Note that this does *not* create a table in the DB.
59
+
60
+ `target` is a relation with all the same fields as `existing`, but guaranteed
61
+ to be an actual `BaseRelation` object. #}
62
+ {%- set target = this.incorporate(type='table') -%}
63
+ {%- set existing = load_relation(this) -%}
64
+ {%- set tmp_relation_type = dbt_firebolt_get_tmp_relation_type(strategy) -%}
65
+ {%- set new_records = make_temp_relation(target) -%}
66
+ {{ drop_relation_if_exists(new_records) }}
67
+ {%- set new_records = new_records.incorporate(type=tmp_relation_type) -%}
68
+ {%- set on_schema_change = incremental_validate_on_schema_change(
69
+ config.get('on_schema_change'),
70
+ default='ignore') -%}
71
+
72
+ -- `BEGIN` happens here:
73
+ {{ run_hooks(pre_hooks, inside_transaction=True) }}
74
+ {%- set partition_by = config.get('partition_by') -%}
75
+ {# First check whether we want to full refresh for existing view or config reasons. #}
76
+ {%- if strategy == 'insert_overwrite' and not partition_by -%}
77
+ {% do exceptions.raise_compiler_error('`insert_overwrite` is specified for model %s, '
78
+ 'but `partition_by` is not. `insert_overwrite` '
79
+ 'requires a partition.' % (target)) %}
80
+ {%- endif -%}
81
+ {%- set do_full_refresh = (should_full_refresh()
82
+ or existing.is_view) %}
83
+ {% if existing is none %}
84
+ {%- set build_sql = create_table_as(False, target, sql) -%}
85
+ {%- elif do_full_refresh -%}
86
+ {{ drop_relation_if_exists(existing) }}
87
+ {%- set build_sql = create_table_as(False, target, sql) -%}
88
+ {%- else -%}
89
+ {# Actually do the incremental query here. #}
90
+ {# Instantiate new objects in dbt's internal list. Have to
91
+ run this query so dbt can query the DB to get the columns in
92
+ new_records. #}
93
+ {%- if tmp_relation_type=='view' -%}
94
+ {%- do run_query(create_view_as(new_records, sql)) -%}
95
+ {%- else -%}
96
+ {%- do run_query(create_table_as(True, new_records, sql)) -%}
97
+ {%- endif -%}
98
+ {# All errors involving schema changes are dealt with in `process_schema_changes`. #}
99
+ {%- set dest_columns = process_schema_changes(on_schema_change,
100
+ new_records,
101
+ existing) -%}
102
+ {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}
103
+ {%- set build_sql = get_incremental_sql(strategy,
104
+ new_records,
105
+ target,
106
+ unique_key,
107
+ dest_columns,
108
+ incremental_predicates) -%}
109
+ {%- endif -%}
110
+ {%- call statement("main") -%}
111
+ {{ build_sql }}
112
+ {%- endcall -%}
113
+
114
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
115
+ {% do apply_grants(target_relation, grant_config, should_revoke) %}
116
+
117
+ {# Todo: figure out what persist_docs and create_indexes do. #}
118
+ {%- do persist_docs(target, model) -%}
119
+ {%- if existing is none
120
+ or existing.is_view
121
+ or should_full_refresh() -%}
122
+ {%- do create_indexes(target) -%}
123
+ {%- endif %}
124
+
125
+ {{ drop_relation_if_exists(new_records) }}
126
+
127
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
128
+
129
+ {{ return({'relations': [target]}) }}
130
+
131
+ {%- endmaterialization %}
@@ -0,0 +1,14 @@
1
+ {% macro is_incremental() %}
2
+ {#- Return True if model is incremental, otherwise False. -#}
3
+ {% if not execute %}
4
+ {{ return(False) }}
5
+ {% else %}
6
+ {% set relation = adapter.get_relation(this.database,
7
+ this.schema,
8
+ this.table) %}
9
+ {{ return(relation is not none
10
+ and relation.type == 'table'
11
+ and model.config.materialized == 'incremental'
12
+ and not should_full_refresh()) }}
13
+ {% endif %}
14
+ {% endmacro %}
@@ -0,0 +1,38 @@
1
+ {% macro firebolt__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
2
+
3
+ {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
4
+
5
+ {% if unique_key %}
6
+ {% if unique_key is sequence and unique_key is not string %}
7
+ delete from {{ target }}
8
+ where
9
+ ({{ get_quoted_csv(unique_key) }}) in (
10
+ select {{ get_quoted_csv(unique_key) }}
11
+ from {{ source }}
12
+ )
13
+ {% if incremental_predicates %}
14
+ {% for predicate in incremental_predicates %}
15
+ and {{ predicate }}
16
+ {% endfor %}
17
+ {% endif %};
18
+ {% else %}
19
+ delete from {{ target }}
20
+ where (
21
+ {{ unique_key }}) in (
22
+ select ({{ unique_key }})
23
+ from {{ source }}
24
+ )
25
+ {%- if incremental_predicates %}
26
+ {% for predicate in incremental_predicates %}
27
+ and {{ predicate }}
28
+ {% endfor %}
29
+ {%- endif -%};
30
+
31
+ {% endif %}
32
+ {% endif %}
33
+
34
+ insert into {{ target }} ({{ dest_cols_csv }})
35
+ select {{ dest_cols_csv }}
36
+ from {{ source }}
37
+
38
+ {%- endmacro %}
@@ -0,0 +1,90 @@
1
+ {% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
2
+ {#
3
+ Ensure that the value of `on_schema_change` in dbt_project models is valid.
4
+ Firebolt doesnt currently support `sync_all_columns`, so this is excluded
5
+ from valid columns.
6
+ #}
7
+ {% if on_schema_change == 'sync_all_columns' %}
8
+ {% do exceptions.raise_compiler_error(
9
+ 'Firebolt does not support the on_schema_change value "sync_all_columns."') %}
10
+ {{ return(default) }}
11
+ {% else %}
12
+ {{ return(on_schema_change) }}
13
+ {% endif %}
14
+ {% endmacro %}
15
+
16
+
17
+ {% macro check_for_schema_changes(source_relation, target_relation) %}
18
+ {#
19
+ Return a dict = {
20
+ 'schema_changed': schema_changed,
21
+ 'source_not_in_target': source_not_in_target (List[FireboltColumn]),
22
+ 'target_not_in_source': target_not_in_source (List[FireboltColumn]),
23
+ 'source_columns': source_columns (List[FireboltColumn]),
24
+ 'target_columns': target_columns (List[FireboltColumn]),
25
+ 'new_target_types': new_target_types,
26
+ 'common_columns': common_colums
27
+ }
28
+ Args:
29
+ source_relation: dbt Relation
30
+ target_relation: dbt Relation
31
+ #}
32
+ {%- set schema_changed = False -%}
33
+ {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}
34
+ {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
35
+ {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
36
+ {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
37
+ {%- set new_target_types = diff_column_data_types(source_columns, target_columns) -%}
38
+ {%- set common_columns = intersect_columns(source_columns, target_columns) -%}
39
+ {% if source_not_in_target != [] %}
40
+ {%- set schema_changed = True -%}
41
+ {% elif target_not_in_source != [] or new_target_types != [] %}
42
+ {%- set schema_changed = True -%}
43
+ {% elif new_target_types != [] %}
44
+ {%- set schema_changed = True -%}
45
+ {% endif %}
46
+ {% set changes_dict = {
47
+ 'schema_changed': schema_changed,
48
+ 'source_not_in_target': source_not_in_target,
49
+ 'target_not_in_source': target_not_in_source,
50
+ 'source_columns': source_columns,
51
+ 'target_columns': target_columns,
52
+ 'new_target_types': new_target_types,
53
+ 'common_columns': common_columns
54
+ } %}
55
+ {{ return(changes_dict) }}
56
+ {% endmacro %}
57
+
58
+
59
+ {% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
60
+ {#
61
+ Check for schema changes. Error out if appropriate, else return the list of columns
62
+ that will be transferred from source to target, i.e. the intersection of the columns.
63
+ #}
64
+ {% if on_schema_change == 'sync_all_columns' %}
65
+ {% do exceptions.raise_compiler_error(
66
+ 'Firebolt does not allow an `on_schema_change` value of `sync_all_columns`.') %}
67
+ {% endif %}
68
+ {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
69
+ {% if schema_changes_dict['schema_changed'] %}
70
+ {% if on_schema_change == 'fail' %}
71
+ {% do exceptions.raise_compiler_error(
72
+ 'A schema change was detected and `on_schema_change` was set to "fail".') %}
73
+ {% else %}
74
+ {% set fail_msg %}
75
+ Schema changes detected on this incremental!
76
+ Either revert the change or run the model with the --full-refresh flag
77
+ on the command line to recreate the model with the new schema definition.
78
+ Running with the --full-refresh flag drops and recreates the database object.
79
+
80
+ Additional troubleshooting context:
81
+ Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}
82
+ Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}
83
+ New column types: {{ schema_changes_dict['new_target_types'] }}
84
+ {% endset %}
85
+
86
+ {% do exceptions.raise_compiler_error(fail_msg) %}
87
+ {% endif %}
88
+ {% endif %}
89
+ {{ return(schema_changes_dict['common_columns']) }}
90
+ {% endmacro %}