dbt-firebolt 1.9.0__py3-none-any.whl → 1.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt/adapters/firebolt/__init__.py +1 -1
- dbt/adapters/firebolt/connections.py +4 -3
- dbt/adapters/firebolt/impl.py +2 -3
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.2.dist-info}/METADATA +3 -3
- dbt_firebolt-1.9.2.dist-info/RECORD +13 -0
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.2.dist-info}/WHEEL +1 -1
- dbt/include/firebolt/dbt_project.yml +0 -6
- dbt/include/firebolt/macros/adapters/apply_grants.sql +0 -37
- dbt/include/firebolt/macros/adapters/relation.sql +0 -49
- dbt/include/firebolt/macros/adapters.sql +0 -197
- dbt/include/firebolt/macros/catalog.sql +0 -103
- dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql +0 -137
- dbt/include/firebolt/macros/dbt_external_tables/dropif.sql +0 -6
- dbt/include/firebolt/macros/dbt_external_tables/get_external_build_plan.sql +0 -18
- dbt/include/firebolt/macros/materializations/clone.sql +0 -3
- dbt/include/firebolt/macros/materializations/materialized_view.sql +0 -3
- dbt/include/firebolt/macros/materializations/models/incremental/column_helpers.sql +0 -21
- dbt/include/firebolt/macros/materializations/models/incremental/incremental.sql +0 -131
- dbt/include/firebolt/macros/materializations/models/incremental/is_incremental.sql +0 -14
- dbt/include/firebolt/macros/materializations/models/incremental/merge.sql +0 -38
- dbt/include/firebolt/macros/materializations/models/incremental/on_schema_change.sql +0 -90
- dbt/include/firebolt/macros/materializations/models/incremental/strategies.sql +0 -132
- dbt/include/firebolt/macros/materializations/seed.sql +0 -42
- dbt/include/firebolt/macros/materializations/snapshot_merge.sql +0 -19
- dbt/include/firebolt/macros/materializations/table.sql +0 -40
- dbt/include/firebolt/macros/materializations/test.sql +0 -15
- dbt/include/firebolt/macros/materializations/view.sql +0 -39
- dbt/include/firebolt/macros/relations/materialized_view/alter.sql +0 -11
- dbt/include/firebolt/macros/relations/materialized_view/create.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/describe.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/refresh.sql +0 -3
- dbt/include/firebolt/macros/relations/table/create.sql +0 -78
- dbt/include/firebolt/macros/relations/table/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/table/rename.sql +0 -6
- dbt/include/firebolt/macros/relations/table/replace.sql +0 -3
- dbt/include/firebolt/macros/relations/view/create.sql +0 -16
- dbt/include/firebolt/macros/relations/view/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/view/rename.sql +0 -6
- dbt/include/firebolt/macros/relations/view/replace.sql +0 -3
- dbt/include/firebolt/macros/utils/array_append.sql +0 -3
- dbt/include/firebolt/macros/utils/array_concat.sql +0 -3
- dbt/include/firebolt/macros/utils/array_construct.sql +0 -3
- dbt/include/firebolt/macros/utils/bool_or.sql +0 -5
- dbt/include/firebolt/macros/utils/cast_bool_to_text.sql +0 -7
- dbt/include/firebolt/macros/utils/dateadd.sql +0 -9
- dbt/include/firebolt/macros/utils/datediff.sql +0 -9
- dbt/include/firebolt/macros/utils/except.sql +0 -6
- dbt/include/firebolt/macros/utils/intersect.sql +0 -6
- dbt/include/firebolt/macros/utils/listagg.sql +0 -27
- dbt/include/firebolt/macros/utils/position.sql +0 -3
- dbt/include/firebolt/macros/utils/right.sql +0 -12
- dbt/include/firebolt/macros/utils/split_part.sql +0 -14
- dbt/include/firebolt/macros/utils/timestamps.sql +0 -14
- dbt_firebolt-1.9.0.dist-info/RECORD +0 -61
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.2.dist-info}/LICENSE +0 -0
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.2.dist-info}/top_level.txt +0 -0
@@ -20,7 +20,7 @@ from dbt_common.exceptions import (
|
|
20
20
|
)
|
21
21
|
from firebolt.client import DEFAULT_API_URL
|
22
22
|
from firebolt.client.auth import Auth, ClientCredentials, UsernamePassword
|
23
|
-
from firebolt.db import ARRAY, DECIMAL
|
23
|
+
from firebolt.db import ARRAY, DECIMAL, ExtendedType
|
24
24
|
from firebolt.db import connect as sdk_connect
|
25
25
|
from firebolt.db.connection import Connection as SDKConnection
|
26
26
|
from firebolt.db.cursor import Cursor
|
@@ -191,8 +191,8 @@ class FireboltConnectionManager(SQLConnectionManager):
|
|
191
191
|
raise NotImplementedError('`cancel` is not implemented for this adapter!')
|
192
192
|
|
193
193
|
@classmethod
|
194
|
-
def data_type_code_to_name(
|
195
|
-
cls, type_code: Union[type,
|
194
|
+
def data_type_code_to_name(
|
195
|
+
cls, type_code: Union[type, ExtendedType] # type: ignore[override] # FIR-29423
|
196
196
|
) -> str:
|
197
197
|
"""
|
198
198
|
Convert a Firebolt data type code to a string representing the data type.
|
@@ -218,6 +218,7 @@ class FireboltConnectionManager(SQLConnectionManager):
|
|
218
218
|
return 'bytea'
|
219
219
|
else:
|
220
220
|
return 'text'
|
221
|
+
return 'text'
|
221
222
|
|
222
223
|
|
223
224
|
def _determine_auth(credentials: FireboltCredentials) -> Auth:
|
dbt/adapters/firebolt/impl.py
CHANGED
@@ -120,13 +120,12 @@ class FireboltIndexConfig(dbtClassMixin):
|
|
120
120
|
'for join indexes.'
|
121
121
|
)
|
122
122
|
if index_config.index_type.upper() == 'AGGREGATING' and not (
|
123
|
-
index_config.
|
123
|
+
index_config.aggregation
|
124
124
|
):
|
125
125
|
raise CompilationError(
|
126
126
|
'Invalid aggregating index definition:\n'
|
127
127
|
f' Got: {index_config}.\n'
|
128
|
-
'
|
129
|
-
'for aggregating indexes.'
|
128
|
+
' aggregation must be specified for aggregating indexes.'
|
130
129
|
)
|
131
130
|
return index_config
|
132
131
|
except ValidationError as exc:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
|
-
Name:
|
3
|
-
Version: 1.9.
|
2
|
+
Name: dbt_firebolt
|
3
|
+
Version: 1.9.2
|
4
4
|
Summary: The Firebolt adapter plugin for dbt (data build tool)
|
5
5
|
Home-page: https://github.com/firebolt-db/dbt-firebolt
|
6
6
|
Author: Firebolt
|
@@ -21,7 +21,7 @@ Description-Content-Type: text/markdown
|
|
21
21
|
License-File: LICENSE
|
22
22
|
Requires-Dist: dbt-adapters <2.0,>=1.0
|
23
23
|
Requires-Dist: dbt-core >=1.8.0
|
24
|
-
Requires-Dist: firebolt-sdk >=1.
|
24
|
+
Requires-Dist: firebolt-sdk >=1.8.1
|
25
25
|
Requires-Dist: pydantic >=0.23
|
26
26
|
Provides-Extra: dev
|
27
27
|
Requires-Dist: allure-pytest ==2.* ; extra == 'dev'
|
@@ -0,0 +1,13 @@
|
|
1
|
+
dbt/adapters/firebolt/__init__.py,sha256=BqeuV6kaWEP3BG901X9rr9ifw8y-qBwHuB2ulefwbHo,411
|
2
|
+
dbt/adapters/firebolt/__version__.py,sha256=zRlZGglif76ZVuWWSjsH_MMPgtVQqmj-SryYJW25FL4,69
|
3
|
+
dbt/adapters/firebolt/column.py,sha256=COo_wjhCFgS3GFcPIPcoq7WAWgzN6DB2XqG-gk51WBc,539
|
4
|
+
dbt/adapters/firebolt/connections.py,sha256=dNdha5dDEYUPhLvlSq8_ftbBA1hiqnUCBukt6wLuZH4,7976
|
5
|
+
dbt/adapters/firebolt/impl.py,sha256=eIqnXzoMoyHq2lp1yMz7O62WJ0apM16H8IssTTwDMiw,15075
|
6
|
+
dbt/adapters/firebolt/relation.py,sha256=Xg3Nrjw3UrF_qwnuGbPT97rSXRiDP1GlIAoBF4b7QnY,1922
|
7
|
+
dbt/adapters/firebolt/relation_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
dbt/include/firebolt/__init__.py,sha256=vBGWeG-dHHkimfnX8axBJ4IgAowFw8xADmo6Auzn2xc,52
|
9
|
+
dbt_firebolt-1.9.2.dist-info/LICENSE,sha256=Nn0EGvW3qmoZpBV_JVM3iPukFf3RiNCIizrWe_2oTHk,11354
|
10
|
+
dbt_firebolt-1.9.2.dist-info/METADATA,sha256=wuRMDg_FlCCBHiC90xZqEJ0OWE27FVJYeUvC83YACss,5237
|
11
|
+
dbt_firebolt-1.9.2.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
12
|
+
dbt_firebolt-1.9.2.dist-info/top_level.txt,sha256=B2YH4he17ajilEWOGCKHbRcEJlCuZKwCcgFcLPntLsE,4
|
13
|
+
dbt_firebolt-1.9.2.dist-info/RECORD,,
|
@@ -1,37 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_show_grant_sql(relation) %}
|
2
|
-
{# Usually called from apply_grants. Should not be called directly. #}
|
3
|
-
{{ adapter.raise_grant_error() }}
|
4
|
-
{% endmacro %}
|
5
|
-
|
6
|
-
|
7
|
-
{%- macro firebolt__get_grant_sql(relation, privilege, grantee) -%}
|
8
|
-
{# Usually called from apply_grants. Should not be called directly. #}
|
9
|
-
{{ adapter.raise_grant_error() }}
|
10
|
-
{%- endmacro -%}
|
11
|
-
|
12
|
-
|
13
|
-
{%- macro firebolt__get_revoke_sql(relation, privilege, grantee) -%}
|
14
|
-
{# Usually called from apply_grants. Should not be called directly. #}
|
15
|
-
{{ adapter.raise_grant_error() }}
|
16
|
-
{%- endmacro -%}
|
17
|
-
|
18
|
-
|
19
|
-
{% macro firebolt__copy_grants() %}
|
20
|
-
{{ return(True) }}
|
21
|
-
{% endmacro %}
|
22
|
-
|
23
|
-
|
24
|
-
{% macro firebolt__apply_grants(relation, grant_config, should_revoke) %}
|
25
|
-
{% if grant_config %}
|
26
|
-
{{ adapter.raise_grant_error() }}
|
27
|
-
{% endif %}
|
28
|
-
{% endmacro %}
|
29
|
-
|
30
|
-
|
31
|
-
{%- macro firebolt__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}
|
32
|
-
{# Firebolt does not support DCL statements yet #}
|
33
|
-
{% if grant_config %}
|
34
|
-
{{ adapter.raise_grant_error() }}
|
35
|
-
{% endif %}
|
36
|
-
{{ return([]) }}
|
37
|
-
{%- endmacro %}
|
@@ -1,49 +0,0 @@
|
|
1
|
-
{% macro firebolt__rename_relation(source, target) -%}
|
2
|
-
{# Create new relation, do CTAS to swap it, remove original relation.
|
3
|
-
Must check whether relation is view or table, because process is
|
4
|
-
different for the two. #}
|
5
|
-
{% set all_relations = (list_relations_without_caching(source)) %}
|
6
|
-
{% set tables = adapter.filter_table(all_relations, 'type', 'table') %}
|
7
|
-
{% set views = adapter.filter_table(all_relations, 'type', 'view') %}
|
8
|
-
{% if (views.rows | length) > 0 %}
|
9
|
-
{% call statement('view_definition', fetch_result=True) %}
|
10
|
-
|
11
|
-
SELECT view_definition FROM information_schema.views
|
12
|
-
WHERE table_name = '{{ source.identifier }}'
|
13
|
-
{%- endcall %}
|
14
|
-
{% if load_result('view_definition')['data'] %}
|
15
|
-
{# For some reason this ocassionally returns an empty set.
|
16
|
-
In that case do nothing. Don't even ask why I have to jump through
|
17
|
-
the stupid hoops to get the value out of the result in the next line. #}
|
18
|
-
{% set view_ddl = load_result('view_definition')['data'][0][0] %}
|
19
|
-
{% set view_ddl = view_ddl.replace(source.identifier,
|
20
|
-
target.identifier) %}
|
21
|
-
{% call statement('new_view') %}
|
22
|
-
|
23
|
-
DROP VIEW {{ source.identifier }};
|
24
|
-
{{ view_ddl }};
|
25
|
-
{% endcall %}
|
26
|
-
{% endif %}
|
27
|
-
{% elif (tables.rows | length) > 0 %}
|
28
|
-
{# There were no views, so retrieve table. (There must be a table.) #}
|
29
|
-
{% set table_type = adapter.filter_table(all_relations, 'table_type', '') %}
|
30
|
-
{% call statement('tables') %}
|
31
|
-
|
32
|
-
CREATE {{ table_type }} TABLE {{ target.identifier }} AS
|
33
|
-
SELECT * FROM {{ source.identifier }};
|
34
|
-
DROP TABLE {{ source.identifier }};
|
35
|
-
{%- endcall %}
|
36
|
-
{% endif %}
|
37
|
-
{% endmacro %}
|
38
|
-
|
39
|
-
|
40
|
-
{% macro firebolt__drop_relation(relation) -%}
|
41
|
-
{#-
|
42
|
-
Drop relation. Drop both table and view because relation doesn't always
|
43
|
-
have a table_type specified.
|
44
|
-
#}
|
45
|
-
{% call statement('drop') %}
|
46
|
-
|
47
|
-
DROP {{ relation.type | upper }} IF EXISTS {{ relation }} CASCADE;
|
48
|
-
{% endcall %}
|
49
|
-
{% endmacro %}
|
@@ -1,197 +0,0 @@
|
|
1
|
-
{% macro firebolt__drop_schema(schema_relation) -%}
|
2
|
-
{# Drop tables and views for schema_relation.
|
3
|
-
Args:
|
4
|
-
schema_relation (dict): Contains values for database and schema.
|
5
|
-
Until schemas are supported this macro will drop all tables and views.
|
6
|
-
#}
|
7
|
-
{% set all_relations = (list_relations_without_caching(schema_relation)) %}
|
8
|
-
|
9
|
-
{% set views = adapter.filter_table(all_relations, 'type', 'view') %}
|
10
|
-
{% set tables = adapter.filter_table(all_relations, 'type', 'table') %}
|
11
|
-
{% do drop_relations_loop(views) %}
|
12
|
-
{% do drop_relations_loop(tables) %}
|
13
|
-
{% endmacro %}
|
14
|
-
|
15
|
-
|
16
|
-
{% macro drop_relations_loop(relations) %}
|
17
|
-
{% for row in relations %}
|
18
|
-
{%- set relation = api.Relation.create(database=target.database,
|
19
|
-
schema=target.schema,
|
20
|
-
identifier=row[1],
|
21
|
-
type=row[3]) -%}
|
22
|
-
{{ adapter.drop_relation(relation) }}
|
23
|
-
{%- endfor %}
|
24
|
-
{% endmacro %}
|
25
|
-
|
26
|
-
|
27
|
-
{% macro firebolt__list_schemas(database) %}
|
28
|
-
{# Return current schema. Name is a misnomer.
|
29
|
-
TODO: Should this actually return all schemas? #}
|
30
|
-
{% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
|
31
|
-
|
32
|
-
SELECT 'public' AS schema
|
33
|
-
{% endcall %}
|
34
|
-
{{ return(load_result('list_schemas').table) }}
|
35
|
-
{% endmacro %}
|
36
|
-
|
37
|
-
|
38
|
-
{% macro firebolt__create_schema(relation) -%}
|
39
|
-
{# stub. Not yet supported in Firebolt. #}
|
40
|
-
{%- call statement('create_schema') %}
|
41
|
-
|
42
|
-
SELECT 'create_schema'
|
43
|
-
{% endcall %}
|
44
|
-
{% endmacro %}
|
45
|
-
|
46
|
-
|
47
|
-
{% macro firebolt__alter_column_type(relation, column_name, new_column_type) -%}
|
48
|
-
{# Stub: alter statements not currently available in Firebolt. #}
|
49
|
-
{% call statement('alter_column_type') %}
|
50
|
-
|
51
|
-
SELECT 'alter_column_type'
|
52
|
-
{% endcall %}
|
53
|
-
{% endmacro %}
|
54
|
-
|
55
|
-
|
56
|
-
{% macro firebolt__check_schema_exists(information_schema, schema) -%}
|
57
|
-
{# Stub. Will be replaced by query later. #}
|
58
|
-
{% call statement('check_schema_exists', fetch_result=True, auto_begin=True) %}
|
59
|
-
|
60
|
-
SELECT 'schema_exists'
|
61
|
-
{% endcall %}
|
62
|
-
{{ return(load_result('check_schema_exists').table) }}
|
63
|
-
{% endmacro %}
|
64
|
-
|
65
|
-
|
66
|
-
{% macro make_create_index_sql(relation,
|
67
|
-
index_name,
|
68
|
-
create_statement,
|
69
|
-
spine_col,
|
70
|
-
other_col) -%}
|
71
|
-
{# Create and return SQL for generating a join or aggregating index.
|
72
|
-
Args:
|
73
|
-
relation (dict):
|
74
|
-
index_name (str): name of the index
|
75
|
-
create_statement (str): either "CREATE JOIN INDEX" or
|
76
|
-
"CREATE AND GENERATE AGGREGATING INDEX"
|
77
|
-
spine_col ([str]):
|
78
|
-
if agg index, key columns
|
79
|
-
if join index, join column
|
80
|
-
other_col ([str]):
|
81
|
-
if agg index, aggregating columns
|
82
|
-
if join index, dimension column
|
83
|
-
#}
|
84
|
-
{{ create_statement }} "{{ index_name }}" ON {{ relation }} (
|
85
|
-
{% if spine_col is iterable and spine_col is not string -%}
|
86
|
-
{{ spine_col | join(', ') }},
|
87
|
-
{% else -%}
|
88
|
-
{{ spine_col }},
|
89
|
-
{% endif -%}
|
90
|
-
{% if other_col is iterable and other_col is not string -%}
|
91
|
-
{{ other_col | join(', ') }}
|
92
|
-
{%- else -%}
|
93
|
-
{{ other_col }}
|
94
|
-
{%- endif -%}
|
95
|
-
);
|
96
|
-
{% endmacro %}
|
97
|
-
|
98
|
-
|
99
|
-
{% macro drop_index(index_name, index_type) -%}
|
100
|
-
{# Drop aggregating or join index. #}
|
101
|
-
{% call statement('drop_index', auto_begin=False) %}
|
102
|
-
|
103
|
-
DROP {{ index_type | upper }} INDEX "{{ index_name }}"
|
104
|
-
{% endcall %}
|
105
|
-
{% endmacro %}
|
106
|
-
|
107
|
-
|
108
|
-
{% macro firebolt__get_create_index_sql(relation, index_dict) -%}
|
109
|
-
{# Return aggregating or join index SQL string. #}
|
110
|
-
{# Parse index inputs and send parsed input to make_create_index_sql #}
|
111
|
-
{%- set index_config = adapter.parse_index(index_dict) -%}
|
112
|
-
{%- set index_name = index_config.render_name(relation) -%}
|
113
|
-
{%- set index_type = index_config.index_type | upper -%}
|
114
|
-
{%- if index_type == "JOIN" -%}
|
115
|
-
{{ make_create_index_sql(relation,
|
116
|
-
index_name,
|
117
|
-
"CREATE JOIN INDEX",
|
118
|
-
index_config.join_columns,
|
119
|
-
index_config.dimension_column) }}
|
120
|
-
{%- elif index_type == "AGGREGATING" -%}
|
121
|
-
{{ make_create_index_sql(relation,
|
122
|
-
index_name,
|
123
|
-
"CREATE AND GENERATE AGGREGATING INDEX",
|
124
|
-
index_config.key_columns,
|
125
|
-
index_config.aggregation) }}
|
126
|
-
{%- endif -%}
|
127
|
-
{%- endmacro %}
|
128
|
-
|
129
|
-
|
130
|
-
{% macro sql_convert_columns_in_relation_firebolt(rows) -%}
|
131
|
-
{% set columns = [] %}
|
132
|
-
{% for row in rows %}
|
133
|
-
{% do columns.append(api.Column(*row)) %}
|
134
|
-
{% endfor %}
|
135
|
-
{{ return(columns) }}
|
136
|
-
{% endmacro %}
|
137
|
-
|
138
|
-
|
139
|
-
{% macro firebolt__get_columns_in_relation(relation) -%}
|
140
|
-
{#-
|
141
|
-
Return column information for table identified by relation as
|
142
|
-
List[FireboltColumn].
|
143
|
-
Args: relation: dbt Relation
|
144
|
-
-#}
|
145
|
-
{% set sql %}
|
146
|
-
SELECT column_name, data_type from information_schema.columns
|
147
|
-
WHERE table_name = '{{ relation.identifier }}'
|
148
|
-
{% endset %}
|
149
|
-
{%- set result = run_query(sql) -%}
|
150
|
-
{% set columns = [] %}
|
151
|
-
{% for row in result %}
|
152
|
-
{% do columns.append(adapter.get_column_class().from_description(row['column_name'],
|
153
|
-
adapter.resolve_special_columns(row['data_type']))) %}
|
154
|
-
{% endfor %}
|
155
|
-
{% do return(columns) %}
|
156
|
-
{% endmacro %}
|
157
|
-
|
158
|
-
|
159
|
-
{% macro firebolt__list_relations_without_caching(relation) %}
|
160
|
-
{# Return all views and tables as agate table.
|
161
|
-
Args:
|
162
|
-
relation (dict): Contains values for database and schema.
|
163
|
-
|
164
|
-
dbt has a relations cache. Using this macro will list all
|
165
|
-
the relations in the current schema using a direct DB query,
|
166
|
-
rather than checking the cache. So the name is a misnomer. Should
|
167
|
-
be list_relations_bypassing_cache or something.
|
168
|
-
#}
|
169
|
-
{% call statement('list_tables_without_caching', fetch_result=True) %}
|
170
|
-
|
171
|
-
SELECT
|
172
|
-
table_catalog AS "database",
|
173
|
-
table_name AS "name",
|
174
|
-
'{{ relation.schema }}' AS "schema",
|
175
|
-
CASE
|
176
|
-
WHEN table_type = 'VIEW' THEN 'view'
|
177
|
-
ELSE 'table'
|
178
|
-
END AS "type"
|
179
|
-
FROM
|
180
|
-
information_schema.tables
|
181
|
-
{% endcall %}
|
182
|
-
{% set info_table = load_result('list_tables_without_caching').table %}
|
183
|
-
{{ return(info_table) }}
|
184
|
-
{% endmacro %}
|
185
|
-
|
186
|
-
{% macro firebolt__truncate_relation(relation) -%}
|
187
|
-
{#
|
188
|
-
Truncate relation. Actual macro is drop_relation in ./adapters/relation.sql.
|
189
|
-
#}
|
190
|
-
|
191
|
-
{# Firebolt doesn't currently support TRUNCATE, so DROP CASCADE.
|
192
|
-
This should only be called from reset_csv_table, where it's followed by
|
193
|
-
`create_csv_table`, so not recreating the table here. To retrieve old code,
|
194
|
-
see commit f9984f6d61b8a1b877bc107b102eeb30eba54f35
|
195
|
-
This will be replaced by `CREATE OR REPLACE`. #}
|
196
|
-
{{ adapter.drop_relation(relation) }}
|
197
|
-
{% endmacro %}
|
@@ -1,103 +0,0 @@
|
|
1
|
-
{# This is for building docs. Right now it's an incomplete description of
|
2
|
-
the columns (for instance, `is_nullable` is missing) but more could be added later. #}
|
3
|
-
|
4
|
-
{% macro firebolt__get_catalog(information_schema, schemas) -%}
|
5
|
-
|
6
|
-
{% set query %}
|
7
|
-
with tables as (
|
8
|
-
{{ firebolt__get_catalog_tables_sql(information_schema) }}
|
9
|
-
{{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }}
|
10
|
-
),
|
11
|
-
columns as (
|
12
|
-
{{ firebolt__get_catalog_columns_sql(information_schema) }}
|
13
|
-
{{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }}
|
14
|
-
)
|
15
|
-
{{ firebolt__get_catalog_results_sql() }}
|
16
|
-
{%- endset -%}
|
17
|
-
|
18
|
-
{{ return(run_query(query)) }}
|
19
|
-
|
20
|
-
{%- endmacro %}
|
21
|
-
|
22
|
-
{% macro firebolt__get_catalog_relations(information_schema, relations) -%}
|
23
|
-
|
24
|
-
{% set query %}
|
25
|
-
with tables as (
|
26
|
-
{{ firebolt__get_catalog_tables_sql(information_schema) }}
|
27
|
-
{{ firebolt__get_catalog_relations_where_clause_sql(relations) }}
|
28
|
-
),
|
29
|
-
columns as (
|
30
|
-
{{ firebolt__get_catalog_columns_sql(information_schema) }}
|
31
|
-
{{ firebolt__get_catalog_relations_where_clause_sql(relations) }}
|
32
|
-
)
|
33
|
-
{{ firebolt__get_catalog_results_sql() }}
|
34
|
-
{%- endset -%}
|
35
|
-
|
36
|
-
{{ return(run_query(query)) }}
|
37
|
-
|
38
|
-
{%- endmacro %}
|
39
|
-
|
40
|
-
|
41
|
-
{% macro firebolt__get_catalog_tables_sql(information_schema) -%}
|
42
|
-
SELECT
|
43
|
-
tbls.table_catalog AS table_database,
|
44
|
-
tbls.table_schema as table_schema,
|
45
|
-
table_type,
|
46
|
-
tbls.table_name as table_name,
|
47
|
-
CASE
|
48
|
-
WHEN table_type = 'VIEW' THEN 'VIEW'
|
49
|
-
ELSE 'TABLE'
|
50
|
-
END AS relation_type
|
51
|
-
FROM
|
52
|
-
information_schema.tables tbls
|
53
|
-
{%- endmacro %}
|
54
|
-
|
55
|
-
|
56
|
-
{% macro firebolt__get_catalog_columns_sql(information_schema) -%}
|
57
|
-
select
|
58
|
-
table_catalog as "table_database",
|
59
|
-
table_schema as "table_schema",
|
60
|
-
table_name as "table_name",
|
61
|
-
column_name as "column_name",
|
62
|
-
ordinal_position as "column_index",
|
63
|
-
data_type as "column_type"
|
64
|
-
from information_schema.columns
|
65
|
-
{%- endmacro %}
|
66
|
-
|
67
|
-
{% macro firebolt__get_catalog_results_sql() -%}
|
68
|
-
SELECT *
|
69
|
-
FROM tables
|
70
|
-
JOIN columns USING ("table_database", "table_schema", "table_name")
|
71
|
-
ORDER BY "column_index"
|
72
|
-
{%- endmacro %}
|
73
|
-
|
74
|
-
|
75
|
-
{% macro firebolt__get_catalog_schemas_where_clause_sql(schemas) -%}
|
76
|
-
WHERE ({%- for schema in schemas -%}
|
77
|
-
UPPER("table_schema") = UPPER('{{ schema }}'){%- if not loop.last %} OR {% endif -%}
|
78
|
-
{%- endfor -%})
|
79
|
-
{%- endmacro %}
|
80
|
-
|
81
|
-
|
82
|
-
{% macro firebolt__get_catalog_relations_where_clause_sql(relations) -%}
|
83
|
-
WHERE (
|
84
|
-
{%- for relation in relations -%}
|
85
|
-
{% if relation.schema and relation.identifier %}
|
86
|
-
(
|
87
|
-
UPPER("table_schema") = UPPER('{{ relation.schema }}')
|
88
|
-
AND UPPER("table_name") = UPPER('{{ relation.identifier }}')
|
89
|
-
)
|
90
|
-
{% elif relation.schema %}
|
91
|
-
(
|
92
|
-
UPPER("table_schema") = UPPER('{{ relation.schema }}')
|
93
|
-
)
|
94
|
-
{% else %}
|
95
|
-
{% do exceptions.raise_compiler_error(
|
96
|
-
'`get_catalog_relations` requires a list of relations, each with a schema'
|
97
|
-
) %}
|
98
|
-
{% endif %}
|
99
|
-
|
100
|
-
{%- if not loop.last %} OR {% endif -%}
|
101
|
-
{%- endfor -%}
|
102
|
-
)
|
103
|
-
{%- endmacro %}
|
@@ -1,137 +0,0 @@
|
|
1
|
-
{% macro firebolt__create_external_table(source_node) %}
|
2
|
-
{% if source_node.external.strategy == 'copy' %}
|
3
|
-
{{ firebolt__create_with_copy_from(source_node) }}
|
4
|
-
{% else %}
|
5
|
-
{{ firebolt__create_with_external_table(source_node) }}
|
6
|
-
{% endif %}
|
7
|
-
{% endmacro %}
|
8
|
-
|
9
|
-
{% macro firebolt__create_with_external_table(source_node) %}
|
10
|
-
{%- set external = source_node.external -%}
|
11
|
-
{%- if 'partitions' in external -%}
|
12
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
13
|
-
external.partitions) -%}
|
14
|
-
{%- else -%}
|
15
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
16
|
-
[]) -%}
|
17
|
-
{%- endif -%}
|
18
|
-
{%- set credentials = external.credentials -%}
|
19
|
-
{# Leaving out "IF NOT EXISTS" because this should only be called by
|
20
|
-
if no DROP IF is necessary. #}
|
21
|
-
CREATE EXTERNAL TABLE {{source(source_node.source_name, source_node.name)}} (
|
22
|
-
{%- for column in columns -%}
|
23
|
-
{{ column }}
|
24
|
-
{{- ',' if not loop.last }}
|
25
|
-
{% endfor -%}
|
26
|
-
)
|
27
|
-
{% if external.url %} URL = '{{external.url}}' {%- endif %}
|
28
|
-
{%- if credentials and credentials.internal_role_arn %}
|
29
|
-
CREDENTIALS = (AWS_ROLE_ARN = '{{credentials.internal_role_arn}}'
|
30
|
-
{%- if credentials.external_role_id %}
|
31
|
-
AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}'
|
32
|
-
{%- endif -%}
|
33
|
-
)
|
34
|
-
{% elif credentials and credentials.aws_key_id %}
|
35
|
-
CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}'
|
36
|
-
AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
|
37
|
-
{%- endif %}
|
38
|
-
{%- if external.object_pattern -%} OBJECT_PATTERN = '{{external.object_pattern}}' {%- endif %}
|
39
|
-
{% if external.object_patterns -%}
|
40
|
-
OBJECT_PATTERN =
|
41
|
-
{%- for obj in external.object_patterns -%}
|
42
|
-
{{ obj }}
|
43
|
-
{{- ',' if not loop.last }}
|
44
|
-
{%- endfor %}
|
45
|
-
{%- endif %}
|
46
|
-
{%- if external.compression -%} COMPRESSION = {{external.compression}} {%- endif %}
|
47
|
-
TYPE = {{ external.type }}
|
48
|
-
{% endmacro %}
|
49
|
-
|
50
|
-
{% macro firebolt__create_with_copy_from(source_node) %}
|
51
|
-
{# COPY FROM is only available in Firebolt 2.0. #}
|
52
|
-
{%- set external = source_node.external -%}
|
53
|
-
{%- set credentials = external.credentials -%}
|
54
|
-
{%- set options = external.options -%}
|
55
|
-
{%- set csv_options = options.csv_options -%}
|
56
|
-
{%- set error_file_credentials = options.error_file_credentials -%}
|
57
|
-
|
58
|
-
{# There are no partitions, but this formats the columns correctly. #}
|
59
|
-
{%- if 'partitions' in external -%}
|
60
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
61
|
-
external.partitions) -%}
|
62
|
-
{%- else -%}
|
63
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
64
|
-
[]) -%}
|
65
|
-
{%- endif -%}
|
66
|
-
COPY INTO {{source(source_node.source_name, source_node.name)}}
|
67
|
-
{%- if columns and columns | length > 0 %}
|
68
|
-
(
|
69
|
-
{%- for column in columns -%}
|
70
|
-
{{ column.name }}
|
71
|
-
{%- if column.default %} DEFAULT {{ column.default }}{% endif %}
|
72
|
-
{%- if column.source_column_name %} {{ '$' ~ loop.index0 }}{% endif %}
|
73
|
-
{{- ',' if not loop.last }}
|
74
|
-
{%- endfor -%}
|
75
|
-
)
|
76
|
-
{%- endif %}
|
77
|
-
FROM '{{external.url}}'
|
78
|
-
{%- if options %}
|
79
|
-
WITH
|
80
|
-
{%- if options.object_pattern %}
|
81
|
-
PATTERN = '{{options.object_pattern}}'
|
82
|
-
{%- endif %}
|
83
|
-
{%- if options.type %}
|
84
|
-
TYPE = {{ options.type }}
|
85
|
-
{%- endif %}
|
86
|
-
{%- if options.auto_create %}
|
87
|
-
AUTO_CREATE = {{ options.auto_create | upper }}
|
88
|
-
{%- endif %}
|
89
|
-
{%- if options.allow_column_mismatch %}
|
90
|
-
ALLOW_COLUMN_MISMATCH = {{ options.allow_column_mismatch | upper }}
|
91
|
-
{%- endif %}
|
92
|
-
{%- if options.error_file %}
|
93
|
-
ERROR_FILE = '{{ options.error_file }}'
|
94
|
-
{%- endif %}
|
95
|
-
{%- if error_file_credentials %}
|
96
|
-
ERROR_FILE_CREDENTIALS = (AWS_KEY_ID = '{{ error_file_credentials.aws_key_id }}' AWS_SECRET_KEY = '{{ error_file_credentials.aws_secret_key }}')
|
97
|
-
{%- endif %}
|
98
|
-
{%- if options.max_errors_per_file %}
|
99
|
-
MAX_ERRORS_PER_FILE = {{ options.max_errors_per_file }}
|
100
|
-
{%- endif %}
|
101
|
-
{%- if csv_options %}
|
102
|
-
{%- if csv_options.header %}
|
103
|
-
HEADER = {{ csv_options.header | upper }}
|
104
|
-
{%- endif %}
|
105
|
-
{%- if csv_options.delimiter %}
|
106
|
-
DELIMITER = '{{ csv_options.delimiter }}'
|
107
|
-
{%- endif %}
|
108
|
-
{%- if csv_options.newline %}
|
109
|
-
NEWLINE = '{{ csv_options.newline }}'
|
110
|
-
{%- endif %}
|
111
|
-
{%- if csv_options.quote %}
|
112
|
-
QUOTE = {{ csv_options.quote }}
|
113
|
-
{%- endif %}
|
114
|
-
{%- if csv_options.escape %}
|
115
|
-
ESCAPE = '{{ csv_options.escape }}'
|
116
|
-
{%- endif %}
|
117
|
-
{%- if csv_options.null_string %}
|
118
|
-
NULL_STRING = '{{ csv_options.null_string }}'
|
119
|
-
{%- endif %}
|
120
|
-
{%- if csv_options.empty_field_as_null %}
|
121
|
-
EMPTY_FIELD_AS_NULL = {{ csv_options.empty_field_as_null | upper }}
|
122
|
-
{%- endif %}
|
123
|
-
{%- if csv_options.skip_blank_lines %}
|
124
|
-
SKIP_BLANK_LINES = {{ csv_options.skip_blank_lines | upper }}
|
125
|
-
{%- endif %}
|
126
|
-
{%- if csv_options.date_format %}
|
127
|
-
DATE_FORMAT = '{{ csv_options.date_format }}'
|
128
|
-
{%- endif %}
|
129
|
-
{%- if csv_options.timestamp_format %}
|
130
|
-
TIMESTAMP_FORMAT = '{{ csv_options.timestamp_format }}'
|
131
|
-
{%- endif %}
|
132
|
-
{%- endif %}
|
133
|
-
{%- endif %}
|
134
|
-
{%- if credentials %}
|
135
|
-
CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
|
136
|
-
{%- endif %}
|
137
|
-
{% endmacro %}
|
@@ -1,18 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_external_build_plan(source_node) %}
|
2
|
-
{% set build_plan = [] %}
|
3
|
-
{% set old_relation = adapter.get_relation(
|
4
|
-
database = source_node.database,
|
5
|
-
schema = source_node.schema,
|
6
|
-
identifier = source_node.identifier
|
7
|
-
) %}
|
8
|
-
{# var(variable, Boolean) defaults to Boolean value if variable isnt set.
|
9
|
-
Firebolt doesn't do refresh because we don't need to—external tables will always
|
10
|
-
pull most recent data from S3, so the default action below is to skip. #}
|
11
|
-
{% if old_relation is none or var('ext_full_refresh', false) %}
|
12
|
-
{% set build_plan = build_plan + [dropif(source_node),
|
13
|
-
create_external_table(source_node)] %}
|
14
|
-
{% else %}
|
15
|
-
{% set build_plan = dbt_external_tables.refresh_external_table(source_node) %}
|
16
|
-
{% endif %}
|
17
|
-
{% do return(build_plan) %}
|
18
|
-
{% endmacro %}
|