dbt-firebolt 1.9.0__py3-none-any.whl → 1.9.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- dbt/adapters/firebolt/__init__.py +1 -1
- dbt/adapters/firebolt/connections.py +4 -2
- dbt/adapters/firebolt/impl.py +2 -3
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/METADATA +2 -2
- dbt_firebolt-1.9.1.dist-info/RECORD +13 -0
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/WHEEL +1 -1
- dbt/include/firebolt/dbt_project.yml +0 -6
- dbt/include/firebolt/macros/adapters/apply_grants.sql +0 -37
- dbt/include/firebolt/macros/adapters/relation.sql +0 -49
- dbt/include/firebolt/macros/adapters.sql +0 -197
- dbt/include/firebolt/macros/catalog.sql +0 -103
- dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql +0 -137
- dbt/include/firebolt/macros/dbt_external_tables/dropif.sql +0 -6
- dbt/include/firebolt/macros/dbt_external_tables/get_external_build_plan.sql +0 -18
- dbt/include/firebolt/macros/materializations/clone.sql +0 -3
- dbt/include/firebolt/macros/materializations/materialized_view.sql +0 -3
- dbt/include/firebolt/macros/materializations/models/incremental/column_helpers.sql +0 -21
- dbt/include/firebolt/macros/materializations/models/incremental/incremental.sql +0 -131
- dbt/include/firebolt/macros/materializations/models/incremental/is_incremental.sql +0 -14
- dbt/include/firebolt/macros/materializations/models/incremental/merge.sql +0 -38
- dbt/include/firebolt/macros/materializations/models/incremental/on_schema_change.sql +0 -90
- dbt/include/firebolt/macros/materializations/models/incremental/strategies.sql +0 -132
- dbt/include/firebolt/macros/materializations/seed.sql +0 -42
- dbt/include/firebolt/macros/materializations/snapshot_merge.sql +0 -19
- dbt/include/firebolt/macros/materializations/table.sql +0 -40
- dbt/include/firebolt/macros/materializations/test.sql +0 -15
- dbt/include/firebolt/macros/materializations/view.sql +0 -39
- dbt/include/firebolt/macros/relations/materialized_view/alter.sql +0 -11
- dbt/include/firebolt/macros/relations/materialized_view/create.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/describe.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/materialized_view/refresh.sql +0 -3
- dbt/include/firebolt/macros/relations/table/create.sql +0 -78
- dbt/include/firebolt/macros/relations/table/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/table/rename.sql +0 -6
- dbt/include/firebolt/macros/relations/table/replace.sql +0 -3
- dbt/include/firebolt/macros/relations/view/create.sql +0 -16
- dbt/include/firebolt/macros/relations/view/drop.sql +0 -3
- dbt/include/firebolt/macros/relations/view/rename.sql +0 -6
- dbt/include/firebolt/macros/relations/view/replace.sql +0 -3
- dbt/include/firebolt/macros/utils/array_append.sql +0 -3
- dbt/include/firebolt/macros/utils/array_concat.sql +0 -3
- dbt/include/firebolt/macros/utils/array_construct.sql +0 -3
- dbt/include/firebolt/macros/utils/bool_or.sql +0 -5
- dbt/include/firebolt/macros/utils/cast_bool_to_text.sql +0 -7
- dbt/include/firebolt/macros/utils/dateadd.sql +0 -9
- dbt/include/firebolt/macros/utils/datediff.sql +0 -9
- dbt/include/firebolt/macros/utils/except.sql +0 -6
- dbt/include/firebolt/macros/utils/intersect.sql +0 -6
- dbt/include/firebolt/macros/utils/listagg.sql +0 -27
- dbt/include/firebolt/macros/utils/position.sql +0 -3
- dbt/include/firebolt/macros/utils/right.sql +0 -12
- dbt/include/firebolt/macros/utils/split_part.sql +0 -14
- dbt/include/firebolt/macros/utils/timestamps.sql +0 -14
- dbt_firebolt-1.9.0.dist-info/RECORD +0 -61
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/LICENSE +0 -0
- {dbt_firebolt-1.9.0.dist-info → dbt_firebolt-1.9.1.dist-info}/top_level.txt +0 -0
@@ -20,6 +20,7 @@ from dbt_common.exceptions import (
|
|
20
20
|
)
|
21
21
|
from firebolt.client import DEFAULT_API_URL
|
22
22
|
from firebolt.client.auth import Auth, ClientCredentials, UsernamePassword
|
23
|
+
from firebolt.common._types import ExtendedType
|
23
24
|
from firebolt.db import ARRAY, DECIMAL
|
24
25
|
from firebolt.db import connect as sdk_connect
|
25
26
|
from firebolt.db.connection import Connection as SDKConnection
|
@@ -191,8 +192,8 @@ class FireboltConnectionManager(SQLConnectionManager):
|
|
191
192
|
raise NotImplementedError('`cancel` is not implemented for this adapter!')
|
192
193
|
|
193
194
|
@classmethod
|
194
|
-
def data_type_code_to_name(
|
195
|
-
cls, type_code: Union[type,
|
195
|
+
def data_type_code_to_name(
|
196
|
+
cls, type_code: Union[type, ExtendedType] # type: ignore[override] # FIR-29423
|
196
197
|
) -> str:
|
197
198
|
"""
|
198
199
|
Convert a Firebolt data type code to a string representing the data type.
|
@@ -218,6 +219,7 @@ class FireboltConnectionManager(SQLConnectionManager):
|
|
218
219
|
return 'bytea'
|
219
220
|
else:
|
220
221
|
return 'text'
|
222
|
+
return 'text'
|
221
223
|
|
222
224
|
|
223
225
|
def _determine_auth(credentials: FireboltCredentials) -> Auth:
|
dbt/adapters/firebolt/impl.py
CHANGED
@@ -120,13 +120,12 @@ class FireboltIndexConfig(dbtClassMixin):
|
|
120
120
|
'for join indexes.'
|
121
121
|
)
|
122
122
|
if index_config.index_type.upper() == 'AGGREGATING' and not (
|
123
|
-
index_config.
|
123
|
+
index_config.aggregation
|
124
124
|
):
|
125
125
|
raise CompilationError(
|
126
126
|
'Invalid aggregating index definition:\n'
|
127
127
|
f' Got: {index_config}.\n'
|
128
|
-
'
|
129
|
-
'for aggregating indexes.'
|
128
|
+
' aggregation must be specified for aggregating indexes.'
|
130
129
|
)
|
131
130
|
return index_config
|
132
131
|
except ValidationError as exc:
|
@@ -0,0 +1,13 @@
|
|
1
|
+
dbt/adapters/firebolt/__init__.py,sha256=jTWhFz0VsnTZFi4djXBOhAHvUJaeQvrHDSiVpDTfKM4,411
|
2
|
+
dbt/adapters/firebolt/__version__.py,sha256=zRlZGglif76ZVuWWSjsH_MMPgtVQqmj-SryYJW25FL4,69
|
3
|
+
dbt/adapters/firebolt/column.py,sha256=COo_wjhCFgS3GFcPIPcoq7WAWgzN6DB2XqG-gk51WBc,539
|
4
|
+
dbt/adapters/firebolt/connections.py,sha256=-R97hfke32n4j0cTUwd4A5xAwD8PqXVgnG7BPetQKcg,8010
|
5
|
+
dbt/adapters/firebolt/impl.py,sha256=eIqnXzoMoyHq2lp1yMz7O62WJ0apM16H8IssTTwDMiw,15075
|
6
|
+
dbt/adapters/firebolt/relation.py,sha256=Xg3Nrjw3UrF_qwnuGbPT97rSXRiDP1GlIAoBF4b7QnY,1922
|
7
|
+
dbt/adapters/firebolt/relation_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
dbt/include/firebolt/__init__.py,sha256=vBGWeG-dHHkimfnX8axBJ4IgAowFw8xADmo6Auzn2xc,52
|
9
|
+
dbt_firebolt-1.9.1.dist-info/LICENSE,sha256=Nn0EGvW3qmoZpBV_JVM3iPukFf3RiNCIizrWe_2oTHk,11354
|
10
|
+
dbt_firebolt-1.9.1.dist-info/METADATA,sha256=Vay4PDqyfZmqmUUuWUA9A4ChEngOmwZ-o5e-M6DLkYk,5237
|
11
|
+
dbt_firebolt-1.9.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
12
|
+
dbt_firebolt-1.9.1.dist-info/top_level.txt,sha256=B2YH4he17ajilEWOGCKHbRcEJlCuZKwCcgFcLPntLsE,4
|
13
|
+
dbt_firebolt-1.9.1.dist-info/RECORD,,
|
@@ -1,37 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_show_grant_sql(relation) %}
|
2
|
-
{# Usually called from apply_grants. Should not be called directly. #}
|
3
|
-
{{ adapter.raise_grant_error() }}
|
4
|
-
{% endmacro %}
|
5
|
-
|
6
|
-
|
7
|
-
{%- macro firebolt__get_grant_sql(relation, privilege, grantee) -%}
|
8
|
-
{# Usually called from apply_grants. Should not be called directly. #}
|
9
|
-
{{ adapter.raise_grant_error() }}
|
10
|
-
{%- endmacro -%}
|
11
|
-
|
12
|
-
|
13
|
-
{%- macro firebolt__get_revoke_sql(relation, privilege, grantee) -%}
|
14
|
-
{# Usually called from apply_grants. Should not be called directly. #}
|
15
|
-
{{ adapter.raise_grant_error() }}
|
16
|
-
{%- endmacro -%}
|
17
|
-
|
18
|
-
|
19
|
-
{% macro firebolt__copy_grants() %}
|
20
|
-
{{ return(True) }}
|
21
|
-
{% endmacro %}
|
22
|
-
|
23
|
-
|
24
|
-
{% macro firebolt__apply_grants(relation, grant_config, should_revoke) %}
|
25
|
-
{% if grant_config %}
|
26
|
-
{{ adapter.raise_grant_error() }}
|
27
|
-
{% endif %}
|
28
|
-
{% endmacro %}
|
29
|
-
|
30
|
-
|
31
|
-
{%- macro firebolt__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}
|
32
|
-
{# Firebolt does not support DCL statements yet #}
|
33
|
-
{% if grant_config %}
|
34
|
-
{{ adapter.raise_grant_error() }}
|
35
|
-
{% endif %}
|
36
|
-
{{ return([]) }}
|
37
|
-
{%- endmacro %}
|
@@ -1,49 +0,0 @@
|
|
1
|
-
{% macro firebolt__rename_relation(source, target) -%}
|
2
|
-
{# Create new relation, do CTAS to swap it, remove original relation.
|
3
|
-
Must check whether relation is view or table, because process is
|
4
|
-
different for the two. #}
|
5
|
-
{% set all_relations = (list_relations_without_caching(source)) %}
|
6
|
-
{% set tables = adapter.filter_table(all_relations, 'type', 'table') %}
|
7
|
-
{% set views = adapter.filter_table(all_relations, 'type', 'view') %}
|
8
|
-
{% if (views.rows | length) > 0 %}
|
9
|
-
{% call statement('view_definition', fetch_result=True) %}
|
10
|
-
|
11
|
-
SELECT view_definition FROM information_schema.views
|
12
|
-
WHERE table_name = '{{ source.identifier }}'
|
13
|
-
{%- endcall %}
|
14
|
-
{% if load_result('view_definition')['data'] %}
|
15
|
-
{# For some reason this ocassionally returns an empty set.
|
16
|
-
In that case do nothing. Don't even ask why I have to jump through
|
17
|
-
the stupid hoops to get the value out of the result in the next line. #}
|
18
|
-
{% set view_ddl = load_result('view_definition')['data'][0][0] %}
|
19
|
-
{% set view_ddl = view_ddl.replace(source.identifier,
|
20
|
-
target.identifier) %}
|
21
|
-
{% call statement('new_view') %}
|
22
|
-
|
23
|
-
DROP VIEW {{ source.identifier }};
|
24
|
-
{{ view_ddl }};
|
25
|
-
{% endcall %}
|
26
|
-
{% endif %}
|
27
|
-
{% elif (tables.rows | length) > 0 %}
|
28
|
-
{# There were no views, so retrieve table. (There must be a table.) #}
|
29
|
-
{% set table_type = adapter.filter_table(all_relations, 'table_type', '') %}
|
30
|
-
{% call statement('tables') %}
|
31
|
-
|
32
|
-
CREATE {{ table_type }} TABLE {{ target.identifier }} AS
|
33
|
-
SELECT * FROM {{ source.identifier }};
|
34
|
-
DROP TABLE {{ source.identifier }};
|
35
|
-
{%- endcall %}
|
36
|
-
{% endif %}
|
37
|
-
{% endmacro %}
|
38
|
-
|
39
|
-
|
40
|
-
{% macro firebolt__drop_relation(relation) -%}
|
41
|
-
{#-
|
42
|
-
Drop relation. Drop both table and view because relation doesn't always
|
43
|
-
have a table_type specified.
|
44
|
-
#}
|
45
|
-
{% call statement('drop') %}
|
46
|
-
|
47
|
-
DROP {{ relation.type | upper }} IF EXISTS {{ relation }} CASCADE;
|
48
|
-
{% endcall %}
|
49
|
-
{% endmacro %}
|
@@ -1,197 +0,0 @@
|
|
1
|
-
{% macro firebolt__drop_schema(schema_relation) -%}
|
2
|
-
{# Drop tables and views for schema_relation.
|
3
|
-
Args:
|
4
|
-
schema_relation (dict): Contains values for database and schema.
|
5
|
-
Until schemas are supported this macro will drop all tables and views.
|
6
|
-
#}
|
7
|
-
{% set all_relations = (list_relations_without_caching(schema_relation)) %}
|
8
|
-
|
9
|
-
{% set views = adapter.filter_table(all_relations, 'type', 'view') %}
|
10
|
-
{% set tables = adapter.filter_table(all_relations, 'type', 'table') %}
|
11
|
-
{% do drop_relations_loop(views) %}
|
12
|
-
{% do drop_relations_loop(tables) %}
|
13
|
-
{% endmacro %}
|
14
|
-
|
15
|
-
|
16
|
-
{% macro drop_relations_loop(relations) %}
|
17
|
-
{% for row in relations %}
|
18
|
-
{%- set relation = api.Relation.create(database=target.database,
|
19
|
-
schema=target.schema,
|
20
|
-
identifier=row[1],
|
21
|
-
type=row[3]) -%}
|
22
|
-
{{ adapter.drop_relation(relation) }}
|
23
|
-
{%- endfor %}
|
24
|
-
{% endmacro %}
|
25
|
-
|
26
|
-
|
27
|
-
{% macro firebolt__list_schemas(database) %}
|
28
|
-
{# Return current schema. Name is a misnomer.
|
29
|
-
TODO: Should this actually return all schemas? #}
|
30
|
-
{% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
|
31
|
-
|
32
|
-
SELECT 'public' AS schema
|
33
|
-
{% endcall %}
|
34
|
-
{{ return(load_result('list_schemas').table) }}
|
35
|
-
{% endmacro %}
|
36
|
-
|
37
|
-
|
38
|
-
{% macro firebolt__create_schema(relation) -%}
|
39
|
-
{# stub. Not yet supported in Firebolt. #}
|
40
|
-
{%- call statement('create_schema') %}
|
41
|
-
|
42
|
-
SELECT 'create_schema'
|
43
|
-
{% endcall %}
|
44
|
-
{% endmacro %}
|
45
|
-
|
46
|
-
|
47
|
-
{% macro firebolt__alter_column_type(relation, column_name, new_column_type) -%}
|
48
|
-
{# Stub: alter statements not currently available in Firebolt. #}
|
49
|
-
{% call statement('alter_column_type') %}
|
50
|
-
|
51
|
-
SELECT 'alter_column_type'
|
52
|
-
{% endcall %}
|
53
|
-
{% endmacro %}
|
54
|
-
|
55
|
-
|
56
|
-
{% macro firebolt__check_schema_exists(information_schema, schema) -%}
|
57
|
-
{# Stub. Will be replaced by query later. #}
|
58
|
-
{% call statement('check_schema_exists', fetch_result=True, auto_begin=True) %}
|
59
|
-
|
60
|
-
SELECT 'schema_exists'
|
61
|
-
{% endcall %}
|
62
|
-
{{ return(load_result('check_schema_exists').table) }}
|
63
|
-
{% endmacro %}
|
64
|
-
|
65
|
-
|
66
|
-
{% macro make_create_index_sql(relation,
|
67
|
-
index_name,
|
68
|
-
create_statement,
|
69
|
-
spine_col,
|
70
|
-
other_col) -%}
|
71
|
-
{# Create and return SQL for generating a join or aggregating index.
|
72
|
-
Args:
|
73
|
-
relation (dict):
|
74
|
-
index_name (str): name of the index
|
75
|
-
create_statement (str): either "CREATE JOIN INDEX" or
|
76
|
-
"CREATE AND GENERATE AGGREGATING INDEX"
|
77
|
-
spine_col ([str]):
|
78
|
-
if agg index, key columns
|
79
|
-
if join index, join column
|
80
|
-
other_col ([str]):
|
81
|
-
if agg index, aggregating columns
|
82
|
-
if join index, dimension column
|
83
|
-
#}
|
84
|
-
{{ create_statement }} "{{ index_name }}" ON {{ relation }} (
|
85
|
-
{% if spine_col is iterable and spine_col is not string -%}
|
86
|
-
{{ spine_col | join(', ') }},
|
87
|
-
{% else -%}
|
88
|
-
{{ spine_col }},
|
89
|
-
{% endif -%}
|
90
|
-
{% if other_col is iterable and other_col is not string -%}
|
91
|
-
{{ other_col | join(', ') }}
|
92
|
-
{%- else -%}
|
93
|
-
{{ other_col }}
|
94
|
-
{%- endif -%}
|
95
|
-
);
|
96
|
-
{% endmacro %}
|
97
|
-
|
98
|
-
|
99
|
-
{% macro drop_index(index_name, index_type) -%}
|
100
|
-
{# Drop aggregating or join index. #}
|
101
|
-
{% call statement('drop_index', auto_begin=False) %}
|
102
|
-
|
103
|
-
DROP {{ index_type | upper }} INDEX "{{ index_name }}"
|
104
|
-
{% endcall %}
|
105
|
-
{% endmacro %}
|
106
|
-
|
107
|
-
|
108
|
-
{% macro firebolt__get_create_index_sql(relation, index_dict) -%}
|
109
|
-
{# Return aggregating or join index SQL string. #}
|
110
|
-
{# Parse index inputs and send parsed input to make_create_index_sql #}
|
111
|
-
{%- set index_config = adapter.parse_index(index_dict) -%}
|
112
|
-
{%- set index_name = index_config.render_name(relation) -%}
|
113
|
-
{%- set index_type = index_config.index_type | upper -%}
|
114
|
-
{%- if index_type == "JOIN" -%}
|
115
|
-
{{ make_create_index_sql(relation,
|
116
|
-
index_name,
|
117
|
-
"CREATE JOIN INDEX",
|
118
|
-
index_config.join_columns,
|
119
|
-
index_config.dimension_column) }}
|
120
|
-
{%- elif index_type == "AGGREGATING" -%}
|
121
|
-
{{ make_create_index_sql(relation,
|
122
|
-
index_name,
|
123
|
-
"CREATE AND GENERATE AGGREGATING INDEX",
|
124
|
-
index_config.key_columns,
|
125
|
-
index_config.aggregation) }}
|
126
|
-
{%- endif -%}
|
127
|
-
{%- endmacro %}
|
128
|
-
|
129
|
-
|
130
|
-
{% macro sql_convert_columns_in_relation_firebolt(rows) -%}
|
131
|
-
{% set columns = [] %}
|
132
|
-
{% for row in rows %}
|
133
|
-
{% do columns.append(api.Column(*row)) %}
|
134
|
-
{% endfor %}
|
135
|
-
{{ return(columns) }}
|
136
|
-
{% endmacro %}
|
137
|
-
|
138
|
-
|
139
|
-
{% macro firebolt__get_columns_in_relation(relation) -%}
|
140
|
-
{#-
|
141
|
-
Return column information for table identified by relation as
|
142
|
-
List[FireboltColumn].
|
143
|
-
Args: relation: dbt Relation
|
144
|
-
-#}
|
145
|
-
{% set sql %}
|
146
|
-
SELECT column_name, data_type from information_schema.columns
|
147
|
-
WHERE table_name = '{{ relation.identifier }}'
|
148
|
-
{% endset %}
|
149
|
-
{%- set result = run_query(sql) -%}
|
150
|
-
{% set columns = [] %}
|
151
|
-
{% for row in result %}
|
152
|
-
{% do columns.append(adapter.get_column_class().from_description(row['column_name'],
|
153
|
-
adapter.resolve_special_columns(row['data_type']))) %}
|
154
|
-
{% endfor %}
|
155
|
-
{% do return(columns) %}
|
156
|
-
{% endmacro %}
|
157
|
-
|
158
|
-
|
159
|
-
{% macro firebolt__list_relations_without_caching(relation) %}
|
160
|
-
{# Return all views and tables as agate table.
|
161
|
-
Args:
|
162
|
-
relation (dict): Contains values for database and schema.
|
163
|
-
|
164
|
-
dbt has a relations cache. Using this macro will list all
|
165
|
-
the relations in the current schema using a direct DB query,
|
166
|
-
rather than checking the cache. So the name is a misnomer. Should
|
167
|
-
be list_relations_bypassing_cache or something.
|
168
|
-
#}
|
169
|
-
{% call statement('list_tables_without_caching', fetch_result=True) %}
|
170
|
-
|
171
|
-
SELECT
|
172
|
-
table_catalog AS "database",
|
173
|
-
table_name AS "name",
|
174
|
-
'{{ relation.schema }}' AS "schema",
|
175
|
-
CASE
|
176
|
-
WHEN table_type = 'VIEW' THEN 'view'
|
177
|
-
ELSE 'table'
|
178
|
-
END AS "type"
|
179
|
-
FROM
|
180
|
-
information_schema.tables
|
181
|
-
{% endcall %}
|
182
|
-
{% set info_table = load_result('list_tables_without_caching').table %}
|
183
|
-
{{ return(info_table) }}
|
184
|
-
{% endmacro %}
|
185
|
-
|
186
|
-
{% macro firebolt__truncate_relation(relation) -%}
|
187
|
-
{#
|
188
|
-
Truncate relation. Actual macro is drop_relation in ./adapters/relation.sql.
|
189
|
-
#}
|
190
|
-
|
191
|
-
{# Firebolt doesn't currently support TRUNCATE, so DROP CASCADE.
|
192
|
-
This should only be called from reset_csv_table, where it's followed by
|
193
|
-
`create_csv_table`, so not recreating the table here. To retrieve old code,
|
194
|
-
see commit f9984f6d61b8a1b877bc107b102eeb30eba54f35
|
195
|
-
This will be replaced by `CREATE OR REPLACE`. #}
|
196
|
-
{{ adapter.drop_relation(relation) }}
|
197
|
-
{% endmacro %}
|
@@ -1,103 +0,0 @@
|
|
1
|
-
{# This is for building docs. Right now it's an incomplete description of
|
2
|
-
the columns (for instance, `is_nullable` is missing) but more could be added later. #}
|
3
|
-
|
4
|
-
{% macro firebolt__get_catalog(information_schema, schemas) -%}
|
5
|
-
|
6
|
-
{% set query %}
|
7
|
-
with tables as (
|
8
|
-
{{ firebolt__get_catalog_tables_sql(information_schema) }}
|
9
|
-
{{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }}
|
10
|
-
),
|
11
|
-
columns as (
|
12
|
-
{{ firebolt__get_catalog_columns_sql(information_schema) }}
|
13
|
-
{{ firebolt__get_catalog_schemas_where_clause_sql(schemas) }}
|
14
|
-
)
|
15
|
-
{{ firebolt__get_catalog_results_sql() }}
|
16
|
-
{%- endset -%}
|
17
|
-
|
18
|
-
{{ return(run_query(query)) }}
|
19
|
-
|
20
|
-
{%- endmacro %}
|
21
|
-
|
22
|
-
{% macro firebolt__get_catalog_relations(information_schema, relations) -%}
|
23
|
-
|
24
|
-
{% set query %}
|
25
|
-
with tables as (
|
26
|
-
{{ firebolt__get_catalog_tables_sql(information_schema) }}
|
27
|
-
{{ firebolt__get_catalog_relations_where_clause_sql(relations) }}
|
28
|
-
),
|
29
|
-
columns as (
|
30
|
-
{{ firebolt__get_catalog_columns_sql(information_schema) }}
|
31
|
-
{{ firebolt__get_catalog_relations_where_clause_sql(relations) }}
|
32
|
-
)
|
33
|
-
{{ firebolt__get_catalog_results_sql() }}
|
34
|
-
{%- endset -%}
|
35
|
-
|
36
|
-
{{ return(run_query(query)) }}
|
37
|
-
|
38
|
-
{%- endmacro %}
|
39
|
-
|
40
|
-
|
41
|
-
{% macro firebolt__get_catalog_tables_sql(information_schema) -%}
|
42
|
-
SELECT
|
43
|
-
tbls.table_catalog AS table_database,
|
44
|
-
tbls.table_schema as table_schema,
|
45
|
-
table_type,
|
46
|
-
tbls.table_name as table_name,
|
47
|
-
CASE
|
48
|
-
WHEN table_type = 'VIEW' THEN 'VIEW'
|
49
|
-
ELSE 'TABLE'
|
50
|
-
END AS relation_type
|
51
|
-
FROM
|
52
|
-
information_schema.tables tbls
|
53
|
-
{%- endmacro %}
|
54
|
-
|
55
|
-
|
56
|
-
{% macro firebolt__get_catalog_columns_sql(information_schema) -%}
|
57
|
-
select
|
58
|
-
table_catalog as "table_database",
|
59
|
-
table_schema as "table_schema",
|
60
|
-
table_name as "table_name",
|
61
|
-
column_name as "column_name",
|
62
|
-
ordinal_position as "column_index",
|
63
|
-
data_type as "column_type"
|
64
|
-
from information_schema.columns
|
65
|
-
{%- endmacro %}
|
66
|
-
|
67
|
-
{% macro firebolt__get_catalog_results_sql() -%}
|
68
|
-
SELECT *
|
69
|
-
FROM tables
|
70
|
-
JOIN columns USING ("table_database", "table_schema", "table_name")
|
71
|
-
ORDER BY "column_index"
|
72
|
-
{%- endmacro %}
|
73
|
-
|
74
|
-
|
75
|
-
{% macro firebolt__get_catalog_schemas_where_clause_sql(schemas) -%}
|
76
|
-
WHERE ({%- for schema in schemas -%}
|
77
|
-
UPPER("table_schema") = UPPER('{{ schema }}'){%- if not loop.last %} OR {% endif -%}
|
78
|
-
{%- endfor -%})
|
79
|
-
{%- endmacro %}
|
80
|
-
|
81
|
-
|
82
|
-
{% macro firebolt__get_catalog_relations_where_clause_sql(relations) -%}
|
83
|
-
WHERE (
|
84
|
-
{%- for relation in relations -%}
|
85
|
-
{% if relation.schema and relation.identifier %}
|
86
|
-
(
|
87
|
-
UPPER("table_schema") = UPPER('{{ relation.schema }}')
|
88
|
-
AND UPPER("table_name") = UPPER('{{ relation.identifier }}')
|
89
|
-
)
|
90
|
-
{% elif relation.schema %}
|
91
|
-
(
|
92
|
-
UPPER("table_schema") = UPPER('{{ relation.schema }}')
|
93
|
-
)
|
94
|
-
{% else %}
|
95
|
-
{% do exceptions.raise_compiler_error(
|
96
|
-
'`get_catalog_relations` requires a list of relations, each with a schema'
|
97
|
-
) %}
|
98
|
-
{% endif %}
|
99
|
-
|
100
|
-
{%- if not loop.last %} OR {% endif -%}
|
101
|
-
{%- endfor -%}
|
102
|
-
)
|
103
|
-
{%- endmacro %}
|
@@ -1,137 +0,0 @@
|
|
1
|
-
{% macro firebolt__create_external_table(source_node) %}
|
2
|
-
{% if source_node.external.strategy == 'copy' %}
|
3
|
-
{{ firebolt__create_with_copy_from(source_node) }}
|
4
|
-
{% else %}
|
5
|
-
{{ firebolt__create_with_external_table(source_node) }}
|
6
|
-
{% endif %}
|
7
|
-
{% endmacro %}
|
8
|
-
|
9
|
-
{% macro firebolt__create_with_external_table(source_node) %}
|
10
|
-
{%- set external = source_node.external -%}
|
11
|
-
{%- if 'partitions' in external -%}
|
12
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
13
|
-
external.partitions) -%}
|
14
|
-
{%- else -%}
|
15
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
16
|
-
[]) -%}
|
17
|
-
{%- endif -%}
|
18
|
-
{%- set credentials = external.credentials -%}
|
19
|
-
{# Leaving out "IF NOT EXISTS" because this should only be called by
|
20
|
-
if no DROP IF is necessary. #}
|
21
|
-
CREATE EXTERNAL TABLE {{source(source_node.source_name, source_node.name)}} (
|
22
|
-
{%- for column in columns -%}
|
23
|
-
{{ column }}
|
24
|
-
{{- ',' if not loop.last }}
|
25
|
-
{% endfor -%}
|
26
|
-
)
|
27
|
-
{% if external.url %} URL = '{{external.url}}' {%- endif %}
|
28
|
-
{%- if credentials and credentials.internal_role_arn %}
|
29
|
-
CREDENTIALS = (AWS_ROLE_ARN = '{{credentials.internal_role_arn}}'
|
30
|
-
{%- if credentials.external_role_id %}
|
31
|
-
AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}'
|
32
|
-
{%- endif -%}
|
33
|
-
)
|
34
|
-
{% elif credentials and credentials.aws_key_id %}
|
35
|
-
CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}'
|
36
|
-
AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
|
37
|
-
{%- endif %}
|
38
|
-
{%- if external.object_pattern -%} OBJECT_PATTERN = '{{external.object_pattern}}' {%- endif %}
|
39
|
-
{% if external.object_patterns -%}
|
40
|
-
OBJECT_PATTERN =
|
41
|
-
{%- for obj in external.object_patterns -%}
|
42
|
-
{{ obj }}
|
43
|
-
{{- ',' if not loop.last }}
|
44
|
-
{%- endfor %}
|
45
|
-
{%- endif %}
|
46
|
-
{%- if external.compression -%} COMPRESSION = {{external.compression}} {%- endif %}
|
47
|
-
TYPE = {{ external.type }}
|
48
|
-
{% endmacro %}
|
49
|
-
|
50
|
-
{% macro firebolt__create_with_copy_from(source_node) %}
|
51
|
-
{# COPY FROM is only available in Firebolt 2.0. #}
|
52
|
-
{%- set external = source_node.external -%}
|
53
|
-
{%- set credentials = external.credentials -%}
|
54
|
-
{%- set options = external.options -%}
|
55
|
-
{%- set csv_options = options.csv_options -%}
|
56
|
-
{%- set error_file_credentials = options.error_file_credentials -%}
|
57
|
-
|
58
|
-
{# There are no partitions, but this formats the columns correctly. #}
|
59
|
-
{%- if 'partitions' in external -%}
|
60
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
61
|
-
external.partitions) -%}
|
62
|
-
{%- else -%}
|
63
|
-
{%- set columns = adapter.make_field_partition_pairs(source_node.columns.values(),
|
64
|
-
[]) -%}
|
65
|
-
{%- endif -%}
|
66
|
-
COPY INTO {{source(source_node.source_name, source_node.name)}}
|
67
|
-
{%- if columns and columns | length > 0 %}
|
68
|
-
(
|
69
|
-
{%- for column in columns -%}
|
70
|
-
{{ column.name }}
|
71
|
-
{%- if column.default %} DEFAULT {{ column.default }}{% endif %}
|
72
|
-
{%- if column.source_column_name %} {{ '$' ~ loop.index0 }}{% endif %}
|
73
|
-
{{- ',' if not loop.last }}
|
74
|
-
{%- endfor -%}
|
75
|
-
)
|
76
|
-
{%- endif %}
|
77
|
-
FROM '{{external.url}}'
|
78
|
-
{%- if options %}
|
79
|
-
WITH
|
80
|
-
{%- if options.object_pattern %}
|
81
|
-
PATTERN = '{{options.object_pattern}}'
|
82
|
-
{%- endif %}
|
83
|
-
{%- if options.type %}
|
84
|
-
TYPE = {{ options.type }}
|
85
|
-
{%- endif %}
|
86
|
-
{%- if options.auto_create %}
|
87
|
-
AUTO_CREATE = {{ options.auto_create | upper }}
|
88
|
-
{%- endif %}
|
89
|
-
{%- if options.allow_column_mismatch %}
|
90
|
-
ALLOW_COLUMN_MISMATCH = {{ options.allow_column_mismatch | upper }}
|
91
|
-
{%- endif %}
|
92
|
-
{%- if options.error_file %}
|
93
|
-
ERROR_FILE = '{{ options.error_file }}'
|
94
|
-
{%- endif %}
|
95
|
-
{%- if error_file_credentials %}
|
96
|
-
ERROR_FILE_CREDENTIALS = (AWS_KEY_ID = '{{ error_file_credentials.aws_key_id }}' AWS_SECRET_KEY = '{{ error_file_credentials.aws_secret_key }}')
|
97
|
-
{%- endif %}
|
98
|
-
{%- if options.max_errors_per_file %}
|
99
|
-
MAX_ERRORS_PER_FILE = {{ options.max_errors_per_file }}
|
100
|
-
{%- endif %}
|
101
|
-
{%- if csv_options %}
|
102
|
-
{%- if csv_options.header %}
|
103
|
-
HEADER = {{ csv_options.header | upper }}
|
104
|
-
{%- endif %}
|
105
|
-
{%- if csv_options.delimiter %}
|
106
|
-
DELIMITER = '{{ csv_options.delimiter }}'
|
107
|
-
{%- endif %}
|
108
|
-
{%- if csv_options.newline %}
|
109
|
-
NEWLINE = '{{ csv_options.newline }}'
|
110
|
-
{%- endif %}
|
111
|
-
{%- if csv_options.quote %}
|
112
|
-
QUOTE = {{ csv_options.quote }}
|
113
|
-
{%- endif %}
|
114
|
-
{%- if csv_options.escape %}
|
115
|
-
ESCAPE = '{{ csv_options.escape }}'
|
116
|
-
{%- endif %}
|
117
|
-
{%- if csv_options.null_string %}
|
118
|
-
NULL_STRING = '{{ csv_options.null_string }}'
|
119
|
-
{%- endif %}
|
120
|
-
{%- if csv_options.empty_field_as_null %}
|
121
|
-
EMPTY_FIELD_AS_NULL = {{ csv_options.empty_field_as_null | upper }}
|
122
|
-
{%- endif %}
|
123
|
-
{%- if csv_options.skip_blank_lines %}
|
124
|
-
SKIP_BLANK_LINES = {{ csv_options.skip_blank_lines | upper }}
|
125
|
-
{%- endif %}
|
126
|
-
{%- if csv_options.date_format %}
|
127
|
-
DATE_FORMAT = '{{ csv_options.date_format }}'
|
128
|
-
{%- endif %}
|
129
|
-
{%- if csv_options.timestamp_format %}
|
130
|
-
TIMESTAMP_FORMAT = '{{ csv_options.timestamp_format }}'
|
131
|
-
{%- endif %}
|
132
|
-
{%- endif %}
|
133
|
-
{%- endif %}
|
134
|
-
{%- if credentials %}
|
135
|
-
CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}')
|
136
|
-
{%- endif %}
|
137
|
-
{% endmacro %}
|
@@ -1,18 +0,0 @@
|
|
1
|
-
{% macro firebolt__get_external_build_plan(source_node) %}
|
2
|
-
{% set build_plan = [] %}
|
3
|
-
{% set old_relation = adapter.get_relation(
|
4
|
-
database = source_node.database,
|
5
|
-
schema = source_node.schema,
|
6
|
-
identifier = source_node.identifier
|
7
|
-
) %}
|
8
|
-
{# var(variable, Boolean) defaults to Boolean value if variable isnt set.
|
9
|
-
Firebolt doesn't do refresh because we don't need to—external tables will always
|
10
|
-
pull most recent data from S3, so the default action below is to skip. #}
|
11
|
-
{% if old_relation is none or var('ext_full_refresh', false) %}
|
12
|
-
{% set build_plan = build_plan + [dropif(source_node),
|
13
|
-
create_external_table(source_node)] %}
|
14
|
-
{% else %}
|
15
|
-
{% set build_plan = dbt_external_tables.refresh_external_table(source_node) %}
|
16
|
-
{% endif %}
|
17
|
-
{% do return(build_plan) %}
|
18
|
-
{% endmacro %}
|
@@ -1,21 +0,0 @@
|
|
1
|
-
{% macro intersect_columns(source_columns, target_columns) %}
|
2
|
-
{# Return a List[FireboltColumn] of columns that appear in both source and target.
|
3
|
-
Args:
|
4
|
-
source_columns: List[FireboltColumn]
|
5
|
-
target_columns: List[FireboltColumn]
|
6
|
-
#}
|
7
|
-
{% set result = [] %}
|
8
|
-
{# Note I'm using `column` and not `name` below, as name is a getter
|
9
|
-
and column is the actual field. #}
|
10
|
-
{% set source_names = source_columns | map(attribute = 'column') | list %}
|
11
|
-
{% set target_names = target_columns | map(attribute = 'column') | list %}
|
12
|
-
{# Check whether the name attribute exists in the target - this does
|
13
|
-
not perform a data type check. This is O(m•n), but cardinality
|
14
|
-
of columns is probably low enough that it doesn't matter. #}
|
15
|
-
{% for sc in source_columns %}
|
16
|
-
{% if sc.name in target_names %}
|
17
|
-
{{ result.append(sc) }}
|
18
|
-
{% endif %}
|
19
|
-
{% endfor %}
|
20
|
-
{{ return(result) }}
|
21
|
-
{% endmacro %}
|