dbt-firebolt 1.8.2__py3-none-any.whl → 1.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,7 @@ from dbt.adapters.firebolt.connections import (
7
7
  from dbt.adapters.firebolt.impl import FireboltAdapter
8
8
  from dbt.include import firebolt
9
9
 
10
- __version__ = "1.8.2"
10
+ __version__ = "1.9.0"
11
11
 
12
12
  Plugin = AdapterPlugin(
13
13
  adapter=FireboltAdapter, # type: ignore
@@ -39,7 +39,6 @@ class FireboltCredentials(Credentials):
39
39
  client_id: Optional[str] = None
40
40
  client_secret: Optional[str] = None
41
41
  api_endpoint: Optional[str] = DEFAULT_API_URL
42
- driver: str = 'com.firebolt.FireboltDriver'
43
42
  engine_name: Optional[str] = None
44
43
  account_name: Optional[str] = None
45
44
  retries: int = 1
@@ -1,7 +1,6 @@
1
1
  import re
2
2
  import time
3
3
  from dataclasses import dataclass
4
- from datetime import datetime
5
4
  from typing import Any, List, Mapping, Optional, Union
6
5
 
7
6
  import agate
@@ -29,21 +28,45 @@ from dbt.adapters.firebolt.connections import FireboltConnectionManager
29
28
  from dbt.adapters.firebolt.relation import FireboltRelation
30
29
 
31
30
 
31
+ def quote_columns(columns: Union[str, List[str]]) -> Union[str, List[str]]:
32
+ if isinstance(columns, str):
33
+ return f'"{columns}"'
34
+ quoted_columns = []
35
+ for col in columns:
36
+ if col.startswith('"') and col.endswith('"'):
37
+ quoted_columns.append(col)
38
+ else:
39
+ quoted_columns.append(f'"{col}"')
40
+ return quoted_columns
41
+
42
+
32
43
  @dataclass
33
44
  class FireboltIndexConfig(dbtClassMixin):
34
45
  index_type: str
46
+ index_name: Optional[str] = None
35
47
  join_columns: Optional[Union[str, List[str]]] = None
36
48
  key_columns: Optional[Union[str, List[str]]] = None
37
49
  dimension_column: Optional[Union[str, List[str]]] = None
38
50
  aggregation: Optional[Union[str, List[str]]] = None
39
51
 
52
+ def __post_init__(self) -> None:
53
+ # quote unquoted columns
54
+ if self.join_columns:
55
+ self.join_columns = quote_columns(self.join_columns)
56
+ if self.key_columns:
57
+ self.key_columns = quote_columns(self.key_columns)
58
+ if self.dimension_column:
59
+ self.dimension_column = quote_columns(self.dimension_column)
60
+
40
61
  def render_name(self, relation: FireboltRelation) -> str:
41
62
  """
42
63
  Name an index according to the following format, joined by `_`:
43
64
  index type, relation name, key/join columns, timestamp (unix & UTC)
44
65
  example index name: join_my_model_customer_id_1633504263.
45
66
  """
46
- now_unix = str(int(time.mktime(datetime.utcnow().timetuple())))
67
+ if self.index_name:
68
+ return self.index_name
69
+ now_unix = str(int(time.time()))
47
70
  # If column_names is a list with > 1 members, join with _,
48
71
  # otherwise do not. We were getting index names like
49
72
  # join__idx__emf_customers__f_i_r_s_t___n_a_m_e__165093112.
@@ -51,14 +74,23 @@ class FireboltIndexConfig(dbtClassMixin):
51
74
  spine_col = (
52
75
  '_'.join(column_names) if isinstance(column_names, list) else column_names
53
76
  )
77
+ # Additional hash to ensure uniqueness after spaces are removed
78
+ column_hash = str(hash(spine_col))[:5]
54
79
  inputs = [
55
80
  f'{self.index_type}_idx',
56
81
  relation.identifier,
57
82
  spine_col,
83
+ column_hash,
58
84
  now_unix,
59
85
  ]
60
- string = '__'.join([x for x in inputs if x is not None])
61
- return string
86
+ index_name = '__'.join([x for x in inputs if x is not None])
87
+ if len(index_name) > 255:
88
+ # Firebolt index names must be <= 255 characters.
89
+ # If the index name is too long, hash it.
90
+ return f'{self.index_type}_idx_{hash(index_name)}'
91
+ # Remove any spaces or quotes from the index name.
92
+ index_name = index_name.replace(' ', '_').replace('"', '')
93
+ return index_name
62
94
 
63
95
  @classmethod
64
96
  def parse(cls, raw_index: Optional[Mapping]) -> Optional['FireboltIndexConfig']:
@@ -274,7 +306,6 @@ class FireboltAdapter(SQLAdapter):
274
306
  columns: the number of rows that are different between the two
275
307
  relations and the number of mismatched rows.
276
308
  """
277
- # This method only really exists for test reasons.
278
309
  names: List[str]
279
310
  if column_names is None:
280
311
  columns = self.get_columns_in_relation(relation_a)
@@ -68,8 +68,8 @@
68
68
  (
69
69
  {%- for column in columns -%}
70
70
  {{ column.name }}
71
- {%- if column.default is not none %} DEFAULT {{ column.default }}{% endif %}
72
- {%- if column.source_column_name is not none %} {{ '$' ~ loop.index0 }}{% endif %}
71
+ {%- if column.default %} DEFAULT {{ column.default }}{% endif %}
72
+ {%- if column.source_column_name %} {{ '$' ~ loop.index0 }}{% endif %}
73
73
  {{- ',' if not loop.last }}
74
74
  {%- endfor -%}
75
75
  )
@@ -83,10 +83,10 @@
83
83
  {%- if options.type %}
84
84
  TYPE = {{ options.type }}
85
85
  {%- endif %}
86
- {%- if options.auto_create is not none %}
86
+ {%- if options.auto_create %}
87
87
  AUTO_CREATE = {{ options.auto_create | upper }}
88
88
  {%- endif %}
89
- {%- if options.allow_column_mismatch is not none %}
89
+ {%- if options.allow_column_mismatch %}
90
90
  ALLOW_COLUMN_MISMATCH = {{ options.allow_column_mismatch | upper }}
91
91
  {%- endif %}
92
92
  {%- if options.error_file %}
@@ -99,7 +99,7 @@
99
99
  MAX_ERRORS_PER_FILE = {{ options.max_errors_per_file }}
100
100
  {%- endif %}
101
101
  {%- if csv_options %}
102
- {%- if csv_options.header is not none %}
102
+ {%- if csv_options.header %}
103
103
  HEADER = {{ csv_options.header | upper }}
104
104
  {%- endif %}
105
105
  {%- if csv_options.delimiter %}
@@ -117,10 +117,10 @@
117
117
  {%- if csv_options.null_string %}
118
118
  NULL_STRING = '{{ csv_options.null_string }}'
119
119
  {%- endif %}
120
- {%- if csv_options.empty_field_as_null is not none %}
120
+ {%- if csv_options.empty_field_as_null %}
121
121
  EMPTY_FIELD_AS_NULL = {{ csv_options.empty_field_as_null | upper }}
122
122
  {%- endif %}
123
- {%- if csv_options.skip_blank_lines is not none %}
123
+ {%- if csv_options.skip_blank_lines %}
124
124
  SKIP_BLANK_LINES = {{ csv_options.skip_blank_lines | upper }}
125
125
  {%- endif %}
126
126
  {%- if csv_options.date_format %}
@@ -0,0 +1,19 @@
1
+ {% macro firebolt__snapshot_merge_sql(target, source, insert_cols) -%}
2
+ {%- set insert_cols_csv = insert_cols | join(', ') -%}
3
+
4
+ UPDATE {{ target.render() }} AS DBT_INTERNAL_DEST
5
+ SET dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to
6
+ FROM {{ source }} AS DBT_INTERNAL_SOURCE
7
+ WHERE DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id
8
+ AND DBT_INTERNAL_DEST.dbt_valid_to IS NULL
9
+ AND DBT_INTERNAL_SOURCE.dbt_change_type IN ('update', 'delete');
10
+
11
+ INSERT INTO {{ target.render() }} ({{ insert_cols_csv }})
12
+ SELECT {{ insert_cols_csv }}
13
+ FROM {{ source }} AS DBT_INTERNAL_SOURCE
14
+ WHERE DBT_INTERNAL_SOURCE.dbt_scd_id NOT IN (
15
+ SELECT dbt_scd_id FROM {{ target.render() }}
16
+ )
17
+ AND DBT_INTERNAL_SOURCE.dbt_change_type = 'insert';
18
+
19
+ {% endmacro %}
@@ -24,6 +24,10 @@
24
24
  ) }}
25
25
  {%- endif -%}
26
26
 
27
+ {%- if temporary -%}
28
+ {%- do adapter.drop_relation(relation) -%}
29
+ {%- endif -%}
30
+
27
31
  {%- set table_type = config.get(
28
32
  'table_type',
29
33
  default = 'dimension'
@@ -49,9 +53,9 @@
49
53
 
50
54
  {%- if primary_index %}
51
55
  primary INDEX {% if primary_index is iterable and primary_index is not string %}
52
- {{ primary_index | join(', ') }}
56
+ "{{ primary_index | join('", "') }}"
53
57
  {%- else -%}
54
- {{ primary_index }}
58
+ "{{ primary_index }}"
55
59
  {%- endif -%}
56
60
  {%- endif -%}
57
61
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dbt-firebolt
3
- Version: 1.8.2
3
+ Version: 1.9.0
4
4
  Summary: The Firebolt adapter plugin for dbt (data build tool)
5
5
  Home-page: https://github.com/firebolt-db/dbt-firebolt
6
6
  Author: Firebolt
@@ -13,6 +13,9 @@ Classifier: Programming Language :: Python :: 3
13
13
  Classifier: Programming Language :: Python :: 3 :: Only
14
14
  Classifier: Programming Language :: Python :: 3.8
15
15
  Classifier: Programming Language :: Python :: 3.9
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
16
19
  Requires-Python: >=3.8
17
20
  Description-Content-Type: text/markdown
18
21
  License-File: LICENSE
@@ -24,7 +27,7 @@ Provides-Extra: dev
24
27
  Requires-Dist: allure-pytest ==2.* ; extra == 'dev'
25
28
  Requires-Dist: dbt-tests-adapter ~=1.6 ; extra == 'dev'
26
29
  Requires-Dist: mypy ==1.4.1 ; extra == 'dev'
27
- Requires-Dist: pre-commit ==2.15.0 ; extra == 'dev'
30
+ Requires-Dist: pre-commit ==3.5.0 ; extra == 'dev'
28
31
  Requires-Dist: pytest ==7.* ; extra == 'dev'
29
32
 
30
33
  <img width="1113" alt="Screen Shot 2021-12-10 at 1 09 09 PM" src="https://user-images.githubusercontent.com/7674553/145641621-a7dabe78-da92-4f0a-bbd2-54ccf7f34b57.png">
@@ -75,7 +78,7 @@ The table below shows which dbt and Firebolt features are supported by the adapt
75
78
  | Incremental materializations - insert_overwrite | :white_check_mark: |
76
79
  | Incremental materializations - delete+insert | :white_check_mark: |
77
80
  | Incremental materializations - merge | :x: |
78
- | Snapshots | :x: |
81
+ | Snapshots | :white_check_mark: |
79
82
  | Seeds | :white_check_mark: |
80
83
  | Tests | :white_check_mark: |
81
84
  | Documentation | :white_check_mark: |
@@ -1,8 +1,8 @@
1
- dbt/adapters/firebolt/__init__.py,sha256=4WyLJ7pu9-HKBdCnohQRVqNBbOM6_nFbyOXxAhM2e_g,411
1
+ dbt/adapters/firebolt/__init__.py,sha256=JjZII9vG02tYg2Mx_VhNAvHCmaGklk5xXNQhOpqQcOM,411
2
2
  dbt/adapters/firebolt/__version__.py,sha256=zRlZGglif76ZVuWWSjsH_MMPgtVQqmj-SryYJW25FL4,69
3
3
  dbt/adapters/firebolt/column.py,sha256=COo_wjhCFgS3GFcPIPcoq7WAWgzN6DB2XqG-gk51WBc,539
4
- dbt/adapters/firebolt/connections.py,sha256=r7imGPka0klw1ev-5GTSIwUbO-Df_5DE4Ss-9OfLBMk,7990
5
- dbt/adapters/firebolt/impl.py,sha256=vHK7AfpmMdZMCHaky3IEufl18IYFPs9mdvdmTpMrUh4,13939
4
+ dbt/adapters/firebolt/connections.py,sha256=_i0wPE6LWmLGo_C50UhPAU8IMUi_I2HU23fBZtfhi0Q,7942
5
+ dbt/adapters/firebolt/impl.py,sha256=xlmGCDezIsUKiQ4gQFR60h_QtvSgW1lOacktv8Lt3ow,15143
6
6
  dbt/adapters/firebolt/relation.py,sha256=Xg3Nrjw3UrF_qwnuGbPT97rSXRiDP1GlIAoBF4b7QnY,1922
7
7
  dbt/adapters/firebolt/relation_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
8
  dbt/include/firebolt/__init__.py,sha256=vBGWeG-dHHkimfnX8axBJ4IgAowFw8xADmo6Auzn2xc,52
@@ -11,12 +11,13 @@ dbt/include/firebolt/macros/adapters.sql,sha256=NiTW87XQmC8gGahY6LCw7Fd6Brgin5qB
11
11
  dbt/include/firebolt/macros/catalog.sql,sha256=LLhjI9oEoMEQKQfE3LJ5nWS9-u3bwBAYBz4ukrsg5T8,3350
12
12
  dbt/include/firebolt/macros/adapters/apply_grants.sql,sha256=nEmyDs0K0HENxxMaY8v-5oifIXDMrysmuUb_bo9bCuY,1095
13
13
  dbt/include/firebolt/macros/adapters/relation.sql,sha256=1nQZg_vwpGYwFI2EQvHJA5CqFWk7h3w3Yq2uvYXNR5E,2017
14
- dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql,sha256=x7awixF-k8neugbKp7_QVb7PhBlQnsaIvvdBtJEeJqE,6153
14
+ dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql,sha256=MeFyR0oMjQdc0GlUABx6DDm5t76PdcdmXKWGVThi4EE,6069
15
15
  dbt/include/firebolt/macros/dbt_external_tables/dropif.sql,sha256=-kZzUVgC6Bcof7tZ9VcDK427agylMHeG-yA8NqSeWkw,167
16
16
  dbt/include/firebolt/macros/dbt_external_tables/get_external_build_plan.sql,sha256=BWNSuqB9t0hxbxL9CbIc0UFTBBarQMtGfHv9LLAbqSI,894
17
17
  dbt/include/firebolt/macros/materializations/clone.sql,sha256=SKPc_tG0tF2HRKlo1-CQPvxVWGKHpXDwxvubsCwKTG8,79
18
18
  dbt/include/firebolt/macros/materializations/materialized_view.sql,sha256=pdWGMP25o0RqNINx1hbn1Q4dTNcdk0Dgrlv3K8XQAbs,202
19
19
  dbt/include/firebolt/macros/materializations/seed.sql,sha256=O622lwmzRGeAfaNfRpjR183ARFltILteznwgXZfPXMA,1564
20
+ dbt/include/firebolt/macros/materializations/snapshot_merge.sql,sha256=9Y4tGWkqkBDt53QHKxJPFALboKQxDJ4ZaSrLwNyRgIs,795
20
21
  dbt/include/firebolt/macros/materializations/table.sql,sha256=XY0D-df9EoWOBF3kCT3TCE3Rko7bvvfuztQT0-Pqlfs,1752
21
22
  dbt/include/firebolt/macros/materializations/test.sql,sha256=Wol19pNIfkPRTpVUYKAGbkipYOMZjixnAaoZ1chuP88,488
22
23
  dbt/include/firebolt/macros/materializations/view.sql,sha256=0xg2mO-1UWtZKj2Wlplgm7s93l-FKBswajiBOqPu2Og,1706
@@ -31,7 +32,7 @@ dbt/include/firebolt/macros/relations/materialized_view/create.sql,sha256=xthF0o
31
32
  dbt/include/firebolt/macros/relations/materialized_view/describe.sql,sha256=uMJWY-9P-49YjhwH_F1Co4szia0oNAVYqJIv2IVGzM8,164
32
33
  dbt/include/firebolt/macros/relations/materialized_view/drop.sql,sha256=Ya03cid5h05t3-SIF4JndKtMMIctD46OX9gPVt9jhVw,160
33
34
  dbt/include/firebolt/macros/relations/materialized_view/refresh.sql,sha256=9s4e6cjT9Dx9nCCJPMlhV5SkzHX8574cdIJUhSfNmU8,163
34
- dbt/include/firebolt/macros/relations/table/create.sql,sha256=4qK5r8EODXDjsBZwbHupLFfp3cDT4JkQkOuH94L9KVQ,2406
35
+ dbt/include/firebolt/macros/relations/table/create.sql,sha256=5FK3XOICE-9LGojHzh0pjCy5wO-RKyeytzkEQmYT7zY,2507
35
36
  dbt/include/firebolt/macros/relations/table/drop.sql,sha256=3W0SMqmlDizmZqn-QZvqXeJ9lG2KtooiFvwac3WzSzQ,110
36
37
  dbt/include/firebolt/macros/relations/table/rename.sql,sha256=5dmKvuiSnMTARuthnDJBUOORdQCIeEuGMLoyZlgQxVU,185
37
38
  dbt/include/firebolt/macros/relations/table/replace.sql,sha256=bw08YDtLMhkM4dO4nYXZ32U50lcnTG5Axb-Lls7XpB0,132
@@ -53,8 +54,8 @@ dbt/include/firebolt/macros/utils/position.sql,sha256=WPo9_bhvNYJooEAsHC9OcnNAwU
53
54
  dbt/include/firebolt/macros/utils/right.sql,sha256=_mm1_2MvlOH4O6CmYhgvVxMLfDxAvgv-EMwZ8OBOq-I,254
54
55
  dbt/include/firebolt/macros/utils/split_part.sql,sha256=5dUlbx3Pt1iWWaIlxiXyYUwUqzXuLLXMB-1aGJHNk4o,464
55
56
  dbt/include/firebolt/macros/utils/timestamps.sql,sha256=22g-QpJjBuBUQOHNpQ_TuMRa-cHxaxXPv8ItEUo-vEk,397
56
- dbt_firebolt-1.8.2.dist-info/LICENSE,sha256=Nn0EGvW3qmoZpBV_JVM3iPukFf3RiNCIizrWe_2oTHk,11354
57
- dbt_firebolt-1.8.2.dist-info/METADATA,sha256=mFMHiN_VwI-l0-MZ-KoCVha61CJDRS8WyHOnLJkZiEA,5070
58
- dbt_firebolt-1.8.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
59
- dbt_firebolt-1.8.2.dist-info/top_level.txt,sha256=B2YH4he17ajilEWOGCKHbRcEJlCuZKwCcgFcLPntLsE,4
60
- dbt_firebolt-1.8.2.dist-info/RECORD,,
57
+ dbt_firebolt-1.9.0.dist-info/LICENSE,sha256=Nn0EGvW3qmoZpBV_JVM3iPukFf3RiNCIizrWe_2oTHk,11354
58
+ dbt_firebolt-1.9.0.dist-info/METADATA,sha256=cusaAXb1pygjqHsGIE6sk-Xk56vD8NOhOnVzWhWn-_M,5237
59
+ dbt_firebolt-1.9.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
60
+ dbt_firebolt-1.9.0.dist-info/top_level.txt,sha256=B2YH4he17ajilEWOGCKHbRcEJlCuZKwCcgFcLPntLsE,4
61
+ dbt_firebolt-1.9.0.dist-info/RECORD,,