dbt-adapters 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-adapters might be problematic. Click here for more details.

Files changed (147) hide show
  1. dbt/__init__.py +0 -0
  2. dbt/adapters/__about__.py +1 -0
  3. dbt/adapters/__init__.py +7 -0
  4. dbt/adapters/base/README.md +13 -0
  5. dbt/adapters/base/__init__.py +15 -0
  6. dbt/adapters/base/column.py +166 -0
  7. dbt/adapters/base/connections.py +426 -0
  8. dbt/adapters/base/impl.py +1654 -0
  9. dbt/adapters/base/meta.py +131 -0
  10. dbt/adapters/base/plugin.py +32 -0
  11. dbt/adapters/base/query_headers.py +101 -0
  12. dbt/adapters/base/relation.py +471 -0
  13. dbt/adapters/cache.py +521 -0
  14. dbt/adapters/capability.py +52 -0
  15. dbt/adapters/clients/__init__.py +0 -0
  16. dbt/adapters/clients/jinja.py +24 -0
  17. dbt/adapters/contracts/__init__.py +0 -0
  18. dbt/adapters/contracts/connection.py +228 -0
  19. dbt/adapters/contracts/macros.py +11 -0
  20. dbt/adapters/contracts/relation.py +125 -0
  21. dbt/adapters/events/README.md +57 -0
  22. dbt/adapters/events/__init__.py +0 -0
  23. dbt/adapters/events/adapter_types.proto +517 -0
  24. dbt/adapters/events/adapter_types_pb2.py +208 -0
  25. dbt/adapters/events/base_types.py +40 -0
  26. dbt/adapters/events/logging.py +83 -0
  27. dbt/adapters/events/types.py +423 -0
  28. dbt/adapters/exceptions/__init__.py +40 -0
  29. dbt/adapters/exceptions/alias.py +24 -0
  30. dbt/adapters/exceptions/cache.py +68 -0
  31. dbt/adapters/exceptions/compilation.py +255 -0
  32. dbt/adapters/exceptions/connection.py +16 -0
  33. dbt/adapters/exceptions/database.py +51 -0
  34. dbt/adapters/factory.py +246 -0
  35. dbt/adapters/protocol.py +173 -0
  36. dbt/adapters/reference_keys.py +39 -0
  37. dbt/adapters/relation_configs/README.md +25 -0
  38. dbt/adapters/relation_configs/__init__.py +12 -0
  39. dbt/adapters/relation_configs/config_base.py +44 -0
  40. dbt/adapters/relation_configs/config_change.py +24 -0
  41. dbt/adapters/relation_configs/config_validation.py +57 -0
  42. dbt/adapters/sql/__init__.py +2 -0
  43. dbt/adapters/sql/connections.py +195 -0
  44. dbt/adapters/sql/impl.py +273 -0
  45. dbt/adapters/utils.py +69 -0
  46. dbt/include/global_project/__init__.py +4 -0
  47. dbt/include/global_project/dbt_project.yml +7 -0
  48. dbt/include/global_project/docs/overview.md +43 -0
  49. dbt/include/global_project/macros/adapters/apply_grants.sql +167 -0
  50. dbt/include/global_project/macros/adapters/columns.sql +137 -0
  51. dbt/include/global_project/macros/adapters/freshness.sql +16 -0
  52. dbt/include/global_project/macros/adapters/indexes.sql +41 -0
  53. dbt/include/global_project/macros/adapters/metadata.sql +96 -0
  54. dbt/include/global_project/macros/adapters/persist_docs.sql +33 -0
  55. dbt/include/global_project/macros/adapters/relation.sql +79 -0
  56. dbt/include/global_project/macros/adapters/schema.sql +20 -0
  57. dbt/include/global_project/macros/adapters/show.sql +22 -0
  58. dbt/include/global_project/macros/adapters/timestamps.sql +44 -0
  59. dbt/include/global_project/macros/adapters/validate_sql.sql +10 -0
  60. dbt/include/global_project/macros/etc/datetime.sql +62 -0
  61. dbt/include/global_project/macros/etc/statement.sql +52 -0
  62. dbt/include/global_project/macros/generic_test_sql/accepted_values.sql +27 -0
  63. dbt/include/global_project/macros/generic_test_sql/not_null.sql +9 -0
  64. dbt/include/global_project/macros/generic_test_sql/relationships.sql +23 -0
  65. dbt/include/global_project/macros/generic_test_sql/unique.sql +12 -0
  66. dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql +36 -0
  67. dbt/include/global_project/macros/get_custom_name/get_custom_database.sql +32 -0
  68. dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql +60 -0
  69. dbt/include/global_project/macros/materializations/configs.sql +21 -0
  70. dbt/include/global_project/macros/materializations/hooks.sql +35 -0
  71. dbt/include/global_project/macros/materializations/models/clone/can_clone_table.sql +7 -0
  72. dbt/include/global_project/macros/materializations/models/clone/clone.sql +67 -0
  73. dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql +7 -0
  74. dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql +80 -0
  75. dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +92 -0
  76. dbt/include/global_project/macros/materializations/models/incremental/is_incremental.sql +13 -0
  77. dbt/include/global_project/macros/materializations/models/incremental/merge.sql +131 -0
  78. dbt/include/global_project/macros/materializations/models/incremental/on_schema_change.sql +144 -0
  79. dbt/include/global_project/macros/materializations/models/incremental/strategies.sql +79 -0
  80. dbt/include/global_project/macros/materializations/models/materialized_view.sql +121 -0
  81. dbt/include/global_project/macros/materializations/models/table.sql +64 -0
  82. dbt/include/global_project/macros/materializations/models/view.sql +72 -0
  83. dbt/include/global_project/macros/materializations/seeds/helpers.sql +128 -0
  84. dbt/include/global_project/macros/materializations/seeds/seed.sql +60 -0
  85. dbt/include/global_project/macros/materializations/snapshots/helpers.sql +181 -0
  86. dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +99 -0
  87. dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +25 -0
  88. dbt/include/global_project/macros/materializations/snapshots/strategies.sql +174 -0
  89. dbt/include/global_project/macros/materializations/tests/helpers.sql +14 -0
  90. dbt/include/global_project/macros/materializations/tests/test.sql +60 -0
  91. dbt/include/global_project/macros/materializations/tests/where_subquery.sql +15 -0
  92. dbt/include/global_project/macros/python_model/python.sql +103 -0
  93. dbt/include/global_project/macros/relations/column/columns_spec_ddl.sql +89 -0
  94. dbt/include/global_project/macros/relations/create.sql +23 -0
  95. dbt/include/global_project/macros/relations/create_backup.sql +17 -0
  96. dbt/include/global_project/macros/relations/create_intermediate.sql +17 -0
  97. dbt/include/global_project/macros/relations/drop.sql +41 -0
  98. dbt/include/global_project/macros/relations/drop_backup.sql +14 -0
  99. dbt/include/global_project/macros/relations/materialized_view/alter.sql +55 -0
  100. dbt/include/global_project/macros/relations/materialized_view/create.sql +10 -0
  101. dbt/include/global_project/macros/relations/materialized_view/drop.sql +14 -0
  102. dbt/include/global_project/macros/relations/materialized_view/refresh.sql +9 -0
  103. dbt/include/global_project/macros/relations/materialized_view/rename.sql +10 -0
  104. dbt/include/global_project/macros/relations/materialized_view/replace.sql +10 -0
  105. dbt/include/global_project/macros/relations/rename.sql +35 -0
  106. dbt/include/global_project/macros/relations/rename_intermediate.sql +14 -0
  107. dbt/include/global_project/macros/relations/replace.sql +50 -0
  108. dbt/include/global_project/macros/relations/schema.sql +8 -0
  109. dbt/include/global_project/macros/relations/table/create.sql +60 -0
  110. dbt/include/global_project/macros/relations/table/drop.sql +14 -0
  111. dbt/include/global_project/macros/relations/table/rename.sql +10 -0
  112. dbt/include/global_project/macros/relations/table/replace.sql +10 -0
  113. dbt/include/global_project/macros/relations/view/create.sql +27 -0
  114. dbt/include/global_project/macros/relations/view/drop.sql +14 -0
  115. dbt/include/global_project/macros/relations/view/rename.sql +10 -0
  116. dbt/include/global_project/macros/relations/view/replace.sql +66 -0
  117. dbt/include/global_project/macros/utils/any_value.sql +9 -0
  118. dbt/include/global_project/macros/utils/array_append.sql +8 -0
  119. dbt/include/global_project/macros/utils/array_concat.sql +7 -0
  120. dbt/include/global_project/macros/utils/array_construct.sql +12 -0
  121. dbt/include/global_project/macros/utils/bool_or.sql +9 -0
  122. dbt/include/global_project/macros/utils/cast_bool_to_text.sql +7 -0
  123. dbt/include/global_project/macros/utils/concat.sql +7 -0
  124. dbt/include/global_project/macros/utils/data_types.sql +129 -0
  125. dbt/include/global_project/macros/utils/date_spine.sql +75 -0
  126. dbt/include/global_project/macros/utils/date_trunc.sql +7 -0
  127. dbt/include/global_project/macros/utils/dateadd.sql +14 -0
  128. dbt/include/global_project/macros/utils/datediff.sql +14 -0
  129. dbt/include/global_project/macros/utils/escape_single_quotes.sql +8 -0
  130. dbt/include/global_project/macros/utils/except.sql +9 -0
  131. dbt/include/global_project/macros/utils/generate_series.sql +53 -0
  132. dbt/include/global_project/macros/utils/hash.sql +7 -0
  133. dbt/include/global_project/macros/utils/intersect.sql +9 -0
  134. dbt/include/global_project/macros/utils/last_day.sql +15 -0
  135. dbt/include/global_project/macros/utils/length.sql +11 -0
  136. dbt/include/global_project/macros/utils/listagg.sql +30 -0
  137. dbt/include/global_project/macros/utils/literal.sql +7 -0
  138. dbt/include/global_project/macros/utils/position.sql +11 -0
  139. dbt/include/global_project/macros/utils/replace.sql +14 -0
  140. dbt/include/global_project/macros/utils/right.sql +12 -0
  141. dbt/include/global_project/macros/utils/safe_cast.sql +9 -0
  142. dbt/include/global_project/macros/utils/split_part.sql +26 -0
  143. dbt/include/global_project/tests/generic/builtin.sql +30 -0
  144. dbt_adapters-0.1.0a1.dist-info/METADATA +81 -0
  145. dbt_adapters-0.1.0a1.dist-info/RECORD +147 -0
  146. dbt_adapters-0.1.0a1.dist-info/WHEEL +4 -0
  147. dbt_adapters-0.1.0a1.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,144 @@
1
+ {% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
2
+
3
+ {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}
4
+
5
+ {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}
6
+ {% do log(log_message) %}
7
+
8
+ {{ return(default) }}
9
+
10
+ {% else %}
11
+
12
+ {{ return(on_schema_change) }}
13
+
14
+ {% endif %}
15
+
16
+ {% endmacro %}
17
+
18
+
19
+ {% macro check_for_schema_changes(source_relation, target_relation) %}
20
+
21
+ {% set schema_changed = False %}
22
+
23
+ {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}
24
+ {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
25
+ {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
26
+ {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
27
+
28
+ {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}
29
+
30
+ {% if source_not_in_target != [] %}
31
+ {% set schema_changed = True %}
32
+ {% elif target_not_in_source != [] or new_target_types != [] %}
33
+ {% set schema_changed = True %}
34
+ {% elif new_target_types != [] %}
35
+ {% set schema_changed = True %}
36
+ {% endif %}
37
+
38
+ {% set changes_dict = {
39
+ 'schema_changed': schema_changed,
40
+ 'source_not_in_target': source_not_in_target,
41
+ 'target_not_in_source': target_not_in_source,
42
+ 'source_columns': source_columns,
43
+ 'target_columns': target_columns,
44
+ 'new_target_types': new_target_types
45
+ } %}
46
+
47
+ {% set msg %}
48
+ In {{ target_relation }}:
49
+ Schema changed: {{ schema_changed }}
50
+ Source columns not in target: {{ source_not_in_target }}
51
+ Target columns not in source: {{ target_not_in_source }}
52
+ New column types: {{ new_target_types }}
53
+ {% endset %}
54
+
55
+ {% do log(msg) %}
56
+
57
+ {{ return(changes_dict) }}
58
+
59
+ {% endmacro %}
60
+
61
+
62
+ {% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
63
+
64
+ {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}
65
+
66
+ {%- if on_schema_change == 'append_new_columns'-%}
67
+ {%- if add_to_target_arr | length > 0 -%}
68
+ {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}
69
+ {%- endif -%}
70
+
71
+ {% elif on_schema_change == 'sync_all_columns' %}
72
+ {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}
73
+ {%- set new_target_types = schema_changes_dict['new_target_types'] -%}
74
+
75
+ {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}
76
+ {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}
77
+ {% endif %}
78
+
79
+ {% if new_target_types != [] %}
80
+ {% for ntt in new_target_types %}
81
+ {% set column_name = ntt['column_name'] %}
82
+ {% set new_type = ntt['new_type'] %}
83
+ {% do alter_column_type(target_relation, column_name, new_type) %}
84
+ {% endfor %}
85
+ {% endif %}
86
+
87
+ {% endif %}
88
+
89
+ {% set schema_change_message %}
90
+ In {{ target_relation }}:
91
+ Schema change approach: {{ on_schema_change }}
92
+ Columns added: {{ add_to_target_arr }}
93
+ Columns removed: {{ remove_from_target_arr }}
94
+ Data types changed: {{ new_target_types }}
95
+ {% endset %}
96
+
97
+ {% do log(schema_change_message) %}
98
+
99
+ {% endmacro %}
100
+
101
+
102
+ {% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
103
+
104
+ {% if on_schema_change == 'ignore' %}
105
+
106
+ {{ return({}) }}
107
+
108
+ {% else %}
109
+
110
+ {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
111
+
112
+ {% if schema_changes_dict['schema_changed'] %}
113
+
114
+ {% if on_schema_change == 'fail' %}
115
+
116
+ {% set fail_msg %}
117
+ The source and target schemas on this incremental model are out of sync!
118
+ They can be reconciled in several ways:
119
+ - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.
120
+ - Re-run the incremental model with `full_refresh: True` to update the target schema.
121
+ - update the schema manually and re-run the process.
122
+
123
+ Additional troubleshooting context:
124
+ Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}
125
+ Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}
126
+ New column types: {{ schema_changes_dict['new_target_types'] }}
127
+ {% endset %}
128
+
129
+ {% do exceptions.raise_compiler_error(fail_msg) %}
130
+
131
+ {# -- unless we ignore, run the sync operation per the config #}
132
+ {% else %}
133
+
134
+ {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
135
+
136
+ {% endif %}
137
+
138
+ {% endif %}
139
+
140
+ {{ return(schema_changes_dict['source_columns']) }}
141
+
142
+ {% endif %}
143
+
144
+ {% endmacro %}
@@ -0,0 +1,79 @@
1
+ {% macro get_incremental_append_sql(arg_dict) %}
2
+
3
+ {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}
4
+
5
+ {% endmacro %}
6
+
7
+
8
+ {% macro default__get_incremental_append_sql(arg_dict) %}
9
+
10
+ {% do return(get_insert_into_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["dest_columns"])) %}
11
+
12
+ {% endmacro %}
13
+
14
+
15
+ {# snowflake #}
16
+ {% macro get_incremental_delete_insert_sql(arg_dict) %}
17
+
18
+ {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}
19
+
20
+ {% endmacro %}
21
+
22
+ {% macro default__get_incremental_delete_insert_sql(arg_dict) %}
23
+
24
+ {% do return(get_delete_insert_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["unique_key"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %}
25
+
26
+ {% endmacro %}
27
+
28
+
29
+ {# snowflake, bigquery, spark #}
30
+ {% macro get_incremental_merge_sql(arg_dict) %}
31
+
32
+ {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}
33
+
34
+ {% endmacro %}
35
+
36
+ {% macro default__get_incremental_merge_sql(arg_dict) %}
37
+
38
+ {% do return(get_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["unique_key"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %}
39
+
40
+ {% endmacro %}
41
+
42
+
43
+ {% macro get_incremental_insert_overwrite_sql(arg_dict) %}
44
+
45
+ {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}
46
+
47
+ {% endmacro %}
48
+
49
+ {% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}
50
+
51
+ {% do return(get_insert_overwrite_merge_sql(arg_dict["target_relation"], arg_dict["temp_relation"], arg_dict["dest_columns"], arg_dict["incremental_predicates"])) %}
52
+
53
+ {% endmacro %}
54
+
55
+
56
+ {% macro get_incremental_default_sql(arg_dict) %}
57
+
58
+ {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}
59
+
60
+ {% endmacro %}
61
+
62
+ {% macro default__get_incremental_default_sql(arg_dict) %}
63
+
64
+ {% do return(get_incremental_append_sql(arg_dict)) %}
65
+
66
+ {% endmacro %}
67
+
68
+
69
+ {% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}
70
+
71
+ {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
72
+
73
+ insert into {{ target_relation }} ({{ dest_cols_csv }})
74
+ (
75
+ select {{ dest_cols_csv }}
76
+ from {{ temp_relation }}
77
+ )
78
+
79
+ {% endmacro %}
@@ -0,0 +1,121 @@
1
+ {% materialization materialized_view, default %}
2
+ {% set existing_relation = load_cached_relation(this) %}
3
+ {% set target_relation = this.incorporate(type=this.MaterializedView) %}
4
+ {% set intermediate_relation = make_intermediate_relation(target_relation) %}
5
+ {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}
6
+ {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}
7
+
8
+ {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}
9
+
10
+ {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}
11
+
12
+ {% if build_sql == '' %}
13
+ {{ materialized_view_execute_no_op(target_relation) }}
14
+ {% else %}
15
+ {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}
16
+ {% endif %}
17
+
18
+ {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}
19
+
20
+ {{ return({'relations': [target_relation]}) }}
21
+
22
+ {% endmaterialization %}
23
+
24
+
25
+ {% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}
26
+
27
+ -- backup_relation and intermediate_relation should not already exist in the database
28
+ -- it's possible these exist because of a previous run that exited unexpectedly
29
+ {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}
30
+ {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}
31
+
32
+ -- drop the temp relations if they exist already in the database
33
+ {{ drop_relation_if_exists(preexisting_backup_relation) }}
34
+ {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
35
+
36
+ {{ run_hooks(pre_hooks, inside_transaction=False) }}
37
+
38
+ {% endmacro %}
39
+
40
+
41
+ {% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}
42
+
43
+ -- drop the temp relations if they exist to leave the database clean for the next run
44
+ {{ drop_relation_if_exists(backup_relation) }}
45
+ {{ drop_relation_if_exists(intermediate_relation) }}
46
+
47
+ {{ run_hooks(post_hooks, inside_transaction=False) }}
48
+
49
+ {% endmacro %}
50
+
51
+
52
+ {% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}
53
+
54
+ {% set full_refresh_mode = should_full_refresh() %}
55
+
56
+ -- determine the scenario we're in: create, full_refresh, alter, refresh data
57
+ {% if existing_relation is none %}
58
+ {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}
59
+ {% elif full_refresh_mode or not existing_relation.is_materialized_view %}
60
+ {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}
61
+ {% else %}
62
+
63
+ -- get config options
64
+ {% set on_configuration_change = config.get('on_configuration_change') %}
65
+ {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}
66
+
67
+ {% if configuration_changes is none %}
68
+ {% set build_sql = refresh_materialized_view(target_relation) %}
69
+
70
+ {% elif on_configuration_change == 'apply' %}
71
+ {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}
72
+ {% elif on_configuration_change == 'continue' %}
73
+ {% set build_sql = '' %}
74
+ {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }}
75
+ {% elif on_configuration_change == 'fail' %}
76
+ {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }}
77
+
78
+ {% else %}
79
+ -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'
80
+ {{ exceptions.raise_compiler_error("Unexpected configuration scenario") }}
81
+
82
+ {% endif %}
83
+
84
+ {% endif %}
85
+
86
+ {% do return(build_sql) %}
87
+
88
+ {% endmacro %}
89
+
90
+
91
+ {% macro materialized_view_execute_no_op(target_relation) %}
92
+ {% do store_raw_result(
93
+ name="main",
94
+ message="skip " ~ target_relation,
95
+ code="skip",
96
+ rows_affected="-1"
97
+ ) %}
98
+ {% endmacro %}
99
+
100
+
101
+ {% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}
102
+
103
+ -- `BEGIN` happens here:
104
+ {{ run_hooks(pre_hooks, inside_transaction=True) }}
105
+
106
+ {% set grant_config = config.get('grants') %}
107
+
108
+ {% call statement(name="main") %}
109
+ {{ build_sql }}
110
+ {% endcall %}
111
+
112
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
113
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
114
+
115
+ {% do persist_docs(target_relation, model) %}
116
+
117
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
118
+
119
+ {{ adapter.commit() }}
120
+
121
+ {% endmacro %}
@@ -0,0 +1,64 @@
1
+ {% materialization table, default %}
2
+
3
+ {%- set existing_relation = load_cached_relation(this) -%}
4
+ {%- set target_relation = this.incorporate(type='table') %}
5
+ {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}
6
+ -- the intermediate_relation should not already exist in the database; get_relation
7
+ -- will return None in that case. Otherwise, we get a relation that we can drop
8
+ -- later, before we try to use this name for the current operation
9
+ {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}
10
+ /*
11
+ See ../view/view.sql for more information about this relation.
12
+ */
13
+ {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}
14
+ {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
15
+ -- as above, the backup_relation should not already exist
16
+ {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
17
+ -- grab current tables grants config for comparision later on
18
+ {% set grant_config = config.get('grants') %}
19
+
20
+ -- drop the temp relations if they exist already in the database
21
+ {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
22
+ {{ drop_relation_if_exists(preexisting_backup_relation) }}
23
+
24
+ {{ run_hooks(pre_hooks, inside_transaction=False) }}
25
+
26
+ -- `BEGIN` happens here:
27
+ {{ run_hooks(pre_hooks, inside_transaction=True) }}
28
+
29
+ -- build model
30
+ {% call statement('main') -%}
31
+ {{ get_create_table_as_sql(False, intermediate_relation, sql) }}
32
+ {%- endcall %}
33
+
34
+ -- cleanup
35
+ {% if existing_relation is not none %}
36
+ /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped
37
+ since the variable was first set. */
38
+ {% set existing_relation = load_cached_relation(existing_relation) %}
39
+ {% if existing_relation is not none %}
40
+ {{ adapter.rename_relation(existing_relation, backup_relation) }}
41
+ {% endif %}
42
+ {% endif %}
43
+
44
+ {{ adapter.rename_relation(intermediate_relation, target_relation) }}
45
+
46
+ {% do create_indexes(target_relation) %}
47
+
48
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
49
+
50
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
51
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
52
+
53
+ {% do persist_docs(target_relation, model) %}
54
+
55
+ -- `COMMIT` happens here
56
+ {{ adapter.commit() }}
57
+
58
+ -- finally, drop the existing/backup relation after the commit
59
+ {{ drop_relation_if_exists(backup_relation) }}
60
+
61
+ {{ run_hooks(post_hooks, inside_transaction=False) }}
62
+
63
+ {{ return({'relations': [target_relation]}) }}
64
+ {% endmaterialization %}
@@ -0,0 +1,72 @@
1
+ {%- materialization view, default -%}
2
+
3
+ {%- set existing_relation = load_cached_relation(this) -%}
4
+ {%- set target_relation = this.incorporate(type='view') -%}
5
+ {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}
6
+
7
+ -- the intermediate_relation should not already exist in the database; get_relation
8
+ -- will return None in that case. Otherwise, we get a relation that we can drop
9
+ -- later, before we try to use this name for the current operation
10
+ {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}
11
+ /*
12
+ This relation (probably) doesn't exist yet. If it does exist, it's a leftover from
13
+ a previous run, and we're going to try to drop it immediately. At the end of this
14
+ materialization, we're going to rename the "existing_relation" to this identifier,
15
+ and then we're going to drop it. In order to make sure we run the correct one of:
16
+ - drop view ...
17
+ - drop table ...
18
+
19
+ We need to set the type of this relation to be the type of the existing_relation, if it exists,
20
+ or else "view" as a sane default if it does not. Note that if the existing_relation does not
21
+ exist, then there is nothing to move out of the way and subsequentally drop. In that case,
22
+ this relation will be effectively unused.
23
+ */
24
+ {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}
25
+ {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
26
+ -- as above, the backup_relation should not already exist
27
+ {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
28
+ -- grab current tables grants config for comparision later on
29
+ {% set grant_config = config.get('grants') %}
30
+
31
+ {{ run_hooks(pre_hooks, inside_transaction=False) }}
32
+
33
+ -- drop the temp relations if they exist already in the database
34
+ {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
35
+ {{ drop_relation_if_exists(preexisting_backup_relation) }}
36
+
37
+ -- `BEGIN` happens here:
38
+ {{ run_hooks(pre_hooks, inside_transaction=True) }}
39
+
40
+ -- build model
41
+ {% call statement('main') -%}
42
+ {{ get_create_view_as_sql(intermediate_relation, sql) }}
43
+ {%- endcall %}
44
+
45
+ -- cleanup
46
+ -- move the existing view out of the way
47
+ {% if existing_relation is not none %}
48
+ /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped
49
+ since the variable was first set. */
50
+ {% set existing_relation = load_cached_relation(existing_relation) %}
51
+ {% if existing_relation is not none %}
52
+ {{ adapter.rename_relation(existing_relation, backup_relation) }}
53
+ {% endif %}
54
+ {% endif %}
55
+ {{ adapter.rename_relation(intermediate_relation, target_relation) }}
56
+
57
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
58
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
59
+
60
+ {% do persist_docs(target_relation, model) %}
61
+
62
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
63
+
64
+ {{ adapter.commit() }}
65
+
66
+ {{ drop_relation_if_exists(backup_relation) }}
67
+
68
+ {{ run_hooks(post_hooks, inside_transaction=False) }}
69
+
70
+ {{ return({'relations': [target_relation]}) }}
71
+
72
+ {%- endmaterialization -%}
@@ -0,0 +1,128 @@
1
+
2
+ {% macro create_csv_table(model, agate_table) -%}
3
+ {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}
4
+ {%- endmacro %}
5
+
6
+ {% macro default__create_csv_table(model, agate_table) %}
7
+ {%- set column_override = model['config'].get('column_types', {}) -%}
8
+ {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
9
+
10
+ {% set sql %}
11
+ create table {{ this.render() }} (
12
+ {%- for col_name in agate_table.column_names -%}
13
+ {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
14
+ {%- set type = column_override.get(col_name, inferred_type) -%}
15
+ {%- set column_name = (col_name | string) -%}
16
+ {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
17
+ {%- endfor -%}
18
+ )
19
+ {% endset %}
20
+
21
+ {% call statement('_') -%}
22
+ {{ sql }}
23
+ {%- endcall %}
24
+
25
+ {{ return(sql) }}
26
+ {% endmacro %}
27
+
28
+
29
+ {% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
30
+ {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}
31
+ {%- endmacro %}
32
+
33
+ {% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
34
+ {% set sql = "" %}
35
+ {% if full_refresh %}
36
+ {{ adapter.drop_relation(old_relation) }}
37
+ {% set sql = create_csv_table(model, agate_table) %}
38
+ {% else %}
39
+ {{ adapter.truncate_relation(old_relation) }}
40
+ {% set sql = "truncate table " ~ old_relation %}
41
+ {% endif %}
42
+
43
+ {{ return(sql) }}
44
+ {% endmacro %}
45
+
46
+
47
+ {% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}
48
+ {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}
49
+ {% endmacro %}
50
+
51
+ {% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}
52
+ {{ create_or_truncate_sql }};
53
+ -- dbt seed --
54
+ {{ insert_sql }}
55
+ {% endmacro %}
56
+
57
+
58
+ {% macro get_binding_char() -%}
59
+ {{ adapter.dispatch('get_binding_char', 'dbt')() }}
60
+ {%- endmacro %}
61
+
62
+ {% macro default__get_binding_char() %}
63
+ {{ return('%s') }}
64
+ {% endmacro %}
65
+
66
+
67
+ {% macro get_batch_size() -%}
68
+ {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}
69
+ {%- endmacro %}
70
+
71
+ {% macro default__get_batch_size() %}
72
+ {{ return(10000) }}
73
+ {% endmacro %}
74
+
75
+
76
+ {% macro get_seed_column_quoted_csv(model, column_names) %}
77
+ {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
78
+ {% set quoted = [] %}
79
+ {% for col in column_names -%}
80
+ {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}
81
+ {%- endfor %}
82
+
83
+ {%- set dest_cols_csv = quoted | join(', ') -%}
84
+ {{ return(dest_cols_csv) }}
85
+ {% endmacro %}
86
+
87
+
88
+ {% macro load_csv_rows(model, agate_table) -%}
89
+ {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}
90
+ {%- endmacro %}
91
+
92
+ {% macro default__load_csv_rows(model, agate_table) %}
93
+
94
+ {% set batch_size = get_batch_size() %}
95
+
96
+ {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
97
+ {% set bindings = [] %}
98
+
99
+ {% set statements = [] %}
100
+
101
+ {% for chunk in agate_table.rows | batch(batch_size) %}
102
+ {% set bindings = [] %}
103
+
104
+ {% for row in chunk %}
105
+ {% do bindings.extend(row) %}
106
+ {% endfor %}
107
+
108
+ {% set sql %}
109
+ insert into {{ this.render() }} ({{ cols_sql }}) values
110
+ {% for row in chunk -%}
111
+ ({%- for column in agate_table.column_names -%}
112
+ {{ get_binding_char() }}
113
+ {%- if not loop.last%},{%- endif %}
114
+ {%- endfor -%})
115
+ {%- if not loop.last%},{%- endif %}
116
+ {%- endfor %}
117
+ {% endset %}
118
+
119
+ {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
120
+
121
+ {% if loop.index0 == 0 %}
122
+ {% do statements.append(sql) %}
123
+ {% endif %}
124
+ {% endfor %}
125
+
126
+ {# Return SQL so we can render it out into the compiled files #}
127
+ {{ return(statements[0]) }}
128
+ {% endmacro %}
@@ -0,0 +1,60 @@
1
+ {% materialization seed, default %}
2
+
3
+ {%- set identifier = model['alias'] -%}
4
+ {%- set full_refresh_mode = (should_full_refresh()) -%}
5
+
6
+ {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
7
+
8
+ {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
9
+ {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
10
+
11
+ {%- set grant_config = config.get('grants') -%}
12
+ {%- set agate_table = load_agate_table() -%}
13
+ -- grab current tables grants config for comparison later on
14
+
15
+ {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}
16
+
17
+ {{ run_hooks(pre_hooks, inside_transaction=False) }}
18
+
19
+ -- `BEGIN` happens here:
20
+ {{ run_hooks(pre_hooks, inside_transaction=True) }}
21
+
22
+ -- build model
23
+ {% set create_table_sql = "" %}
24
+ {% if exists_as_view %}
25
+ {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }}
26
+ {% elif exists_as_table %}
27
+ {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}
28
+ {% else %}
29
+ {% set create_table_sql = create_csv_table(model, agate_table) %}
30
+ {% endif %}
31
+
32
+ {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}
33
+ {% set rows_affected = (agate_table.rows | length) %}
34
+ {% set sql = load_csv_rows(model, agate_table) %}
35
+
36
+ {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}
37
+ {{ get_csv_sql(create_table_sql, sql) }};
38
+ {% endcall %}
39
+
40
+ {% set target_relation = this.incorporate(type='table') %}
41
+
42
+ {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}
43
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
44
+
45
+ {% do persist_docs(target_relation, model) %}
46
+
47
+ {% if full_refresh_mode or not exists_as_table %}
48
+ {% do create_indexes(target_relation) %}
49
+ {% endif %}
50
+
51
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
52
+
53
+ -- `COMMIT` happens here
54
+ {{ adapter.commit() }}
55
+
56
+ {{ run_hooks(post_hooks, inside_transaction=False) }}
57
+
58
+ {{ return({'relations': [target_relation]}) }}
59
+
60
+ {% endmaterialization %}