dbt-adapters 1.22.2__py3-none-any.whl

dbt/include/global_project/macros/materializations/models/materialized_view.sql

@@ -0,0 +1,121 @@
+{% materialization materialized_view, default %}
+    {% set existing_relation = load_cached_relation(this) %}
+    {% set target_relation = this.incorporate(type=this.MaterializedView) %}
+    {% set intermediate_relation = make_intermediate_relation(target_relation) %}
+    {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}
+    {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}
+
+    {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}
+
+        {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}
+
+        {% if build_sql == '' %}
+            {{ materialized_view_execute_no_op(target_relation) }}
+        {% else %}
+            {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}
+        {% endif %}
+
+    {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}
+
+    {{ return({'relations': [target_relation]}) }}
+
+{% endmaterialization %}
+
+
+{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}
+
+    -- backup_relation and intermediate_relation should not already exist in the database
+    -- it's possible these exist because of a previous run that exited unexpectedly
+    {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}
+    {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}
+
+    -- drop the temp relations if they exist already in the database
+    {{ drop_relation_if_exists(preexisting_backup_relation) }}
+    {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+
+    {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+{% endmacro %}
+
+
+{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}
+
+    -- drop the temp relations if they exist to leave the database clean for the next run
+    {{ drop_relation_if_exists(backup_relation) }}
+    {{ drop_relation_if_exists(intermediate_relation) }}
+
+    {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+{% endmacro %}
+
+
+{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}
+
+    {% set full_refresh_mode = should_full_refresh() %}
+
+    -- determine the scenario we're in: create, full_refresh, alter, refresh data
+    {% if existing_relation is none %}
+        {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}
+    {% elif full_refresh_mode or not existing_relation.is_materialized_view %}
+        {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}
+    {% else %}
+
+        -- get config options
+        {% set on_configuration_change = config.get('on_configuration_change') %}
+        {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}
+
+        {% if configuration_changes is none %}
+            {% set build_sql = refresh_materialized_view(target_relation) %}
+
+        {% elif on_configuration_change == 'apply' %}
+            {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}
+        {% elif on_configuration_change == 'continue' %}
+            {% set build_sql = '' %}
+            {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation.render() ~ "`") }}
+        {% elif on_configuration_change == 'fail' %}
+            {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation.render() ~ "`") }}
+
+        {% else %}
+            -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'
+            {{ exceptions.raise_compiler_error("Unexpected configuration scenario") }}
+
+        {% endif %}
+
+    {% endif %}
+
+    {% do return(build_sql) %}
+
+{% endmacro %}
+
+
+{% macro materialized_view_execute_no_op(target_relation) %}
+    {% do store_raw_result(
+        name="main",
+        message="skip " ~ target_relation,
+        code="skip",
+        rows_affected="-1"
+    ) %}
+{% endmacro %}
+
+
+{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}
+
+    -- `BEGIN` happens here:
+    {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+    {% set grant_config = config.get('grants') %}
+
+    {% call statement(name="main") %}
+        {{ build_sql }}
+    {% endcall %}
+
+    {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+    {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+    {% do persist_docs(target_relation, model) %}
+
+    {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+    {{ adapter.commit() }}
+
+{% endmacro %}

dbt/include/global_project/macros/materializations/models/table.sql

@@ -0,0 +1,64 @@
+{% materialization table, default %}
+
+  {%- set existing_relation = load_cached_relation(this) -%}
+  {%- set target_relation = this.incorporate(type='table') %}
+  {%- set intermediate_relation =  make_intermediate_relation(target_relation) -%}
+  -- the intermediate_relation should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation
+  {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}
+  /*
+      See ../view/view.sql for more information about this relation.
+  */
+  {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}
+  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
+  -- as above, the backup_relation should not already exist
+  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+  -- grab current tables grants config for comparision later on
+  {% set grant_config = config.get('grants') %}
+
+  -- drop the temp relations if they exist already in the database
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% call statement('main') -%}
+    {{ get_create_table_as_sql(False, intermediate_relation, sql) }}
+  {%- endcall %}
+
+  {% do create_indexes(intermediate_relation) %}
+
+  -- cleanup
+  {% if existing_relation is not none %}
+     /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped
+        since the variable was first set. */
+    {% set existing_relation = load_cached_relation(existing_relation) %}
+    {% if existing_relation is not none %}
+        {{ adapter.rename_relation(existing_relation, backup_relation) }}
+    {% endif %}
+  {% endif %}
+
+  {{ adapter.rename_relation(intermediate_relation, target_relation) }}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  -- `COMMIT` happens here
+  {{ adapter.commit() }}
+
+  -- finally, drop the existing/backup relation after the commit
+  {{ drop_relation_if_exists(backup_relation) }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+{% endmaterialization %}

dbt/include/global_project/macros/materializations/models/view.sql

@@ -0,0 +1,72 @@
+{%- materialization view, default -%}
+
+  {%- set existing_relation = load_cached_relation(this) -%}
+  {%- set target_relation = this.incorporate(type='view') -%}
+  {%- set intermediate_relation =  make_intermediate_relation(target_relation) -%}
+
+  -- the intermediate_relation should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation
+  {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}
+  /*
+     This relation (probably) doesn't exist yet. If it does exist, it's a leftover from
+     a previous run, and we're going to try to drop it immediately. At the end of this
+     materialization, we're going to rename the "existing_relation" to this identifier,
+     and then we're going to drop it. In order to make sure we run the correct one of:
+       - drop view ...
+       - drop table ...
+
+     We need to set the type of this relation to be the type of the existing_relation, if it exists,
+     or else "view" as a sane default if it does not. Note that if the existing_relation does not
+     exist, then there is nothing to move out of the way and subsequentally drop. In that case,
+     this relation will be effectively unused.
+  */
+  {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}
+  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
+  -- as above, the backup_relation should not already exist
+  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+  -- grab current tables grants config for comparision later on
+  {% set grant_config = config.get('grants') %}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- drop the temp relations if they exist already in the database
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% call statement('main') -%}
+    {{ get_create_view_as_sql(intermediate_relation, sql) }}
+  {%- endcall %}
+
+  -- cleanup
+  -- move the existing view out of the way
+  {% if existing_relation is not none %}
+     /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped
+        since the variable was first set. */
+    {% set existing_relation = load_cached_relation(existing_relation) %}
+    {% if existing_relation is not none %}
+        {{ adapter.rename_relation(existing_relation, backup_relation) }}
+    {% endif %}
+  {% endif %}
+  {{ adapter.rename_relation(intermediate_relation, target_relation) }}
+
+  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  {{ adapter.commit() }}
+
+  {{ drop_relation_if_exists(backup_relation) }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{%- endmaterialization -%}

dbt/include/global_project/macros/materializations/seeds/helpers.sql

@@ -0,0 +1,128 @@
+
+{% macro create_csv_table(model, agate_table) -%}
+  {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}
+{%- endmacro %}
+
+{% macro default__create_csv_table(model, agate_table) %}
+  {%- set column_override = model['config'].get('column_types', {}) -%}
+  {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
+
+  {% set sql %}
+    create table {{ this.render() }} (
+        {%- for col_name in agate_table.column_names -%}
+            {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
+            {%- set type = column_override.get(col_name, inferred_type) -%}
+            {%- set column_name = (col_name | string) -%}
+            {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
+        {%- endfor -%}
+    )
+  {% endset %}
+
+  {% call statement('_') -%}
+    {{ sql }}
+  {%- endcall %}
+
+  {{ return(sql) }}
+{% endmacro %}
+
+
+{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
+  {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}
+{%- endmacro %}
+
+{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
+    {% set sql = "" %}
+    {% if full_refresh %}
+        {{ adapter.drop_relation(old_relation) }}
+        {% set sql = create_csv_table(model, agate_table) %}
+    {% else %}
+        {{ adapter.truncate_relation(old_relation) }}
+        {% set sql = "truncate table " ~ old_relation.render() %}
+    {% endif %}
+
+    {{ return(sql) }}
+{% endmacro %}
+
+
+{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}
+    {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}
+{% endmacro %}
+
+{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}
+    {{ create_or_truncate_sql }};
+    -- dbt seed --
+    {{ insert_sql }}
+{% endmacro %}
+
+
+{% macro get_binding_char() -%}
+  {{ adapter.dispatch('get_binding_char', 'dbt')() }}
+{%- endmacro %}
+
+{% macro default__get_binding_char() %}
+  {{ return('%s') }}
+{% endmacro %}
+
+
+{% macro get_batch_size() -%}
+  {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}
+{%- endmacro %}
+
+{% macro default__get_batch_size() %}
+  {{ return(10000) }}
+{% endmacro %}
+
+
+{% macro get_seed_column_quoted_csv(model, column_names) %}
+  {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
+    {% set quoted = [] %}
+    {% for col in column_names -%}
+        {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}
+    {%- endfor %}
+
+    {%- set dest_cols_csv = quoted | join(', ') -%}
+    {{ return(dest_cols_csv) }}
+{% endmacro %}
+
+
+{% macro load_csv_rows(model, agate_table) -%}
+  {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}
+{%- endmacro %}
+
+{% macro default__load_csv_rows(model, agate_table) %}
+
+  {% set batch_size = get_batch_size() %}
+
+  {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+  {% set bindings = [] %}
+
+  {% set statements = [] %}
+
+  {% for chunk in agate_table.rows | batch(batch_size) %}
+      {% set bindings = [] %}
+
+      {% for row in chunk %}
+          {% do bindings.extend(row) %}
+      {% endfor %}
+
+      {% set sql %}
+          insert into {{ this.render() }} ({{ cols_sql }}) values
+          {% for row in chunk -%}
+              ({%- for column in agate_table.column_names -%}
+                  {{ get_binding_char() }}
+                  {%- if not loop.last%},{%- endif %}
+              {%- endfor -%})
+              {%- if not loop.last%},{%- endif %}
+          {%- endfor %}
+      {% endset %}
+
+      {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+
+      {% if loop.index0 == 0 %}
+          {% do statements.append(sql) %}
+      {% endif %}
+  {% endfor %}
+
+  {# Return SQL so we can render it out into the compiled files #}
+  {{ return(statements[0]) }}
+{% endmacro %}

dbt/include/global_project/macros/materializations/seeds/seed.sql

@@ -0,0 +1,60 @@
+{% materialization seed, default %}
+
+  {%- set identifier = model['alias'] -%}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+
+  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
+  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+
+  {%- set grant_config = config.get('grants') -%}
+  {%- set agate_table = load_agate_table() -%}
+  -- grab current tables grants config for comparison later on
+
+  {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% set create_table_sql = "" %}
+  {% if exists_as_view %}
+    {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation.render())) }}
+  {% elif exists_as_table %}
+    {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}
+  {% else %}
+    {% set create_table_sql = create_csv_table(model, agate_table) %}
+  {% endif %}
+
+  {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}
+  {% set rows_affected = (agate_table.rows | length) %}
+  {% set sql = load_csv_rows(model, agate_table) %}
+
+  {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}
+    {{ get_csv_sql(create_table_sql, sql) }};
+  {% endcall %}
+
+  {% set target_relation = this.incorporate(type='table') %}
+
+  {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  {% if full_refresh_mode or not exists_as_table %}
+    {% do create_indexes(target_relation) %}
+  {% endif %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {{ adapter.commit() }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{% endmaterialization %}