dbt-adapters 0.1.0a1__py3-none-any.whl

dbt/include/global_project/macros/generic_test_sql/unique.sql

@@ -0,0 +1,12 @@
+{% macro default__test_unique(model, column_name) %}
+
+select
+    {{ column_name }} as unique_field,
+    count(*) as n_records
+
+from {{ model }}
+where {{ column_name }} is not null
+group by {{ column_name }}
+having count(*) > 1
+
+{% endmacro %}

dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql

@@ -0,0 +1,36 @@
+
+{#
+    Renders a alias name given a custom alias name. If the custom
+    alias name is none, then the resulting alias is just the filename of the
+    model. If an alias override is specified, then that is used.
+
+    This macro can be overriden in projects to define different semantics
+    for rendering a alias name.
+
+    Arguments:
+    custom_alias_name: The custom alias name specified for a model, or none
+    node: The available node that an alias is being generated for, or none
+
+#}
+
+{% macro generate_alias_name(custom_alias_name=none, node=none) -%}
+    {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}
+{%- endmacro %}
+
+{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}
+
+    {%- if custom_alias_name -%}
+
+        {{ custom_alias_name | trim }}
+
+    {%- elif node.version -%}
+
+        {{ return(node.name ~ "_v" ~ (node.version | replace(".", "_"))) }}
+
+    {%- else -%}
+
+        {{ node.name }}
+
+    {%- endif -%}
+
+{%- endmacro %}

dbt/include/global_project/macros/get_custom_name/get_custom_database.sql

@@ -0,0 +1,32 @@
+{#
+    Renders a database name given a custom database name. If the custom
+    database name is none, then the resulting database is just the "database"
+    value in the specified target. If a database override is specified, then
+    the custom database name is used instead of the default "database" value.
+
+    This macro can be overriden in projects to define different semantics
+    for rendering a database name.
+
+    Arguments:
+    custom_database_name: The custom database name specified for a model, or none
+    node: The node the database is being generated for
+
+#}
+
+{% macro generate_database_name(custom_database_name=none, node=none) -%}
+    {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}
+{%- endmacro %}
+
+{% macro default__generate_database_name(custom_database_name=none, node=none) -%}
+    {%- set default_database = target.database -%}
+    {%- if custom_database_name is none -%}
+
+        {{ default_database }}
+
+    {%- else -%}
+
+        {{ custom_database_name }}
+
+    {%- endif -%}
+
+{%- endmacro %}

dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql

@@ -0,0 +1,60 @@
+
+{#
+    Renders a schema name given a custom schema name. If the custom
+    schema name is none, then the resulting schema is just the "schema"
+    value in the specified target. If a schema override is specified, then
+    the resulting schema is the default schema concatenated with the
+    custom schema.
+
+    This macro can be overriden in projects to define different semantics
+    for rendering a schema name.
+
+    Arguments:
+    custom_schema_name: The custom schema name specified for a model, or none
+    node: The node the schema is being generated for
+
+#}
+{% macro generate_schema_name(custom_schema_name=none, node=none) -%}
+    {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}
+{% endmacro %}
+
+{% macro default__generate_schema_name(custom_schema_name, node) -%}
+
+    {%- set default_schema = target.schema -%}
+    {%- if custom_schema_name is none -%}
+
+        {{ default_schema }}
+
+    {%- else -%}
+
+        {{ default_schema }}_{{ custom_schema_name | trim }}
+
+    {%- endif -%}
+
+{%- endmacro %}
+
+
+{#
+    Renders a schema name given a custom schema name. In production, this macro
+    will render out the overriden schema name for a model. Otherwise, the default
+    schema specified in the active target is used.
+
+    Arguments:
+    custom_schema_name: The custom schema name specified for a model, or none
+    node: The node the schema is being generated for
+
+#}
+{% macro generate_schema_name_for_env(custom_schema_name, node) -%}
+
+    {%- set default_schema = target.schema -%}
+    {%- if target.name == 'prod' and custom_schema_name is not none -%}
+
+        {{ custom_schema_name | trim }}
+
+    {%- else -%}
+
+        {{ default_schema }}
+
+    {%- endif -%}
+
+{%- endmacro %}

dbt/include/global_project/macros/materializations/configs.sql

@@ -0,0 +1,21 @@
+{% macro set_sql_header(config) -%}
+  {{ config.set('sql_header', caller()) }}
+{%- endmacro %}
+
+
+{% macro should_full_refresh() %}
+  {% set config_full_refresh = config.get('full_refresh') %}
+  {% if config_full_refresh is none %}
+    {% set config_full_refresh = flags.FULL_REFRESH %}
+  {% endif %}
+  {% do return(config_full_refresh) %}
+{% endmacro %}
+
+
+{% macro should_store_failures() %}
+  {% set config_store_failures = config.get('store_failures') %}
+  {% if config_store_failures is none %}
+    {% set config_store_failures = flags.STORE_FAILURES %}
+  {% endif %}
+  {% do return(config_store_failures) %}
+{% endmacro %}

dbt/include/global_project/macros/materializations/hooks.sql

@@ -0,0 +1,35 @@
+{% macro run_hooks(hooks, inside_transaction=True) %}
+  {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction)  %}
+    {% if not inside_transaction and loop.first %}
+      {% call statement(auto_begin=inside_transaction) %}
+        commit;
+      {% endcall %}
+    {% endif %}
+    {% set rendered = render(hook.get('sql')) | trim %}
+    {% if (rendered | length) > 0 %}
+      {% call statement(auto_begin=inside_transaction) %}
+        {{ rendered }}
+      {% endcall %}
+    {% endif %}
+  {% endfor %}
+{% endmacro %}
+
+
+{% macro make_hook_config(sql, inside_transaction) %}
+    {{ tojson({"sql": sql, "transaction": inside_transaction}) }}
+{% endmacro %}
+
+
+{% macro before_begin(sql) %}
+    {{ make_hook_config(sql, inside_transaction=False) }}
+{% endmacro %}
+
+
+{% macro in_transaction(sql) %}
+    {{ make_hook_config(sql, inside_transaction=True) }}
+{% endmacro %}
+
+
+{% macro after_commit(sql) %}
+    {{ make_hook_config(sql, inside_transaction=False) }}
+{% endmacro %}

dbt/include/global_project/macros/materializations/models/clone/can_clone_table.sql

@@ -0,0 +1,7 @@
+{% macro can_clone_table() %}
+    {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}
+{% endmacro %}
+
+{% macro default__can_clone_table() %}
+    {{ return(False) }}
+{% endmacro %}

dbt/include/global_project/macros/materializations/models/clone/clone.sql

@@ -0,0 +1,67 @@
+{%- materialization clone, default -%}
+
+  {%- set relations = {'relations': []} -%}
+
+  {%- if not defer_relation -%}
+      -- nothing to do
+      {{ log("No relation found in state manifest for " ~ model.unique_id, info=True) }}
+      {{ return(relations) }}
+  {%- endif -%}
+
+  {%- set existing_relation = load_cached_relation(this) -%}
+
+  {%- if existing_relation and not flags.FULL_REFRESH -%}
+      -- noop!
+      {{ log("Relation " ~ existing_relation ~ " already exists", info=True) }}
+      {{ return(relations) }}
+  {%- endif -%}
+
+  {%- set other_existing_relation = load_cached_relation(defer_relation) -%}
+
+  -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table
+  -- Otherwise, this will be a view
+
+  {% set can_clone_table = can_clone_table() %}
+
+  {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}
+
+      {%- set target_relation = this.incorporate(type='table') -%}
+      {% if existing_relation is not none and not existing_relation.is_table %}
+        {{ log("Dropping relation " ~ existing_relation ~ " because it is of type " ~ existing_relation.type) }}
+        {{ drop_relation_if_exists(existing_relation) }}
+      {% endif %}
+
+      -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'
+      {% call statement('main') %}
+          {% if target_relation and defer_relation and target_relation == defer_relation %}
+              {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation) }}
+          {% else %}
+              {{ create_or_replace_clone(target_relation, defer_relation) }}
+          {% endif %}
+
+      {% endcall %}
+
+      {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+      {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+      {% do persist_docs(target_relation, model) %}
+
+      {{ return({'relations': [target_relation]}) }}
+
+  {%- else -%}
+
+      {%- set target_relation = this.incorporate(type='view') -%}
+
+      -- reuse the view materialization
+      -- TODO: support actual dispatch for materialization macros
+      -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799
+      {% set search_name = "materialization_view_" ~ adapter.type() %}
+      {% if not search_name in context %}
+          {% set search_name = "materialization_view_default" %}
+      {% endif %}
+      {% set materialization_macro = context[search_name] %}
+      {% set relations = materialization_macro() %}
+      {{ return(relations) }}
+
+  {%- endif -%}
+
+{%- endmaterialization -%}

dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql

@@ -0,0 +1,7 @@
+{% macro create_or_replace_clone(this_relation, defer_relation) %}
+    {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}
+{% endmacro %}
+
+{% macro default__create_or_replace_clone(this_relation, defer_relation) %}
+    create or replace table {{ this_relation }} clone {{ defer_relation }}
+{% endmacro %}

dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql

@@ -0,0 +1,80 @@
+/* {#
+    Helper macros for internal use with incremental materializations.
+    Use with care if calling elsewhere.
+#} */
+
+
+{% macro get_quoted_csv(column_names) %}
+
+    {% set quoted = [] %}
+    {% for col in column_names -%}
+        {%- do quoted.append(adapter.quote(col)) -%}
+    {%- endfor %}
+
+    {%- set dest_cols_csv = quoted | join(', ') -%}
+    {{ return(dest_cols_csv) }}
+
+{% endmacro %}
+
+
+{% macro diff_columns(source_columns, target_columns) %}
+
+  {% set result = [] %}
+  {% set source_names = source_columns | map(attribute = 'column') | list %}
+  {% set target_names = target_columns | map(attribute = 'column') | list %}
+
+   {# --check whether the name attribute exists in the target - this does not perform a data type check #}
+   {% for sc in source_columns %}
+     {% if sc.name not in target_names %}
+        {{ result.append(sc) }}
+     {% endif %}
+   {% endfor %}
+
+  {{ return(result) }}
+
+{% endmacro %}
+
+
+{% macro diff_column_data_types(source_columns, target_columns) %}
+
+  {% set result = [] %}
+  {% for sc in source_columns %}
+    {% set tc = target_columns | selectattr("name", "equalto", sc.name) | list | first %}
+    {% if tc %}
+      {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}
+        {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}
+      {% endif %}
+    {% endif %}
+  {% endfor %}
+
+  {{ return(result) }}
+
+{% endmacro %}
+
+{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}
+  {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}
+{% endmacro %}
+
+{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}
+  {%- set default_cols = dest_columns | map(attribute="quoted") | list -%}
+
+  {%- if merge_update_columns and merge_exclude_columns -%}
+    {{ exceptions.raise_compiler_error(
+        'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'
+    )}}
+  {%- elif merge_update_columns -%}
+    {%- set update_columns = merge_update_columns -%}
+  {%- elif merge_exclude_columns -%}
+    {%- set update_columns = [] -%}
+    {%- for column in dest_columns -%}
+      {% if column.column | lower not in merge_exclude_columns | map("lower") | list %}
+        {%- do update_columns.append(column.quoted) -%}
+      {% endif %}
+    {%- endfor -%}
+  {%- else -%}
+    {%- set update_columns = default_cols -%}
+  {%- endif -%}
+
+  {{ return(update_columns) }}
+
+{% endmacro %}

dbt/include/global_project/macros/materializations/models/incremental/incremental.sql

@@ -0,0 +1,92 @@
+
+{% materialization incremental, default -%}
+
+  -- relations
+  {%- set existing_relation = load_cached_relation(this) -%}
+  {%- set target_relation = this.incorporate(type='table') -%}
+  {%- set temp_relation = make_temp_relation(target_relation)-%}
+  {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}
+  {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}
+  {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
+
+  -- configs
+  {%- set unique_key = config.get('unique_key') -%}
+  {%- set full_refresh_mode = (should_full_refresh()  or existing_relation.is_view) -%}
+  {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
+
+  -- the temp_ and backup_ relations should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation. This has to happen before
+  -- BEGIN, in a separate transaction
+  {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
+  {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
+   -- grab current tables grants config for comparision later on
+  {% set grant_config = config.get('grants') %}
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  {% set to_drop = [] %}
+
+  {% if existing_relation is none %}
+      {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}
+  {% elif full_refresh_mode %}
+      {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}
+      {% set need_swap = true %}
+  {% else %}
+    {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}
+    {% do adapter.expand_target_column_types(
+             from_relation=temp_relation,
+             to_relation=target_relation) %}
+    {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}
+    {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}
+    {% if not dest_columns %}
+      {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
+    {% endif %}
+
+    {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
+    {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
+    {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}
+    {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
+    {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
+    {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
+
+  {% endif %}
+
+  {% call statement("main") %}
+      {{ build_sql }}
+  {% endcall %}
+
+  {% if need_swap %}
+      {% do adapter.rename_relation(target_relation, backup_relation) %}
+      {% do adapter.rename_relation(intermediate_relation, target_relation) %}
+      {% do to_drop.append(backup_relation) %}
+  {% endif %}
+
+  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
+    {% do create_indexes(target_relation) %}
+  {% endif %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {% do adapter.commit() %}
+
+  {% for rel in to_drop %}
+      {% do adapter.drop_relation(rel) %}
+  {% endfor %}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{%- endmaterialization %}

dbt/include/global_project/macros/materializations/models/incremental/is_incremental.sql

@@ -0,0 +1,13 @@
+
+{% macro is_incremental() %}
+    {#-- do not run introspective queries in parsing #}
+    {% if not execute %}
+        {{ return(False) }}
+    {% else %}
+        {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
+        {{ return(relation is not none
+                  and relation.type == 'table'
+                  and model.config.materialized == 'incremental'
+                  and not should_full_refresh()) }}
+    {% endif %}
+{% endmacro %}

dbt/include/global_project/macros/materializations/models/incremental/merge.sql

@@ -0,0 +1,131 @@
+{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}
+   -- back compat for old kwarg name
+  {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}
+  {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}
+{%- endmacro %}
+
+{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}
+    {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}
+    {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
+    {%- set merge_update_columns = config.get('merge_update_columns') -%}
+    {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}
+    {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}
+    {%- set sql_header = config.get('sql_header', none) -%}
+
+    {% if unique_key %}
+        {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}
+            {% for key in unique_key %}
+                {% set this_key_match %}
+                    DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}
+                {% endset %}
+                {% do predicates.append(this_key_match) %}
+            {% endfor %}
+        {% else %}
+            {% set unique_key_match %}
+                DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}
+            {% endset %}
+            {% do predicates.append(unique_key_match) %}
+        {% endif %}
+    {% else %}
+        {% do predicates.append('FALSE') %}
+    {% endif %}
+
+    {{ sql_header if sql_header is not none }}
+
+    merge into {{ target }} as DBT_INTERNAL_DEST
+        using {{ source }} as DBT_INTERNAL_SOURCE
+        on {{"(" ~ predicates | join(") and (") ~ ")"}}
+
+    {% if unique_key %}
+    when matched then update set
+        {% for column_name in update_columns -%}
+            {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}
+            {%- if not loop.last %}, {%- endif %}
+        {%- endfor %}
+    {% endif %}
+
+    when not matched then insert
+        ({{ dest_cols_csv }})
+    values
+        ({{ dest_cols_csv }})
+
+{% endmacro %}
+
+
+{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
+  {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}
+{%- endmacro %}
+
+{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}
+
+    {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
+
+    {% if unique_key %}
+        {% if unique_key is sequence and unique_key is not string %}
+            delete from {{target }}
+            using {{ source }}
+            where (
+                {% for key in unique_key %}
+                    {{ source }}.{{ key }} = {{ target }}.{{ key }}
+                    {{ "and " if not loop.last}}
+                {% endfor %}
+                {% if incremental_predicates %}
+                    {% for predicate in incremental_predicates %}
+                        and {{ predicate }}
+                    {% endfor %}
+                {% endif %}
+            );
+        {% else %}
+            delete from {{ target }}
+            where (
+                {{ unique_key }}) in (
+                select ({{ unique_key }})
+                from {{ source }}
+            )
+            {%- if incremental_predicates %}
+                {% for predicate in incremental_predicates %}
+                    and {{ predicate }}
+                {% endfor %}
+            {%- endif -%};
+
+        {% endif %}
+    {% endif %}
+
+    insert into {{ target }} ({{ dest_cols_csv }})
+    (
+        select {{ dest_cols_csv }}
+        from {{ source }}
+    )
+
+{%- endmacro %}
+
+
+{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}
+  {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}
+{%- endmacro %}
+
+{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}
+    {#-- The only time include_sql_header is True: --#}
+    {#-- BigQuery + insert_overwrite strategy + "static" partitions config --#}
+    {#-- We should consider including the sql header at the materialization level instead --#}
+
+    {%- set predicates = [] if predicates is none else [] + predicates -%}
+    {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
+    {%- set sql_header = config.get('sql_header', none) -%}
+
+    {{ sql_header if sql_header is not none and include_sql_header }}
+
+    merge into {{ target }} as DBT_INTERNAL_DEST
+        using {{ source }} as DBT_INTERNAL_SOURCE
+        on FALSE
+
+    when not matched by source
+        {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}
+        then delete
+
+    when not matched then insert
+        ({{ dest_cols_csv }})
+    values
+        ({{ dest_cols_csv }})
+
+{% endmacro %}