dbt-adapters 0.1.0a1__py3-none-any.whl

dbt/adapters/sql/impl.py

@@ -0,0 +1,273 @@
+from typing import Any, List, Optional, Tuple, Type
+
+import agate
+from dbt_common.events.functions import fire_event
+
+from dbt.adapters.base import BaseAdapter, BaseRelation, available
+from dbt.adapters.cache import _make_ref_key_dict
+from dbt.adapters.contracts.connection import AdapterResponse, Connection
+from dbt.adapters.events.types import ColTypeChange, SchemaCreation, SchemaDrop
+from dbt.adapters.exceptions import RelationTypeNullError
+from dbt.adapters.sql.connections import SQLConnectionManager
+
+
+LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching"
+GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation"
+LIST_SCHEMAS_MACRO_NAME = "list_schemas"
+CHECK_SCHEMA_EXISTS_MACRO_NAME = "check_schema_exists"
+CREATE_SCHEMA_MACRO_NAME = "create_schema"
+DROP_SCHEMA_MACRO_NAME = "drop_schema"
+RENAME_RELATION_MACRO_NAME = "rename_relation"
+TRUNCATE_RELATION_MACRO_NAME = "truncate_relation"
+DROP_RELATION_MACRO_NAME = "drop_relation"
+ALTER_COLUMN_TYPE_MACRO_NAME = "alter_column_type"
+VALIDATE_SQL_MACRO_NAME = "validate_sql"
+
+
+class SQLAdapter(BaseAdapter):
+    """The default adapter with the common agate conversions and some SQL
+    methods was implemented. This adapter has a different much shorter list of
+    methods to implement, but some more macros that must be implemented.
+
+    To implement a macro, implement "${adapter_type}__${macro_name}". in the
+    adapter's internal project.
+
+    Methods to implement:
+        - date_function
+
+    Macros to implement:
+        - get_catalog
+        - list_relations_without_caching
+        - get_columns_in_relation
+    """
+
+    ConnectionManager: Type[SQLConnectionManager]
+    connections: SQLConnectionManager
+
+    @available.parse(lambda *a, **k: (None, None))
+    def add_query(
+        self,
+        sql: str,
+        auto_begin: bool = True,
+        bindings: Optional[Any] = None,
+        abridge_sql_log: bool = False,
+    ) -> Tuple[Connection, Any]:
+        """Add a query to the current transaction. A thin wrapper around
+        ConnectionManager.add_query.
+
+        :param sql: The SQL query to add
+        :param auto_begin: If set and there is no transaction in progress,
+            begin a new one.
+        :param bindings: An optional list of bindings for the query.
+        :param abridge_sql_log: If set, limit the raw sql logged to 512
+            characters
+        """
+        return self.connections.add_query(sql, auto_begin, bindings, abridge_sql_log)
+
+    @classmethod
+    def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "text"
+
+    @classmethod
+    def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        # TODO CT-211
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))  # type: ignore[attr-defined]
+        return "float8" if decimals else "integer"
+
+    @classmethod
+    def convert_integer_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "integer"
+
+    @classmethod
+    def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "boolean"
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "timestamp without time zone"
+
+    @classmethod
+    def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "date"
+
+    @classmethod
+    def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "time"
+
+    @classmethod
+    def is_cancelable(cls) -> bool:
+        return True
+
+    def expand_column_types(self, goal, current):
+        reference_columns = {c.name: c for c in self.get_columns_in_relation(goal)}
+
+        target_columns = {c.name: c for c in self.get_columns_in_relation(current)}
+
+        for column_name, reference_column in reference_columns.items():
+            target_column = target_columns.get(column_name)
+
+            if target_column is not None and target_column.can_expand_to(reference_column):
+                col_string_size = reference_column.string_size()
+                new_type = self.Column.string_type(col_string_size)
+                fire_event(
+                    ColTypeChange(
+                        orig_type=target_column.data_type,
+                        new_type=new_type,
+                        table=_make_ref_key_dict(current),
+                    )
+                )
+
+                self.alter_column_type(current, column_name, new_type)
+
+    def alter_column_type(self, relation, column_name, new_column_type) -> None:
+        """
+        1. Create a new column (w/ temp name and correct type)
+        2. Copy data over to it
+        3. Drop the existing column (cascade!)
+        4. Rename the new column to existing column
+        """
+        kwargs = {
+            "relation": relation,
+            "column_name": column_name,
+            "new_column_type": new_column_type,
+        }
+        self.execute_macro(ALTER_COLUMN_TYPE_MACRO_NAME, kwargs=kwargs)
+
+    def drop_relation(self, relation):
+        if relation.type is None:
+            raise RelationTypeNullError(relation)
+
+        self.cache_dropped(relation)
+        self.execute_macro(DROP_RELATION_MACRO_NAME, kwargs={"relation": relation})
+
+    def truncate_relation(self, relation):
+        self.execute_macro(TRUNCATE_RELATION_MACRO_NAME, kwargs={"relation": relation})
+
+    def rename_relation(self, from_relation, to_relation):
+        self.cache_renamed(from_relation, to_relation)
+
+        kwargs = {"from_relation": from_relation, "to_relation": to_relation}
+        self.execute_macro(RENAME_RELATION_MACRO_NAME, kwargs=kwargs)
+
+    def get_columns_in_relation(self, relation):
+        return self.execute_macro(
+            GET_COLUMNS_IN_RELATION_MACRO_NAME, kwargs={"relation": relation}
+        )
+
+    def create_schema(self, relation: BaseRelation) -> None:
+        relation = relation.without_identifier()
+        fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
+        kwargs = {
+            "relation": relation,
+        }
+        self.execute_macro(CREATE_SCHEMA_MACRO_NAME, kwargs=kwargs)
+        self.commit_if_has_connection()
+        # we can't update the cache here, as if the schema already existed we
+        # don't want to (incorrectly) say that it's empty
+
+    def drop_schema(self, relation: BaseRelation) -> None:
+        relation = relation.without_identifier()
+        fire_event(SchemaDrop(relation=_make_ref_key_dict(relation)))
+        kwargs = {
+            "relation": relation,
+        }
+        self.execute_macro(DROP_SCHEMA_MACRO_NAME, kwargs=kwargs)
+        self.commit_if_has_connection()
+        # we can update the cache here
+        self.cache.drop_schema(relation.database, relation.schema)
+
+    def list_relations_without_caching(
+        self,
+        schema_relation: BaseRelation,
+    ) -> List[BaseRelation]:
+        kwargs = {"schema_relation": schema_relation}
+        results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs)
+
+        relations = []
+        quote_policy = {"database": True, "schema": True, "identifier": True}
+        for _database, name, _schema, _type in results:
+            try:
+                _type = self.Relation.get_relation_type(_type)
+            except ValueError:
+                _type = self.Relation.External
+            relations.append(
+                self.Relation.create(
+                    database=_database,
+                    schema=_schema,
+                    identifier=name,
+                    quote_policy=quote_policy,
+                    type=_type,
+                )
+            )
+        return relations
+
+    @classmethod
+    def quote(self, identifier):
+        return '"{}"'.format(identifier)
+
+    def list_schemas(self, database: str) -> List[str]:
+        results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database})
+
+        return [row[0] for row in results]
+
+    def check_schema_exists(self, database: str, schema: str) -> bool:
+        information_schema = self.Relation.create(
+            database=database,
+            schema=schema,
+            identifier="INFORMATION_SCHEMA",
+            quote_policy=self.config.quoting,
+        ).information_schema()
+
+        kwargs = {"information_schema": information_schema, "schema": schema}
+        results = self.execute_macro(CHECK_SCHEMA_EXISTS_MACRO_NAME, kwargs=kwargs)
+        return results[0][0] > 0
+
+    def validate_sql(self, sql: str) -> AdapterResponse:
+        """Submit the given SQL to the engine for validation, but not execution.
+
+        By default we simply prefix the query with the explain keyword and allow the
+        exceptions thrown by the underlying engine on invalid SQL inputs to bubble up
+        to the exception handler. For adjustments to the explain statement - such as
+        for adapters that have different mechanisms for hinting at query validation
+        or dry-run - callers may be able to override the validate_sql_query macro with
+        the addition of an <adapter>__validate_sql implementation.
+
+        :param sql str: The sql to validate
+        """
+        kwargs = {
+            "sql": sql,
+        }
+        result = self.execute_macro(VALIDATE_SQL_MACRO_NAME, kwargs=kwargs)
+        # The statement macro always returns an AdapterResponse in the output AttrDict's
+        # `response` property, and we preserve the full payload in case we want to
+        # return fetched output for engines where explain plans are emitted as columnar
+        # results. Any macro override that deviates from this behavior may encounter an
+        # assertion error in the runtime.
+        adapter_response = result.response  # type: ignore[attr-defined]
+        assert isinstance(adapter_response, AdapterResponse), (
+            f"Expected AdapterResponse from validate_sql macro execution, "
+            f"got {type(adapter_response)}."
+        )
+        return adapter_response
+
+    # This is for use in the test suite
+    def run_sql_for_tests(self, sql, fetch, conn):
+        cursor = conn.handle.cursor()
+        try:
+            cursor.execute(sql)
+            if hasattr(conn.handle, "commit"):
+                conn.handle.commit()
+            if fetch == "one":
+                return cursor.fetchone()
+            elif fetch == "all":
+                return cursor.fetchall()
+            else:
+                return
+        except BaseException as e:
+            if conn.handle and not getattr(conn.handle, "closed", True):
+                conn.handle.rollback()
+            print(sql)
+            print(e)
+            raise
+        finally:
+            conn.transaction_open = False

dbt/adapters/utils.py

@@ -0,0 +1,69 @@
+from typing import Mapping, Sequence, Any, Dict, List
+
+from dbt.adapters.exceptions import DuplicateAliasError
+
+
+class Translator:
+    def __init__(self, aliases: Mapping[str, str], recursive: bool = False) -> None:
+        self.aliases = aliases
+        self.recursive = recursive
+
+    def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]:
+        result: Dict[str, Any] = {}
+
+        for key, value in kwargs.items():
+            canonical_key = self.aliases.get(key, key)
+            if canonical_key in result:
+                raise DuplicateAliasError(kwargs, self.aliases, canonical_key)
+            result[canonical_key] = self.translate_value(value)
+        return result
+
+    def translate_sequence(self, value: Sequence[Any]) -> List[Any]:
+        return [self.translate_value(v) for v in value]
+
+    def translate_value(self, value: Any) -> Any:
+        if self.recursive:
+            if isinstance(value, Mapping):
+                return self.translate_mapping(value)
+            elif isinstance(value, (list, tuple)):
+                return self.translate_sequence(value)
+        return value
+
+    def translate(self, value: Mapping[str, Any]) -> Dict[str, Any]:
+        try:
+            return self.translate_mapping(value)
+        except RuntimeError as exc:
+            if "maximum recursion depth exceeded" in str(exc):
+                raise RecursionError("Cycle detected in a value passed to translate!")
+            raise
+
+
+def translate_aliases(
+    kwargs: Dict[str, Any],
+    aliases: Dict[str, str],
+    recurse: bool = False,
+) -> Dict[str, Any]:
+    """Given a dict of keyword arguments and a dict mapping aliases to their
+    canonical values, canonicalize the keys in the kwargs dict.
+
+    If recurse is True, perform this operation recursively.
+
+    :returns: A dict containing all the values in kwargs referenced by their
+        canonical key.
+    :raises: `AliasError`, if a canonical key is defined more than once.
+    """
+    translator = Translator(aliases, recurse)
+    return translator.translate(kwargs)
+
+
+# some types need to make constants available to the jinja context as
+# attributes, and regular properties only work with objects. maybe this should
+# be handled by the RelationProxy?
+
+
+class classproperty(object):
+    def __init__(self, func) -> None:
+        self.func = func
+
+    def __get__(self, obj, objtype):
+        return self.func(objtype)

dbt/include/global_project/__init__.py

@@ -0,0 +1,4 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)
+PROJECT_NAME = "dbt"

dbt/include/global_project/dbt_project.yml

@@ -0,0 +1,7 @@
+config-version: 2
+name: dbt
+version: 1.0
+
+docs-paths: ["docs"]
+macro-paths: ["macros"]
+test-paths: ["tests"]

dbt/include/global_project/docs/overview.md

@@ -0,0 +1,43 @@
+
+{% docs __overview__ %}
+
+### Welcome!
+
+Welcome to the auto-generated documentation for your dbt project!
+
+### Navigation
+
+You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models
+in your project.
+
+#### Project Tab
+The `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the
+models defined in your dbt project, as well as models imported from dbt packages.
+
+#### Database Tab
+The `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view
+shows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown
+in this interface, as they do not exist in the database.
+
+### Graph Exploration
+You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.
+
+On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`
+button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,
+or are built from, the model you're exploring.
+
+Once expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the
+models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).
+
+Note that you can also right-click on models to interactively filter and explore the graph.
+
+---
+
+### More information
+
+- [What is dbt](https://docs.getdbt.com/docs/introduction)?
+- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)
+- [Installation](https://docs.getdbt.com/docs/installation)
+- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion
+
+{% enddocs %}

dbt/include/global_project/macros/adapters/apply_grants.sql

@@ -0,0 +1,167 @@
+{# ------- BOOLEAN MACROS --------- #}
+
+{#
+  -- COPY GRANTS
+  -- When a relational object (view or table) is replaced in this database,
+  -- do previous grants carry over to the new object? This may depend on:
+  --    whether we use alter-rename-swap versus CREATE OR REPLACE
+  --    user-supplied configuration (e.g. copy_grants on Snowflake)
+  -- By default, play it safe, assume TRUE: that grants ARE copied over.
+  -- This means dbt will first "show" current grants and then calculate diffs.
+  -- It may require an additional query than is strictly necessary,
+  -- but better safe than sorry.
+#}
+
+{% macro copy_grants() %}
+    {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}
+{% endmacro %}
+
+{% macro default__copy_grants() %}
+    {{ return(True) }}
+{% endmacro %}
+
+
+{#
+  -- SUPPORT MULTIPLE GRANTEES PER DCL STATEMENT
+  -- Does this database support 'grant {privilege} to {grantee_1}, {grantee_2}, ...'
+  -- Or must these be separate statements:
+  --  `grant {privilege} to {grantee_1}`;
+  --  `grant {privilege} to {grantee_2}`;
+  -- By default, pick the former, because it's what we prefer when available.
+#}
+
+{% macro support_multiple_grantees_per_dcl_statement() %}
+    {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}
+{% endmacro %}
+
+{%- macro default__support_multiple_grantees_per_dcl_statement() -%}
+    {{ return(True) }}
+{%- endmacro -%}
+
+
+{% macro should_revoke(existing_relation, full_refresh_mode=True) %}
+
+    {% if not existing_relation %}
+        {#-- The table doesn't already exist, so no grants to copy over --#}
+        {{ return(False) }}
+    {% elif full_refresh_mode %}
+        {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}
+        {{ return(copy_grants()) }}
+    {% else %}
+        {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}
+        {{ return(True) }}
+    {% endif %}
+
+{% endmacro %}
+
+{# ------- DCL STATEMENT TEMPLATES --------- #}
+
+{% macro get_show_grant_sql(relation) %}
+    {{ return(adapter.dispatch("get_show_grant_sql", "dbt")(relation)) }}
+{% endmacro %}
+
+{% macro default__get_show_grant_sql(relation) %}
+    show grants on {{ relation }}
+{% endmacro %}
+
+
+{% macro get_grant_sql(relation, privilege, grantees) %}
+    {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}
+{% endmacro %}
+
+{%- macro default__get_grant_sql(relation, privilege, grantees) -%}
+    grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}
+{%- endmacro -%}
+
+
+{% macro get_revoke_sql(relation, privilege, grantees) %}
+    {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}
+{% endmacro %}
+
+{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}
+    revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}
+{%- endmacro -%}
+
+
+{# ------- RUNTIME APPLICATION --------- #}
+
+{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}
+    {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}
+{% endmacro %}
+
+{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}
+    {#
+      -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.
+      -- Depending on whether this database supports multiple grantees per statement, pass in the list of
+      -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.
+      -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`
+    #}
+    {%- set dcl_statements = [] -%}
+    {%- for privilege, grantees in grant_config.items() %}
+        {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}
+          {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}
+          {%- do dcl_statements.append(dcl) -%}
+        {%- else -%}
+          {%- for grantee in grantees -%}
+              {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}
+              {%- do dcl_statements.append(dcl) -%}
+          {% endfor -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{ return(dcl_statements) }}
+{%- endmacro %}
+
+
+{% macro call_dcl_statements(dcl_statement_list) %}
+    {{ return(adapter.dispatch("call_dcl_statements", "dbt")(dcl_statement_list)) }}
+{% endmacro %}
+
+{% macro default__call_dcl_statements(dcl_statement_list) %}
+    {#
+      -- By default, supply all grant + revoke statements in a single semicolon-separated block,
+      -- so that they're all processed together.
+
+      -- Some databases do not support this. Those adapters will need to override this macro
+      -- to run each statement individually.
+    #}
+    {% call statement('grants') %}
+        {% for dcl_statement in dcl_statement_list %}
+            {{ dcl_statement }};
+        {% endfor %}
+    {% endcall %}
+{% endmacro %}
+
+
+{% macro apply_grants(relation, grant_config, should_revoke) %}
+    {{ return(adapter.dispatch("apply_grants", "dbt")(relation, grant_config, should_revoke)) }}
+{% endmacro %}
+
+{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}
+    {#-- If grant_config is {} or None, this is a no-op --#}
+    {% if grant_config %}
+        {% if should_revoke %}
+            {#-- We think previous grants may have carried over --#}
+            {#-- Show current grants and calculate diffs --#}
+            {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}
+            {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}
+            {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}
+            {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}
+            {% if not (needs_granting or needs_revoking) %}
+                {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}
+            {% endif %}
+        {% else %}
+            {#-- We don't think there's any chance of previous grants having carried over. --#}
+            {#-- Jump straight to granting what the user has configured. --#}
+            {% set needs_revoking = {} %}
+            {% set needs_granting = grant_config %}
+        {% endif %}
+        {% if needs_granting or needs_revoking %}
+            {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}
+            {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}
+            {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}
+            {% if dcl_statement_list %}
+                {{ call_dcl_statements(dcl_statement_list) }}
+            {% endif %}
+        {% endif %}
+    {% endif %}
+{% endmacro %}