dbt-adapters 1.22.2__py3-none-any.whl

dbt/adapters/sql/connections.py

@@ -0,0 +1,263 @@
+import abc
+import time
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    TYPE_CHECKING,
+    Type,
+)
+
+from dbt_common.events.contextvars import get_node_info
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtInternalError, NotImplementedError
+from dbt_common.utils import cast_to_str
+
+from dbt.adapters.base import BaseConnectionManager
+from dbt.adapters.contracts.connection import (
+    AdapterResponse,
+    Connection,
+    ConnectionState,
+)
+from dbt.adapters.events.types import (
+    ConnectionUsed,
+    SQLCommit,
+    SQLQuery,
+    SQLQueryStatus,
+    AdapterEventDebug,
+)
+
+if TYPE_CHECKING:
+    import agate
+
+
+class SQLConnectionManager(BaseConnectionManager):
+    """The default connection manager with some common SQL methods implemented.
+
+    Methods to implement:
+        - exception_handler
+        - cancel
+        - get_response
+        - open
+    """
+
+    @abc.abstractmethod
+    def cancel(self, connection: Connection):
+        """Cancel the given connection."""
+        raise NotImplementedError("`cancel` is not implemented for this adapter!")
+
+    def cancel_open(self) -> List[str]:
+        names = []
+        this_connection = self.get_if_exists()
+        with self.lock:
+            for connection in self.thread_connections.values():
+                if connection is this_connection:
+                    continue
+
+                # if the connection failed, the handle will be None so we have
+                # nothing to cancel.
+                if connection.handle is not None and connection.state == ConnectionState.OPEN:
+                    self.cancel(connection)
+                if connection.name is not None:
+                    names.append(connection.name)
+        return names
+
+    def add_query(
+        self,
+        sql: str,
+        auto_begin: bool = True,
+        bindings: Optional[Any] = None,
+        abridge_sql_log: bool = False,
+        retryable_exceptions: Tuple[Type[Exception], ...] = tuple(),
+        retry_limit: int = 1,
+    ) -> Tuple[Connection, Any]:
+        """
+        Retry function encapsulated here to avoid commitment to some
+        user-facing interface. Right now, Redshift commits to a 1 second
+        retry timeout so this serves as a default.
+        """
+
+        def _execute_query_with_retry(
+            cursor: Any,
+            sql: str,
+            bindings: Optional[Any],
+            retryable_exceptions: Tuple[Type[Exception], ...],
+            retry_limit: int,
+            attempt: int,
+        ):
+            """
+            A success sees the try exit cleanly and avoid any recursive
+            retries. Failure begins a sleep and retry routine.
+            """
+            try:
+                cursor.execute(sql, bindings)
+            except retryable_exceptions as e:
+                # Cease retries and fail when limit is hit.
+                if attempt >= retry_limit:
+                    raise e
+
+                fire_event(
+                    AdapterEventDebug(
+                        base_msg=f"Got a retryable error {type(e)}. {retry_limit - attempt} retries left. "
+                        f"Retrying in 1 second.\nError:\n{e}"
+                    )
+                )
+                time.sleep(1)
+
+                return _execute_query_with_retry(
+                    cursor=cursor,
+                    sql=sql,
+                    bindings=bindings,
+                    retryable_exceptions=retryable_exceptions,
+                    retry_limit=retry_limit,
+                    attempt=attempt + 1,
+                )
+
+        connection = self.get_thread_connection()
+        if auto_begin and connection.transaction_open is False:
+            self.begin()
+        fire_event(
+            ConnectionUsed(
+                conn_type=self.TYPE,
+                conn_name=cast_to_str(connection.name),
+                node_info=get_node_info(),
+            )
+        )
+
+        with self.exception_handler(sql):
+            if abridge_sql_log:
+                log_sql = "{}...".format(sql[:512])
+            else:
+                log_sql = sql
+
+            fire_event(
+                SQLQuery(
+                    conn_name=cast_to_str(connection.name),
+                    sql=log_sql,
+                    node_info=get_node_info(),
+                )
+            )
+
+            pre = time.perf_counter()
+
+            cursor = connection.handle.cursor()
+            _execute_query_with_retry(
+                cursor=cursor,
+                sql=sql,
+                bindings=bindings,
+                retryable_exceptions=retryable_exceptions,
+                retry_limit=retry_limit,
+                attempt=1,
+            )
+
+            result = self.get_response(cursor)
+
+            fire_event(
+                SQLQueryStatus(
+                    status=str(result),
+                    elapsed=time.perf_counter() - pre,
+                    node_info=get_node_info(),
+                    query_id=result.query_id,
+                )
+            )
+
+            return connection, cursor
+
+    @classmethod
+    @abc.abstractmethod
+    def get_response(cls, cursor: Any) -> AdapterResponse:
+        """Get the status of the cursor."""
+        raise NotImplementedError("`get_response` is not implemented for this adapter!")
+
+    @classmethod
+    def process_results(
+        cls, column_names: Iterable[str], rows: Iterable[Any]
+    ) -> Iterator[Dict[str, Any]]:
+        unique_col_names = dict()  # type: ignore[var-annotated]
+        for idx in range(len(column_names)):  # type: ignore[arg-type]
+            col_name = column_names[idx]  # type: ignore[index]
+            if col_name in unique_col_names:
+                unique_col_names[col_name] += 1
+                column_names[idx] = f"{col_name}_{unique_col_names[col_name]}"  # type: ignore[index] # noqa
+            else:
+                unique_col_names[column_names[idx]] = 1  # type: ignore[index]
+
+        for row in rows:
+            yield dict(zip(column_names, row))
+
+    @classmethod
+    def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> "agate.Table":
+        from dbt_common.clients.agate_helper import table_from_data_flat
+
+        data: Iterable[Any] = []
+        column_names: List[str] = []
+
+        if cursor.description is not None:
+            column_names = [col[0] for col in cursor.description]
+            if limit:
+                rows = cursor.fetchmany(limit)
+            else:
+                rows = cursor.fetchall()
+            data = cls.process_results(column_names, rows)
+
+        return table_from_data_flat(data, column_names)
+
+    def execute(
+        self,
+        sql: str,
+        auto_begin: bool = False,
+        fetch: bool = False,
+        limit: Optional[int] = None,
+    ) -> Tuple[AdapterResponse, "agate.Table"]:
+        from dbt_common.clients.agate_helper import empty_table
+
+        sql = self._add_query_comment(sql)
+        _, cursor = self.add_query(sql, auto_begin)
+        response = self.get_response(cursor)
+        if fetch:
+            table = self.get_result_from_cursor(cursor, limit)
+        else:
+            table = empty_table()
+        return response, table
+
+    def add_begin_query(self):
+        return self.add_query("BEGIN", auto_begin=False)
+
+    def add_commit_query(self):
+        return self.add_query("COMMIT", auto_begin=False)
+
+    def add_select_query(self, sql: str) -> Tuple[Connection, Any]:
+        sql = self._add_query_comment(sql)
+        return self.add_query(sql, auto_begin=False)
+
+    def begin(self):
+        connection = self.get_thread_connection()
+        if connection.transaction_open is True:
+            raise DbtInternalError(
+                'Tried to begin a new transaction on connection "{}", but '
+                "it already had one open!".format(connection.name)
+            )
+
+        self.add_begin_query()
+
+        connection.transaction_open = True
+        return connection
+
+    def commit(self):
+        connection = self.get_thread_connection()
+        if connection.transaction_open is False:
+            raise DbtInternalError(
+                'Tried to commit transaction on connection "{}", but '
+                "it does not have one open!".format(connection.name)
+            )
+
+        fire_event(SQLCommit(conn_name=connection.name, node_info=get_node_info()))
+        self.add_commit_query()
+
+        connection.transaction_open = False
+
+        return connection

dbt/adapters/sql/impl.py

@@ -0,0 +1,286 @@
+from typing import Any, List, Optional, Tuple, Type, TYPE_CHECKING
+
+from dbt_common.events.functions import fire_event
+from dbt_common.record import record_function
+
+from dbt.adapters.base import BaseAdapter, BaseRelation, available
+from dbt.adapters.cache import _make_ref_key_dict
+from dbt.adapters.contracts.connection import AdapterResponse, Connection
+from dbt.adapters.events.types import ColTypeChange, SchemaCreation, SchemaDrop
+from dbt.adapters.exceptions import RelationTypeNullError
+from dbt.adapters.record.base import AdapterTestSqlRecord, AdapterAddQueryRecord
+from dbt.adapters.sql.connections import SQLConnectionManager
+
+LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching"
+GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation"
+LIST_SCHEMAS_MACRO_NAME = "list_schemas"
+CHECK_SCHEMA_EXISTS_MACRO_NAME = "check_schema_exists"
+CREATE_SCHEMA_MACRO_NAME = "create_schema"
+DROP_SCHEMA_MACRO_NAME = "drop_schema"
+RENAME_RELATION_MACRO_NAME = "rename_relation"
+TRUNCATE_RELATION_MACRO_NAME = "truncate_relation"
+DROP_RELATION_MACRO_NAME = "drop_relation"
+ALTER_COLUMN_TYPE_MACRO_NAME = "alter_column_type"
+VALIDATE_SQL_MACRO_NAME = "validate_sql"
+
+if TYPE_CHECKING:
+    import agate
+
+
+class SQLAdapter(BaseAdapter):
+    """The default adapter with the common agate conversions and some SQL
+    methods was implemented. This adapter has a different much shorter list of
+    methods to implement, but some more macros that must be implemented.
+
+    To implement a macro, implement "${adapter_type}__${macro_name}". in the
+    adapter's internal project.
+
+    Methods to implement:
+        - date_function
+
+    Macros to implement:
+        - get_catalog
+        - list_relations_without_caching
+        - get_columns_in_relation
+        - get_catalog_for_single_relation
+    """
+
+    ConnectionManager: Type[SQLConnectionManager]
+    connections: SQLConnectionManager
+
+    @available.parse(lambda *a, **k: (None, None))
+    @record_function(
+        AdapterAddQueryRecord, method=True, index_on_thread_id=True, id_field_name="thread_id"
+    )
+    def add_query(
+        self,
+        sql: str,
+        auto_begin: bool = True,
+        bindings: Optional[Any] = None,
+        abridge_sql_log: bool = False,
+    ) -> Tuple[Connection, Any]:
+        """Add a query to the current transaction. A thin wrapper around
+        ConnectionManager.add_query.
+
+        :param sql: The SQL query to add
+        :param auto_begin: If set and there is no transaction in progress,
+            begin a new one.
+        :param bindings: An optional list of bindings for the query.
+        :param abridge_sql_log: If set, limit the raw sql logged to 512
+            characters
+        """
+        return self.connections.add_query(sql, auto_begin, bindings, abridge_sql_log)
+
+    @classmethod
+    def convert_text_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return "text"
+
+    @classmethod
+    def convert_number_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        import agate
+
+        # TODO CT-211
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
+        return "float8" if decimals else "integer"
+
+    @classmethod
+    def convert_integer_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return "integer"
+
+    @classmethod
+    def convert_boolean_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return "boolean"
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return "timestamp without time zone"
+
+    @classmethod
+    def convert_date_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return "date"
+
+    @classmethod
+    def convert_time_type(cls, agate_table: "agate.Table", col_idx: int) -> str:
+        return "time"
+
+    @classmethod
+    def is_cancelable(cls) -> bool:
+        return True
+
+    def expand_column_types(self, goal, current):
+        reference_columns = {c.name: c for c in self.get_columns_in_relation(goal)}
+
+        target_columns = {c.name: c for c in self.get_columns_in_relation(current)}
+
+        for column_name, reference_column in reference_columns.items():
+            target_column = target_columns.get(column_name)
+
+            if target_column is not None and target_column.can_expand_to(reference_column):
+                col_string_size = reference_column.string_size()
+                new_type = self.Column.string_type(col_string_size)
+                fire_event(
+                    ColTypeChange(
+                        orig_type=target_column.data_type,
+                        new_type=new_type,
+                        table=_make_ref_key_dict(current),
+                    )
+                )
+
+                self.alter_column_type(current, column_name, new_type)
+
+    def alter_column_type(self, relation, column_name, new_column_type) -> None:
+        """
+        1. Create a new column (w/ temp name and correct type)
+        2. Copy data over to it
+        3. Drop the existing column (cascade!)
+        4. Rename the new column to existing column
+        """
+        kwargs = {
+            "relation": relation,
+            "column_name": column_name,
+            "new_column_type": new_column_type,
+        }
+        self.execute_macro(ALTER_COLUMN_TYPE_MACRO_NAME, kwargs=kwargs)
+
+    def drop_relation(self, relation):
+        if relation.type is None:
+            raise RelationTypeNullError(relation)
+
+        self.cache_dropped(relation)
+        self.execute_macro(DROP_RELATION_MACRO_NAME, kwargs={"relation": relation})
+
+    def truncate_relation(self, relation):
+        self.execute_macro(TRUNCATE_RELATION_MACRO_NAME, kwargs={"relation": relation})
+
+    def rename_relation(self, from_relation, to_relation):
+        self.cache_renamed(from_relation, to_relation)
+
+        kwargs = {"from_relation": from_relation, "to_relation": to_relation}
+        self.execute_macro(RENAME_RELATION_MACRO_NAME, kwargs=kwargs)
+
+    def get_columns_in_relation(self, relation):
+        return self.execute_macro(
+            GET_COLUMNS_IN_RELATION_MACRO_NAME, kwargs={"relation": relation}
+        )
+
+    def create_schema(self, relation: BaseRelation) -> None:
+        relation = relation.without_identifier()
+        fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
+        kwargs = {
+            "relation": relation,
+        }
+        self.execute_macro(CREATE_SCHEMA_MACRO_NAME, kwargs=kwargs)
+        self.commit_if_has_connection()
+        # we can't update the cache here, as if the schema already existed we
+        # don't want to (incorrectly) say that it's empty
+
+    def drop_schema(self, relation: BaseRelation) -> None:
+        relation = relation.without_identifier()
+        fire_event(SchemaDrop(relation=_make_ref_key_dict(relation)))
+        kwargs = {
+            "relation": relation,
+        }
+        self.execute_macro(DROP_SCHEMA_MACRO_NAME, kwargs=kwargs)
+        self.commit_if_has_connection()
+        # we can update the cache here
+        self.cache.drop_schema(relation.database, relation.schema)
+
+    def list_relations_without_caching(
+        self,
+        schema_relation: BaseRelation,
+    ) -> List[BaseRelation]:
+        kwargs = {"schema_relation": schema_relation}
+        results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs)
+
+        relations = []
+        quote_policy = {"database": True, "schema": True, "identifier": True}
+        for _database, name, _schema, _type in results:
+            try:
+                _type = self.Relation.get_relation_type(_type)
+            except ValueError:
+                _type = self.Relation.External
+            relations.append(
+                self.Relation.create(
+                    database=_database,
+                    schema=_schema,
+                    identifier=name,
+                    quote_policy=quote_policy,
+                    type=_type,
+                )
+            )
+        return relations
+
+    @classmethod
+    def quote(self, identifier):
+        return '"{}"'.format(identifier)
+
+    def list_schemas(self, database: str) -> List[str]:
+        results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database})
+
+        return [row[0] for row in results]
+
+    def check_schema_exists(self, database: str, schema: str) -> bool:
+        information_schema = self.Relation.create(
+            database=database,
+            schema=schema,
+            identifier="INFORMATION_SCHEMA",
+            quote_policy=self.config.quoting,
+        ).information_schema()
+
+        kwargs = {"information_schema": information_schema, "schema": schema}
+        results = self.execute_macro(CHECK_SCHEMA_EXISTS_MACRO_NAME, kwargs=kwargs)
+        return results[0][0] > 0
+
+    def validate_sql(self, sql: str) -> AdapterResponse:
+        """Submit the given SQL to the engine for validation, but not execution.
+
+        By default we simply prefix the query with the explain keyword and allow the
+        exceptions thrown by the underlying engine on invalid SQL inputs to bubble up
+        to the exception handler. For adjustments to the explain statement - such as
+        for adapters that have different mechanisms for hinting at query validation
+        or dry-run - callers may be able to override the validate_sql_query macro with
+        the addition of an <adapter>__validate_sql implementation.
+
+        :param sql str: The sql to validate
+        """
+        kwargs = {
+            "sql": sql,
+        }
+        result = self.execute_macro(VALIDATE_SQL_MACRO_NAME, kwargs=kwargs)
+        # The statement macro always returns an AdapterResponse in the output AttrDict's
+        # `response` property, and we preserve the full payload in case we want to
+        # return fetched output for engines where explain plans are emitted as columnar
+        # results. Any macro override that deviates from this behavior may encounter an
+        # assertion error in the runtime.
+        adapter_response = result.response
+        assert isinstance(adapter_response, AdapterResponse), (
+            f"Expected AdapterResponse from validate_sql macro execution, "
+            f"got {type(adapter_response)}."
+        )
+        return adapter_response
+
+    # This is for use in the test suite
+    @available
+    @record_function(
+        AdapterTestSqlRecord, method=True, index_on_thread_id=True, id_field_name="thread_id"
+    )
+    def run_sql_for_tests(self, sql, fetch, conn):
+        cursor = conn.handle.cursor()
+        try:
+            cursor.execute(sql)
+            if hasattr(conn.handle, "commit"):
+                conn.handle.commit()
+            if fetch == "one":
+                return cursor.fetchone()
+            elif fetch == "all":
+                return cursor.fetchall()
+            else:
+                return
+        except BaseException as e:
+            if conn.handle and not getattr(conn.handle, "closed", True):
+                conn.handle.rollback()
+            print(sql)
+            print(e)
+            raise
+        finally:
+            conn.transaction_open = False

dbt/adapters/utils.py

@@ -0,0 +1,69 @@
+from typing import Mapping, Sequence, Any, Dict, List
+
+from dbt.adapters.exceptions import DuplicateAliasError
+
+
+class Translator:
+    def __init__(self, aliases: Mapping[str, str], recursive: bool = False) -> None:
+        self.aliases = aliases
+        self.recursive = recursive
+
+    def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]:
+        result: Dict[str, Any] = {}
+
+        for key, value in kwargs.items():
+            canonical_key = self.aliases.get(key, key)
+            if canonical_key in result:
+                raise DuplicateAliasError(kwargs, self.aliases, canonical_key)
+            result[canonical_key] = self.translate_value(value)
+        return result
+
+    def translate_sequence(self, value: Sequence[Any]) -> List[Any]:
+        return [self.translate_value(v) for v in value]
+
+    def translate_value(self, value: Any) -> Any:
+        if self.recursive:
+            if isinstance(value, Mapping):
+                return self.translate_mapping(value)
+            elif isinstance(value, (list, tuple)):
+                return self.translate_sequence(value)
+        return value
+
+    def translate(self, value: Mapping[str, Any]) -> Dict[str, Any]:
+        try:
+            return self.translate_mapping(value)
+        except RuntimeError as exc:
+            if "maximum recursion depth exceeded" in str(exc):
+                raise RecursionError("Cycle detected in a value passed to translate!")
+            raise
+
+
+def translate_aliases(
+    kwargs: Dict[str, Any],
+    aliases: Dict[str, str],
+    recurse: bool = False,
+) -> Dict[str, Any]:
+    """Given a dict of keyword arguments and a dict mapping aliases to their
+    canonical values, canonicalize the keys in the kwargs dict.
+
+    If recurse is True, perform this operation recursively.
+
+    :returns: A dict containing all the values in kwargs referenced by their
+        canonical key.
+    :raises: `AliasError`, if a canonical key is defined more than once.
+    """
+    translator = Translator(aliases, recurse)
+    return translator.translate(kwargs)
+
+
+# some types need to make constants available to the jinja context as
+# attributes, and regular properties only work with objects. maybe this should
+# be handled by the RelationProxy?
+
+
+class classproperty(object):
+    def __init__(self, func) -> None:
+        self.func = func
+
+    def __get__(self, obj, objtype):
+        return self.func(objtype)

dbt/include/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

dbt/include/global_project/__init__.py

@@ -0,0 +1,4 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)
+PROJECT_NAME = "dbt"

dbt/include/global_project/dbt_project.yml

@@ -0,0 +1,7 @@
+config-version: 2
+name: dbt
+version: 1.0
+
+docs-paths: ["docs"]
+macro-paths: ["macros"]
+test-paths: ["tests"]

dbt/include/global_project/docs/overview.md

@@ -0,0 +1,43 @@
+
+{% docs __overview__ %}
+
+### Welcome!
+
+Welcome to the auto-generated documentation for your dbt project!
+
+### Navigation
+
+You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models
+in your project.
+
+#### Project Tab
+The `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the
+models defined in your dbt project, as well as models imported from dbt packages.
+
+#### Database Tab
+The `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view
+shows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown
+in this interface, as they do not exist in the database.
+
+### Graph Exploration
+You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.
+
+On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`
+button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,
+or are built from, the model you're exploring.
+
+Once expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the
+models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).
+
+Note that you can also right-click on models to interactively filter and explore the graph.
+
+---
+
+### More information
+
+- [What is dbt](https://docs.getdbt.com/docs/introduction)?
+- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)
+- [Installation](https://docs.getdbt.com/docs/installation)
+- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion
+
+{% enddocs %}