dbt-adapters 1.22.2__py3-none-any.whl

dbt/adapters/catalogs/_integration.py

@@ -0,0 +1,113 @@
+import abc
+from typing import Any, Dict, Optional
+from typing_extensions import Protocol
+
+from dbt.adapters.contracts.relation import RelationConfig
+
+
+class CatalogIntegrationConfig(Protocol):
+    """
+    Represents the user configuration required to describe a catalog integration
+
+    This class serves as a blueprint for catalog integration configurations,
+    providing details about the catalog type, name, and other optional
+    properties necessary for integration. It is designed to be used with
+    any implementation that requires a catalog configuration protocol,
+    ensuring a standardized structure and attributes are in place.
+
+    Attributes:
+        name (str): the name of the catalog integration in the dbt project, e.g. "my_iceberg_operational_data"
+            - a unique name for this catalog integration to be referenced in a model configuration
+        catalog_type (str): the type of the catalog integration in the data platform, e.g. "iceberg_rest"
+            - this is required for dbt to determine the correct method for parsing user configuration
+            - usually a combination of the catalog and the way in which the data platform interacts with it
+        catalog_name (Optional[str]): the name of the catalog integration in the data platform, e.g. "my_favorite_iceberg_catalog"
+            - this is required for dbt to correctly reference catalogs by name from model configuration
+            - expected to be unique within the data platform, but many dbt catalog integrations can share the same catalog name
+        table_format (Optional[str]): the table format this catalog uses
+            - this is commonly unique to each catalog type, and should only be required from the user for catalogs that support multiple formats
+        external_volume (Optional[str]): external storage volume identifier
+            - while this is a separate concept from catalogs, we feel it is more user-friendly to group it with the catalog configuration
+            - it's possible to use a default external volume at the user, database, or account level, hence this is optional
+            - a result of this grouping is that there can only be one external volume per catalog integration, but many catalogs can share the same volume
+            - a user should create a new dbt catalog if they want to use a different external volume for a given catalog integration
+        adapter_properties (Optional[Dict[str, Any]]):
+            - additional, adapter-specific properties are nested here to avoid future collision when expanding the catalog integration protocol
+    """
+
+    name: str
+    catalog_type: str
+    catalog_name: Optional[str]
+    table_format: Optional[str]
+    external_volume: Optional[str]
+    file_format: Optional[str]
+    adapter_properties: Dict[str, Any]
+
+
+class CatalogRelation(Protocol):
+    catalog_name: Optional[str]
+    table_format: Optional[str]
+    external_volume: Optional[str]
+    file_format: Optional[str]
+
+
+class CatalogIntegration(abc.ABC):
+    """
+    Represent a catalog integration for a given user config
+
+    This class should be implemented by specific catalog integration types in an adapter.
+    A catalog integration is a specific platform's way of interacting with a specific catalog.
+
+    Attributes:
+        name (str): the name of the catalog integration in the dbt project, e.g. "my_iceberg_operational_data"
+            - a unique name for this catalog integration to be referenced in a model configuration
+        catalog_type (str): the type of the catalog integration in the data platform, e.g. "iceberg_rest"
+            - this is a name for this particular implementation of the catalog integration, hence it is a class attribute
+        catalog_name (Optional[str]): the name of the catalog integration in the data platform, e.g. "my_favorite_iceberg_catalog"
+            - this is required for dbt to correctly reference catalogs by name from model configuration
+            - expected to be unique within the data platform, but many dbt catalog integrations can share the same catalog name
+        table_format (Optional[str]): the table format this catalog uses
+            - this is commonly unique to each catalog type, and should only be required from the user for catalogs that support multiple formats
+        external_volume (Optional[str]): external storage volume identifier
+            - while this is a separate concept from catalogs, we feel it is more user-friendly to group it with the catalog configuration
+            - it's possible to use a default external volume at the user, database, or account level, hence this is optional
+            - a result of this grouping is that there can only be one external volume per catalog integration, but many catalogs can share the same volume
+            - a user should create a new dbt catalog if they want to use a different external volume for a given catalog integration
+        allows_writes (bool): identifies whether this catalog integration supports writes
+            - this is required for dbt to correctly identify whether a catalog is writable during parse time
+            - this is determined by the catalog integration type, hence it is a class attribute
+    """
+
+    catalog_type: str
+    table_format: Optional[str] = None
+    file_format: Optional[str] = None
+    allows_writes: bool = False
+
+    def __init__(self, config: CatalogIntegrationConfig) -> None:
+        # table_format is often fixed for a catalog type, allow it to be defined at the class level
+        if config.table_format is not None:
+            self.table_format = config.table_format
+        self.name: str = config.name
+        self.catalog_name: Optional[str] = config.catalog_name
+        self.external_volume: Optional[str] = config.external_volume
+        self.file_format: Optional[str] = config.file_format
+
+    def build_relation(self, config: RelationConfig) -> CatalogRelation:
+        """
+        Builds relation configuration within the context of this catalog integration.
+
+        This method is a placeholder and must be implemented in subclasses to provide
+        custom logic for building a relation.
+
+        Args:
+            config: User-provided model configuration.
+
+        Returns:
+            A `CatalogRelation` object constructed based on the input configuration.
+
+        Raises:
+            NotImplementedError: Raised when this method is not implemented in a subclass.
+        """
+        raise NotImplementedError(
+            f"`{self.__class__.__name__}.build_relation` must be implemented to use this feature"
+        )

dbt/adapters/clients/jinja.py

@@ -0,0 +1,24 @@
+from typing import Any, Dict
+
+from dbt_common.clients.jinja import BaseMacroGenerator, get_environment
+
+
+class QueryStringGenerator(BaseMacroGenerator):
+    def __init__(self, template_str: str, context: Dict[str, Any]) -> None:
+        super().__init__(context)
+        self.template_str: str = template_str
+        env = get_environment()
+        self.template = env.from_string(
+            self.template_str,
+            globals=self.context,
+        )
+
+    def get_name(self) -> str:
+        return "query_comment_macro"
+
+    def get_template(self):
+        """Don't use the template cache, we don't have a node"""
+        return self.template
+
+    def __call__(self, connection_name: str, node) -> str:
+        return str(self.call_macro(connection_name, node))

dbt/adapters/contracts/connection.py

@@ -0,0 +1,229 @@
+import abc
+from dataclasses import dataclass, field
+import itertools
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+)
+
+from dbt_common.contracts.util import Replaceable
+from dbt_common.dataclass_schema import (
+    ExtensibleDbtClassMixin,
+    StrEnum,
+    ValidatedStringMixin,
+    dbtClassMixin,
+)
+
+# TODO: this is a very bad dependency - shared global state
+from dbt_common.events.contextvars import get_node_info
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtInternalError
+from dbt_common.utils import md5
+from mashumaro.jsonschema.annotations import Pattern
+from typing_extensions import Protocol, Annotated
+
+from dbt.adapters.events.types import NewConnectionOpening
+from dbt.adapters.utils import translate_aliases
+
+
+class Identifier(ValidatedStringMixin):
+    ValidationRegex = r"^[A-Za-z_][A-Za-z0-9_]+$"
+
+
+@dataclass
+class AdapterResponse(dbtClassMixin):
+    _message: str
+    code: Optional[str] = None
+    rows_affected: Optional[int] = None
+    query_id: Optional[str] = None
+
+    def __str__(self):
+        return self._message
+
+
+class ConnectionState(StrEnum):
+    INIT = "init"
+    OPEN = "open"
+    CLOSED = "closed"
+    FAIL = "fail"
+
+
+@dataclass(init=False)
+class Connection(ExtensibleDbtClassMixin, Replaceable):
+    # Annotated is used by mashumaro for jsonschema generation
+    type: Annotated[Identifier, Pattern(r"^[A-Za-z_][A-Za-z0-9_]+$")]
+    name: Optional[str] = None
+    state: ConnectionState = ConnectionState.INIT  # type: ignore
+    transaction_open: bool = False
+    _handle: Optional[Any] = None
+    _credentials: Optional[Any] = None
+
+    def __init__(
+        self,
+        type: Identifier,
+        name: Optional[str],
+        credentials: dbtClassMixin,
+        state: ConnectionState = ConnectionState.INIT,  # type: ignore
+        transaction_open: bool = False,
+        handle: Optional[Any] = None,
+    ) -> None:
+        self.type = type
+        self.name = name
+        self.state = state
+        self.credentials = credentials
+        self.transaction_open = transaction_open
+        self.handle = handle
+
+    @property
+    def credentials(self):
+        return self._credentials
+
+    @credentials.setter
+    def credentials(self, value):
+        self._credentials = value
+
+    @property
+    def handle(self):
+        if isinstance(self._handle, LazyHandle):
+            try:
+                # this will actually change 'self._handle'.
+                self._handle.resolve(self)
+            except RecursionError as exc:
+                raise DbtInternalError(
+                    "A connection's open() method attempted to read the handle value"
+                ) from exc
+        return self._handle
+
+    @handle.setter
+    def handle(self, value):
+        self._handle = value
+
+
+class LazyHandle:
+    """The opener must be a callable that takes a Connection object and opens the
+    connection, updating the handle on the Connection.
+    """
+
+    def __init__(self, opener: Callable[[Connection], Connection]) -> None:
+        self.opener = opener
+
+    def resolve(self, connection: Connection) -> Connection:
+        fire_event(
+            NewConnectionOpening(connection_state=connection.state, node_info=get_node_info())
+        )
+        return self.opener(connection)
+
+
+# see https://github.com/python/mypy/issues/4717#issuecomment-373932080
+# and https://github.com/python/mypy/issues/5374
+# for why we have type: ignore. Maybe someday dataclasses + abstract classes
+# will work.
+@dataclass
+class Credentials(ExtensibleDbtClassMixin, Replaceable, metaclass=abc.ABCMeta):
+    database: str
+    schema: str
+    _ALIASES: ClassVar[Dict[str, str]] = field(default={}, init=False)
+
+    @abc.abstractproperty
+    def type(self) -> str:
+        raise NotImplementedError("type not implemented for base credentials class")
+
+    @property
+    def unique_field(self) -> str:
+        """Hashed and included in anonymous telemetry to track adapter adoption.
+        Return the field from Credentials that can uniquely identify
+        one team/organization using this adapter
+        """
+        raise NotImplementedError("unique_field not implemented for base credentials class")
+
+    def hashed_unique_field(self) -> str:
+        return md5(self.unique_field)
+
+    def connection_info(self, *, with_aliases: bool = False) -> Iterable[Tuple[str, Any]]:
+        """Return an ordered iterator of key/value pairs for pretty-printing."""
+        as_dict = self.to_dict(omit_none=False)
+        connection_keys = set(self._connection_keys())
+        aliases: List[str] = []
+        if with_aliases:
+            aliases = [k for k, v in self._ALIASES.items() if v in connection_keys]
+        for key in itertools.chain(self._connection_keys(), aliases):
+            if key in as_dict:
+                yield key, as_dict[key]
+
+    @abc.abstractmethod
+    def _connection_keys(self) -> Tuple[str, ...]:
+        raise NotImplementedError
+
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        # Need to fixup dbname => database, pass => password
+        data = cls.translate_aliases(data)
+        return data
+
+    @classmethod
+    def translate_aliases(cls, kwargs: Dict[str, Any], recurse: bool = False) -> Dict[str, Any]:
+        return translate_aliases(kwargs, cls._ALIASES, recurse)
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
+        # no super() -- do we need it?
+        if self._ALIASES:
+            dct.update(
+                {
+                    new_name: dct[canonical_name]
+                    for new_name, canonical_name in self._ALIASES.items()
+                    if canonical_name in dct
+                }
+            )
+        return dct
+
+
+class HasCredentials(Protocol):
+    credentials: Credentials
+    profile_name: str
+    target_name: str
+    threads: int
+
+    def to_target_dict(self):
+        raise NotImplementedError("to_target_dict not implemented")
+
+
+DEFAULT_QUERY_COMMENT = """
+{%- set comment_dict = {} -%}
+{%- do comment_dict.update(
+    app='dbt',
+    dbt_version=dbt_version,
+    profile_name=target.get('profile_name'),
+    target_name=target.get('target_name'),
+) -%}
+{%- if node is not none -%}
+  {%- do comment_dict.update(
+    node_id=node.unique_id,
+  ) -%}
+{% else %}
+  {# in the node context, the connection name is the node_id #}
+  {%- do comment_dict.update(connection_name=connection_name) -%}
+{%- endif -%}
+{{ return(tojson(comment_dict)) }}
+"""
+
+
+@dataclass
+class QueryComment(dbtClassMixin):
+    comment: str = DEFAULT_QUERY_COMMENT
+    append: Optional[bool] = None
+    job_label: bool = field(default=False, metadata={"alias": "job-label"})
+
+
+class AdapterRequiredConfig(HasCredentials, Protocol):
+    project_name: str
+    query_comment: QueryComment
+    cli_vars: Dict[str, Any]
+    target_path: str
+    log_cache_events: bool

dbt/adapters/contracts/macros.py

@@ -0,0 +1,11 @@
+from typing import Optional
+
+from dbt_common.clients.jinja import MacroProtocol
+from typing_extensions import Protocol
+
+
+class MacroResolverProtocol(Protocol):
+    def find_macro_by_name(
+        self, name: str, root_project_name: str, package: Optional[str]
+    ) -> Optional[MacroProtocol]:
+        raise NotImplementedError("find_macro_by_name not implemented")

dbt/adapters/contracts/relation.py

@@ -0,0 +1,160 @@
+from abc import ABC
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Dict, Optional, Any, Union, List
+
+
+from dbt_common.contracts.config.materialization import OnConfigurationChangeOption
+from dbt_common.contracts.util import Replaceable
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+from dbt_common.exceptions import CompilationError, DataclassNotDictError
+from dbt_common.utils import deep_merge
+from typing_extensions import Protocol
+
+
+class RelationType(StrEnum):
+    Table = "table"
+    View = "view"
+    CTE = "cte"
+    MaterializedView = "materialized_view"
+    Ephemeral = "ephemeral"
+    # this is a "catch all" that is better than `None` == external to anything dbt is aware of
+    External = "external"
+    PointerTable = "pointer_table"
+    Function = "function"
+
+
+class MaterializationContract(Protocol):
+    enforced: bool
+    alias_types: bool
+
+
+class MaterializationConfig(Mapping, ABC):
+    materialized: str
+    incremental_strategy: Optional[str]
+    persist_docs: Dict[str, Any]
+    column_types: Dict[str, Any]
+    full_refresh: Optional[bool]
+    quoting: Dict[str, Any]
+    unique_key: Union[str, List[str], None]
+    on_schema_change: Optional[str]
+    on_configuration_change: OnConfigurationChangeOption
+    contract: MaterializationContract
+    extra: Dict[str, Any]
+
+    def __contains__(self, item): ...
+
+    def __delitem__(self, key): ...
+
+
+class RelationConfig(Protocol):
+    resource_type: str
+    name: str
+    description: str
+    database: str
+    schema: str
+    identifier: str
+    compiled_code: Optional[str]
+    meta: Dict[str, Any]
+    tags: List[str]
+    quoting_dict: Dict[str, bool]
+    config: Optional[MaterializationConfig]
+
+
+class ComponentName(StrEnum):
+    Database = "database"
+    Schema = "schema"
+    Identifier = "identifier"
+
+
+class HasQuoting(Protocol):
+    quoting: Dict[str, bool]
+
+
+class FakeAPIObject(dbtClassMixin, Replaceable, Mapping):
+    # override the mapping truthiness, len is always >1
+    def __bool__(self):
+        return True
+
+    def __getitem__(self, key):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            raise KeyError(key) from None
+
+    def __iter__(self):
+        raise DataclassNotDictError(self)
+
+    def __len__(self):
+        raise DataclassNotDictError(self)
+
+    def incorporate(self, **kwargs):
+        value = self.to_dict(omit_none=True)
+        value = deep_merge(value, kwargs)
+        return self.from_dict(value)
+
+
+@dataclass
+class Policy(FakeAPIObject):
+    database: bool = True
+    schema: bool = True
+    identifier: bool = True
+
+    def get_part(self, key: ComponentName) -> bool:
+        if key == ComponentName.Database:
+            return self.database
+        elif key == ComponentName.Schema:
+            return self.schema
+        elif key == ComponentName.Identifier:
+            return self.identifier
+        else:
+            raise ValueError(
+                "Got a key of {}, expected one of {}".format(key, list(ComponentName))
+            )
+
+    def replace_dict(self, dct: Dict[ComponentName, bool]):
+        kwargs: Dict[str, bool] = {}
+        for k, v in dct.items():
+            kwargs[str(k)] = v
+        return self.replace(**kwargs)
+
+
+@dataclass
+class Path(FakeAPIObject):
+    database: Optional[str] = None
+    schema: Optional[str] = None
+    identifier: Optional[str] = None
+
+    def __post_init__(self):
+        # handle pesky jinja2.Undefined sneaking in here and messing up rende
+        if not isinstance(self.database, (type(None), str)):
+            raise CompilationError("Got an invalid path database: {}".format(self.database))
+        if not isinstance(self.schema, (type(None), str)):
+            raise CompilationError("Got an invalid path schema: {}".format(self.schema))
+        if not isinstance(self.identifier, (type(None), str)):
+            raise CompilationError("Got an invalid path identifier: {}".format(self.identifier))
+
+    def get_lowered_part(self, key: ComponentName) -> Optional[str]:
+        part = self.get_part(key)
+        if part is not None:
+            part = part.lower()
+        return part
+
+    def get_part(self, key: ComponentName) -> Optional[str]:
+        if key == ComponentName.Database:
+            return self.database
+        elif key == ComponentName.Schema:
+            return self.schema
+        elif key == ComponentName.Identifier:
+            return self.identifier
+        else:
+            raise ValueError(
+                "Got a key of {}, expected one of {}".format(key, list(ComponentName))
+            )
+
+    def replace_dict(self, dct: Dict[ComponentName, str]):
+        kwargs: Dict[str, str] = {}
+        for k, v in dct.items():
+            kwargs[str(k)] = v
+        return self.replace(**kwargs)

dbt/adapters/events/README.md

@@ -0,0 +1,51 @@
+# Events Module
+The Events module is responsible for communicating internal dbt structures into a consumable interface. Because the "event" classes are based entirely on protobuf definitions, the interface is really clearly defined, whether or not protobufs are used to consume it. We use Betterproto for compiling the protobuf message definitions into Python classes.
+
+# Using the Events Module
+The event module provides types that represent what is happening in dbt in `events.types`. These types are intended to represent an exhaustive list of all things happening within dbt that will need to be logged, streamed, or printed. To fire an event, `common.events.functions::fire_event` is the entry point to the module from everywhere in dbt.
+
+# Logging
+When events are processed via `fire_event`, nearly everything is logged. Whether or not the user has enabled the debug flag, all debug messages are still logged to the file. However, some events are particularly time consuming to construct because they return a huge amount of data. Today, the only messages in this category are cache events and are only logged if the `--log-cache-events` flag is on. This is important because these messages should not be created unless they are going to be logged, because they cause a noticable performance degredation. These events use a "fire_event_if" functions.
+
+# Adding a New Event
+All protos have been moved into the central protos repository. To edit an event proto, edit https://github.com/dbt-labs/proto-python-public or open an issue on that repository.
+
+## Required for Every Event
+
+- a method `code`, that's unique across events
+- assign a log level by using the Level mixin: `DebugLevel`, `InfoLevel`, `WarnLevel`, or `ErrorLevel`
+- a message()
+
+Example
+```
+class PartialParsingDeletedExposure(DebugLevel):
+    def code(self):
+        return "I049"
+
+    def message(self) -> str:
+        return f"Partial parsing: deleted exposure {self.unique_id}"
+
+```
+
+
+# Adapter Maintainers
+To integrate existing log messages from adapters, you likely have a line of code like this in your adapter already:
+```python
+from dbt.logger import GLOBAL_LOGGER as logger
+```
+
+Simply change it to these two lines with your adapter's database name, and all your existing call sites will now use the new system for v1.0:
+
+```python
+
+from dbt.adapters.events.logging import AdapterLogger
+
+logger = AdapterLogger("<database name>")
+# e.g. AdapterLogger("Snowflake")
+```
+
+## Compiling types.proto
+
+After adding a new message in `adapter_types.proto`, either:
+- In the repository root directory: `make adapter_proto_types`
+- In the `core/dbt/adapters/events` directory: `protoc -I=. --python_out=. types.proto`

dbt/adapters/events/adapter_types_pb2.py

@@ -0,0 +1,2 @@
+# preserving import path during dbtlabs.proto refactor
+from dbtlabs.proto.public.v1.fields.adapter_types_pb2 import *  # noqa

dbt/adapters/events/base_types.py

@@ -0,0 +1,36 @@
+from dbt_common.events.base_types import BaseEvent
+from dbt_common.events.base_types import DebugLevel as CommonDebugLevel
+from dbt_common.events.base_types import DynamicLevel as CommonDynamicLevel
+from dbt_common.events.base_types import ErrorLevel as CommonErrorLevel
+from dbt_common.events.base_types import InfoLevel as CommonInfoLevel
+from dbt_common.events.base_types import TestLevel as CommonTestLevel
+from dbt_common.events.base_types import WarnLevel as CommonWarnLevel
+from dbt.adapters.events import adapter_types_pb2
+
+
+class AdapterBaseEvent(BaseEvent):
+    PROTO_TYPES_MODULE = adapter_types_pb2
+
+
+class DynamicLevel(CommonDynamicLevel, AdapterBaseEvent):
+    pass
+
+
+class TestLevel(CommonTestLevel, AdapterBaseEvent):
+    pass
+
+
+class DebugLevel(CommonDebugLevel, AdapterBaseEvent):
+    pass
+
+
+class InfoLevel(CommonInfoLevel, AdapterBaseEvent):
+    pass
+
+
+class WarnLevel(CommonWarnLevel, AdapterBaseEvent):
+    pass
+
+
+class ErrorLevel(CommonErrorLevel, AdapterBaseEvent):
+    pass