dvt-core 1.11.0b4__py3-none-any.whl

dvt/context/exceptions_jinja.py

@@ -0,0 +1,179 @@
+import functools
+from typing import NoReturn
+
+from dvt.events.types import JinjaLogWarning, SnapshotTimestampWarning
+from dvt.exceptions import (
+    AmbiguousAliasError,
+    AmbiguousCatalogMatchError,
+    CompilationError,
+    ContractError,
+    DependencyError,
+    DependencyNotFoundError,
+    DuplicatePatchPathError,
+    DuplicateResourceNameError,
+    FailFastError,
+    MissingRelationError,
+    PropertyYMLError,
+    env_secrets,
+    scrub_secrets,
+)
+
+from dbt.adapters.exceptions import (
+    ColumnTypeMissingError,
+    MissingConfigError,
+    MissingMaterializationError,
+    RelationWrongTypeError,
+)
+from dbt.adapters.exceptions.cache import CacheInconsistencyError
+from dbt_common.events.functions import warn_or_error
+from dbt_common.exceptions import (
+    DataclassNotDictError,
+    DbtDatabaseError,
+    DbtRuntimeError,
+    NotImplementedError,
+)
+
+
+def warn(msg, node=None):
+    warn_or_error(JinjaLogWarning(msg=msg), node=node)
+    return ""
+
+
+def missing_config(model, name) -> NoReturn:
+    raise MissingConfigError(unique_id=model.unique_id, name=name)
+
+
+def missing_materialization(model, adapter_type) -> NoReturn:
+    raise MissingMaterializationError(
+        materialization=model.config.materialized, adapter_type=adapter_type
+    )
+
+
+def missing_relation(relation, model=None) -> NoReturn:
+    raise MissingRelationError(relation, model)
+
+
+def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn:
+    raise AmbiguousAliasError(node_1, node_2, duped_name)
+
+
+def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn:
+    raise AmbiguousCatalogMatchError(unique_id, match_1, match_2)
+
+
+def raise_cache_inconsistent(message) -> NoReturn:
+    raise CacheInconsistencyError(message)
+
+
+def raise_dataclass_not_dict(obj) -> NoReturn:
+    raise DataclassNotDictError(obj)
+
+
+def raise_compiler_error(msg, node=None) -> NoReturn:
+    raise CompilationError(msg, node)
+
+
+def raise_contract_error(yaml_columns, sql_columns) -> NoReturn:
+    raise ContractError(yaml_columns, sql_columns)
+
+
+def raise_database_error(msg, node=None) -> NoReturn:
+    raise DbtDatabaseError(msg, node)
+
+
+def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn:
+    raise DependencyNotFoundError(node, node_description, required_pkg)
+
+
+def raise_dependency_error(msg) -> NoReturn:
+    raise DependencyError(scrub_secrets(msg, env_secrets()))
+
+
+def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn:
+    raise DuplicatePatchPathError(patch_1, existing_patch_path)
+
+
+def raise_duplicate_resource_name(node_1, node_2) -> NoReturn:
+    raise DuplicateResourceNameError(node_1, node_2)
+
+
+def raise_invalid_property_yml_version(path, issue) -> NoReturn:
+    raise PropertyYMLError(path, issue)
+
+
+def raise_not_implemented(msg) -> NoReturn:
+    raise NotImplementedError(msg)
+
+
+def relation_wrong_type(relation, expected_type, model=None) -> NoReturn:
+    raise RelationWrongTypeError(relation, expected_type, model)
+
+
+def column_type_missing(column_names) -> NoReturn:
+    raise ColumnTypeMissingError(column_names)
+
+
+def raise_fail_fast_error(msg, node=None) -> NoReturn:
+    raise FailFastError(msg, node=node)
+
+
+def warn_snapshot_timestamp_data_types(
+    snapshot_time_data_type: str, updated_at_data_type: str
+) -> None:
+    warn_or_error(
+        SnapshotTimestampWarning(
+            snapshot_time_data_type=snapshot_time_data_type,
+            updated_at_data_type=updated_at_data_type,
+        )
+    )
+
+
+# Update this when a new function should be added to the
+# dbt context's `exceptions` key!
+CONTEXT_EXPORTS = {
+    fn.__name__: fn
+    for fn in [
+        warn,
+        missing_config,
+        missing_materialization,
+        missing_relation,
+        raise_ambiguous_alias,
+        raise_ambiguous_catalog_match,
+        raise_cache_inconsistent,
+        raise_dataclass_not_dict,
+        raise_compiler_error,
+        raise_database_error,
+        raise_dep_not_found,
+        raise_dependency_error,
+        raise_duplicate_patch_name,
+        raise_duplicate_resource_name,
+        raise_invalid_property_yml_version,
+        raise_not_implemented,
+        relation_wrong_type,
+        raise_contract_error,
+        column_type_missing,
+        raise_fail_fast_error,
+        warn_snapshot_timestamp_data_types,
+    ]
+}
+
+
+# wraps context based exceptions in node info
+def wrapper(model):
+    def wrap(func):
+        @functools.wraps(func)
+        def inner(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except DbtRuntimeError as exc:
+                exc.add_node(model)
+                raise exc
+
+        return inner
+
+    return wrap
+
+
+def wrapped_exports(model):
+    wrap = wrapper(model)
+    return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()}

dvt/context/macro_resolver.py

@@ -0,0 +1,195 @@
+from typing import Dict, MutableMapping, Optional
+
+from dvt.clients.jinja import MacroGenerator
+from dvt.contracts.graph.nodes import Macro
+from dvt.exceptions import DuplicateMacroNameError, PackageNotFoundForMacroError
+from dvt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME
+
+MacroNamespace = Dict[str, Macro]
+
+
+# This class builds the MacroResolver by adding macros
+# to various categories for finding macros in the right order,
+# so that higher precedence macros are found first.
+# This functionality is also provided by the MacroNamespace,
+# but the intention is to eventually replace that class.
+# This enables us to get the macro unique_id without
+# processing every macro in the project.
+# Note: the root project macros override everything in the
+# dbt internal projects. External projects (dependencies) will
+# use their own macros first, then pull from the root project
+# followed by dbt internal projects.
+class MacroResolver:
+    def __init__(
+        self,
+        macros: MutableMapping[str, Macro],
+        root_project_name: str,
+        internal_package_names,
+    ) -> None:
+        self.root_project_name = root_project_name
+        self.macros = macros
+        # internal packages comes from get_adapter_package_names
+        self.internal_package_names = internal_package_names
+
+        # To be filled in from macros.
+        self.internal_packages: Dict[str, MacroNamespace] = {}
+        self.packages: Dict[str, MacroNamespace] = {}
+        self.root_package_macros: MacroNamespace = {}
+
+        # add the macros to internal_packages, packages, and root packages
+        self.add_macros()
+        self._build_internal_packages_namespace()
+        self._build_macros_by_name()
+
+    def _build_internal_packages_namespace(self) -> None:
+        # Iterate in reverse-order and overwrite: the packages that are first
+        # in the list are the ones we want to "win".
+        self.internal_packages_namespace: MacroNamespace = {}
+        for pkg in reversed(self.internal_package_names):
+            if pkg in self.internal_packages:
+                # Turn the internal packages into a flat namespace
+                self.internal_packages_namespace.update(self.internal_packages[pkg])
+
+    # search order:
+    # local_namespace (package of particular node), not including
+    #    the internal packages or the root package
+    #    This means that within an extra package, it uses its own macros
+    # root package namespace
+    # non-internal packages (that aren't local or root)
+    # dbt internal packages
+    def _build_macros_by_name(self) -> None:
+        macros_by_name = {}
+
+        # all internal packages (already in the right order)
+        for macro in self.internal_packages_namespace.values():
+            macros_by_name[macro.name] = macro
+
+        # non-internal packages
+        for fnamespace in self.packages.values():
+            for macro in fnamespace.values():
+                macros_by_name[macro.name] = macro
+
+        # root package macros
+        for macro in self.root_package_macros.values():
+            macros_by_name[macro.name] = macro
+
+        self.macros_by_name = macros_by_name
+
+    def _add_macro_to(
+        self,
+        package_namespaces: Dict[str, MacroNamespace],
+        macro: Macro,
+    ) -> None:
+        if macro.package_name in package_namespaces:
+            namespace = package_namespaces[macro.package_name]
+        else:
+            namespace = {}
+            package_namespaces[macro.package_name] = namespace
+
+        if macro.name in namespace:
+            raise DuplicateMacroNameError(macro, macro, macro.package_name)
+        package_namespaces[macro.package_name][macro.name] = macro
+
+    def add_macro(self, macro: Macro) -> None:
+        macro_name: str = macro.name
+
+        # internal macros (from plugins) will be processed separately from
+        # project macros, so store them in a different place
+        if macro.package_name in self.internal_package_names:
+            self._add_macro_to(self.internal_packages, macro)
+        else:
+            # if it's not an internal package
+            self._add_macro_to(self.packages, macro)
+            # add to root_package_macros if it's in the root package
+            if macro.package_name == self.root_project_name:
+                self.root_package_macros[macro_name] = macro
+
+    def add_macros(self) -> None:
+        for macro in self.macros.values():
+            self.add_macro(macro)
+
+    def get_macro(self, local_package, macro_name) -> Optional[Macro]:
+        local_package_macros = {}
+        # If the macro is explicitly prefixed with an internal namespace
+        # (e.g. 'dbt.some_macro'), look there first
+        if local_package in self.internal_package_names:
+            local_package_macros = self.internal_packages[local_package]
+        # If the macro is explicitly prefixed with a different package name
+        # (e.g. 'dbt_utils.some_macro'), look there first
+        if local_package not in self.internal_package_names and local_package in self.packages:
+            local_package_macros = self.packages[local_package]
+        # First: search the specified package for this macro
+        if macro_name in local_package_macros:
+            return local_package_macros[macro_name]
+        # Now look up in the standard search order
+        if macro_name in self.macros_by_name:
+            return self.macros_by_name[macro_name]
+        return None
+
+    def get_macro_id(self, local_package, macro_name) -> Optional[str]:
+        macro = self.get_macro(local_package, macro_name)
+        if macro is None:
+            return None
+        else:
+            return macro.unique_id
+
+
+# Currently this is just used by test processing in the schema
+# parser (in connection with the MacroResolver). Future work
+# will extend the use of these classes to other parsing areas.
+# One of the features of this class compared to the MacroNamespace
+# is that you can limit the number of macros provided to the
+# context dictionary in the 'to_dict' manifest method.
+class TestMacroNamespace:
+    def __init__(self, macro_resolver, ctx, node, thread_ctx, depends_on_macros):
+        self.macro_resolver = macro_resolver
+        self.ctx = ctx
+        self.node = node  # can be none
+        self.thread_ctx = thread_ctx
+        self.local_namespace = {}
+        self.project_namespace = {}
+        if depends_on_macros:
+            dep_macros = []
+            self.recursively_get_depends_on_macros(depends_on_macros, dep_macros)
+            for macro_unique_id in dep_macros:
+                if macro_unique_id in self.macro_resolver.macros:
+                    # Split up the macro unique_id to get the project_name
+                    (_, project_name, macro_name) = macro_unique_id.split(".")
+                    # Save the plain macro_name in the local_namespace
+                    macro = self.macro_resolver.macros[macro_unique_id]
+                    macro_gen = MacroGenerator(
+                        macro,
+                        self.ctx,
+                        self.node,
+                        self.thread_ctx,
+                    )
+                    self.local_namespace[macro_name] = macro_gen
+                    # We also need the two part macro name
+                    if project_name not in self.project_namespace:
+                        self.project_namespace[project_name] = {}
+                    self.project_namespace[project_name][macro_name] = macro_gen
+
+    def recursively_get_depends_on_macros(self, depends_on_macros, dep_macros):
+        for macro_unique_id in depends_on_macros:
+            if macro_unique_id in dep_macros:
+                continue
+            dep_macros.append(macro_unique_id)
+            if macro_unique_id in self.macro_resolver.macros:
+                macro = self.macro_resolver.macros[macro_unique_id]
+                if macro.depends_on.macros:
+                    self.recursively_get_depends_on_macros(macro.depends_on.macros, dep_macros)
+
+    def get_from_package(self, package_name: Optional[str], name: str) -> Optional[MacroGenerator]:
+        macro = None
+        if package_name is None:
+            macro = self.macro_resolver.macros_by_name.get(name)
+        elif package_name == GLOBAL_PROJECT_NAME:
+            macro = self.macro_resolver.internal_packages_namespace.get(name)
+        elif package_name in self.macro_resolver.packages:
+            macro = self.macro_resolver.packages[package_name].get(name)
+        else:
+            raise PackageNotFoundForMacroError(package_name)
+        if not macro:
+            return None
+        macro_func = MacroGenerator(macro, self.ctx, self.node, self.thread_ctx)
+        return macro_func

dvt/context/macros.py

@@ -0,0 +1,171 @@
+from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Set, Union
+
+from dvt.clients.jinja import MacroGenerator, MacroStack
+from dvt.contracts.graph.nodes import Macro
+from dvt.exceptions import DuplicateMacroNameError, PackageNotFoundForMacroError
+from dvt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME
+
+FlatNamespace = Dict[str, MacroGenerator]
+NamespaceMember = Union[FlatNamespace, MacroGenerator]
+FullNamespace = Dict[str, NamespaceMember]
+
+
+# The point of this class is to collect the various macros
+# and provide the ability to flatten them into the ManifestContexts
+# that are created for jinja, so that macro calls can be resolved.
+# Creates special iterators and _keys methods to flatten the lists.
+# When this class is created it has a static 'local_namespace' which
+# depends on the package of the node, so it only works for one
+# particular local package at a time for "flattening" into a context.
+# 'get_by_package' should work for any macro.
+class MacroNamespace(Mapping):
+    def __init__(
+        self,
+        global_namespace: FlatNamespace,  # root package macros
+        local_namespace: FlatNamespace,  # packages for *this* node
+        global_project_namespace: FlatNamespace,  # internal packages
+        packages: Dict[str, FlatNamespace],  # non-internal packages
+    ):
+        self.global_namespace: FlatNamespace = global_namespace
+        self.local_namespace: FlatNamespace = local_namespace
+        self.packages: Dict[str, FlatNamespace] = packages
+        self.global_project_namespace: FlatNamespace = global_project_namespace
+
+    def _search_order(self) -> Iterable[Union[FullNamespace, FlatNamespace]]:
+        yield self.local_namespace  # local package
+        yield self.global_namespace  # root package
+        # TODO CT-211
+        yield self.packages  # type: ignore[misc] # non-internal packages
+        yield {
+            # TODO CT-211
+            GLOBAL_PROJECT_NAME: self.global_project_namespace,  # type: ignore[misc] # dbt
+        }
+        yield self.global_project_namespace  # other internal project besides dbt
+
+    # provides special keys method for MacroNamespace iterator
+    # returns keys from local_namespace, global_namespace, packages,
+    # global_project_namespace
+    def _keys(self) -> Set[str]:
+        keys: Set[str] = set()
+        for search in self._search_order():
+            keys.update(search)
+        return keys
+
+    # special iterator using special keys
+    def __iter__(self) -> Iterator[str]:
+        for key in self._keys():
+            yield key
+
+    def __len__(self):
+        return len(self._keys())
+
+    def __getitem__(self, key: str) -> NamespaceMember:
+        for dct in self._search_order():
+            if key in dct:
+                return dct[key]
+        raise KeyError(key)
+
+    def get_from_package(self, package_name: Optional[str], name: str) -> Optional[MacroGenerator]:
+        if package_name is None:
+            return self.get(name)
+        elif package_name == GLOBAL_PROJECT_NAME:
+            return self.global_project_namespace.get(name)
+        elif package_name in self.packages:
+            return self.packages[package_name].get(name)
+        else:
+            raise PackageNotFoundForMacroError(package_name)
+
+
+# This class builds the MacroNamespace by adding macros to
+# internal_packages or packages, and locals/globals.
+# Call 'build_namespace' to return a MacroNamespace.
+# This is used by ManifestContext (and subclasses)
+class MacroNamespaceBuilder:
+    def __init__(
+        self,
+        root_package: str,
+        search_package: str,
+        thread_ctx: MacroStack,
+        internal_packages: List[str],
+        node: Optional[Any] = None,
+    ) -> None:
+        self.root_package = root_package
+        self.search_package = search_package
+        # internal packages comes from get_adapter_package_names
+        self.internal_package_names = set(internal_packages)
+        self.internal_package_names_order = internal_packages
+        # macro_func is added here if in root package, since
+        # the root package acts as a "global" namespace, overriding
+        # everything else except local external package macro calls
+        self.globals: FlatNamespace = {}
+        # macro_func is added here if it's the package for this node
+        self.locals: FlatNamespace = {}
+        # Create a dictionary of [package name][macro name] =
+        #     MacroGenerator object which acts like a function
+        self.internal_packages: Dict[str, FlatNamespace] = {}
+        self.packages: Dict[str, FlatNamespace] = {}
+        self.thread_ctx = thread_ctx
+        self.node = node
+
+    def _add_macro_to(
+        self,
+        hierarchy: Dict[str, FlatNamespace],
+        macro: Macro,
+        macro_func: MacroGenerator,
+    ):
+        if macro.package_name in hierarchy:
+            namespace = hierarchy[macro.package_name]
+        else:
+            namespace = {}
+            hierarchy[macro.package_name] = namespace
+
+        if macro.name in namespace:
+            raise DuplicateMacroNameError(macro_func.macro, macro, macro.package_name)
+        hierarchy[macro.package_name][macro.name] = macro_func
+
+    def add_macro(self, macro: Macro, ctx: Dict[str, Any]) -> None:
+        macro_name: str = macro.name
+
+        # MacroGenerator is in clients/jinja.py
+        # a MacroGenerator object is a callable object that will
+        # execute the MacroGenerator.__call__ function
+        macro_func: MacroGenerator = MacroGenerator(macro, ctx, self.node, self.thread_ctx)
+
+        # internal macros (from plugins) will be processed separately from
+        # project macros, so store them in a different place
+        if macro.package_name in self.internal_package_names:
+            self._add_macro_to(self.internal_packages, macro, macro_func)
+        else:
+            # if it's not an internal package
+            self._add_macro_to(self.packages, macro, macro_func)
+            # add to locals if it's the package this node is in
+            if macro.package_name == self.search_package:
+                self.locals[macro_name] = macro_func
+            # add to globals if it's in the root package
+            elif macro.package_name == self.root_package:
+                self.globals[macro_name] = macro_func
+
+    def add_macros(self, macros: Iterable[Macro], ctx: Dict[str, Any]) -> None:
+        for macro in macros:
+            self.add_macro(macro, ctx)
+
+    def build_namespace(
+        self, macros_by_package: Dict[str, Dict[str, Macro]], ctx: Dict[str, Any]
+    ) -> MacroNamespace:
+        for package in macros_by_package.values():
+            self.add_macros(package.values(), ctx)
+
+        # Iterate in reverse-order and overwrite: the packages that are first
+        # in the list are the ones we want to "win".
+        global_project_namespace: FlatNamespace = {}
+        for pkg in reversed(self.internal_package_names_order):
+            if pkg in self.internal_packages:
+                # add the macros pointed to by this package name
+                global_project_namespace.update(self.internal_packages[pkg])
+
+        return MacroNamespace(
+            global_namespace=self.globals,  # root package macros
+            local_namespace=self.locals,  # packages for *this* node
+            global_project_namespace=global_project_namespace,  # internal packages
+            packages=self.packages,  # non internal_packages
+        )

dvt/context/manifest.py

@@ -0,0 +1,73 @@
+from typing import List
+
+from dvt.clients.jinja import MacroStack
+from dvt.context.macro_resolver import TestMacroNamespace
+from dvt.contracts.graph.manifest import Manifest
+
+from dbt.adapters.contracts.connection import AdapterRequiredConfig
+
+from .base import contextproperty
+from .configured import ConfiguredContext
+from .macros import MacroNamespace, MacroNamespaceBuilder
+
+
+class ManifestContext(ConfiguredContext):
+    """The Macro context has everything in the target context, plus the macros
+    in the manifest.
+
+    The given macros can override any previous context values, which will be
+    available as if they were accessed relative to the package name.
+    """
+
+    # subclasses are QueryHeaderContext and ProviderContext
+    def __init__(
+        self,
+        config: AdapterRequiredConfig,
+        manifest: Manifest,
+        search_package: str,
+    ) -> None:
+        super().__init__(config)
+        self.manifest = manifest
+        # this is the package of the node for which this context was built
+        self.search_package = search_package
+        self.macro_stack = MacroStack()
+        # This namespace is used by the BaseDatabaseWrapper in jinja rendering.
+        # The namespace is passed to it when it's constructed. It expects
+        # to be able to do: namespace.get_from_package(..)
+        self.namespace = self._build_namespace()
+
+    def _build_namespace(self) -> MacroNamespace:
+        # this takes all the macros in the manifest and adds them
+        # to the MacroNamespaceBuilder stored in self.namespace
+        builder = self._get_namespace_builder()
+        return builder.build_namespace(self.manifest.get_macros_by_package(), self._ctx)
+
+    def _get_namespace_builder(self) -> MacroNamespaceBuilder:
+        # avoid an import loop
+        from dbt.adapters.factory import get_adapter_package_names
+
+        internal_packages: List[str] = get_adapter_package_names(self.config.credentials.type)
+        return MacroNamespaceBuilder(
+            self.config.project_name,
+            self.search_package,
+            self.macro_stack,
+            internal_packages,
+            None,
+        )
+
+    # This does not use the Mashumaro code
+    def to_dict(self):
+        dct = super().to_dict()
+        # This moves all of the macros in the 'namespace' into top level
+        # keys in the manifest dictionary
+        if isinstance(self.namespace, TestMacroNamespace):
+            dct.update(self.namespace.local_namespace)
+            dct.update(self.namespace.project_namespace)
+        else:
+            dct.update(self.namespace)
+
+        return dct
+
+    @contextproperty()
+    def context_macro_stack(self):
+        return self.macro_stack