dvt-core 1.11.0b4__py3-none-any.whl

dvt/parser/generic_test_builders.py

@@ -0,0 +1,334 @@
+import re
+from copy import deepcopy
+from typing import Any, Dict, Generic, List, Optional, Tuple
+
+from dvt import deprecations
+from dvt.artifacts.resources import NodeVersion
+from dvt.clients.jinja import GENERIC_TEST_KWARGS_NAME, get_rendered
+from dvt.contracts.graph.nodes import UnpatchedSourceDefinition
+from dvt.contracts.graph.unparsed import UnparsedModelUpdate, UnparsedNodeUpdate
+from dvt.exceptions import (
+    CustomMacroPopulatingConfigValueError,
+    SameKeyNestedError,
+    TagNotStringError,
+    TagsNotListOfStringsError,
+    TestArgIncludesModelError,
+    TestArgsNotDictError,
+    TestDefinitionDictLengthError,
+    TestNameNotStringError,
+    TestTypeError,
+    UnexpectedTestNamePatternError,
+)
+from dvt.flags import get_flags
+from dvt.parser.common import Testable
+from dvt.utils import md5
+
+from dbt_common.exceptions.macros import UndefinedMacroError
+
+
+def synthesize_generic_test_names(
+    test_type: str, test_name: str, args: Dict[str, Any]
+) -> Tuple[str, str]:
+    # Using the type, name, and arguments to this generic test, synthesize a (hopefully) unique name
+    # Will not be unique if multiple tests have same name + arguments, and only configs differ
+    # Returns a shorter version (hashed/truncated, for the compiled file)
+    # as well as the full name (for the unique_id + FQN)
+    flat_args = []
+    for arg_name in sorted(args):
+        # the model is already embedded in the name, so skip it
+        if arg_name == "model":
+            continue
+        arg_val = args[arg_name]
+
+        if isinstance(arg_val, dict):
+            parts = list(arg_val.values())
+        elif isinstance(arg_val, (list, tuple)):
+            parts = list(arg_val)
+        else:
+            parts = [arg_val]
+
+        flat_args.extend([str(part) for part in parts])
+
+    clean_flat_args = [re.sub("[^0-9a-zA-Z_]+", "_", arg) for arg in flat_args]
+    unique = "__".join(clean_flat_args)
+
+    # for the file path + alias, the name must be <64 characters
+    # if the full name is too long, include the first 30 identifying chars plus
+    # a 32-character hash of the full contents
+
+    test_identifier = "{}_{}".format(test_type, test_name)
+    full_name = "{}_{}".format(test_identifier, unique)
+
+    if len(full_name) >= 64:
+        test_trunc_identifier = test_identifier[:30]
+        label = md5(full_name)
+        short_name = "{}_{}".format(test_trunc_identifier, label)
+    else:
+        short_name = full_name
+
+    return short_name, full_name
+
+
+class TestBuilder(Generic[Testable]):
+    """An object to hold assorted test settings and perform basic parsing
+
+    Test names have the following pattern:
+        - the test name itself may be namespaced (package.test)
+        - or it may not be namespaced (test)
+
+    """
+
+    # The 'test_name' is used to find the 'macro' that implements the test
+    TEST_NAME_PATTERN = re.compile(
+        r"((?P<test_namespace>([a-zA-Z_][0-9a-zA-Z_]*))\.)?"
+        r"(?P<test_name>([a-zA-Z_][0-9a-zA-Z_]*))"
+    )
+    # args in the test entry representing test configs
+    CONFIG_ARGS = (
+        "severity",
+        "tags",
+        "enabled",
+        "where",
+        "limit",
+        "warn_if",
+        "error_if",
+        "fail_calc",
+        "store_failures",
+        "store_failures_as",
+        "meta",
+        "database",
+        "schema",
+        "alias",
+    )
+
+    def __init__(
+        self,
+        data_test: Dict[str, Any],
+        target: Testable,
+        package_name: str,
+        render_ctx: Dict[str, Any],
+        column_name: Optional[str] = None,
+        version: Optional[NodeVersion] = None,
+    ) -> None:
+        test_name, test_args = self.extract_test_args(
+            data_test, target.original_file_path, target.name, column_name, package_name
+        )
+        self.args: Dict[str, Any] = test_args
+        if "model" in self.args:
+            raise TestArgIncludesModelError()
+        self.package_name: str = package_name
+        self.target: Testable = target
+        self.version: Optional[NodeVersion] = version
+        self.render_ctx: Dict[str, Any] = render_ctx
+        self.column_name: Optional[str] = column_name
+        self.args["model"] = self.build_model_str()
+
+        match = self.TEST_NAME_PATTERN.match(test_name)
+        if match is None:
+            raise UnexpectedTestNamePatternError(test_name)
+
+        groups = match.groupdict()
+        self.name: str = groups["test_name"]
+        self.namespace: str = groups["test_namespace"]
+        self.config: Dict[str, Any] = {}
+        # Process legacy args
+        self.config.update(self._process_legacy_args())
+
+        # Process config args if present
+        if "config" in self.args:
+            self.config.update(self._render_values(self.args.pop("config", {})))
+
+        if self.namespace is not None:
+            self.package_name = self.namespace
+
+        # If the user has provided a description for this generic test, use it
+        # Then delete the "description" argument to:
+        # 1. Avoid passing it into the test macro
+        # 2. Avoid passing it into the test name synthesis
+        # Otherwise, use an empty string
+        self.description: str = ""
+
+        if "description" in self.args:
+            self.description = self.args["description"]
+            del self.args["description"]
+
+        # If the user has provided a custom name for this generic test, use it
+        # Then delete the "name" argument to avoid passing it into the test macro
+        # Otherwise, use an auto-generated name synthesized from test inputs
+        self.compiled_name: str = ""
+        self.fqn_name: str = ""
+
+        if "name" in self.args:
+            # Assign the user-defined name here, which will be checked for uniqueness later
+            # we will raise an error if two tests have same name for same model + column combo
+            self.compiled_name = self.args["name"]
+            self.fqn_name = self.args["name"]
+            del self.args["name"]
+        else:
+            short_name, full_name = self.get_synthetic_test_names()
+            self.compiled_name = short_name
+            self.fqn_name = full_name
+            # use hashed name as alias if full name is too long
+            if short_name != full_name and "alias" not in self.config:
+                self.config["alias"] = short_name
+
+    def _process_legacy_args(self):
+        config = {}
+        for key in self.CONFIG_ARGS:
+            value = self.args.pop(key, None)
+            if value and "config" in self.args and key in self.args["config"]:
+                raise SameKeyNestedError()
+            if not value and "config" in self.args:
+                value = self.args["config"].pop(key, None)
+            config[key] = value
+
+        return self._render_values(config)
+
+    def _render_values(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        rendered_config = {}
+        for key, value in config.items():
+            if isinstance(value, str):
+                try:
+                    value = get_rendered(value, self.render_ctx, native=True)
+                except UndefinedMacroError as e:
+                    raise CustomMacroPopulatingConfigValueError(
+                        target_name=self.target.name,
+                        column_name=self.column_name,
+                        name=self.name,
+                        key=key,
+                        err_msg=e.msg,
+                    )
+            if value is not None:
+                rendered_config[key] = value
+        return rendered_config
+
+    def _bad_type(self) -> TypeError:
+        return TypeError('invalid target type "{}"'.format(type(self.target)))
+
+    @staticmethod
+    def extract_test_args(
+        data_test, file_path, resource_name=None, column_name=None, package_name=None
+    ) -> Tuple[str, Dict[str, Any]]:
+        if not isinstance(data_test, dict):
+            raise TestTypeError(data_test)
+
+        # If the test is a dictionary with top-level keys, the test name is "test_name"
+        # and the rest are arguments
+        # {'name': 'my_favorite_test', 'test_name': 'unique', 'config': {'where': '1=1'}}
+        if "test_name" in data_test.keys():
+            test_name = data_test.pop("test_name")
+            test_args = data_test
+        # If the test is a nested dictionary with one top-level key, the test name
+        # is the dict name, and nested keys are arguments
+        # {'unique': {'name': 'my_favorite_test', 'config': {'where': '1=1'}}}
+        else:
+            data_test = list(data_test.items())
+            if len(data_test) != 1:
+                raise TestDefinitionDictLengthError(data_test)
+            test_name, test_args = data_test[0]
+
+        if not isinstance(test_args, dict):
+            raise TestArgsNotDictError(test_args)
+        if not isinstance(test_name, str):
+            raise TestNameNotStringError(test_name)
+        test_args = deepcopy(test_args)
+        if column_name is not None:
+            test_args["column_name"] = column_name
+
+        # Extract kwargs when they are nested under new 'arguments' property separately from 'config' if require_generic_test_arguments_property is enabled
+        if get_flags().require_generic_test_arguments_property:
+            arguments = test_args.pop("arguments", {})
+            if not arguments and any(
+                k not in ("config", "column_name", "description", "name") for k in test_args.keys()
+            ):
+                resource = (
+                    f"'{resource_name}' in package '{package_name}'"
+                    if package_name
+                    else f"'{resource_name}'"
+                )
+                deprecations.warn(
+                    "missing-arguments-property-in-generic-test-deprecation",
+                    test_name=f"`{test_name}` defined on {resource} ({file_path})",
+                )
+            if isinstance(arguments, dict):
+                test_args = {**test_args, **arguments}
+        elif "arguments" in test_args:
+            deprecations.warn(
+                "arguments-property-in-generic-test-deprecation",
+                test_name=f"`{test_name}` ({test_args['arguments']})",
+            )
+
+        return test_name, test_args
+
+    def tags(self) -> List[str]:
+        tags = self.config.get("tags", [])
+        if isinstance(tags, str):
+            tags = [tags]
+        if not isinstance(tags, list):
+            raise TagsNotListOfStringsError(tags)
+        for tag in tags:
+            if not isinstance(tag, str):
+                raise TagNotStringError(tag)
+        return tags[:]
+
+    def macro_name(self) -> str:
+        macro_name = "test_{}".format(self.name)
+        if self.namespace is not None:
+            macro_name = "{}.{}".format(self.namespace, macro_name)
+        return macro_name
+
+    def get_synthetic_test_names(self) -> Tuple[str, str]:
+        # Returns two names: shorter (for the compiled file), full (for the unique_id + FQN)
+        target_name = self.target.name
+        if isinstance(self.target, UnparsedModelUpdate):
+            name = self.name
+            if self.version:
+                target_name = f"{self.target.name}_v{self.version}"
+        elif isinstance(self.target, UnparsedNodeUpdate):
+            name = self.name
+        elif isinstance(self.target, UnpatchedSourceDefinition):
+            name = "source_" + self.name
+        else:
+            raise self._bad_type()
+        if self.namespace is not None:
+            name = "{}_{}".format(self.namespace, name)
+        return synthesize_generic_test_names(name, target_name, self.args)
+
+    def construct_config(self) -> str:
+        configs = ",".join(
+            [
+                f"{key}="
+                + (
+                    ('"' + value.replace('"', '\\"') + '"')
+                    if isinstance(value, str)
+                    else str(value)
+                )
+                for key, value in self.config.items()
+            ]
+        )
+        if configs:
+            return f"{{{{ config({configs}) }}}}"
+        else:
+            return ""
+
+    # this is the 'raw_code' that's used in 'render_update' and execution
+    # of the test macro
+    def build_raw_code(self) -> str:
+        return ("{{{{ {macro}(**{kwargs_name}) }}}}{config}").format(
+            macro=self.macro_name(),
+            config=self.construct_config(),
+            kwargs_name=GENERIC_TEST_KWARGS_NAME,
+        )
+
+    def build_model_str(self):
+        targ = self.target
+        if isinstance(self.target, UnparsedModelUpdate):
+            if self.version:
+                target_str = f"ref('{targ.name}', version='{self.version}')"
+            else:
+                target_str = f"ref('{targ.name}')"
+        elif isinstance(self.target, UnparsedNodeUpdate):
+            target_str = f"ref('{targ.name}')"
+        elif isinstance(self.target, UnpatchedSourceDefinition):
+            target_str = f"source('{targ.source.name}', '{targ.table.name}')"
+        return f"{{{{ get_where_subquery({target_str}) }}}}"

dvt/parser/hooks.py

@@ -0,0 +1,119 @@
+from dataclasses import dataclass
+from typing import Iterable, Iterator, List, Tuple, Union
+
+from dvt.context.context_config import ContextConfig
+from dvt.contracts.files import FilePath
+from dvt.contracts.graph.nodes import HookNode
+from dvt.node_types import NodeType, RunHookType
+from dvt.parser.base import SimpleParser
+from dvt.parser.search import FileBlock
+from dvt.utils import get_pseudo_hook_path
+
+from dbt_common.exceptions import DbtInternalError
+
+
+@dataclass
+class HookBlock(FileBlock):
+    project: str
+    value: str
+    index: int
+    hook_type: RunHookType
+
+    @property
+    def contents(self):
+        return self.value
+
+    @property
+    def name(self):
+        return "{}-{!s}-{!s}".format(self.project, self.hook_type, self.index)
+
+
+class HookSearcher(Iterable[HookBlock]):
+    def __init__(self, project, source_file, hook_type) -> None:
+        self.project = project
+        self.source_file = source_file
+        self.hook_type = hook_type
+
+    def _hook_list(self, hooks: Union[str, List[str], Tuple[str, ...]]) -> List[str]:
+        if isinstance(hooks, tuple):
+            hooks = list(hooks)
+        elif not isinstance(hooks, list):
+            hooks = [hooks]
+        return hooks
+
+    def get_hook_defs(self) -> List[str]:
+        if self.hook_type == RunHookType.Start:
+            hooks = self.project.on_run_start
+        elif self.hook_type == RunHookType.End:
+            hooks = self.project.on_run_end
+        else:
+            raise DbtInternalError(
+                'hook_type must be one of "{}" or "{}" (got {})'.format(
+                    RunHookType.Start, RunHookType.End, self.hook_type
+                )
+            )
+        return self._hook_list(hooks)
+
+    def __iter__(self) -> Iterator[HookBlock]:
+        hooks = self.get_hook_defs()
+        for index, hook in enumerate(hooks):
+            yield HookBlock(
+                file=self.source_file,
+                project=self.project.project_name,
+                value=hook,
+                index=index,
+                hook_type=self.hook_type,
+            )
+
+
+class HookParser(SimpleParser[HookBlock, HookNode]):
+
+    # Hooks are only in the dbt_project.yml file for the project
+    def get_path(self) -> FilePath:
+        # There ought to be an existing file object for this, but
+        # until that is implemented use a dummy modification time
+        path = FilePath(
+            project_root=self.project.project_root,
+            searched_path=".",
+            relative_path="dbt_project.yml",
+            modification_time=0.0,
+        )
+        return path
+
+    def parse_from_dict(self, dct, validate=True) -> HookNode:
+        if validate:
+            HookNode.validate(dct)
+        return HookNode.from_dict(dct)
+
+    @classmethod
+    def get_compiled_path(cls, block: HookBlock):
+        return get_pseudo_hook_path(block.name)
+
+    def _create_parsetime_node(
+        self,
+        block: HookBlock,
+        path: str,
+        config: ContextConfig,
+        fqn: List[str],
+        name=None,
+        **kwargs,
+    ) -> HookNode:
+
+        return super()._create_parsetime_node(
+            block=block,
+            path=path,
+            config=config,
+            fqn=fqn,
+            index=block.index,
+            name=name,
+            tags=[str(block.hook_type)],
+        )
+
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Operation
+
+    def parse_file(self, block: FileBlock) -> None:
+        for hook_type in RunHookType:
+            for hook in HookSearcher(self.project, block.file, hook_type):
+                self.parse_node(hook)

dvt/parser/macros.py

@@ -0,0 +1,137 @@
+from typing import Iterable, List
+
+import jinja2
+from dvt.artifacts.resources import MacroArgument
+from dvt.clients.jinja import get_supported_languages
+from dvt.contracts.files import FilePath, SourceFile
+from dvt.contracts.graph.nodes import Macro
+from dvt.contracts.graph.unparsed import UnparsedMacro
+from dvt.exceptions import ParsingError
+from dvt.flags import get_flags
+from dvt.node_types import NodeType
+from dvt.parser.base import BaseParser
+from dvt.parser.search import FileBlock, filesystem_search
+
+from dbt_common.clients import jinja
+from dbt_common.clients._jinja_blocks import ExtractWarning
+from dbt_common.utils import MACRO_PREFIX
+
+
+class MacroParser(BaseParser[Macro]):
+    # This is only used when creating a MacroManifest separate
+    # from the normal parsing flow.
+    def get_paths(self) -> List[FilePath]:
+        return filesystem_search(
+            project=self.project, relative_dirs=self.project.macro_paths, extension=".sql"
+        )
+
+    @property
+    def resource_type(self) -> NodeType:
+        return NodeType.Macro
+
+    @classmethod
+    def get_compiled_path(cls, block: FileBlock):
+        return block.path.relative_path
+
+    def parse_macro(self, block: jinja.BlockTag, base_node: UnparsedMacro, name: str) -> Macro:
+        unique_id = self.generate_unique_id(name)
+        macro_sql = block.full_block or ""
+
+        return Macro(
+            path=base_node.path,
+            macro_sql=macro_sql,
+            original_file_path=base_node.original_file_path,
+            package_name=base_node.package_name,
+            resource_type=base_node.resource_type,
+            name=name,
+            unique_id=unique_id,
+        )
+
+    def parse_unparsed_macros(self, base_node: UnparsedMacro) -> Iterable[Macro]:
+        # This is a bit of a hack to get the file path to the deprecation
+        def wrap_handle_extract_warning(warning: ExtractWarning) -> None:
+            self._handle_extract_warning(warning=warning, file=base_node.original_file_path)
+
+        try:
+            blocks: List[jinja.BlockTag] = [
+                t
+                for t in jinja.extract_toplevel_blocks(
+                    base_node.raw_code,
+                    allowed_blocks={"macro", "materialization", "test", "data_test"},
+                    collect_raw_data=False,
+                    warning_callback=wrap_handle_extract_warning,
+                )
+                if isinstance(t, jinja.BlockTag)
+            ]
+        except ParsingError as exc:
+            exc.add_node(base_node)
+            raise
+
+        for block in blocks:
+            try:
+                ast = jinja.parse(block.full_block)
+            except ParsingError as e:
+                e.add_node(base_node)
+                raise
+
+            if (
+                isinstance(ast, jinja2.nodes.Template)
+                and hasattr(ast, "body")
+                and len(ast.body) == 1
+                and isinstance(ast.body[0], jinja2.nodes.Macro)
+            ):
+                # If the top level node in the Template is a Macro, things look
+                # good and this is much faster than traversing the full ast, as
+                # in the following else clause. It's not clear if that traversal
+                # is ever really needed.
+                macro = ast.body[0]
+            else:
+                macro_nodes = list(ast.find_all(jinja2.nodes.Macro))
+
+                if len(macro_nodes) != 1:
+                    # things have gone disastrously wrong, we thought we only
+                    # parsed one block!
+                    raise ParsingError(
+                        f"Found multiple macros in {block.full_block}, expected 1", node=base_node
+                    )
+
+                macro = macro_nodes[0]
+
+            if not macro.name.startswith(MACRO_PREFIX):
+                continue
+
+            name: str = macro.name.replace(MACRO_PREFIX, "")
+            node = self.parse_macro(block, base_node, name)
+
+            if getattr(get_flags(), "validate_macro_args", False):
+                node.arguments = self._extract_args(macro)
+
+            # get supported_languages for materialization macro
+            if block.block_type_name == "materialization":
+                node.supported_languages = get_supported_languages(macro)
+            yield node
+
+    def _extract_args(self, macro) -> List[MacroArgument]:
+        try:
+            return list([MacroArgument(name=arg.name) for arg in macro.args])
+        except Exception:
+            return []
+
+    def parse_file(self, block: FileBlock):
+        assert isinstance(block.file, SourceFile)
+        source_file = block.file
+        assert isinstance(source_file.contents, str)
+        original_file_path = source_file.path.original_file_path
+
+        # this is really only used for error messages
+        base_node = UnparsedMacro(
+            path=original_file_path,
+            original_file_path=original_file_path,
+            package_name=self.project.project_name,
+            raw_code=source_file.contents,
+            resource_type=NodeType.Macro,
+            language="sql",
+        )
+
+        for node in self.parse_unparsed_macros(base_node):
+            self.manifest.add_macro(block.file, node)