dvt-core 1.11.0b4__py3-none-any.whl

dvt/clients/git.py

@@ -0,0 +1,164 @@
+import os.path
+import re
+
+from dvt.events.types import (
+    GitNothingToDo,
+    GitProgressCheckedOutAt,
+    GitProgressCheckoutRevision,
+    GitProgressPullingNewDependency,
+    GitProgressUpdatedCheckoutRange,
+    GitProgressUpdatingExistingDependency,
+    GitSparseCheckoutSubdirectory,
+)
+from dvt.exceptions import (
+    CommandResultError,
+    DbtRuntimeError,
+    GitCheckoutError,
+    GitCloningError,
+    UnknownGitCloningProblemError,
+)
+from packaging import version
+
+from dbt_common.clients.system import rmdir, run_cmd
+from dbt_common.events.functions import fire_event
+
+
+def _is_commit(revision: str) -> bool:
+    # match SHA-1 git commit
+    return bool(re.match(r"\b[0-9a-f]{40}\b", revision))
+
+
+def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None):
+    has_revision = revision is not None
+    is_commit = _is_commit(revision or "")
+
+    clone_cmd = ["git", "clone", "--depth", "1"]
+    if subdirectory:
+        fire_event(GitSparseCheckoutSubdirectory(subdir=subdirectory))
+        out, _ = run_cmd(cwd, ["git", "--version"], env={"LC_ALL": "C"})
+        git_version = version.parse(re.search(r"\d+\.\d+\.\d+", out.decode("utf-8")).group(0))
+        if not git_version >= version.parse("2.25.0"):
+            # 2.25.0 introduces --sparse
+            raise RuntimeError(
+                "Please update your git version to pull a dbt package "
+                "from a subdirectory: your version is {}, >= 2.25.0 needed".format(git_version)
+            )
+        clone_cmd.extend(["--filter=blob:none", "--sparse"])
+
+    if has_revision and not is_commit:
+        clone_cmd.extend(["--branch", revision])
+
+    clone_cmd.append(repo)
+
+    if dirname is not None:
+        clone_cmd.append(dirname)
+    try:
+        result = run_cmd(cwd, clone_cmd, env={"LC_ALL": "C"})
+    except CommandResultError as exc:
+        raise GitCloningError(repo, revision, exc)
+
+    if subdirectory:
+        cwd_subdir = os.path.join(cwd, dirname or "")
+        clone_cmd_subdir = ["git", "sparse-checkout", "set", subdirectory]
+        try:
+            run_cmd(cwd_subdir, clone_cmd_subdir)
+        except CommandResultError as exc:
+            raise GitCloningError(repo, revision, exc)
+
+    if remove_git_dir:
+        rmdir(os.path.join(dirname, ".git"))
+
+    return result
+
+
+def list_tags(cwd):
+    out, err = run_cmd(cwd, ["git", "tag", "--list"], env={"LC_ALL": "C"})
+    tags = out.decode("utf-8").strip().split("\n")
+    return tags
+
+
+def _checkout(cwd, repo, revision):
+    fire_event(GitProgressCheckoutRevision(revision=revision))
+
+    fetch_cmd = ["git", "fetch", "origin", "--depth", "1"]
+
+    if _is_commit(revision):
+        run_cmd(cwd, fetch_cmd + [revision])
+    else:
+        run_cmd(cwd, ["git", "remote", "set-branches", "origin", revision])
+        run_cmd(cwd, fetch_cmd + ["--tags", revision])
+
+    if _is_commit(revision):
+        spec = revision
+    # Prefer tags to branches if one exists
+    elif revision in list_tags(cwd):
+        spec = "tags/{}".format(revision)
+    else:
+        spec = "origin/{}".format(revision)
+
+    out, err = run_cmd(cwd, ["git", "reset", "--hard", spec], env={"LC_ALL": "C"})
+    return out, err
+
+
+def checkout(cwd, repo, revision=None):
+    if revision is None:
+        revision = "HEAD"
+    try:
+        return _checkout(cwd, repo, revision)
+    except CommandResultError as exc:
+        raise GitCheckoutError(repo=repo, revision=revision, error=exc)
+
+
+def get_current_sha(cwd):
+    out, err = run_cmd(cwd, ["git", "rev-parse", "HEAD"], env={"LC_ALL": "C"})
+
+    return out.decode("utf-8").strip()
+
+
+def remove_remote(cwd):
+    return run_cmd(cwd, ["git", "remote", "rm", "origin"], env={"LC_ALL": "C"})
+
+
+def clone_and_checkout(
+    repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None
+):
+    exists = None
+    try:
+        _, err = clone(
+            repo,
+            cwd,
+            dirname=dirname,
+            remove_git_dir=remove_git_dir,
+            subdirectory=subdirectory,
+        )
+    except CommandResultError as exc:
+        err = exc.stderr
+        exists = re.match("fatal: destination path '(.+)' already exists", err)
+        if not exists:
+            raise UnknownGitCloningProblemError(repo)
+
+    directory = None
+    start_sha = None
+    if exists:
+        directory = exists.group(1)
+        fire_event(GitProgressUpdatingExistingDependency(dir=directory))
+    else:
+        matches = re.match("Cloning into '(.+)'", err.decode("utf-8"))
+        if matches is None:
+            raise DbtRuntimeError(f'Error cloning {repo} - never saw "Cloning into ..." from git')
+        directory = matches.group(1)
+        fire_event(GitProgressPullingNewDependency(dir=directory))
+    full_path = os.path.join(cwd, directory)
+    start_sha = get_current_sha(full_path)
+    checkout(full_path, repo, revision)
+    end_sha = get_current_sha(full_path)
+    if exists:
+        if start_sha == end_sha:
+            fire_event(GitNothingToDo(sha=start_sha[:7]))
+        else:
+            fire_event(
+                GitProgressUpdatedCheckoutRange(start_sha=start_sha[:7], end_sha=end_sha[:7])
+            )
+    else:
+        fire_event(GitProgressCheckedOutAt(end_sha=end_sha[:7]))
+    return os.path.join(directory, subdirectory or "")

dvt/clients/jinja.py

@@ -0,0 +1,206 @@
+import re
+import threading
+from contextlib import contextmanager
+from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union
+
+import jinja2
+import jinja2.ext
+import jinja2.nativetypes  # type: ignore
+import jinja2.nodes
+import jinja2.parser
+import jinja2.sandbox
+from dvt.contracts.graph.nodes import GenericTestNode
+from dvt.exceptions import (
+    DbtInternalError,
+    MaterializtionMacroNotUsedError,
+    NoSupportedLanguagesFoundError,
+)
+from dvt.node_types import ModelLanguage
+
+from dbt_common.clients.jinja import (
+    CallableMacroGenerator,
+    MacroProtocol,
+    get_template,
+    render_template,
+)
+from dbt_common.utils import deep_map_render
+
+SUPPORTED_LANG_ARG = jinja2.nodes.Name("supported_languages", "param")
+
+
+class MacroStack(threading.local):
+    def __init__(self):
+        super().__init__()
+        self.call_stack = []
+
+    @property
+    def depth(self) -> int:
+        return len(self.call_stack)
+
+    def push(self, name):
+        self.call_stack.append(name)
+
+    def pop(self, name):
+        got = self.call_stack.pop()
+        if got != name:
+            raise DbtInternalError(f"popped {got}, expected {name}")
+
+
+class MacroGenerator(CallableMacroGenerator):
+    def __init__(
+        self,
+        macro: MacroProtocol,
+        context: Optional[Dict[str, Any]] = None,
+        node: Optional[Any] = None,
+        stack: Optional[MacroStack] = None,
+    ) -> None:
+        super().__init__(macro, context)
+        self.node = node
+        self.stack = stack
+
+    # This adds the macro's unique id to the node's 'depends_on'
+    @contextmanager
+    def track_call(self):
+        # This is only called from __call__
+        if self.stack is None:
+            yield
+        else:
+            unique_id = self.macro.unique_id
+            depth = self.stack.depth
+            # only mark depth=0 as a dependency, when creating this dependency we don't pass in stack
+            if depth == 0 and self.node:
+                self.node.depends_on.add_macro(unique_id)
+            self.stack.push(unique_id)
+            try:
+                yield
+            finally:
+                self.stack.pop(unique_id)
+
+    # this makes MacroGenerator objects callable like functions
+    def __call__(self, *args, **kwargs):
+        with self.track_call():
+            return self.call_macro(*args, **kwargs)
+
+
+class UnitTestMacroGenerator(MacroGenerator):
+    # this makes UnitTestMacroGenerator objects callable like functions
+    def __init__(
+        self,
+        macro_generator: MacroGenerator,
+        call_return_value: Any,
+    ) -> None:
+        super().__init__(
+            macro_generator.macro,
+            macro_generator.context,
+            macro_generator.node,
+            macro_generator.stack,
+        )
+        self.call_return_value = call_return_value
+
+    def __call__(self, *args, **kwargs):
+        with self.track_call():
+            return self.call_return_value
+
+
+# performance note: Local benmcharking (so take it with a big grain of salt!)
+# on this indicates that it is is on average slightly slower than
+# checking two separate patterns, but the standard deviation is smaller with
+# one pattern. The time difference between the two was ~2 std deviations, which
+# is small enough that I've just chosen the more readable option.
+_HAS_RENDER_CHARS_PAT = re.compile(r"({[{%#]|[#}%]})")
+
+_render_cache: Dict[str, Any] = dict()
+
+
+def get_rendered(
+    string: str,
+    ctx: Dict[str, Any],
+    node=None,
+    capture_macros: bool = False,
+    native: bool = False,
+) -> Any:
+    # performance optimization: if there are no jinja control characters in the
+    # string, we can just return the input. Fall back to jinja if the type is
+    # not a string or if native rendering is enabled (so '1' -> 1, etc...)
+    # If this is desirable in the native env as well, we could handle the
+    # native=True case by passing the input string to ast.literal_eval, like
+    # the native renderer does.
+    has_render_chars = not isinstance(string, str) or _HAS_RENDER_CHARS_PAT.search(string)
+
+    if not has_render_chars:
+        if not native:
+            return string
+        elif string in _render_cache:
+            return _render_cache[string]
+
+    template = get_template(
+        string,
+        ctx,
+        node,
+        capture_macros=capture_macros,
+        native=native,
+    )
+
+    rendered = render_template(template, ctx, node)
+
+    if not has_render_chars and native:
+        _render_cache[string] = rendered
+
+    return rendered
+
+
+def undefined_error(msg) -> NoReturn:
+    raise jinja2.exceptions.UndefinedError(msg)
+
+
+GENERIC_TEST_KWARGS_NAME = "_dbt_generic_test_kwargs"
+
+
+def add_rendered_test_kwargs(
+    context: Dict[str, Any],
+    node: GenericTestNode,
+    capture_macros: bool = False,
+) -> None:
+    """Render each of the test kwargs in the given context using the native
+    renderer, then insert that value into the given context as the special test
+    keyword arguments member.
+    """
+    looks_like_func = r"^\s*(env_var|ref|var|source|doc)\s*\(.+\)\s*$"
+
+    def _convert_function(value: Any, keypath: Tuple[Union[str, int], ...]) -> Any:
+        if isinstance(value, str):
+            if keypath == ("column_name",):
+                # special case: Don't render column names as native, make them
+                # be strings
+                return value
+
+            if re.match(looks_like_func, value) is not None:
+                # curly braces to make rendering happy
+                value = f"{{{{ {value} }}}}"
+
+            value = get_rendered(value, context, node, capture_macros=capture_macros, native=True)
+
+        return value
+
+    # The test_metadata.kwargs come from the test builder, and were set
+    # when the test node was created in _parse_generic_test.
+    kwargs = deep_map_render(_convert_function, node.test_metadata.kwargs)
+    context[GENERIC_TEST_KWARGS_NAME] = kwargs
+
+
+def get_supported_languages(node: jinja2.nodes.Macro) -> List[ModelLanguage]:
+    if "materialization" not in node.name:
+        raise MaterializtionMacroNotUsedError(node=node)
+
+    no_kwargs = not node.defaults
+    no_langs_found = SUPPORTED_LANG_ARG not in node.args
+
+    if no_kwargs or no_langs_found:
+        raise NoSupportedLanguagesFoundError(node=node)
+
+    lang_idx = node.args.index(SUPPORTED_LANG_ARG)
+    # indexing defaults from the end
+    # since supported_languages is a kwarg, and kwargs are at always after args
+    return [
+        ModelLanguage[item.value] for item in node.defaults[-(len(node.args) - lang_idx)].items
+    ]

dvt/clients/jinja_static.py

@@ -0,0 +1,245 @@
+import typing
+from typing import Any, Dict, List, Optional, Union
+
+import jinja2
+from dvt.artifacts.resources import RefArgs
+from dvt.exceptions import MacroNamespaceNotStringError, ParsingError
+
+from dbt_common.clients.jinja import get_environment
+from dbt_common.exceptions.macros import MacroNameNotStringError
+from dbt_common.tests import test_caching_enabled
+from dbt_extractor import ExtractionError, py_extract_from_source  # type: ignore
+
+if typing.TYPE_CHECKING:
+    from dvt.context.providers import ParseDatabaseWrapper
+
+
+_TESTING_MACRO_CACHE: Dict[str, Any] = {}
+
+
+def statically_extract_has_name_this(source: str) -> bool:
+    """Checks whether the raw jinja has any references to `this`"""
+    env = get_environment(None, capture_macros=True)
+    parsed = env.parse(source)
+    names = tuple(parsed.find_all(jinja2.nodes.Name))
+
+    for name in names:
+        if hasattr(name, "name") and name.name == "this":
+            return True
+    return False
+
+
+def statically_extract_macro_calls(
+    source: str, ctx: Dict[str, Any], db_wrapper: Optional["ParseDatabaseWrapper"] = None
+) -> List[str]:
+    # set 'capture_macros' to capture undefined
+    env = get_environment(None, capture_macros=True)
+
+    global _TESTING_MACRO_CACHE
+    if test_caching_enabled() and source in _TESTING_MACRO_CACHE:
+        parsed = _TESTING_MACRO_CACHE.get(source, None)
+        func_calls = getattr(parsed, "_dbt_cached_calls")
+    else:
+        parsed = env.parse(source)
+        func_calls = tuple(parsed.find_all(jinja2.nodes.Call))
+
+        if test_caching_enabled():
+            _TESTING_MACRO_CACHE[source] = parsed
+            setattr(parsed, "_dbt_cached_calls", func_calls)
+
+    standard_calls = ["source", "ref", "config"]
+    possible_macro_calls = []
+    for func_call in func_calls:
+        func_name = None
+        if hasattr(func_call, "node") and hasattr(func_call.node, "name"):
+            func_name = func_call.node.name
+        else:
+            if (
+                hasattr(func_call, "node")
+                and hasattr(func_call.node, "node")
+                and type(func_call.node.node).__name__ == "Name"
+                and hasattr(func_call.node, "attr")
+            ):
+                package_name = func_call.node.node.name
+                macro_name = func_call.node.attr
+                if package_name == "adapter":
+                    if macro_name == "dispatch":
+                        ad_macro_calls = statically_parse_adapter_dispatch(
+                            func_call, ctx, db_wrapper
+                        )
+                        possible_macro_calls.extend(ad_macro_calls)
+                    else:
+                        # This skips calls such as adapter.parse_index
+                        continue
+                else:
+                    func_name = f"{package_name}.{macro_name}"
+            else:
+                continue
+        if not func_name:
+            continue
+        if func_name in standard_calls:
+            continue
+        elif ctx.get(func_name):
+            continue
+        else:
+            if func_name not in possible_macro_calls:
+                possible_macro_calls.append(func_name)
+
+    return possible_macro_calls
+
+
+def statically_parse_adapter_dispatch(
+    func_call, ctx: Dict[str, Any], db_wrapper: Optional["ParseDatabaseWrapper"]
+) -> List[str]:
+    possible_macro_calls = []
+    # This captures an adapter.dispatch('<macro_name>') call.
+
+    func_name = None
+    # macro_name positional argument
+    if len(func_call.args) > 0:
+        func_name = func_call.args[0].value
+    if func_name:
+        possible_macro_calls.append(func_name)
+
+    # packages positional argument
+    macro_namespace = None
+    packages_arg = None
+    packages_arg_type = None
+
+    if len(func_call.args) > 1:
+        packages_arg = func_call.args[1]
+        # This can be a List or a Call
+        packages_arg_type = type(func_call.args[1]).__name__
+
+    # keyword arguments
+    if func_call.kwargs:
+        for kwarg in func_call.kwargs:
+            if kwarg.key == "macro_name":
+                # This will remain to enable static resolution
+                if type(kwarg.value).__name__ == "Const":
+                    func_name = kwarg.value.value
+                    possible_macro_calls.append(func_name)
+                else:
+                    raise MacroNameNotStringError(kwarg_value=kwarg.value.value)
+            elif kwarg.key == "macro_namespace":
+                # This will remain to enable static resolution
+                kwarg_type = type(kwarg.value).__name__
+                if kwarg_type == "Const":
+                    macro_namespace = kwarg.value.value
+                else:
+                    raise MacroNamespaceNotStringError(kwarg_type)
+
+    # positional arguments
+    if packages_arg:
+        if packages_arg_type == "List":
+            # This will remain to enable static resolution
+            packages = []
+            for item in packages_arg.items:
+                packages.append(item.value)
+        elif packages_arg_type == "Const":
+            # This will remain to enable static resolution
+            macro_namespace = packages_arg.value
+
+    if db_wrapper:
+        macro = db_wrapper.dispatch(func_name, macro_namespace=macro_namespace).macro
+        func_name = f"{macro.package_name}.{macro.name}"  # type: ignore[attr-defined]
+        possible_macro_calls.append(func_name)
+    else:  # this is only for tests/unit/test_macro_calls.py
+        if macro_namespace:
+            packages = [macro_namespace]
+        else:
+            packages = []
+        for package_name in packages:
+            possible_macro_calls.append(f"{package_name}.{func_name}")
+
+    return possible_macro_calls
+
+
+def statically_parse_ref_or_source(expression: str) -> Union[RefArgs, List[str]]:
+    """
+    Returns a RefArgs or List[str] object, corresponding to ref or source respectively, given an input jinja expression.
+
+    input: str representing how input node is referenced in tested model sql
+        * examples:
+        - "ref('my_model_a')"
+        - "ref('my_model_a', version=3)"
+        - "ref('package', 'my_model_a', version=3)"
+        - "source('my_source_schema', 'my_source_name')"
+
+    If input is not a well-formed jinja ref or source expression, a ParsingError is raised.
+    """
+    ref_or_source: Union[RefArgs, List[str]]
+
+    try:
+        statically_parsed = py_extract_from_source(f"{{{{ {expression} }}}}")
+    except ExtractionError:
+        raise ParsingError(f"Invalid jinja expression: {expression}")
+
+    if statically_parsed.get("refs"):
+        raw_ref = list(statically_parsed["refs"])[0]
+        ref_or_source = RefArgs(
+            package=raw_ref.get("package"),
+            name=raw_ref.get("name"),
+            version=raw_ref.get("version"),
+        )
+    elif statically_parsed.get("sources"):
+        source_name, source_table_name = list(statically_parsed["sources"])[0]
+        ref_or_source = [source_name, source_table_name]
+    else:
+        raise ParsingError(f"Invalid ref or source expression: {expression}")
+
+    return ref_or_source
+
+
+def statically_parse_unrendered_config(string: str) -> Optional[Dict[str, Any]]:
+    """
+    Given a string with jinja, extract an unrendered config call.
+    If no config call is present, returns None.
+
+    For example, given:
+    "{{ config(materialized=env_var('DBT_TEST_STATE_MODIFIED')) }}\nselect 1 as id"
+    returns: {'materialized': "Keyword(key='materialized', value=Call(node=Name(name='env_var', ctx='load'), args=[Const(value='DBT_TEST_STATE_MODIFIED')], kwargs=[], dyn_args=None, dyn_kwargs=None))"}
+
+    No config call:
+    "select 1 as id"
+    returns: None
+    """
+    # Return early to avoid creating jinja environemt if no config call in input string
+    if "config(" not in string:
+        return None
+
+    # set 'capture_macros' to capture undefined
+    env = get_environment(None, capture_macros=True)
+
+    global _TESTING_MACRO_CACHE
+    if test_caching_enabled() and _TESTING_MACRO_CACHE and string in _TESTING_MACRO_CACHE:
+        parsed = _TESTING_MACRO_CACHE.get(string, None)
+        func_calls = getattr(parsed, "_dbt_cached_calls")
+    else:
+        parsed = env.parse(string)
+        func_calls = tuple(parsed.find_all(jinja2.nodes.Call))
+
+    config_func_calls = list(
+        filter(
+            lambda f: hasattr(f, "node") and hasattr(f.node, "name") and f.node.name == "config",
+            func_calls,
+        )
+    )
+    # There should only be one {{ config(...) }} call per input
+    config_func_call = config_func_calls[0] if config_func_calls else None
+
+    if not config_func_call:
+        return None
+
+    unrendered_config = {}
+    for kwarg in config_func_call.kwargs:
+        unrendered_config[kwarg.key] = construct_static_kwarg_value(kwarg)
+
+    return unrendered_config
+
+
+def construct_static_kwarg_value(kwarg) -> str:
+    # Instead of trying to re-assemble complex kwarg value, simply stringify the value.
+    # This is still useful to be able to detect changes in unrendered configs, even if it is
+    # not an exact representation of the user input.
+    return str(kwarg)