dvt-core 1.11.0b4__py3-none-any.whl

dvt/config/runtime.py

@@ -0,0 +1,491 @@
+import itertools
+import os
+from copy import deepcopy
+from dataclasses import dataclass
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    Mapping,
+    MutableSet,
+    Optional,
+    Tuple,
+    Type,
+)
+
+from dvt import tracking
+from dvt.artifacts.resources import Quoting
+from dvt.config.project import load_raw_project
+from dvt.contracts.graph.manifest import ManifestMetadata
+from dvt.contracts.project import Configuration
+from dvt.events.types import UnusedResourceConfigPath
+from dvt.exceptions import (
+    ConfigContractBrokenError,
+    DbtProjectError,
+    DbtRuntimeError,
+    NonUniquePackageNameError,
+    UninstalledPackagesFoundError,
+)
+from dvt.flags import get_flags
+
+from dbt.adapters.contracts.connection import (
+    AdapterRequiredConfig,
+    Credentials,
+    HasCredentials,
+)
+from dbt.adapters.contracts.relation import ComponentName
+from dbt.adapters.factory import get_include_paths, get_relation_class_by_name
+from dbt_common.dataclass_schema import ValidationError
+from dbt_common.events.functions import warn_or_error
+from dbt_common.helper_types import DictDefaultEmptyStr, FQNPath, PathSet
+
+from .profile import Profile
+from .project import Project
+from .renderer import DbtProjectYamlRenderer, ProfileRenderer
+
+
+# Called by RuntimeConfig.collect_parts class method
+def load_project(
+    project_root: str,
+    version_check: bool,
+    profile: HasCredentials,
+    cli_vars: Optional[Dict[str, Any]] = None,
+    validate: bool = False,
+) -> Project:
+    # get the project with all of the provided information
+    project_renderer = DbtProjectYamlRenderer(profile, cli_vars)
+    project = Project.from_project_root(
+        project_root, project_renderer, verify_version=version_check, validate=validate
+    )
+
+    # Save env_vars encountered in rendering for partial parsing
+    project.project_env_vars = project_renderer.ctx_obj.env_vars
+    return project
+
+
+def load_profile(
+    project_root: str,
+    cli_vars: Dict[str, Any],
+    profile_name_override: Optional[str] = None,
+    target_override: Optional[str] = None,
+    threads_override: Optional[int] = None,
+) -> Profile:
+    raw_project = load_raw_project(project_root)
+    raw_profile_name = raw_project.get("profile")
+    profile_renderer = ProfileRenderer(cli_vars)
+    profile_name = profile_renderer.render_value(raw_profile_name)
+    profile = Profile.render(
+        profile_renderer, profile_name, profile_name_override, target_override, threads_override
+    )
+    # Save env_vars encountered in rendering for partial parsing
+    profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
+    return profile
+
+
+def _project_quoting_dict(proj: Project, profile: Profile) -> Dict[ComponentName, bool]:
+    src: Dict[str, Any] = profile.credentials.translate_aliases(proj.quoting)
+    result: Dict[ComponentName, bool] = {}
+    for key in ComponentName:
+        if key in src:
+            value = src[key]
+            if isinstance(value, bool):
+                result[key] = value
+    return result
+
+
+@dataclass
+class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
+    args: Any
+    profile_name: str
+    cli_vars: Dict[str, Any]
+    dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None
+
+    def __post_init__(self):
+        self.validate()
+
+    @classmethod
+    def get_profile(
+        cls,
+        project_root: str,
+        cli_vars: Dict[str, Any],
+        args: Any,
+    ) -> Profile:
+        return load_profile(
+            project_root,
+            cli_vars,
+            args.profile,
+            args.target,
+            args.threads,
+        )
+
+    # Called by 'new_project' and 'from_args'
+    @classmethod
+    def from_parts(
+        cls,
+        project: Project,
+        profile: Profile,
+        args: Any,
+        dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None,
+    ) -> "RuntimeConfig":
+        """Instantiate a RuntimeConfig from its components.
+
+        :param profile: A parsed dbt Profile.
+        :param project: A parsed dbt Project.
+        :param args: The parsed command-line arguments.
+        :returns RuntimeConfig: The new configuration.
+        """
+        quoting: Dict[str, Any] = (
+            get_relation_class_by_name(profile.credentials.type)
+            .get_default_quote_policy()
+            .replace_dict(_project_quoting_dict(project, profile))
+        ).to_dict(omit_none=True)
+
+        cli_vars: Dict[str, Any] = getattr(args, "vars", {})
+        log_cache_events: bool = getattr(args, "log_cache_events", profile.log_cache_events)
+
+        return cls(
+            project_name=project.project_name,
+            version=project.version,
+            project_root=project.project_root,
+            model_paths=project.model_paths,
+            macro_paths=project.macro_paths,
+            seed_paths=project.seed_paths,
+            test_paths=project.test_paths,
+            analysis_paths=project.analysis_paths,
+            docs_paths=project.docs_paths,
+            asset_paths=project.asset_paths,
+            function_paths=project.function_paths,
+            target_path=project.target_path,
+            snapshot_paths=project.snapshot_paths,
+            clean_targets=project.clean_targets,
+            log_path=project.log_path,
+            packages_install_path=project.packages_install_path,
+            packages_specified_path=project.packages_specified_path,
+            quoting=quoting,
+            models=project.models,
+            on_run_start=project.on_run_start,
+            on_run_end=project.on_run_end,
+            dispatch=project.dispatch,
+            seeds=project.seeds,
+            snapshots=project.snapshots,
+            dbt_version=project.dbt_version,
+            packages=project.packages,
+            manifest_selectors=project.manifest_selectors,
+            selectors=project.selectors,
+            query_comment=project.query_comment,
+            sources=project.sources,
+            data_tests=project.data_tests,
+            unit_tests=project.unit_tests,
+            metrics=project.metrics,
+            semantic_models=project.semantic_models,
+            saved_queries=project.saved_queries,
+            exposures=project.exposures,
+            functions=project.functions,
+            vars=project.vars,
+            config_version=project.config_version,
+            unrendered=project.unrendered,
+            project_env_vars=project.project_env_vars,
+            restrict_access=project.restrict_access,
+            profile_env_vars=profile.profile_env_vars,
+            profile_name=profile.profile_name,
+            target_name=profile.target_name,
+            threads=profile.threads,
+            credentials=profile.credentials,
+            args=args,
+            cli_vars=cli_vars,
+            log_cache_events=log_cache_events,
+            dependencies=dependencies,
+            dbt_cloud=project.dbt_cloud,
+            flags=project.flags,
+        )
+
+    # Called by 'load_projects' in this class
+    def new_project(self, project_root: str) -> "RuntimeConfig":
+        """Given a new project root, read in its project dictionary, supply the
+        existing project's profile info, and create a new project file.
+
+        :param project_root: A filepath to a dbt project.
+        :raises DbtProfileError: If the profile is invalid.
+        :raises DbtProjectError: If project is missing or invalid.
+        :returns: The new configuration.
+        """
+        # copy profile
+        profile = Profile(**self.to_profile_info())
+        profile.validate()
+
+        # load the new project and its packages. Don't pass cli variables.
+        renderer = DbtProjectYamlRenderer(profile)
+        project = Project.from_project_root(
+            project_root,
+            renderer,
+            verify_version=bool(getattr(self.args, "VERSION_CHECK", True)),
+        )
+
+        runtime_config = self.from_parts(
+            project=project,
+            profile=profile,
+            args=deepcopy(self.args),
+        )
+        # force our quoting back onto the new project.
+        runtime_config.quoting = deepcopy(self.quoting)
+        return runtime_config
+
+    def serialize(self) -> Dict[str, Any]:
+        """Serialize the full configuration to a single dictionary. For any
+        instance that has passed validate() (which happens in __init__), it
+        matches the Configuration contract.
+
+        Note that args are not serialized.
+
+        :returns dict: The serialized configuration.
+        """
+        result = self.to_project_config(with_packages=True)
+        result.update(self.to_profile_info(serialize_credentials=True))
+        result["cli_vars"] = deepcopy(self.cli_vars)
+        return result
+
+    def validate(self):
+        """Validate the configuration against its contract.
+
+        :raises DbtProjectError: If the configuration fails validation.
+        """
+        try:
+            Configuration.validate(self.serialize())
+        except ValidationError as e:
+            raise ConfigContractBrokenError(e) from e
+
+    # Called by RuntimeConfig.from_args
+    @classmethod
+    def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]:
+        # profile_name from the project
+        project_root = args.project_dir if args.project_dir else os.getcwd()
+        cli_vars: Dict[str, Any] = getattr(args, "vars", {})
+        profile = cls.get_profile(
+            project_root,
+            cli_vars,
+            args,
+        )
+        flags = get_flags()
+        project = load_project(project_root, bool(flags.VERSION_CHECK), profile, cli_vars)
+        return project, profile
+
+    # Called in task/base.py, in BaseTask.from_args
+    @classmethod
+    def from_args(cls, args: Any) -> "RuntimeConfig":
+        """Given arguments, read in dbt_project.yml from the current directory,
+        read in packages.yml if it exists, and use them to find the profile to
+        load.
+
+        :param args: The arguments as parsed from the cli.
+        :raises DbtProjectError: If the project is invalid or missing.
+        :raises DbtProfileError: If the profile is invalid or missing.
+        :raises DbtValidationError: If the cli variables are invalid.
+        """
+        project, profile = cls.collect_parts(args)
+
+        return cls.from_parts(
+            project=project,
+            profile=profile,
+            args=args,
+        )
+
+    def get_metadata(self) -> ManifestMetadata:
+        return ManifestMetadata(
+            project_name=self.project_name,
+            project_id=self.hashed_name(),
+            user_id=tracking.active_user.id if tracking.active_user else None,
+            send_anonymous_usage_stats=(
+                get_flags().SEND_ANONYMOUS_USAGE_STATS if tracking.active_user else None
+            ),
+            adapter_type=self.credentials.type,
+            quoting=Quoting(
+                database=self.quoting.get("database", None),
+                schema=self.quoting.get("schema", None),
+                identifier=self.quoting.get("identifier", None),
+                column=self.quoting.get("column", None),
+            ),
+            run_started_at=(
+                tracking.active_user.run_started_at if tracking.active_user is not None else None
+            ),
+        )
+
+    def _get_v2_config_paths(
+        self,
+        config,
+        path: FQNPath,
+        paths: MutableSet[FQNPath],
+    ) -> PathSet:
+        for key, value in config.items():
+            if isinstance(value, dict) and not key.startswith("+"):
+                self._get_config_paths(value, path + (key,), paths)
+            else:
+                paths.add(path)
+        return frozenset(paths)
+
+    def _get_config_paths(
+        self,
+        config: Dict[str, Any],
+        path: FQNPath = (),
+        paths: Optional[MutableSet[FQNPath]] = None,
+    ) -> PathSet:
+        if paths is None:
+            paths = set()
+
+        for key, value in config.items():
+            if isinstance(value, dict) and not key.startswith("+"):
+                self._get_v2_config_paths(value, path + (key,), paths)
+            else:
+                paths.add(path)
+        return frozenset(paths)
+
+    def get_resource_config_paths(self) -> Dict[str, PathSet]:
+        """Return a dictionary with resource type keys whose values are
+        lists of lists of strings, where each inner list of strings represents
+        a configured path in the resource.
+        """
+        return {
+            "models": self._get_config_paths(self.models),
+            "seeds": self._get_config_paths(self.seeds),
+            "snapshots": self._get_config_paths(self.snapshots),
+            "sources": self._get_config_paths(self.sources),
+            "data_tests": self._get_config_paths(self.data_tests),
+            "unit_tests": self._get_config_paths(self.unit_tests),
+            "metrics": self._get_config_paths(self.metrics),
+            "semantic_models": self._get_config_paths(self.semantic_models),
+            "saved_queries": self._get_config_paths(self.saved_queries),
+            "exposures": self._get_config_paths(self.exposures),
+            "functions": self._get_config_paths(self.functions),
+        }
+
+    def warn_for_unused_resource_config_paths(
+        self,
+        resource_fqns: Mapping[str, PathSet],
+        disabled: PathSet,
+    ) -> None:
+        """Return a list of lists of strings, where each inner list of strings
+        represents a type + FQN path of a resource configuration that is not
+        used.
+        """
+        disabled_fqns = frozenset(tuple(fqn) for fqn in disabled)
+        resource_config_paths = self.get_resource_config_paths()
+        unused_resource_config_paths = []
+        for resource_type, config_paths in resource_config_paths.items():
+            used_fqns = resource_fqns.get(resource_type, frozenset())
+            fqns = used_fqns | disabled_fqns
+
+            for config_path in config_paths:
+                if not _is_config_used(config_path, fqns):
+                    resource_path = ".".join(i for i in ((resource_type,) + config_path))
+                    unused_resource_config_paths.append(resource_path)
+
+        if len(unused_resource_config_paths) == 0:
+            return
+
+        warn_or_error(UnusedResourceConfigPath(unused_config_paths=unused_resource_config_paths))
+
+    def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]:
+        if self.dependencies is None:
+            all_projects = {self.project_name: self}
+            internal_packages = get_include_paths(self.credentials.type)
+            if base_only:
+                # Test setup -- we want to load macros without dependencies
+                project_paths = itertools.chain(internal_packages)
+            else:
+                # raise exception if fewer installed packages than in packages.yml
+                count_packages_specified = len(self.packages.packages)  # type: ignore
+                count_packages_installed = len(tuple(self._get_project_directories()))
+                if count_packages_specified > count_packages_installed:
+                    raise UninstalledPackagesFoundError(
+                        count_packages_specified,
+                        count_packages_installed,
+                        self.packages_specified_path,
+                        self.packages_install_path,
+                    )
+                project_paths = itertools.chain(internal_packages, self._get_project_directories())
+            for project_name, project in self.load_projects(project_paths):
+                if project_name in all_projects:
+                    raise NonUniquePackageNameError(project_name)
+                all_projects[project_name] = project
+            self.dependencies = all_projects
+        return self.dependencies
+
+    def clear_dependencies(self):
+        self.dependencies = None
+
+    # Called by 'load_dependencies' in this class
+    def load_projects(self, paths: Iterable[Path]) -> Iterator[Tuple[str, "RuntimeConfig"]]:
+        for path in paths:
+            try:
+                project = self.new_project(str(path))
+            except DbtProjectError as e:
+                raise DbtProjectError(
+                    f"Failed to read package: {e}",
+                    result_type="invalid_project",
+                    path=path,
+                ) from e
+            else:
+                yield project.project_name, project
+
+    def _get_project_directories(self) -> Iterator[Path]:
+        root = Path(self.project_root) / self.packages_install_path
+
+        if root.exists():
+            for path in root.iterdir():
+                if path.is_dir() and not path.name.startswith("__"):
+                    yield path
+
+
+class UnsetCredentials(Credentials):
+    def __init__(self) -> None:
+        super().__init__("", "")
+
+    @property
+    def type(self):
+        return None
+
+    @property
+    def unique_field(self):
+        return None
+
+    def connection_info(self, *args, **kwargs):
+        return {}
+
+    def _connection_keys(self):
+        return ()
+
+
+# This is used by commands which do not require
+# a profile, i.e. dbt deps and clean
+class UnsetProfile(Profile):
+    def __init__(self):
+        self.credentials = UnsetCredentials()
+        self.profile_name = ""
+        self.target_name = ""
+        self.threads = -1
+
+    def to_target_dict(self):
+        return DictDefaultEmptyStr({})
+
+    def __getattribute__(self, name):
+        if name in {"profile_name", "target_name", "threads"}:
+            raise DbtRuntimeError(f'Error: disallowed attribute "{name}" - no profile!')
+
+        return Profile.__getattribute__(self, name)
+
+
+UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE = """\
+Configuration paths exist in your dbt_project.yml file which do not \
+apply to any resources.
+There are {} unused configuration paths:
+{}
+"""
+
+
+def _is_config_used(path, fqns):
+    if fqns:
+        for fqn in fqns:
+            if len(path) <= len(fqn) and fqn[: len(path)] == path:
+                return True
+    return False

dvt/config/selectors.py

@@ -0,0 +1,209 @@
+from copy import deepcopy
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+from dvt.clients.yaml_helper import Dumper, Loader, load_yaml_text, yaml  # noqa: F401
+from dvt.contracts.selection import SelectorFile
+from dvt.exceptions import DbtSelectorsError
+from dvt.graph import SelectionSpec, parse_from_selectors_definition
+from dvt.graph.selector_spec import SelectionCriteria
+
+from dbt_common.clients.system import (
+    load_file_contents,
+    path_exists,
+    resolve_path_from_base,
+)
+from dbt_common.dataclass_schema import ValidationError
+from dbt_common.exceptions import DbtRuntimeError
+
+from .renderer import BaseRenderer
+
+MALFORMED_SELECTOR_ERROR = """\
+The selectors.yml file in this project is malformed. Please double check
+the contents of this file and fix any errors before retrying.
+
+You can find more information on the syntax for this file here:
+https://docs.getdbt.com/reference/node-selection/yaml-selectors
+
+Validator Error:
+{error}
+"""
+
+
+class SelectorConfig(Dict[str, Dict[str, Union[SelectionSpec, bool]]]):
+    @classmethod
+    def selectors_from_dict(cls, data: Dict[str, Any]) -> "SelectorConfig":
+        try:
+            SelectorFile.validate(data)
+            selector_file = SelectorFile.from_dict(data)
+            validate_selector_default(selector_file)
+            selectors = parse_from_selectors_definition(selector_file)
+        except ValidationError as exc:
+            yaml_sel_cfg = yaml.dump(exc.instance)
+            raise DbtSelectorsError(
+                f"Could not parse selector file data: \n{yaml_sel_cfg}\n"
+                f"Valid root-level selector definitions: "
+                f"union, intersection, string, dictionary. No lists. "
+                f"\nhttps://docs.getdbt.com/reference/node-selection/"
+                f"yaml-selectors",
+                result_type="invalid_selector",
+            ) from exc
+        except DbtRuntimeError as exc:
+            raise DbtSelectorsError(
+                f"Could not read selector file data: {exc}",
+                result_type="invalid_selector",
+            ) from exc
+
+        return cls(selectors)
+
+    @classmethod
+    def render_from_dict(
+        cls,
+        data: Dict[str, Any],
+        renderer: BaseRenderer,
+    ) -> "SelectorConfig":
+        try:
+            rendered = renderer.render_data(data)
+        except (ValidationError, DbtRuntimeError) as exc:
+            raise DbtSelectorsError(
+                f"Could not render selector data: {exc}",
+                result_type="invalid_selector",
+            ) from exc
+        return cls.selectors_from_dict(rendered)
+
+    @classmethod
+    def from_path(
+        cls,
+        path: Path,
+        renderer: BaseRenderer,
+    ) -> "SelectorConfig":
+        try:
+            data = load_yaml_text(load_file_contents(str(path)))
+            if data is None:
+                raise ValidationError("No data found in selector file at path: {path}")
+        except (ValidationError, DbtRuntimeError) as exc:
+            raise DbtSelectorsError(
+                f"Could not read selector file: {exc}",
+                result_type="invalid_selector",
+                path=path,
+            ) from exc
+
+        try:
+            return cls.render_from_dict(data, renderer)
+        except DbtSelectorsError as exc:
+            exc.path = path
+            raise
+
+
+def selector_data_from_root(project_root: str) -> Optional[Dict[str, Any]]:
+    selector_filepath = resolve_path_from_base("selectors.yml", project_root)
+
+    if path_exists(selector_filepath):
+        selectors_dict = load_yaml_text(load_file_contents(selector_filepath))
+    else:
+        selectors_dict = None
+    return selectors_dict
+
+
+def selector_config_from_data(selectors_data: Dict[str, Any]) -> SelectorConfig:
+    if not selectors_data:
+        selectors_data = {"selectors": []}
+
+    try:
+        selectors = SelectorConfig.selectors_from_dict(selectors_data)
+    except ValidationError as e:
+        raise DbtSelectorsError(
+            MALFORMED_SELECTOR_ERROR.format(error=str(e.message)),
+            result_type="invalid_selector",
+        ) from e
+    return selectors
+
+
+def validate_selector_default(selector_file: SelectorFile) -> None:
+    """Check if a selector.yml file has more than 1 default key set to true"""
+    default_set: bool = False
+    default_selector_name: Union[str, None] = None
+
+    for selector in selector_file.selectors:
+        if selector.default is True and default_set is False:
+            default_set = True
+            default_selector_name = selector.name
+            continue
+        if selector.default is True and default_set is True:
+            raise DbtSelectorsError(
+                "Error when parsing the selector file. "
+                "Found multiple selectors with `default: true`:"
+                f"{default_selector_name} and {selector.name}"
+            )
+
+
+# These are utilities to clean up the dictionary created from
+# selectors.yml by turning the cli-string format entries into
+# normalized dictionary entries. It parallels the flow in
+# dbt/graph/cli.py. If changes are made there, it might
+# be necessary to make changes here. Ideally it would be
+# good to combine the two flows into one at some point.
+class SelectorDict:
+    @classmethod
+    def parse_dict_definition(cls, definition, selector_dict={}):
+        key = list(definition)[0]
+        value = definition[key]
+        if isinstance(value, list):
+            new_values = []
+            for sel_def in value:
+                new_value = cls.parse_from_definition(sel_def, selector_dict=selector_dict)
+                new_values.append(new_value)
+            value = new_values
+        if key == "exclude":
+            definition = {key: value}
+        elif len(definition) == 1:
+            definition = {"method": key, "value": value}
+        elif key == "method" and value == "selector":
+            sel_def = definition.get("value")
+            if sel_def not in selector_dict:
+                raise DbtSelectorsError(f"Existing selector definition for {sel_def} not found.")
+            return selector_dict[definition["value"]]["definition"]
+        return definition
+
+    @classmethod
+    def parse_a_definition(cls, def_type, definition, selector_dict={}):
+        # this definition must be a list
+        new_dict = {def_type: []}
+        for sel_def in definition[def_type]:
+            if isinstance(sel_def, dict):
+                sel_def = cls.parse_from_definition(sel_def, selector_dict=selector_dict)
+                new_dict[def_type].append(sel_def)
+            elif isinstance(sel_def, str):
+                sel_def = SelectionCriteria.dict_from_single_spec(sel_def)
+                new_dict[def_type].append(sel_def)
+            else:
+                new_dict[def_type].append(sel_def)
+        return new_dict
+
+    @classmethod
+    def parse_from_definition(cls, definition, selector_dict={}):
+        if isinstance(definition, str):
+            definition = SelectionCriteria.dict_from_single_spec(definition)
+        elif "union" in definition:
+            definition = cls.parse_a_definition("union", definition, selector_dict=selector_dict)
+        elif "intersection" in definition:
+            definition = cls.parse_a_definition(
+                "intersection", definition, selector_dict=selector_dict
+            )
+        elif isinstance(definition, dict):
+            definition = cls.parse_dict_definition(definition, selector_dict=selector_dict)
+        return definition
+
+    # This is the normal entrypoint of this code. Give it the
+    # list of selectors generated from the selectors.yml file.
+    @classmethod
+    def parse_from_selectors_list(cls, selectors):
+        selector_dict = {}
+        for selector in selectors:
+            sel_name = selector["name"]
+            selector_dict[sel_name] = selector
+            definition = cls.parse_from_definition(
+                selector["definition"], selector_dict=deepcopy(selector_dict)
+            )
+            selector_dict[sel_name]["definition"] = definition
+        return selector_dict