dvt-core 0.52.2__cp310-cp310-macosx_10_9_x86_64.whl

dbt/config/selectors.py

@@ -0,0 +1,208 @@
+from copy import deepcopy
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+from dbt.clients.yaml_helper import Dumper, Loader, load_yaml_text, yaml  # noqa: F401
+from dbt.contracts.selection import SelectorFile
+from dbt.exceptions import DbtSelectorsError
+from dbt.graph import SelectionSpec, parse_from_selectors_definition
+from dbt.graph.selector_spec import SelectionCriteria
+from dbt_common.clients.system import (
+    load_file_contents,
+    path_exists,
+    resolve_path_from_base,
+)
+from dbt_common.dataclass_schema import ValidationError
+from dbt_common.exceptions import DbtRuntimeError
+
+from .renderer import BaseRenderer
+
+MALFORMED_SELECTOR_ERROR = """\
+The selectors.yml file in this project is malformed. Please double check
+the contents of this file and fix any errors before retrying.
+
+You can find more information on the syntax for this file here:
+https://docs.getdbt.com/reference/node-selection/yaml-selectors
+
+Validator Error:
+{error}
+"""
+
+
+class SelectorConfig(Dict[str, Dict[str, Union[SelectionSpec, bool]]]):
+    @classmethod
+    def selectors_from_dict(cls, data: Dict[str, Any]) -> "SelectorConfig":
+        try:
+            SelectorFile.validate(data)
+            selector_file = SelectorFile.from_dict(data)
+            validate_selector_default(selector_file)
+            selectors = parse_from_selectors_definition(selector_file)
+        except ValidationError as exc:
+            yaml_sel_cfg = yaml.dump(exc.instance)
+            raise DbtSelectorsError(
+                f"Could not parse selector file data: \n{yaml_sel_cfg}\n"
+                f"Valid root-level selector definitions: "
+                f"union, intersection, string, dictionary. No lists. "
+                f"\nhttps://docs.getdbt.com/reference/node-selection/"
+                f"yaml-selectors",
+                result_type="invalid_selector",
+            ) from exc
+        except DbtRuntimeError as exc:
+            raise DbtSelectorsError(
+                f"Could not read selector file data: {exc}",
+                result_type="invalid_selector",
+            ) from exc
+
+        return cls(selectors)
+
+    @classmethod
+    def render_from_dict(
+        cls,
+        data: Dict[str, Any],
+        renderer: BaseRenderer,
+    ) -> "SelectorConfig":
+        try:
+            rendered = renderer.render_data(data)
+        except (ValidationError, DbtRuntimeError) as exc:
+            raise DbtSelectorsError(
+                f"Could not render selector data: {exc}",
+                result_type="invalid_selector",
+            ) from exc
+        return cls.selectors_from_dict(rendered)
+
+    @classmethod
+    def from_path(
+        cls,
+        path: Path,
+        renderer: BaseRenderer,
+    ) -> "SelectorConfig":
+        try:
+            data = load_yaml_text(load_file_contents(str(path)))
+            if data is None:
+                raise ValidationError("No data found in selector file at path: {path}")
+        except (ValidationError, DbtRuntimeError) as exc:
+            raise DbtSelectorsError(
+                f"Could not read selector file: {exc}",
+                result_type="invalid_selector",
+                path=path,
+            ) from exc
+
+        try:
+            return cls.render_from_dict(data, renderer)
+        except DbtSelectorsError as exc:
+            exc.path = path
+            raise
+
+
+def selector_data_from_root(project_root: str) -> Optional[Dict[str, Any]]:
+    selector_filepath = resolve_path_from_base("selectors.yml", project_root)
+
+    if path_exists(selector_filepath):
+        selectors_dict = load_yaml_text(load_file_contents(selector_filepath))
+    else:
+        selectors_dict = None
+    return selectors_dict
+
+
+def selector_config_from_data(selectors_data: Dict[str, Any]) -> SelectorConfig:
+    if not selectors_data:
+        selectors_data = {"selectors": []}
+
+    try:
+        selectors = SelectorConfig.selectors_from_dict(selectors_data)
+    except ValidationError as e:
+        raise DbtSelectorsError(
+            MALFORMED_SELECTOR_ERROR.format(error=str(e.message)),
+            result_type="invalid_selector",
+        ) from e
+    return selectors
+
+
+def validate_selector_default(selector_file: SelectorFile) -> None:
+    """Check if a selector.yml file has more than 1 default key set to true"""
+    default_set: bool = False
+    default_selector_name: Union[str, None] = None
+
+    for selector in selector_file.selectors:
+        if selector.default is True and default_set is False:
+            default_set = True
+            default_selector_name = selector.name
+            continue
+        if selector.default is True and default_set is True:
+            raise DbtSelectorsError(
+                "Error when parsing the selector file. "
+                "Found multiple selectors with `default: true`:"
+                f"{default_selector_name} and {selector.name}"
+            )
+
+
+# These are utilities to clean up the dictionary created from
+# selectors.yml by turning the cli-string format entries into
+# normalized dictionary entries. It parallels the flow in
+# dbt/graph/cli.py. If changes are made there, it might
+# be necessary to make changes here. Ideally it would be
+# good to combine the two flows into one at some point.
+class SelectorDict:
+    @classmethod
+    def parse_dict_definition(cls, definition, selector_dict={}):
+        key = list(definition)[0]
+        value = definition[key]
+        if isinstance(value, list):
+            new_values = []
+            for sel_def in value:
+                new_value = cls.parse_from_definition(sel_def, selector_dict=selector_dict)
+                new_values.append(new_value)
+            value = new_values
+        if key == "exclude":
+            definition = {key: value}
+        elif len(definition) == 1:
+            definition = {"method": key, "value": value}
+        elif key == "method" and value == "selector":
+            sel_def = definition.get("value")
+            if sel_def not in selector_dict:
+                raise DbtSelectorsError(f"Existing selector definition for {sel_def} not found.")
+            return selector_dict[definition["value"]]["definition"]
+        return definition
+
+    @classmethod
+    def parse_a_definition(cls, def_type, definition, selector_dict={}):
+        # this definition must be a list
+        new_dict = {def_type: []}
+        for sel_def in definition[def_type]:
+            if isinstance(sel_def, dict):
+                sel_def = cls.parse_from_definition(sel_def, selector_dict=selector_dict)
+                new_dict[def_type].append(sel_def)
+            elif isinstance(sel_def, str):
+                sel_def = SelectionCriteria.dict_from_single_spec(sel_def)
+                new_dict[def_type].append(sel_def)
+            else:
+                new_dict[def_type].append(sel_def)
+        return new_dict
+
+    @classmethod
+    def parse_from_definition(cls, definition, selector_dict={}):
+        if isinstance(definition, str):
+            definition = SelectionCriteria.dict_from_single_spec(definition)
+        elif "union" in definition:
+            definition = cls.parse_a_definition("union", definition, selector_dict=selector_dict)
+        elif "intersection" in definition:
+            definition = cls.parse_a_definition(
+                "intersection", definition, selector_dict=selector_dict
+            )
+        elif isinstance(definition, dict):
+            definition = cls.parse_dict_definition(definition, selector_dict=selector_dict)
+        return definition
+
+    # This is the normal entrypoint of this code. Give it the
+    # list of selectors generated from the selectors.yml file.
+    @classmethod
+    def parse_from_selectors_list(cls, selectors):
+        selector_dict = {}
+        for selector in selectors:
+            sel_name = selector["name"]
+            selector_dict[sel_name] = selector
+            definition = cls.parse_from_definition(
+                selector["definition"], selector_dict=deepcopy(selector_dict)
+            )
+            selector_dict[sel_name]["definition"] = definition
+        return selector_dict

dbt/config/utils.py

@@ -0,0 +1,77 @@
+from typing import Any, Dict, Optional
+
+from dbt import deprecations
+from dbt.clients import yaml_helper
+from dbt.events.types import InvalidOptionYAML
+from dbt.exceptions import DbtExclusivePropertyUseError, OptionNotYamlDictError
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtValidationError
+
+
+def parse_cli_vars(var_string: str) -> Dict[str, Any]:
+    return parse_cli_yaml_string(var_string, "vars")
+
+
+def parse_cli_yaml_string(var_string: str, cli_option_name: str) -> Dict[str, Any]:
+    try:
+        cli_vars = yaml_helper.load_yaml_text(var_string)
+        var_type = type(cli_vars)
+        if cli_vars is not None and var_type is dict:
+            return cli_vars
+        else:
+            raise OptionNotYamlDictError(var_type, cli_option_name)
+    except (DbtValidationError, OptionNotYamlDictError):
+        fire_event(InvalidOptionYAML(option_name=cli_option_name))
+        raise
+
+
+def exclusive_primary_alt_value_setting(
+    dictionary: Optional[Dict[str, Any]],
+    primary: str,
+    alt: str,
+    parent_config: Optional[str] = None,
+) -> None:
+    """Munges in place under the primary the options for the primary and alt values
+
+    Sometimes we allow setting something via TWO keys, but not at the same time. If both the primary
+    key and alt key have values, an error gets raised. If the alt key has values, then we update
+    the dictionary to ensure the primary key contains the values. If neither are set, nothing happens.
+    """
+
+    if dictionary is None:
+        return
+
+    primary_options = dictionary.get(primary)
+    alt_options = dictionary.get(alt)
+
+    if primary_options and alt_options:
+        where = f" in `{parent_config}`" if parent_config is not None else ""
+        raise DbtExclusivePropertyUseError(
+            f"Only `{alt}` or `{primary}` can be specified{where}, not both"
+        )
+
+    if alt in dictionary:
+        alt_value = dictionary.pop(alt)
+        dictionary[primary] = alt_value
+
+
+def normalize_warn_error_options(warn_error_options: Dict[str, Any]) -> None:
+    has_include = "include" in warn_error_options
+    has_exclude = "exclude" in warn_error_options
+
+    if has_include or has_exclude:
+        deprecations.buffer(
+            "weo-include-exclude-deprecation",
+            found_include=has_include,
+            found_exclude=has_exclude,
+        )
+
+    exclusive_primary_alt_value_setting(
+        warn_error_options, "error", "include", "warn_error_options"
+    )
+    exclusive_primary_alt_value_setting(
+        warn_error_options, "warn", "exclude", "warn_error_options"
+    )
+    for key in ("error", "warn", "silence"):
+        if key in warn_error_options and warn_error_options[key] is None:
+            warn_error_options[key] = []

dbt/constants.py

@@ -0,0 +1,28 @@
+from dbt_semantic_interfaces.type_enums import TimeGranularity
+
+DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
+
+SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
+
+MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
+MAXIMUM_SEED_SIZE_NAME = "1MB"
+
+PIN_PACKAGE_URL = (
+    "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"
+)
+
+DBT_PROJECT_FILE_NAME = "dbt_project.yml"
+PACKAGES_FILE_NAME = "packages.yml"
+DEPENDENCIES_FILE_NAME = "dependencies.yml"
+PACKAGE_LOCK_FILE_NAME = "package-lock.yml"
+MANIFEST_FILE_NAME = "manifest.json"
+SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json"
+LEGACY_TIME_SPINE_MODEL_NAME = "metricflow_time_spine"
+LEGACY_TIME_SPINE_GRANULARITY = TimeGranularity.DAY
+MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY = TimeGranularity.DAY
+PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
+PACKAGE_LOCK_HASH_KEY = "sha1_hash"
+CATALOGS_FILE_NAME = "catalogs.yml"
+RUN_RESULTS_FILE_NAME = "run_results.json"
+CATALOG_FILENAME = "catalog.json"
+SOURCE_RESULT_FILE_NAME = "sources.json"