dvt-core 1.11.0b4__py3-none-any.whl

dvt/flags.py

@@ -0,0 +1,89 @@
+# Do not import the os package because we expose this package in jinja
+from argparse import Namespace
+from pathlib import Path
+
+# this roughly follows the patten of EVENT_MANAGER in dbt/common/events/functions.py
+# During de-globlization, we'll need to handle both similarly
+# Match USE_COLORS default with default in dbt.cli.params.use_colors for use in --version
+GLOBAL_FLAGS = Namespace(USE_COLORS=True)  # type: ignore
+
+
+def set_flags(flags):
+    global GLOBAL_FLAGS
+    GLOBAL_FLAGS = flags
+
+
+def get_flags():
+    return GLOBAL_FLAGS
+
+
+def set_from_args(args: Namespace, project_flags):
+    global GLOBAL_FLAGS
+    from dvt.cli.flags import Flags, convert_config
+    from dvt.cli.main import cli
+
+    # we set attributes of args after initialize the flags, but project_flags
+    # is being read in the Flags constructor, so we need to read it here and pass in
+    # to make sure we use the correct project_flags
+    profiles_dir = getattr(args, "PROFILES_DIR", None) or getattr(args, "profiles_dir", None)
+    project_dir = getattr(args, "PROJECT_DIR", None) or getattr(args, "project_dir", None)
+    if profiles_dir and project_dir:
+        from dvt.config.project import read_project_flags
+
+        project_flags = read_project_flags(project_dir, profiles_dir)
+
+    # make a dummy context to get the flags, totally arbitrary
+    ctx = cli.make_context("run", ["run"])
+    flags = Flags(ctx, project_flags)
+    for arg_name, args_param_value in vars(args).items():
+        args_param_value = convert_config(arg_name, args_param_value)
+        object.__setattr__(flags, arg_name.upper(), args_param_value)
+        object.__setattr__(flags, arg_name.lower(), args_param_value)
+    flags.set_common_global_flags()
+    GLOBAL_FLAGS = flags  # type: ignore
+
+
+def get_flag_dict():
+    flag_attr = {
+        "use_experimental_parser",
+        "static_parser",
+        "warn_error",
+        "warn_error_options",
+        "write_json",
+        "partial_parse",
+        "use_colors",
+        "profiles_dir",
+        "debug",
+        "log_format",
+        "version_check",
+        "fail_fast",
+        "send_anonymous_usage_stats",
+        "printer_width",
+        "indirect_selection",
+        "log_cache_events",
+        "quiet",
+        "no_print",
+        "cache_selected_only",
+        "introspect",
+        "target_path",
+        "log_path",
+        "invocation_command",
+        "empty",
+    }
+    return {key: getattr(GLOBAL_FLAGS, key.upper(), None) for key in flag_attr}
+
+
+# This is used by core/dbt/context/base.py to return a flag object
+# in Jinja.
+def get_flag_obj():
+    new_flags = Namespace()
+    for key, val in get_flag_dict().items():
+        if isinstance(val, Path):
+            val = str(val)
+        setattr(new_flags, key.upper(), val)
+    # The following 3 are CLI arguments only so they're not full-fledged flags,
+    # but we put in flags for users.
+    setattr(new_flags, "FULL_REFRESH", getattr(GLOBAL_FLAGS, "FULL_REFRESH", None))
+    setattr(new_flags, "STORE_FAILURES", getattr(GLOBAL_FLAGS, "STORE_FAILURES", None))
+    setattr(new_flags, "WHICH", getattr(GLOBAL_FLAGS, "WHICH", None))
+    return new_flags

dvt/graph/__init__.py

@@ -0,0 +1,11 @@
+from .cli import parse_difference, parse_from_selectors_definition  # noqa: F401
+from .graph import Graph, UniqueId  # noqa: F401
+from .queue import GraphQueue  # noqa: F401
+from .selector import NodeSelector, ResourceTypeSelector  # noqa: F401
+from .selector_spec import (  # noqa: F401
+    SelectionCriteria,
+    SelectionDifference,
+    SelectionIntersection,
+    SelectionSpec,
+    SelectionUnion,
+)

dvt/graph/cli.py

@@ -0,0 +1,248 @@
+# special support for CLI argument parsing.
+# TODO: Remove as part of https://github.com/dbt-labs/dbt-core/issues/6701
+import itertools
+from copy import deepcopy
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from dvt.clients.yaml_helper import Dumper, Loader, yaml  # noqa: F401
+from dvt.contracts.selection import SelectorDefinition, SelectorFile
+from dvt.flags import get_flags
+
+from dbt_common.exceptions import DbtInternalError, DbtValidationError
+
+from .selector_spec import (
+    IndirectSelection,
+    SelectionCriteria,
+    SelectionDifference,
+    SelectionIntersection,
+    SelectionSpec,
+    SelectionUnion,
+)
+
+INTERSECTION_DELIMITER = ","
+
+DEFAULT_INCLUDES: List[str] = ["fqn:*", "source:*", "exposure:*", "metric:*", "semantic_model:*"]
+DEFAULT_EXCLUDES: List[str] = []
+
+
+def parse_union(
+    components: List[str],
+    expect_exists: bool,
+) -> SelectionUnion:
+    # turn ['a b', 'c'] -> ['a', 'b', 'c']
+    raw_specs = itertools.chain.from_iterable(r.split(" ") for r in components)
+    union_components: List[SelectionSpec] = []
+    flags = get_flags()
+    # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
+    for raw_spec in raw_specs:
+        intersection_components: List[SelectionSpec] = [
+            SelectionCriteria.from_single_spec(part)
+            for part in raw_spec.split(INTERSECTION_DELIMITER)
+        ]
+        union_components.append(
+            SelectionIntersection(
+                components=intersection_components,
+                expect_exists=expect_exists,
+                raw=raw_spec,
+                indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION),
+            )
+        )
+    return SelectionUnion(
+        components=union_components,
+        expect_exists=False,
+        raw=components,
+        indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION),
+    )
+
+
+def parse_union_from_default(raw: Optional[List[str]], default: List[str]) -> SelectionUnion:
+    components: List[str]
+    expect_exists: bool
+    if raw is None:
+        return parse_union(components=default, expect_exists=False)
+    else:
+        return parse_union(components=raw, expect_exists=True)
+
+
+def parse_difference(
+    include: Optional[List[str]], exclude: Optional[List[str]]
+) -> SelectionDifference:
+
+    if include == ():
+        include = None
+
+    included = parse_union_from_default(include, DEFAULT_INCLUDES)
+    excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES)
+    return SelectionDifference(components=[included, excluded])
+
+
+RawDefinition = Union[str, Dict[str, Any]]
+
+
+def _get_list_dicts(dct: Dict[str, Any], key: str) -> List[RawDefinition]:
+    result: List[RawDefinition] = []
+    if key not in dct:
+        raise DbtInternalError(f"Expected to find key {key} in dict, only found {list(dct)}")
+    values = dct[key]
+    if not isinstance(values, list):
+        raise DbtValidationError(f'Invalid value for key "{key}". Expected a list.')
+    for value in values:
+        if isinstance(value, dict):
+            for value_key in value:
+                if not isinstance(value_key, str):
+                    raise DbtValidationError(
+                        f'Expected all keys to "{key}" dict to be strings, '
+                        f'but "{value_key}" is a "{type(value_key)}"'
+                    )
+            result.append(value)
+        elif isinstance(value, str):
+            result.append(value)
+        else:
+            raise DbtValidationError(
+                f'Invalid value type {type(value)} in key "{key}", expected '
+                f"dict or str (value: {value})."
+            )
+
+    return result
+
+
+def _parse_exclusions(definition, result={}) -> Optional[SelectionSpec]:
+    exclusions = _get_list_dicts(definition, "exclude")
+    parsed_exclusions = [parse_from_definition(excl, result=result) for excl in exclusions]
+    if len(parsed_exclusions) == 1:
+        return parsed_exclusions[0]
+    elif len(parsed_exclusions) > 1:
+        return SelectionUnion(components=parsed_exclusions, raw=exclusions)
+    else:
+        return None
+
+
+def _parse_include_exclude_subdefs(
+    definitions: List[RawDefinition], result={}
+) -> Tuple[List[SelectionSpec], Optional[SelectionSpec]]:
+    include_parts: List[SelectionSpec] = []
+    diff_arg: Optional[SelectionSpec] = None
+
+    for definition in definitions:
+        if isinstance(definition, dict) and "exclude" in definition:
+            # do not allow multiple exclude: defs at the same level
+            if diff_arg is not None:
+                yaml_sel_cfg = yaml.dump(definition)
+                raise DbtValidationError(
+                    f"You cannot provide multiple exclude arguments to the "
+                    f"same selector set operator:\n{yaml_sel_cfg}"
+                )
+            diff_arg = _parse_exclusions(definition, result=result)
+        else:
+            include_parts.append(parse_from_definition(definition, result=result))
+
+    return (include_parts, diff_arg)
+
+
+def parse_union_definition(definition: Dict[str, Any], result={}) -> SelectionSpec:
+    union_def_parts = _get_list_dicts(definition, "union")
+    include, exclude = _parse_include_exclude_subdefs(union_def_parts, result=result)
+
+    union = SelectionUnion(components=include)
+
+    if exclude is None:
+        union.raw = definition
+        return union
+    else:
+        return SelectionDifference(components=[union, exclude], raw=definition)
+
+
+def parse_intersection_definition(definition: Dict[str, Any], result={}) -> SelectionSpec:
+    intersection_def_parts = _get_list_dicts(definition, "intersection")
+    include, exclude = _parse_include_exclude_subdefs(intersection_def_parts, result=result)
+    intersection = SelectionIntersection(components=include)
+
+    if exclude is None:
+        intersection.raw = definition
+        return intersection
+    else:
+        return SelectionDifference(components=[intersection, exclude], raw=definition)
+
+
+def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpec:
+    diff_arg: Optional[SelectionSpec] = None
+    if len(definition) == 1:
+        key = list(definition)[0]
+        value = definition[key]
+        if not isinstance(key, str):
+            raise DbtValidationError(
+                f'Expected definition key to be a "str", got one of type ' f'"{type(key)}" ({key})'
+            )
+        dct = {
+            "method": key,
+            "value": value,
+        }
+    elif definition.get("method") == "selector":
+        sel_def = definition.get("value")
+        if sel_def not in result:
+            raise DbtValidationError(f"Existing selector definition for {sel_def} not found.")
+        return result[definition["value"]]["definition"]
+    elif "method" in definition and "value" in definition:
+        dct = definition
+        if "exclude" in definition:
+            diff_arg = _parse_exclusions(definition, result=result)
+            dct = {k: v for k, v in dct.items() if k != "exclude"}
+    else:
+        raise DbtValidationError(
+            f'Expected either 1 key or else "method" '
+            f'and "value" keys, but got {list(definition)}'
+        )
+
+    # if key isn't a valid method name, this will raise
+    base = SelectionCriteria.selection_criteria_from_dict(definition, dct)
+    if diff_arg is None:
+        return base
+    else:
+        return SelectionDifference(components=[base, diff_arg])
+
+
+def parse_from_definition(
+    definition: RawDefinition,
+    rootlevel=False,
+    result: Dict[str, Dict[str, Union[SelectionSpec, bool]]] = {},
+) -> SelectionSpec:
+
+    if (
+        isinstance(definition, dict)
+        and ("union" in definition or "intersection" in definition)
+        and rootlevel
+        and len(definition) > 1
+    ):
+        keys = ",".join(definition.keys())
+        raise DbtValidationError(
+            f"Only a single 'union' or 'intersection' key is allowed "
+            f"in a root level selector definition; found {keys}."
+        )
+    if isinstance(definition, str):
+        return SelectionCriteria.from_single_spec(definition)
+    elif "union" in definition:
+        return parse_union_definition(definition, result=result)
+    elif "intersection" in definition:
+        return parse_intersection_definition(definition, result=result)
+    elif isinstance(definition, dict):
+        return parse_dict_definition(definition, result=result)
+    else:
+        raise DbtValidationError(
+            f"Expected to find union, intersection, str or dict, instead "
+            f"found {type(definition)}: {definition}"
+        )
+
+
+def parse_from_selectors_definition(
+    source: SelectorFile,
+) -> Dict[str, Dict[str, Union[SelectionSpec, bool]]]:
+    result: Dict[str, Dict[str, Union[SelectionSpec, bool]]] = {}
+    selector: SelectorDefinition
+    for selector in source.selectors:
+        result[selector.name] = {
+            "default": selector.default,
+            "definition": parse_from_definition(
+                selector.definition, rootlevel=True, result=deepcopy(result)
+            ),
+        }
+    return result

dvt/graph/graph.py

@@ -0,0 +1,172 @@
+from functools import partial
+from itertools import product
+from typing import Iterable, Iterator, NewType, Optional, Set
+
+import networkx as nx  # type: ignore
+
+from dbt_common.exceptions import DbtInternalError
+
+UniqueId = NewType("UniqueId", str)
+
+
+class Graph:
+    """A wrapper around the networkx graph that understands SelectionCriteria
+    and how they interact with the graph.
+    """
+
+    def __init__(self, graph) -> None:
+        self.graph: nx.DiGraph = graph
+
+    def nodes(self) -> Set[UniqueId]:
+        return set(self.graph.nodes())
+
+    def edges(self):
+        return self.graph.edges()
+
+    def __iter__(self) -> Iterator[UniqueId]:
+        return iter(self.graph.nodes())
+
+    def ancestors(self, node: UniqueId, max_depth: Optional[int] = None) -> Set[UniqueId]:
+        """Returns all nodes having a path to `node` in `graph`"""
+        if not self.graph.has_node(node):
+            raise DbtInternalError(f"Node {node} not found in the graph!")
+        filtered_graph = self.exclude_edge_type("parent_test")
+        return {
+            child
+            for _, child in nx.bfs_edges(filtered_graph, node, reverse=True, depth_limit=max_depth)
+        }
+
+    def descendants(self, node: UniqueId, max_depth: Optional[int] = None) -> Set[UniqueId]:
+        """Returns all nodes reachable from `node` in `graph`"""
+        if not self.graph.has_node(node):
+            raise DbtInternalError(f"Node {node} not found in the graph!")
+        filtered_graph = self.exclude_edge_type("parent_test")
+        return {child for _, child in nx.bfs_edges(filtered_graph, node, depth_limit=max_depth)}
+
+    def exclude_edge_type(self, edge_type_to_exclude):
+        return nx.subgraph_view(
+            self.graph,
+            filter_edge=partial(self.filter_edges_by_type, edge_type=edge_type_to_exclude),
+        )
+
+    def filter_edges_by_type(self, first_node, second_node, edge_type):
+        return self.graph.get_edge_data(first_node, second_node).get("edge_type") != edge_type
+
+    def select_childrens_parents(self, selected: Set[UniqueId]) -> Set[UniqueId]:
+        ancestors_for = self.select_children(selected) | selected
+        return self.select_parents(ancestors_for) | ancestors_for
+
+    def select_children(
+        self, selected: Set[UniqueId], max_depth: Optional[int] = None
+    ) -> Set[UniqueId]:
+        """Returns all nodes which are descendants of the 'selected' set.
+        Nodes in the 'selected' set are counted as children only if
+        they are descendants of other nodes in the 'selected' set."""
+        children: Set[UniqueId] = set()
+        i = 0
+        while len(selected) > 0 and (max_depth is None or i < max_depth):
+            next_layer: Set[UniqueId] = set()
+            for node in selected:
+                next_layer.update(
+                    iter(
+                        e[1]
+                        for e in self.graph.out_edges(node)
+                        if e[1] not in children
+                        and self.filter_edges_by_type(e[0], e[1], "parent_test")
+                    )
+                )
+            children.update(next_layer)
+            selected = next_layer
+            i += 1
+
+        return children
+
+    def select_parents(
+        self, selected: Set[UniqueId], max_depth: Optional[int] = None
+    ) -> Set[UniqueId]:
+        """Returns all nodes which are ancestors of the 'selected' set.
+        Nodes in the 'selected' set are counted as parents only if
+        they are ancestors of other nodes in the 'selected' set."""
+        parents: Set[UniqueId] = set()
+        i = 0
+        while len(selected) > 0 and (max_depth is None or i < max_depth):
+            next_layer: Set[UniqueId] = set()
+            for node in selected:
+                next_layer.update(
+                    iter(
+                        e[0]
+                        for e in self.graph.in_edges(node)
+                        if e[0] not in parents
+                        and self.filter_edges_by_type(e[0], e[1], "parent_test")
+                    )
+                )
+            parents.update(next_layer)
+            selected = next_layer
+            i += 1
+
+        return parents
+
+    def select_successors(self, selected: Set[UniqueId]) -> Set[UniqueId]:
+        successors: Set[UniqueId] = set()
+        for node in selected:
+            successors.update(self.graph.successors(node))
+        return successors
+
+    def get_subset_graph(self, selected: Iterable[UniqueId]) -> "Graph":
+        """Create and return a new graph that is a shallow copy of the graph,
+        but with only the nodes in include_nodes. Transitive edges across
+        removed nodes are preserved as explicit new edges.
+        """
+
+        new_graph: nx.DiGraph = self.graph.copy()
+        include_nodes: Set[UniqueId] = set(selected)
+
+        still_removing: bool = True
+        while still_removing:
+            nodes_to_remove = list(
+                node
+                for node in new_graph
+                if node not in include_nodes
+                and (new_graph.in_degree(node) * new_graph.out_degree(node)) == 0
+            )
+            if len(nodes_to_remove) == 0:
+                still_removing = False
+            else:
+                new_graph.remove_nodes_from(nodes_to_remove)
+
+        # sort remaining nodes by degree
+        remaining_nodes = list(new_graph.nodes())
+        remaining_nodes.sort(
+            key=lambda node: new_graph.in_degree(node) * new_graph.out_degree(node)
+        )
+
+        for node in remaining_nodes:
+            if node not in include_nodes:
+                source_nodes = [x for x, _ in new_graph.in_edges(node)]
+                target_nodes = [x for _, x in new_graph.out_edges(node)]
+
+                new_edges = product(source_nodes, target_nodes)
+                non_cyclic_new_edges = [
+                    (source, target)
+                    for source, target in new_edges
+                    if source != target and not new_graph.has_edge(source, target)
+                ]  # removes cyclic refs and edges already existing in new graph
+
+                new_graph.add_edges_from(non_cyclic_new_edges)
+                new_graph.remove_node(node)
+
+        for node in include_nodes:
+            if node not in new_graph:
+                raise ValueError(
+                    "Couldn't find model '{}' -- does it exist or is it disabled?".format(node)
+                )
+
+        return Graph(new_graph)
+
+    def subgraph(self, nodes: Iterable[UniqueId]) -> "Graph":
+        # Take the original networkx graph and return a subgraph containing only
+        # the selected unique_id nodes.
+        return Graph(self.graph.subgraph(nodes))
+
+    def get_dependent_nodes(self, node: UniqueId):
+        return nx.descendants(self.graph, node)