dvt-core 0.59.0a51__py3-none-any.whl

dbt/graph/selector_spec.py

@@ -0,0 +1,222 @@
+import os
+import re
+from abc import ABCMeta, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union
+
+from dbt.exceptions import InvalidSelectorError
+from dbt.flags import get_flags
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+from dbt_common.exceptions import DbtRuntimeError
+
+from .graph import UniqueId
+from .selector_methods import MethodName
+
+RAW_SELECTOR_PATTERN = re.compile(
+    r"\A"
+    r"(?P<childrens_parents>(\@))?"
+    r"(?P<parents>((?P<parents_depth>(\d*))\+))?"
+    r"((?P<method>([\w.]+)):)?(?P<value>(.*?))"
+    r"(?P<children>(\+(?P<children_depth>(\d*))))?"
+    r"\Z"
+)
+SELECTOR_METHOD_SEPARATOR = "."
+
+
+class IndirectSelection(StrEnum):
+    Eager = "eager"
+    Cautious = "cautious"
+    Buildable = "buildable"
+    Empty = "empty"
+
+
+def _probably_path(value: str):
+    """Decide if the value is probably a path. Windows has two path separators, so
+    we should check both sep ('\\') and altsep ('/') there.
+    """
+    if os.path.sep in value:
+        return True
+    elif os.path.altsep is not None and os.path.altsep in value:
+        return True
+    else:
+        return False
+
+
+def _match_to_int(match: Dict[str, str], key: str) -> Optional[int]:
+    raw = match.get(key)
+    # turn the empty string into None, too.
+    if not raw:
+        return None
+    try:
+        return int(raw)
+    except ValueError as exc:
+        raise DbtRuntimeError(f"Invalid node spec - could not handle parent depth {raw}") from exc
+
+
+SelectionSpec = Union[
+    "SelectionCriteria",
+    "SelectionIntersection",
+    "SelectionDifference",
+    "SelectionUnion",
+]
+
+
+@dataclass
+class SelectionCriteria:
+    raw: Any
+    method: MethodName
+    method_arguments: List[str]
+    value: Any
+    childrens_parents: bool
+    parents: bool
+    parents_depth: Optional[int]
+    children: bool
+    children_depth: Optional[int]
+    indirect_selection: IndirectSelection = IndirectSelection.Eager
+
+    def __post_init__(self):
+        if self.children and self.childrens_parents:
+            raise DbtRuntimeError(
+                f'Invalid node spec {self.raw} - "@" prefix and "+" suffix ' "are incompatible"
+            )
+
+    @classmethod
+    def default_method(cls, value: str) -> MethodName:
+        if _probably_path(value):
+            return MethodName.Path
+        elif value.lower().endswith((".sql", ".py", ".csv")):
+            return MethodName.File
+        else:
+            return MethodName.FQN
+
+    @classmethod
+    def parse_method(cls, groupdict: Dict[str, Any]) -> Tuple[MethodName, List[str]]:
+        raw_method = groupdict.get("method")
+        if raw_method is None:
+            return cls.default_method(groupdict["value"]), []
+
+        method_parts: List[str] = raw_method.split(SELECTOR_METHOD_SEPARATOR)
+        try:
+            method_name = MethodName(method_parts[0])
+        except ValueError as exc:
+            raise InvalidSelectorError(f"'{method_parts[0]}' is not a valid method name") from exc
+
+        # Following is for cases like config.severity and config.materialized
+        method_arguments: List[str] = method_parts[1:]
+
+        return method_name, method_arguments
+
+    @classmethod
+    def selection_criteria_from_dict(
+        cls,
+        raw: Any,
+        dct: Dict[str, Any],
+    ) -> "SelectionCriteria":
+        if "value" not in dct:
+            raise DbtRuntimeError(f'Invalid node spec "{raw}" - no search value!')
+        method_name, method_arguments = cls.parse_method(dct)
+
+        parents_depth = _match_to_int(dct, "parents_depth")
+        children_depth = _match_to_int(dct, "children_depth")
+
+        # If defined field in selector, override CLI flag
+        indirect_selection = IndirectSelection(
+            dct.get("indirect_selection", getattr(get_flags(), "INDIRECT_SELECTION", "eager"))
+        )
+
+        return cls(
+            raw=raw,
+            method=method_name,
+            method_arguments=method_arguments,
+            value=dct["value"],
+            childrens_parents=bool(dct.get("childrens_parents")),
+            parents=bool(dct.get("parents")),
+            parents_depth=parents_depth,
+            children=bool(dct.get("children")),
+            children_depth=children_depth,
+            indirect_selection=indirect_selection,
+        )
+
+    @classmethod
+    def dict_from_single_spec(cls, raw: str):
+        result = RAW_SELECTOR_PATTERN.match(raw)
+        if result is None:
+            return {"error": "Invalid selector spec"}
+        dct: Dict[str, Any] = result.groupdict()
+        method_name, method_arguments = cls.parse_method(dct)
+        meth_name = str(method_name)
+        if method_arguments:
+            meth_name += "." + ".".join(method_arguments)
+        dct["method"] = meth_name
+        dct = {k: v for k, v in dct.items() if (v is not None and v != "")}
+        if "childrens_parents" in dct:
+            dct["childrens_parents"] = bool(dct.get("childrens_parents"))
+        if "parents" in dct:
+            dct["parents"] = bool(dct.get("parents"))
+        if "children" in dct:
+            dct["children"] = bool(dct.get("children"))
+        return dct
+
+    @classmethod
+    def from_single_spec(cls, raw: str) -> "SelectionCriteria":
+        result = RAW_SELECTOR_PATTERN.match(raw)
+        if result is None:
+            # bad spec!
+            raise DbtRuntimeError(f'Invalid selector spec "{raw}"')
+
+        return cls.selection_criteria_from_dict(raw, result.groupdict())
+
+
+class BaseSelectionGroup(dbtClassMixin, Iterable[SelectionSpec], metaclass=ABCMeta):
+    def __init__(
+        self,
+        components: Iterable[SelectionSpec],
+        indirect_selection: IndirectSelection = IndirectSelection.Eager,
+        expect_exists: bool = False,
+        raw: Any = None,
+    ) -> None:
+        self.components: List[SelectionSpec] = list(components)
+        self.expect_exists = expect_exists
+        self.raw = raw
+        self.indirect_selection = indirect_selection
+
+    def __iter__(self) -> Iterator[SelectionSpec]:
+        for component in self.components:
+            yield component
+
+    @abstractmethod
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        raise NotImplementedError("_combine_selections not implemented!")
+
+    def combined(self, selections: List[Set[UniqueId]]) -> Set[UniqueId]:
+        if not selections:
+            return set()
+
+        return self.combine_selections(selections)
+
+
+class SelectionIntersection(BaseSelectionGroup):
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        return set.intersection(*selections)
+
+
+class SelectionDifference(BaseSelectionGroup):
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        return set.difference(*selections)
+
+
+class SelectionUnion(BaseSelectionGroup):
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        return set.union(*selections)

dbt/graph/thread_pool.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from multiprocessing.pool import ThreadPool
+
+
+class DbtThreadPool(ThreadPool):
+    """A ThreadPool that tracks whether or not it's been closed"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.closed = False
+
+    def close(self):
+        self.closed = True
+        super().close()
+
+    def is_closed(self):
+        return self.closed

dbt/hooks.py

@@ -0,0 +1,21 @@
+import json
+from typing import Any, Dict, Union
+
+from dbt_common.dataclass_schema import StrEnum
+
+
+class ModelHookType(StrEnum):
+    PreHook = "pre-hook"
+    PostHook = "post-hook"
+
+
+def get_hook_dict(source: Union[str, Dict[str, Any]]) -> Dict[str, Any]:
+    """From a source string-or-dict, get a dictionary that can be passed to
+    Hook.from_dict
+    """
+    if isinstance(source, dict):
+        return source
+    try:
+        return json.loads(source)
+    except ValueError:
+        return {"sql": source}

dbt/include/README.md

@@ -0,0 +1,49 @@
+# Include Module
+
+The Include module is responsible for the starter project scaffold.
+
+# Directories
+
+## `starter_project`
+Produces the default project after running the `dbt init` command for the CLI. `dbt-cloud` initializes the project by using [dbt-starter-project](https://github.com/dbt-labs/dbt-starter-project).
+
+# adapter.dispatch
+Packages (e.g. `include` directories of adapters, any [hub](https://hub.getdbt.com/)-hosted package) can be interpreted as namespaces of functions a.k.a macros. In `dbt`'s macrospace, we take advantage of the multiple dispatch programming language concept. In short, multiple dispatch supports dynamic searching for a function across several namespaces—usually in a manually specified manner/order.
+
+Adapters can have their own implementation of the same macro X. For example, a macro executed by `dbt-redshift` may need a specific implementation different from `dbt-snowflake`'s macro. We use multiple dispatch via `adapter.dispatch`, a Jinja function, which enables polymorphic macro invocations. The chosen implementation is selected according to what the `adapter` object is set to at runtime (it could be for redshift, postgres, and so on).
+
+For more on this object, check out the dbt docs [here](https://docs.getdbt.com/reference/dbt-jinja-functions/adapter).
+
+# dbt and database adapter python package interop
+
+Let’s say we have a fictional python app named `dbt-core` with this structure
+
+```
+dbt
+├── adapters
+│   └── base.py
+├── cli.py
+└── main.py
+```
+
+`pip install dbt-core` will install this application in my python environment, maintaining the same structure. Note that `dbt.adapters` only contains a `base.py`. In this example, we can assume that base.py includes an abstract class for creating connections. Let’s say we wanted to create an postgres adapter that this app could use, and can be installed independently. We can create a python package with the following structure called `dbt-postgres`
+```
+dbt
+└── adapters
+    └── postgres
+        └── impl.py
+```
+
+`pip install dbt-postgres` will install this package in the python environment, maintaining the same structure again. Let’s say `impl.py` imports `dbt.adapters.base` and implements a concrete class inheriting from the abstract class in `base.py` from the `dbt-core` package. Since our top level package is named the same in both packages, `pip` will put this in the same place. We end up with this installed in our python environment.
+
+```
+dbt
+├── adapters
+│   ├── base.py
+│   └── postgres
+│       └── impl.py
+├── cli.py
+└── main.py
+```
+
+`dbt.adapters` now has a postgres module that dbt can easily find and call directly. dbt and its adapters follows the same type of file structure convention. This is the magic that allows you to import `dbt.*` in database adapters, and using a factory pattern in dbt-core, we can create instances of concrete classes defined in the database adapter packages (for creating connections, defining database configuration, defining credentials, etc.)

dbt/include/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)