dvt-core 1.11.0b4__py3-none-any.whl

dvt/graph/selector_spec.py

@@ -0,0 +1,223 @@
+import os
+import re
+from abc import ABCMeta, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union
+
+from dvt.exceptions import InvalidSelectorError
+from dvt.flags import get_flags
+
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+from dbt_common.exceptions import DbtRuntimeError
+
+from .graph import UniqueId
+from .selector_methods import MethodName
+
+RAW_SELECTOR_PATTERN = re.compile(
+    r"\A"
+    r"(?P<childrens_parents>(\@))?"
+    r"(?P<parents>((?P<parents_depth>(\d*))\+))?"
+    r"((?P<method>([\w.]+)):)?(?P<value>(.*?))"
+    r"(?P<children>(\+(?P<children_depth>(\d*))))?"
+    r"\Z"
+)
+SELECTOR_METHOD_SEPARATOR = "."
+
+
+class IndirectSelection(StrEnum):
+    Eager = "eager"
+    Cautious = "cautious"
+    Buildable = "buildable"
+    Empty = "empty"
+
+
+def _probably_path(value: str):
+    """Decide if the value is probably a path. Windows has two path separators, so
+    we should check both sep ('\\') and altsep ('/') there.
+    """
+    if os.path.sep in value:
+        return True
+    elif os.path.altsep is not None and os.path.altsep in value:
+        return True
+    else:
+        return False
+
+
+def _match_to_int(match: Dict[str, str], key: str) -> Optional[int]:
+    raw = match.get(key)
+    # turn the empty string into None, too.
+    if not raw:
+        return None
+    try:
+        return int(raw)
+    except ValueError as exc:
+        raise DbtRuntimeError(f"Invalid node spec - could not handle parent depth {raw}") from exc
+
+
+SelectionSpec = Union[
+    "SelectionCriteria",
+    "SelectionIntersection",
+    "SelectionDifference",
+    "SelectionUnion",
+]
+
+
+@dataclass
+class SelectionCriteria:
+    raw: Any
+    method: MethodName
+    method_arguments: List[str]
+    value: Any
+    childrens_parents: bool
+    parents: bool
+    parents_depth: Optional[int]
+    children: bool
+    children_depth: Optional[int]
+    indirect_selection: IndirectSelection = IndirectSelection.Eager
+
+    def __post_init__(self):
+        if self.children and self.childrens_parents:
+            raise DbtRuntimeError(
+                f'Invalid node spec {self.raw} - "@" prefix and "+" suffix ' "are incompatible"
+            )
+
+    @classmethod
+    def default_method(cls, value: str) -> MethodName:
+        if _probably_path(value):
+            return MethodName.Path
+        elif value.lower().endswith((".sql", ".py", ".csv")):
+            return MethodName.File
+        else:
+            return MethodName.FQN
+
+    @classmethod
+    def parse_method(cls, groupdict: Dict[str, Any]) -> Tuple[MethodName, List[str]]:
+        raw_method = groupdict.get("method")
+        if raw_method is None:
+            return cls.default_method(groupdict["value"]), []
+
+        method_parts: List[str] = raw_method.split(SELECTOR_METHOD_SEPARATOR)
+        try:
+            method_name = MethodName(method_parts[0])
+        except ValueError as exc:
+            raise InvalidSelectorError(f"'{method_parts[0]}' is not a valid method name") from exc
+
+        # Following is for cases like config.severity and config.materialized
+        method_arguments: List[str] = method_parts[1:]
+
+        return method_name, method_arguments
+
+    @classmethod
+    def selection_criteria_from_dict(
+        cls,
+        raw: Any,
+        dct: Dict[str, Any],
+    ) -> "SelectionCriteria":
+        if "value" not in dct:
+            raise DbtRuntimeError(f'Invalid node spec "{raw}" - no search value!')
+        method_name, method_arguments = cls.parse_method(dct)
+
+        parents_depth = _match_to_int(dct, "parents_depth")
+        children_depth = _match_to_int(dct, "children_depth")
+
+        # If defined field in selector, override CLI flag
+        indirect_selection = IndirectSelection(
+            dct.get("indirect_selection", get_flags().INDIRECT_SELECTION)
+        )
+
+        return cls(
+            raw=raw,
+            method=method_name,
+            method_arguments=method_arguments,
+            value=dct["value"],
+            childrens_parents=bool(dct.get("childrens_parents")),
+            parents=bool(dct.get("parents")),
+            parents_depth=parents_depth,
+            children=bool(dct.get("children")),
+            children_depth=children_depth,
+            indirect_selection=indirect_selection,
+        )
+
+    @classmethod
+    def dict_from_single_spec(cls, raw: str):
+        result = RAW_SELECTOR_PATTERN.match(raw)
+        if result is None:
+            return {"error": "Invalid selector spec"}
+        dct: Dict[str, Any] = result.groupdict()
+        method_name, method_arguments = cls.parse_method(dct)
+        meth_name = str(method_name)
+        if method_arguments:
+            meth_name += "." + ".".join(method_arguments)
+        dct["method"] = meth_name
+        dct = {k: v for k, v in dct.items() if (v is not None and v != "")}
+        if "childrens_parents" in dct:
+            dct["childrens_parents"] = bool(dct.get("childrens_parents"))
+        if "parents" in dct:
+            dct["parents"] = bool(dct.get("parents"))
+        if "children" in dct:
+            dct["children"] = bool(dct.get("children"))
+        return dct
+
+    @classmethod
+    def from_single_spec(cls, raw: str) -> "SelectionCriteria":
+        result = RAW_SELECTOR_PATTERN.match(raw)
+        if result is None:
+            # bad spec!
+            raise DbtRuntimeError(f'Invalid selector spec "{raw}"')
+
+        return cls.selection_criteria_from_dict(raw, result.groupdict())
+
+
+class BaseSelectionGroup(dbtClassMixin, Iterable[SelectionSpec], metaclass=ABCMeta):
+    def __init__(
+        self,
+        components: Iterable[SelectionSpec],
+        indirect_selection: IndirectSelection = IndirectSelection.Eager,
+        expect_exists: bool = False,
+        raw: Any = None,
+    ) -> None:
+        self.components: List[SelectionSpec] = list(components)
+        self.expect_exists = expect_exists
+        self.raw = raw
+        self.indirect_selection = indirect_selection
+
+    def __iter__(self) -> Iterator[SelectionSpec]:
+        for component in self.components:
+            yield component
+
+    @abstractmethod
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        raise NotImplementedError("_combine_selections not implemented!")
+
+    def combined(self, selections: List[Set[UniqueId]]) -> Set[UniqueId]:
+        if not selections:
+            return set()
+
+        return self.combine_selections(selections)
+
+
+class SelectionIntersection(BaseSelectionGroup):
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        return set.intersection(*selections)
+
+
+class SelectionDifference(BaseSelectionGroup):
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        return set.difference(*selections)
+
+
+class SelectionUnion(BaseSelectionGroup):
+    def combine_selections(
+        self,
+        selections: List[Set[UniqueId]],
+    ) -> Set[UniqueId]:
+        return set.union(*selections)

dvt/graph/thread_pool.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from multiprocessing.pool import ThreadPool
+
+
+class DbtThreadPool(ThreadPool):
+    """A ThreadPool that tracks whether or not it's been closed"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.closed = False
+
+    def close(self):
+        self.closed = True
+        super().close()
+
+    def is_closed(self):
+        return self.closed

dvt/hooks.py

@@ -0,0 +1,21 @@
+import json
+from typing import Any, Dict, Union
+
+from dbt_common.dataclass_schema import StrEnum
+
+
+class ModelHookType(StrEnum):
+    PreHook = "pre-hook"
+    PostHook = "post-hook"
+
+
+def get_hook_dict(source: Union[str, Dict[str, Any]]) -> Dict[str, Any]:
+    """From a source string-or-dict, get a dictionary that can be passed to
+    Hook.from_dict
+    """
+    if isinstance(source, dict):
+        return source
+    try:
+        return json.loads(source)
+    except ValueError:
+        return {"sql": source}

dvt/include/README.md

@@ -0,0 +1,49 @@
+# Include Module
+
+The Include module is responsible for the starter project scaffold.
+
+# Directories
+
+## `starter_project`
+Produces the default project after running the `dbt init` command for the CLI. `dbt-cloud` initializes the project by using [dbt-starter-project](https://github.com/dbt-labs/dbt-starter-project).
+
+# adapter.dispatch
+Packages (e.g. `include` directories of adapters, any [hub](https://hub.getdbt.com/)-hosted package) can be interpreted as namespaces of functions a.k.a macros. In `dbt`'s macrospace, we take advantage of the multiple dispatch programming language concept. In short, multiple dispatch supports dynamic searching for a function across several namespaces—usually in a manually specified manner/order.
+
+Adapters can have their own implementation of the same macro X. For example, a macro executed by `dbt-redshift` may need a specific implementation different from `dbt-snowflake`'s macro. We use multiple dispatch via `adapter.dispatch`, a Jinja function, which enables polymorphic macro invocations. The chosen implementation is selected according to what the `adapter` object is set to at runtime (it could be for redshift, postgres, and so on).
+
+For more on this object, check out the dbt docs [here](https://docs.getdbt.com/reference/dbt-jinja-functions/adapter).
+
+# dbt and database adapter python package interop
+
+Let’s say we have a fictional python app named `dbt-core` with this structure
+
+```
+dbt
+├── adapters
+│   └── base.py
+├── cli.py
+└── main.py
+```
+
+`pip install dbt-core` will install this application in my python environment, maintaining the same structure. Note that `dbt.adapters` only contains a `base.py`. In this example, we can assume that base.py includes an abstract class for creating connections. Let’s say we wanted to create an postgres adapter that this app could use, and can be installed independently. We can create a python package with the following structure called `dbt-postgres`
+```
+dbt
+└── adapters
+    └── postgres
+        └── impl.py
+```
+
+`pip install dbt-postgres` will install this package in the python environment, maintaining the same structure again. Let’s say `impl.py` imports `dbt.adapters.base` and implements a concrete class inheriting from the abstract class in `base.py` from the `dbt-core` package. Since our top level package is named the same in both packages, `pip` will put this in the same place. We end up with this installed in our python environment.
+
+```
+dbt
+├── adapters
+│   ├── base.py
+│   └── postgres
+│       └── impl.py
+├── cli.py
+└── main.py
+```
+
+`dbt.adapters` now has a postgres module that dbt can easily find and call directly. dbt and its adapters follows the same type of file structure convention. This is the magic that allows you to import `dbt.*` in database adapters, and using a factory pattern in dbt-core, we can create instances of concrete classes defined in the database adapter packages (for creating connections, defining database configuration, defining credentials, etc.)

dvt/include/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

dvt/include/global_project.py

@@ -0,0 +1,4 @@
+# Global project configuration for dvt-core
+# This defines the name of the global/internal project namespace
+
+PROJECT_NAME = "dbt"  # Keep as "dbt" for compatibility with adapter ecosystem

dvt/include/starter_project/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

dvt/include/starter_project/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

dvt/include/starter_project/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

dvt/include/starter_project/dvt_project.yml

@@ -0,0 +1,36 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: '{project_name}'
+version: '1.0.0'
+
+# This setting configures which "profile" dbt uses for this project.
+profile: '{profile_name}'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{{{ config(...) }}}}` macro.
+models:
+  {project_name}:
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view

dvt/include/starter_project/models/example/my_first_dbt_model.sql

@@ -0,0 +1,27 @@
+
+/*
+    Welcome to your first dbt model!
+    Did you know that you can also configure models directly within SQL files?
+    This will override configurations stated in dbt_project.yml
+
+    Try changing "table" to "view" below
+*/
+
+{{ config(materialized='table') }}
+
+with source_data as (
+
+    select 1 as id
+    union all
+    select null as id
+
+)
+
+select *
+from source_data
+
+/*
+    Uncomment the line below to remove records with null `id` values
+*/
+
+-- where id is not null

dvt/include/starter_project/models/example/my_second_dbt_model.sql

@@ -0,0 +1,6 @@
+
+-- Use the `ref` function to select from other models
+
+select *
+from {{ ref('my_first_dbt_model') }}
+where id = 1

dvt/include/starter_project/models/example/schema.yml

@@ -0,0 +1,21 @@
+
+version: 2
+
+models:
+  - name: my_first_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        data_tests:
+          - unique
+          - not_null
+
+  - name: my_second_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        data_tests:
+          - unique
+          - not_null

dvt/internal_deprecations.py

@@ -0,0 +1,27 @@
+import functools
+from typing import Optional
+
+from dvt.events.types import InternalDeprecation
+
+from dbt_common.events.functions import warn_or_error
+
+
+def deprecated(suggested_action: str, version: str, reason: Optional[str]):
+    def inner(func):
+        @functools.wraps(func)
+        def wrapped(*args, **kwargs):
+            name = func.__name__
+
+            warn_or_error(
+                InternalDeprecation(
+                    name=name,
+                    suggested_action=suggested_action,
+                    version=version,
+                    reason=reason,
+                )
+            )  # TODO: pass in event?
+            return func(*args, **kwargs)
+
+        return wrapped
+
+    return inner

dvt/jsonschemas/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+JSONSCHEMAS_PATH = os.path.dirname(__file__)