dvt-core 0.52.2__cp310-cp310-macosx_10_9_x86_64.whl

dbt/task/build.py

@@ -0,0 +1,197 @@
+from typing import Dict, List, Optional, Set, Type
+
+from dbt.artifacts.schemas.results import NodeStatus
+from dbt.artifacts.schemas.run import RunResult
+from dbt.cli.flags import Flags
+from dbt.config.runtime import RuntimeConfig
+from dbt.contracts.graph.manifest import Manifest
+from dbt.exceptions import DbtInternalError
+from dbt.graph import Graph, GraphQueue, ResourceTypeSelector
+from dbt.node_types import NodeType
+from dbt.runners import ExposureRunner as exposure_runner
+from dbt.runners import SavedQueryRunner as saved_query_runner
+from dbt.task.base import BaseRunner, resource_types_from_args
+from dbt.task.run import MicrobatchModelRunner
+
+from .function import FunctionRunner as function_runner
+from .run import ModelRunner as run_model_runner
+from .run import RunTask
+from .seed import SeedRunner as seed_runner
+from .snapshot import SnapshotRunner as snapshot_model_runner
+from .test import TestRunner as test_runner
+
+
+class BuildTask(RunTask):
+    """The Build task processes all assets of a given process and attempts to
+    'build' them in an opinionated fashion.  Every resource type outlined in
+    RUNNER_MAP will be processed by the mapped runners class.
+
+    I.E. a resource of type Model is handled by the ModelRunner which is
+    imported as run_model_runner."""
+
+    MARK_DEPENDENT_ERRORS_STATUSES = [
+        NodeStatus.Error,
+        NodeStatus.Fail,
+        NodeStatus.Skipped,
+        NodeStatus.PartialSuccess,
+    ]
+
+    RUNNER_MAP = {
+        NodeType.Model: run_model_runner,
+        NodeType.Snapshot: snapshot_model_runner,
+        NodeType.Seed: seed_runner,
+        NodeType.Test: test_runner,
+        NodeType.Unit: test_runner,
+        NodeType.SavedQuery: saved_query_runner,
+        NodeType.Exposure: exposure_runner,
+        NodeType.Function: function_runner,
+    }
+    ALL_RESOURCE_VALUES = frozenset({x for x in RUNNER_MAP.keys()})
+
+    def __init__(self, args: Flags, config: RuntimeConfig, manifest: Manifest) -> None:
+        super().__init__(args, config, manifest)
+        self.selected_unit_tests: Set = set()
+        self.model_to_unit_test_map: Dict[str, List] = {}
+
+    def resource_types(self, no_unit_tests: bool = False) -> List[NodeType]:
+        resource_types = resource_types_from_args(
+            self.args, set(self.ALL_RESOURCE_VALUES), set(self.ALL_RESOURCE_VALUES)
+        )
+
+        # First we get selected_nodes including unit tests, then without,
+        # and do a set difference.
+        if no_unit_tests is True and NodeType.Unit in resource_types:
+            resource_types.remove(NodeType.Unit)
+        return list(resource_types)
+
+    # overrides get_graph_queue in runnable.py
+    def get_graph_queue(self) -> GraphQueue:
+        # Following uses self.selection_arg and self.exclusion_arg
+        spec = self.get_selection_spec()
+
+        # selector including unit tests
+        full_selector = self.get_node_selector(no_unit_tests=False)
+        # selected node unique_ids with unit_tests
+        full_selected_nodes = full_selector.get_selected(spec)
+
+        # This selector removes the unit_tests from the selector
+        selector_wo_unit_tests = self.get_node_selector(no_unit_tests=True)
+        # selected node unique_ids without unit_tests
+        selected_nodes_wo_unit_tests = selector_wo_unit_tests.get_selected(spec)
+
+        # Get the difference in the sets of nodes with and without unit tests and
+        # save it
+        selected_unit_tests = full_selected_nodes - selected_nodes_wo_unit_tests
+        self.selected_unit_tests = selected_unit_tests
+        self.build_model_to_unit_test_map(selected_unit_tests)
+
+        # get_graph_queue in the selector will remove NodeTypes not specified
+        # in the node_selector (filter_selection).
+        return selector_wo_unit_tests.get_graph_queue(spec)
+
+    # overrides handle_job_queue in runnable.py
+    def handle_job_queue(self, pool, callback):
+        if self.run_count == 0:
+            self.num_nodes = self.num_nodes + len(self.selected_unit_tests)
+        node = self.job_queue.get()
+        if (
+            node.resource_type == NodeType.Model
+            and self.model_to_unit_test_map
+            and node.unique_id in self.model_to_unit_test_map
+        ):
+            self.handle_model_with_unit_tests_node(node, pool, callback)
+
+        else:
+            self.handle_job_queue_node(node, pool, callback)
+
+    def handle_model_with_unit_tests_node(self, node, pool, callback):
+        self._raise_set_error()
+        args = [node, pool]
+        if self.config.args.single_threaded:
+            callback(self.call_model_and_unit_tests_runner(*args))
+        else:
+            pool.apply_async(self.call_model_and_unit_tests_runner, args=args, callback=callback)
+
+    def call_model_and_unit_tests_runner(self, node, pool) -> RunResult:
+        assert self.manifest
+        for unit_test_unique_id in self.model_to_unit_test_map[node.unique_id]:
+            unit_test_node = self.manifest.unit_tests[unit_test_unique_id]
+            unit_test_runner = self.get_runner(unit_test_node)
+            # If the model is marked skip, also skip the unit tests
+            if node.unique_id in self._skipped_children:
+                # cause is only for ephemeral nodes
+                unit_test_runner.do_skip(cause=None)
+            result = self.call_runner(unit_test_runner)
+            self._handle_result(result)
+            if result.status in self.MARK_DEPENDENT_ERRORS_STATUSES:
+                # The _skipped_children dictionary can contain a run_result for ephemeral nodes,
+                # but that should never be the case here.
+                self._skipped_children[node.unique_id] = None
+        runner = self.get_runner(node)
+        if runner.node.unique_id in self._skipped_children:
+            cause = self._skipped_children.pop(runner.node.unique_id)
+            runner.do_skip(cause=cause)
+
+        if isinstance(runner, MicrobatchModelRunner):
+            runner.set_parent_task(self)
+            runner.set_pool(pool)
+
+        return self.call_runner(runner)
+
+    # handle non-model-plus-unit-tests nodes
+    def handle_job_queue_node(self, node, pool, callback):
+        self._raise_set_error()
+        runner = self.get_runner(node)
+        # we finally know what we're running! Make sure we haven't decided
+        # to skip it due to upstream failures
+        if runner.node.unique_id in self._skipped_children:
+            cause = self._skipped_children.pop(runner.node.unique_id)
+            runner.do_skip(cause=cause)
+
+        if isinstance(runner, MicrobatchModelRunner):
+            runner.set_parent_task(self)
+            runner.set_pool(pool)
+
+        args = [runner]
+        self._submit(pool, args, callback)
+
+    # Make a map of model unique_ids to selected unit test unique_ids,
+    # for processing before the model.
+    def build_model_to_unit_test_map(self, selected_unit_tests):
+        dct = {}
+        for unit_test_unique_id in selected_unit_tests:
+            unit_test = self.manifest.unit_tests[unit_test_unique_id]
+            model_unique_id = unit_test.depends_on.nodes[0]
+            if model_unique_id not in dct:
+                dct[model_unique_id] = []
+            dct[model_unique_id].append(unit_test.unique_id)
+        self.model_to_unit_test_map = dct
+
+    # We return two different kinds of selectors, one with unit tests and one without
+    def get_node_selector(self, no_unit_tests=False) -> ResourceTypeSelector:
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to get node selection")
+
+        resource_types = self.resource_types(no_unit_tests)
+
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=resource_types,
+        )
+
+    def get_runner_type(self, node) -> Optional[Type[BaseRunner]]:
+        if (
+            node.resource_type == NodeType.Model
+            and super().get_runner_type(node) == MicrobatchModelRunner
+        ):
+            return MicrobatchModelRunner
+
+        return self.RUNNER_MAP.get(node.resource_type)
+
+    # Special build compile_manifest method to pass add_test_edges to the compiler
+    def compile_manifest(self) -> None:
+        if self.manifest is None:
+            raise DbtInternalError("compile_manifest called before manifest was loaded")
+        self.graph: Graph = self.compiler.compile(self.manifest, add_test_edges=True)

dbt/task/clean.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+from shutil import rmtree
+
+from dbt import deprecations
+from dbt.cli.flags import Flags
+from dbt.config.project import Project
+from dbt.events.types import CheckCleanPath, ConfirmCleanPath, FinishedCleanPaths
+from dbt.task.base import BaseTask, move_to_nearest_project_dir
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtRuntimeError
+
+
+class CleanTask(BaseTask):
+    def __init__(self, args: Flags, config: Project):
+        super().__init__(args)
+        self.config = config
+        self.project = config
+
+    def run(self) -> None:
+        """
+        This function takes all the paths in the target file
+        and cleans the project paths that are not protected.
+        """
+        project_dir = move_to_nearest_project_dir(self.args.project_dir)
+
+        potential_clean_paths = set(Path(p).resolve() for p in self.project.clean_targets)
+        source_paths = set(
+            Path(p).resolve() for p in (*self.project.all_source_paths, *self.project.test_paths)
+        )
+        clean_paths = potential_clean_paths.difference(source_paths)
+
+        if potential_clean_paths != clean_paths:
+            raise DbtRuntimeError(
+                f"dbt will not clean the following source paths: {[str(s) for s in source_paths.intersection(potential_clean_paths)]}"
+            )
+
+        paths_outside_project = set(
+            path for path in clean_paths if project_dir not in path.absolute().parents
+        )
+        if paths_outside_project and self.args.clean_project_files_only:
+            raise DbtRuntimeError(
+                f"dbt will not clean the following directories outside the project: {[str(p) for p in paths_outside_project]}"
+            )
+
+        if (
+            "dbt_modules" in self.project.clean_targets
+            and self.config.packages_install_path not in self.config.clean_targets
+        ):
+            deprecations.warn("install-packages-path")
+
+        for path in clean_paths:
+            fire_event(CheckCleanPath(path=str(path)))
+            rmtree(path, True)
+            fire_event(ConfirmCleanPath(path=str(path)))
+
+        fire_event(FinishedCleanPaths())

dbt/task/clone.py

@@ -0,0 +1,161 @@
+import threading
+from typing import AbstractSet, Any, Collection, Iterable, List, Optional, Set, Type
+
+from dbt.adapters.base import BaseAdapter, BaseRelation
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.schemas.run import RunResult, RunStatus
+from dbt.clients.jinja import MacroGenerator
+from dbt.context.providers import generate_runtime_model_context
+from dbt.contracts.graph.manifest import Manifest
+from dbt.graph import ResourceTypeSelector
+from dbt.node_types import REFABLE_NODE_TYPES
+from dbt.task.base import BaseRunner, resource_types_from_args
+from dbt.task.run import _validate_materialization_relations_dict
+from dbt.task.runnable import GraphRunnableMode, GraphRunnableTask
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.exceptions import CompilationError, DbtInternalError
+
+
+class CloneRunner(BaseRunner):
+    def before_execute(self) -> None:
+        pass
+
+    def after_execute(self, result) -> None:
+        pass
+
+    def _build_run_model_result(self, model, context):
+        result = context["load_result"]("main")
+        if result:
+            status = RunStatus.Success
+            message = str(result.response)
+        else:
+            status = RunStatus.Success
+            message = "No-op"
+        adapter_response = {}
+        if result and isinstance(result.response, dbtClassMixin):
+            adapter_response = result.response.to_dict(omit_none=True)
+        return RunResult(
+            node=model,
+            status=status,
+            timing=[],
+            thread_id=threading.current_thread().name,
+            execution_time=0,
+            message=message,
+            adapter_response=adapter_response,
+            failures=None,
+            batch_results=None,
+        )
+
+    def compile(self, manifest: Manifest):
+        # no-op
+        return self.node
+
+    def _materialization_relations(self, result: Any, model) -> List[BaseRelation]:
+        if isinstance(result, str):
+            msg = (
+                'The materialization ("{}") did not explicitly return a '
+                "list of relations to add to the cache.".format(str(model.get_materialization()))
+            )
+            raise CompilationError(msg, node=model)
+
+        if isinstance(result, dict):
+            return _validate_materialization_relations_dict(result, model)
+
+        msg = (
+            "Invalid return value from materialization, expected a dict "
+            'with key "relations", got: {}'.format(str(result))
+        )
+        raise CompilationError(msg, node=model)
+
+    def execute(self, model, manifest):
+        context = generate_runtime_model_context(model, self.config, manifest)
+        materialization_macro = manifest.find_materialization_macro_by_name(
+            self.config.project_name, "clone", self.adapter.type()
+        )
+
+        if "config" not in context:
+            raise DbtInternalError(
+                "Invalid materialization context generated, missing config: {}".format(context)
+            )
+
+        context_config = context["config"]
+
+        hook_ctx = self.adapter.pre_model_hook(context_config)
+        try:
+            result = MacroGenerator(materialization_macro, context)()
+        finally:
+            self.adapter.post_model_hook(context_config, hook_ctx)
+
+        for relation in self._materialization_relations(result, model):
+            self.adapter.cache_added(relation.incorporate(dbt_created=True))
+
+        return self._build_run_model_result(model, context)
+
+
+class CloneTask(GraphRunnableTask):
+    def raise_on_first_error(self) -> bool:
+        return False
+
+    def get_run_mode(self) -> GraphRunnableMode:
+        return GraphRunnableMode.Independent
+
+    def _get_deferred_manifest(self) -> Optional[Manifest]:
+        # Unlike other commands, 'clone' always requires a state manifest
+        # Load previous state, regardless of whether --defer flag has been set
+        return self._get_previous_state()
+
+    def get_model_schemas(self, adapter, selected_uids: Iterable[str]) -> Set[BaseRelation]:
+        if self.manifest is None:
+            raise DbtInternalError("manifest was None in get_model_schemas")
+        result: Set[BaseRelation] = set()
+
+        for node in self.manifest.nodes.values():
+            if node.unique_id not in selected_uids:
+                continue
+            if node.is_relational and not node.is_ephemeral:
+                relation = adapter.Relation.create_from(self.config, node)
+                result.add(relation.without_identifier())
+
+                # cache the 'other' schemas too!
+                if node.defer_relation:  # type: ignore
+                    other_relation = adapter.Relation.create_from(
+                        self.config, node.defer_relation  # type: ignore
+                    )
+                    result.add(other_relation.without_identifier())
+
+        return result
+
+    def before_run(self, adapter: BaseAdapter, selected_uids: AbstractSet[str]) -> RunStatus:
+        with adapter.connection_named("master"):
+            self.defer_to_manifest()
+            # only create target schemas, but also cache defer_relation schemas
+            schemas_to_create = super().get_model_schemas(adapter, selected_uids)
+            self.create_schemas(adapter, schemas_to_create)
+            schemas_to_cache = self.get_model_schemas(adapter, selected_uids)
+            self.populate_adapter_cache(adapter, schemas_to_cache)
+            return RunStatus.Success
+
+    @property
+    def resource_types(self) -> List[NodeType]:
+        resource_types: Collection[NodeType] = resource_types_from_args(
+            self.args, set(REFABLE_NODE_TYPES), set(REFABLE_NODE_TYPES)
+        )
+
+        # filter out any non-refable node types
+        resource_types = [rt for rt in resource_types if rt in REFABLE_NODE_TYPES]
+        return list(resource_types)
+
+    def get_node_selector(self) -> ResourceTypeSelector:
+        resource_types = self.resource_types
+
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to get perform node selection")
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=resource_types,
+        )
+
+    def get_runner_type(self, _) -> Optional[Type[BaseRunner]]:
+        return CloneRunner

dbt/task/compile.py

@@ -0,0 +1,150 @@
+import threading
+from typing import Optional, Type
+
+from dbt.artifacts.schemas.run import RunResult, RunStatus
+from dbt.contracts.graph.manifest import Manifest
+from dbt.events.types import CompiledNode, ParseInlineNodeError
+from dbt.flags import get_flags
+from dbt.graph import ResourceTypeSelector
+from dbt.node_types import EXECUTABLE_NODE_TYPES, NodeType
+from dbt.parser.manifest import process_node
+from dbt.parser.sql import SqlBlockParser
+from dbt.task.base import BaseRunner
+from dbt.task.runnable import GraphRunnableTask
+from dbt_common.events.base_types import EventLevel
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import Note
+from dbt_common.exceptions import CompilationError
+from dbt_common.exceptions import DbtBaseException as DbtException
+from dbt_common.exceptions import DbtInternalError
+
+
+class CompileRunner(BaseRunner):
+    def before_execute(self) -> None:
+        pass
+
+    def after_execute(self, result) -> None:
+        pass
+
+    def execute(self, compiled_node, manifest):
+        return RunResult(
+            node=compiled_node,
+            status=RunStatus.Success,
+            timing=[],
+            thread_id=threading.current_thread().name,
+            execution_time=0,
+            message=None,
+            adapter_response={},
+            failures=None,
+            batch_results=None,
+        )
+
+    def compile(self, manifest: Manifest):
+        return self.compiler.compile_node(self.node, manifest, {})
+
+
+class CompileTask(GraphRunnableTask):
+    # We add a new inline node to the manifest during initialization
+    # it should be removed before the task is complete
+    _inline_node_id = None
+
+    def raise_on_first_error(self) -> bool:
+        return True
+
+    def get_node_selector(self) -> ResourceTypeSelector:
+        if getattr(self.args, "inline", None):
+            resource_types = [NodeType.SqlOperation]
+        else:
+            resource_types = EXECUTABLE_NODE_TYPES
+
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to get perform node selection")
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=resource_types,
+        )
+
+    def get_runner_type(self, _) -> Optional[Type[BaseRunner]]:
+        return CompileRunner
+
+    def task_end_messages(self, results) -> None:
+        is_inline = bool(getattr(self.args, "inline", None))
+        output_format = getattr(self.args, "output", "text")
+
+        if is_inline:
+            matched_results = [result for result in results if result.node.name == "inline_query"]
+        elif self.selection_arg:
+            matched_results = []
+            for result in results:
+                if result.node.name in self.selection_arg[0]:
+                    matched_results.append(result)
+                else:
+                    fire_event(
+                        Note(msg=f"Excluded node '{result.node.name}' from results"),
+                        EventLevel.DEBUG,
+                    )
+        # No selector passed, compiling all nodes
+        else:
+            matched_results = []
+
+        for result in matched_results:
+            fire_event(
+                CompiledNode(
+                    node_name=result.node.name,
+                    compiled=result.node.compiled_code,
+                    is_inline=is_inline,
+                    output_format=output_format,
+                    unique_id=result.node.unique_id,
+                    quiet=get_flags().QUIET,
+                )
+            )
+
+    def _runtime_initialize(self):
+        if getattr(self.args, "inline", None):
+            try:
+                block_parser = SqlBlockParser(
+                    project=self.config, manifest=self.manifest, root_project=self.config
+                )
+                sql_node = block_parser.parse_remote(self.args.inline, "inline_query")
+                process_node(self.config, self.manifest, sql_node)
+                # Special hack to remove disabled, if it's there. This would only happen
+                # if all models are disabled in dbt_project
+                if sql_node.config.enabled is False:
+                    sql_node.config.enabled = True
+                    self.manifest.disabled.pop(sql_node.unique_id)
+                    self.manifest.nodes[sql_node.unique_id] = sql_node
+                # keep track of the node added to the manifest
+                self._inline_node_id = sql_node.unique_id
+            except CompilationError as exc:
+                fire_event(
+                    ParseInlineNodeError(
+                        exc=str(exc.msg),
+                        node_info={
+                            "node_path": "sql/inline_query",
+                            "node_name": "inline_query",
+                            "unique_id": "sqloperation.test.inline_query",
+                            "node_status": "failed",
+                        },
+                    )
+                )
+                raise DbtException("Error parsing inline query")
+        super()._runtime_initialize()
+
+    def after_run(self, adapter, results) -> None:
+        # remove inline node from manifest
+        if self._inline_node_id:
+            self.manifest.nodes.pop(self._inline_node_id)
+            self._inline_node_id = None
+        super().after_run(adapter, results)
+
+    def _handle_result(self, result) -> None:
+        super()._handle_result(result)
+
+        if (
+            result.node.is_ephemeral_model
+            and type(self) is CompileTask
+            and (self.args.select or getattr(self.args, "inline", None))
+        ):
+            self.node_results.append(result)