dvt-core 1.11.0b4__py3-none-any.whl

dvt/task/clean.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+from shutil import rmtree
+
+from dvt import deprecations
+from dvt.cli.flags import Flags
+from dvt.config.project import Project
+from dvt.events.types import CheckCleanPath, ConfirmCleanPath, FinishedCleanPaths
+from dvt.task.base import BaseTask, move_to_nearest_project_dir
+
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtRuntimeError
+
+
+class CleanTask(BaseTask):
+    def __init__(self, args: Flags, config: Project):
+        super().__init__(args)
+        self.config = config
+        self.project = config
+
+    def run(self) -> None:
+        """
+        This function takes all the paths in the target file
+        and cleans the project paths that are not protected.
+        """
+        project_dir = move_to_nearest_project_dir(self.args.project_dir)
+
+        potential_clean_paths = set(Path(p).resolve() for p in self.project.clean_targets)
+        source_paths = set(
+            Path(p).resolve() for p in (*self.project.all_source_paths, *self.project.test_paths)
+        )
+        clean_paths = potential_clean_paths.difference(source_paths)
+
+        if potential_clean_paths != clean_paths:
+            raise DbtRuntimeError(
+                f"dbt will not clean the following source paths: {[str(s) for s in source_paths.intersection(potential_clean_paths)]}"
+            )
+
+        paths_outside_project = set(
+            path for path in clean_paths if project_dir not in path.absolute().parents
+        )
+        if paths_outside_project and self.args.clean_project_files_only:
+            raise DbtRuntimeError(
+                f"dbt will not clean the following directories outside the project: {[str(p) for p in paths_outside_project]}"
+            )
+
+        if (
+            "dbt_modules" in self.project.clean_targets
+            and self.config.packages_install_path not in self.config.clean_targets
+        ):
+            deprecations.warn("install-packages-path")
+
+        for path in clean_paths:
+            fire_event(CheckCleanPath(path=str(path)))
+            rmtree(path, True)
+            fire_event(ConfirmCleanPath(path=str(path)))
+
+        fire_event(FinishedCleanPaths())

dvt/task/clone.py

@@ -0,0 +1,162 @@
+import threading
+from typing import AbstractSet, Any, Collection, Iterable, List, Optional, Set, Type
+
+from dvt.artifacts.resources.types import NodeType
+from dvt.artifacts.schemas.run import RunResult, RunStatus
+from dvt.clients.jinja import MacroGenerator
+from dvt.context.providers import generate_runtime_model_context
+from dvt.contracts.graph.manifest import Manifest
+from dvt.graph import ResourceTypeSelector
+from dvt.node_types import REFABLE_NODE_TYPES
+from dvt.task.base import BaseRunner, resource_types_from_args
+from dvt.task.run import _validate_materialization_relations_dict
+from dvt.task.runnable import GraphRunnableMode, GraphRunnableTask
+
+from dbt.adapters.base import BaseAdapter, BaseRelation
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.exceptions import CompilationError, DbtInternalError
+
+
+class CloneRunner(BaseRunner):
+    def before_execute(self) -> None:
+        pass
+
+    def after_execute(self, result) -> None:
+        pass
+
+    def _build_run_model_result(self, model, context):
+        result = context["load_result"]("main")
+        if result:
+            status = RunStatus.Success
+            message = str(result.response)
+        else:
+            status = RunStatus.Success
+            message = "No-op"
+        adapter_response = {}
+        if result and isinstance(result.response, dbtClassMixin):
+            adapter_response = result.response.to_dict(omit_none=True)
+        return RunResult(
+            node=model,
+            status=status,
+            timing=[],
+            thread_id=threading.current_thread().name,
+            execution_time=0,
+            message=message,
+            adapter_response=adapter_response,
+            failures=None,
+            batch_results=None,
+        )
+
+    def compile(self, manifest: Manifest):
+        # no-op
+        return self.node
+
+    def _materialization_relations(self, result: Any, model) -> List[BaseRelation]:
+        if isinstance(result, str):
+            msg = (
+                'The materialization ("{}") did not explicitly return a '
+                "list of relations to add to the cache.".format(str(model.get_materialization()))
+            )
+            raise CompilationError(msg, node=model)
+
+        if isinstance(result, dict):
+            return _validate_materialization_relations_dict(result, model)
+
+        msg = (
+            "Invalid return value from materialization, expected a dict "
+            'with key "relations", got: {}'.format(str(result))
+        )
+        raise CompilationError(msg, node=model)
+
+    def execute(self, model, manifest):
+        context = generate_runtime_model_context(model, self.config, manifest)
+        materialization_macro = manifest.find_materialization_macro_by_name(
+            self.config.project_name, "clone", self.adapter.type()
+        )
+
+        if "config" not in context:
+            raise DbtInternalError(
+                "Invalid materialization context generated, missing config: {}".format(context)
+            )
+
+        context_config = context["config"]
+
+        hook_ctx = self.adapter.pre_model_hook(context_config)
+        try:
+            result = MacroGenerator(materialization_macro, context)()
+        finally:
+            self.adapter.post_model_hook(context_config, hook_ctx)
+
+        for relation in self._materialization_relations(result, model):
+            self.adapter.cache_added(relation.incorporate(dbt_created=True))
+
+        return self._build_run_model_result(model, context)
+
+
+class CloneTask(GraphRunnableTask):
+    def raise_on_first_error(self) -> bool:
+        return False
+
+    def get_run_mode(self) -> GraphRunnableMode:
+        return GraphRunnableMode.Independent
+
+    def _get_deferred_manifest(self) -> Optional[Manifest]:
+        # Unlike other commands, 'clone' always requires a state manifest
+        # Load previous state, regardless of whether --defer flag has been set
+        return self._get_previous_state()
+
+    def get_model_schemas(self, adapter, selected_uids: Iterable[str]) -> Set[BaseRelation]:
+        if self.manifest is None:
+            raise DbtInternalError("manifest was None in get_model_schemas")
+        result: Set[BaseRelation] = set()
+
+        for node in self.manifest.nodes.values():
+            if node.unique_id not in selected_uids:
+                continue
+            if node.is_relational and not node.is_ephemeral:
+                relation = adapter.Relation.create_from(self.config, node)
+                result.add(relation.without_identifier())
+
+                # cache the 'other' schemas too!
+                if node.defer_relation:  # type: ignore
+                    other_relation = adapter.Relation.create_from(
+                        self.config, node.defer_relation  # type: ignore
+                    )
+                    result.add(other_relation.without_identifier())
+
+        return result
+
+    def before_run(self, adapter: BaseAdapter, selected_uids: AbstractSet[str]) -> RunStatus:
+        with adapter.connection_named("master"):
+            self.defer_to_manifest()
+            # only create target schemas, but also cache defer_relation schemas
+            schemas_to_create = super().get_model_schemas(adapter, selected_uids)
+            self.create_schemas(adapter, schemas_to_create)
+            schemas_to_cache = self.get_model_schemas(adapter, selected_uids)
+            self.populate_adapter_cache(adapter, schemas_to_cache)
+            return RunStatus.Success
+
+    @property
+    def resource_types(self) -> List[NodeType]:
+        resource_types: Collection[NodeType] = resource_types_from_args(
+            self.args, set(REFABLE_NODE_TYPES), set(REFABLE_NODE_TYPES)
+        )
+
+        # filter out any non-refable node types
+        resource_types = [rt for rt in resource_types if rt in REFABLE_NODE_TYPES]
+        return list(resource_types)
+
+    def get_node_selector(self) -> ResourceTypeSelector:
+        resource_types = self.resource_types
+
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to get perform node selection")
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=resource_types,
+        )
+
+    def get_runner_type(self, _) -> Optional[Type[BaseRunner]]:
+        return CloneRunner

dvt/task/compile.py

@@ -0,0 +1,151 @@
+import threading
+from typing import Optional, Type
+
+from dvt.artifacts.schemas.run import RunResult, RunStatus
+from dvt.contracts.graph.manifest import Manifest
+from dvt.events.types import CompiledNode, ParseInlineNodeError
+from dvt.flags import get_flags
+from dvt.graph import ResourceTypeSelector
+from dvt.node_types import EXECUTABLE_NODE_TYPES, NodeType
+from dvt.parser.manifest import process_node
+from dvt.parser.sql import SqlBlockParser
+from dvt.task.base import BaseRunner
+from dvt.task.runnable import GraphRunnableTask
+
+from dbt_common.events.base_types import EventLevel
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import Note
+from dbt_common.exceptions import CompilationError
+from dbt_common.exceptions import DbtBaseException as DbtException
+from dbt_common.exceptions import DbtInternalError
+
+
+class CompileRunner(BaseRunner):
+    def before_execute(self) -> None:
+        pass
+
+    def after_execute(self, result) -> None:
+        pass
+
+    def execute(self, compiled_node, manifest):
+        return RunResult(
+            node=compiled_node,
+            status=RunStatus.Success,
+            timing=[],
+            thread_id=threading.current_thread().name,
+            execution_time=0,
+            message=None,
+            adapter_response={},
+            failures=None,
+            batch_results=None,
+        )
+
+    def compile(self, manifest: Manifest):
+        return self.compiler.compile_node(self.node, manifest, {})
+
+
+class CompileTask(GraphRunnableTask):
+    # We add a new inline node to the manifest during initialization
+    # it should be removed before the task is complete
+    _inline_node_id = None
+
+    def raise_on_first_error(self) -> bool:
+        return True
+
+    def get_node_selector(self) -> ResourceTypeSelector:
+        if getattr(self.args, "inline", None):
+            resource_types = [NodeType.SqlOperation]
+        else:
+            resource_types = EXECUTABLE_NODE_TYPES
+
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to get perform node selection")
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=resource_types,
+        )
+
+    def get_runner_type(self, _) -> Optional[Type[BaseRunner]]:
+        return CompileRunner
+
+    def task_end_messages(self, results) -> None:
+        is_inline = bool(getattr(self.args, "inline", None))
+        output_format = getattr(self.args, "output", "text")
+
+        if is_inline:
+            matched_results = [result for result in results if result.node.name == "inline_query"]
+        elif self.selection_arg:
+            matched_results = []
+            for result in results:
+                if result.node.name in self.selection_arg[0]:
+                    matched_results.append(result)
+                else:
+                    fire_event(
+                        Note(msg=f"Excluded node '{result.node.name}' from results"),
+                        EventLevel.DEBUG,
+                    )
+        # No selector passed, compiling all nodes
+        else:
+            matched_results = []
+
+        for result in matched_results:
+            fire_event(
+                CompiledNode(
+                    node_name=result.node.name,
+                    compiled=result.node.compiled_code,
+                    is_inline=is_inline,
+                    output_format=output_format,
+                    unique_id=result.node.unique_id,
+                    quiet=get_flags().QUIET,
+                )
+            )
+
+    def _runtime_initialize(self):
+        if getattr(self.args, "inline", None):
+            try:
+                block_parser = SqlBlockParser(
+                    project=self.config, manifest=self.manifest, root_project=self.config
+                )
+                sql_node = block_parser.parse_remote(self.args.inline, "inline_query")
+                process_node(self.config, self.manifest, sql_node)
+                # Special hack to remove disabled, if it's there. This would only happen
+                # if all models are disabled in dbt_project
+                if sql_node.config.enabled is False:
+                    sql_node.config.enabled = True
+                    self.manifest.disabled.pop(sql_node.unique_id)
+                    self.manifest.nodes[sql_node.unique_id] = sql_node
+                # keep track of the node added to the manifest
+                self._inline_node_id = sql_node.unique_id
+            except CompilationError as exc:
+                fire_event(
+                    ParseInlineNodeError(
+                        exc=str(exc.msg),
+                        node_info={
+                            "node_path": "sql/inline_query",
+                            "node_name": "inline_query",
+                            "unique_id": "sqloperation.test.inline_query",
+                            "node_status": "failed",
+                        },
+                    )
+                )
+                raise DbtException("Error parsing inline query")
+        super()._runtime_initialize()
+
+    def after_run(self, adapter, results) -> None:
+        # remove inline node from manifest
+        if self._inline_node_id:
+            self.manifest.nodes.pop(self._inline_node_id)
+            self._inline_node_id = None
+        super().after_run(adapter, results)
+
+    def _handle_result(self, result) -> None:
+        super()._handle_result(result)
+
+        if (
+            result.node.is_ephemeral_model
+            and type(self) is CompileTask
+            and (self.args.select or getattr(self.args, "inline", None))
+        ):
+            self.node_results.append(result)