dvt-core 1.11.0b4__py3-none-any.whl

dvt/task/docs/serve.py

@@ -0,0 +1,28 @@
+import os
+import shutil
+import socketserver
+import webbrowser
+from http.server import SimpleHTTPRequestHandler
+
+import click
+from dvt.task.base import ConfiguredTask
+from dvt.task.docs import DOCS_INDEX_FILE_PATH
+
+
+class ServeTask(ConfiguredTask):
+    def run(self):
+        os.chdir(self.config.project_target_path)
+        shutil.copyfile(DOCS_INDEX_FILE_PATH, "index.html")
+
+        port = self.args.port
+        host = self.args.host
+
+        if self.args.browser:
+            webbrowser.open_new_tab(f"http://localhost:{port}")
+
+        with socketserver.TCPServer((host, port), SimpleHTTPRequestHandler) as httpd:
+            click.echo(f"Serving docs at {port}")
+            click.echo(f"To access from your browser, navigate to: http://localhost:{port}")
+            click.echo("\n\n")
+            click.echo("Press Ctrl+C to exit.")
+            httpd.serve_forever()

dvt/task/freshness.py

@@ -0,0 +1,323 @@
+import os
+import threading
+import time
+from typing import AbstractSet, Dict, List, Optional, Type
+
+from dvt import deprecations
+from dvt.artifacts.schemas.freshness import (
+    FreshnessResult,
+    FreshnessStatus,
+    PartialSourceFreshnessResult,
+    SourceFreshnessResult,
+)
+from dvt.clients import jinja
+from dvt.constants import SOURCE_RESULT_FILE_NAME
+from dvt.context.providers import RuntimeProvider, SourceContext
+from dvt.contracts.graph.manifest import Manifest
+from dvt.contracts.graph.nodes import HookNode, SourceDefinition
+from dvt.contracts.results import RunStatus
+from dvt.events.types import FreshnessCheckComplete, LogFreshnessResult, LogStartLine
+from dvt.graph import ResourceTypeSelector
+from dvt.node_types import NodeType, RunHookType
+
+from dbt.adapters.base import BaseAdapter
+from dbt.adapters.base.impl import FreshnessResponse
+from dbt.adapters.base.relation import BaseRelation
+from dbt.adapters.capability import Capability
+from dbt.adapters.contracts.connection import AdapterResponse
+from dbt_common.events.base_types import EventLevel
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import Note
+from dbt_common.exceptions import DbtInternalError, DbtRuntimeError
+
+from .base import BaseRunner
+from .printer import print_run_result_error
+from .run import RunTask
+
+
+class FreshnessRunner(BaseRunner):
+    def __init__(self, config, adapter, node, node_index, num_nodes) -> None:
+        super().__init__(config, adapter, node, node_index, num_nodes)
+        self._metadata_freshness_cache: Dict[BaseRelation, FreshnessResult] = {}
+
+    def set_metadata_freshness_cache(
+        self, metadata_freshness_cache: Dict[BaseRelation, FreshnessResult]
+    ) -> None:
+        self._metadata_freshness_cache = metadata_freshness_cache
+
+    def on_skip(self):
+        raise DbtRuntimeError("Freshness: nodes cannot be skipped!")
+
+    def before_execute(self) -> None:
+        description = "freshness of {0.source_name}.{0.name}".format(self.node)
+        fire_event(
+            LogStartLine(
+                description=description,
+                index=self.node_index,
+                total=self.num_nodes,
+                node_info=self.node.node_info,
+            )
+        )
+
+    def after_execute(self, result) -> None:
+        if hasattr(result, "node"):
+            source_name = result.node.source_name
+            table_name = result.node.name
+        else:
+            source_name = result.source_name
+            table_name = result.table_name
+        level = LogFreshnessResult.status_to_level(str(result.status))
+        fire_event(
+            LogFreshnessResult(
+                status=result.status,
+                source_name=source_name,
+                table_name=table_name,
+                index=self.node_index,
+                total=self.num_nodes,
+                execution_time=result.execution_time,
+                node_info=self.node.node_info,
+            ),
+            level=level,
+        )
+
+    def error_result(self, node, message, start_time, timing_info):
+        return self._build_run_result(
+            node=node,
+            start_time=start_time,
+            status=FreshnessStatus.RuntimeErr,
+            timing_info=timing_info,
+            message=message,
+        )
+
+    def _build_run_result(self, node, start_time, status, timing_info, message):
+        execution_time = time.time() - start_time
+        thread_id = threading.current_thread().name
+        return PartialSourceFreshnessResult(
+            status=status,
+            thread_id=thread_id,
+            execution_time=execution_time,
+            timing=timing_info,
+            message=message,
+            node=node,
+            adapter_response={},
+            failures=None,
+        )
+
+    def from_run_result(self, result, start_time, timing_info):
+        result.execution_time = time.time() - start_time
+        result.timing.extend(timing_info)
+        return result
+
+    def execute(self, compiled_node, manifest):
+        relation = self.adapter.Relation.create_from(self.config, compiled_node)
+        # given a Source, calculate its freshness.
+        with self.adapter.connection_named(compiled_node.unique_id, compiled_node):
+            self.adapter.clear_transaction()
+            adapter_response: Optional[AdapterResponse] = None
+            freshness: Optional[FreshnessResponse] = None
+
+            if compiled_node.loaded_at_query is not None:
+                # within the context user can have access to `this`, `source_node`(`model` will point to the same thing),  etc
+                compiled_code = jinja.get_rendered(
+                    compiled_node.loaded_at_query,
+                    SourceContext(
+                        compiled_node, self.config, manifest, RuntimeProvider(), None
+                    ).to_dict(),
+                    compiled_node,
+                )
+                adapter_response, freshness = self.adapter.calculate_freshness_from_custom_sql(
+                    relation,
+                    compiled_code,
+                    macro_resolver=manifest,
+                )
+                status = compiled_node.freshness.status(freshness["age"])
+            elif compiled_node.loaded_at_field is not None:
+                adapter_response, freshness = self.adapter.calculate_freshness(
+                    relation,
+                    compiled_node.loaded_at_field,
+                    compiled_node.freshness.filter,
+                    macro_resolver=manifest,
+                )
+
+                status = compiled_node.freshness.status(freshness["age"])
+            elif self.adapter.supports(Capability.TableLastModifiedMetadata):
+                if compiled_node.freshness.filter is not None:
+                    fire_event(
+                        Note(
+                            msg=f"A filter cannot be applied to a metadata freshness check on source '{compiled_node.name}'."
+                        ),
+                        EventLevel.WARN,
+                    )
+
+                metadata_source = self.adapter.Relation.create_from(self.config, compiled_node)
+                if metadata_source in self._metadata_freshness_cache:
+                    freshness = self._metadata_freshness_cache[metadata_source]
+                else:
+                    adapter_response, freshness = self.adapter.calculate_freshness_from_metadata(
+                        relation,
+                        macro_resolver=manifest,
+                    )
+
+                status = compiled_node.freshness.status(freshness["age"])
+            else:
+                raise DbtRuntimeError(
+                    f"Could not compute freshness for source {compiled_node.name}: no 'loaded_at_field' provided and {self.adapter.type()} adapter does not support metadata-based freshness checks."
+                )
+        # adapter_response was not returned in previous versions, so this will be None
+        # we cannot call to_dict() on NoneType
+        if adapter_response:
+            adapter_response = adapter_response.to_dict(omit_none=True)
+
+        return SourceFreshnessResult(
+            node=compiled_node,
+            status=status,
+            thread_id=threading.current_thread().name,
+            timing=[],
+            execution_time=0,
+            message=None,
+            adapter_response=adapter_response or {},
+            failures=None,
+            **freshness,
+        )
+
+    def compile(self, manifest: Manifest):
+        if self.node.resource_type != NodeType.Source:
+            # should be unreachable...
+            raise DbtRuntimeError("freshness runner: got a non-Source")
+        # we don't do anything interesting when we compile a source node
+        return self.node
+
+
+class FreshnessSelector(ResourceTypeSelector):
+    def node_is_match(self, node):
+        if not super().node_is_match(node):
+            return False
+        if not isinstance(node, SourceDefinition):
+            return False
+        return node.has_freshness
+
+
+class FreshnessTask(RunTask):
+    def __init__(self, args, config, manifest) -> None:
+        super().__init__(args, config, manifest)
+
+        if self.args.output:
+            deprecations.warn(
+                "custom-output-path-in-source-freshness-deprecation", path=str(self.args.output)
+            )
+
+        self._metadata_freshness_cache: Dict[BaseRelation, FreshnessResult] = {}
+
+    def result_path(self) -> str:
+        if self.args.output:
+            return os.path.realpath(self.args.output)
+        else:
+            return os.path.join(self.config.project_target_path, SOURCE_RESULT_FILE_NAME)
+
+    def raise_on_first_error(self) -> bool:
+        return False
+
+    def get_node_selector(self):
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to get perform node selection")
+        return FreshnessSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=[NodeType.Source],
+        )
+
+    def before_run(self, adapter: BaseAdapter, selected_uids: AbstractSet[str]) -> RunStatus:
+        populate_metadata_freshness_cache_status = RunStatus.Success
+
+        before_run_status = super().before_run(adapter, selected_uids)
+
+        if before_run_status == RunStatus.Success and adapter.supports(
+            Capability.TableLastModifiedMetadataBatch
+        ):
+            populate_metadata_freshness_cache_status = self.populate_metadata_freshness_cache(
+                adapter, selected_uids
+            )
+
+        if (
+            before_run_status == RunStatus.Success
+            and populate_metadata_freshness_cache_status == RunStatus.Success
+        ):
+            return RunStatus.Success
+        else:
+            return RunStatus.Error
+
+    def get_runner(self, node) -> BaseRunner:
+        freshness_runner = super().get_runner(node)
+        assert isinstance(freshness_runner, FreshnessRunner)
+        freshness_runner.set_metadata_freshness_cache(self._metadata_freshness_cache)
+        return freshness_runner
+
+    def get_runner_type(self, _) -> Optional[Type[BaseRunner]]:
+        return FreshnessRunner
+
+    def get_result(self, results, elapsed_time, generated_at):
+        return FreshnessResult.from_node_results(
+            elapsed_time=elapsed_time, generated_at=generated_at, results=results
+        )
+
+    def task_end_messages(self, results) -> None:
+        for result in results:
+            if result.status in (
+                FreshnessStatus.Error,
+                FreshnessStatus.RuntimeErr,
+                RunStatus.Error,
+            ):
+                print_run_result_error(result)
+
+        fire_event(FreshnessCheckComplete())
+
+    def get_hooks_by_type(self, hook_type: RunHookType) -> List[HookNode]:
+        hooks = super().get_hooks_by_type(hook_type)
+        if self.args.source_freshness_run_project_hooks:
+            return hooks
+        else:
+            if hooks:
+                deprecations.warn("source-freshness-project-hooks")
+            return []
+
+    def populate_metadata_freshness_cache(
+        self, adapter, selected_uids: AbstractSet[str]
+    ) -> RunStatus:
+        if self.manifest is None:
+            raise DbtInternalError("Manifest must be set to populate metadata freshness cache")
+
+        batch_metadata_sources: List[BaseRelation] = []
+        for selected_source_uid in list(selected_uids):
+            source = self.manifest.sources.get(selected_source_uid)
+            if source and source.loaded_at_field is None:
+                metadata_source = adapter.Relation.create_from(self.config, source)
+                batch_metadata_sources.append(metadata_source)
+
+        fire_event(
+            Note(
+                msg=f"Pulling freshness from warehouse metadata tables for {len(batch_metadata_sources)} sources"
+            ),
+            EventLevel.INFO,
+        )
+
+        try:
+            _, metadata_freshness_results = adapter.calculate_freshness_from_metadata_batch(
+                batch_metadata_sources
+            )
+            self._metadata_freshness_cache.update(metadata_freshness_results)
+            return RunStatus.Success
+        except Exception as e:
+            # This error handling is intentionally very coarse.
+            # If anything goes wrong during batch metadata calculation, we can safely
+            # leave _metadata_freshness_cache unpopulated.
+            # Downstream, this will be gracefully handled as a cache miss and non-batch
+            # metadata-based freshness will still be performed on a source-by-source basis.
+            fire_event(
+                Note(msg=f"Metadata freshness could not be computed in batch: {e}"),
+                EventLevel.WARN,
+            )
+            return RunStatus.Error
+
+    def get_freshness_metadata_cache(self) -> Dict[BaseRelation, FreshnessResult]:
+        return self._metadata_freshness_cache

dvt/task/function.py

@@ -0,0 +1,122 @@
+import threading
+from typing import Any, Dict
+
+from dvt.artifacts.schemas.results import NodeStatus, RunStatus
+from dvt.artifacts.schemas.run import RunResult
+from dvt.clients.jinja import MacroGenerator
+from dvt.context.providers import generate_runtime_function_context
+from dvt.contracts.graph.manifest import Manifest
+from dvt.contracts.graph.nodes import FunctionNode
+from dvt.events.types import LogFunctionResult, LogStartLine
+from dvt.task import group_lookup
+from dvt.task.compile import CompileRunner
+
+from dbt.adapters.exceptions import MissingMaterializationError
+from dbt_common.clients.jinja import MacroProtocol
+from dbt_common.events.base_types import EventLevel
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtValidationError
+
+
+class FunctionRunner(CompileRunner):
+
+    def __init__(self, config, adapter, node, node_index: int, num_nodes: int) -> None:
+        super().__init__(config, adapter, node, node_index, num_nodes)
+
+        # doing this gives us type hints for the node :D
+        assert isinstance(node, FunctionNode)
+        self.node = node
+
+    def describe_node(self) -> str:
+        return f"function {self.node.name}"  # TODO: add more info, similar to SeedRunner.describe_node
+
+    def before_execute(self) -> None:
+        fire_event(
+            LogStartLine(
+                description=self.describe_node(),
+                index=self.node_index,
+                total=self.num_nodes,
+                node_info=self.node.node_info,
+            )
+        )
+
+    def _get_materialization_macro(
+        self, compiled_node: FunctionNode, manifest: Manifest
+    ) -> MacroProtocol:
+        materialization_macro = manifest.find_materialization_macro_by_name(
+            self.config.project_name, compiled_node.get_materialization(), self.adapter.type()
+        )
+        if materialization_macro is None:
+            raise MissingMaterializationError(
+                materialization=compiled_node.get_materialization(),
+                adapter_type=self.adapter.type(),
+            )
+
+        return materialization_macro
+
+    def _check_lang_supported(
+        self, compiled_node: FunctionNode, materialization_macro: MacroProtocol
+    ) -> None:
+        # TODO: This function and its typing is a bit wonky, we should fix it
+        # Specifically, a MacroProtocol doesn't have a supported_languags attribute, but a macro does. We're acting
+        # like the materialization_macro might not have a supported_languages attribute, but we access it in an unguarded manner.
+        # So are we guaranteed to always have a Macro here? (because a Macro always has a supported_languages attribute)
+        # This logic is a copy of of the logic in the run.py file, so the same logical conundrum applies there. Also perhaps
+        # we can refactor to having one definition, and maybe a logically consistent one...
+        mat_has_supported_langs = hasattr(materialization_macro, "supported_languages")
+        function_lang_supported = compiled_node.language in materialization_macro.supported_languages  # type: ignore
+        if mat_has_supported_langs and not function_lang_supported:
+            str_langs = [str(lang) for lang in materialization_macro.supported_languages]  # type: ignore
+            raise DbtValidationError(
+                f'Materialization "{materialization_macro.name}" only supports languages {str_langs}; '
+                f'got "{compiled_node.language}"'
+            )
+
+    def build_result(self, compiled_node: FunctionNode, context: Dict[str, Any]) -> RunResult:
+        loaded_result = context["load_result"]("main")
+
+        return RunResult(
+            node=compiled_node,
+            status=RunStatus.Success,
+            timing=[],
+            thread_id=threading.current_thread().name,
+            # This gets set later in `from_run_result` called by `BaseRunner.safe_run`
+            execution_time=0.0,
+            message=str(loaded_result.response),
+            adapter_response=loaded_result.response.to_dict(omit_none=True),
+            failures=loaded_result.get("failures"),
+            batch_results=None,
+        )
+
+    def execute(self, compiled_node: FunctionNode, manifest: Manifest) -> RunResult:
+        materialization_macro = self._get_materialization_macro(compiled_node, manifest)
+        self._check_lang_supported(compiled_node, materialization_macro)
+        context = generate_runtime_function_context(compiled_node, self.config, manifest)
+
+        MacroGenerator(materialization_macro, context=context)()
+
+        return self.build_result(compiled_node, context)
+
+    def after_execute(self, result: RunResult) -> None:
+        self.print_result_line(result)
+
+    # def compile() defined on CompileRunner
+
+    def print_result_line(self, result: RunResult) -> None:
+        node = result.node
+        assert isinstance(node, FunctionNode)
+
+        group = group_lookup.get(node.unique_id)
+        level = EventLevel.ERROR if result.status == NodeStatus.Error else EventLevel.INFO
+        fire_event(
+            LogFunctionResult(
+                description=self.describe_node(),
+                status=result.status,
+                index=self.node_index,
+                total=self.num_nodes,
+                execution_time=result.execution_time,
+                node_info=self.node.node_info,
+                group=group,
+            ),
+            level=level,
+        )

dvt/task/group_lookup.py

@@ -0,0 +1,46 @@
+from typing import AbstractSet, Dict, Optional, Union
+
+from dvt.contracts.graph.manifest import Manifest
+from dvt.contracts.graph.nodes import Group
+
+_node_id_to_group_name_map: Dict[str, str] = {}
+_group_name_to_group_map: Dict[str, Group] = {}
+
+
+def init(manifest: Optional[Manifest], selected_ids: AbstractSet[str]) -> None:
+    if not manifest:
+        return
+
+    if not manifest.groups:
+        return
+
+    if not hasattr(manifest, "group_map"):
+        manifest.build_group_map()
+
+    _every_group_name_to_group_map = {v.name: v for v in manifest.groups.values()}
+
+    for group_name, node_ids in manifest.group_map.items():
+        for node_id in node_ids:
+            # only add node to lookup if it's selected
+            if node_id in selected_ids:
+                _node_id_to_group_name_map[node_id] = group_name
+
+                # only add group to lookup if it's not already there and if node is selected
+                if group_name not in _group_name_to_group_map:
+                    _group_name_to_group_map[group_name] = _every_group_name_to_group_map[
+                        group_name
+                    ]
+
+
+def get(node_id: str) -> Optional[Dict[str, Union[str, Dict[str, str]]]]:
+    group_name = _node_id_to_group_name_map.get(node_id)
+
+    if group_name is None:
+        return None
+
+    group = _group_name_to_group_map.get(group_name)
+
+    if group is None:
+        return None
+
+    return group.to_logging_dict()