dvt-core 1.11.0b4__py3-none-any.whl

dvt/task/printer.py

@@ -0,0 +1,176 @@
+from typing import Dict, Optional, Union
+
+from dvt.artifacts.schemas.results import NodeStatus
+from dvt.contracts.graph.nodes import Exposure
+from dvt.events.types import (
+    CheckNodeTestFailure,
+    EndOfRunSummary,
+    RunResultError,
+    RunResultErrorNoMessage,
+    RunResultFailure,
+    RunResultWarning,
+    RunResultWarningMessage,
+    SQLCompiledPath,
+    StatsLine,
+)
+from dvt.node_types import NodeType
+from dvt.task import group_lookup
+
+from dbt_common.events.base_types import EventLevel
+from dbt_common.events.format import pluralize
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import Formatting
+
+
+def get_counts(flat_nodes) -> str:
+    counts: Dict[str, int] = {}
+
+    for node in flat_nodes:
+        t = node.resource_type
+
+        if node.resource_type == NodeType.Model:
+            t = "{} {}".format(node.get_materialization(), t)
+        elif node.resource_type == NodeType.Operation:
+            t = "project hook"
+
+        counts[t] = counts.get(t, 0) + 1
+
+    sorted_items = sorted(counts.items(), key=lambda x: x[0])
+    stat_line = ", ".join([pluralize(v, k).replace("_", " ") for k, v in sorted_items])
+
+    return stat_line
+
+
+def interpret_run_result(result) -> str:
+    if result.status in (NodeStatus.Error, NodeStatus.Fail, NodeStatus.PartialSuccess):
+        return "error"
+    elif result.status == NodeStatus.Skipped:
+        return "skip"
+    elif result.status == NodeStatus.Warn:
+        return "warn"
+    elif result.status in (NodeStatus.Pass, NodeStatus.Success):
+        return "pass"
+    elif result.status == NodeStatus.NoOp:
+        return "noop"
+    else:
+        raise RuntimeError(f"unhandled result {result}")
+
+
+def print_run_status_line(results) -> None:
+    stats = {
+        "error": 0,
+        "skip": 0,
+        "pass": 0,
+        "warn": 0,
+        "noop": 0,
+        "total": 0,
+    }
+
+    for r in results:
+        result_type = interpret_run_result(r)
+        stats[result_type] += 1
+        stats["total"] += 1
+
+    fire_event(Formatting(""))
+    fire_event(StatsLine(stats=stats))
+
+
+def print_run_result_error(
+    result,
+    newline: bool = True,
+    is_warning: bool = False,
+    group: Optional[Dict[str, Union[str, Dict[str, str]]]] = None,
+) -> None:
+    # set node_info for logging events
+    node_info = None
+    if hasattr(result, "node") and result.node:
+        node_info = result.node.node_info
+    if result.status in (NodeStatus.Fail, NodeStatus.Error) or (
+        is_warning and result.status == NodeStatus.Warn
+    ):
+        if newline:
+            fire_event(Formatting(""))
+        if is_warning:
+            fire_event(
+                RunResultWarning(
+                    resource_type=result.node.resource_type,
+                    node_name=result.node.name,
+                    path=result.node.original_file_path,
+                    node_info=node_info,
+                    group=group,
+                )
+            )
+        else:
+            fire_event(
+                RunResultFailure(
+                    resource_type=result.node.resource_type,
+                    node_name=result.node.name,
+                    path=result.node.original_file_path,
+                    node_info=node_info,
+                    group=group,
+                )
+            )
+
+        if result.message:
+            if is_warning:
+                fire_event(RunResultWarningMessage(msg=result.message, node_info=node_info))
+            else:
+                fire_event(RunResultError(msg=result.message, node_info=node_info, group=group))
+        else:
+            fire_event(RunResultErrorNoMessage(status=result.status, node_info=node_info))
+
+        if getattr(result.node, "compiled_path", None):
+            fire_event(Formatting(""))
+            fire_event(SQLCompiledPath(path=result.node.compiled_path, node_info=node_info))
+
+        if getattr(result.node, "should_store_failures", None):
+            fire_event(Formatting(""))
+            fire_event(
+                CheckNodeTestFailure(relation_name=result.node.relation_name, node_info=node_info)
+            )
+    elif result.status == NodeStatus.Skipped and result.message is not None:
+        if newline:
+            fire_event(Formatting(""), level=EventLevel.DEBUG)
+        fire_event(RunResultError(msg=result.message), level=EventLevel.DEBUG)
+    elif result.message is not None:
+        if newline:
+            fire_event(Formatting(""))
+        fire_event(RunResultError(msg=result.message, node_info=node_info, group=group))
+
+
+def print_run_end_messages(results, keyboard_interrupt: bool = False) -> None:
+    errors, warnings, partial_successes = [], [], []
+    for r in results:
+        if r.status in (NodeStatus.RuntimeErr, NodeStatus.Error, NodeStatus.Fail):
+            errors.append(r)
+        elif r.status == NodeStatus.Skipped and r.message:
+            if isinstance(r.node, Exposure):
+                # Don't include exposure skips in errors list
+                continue
+            else:
+                # This means we skipped a node because of an issue upstream, so include it as an error
+                errors.append(r)
+        elif r.status == NodeStatus.Warn:
+            warnings.append(r)
+        elif r.status == NodeStatus.PartialSuccess:
+            partial_successes.append(r)
+
+    fire_event(Formatting(""))
+    fire_event(
+        EndOfRunSummary(
+            num_errors=len(errors),
+            num_warnings=len(warnings),
+            num_partial_success=len(partial_successes),
+            keyboard_interrupt=keyboard_interrupt,
+        )
+    )
+
+    for error in errors:
+        group = group_lookup.get(error.node.unique_id) if hasattr(error, "node") else None
+        print_run_result_error(error, is_warning=False, group=group)
+
+    for warning in warnings:
+        group = group_lookup.get(warning.node.unique_id) if hasattr(warning, "node") else None
+        print_run_result_error(warning, is_warning=True, group=group)
+
+    print_run_status_line(results)

dvt/task/profiles.py

@@ -0,0 +1,256 @@
+"""
+DVT Profiles Tasks
+
+This module implements profile management commands:
+- list: List all configured profiles
+- show: Show details of a specific profile
+- test: Test one or all profiles
+"""
+
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from dvt.cli.flags import Flags
+from dvt.config.profiles_v2 import ProfileRegistry, load_unified_profiles
+from dvt.events.types import Note
+from dvt.mp_context import get_mp_context
+from dvt.task.base import BaseTask
+
+from dbt_common.events.functions import fire_event
+from dbt_common.ui import green, red, yellow
+
+
+class ProfilesListTask(BaseTask):
+    """Task to list all configured profiles."""
+
+    def __init__(self, args: Flags) -> None:
+        super().__init__(args)
+        self.profiles_dir = args.PROFILES_DIR
+        self.profile_path = os.path.join(self.profiles_dir, "profiles.yml")
+
+    def run(self) -> bool:
+        """List all configured profiles."""
+        fire_event(Note(msg=""))
+        fire_event(Note(msg="=" * 60))
+        fire_event(Note(msg="DVT Profiles"))
+        fire_event(Note(msg="=" * 60))
+        fire_event(Note(msg=f"Using profiles.yml file at {self.profile_path}"))
+        fire_event(Note(msg=""))
+
+        # Load unified profiles
+        project_dir = Path(self.args.PROJECT_DIR) if self.args.PROJECT_DIR else None
+        unified_profiles = load_unified_profiles(project_dir)
+
+        # Create registry
+        registry = ProfileRegistry(unified_profiles)
+
+        # Get all profiles
+        all_profiles = registry.list_all_profiles()
+
+        if not all_profiles:
+            fire_event(Note(msg=red("No profiles found in profiles.yml")))
+            return False
+
+        fire_event(Note(msg=f"Found {len(all_profiles)} profile(s):\n"))
+
+        # List each profile with adapter type
+        for profile_name in all_profiles:
+            profile_config = registry.get_or_create_profile(profile_name)
+            if profile_config:
+                adapter_type = profile_config.get("type", "unknown")
+                fire_event(Note(msg=f"  • {green(profile_name)} ({adapter_type})"))
+            else:
+                fire_event(Note(msg=f"  • {red(profile_name)} (error loading)"))
+
+        fire_event(Note(msg=""))
+        fire_event(Note(msg=f"Use 'dvt profiles show <profile_name>' to see details"))
+        fire_event(Note(msg=f"Use 'dvt profiles test [profile_name]' to test connections"))
+        fire_event(Note(msg=""))
+
+        return True
+
+    def interpret_results(self, results) -> bool:
+        return results
+
+
+class ProfilesShowTask(BaseTask):
+    """Task to show details of a specific profile."""
+
+    def __init__(self, args: Flags, profile_name: str) -> None:
+        super().__init__(args)
+        self.profile_name = profile_name
+        self.profiles_dir = args.PROFILES_DIR
+        self.profile_path = os.path.join(self.profiles_dir, "profiles.yml")
+
+    def run(self) -> bool:
+        """Show details of a specific profile."""
+        fire_event(Note(msg=""))
+        fire_event(Note(msg="=" * 60))
+        fire_event(Note(msg=f"Profile: {self.profile_name}"))
+        fire_event(Note(msg="=" * 60))
+        fire_event(Note(msg=f"Using profiles.yml file at {self.profile_path}"))
+        fire_event(Note(msg=""))
+
+        # Load unified profiles
+        project_dir = Path(self.args.PROJECT_DIR) if self.args.PROJECT_DIR else None
+        unified_profiles = load_unified_profiles(project_dir)
+
+        # Create registry
+        registry = ProfileRegistry(unified_profiles)
+
+        # Get profile config
+        profile_config = registry.get_or_create_profile(self.profile_name)
+        if not profile_config:
+            fire_event(Note(msg=red(f"Profile '{self.profile_name}' not found")))
+            fire_event(Note(msg=""))
+            fire_event(Note(msg="Available profiles:"))
+            for name in registry.list_all_profiles():
+                fire_event(Note(msg=f"  • {name}"))
+            fire_event(Note(msg=""))
+            return False
+
+        # Display profile details
+        adapter_type = profile_config.get("type", "unknown")
+        fire_event(Note(msg=f"Adapter Type: {green(adapter_type)}"))
+        fire_event(Note(msg=""))
+        fire_event(Note(msg="Configuration:"))
+
+        # Show all config (except sensitive fields)
+        sensitive_fields = {"password", "token", "private_key", "api_key", "secret"}
+        for key, value in sorted(profile_config.items()):
+            if key.lower() in sensitive_fields:
+                fire_event(Note(msg=f"  {key}: {yellow('***REDACTED***')}"))
+            else:
+                fire_event(Note(msg=f"  {key}: {value}"))
+
+        fire_event(Note(msg=""))
+        fire_event(
+            Note(msg=f"Use 'dvt profiles test {self.profile_name}' to test this connection")
+        )
+        fire_event(Note(msg=""))
+
+        return True
+
+    def interpret_results(self, results) -> bool:
+        return results
+
+
+class ProfilesTestTask(BaseTask):
+    """Task to test one or all profiles."""
+
+    def __init__(self, args: Flags, profile_name: Optional[str] = None) -> None:
+        super().__init__(args)
+        self.profile_name = profile_name
+        self.profiles_dir = args.PROFILES_DIR
+        self.profile_path = os.path.join(self.profiles_dir, "profiles.yml")
+
+    def run(self) -> bool:
+        """Test one or all profiles."""
+        from dvt.config import Profile
+        from dvt.config.renderer import ProfileRenderer
+
+        from dbt.adapters.factory import get_adapter, register_adapter
+
+        fire_event(Note(msg=""))
+        fire_event(Note(msg="=" * 60))
+        if self.profile_name:
+            fire_event(Note(msg=f"Testing Profile: {self.profile_name}"))
+        else:
+            fire_event(Note(msg="Testing All Profiles"))
+        fire_event(Note(msg="=" * 60))
+        fire_event(Note(msg=f"Using profiles.yml file at {self.profile_path}"))
+        fire_event(Note(msg=""))
+
+        # Load unified profiles
+        project_dir = Path(self.args.PROJECT_DIR) if self.args.PROJECT_DIR else None
+        unified_profiles = load_unified_profiles(project_dir)
+
+        # Create registry
+        registry = ProfileRegistry(unified_profiles)
+
+        # Determine which profiles to test
+        if self.profile_name:
+            profiles_to_test = [self.profile_name]
+        else:
+            profiles_to_test = registry.list_all_profiles()
+
+        if not profiles_to_test:
+            fire_event(Note(msg=red("No profiles to test")))
+            return False
+
+        # Test each profile
+        results = {}
+        for profile_name in profiles_to_test:
+            fire_event(Note(msg=f"Testing {profile_name}..."))
+
+            try:
+                # Get profile config
+                profile_config = registry.get_or_create_profile(profile_name)
+                if not profile_config:
+                    fire_event(Note(msg=red(f"  ✗ Profile '{profile_name}' not found")))
+                    results[profile_name] = False
+                    continue
+
+                # Show adapter type
+                adapter_type = profile_config.get("type", "unknown")
+                fire_event(Note(msg=f"  Adapter: {adapter_type}"))
+
+                # Create a Profile object for testing
+                profile_data = {
+                    "outputs": {
+                        "test": profile_config,
+                    },
+                    "target": "test",
+                }
+
+                renderer = ProfileRenderer({})
+                test_profile = Profile.from_raw_profile_info(
+                    raw_profile=profile_data,
+                    profile_name=profile_name,
+                    target_override="test",
+                    renderer=renderer,
+                )
+
+                # Test connection
+                register_adapter(test_profile, get_mp_context())
+                adapter = get_adapter(test_profile)
+
+                try:
+                    with adapter.connection_named("debug"):
+                        adapter.debug_query()
+                    fire_event(Note(msg=green(f"  ✓ Connection successful")))
+                    results[profile_name] = True
+                except Exception as exc:
+                    fire_event(Note(msg=red(f"  ✗ Connection failed: {str(exc)}")))
+                    results[profile_name] = False
+
+            except Exception as e:
+                fire_event(Note(msg=red(f"  ✗ Error: {str(e)}")))
+                results[profile_name] = False
+
+            fire_event(Note(msg=""))
+
+        # Summary
+        success_count = sum(1 for success in results.values() if success)
+        fail_count = len(results) - success_count
+
+        fire_event(Note(msg="=" * 60))
+        fire_event(Note(msg="Summary"))
+        fire_event(Note(msg="=" * 60))
+        fire_event(Note(msg=f"Total: {len(results)}"))
+        fire_event(Note(msg=green(f"Passed: {success_count}")))
+        if fail_count > 0:
+            fire_event(Note(msg=red(f"Failed: {fail_count}")))
+
+        fire_event(Note(msg=""))
+        for profile_name, success in results.items():
+            status = green("✓") if success else red("✗")
+            fire_event(Note(msg=f"  {status} {profile_name}"))
+
+        fire_event(Note(msg=""))
+
+        return fail_count == 0
+
+    def interpret_results(self, results) -> bool:
+        return results

dvt/task/retry.py

@@ -0,0 +1,175 @@
+from pathlib import Path
+
+from click import get_current_context
+from click.core import ParameterSource
+from dvt.artifacts.schemas.results import NodeStatus
+from dvt.cli.flags import Flags
+from dvt.cli.types import Command as CliCommand
+from dvt.config import RuntimeConfig
+from dvt.constants import RUN_RESULTS_FILE_NAME
+from dvt.contracts.state import load_result_state
+from dvt.flags import get_flags, set_flags
+from dvt.graph import GraphQueue
+from dvt.parser.manifest import parse_manifest
+from dvt.task.base import ConfiguredTask
+from dvt.task.build import BuildTask
+from dvt.task.clone import CloneTask
+from dvt.task.compile import CompileTask
+from dvt.task.docs.generate import GenerateTask
+from dvt.task.run import RunTask
+from dvt.task.run_operation import RunOperationTask
+from dvt.task.seed import SeedTask
+from dvt.task.snapshot import SnapshotTask
+from dvt.task.test import TestTask
+
+from dbt_common.exceptions import DbtRuntimeError
+
+RETRYABLE_STATUSES = {
+    NodeStatus.Error,
+    NodeStatus.Fail,
+    NodeStatus.Skipped,
+    NodeStatus.RuntimeErr,
+    NodeStatus.PartialSuccess,
+}
+IGNORE_PARENT_FLAGS = {
+    "log_path",
+    "output_path",
+    "profiles_dir",
+    "profiles_dir_exists_false",
+    "project_dir",
+    "defer_state",
+    "deprecated_state",
+    "target_path",
+    "warn_error",
+}
+
+ALLOW_CLI_OVERRIDE_FLAGS = {"vars", "threads"}
+
+TASK_DICT = {
+    "build": BuildTask,
+    "compile": CompileTask,
+    "clone": CloneTask,
+    "generate": GenerateTask,
+    "seed": SeedTask,
+    "snapshot": SnapshotTask,
+    "test": TestTask,
+    "run": RunTask,
+    "run-operation": RunOperationTask,
+}
+
+CMD_DICT = {
+    "build": CliCommand.BUILD,
+    "compile": CliCommand.COMPILE,
+    "clone": CliCommand.CLONE,
+    "generate": CliCommand.DOCS_GENERATE,
+    "seed": CliCommand.SEED,
+    "snapshot": CliCommand.SNAPSHOT,
+    "test": CliCommand.TEST,
+    "run": CliCommand.RUN,
+    "run-operation": CliCommand.RUN_OPERATION,
+}
+
+
+class RetryTask(ConfiguredTask):
+    def __init__(self, args: Flags, config: RuntimeConfig) -> None:
+        # load previous run results
+        state_path = args.state or config.target_path
+        self.previous_results = load_result_state(
+            Path(config.project_root) / Path(state_path) / RUN_RESULTS_FILE_NAME
+        )
+        if not self.previous_results:
+            raise DbtRuntimeError(
+                f"Could not find previous run in '{state_path}' target directory"
+            )
+        self.previous_args = self.previous_results.args
+        self.previous_command_name = self.previous_args.get("which")
+
+        # Reslove flags and config
+        if args.warn_error:
+            RETRYABLE_STATUSES.add(NodeStatus.Warn)
+
+        cli_command = CMD_DICT.get(self.previous_command_name)  # type: ignore
+        # Remove these args when their default values are present, otherwise they'll raise an exception
+        args_to_remove = {
+            "show": lambda x: True,
+            "resource_types": lambda x: x == [],
+            "warn_error_options": lambda x: x == {"warn": [], "error": [], "silence": []},
+        }
+        for k, v in args_to_remove.items():
+            if k in self.previous_args and v(self.previous_args[k]):
+                del self.previous_args[k]
+        previous_args = {
+            k: v for k, v in self.previous_args.items() if k not in IGNORE_PARENT_FLAGS
+        }
+        click_context = get_current_context()
+        current_args = {
+            k: v
+            for k, v in args.__dict__.items()
+            if k in IGNORE_PARENT_FLAGS
+            or (
+                click_context.get_parameter_source(k) == ParameterSource.COMMANDLINE
+                and k in ALLOW_CLI_OVERRIDE_FLAGS
+            )
+        }
+        combined_args = {**previous_args, **current_args}
+        retry_flags = Flags.from_dict(cli_command, combined_args)  # type: ignore
+        set_flags(retry_flags)
+        retry_config = RuntimeConfig.from_args(args=retry_flags)
+
+        # Parse manifest using resolved config/flags
+        manifest = parse_manifest(retry_config, False, True, retry_flags.write_json, [])  # type: ignore
+        super().__init__(args, retry_config, manifest)
+        self.task_class = TASK_DICT.get(self.previous_command_name)  # type: ignore
+
+    def run(self):
+        unique_ids = {
+            result.unique_id
+            for result in self.previous_results.results
+            if result.status in RETRYABLE_STATUSES
+            and not (
+                self.previous_command_name != "run-operation"
+                and result.unique_id.startswith("operation.")
+            )
+        }
+
+        # We need this so that re-running of a microbatch model will only rerun
+        # batches that previously failed. Note _explicitly_ do no pass the
+        # batch info if there were _no_ successful batches previously. This is
+        # because passing the batch info _forces_ the microbatch process into
+        # _incremental_ model, and it may be that we need to be in full refresh
+        # mode which is only handled if previous_batch_results _isn't_ passed for a node
+        batch_map = {
+            result.unique_id: result.batch_results
+            for result in self.previous_results.results
+            if result.batch_results is not None
+            and len(result.batch_results.successful) != 0
+            and len(result.batch_results.failed) > 0
+            and not (
+                self.previous_command_name != "run-operation"
+                and result.unique_id.startswith("operation.")
+            )
+        }
+
+        class TaskWrapper(self.task_class):
+            def get_graph_queue(self):
+                new_graph = self.graph.get_subset_graph(unique_ids)
+                return GraphQueue(
+                    new_graph.graph,
+                    self.manifest,
+                    unique_ids,
+                )
+
+        task = TaskWrapper(
+            get_flags(),
+            self.config,
+            self.manifest,
+        )
+
+        if self.task_class == RunTask:
+            task.batch_map = batch_map
+
+        return_value = task.run()
+        return return_value
+
+    def interpret_results(self, *args, **kwargs):
+        return self.task_class.interpret_results(*args, **kwargs)