dvt-core 1.11.0b4__py3-none-any.whl

dvt/artifacts/schemas/results.py

@@ -0,0 +1,148 @@
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+from dvt.contracts.graph.nodes import ResultNode
+
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+from dbt_common.events.helpers import datetime_to_json_string
+from dbt_common.utils import cast_to_int, cast_to_str
+
+
+@dataclass
+class TimingInfo(dbtClassMixin):
+    """
+    Represents a step in the execution of a node.
+    `name` should be one of: compile, execute, or other
+    Do not call directly, use `collect_timing_info` instead.
+    """
+
+    name: str
+    started_at: Optional[datetime] = None
+    completed_at: Optional[datetime] = None
+
+    def begin(self):
+        self.started_at = datetime.now(timezone.utc).replace(tzinfo=None)
+
+    def end(self):
+        self.completed_at = datetime.now(timezone.utc).replace(tzinfo=None)
+
+    def to_msg_dict(self):
+        msg_dict = {"name": str(self.name)}
+        if self.started_at:
+            msg_dict["started_at"] = datetime_to_json_string(self.started_at)
+        if self.completed_at:
+            msg_dict["completed_at"] = datetime_to_json_string(self.completed_at)
+        return msg_dict
+
+
+# This is a context manager
+class collect_timing_info:
+    def __init__(self, name: str, callback: Callable[[TimingInfo], None]) -> None:
+        self.timing_info = TimingInfo(name=name)
+        self.callback = callback
+
+    def __enter__(self):
+        self.timing_info.begin()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.timing_info.end()
+        self.callback(self.timing_info)
+
+
+class RunningStatus(StrEnum):
+    Started = "started"
+    Compiling = "compiling"
+    Executing = "executing"
+
+
+class NodeStatus(StrEnum):
+    Success = "success"
+    Error = "error"
+    Fail = "fail"
+    Warn = "warn"
+    Skipped = "skipped"
+    PartialSuccess = "partial success"
+    Pass = "pass"
+    RuntimeErr = "runtime error"
+    NoOp = "no-op"
+
+
+class RunStatus(StrEnum):
+    Success = NodeStatus.Success
+    Error = NodeStatus.Error
+    Skipped = NodeStatus.Skipped
+    PartialSuccess = NodeStatus.PartialSuccess
+    NoOp = NodeStatus.NoOp
+
+
+class TestStatus(StrEnum):
+    __test__ = False
+    Pass = NodeStatus.Pass
+    Error = NodeStatus.Error
+    Fail = NodeStatus.Fail
+    Warn = NodeStatus.Warn
+    Skipped = NodeStatus.Skipped
+
+
+class FreshnessStatus(StrEnum):
+    Pass = NodeStatus.Pass
+    Warn = NodeStatus.Warn
+    Error = NodeStatus.Error
+    RuntimeErr = NodeStatus.RuntimeErr
+
+
+@dataclass
+class BaseResult(dbtClassMixin):
+    status: Union[RunStatus, TestStatus, FreshnessStatus]
+    timing: List[TimingInfo]
+    thread_id: str
+    execution_time: float
+    adapter_response: Dict[str, Any]
+    message: Optional[str]
+    failures: Optional[int]
+
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        if "message" not in data:
+            data["message"] = None
+        if "failures" not in data:
+            data["failures"] = None
+        return data
+
+    def to_msg_dict(self):
+        msg_dict = {
+            "status": str(self.status),
+            "message": cast_to_str(self.message),
+            "thread": self.thread_id,
+            "execution_time": self.execution_time,
+            "num_failures": cast_to_int(self.failures),
+            "timing_info": [ti.to_msg_dict() for ti in self.timing],
+            "adapter_response": self.adapter_response,
+        }
+        return msg_dict
+
+
+@dataclass
+class NodeResult(BaseResult):
+    node: ResultNode
+
+
+@dataclass
+class ExecutionResult(dbtClassMixin):
+    results: Sequence[BaseResult]
+    elapsed_time: float
+
+    def __len__(self):
+        return len(self.results)
+
+    def __iter__(self):
+        return iter(self.results)
+
+    def __getitem__(self, idx):
+        return self.results[idx]
+
+
+# due to issues with typing.Union collapsing subclasses, this can't subclass
+# PartialResult

dvt/artifacts/schemas/run/__init__.py

@@ -0,0 +1,2 @@
+# alias to latest
+from dvt.artifacts.schemas.run.v5.run import *  # noqa

dvt/artifacts/schemas/run/v5/run.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import copy
+import threading
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, Iterable, Optional, Sequence, Tuple
+
+# https://github.com/dbt-labs/dbt-core/issues/10098
+# Needed for Mashumaro serialization of RunResult below
+# TODO: investigate alternative approaches to restore conditional import
+# if TYPE_CHECKING:
+import agate
+from dvt.artifacts.resources import CompiledResource
+from dvt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    get_artifact_schema_version,
+    schema_version,
+)
+from dvt.artifacts.schemas.batch_results import BatchResults
+from dvt.artifacts.schemas.results import (
+    BaseResult,
+    ExecutionResult,
+    NodeResult,
+    ResultNode,
+    RunStatus,
+)
+from dvt.exceptions import scrub_secrets
+
+from dbt_common.clients.system import write_json
+from dbt_common.constants import SECRET_ENV_PREFIX
+
+
+@dataclass
+class RunResult(NodeResult):
+    agate_table: Optional["agate.Table"] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    batch_results: Optional[BatchResults] = None
+
+    @property
+    def skipped(self):
+        return self.status == RunStatus.Skipped
+
+    @classmethod
+    def from_node(cls, node: ResultNode, status: RunStatus, message: Optional[str]):
+        thread_id = threading.current_thread().name
+        return RunResult(
+            status=status,
+            thread_id=thread_id,
+            execution_time=0,
+            timing=[],
+            message=message,
+            node=node,
+            adapter_response={},
+            failures=None,
+            batch_results=None,
+        )
+
+
+@dataclass
+class RunResultsMetadata(BaseArtifactMetadata):
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(RunResultsArtifact.dbt_schema_version)
+    )
+
+
+@dataclass
+class RunResultOutput(BaseResult):
+    unique_id: str
+    compiled: Optional[bool]
+    compiled_code: Optional[str]
+    relation_name: Optional[str]
+    batch_results: Optional[BatchResults] = None
+
+
+def process_run_result(result: RunResult) -> RunResultOutput:
+
+    compiled = isinstance(result.node, CompiledResource)
+
+    return RunResultOutput(
+        unique_id=result.node.unique_id,
+        status=result.status,
+        timing=result.timing,
+        thread_id=result.thread_id,
+        execution_time=result.execution_time,
+        message=result.message,
+        adapter_response=result.adapter_response,
+        failures=result.failures,
+        batch_results=result.batch_results,
+        compiled=result.node.compiled if compiled else None,  # type:ignore
+        compiled_code=result.node.compiled_code if compiled else None,  # type:ignore
+        relation_name=result.node.relation_name if compiled else None,  # type:ignore
+    )
+
+
+@dataclass
+class RunExecutionResult(
+    ExecutionResult,
+):
+    results: Sequence[RunResult]
+    args: Dict[str, Any] = field(default_factory=dict)
+    generated_at: datetime = field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)
+    )
+
+    def write(self, path: str):
+        writable = RunResultsArtifact.from_execution_results(
+            results=self.results,
+            elapsed_time=self.elapsed_time,
+            generated_at=self.generated_at,
+            args=self.args,
+        )
+        writable.write(path)
+
+
+@dataclass
+@schema_version("run-results", 6)
+class RunResultsArtifact(ExecutionResult, ArtifactMixin):
+    results: Sequence[RunResultOutput]
+    args: Dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_execution_results(
+        cls,
+        results: Sequence[RunResult],
+        elapsed_time: float,
+        generated_at: datetime,
+        args: Dict,
+    ):
+        processed_results = [
+            process_run_result(result) for result in results if isinstance(result, RunResult)
+        ]
+        meta = RunResultsMetadata(
+            dbt_schema_version=str(cls.dbt_schema_version),
+            generated_at=generated_at,
+        )
+
+        secret_vars = [
+            v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()
+        ]
+
+        scrubbed_args = copy.deepcopy(args)
+
+        # scrub secrets in invocation command
+        scrubbed_args["invocation_command"] = scrub_secrets(
+            scrubbed_args["invocation_command"], secret_vars
+        )
+
+        # scrub secrets in vars dict
+        scrubbed_args["vars"] = {
+            k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items()
+        }
+
+        return cls(
+            metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args
+        )
+
+    @classmethod
+    def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
+        return [
+            ("run-results", 4),
+            ("run-results", 5),
+        ]
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This overrides the "upgrade_schema_version" call in VersionedSchema (via
+        ArtifactMixin) to modify the dictionary passed in from earlier versions of the run_results.
+        """
+        run_results_schema_version = get_artifact_schema_version(data)
+        # If less than the current version (v5), preprocess contents to match latest schema version
+        if run_results_schema_version <= 5:
+            # In v5, we added 'compiled' attributes to each result entry
+            # Going forward, dbt expects these to be populated
+            for result in data["results"]:
+                result["compiled"] = False
+                result["compiled_code"] = ""
+                result["relation_name"] = ""
+        return cls.from_dict(data)
+
+    def write(self, path: str):
+        write_json(path, self.to_dict(omit_none=False))

dvt/artifacts/schemas/upgrades/__init__.py

@@ -0,0 +1,4 @@
+from dvt.artifacts.schemas.upgrades.upgrade_manifest import upgrade_manifest_json
+from dvt.artifacts.schemas.upgrades.upgrade_manifest_dbt_version import (
+    upgrade_manifest_json_dbt_version,
+)

dvt/artifacts/schemas/upgrades/upgrade_manifest.py

@@ -0,0 +1,174 @@
+def rename_sql_attr(node_content: dict) -> dict:
+    if "raw_sql" in node_content:
+        node_content["raw_code"] = node_content.pop("raw_sql")
+    if "compiled_sql" in node_content:
+        node_content["compiled_code"] = node_content.pop("compiled_sql")
+    node_content["language"] = "sql"
+    return node_content
+
+
+def upgrade_ref_content(node_content: dict) -> dict:
+    # In v1.5 we switched Node.refs from List[List[str]] to List[Dict[str, Union[NodeVersion, str]]]
+    # Previous versions did not have a version keyword argument for ref
+    if "refs" in node_content:
+        upgraded_refs = []
+        for ref in node_content["refs"]:
+            if isinstance(ref, list):
+                if len(ref) == 1:
+                    upgraded_refs.append({"package": None, "name": ref[0], "version": None})
+                else:
+                    upgraded_refs.append({"package": ref[0], "name": ref[1], "version": None})
+        node_content["refs"] = upgraded_refs
+    return node_content
+
+
+def upgrade_node_content(node_content):
+    rename_sql_attr(node_content)
+    upgrade_ref_content(node_content)
+    if node_content["resource_type"] != "seed" and "root_path" in node_content:
+        del node_content["root_path"]
+
+
+def upgrade_seed_content(node_content):
+    # Remove compilation related attributes
+    for attr_name in (
+        "language",
+        "refs",
+        "sources",
+        "metrics",
+        "compiled_path",
+        "compiled",
+        "compiled_code",
+        "extra_ctes_injected",
+        "extra_ctes",
+        "relation_name",
+    ):
+        if attr_name in node_content:
+            del node_content[attr_name]
+        # In v1.4, we switched SeedNode.depends_on from DependsOn to MacroDependsOn
+        node_content.get("depends_on", {}).pop("nodes", None)
+
+
+def drop_v9_and_prior_metrics(manifest: dict) -> None:
+    manifest["metrics"] = {}
+    filtered_disabled_entries = {}
+    for entry_name, resource_list in manifest.get("disabled", {}).items():
+        filtered_resource_list = []
+        for resource in resource_list:
+            if resource.get("resource_type") != "metric":
+                filtered_resource_list.append(resource)
+        filtered_disabled_entries[entry_name] = filtered_resource_list
+
+    manifest["disabled"] = filtered_disabled_entries
+
+
+def _convert_dct_with_filter(v10_dct_with_opt_filter):
+    """Upgrage the filter object from v10 to v11.
+
+    v10 filters from a serialized manifest looked like:
+    {..., 'filter': {'where_sql_template': '<filter_value>'}}
+    whereas v11 filters look like:
+    {..., 'filter': {'where_filters': [{'where_sql_template': '<filter_value>'}, ...]}}
+    """
+    if v10_dct_with_opt_filter is not None and v10_dct_with_opt_filter.get("filter") is not None:
+        v10_dct_with_opt_filter["filter"] = {"where_filters": [v10_dct_with_opt_filter["filter"]]}
+
+
+def _convert_metric(v10_metric_dict):
+    """Upgrades a v10 metric object to a v11 metric object.
+
+    Specifcally the following properties change
+    1. metric.filter
+    2. metric.type_params.measure.filter
+    3. metric.type_params.input_measures[x].filter
+    4. metric.type_params.numerator.filter
+    5. metric.type_params.denominator.filter
+    6. metric.type_params.metrics[x].filter"
+    """
+
+    # handles top level metric filter
+    _convert_dct_with_filter(v10_metric_dict)
+
+    type_params = v10_metric_dict.get("type_params")
+    if type_params is not None:
+        _convert_dct_with_filter(type_params.get("measure"))
+        _convert_dct_with_filter(type_params.get("numerator"))
+        _convert_dct_with_filter(type_params.get("denominator"))
+
+        # handles metric.type_params.input_measures[x].filter
+        input_measures = type_params.get("input_measures")
+        if input_measures is not None:
+            for input_measure in input_measures:
+                _convert_dct_with_filter(input_measure)
+
+        # handles metric.type_params.metrics[x].filter
+        metrics = type_params.get("metrics")
+        if metrics is not None:
+            for metric in metrics:
+                _convert_dct_with_filter(metric)
+
+
+def upgrade_v10_metric_filters(manifest: dict):
+    """Handles metric filters changes from v10 to v11."""
+
+    metrics = manifest.get("metrics", {})
+    for metric in metrics.values():
+        _convert_metric(metric)
+
+    disabled_nodes = manifest.get("disabled", {})
+    for unique_id, nodes in disabled_nodes.items():
+        if unique_id.split(".")[0] == "metric":
+            for node in nodes:
+                _convert_metric(node)
+
+
+def upgrade_manifest_json(manifest: dict, manifest_schema_version: int) -> dict:
+    # this should remain 9 while the check in `upgrade_schema_version` may change
+    if manifest_schema_version <= 9:
+        drop_v9_and_prior_metrics(manifest=manifest)
+    elif manifest_schema_version == 10:
+        upgrade_v10_metric_filters(manifest=manifest)
+
+    for node_content in manifest.get("nodes", {}).values():
+        upgrade_node_content(node_content)
+        if node_content["resource_type"] == "seed":
+            upgrade_seed_content(node_content)
+    for disabled in manifest.get("disabled", {}).values():
+        # There can be multiple disabled nodes for the same unique_id
+        # so make sure all the nodes get the attr renamed
+        for node_content in disabled:
+            upgrade_node_content(node_content)
+            if node_content["resource_type"] == "seed":
+                upgrade_seed_content(node_content)
+    # add group key
+    if "groups" not in manifest:
+        manifest["groups"] = {}
+    if "group_map" not in manifest:
+        manifest["group_map"] = {}
+    # add unit_tests key
+    if "unit_tests" not in manifest:
+        manifest["unit_tests"] = {}
+    for metric_content in manifest.get("metrics", {}).values():
+        # handle attr renames + value translation ("expression" -> "derived")
+        metric_content = upgrade_ref_content(metric_content)
+        if "root_path" in metric_content:
+            del metric_content["root_path"]
+    for exposure_content in manifest.get("exposures", {}).values():
+        exposure_content = upgrade_ref_content(exposure_content)
+        if "root_path" in exposure_content:
+            del exposure_content["root_path"]
+    for source_content in manifest.get("sources", {}).values():
+        if "root_path" in source_content:
+            del source_content["root_path"]
+    for macro_content in manifest.get("macros", {}).values():
+        if "root_path" in macro_content:
+            del macro_content["root_path"]
+    for doc_content in manifest.get("docs", {}).values():
+        if "root_path" in doc_content:
+            del doc_content["root_path"]
+        doc_content["resource_type"] = "doc"
+    if "semantic_models" not in manifest:
+        manifest["semantic_models"] = {}
+    if "saved_queries" not in manifest:
+        manifest["saved_queries"] = {}
+    return manifest

dvt/artifacts/schemas/upgrades/upgrade_manifest_dbt_version.py

@@ -0,0 +1,2 @@
+def upgrade_manifest_json_dbt_version(manifest: dict) -> dict:
+    return manifest

dvt/artifacts/utils/validation.py

@@ -0,0 +1,153 @@
+import re
+
+HTML_COLORS = [
+    "aliceblue",
+    "antiquewhite",
+    "aqua",
+    "aquamarine",
+    "azure",
+    "beige",
+    "bisque",
+    "black",
+    "blanchedalmond",
+    "blue",
+    "blueviolet",
+    "brown",
+    "burlywood",
+    "cadetblue",
+    "chartreuse",
+    "chocolate",
+    "coral",
+    "cornflowerblue",
+    "cornsilk",
+    "crimson",
+    "cyan",
+    "darkblue",
+    "darkcyan",
+    "darkgoldenrod",
+    "darkgray",
+    "darkgreen",
+    "darkkhaki",
+    "darkmagenta",
+    "darkolivegreen",
+    "darkorange",
+    "darkorchid",
+    "darkred",
+    "darksalmon",
+    "darkseagreen",
+    "darkslateblue",
+    "darkslategray",
+    "darkturquoise",
+    "darkviolet",
+    "deeppink",
+    "deepskyblue",
+    "dimgray",
+    "dodgerblue",
+    "firebrick",
+    "floralwhite",
+    "forestgreen",
+    "fuchsia",
+    "gainsboro",
+    "ghostwhite",
+    "gold",
+    "goldenrod",
+    "gray",
+    "green",
+    "greenyellow",
+    "honeydew",
+    "hotpink",
+    "indianred",
+    "indigo",
+    "ivory",
+    "khaki",
+    "lavender",
+    "lavenderblush",
+    "lawngreen",
+    "lemonchiffon",
+    "lightblue",
+    "lightcoral",
+    "lightcyan",
+    "lightgoldenrodyellow",
+    "lightgray",
+    "lightgreen",
+    "lightpink",
+    "lightsalmon",
+    "lightsalmon",
+    "lightseagreen",
+    "lightskyblue",
+    "lightslategray",
+    "lightsteelblue",
+    "lightyellow",
+    "lime",
+    "limegreen",
+    "linen",
+    "magenta",
+    "maroon",
+    "mediumaquamarine",
+    "mediumblue",
+    "mediumorchid",
+    "mediumpurple",
+    "mediumseagreen",
+    "mediumslateblue",
+    "mediumslateblue",
+    "mediumspringgreen",
+    "mediumturquoise",
+    "mediumvioletred",
+    "midnightblue",
+    "mintcream",
+    "mistyrose",
+    "moccasin",
+    "navajowhite",
+    "navy",
+    "oldlace",
+    "olive",
+    "olivedrab",
+    "orange",
+    "orangered",
+    "orchid",
+    "palegoldenrod",
+    "palegreen",
+    "paleturquoise",
+    "palevioletred",
+    "papayawhip",
+    "peachpuff",
+    "peru",
+    "pink",
+    "plum",
+    "powderblue",
+    "purple",
+    "rebeccapurple",
+    "red",
+    "rosybrown",
+    "royalblue",
+    "saddlebrown",
+    "salmon",
+    "sandybrown",
+    "seagreen",
+    "seashell",
+    "sienna",
+    "silver",
+    "skyblue",
+    "slateblue",
+    "slategray",
+    "snow",
+    "springgreen",
+    "steelblue",
+    "tan",
+    "teal",
+    "thistle",
+    "tomato",
+    "turquoise",
+    "violet",
+    "wheat",
+    "white",
+    "whitesmoke",
+    "yellow",
+    "yellowgreen",
+]
+
+
+def validate_color(color: str) -> bool:
+    match_hex = re.search(r"^#(?:[0-9a-f]{3}){1,2}$", color.lower())
+    match_html_color_name = color.lower() in HTML_COLORS
+    return bool(match_hex or match_html_color_name)

dvt/cli/__init__.py

@@ -0,0 +1 @@
+from .main import cli as dbt_cli  # noqa

dvt/cli/context.py

@@ -0,0 +1,16 @@
+from typing import Optional
+
+import click
+from dvt.cli.main import cli as dbt
+
+
+def make_context(args, command=dbt) -> Optional[click.Context]:
+    try:
+        ctx = command.make_context(command.name, args)
+    except click.exceptions.Exit:
+        return None
+
+    ctx.invoked_subcommand = ctx.protected_args[0] if ctx.protected_args else None
+    ctx.obj = {}
+
+    return ctx