dvt-core 0.52.2__cp310-cp310-macosx_10_9_x86_64.whl

dbt/artifacts/schemas/freshness/v3/freshness.py

@@ -0,0 +1,158 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from dbt.artifacts.resources import FreshnessThreshold
+from dbt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    VersionedSchema,
+    schema_version,
+)
+from dbt.artifacts.schemas.results import (
+    ExecutionResult,
+    FreshnessStatus,
+    NodeResult,
+    TimingInfo,
+)
+from dbt.contracts.graph.nodes import SourceDefinition
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+from dbt_common.exceptions import DbtInternalError
+
+
+@dataclass
+class SourceFreshnessResult(NodeResult):
+    node: SourceDefinition
+    status: FreshnessStatus
+    max_loaded_at: datetime
+    snapshotted_at: datetime
+    age: float
+
+    @property
+    def skipped(self):
+        return False
+
+
+@dataclass
+class PartialSourceFreshnessResult(NodeResult):
+    status: FreshnessStatus
+
+    @property
+    def skipped(self):
+        return False
+
+
+FreshnessNodeResult = Union[PartialSourceFreshnessResult, SourceFreshnessResult]
+
+
+@dataclass
+class FreshnessMetadata(BaseArtifactMetadata):
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(FreshnessExecutionResultArtifact.dbt_schema_version)
+    )
+
+
+@dataclass
+class FreshnessResult(ExecutionResult):
+    metadata: FreshnessMetadata
+    results: Sequence[FreshnessNodeResult]
+
+    @classmethod
+    def from_node_results(
+        cls,
+        results: List[FreshnessNodeResult],
+        elapsed_time: float,
+        generated_at: datetime,
+    ):
+        meta = FreshnessMetadata(generated_at=generated_at)
+        return cls(metadata=meta, results=results, elapsed_time=elapsed_time)
+
+    def write(self, path):
+        FreshnessExecutionResultArtifact.from_result(self).write(path)
+
+
+@dataclass
+class SourceFreshnessOutput(dbtClassMixin):
+    unique_id: str
+    max_loaded_at: datetime
+    snapshotted_at: datetime
+    max_loaded_at_time_ago_in_s: float
+    status: FreshnessStatus
+    criteria: FreshnessThreshold
+    adapter_response: Dict[str, Any]
+    timing: List[TimingInfo]
+    thread_id: str
+    execution_time: float
+
+
+class FreshnessErrorEnum(StrEnum):
+    runtime_error = "runtime error"
+
+
+@dataclass
+class SourceFreshnessRuntimeError(dbtClassMixin):
+    unique_id: str
+    error: Optional[Union[str, int]]
+    status: FreshnessErrorEnum
+
+
+FreshnessNodeOutput = Union[SourceFreshnessRuntimeError, SourceFreshnessOutput]
+
+
+@dataclass
+@schema_version("sources", 3)
+class FreshnessExecutionResultArtifact(
+    ArtifactMixin,
+    VersionedSchema,
+):
+    metadata: FreshnessMetadata
+    results: Sequence[FreshnessNodeOutput]
+    elapsed_time: float
+
+    @classmethod
+    def from_result(cls, base: FreshnessResult):
+        processed = [
+            process_freshness_result(r)
+            for r in base.results
+            if isinstance(r, SourceFreshnessResult)
+        ]
+        return cls(
+            metadata=base.metadata,
+            results=processed,
+            elapsed_time=base.elapsed_time,
+        )
+
+
+def process_freshness_result(result: FreshnessNodeResult) -> FreshnessNodeOutput:
+    unique_id = result.node.unique_id
+    if result.status == FreshnessStatus.RuntimeErr:
+        return SourceFreshnessRuntimeError(
+            unique_id=unique_id,
+            error=result.message,
+            status=FreshnessErrorEnum.runtime_error,
+        )
+
+    # we know that this must be a SourceFreshnessResult
+    if not isinstance(result, SourceFreshnessResult):
+        raise DbtInternalError(
+            "Got {} instead of a SourceFreshnessResult for a "
+            "non-error result in freshness execution!".format(type(result))
+        )
+    # if we're here, we must have a non-None freshness threshold
+    criteria = result.node.freshness
+    if criteria is None:
+        raise DbtInternalError(
+            "Somehow evaluated a freshness result for a source that has no freshness criteria!"
+        )
+    return SourceFreshnessOutput(
+        unique_id=unique_id,
+        max_loaded_at=result.max_loaded_at,
+        snapshotted_at=result.snapshotted_at,
+        max_loaded_at_time_ago_in_s=result.age,
+        status=result.status,
+        criteria=criteria,
+        adapter_response=result.adapter_response,
+        timing=result.timing,
+        thread_id=result.thread_id,
+        execution_time=result.execution_time,
+    )

dbt/artifacts/schemas/manifest/__init__.py

@@ -0,0 +1,2 @@
+# alias to latest
+from dbt.artifacts.schemas.manifest.v12.manifest import *  # noqa

dbt/artifacts/schemas/manifest/v12/manifest.py

@@ -0,0 +1,211 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
+from uuid import UUID
+
+from dbt import tracking
+from dbt.artifacts.resources import (
+    Analysis,
+    Documentation,
+    Exposure,
+    Function,
+    GenericTest,
+    Group,
+    HookNode,
+    Macro,
+    Metric,
+    Model,
+    SavedQuery,
+    Seed,
+    SemanticModel,
+    SingularTest,
+    Snapshot,
+    SourceDefinition,
+    SqlOperation,
+    UnitTestDefinition,
+)
+from dbt.artifacts.resources.v1.components import Quoting
+from dbt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    get_artifact_dbt_version,
+    get_artifact_schema_version,
+    schema_version,
+)
+from dbt.artifacts.schemas.upgrades import (
+    upgrade_manifest_json,
+    upgrade_manifest_json_dbt_version,
+)
+from dbt.version import __version__
+from dbt_common.exceptions import DbtInternalError
+
+NodeEdgeMap = Dict[str, List[str]]
+UniqueID = str
+ManifestResource = Union[
+    Seed, Analysis, SingularTest, HookNode, Model, SqlOperation, GenericTest, Snapshot, Function
+]
+DisabledManifestResource = Union[
+    ManifestResource,
+    SourceDefinition,
+    Exposure,
+    Metric,
+    SavedQuery,
+    SemanticModel,
+    UnitTestDefinition,
+]
+
+
+@dataclass
+class ManifestMetadata(BaseArtifactMetadata):
+    """Metadata for the manifest."""
+
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(WritableManifest.dbt_schema_version)
+    )
+    project_name: Optional[str] = field(
+        default=None,
+        metadata={
+            "description": "Name of the root project",
+        },
+    )
+    project_id: Optional[str] = field(
+        default=None,
+        metadata={
+            "description": "A unique identifier for the project, hashed from the project name",
+        },
+    )
+    user_id: Optional[UUID] = field(
+        default=None,
+        metadata={
+            "description": "A unique identifier for the user",
+        },
+    )
+    send_anonymous_usage_stats: Optional[bool] = field(
+        default=None,
+        metadata=dict(
+            description=("Whether dbt is configured to send anonymous usage statistics")
+        ),
+    )
+    adapter_type: Optional[str] = field(
+        default=None,
+        metadata=dict(description="The type name of the adapter"),
+    )
+    quoting: Optional[Quoting] = field(
+        default_factory=Quoting,
+        metadata=dict(description="The quoting configuration for the project"),
+    )
+    run_started_at: Optional[datetime] = field(
+        default=tracking.active_user.run_started_at if tracking.active_user is not None else None,
+        metadata=dict(description="The timestamp when the run started"),
+    )
+
+    @classmethod
+    def default(cls):
+        return cls(
+            dbt_schema_version=str(WritableManifest.dbt_schema_version),
+        )
+
+
+@dataclass
+@schema_version("manifest", 12)
+class WritableManifest(ArtifactMixin):
+    nodes: Mapping[UniqueID, ManifestResource] = field(
+        metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
+    )
+    sources: Mapping[UniqueID, SourceDefinition] = field(
+        metadata=dict(description=("The sources defined in the dbt project and its dependencies"))
+    )
+    macros: Mapping[UniqueID, Macro] = field(
+        metadata=dict(description=("The macros defined in the dbt project and its dependencies"))
+    )
+    docs: Mapping[UniqueID, Documentation] = field(
+        metadata=dict(description=("The docs defined in the dbt project and its dependencies"))
+    )
+    exposures: Mapping[UniqueID, Exposure] = field(
+        metadata=dict(
+            description=("The exposures defined in the dbt project and its dependencies")
+        )
+    )
+    metrics: Mapping[UniqueID, Metric] = field(
+        metadata=dict(description=("The metrics defined in the dbt project and its dependencies"))
+    )
+    groups: Mapping[UniqueID, Group] = field(
+        metadata=dict(description=("The groups defined in the dbt project"))
+    )
+    selectors: Mapping[UniqueID, Any] = field(
+        metadata=dict(description=("The selectors defined in selectors.yml"))
+    )
+    disabled: Optional[Mapping[UniqueID, List[DisabledManifestResource]]] = field(
+        metadata=dict(description="A mapping of the disabled nodes in the target")
+    )
+    parent_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from child nodes to their dependencies",
+        )
+    )
+    child_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from parent nodes to their dependents",
+        )
+    )
+    group_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from group names to their nodes",
+        )
+    )
+    saved_queries: Mapping[UniqueID, SavedQuery] = field(
+        metadata=dict(description=("The saved queries defined in the dbt project"))
+    )
+    semantic_models: Mapping[UniqueID, SemanticModel] = field(
+        metadata=dict(description=("The semantic models defined in the dbt project"))
+    )
+    metadata: ManifestMetadata = field(
+        metadata=dict(
+            description="Metadata about the manifest",
+        )
+    )
+    unit_tests: Mapping[UniqueID, UnitTestDefinition] = field(
+        metadata=dict(
+            description="The unit tests defined in the project",
+        )
+    )
+    functions: Mapping[UniqueID, Function] = field(
+        default_factory=dict,
+        metadata=dict(description=("The functions defined in the dbt project")),
+    )
+
+    @classmethod
+    def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
+        return [
+            ("manifest", 4),
+            ("manifest", 5),
+            ("manifest", 6),
+            ("manifest", 7),
+            ("manifest", 8),
+            ("manifest", 9),
+            ("manifest", 10),
+            ("manifest", 11),
+        ]
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This overrides the "upgrade_schema_version" call in VersionedSchema (via
+        ArtifactMixin) to modify the dictionary passed in from earlier versions of the manifest."""
+        manifest_schema_version = get_artifact_schema_version(data)
+        if manifest_schema_version < cls.dbt_schema_version.version:
+            data = upgrade_manifest_json(data, manifest_schema_version)
+
+        manifest_dbt_version = get_artifact_dbt_version(data)
+        if manifest_dbt_version and manifest_dbt_version != __version__:
+            data = upgrade_manifest_json_dbt_version(data)
+        return cls.from_dict(data)
+
+    @classmethod
+    def validate(cls, _):
+        # When dbt try to load an artifact with additional optional fields
+        # that are not present in the schema, from_dict will work fine.
+        # As long as validate is not called, the schema will not be enforced.
+        # This is intentional, as it allows for safer schema upgrades.
+        raise DbtInternalError(
+            "The WritableManifest should never be validated directly to allow for schema upgrades."
+        )

dbt/artifacts/schemas/results.py

@@ -0,0 +1,147 @@
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+from dbt.contracts.graph.nodes import ResultNode
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+from dbt_common.events.helpers import datetime_to_json_string
+from dbt_common.utils import cast_to_int, cast_to_str
+
+
+@dataclass
+class TimingInfo(dbtClassMixin):
+    """
+    Represents a step in the execution of a node.
+    `name` should be one of: compile, execute, or other
+    Do not call directly, use `collect_timing_info` instead.
+    """
+
+    name: str
+    started_at: Optional[datetime] = None
+    completed_at: Optional[datetime] = None
+
+    def begin(self):
+        self.started_at = datetime.now(timezone.utc).replace(tzinfo=None)
+
+    def end(self):
+        self.completed_at = datetime.now(timezone.utc).replace(tzinfo=None)
+
+    def to_msg_dict(self):
+        msg_dict = {"name": str(self.name)}
+        if self.started_at:
+            msg_dict["started_at"] = datetime_to_json_string(self.started_at)
+        if self.completed_at:
+            msg_dict["completed_at"] = datetime_to_json_string(self.completed_at)
+        return msg_dict
+
+
+# This is a context manager
+class collect_timing_info:
+    def __init__(self, name: str, callback: Callable[[TimingInfo], None]) -> None:
+        self.timing_info = TimingInfo(name=name)
+        self.callback = callback
+
+    def __enter__(self):
+        self.timing_info.begin()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.timing_info.end()
+        self.callback(self.timing_info)
+
+
+class RunningStatus(StrEnum):
+    Started = "started"
+    Compiling = "compiling"
+    Executing = "executing"
+
+
+class NodeStatus(StrEnum):
+    Success = "success"
+    Error = "error"
+    Fail = "fail"
+    Warn = "warn"
+    Skipped = "skipped"
+    PartialSuccess = "partial success"
+    Pass = "pass"
+    RuntimeErr = "runtime error"
+    NoOp = "no-op"
+
+
+class RunStatus(StrEnum):
+    Success = NodeStatus.Success
+    Error = NodeStatus.Error
+    Skipped = NodeStatus.Skipped
+    PartialSuccess = NodeStatus.PartialSuccess
+    NoOp = NodeStatus.NoOp
+
+
+class TestStatus(StrEnum):
+    __test__ = False
+    Pass = NodeStatus.Pass
+    Error = NodeStatus.Error
+    Fail = NodeStatus.Fail
+    Warn = NodeStatus.Warn
+    Skipped = NodeStatus.Skipped
+
+
+class FreshnessStatus(StrEnum):
+    Pass = NodeStatus.Pass
+    Warn = NodeStatus.Warn
+    Error = NodeStatus.Error
+    RuntimeErr = NodeStatus.RuntimeErr
+
+
+@dataclass
+class BaseResult(dbtClassMixin):
+    status: Union[RunStatus, TestStatus, FreshnessStatus]
+    timing: List[TimingInfo]
+    thread_id: str
+    execution_time: float
+    adapter_response: Dict[str, Any]
+    message: Optional[str]
+    failures: Optional[int]
+
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        if "message" not in data:
+            data["message"] = None
+        if "failures" not in data:
+            data["failures"] = None
+        return data
+
+    def to_msg_dict(self):
+        msg_dict = {
+            "status": str(self.status),
+            "message": cast_to_str(self.message),
+            "thread": self.thread_id,
+            "execution_time": self.execution_time,
+            "num_failures": cast_to_int(self.failures),
+            "timing_info": [ti.to_msg_dict() for ti in self.timing],
+            "adapter_response": self.adapter_response,
+        }
+        return msg_dict
+
+
+@dataclass
+class NodeResult(BaseResult):
+    node: ResultNode
+
+
+@dataclass
+class ExecutionResult(dbtClassMixin):
+    results: Sequence[BaseResult]
+    elapsed_time: float
+
+    def __len__(self):
+        return len(self.results)
+
+    def __iter__(self):
+        return iter(self.results)
+
+    def __getitem__(self, idx):
+        return self.results[idx]
+
+
+# due to issues with typing.Union collapsing subclasses, this can't subclass
+# PartialResult

dbt/artifacts/schemas/run/__init__.py

@@ -0,0 +1,2 @@
+# alias to latest
+from dbt.artifacts.schemas.run.v5.run import *  # noqa

dbt/artifacts/schemas/run/v5/run.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import copy
+import threading
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, Iterable, Optional, Sequence, Tuple
+
+# https://github.com/dbt-labs/dbt-core/issues/10098
+# Needed for Mashumaro serialization of RunResult below
+# TODO: investigate alternative approaches to restore conditional import
+# if TYPE_CHECKING:
+import agate
+
+from dbt.artifacts.resources import CompiledResource
+from dbt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    get_artifact_schema_version,
+    schema_version,
+)
+from dbt.artifacts.schemas.batch_results import BatchResults
+from dbt.artifacts.schemas.results import (
+    BaseResult,
+    ExecutionResult,
+    NodeResult,
+    ResultNode,
+    RunStatus,
+)
+from dbt.exceptions import scrub_secrets
+from dbt_common.clients.system import write_json
+from dbt_common.constants import SECRET_ENV_PREFIX
+
+
+@dataclass
+class RunResult(NodeResult):
+    agate_table: Optional["agate.Table"] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
+    batch_results: Optional[BatchResults] = None
+
+    @property
+    def skipped(self):
+        return self.status == RunStatus.Skipped
+
+    @classmethod
+    def from_node(cls, node: ResultNode, status: RunStatus, message: Optional[str]):
+        thread_id = threading.current_thread().name
+        return RunResult(
+            status=status,
+            thread_id=thread_id,
+            execution_time=0,
+            timing=[],
+            message=message,
+            node=node,
+            adapter_response={},
+            failures=None,
+            batch_results=None,
+        )
+
+
+@dataclass
+class RunResultsMetadata(BaseArtifactMetadata):
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(RunResultsArtifact.dbt_schema_version)
+    )
+
+
+@dataclass
+class RunResultOutput(BaseResult):
+    unique_id: str
+    compiled: Optional[bool]
+    compiled_code: Optional[str]
+    relation_name: Optional[str]
+    batch_results: Optional[BatchResults] = None
+
+
+def process_run_result(result: RunResult) -> RunResultOutput:
+
+    compiled = isinstance(result.node, CompiledResource)
+
+    return RunResultOutput(
+        unique_id=result.node.unique_id,
+        status=result.status,
+        timing=result.timing,
+        thread_id=result.thread_id,
+        execution_time=result.execution_time,
+        message=result.message,
+        adapter_response=result.adapter_response,
+        failures=result.failures,
+        batch_results=result.batch_results,
+        compiled=result.node.compiled if compiled else None,  # type:ignore
+        compiled_code=result.node.compiled_code if compiled else None,  # type:ignore
+        relation_name=result.node.relation_name if compiled else None,  # type:ignore
+    )
+
+
+@dataclass
+class RunExecutionResult(
+    ExecutionResult,
+):
+    results: Sequence[RunResult]
+    args: Dict[str, Any] = field(default_factory=dict)
+    generated_at: datetime = field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)
+    )
+
+    def write(self, path: str):
+        writable = RunResultsArtifact.from_execution_results(
+            results=self.results,
+            elapsed_time=self.elapsed_time,
+            generated_at=self.generated_at,
+            args=self.args,
+        )
+        writable.write(path)
+
+
+@dataclass
+@schema_version("run-results", 6)
+class RunResultsArtifact(ExecutionResult, ArtifactMixin):
+    results: Sequence[RunResultOutput]
+    args: Dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_execution_results(
+        cls,
+        results: Sequence[RunResult],
+        elapsed_time: float,
+        generated_at: datetime,
+        args: Dict,
+    ):
+        processed_results = [
+            process_run_result(result) for result in results if isinstance(result, RunResult)
+        ]
+        meta = RunResultsMetadata(
+            dbt_schema_version=str(cls.dbt_schema_version),
+            generated_at=generated_at,
+        )
+
+        secret_vars = [
+            v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()
+        ]
+
+        scrubbed_args = copy.deepcopy(args)
+
+        # scrub secrets in invocation command
+        scrubbed_args["invocation_command"] = scrub_secrets(
+            scrubbed_args["invocation_command"], secret_vars
+        )
+
+        # scrub secrets in vars dict
+        scrubbed_args["vars"] = {
+            k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items()
+        }
+
+        return cls(
+            metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args
+        )
+
+    @classmethod
+    def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
+        return [
+            ("run-results", 4),
+            ("run-results", 5),
+        ]
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This overrides the "upgrade_schema_version" call in VersionedSchema (via
+        ArtifactMixin) to modify the dictionary passed in from earlier versions of the run_results.
+        """
+        run_results_schema_version = get_artifact_schema_version(data)
+        # If less than the current version (v5), preprocess contents to match latest schema version
+        if run_results_schema_version <= 5:
+            # In v5, we added 'compiled' attributes to each result entry
+            # Going forward, dbt expects these to be populated
+            for result in data["results"]:
+                result["compiled"] = False
+                result["compiled_code"] = ""
+                result["relation_name"] = ""
+        return cls.from_dict(data)
+
+    def write(self, path: str):
+        write_json(path, self.to_dict(omit_none=False))