dvt-core 1.11.0b4__py3-none-any.whl

dvt/artifacts/schemas/base.py

@@ -0,0 +1,191 @@
+import dataclasses
+import functools
+from datetime import datetime, timezone
+from typing import Any, ClassVar, Dict, Optional, Type, TypeVar
+
+from dvt.artifacts.exceptions import IncompatibleSchemaError
+from dvt.version import __version__
+from mashumaro.jsonschema import build_json_schema
+from mashumaro.jsonschema.dialects import DRAFT_2020_12
+
+from dbt_common.clients.system import read_json, write_json
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.events.functions import get_metadata_vars
+from dbt_common.exceptions import DbtInternalError, DbtRuntimeError
+from dbt_common.invocation import get_invocation_id, get_invocation_started_at
+
+BASE_SCHEMAS_URL = "https://schemas.getdbt.com/"
+SCHEMA_PATH = "dbt/{name}/v{version}.json"
+
+
+@dataclasses.dataclass
+class SchemaVersion:
+    name: str
+    version: int
+
+    @property
+    def path(self) -> str:
+        return SCHEMA_PATH.format(name=self.name, version=self.version)
+
+    def __str__(self) -> str:
+        return BASE_SCHEMAS_URL + self.path
+
+
+class Writable:
+    def write(self, path: str):
+        write_json(path, self.to_dict(omit_none=False, context={"artifact": True}))  # type: ignore
+
+
+class Readable:
+    @classmethod
+    def read(cls, path: str):
+        try:
+            data = read_json(path)
+        except (EnvironmentError, ValueError) as exc:
+            raise DbtRuntimeError(
+                f'Could not read {cls.__name__} at "{path}" as JSON: {exc}'
+            ) from exc
+
+        return cls.from_dict(data)  # type: ignore
+
+
+# This is used in the ManifestMetadata, RunResultsMetadata, RunOperationResultMetadata,
+# FreshnessMetadata, and CatalogMetadata classes
+@dataclasses.dataclass
+class BaseArtifactMetadata(dbtClassMixin):
+    dbt_schema_version: str
+    dbt_version: str = __version__
+    generated_at: datetime = dataclasses.field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)
+    )
+    invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id)
+    invocation_started_at: Optional[datetime] = dataclasses.field(
+        default_factory=get_invocation_started_at
+    )
+    env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars)
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+        if dct["generated_at"] and dct["generated_at"].endswith("+00:00"):
+            dct["generated_at"] = dct["generated_at"].replace("+00:00", "") + "Z"
+        return dct
+
+
+# This is used as a class decorator to set the schema_version in the
+# 'dbt_schema_version' class attribute. (It's copied into the metadata objects.)
+# Name attributes of SchemaVersion in classes with the 'schema_version' decorator:
+#   manifest
+#   run-results
+#   run-operation-result
+#   sources
+#   catalog
+#   remote-compile-result
+#   remote-execution-result
+#   remote-run-result
+S = TypeVar("S", bound="VersionedSchema")
+
+
+def schema_version(name: str, version: int):
+    def inner(cls: Type[S]):
+        cls.dbt_schema_version = SchemaVersion(
+            name=name,
+            version=version,
+        )
+        return cls
+
+    return inner
+
+
+# This is used in the ArtifactMixin and RemoteCompileResultMixin classes
+@dataclasses.dataclass
+class VersionedSchema(dbtClassMixin):
+    dbt_schema_version: ClassVar[SchemaVersion]
+
+    @classmethod
+    @functools.lru_cache
+    def json_schema(cls) -> Dict[str, Any]:
+        json_schema_obj = build_json_schema(cls, dialect=DRAFT_2020_12, with_dialect_uri=True)
+        json_schema = json_schema_obj.to_dict()
+        json_schema["$id"] = str(cls.dbt_schema_version)
+        return json_schema
+
+    @classmethod
+    def is_compatible_version(cls, schema_version):
+        compatible_versions = [str(cls.dbt_schema_version)]
+        if hasattr(cls, "compatible_previous_versions"):
+            for name, version in cls.compatible_previous_versions():
+                compatible_versions.append(str(SchemaVersion(name, version)))
+        return str(schema_version) in compatible_versions
+
+    @classmethod
+    def read_and_check_versions(cls, path: str):
+        try:
+            data = read_json(path)
+        except (EnvironmentError, ValueError) as exc:
+            raise DbtRuntimeError(
+                f'Could not read {cls.__name__} at "{path}" as JSON: {exc}'
+            ) from exc
+
+        # Check metadata version. There is a class variable 'dbt_schema_version', but
+        # that doesn't show up in artifacts, where it only exists in the 'metadata'
+        # dictionary.
+        if hasattr(cls, "dbt_schema_version"):
+            if "metadata" in data and "dbt_schema_version" in data["metadata"]:
+                previous_schema_version = data["metadata"]["dbt_schema_version"]
+                # cls.dbt_schema_version is a SchemaVersion object
+                if not cls.is_compatible_version(previous_schema_version):
+                    raise IncompatibleSchemaError(
+                        expected=str(cls.dbt_schema_version),
+                        found=previous_schema_version,
+                    )
+
+        return cls.upgrade_schema_version(data)
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This will modify the data (dictionary) passed in to match the current
+        artifact schema code, if necessary. This is the default method, which
+        just returns the instantiated object via from_dict."""
+        return cls.from_dict(data)
+
+
+T = TypeVar("T", bound="ArtifactMixin")
+
+
+# metadata should really be a Generic[T_M] where T_M is a TypeVar bound to
+# BaseArtifactMetadata. Unfortunately this isn't possible due to a mypy issue:
+# https://github.com/python/mypy/issues/7520
+# This is used in the WritableManifest, RunResultsArtifact, RunOperationResultsArtifact,
+# and CatalogArtifact
+@dataclasses.dataclass(init=False)
+class ArtifactMixin(VersionedSchema, Writable, Readable):
+    metadata: BaseArtifactMetadata
+
+    @classmethod
+    def validate(cls, data):
+        super().validate(data)
+        if cls.dbt_schema_version is None:
+            raise DbtInternalError("Cannot call from_dict with no schema version!")
+
+
+def get_artifact_schema_version(dct: dict) -> int:
+    schema_version = dct.get("metadata", {}).get("dbt_schema_version", None)
+    if not schema_version:
+        raise ValueError("Artifact is missing schema version")
+
+    # schema_version is in this format: https://schemas.getdbt.com/dbt/manifest/v10.json
+    # What the code below is doing:
+    # 1. Split on "/" – v10.json
+    # 2. Split on "." – v10
+    # 3. Skip first character – 10
+    # 4. Convert to int
+    # TODO: If this gets more complicated, turn into a regex
+    return int(schema_version.split("/")[-1].split(".")[0][1:])
+
+
+def get_artifact_dbt_version(dct: dict) -> Optional[str]:
+    dbt_version = dct.get("metadata", {}).get("dbt_version", None)
+    if dbt_version is None:
+        return None
+
+    return str(dbt_version)

dvt/artifacts/schemas/batch_results.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Tuple
+
+from dbt_common.dataclass_schema import dbtClassMixin
+
+BatchType = Tuple[datetime, datetime]
+
+
+@dataclass
+class BatchResults(dbtClassMixin):
+    successful: List[BatchType] = field(default_factory=list)
+    failed: List[BatchType] = field(default_factory=list)
+
+    def __add__(self, other: BatchResults) -> BatchResults:
+        return BatchResults(
+            successful=self.successful + other.successful,
+            failed=self.failed + other.failed,
+        )
+
+    def __len__(self):
+        return len(self.successful) + len(self.failed)

dvt/artifacts/schemas/catalog/__init__.py

@@ -0,0 +1,12 @@
+# alias to latest
+from dvt.artifacts.schemas.catalog.v1.catalog import *  # noqa
+
+from dbt_common.contracts.metadata import (
+    CatalogKey,
+    CatalogTable,
+    ColumnMap,
+    ColumnMetadata,
+    StatsDict,
+    StatsItem,
+    TableMetadata,
+)

dvt/artifacts/schemas/catalog/v1/catalog.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+from dvt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    schema_version,
+)
+
+from dbt_common.contracts.metadata import CatalogTable
+from dbt_common.dataclass_schema import dbtClassMixin
+
+Primitive = Union[bool, str, float, None]
+PrimitiveDict = Dict[str, Primitive]
+
+
+@dataclass
+class CatalogMetadata(BaseArtifactMetadata):
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(CatalogArtifact.dbt_schema_version)
+    )
+
+
+@dataclass
+class CatalogResults(dbtClassMixin):
+    nodes: Dict[str, CatalogTable]
+    sources: Dict[str, CatalogTable]
+    errors: Optional[List[str]] = None
+    _compile_results: Optional[Any] = None
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+        if "_compile_results" in dct:
+            del dct["_compile_results"]
+        return dct
+
+
+@dataclass
+@schema_version("catalog", 1)
+class CatalogArtifact(CatalogResults, ArtifactMixin):
+    metadata: CatalogMetadata
+
+    @classmethod
+    def from_results(
+        cls,
+        generated_at: datetime,
+        nodes: Dict[str, CatalogTable],
+        sources: Dict[str, CatalogTable],
+        compile_results: Optional[Any],
+        errors: Optional[List[str]],
+    ) -> "CatalogArtifact":
+        meta = CatalogMetadata(generated_at=generated_at)
+        return cls(
+            metadata=meta,
+            nodes=nodes,
+            sources=sources,
+            errors=errors,
+            _compile_results=compile_results,
+        )

dvt/artifacts/schemas/freshness/__init__.py

@@ -0,0 +1 @@
+from dvt.artifacts.schemas.freshness.v3.freshness import *  # noqa

dvt/artifacts/schemas/freshness/v3/freshness.py

@@ -0,0 +1,159 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from dvt.artifacts.resources import FreshnessThreshold
+from dvt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    VersionedSchema,
+    schema_version,
+)
+from dvt.artifacts.schemas.results import (
+    ExecutionResult,
+    FreshnessStatus,
+    NodeResult,
+    TimingInfo,
+)
+from dvt.contracts.graph.nodes import SourceDefinition
+
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+from dbt_common.exceptions import DbtInternalError
+
+
+@dataclass
+class SourceFreshnessResult(NodeResult):
+    node: SourceDefinition
+    status: FreshnessStatus
+    max_loaded_at: datetime
+    snapshotted_at: datetime
+    age: float
+
+    @property
+    def skipped(self):
+        return False
+
+
+@dataclass
+class PartialSourceFreshnessResult(NodeResult):
+    status: FreshnessStatus
+
+    @property
+    def skipped(self):
+        return False
+
+
+FreshnessNodeResult = Union[PartialSourceFreshnessResult, SourceFreshnessResult]
+
+
+@dataclass
+class FreshnessMetadata(BaseArtifactMetadata):
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(FreshnessExecutionResultArtifact.dbt_schema_version)
+    )
+
+
+@dataclass
+class FreshnessResult(ExecutionResult):
+    metadata: FreshnessMetadata
+    results: Sequence[FreshnessNodeResult]
+
+    @classmethod
+    def from_node_results(
+        cls,
+        results: List[FreshnessNodeResult],
+        elapsed_time: float,
+        generated_at: datetime,
+    ):
+        meta = FreshnessMetadata(generated_at=generated_at)
+        return cls(metadata=meta, results=results, elapsed_time=elapsed_time)
+
+    def write(self, path):
+        FreshnessExecutionResultArtifact.from_result(self).write(path)
+
+
+@dataclass
+class SourceFreshnessOutput(dbtClassMixin):
+    unique_id: str
+    max_loaded_at: datetime
+    snapshotted_at: datetime
+    max_loaded_at_time_ago_in_s: float
+    status: FreshnessStatus
+    criteria: FreshnessThreshold
+    adapter_response: Dict[str, Any]
+    timing: List[TimingInfo]
+    thread_id: str
+    execution_time: float
+
+
+class FreshnessErrorEnum(StrEnum):
+    runtime_error = "runtime error"
+
+
+@dataclass
+class SourceFreshnessRuntimeError(dbtClassMixin):
+    unique_id: str
+    error: Optional[Union[str, int]]
+    status: FreshnessErrorEnum
+
+
+FreshnessNodeOutput = Union[SourceFreshnessRuntimeError, SourceFreshnessOutput]
+
+
+@dataclass
+@schema_version("sources", 3)
+class FreshnessExecutionResultArtifact(
+    ArtifactMixin,
+    VersionedSchema,
+):
+    metadata: FreshnessMetadata
+    results: Sequence[FreshnessNodeOutput]
+    elapsed_time: float
+
+    @classmethod
+    def from_result(cls, base: FreshnessResult):
+        processed = [
+            process_freshness_result(r)
+            for r in base.results
+            if isinstance(r, SourceFreshnessResult)
+        ]
+        return cls(
+            metadata=base.metadata,
+            results=processed,
+            elapsed_time=base.elapsed_time,
+        )
+
+
+def process_freshness_result(result: FreshnessNodeResult) -> FreshnessNodeOutput:
+    unique_id = result.node.unique_id
+    if result.status == FreshnessStatus.RuntimeErr:
+        return SourceFreshnessRuntimeError(
+            unique_id=unique_id,
+            error=result.message,
+            status=FreshnessErrorEnum.runtime_error,
+        )
+
+    # we know that this must be a SourceFreshnessResult
+    if not isinstance(result, SourceFreshnessResult):
+        raise DbtInternalError(
+            "Got {} instead of a SourceFreshnessResult for a "
+            "non-error result in freshness execution!".format(type(result))
+        )
+    # if we're here, we must have a non-None freshness threshold
+    criteria = result.node.freshness
+    if criteria is None:
+        raise DbtInternalError(
+            "Somehow evaluated a freshness result for a source that has no freshness criteria!"
+        )
+    return SourceFreshnessOutput(
+        unique_id=unique_id,
+        max_loaded_at=result.max_loaded_at,
+        snapshotted_at=result.snapshotted_at,
+        max_loaded_at_time_ago_in_s=result.age,
+        status=result.status,
+        criteria=criteria,
+        adapter_response=result.adapter_response,
+        timing=result.timing,
+        thread_id=result.thread_id,
+        execution_time=result.execution_time,
+    )

dvt/artifacts/schemas/manifest/__init__.py

@@ -0,0 +1,2 @@
+# alias to latest
+from dvt.artifacts.schemas.manifest.v12.manifest import *  # noqa

dvt/artifacts/schemas/manifest/v12/manifest.py

@@ -0,0 +1,212 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union
+from uuid import UUID
+
+from dvt import tracking
+from dvt.artifacts.resources import (
+    Analysis,
+    Documentation,
+    Exposure,
+    Function,
+    GenericTest,
+    Group,
+    HookNode,
+    Macro,
+    Metric,
+    Model,
+    SavedQuery,
+    Seed,
+    SemanticModel,
+    SingularTest,
+    Snapshot,
+    SourceDefinition,
+    SqlOperation,
+    UnitTestDefinition,
+)
+from dvt.artifacts.resources.v1.components import Quoting
+from dvt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    get_artifact_dbt_version,
+    get_artifact_schema_version,
+    schema_version,
+)
+from dvt.artifacts.schemas.upgrades import (
+    upgrade_manifest_json,
+    upgrade_manifest_json_dbt_version,
+)
+from dvt.version import __version__
+
+from dbt_common.exceptions import DbtInternalError
+
+NodeEdgeMap = Dict[str, List[str]]
+UniqueID = str
+ManifestResource = Union[
+    Seed, Analysis, SingularTest, HookNode, Model, SqlOperation, GenericTest, Snapshot, Function
+]
+DisabledManifestResource = Union[
+    ManifestResource,
+    SourceDefinition,
+    Exposure,
+    Metric,
+    SavedQuery,
+    SemanticModel,
+    UnitTestDefinition,
+]
+
+
+@dataclass
+class ManifestMetadata(BaseArtifactMetadata):
+    """Metadata for the manifest."""
+
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(WritableManifest.dbt_schema_version)
+    )
+    project_name: Optional[str] = field(
+        default=None,
+        metadata={
+            "description": "Name of the root project",
+        },
+    )
+    project_id: Optional[str] = field(
+        default=None,
+        metadata={
+            "description": "A unique identifier for the project, hashed from the project name",
+        },
+    )
+    user_id: Optional[UUID] = field(
+        default=None,
+        metadata={
+            "description": "A unique identifier for the user",
+        },
+    )
+    send_anonymous_usage_stats: Optional[bool] = field(
+        default=None,
+        metadata=dict(
+            description=("Whether dbt is configured to send anonymous usage statistics")
+        ),
+    )
+    adapter_type: Optional[str] = field(
+        default=None,
+        metadata=dict(description="The type name of the adapter"),
+    )
+    quoting: Optional[Quoting] = field(
+        default_factory=Quoting,
+        metadata=dict(description="The quoting configuration for the project"),
+    )
+    run_started_at: Optional[datetime] = field(
+        default=tracking.active_user.run_started_at if tracking.active_user is not None else None,
+        metadata=dict(description="The timestamp when the run started"),
+    )
+
+    @classmethod
+    def default(cls):
+        return cls(
+            dbt_schema_version=str(WritableManifest.dbt_schema_version),
+        )
+
+
+@dataclass
+@schema_version("manifest", 12)
+class WritableManifest(ArtifactMixin):
+    nodes: Mapping[UniqueID, ManifestResource] = field(
+        metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
+    )
+    sources: Mapping[UniqueID, SourceDefinition] = field(
+        metadata=dict(description=("The sources defined in the dbt project and its dependencies"))
+    )
+    macros: Mapping[UniqueID, Macro] = field(
+        metadata=dict(description=("The macros defined in the dbt project and its dependencies"))
+    )
+    docs: Mapping[UniqueID, Documentation] = field(
+        metadata=dict(description=("The docs defined in the dbt project and its dependencies"))
+    )
+    exposures: Mapping[UniqueID, Exposure] = field(
+        metadata=dict(
+            description=("The exposures defined in the dbt project and its dependencies")
+        )
+    )
+    metrics: Mapping[UniqueID, Metric] = field(
+        metadata=dict(description=("The metrics defined in the dbt project and its dependencies"))
+    )
+    groups: Mapping[UniqueID, Group] = field(
+        metadata=dict(description=("The groups defined in the dbt project"))
+    )
+    selectors: Mapping[UniqueID, Any] = field(
+        metadata=dict(description=("The selectors defined in selectors.yml"))
+    )
+    disabled: Optional[Mapping[UniqueID, List[DisabledManifestResource]]] = field(
+        metadata=dict(description="A mapping of the disabled nodes in the target")
+    )
+    parent_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from child nodes to their dependencies",
+        )
+    )
+    child_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from parent nodes to their dependents",
+        )
+    )
+    group_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from group names to their nodes",
+        )
+    )
+    saved_queries: Mapping[UniqueID, SavedQuery] = field(
+        metadata=dict(description=("The saved queries defined in the dbt project"))
+    )
+    semantic_models: Mapping[UniqueID, SemanticModel] = field(
+        metadata=dict(description=("The semantic models defined in the dbt project"))
+    )
+    metadata: ManifestMetadata = field(
+        metadata=dict(
+            description="Metadata about the manifest",
+        )
+    )
+    unit_tests: Mapping[UniqueID, UnitTestDefinition] = field(
+        metadata=dict(
+            description="The unit tests defined in the project",
+        )
+    )
+    functions: Mapping[UniqueID, Function] = field(
+        default_factory=dict,
+        metadata=dict(description=("The functions defined in the dbt project")),
+    )
+
+    @classmethod
+    def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
+        return [
+            ("manifest", 4),
+            ("manifest", 5),
+            ("manifest", 6),
+            ("manifest", 7),
+            ("manifest", 8),
+            ("manifest", 9),
+            ("manifest", 10),
+            ("manifest", 11),
+        ]
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This overrides the "upgrade_schema_version" call in VersionedSchema (via
+        ArtifactMixin) to modify the dictionary passed in from earlier versions of the manifest."""
+        manifest_schema_version = get_artifact_schema_version(data)
+        if manifest_schema_version < cls.dbt_schema_version.version:
+            data = upgrade_manifest_json(data, manifest_schema_version)
+
+        manifest_dbt_version = get_artifact_dbt_version(data)
+        if manifest_dbt_version and manifest_dbt_version != __version__:
+            data = upgrade_manifest_json_dbt_version(data)
+        return cls.from_dict(data)
+
+    @classmethod
+    def validate(cls, _):
+        # When dbt try to load an artifact with additional optional fields
+        # that are not present in the schema, from_dict will work fine.
+        # As long as validate is not called, the schema will not be enforced.
+        # This is intentional, as it allows for safer schema upgrades.
+        raise DbtInternalError(
+            "The WritableManifest should never be validated directly to allow for schema upgrades."
+        )