dvt-core 0.52.2__cp310-cp310-macosx_10_9_x86_64.whl

dbt/artifacts/resources/v1/singular_test.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass, field
+from typing import Literal
+
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import CompiledResource
+from dbt.artifacts.resources.v1.config import TestConfig
+
+
+@dataclass
+class SingularTest(CompiledResource):
+    resource_type: Literal[NodeType.Test]
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type: ignore

dbt/artifacts/resources/v1/snapshot.py

@@ -0,0 +1,91 @@
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Optional, Union
+
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import CompiledResource, DeferRelation
+from dbt.artifacts.resources.v1.config import NodeConfig
+from dbt_common.dataclass_schema import ValidationError, dbtClassMixin
+
+
+@dataclass
+class SnapshotMetaColumnNames(dbtClassMixin):
+    dbt_valid_to: Optional[str] = None
+    dbt_valid_from: Optional[str] = None
+    dbt_scd_id: Optional[str] = None
+    dbt_updated_at: Optional[str] = None
+    dbt_is_deleted: Optional[str] = None
+
+
+@dataclass
+class SnapshotConfig(NodeConfig):
+    materialized: str = "snapshot"
+    strategy: Optional[str] = None
+    unique_key: Union[str, List[str], None] = None
+    target_schema: Optional[str] = None
+    target_database: Optional[str] = None
+    updated_at: Optional[str] = None
+    # Not using Optional because of serialization issues with a Union of str and List[str]
+    check_cols: Union[str, List[str], None] = None
+    snapshot_meta_column_names: SnapshotMetaColumnNames = field(
+        default_factory=SnapshotMetaColumnNames
+    )
+    dbt_valid_to_current: Optional[str] = None
+
+    @property
+    def snapshot_table_column_names(self):
+        return {
+            "dbt_valid_from": self.snapshot_meta_column_names.dbt_valid_from or "dbt_valid_from",
+            "dbt_valid_to": self.snapshot_meta_column_names.dbt_valid_to or "dbt_valid_to",
+            "dbt_scd_id": self.snapshot_meta_column_names.dbt_scd_id or "dbt_scd_id",
+            "dbt_updated_at": self.snapshot_meta_column_names.dbt_updated_at or "dbt_updated_at",
+            "dbt_is_deleted": self.snapshot_meta_column_names.dbt_is_deleted or "dbt_is_deleted",
+        }
+
+    def final_validate(self):
+        if not self.strategy or not self.unique_key:
+            raise ValidationError(
+                "Snapshots must be configured with a 'strategy' and 'unique_key'."
+            )
+        if self.strategy == "check":
+            if not self.check_cols:
+                raise ValidationError(
+                    "A snapshot configured with the check strategy must "
+                    "specify a check_cols configuration."
+                )
+            if isinstance(self.check_cols, str) and self.check_cols != "all":
+                raise ValidationError(
+                    f"Invalid value for 'check_cols': {self.check_cols}. "
+                    "Expected 'all' or a list of strings."
+                )
+        elif self.strategy == "timestamp":
+            if not self.updated_at:
+                raise ValidationError(
+                    "A snapshot configured with the timestamp strategy "
+                    "must specify an updated_at configuration."
+                )
+            if self.check_cols:
+                raise ValidationError("A 'timestamp' snapshot should not have 'check_cols'")
+        # If the strategy is not 'check' or 'timestamp' it's a custom strategy,
+        # formerly supported with GenericSnapshotConfig
+
+        if self.materialized and self.materialized != "snapshot":
+            raise ValidationError("A snapshot must have a materialized value of 'snapshot'")
+
+    # Called by "calculate_node_config_dict" in ContextConfigGenerator
+    def finalize_and_validate(self):
+        data = self.to_dict(omit_none=True)
+        self.validate(data)
+        return self.from_dict(data)
+
+
+@dataclass
+class Snapshot(CompiledResource):
+    resource_type: Literal[NodeType.Snapshot]
+    config: SnapshotConfig
+    defer_relation: Optional[DeferRelation] = None
+
+    def __post_serialize__(self, dct, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+        if context and context.get("artifact") and "defer_relation" in dct:
+            del dct["defer_relation"]
+        return dct

dbt/artifacts/resources/v1/source_definition.py

@@ -0,0 +1,84 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from dbt.artifacts.resources.base import GraphResource
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import (
+    ColumnInfo,
+    FreshnessThreshold,
+    HasRelationMetadata,
+    Quoting,
+)
+from dbt.artifacts.resources.v1.config import BaseConfig, MergeBehavior
+from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed
+from dbt_common.contracts.util import Mergeable
+from dbt_common.exceptions import CompilationError
+
+
+@dataclass
+class SourceConfig(BaseConfig):
+    enabled: bool = True
+    event_time: Any = None
+    freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold)
+    loaded_at_field: Optional[str] = None
+    loaded_at_query: Optional[str] = None
+    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
+    tags: List[str] = field(default_factory=list)
+
+
+@dataclass
+class ExternalPartition(AdditionalPropertiesAllowed):
+    name: str = ""
+    description: str = ""
+    data_type: str = ""
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        if self.name == "" or self.data_type == "":
+            raise CompilationError("External partition columns must have names and data types")
+
+
+@dataclass
+class ExternalTable(AdditionalPropertiesAllowed, Mergeable):
+    location: Optional[str] = None
+    file_format: Optional[str] = None
+    row_format: Optional[str] = None
+    tbl_properties: Optional[str] = None
+    partitions: Optional[Union[List[str], List[ExternalPartition]]] = None
+
+    def __bool__(self):
+        return self.location is not None
+
+
+@dataclass
+class ParsedSourceMandatory(GraphResource, HasRelationMetadata):
+    source_name: str
+    source_description: str
+    loader: str
+    identifier: str
+    resource_type: Literal[NodeType.Source]
+
+
+@dataclass
+class SourceDefinition(ParsedSourceMandatory):
+    quoting: Quoting = field(default_factory=Quoting)
+    loaded_at_field: Optional[str] = None
+    loaded_at_query: Optional[str] = None
+    freshness: Optional[FreshnessThreshold] = None
+    external: Optional[ExternalTable] = None
+    description: str = ""
+    columns: Dict[str, ColumnInfo] = field(default_factory=dict)
+    meta: Dict[str, Any] = field(default_factory=dict)
+    source_meta: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+    config: SourceConfig = field(default_factory=SourceConfig)
+    patch_path: Optional[str] = None
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    relation_name: Optional[str] = None
+    created_at: float = field(default_factory=lambda: time.time())
+    unrendered_database: Optional[str] = None
+    unrendered_schema: Optional[str] = None
+    doc_blocks: List[str] = field(default_factory=list)
+    # DVT: Connection name from profiles.yml for this source
+    connection: Optional[str] = None

dbt/artifacts/resources/v1/sql_operation.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import Literal
+
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import CompiledResource
+
+
+@dataclass
+class SqlOperation(CompiledResource):
+    resource_type: Literal[NodeType.SqlOperation]

dbt/artifacts/resources/v1/unit_test_definition.py

@@ -0,0 +1,77 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from dbt.artifacts.resources import DependsOn, NodeVersion
+from dbt.artifacts.resources.base import GraphResource
+from dbt.artifacts.resources.v1.config import list_str, metas
+from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
+from dbt_common.contracts.config.metadata import ShowBehavior
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+
+
+@dataclass
+class UnitTestConfig(BaseConfig):
+    tags: Union[str, List[str]] = field(
+        default_factory=list_str,
+        metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
+    )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    enabled: bool = True
+
+
+class UnitTestFormat(StrEnum):
+    CSV = "csv"
+    Dict = "dict"
+    SQL = "sql"
+
+
+@dataclass
+class UnitTestInputFixture(dbtClassMixin):
+    input: str
+    rows: Optional[Union[str, List[Dict[str, Any]]]] = None
+    format: UnitTestFormat = UnitTestFormat.Dict
+    fixture: Optional[str] = None
+
+
+@dataclass
+class UnitTestOverrides(dbtClassMixin):
+    macros: Dict[str, Any] = field(default_factory=dict)
+    vars: Dict[str, Any] = field(default_factory=dict)
+    env_vars: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class UnitTestNodeVersions(dbtClassMixin):
+    include: Optional[List[NodeVersion]] = None
+    exclude: Optional[List[NodeVersion]] = None
+
+
+@dataclass
+class UnitTestOutputFixture(dbtClassMixin):
+    rows: Optional[Union[str, List[Dict[str, Any]]]] = None
+    format: UnitTestFormat = UnitTestFormat.Dict
+    fixture: Optional[str] = None
+
+
+@dataclass
+class UnitTestDefinitionMandatory:
+    model: str
+    given: Sequence[UnitTestInputFixture]
+    expect: UnitTestOutputFixture
+
+
+@dataclass
+class UnitTestDefinition(GraphResource, UnitTestDefinitionMandatory):
+    description: str = ""
+    overrides: Optional[UnitTestOverrides] = None
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    config: UnitTestConfig = field(default_factory=UnitTestConfig)
+    checksum: Optional[str] = None
+    schema: Optional[str] = None
+    created_at: float = field(default_factory=lambda: time.time())
+    versions: Optional[UnitTestNodeVersions] = None
+    version: Optional[NodeVersion] = None

dbt/artifacts/schemas/base.py

@@ -0,0 +1,191 @@
+import dataclasses
+import functools
+from datetime import datetime, timezone
+from typing import Any, ClassVar, Dict, Optional, Type, TypeVar
+
+from mashumaro.jsonschema import build_json_schema
+from mashumaro.jsonschema.dialects import DRAFT_2020_12
+
+from dbt.artifacts.exceptions import IncompatibleSchemaError
+from dbt.version import __version__
+from dbt_common.clients.system import read_json, write_json
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.events.functions import get_metadata_vars
+from dbt_common.exceptions import DbtInternalError, DbtRuntimeError
+from dbt_common.invocation import get_invocation_id, get_invocation_started_at
+
+BASE_SCHEMAS_URL = "https://schemas.getdbt.com/"
+SCHEMA_PATH = "dbt/{name}/v{version}.json"
+
+
+@dataclasses.dataclass
+class SchemaVersion:
+    name: str
+    version: int
+
+    @property
+    def path(self) -> str:
+        return SCHEMA_PATH.format(name=self.name, version=self.version)
+
+    def __str__(self) -> str:
+        return BASE_SCHEMAS_URL + self.path
+
+
+class Writable:
+    def write(self, path: str):
+        write_json(path, self.to_dict(omit_none=False, context={"artifact": True}))  # type: ignore
+
+
+class Readable:
+    @classmethod
+    def read(cls, path: str):
+        try:
+            data = read_json(path)
+        except (EnvironmentError, ValueError) as exc:
+            raise DbtRuntimeError(
+                f'Could not read {cls.__name__} at "{path}" as JSON: {exc}'
+            ) from exc
+
+        return cls.from_dict(data)  # type: ignore
+
+
+# This is used in the ManifestMetadata, RunResultsMetadata, RunOperationResultMetadata,
+# FreshnessMetadata, and CatalogMetadata classes
+@dataclasses.dataclass
+class BaseArtifactMetadata(dbtClassMixin):
+    dbt_schema_version: str
+    dbt_version: str = __version__
+    generated_at: datetime = dataclasses.field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)
+    )
+    invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id)
+    invocation_started_at: Optional[datetime] = dataclasses.field(
+        default_factory=get_invocation_started_at
+    )
+    env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars)
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+        if dct["generated_at"] and dct["generated_at"].endswith("+00:00"):
+            dct["generated_at"] = dct["generated_at"].replace("+00:00", "") + "Z"
+        return dct
+
+
+# This is used as a class decorator to set the schema_version in the
+# 'dbt_schema_version' class attribute. (It's copied into the metadata objects.)
+# Name attributes of SchemaVersion in classes with the 'schema_version' decorator:
+#   manifest
+#   run-results
+#   run-operation-result
+#   sources
+#   catalog
+#   remote-compile-result
+#   remote-execution-result
+#   remote-run-result
+S = TypeVar("S", bound="VersionedSchema")
+
+
+def schema_version(name: str, version: int):
+    def inner(cls: Type[S]):
+        cls.dbt_schema_version = SchemaVersion(
+            name=name,
+            version=version,
+        )
+        return cls
+
+    return inner
+
+
+# This is used in the ArtifactMixin and RemoteCompileResultMixin classes
+@dataclasses.dataclass
+class VersionedSchema(dbtClassMixin):
+    dbt_schema_version: ClassVar[SchemaVersion]
+
+    @classmethod
+    @functools.lru_cache
+    def json_schema(cls) -> Dict[str, Any]:
+        json_schema_obj = build_json_schema(cls, dialect=DRAFT_2020_12, with_dialect_uri=True)
+        json_schema = json_schema_obj.to_dict()
+        json_schema["$id"] = str(cls.dbt_schema_version)
+        return json_schema
+
+    @classmethod
+    def is_compatible_version(cls, schema_version):
+        compatible_versions = [str(cls.dbt_schema_version)]
+        if hasattr(cls, "compatible_previous_versions"):
+            for name, version in cls.compatible_previous_versions():
+                compatible_versions.append(str(SchemaVersion(name, version)))
+        return str(schema_version) in compatible_versions
+
+    @classmethod
+    def read_and_check_versions(cls, path: str):
+        try:
+            data = read_json(path)
+        except (EnvironmentError, ValueError) as exc:
+            raise DbtRuntimeError(
+                f'Could not read {cls.__name__} at "{path}" as JSON: {exc}'
+            ) from exc
+
+        # Check metadata version. There is a class variable 'dbt_schema_version', but
+        # that doesn't show up in artifacts, where it only exists in the 'metadata'
+        # dictionary.
+        if hasattr(cls, "dbt_schema_version"):
+            if "metadata" in data and "dbt_schema_version" in data["metadata"]:
+                previous_schema_version = data["metadata"]["dbt_schema_version"]
+                # cls.dbt_schema_version is a SchemaVersion object
+                if not cls.is_compatible_version(previous_schema_version):
+                    raise IncompatibleSchemaError(
+                        expected=str(cls.dbt_schema_version),
+                        found=previous_schema_version,
+                    )
+
+        return cls.upgrade_schema_version(data)
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This will modify the data (dictionary) passed in to match the current
+        artifact schema code, if necessary. This is the default method, which
+        just returns the instantiated object via from_dict."""
+        return cls.from_dict(data)
+
+
+T = TypeVar("T", bound="ArtifactMixin")
+
+
+# metadata should really be a Generic[T_M] where T_M is a TypeVar bound to
+# BaseArtifactMetadata. Unfortunately this isn't possible due to a mypy issue:
+# https://github.com/python/mypy/issues/7520
+# This is used in the WritableManifest, RunResultsArtifact, RunOperationResultsArtifact,
+# and CatalogArtifact
+@dataclasses.dataclass(init=False)
+class ArtifactMixin(VersionedSchema, Writable, Readable):
+    metadata: BaseArtifactMetadata
+
+    @classmethod
+    def validate(cls, data):
+        super().validate(data)
+        if cls.dbt_schema_version is None:
+            raise DbtInternalError("Cannot call from_dict with no schema version!")
+
+
+def get_artifact_schema_version(dct: dict) -> int:
+    schema_version = dct.get("metadata", {}).get("dbt_schema_version", None)
+    if not schema_version:
+        raise ValueError("Artifact is missing schema version")
+
+    # schema_version is in this format: https://schemas.getdbt.com/dbt/manifest/v10.json
+    # What the code below is doing:
+    # 1. Split on "/" – v10.json
+    # 2. Split on "." – v10
+    # 3. Skip first character – 10
+    # 4. Convert to int
+    # TODO: If this gets more complicated, turn into a regex
+    return int(schema_version.split("/")[-1].split(".")[0][1:])
+
+
+def get_artifact_dbt_version(dct: dict) -> Optional[str]:
+    dbt_version = dct.get("metadata", {}).get("dbt_version", None)
+    if dbt_version is None:
+        return None
+
+    return str(dbt_version)

dbt/artifacts/schemas/batch_results.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Tuple
+
+from dbt_common.dataclass_schema import dbtClassMixin
+
+BatchType = Tuple[datetime, datetime]
+
+
+@dataclass
+class BatchResults(dbtClassMixin):
+    successful: List[BatchType] = field(default_factory=list)
+    failed: List[BatchType] = field(default_factory=list)
+
+    def __add__(self, other: BatchResults) -> BatchResults:
+        return BatchResults(
+            successful=self.successful + other.successful,
+            failed=self.failed + other.failed,
+        )
+
+    def __len__(self):
+        return len(self.successful) + len(self.failed)

dbt/artifacts/schemas/catalog/__init__.py

@@ -0,0 +1,11 @@
+# alias to latest
+from dbt.artifacts.schemas.catalog.v1.catalog import *  # noqa
+from dbt_common.contracts.metadata import (
+    CatalogKey,
+    CatalogTable,
+    ColumnMap,
+    ColumnMetadata,
+    StatsDict,
+    StatsItem,
+    TableMetadata,
+)

dbt/artifacts/schemas/catalog/v1/catalog.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+from dbt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    schema_version,
+)
+from dbt_common.contracts.metadata import CatalogTable
+from dbt_common.dataclass_schema import dbtClassMixin
+
+Primitive = Union[bool, str, float, None]
+PrimitiveDict = Dict[str, Primitive]
+
+
+@dataclass
+class CatalogMetadata(BaseArtifactMetadata):
+    dbt_schema_version: str = field(
+        default_factory=lambda: str(CatalogArtifact.dbt_schema_version)
+    )
+
+
+@dataclass
+class CatalogResults(dbtClassMixin):
+    nodes: Dict[str, CatalogTable]
+    sources: Dict[str, CatalogTable]
+    errors: Optional[List[str]] = None
+    _compile_results: Optional[Any] = None
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+        if "_compile_results" in dct:
+            del dct["_compile_results"]
+        return dct
+
+
+@dataclass
+@schema_version("catalog", 1)
+class CatalogArtifact(CatalogResults, ArtifactMixin):
+    metadata: CatalogMetadata
+
+    @classmethod
+    def from_results(
+        cls,
+        generated_at: datetime,
+        nodes: Dict[str, CatalogTable],
+        sources: Dict[str, CatalogTable],
+        compile_results: Optional[Any],
+        errors: Optional[List[str]],
+    ) -> "CatalogArtifact":
+        meta = CatalogMetadata(generated_at=generated_at)
+        return cls(
+            metadata=meta,
+            nodes=nodes,
+            sources=sources,
+            errors=errors,
+            _compile_results=compile_results,
+        )

dbt/artifacts/schemas/freshness/__init__.py

@@ -0,0 +1 @@
+from dbt.artifacts.schemas.freshness.v3.freshness import *  # noqa