dvt-core 1.11.0b4__py3-none-any.whl

dvt/artifacts/resources/v1/components.py

@@ -0,0 +1,275 @@
+import time
+from dataclasses import dataclass, field
+from datetime import timedelta
+from typing import Any, Dict, List, Optional, Union
+
+from dvt.artifacts.resources.base import Docs, FileHash, GraphResource
+from dvt.artifacts.resources.types import NodeType, TimePeriod
+from dvt.artifacts.resources.v1.config import NodeConfig
+
+from dbt_common.contracts.config.base import BaseConfig, MergeBehavior
+from dbt_common.contracts.config.properties import AdditionalPropertiesMixin
+from dbt_common.contracts.constraints import ColumnLevelConstraint
+from dbt_common.contracts.util import Mergeable
+from dbt_common.dataclass_schema import ExtensibleDbtClassMixin, dbtClassMixin
+from dbt_semantic_interfaces.type_enums import TimeGranularity
+
+NodeVersion = Union[str, float]
+
+
+def _backcompat_doc_blocks(doc_blocks: Any) -> List[str]:
+    """
+    Make doc_blocks backwards-compatible for scenarios where a user specifies `doc_blocks` on a model or column.
+    Mashumaro will raise a serialization error if the specified `doc_blocks` isn't a list of strings.
+    In such a scenario, this method returns an empty list to avoid a serialization error.
+    Further along, `_get_doc_blocks` in `manifest.py` populates the correct `doc_blocks` for the happy path.
+    """
+
+    if isinstance(doc_blocks, list) and all(isinstance(x, str) for x in doc_blocks):
+        return doc_blocks
+
+    return []
+
+
+@dataclass
+class MacroDependsOn(dbtClassMixin):
+    macros: List[str] = field(default_factory=list)
+
+    # 'in' on lists is O(n) so this is O(n^2) for # of macros
+    def add_macro(self, value: str):
+        if value not in self.macros:
+            self.macros.append(value)
+
+
+@dataclass
+class DependsOn(MacroDependsOn):
+    nodes: List[str] = field(default_factory=list)
+
+    def add_node(self, value: str):
+        if value not in self.nodes:
+            self.nodes.append(value)
+
+
+@dataclass
+class RefArgs(dbtClassMixin):
+    name: str
+    package: Optional[str] = None
+    version: Optional[NodeVersion] = None
+
+    @property
+    def positional_args(self) -> List[str]:
+        if self.package:
+            return [self.package, self.name]
+        else:
+            return [self.name]
+
+    @property
+    def keyword_args(self) -> Dict[str, Optional[NodeVersion]]:
+        if self.version:
+            return {"version": self.version}
+        else:
+            return {}
+
+
+@dataclass
+class ColumnConfig(BaseConfig):
+    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
+    tags: List[str] = field(default_factory=list)
+
+
+@dataclass
+class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin):
+    """Used in all ManifestNodes and SourceDefinition"""
+
+    name: str
+    description: str = ""
+    meta: Dict[str, Any] = field(default_factory=dict)
+    data_type: Optional[str] = None
+    constraints: List[ColumnLevelConstraint] = field(default_factory=list)
+    quote: Optional[bool] = None
+    config: ColumnConfig = field(default_factory=ColumnConfig)
+    tags: List[str] = field(default_factory=list)
+    _extra: Dict[str, Any] = field(default_factory=dict)
+    granularity: Optional[TimeGranularity] = None
+    doc_blocks: List[str] = field(default_factory=list)
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None) -> dict:
+        dct = super().__post_serialize__(dct, context)
+        dct["doc_blocks"] = _backcompat_doc_blocks(dct["doc_blocks"])
+        return dct
+
+
+@dataclass
+class InjectedCTE(dbtClassMixin):
+    """Used in CompiledNodes as part of ephemeral model processing"""
+
+    id: str
+    sql: str
+
+
+@dataclass
+class Contract(dbtClassMixin):
+    enforced: bool = False
+    alias_types: bool = True
+    checksum: Optional[str] = None
+
+
+@dataclass
+class Quoting(dbtClassMixin, Mergeable):
+    database: Optional[bool] = None
+    schema: Optional[bool] = None
+    identifier: Optional[bool] = None
+    column: Optional[bool] = None
+
+
+@dataclass
+class Time(dbtClassMixin, Mergeable):
+    count: Optional[int] = None
+    period: Optional[TimePeriod] = None
+
+    def exceeded(self, actual_age: float) -> bool:
+        if self.period is None or self.count is None:
+            return False
+        kwargs: Dict[str, int] = {self.period.plural(): self.count}
+        difference = timedelta(**kwargs).total_seconds()
+        return actual_age > difference
+
+    def __bool__(self):
+        return self.count is not None and self.period is not None
+
+
+@dataclass
+class FreshnessThreshold(dbtClassMixin, Mergeable):
+    warn_after: Optional[Time] = field(default_factory=Time)
+    error_after: Optional[Time] = field(default_factory=Time)
+    filter: Optional[str] = None
+
+    def status(self, age: float) -> "dbt.artifacts.schemas.results.FreshnessStatus":  # type: ignore # noqa F821
+        from dvt.artifacts.schemas.results import FreshnessStatus
+
+        if self.error_after and self.error_after.exceeded(age):
+            return FreshnessStatus.Error
+        elif self.warn_after and self.warn_after.exceeded(age):
+            return FreshnessStatus.Warn
+        else:
+            return FreshnessStatus.Pass
+
+    def __bool__(self):
+        return bool(self.warn_after) or bool(self.error_after)
+
+
+@dataclass
+class HasRelationMetadata(dbtClassMixin):
+    database: Optional[str]
+    schema: str
+
+    # Can't set database to None like it ought to be
+    # because it messes up the subclasses and default parameters
+    # so hack it here
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        if "database" not in data:
+            data["database"] = None
+        return data
+
+    @property
+    def quoting_dict(self) -> Dict[str, bool]:
+        if hasattr(self, "quoting"):
+            return self.quoting.to_dict(omit_none=True)
+        else:
+            return {}
+
+
+@dataclass
+class DeferRelation(HasRelationMetadata):
+    alias: str
+    relation_name: Optional[str]
+    # The rest of these fields match RelationConfig protocol exactly
+    resource_type: NodeType
+    name: str
+    description: str
+    compiled_code: Optional[str]
+    meta: Dict[str, Any]
+    tags: List[str]
+    config: Optional[NodeConfig]
+
+    @property
+    def identifier(self):
+        return self.alias
+
+
+@dataclass
+class ParsedResourceMandatory(GraphResource, HasRelationMetadata):
+    alias: str
+    checksum: FileHash
+    config: NodeConfig = field(default_factory=NodeConfig)
+
+    @property
+    def identifier(self):
+        return self.alias
+
+
+@dataclass
+class ParsedResource(ParsedResourceMandatory):
+    tags: List[str] = field(default_factory=list)
+    description: str = field(default="")
+    columns: Dict[str, ColumnInfo] = field(default_factory=dict)
+    meta: Dict[str, Any] = field(default_factory=dict)
+    group: Optional[str] = None
+    docs: Docs = field(default_factory=Docs)
+    patch_path: Optional[str] = None
+    build_path: Optional[str] = None
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    created_at: float = field(default_factory=lambda: time.time())
+    config_call_dict: Dict[str, Any] = field(default_factory=dict)
+    unrendered_config_call_dict: Dict[str, Any] = field(default_factory=dict)
+    relation_name: Optional[str] = None
+    raw_code: str = ""
+    doc_blocks: List[str] = field(default_factory=list)
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+
+        if context and context.get("artifact") and "config_call_dict" in dct:
+            del dct["config_call_dict"]
+        if context and context.get("artifact") and "unrendered_config_call_dict" in dct:
+            del dct["unrendered_config_call_dict"]
+
+        dct["doc_blocks"] = _backcompat_doc_blocks(dct["doc_blocks"])
+
+        return dct
+
+
+@dataclass
+class CompiledResource(ParsedResource):
+    """Contains attributes necessary for SQL files and nodes with refs, sources, etc,
+    so all ManifestNodes except SeedNode."""
+
+    language: str = "sql"
+    refs: List[RefArgs] = field(default_factory=list)
+    sources: List[List[str]] = field(default_factory=list)
+    metrics: List[List[str]] = field(default_factory=list)
+    functions: List[List[str]] = field(default_factory=list)
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    compiled_path: Optional[str] = None
+    compiled: bool = False
+    compiled_code: Optional[str] = None
+    extra_ctes_injected: bool = False
+    extra_ctes: List[InjectedCTE] = field(default_factory=list)
+    _pre_injected_sql: Optional[str] = None
+    contract: Contract = field(default_factory=Contract)
+
+    def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+        if "_pre_injected_sql" in dct:
+            del dct["_pre_injected_sql"]
+        # Remove compiled attributes
+        if "compiled" in dct and dct["compiled"] is False:
+            del dct["compiled"]
+            del dct["extra_ctes_injected"]
+            del dct["extra_ctes"]
+            # "omit_none" means these might not be in the dictionary
+            if "compiled_code" in dct:
+                del dct["compiled_code"]
+        return dct

dvt/artifacts/resources/v1/config.py

@@ -0,0 +1,282 @@
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+from dvt import hooks
+from dvt.artifacts.resources.base import Docs
+from dvt.artifacts.resources.types import ModelHookType
+from dvt.artifacts.utils.validation import validate_color
+from mashumaro.jsonschema.annotations import Pattern
+from typing_extensions import Annotated
+
+from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
+from dbt_common.contracts.config.materialization import OnConfigurationChangeOption
+from dbt_common.contracts.config.metadata import Metadata, ShowBehavior
+from dbt_common.dataclass_schema import ValidationError, dbtClassMixin
+
+
+def list_str() -> List[str]:
+    return []
+
+
+class Severity(str):
+    pass
+
+
+def metas(*metas: Metadata) -> Dict[str, Any]:
+    existing: Dict[str, Any] = {}
+    for m in metas:
+        existing = m.meta(existing)
+    return existing
+
+
+@dataclass
+class ContractConfig(dbtClassMixin):
+    enforced: bool = False
+    alias_types: bool = True
+
+
+@dataclass
+class Hook(dbtClassMixin):
+    sql: str
+    transaction: bool = True
+    index: Optional[int] = None
+
+
+@dataclass
+class NodeAndTestConfig(BaseConfig):
+    enabled: bool = True
+    # these fields are included in serialized output, but are not part of
+    # config comparison (they are part of database_representation)
+    alias: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    schema: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    database: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    tags: Union[List[str], str] = field(
+        default_factory=list_str,
+        metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
+    )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+
+
+@dataclass
+class NodeConfig(NodeAndTestConfig):
+    # Note: if any new fields are added with MergeBehavior, also update the
+    # 'mergebehavior' dictionary
+    materialized: str = "view"
+    incremental_strategy: Optional[str] = None
+    batch_size: Any = None
+    lookback: Any = 1
+    begin: Any = None
+    persist_docs: Dict[str, Any] = field(default_factory=dict)
+    post_hook: List[Hook] = field(
+        default_factory=list,
+        metadata={"merge": MergeBehavior.Append, "alias": "post-hook"},
+    )
+    pre_hook: List[Hook] = field(
+        default_factory=list,
+        metadata={"merge": MergeBehavior.Append, "alias": "pre-hook"},
+    )
+    quoting: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    # This is actually only used by seeds. Should it be available to others?
+    # That would be a breaking change!
+    column_types: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    full_refresh: Optional[bool] = None
+    # 'unique_key' doesn't use 'Optional' because typing.get_type_hints was
+    # sometimes getting the Union order wrong, causing serialization failures.
+    unique_key: Union[str, List[str], None] = None
+    on_schema_change: Optional[str] = "ignore"
+    on_configuration_change: OnConfigurationChangeOption = field(
+        default_factory=OnConfigurationChangeOption.default
+    )
+    grants: Dict[str, Any] = field(
+        default_factory=dict, metadata=MergeBehavior.DictKeyAppend.meta()
+    )
+    packages: List[str] = field(
+        default_factory=list,
+        metadata=MergeBehavior.Append.meta(),
+    )
+    docs: Docs = field(
+        default_factory=Docs,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    contract: ContractConfig = field(
+        default_factory=ContractConfig,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    event_time: Any = None
+    concurrent_batches: Any = None
+
+    # DVT-specific configuration fields
+    compute_engine: Optional[str] = field(
+        default=None,
+        metadata=MergeBehavior.Clobber.meta(),
+    )
+    pushdown_enabled: Optional[bool] = field(
+        default=None,
+        metadata=MergeBehavior.Clobber.meta(),
+    )
+    target_profile: Optional[str] = field(
+        default=None,
+        metadata=MergeBehavior.Clobber.meta(),
+    )
+
+    def __post_init__(self):
+        # we validate that node_color has a suitable value to prevent dbt-docs from crashing
+        if self.docs.node_color:
+            node_color = self.docs.node_color
+            if not validate_color(node_color):
+                raise ValidationError(
+                    f"Invalid color name for docs.node_color: {node_color}. "
+                    "It is neither a valid HTML color name nor a valid HEX code."
+                )
+
+        if (
+            self.contract.enforced
+            and self.materialized == "incremental"
+            and self.on_schema_change not in ("append_new_columns", "fail")
+        ):
+            raise ValidationError(
+                f"Invalid value for on_schema_change: {self.on_schema_change}. Models "
+                "materialized as incremental with contracts enabled must set "
+                "on_schema_change to 'append_new_columns' or 'fail'"
+            )
+
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        for key in ModelHookType:
+            if key in data:
+                data[key] = [hooks.get_hook_dict(h) for h in data[key]]
+        return data
+
+
+SEVERITY_PATTERN = r"^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$"
+
+
+@dataclass
+class TestConfig(NodeAndTestConfig):
+    __test__ = False
+
+    # this is repeated because of a different default
+    schema: Optional[str] = field(
+        default="dbt_test__audit",
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    materialized: str = "test"
+    # Annotated is used by mashumaro for jsonschema generation
+    severity: Annotated[Severity, Pattern(SEVERITY_PATTERN)] = Severity("ERROR")
+    store_failures: Optional[bool] = None
+    store_failures_as: Optional[str] = None
+    where: Optional[str] = None
+    limit: Optional[int] = None
+    fail_calc: str = "count(*)"
+    warn_if: str = "!= 0"
+    error_if: str = "!= 0"
+
+    def finalize_and_validate(self):
+        """
+        The presence of a setting for `store_failures_as` overrides any existing setting for `store_failures`,
+        regardless of level of granularity. If `store_failures_as` is not set, then `store_failures` takes effect.
+        At the time of implementation, `store_failures = True` would always create a table; the user could not
+        configure this. Hence, if `store_failures = True` and `store_failures_as` is not specified, then it
+        should be set to "table" to mimic the existing functionality.
+
+        A side effect of this overriding functionality is that `store_failures_as="view"` at the project
+        level cannot be turned off at the model level without setting both `store_failures_as` and
+        `store_failures`. The former would cascade down and override `store_failures=False`. The proposal
+        is to include "ephemeral" as a value for `store_failures_as`, which effectively sets
+        `store_failures=False`.
+
+        The exception handling for this is tricky. If we raise an exception here, the entire run fails at
+        parse time. We would rather well-formed models run successfully, leaving only exceptions to be rerun
+        if necessary. Hence, the exception needs to be raised in the test materialization. In order to do so,
+        we need to make sure that we go down the `store_failures = True` route with the invalid setting for
+        `store_failures_as`. This results in the `.get()` defaulted to `True` below, instead of a normal
+        dictionary lookup as is done in the `if` block. Refer to the test materialization for the
+        exception that is raise as a result of an invalid value.
+
+        The intention of this block is to behave as if `store_failures_as` is the only setting,
+        but still allow for backwards compatibility for `store_failures`.
+        See https://github.com/dbt-labs/dbt-core/issues/6914 for more information.
+        """
+        super().finalize_and_validate()
+
+        # if `store_failures_as` is not set, it gets set by `store_failures`
+        # the settings below mimic existing behavior prior to `store_failures_as`
+        get_store_failures_as_map = {
+            True: "table",
+            False: "ephemeral",
+            None: None,
+        }
+
+        # if `store_failures_as` is set, it dictates what `store_failures` gets set to
+        # the settings below overrides whatever `store_failures` is set to by the user
+        get_store_failures_map = {
+            "ephemeral": False,
+            "table": True,
+            "view": True,
+        }
+
+        if self.store_failures_as is None:
+            self.store_failures_as = get_store_failures_as_map[self.store_failures]
+        else:
+            self.store_failures = get_store_failures_map.get(self.store_failures_as, True)
+
+        return self
+
+    @classmethod
+    def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> bool:
+        """This is like __eq__, except it explicitly checks certain fields."""
+        modifiers = [
+            "severity",
+            "where",
+            "limit",
+            "fail_calc",
+            "warn_if",
+            "error_if",
+            "store_failures",
+            "store_failures_as",
+        ]
+
+        seen = set()
+        for _, target_name in cls._get_fields():
+            key = target_name
+            seen.add(key)
+            if key in modifiers:
+                if not cls.compare_key(unrendered, other, key):
+                    return False
+        return True
+
+    @classmethod
+    def validate(cls, data):
+        if data.get("severity") and not re.match(SEVERITY_PATTERN, data.get("severity")):
+            raise ValidationError(
+                f"Severity must be either 'warn' or 'error'. Got '{data.get('severity')}'"
+            )
+
+        super().validate(data)
+
+        if data.get("materialized") and data.get("materialized") != "test":
+            raise ValidationError("A test must have a materialized value of 'test'")

dvt/artifacts/resources/v1/documentation.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass
+from typing import Literal
+
+from dvt.artifacts.resources.base import BaseResource
+from dvt.artifacts.resources.types import NodeType
+
+
+@dataclass
+class Documentation(BaseResource):
+    resource_type: Literal[NodeType.Documentation]
+    block_contents: str

dvt/artifacts/resources/v1/exposure.py

@@ -0,0 +1,52 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from dvt.artifacts.resources.base import GraphResource
+from dvt.artifacts.resources.types import NodeType
+from dvt.artifacts.resources.v1.components import DependsOn, RefArgs
+from dvt.artifacts.resources.v1.owner import Owner
+
+from dbt_common.contracts.config.base import BaseConfig
+from dbt_common.dataclass_schema import StrEnum
+
+
+class ExposureType(StrEnum):
+    Dashboard = "dashboard"
+    Notebook = "notebook"
+    Analysis = "analysis"
+    ML = "ml"
+    Application = "application"
+
+
+class MaturityType(StrEnum):
+    Low = "low"
+    Medium = "medium"
+    High = "high"
+
+
+@dataclass
+class ExposureConfig(BaseConfig):
+    enabled: bool = True
+    tags: List[str] = field(default_factory=list)
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Exposure(GraphResource):
+    type: ExposureType
+    owner: Owner
+    resource_type: Literal[NodeType.Exposure]
+    description: str = ""
+    label: Optional[str] = None
+    maturity: Optional[MaturityType] = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+    config: ExposureConfig = field(default_factory=ExposureConfig)
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    url: Optional[str] = None
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    refs: List[RefArgs] = field(default_factory=list)
+    sources: List[List[str]] = field(default_factory=list)
+    metrics: List[List[str]] = field(default_factory=list)
+    created_at: float = field(default_factory=lambda: time.time())

dvt/artifacts/resources/v1/function.py

@@ -0,0 +1,53 @@
+from dataclasses import dataclass, field
+from typing import List, Literal, Optional
+
+from dvt.artifacts.resources.types import FunctionType, FunctionVolatility, NodeType
+from dvt.artifacts.resources.v1.components import CompiledResource
+from dvt.artifacts.resources.v1.config import NodeConfig
+
+from dbt_common.dataclass_schema import dbtClassMixin
+
+# =============
+# Function config, and supporting classes
+# =============
+
+
+@dataclass
+class FunctionConfig(NodeConfig):
+    # The fact that this is a property, that can be changed, seems wrong.
+    # A function's materialization should never be changed, so why allow for it?
+    materialized: str = "function"
+    type: FunctionType = FunctionType.Scalar
+    volatility: Optional[FunctionVolatility] = None
+    runtime_version: Optional[str] = None
+    entry_point: Optional[str] = None
+
+
+# =============
+# Function resource, and supporting classes
+# =============
+
+
+@dataclass
+class FunctionArgument(dbtClassMixin):
+    name: str
+    data_type: str
+    description: Optional[str] = None
+
+
+@dataclass
+class FunctionReturns(dbtClassMixin):
+    data_type: str
+    description: Optional[str] = None
+
+
+@dataclass
+class FunctionMandatory(dbtClassMixin):
+    returns: FunctionReturns
+
+
+@dataclass
+class Function(CompiledResource, FunctionMandatory):
+    resource_type: Literal[NodeType.Function]
+    config: FunctionConfig
+    arguments: List[FunctionArgument] = field(default_factory=list)