dvt-core 0.59.0a51__py3-none-any.whl

dbt/artifacts/resources/v1/config.py

@@ -0,0 +1,277 @@
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+from mashumaro.jsonschema.annotations import Pattern
+from typing_extensions import Annotated
+
+from dbt import hooks
+from dbt.artifacts.resources.base import Docs
+from dbt.artifacts.resources.types import ModelHookType
+from dbt.artifacts.utils.validation import validate_color
+from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
+from dbt_common.contracts.config.materialization import OnConfigurationChangeOption
+from dbt_common.contracts.config.metadata import Metadata, ShowBehavior
+from dbt_common.dataclass_schema import ValidationError, dbtClassMixin
+
+
+def list_str() -> List[str]:
+    return []
+
+
+class Severity(str):
+    pass
+
+
+def metas(*metas: Metadata) -> Dict[str, Any]:
+    existing: Dict[str, Any] = {}
+    for m in metas:
+        existing = m.meta(existing)
+    return existing
+
+
+@dataclass
+class ContractConfig(dbtClassMixin):
+    enforced: bool = False
+    alias_types: bool = True
+
+
+@dataclass
+class Hook(dbtClassMixin):
+    sql: str
+    transaction: bool = True
+    index: Optional[int] = None
+
+
+@dataclass
+class NodeAndTestConfig(BaseConfig):
+    enabled: bool = True
+    # these fields are included in serialized output, but are not part of
+    # config comparison (they are part of database_representation)
+    alias: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    schema: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    database: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    tags: Union[List[str], str] = field(
+        default_factory=list_str,
+        metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
+    )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+
+
+@dataclass
+class NodeConfig(NodeAndTestConfig):
+    # Note: if any new fields are added with MergeBehavior, also update the
+    # 'mergebehavior' dictionary
+    materialized: str = "view"
+    incremental_strategy: Optional[str] = None
+    batch_size: Any = None
+    lookback: Any = 1
+    begin: Any = None
+    persist_docs: Dict[str, Any] = field(default_factory=dict)
+    post_hook: List[Hook] = field(
+        default_factory=list,
+        metadata={"merge": MergeBehavior.Append, "alias": "post-hook"},
+    )
+    pre_hook: List[Hook] = field(
+        default_factory=list,
+        metadata={"merge": MergeBehavior.Append, "alias": "pre-hook"},
+    )
+    quoting: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    # This is actually only used by seeds. Should it be available to others?
+    # That would be a breaking change!
+    column_types: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    full_refresh: Optional[bool] = None
+    # 'unique_key' doesn't use 'Optional' because typing.get_type_hints was
+    # sometimes getting the Union order wrong, causing serialization failures.
+    unique_key: Union[str, List[str], None] = None
+    on_schema_change: Optional[str] = "ignore"
+    on_configuration_change: OnConfigurationChangeOption = field(
+        default_factory=OnConfigurationChangeOption.default
+    )
+    grants: Dict[str, Any] = field(
+        default_factory=dict, metadata=MergeBehavior.DictKeyAppend.meta()
+    )
+    packages: List[str] = field(
+        default_factory=list,
+        metadata=MergeBehavior.Append.meta(),
+    )
+    docs: Docs = field(
+        default_factory=Docs,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    contract: ContractConfig = field(
+        default_factory=ContractConfig,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    event_time: Any = None
+    concurrent_batches: Any = None
+    # DVT-specific configuration fields
+    target: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )  # Target connection for materialization
+    compute: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )  # Compute engine override (spark-local/spark-cluster/etc.)
+
+    def __post_init__(self):
+        # we validate that node_color has a suitable value to prevent dbt-docs from crashing
+        if self.docs.node_color:
+            node_color = self.docs.node_color
+            if not validate_color(node_color):
+                raise ValidationError(
+                    f"Invalid color name for docs.node_color: {node_color}. "
+                    "It is neither a valid HTML color name nor a valid HEX code."
+                )
+
+        if (
+            self.contract.enforced
+            and self.materialized == "incremental"
+            and self.on_schema_change not in ("append_new_columns", "fail")
+        ):
+            raise ValidationError(
+                f"Invalid value for on_schema_change: {self.on_schema_change}. Models "
+                "materialized as incremental with contracts enabled must set "
+                "on_schema_change to 'append_new_columns' or 'fail'"
+            )
+
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        for key in ModelHookType:
+            if key in data:
+                data[key] = [hooks.get_hook_dict(h) for h in data[key]]
+        return data
+
+
+SEVERITY_PATTERN = r"^([Ww][Aa][Rr][Nn]|[Ee][Rr][Rr][Oo][Rr])$"
+
+
+@dataclass
+class TestConfig(NodeAndTestConfig):
+    __test__ = False
+
+    # this is repeated because of a different default
+    schema: Optional[str] = field(
+        default="dbt_test__audit",
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    materialized: str = "test"
+    # Annotated is used by mashumaro for jsonschema generation
+    severity: Annotated[Severity, Pattern(SEVERITY_PATTERN)] = Severity("ERROR")
+    store_failures: Optional[bool] = None
+    store_failures_as: Optional[str] = None
+    where: Optional[str] = None
+    limit: Optional[int] = None
+    fail_calc: str = "count(*)"
+    warn_if: str = "!= 0"
+    error_if: str = "!= 0"
+
+    def finalize_and_validate(self):
+        """
+        The presence of a setting for `store_failures_as` overrides any existing setting for `store_failures`,
+        regardless of level of granularity. If `store_failures_as` is not set, then `store_failures` takes effect.
+        At the time of implementation, `store_failures = True` would always create a table; the user could not
+        configure this. Hence, if `store_failures = True` and `store_failures_as` is not specified, then it
+        should be set to "table" to mimic the existing functionality.
+
+        A side effect of this overriding functionality is that `store_failures_as="view"` at the project
+        level cannot be turned off at the model level without setting both `store_failures_as` and
+        `store_failures`. The former would cascade down and override `store_failures=False`. The proposal
+        is to include "ephemeral" as a value for `store_failures_as`, which effectively sets
+        `store_failures=False`.
+
+        The exception handling for this is tricky. If we raise an exception here, the entire run fails at
+        parse time. We would rather well-formed models run successfully, leaving only exceptions to be rerun
+        if necessary. Hence, the exception needs to be raised in the test materialization. In order to do so,
+        we need to make sure that we go down the `store_failures = True` route with the invalid setting for
+        `store_failures_as`. This results in the `.get()` defaulted to `True` below, instead of a normal
+        dictionary lookup as is done in the `if` block. Refer to the test materialization for the
+        exception that is raise as a result of an invalid value.
+
+        The intention of this block is to behave as if `store_failures_as` is the only setting,
+        but still allow for backwards compatibility for `store_failures`.
+        See https://github.com/dbt-labs/dbt-core/issues/6914 for more information.
+        """
+        super().finalize_and_validate()
+
+        # if `store_failures_as` is not set, it gets set by `store_failures`
+        # the settings below mimic existing behavior prior to `store_failures_as`
+        get_store_failures_as_map = {
+            True: "table",
+            False: "ephemeral",
+            None: None,
+        }
+
+        # if `store_failures_as` is set, it dictates what `store_failures` gets set to
+        # the settings below overrides whatever `store_failures` is set to by the user
+        get_store_failures_map = {
+            "ephemeral": False,
+            "table": True,
+            "view": True,
+        }
+
+        if self.store_failures_as is None:
+            self.store_failures_as = get_store_failures_as_map[self.store_failures]
+        else:
+            self.store_failures = get_store_failures_map.get(self.store_failures_as, True)
+
+        return self
+
+    @classmethod
+    def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> bool:
+        """This is like __eq__, except it explicitly checks certain fields."""
+        modifiers = [
+            "severity",
+            "where",
+            "limit",
+            "fail_calc",
+            "warn_if",
+            "error_if",
+            "store_failures",
+            "store_failures_as",
+        ]
+
+        seen = set()
+        for _, target_name in cls._get_fields():
+            key = target_name
+            seen.add(key)
+            if key in modifiers:
+                if not cls.compare_key(unrendered, other, key):
+                    return False
+        return True
+
+    @classmethod
+    def validate(cls, data):
+        if data.get("severity") and not re.match(SEVERITY_PATTERN, data.get("severity")):
+            raise ValidationError(
+                f"Severity must be either 'warn' or 'error'. Got '{data.get('severity')}'"
+            )
+
+        super().validate(data)
+
+        if data.get("materialized") and data.get("materialized") != "test":
+            raise ValidationError("A test must have a materialized value of 'test'")

dbt/artifacts/resources/v1/documentation.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass
+from typing import Literal
+
+from dbt.artifacts.resources.base import BaseResource
+from dbt.artifacts.resources.types import NodeType
+
+
+@dataclass
+class Documentation(BaseResource):
+    resource_type: Literal[NodeType.Documentation]
+    block_contents: str

dbt/artifacts/resources/v1/exposure.py

@@ -0,0 +1,51 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from dbt.artifacts.resources.base import GraphResource
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import DependsOn, RefArgs
+from dbt.artifacts.resources.v1.owner import Owner
+from dbt_common.contracts.config.base import BaseConfig
+from dbt_common.dataclass_schema import StrEnum
+
+
+class ExposureType(StrEnum):
+    Dashboard = "dashboard"
+    Notebook = "notebook"
+    Analysis = "analysis"
+    ML = "ml"
+    Application = "application"
+
+
+class MaturityType(StrEnum):
+    Low = "low"
+    Medium = "medium"
+    High = "high"
+
+
+@dataclass
+class ExposureConfig(BaseConfig):
+    enabled: bool = True
+    tags: List[str] = field(default_factory=list)
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Exposure(GraphResource):
+    type: ExposureType
+    owner: Owner
+    resource_type: Literal[NodeType.Exposure]
+    description: str = ""
+    label: Optional[str] = None
+    maturity: Optional[MaturityType] = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+    config: ExposureConfig = field(default_factory=ExposureConfig)
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    url: Optional[str] = None
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    refs: List[RefArgs] = field(default_factory=list)
+    sources: List[List[str]] = field(default_factory=list)
+    metrics: List[List[str]] = field(default_factory=list)
+    created_at: float = field(default_factory=lambda: time.time())

dbt/artifacts/resources/v1/function.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass, field
+from typing import List, Literal, Optional
+
+from dbt.artifacts.resources.types import FunctionType, FunctionVolatility, NodeType
+from dbt.artifacts.resources.v1.components import CompiledResource
+from dbt.artifacts.resources.v1.config import NodeConfig
+from dbt_common.dataclass_schema import dbtClassMixin
+
+# =============
+# Function config, and supporting classes
+# =============
+
+
+@dataclass
+class FunctionConfig(NodeConfig):
+    # The fact that this is a property, that can be changed, seems wrong.
+    # A function's materialization should never be changed, so why allow for it?
+    materialized: str = "function"
+    type: FunctionType = FunctionType.Scalar
+    volatility: Optional[FunctionVolatility] = None
+    runtime_version: Optional[str] = None
+    entry_point: Optional[str] = None
+
+
+# =============
+# Function resource, and supporting classes
+# =============
+
+
+@dataclass
+class FunctionArgument(dbtClassMixin):
+    name: str
+    data_type: str
+    description: Optional[str] = None
+
+
+@dataclass
+class FunctionReturns(dbtClassMixin):
+    data_type: str
+    description: Optional[str] = None
+
+
+@dataclass
+class FunctionMandatory(dbtClassMixin):
+    returns: FunctionReturns
+
+
+@dataclass
+class Function(CompiledResource, FunctionMandatory):
+    resource_type: Literal[NodeType.Function]
+    config: FunctionConfig
+    arguments: List[FunctionArgument] = field(default_factory=list)

dbt/artifacts/resources/v1/generic_test.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Literal, Optional
+
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import CompiledResource
+from dbt.artifacts.resources.v1.config import TestConfig
+from dbt_common.dataclass_schema import dbtClassMixin
+
+
+@dataclass
+class TestMetadata(dbtClassMixin):
+    __test__ = False
+
+    name: str = "test"  # dummy default to allow default in GenericTestNode. Should always be set.
+    # kwargs are the args that are left in the test builder after
+    # removing configs. They are set from the test builder when
+    # the test node is created.
+    kwargs: Dict[str, Any] = field(default_factory=dict)
+    namespace: Optional[str] = None
+
+
+@dataclass
+class GenericTest(CompiledResource):
+    resource_type: Literal[NodeType.Test]
+    column_name: Optional[str] = None
+    file_key_name: Optional[str] = None
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type: ignore
+    attached_node: Optional[str] = None
+    test_metadata: TestMetadata = field(default_factory=TestMetadata)

dbt/artifacts/resources/v1/group.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Literal, Optional
+
+from dbt.artifacts.resources.base import BaseResource
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.owner import Owner
+from dbt_common.contracts.config.base import BaseConfig, MergeBehavior
+
+
+@dataclass
+class GroupConfig(BaseConfig):
+    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
+
+
+@dataclass
+class Group(BaseResource):
+    name: str
+    owner: Owner
+    resource_type: Literal[NodeType.Group]
+    description: Optional[str] = None
+    config: GroupConfig = field(default_factory=GroupConfig)

dbt/artifacts/resources/v1/hook.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass
+from typing import Literal, Optional
+
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import CompiledResource
+
+
+@dataclass
+class HookNode(CompiledResource):
+    resource_type: Literal[NodeType.Operation]
+    index: Optional[int] = None

dbt/artifacts/resources/v1/macro.py

@@ -0,0 +1,29 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from dbt.artifacts.resources.base import BaseResource, Docs
+from dbt.artifacts.resources.types import ModelLanguage, NodeType
+from dbt.artifacts.resources.v1.components import MacroDependsOn
+from dbt_common.dataclass_schema import dbtClassMixin
+
+
+@dataclass
+class MacroArgument(dbtClassMixin):
+    name: str
+    type: Optional[str] = None
+    description: str = ""
+
+
+@dataclass
+class Macro(BaseResource):
+    macro_sql: str
+    resource_type: Literal[NodeType.Macro]
+    depends_on: MacroDependsOn = field(default_factory=MacroDependsOn)
+    description: str = ""
+    meta: Dict[str, Any] = field(default_factory=dict)
+    docs: Docs = field(default_factory=Docs)
+    patch_path: Optional[str] = None
+    arguments: List[MacroArgument] = field(default_factory=list)
+    created_at: float = field(default_factory=lambda: time.time())
+    supported_languages: Optional[List[ModelLanguage]] = None

dbt/artifacts/resources/v1/metric.py

@@ -0,0 +1,172 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from dbt.artifacts.resources.base import GraphResource
+from dbt.artifacts.resources.types import NodeType
+from dbt.artifacts.resources.v1.components import DependsOn, RefArgs
+from dbt.artifacts.resources.v1.semantic_layer_components import (
+    MeasureAggregationParameters,
+    NonAdditiveDimension,
+    SourceFileMetadata,
+    WhereFilterIntersection,
+)
+from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_semantic_interfaces.references import MeasureReference, MetricReference
+from dbt_semantic_interfaces.type_enums import (
+    AggregationType,
+    ConversionCalculationType,
+    MetricType,
+    PeriodAggregation,
+    TimeGranularity,
+)
+
+"""
+The following classes are dataclasses which are used to construct the Metric
+node in dbt-core. Additionally, these classes need to at a minimum support
+what is specified in their protocol definitions in dbt-semantic-interfaces.
+Their protocol definitions can be found here:
+https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/metric.py
+"""
+
+
+@dataclass
+class MetricInputMeasure(dbtClassMixin):
+    name: str
+    filter: Optional[WhereFilterIntersection] = None
+    alias: Optional[str] = None
+    join_to_timespine: bool = False
+    fill_nulls_with: Optional[int] = None
+
+    def measure_reference(self) -> MeasureReference:
+        return MeasureReference(element_name=self.name)
+
+    def post_aggregation_measure_reference(self) -> MeasureReference:
+        return MeasureReference(element_name=self.alias or self.name)
+
+
+@dataclass
+class MetricTimeWindow(dbtClassMixin):
+    count: int
+    granularity: str
+
+    @property
+    def window_string(self) -> str:  # noqa: D
+        return f"{self.count} {self.granularity}"
+
+    @property
+    def is_standard_granularity(self) -> bool:  # noqa: D
+        return self.granularity.casefold() in {item.value.casefold() for item in TimeGranularity}
+
+
+@dataclass
+class MetricInput(dbtClassMixin):
+    name: str
+    filter: Optional[WhereFilterIntersection] = None
+    alias: Optional[str] = None
+    offset_window: Optional[MetricTimeWindow] = None
+    offset_to_grain: Optional[str] = None
+
+    def as_reference(self) -> MetricReference:
+        return MetricReference(element_name=self.name)
+
+    def post_aggregation_reference(self) -> MetricReference:
+        return MetricReference(element_name=self.alias or self.name)
+
+
+@dataclass
+class ConstantPropertyInput(dbtClassMixin):
+    base_property: str
+    conversion_property: str
+
+
+@dataclass
+class ConversionTypeParams(dbtClassMixin):
+    base_measure: MetricInputMeasure
+    conversion_measure: MetricInputMeasure
+    entity: str
+    calculation: ConversionCalculationType = ConversionCalculationType.CONVERSION_RATE
+    window: Optional[MetricTimeWindow] = None
+    constant_properties: Optional[List[ConstantPropertyInput]] = None
+
+
+@dataclass
+class CumulativeTypeParams(dbtClassMixin):
+    window: Optional[MetricTimeWindow] = None
+    grain_to_date: Optional[str] = None
+    period_agg: PeriodAggregation = PeriodAggregation.FIRST
+    metric: Optional[MetricInput] = None
+
+
+@dataclass
+class MetricAggregationParams(dbtClassMixin):
+    semantic_model: str
+    agg: AggregationType
+    agg_params: Optional[MeasureAggregationParameters] = None
+    agg_time_dimension: Optional[str] = None
+    non_additive_dimension: Optional[NonAdditiveDimension] = None
+    expr: Optional[str] = None
+
+
+@dataclass
+class MetricTypeParams(dbtClassMixin):
+    measure: Optional[MetricInputMeasure] = None
+    input_measures: List[MetricInputMeasure] = field(default_factory=list)
+    numerator: Optional[MetricInput] = None
+    denominator: Optional[MetricInput] = None
+    expr: Optional[str] = None
+    window: Optional[MetricTimeWindow] = None
+    grain_to_date: Optional[TimeGranularity] = (
+        None  # legacy, use cumulative_type_params.grain_to_date
+    )
+    metrics: Optional[List[MetricInput]] = None
+    conversion_type_params: Optional[ConversionTypeParams] = None
+    cumulative_type_params: Optional[CumulativeTypeParams] = None
+    metric_aggregation_params: Optional[MetricAggregationParams] = None
+
+
+@dataclass
+class MetricConfig(BaseConfig):
+    enabled: bool = True
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+
+    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
+
+
+@dataclass
+class Metric(GraphResource):
+    name: str
+    description: str
+    label: str
+    type: MetricType
+    type_params: MetricTypeParams
+    filter: Optional[WhereFilterIntersection] = None
+    metadata: Optional[SourceFileMetadata] = None
+    time_granularity: Optional[str] = None
+    resource_type: Literal[NodeType.Metric]
+    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
+    tags: List[str] = field(default_factory=list)
+    config: MetricConfig = field(default_factory=MetricConfig)
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    sources: List[List[str]] = field(default_factory=list)
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    refs: List[RefArgs] = field(default_factory=list)
+    metrics: List[List[str]] = field(default_factory=list)
+    created_at: float = field(default_factory=lambda: time.time())
+    group: Optional[str] = None
+
+    @property
+    def input_measures(self) -> List[MetricInputMeasure]:
+        return self.type_params.input_measures
+
+    @property
+    def measure_references(self) -> List[MeasureReference]:
+        return [x.measure_reference() for x in self.input_measures]
+
+    @property
+    def input_metrics(self) -> List[MetricInput]:
+        return self.type_params.metrics or []