dvt-core 1.11.0b4__py3-none-any.whl

dvt/artifacts/resources/v1/semantic_model.py

@@ -0,0 +1,315 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence
+
+from dvt.artifacts.resources import SourceFileMetadata
+from dvt.artifacts.resources.base import GraphResource
+from dvt.artifacts.resources.v1.components import DependsOn, RefArgs
+from dvt.artifacts.resources.v1.metric import Metric
+from dvt.artifacts.resources.v1.semantic_layer_components import (
+    MeasureAggregationParameters,
+    NonAdditiveDimension,
+)
+
+from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_semantic_interfaces.references import (
+    DimensionReference,
+    EntityReference,
+    LinkableElementReference,
+    MeasureReference,
+    SemanticModelReference,
+    TimeDimensionReference,
+)
+from dbt_semantic_interfaces.type_enums import (
+    AggregationType,
+    DimensionType,
+    EntityType,
+    MetricType,
+    TimeGranularity,
+)
+
+"""
+The classes in this file are dataclasses which are used to construct the Semantic
+Model node in dbt-core. Additionally, these classes need to at a minimum support
+what is specified in their protocol definitions in dbt-semantic-interfaces.
+Their protocol definitions can be found here:
+https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/semantic_model.py
+"""
+
+
+@dataclass
+class SemanticLayerElementConfig(dbtClassMixin):
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+
+
+@dataclass
+class Defaults(dbtClassMixin):
+    agg_time_dimension: Optional[str] = None
+
+
+@dataclass
+class NodeRelation(dbtClassMixin):
+    alias: str
+    schema_name: str  # TODO: Could this be called simply "schema" so we could reuse StateRelation?
+    database: Optional[str] = None
+    relation_name: Optional[str] = ""
+
+
+# ====================================
+# Dimension objects
+# Dimension protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/dimension.py
+# ====================================
+
+
+@dataclass
+class DimensionValidityParams(dbtClassMixin):
+    is_start: bool = False
+    is_end: bool = False
+
+
+@dataclass
+class DimensionTypeParams(dbtClassMixin):
+    time_granularity: TimeGranularity
+    validity_params: Optional[DimensionValidityParams] = None
+
+
+@dataclass
+class Dimension(dbtClassMixin):
+    name: str
+    type: DimensionType
+    description: Optional[str] = None
+    label: Optional[str] = None
+    is_partition: bool = False
+    type_params: Optional[DimensionTypeParams] = None
+    expr: Optional[str] = None
+    metadata: Optional[SourceFileMetadata] = None
+    config: Optional[SemanticLayerElementConfig] = None
+
+    @property
+    def reference(self) -> DimensionReference:
+        return DimensionReference(element_name=self.name)
+
+    @property
+    def time_dimension_reference(self) -> Optional[TimeDimensionReference]:
+        if self.type == DimensionType.TIME:
+            return TimeDimensionReference(element_name=self.name)
+        else:
+            return None
+
+    @property
+    def validity_params(self) -> Optional[DimensionValidityParams]:
+        if self.type_params:
+            return self.type_params.validity_params
+        else:
+            return None
+
+
+# ====================================
+# Entity objects
+# Entity protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/entity.py
+# ====================================
+
+
+@dataclass
+class Entity(dbtClassMixin):
+    name: str
+    type: EntityType
+    description: Optional[str] = None
+    label: Optional[str] = None
+    role: Optional[str] = None
+    expr: Optional[str] = None
+    config: Optional[SemanticLayerElementConfig] = None
+
+    @property
+    def reference(self) -> EntityReference:
+        return EntityReference(element_name=self.name)
+
+    @property
+    def is_linkable_entity_type(self) -> bool:
+        return self.type in (EntityType.PRIMARY, EntityType.UNIQUE, EntityType.NATURAL)
+
+
+# ====================================
+# Measure object
+# Measure protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/measure.py
+# ====================================
+
+
+@dataclass
+class Measure(dbtClassMixin):
+    name: str
+    agg: AggregationType
+    description: Optional[str] = None
+    label: Optional[str] = None
+    create_metric: bool = False
+    expr: Optional[str] = None
+    agg_params: Optional[MeasureAggregationParameters] = None
+    non_additive_dimension: Optional[NonAdditiveDimension] = None
+    agg_time_dimension: Optional[str] = None
+    config: Optional[SemanticLayerElementConfig] = None
+
+    @property
+    def reference(self) -> MeasureReference:
+        return MeasureReference(element_name=self.name)
+
+
+# ====================================
+# SemanticModel final parts
+# ====================================
+
+
+@dataclass
+class SemanticModelConfig(BaseConfig):
+    enabled: bool = True
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+
+
+@dataclass
+class SemanticModel(GraphResource):
+    model: str
+    node_relation: Optional[NodeRelation]
+    description: Optional[str] = None
+    label: Optional[str] = None
+    defaults: Optional[Defaults] = None
+    entities: Sequence[Entity] = field(default_factory=list)
+    measures: Sequence[Measure] = field(default_factory=list)
+    dimensions: Sequence[Dimension] = field(default_factory=list)
+    metadata: Optional[SourceFileMetadata] = None
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    refs: List[RefArgs] = field(default_factory=list)
+    created_at: float = field(default_factory=lambda: time.time())
+    config: SemanticModelConfig = field(default_factory=SemanticModelConfig)
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    primary_entity: Optional[str] = None
+    group: Optional[str] = None
+
+    @property
+    def entity_references(self) -> List[LinkableElementReference]:
+        return [entity.reference for entity in self.entities]
+
+    @property
+    def dimension_references(self) -> List[LinkableElementReference]:
+        return [dimension.reference for dimension in self.dimensions]
+
+    @property
+    def measure_references(self) -> List[MeasureReference]:
+        return [measure.reference for measure in self.measures]
+
+    @property
+    def has_validity_dimensions(self) -> bool:
+        return any([dim.validity_params is not None for dim in self.dimensions])
+
+    @property
+    def validity_start_dimension(self) -> Optional[Dimension]:
+        validity_start_dims = [
+            dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_start
+        ]
+        if not validity_start_dims:
+            return None
+        return validity_start_dims[0]
+
+    @property
+    def validity_end_dimension(self) -> Optional[Dimension]:
+        validity_end_dims = [
+            dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_end
+        ]
+        if not validity_end_dims:
+            return None
+        return validity_end_dims[0]
+
+    @property
+    def partitions(self) -> List[Dimension]:  # noqa: D
+        return [dim for dim in self.dimensions or [] if dim.is_partition]
+
+    @property
+    def partition(self) -> Optional[Dimension]:
+        partitions = self.partitions
+        if not partitions:
+            return None
+        return partitions[0]
+
+    @property
+    def reference(self) -> SemanticModelReference:
+        return SemanticModelReference(semantic_model_name=self.name)
+
+    def checked_agg_time_dimension_for_measure(
+        self, measure_reference: MeasureReference
+    ) -> TimeDimensionReference:
+        measure: Optional[Measure] = None
+        for measure in self.measures:
+            if measure.reference == measure_reference:
+                measure = measure
+
+        assert (
+            measure is not None
+        ), f"No measure with name ({measure_reference.element_name}) in semantic_model with name ({self.name})"
+
+        default_agg_time_dimension = (
+            self.defaults.agg_time_dimension if self.defaults is not None else None
+        )
+
+        agg_time_dimension_name = measure.agg_time_dimension or default_agg_time_dimension
+        assert agg_time_dimension_name is not None, (
+            f"Aggregation time dimension for measure {measure.name} on semantic model {self.name} is not set! "
+            "To fix this either specify a default `agg_time_dimension` for the semantic model or define an "
+            "`agg_time_dimension` on the measure directly."
+        )
+        return TimeDimensionReference(element_name=agg_time_dimension_name)
+
+    def checked_agg_time_dimension_for_simple_metric(
+        self, metric: Metric
+    ) -> TimeDimensionReference:
+        assert (
+            metric.type == MetricType.SIMPLE
+        ), "Only simple metrics can have an agg time dimension."
+        metric_agg_params = metric.type_params.metric_aggregation_params
+        # There are validations elsewhere to check this for metrics and provide messaging for it.
+        assert metric_agg_params, "Simple metrics must have metric_aggregation_params."
+        # This indicates a validation bug / dev error, not a user error that should appear
+        # in a user's YAML.
+        assert (
+            metric_agg_params.semantic_model == self.name
+        ), "Cannot retrieve the agg time dimension for a metric from a different model "
+        f"than the one that the metric belongs to. Metric `{metric.name}` belongs to model "
+        f"`{metric_agg_params.semantic_model}`, but we requested the agg time dimension from model `{self.name}`."
+
+        metric_time_dimension_name = None
+        if (
+            metric.type_params
+            and metric.type_params.metric_aggregation_params
+            and metric.type_params.metric_aggregation_params.agg_time_dimension
+        ):
+            metric_time_dimension_name = (
+                metric.type_params.metric_aggregation_params.agg_time_dimension
+            )
+
+        default_agg_time_dimension = (
+            self.defaults.agg_time_dimension if self.defaults is not None else None
+        )
+        agg_time_dimension_name = metric_time_dimension_name or default_agg_time_dimension
+
+        assert agg_time_dimension_name is not None, (
+            f"Aggregation time dimension for metric {metric.name} is not set! This should either be set directly on "
+            f"the metric specification in the model, or else defaulted to the time dimension in the data "
+            f"source containing the metric."
+        )
+        return TimeDimensionReference(element_name=agg_time_dimension_name)
+
+    @property
+    def primary_entity_reference(self) -> Optional[EntityReference]:
+        return (
+            EntityReference(element_name=self.primary_entity)
+            if self.primary_entity is not None
+            else None
+        )

dvt/artifacts/resources/v1/singular_test.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass, field
+from typing import Literal
+
+from dvt.artifacts.resources.types import NodeType
+from dvt.artifacts.resources.v1.components import CompiledResource
+from dvt.artifacts.resources.v1.config import TestConfig
+
+
+@dataclass
+class SingularTest(CompiledResource):
+    resource_type: Literal[NodeType.Test]
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type: ignore

dvt/artifacts/resources/v1/snapshot.py

@@ -0,0 +1,92 @@
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Optional, Union
+
+from dvt.artifacts.resources.types import NodeType
+from dvt.artifacts.resources.v1.components import CompiledResource, DeferRelation
+from dvt.artifacts.resources.v1.config import NodeConfig
+
+from dbt_common.dataclass_schema import ValidationError, dbtClassMixin
+
+
+@dataclass
+class SnapshotMetaColumnNames(dbtClassMixin):
+    dbt_valid_to: Optional[str] = None
+    dbt_valid_from: Optional[str] = None
+    dbt_scd_id: Optional[str] = None
+    dbt_updated_at: Optional[str] = None
+    dbt_is_deleted: Optional[str] = None
+
+
+@dataclass
+class SnapshotConfig(NodeConfig):
+    materialized: str = "snapshot"
+    strategy: Optional[str] = None
+    unique_key: Union[str, List[str], None] = None
+    target_schema: Optional[str] = None
+    target_database: Optional[str] = None
+    updated_at: Optional[str] = None
+    # Not using Optional because of serialization issues with a Union of str and List[str]
+    check_cols: Union[str, List[str], None] = None
+    snapshot_meta_column_names: SnapshotMetaColumnNames = field(
+        default_factory=SnapshotMetaColumnNames
+    )
+    dbt_valid_to_current: Optional[str] = None
+
+    @property
+    def snapshot_table_column_names(self):
+        return {
+            "dbt_valid_from": self.snapshot_meta_column_names.dbt_valid_from or "dbt_valid_from",
+            "dbt_valid_to": self.snapshot_meta_column_names.dbt_valid_to or "dbt_valid_to",
+            "dbt_scd_id": self.snapshot_meta_column_names.dbt_scd_id or "dbt_scd_id",
+            "dbt_updated_at": self.snapshot_meta_column_names.dbt_updated_at or "dbt_updated_at",
+            "dbt_is_deleted": self.snapshot_meta_column_names.dbt_is_deleted or "dbt_is_deleted",
+        }
+
+    def final_validate(self):
+        if not self.strategy or not self.unique_key:
+            raise ValidationError(
+                "Snapshots must be configured with a 'strategy' and 'unique_key'."
+            )
+        if self.strategy == "check":
+            if not self.check_cols:
+                raise ValidationError(
+                    "A snapshot configured with the check strategy must "
+                    "specify a check_cols configuration."
+                )
+            if isinstance(self.check_cols, str) and self.check_cols != "all":
+                raise ValidationError(
+                    f"Invalid value for 'check_cols': {self.check_cols}. "
+                    "Expected 'all' or a list of strings."
+                )
+        elif self.strategy == "timestamp":
+            if not self.updated_at:
+                raise ValidationError(
+                    "A snapshot configured with the timestamp strategy "
+                    "must specify an updated_at configuration."
+                )
+            if self.check_cols:
+                raise ValidationError("A 'timestamp' snapshot should not have 'check_cols'")
+        # If the strategy is not 'check' or 'timestamp' it's a custom strategy,
+        # formerly supported with GenericSnapshotConfig
+
+        if self.materialized and self.materialized != "snapshot":
+            raise ValidationError("A snapshot must have a materialized value of 'snapshot'")
+
+    # Called by "calculate_node_config_dict" in ContextConfigGenerator
+    def finalize_and_validate(self):
+        data = self.to_dict(omit_none=True)
+        self.validate(data)
+        return self.from_dict(data)
+
+
+@dataclass
+class Snapshot(CompiledResource):
+    resource_type: Literal[NodeType.Snapshot]
+    config: SnapshotConfig
+    defer_relation: Optional[DeferRelation] = None
+
+    def __post_serialize__(self, dct, context: Optional[Dict] = None):
+        dct = super().__post_serialize__(dct, context)
+        if context and context.get("artifact") and "defer_relation" in dct:
+            del dct["defer_relation"]
+        return dct

dvt/artifacts/resources/v1/source_definition.py

@@ -0,0 +1,85 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from dvt.artifacts.resources.base import GraphResource
+from dvt.artifacts.resources.types import NodeType
+from dvt.artifacts.resources.v1.components import (
+    ColumnInfo,
+    FreshnessThreshold,
+    HasRelationMetadata,
+    Quoting,
+)
+from dvt.artifacts.resources.v1.config import BaseConfig, MergeBehavior
+
+from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed
+from dbt_common.contracts.util import Mergeable
+from dbt_common.exceptions import CompilationError
+
+
+@dataclass
+class SourceConfig(BaseConfig):
+    enabled: bool = True
+    event_time: Any = None
+    freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold)
+    loaded_at_field: Optional[str] = None
+    loaded_at_query: Optional[str] = None
+    meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
+    tags: List[str] = field(default_factory=list)
+
+
+@dataclass
+class ExternalPartition(AdditionalPropertiesAllowed):
+    name: str = ""
+    description: str = ""
+    data_type: str = ""
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        if self.name == "" or self.data_type == "":
+            raise CompilationError("External partition columns must have names and data types")
+
+
+@dataclass
+class ExternalTable(AdditionalPropertiesAllowed, Mergeable):
+    location: Optional[str] = None
+    file_format: Optional[str] = None
+    row_format: Optional[str] = None
+    tbl_properties: Optional[str] = None
+    partitions: Optional[Union[List[str], List[ExternalPartition]]] = None
+
+    def __bool__(self):
+        return self.location is not None
+
+
+@dataclass
+class ParsedSourceMandatory(GraphResource, HasRelationMetadata):
+    source_name: str
+    source_description: str
+    loader: str
+    identifier: str
+    resource_type: Literal[NodeType.Source]
+
+
+@dataclass
+class SourceDefinition(ParsedSourceMandatory):
+    quoting: Quoting = field(default_factory=Quoting)
+    loaded_at_field: Optional[str] = None
+    loaded_at_query: Optional[str] = None
+    freshness: Optional[FreshnessThreshold] = None
+    external: Optional[ExternalTable] = None
+    description: str = ""
+    columns: Dict[str, ColumnInfo] = field(default_factory=dict)
+    meta: Dict[str, Any] = field(default_factory=dict)
+    source_meta: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
+    config: SourceConfig = field(default_factory=SourceConfig)
+    patch_path: Optional[str] = None
+    unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    relation_name: Optional[str] = None
+    created_at: float = field(default_factory=lambda: time.time())
+    unrendered_database: Optional[str] = None
+    unrendered_schema: Optional[str] = None
+    doc_blocks: List[str] = field(default_factory=list)
+    # DVT-specific: profile reference for multi-source support
+    profile: Optional[str] = None

dvt/artifacts/resources/v1/sql_operation.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import Literal
+
+from dvt.artifacts.resources.types import NodeType
+from dvt.artifacts.resources.v1.components import CompiledResource
+
+
+@dataclass
+class SqlOperation(CompiledResource):
+    resource_type: Literal[NodeType.SqlOperation]

dvt/artifacts/resources/v1/unit_test_definition.py

@@ -0,0 +1,78 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from dvt.artifacts.resources import DependsOn, NodeVersion
+from dvt.artifacts.resources.base import GraphResource
+from dvt.artifacts.resources.v1.config import list_str, metas
+
+from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
+from dbt_common.contracts.config.metadata import ShowBehavior
+from dbt_common.dataclass_schema import StrEnum, dbtClassMixin
+
+
+@dataclass
+class UnitTestConfig(BaseConfig):
+    tags: Union[str, List[str]] = field(
+        default_factory=list_str,
+        metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
+    )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    enabled: bool = True
+
+
+class UnitTestFormat(StrEnum):
+    CSV = "csv"
+    Dict = "dict"
+    SQL = "sql"
+
+
+@dataclass
+class UnitTestInputFixture(dbtClassMixin):
+    input: str
+    rows: Optional[Union[str, List[Dict[str, Any]]]] = None
+    format: UnitTestFormat = UnitTestFormat.Dict
+    fixture: Optional[str] = None
+
+
+@dataclass
+class UnitTestOverrides(dbtClassMixin):
+    macros: Dict[str, Any] = field(default_factory=dict)
+    vars: Dict[str, Any] = field(default_factory=dict)
+    env_vars: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class UnitTestNodeVersions(dbtClassMixin):
+    include: Optional[List[NodeVersion]] = None
+    exclude: Optional[List[NodeVersion]] = None
+
+
+@dataclass
+class UnitTestOutputFixture(dbtClassMixin):
+    rows: Optional[Union[str, List[Dict[str, Any]]]] = None
+    format: UnitTestFormat = UnitTestFormat.Dict
+    fixture: Optional[str] = None
+
+
+@dataclass
+class UnitTestDefinitionMandatory:
+    model: str
+    given: Sequence[UnitTestInputFixture]
+    expect: UnitTestOutputFixture
+
+
+@dataclass
+class UnitTestDefinition(GraphResource, UnitTestDefinitionMandatory):
+    description: str = ""
+    overrides: Optional[UnitTestOverrides] = None
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    config: UnitTestConfig = field(default_factory=UnitTestConfig)
+    checksum: Optional[str] = None
+    schema: Optional[str] = None
+    created_at: float = field(default_factory=lambda: time.time())
+    versions: Optional[UnitTestNodeVersions] = None
+    version: Optional[NodeVersion] = None