dvt-core 1.11.0b4__py3-none-any.whl

dvt/contracts/project.py

@@ -0,0 +1,417 @@
+from dataclasses import dataclass, field
+from typing import Any, ClassVar, Dict, List, Optional, Union
+
+from dvt.contracts.util import Identifier, list_str
+from mashumaro.jsonschema.annotations import Pattern
+from mashumaro.types import SerializableType
+from typing_extensions import Annotated
+
+from dbt.adapters.contracts.connection import QueryComment
+from dbt_common.contracts.util import Mergeable
+from dbt_common.dataclass_schema import (
+    ExtensibleDbtClassMixin,
+    ValidationError,
+    dbtClassMixin,
+    dbtMashConfig,
+)
+from dbt_common.helper_types import NoValue
+
+DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True
+
+
+class SemverString(str, SerializableType):
+    def _serialize(self) -> str:
+        return self
+
+    @classmethod
+    def _deserialize(cls, value: str) -> "SemverString":
+        return SemverString(value)
+
+
+# This supports full semver, but also allows for 2 group version numbers, (allows '1.0').
+sem_ver_pattern = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)(\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)?$"
+
+
+@dataclass
+class Quoting(dbtClassMixin, Mergeable):
+    schema: Optional[bool] = None
+    database: Optional[bool] = None
+    project: Optional[bool] = None
+    identifier: Optional[bool] = None
+    snowflake_ignore_case: Optional[bool] = None
+
+
+@dataclass
+class Package(dbtClassMixin):
+
+    # Exclude {'name': None} from to_dict result to avoid changing sha1_hash result
+    # when user has not changed their 'packages' configuration.
+    def __post_serialize__(self, data, context: Optional[Dict]):
+        if "name" in data.keys() and data["name"] is None:
+            data.pop("name")
+            return data
+        return data
+
+
+@dataclass
+class LocalPackage(Package):
+    local: str
+    unrendered: Dict[str, Any] = field(default_factory=dict)
+    name: Optional[str] = None
+
+
+# `float` also allows `int`, according to PEP484 (and jsonschema!)
+RawVersion = Union[str, float]
+
+
+@dataclass
+class TarballPackage(Package):
+    tarball: str
+    name: str
+    unrendered: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class GitPackage(Package):
+    git: str
+    revision: Optional[RawVersion] = None
+    warn_unpinned: Optional[bool] = field(default=None, metadata={"alias": "warn-unpinned"})
+    subdirectory: Optional[str] = None
+    unrendered: Dict[str, Any] = field(default_factory=dict)
+    name: Optional[str] = None
+
+    def get_revisions(self) -> List[str]:
+        if self.revision is None:
+            return []
+        else:
+            return [str(self.revision)]
+
+
+@dataclass
+class PrivatePackage(Package):
+    private: str
+    provider: Optional[str] = None
+    revision: Optional[RawVersion] = None
+    warn_unpinned: Optional[bool] = field(default=None, metadata={"alias": "warn-unpinned"})
+    subdirectory: Optional[str] = None
+    unrendered: Dict[str, Any] = field(default_factory=dict)
+    name: Optional[str] = None
+
+
+@dataclass
+class RegistryPackage(Package):
+    package: str
+    version: Union[RawVersion, List[RawVersion]]
+    install_prerelease: Optional[bool] = False
+    unrendered: Dict[str, Any] = field(default_factory=dict)
+    name: Optional[str] = None
+
+    def get_versions(self) -> List[str]:
+        if isinstance(self.version, list):
+            return [str(v) for v in self.version]
+        else:
+            return [str(self.version)]
+
+
+PackageSpec = Union[LocalPackage, TarballPackage, GitPackage, RegistryPackage, PrivatePackage]
+
+
+@dataclass
+class PackageConfig(dbtClassMixin):
+    packages: List[PackageSpec]
+
+    @classmethod
+    def validate(cls, data):
+        for package in data.get("packages", data):
+            # This can happen when the target is a variable that is not filled and results in hangs
+            if isinstance(package, dict):
+                if package.get("package") == "":
+                    raise ValidationError(
+                        "A hub package is missing the value. It is a required property."
+                    )
+                if package.get("local") == "":
+                    raise ValidationError(
+                        "A local package is missing the value. It is a required property."
+                    )
+                if package.get("git") == "":
+                    raise ValidationError(
+                        "A git package is missing the value. It is a required property."
+                    )
+            if isinstance(package, dict) and package.get("package"):
+                if not package["version"]:
+                    raise ValidationError(
+                        f"{package['package']} is missing the version. When installing from the Hub "
+                        "package index, version is a required property"
+                    )
+                if "/" not in package["package"]:
+                    raise ValidationError(
+                        f"{package['package']} was not found in the package index. Packages on the index "
+                        "require a namespace, e.g dbt-labs/dbt_utils"
+                    )
+        super().validate(data)
+
+
+@dataclass
+class ProjectPackageMetadata:
+    name: str
+    packages: List[PackageSpec]
+
+    @classmethod
+    def from_project(cls, project):
+        return cls(name=project.project_name, packages=project.packages.packages)
+
+
+@dataclass
+class Downloads(ExtensibleDbtClassMixin):
+    tarball: str
+
+
+@dataclass
+class RegistryPackageMetadata(
+    ExtensibleDbtClassMixin,
+    ProjectPackageMetadata,
+):
+    downloads: Downloads
+
+
+# A list of all the reserved words that packages may not have as names.
+BANNED_PROJECT_NAMES = {
+    "_sql_results",
+    "adapter",
+    "api",
+    "column",
+    "config",
+    "context",
+    "database",
+    "env",
+    "env_var",
+    "exceptions",
+    "execute",
+    "flags",
+    "fromjson",
+    "fromyaml",
+    "graph",
+    "invocation_id",
+    "load_agate_table",
+    "load_result",
+    "log",
+    "model",
+    "modules",
+    "post_hooks",
+    "pre_hooks",
+    "ref",
+    "render",
+    "return",
+    "run_started_at",
+    "schema",
+    "source",
+    "sql",
+    "sql_now",
+    "store_result",
+    "store_raw_result",
+    "target",
+    "this",
+    "tojson",
+    "toyaml",
+    "try_or_compiler_error",
+    "var",
+    "write",
+}
+
+
+@dataclass
+class Project(dbtClassMixin):
+    _hyphenated: ClassVar[bool] = True
+    # Annotated is used by mashumaro for jsonschema generation
+    name: Annotated[Identifier, Pattern(r"^[^\d\W]\w*$")]
+    config_version: Optional[int] = 2
+    # Annotated is used by mashumaro for jsonschema generation
+    version: Optional[Union[Annotated[SemverString, Pattern(sem_ver_pattern)], float]] = None
+    project_root: Optional[str] = None
+    source_paths: Optional[List[str]] = None
+    model_paths: Optional[List[str]] = None
+    macro_paths: Optional[List[str]] = None
+    data_paths: Optional[List[str]] = None  # deprecated
+    seed_paths: Optional[List[str]] = None
+    test_paths: Optional[List[str]] = None
+    analysis_paths: Optional[List[str]] = None
+    docs_paths: Optional[List[str]] = None
+    asset_paths: Optional[List[str]] = None
+    function_paths: Optional[List[str]] = None
+    target_path: Optional[str] = None
+    snapshot_paths: Optional[List[str]] = None
+    clean_targets: Optional[List[str]] = None
+    profile: Optional[str] = None
+    log_path: Optional[str] = None
+    packages_install_path: Optional[str] = None
+    quoting: Optional[Quoting] = None
+    on_run_start: Optional[List[str]] = field(default_factory=list_str)
+    on_run_end: Optional[List[str]] = field(default_factory=list_str)
+    require_dbt_version: Optional[Union[List[str], str]] = None
+    dispatch: List[Dict[str, Any]] = field(default_factory=list)
+    models: Dict[str, Any] = field(default_factory=dict)
+    seeds: Dict[str, Any] = field(default_factory=dict)
+    snapshots: Dict[str, Any] = field(default_factory=dict)
+    analyses: Dict[str, Any] = field(default_factory=dict)
+    sources: Dict[str, Any] = field(default_factory=dict)
+    tests: Dict[str, Any] = field(default_factory=dict)  # deprecated
+    data_tests: Dict[str, Any] = field(default_factory=dict)
+    unit_tests: Dict[str, Any] = field(default_factory=dict)
+    metrics: Dict[str, Any] = field(default_factory=dict)
+    semantic_models: Dict[str, Any] = field(default_factory=dict)
+    saved_queries: Dict[str, Any] = field(default_factory=dict)
+    exposures: Dict[str, Any] = field(default_factory=dict)
+    functions: Dict[str, Any] = field(default_factory=dict)
+    vars: Optional[Dict[str, Any]] = field(
+        default=None,
+        metadata=dict(
+            description="map project names to their vars override dicts",
+        ),
+    )
+    packages: List[PackageSpec] = field(default_factory=list)
+    query_comment: Optional[Union[QueryComment, NoValue, str]] = field(default_factory=NoValue)
+    restrict_access: bool = False
+    dbt_cloud: Optional[Dict[str, Any]] = None
+    flags: Dict[str, Any] = field(default_factory=dict)
+
+    class Config(dbtMashConfig):
+        # These tell mashumaro to use aliases for jsonschema and for "from_dict"
+        aliases = {
+            "config_version": "config-version",
+            "project_root": "project-root",
+            "source_paths": "source-paths",
+            "model_paths": "model-paths",
+            "macro_paths": "macro-paths",
+            "data_paths": "data-paths",
+            "seed_paths": "seed-paths",
+            "test_paths": "test-paths",
+            "analysis_paths": "analysis-paths",
+            "docs_paths": "docs-paths",
+            "asset_paths": "asset-paths",
+            "function_paths": "function-paths",
+            "target_path": "target-path",
+            "snapshot_paths": "snapshot-paths",
+            "clean_targets": "clean-targets",
+            "log_path": "log-path",
+            "packages_install_path": "packages-install-path",
+            "on_run_start": "on-run-start",
+            "on_run_end": "on-run-end",
+            "require_dbt_version": "require-dbt-version",
+            "query_comment": "query-comment",
+            "restrict_access": "restrict-access",
+            "semantic_models": "semantic-models",
+            "saved_queries": "saved-queries",
+            "dbt_cloud": "dbt-cloud",
+        }
+
+    @classmethod
+    def validate(cls, data):
+        super().validate(data)
+        if data["name"] in BANNED_PROJECT_NAMES:
+            raise ValidationError(f"Invalid project name: {data['name']} is a reserved word")
+        # validate dispatch config
+        if "dispatch" in data and data["dispatch"]:
+            entries = data["dispatch"]
+            for entry in entries:
+                if (
+                    "macro_namespace" not in entry
+                    or "search_order" not in entry
+                    or not isinstance(entry["search_order"], list)
+                ):
+                    raise ValidationError(f"Invalid project dispatch config: {entry}")
+        if "dbt_cloud" in data and not isinstance(data["dbt_cloud"], dict):
+            raise ValidationError(
+                f"Invalid dbt_cloud config. Expected a 'dict' but got '{type(data['dbt_cloud'])}'"
+            )
+        if data.get("tests", None) and data.get("data_tests", None):
+            raise ValidationError(
+                "Invalid project config: cannot have both 'tests' and 'data_tests' defined"
+            )
+
+
+@dataclass
+class ProjectFlags(ExtensibleDbtClassMixin):
+    cache_selected_only: Optional[bool] = None
+    debug: Optional[bool] = None
+    fail_fast: Optional[bool] = None
+    indirect_selection: Optional[str] = None
+    log_format: Optional[str] = None
+    log_format_file: Optional[str] = None
+    log_level: Optional[str] = None
+    log_level_file: Optional[str] = None
+    partial_parse: Optional[bool] = None
+    populate_cache: Optional[bool] = None
+    printer_width: Optional[int] = None
+    send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS
+    static_parser: Optional[bool] = None
+    use_colors: Optional[bool] = None
+    use_colors_file: Optional[bool] = None
+    use_experimental_parser: Optional[bool] = None
+    version_check: Optional[bool] = None
+    warn_error: Optional[bool] = None
+    warn_error_options: Optional[Dict[str, Union[str, List[str]]]] = None
+    write_json: Optional[bool] = None
+
+    # legacy behaviors - https://github.com/dbt-labs/dbt-core/blob/main/docs/guides/behavior-change-flags.md
+    require_batched_execution_for_custom_microbatch_strategy: bool = False
+    require_event_names_in_deprecations: bool = False
+    require_explicit_package_overrides_for_builtin_materializations: bool = True
+    require_resource_names_without_spaces: bool = True
+    source_freshness_run_project_hooks: bool = True
+    skip_nodes_if_on_run_start_fails: bool = False
+    state_modified_compare_more_unrendered_values: bool = False
+    state_modified_compare_vars: bool = False
+    require_yaml_configuration_for_mf_time_spines: bool = False
+    require_nested_cumulative_type_params: bool = False
+    validate_macro_args: bool = False
+    require_all_warnings_handled_by_warn_error: bool = False
+    require_generic_test_arguments_property: bool = True
+
+    @property
+    def project_only_flags(self) -> Dict[str, Any]:
+        return {
+            "require_batched_execution_for_custom_microbatch_strategy": self.require_batched_execution_for_custom_microbatch_strategy,
+            "require_explicit_package_overrides_for_builtin_materializations": self.require_explicit_package_overrides_for_builtin_materializations,
+            "require_resource_names_without_spaces": self.require_resource_names_without_spaces,
+            "source_freshness_run_project_hooks": self.source_freshness_run_project_hooks,
+            "skip_nodes_if_on_run_start_fails": self.skip_nodes_if_on_run_start_fails,
+            "state_modified_compare_more_unrendered_values": self.state_modified_compare_more_unrendered_values,
+            "state_modified_compare_vars": self.state_modified_compare_vars,
+            "require_yaml_configuration_for_mf_time_spines": self.require_yaml_configuration_for_mf_time_spines,
+            "require_nested_cumulative_type_params": self.require_nested_cumulative_type_params,
+            "validate_macro_args": self.validate_macro_args,
+            "require_all_warnings_handled_by_warn_error": self.require_all_warnings_handled_by_warn_error,
+            "require_generic_test_arguments_property": self.require_generic_test_arguments_property,
+        }
+
+
+@dataclass
+class ProfileConfig(dbtClassMixin):
+    profile_name: str
+    target_name: str
+    threads: int
+    # TODO: make this a dynamic union of some kind?
+    credentials: Optional[Dict[str, Any]]
+
+
+@dataclass
+class ConfiguredQuoting(Quoting):
+    identifier: bool = True
+    schema: bool = True
+    database: Optional[bool] = None
+    project: Optional[bool] = None
+    snowflake_ignore_case: Optional[bool] = None
+
+
+@dataclass
+class Configuration(Project, ProfileConfig):
+    cli_vars: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata={"preserve_underscore": True},
+    )
+    quoting: Optional[ConfiguredQuoting] = None
+
+
+@dataclass
+class ProjectList(dbtClassMixin):
+    projects: Dict[str, Project]

dvt/contracts/results.py

@@ -0,0 +1,53 @@
+# flake8: noqa
+
+# This file is temporary, in order to not break various adapter tests, etc, until
+# they are updated to use the new locations.
+
+from dvt.artifacts.schemas.base import (
+    ArtifactMixin,
+    BaseArtifactMetadata,
+    VersionedSchema,
+    schema_version,
+)
+from dvt.artifacts.schemas.catalog import (
+    CatalogArtifact,
+    CatalogKey,
+    CatalogMetadata,
+    CatalogResults,
+    CatalogTable,
+    ColumnMetadata,
+    StatsItem,
+    TableMetadata,
+)
+from dvt.artifacts.schemas.freshness import (
+    FreshnessErrorEnum,
+    FreshnessExecutionResultArtifact,
+    FreshnessMetadata,
+    FreshnessNodeOutput,
+    FreshnessNodeResult,
+    FreshnessResult,
+    PartialSourceFreshnessResult,
+    SourceFreshnessOutput,
+    SourceFreshnessResult,
+    SourceFreshnessRuntimeError,
+    process_freshness_result,
+)
+from dvt.artifacts.schemas.results import (
+    BaseResult,
+    ExecutionResult,
+    FreshnessStatus,
+    NodeResult,
+    NodeStatus,
+    RunningStatus,
+    RunStatus,
+    TestStatus,
+    TimingInfo,
+    collect_timing_info,
+)
+from dvt.artifacts.schemas.run import (
+    RunExecutionResult,
+    RunResult,
+    RunResultsArtifact,
+    RunResultsMetadata,
+    process_run_result,
+)

dvt/contracts/selection.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import Any, Dict, List, Union
+
+from dbt_common.dataclass_schema import dbtClassMixin
+
+
+@dataclass
+class SelectorDefinition(dbtClassMixin):
+    name: str
+    definition: Union[str, Dict[str, Any]]
+    description: str = ""
+    default: bool = False
+
+
+@dataclass
+class SelectorFile(dbtClassMixin):
+    selectors: List[SelectorDefinition]
+    version: int = 2
+
+
+# @dataclass
+# class SelectorCollection:
+#     packages: Dict[str, List[SelectorFile]] = field(default_factory=dict)

dvt/contracts/sql.py

@@ -0,0 +1,86 @@
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Sequence
+
+from dvt.artifacts.schemas.base import VersionedSchema, schema_version
+from dvt.artifacts.schemas.results import ExecutionResult, TimingInfo
+from dvt.artifacts.schemas.run import RunExecutionResult, RunResult, RunResultsArtifact
+from dvt.contracts.graph.nodes import ResultNode
+from dvt.events.types import ArtifactWritten
+
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.events.functions import fire_event
+
+TaskTags = Optional[Dict[str, Any]]
+TaskID = uuid.UUID
+
+# Outputs
+
+
+@dataclass
+class RemoteCompileResultMixin(VersionedSchema):
+    raw_code: str
+    compiled_code: str
+    node: ResultNode
+    timing: List[TimingInfo]
+
+
+@dataclass
+@schema_version("remote-compile-result", 1)
+class RemoteCompileResult(RemoteCompileResultMixin):
+    generated_at: datetime = field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)
+    )
+
+    @property
+    def error(self) -> None:
+        # TODO: Can we delete this? It's never set anywhere else and never accessed
+        return None
+
+
+@dataclass
+@schema_version("remote-execution-result", 1)
+class RemoteExecutionResult(ExecutionResult):
+    results: Sequence[RunResult]
+    args: Dict[str, Any] = field(default_factory=dict)
+    generated_at: datetime = field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)
+    )
+
+    def write(self, path: str) -> None:
+        writable = RunResultsArtifact.from_execution_results(
+            generated_at=self.generated_at,
+            results=self.results,
+            elapsed_time=self.elapsed_time,
+            args=self.args,
+        )
+        writable.write(path)
+        fire_event(ArtifactWritten(artifact_type=writable.__class__.__name__, artifact_path=path))
+
+    @classmethod
+    def from_local_result(
+        cls,
+        base: RunExecutionResult,
+    ) -> "RemoteExecutionResult":
+        return cls(
+            generated_at=base.generated_at,
+            results=base.results,
+            elapsed_time=base.elapsed_time,
+            args=base.args,
+        )
+
+
+@dataclass
+class ResultTable(dbtClassMixin):
+    column_names: List[str]
+    rows: List[Any]
+
+
+@dataclass
+@schema_version("remote-run-result", 1)
+class RemoteRunResult(RemoteCompileResultMixin):
+    table: ResultTable
+    generated_at: datetime = field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)
+    )

dvt/contracts/state.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+from typing import Optional
+
+from dvt.artifacts.exceptions import IncompatibleSchemaError
+from dvt.artifacts.schemas.freshness import FreshnessExecutionResultArtifact
+from dvt.artifacts.schemas.manifest import WritableManifest
+from dvt.artifacts.schemas.run import RunResultsArtifact
+from dvt.constants import RUN_RESULTS_FILE_NAME
+from dvt.contracts.graph.manifest import Manifest
+from dvt.events.types import WarnStateTargetEqual
+
+from dbt_common.events.functions import fire_event
+
+
+def load_result_state(results_path) -> Optional[RunResultsArtifact]:
+    if results_path.exists() and results_path.is_file():
+        try:
+            return RunResultsArtifact.read_and_check_versions(str(results_path))
+        except IncompatibleSchemaError as exc:
+            exc.add_filename(str(results_path))
+            raise
+    return None
+
+
+class PreviousState:
+    def __init__(self, state_path: Path, target_path: Path, project_root: Path) -> None:
+        self.state_path: Path = state_path
+        self.target_path: Path = target_path
+        self.project_root: Path = project_root
+        self.manifest: Optional[Manifest] = None
+        self.results: Optional[RunResultsArtifact] = None
+        self.sources: Optional[FreshnessExecutionResultArtifact] = None
+        self.sources_current: Optional[FreshnessExecutionResultArtifact] = None
+
+        if self.state_path == self.target_path:
+            fire_event(WarnStateTargetEqual(state_path=str(self.state_path)))
+
+        # Note: if state_path is absolute, project_root will be ignored.
+        manifest_path = self.project_root / self.state_path / "manifest.json"
+        if manifest_path.exists() and manifest_path.is_file():
+            try:
+                writable_manifest = WritableManifest.read_and_check_versions(str(manifest_path))
+                self.manifest = Manifest.from_writable_manifest(writable_manifest)
+            except IncompatibleSchemaError as exc:
+                exc.add_filename(str(manifest_path))
+                raise
+
+        results_path = self.project_root / self.state_path / RUN_RESULTS_FILE_NAME
+        self.results = load_result_state(results_path)
+
+        sources_path = self.project_root / self.state_path / "sources.json"
+        if sources_path.exists() and sources_path.is_file():
+            try:
+                self.sources = FreshnessExecutionResultArtifact.read_and_check_versions(
+                    str(sources_path)
+                )
+            except IncompatibleSchemaError as exc:
+                exc.add_filename(str(sources_path))
+                raise
+
+        sources_current_path = self.project_root / self.target_path / "sources.json"
+        if sources_current_path.exists() and sources_current_path.is_file():
+            try:
+                self.sources_current = FreshnessExecutionResultArtifact.read_and_check_versions(
+                    str(sources_current_path)
+                )
+            except IncompatibleSchemaError as exc:
+                exc.add_filename(str(sources_current_path))
+                raise

dvt/contracts/util.py

@@ -0,0 +1,46 @@
+from typing import Any, List, Tuple
+
+# Leave imports of `Mergeable` to preserve import paths
+from dbt_common.contracts.util import Mergeable  # noqa:F401
+from dbt_common.dataclass_schema import ValidatedStringMixin, ValidationError
+
+SourceKey = Tuple[str, str]
+
+
+def list_str() -> List[str]:
+    """Mypy gets upset about stuff like:
+
+    from dataclasses import dataclass, field
+    from typing import Optional, List
+
+    @dataclass
+    class Foo:
+        x: Optional[List[str]] = field(default_factory=list)
+
+
+    Because `list` could be any kind of list, I guess
+    """
+    return []
+
+
+class Identifier(ValidatedStringMixin):
+    """Our definition of a valid Identifier is the same as what's valid for an unquoted database table name.
+
+    That is:
+    1. It can contain a-z, A-Z, 0-9, and _
+    1. It cannot start with a number
+    """
+
+    ValidationRegex = r"^[^\d\W]\w*$"
+
+    @classmethod
+    def is_valid(cls, value: Any) -> bool:
+        if not isinstance(value, str):
+            return False
+
+        try:
+            cls.validate(value)
+        except ValidationError:
+            return False
+
+        return True