dvt-core 1.11.0b4__py3-none-any.whl

dvt/deprecations.py

@@ -0,0 +1,347 @@
+import abc
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any, Callable, ClassVar, DefaultDict, Dict, List, Optional
+
+import dvt.tracking
+from dvt.events import types as core_types
+from dvt.flags import get_flags
+
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.events.functions import fire_event, warn_or_error
+from dbt_common.events.types import Note
+
+
+class DBTDeprecation:
+    _name: ClassVar[Optional[str]] = None
+    _event: ClassVar[Optional[str]] = None
+    _is_preview: ClassVar[bool] = False
+
+    @property
+    def name(self) -> str:
+        if self._name is not None:
+            return self._name
+        raise NotImplementedError("name not implemented for {}".format(self))
+
+    def track_deprecation_warn(self) -> None:
+        if dbt.tracking.active_user is not None:
+            dbt.tracking.track_deprecation_warn({"deprecation_name": self.name})
+
+    @property
+    def event(self) -> abc.ABCMeta:
+        if self._event is not None:
+            module_path = core_types
+            class_name = self._event
+
+            try:
+                return getattr(module_path, class_name)
+            except AttributeError:
+                msg = f"Event Class `{class_name}` is not defined in `{module_path}`"
+                raise NameError(msg)
+        raise NotImplementedError("event not implemented for {}".format(self._event))
+
+    def preview(self, base_event: abc.ABCMeta) -> None:
+        note_event = Note(msg=base_event.message())  # type: ignore
+        fire_event(note_event)
+
+    def show(self, *args, **kwargs) -> None:
+        if self._is_preview:
+            base_event = self.event(**kwargs)
+            self.preview(base_event)
+        else:
+            flags = get_flags()
+            if self.name not in active_deprecations or flags.show_all_deprecations:
+                event = self.event(**kwargs)
+                warn_or_error(event)
+                self.track_deprecation_warn()
+
+            active_deprecations[self.name] += 1
+
+
+class PackageRedirectDeprecation(DBTDeprecation):
+    _name = "package-redirect"
+    _event = "PackageRedirectDeprecation"
+
+
+class PackageInstallPathDeprecation(DBTDeprecation):
+    _name = "install-packages-path"
+    _event = "PackageInstallPathDeprecation"
+
+
+# deprecations with a pattern of `project-config-*` for the name are not hardcoded
+# they are called programatically via the pattern below
+class ConfigSourcePathDeprecation(DBTDeprecation):
+    _name = "project-config-source-paths"
+    _event = "ConfigSourcePathDeprecation"
+
+
+class ConfigDataPathDeprecation(DBTDeprecation):
+    _name = "project-config-data-paths"
+    _event = "ConfigDataPathDeprecation"
+
+
+class ConfigLogPathDeprecation(DBTDeprecation):
+    _name = "project-config-log-path"
+    _event = "ConfigLogPathDeprecation"
+
+
+class ConfigTargetPathDeprecation(DBTDeprecation):
+    _name = "project-config-target-path"
+    _event = "ConfigTargetPathDeprecation"
+
+
+def renamed_method(old_name: str, new_name: str):
+    class AdapterDeprecationWarning(DBTDeprecation):
+        _name = "adapter:{}".format(old_name)
+        _event = "AdapterDeprecationWarning"
+
+    dep = AdapterDeprecationWarning()
+    deprecations_list.append(dep)
+    deprecations[dep.name] = dep
+
+
+class MetricAttributesRenamed(DBTDeprecation):
+    _name = "metric-attr-renamed"
+    _event = "MetricAttributesRenamed"
+
+
+class ExposureNameDeprecation(DBTDeprecation):
+    _name = "exposure-name"
+    _event = "ExposureNameDeprecation"
+
+
+class CollectFreshnessReturnSignature(DBTDeprecation):
+    _name = "collect-freshness-return-signature"
+    _event = "CollectFreshnessReturnSignature"
+
+
+class ProjectFlagsMovedDeprecation(DBTDeprecation):
+    _name = "project-flags-moved"
+    _event = "ProjectFlagsMovedDeprecation"
+
+
+class PackageMaterializationOverrideDeprecation(DBTDeprecation):
+    _name = "package-materialization-override"
+    _event = "PackageMaterializationOverrideDeprecation"
+
+
+class ResourceNamesWithSpacesDeprecation(DBTDeprecation):
+    _name = "resource-names-with-spaces"
+    _event = "ResourceNamesWithSpacesDeprecation"
+
+
+class SourceFreshnessProjectHooksNotRun(DBTDeprecation):
+    _name = "source-freshness-project-hooks"
+    _event = "SourceFreshnessProjectHooksNotRun"
+
+
+class MFTimespineWithoutYamlConfigurationDeprecation(DBTDeprecation):
+    _name = "mf-timespine-without-yaml-configuration"
+    _event = "MFTimespineWithoutYamlConfigurationDeprecation"
+
+
+class MFCumulativeTypeParamsDeprecation(DBTDeprecation):
+    _name = "mf-cumulative-type-params-deprecation"
+    _event = "MFCumulativeTypeParamsDeprecation"
+
+
+class MicrobatchMacroOutsideOfBatchesDeprecation(DBTDeprecation):
+    _name = "microbatch-macro-outside-of-batches-deprecation"
+    _event = "MicrobatchMacroOutsideOfBatchesDeprecation"
+
+
+class GenericJSONSchemaValidationDeprecation(DBTDeprecation):
+    _name = "generic-json-schema-validation-deprecation"
+    _event = "GenericJSONSchemaValidationDeprecation"
+    _is_preview = True
+
+
+class UnexpectedJinjaBlockDeprecation(DBTDeprecation):
+    _name = "unexpected-jinja-block-deprecation"
+    _event = "UnexpectedJinjaBlockDeprecation"
+
+
+class DuplicateYAMLKeysDeprecation(DBTDeprecation):
+    _name = "duplicate-yaml-keys-deprecation"
+    _event = "DuplicateYAMLKeysDeprecation"
+
+
+class CustomTopLevelKeyDeprecation(DBTDeprecation):
+    _name = "custom-top-level-key-deprecation"
+    _event = "CustomTopLevelKeyDeprecation"
+
+
+class CustomKeyInConfigDeprecation(DBTDeprecation):
+    _name = "custom-key-in-config-deprecation"
+    _event = "CustomKeyInConfigDeprecation"
+
+
+class CustomKeyInObjectDeprecation(DBTDeprecation):
+    _name = "custom-key-in-object-deprecation"
+    _event = "CustomKeyInObjectDeprecation"
+
+
+class WEOInlcudeExcludeDeprecation(DBTDeprecation):
+    _name = "weo-include-exclude-deprecation"
+    _event = "WEOIncludeExcludeDeprecation"
+
+
+class CustomOutputPathInSourceFreshnessDeprecation(DBTDeprecation):
+    _name = "custom-output-path-in-source-freshness-deprecation"
+    _event = "CustomOutputPathInSourceFreshnessDeprecation"
+
+
+class SourceOverrideDeprecation(DBTDeprecation):
+    _name = "source-override-deprecation"
+    _event = "SourceOverrideDeprecation"
+
+
+class PropertyMovedToConfigDeprecation(DBTDeprecation):
+    _name = "property-moved-to-config-deprecation"
+    _event = "PropertyMovedToConfigDeprecation"
+
+
+class ModelParamUsageDeprecation(DBTDeprecation):
+    _name = "model-param-usage-deprecation"
+    _event = "ModelParamUsageDeprecation"
+
+
+class EnvironmentVariableNamespaceDeprecation(DBTDeprecation):
+    _name = "environment-variable-namespace-deprecation"
+    _event = "EnvironmentVariableNamespaceDeprecation"
+
+
+class MissingPlusPrefixDeprecation(DBTDeprecation):
+    _name = "missing-plus-prefix-in-config-deprecation"
+    _event = "MissingPlusPrefixDeprecation"
+
+
+class ArgumentsPropertyInGenericTestDeprecation(DBTDeprecation):
+    _name = "arguments-property-in-generic-test-deprecation"
+    _event = "ArgumentsPropertyInGenericTestDeprecation"
+
+
+class MissingArgumentsPropertyInGenericTestDeprecation(DBTDeprecation):
+    _name = "missing-arguments-property-in-generic-test-deprecation"
+    _event = "MissingArgumentsPropertyInGenericTestDeprecation"
+
+
+class ModulesItertoolsUsageDeprecation(DBTDeprecation):
+    _name = "modules-itertools-usage-deprecation"
+    _event = "ModulesItertoolsUsageDeprecation"
+
+
+def renamed_env_var(old_name: str, new_name: str):
+    class EnvironmentVariableRenamed(DBTDeprecation):
+        _name = f"environment-variable-renamed:{old_name}"
+        _event = "EnvironmentVariableRenamed"
+
+    dep = EnvironmentVariableRenamed()
+    deprecations_list.append(dep)
+    deprecations[dep.name] = dep
+
+    def cb():
+        dep.show(old_name=old_name, new_name=new_name)
+
+    return cb
+
+
+def warn(name: str, *args, **kwargs) -> None:
+    if name not in deprecations:
+        # this should (hopefully) never happen
+        raise RuntimeError("Error showing deprecation warning: {}".format(name))
+
+    deprecations[name].show(*args, **kwargs)
+
+
+def buffer(name: str, *args, **kwargs):
+    def show_callback():
+        deprecations[name].show(*args, **kwargs)
+
+    buffered_deprecations.append(show_callback)
+
+
+def show_deprecations_summary() -> None:
+    summaries: List[Dict[str, Any]] = []
+    for deprecation, occurrences in active_deprecations.items():
+        deprecation_event = deprecations[deprecation].event()
+        summaries.append(
+            DeprecationSummary(
+                event_name=deprecation_event.__name__,
+                event_code=deprecation_event.code(),
+                occurrences=occurrences,
+            ).to_msg_dict()
+        )
+
+    if len(summaries) > 0:
+        show_all_hint = not get_flags().show_all_deprecations
+        warn_or_error(
+            core_types.DeprecationsSummary(summaries=summaries, show_all_hint=show_all_hint)
+        )
+
+
+# these are globally available
+# since modules are only imported once, active_deprecations is a singleton
+
+active_deprecations: DefaultDict[str, int] = defaultdict(int)
+
+deprecations_list: List[DBTDeprecation] = [
+    PackageRedirectDeprecation(),
+    PackageInstallPathDeprecation(),
+    ConfigSourcePathDeprecation(),
+    ConfigDataPathDeprecation(),
+    ExposureNameDeprecation(),
+    ConfigLogPathDeprecation(),
+    ConfigTargetPathDeprecation(),
+    CollectFreshnessReturnSignature(),
+    ProjectFlagsMovedDeprecation(),
+    PackageMaterializationOverrideDeprecation(),
+    ResourceNamesWithSpacesDeprecation(),
+    SourceFreshnessProjectHooksNotRun(),
+    MFTimespineWithoutYamlConfigurationDeprecation(),
+    MFCumulativeTypeParamsDeprecation(),
+    MicrobatchMacroOutsideOfBatchesDeprecation(),
+    GenericJSONSchemaValidationDeprecation(),
+    UnexpectedJinjaBlockDeprecation(),
+    DuplicateYAMLKeysDeprecation(),
+    CustomTopLevelKeyDeprecation(),
+    CustomKeyInConfigDeprecation(),
+    CustomKeyInObjectDeprecation(),
+    CustomOutputPathInSourceFreshnessDeprecation(),
+    PropertyMovedToConfigDeprecation(),
+    ModelParamUsageDeprecation(),
+    WEOInlcudeExcludeDeprecation(),
+    SourceOverrideDeprecation(),
+    EnvironmentVariableNamespaceDeprecation(),
+    MissingPlusPrefixDeprecation(),
+    ArgumentsPropertyInGenericTestDeprecation(),
+    MissingArgumentsPropertyInGenericTestDeprecation(),
+    ModulesItertoolsUsageDeprecation(),
+]
+
+deprecations: Dict[str, DBTDeprecation] = {d.name: d for d in deprecations_list}
+
+buffered_deprecations: List[Callable] = []
+
+
+def reset_deprecations():
+    active_deprecations.clear()
+
+
+def fire_buffered_deprecations():
+    [dep_fn() for dep_fn in buffered_deprecations]
+    buffered_deprecations.clear()
+
+
+@dataclass
+class DeprecationSummary(dbtClassMixin):
+    event_name: str
+    event_code: str
+    occurrences: int
+
+    def to_msg_dict(self) -> Dict[str, Any]:
+        return {
+            "event_name": self.event_name,
+            "event_code": self.event_code,
+            "occurrences": self.occurrences,
+        }

dvt/deps/base.py

@@ -0,0 +1,153 @@
+import abc
+import functools
+import os
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Dict, Generic, List, Optional, TypeVar
+
+from dvt.contracts.project import ProjectPackageMetadata
+from dvt.events.types import DepsSetDownloadDirectory
+
+from dbt_common.clients import system
+from dbt_common.events.functions import fire_event
+from dbt_common.utils.connection import connection_exception_retry
+
+DOWNLOADS_PATH = None
+
+
+def get_downloads_path():
+    return DOWNLOADS_PATH
+
+
+@contextmanager
+def downloads_directory():
+    global DOWNLOADS_PATH
+    remove_downloads = False
+    # the user might have set an environment variable. Set it to that, and do
+    # not remove it when finished.
+    if DOWNLOADS_PATH is None:
+        DOWNLOADS_PATH = os.getenv("DBT_DOWNLOADS_DIR")
+        remove_downloads = False
+    # if we are making a per-run temp directory, remove it at the end of
+    # successful runs
+    if DOWNLOADS_PATH is None:
+        DOWNLOADS_PATH = tempfile.mkdtemp(prefix="dbt-downloads-")
+        remove_downloads = True
+
+    system.make_directory(DOWNLOADS_PATH)
+    fire_event(DepsSetDownloadDirectory(path=DOWNLOADS_PATH))
+
+    yield DOWNLOADS_PATH
+
+    if remove_downloads:
+        system.rmtree(DOWNLOADS_PATH)
+        DOWNLOADS_PATH = None
+
+
+class BasePackage(metaclass=abc.ABCMeta):
+    @abc.abstractproperty
+    def name(self) -> str:
+        raise NotImplementedError
+
+    def all_names(self) -> List[str]:
+        return [self.name]
+
+    @abc.abstractmethod
+    def source_type(self) -> str:
+        raise NotImplementedError
+
+
+class PinnedPackage(BasePackage):
+    def __init__(self) -> None:
+        self._cached_metadata: Optional[ProjectPackageMetadata] = None
+
+    def __str__(self) -> str:
+        version = self.get_version()
+        if not version:
+            return self.name
+
+        return "{}@{}".format(self.name, version)
+
+    @abc.abstractmethod
+    def get_version(self) -> Optional[str]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def _fetch_metadata(self, project, renderer):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def install(self, project, renderer):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def nice_version_name(self):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def to_dict(self) -> Dict[str, str]:
+        raise NotImplementedError
+
+    def fetch_metadata(self, project, renderer):
+        if not self._cached_metadata:
+            self._cached_metadata = self._fetch_metadata(project, renderer)
+        return self._cached_metadata
+
+    def get_project_name(self, project, renderer):
+        metadata = self.fetch_metadata(project, renderer)
+        return metadata.name
+
+    def get_installation_path(self, project, renderer):
+        dest_dirname = self.get_project_name(project, renderer)
+        return os.path.join(project.packages_install_path, dest_dirname)
+
+    def get_subdirectory(self):
+        return None
+
+    def _install(self, project, renderer):
+        metadata = self.fetch_metadata(project, renderer)
+
+        tar_name = f"{self.package}.{self.version}.tar.gz"
+        tar_path = (Path(get_downloads_path()) / tar_name).resolve(strict=False)
+        system.make_directory(str(tar_path.parent))
+
+        download_url = metadata.downloads.tarball
+        deps_path = project.packages_install_path
+        package_name = self.get_project_name(project, renderer)
+
+        download_untar_fn = functools.partial(
+            self.download_and_untar, download_url, str(tar_path), deps_path, package_name
+        )
+        connection_exception_retry(download_untar_fn, 5)
+
+    def download_and_untar(self, download_url, tar_path, deps_path, package_name):
+        """
+        Sometimes the download of the files fails and we want to retry.  Sometimes the
+        download appears successful but the file did not make it through as expected
+        (generally due to a github incident).  Either way we want to retry downloading
+        and untarring to see if we can get a success.  Call this within
+        `connection_exception_retry`
+        """
+
+        system.download(download_url, tar_path)
+        system.untar_package(tar_path, deps_path, package_name)
+
+
+SomePinned = TypeVar("SomePinned", bound=PinnedPackage)
+SomeUnpinned = TypeVar("SomeUnpinned", bound="UnpinnedPackage")
+
+
+class UnpinnedPackage(Generic[SomePinned], BasePackage):
+    @classmethod
+    @abc.abstractmethod
+    def from_contract(cls, contract):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def incorporate(self: SomeUnpinned, other: SomeUnpinned) -> SomeUnpinned:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def resolved(self) -> SomePinned:
+        raise NotImplementedError

dvt/deps/git.py

@@ -0,0 +1,196 @@
+import os
+from typing import Dict, List, Optional
+
+from dvt.clients import git
+from dvt.config.project import PartialProject, Project
+from dvt.config.renderer import PackageRenderer
+from dvt.contracts.project import GitPackage, ProjectPackageMetadata
+from dvt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
+from dvt.events.types import DepsScrubbedPackageName, DepsUnpinned, EnsureGitInstalled
+from dvt.exceptions import MultipleVersionGitDepsError
+from dvt.utils import md5
+
+from dbt_common.clients import system
+from dbt_common.events.functions import (
+    env_secrets,
+    fire_event,
+    scrub_secrets,
+    warn_or_error,
+)
+from dbt_common.exceptions import ExecutableError
+
+
+def md5sum(s: str):
+    return md5(s, "latin-1")
+
+
+class GitPackageMixin:
+    def __init__(
+        self,
+        git: str,
+        git_unrendered: str,
+        subdirectory: Optional[str] = None,
+    ) -> None:
+        super().__init__()
+        self.git = git
+        self.git_unrendered = git_unrendered
+        self.subdirectory = subdirectory
+
+    @property
+    def name(self):
+        return f"{self.git}/{self.subdirectory}" if self.subdirectory else self.git
+
+    def source_type(self) -> str:
+        return "git"
+
+
+class GitPinnedPackage(GitPackageMixin, PinnedPackage):
+    def __init__(
+        self,
+        git: str,
+        git_unrendered: str,
+        revision: str,
+        warn_unpinned: bool = True,
+        subdirectory: Optional[str] = None,
+    ) -> None:
+        super().__init__(git, git_unrendered, subdirectory)
+        self.revision = revision
+        self.warn_unpinned = warn_unpinned
+        self.subdirectory = subdirectory
+        self._checkout_name = md5sum(self.name)
+
+    def to_dict(self) -> Dict[str, str]:
+        git_scrubbed = scrub_secrets(self.git_unrendered, env_secrets())
+        if self.git_unrendered != git_scrubbed:
+            warn_or_error(DepsScrubbedPackageName(package_name=git_scrubbed))
+        ret = {
+            "git": git_scrubbed,
+            "revision": self.revision,
+        }
+        if self.subdirectory:
+            ret["subdirectory"] = self.subdirectory
+        return ret
+
+    def get_version(self):
+        return self.revision
+
+    def get_subdirectory(self):
+        return self.subdirectory
+
+    def nice_version_name(self):
+        if self.revision == "HEAD":
+            return "HEAD (default revision)"
+        else:
+            return "revision {}".format(self.revision)
+
+    def _checkout(self):
+        """Performs a shallow clone of the repository into the downloads
+        directory. This function can be called repeatedly. If the project has
+        already been checked out at this version, it will be a no-op. Returns
+        the path to the checked out directory."""
+        try:
+            dir_ = git.clone_and_checkout(
+                self.git,
+                get_downloads_path(),
+                revision=self.revision,
+                dirname=self._checkout_name,
+                subdirectory=self.subdirectory,
+            )
+        except ExecutableError as exc:
+            if exc.cmd and exc.cmd[0] == "git":
+                fire_event(EnsureGitInstalled())
+            raise
+        return os.path.join(get_downloads_path(), dir_)
+
+    def _fetch_metadata(
+        self, project: Project, renderer: PackageRenderer
+    ) -> ProjectPackageMetadata:
+        path = self._checkout()
+
+        # raise warning (or error) if this package is not pinned
+        if (self.revision == "HEAD" or self.revision in ("main", "master")) and self.warn_unpinned:
+            warn_or_error(DepsUnpinned(revision=self.revision, git=self.git))
+
+        # now overwrite 'revision' with actual commit SHA
+        self.revision = git.get_current_sha(path)
+
+        partial = PartialProject.from_project_root(path)
+        return partial.render_package_metadata(renderer)
+
+    def install(self, project, renderer):
+        dest_path = self.get_installation_path(project, renderer)
+        if os.path.exists(dest_path):
+            if system.path_is_symlink(dest_path):
+                system.remove_file(dest_path)
+            else:
+                system.rmdir(dest_path)
+
+        system.move(self._checkout(), dest_path)
+
+
+class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
+    def __init__(
+        self,
+        git: str,
+        git_unrendered: str,
+        revisions: List[str],
+        warn_unpinned: bool = True,
+        subdirectory: Optional[str] = None,
+    ) -> None:
+        super().__init__(git, git_unrendered, subdirectory)
+        self.revisions = revisions
+        self.warn_unpinned = warn_unpinned
+        self.subdirectory = subdirectory
+
+    @classmethod
+    def from_contract(cls, contract: GitPackage) -> "GitUnpinnedPackage":
+        revisions = contract.get_revisions()
+
+        # we want to map None -> True
+        warn_unpinned = contract.warn_unpinned is not False
+        return cls(
+            git=contract.git,
+            git_unrendered=(contract.unrendered.get("git") or contract.git),
+            revisions=revisions,
+            warn_unpinned=warn_unpinned,
+            subdirectory=contract.subdirectory,
+        )
+
+    def all_names(self) -> List[str]:
+        if self.git.endswith(".git"):
+            other = self.git[:-4]
+        else:
+            other = self.git + ".git"
+
+        if self.subdirectory:
+            git_name = f"{self.git}/{self.subdirectory}"
+            other = f"{other}/{self.subdirectory}"
+        else:
+            git_name = self.git
+
+        return [git_name, other]
+
+    def incorporate(self, other: "GitUnpinnedPackage") -> "GitUnpinnedPackage":
+        warn_unpinned = self.warn_unpinned and other.warn_unpinned
+
+        return GitUnpinnedPackage(
+            git=self.git,
+            git_unrendered=self.git_unrendered,
+            revisions=self.revisions + other.revisions,
+            warn_unpinned=warn_unpinned,
+            subdirectory=self.subdirectory,
+        )
+
+    def resolved(self) -> GitPinnedPackage:
+        requested = set(self.revisions)
+        if len(requested) == 0:
+            requested = {"HEAD"}
+        elif len(requested) > 1:
+            raise MultipleVersionGitDepsError(self.name, requested)
+        return GitPinnedPackage(
+            git=self.git,
+            git_unrendered=self.git_unrendered,
+            revision=requested.pop(),
+            warn_unpinned=self.warn_unpinned,
+            subdirectory=self.subdirectory,
+        )