dvt-core 1.11.0b4__py3-none-any.whl

dvt/deps/local.py

@@ -0,0 +1,80 @@
+import shutil
+from typing import Dict
+
+from dvt.config.project import PartialProject, Project
+from dvt.config.renderer import PackageRenderer
+from dvt.contracts.project import LocalPackage, ProjectPackageMetadata
+from dvt.deps.base import PinnedPackage, UnpinnedPackage
+from dvt.events.types import DepsCreatingLocalSymlink, DepsSymlinkNotAvailable
+
+from dbt_common.clients import system
+from dbt_common.events.functions import fire_event
+
+
+class LocalPackageMixin:
+    def __init__(self, local: str) -> None:
+        super().__init__()
+        self.local = local
+
+    @property
+    def name(self):
+        return self.local
+
+    def source_type(self):
+        return "local"
+
+
+class LocalPinnedPackage(LocalPackageMixin, PinnedPackage):
+    def __init__(self, local: str) -> None:
+        super().__init__(local)
+
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "local": self.local,
+        }
+
+    def get_version(self):
+        return None
+
+    def nice_version_name(self):
+        return "<local @ {}>".format(self.local)
+
+    def resolve_path(self, project):
+        return system.resolve_path_from_base(
+            self.local,
+            project.project_root,
+        )
+
+    def _fetch_metadata(
+        self, project: Project, renderer: PackageRenderer
+    ) -> ProjectPackageMetadata:
+        partial = PartialProject.from_project_root(self.resolve_path(project))
+        return partial.render_package_metadata(renderer)
+
+    def install(self, project, renderer):
+        src_path = self.resolve_path(project)
+        dest_path = self.get_installation_path(project, renderer)
+
+        if system.path_exists(dest_path):
+            if not system.path_is_symlink(dest_path):
+                system.rmdir(dest_path)
+            else:
+                system.remove_file(dest_path)
+        try:
+            fire_event(DepsCreatingLocalSymlink())
+            system.make_symlink(src_path, dest_path)
+        except OSError:
+            fire_event(DepsSymlinkNotAvailable())
+            shutil.copytree(src_path, dest_path)
+
+
+class LocalUnpinnedPackage(LocalPackageMixin, UnpinnedPackage[LocalPinnedPackage]):
+    @classmethod
+    def from_contract(cls, contract: LocalPackage) -> "LocalUnpinnedPackage":
+        return cls(local=contract.local)
+
+    def incorporate(self, other: "LocalUnpinnedPackage") -> "LocalUnpinnedPackage":
+        return LocalUnpinnedPackage(local=self.local)
+
+    def resolved(self) -> LocalPinnedPackage:
+        return LocalPinnedPackage(local=self.local)

dvt/deps/registry.py

@@ -0,0 +1,131 @@
+from typing import Dict, List
+
+from dvt.clients import registry
+from dvt.contracts.project import RegistryPackage, RegistryPackageMetadata
+from dvt.deps.base import PinnedPackage, UnpinnedPackage
+from dvt.exceptions import (
+    DependencyError,
+    PackageNotFoundError,
+    PackageVersionNotFoundError,
+)
+from dvt.flags import get_flags
+from dvt.version import get_installed_version
+
+from dbt_common import semver
+from dbt_common.exceptions import VersionsNotCompatibleError
+
+
+class RegistryPackageMixin:
+    def __init__(self, package: str) -> None:
+        super().__init__()
+        self.package = package
+
+    @property
+    def name(self):
+        return self.package
+
+    def source_type(self) -> str:
+        return "hub"
+
+
+class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
+    def __init__(self, package: str, version: str, version_latest: str) -> None:
+        super().__init__(package)
+        self.version = version
+        self.version_latest = version_latest
+
+    @property
+    def name(self):
+        return self.package
+
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "package": self.package,
+            "version": self.version,
+        }
+
+    def source_type(self):
+        return "hub"
+
+    def get_version(self):
+        return self.version
+
+    def get_version_latest(self):
+        return self.version_latest
+
+    def nice_version_name(self):
+        return "version {}".format(self.version)
+
+    def _fetch_metadata(self, project, renderer) -> RegistryPackageMetadata:
+        dct = registry.package_version(self.package, self.version)
+        return RegistryPackageMetadata.from_dict(dct)
+
+    def install(self, project, renderer):
+        self._install(project, renderer)
+
+
+class RegistryUnpinnedPackage(RegistryPackageMixin, UnpinnedPackage[RegistryPinnedPackage]):
+    def __init__(
+        self, package: str, versions: List[semver.VersionSpecifier], install_prerelease: bool
+    ) -> None:
+        super().__init__(package)
+        self.versions = versions
+        self.install_prerelease = install_prerelease
+
+    def _check_in_index(self):
+        index = registry.index_cached()
+        if self.package not in index:
+            raise PackageNotFoundError(self.package)
+
+    @classmethod
+    def from_contract(cls, contract: RegistryPackage) -> "RegistryUnpinnedPackage":
+        raw_version = contract.get_versions()
+
+        versions = [semver.VersionSpecifier.from_version_string(v) for v in raw_version]
+        return cls(
+            package=contract.package,
+            versions=versions,
+            install_prerelease=bool(contract.install_prerelease),
+        )
+
+    def incorporate(self, other: "RegistryUnpinnedPackage") -> "RegistryUnpinnedPackage":
+        return RegistryUnpinnedPackage(
+            package=self.package,
+            install_prerelease=self.install_prerelease,
+            versions=self.versions + other.versions,
+        )
+
+    def resolved(self) -> RegistryPinnedPackage:
+        self._check_in_index()
+        try:
+            range_ = semver.reduce_versions(*self.versions)
+        except VersionsNotCompatibleError as e:
+            new_msg = "Version error for package {}: {}".format(self.name, e)
+            raise DependencyError(new_msg) from e
+        flags = get_flags()
+        should_version_check = bool(flags.VERSION_CHECK)
+        dbt_version = get_installed_version()
+        compatible_versions = registry.get_compatible_versions(
+            self.package, dbt_version, should_version_check
+        )
+        prerelease_version_specified = any(bool(version.prerelease) for version in self.versions)
+        installable = semver.filter_installable(
+            compatible_versions, self.install_prerelease or prerelease_version_specified
+        )
+        if installable:
+            # for now, pick a version and then recurse. later on,
+            # we'll probably want to traverse multiple options
+            # so we can match packages. not going to make a difference
+            # right now.
+            target = semver.resolve_to_specific_version(range_, installable)
+        else:
+            target = None
+        if not target:
+            # raise an exception if no installable target version is found
+            raise PackageVersionNotFoundError(
+                self.package, range_, installable, should_version_check
+            )
+        latest_compatible = installable[-1]
+        return RegistryPinnedPackage(
+            package=self.package, version=target, version_latest=latest_compatible
+        )

dvt/deps/resolver.py

@@ -0,0 +1,149 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterator, List, NoReturn, Set, Type
+
+from dvt.config import Project
+from dvt.config.renderer import PackageRenderer
+from dvt.contracts.project import (
+    GitPackage,
+    LocalPackage,
+    PackageSpec,
+    PrivatePackage,
+    RegistryPackage,
+    TarballPackage,
+)
+from dvt.deps.base import BasePackage, PinnedPackage, UnpinnedPackage
+from dvt.deps.git import GitUnpinnedPackage
+from dvt.deps.local import LocalUnpinnedPackage
+from dvt.deps.registry import RegistryUnpinnedPackage
+from dvt.deps.tarball import TarballUnpinnedPackage
+from dvt.exceptions import (
+    DependencyError,
+    DuplicateDependencyToRootError,
+    DuplicateProjectDependencyError,
+    MismatchedDependencyTypeError,
+)
+
+
+@dataclass
+class PackageListing:
+    packages: Dict[str, UnpinnedPackage] = field(default_factory=dict)
+
+    def __len__(self):
+        return len(self.packages)
+
+    def __bool__(self):
+        return bool(self.packages)
+
+    def _pick_key(self, key: BasePackage) -> str:
+        for name in key.all_names():
+            if name in self.packages:
+                return name
+        return key.name
+
+    def __contains__(self, key: BasePackage):
+        for name in key.all_names():
+            if name in self.packages:
+                return True
+
+    def __getitem__(self, key: BasePackage):
+        key_str: str = self._pick_key(key)
+        return self.packages[key_str]
+
+    def __setitem__(self, key: BasePackage, value):
+        key_str: str = self._pick_key(key)
+        self.packages[key_str] = value
+
+    def _mismatched_types(self, old: UnpinnedPackage, new: UnpinnedPackage) -> NoReturn:
+        raise MismatchedDependencyTypeError(new, old)
+
+    def incorporate(self, package: UnpinnedPackage):
+        key: str = self._pick_key(package)
+        if key in self.packages:
+            existing: UnpinnedPackage = self.packages[key]
+            if not isinstance(existing, type(package)):
+                self._mismatched_types(existing, package)
+            self.packages[key] = existing.incorporate(package)
+        else:
+            self.packages[key] = package
+
+    def update_from(self, src: List[PackageSpec]) -> None:
+        pkg: UnpinnedPackage
+        for contract in src:
+            if isinstance(contract, LocalPackage):
+                pkg = LocalUnpinnedPackage.from_contract(contract)
+            elif isinstance(contract, TarballPackage):
+                pkg = TarballUnpinnedPackage.from_contract(contract)
+            elif isinstance(contract, GitPackage):
+                pkg = GitUnpinnedPackage.from_contract(contract)
+            elif isinstance(contract, PrivatePackage):
+                raise DependencyError(
+                    f'Cannot resolve private package {contract.private} because git provider integration is missing. Please use a "git" package instead.'
+                )
+            elif isinstance(contract, RegistryPackage):
+                pkg = RegistryUnpinnedPackage.from_contract(contract)
+            else:
+                raise DependencyError("Invalid package type {}".format(type(contract)))
+            self.incorporate(pkg)
+
+    @classmethod
+    def from_contracts(cls: Type["PackageListing"], src: List[PackageSpec]) -> "PackageListing":
+        self = cls({})
+        self.update_from(src)
+        return self
+
+    def resolved(self) -> List[PinnedPackage]:
+        return [p.resolved() for p in self.packages.values()]
+
+    def __iter__(self) -> Iterator[UnpinnedPackage]:
+        return iter(self.packages.values())
+
+
+def _check_for_duplicate_project_names(
+    final_deps: List[PinnedPackage],
+    project: Project,
+    renderer: PackageRenderer,
+):
+    seen: Set[str] = set()
+    for package in final_deps:
+        project_name = package.get_project_name(project, renderer)
+        if project_name in seen:
+            raise DuplicateProjectDependencyError(project_name)
+        elif project_name == project.project_name:
+            raise DuplicateDependencyToRootError(project_name)
+        seen.add(project_name)
+
+
+def resolve_packages(
+    packages: List[PackageSpec],
+    project: Project,
+    cli_vars: Dict[str, Any],
+) -> List[PinnedPackage]:
+    pending = PackageListing.from_contracts(packages)
+    final = PackageListing()
+
+    renderer = PackageRenderer(cli_vars)
+
+    while pending:
+        next_pending = PackageListing()
+        # resolve the dependency in question
+        for package in pending:
+            final.incorporate(package)
+            target = final[package].resolved().fetch_metadata(project, renderer)
+            next_pending.update_from(target.packages)
+        pending = next_pending
+
+    resolved = final.resolved()
+    _check_for_duplicate_project_names(resolved, project, renderer)
+    return resolved
+
+
+def resolve_lock_packages(packages: List[PackageSpec]) -> List[PinnedPackage]:
+    lock_packages = PackageListing.from_contracts(packages)
+    final = PackageListing()
+
+    for package in lock_packages:
+        final.incorporate(package)
+
+    resolved = final.resolved()
+
+    return resolved

dvt/deps/tarball.py

@@ -0,0 +1,121 @@
+import functools
+import os
+from pathlib import Path
+from typing import Dict
+
+from dvt.config.project import PartialProject
+from dvt.contracts.project import TarballPackage
+from dvt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
+from dvt.events.types import DepsScrubbedPackageName
+from dvt.exceptions import DependencyError, env_secrets, scrub_secrets
+
+from dbt_common.clients import system
+from dbt_common.events.functions import warn_or_error
+from dbt_common.utils.connection import connection_exception_retry
+
+
+class TarballPackageMixin:
+    def __init__(self, tarball: str, tarball_unrendered: str) -> None:
+        super().__init__()
+        self.tarball = tarball
+        self.tarball_unrendered = tarball_unrendered
+
+    @property
+    def name(self):
+        return self.tarball
+
+    def source_type(self) -> str:
+        return "tarball"
+
+
+class TarballPinnedPackage(TarballPackageMixin, PinnedPackage):
+    def __init__(self, tarball: str, tarball_unrendered: str, package: str) -> None:
+        super().__init__(tarball, tarball_unrendered)
+        self.package = package
+        self.version = "tarball"
+        self.tar_path = os.path.join(Path(get_downloads_path()), self.package)
+        self.untarred_path = f"{self.tar_path}_untarred"
+
+    @property
+    def name(self):
+        return self.package
+
+    def to_dict(self) -> Dict[str, str]:
+        tarball_scrubbed = scrub_secrets(self.tarball_unrendered, env_secrets())
+        if self.tarball_unrendered != tarball_scrubbed:
+            warn_or_error(DepsScrubbedPackageName(package_name=tarball_scrubbed))
+        return {
+            "tarball": tarball_scrubbed,
+            "name": self.package,
+        }
+
+    def get_version(self):
+        return self.version
+
+    def nice_version_name(self):
+        return f"tarball (url: {self.tarball})"
+
+    def _fetch_metadata(self, project, renderer):
+        """Download and untar the project and parse metadata from the project folder."""
+        download_untar_fn = functools.partial(
+            self.download_and_untar, self.tarball, self.tar_path, self.untarred_path, self.name
+        )
+        connection_exception_retry(download_untar_fn, 5)
+
+        tar_contents = os.listdir(self.untarred_path)
+        if len(tar_contents) != 1:
+            raise DependencyError(
+                f"Incorrect structure for package extracted from {self.tarball}."
+                f"The extracted package needs to follow the structure {self.name}/<package_contents>."
+            )
+        child_folder = os.listdir(self.untarred_path)[0]
+
+        self.untarred_path = os.path.join(self.untarred_path, child_folder)
+        partial = PartialProject.from_project_root(self.untarred_path)
+        metadata = partial.render_package_metadata(renderer)
+        metadata.name = self.package if self.package else metadata.name
+        return metadata
+
+    def install(self, project, renderer):
+        download_untar_fn = functools.partial(
+            self.download_and_untar, self.tarball, self.tar_path, self.untarred_path, self.name
+        )
+        connection_exception_retry(download_untar_fn, 5)
+        dest_path = self.get_installation_path(project, renderer)
+        if os.path.exists(dest_path):
+            if system.path_is_symlink(dest_path):
+                system.remove_file(dest_path)
+            else:
+                system.rmdir(dest_path)
+        system.move(self.untarred_path, dest_path)
+
+
+class TarballUnpinnedPackage(TarballPackageMixin, UnpinnedPackage[TarballPinnedPackage]):
+    def __init__(
+        self,
+        tarball: str,
+        tarball_unrendered: str,
+        package: str,
+    ) -> None:
+        super().__init__(tarball, tarball_unrendered)
+        # setup to recycle RegistryPinnedPackage fns
+        self.package = package
+        self.version = "tarball"
+
+    @classmethod
+    def from_contract(cls, contract: TarballPackage) -> "TarballUnpinnedPackage":
+        return cls(
+            tarball=contract.tarball,
+            tarball_unrendered=(contract.unrendered.get("tarball") or contract.tarball),
+            package=contract.name,
+        )
+
+    def incorporate(self, other: "TarballUnpinnedPackage") -> "TarballUnpinnedPackage":
+        return TarballUnpinnedPackage(
+            tarball=self.tarball, tarball_unrendered=self.tarball_unrendered, package=self.package
+        )
+
+    def resolved(self) -> TarballPinnedPackage:
+        return TarballPinnedPackage(
+            tarball=self.tarball, tarball_unrendered=self.tarball_unrendered, package=self.package
+        )

dvt/docs/source/_ext/dbt_click.py

@@ -0,0 +1,118 @@
+import traceback
+import typing as t
+
+import click
+import click.types as click_t
+import dvt.cli.option_types as dbt_t
+from docutils import nodes
+from docutils.parsers.rst import Directive
+
+PARAM_TYPE_MAP = {
+    click_t.BoolParamType: lambda _: "boolean",
+    click_t.Choice: lambda c: f"choice: {c.choices}",
+    click_t.IntParamType: lambda _: "int",
+    click_t.Path: lambda _: "path",
+    click_t.StringParamType: lambda _: "string",
+    dbt_t.YAML: lambda _: "YAML",
+}
+
+
+def format_command(cmd) -> nodes.section:
+    cmd_name = cmd.name.replace("-", "_")
+    section = nodes.section(
+        "",
+        nodes.title(text=f"Command: {cmd_name}"),
+        ids=[cmd_name],
+        names=[cmd_name],
+    )
+    section.extend(format_params(cmd))
+    return section
+
+
+def format_params(cmd) -> t.List[nodes.section]:
+    lines = []
+    for param in cmd.params:
+        uid = f"{cmd.name}|{param.name}"
+        param_section = nodes.section(
+            "",
+            nodes.title(text=param.name),
+            ids=[uid],
+            names=[uid],
+        )
+
+        get_type_str = PARAM_TYPE_MAP.get(type(param.type), lambda _: "unknown")
+        type_str = get_type_str(param.type)
+
+        param_section.append(nodes.paragraph(text=f"Type: {type_str}"))
+        help_txt = getattr(param, "help", None)
+        if help_txt is not None:
+            param_section.append(nodes.paragraph(text=help_txt))
+        lines.append(param_section)
+    return lines
+
+
+def load_module(module_path: str, error) -> t.Union[click.Command, click.Group]:
+    try:
+        module_name, attr_name = module_path.split(":", 1)
+    except ValueError:  # noqa
+        raise error(f'"{module_path}" is not of format "module:parser"')
+
+    try:
+        mod = __import__(module_name, globals(), locals(), [attr_name])
+    except Exception:  # noqa
+        raise error(
+            f'Failed to import "{attr_name}" from "{module_name}". '
+            f"The following exception was raised:\n{traceback.format_exc()}"
+        )
+
+    if not hasattr(mod, attr_name):
+        raise error(f'Module "{module_name}" has no attribute "{attr_name}"')
+
+    parser = getattr(mod, attr_name)
+
+    if not isinstance(parser, (click.Command, click.Group)):
+        raise error(
+            f'"{type(parser)}" of type "{module_path}" is not click.Command'
+            ' or click.Group."click.BaseCommand"'
+        )
+    return parser
+
+
+class DBTClick(Directive):
+    has_content = False
+    required_arguments = 1
+
+    def run(self):
+        section = nodes.section(
+            "",
+            ids=["dbt-section"],
+            names=["dbt-section"],
+        )
+        cmds = self._get_commands(self.arguments[0])
+        for cmd in cmds:
+            command_section = format_command(cmd)
+            section.extend(command_section)
+        return [section]
+
+    def _get_commands(self, module: str) -> t.List[click.Command]:
+        click_group = load_module(module, self.error)
+        if type(click_group) is not click.Group:
+            raise self.error('Type "click.Group" not supported in dbt_click extension')
+        cmd_strs = [cmd for cmd in click_group.commands]
+        cmd_strs.sort()
+        cmds = []
+        for cmd_str in cmd_strs:
+            cmd = click_group.commands.get(cmd_str)
+            if cmd is not None:
+                cmds.append(cmd)
+        return cmds
+
+
+def setup(app) -> t.Dict[str, t.Any]:
+    app.add_directive("dbt_click", DBTClick)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

dvt/docs/source/conf.py

@@ -0,0 +1,32 @@
+import os
+import sys
+import typing as t
+
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+sys.path.insert(0, os.path.abspath("../../.."))
+sys.path.insert(0, os.path.abspath("./_ext"))
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "dbt-core"
+copyright = "2022, dbt Labs"
+author = "dbt Labs"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ["dbt_click"]
+
+templates_path = ["_templates"]
+exclude_patterns: t.List[str] = []
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "alabaster"
+html_static_path = ["_static"]

dvt/env_vars.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from dvt.cli import params
+from dvt.deprecations import warn
+from dvt.exceptions import DbtInternalError
+
+from dbt_common.constants import ENGINE_ENV_PREFIX
+from dbt_common.context import get_invocation_context
+
+# These are env vars that are not in the params module, but are still allowed to be set.
+# New additions to this list should use the new naming scheme, unless they are being added because
+# they already existed, but we didn't know about them previously.
+# TODO: Should at least some of these become (undocumented) cli param options?
+_ADDITIONAL_ENGINE_ENV_VARS: List[str] = [
+    "DBT_INVOCATION_ENV",
+    "DBT_RECORDED_FILE_PATH",
+    "DBT_TEST_STATE_MODIFIED",  # TODO: This is testing related, should we do this differently?
+    "DBT_PACKAGE_HUB_URL",
+    "DBT_DOWNLOAD_DIR",
+    "DBT_PP_FILE_DIFF_TEST",  # TODO: This is testing related, should we do this differently?
+    "DBT_PP_TEST",  # TODO: This is testing related, should we do this differently?
+]
+
+
+@dataclass(frozen=True, init=False)
+class EngineEnvVar:
+    name: str
+    old_name: Optional[str] = None
+
+    def __init__(self, envvar: str) -> None:
+        if envvar.startswith(ENGINE_ENV_PREFIX):
+            object.__setattr__(self, "name", envvar)
+            object.__setattr__(self, "old_name", None)
+        elif envvar.startswith("DBT"):
+            object.__setattr__(self, "name", envvar.replace("DBT", f"{ENGINE_ENV_PREFIX}"))
+            object.__setattr__(self, "old_name", envvar)
+        else:
+            raise DbtInternalError(
+                f"Invalid environment variable: {envvar}, this will only happen if we add a new option to dbt that has an envvar that doesn't start with DBT_ or {ENGINE_ENV_PREFIX}"
+            )
+
+
+# Here we are creating a set of all known engine env vars. This is used in this moduleto create an allow list of dbt
+# engine env vars. We also use it in the cli flags module to cross propagate engine env vars with their old non-engine prefixed names.
+KNOWN_ENGINE_ENV_VARS: set[EngineEnvVar] = {
+    EngineEnvVar(envvar=envvar)
+    for envvar in [*params.KNOWN_ENV_VARS, *_ADDITIONAL_ENGINE_ENV_VARS]
+}
+_ALLOWED_ENV_VARS: set[str] = {envvar.name for envvar in KNOWN_ENGINE_ENV_VARS}
+
+
+def validate_engine_env_vars() -> None:
+    """
+    Validate that any set environment variables that begin with the engine prefix are allowed.
+    """
+    env_vars = get_invocation_context()._env
+    for env_var in env_vars.keys():
+        if env_var.startswith(ENGINE_ENV_PREFIX) and env_var not in _ALLOWED_ENV_VARS:
+            warn(
+                "environment-variable-namespace-deprecation",
+                env_var=env_var,
+                reserved_prefix=ENGINE_ENV_PREFIX,
+            )