dvt-core 1.11.0b4__py3-none-any.whl

dvt/task/deps.py

@@ -0,0 +1,280 @@
+import json
+from hashlib import sha1
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import dvt.deprecations
+import dvt.exceptions
+import dvt.utils
+import yaml
+from dvt.config import Project
+from dvt.config.project import load_yml_dict, package_config_from_data
+from dvt.config.renderer import PackageRenderer
+from dvt.constants import PACKAGE_LOCK_FILE_NAME, PACKAGE_LOCK_HASH_KEY
+from dvt.contracts.project import PackageSpec
+from dvt.deps.base import downloads_directory
+from dvt.deps.registry import RegistryPinnedPackage
+from dvt.deps.resolver import resolve_lock_packages, resolve_packages
+from dvt.events.types import (
+    DepsAddPackage,
+    DepsFoundDuplicatePackage,
+    DepsInstallInfo,
+    DepsListSubdirectory,
+    DepsLockUpdating,
+    DepsNoPackagesFound,
+    DepsNotifyUpdatesAvailable,
+    DepsStartPackageInstall,
+    DepsUpdateAvailable,
+    DepsUpToDate,
+)
+from dvt.task.base import BaseTask, move_to_nearest_project_dir
+
+from dbt_common.clients import system
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import Formatting
+
+
+class dbtPackageDumper(yaml.Dumper):
+    def increase_indent(self, flow=False, indentless=False):
+        return super(dbtPackageDumper, self).increase_indent(flow, False)
+
+
+def _create_sha1_hash(packages: List[PackageSpec]) -> str:
+    """Create a SHA1 hash of the packages list,
+    this is used to determine if the packages for current execution matches
+    the previous lock.
+
+    Args:
+        list[Packages]: list of packages specified that are already rendered
+
+    Returns:
+        str: SHA1 hash of the packages list
+    """
+    package_strs = [json.dumps(package.to_dict(), sort_keys=True) for package in packages]
+    package_strs = sorted(package_strs)
+
+    return sha1("\n".join(package_strs).encode("utf-8")).hexdigest()
+
+
+def _create_packages_yml_entry(package: str, version: Optional[str], source: str) -> dict:
+    """Create a formatted entry to add to `packages.yml` or `package-lock.yml` file
+
+    Args:
+        package (str): Name of package to download
+        version (str): Version of package to download
+        source (str): Source of where to download package from
+
+    Returns:
+        dict: Formatted dict to write to `packages.yml` or `package-lock.yml` file
+    """
+    package_key = source
+    version_key = "version"
+
+    if source == "hub":
+        package_key = "package"
+
+    packages_yml_entry = {package_key: package}
+
+    if source == "git":
+        version_key = "revision"
+
+    if version:
+        if "," in version:
+            version = version.split(",")  # type: ignore
+
+        packages_yml_entry[version_key] = version
+
+    return packages_yml_entry
+
+
+class DepsTask(BaseTask):
+    def __init__(self, args: Any, project: Project) -> None:
+        super().__init__(args=args)
+        # N.B. This is a temporary fix for a bug when using relative paths via
+        # --project-dir with deps.  A larger overhaul of our path handling methods
+        # is needed to fix this the "right" way.
+        # See GH-7615
+        project.project_root = str(Path(project.project_root).resolve())
+        self.project = project
+        self.cli_vars = args.vars
+
+    def track_package_install(
+        self, package_name: str, source_type: str, version: Optional[str]
+    ) -> None:
+        # Hub packages do not need to be hashed, as they are public
+        if source_type == "local":
+            package_name = dbt.utils.md5(package_name)
+            version = "local"
+        elif source_type == "tarball":
+            package_name = dbt.utils.md5(package_name)
+            version = "tarball"
+        elif source_type != "hub":
+            package_name = dbt.utils.md5(package_name)
+            version = dbt.utils.md5(version)
+
+        dbt.tracking.track_package_install(
+            "deps",
+            self.project.hashed_name(),
+            {"name": package_name, "source": source_type, "version": version},
+        )
+
+    def check_for_duplicate_packages(self, packages_yml):
+        """Loop through contents of `packages.yml` to ensure no duplicate package names + versions.
+
+        This duplicate check will take into consideration exact match of a package name, as well as
+        a check to see if a package name exists within a name (i.e. a package name inside a git URL).
+
+        Args:
+            packages_yml (dict): In-memory read of `packages.yml` contents
+
+        Returns:
+            dict: Updated or untouched packages_yml contents
+        """
+        for i, pkg_entry in enumerate(packages_yml["packages"]):
+            for val in pkg_entry.values():
+                if self.args.add_package["name"] in val:
+                    del packages_yml["packages"][i]
+
+                    fire_event(DepsFoundDuplicatePackage(removed_package=pkg_entry))
+
+        return packages_yml
+
+    def add(self):
+        packages_yml_filepath = (
+            f"{self.project.project_root}/{self.project.packages_specified_path}"
+        )
+        if not system.path_exists(packages_yml_filepath):
+            with open(packages_yml_filepath, "w") as package_yml:
+                yaml.safe_dump({"packages": []}, package_yml)
+            fire_event(Formatting("Created packages.yml"))
+
+        new_package_entry = _create_packages_yml_entry(
+            self.args.add_package["name"], self.args.add_package["version"], self.args.source
+        )
+
+        with open(packages_yml_filepath, "r") as user_yml_obj:
+            packages_yml = yaml.safe_load(user_yml_obj)
+            packages_yml = self.check_for_duplicate_packages(packages_yml)
+            packages_yml["packages"].append(new_package_entry)
+
+        self.project.packages.packages = package_config_from_data(packages_yml).packages
+
+        if packages_yml:
+            with open(packages_yml_filepath, "w") as pkg_obj:
+                pkg_obj.write(
+                    yaml.dump(packages_yml, Dumper=dbtPackageDumper, default_flow_style=False)
+                )
+
+                fire_event(
+                    DepsAddPackage(
+                        package_name=self.args.add_package["name"],
+                        version=self.args.add_package["version"],
+                        packages_filepath=packages_yml_filepath,
+                    )
+                )
+
+    def lock(self) -> None:
+        lock_filepath = f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}"
+
+        packages = self.project.packages.packages
+        packages_installed: Dict[str, Any] = {"packages": []}
+
+        if not packages:
+            fire_event(DepsNoPackagesFound())
+            return
+
+        with downloads_directory():
+            resolved_deps = resolve_packages(packages, self.project, self.cli_vars)
+
+        # this loop is to create the package-lock.yml in the same format as original packages.yml
+        # package-lock.yml includes both the stated packages in packages.yml along with dependent packages
+        renderer = PackageRenderer(self.cli_vars)
+        for package in resolved_deps:
+            package_dict = package.to_dict()
+            package_dict["name"] = package.get_project_name(self.project, renderer)
+            packages_installed["packages"].append(package_dict)
+
+        packages_installed[PACKAGE_LOCK_HASH_KEY] = _create_sha1_hash(
+            self.project.packages.packages
+        )
+
+        with open(lock_filepath, "w") as lock_obj:
+            yaml.dump(packages_installed, lock_obj, Dumper=dbtPackageDumper)
+
+        fire_event(DepsLockUpdating(lock_filepath=lock_filepath))
+
+    def run(self) -> None:
+        move_to_nearest_project_dir(self.args.project_dir)
+        if self.args.add_package:
+            self.add()
+
+        # Check lock file exist and generated by the same packages.yml
+        # or dependencies.yml.
+        lock_file_path = f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}"
+        if not system.path_exists(lock_file_path):
+            self.lock()
+        elif self.args.upgrade:
+            self.lock()
+        else:
+            # Check dependency definition is modified or not.
+            current_hash = _create_sha1_hash(self.project.packages.packages)
+            previous_hash = load_yml_dict(lock_file_path).get(PACKAGE_LOCK_HASH_KEY, None)
+            if previous_hash != current_hash:
+                self.lock()
+
+        # Early return when 'dbt deps --lock'
+        # Just resolve packages and write lock file, don't actually install packages
+        if self.args.lock:
+            return
+
+        if system.path_exists(self.project.packages_install_path):
+            system.rmtree(self.project.packages_install_path)
+
+        system.make_directory(self.project.packages_install_path)
+
+        packages_lock_dict = load_yml_dict(f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}")
+
+        renderer = PackageRenderer(self.cli_vars)
+        packages_lock_config = package_config_from_data(
+            renderer.render_data(packages_lock_dict), packages_lock_dict
+        ).packages
+
+        if not packages_lock_config:
+            fire_event(DepsNoPackagesFound())
+            return
+
+        with downloads_directory():
+            lock_defined_deps = resolve_lock_packages(packages_lock_config)
+            renderer = PackageRenderer(self.cli_vars)
+
+            packages_to_upgrade = []
+
+            for package in lock_defined_deps:
+                package_name = package.name
+                source_type = package.source_type()
+                version = package.get_version()
+
+                fire_event(DepsStartPackageInstall(package_name=package_name))
+                package.install(self.project, renderer)
+
+                fire_event(DepsInstallInfo(version_name=package.nice_version_name()))
+
+                if isinstance(package, RegistryPinnedPackage):
+                    version_latest = package.get_version_latest()
+
+                    if version_latest != version:
+                        packages_to_upgrade.append(package_name)
+                        fire_event(DepsUpdateAvailable(version_latest=version_latest))
+                    else:
+                        fire_event(DepsUpToDate())
+
+                if package.get_subdirectory():
+                    fire_event(DepsListSubdirectory(subdirectory=package.get_subdirectory()))
+
+                self.track_package_install(
+                    package_name=package_name, source_type=source_type, version=version
+                )
+
+            if packages_to_upgrade:
+                fire_event(Formatting(""))
+                fire_event(DepsNotifyUpdatesAvailable(packages=packages_to_upgrade))

dvt/task/docs/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+DOCS_INDEX_FILE_PATH = os.path.normpath(os.path.join(os.path.dirname(__file__), "index.html"))

dvt/task/docs/generate.py

@@ -0,0 +1,408 @@
+import os
+import shutil
+from dataclasses import replace
+from datetime import datetime, timezone
+from itertools import chain
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
+
+import agate
+import dvt.compilation
+import dvt.exceptions
+import dvt.utils
+from dvt.artifacts.schemas.catalog import (
+    CatalogArtifact,
+    CatalogKey,
+    CatalogResults,
+    CatalogTable,
+    ColumnMetadata,
+    PrimitiveDict,
+    StatsDict,
+    StatsItem,
+    TableMetadata,
+)
+from dvt.artifacts.schemas.results import NodeStatus
+from dvt.constants import CATALOG_FILENAME, MANIFEST_FILE_NAME
+from dvt.contracts.graph.manifest import Manifest
+from dvt.contracts.graph.nodes import ResultNode
+from dvt.events.types import ArtifactWritten
+from dvt.exceptions import AmbiguousCatalogMatchError
+from dvt.graph import ResourceTypeSelector
+from dvt.graph.graph import UniqueId
+from dvt.node_types import EXECUTABLE_NODE_TYPES, NodeType
+from dvt.parser.manifest import write_manifest
+from dvt.task.compile import CompileTask
+from dvt.task.docs import DOCS_INDEX_FILE_PATH
+from dvt.utils.artifact_upload import add_artifact_produced
+
+import dbt_common.utils.formatting
+from dbt.adapters.events.types import (
+    BuildingCatalog,
+    CannotGenerateDocs,
+    CatalogWritten,
+    WriteCatalogFailure,
+)
+from dbt.adapters.factory import get_adapter
+from dbt_common.clients.system import load_file_contents
+from dbt_common.dataclass_schema import ValidationError
+from dbt_common.events.functions import fire_event
+from dbt_common.exceptions import DbtInternalError
+
+
+def get_stripped_prefix(source: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+    """Go through the source, extracting every key/value pair where the key starts
+    with the given prefix.
+    """
+    cut = len(prefix)
+    return {k[cut:]: v for k, v in source.items() if k.startswith(prefix)}
+
+
+def build_catalog_table(data) -> CatalogTable:
+    # build the new table's metadata + stats
+    metadata = TableMetadata.from_dict(get_stripped_prefix(data, "table_"))
+    stats = format_stats(get_stripped_prefix(data, "stats:"))
+
+    return CatalogTable(
+        metadata=metadata,
+        stats=stats,
+        columns={},
+    )
+
+
+# keys are database name, schema name, table name
+class Catalog(Dict[CatalogKey, CatalogTable]):
+    def __init__(self, columns: List[PrimitiveDict]) -> None:
+        super().__init__()
+        for col in columns:
+            self.add_column(col)
+
+    def get_table(self, data: PrimitiveDict) -> CatalogTable:
+        database = data.get("table_database")
+        if database is None:
+            dkey: Optional[str] = None
+        else:
+            dkey = str(database)
+
+        try:
+            key = CatalogKey(
+                dkey,
+                str(data["table_schema"]),
+                str(data["table_name"]),
+            )
+        except KeyError as exc:
+            raise dbt_common.exceptions.CompilationError(
+                "Catalog information missing required key {} (got {})".format(exc, data)
+            )
+        table: CatalogTable
+        if key in self:
+            table = self[key]
+        else:
+            table = build_catalog_table(data)
+            self[key] = table
+        return table
+
+    def add_column(self, data: PrimitiveDict):
+        table = self.get_table(data)
+        column_data = get_stripped_prefix(data, "column_")
+        # the index should really never be that big so it's ok to end up
+        # serializing this to JSON (2^53 is the max safe value there)
+        column_data["index"] = int(column_data["index"])
+
+        column = ColumnMetadata.from_dict(column_data)
+        table.columns[column.name] = column
+
+    def make_unique_id_map(
+        self, manifest: Manifest, selected_node_ids: Optional[Set[UniqueId]] = None
+    ) -> Tuple[Dict[str, CatalogTable], Dict[str, CatalogTable]]:
+        """
+        Create mappings between CatalogKeys and CatalogTables for nodes and sources, filtered by selected_node_ids.
+
+        By default, selected_node_ids is None and all nodes and sources defined in the manifest are included in the mappings.
+        """
+        nodes: Dict[str, CatalogTable] = {}
+        sources: Dict[str, CatalogTable] = {}
+
+        node_map, source_map = get_unique_id_mapping(manifest)
+        table: CatalogTable
+        for table in self.values():
+            key = table.key()
+            if key in node_map:
+                unique_id = node_map[key]
+                if selected_node_ids is None or unique_id in selected_node_ids:
+                    nodes[unique_id] = replace(table, unique_id=unique_id)
+
+            unique_ids = source_map.get(table.key(), set())
+            for unique_id in unique_ids:
+                if unique_id in sources:
+                    raise AmbiguousCatalogMatchError(
+                        unique_id,
+                        sources[unique_id].to_dict(omit_none=True),
+                        table.to_dict(omit_none=True),
+                    )
+                elif selected_node_ids is None or unique_id in selected_node_ids:
+                    sources[unique_id] = replace(table, unique_id=unique_id)
+        return nodes, sources
+
+
+def format_stats(stats: PrimitiveDict) -> StatsDict:
+    """Given a dictionary following this layout:
+
+        {
+            'encoded:label': 'Encoded',
+            'encoded:value': 'Yes',
+            'encoded:description': 'Indicates if the column is encoded',
+            'encoded:include': True,
+
+            'size:label': 'Size',
+            'size:value': 128,
+            'size:description': 'Size of the table in MB',
+            'size:include': True,
+        }
+
+    format_stats will convert the dict into a StatsDict with keys of 'encoded'
+    and 'size'.
+    """
+    stats_collector: StatsDict = {}
+
+    base_keys = {k.split(":")[0] for k in stats}
+    for key in base_keys:
+        dct: PrimitiveDict = {"id": key}
+        for subkey in ("label", "value", "description", "include"):
+            dct[subkey] = stats["{}:{}".format(key, subkey)]
+
+        try:
+            stats_item = StatsItem.from_dict(dct)
+        except ValidationError:
+            continue
+        if stats_item.include:
+            stats_collector[key] = stats_item
+
+    # we always have a 'has_stats' field, it's never included
+    has_stats = StatsItem(
+        id="has_stats",
+        label="Has Stats?",
+        value=len(stats_collector) > 0,
+        description="Indicates whether there are statistics for this table",
+        include=False,
+    )
+    stats_collector["has_stats"] = has_stats
+    return stats_collector
+
+
+def mapping_key(node: ResultNode) -> CatalogKey:
+    dkey = dbt_common.utils.formatting.lowercase(node.database)
+    return CatalogKey(dkey, node.schema.lower(), node.identifier.lower())
+
+
+def get_unique_id_mapping(
+    manifest: Manifest,
+) -> Tuple[Dict[CatalogKey, str], Dict[CatalogKey, Set[str]]]:
+    # A single relation could have multiple unique IDs pointing to it if a
+    # source were also a node.
+    node_map: Dict[CatalogKey, str] = {}
+    source_map: Dict[CatalogKey, Set[str]] = {}
+    for unique_id, node in manifest.nodes.items():
+        key = mapping_key(node)
+        node_map[key] = unique_id
+
+    for unique_id, source in manifest.sources.items():
+        key = mapping_key(source)
+        if key not in source_map:
+            source_map[key] = set()
+        source_map[key].add(unique_id)
+    return node_map, source_map
+
+
+class GenerateTask(CompileTask):
+    def run(self) -> CatalogArtifact:
+        compile_results = None
+        if self.args.compile:
+            compile_results = CompileTask.run(self)
+            if any(r.status == NodeStatus.Error for r in compile_results):
+                fire_event(CannotGenerateDocs())
+                return CatalogArtifact.from_results(
+                    nodes={},
+                    sources={},
+                    generated_at=datetime.now(timezone.utc).replace(tzinfo=None),
+                    errors=None,
+                    compile_results=compile_results,
+                )
+
+        shutil.copyfile(
+            DOCS_INDEX_FILE_PATH, os.path.join(self.config.project_target_path, "index.html")
+        )
+
+        for asset_path in self.config.asset_paths:
+            to_asset_path = os.path.join(self.config.project_target_path, asset_path)
+
+            if os.path.exists(to_asset_path):
+                shutil.rmtree(to_asset_path)
+
+            from_asset_path = os.path.join(self.config.project_root, asset_path)
+
+            if os.path.exists(from_asset_path):
+                shutil.copytree(from_asset_path, to_asset_path)
+
+        if self.manifest is None:
+            raise DbtInternalError("self.manifest was None in run!")
+
+        selected_node_ids: Optional[Set[UniqueId]] = None
+        if self.args.empty_catalog:
+            catalog_table: agate.Table = agate.Table([])
+            exceptions: List[Exception] = []
+            selected_node_ids = set()
+        else:
+            adapter = get_adapter(self.config)
+            with adapter.connection_named("generate_catalog"):
+                fire_event(BuildingCatalog())
+                # Get a list of relations we need from the catalog
+                relations = None
+                if self.job_queue is not None:
+                    selected_node_ids = self.job_queue.get_selected_nodes()
+                    selected_nodes = self._get_nodes_from_ids(self.manifest, selected_node_ids)
+
+                    # Source selection is handled separately from main job_queue selection because
+                    # SourceDefinition nodes cannot be safely compiled / run by the CompileRunner / CompileTask,
+                    # but should still be included in the catalog based on the selection spec
+                    selected_source_ids = self._get_selected_source_ids()
+                    selected_source_nodes = self._get_nodes_from_ids(
+                        self.manifest, selected_source_ids
+                    )
+                    selected_node_ids.update(selected_source_ids)
+                    selected_nodes.extend(selected_source_nodes)
+
+                    relations = {
+                        adapter.Relation.create_from(adapter.config, node)
+                        for node in selected_nodes
+                    }
+
+                # This generates the catalog as an agate.Table
+                catalogable_nodes = chain(
+                    [
+                        node
+                        for node in self.manifest.nodes.values()
+                        if (node.is_relational and not node.is_ephemeral_model)
+                    ],
+                    self.manifest.sources.values(),
+                )
+                used_schemas = self.manifest.get_used_schemas()
+                catalog_table, exceptions = adapter.get_filtered_catalog(
+                    catalogable_nodes, used_schemas, relations
+                )
+
+        catalog_data: List[PrimitiveDict] = [
+            dict(zip(catalog_table.column_names, map(dbt.utils._coerce_decimal, row)))
+            for row in catalog_table
+        ]
+
+        catalog = Catalog(catalog_data)
+
+        errors: Optional[List[str]] = None
+        if exceptions:
+            errors = [str(e) for e in exceptions]
+
+        nodes, sources = catalog.make_unique_id_map(self.manifest, selected_node_ids)
+        results = self.get_catalog_results(
+            nodes=nodes,
+            sources=sources,
+            generated_at=datetime.now(timezone.utc).replace(tzinfo=None),
+            compile_results=compile_results,
+            errors=errors,
+        )
+
+        catalog_path = os.path.join(self.config.project_target_path, CATALOG_FILENAME)
+        results.write(catalog_path)
+        add_artifact_produced(catalog_path)
+        fire_event(
+            ArtifactWritten(artifact_type=results.__class__.__name__, artifact_path=catalog_path)
+        )
+
+        if self.args.compile:
+            write_manifest(self.manifest, self.config.project_target_path)
+
+        if self.args.static:
+
+            # Read manifest.json and catalog.json
+            read_manifest_data = load_file_contents(
+                os.path.join(self.config.project_target_path, MANIFEST_FILE_NAME)
+            )
+            read_catalog_data = load_file_contents(catalog_path)
+
+            # Create new static index file contents
+            index_data = load_file_contents(DOCS_INDEX_FILE_PATH)
+            index_data = index_data.replace('"MANIFEST.JSON INLINE DATA"', read_manifest_data)
+            index_data = index_data.replace('"CATALOG.JSON INLINE DATA"', read_catalog_data)
+
+            # Write out the new index file
+            static_index_path = os.path.join(self.config.project_target_path, "static_index.html")
+            with open(static_index_path, "wb") as static_index_file:
+                static_index_file.write(bytes(index_data, "utf8"))
+
+        if exceptions:
+            fire_event(WriteCatalogFailure(num_exceptions=len(exceptions)))
+        fire_event(CatalogWritten(path=os.path.abspath(catalog_path)))
+        return results
+
+    def get_node_selector(self) -> ResourceTypeSelector:
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to perform node selection")
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=EXECUTABLE_NODE_TYPES,
+            include_empty_nodes=True,
+        )
+
+    def get_catalog_results(
+        self,
+        nodes: Dict[str, CatalogTable],
+        sources: Dict[str, CatalogTable],
+        generated_at: datetime,
+        compile_results: Optional[Any],
+        errors: Optional[List[str]],
+    ) -> CatalogArtifact:
+        return CatalogArtifact.from_results(
+            generated_at=generated_at,
+            nodes=nodes,
+            sources=sources,
+            compile_results=compile_results,
+            errors=errors,
+        )
+
+    @classmethod
+    def interpret_results(self, results: Optional[CatalogResults]) -> bool:
+        if results is None:
+            return False
+        if results.errors:
+            return False
+        compile_results = results._compile_results
+        if compile_results is None:
+            return True
+
+        return super().interpret_results(compile_results)
+
+    @staticmethod
+    def _get_nodes_from_ids(manifest: Manifest, node_ids: Iterable[str]) -> List[ResultNode]:
+        selected: List[ResultNode] = []
+        for unique_id in node_ids:
+            if unique_id in manifest.nodes:
+                node = manifest.nodes[unique_id]
+                if node.is_relational and not node.is_ephemeral_model:
+                    selected.append(node)
+            elif unique_id in manifest.sources:
+                source = manifest.sources[unique_id]
+                selected.append(source)
+        return selected
+
+    def _get_selected_source_ids(self) -> Set[UniqueId]:
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to perform node selection")
+
+        source_selector = ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=[NodeType.Source],
+        )
+
+        return source_selector.get_graph_queue(self.get_selection_spec()).get_selected_nodes()