dvt-core 0.59.0a51__py3-none-any.whl

dbt/task/deps.py

@@ -0,0 +1,280 @@
+import json
+from hashlib import sha1
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import yaml
+
+import dbt.deprecations
+import dbt.exceptions
+import dbt.utils
+from dbt.config import Project
+from dbt.config.project import load_yml_dict, package_config_from_data
+from dbt.config.renderer import PackageRenderer
+from dbt.constants import PACKAGE_LOCK_FILE_NAME, PACKAGE_LOCK_HASH_KEY
+from dbt.contracts.project import PackageSpec
+from dbt.deps.base import downloads_directory
+from dbt.deps.registry import RegistryPinnedPackage
+from dbt.deps.resolver import resolve_lock_packages, resolve_packages
+from dbt.events.types import (
+    DepsAddPackage,
+    DepsFoundDuplicatePackage,
+    DepsInstallInfo,
+    DepsListSubdirectory,
+    DepsLockUpdating,
+    DepsNoPackagesFound,
+    DepsNotifyUpdatesAvailable,
+    DepsStartPackageInstall,
+    DepsUpdateAvailable,
+    DepsUpToDate,
+)
+from dbt.task.base import BaseTask, move_to_nearest_project_dir
+from dbt_common.clients import system
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import Formatting
+
+
+class dbtPackageDumper(yaml.Dumper):
+    def increase_indent(self, flow=False, indentless=False):
+        return super(dbtPackageDumper, self).increase_indent(flow, False)
+
+
+def _create_sha1_hash(packages: List[PackageSpec]) -> str:
+    """Create a SHA1 hash of the packages list,
+    this is used to determine if the packages for current execution matches
+    the previous lock.
+
+    Args:
+        list[Packages]: list of packages specified that are already rendered
+
+    Returns:
+        str: SHA1 hash of the packages list
+    """
+    package_strs = [json.dumps(package.to_dict(), sort_keys=True) for package in packages]
+    package_strs = sorted(package_strs)
+
+    return sha1("\n".join(package_strs).encode("utf-8")).hexdigest()
+
+
+def _create_packages_yml_entry(package: str, version: Optional[str], source: str) -> dict:
+    """Create a formatted entry to add to `packages.yml` or `package-lock.yml` file
+
+    Args:
+        package (str): Name of package to download
+        version (str): Version of package to download
+        source (str): Source of where to download package from
+
+    Returns:
+        dict: Formatted dict to write to `packages.yml` or `package-lock.yml` file
+    """
+    package_key = source
+    version_key = "version"
+
+    if source == "hub":
+        package_key = "package"
+
+    packages_yml_entry = {package_key: package}
+
+    if source == "git":
+        version_key = "revision"
+
+    if version:
+        if "," in version:
+            version = version.split(",")  # type: ignore
+
+        packages_yml_entry[version_key] = version
+
+    return packages_yml_entry
+
+
+class DepsTask(BaseTask):
+    def __init__(self, args: Any, project: Project) -> None:
+        super().__init__(args=args)
+        # N.B. This is a temporary fix for a bug when using relative paths via
+        # --project-dir with deps.  A larger overhaul of our path handling methods
+        # is needed to fix this the "right" way.
+        # See GH-7615
+        project.project_root = str(Path(project.project_root).resolve())
+        self.project = project
+        self.cli_vars = args.vars
+
+    def track_package_install(
+        self, package_name: str, source_type: str, version: Optional[str]
+    ) -> None:
+        # Hub packages do not need to be hashed, as they are public
+        if source_type == "local":
+            package_name = dbt.utils.md5(package_name)
+            version = "local"
+        elif source_type == "tarball":
+            package_name = dbt.utils.md5(package_name)
+            version = "tarball"
+        elif source_type != "hub":
+            package_name = dbt.utils.md5(package_name)
+            version = dbt.utils.md5(version)
+
+        dbt.tracking.track_package_install(
+            "deps",
+            self.project.hashed_name(),
+            {"name": package_name, "source": source_type, "version": version},
+        )
+
+    def check_for_duplicate_packages(self, packages_yml):
+        """Loop through contents of `packages.yml` to ensure no duplicate package names + versions.
+
+        This duplicate check will take into consideration exact match of a package name, as well as
+        a check to see if a package name exists within a name (i.e. a package name inside a git URL).
+
+        Args:
+            packages_yml (dict): In-memory read of `packages.yml` contents
+
+        Returns:
+            dict: Updated or untouched packages_yml contents
+        """
+        for i, pkg_entry in enumerate(packages_yml["packages"]):
+            for val in pkg_entry.values():
+                if self.args.add_package["name"] in val:
+                    del packages_yml["packages"][i]
+
+                    fire_event(DepsFoundDuplicatePackage(removed_package=pkg_entry))
+
+        return packages_yml
+
+    def add(self):
+        packages_yml_filepath = (
+            f"{self.project.project_root}/{self.project.packages_specified_path}"
+        )
+        if not system.path_exists(packages_yml_filepath):
+            with open(packages_yml_filepath, "w") as package_yml:
+                yaml.safe_dump({"packages": []}, package_yml)
+            fire_event(Formatting("Created packages.yml"))
+
+        new_package_entry = _create_packages_yml_entry(
+            self.args.add_package["name"], self.args.add_package["version"], self.args.source
+        )
+
+        with open(packages_yml_filepath, "r") as user_yml_obj:
+            packages_yml = yaml.safe_load(user_yml_obj)
+            packages_yml = self.check_for_duplicate_packages(packages_yml)
+            packages_yml["packages"].append(new_package_entry)
+
+        self.project.packages.packages = package_config_from_data(packages_yml).packages
+
+        if packages_yml:
+            with open(packages_yml_filepath, "w") as pkg_obj:
+                pkg_obj.write(
+                    yaml.dump(packages_yml, Dumper=dbtPackageDumper, default_flow_style=False)
+                )
+
+                fire_event(
+                    DepsAddPackage(
+                        package_name=self.args.add_package["name"],
+                        version=self.args.add_package["version"],
+                        packages_filepath=packages_yml_filepath,
+                    )
+                )
+
+    def lock(self) -> None:
+        lock_filepath = f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}"
+
+        packages = self.project.packages.packages
+        packages_installed: Dict[str, Any] = {"packages": []}
+
+        if not packages:
+            fire_event(DepsNoPackagesFound())
+            return
+
+        with downloads_directory():
+            resolved_deps = resolve_packages(packages, self.project, self.cli_vars)
+
+        # this loop is to create the package-lock.yml in the same format as original packages.yml
+        # package-lock.yml includes both the stated packages in packages.yml along with dependent packages
+        renderer = PackageRenderer(self.cli_vars)
+        for package in resolved_deps:
+            package_dict = package.to_dict()
+            package_dict["name"] = package.get_project_name(self.project, renderer)
+            packages_installed["packages"].append(package_dict)
+
+        packages_installed[PACKAGE_LOCK_HASH_KEY] = _create_sha1_hash(
+            self.project.packages.packages
+        )
+
+        with open(lock_filepath, "w") as lock_obj:
+            yaml.dump(packages_installed, lock_obj, Dumper=dbtPackageDumper)
+
+        fire_event(DepsLockUpdating(lock_filepath=lock_filepath))
+
+    def run(self) -> None:
+        move_to_nearest_project_dir(self.args.project_dir)
+        if self.args.add_package:
+            self.add()
+
+        # Check lock file exist and generated by the same packages.yml
+        # or dependencies.yml.
+        lock_file_path = f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}"
+        if not system.path_exists(lock_file_path):
+            self.lock()
+        elif self.args.upgrade:
+            self.lock()
+        else:
+            # Check dependency definition is modified or not.
+            current_hash = _create_sha1_hash(self.project.packages.packages)
+            previous_hash = load_yml_dict(lock_file_path).get(PACKAGE_LOCK_HASH_KEY, None)
+            if previous_hash != current_hash:
+                self.lock()
+
+        # Early return when 'dbt deps --lock'
+        # Just resolve packages and write lock file, don't actually install packages
+        if self.args.lock:
+            return
+
+        if system.path_exists(self.project.packages_install_path):
+            system.rmtree(self.project.packages_install_path)
+
+        system.make_directory(self.project.packages_install_path)
+
+        packages_lock_dict = load_yml_dict(f"{self.project.project_root}/{PACKAGE_LOCK_FILE_NAME}")
+
+        renderer = PackageRenderer(self.cli_vars)
+        packages_lock_config = package_config_from_data(
+            renderer.render_data(packages_lock_dict), packages_lock_dict
+        ).packages
+
+        if not packages_lock_config:
+            fire_event(DepsNoPackagesFound())
+            return
+
+        with downloads_directory():
+            lock_defined_deps = resolve_lock_packages(packages_lock_config)
+            renderer = PackageRenderer(self.cli_vars)
+
+            packages_to_upgrade = []
+
+            for package in lock_defined_deps:
+                package_name = package.name
+                source_type = package.source_type()
+                version = package.get_version()
+
+                fire_event(DepsStartPackageInstall(package_name=package_name))
+                package.install(self.project, renderer)
+
+                fire_event(DepsInstallInfo(version_name=package.nice_version_name()))
+
+                if isinstance(package, RegistryPinnedPackage):
+                    version_latest = package.get_version_latest()
+
+                    if version_latest != version:
+                        packages_to_upgrade.append(package_name)
+                        fire_event(DepsUpdateAvailable(version_latest=version_latest))
+                    else:
+                        fire_event(DepsUpToDate())
+
+                if package.get_subdirectory():
+                    fire_event(DepsListSubdirectory(subdirectory=package.get_subdirectory()))
+
+                self.track_package_install(
+                    package_name=package_name, source_type=source_type, version=version
+                )
+
+            if packages_to_upgrade:
+                fire_event(Formatting(""))
+                fire_event(DepsNotifyUpdatesAvailable(packages=packages_to_upgrade))

dbt/task/docs/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+DOCS_INDEX_FILE_PATH = os.path.normpath(os.path.join(os.path.dirname(__file__), "index.html"))

dbt/task/docs/api/__init__.py

@@ -0,0 +1,23 @@
+"""
+DVT Docs Serve API
+
+v0.56.0: FastAPI routers for enhanced docs serve with 3-tab web UI.
+
+Endpoints:
+- /api/catalog/* - Catalog nodes and search
+- /api/profile/* - Profile results and alerts
+- /api/lineage/* - Lineage graph and traversal
+- /api/spark/* - Spark status (local only)
+"""
+
+from dbt.task.docs.api.catalog import router as catalog_router
+from dbt.task.docs.api.profile import router as profile_router
+from dbt.task.docs.api.lineage import router as lineage_router
+from dbt.task.docs.api.spark import router as spark_router
+
+__all__ = [
+    "catalog_router",
+    "profile_router",
+    "lineage_router",
+    "spark_router",
+]

dbt/task/docs/api/catalog.py

@@ -0,0 +1,204 @@
+"""
+DVT Docs Serve - Catalog API
+
+v0.56.0: Catalog nodes, search, and node details.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+try:
+    from fastapi import APIRouter, HTTPException, Query
+except ImportError:
+    # Provide stub for when FastAPI is not installed
+    class APIRouter:
+        def __init__(self, *args, **kwargs):
+            pass
+        def get(self, *args, **kwargs):
+            def decorator(func):
+                return func
+            return decorator
+    class HTTPException(Exception):
+        def __init__(self, status_code, detail):
+            self.status_code = status_code
+            self.detail = detail
+    def Query(*args, **kwargs):
+        return None
+
+
+router = APIRouter(prefix="/api/catalog", tags=["catalog"])
+
+# Will be set by serve.py
+_project_root: Optional[Path] = None
+_store_initialized: bool = False
+
+
+def set_project_root(project_root: Path) -> None:
+    """Set project root for API access."""
+    global _project_root, _store_initialized
+    _project_root = project_root
+    _store_initialized = True  # Assume serve.py already initialized
+
+
+def _get_store():
+    """Get metadata store instance (lazy initialization)."""
+    global _store_initialized
+
+    if _project_root is None:
+        raise HTTPException(status_code=500, detail="Project root not configured")
+
+    try:
+        from dbt.compute.metadata import ProjectMetadataStore
+        store = ProjectMetadataStore(_project_root)
+        # Only initialize if not already done by serve.py
+        if not _store_initialized:
+            store.initialize()
+            _store_initialized = True
+        return store
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Could not open metadata store: {e}")
+
+
+@router.get("/nodes")
+async def list_catalog_nodes(
+    resource_type: Optional[str] = Query(None, description="Filter by type: model, source, test, seed, snapshot"),
+) -> List[Dict[str, Any]]:
+    """
+    List all catalog nodes.
+
+    Returns all nodes from the catalog, optionally filtered by resource type.
+    """
+    store = _get_store()
+
+    try:
+        if resource_type:
+            nodes = store.get_catalog_nodes_by_type(resource_type)
+        else:
+            nodes = store.get_all_catalog_nodes()
+
+        return [
+            {
+                "unique_id": n.unique_id,
+                "resource_type": n.resource_type,
+                "name": n.name,
+                "schema": n.schema_name,
+                "database": n.database,
+                "connection": n.connection_name,
+                "adapter": n.adapter_type,
+                "description": n.description,
+                "icon": n.icon_type,
+                "color": n.color_hex,
+                "materialized": n.materialized,
+                "row_count": n.row_count,
+            }
+            for n in nodes
+        ]
+    finally:
+        store.close()
+
+
+@router.get("/nodes/{unique_id}")
+async def get_catalog_node(unique_id: str) -> Dict[str, Any]:
+    """
+    Get a specific catalog node by unique ID.
+
+    Returns full node details including columns.
+    """
+    import json
+
+    store = _get_store()
+
+    try:
+        node = store.get_catalog_node(unique_id)
+        if not node:
+            raise HTTPException(status_code=404, detail=f"Node not found: {unique_id}")
+
+        return {
+            "unique_id": node.unique_id,
+            "resource_type": node.resource_type,
+            "name": node.name,
+            "schema": node.schema_name,
+            "database": node.database,
+            "connection": node.connection_name,
+            "adapter": node.adapter_type,
+            "description": node.description,
+            "icon": node.icon_type,
+            "color": node.color_hex,
+            "materialized": node.materialized,
+            "tags": json.loads(node.tags) if node.tags else [],
+            "meta": json.loads(node.meta) if node.meta else {},
+            "columns": json.loads(node.columns) if node.columns else [],
+            "row_count": node.row_count,
+            "bytes_stored": node.bytes_stored,
+            "created_at": node.created_at.isoformat() if node.created_at else None,
+            "updated_at": node.updated_at.isoformat() if node.updated_at else None,
+        }
+    finally:
+        store.close()
+
+
+@router.get("/search")
+async def search_catalog(
+    q: str = Query(..., description="Search query"),
+) -> List[Dict[str, Any]]:
+    """
+    Search catalog nodes by name or description.
+
+    Returns matching nodes sorted by relevance.
+    """
+    if not q or len(q) < 2:
+        raise HTTPException(status_code=400, detail="Query must be at least 2 characters")
+
+    store = _get_store()
+
+    try:
+        nodes = store.search_catalog_nodes(q)
+        return [
+            {
+                "unique_id": n.unique_id,
+                "resource_type": n.resource_type,
+                "name": n.name,
+                "schema": n.schema_name,
+                "connection": n.connection_name,
+                "description": n.description,
+                "icon": n.icon_type,
+                "color": n.color_hex,
+            }
+            for n in nodes
+        ]
+    finally:
+        store.close()
+
+
+@router.get("/stats")
+async def get_catalog_stats() -> Dict[str, Any]:
+    """
+    Get catalog statistics.
+
+    Returns counts of nodes by type, connections, etc.
+    """
+    store = _get_store()
+
+    try:
+        all_nodes = store.get_all_catalog_nodes()
+
+        # Count by type
+        type_counts: Dict[str, int] = {}
+        connection_counts: Dict[str, int] = {}
+        adapter_counts: Dict[str, int] = {}
+
+        for node in all_nodes:
+            type_counts[node.resource_type] = type_counts.get(node.resource_type, 0) + 1
+            if node.connection_name:
+                connection_counts[node.connection_name] = connection_counts.get(node.connection_name, 0) + 1
+            if node.adapter_type:
+                adapter_counts[node.adapter_type] = adapter_counts.get(node.adapter_type, 0) + 1
+
+        return {
+            "total_nodes": len(all_nodes),
+            "by_type": type_counts,
+            "by_connection": connection_counts,
+            "by_adapter": adapter_counts,
+        }
+    finally:
+        store.close()