dvt-core 0.52.2__cp310-cp310-macosx_10_9_x86_64.whl

dbt/task/java.py

@@ -0,0 +1,316 @@
+"""
+Java Task Module
+
+Handles DVT java management commands:
+- check: Check Java and show compatibility with installed PySpark
+- search: Find ALL Java installations on the system
+- set: Interactive selection to set JAVA_HOME
+- install: Guide for installing compatible Java
+
+v0.51.3: New module for comprehensive Java management.
+"""
+
+import os
+import platform
+from typing import List, Optional
+
+import click
+
+from dbt.compute.java_compat import (
+    JavaInstallation,
+    find_all_java_installations,
+    get_current_java,
+    get_pyspark_info,
+    get_pyspark_versions_for_java,
+    check_java_pyspark_compatibility,
+    set_java_home_persistent,
+    PYSPARK_JAVA_COMPATIBILITY,
+)
+
+
+class JavaTask:
+    """Task for managing Java installations."""
+
+    def check(self) -> bool:
+        """
+        Check current Java installation and PySpark compatibility.
+
+        Returns:
+            bool: True if Java is compatible with installed PySpark
+        """
+        click.echo()
+        click.echo(click.style("Java Status", fg="cyan", bold=True))
+        click.echo("-" * 40)
+
+        # Get current Java
+        java = get_current_java()
+        if java:
+            click.echo(f"  JAVA_HOME: {java.path}")
+            click.echo(f"  Version:   Java {java.version}")
+            click.echo(f"  Vendor:    {java.vendor}")
+            click.echo(f"  Details:   {java.version_string}")
+        else:
+            click.echo(click.style("  ✗ Java not found!", fg="red"))
+            click.echo()
+            click.echo("  Run 'dvt java search' to find Java installations")
+            click.echo("  Run 'dvt java install' for installation guide")
+            click.echo()
+            return False
+
+        click.echo()
+        click.echo(click.style("PySpark Status", fg="cyan", bold=True))
+        click.echo("-" * 40)
+
+        # Get PySpark info
+        pyspark = get_pyspark_info()
+        if pyspark:
+            click.echo(f"  Version:       {pyspark.version}")
+            click.echo(f"  Required Java: {', '.join(str(v) for v in pyspark.java_supported)}")
+            click.echo(f"  Recommended:   Java {pyspark.java_recommended}")
+        else:
+            click.echo(click.style("  ✗ PySpark not installed!", fg="red"))
+            click.echo()
+            click.echo("  Install with: pip install pyspark")
+            click.echo()
+            return False
+
+        click.echo()
+        click.echo(click.style("Compatibility", fg="cyan", bold=True))
+        click.echo("-" * 40)
+
+        # Check compatibility
+        is_compat, msg = check_java_pyspark_compatibility(java.version, pyspark.major_minor)
+        if is_compat:
+            click.echo(click.style(f"  ✓ {msg}", fg="green"))
+        else:
+            click.echo(click.style(f"  ✗ {msg}", fg="red"))
+            click.echo()
+            click.echo("  Run 'dvt java set' to select a compatible Java version")
+
+        click.echo()
+        return is_compat
+
+    def search(self) -> List[JavaInstallation]:
+        """
+        Find all Java installations on the system.
+
+        Returns:
+            List of JavaInstallation objects
+        """
+        click.echo()
+        click.echo(click.style("Searching for Java installations...", fg="cyan"))
+        click.echo()
+
+        installations = find_all_java_installations()
+
+        if not installations:
+            click.echo(click.style("No Java installations found.", fg="yellow"))
+            click.echo()
+            click.echo("Run 'dvt java install' for installation guide")
+            click.echo()
+            return []
+
+        click.echo(f"Found {len(installations)} Java installation(s):")
+        click.echo()
+
+        # Get PySpark info for compatibility display
+        pyspark = get_pyspark_info()
+
+        for i, inst in enumerate(installations, 1):
+            # Mark current
+            current_marker = click.style(" * CURRENT", fg="green") if inst.is_current else ""
+
+            # Check compatibility with installed PySpark
+            if pyspark:
+                is_compat, _ = check_java_pyspark_compatibility(inst.version, pyspark.major_minor)
+                compat_marker = click.style(" ✓", fg="green") if is_compat else click.style(" ✗", fg="red")
+            else:
+                compat_marker = ""
+
+            click.echo(f"  [{i}] Java {inst.version} ({inst.vendor}){current_marker}{compat_marker}")
+            click.echo(f"      {inst.path}")
+
+            # Show which PySpark versions this Java supports
+            compatible_pyspark = get_pyspark_versions_for_java(inst.version)
+            if compatible_pyspark:
+                click.echo(f"      Compatible with: PySpark {', '.join(compatible_pyspark)}")
+            click.echo()
+
+        return installations
+
+    def set_java_home(self, installation: Optional[JavaInstallation] = None) -> bool:
+        """
+        Interactively select and set JAVA_HOME.
+
+        If no installation provided, presents interactive menu.
+
+        Args:
+            installation: Pre-selected JavaInstallation (optional)
+
+        Returns:
+            bool: True if successful
+        """
+        if installation:
+            # Direct set
+            success, msg = set_java_home_persistent(installation.path)
+            if success:
+                click.echo(click.style(f"✓ JAVA_HOME set to: {installation.path}", fg="green"))
+                click.echo(f"  {msg}")
+            else:
+                click.echo(click.style(f"✗ {msg}", fg="red"))
+            return success
+
+        # Interactive selection
+        installations = find_all_java_installations()
+        if not installations:
+            click.echo(click.style("No Java installations found.", fg="yellow"))
+            click.echo("Run 'dvt java install' for installation guide")
+            return False
+
+        # Get PySpark info for compatibility display
+        pyspark = get_pyspark_info()
+
+        click.echo()
+        click.echo(click.style("Select Java installation:", fg="cyan", bold=True))
+        click.echo()
+
+        for i, inst in enumerate(installations, 1):
+            # Mark current
+            current_marker = click.style(" (current)", fg="blue") if inst.is_current else ""
+
+            # Check compatibility with installed PySpark
+            if pyspark:
+                is_compat, _ = check_java_pyspark_compatibility(inst.version, pyspark.major_minor)
+                if is_compat:
+                    compat_marker = click.style(" ✓ compatible", fg="green")
+                else:
+                    compat_marker = click.style(" ✗ incompatible", fg="red")
+            else:
+                compat_marker = ""
+
+            click.echo(f"  [{i}] Java {inst.version} ({inst.vendor}){current_marker}{compat_marker}")
+            click.echo(f"      {inst.path}")
+            click.echo()
+
+        # Get user choice
+        while True:
+            try:
+                choice = click.prompt("Your choice", type=int)
+                if 1 <= choice <= len(installations):
+                    break
+                click.echo(click.style(f"Please enter a number between 1 and {len(installations)}", fg="yellow"))
+            except click.Abort:
+                click.echo("\nAborted.")
+                return False
+
+        selected = installations[choice - 1]
+
+        # Warn if incompatible with PySpark
+        if pyspark:
+            is_compat, msg = check_java_pyspark_compatibility(selected.version, pyspark.major_minor)
+            if not is_compat:
+                click.echo()
+                click.echo(click.style(f"⚠️  Warning: {msg}", fg="yellow"))
+                if not click.confirm("Continue anyway?"):
+                    return False
+
+        # Set JAVA_HOME
+        success, msg = set_java_home_persistent(selected.path)
+        click.echo()
+        if success:
+            click.echo(click.style(f"✓ JAVA_HOME set to: {selected.path}", fg="green"))
+            click.echo(f"  {msg}")
+        else:
+            click.echo(click.style(f"✗ {msg}", fg="red"))
+
+        return success
+
+    def install_guide(self) -> None:
+        """
+        Show installation guide for compatible Java version.
+
+        Displays platform-specific installation instructions based on
+        the installed PySpark version.
+        """
+        click.echo()
+
+        # Get PySpark info
+        pyspark = get_pyspark_info()
+        if pyspark:
+            click.echo(click.style(f"Java Installation Guide for PySpark {pyspark.version}", fg="cyan", bold=True))
+            click.echo("=" * 60)
+            click.echo()
+            click.echo(f"PySpark {pyspark.major_minor} requires Java: {', '.join(str(v) for v in pyspark.java_supported)}")
+            click.echo(f"Recommended: Java {pyspark.java_recommended}")
+            recommended = pyspark.java_recommended
+        else:
+            click.echo(click.style("Java Installation Guide", fg="cyan", bold=True))
+            click.echo("=" * 60)
+            click.echo()
+            click.echo("PySpark is not installed. Assuming Java 17+ for latest PySpark.")
+            recommended = 17
+
+        click.echo()
+        os_type = platform.system()
+
+        if os_type == "Darwin":  # macOS
+            click.echo(click.style("📦 macOS Installation Options:", fg="yellow", bold=True))
+            click.echo()
+            click.echo("  Option 1: Homebrew (recommended)")
+            click.echo(click.style(f"    brew install openjdk@{recommended}", fg="green"))
+            click.echo()
+            click.echo("  Option 2: SDKMAN (multiple versions)")
+            click.echo(click.style("    curl -s \"https://get.sdkman.io\" | bash", fg="green"))
+            click.echo(click.style(f"    sdk install java {recommended}.0.2-tem", fg="green"))
+            click.echo()
+            click.echo("  Option 3: Download manually")
+            click.echo(click.style("    https://adoptium.net/", fg="blue"))
+            click.echo()
+            click.echo("  After installation:")
+            click.echo(f"    export JAVA_HOME=$(/usr/libexec/java_home -v {recommended})")
+
+        elif os_type == "Linux":
+            click.echo(click.style("📦 Linux Installation Options:", fg="yellow", bold=True))
+            click.echo()
+            click.echo("  Ubuntu/Debian:")
+            click.echo(click.style("    sudo apt-get update", fg="green"))
+            click.echo(click.style(f"    sudo apt-get install openjdk-{recommended}-jdk", fg="green"))
+            click.echo()
+            click.echo("  RHEL/CentOS/Fedora:")
+            click.echo(click.style(f"    sudo dnf install java-{recommended}-openjdk-devel", fg="green"))
+            click.echo()
+            click.echo("  Arch Linux:")
+            click.echo(click.style(f"    sudo pacman -S jdk{recommended}-openjdk", fg="green"))
+            click.echo()
+            click.echo("  SDKMAN (any distro):")
+            click.echo(click.style("    curl -s \"https://get.sdkman.io\" | bash", fg="green"))
+            click.echo(click.style(f"    sdk install java {recommended}.0.2-tem", fg="green"))
+            click.echo()
+            click.echo("  After installation:")
+            click.echo(f"    export JAVA_HOME=/usr/lib/jvm/java-{recommended}-openjdk")
+
+        elif os_type == "Windows":
+            click.echo(click.style("📦 Windows Installation Options:", fg="yellow", bold=True))
+            click.echo()
+            click.echo("  Option 1: Winget (Windows 11/10)")
+            click.echo(click.style(f"    winget install EclipseAdoptium.Temurin.{recommended}.JDK", fg="green"))
+            click.echo()
+            click.echo("  Option 2: Chocolatey")
+            click.echo(click.style(f"    choco install temurin{recommended}", fg="green"))
+            click.echo()
+            click.echo("  Option 3: Scoop")
+            click.echo(click.style("    scoop bucket add java", fg="green"))
+            click.echo(click.style(f"    scoop install temurin{recommended}-jdk", fg="green"))
+            click.echo()
+            click.echo("  Option 4: Download manually")
+            click.echo(click.style("    https://adoptium.net/", fg="blue"))
+            click.echo()
+            click.echo("  After installation:")
+            click.echo("    Set JAVA_HOME in System Environment Variables")
+
+        click.echo()
+        click.echo(click.style("After installing Java:", fg="cyan"))
+        click.echo("  1. Restart your terminal")
+        click.echo("  2. Run 'dvt java search' to verify")
+        click.echo("  3. Run 'dvt java set' to select the installation")
+        click.echo()

dbt/task/list.py

@@ -0,0 +1,236 @@
+import json
+from typing import Iterator, List
+
+from dbt.cli.flags import Flags
+from dbt.config.runtime import RuntimeConfig
+from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.graph.nodes import (
+    Exposure,
+    Metric,
+    SavedQuery,
+    SemanticModel,
+    SourceDefinition,
+    UnitTestDefinition,
+)
+from dbt.events.types import NoNodesSelected
+from dbt.graph import ResourceTypeSelector
+from dbt.node_types import NodeType
+from dbt.task.base import resource_types_from_args
+from dbt.task.runnable import GraphRunnableTask
+from dbt.utils import JSONEncoder
+from dbt_common.events.contextvars import task_contextvars
+from dbt_common.events.functions import fire_event, warn_or_error
+from dbt_common.events.types import PrintEvent
+from dbt_common.exceptions import DbtInternalError, DbtRuntimeError
+
+
+class ListTask(GraphRunnableTask):
+    DEFAULT_RESOURCE_VALUES = frozenset(
+        (
+            NodeType.Model,
+            NodeType.Snapshot,
+            NodeType.Seed,
+            NodeType.Test,
+            NodeType.Source,
+            NodeType.Exposure,
+            NodeType.Metric,
+            NodeType.SavedQuery,
+            NodeType.SemanticModel,
+            NodeType.Unit,
+            NodeType.Function,
+        )
+    )
+    ALL_RESOURCE_VALUES = DEFAULT_RESOURCE_VALUES | frozenset((NodeType.Analysis,))
+    ALLOWED_KEYS = frozenset(
+        (
+            "alias",
+            "name",
+            "package_name",
+            "depends_on",
+            "tags",
+            "config",
+            "resource_type",
+            "source_name",
+            "original_file_path",
+            "unique_id",
+        )
+    )
+
+    def __init__(self, args: Flags, config: RuntimeConfig, manifest: Manifest) -> None:
+        super().__init__(args, config, manifest)
+        if self.args.models:
+            if self.args.select:
+                raise DbtRuntimeError('"models" and "select" are mutually exclusive arguments')
+            if self.args.resource_types:
+                raise DbtRuntimeError(
+                    '"models" and "resource_type" are mutually exclusive ' "arguments"
+                )
+
+    def _iterate_selected_nodes(self):
+        selector = self.get_node_selector()
+        spec = self.get_selection_spec()
+        unique_ids = sorted(selector.get_selected(spec))
+        if not unique_ids:
+            warn_or_error(NoNodesSelected())
+            return
+        if self.manifest is None:
+            raise DbtInternalError("manifest is None in _iterate_selected_nodes")
+        for unique_id in unique_ids:
+            if unique_id in self.manifest.nodes:
+                yield self.manifest.nodes[unique_id]
+            elif unique_id in self.manifest.sources:
+                yield self.manifest.sources[unique_id]
+            elif unique_id in self.manifest.exposures:
+                yield self.manifest.exposures[unique_id]
+            elif unique_id in self.manifest.metrics:
+                yield self.manifest.metrics[unique_id]
+            elif unique_id in self.manifest.semantic_models:
+                yield self.manifest.semantic_models[unique_id]
+            elif unique_id in self.manifest.unit_tests:
+                yield self.manifest.unit_tests[unique_id]
+            elif unique_id in self.manifest.saved_queries:
+                yield self.manifest.saved_queries[unique_id]
+            elif unique_id in self.manifest.functions:
+                yield self.manifest.functions[unique_id]
+            else:
+                raise DbtRuntimeError(
+                    f'Got an unexpected result from node selection: "{unique_id}"'
+                    f"Listing this node type is not yet supported!"
+                )
+
+    def generate_selectors(self):
+        for node in self._iterate_selected_nodes():
+            if node.resource_type == NodeType.Source:
+                assert isinstance(node, SourceDefinition)
+                # sources are searched for by pkg.source_name.table_name
+                source_selector = ".".join([node.package_name, node.source_name, node.name])
+                yield f"source:{source_selector}"
+            elif node.resource_type == NodeType.Exposure:
+                assert isinstance(node, Exposure)
+                # exposures are searched for by pkg.exposure_name
+                exposure_selector = ".".join([node.package_name, node.name])
+                yield f"exposure:{exposure_selector}"
+            elif node.resource_type == NodeType.Metric:
+                assert isinstance(node, Metric)
+                # metrics are searched for by pkg.metric_name
+                metric_selector = ".".join([node.package_name, node.name])
+                yield f"metric:{metric_selector}"
+            elif node.resource_type == NodeType.SavedQuery:
+                assert isinstance(node, SavedQuery)
+                saved_query_selector = ".".join([node.package_name, node.name])
+                yield f"saved_query:{saved_query_selector}"
+            elif node.resource_type == NodeType.SemanticModel:
+                assert isinstance(node, SemanticModel)
+                semantic_model_selector = ".".join([node.package_name, node.name])
+                yield f"semantic_model:{semantic_model_selector}"
+            elif node.resource_type == NodeType.Unit:
+                assert isinstance(node, UnitTestDefinition)
+                unit_test_selector = ".".join([node.package_name, node.versioned_name])
+                yield f"unit_test:{unit_test_selector}"
+            else:
+                # everything else is from `fqn`
+                yield ".".join(node.fqn)
+
+    def generate_names(self):
+        for node in self._iterate_selected_nodes():
+            yield node.search_name
+
+    def _get_nested_value(self, data, key_path):
+        """Get nested value using dot notation (e.g., 'config.materialized')"""
+        keys = key_path.split(".")
+        current = data
+        for key in keys:
+            if isinstance(current, dict) and key in current:
+                current = current[key]
+            else:
+                return None
+        return current
+
+    def generate_json(self):
+        for node in self._iterate_selected_nodes():
+            node_dict = node.to_dict(omit_none=False)
+
+            if self.args.output_keys:
+                # Handle both nested and regular keys
+                result = {}
+                for key in self.args.output_keys:
+                    if "." in key:
+                        # Handle nested key (e.g., 'config.materialized')
+                        value = self._get_nested_value(node_dict, key)
+                        if value is not None:
+                            result[key] = value
+                    else:
+                        # Handle regular key
+                        if key in node_dict:
+                            result[key] = node_dict[key]
+            else:
+                # Use default allowed keys
+                result = {k: v for k, v in node_dict.items() if k in self.ALLOWED_KEYS}
+
+            yield json.dumps(result, cls=JSONEncoder)
+
+    def generate_paths(self) -> Iterator[str]:
+        for node in self._iterate_selected_nodes():
+            yield node.original_file_path
+
+    def run(self):
+        # We set up a context manager here with "task_contextvars" because we
+        # we need the project_root in compile_manifest.
+        with task_contextvars(project_root=self.config.project_root):
+            self.compile_manifest()
+            output = self.args.output
+            if output == "selector":
+                generator = self.generate_selectors
+            elif output == "name":
+                generator = self.generate_names
+            elif output == "json":
+                generator = self.generate_json
+            elif output == "path":
+                generator = self.generate_paths
+            else:
+                raise DbtInternalError("Invalid output {}".format(output))
+
+            return self.output_results(generator())
+
+    def output_results(self, results):
+        """Log, or output a plain, newline-delimited, and ready-to-pipe list of nodes found."""
+        for result in results:
+            self.node_results.append(result)
+            # No formatting, still get to stdout when --quiet is used
+            fire_event(PrintEvent(msg=result))
+        return self.node_results
+
+    @property
+    def resource_types(self) -> List[NodeType]:
+        if self.args.models:
+            return [NodeType.Model]
+
+        resource_types = resource_types_from_args(
+            self.args, set(self.ALL_RESOURCE_VALUES), set(self.DEFAULT_RESOURCE_VALUES)
+        )
+
+        return list(resource_types)
+
+    @property
+    def selection_arg(self):
+        # for backwards compatibility, list accepts both --models and --select,
+        # with slightly different behavior: --models implies --resource-type model
+        if self.args.models:
+            return self.args.models
+        else:
+            return self.args.select
+
+    def get_node_selector(self) -> ResourceTypeSelector:
+        if self.manifest is None or self.graph is None:
+            raise DbtInternalError("manifest and graph must be set to get perform node selection")
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=self.resource_types,
+            include_empty_nodes=True,
+        )
+
+    def interpret_results(self, results):
+        # list command should always return 0 as exit code
+        return True