dvt-core 1.11.0b4__py3-none-any.whl

dvt/__init__.py

@@ -0,0 +1,7 @@
+# N.B.
+# This will add to the package’s __path__ all subdirectories of directories on sys.path named after the package which effectively combines both modules into a single namespace (dbt.adapters)
+# The matching statement is in plugins/postgres/dbt/__init__.py
+
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

dvt/_pydantic_shim.py

@@ -0,0 +1,26 @@
+# type: ignore
+
+"""Shim to allow support for both Pydantic 1 and Pydantic 2.
+
+dbt-core must support both major versions of Pydantic because dbt-core users might be using an environment with
+either version, and we can't restrict them to one or the other. Here, we essentially import all Pydantic objects
+from version 1 that we use. Throughout the repo, we import these objects from this file instead of from Pydantic
+directly, meaning that we essentially only use Pydantic 1 in dbt-core currently, but without forcing that restriction
+on dbt users. The development environment for this repo should be pinned to Pydantic 1 to ensure devs get appropriate
+type hints.
+"""
+
+from importlib.metadata import version
+
+pydantic_version = version("pydantic")
+# Pydantic uses semantic versioning, i.e. <major>.<minor>.<patch>, and we need to know the major
+pydantic_major = pydantic_version.split(".")[0]
+
+if pydantic_major == "1":
+    from pydantic import BaseSettings  # noqa: F401
+elif pydantic_major == "2":
+    from pydantic.v1 import BaseSettings  # noqa: F401
+else:
+    raise RuntimeError(
+        f"Currently only pydantic 1 and 2 are supported, found pydantic {pydantic_version}"
+    )

dvt/adapters/__init__.py

@@ -0,0 +1,16 @@
+"""
+DVT adapter management extensions.
+
+This module extends dbt's adapter system to support multi-profile
+source connections and compute layer integration.
+"""
+
+from dvt.adapters.multi_adapter_manager import (
+    MultiAdapterManager,
+    create_multi_adapter_manager,
+)
+
+__all__ = [
+    "MultiAdapterManager",
+    "create_multi_adapter_manager",
+]

dvt/adapters/multi_adapter_manager.py

@@ -0,0 +1,268 @@
+"""
+Multi-adapter management for DVT.
+
+This module extends dbt's single-adapter model to support multiple
+adapter instances for different profiles (sources + targets).
+"""
+
+from multiprocessing.context import SpawnContext
+from typing import Any, Dict, Optional
+
+from dvt.config.profiles_v2 import (
+    ProfileReference,
+    ProfileRegistry,
+    UnifiedProfileConfig,
+)
+
+from dbt.adapters.factory import FACTORY, get_adapter_class_by_name, load_plugin
+from dbt.adapters.protocol import AdapterProtocol
+from dbt_common.events.base_types import EventLevel
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import Note
+from dbt_common.exceptions import DbtRuntimeError
+
+
+class MultiAdapterManager:
+    """
+    Manages multiple adapter instances for different profiles.
+
+    DVT extends dbt's single-adapter model by allowing:
+    1. Multiple source connections (each with its own profile)
+    2. Target connection (standard dbt output)
+    3. Each profile can use a different adapter type
+
+    Architecture:
+    - Profile name -> Adapter instance mapping
+    - Lazy initialization (create adapter when first referenced)
+    - Cleanup management for all adapters
+    - Integration with dbt's existing adapter factory
+    """
+
+    def __init__(self, unified_profiles: UnifiedProfileConfig, mp_context: SpawnContext):
+        """
+        Initialize MultiAdapterManager.
+
+        Args:
+            unified_profiles: Unified profile configuration
+            mp_context: Multiprocessing context for adapter initialization
+        """
+        self.unified_profiles = unified_profiles
+        self.mp_context = mp_context
+        self._adapters: Dict[str, AdapterProtocol] = {}
+        self._profile_to_adapter_type: Dict[str, str] = {}
+
+    def get_or_create_adapter(
+        self,
+        profile_name: str,
+        adapter_registered_log_level: Optional[EventLevel] = EventLevel.INFO,
+    ) -> AdapterProtocol:
+        """
+        Get or create adapter instance for a profile.
+
+        This is the main entry point for getting adapters by profile name.
+        It handles:
+        1. Checking if adapter already exists (cache)
+        2. Loading profile configuration
+        3. Loading adapter plugin if needed
+        4. Creating and registering the adapter
+        5. Caching for future use
+
+        Args:
+            profile_name: Name of the profile to get adapter for
+            adapter_registered_log_level: Log level for adapter registration
+
+        Returns:
+            Adapter instance
+
+        Raises:
+            DbtRuntimeError: If profile not found or adapter creation fails
+        """
+        # Check cache
+        if profile_name in self._adapters:
+            return self._adapters[profile_name]
+
+        # Get profile configuration
+        profile = self.unified_profiles.get_profile(profile_name)
+        if not profile:
+            raise DbtRuntimeError(
+                f"Profile '{profile_name}' not found in profiles configuration. "
+                f"Available profiles: {', '.join(self.unified_profiles.list_profiles())}"
+            )
+
+        # Ensure adapter plugin is loaded
+        adapter_type = profile.adapter
+        try:
+            load_plugin(adapter_type)
+        except Exception as e:
+            raise DbtRuntimeError(
+                f"Failed to load adapter plugin '{adapter_type}' for profile '{profile_name}': {e}"
+            )
+
+        # Create adapter configuration
+        # For DVT, we need to create a minimal AdapterRequiredConfig
+        # The actual credentials will come from the profile
+        adapter_config = self._create_adapter_config(profile)
+
+        # Check if an adapter of this type already exists in dbt's FACTORY
+        # If so, we can reuse it if the configuration matches
+        # Otherwise, create a new instance
+        try:
+            existing_adapter = FACTORY.lookup_adapter(adapter_type)
+            # Check if credentials match - if so, reuse
+            if self._credentials_match(existing_adapter, profile):
+                # Reusing existing adapter
+                self._adapters[profile_name] = existing_adapter
+                self._profile_to_adapter_type[profile_name] = adapter_type
+                return existing_adapter
+        except KeyError:
+            # No existing adapter of this type
+            pass
+
+        # Create new adapter instance
+        adapter_class = get_adapter_class_by_name(adapter_type)
+        adapter_instance = adapter_class(adapter_config, self.mp_context)  # type: ignore[call-arg]
+
+        # Cache the adapter
+        self._adapters[profile_name] = adapter_instance
+        self._profile_to_adapter_type[profile_name] = adapter_type
+
+        return adapter_instance
+
+    def _create_adapter_config(self, profile: ProfileReference) -> Any:
+        """
+        Create adapter configuration from ProfileReference.
+
+        This converts DVT's ProfileReference to dbt's expected config format.
+        We create a minimal config object that satisfies adapter requirements.
+
+        Args:
+            profile: Profile reference
+
+        Returns:
+            Configuration object for adapter initialization
+        """
+        # Import credentials class for this adapter type
+        adapter_class = get_adapter_class_by_name(profile.adapter)
+        credentials_class = adapter_class.Credentials  # type: ignore[attr-defined]
+
+        # Create credentials instance from profile
+        credentials_dict = {
+            "type": profile.adapter,
+            **profile.credentials,
+        }
+        credentials = credentials_class.from_dict(credentials_dict)  # type: ignore[attr-defined]
+
+        # Create minimal config object
+        # Note: This is a simplified config for DVT multi-adapter support
+        # Full RuntimeConfig would have more fields, but adapters primarily need credentials
+        class MinimalAdapterConfig:
+            def __init__(self, creds, name, threads):
+                self.credentials = creds
+                self.project_name = name
+                self.profile_name = name
+                self.target_name = name
+                self.threads = threads
+                # Required protocol fields
+                self.query_comment: Dict[str, Any] = {}
+                self.cli_vars: Dict[str, Any] = {}
+                self.target_path = "target"
+                self.log_cache_events = False
+
+        config = MinimalAdapterConfig(credentials, profile.name, profile.threads)
+        return config
+
+    def _credentials_match(self, adapter: AdapterProtocol, profile: ProfileReference) -> bool:
+        """
+        Check if adapter's credentials match the profile.
+
+        Args:
+            adapter: Existing adapter instance
+            profile: Profile reference to compare
+
+        Returns:
+            True if credentials match, False otherwise
+        """
+        # Get adapter's credentials
+        adapter_creds = adapter.config.credentials.to_dict()  # type: ignore[attr-defined]
+        profile_creds = {
+            "type": profile.adapter,
+            **profile.credentials,
+        }
+
+        # Compare key fields (excluding transient fields like query comments)
+        key_fields = ["type", "host", "port", "user", "database", "schema"]
+        for field in key_fields:
+            if field in adapter_creds or field in profile_creds:
+                if adapter_creds.get(field) != profile_creds.get(field):
+                    return False
+
+        return True
+
+    def get_adapter_for_source(self, source_node) -> AdapterProtocol:
+        """
+        Get adapter for a source node.
+
+        Args:
+            source_node: Source node with profile reference
+
+        Returns:
+            Adapter instance for the source's profile
+
+        Raises:
+            DbtRuntimeError: If source has no profile or profile not found
+        """
+        if not hasattr(source_node, "profile") or not source_node.profile:
+            raise DbtRuntimeError(
+                f"Source '{source_node.name}' has no profile reference. "
+                "Sources must specify a profile in sources.yml"
+            )
+
+        return self.get_or_create_adapter(source_node.profile)
+
+    def cleanup_all_adapters(self) -> None:
+        """Clean up all managed adapters."""
+        for profile_name, adapter in self._adapters.items():
+            try:
+                adapter.cleanup_connections()  # type: ignore[attr-defined]
+                fire_event(Note(msg="Cleaned up adapter for profile '{}'".format(profile_name)))
+            except Exception as e:
+                fire_event(
+                    Note(
+                        msg="Error cleaning up adapter for profile '{}': {}".format(
+                            profile_name, e
+                        )
+                    )
+                )
+
+        self._adapters.clear()
+        self._profile_to_adapter_type.clear()
+
+    def list_active_adapters(self) -> Dict[str, str]:
+        """
+        List all active adapters.
+
+        Returns:
+            Dictionary mapping profile name to adapter type
+        """
+        return dict(self._profile_to_adapter_type)
+
+    def get_adapter_count(self) -> int:
+        """Get number of active adapters."""
+        return len(self._adapters)
+
+
+def create_multi_adapter_manager(
+    unified_profiles: UnifiedProfileConfig,
+    mp_context: SpawnContext,
+) -> MultiAdapterManager:
+    """
+    Create MultiAdapterManager instance.
+
+    Args:
+        unified_profiles: Unified profile configuration
+        mp_context: Multiprocessing context
+
+    Returns:
+        MultiAdapterManager instance
+    """
+    return MultiAdapterManager(unified_profiles, mp_context)

dvt/artifacts/exceptions/__init__.py

@@ -0,0 +1 @@
+from dvt.artifacts.exceptions.schemas import IncompatibleSchemaError

dvt/artifacts/exceptions/schemas.py

@@ -0,0 +1,31 @@
+from typing import Optional
+
+from dbt_common.exceptions import DbtRuntimeError
+
+
+class IncompatibleSchemaError(DbtRuntimeError):
+    def __init__(self, expected: str, found: Optional[str] = None) -> None:
+        self.expected = expected
+        self.found = found
+        self.filename = "input file"
+
+        super().__init__(msg=self.get_message())
+
+    def add_filename(self, filename: str):
+        self.filename = filename
+        self.msg = self.get_message()
+
+    def get_message(self) -> str:
+        found_str = "nothing"
+        if self.found is not None:
+            found_str = f'"{self.found}"'
+
+        msg = (
+            f'Expected a schema version of "{self.expected}" in '
+            f"{self.filename}, but found {found_str}. Are you running with a "
+            f"different version of dbt?"
+        )
+        return msg
+
+    CODE = 10014
+    MESSAGE = "Incompatible Schema"

dvt/artifacts/resources/__init__.py

@@ -0,0 +1,116 @@
+from dvt.artifacts.resources.base import BaseResource, Docs, FileHash, GraphResource
+from dvt.artifacts.resources.v1.analysis import Analysis
+from dvt.artifacts.resources.v1.catalog import Catalog, CatalogWriteIntegrationConfig
+
+# alias to latest resource definitions
+from dvt.artifacts.resources.v1.components import (
+    ColumnConfig,
+    ColumnInfo,
+    CompiledResource,
+    Contract,
+    DeferRelation,
+    DependsOn,
+    FreshnessThreshold,
+    HasRelationMetadata,
+    InjectedCTE,
+    NodeVersion,
+    ParsedResource,
+    ParsedResourceMandatory,
+    Quoting,
+    RefArgs,
+    Time,
+)
+from dvt.artifacts.resources.v1.config import (
+    Hook,
+    NodeAndTestConfig,
+    NodeConfig,
+    TestConfig,
+    list_str,
+    metas,
+)
+from dvt.artifacts.resources.v1.documentation import Documentation
+from dvt.artifacts.resources.v1.exposure import (
+    Exposure,
+    ExposureConfig,
+    ExposureType,
+    MaturityType,
+)
+from dvt.artifacts.resources.v1.function import (
+    Function,
+    FunctionArgument,
+    FunctionConfig,
+    FunctionMandatory,
+    FunctionReturns,
+)
+from dvt.artifacts.resources.v1.generic_test import GenericTest, TestMetadata
+from dvt.artifacts.resources.v1.group import Group, GroupConfig
+from dvt.artifacts.resources.v1.hook import HookNode
+from dvt.artifacts.resources.v1.macro import Macro, MacroArgument, MacroDependsOn
+from dvt.artifacts.resources.v1.metric import (
+    ConstantPropertyInput,
+    ConversionTypeParams,
+    CumulativeTypeParams,
+    Metric,
+    MetricAggregationParams,
+    MetricConfig,
+    MetricInput,
+    MetricInputMeasure,
+    MetricTimeWindow,
+    MetricTypeParams,
+)
+from dvt.artifacts.resources.v1.model import (
+    CustomGranularity,
+    Model,
+    ModelConfig,
+    ModelFreshness,
+    TimeSpine,
+)
+from dvt.artifacts.resources.v1.owner import Owner
+from dvt.artifacts.resources.v1.saved_query import (
+    Export,
+    ExportConfig,
+    QueryParams,
+    SavedQuery,
+    SavedQueryConfig,
+    SavedQueryMandatory,
+)
+from dvt.artifacts.resources.v1.seed import Seed, SeedConfig
+from dvt.artifacts.resources.v1.semantic_layer_components import (
+    FileSlice,
+    MeasureAggregationParameters,
+    NonAdditiveDimension,
+    SourceFileMetadata,
+    WhereFilter,
+    WhereFilterIntersection,
+)
+from dvt.artifacts.resources.v1.semantic_model import (
+    Defaults,
+    Dimension,
+    DimensionTypeParams,
+    DimensionValidityParams,
+    Entity,
+    Measure,
+    NodeRelation,
+    SemanticLayerElementConfig,
+    SemanticModel,
+    SemanticModelConfig,
+)
+from dvt.artifacts.resources.v1.singular_test import SingularTest
+from dvt.artifacts.resources.v1.snapshot import Snapshot, SnapshotConfig
+from dvt.artifacts.resources.v1.source_definition import (
+    ExternalPartition,
+    ExternalTable,
+    ParsedSourceMandatory,
+    SourceConfig,
+    SourceDefinition,
+)
+from dvt.artifacts.resources.v1.sql_operation import SqlOperation
+from dvt.artifacts.resources.v1.unit_test_definition import (
+    UnitTestConfig,
+    UnitTestDefinition,
+    UnitTestFormat,
+    UnitTestInputFixture,
+    UnitTestNodeVersions,
+    UnitTestOutputFixture,
+    UnitTestOverrides,
+)

dvt/artifacts/resources/base.py

@@ -0,0 +1,68 @@
+import hashlib
+from dataclasses import dataclass
+from typing import List, Optional
+
+from dvt.artifacts.resources.types import NodeType
+
+from dbt_common.dataclass_schema import dbtClassMixin
+
+
+@dataclass
+class BaseResource(dbtClassMixin):
+    name: str
+    resource_type: NodeType
+    package_name: str
+    path: str
+    original_file_path: str
+    unique_id: str
+
+
+@dataclass
+class GraphResource(BaseResource):
+    fqn: List[str]
+
+
+@dataclass
+class FileHash(dbtClassMixin):
+    name: str  # the hash type name
+    checksum: str  # the hashlib.hash_type().hexdigest() of the file contents
+
+    @classmethod
+    def empty(cls):
+        return FileHash(name="none", checksum="")
+
+    @classmethod
+    def path(cls, path: str):
+        return FileHash(name="path", checksum=path)
+
+    def __eq__(self, other):
+        if not isinstance(other, FileHash):
+            return NotImplemented
+
+        if self.name == "none" or self.name != other.name:
+            return False
+
+        return self.checksum == other.checksum
+
+    def compare(self, contents: str) -> bool:
+        """Compare the file contents with the given hash"""
+        if self.name == "none":
+            return False
+
+        return self.from_contents(contents, name=self.name) == self.checksum
+
+    @classmethod
+    def from_contents(cls, contents: str, name="sha256") -> "FileHash":
+        """Create a file hash from the given file contents. The hash is always
+        the utf-8 encoding of the contents given, because dbt only reads files
+        as utf-8.
+        """
+        data = contents.encode("utf-8")
+        checksum = hashlib.new(name, data).hexdigest()
+        return cls(name=name, checksum=checksum)
+
+
+@dataclass
+class Docs(dbtClassMixin):
+    show: bool = True
+    node_color: Optional[str] = None

dvt/artifacts/resources/types.py

@@ -0,0 +1,93 @@
+from dbt_common.dataclass_schema import StrEnum
+
+
+class AccessType(StrEnum):
+    Private = "private"
+    Protected = "protected"
+    Public = "public"
+
+    @classmethod
+    def is_valid(cls, item):
+        try:
+            cls(item)
+        except ValueError:
+            return False
+        return True
+
+
+class NodeType(StrEnum):
+    Model = "model"
+    Analysis = "analysis"
+    Test = "test"  # renamed to 'data_test'; preserved as 'test' here for back-compat
+    Snapshot = "snapshot"
+    Operation = "operation"
+    Seed = "seed"
+    # TODO: rm?
+    RPCCall = "rpc"
+    SqlOperation = "sql_operation"
+    Documentation = "doc"
+    Source = "source"
+    Macro = "macro"
+    Exposure = "exposure"
+    Metric = "metric"
+    Group = "group"
+    SavedQuery = "saved_query"
+    SemanticModel = "semantic_model"
+    Unit = "unit_test"
+    Fixture = "fixture"
+    Function = "function"
+
+    def pluralize(self) -> str:
+        if self is self.Analysis:
+            return "analyses"
+        elif self is self.SavedQuery:
+            return "saved_queries"
+        elif self is self.Test:
+            return "data_tests"
+        return f"{self}s"
+
+
+class RunHookType(StrEnum):
+    Start = "on-run-start"
+    End = "on-run-end"
+
+
+class ModelLanguage(StrEnum):
+    python = "python"
+    sql = "sql"
+
+
+class ModelHookType(StrEnum):
+    PreHook = "pre-hook"
+    PostHook = "post-hook"
+
+
+class TimePeriod(StrEnum):
+    minute = "minute"
+    hour = "hour"
+    day = "day"
+
+    def plural(self) -> str:
+        return str(self) + "s"
+
+
+class BatchSize(StrEnum):
+    hour = "hour"
+    day = "day"
+    month = "month"
+    year = "year"
+
+    def plural(self) -> str:
+        return str(self) + "s"
+
+
+class FunctionType(StrEnum):
+    Scalar = "scalar"
+    Aggregate = "aggregate"
+    Table = "table"
+
+
+class FunctionVolatility(StrEnum):
+    Deterministic = "deterministic"
+    Stable = "stable"
+    NonDeterministic = "non-deterministic"

dvt/artifacts/resources/v1/analysis.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import Literal
+
+from dvt.artifacts.resources.types import NodeType
+from dvt.artifacts.resources.v1.components import CompiledResource
+
+
+@dataclass
+class Analysis(CompiledResource):
+    resource_type: Literal[NodeType.Analysis]

dvt/artifacts/resources/v1/catalog.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from dbt.adapters.catalogs import CatalogIntegrationConfig
+from dbt_common.dataclass_schema import dbtClassMixin
+
+
+@dataclass
+class CatalogWriteIntegrationConfig(CatalogIntegrationConfig):
+    name: str
+    catalog_type: str
+    external_volume: Optional[str] = None
+    table_format: Optional[str] = None
+    catalog_name: Optional[str] = None
+    file_format: Optional[str] = None
+    adapter_properties: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Catalog(dbtClassMixin):
+    name: str
+    active_write_integration: Optional[str] = None
+    write_integrations: List[CatalogWriteIntegrationConfig] = field(default_factory=list)