dagster-dbt 0.27.13__tar.gz → 0.27.15__tar.gz

{dagster_dbt-0.27.13/dagster_dbt.egg-info → dagster_dbt-0.27.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.27.13
+Version: 0.27.15
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
@@ -15,7 +15,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.13
+Requires-Dist: dagster==1.11.15
 Requires-Dist: dbt-core<1.11,>=1.7
 Requires-Dist: Jinja2
 Requires-Dist: networkx

{dagster_dbt-0.27.13 → dagster_dbt-0.27.15}/dagster_dbt/asset_utils.py

@@ -813,7 +813,7 @@ def build_dbt_specs(
 
         # add check specs associated with the asset
         for child_unique_id in child_map.get(unique_id, []):
-            if not child_unique_id.startswith("test"):
+            if child_unique_id not in selected_unique_ids or not child_unique_id.startswith("test"):
                 continue
             check_spec = translator.get_asset_check_spec(
                 asset_spec=spec,

{dagster_dbt-0.27.13 → dagster_dbt-0.27.15}/dagster_dbt/components/dbt_project/component.py

@@ -1,25 +1,31 @@
 import itertools
 import json
+import shutil
 from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from contextvars import ContextVar
 from dataclasses import dataclass, field
 from functools import cached_property
 from pathlib import Path
 from typing import Annotated, Any, Literal, Optional, Union
 
-from dagster import Resolvable
+import dagster as dg
 from dagster._annotations import public
-from dagster._core.definitions.asset_key import AssetKey
-from dagster._core.definitions.definitions_class import Definitions
-from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
 from dagster._utils.cached_method import cached_method
-from dagster.components.component.component import Component
+from dagster.components.component.state_backed_component import StateBackedComponent
 from dagster.components.core.component_tree import ComponentTree
-from dagster.components.core.context import ComponentLoadContext
 from dagster.components.resolved.core_models import OpSpec, ResolutionContext
 from dagster.components.resolved.model import Resolver
 from dagster.components.scaffold.scaffold import scaffold_with
-from dagster.components.utils.translation import TranslationFn, TranslationFnResolver
+from dagster.components.utils.defs_state import DefsStateConfig
+from dagster.components.utils.translation import (
+    ComponentTranslator,
+    TranslationFn,
+    TranslationFnResolver,
+    create_component_translator_cls,
+)
 from dagster_shared import check
+from dagster_shared.serdes.objects.models.defs_state_info import DefsStateManagementType
 from typing_extensions import TypeAlias
 
 from dagster_dbt.asset_decorator import dbt_assets
@@ -38,14 +44,14 @@ from dagster_dbt.utils import ASSET_RESOURCE_TYPES
 
 
 @dataclass(frozen=True)
-class DagsterDbtComponentsTranslatorSettings(DagsterDbtTranslatorSettings):
+class DagsterDbtComponentTranslatorSettings(DagsterDbtTranslatorSettings):
     """Subclass of DagsterDbtTranslatorSettings that enables code references by default."""
 
     enable_code_references: bool = True
 
 
 @dataclass
-class DbtProjectArgs(Resolvable):
+class DbtProjectArgs(dg.Resolvable):
     """Aligns with DbtProject.__new__."""
 
     project_dir: str
@@ -84,11 +90,22 @@ def resolve_dbt_project(context: ResolutionContext, model) -> DbtProject:
 
 DbtMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
 
+_resolution_context: ContextVar[ResolutionContext] = ContextVar("resolution_context")
+
+
+@contextmanager
+def _set_resolution_context(context: ResolutionContext):
+    token = _resolution_context.set(context)
+    try:
+        yield
+    finally:
+        _resolution_context.reset(token)
+
 
 @public
 @scaffold_with(DbtProjectComponentScaffolder)
 @dataclass
-class DbtProjectComponent(Component, Resolvable):
+class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
     """Expose a DBT project to Dagster as a set of assets.
 
     This component assumes that you have already set up a dbt project, for example, the dbt `Jaffle shop <https://github.com/dbt-labs/jaffle-shop>`_. Run `git clone --depth=1 https://github.com/dbt-labs/jaffle-shop.git jaffle_shop && rm -rf jaffle_shop/.git` to copy that project
@@ -125,8 +142,8 @@ class DbtProjectComponent(Component, Resolvable):
                     "--full_refresh",
                     {
                         "--vars": {
-                            "start_date": "{{ context.partition_range_start }}",
-                            "end_date": "{{ context.partition_range_end }}",
+                            "start_date": "{{ partition_range_start }}",
+                            "end_date": "{{ partition_range_end }}",
                         },
                     },
                 ],
@@ -175,7 +192,7 @@ class DbtProjectComponent(Component, Resolvable):
         TranslationFnResolver(template_vars_for_translation_fn=lambda data: {"node": data}),
     ] = None
     translation_settings: Annotated[
-        Optional[DagsterDbtComponentsTranslatorSettings],
+        Optional[DagsterDbtComponentTranslatorSettings],
         Resolver.default(
             description="Allows enabling or disabling various features for translating dbt models in to Dagster assets.",
             examples=[
@@ -184,7 +201,7 @@ class DbtProjectComponent(Component, Resolvable):
                 },
             ],
         ),
-    ] = None
+    ] = field(default_factory=lambda: DagsterDbtComponentTranslatorSettings())
     prepare_if_dev: Annotated[
         bool,
         Resolver.default(
@@ -192,12 +209,35 @@ class DbtProjectComponent(Component, Resolvable):
         ),
     ] = True
 
+    @property
+    def defs_state_config(self) -> DefsStateConfig:
+        return DefsStateConfig(
+            key=f"{self.__class__.__name__}[{self.project.name}]",
+            type=DefsStateManagementType.LOCAL_FILESYSTEM,
+            refresh_if_dev=self.prepare_if_dev,
+        )
+
     @cached_property
-    def translator(self):
-        translation_settings = self.translation_settings or DagsterDbtComponentsTranslatorSettings()
-        if self.translation:
-            return ProxyDagsterDbtTranslator(self.translation, translation_settings)
-        return DagsterDbtTranslator(translation_settings)
+    def translator(self) -> "DagsterDbtTranslator":
+        return DbtProjectComponentTranslator(self, self.translation_settings)
+
+    @cached_property
+    def _base_translator(self) -> "DagsterDbtTranslator":
+        return DagsterDbtTranslator(self.translation_settings)
+
+    def get_asset_spec(
+        self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]
+    ) -> dg.AssetSpec:
+        return self._base_translator.get_asset_spec(manifest, unique_id, project)
+
+    def get_asset_check_spec(
+        self,
+        asset_spec: dg.AssetSpec,
+        manifest: Mapping[str, Any],
+        unique_id: str,
+        project: Optional["DbtProject"],
+    ) -> Optional[dg.AssetCheckSpec]:
+        return self._base_translator.get_asset_check_spec(asset_spec, manifest, unique_id, project)
 
     @cached_property
     def cli_resource(self):
@@ -213,9 +253,19 @@ class DbtProjectComponent(Component, Resolvable):
             exclude=exclude,
         )
 
-    def build_defs(self, context: ComponentLoadContext) -> Definitions:
-        if self.prepare_if_dev:
-            self.project.prepare_if_dev()
+    def write_state_to_path(self, state_path: Path) -> None:
+        # compile the manifest
+        self.project.preparer.prepare(self.project)
+        # move the manifest to the correct path
+        shutil.copyfile(self.project.manifest_path, state_path)
+
+    def build_defs_from_state(
+        self, context: dg.ComponentLoadContext, state_path: Optional[Path]
+    ) -> dg.Definitions:
+        if state_path is not None:
+            shutil.copyfile(state_path, self.project.manifest_path)
+
+        res_ctx = context.resolution_context
 
         @dbt_assets(
             manifest=self.project.manifest_path,
@@ -227,13 +277,13 @@ class DbtProjectComponent(Component, Resolvable):
             exclude=self.exclude,
             backfill_policy=self.op.backfill_policy if self.op else None,
         )
-        def _fn(context: AssetExecutionContext):
-            yield from self.execute(context=context, dbt=self.cli_resource)
+        def _fn(context: dg.AssetExecutionContext):
+            with _set_resolution_context(res_ctx):
+                yield from self.execute(context=context, dbt=self.cli_resource)
 
-        defs = Definitions(assets=[_fn])
-        return defs
+        return dg.Definitions(assets=[_fn])
 
-    def get_cli_args(self, context: AssetExecutionContext) -> list[str]:
+    def get_cli_args(self, context: dg.AssetExecutionContext) -> list[str]:
         # create a resolution scope that includes the partition key and range, if available
         partition_key = context.partition_key if context.has_partition_key else None
         partition_key_range = (
@@ -244,15 +294,15 @@ class DbtProjectComponent(Component, Resolvable):
         except Exception:
             partition_time_window = None
 
-        scope = dict(
-            partition_key=partition_key,
-            partition_key_range=partition_key_range,
-            partition_time_window=partition_time_window,
-        )
-
-        # resolve the cli args with this additional scope
-        resolved_args = ResolutionContext(scope=scope).resolve_value(
-            self.cli_args, as_type=list[str]
+        # resolve the cli args with additional partition-related scope
+        resolved_args = (
+            _resolution_context.get()
+            .with_scope(
+                partition_key=partition_key,
+                partition_key_range=partition_key_range,
+                partition_time_window=partition_time_window,
+            )
+            .resolve_value(self.cli_args, as_type=list[str])
         )
 
         def _normalize_arg(arg: Union[str, dict[str, Any]]) -> list[str]:
@@ -277,7 +327,7 @@ class DbtProjectComponent(Component, Resolvable):
         return normalized_args
 
     def _get_dbt_event_iterator(
-        self, context: AssetExecutionContext, dbt: DbtCliResource
+        self, context: dg.AssetExecutionContext, dbt: DbtCliResource
     ) -> Iterator:
         iterator = dbt.cli(self.get_cli_args(context), context=context).stream()
         if "column_metadata" in self.include_metadata:
@@ -286,7 +336,7 @@ class DbtProjectComponent(Component, Resolvable):
             iterator = iterator.fetch_row_counts()
         return iterator
 
-    def execute(self, context: AssetExecutionContext, dbt: DbtCliResource) -> Iterator:
+    def execute(self, context: dg.AssetExecutionContext, dbt: DbtCliResource) -> Iterator:
         yield from self._get_dbt_event_iterator(context, dbt)
 
     @cached_property
@@ -298,7 +348,7 @@ class DbtProjectComponent(Component, Resolvable):
         return validate_translator(self.translator)
 
     @cached_method
-    def asset_key_for_model(self, model_name: str) -> AssetKey:
+    def asset_key_for_model(self, model_name: str) -> dg.AssetKey:
         dagster_dbt_translator = self._validated_translator
         manifest = self._validated_manifest
 
@@ -318,17 +368,27 @@ class DbtProjectComponent(Component, Resolvable):
         ).key
 
 
-class ProxyDagsterDbtTranslator(DagsterDbtTranslator):
-    # get_description conflicts on Component, so cant make it directly a translator
-
-    def __init__(self, fn: TranslationFn, settings: Optional[DagsterDbtTranslatorSettings]):
-        self._fn = fn
+class DbtProjectComponentTranslator(
+    create_component_translator_cls(DbtProjectComponent, DagsterDbtTranslator),
+    ComponentTranslator[DbtProjectComponent],
+):
+    def __init__(
+        self,
+        component: DbtProjectComponent,
+        settings: Optional[DagsterDbtComponentTranslatorSettings],
+    ):
+        self._component = component
         super().__init__(settings)
 
-    def get_asset_spec(self, manifest, unique_id, project):
-        base_asset_spec = super().get_asset_spec(manifest, unique_id, project)
-        dbt_props = get_node(manifest, unique_id)
-        return self._fn(base_asset_spec, dbt_props)
+    def get_asset_spec(
+        self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]
+    ) -> dg.AssetSpec:
+        base_spec = super().get_asset_spec(manifest, unique_id, project)
+        if self.component.translation is None:
+            return base_spec
+        else:
+            dbt_props = get_node(manifest, unique_id)
+            return self.component.translation(base_spec, dbt_props)
 
 
 def get_projects_from_dbt_component(components: Path) -> list[DbtProject]:

{dagster_dbt-0.27.13 → dagster_dbt-0.27.15}/dagster_dbt/core/dbt_event_iterator.py

@@ -15,6 +15,7 @@ from dagster._core.definitions.asset_checks.asset_check_evaluation import AssetC
 from dagster._core.definitions.metadata import TableMetadataSet, TextMetadataValue
 from dagster._core.errors import DagsterInvalidPropertyError
 from dagster._core.utils import exhaust_iterator_and_yield_results_with_exception, imap
+from dagster._utils import pushd
 from typing_extensions import TypeVar
 
 from dagster_dbt.asset_utils import default_metadata_from_dbt_resource_props
@@ -31,6 +32,7 @@ DbtDagsterEventType = Union[
     Output, AssetMaterialization, AssetCheckResult, AssetObservation, AssetCheckEvaluation
 ]
 
+
 # We define DbtEventIterator as a generic type for the sake of type hinting.
 # This is so that users who inspect the type of the return value of `DbtCliInvocation.stream()`
 # will be able to see the inner type of the iterator, rather than just `DbtEventIterator`.
@@ -63,7 +65,10 @@ def _fetch_column_metadata(
 
     dbt_resource_props = _get_dbt_resource_props_from_event(invocation, event)
 
-    with adapter.connection_named(f"column_metadata_{dbt_resource_props['unique_id']}"):
+    with (
+        pushd(str(invocation.project_dir)),
+        adapter.connection_named(f"column_metadata_{dbt_resource_props['unique_id']}"),
+    ):
         try:
             cols = invocation._get_columns_from_dbt_resource_props(  # noqa: SLF001
                 adapter=adapter, dbt_resource_props=dbt_resource_props
@@ -165,7 +170,10 @@ def _fetch_row_count_metadata(
     relation_name = dbt_resource_props["relation_name"]
 
     try:
-        with adapter.connection_named(f"row_count_{unique_id}"):
+        with (
+            pushd(str(invocation.project_dir)),
+            adapter.connection_named(f"row_count_{unique_id}"),
+        ):
             query_result = adapter.execute(
                 f"""
                     SELECT
@@ -267,11 +275,12 @@ class DbtEventIterator(Iterator[T]):
         """
 
         def _map_fn(event: DbtDagsterEventType) -> DbtDagsterEventType:
-            result = fn(self._dbt_cli_invocation, event)
-            if result is None:
-                return event
+            with pushd(str(self._dbt_cli_invocation.project_dir)):
+                result = fn(self._dbt_cli_invocation, event)
+                if result is None:
+                    return event
 
-            return event.with_metadata({**event.metadata, **result})
+                return event.with_metadata({**event.metadata, **result})
 
         # If the adapter is DuckDB, we need to wait for the dbt CLI process to complete
         # so that the DuckDB lock is released. This is because DuckDB does not allow for

dagster_dbt-0.27.15/dagster_dbt/version.py

@@ -0,0 +1 @@
+__version__ = "0.27.15"

{dagster_dbt-0.27.13 → dagster_dbt-0.27.15/dagster_dbt.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.27.13
+Version: 0.27.15
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
@@ -15,7 +15,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.13
+Requires-Dist: dagster==1.11.15
 Requires-Dist: dbt-core<1.11,>=1.7
 Requires-Dist: Jinja2
 Requires-Dist: networkx

{dagster_dbt-0.27.13 → dagster_dbt-0.27.15}/dagster_dbt.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dagster==1.11.13
+dagster==1.11.15
 dbt-core<1.11,>=1.7
 Jinja2
 networkx

{dagster_dbt-0.27.13 → dagster_dbt-0.27.15}/setup.py

@@ -40,7 +40,7 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.14",
     install_requires=[
-        "dagster==1.11.13",
+        "dagster==1.11.15",
         # Follow the version support constraints for dbt Core: https://docs.getdbt.com/docs/dbt-versions/core
         f"dbt-core>=1.7,<{DBT_CORE_VERSION_UPPER_BOUND}",
         "Jinja2",

dagster_dbt-0.27.13/dagster_dbt/version.py

@@ -1 +0,0 @@
-__version__ = "0.27.13"