dagster-dbt 0.27.16__tar.gz → 0.28.1__tar.gz

{dagster_dbt-0.27.16/dagster_dbt.egg-info → dagster_dbt-0.28.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.27.16
+Version: 0.28.1
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
@@ -15,8 +15,9 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.16
+Requires-Dist: dagster==1.12.1
 Requires-Dist: dbt-core<1.11,>=1.7
+Requires-Dist: gitpython
 Requires-Dist: Jinja2
 Requires-Dist: networkx
 Requires-Dist: orjson

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt/asset_utils.py

@@ -1,11 +1,14 @@
 import hashlib
 import os
+import shutil
+import tempfile
 import textwrap
 from collections import defaultdict
 from collections.abc import Iterable, Mapping, Sequence
 from pathlib import Path
 from typing import TYPE_CHECKING, AbstractSet, Annotated, Any, Final, Optional, Union  # noqa: UP035
 
+import yaml
 from dagster import (
     AssetCheckKey,
     AssetCheckSpec,
@@ -18,7 +21,6 @@ from dagster import (
     DagsterInvalidDefinitionError,
     DagsterInvariantViolationError,
     DefaultScheduleStatus,
-    LegacyFreshnessPolicy,
     OpExecutionContext,
     RunConfig,
     ScheduleDefinition,
@@ -33,6 +35,7 @@ from dagster._core.definitions.metadata import TableMetadataSet
 from dagster._core.errors import DagsterInvalidPropertyError
 from dagster._core.types.dagster_type import Nothing
 from dagster._record import ImportFrom, record
+from dagster_shared.record import replace
 
 from dagster_dbt.dbt_project import DbtProject
 from dagster_dbt.metadata_set import DbtMetadataSet
@@ -56,6 +59,10 @@ DBT_DEFAULT_SELECTOR = ""
 DBT_INDIRECT_SELECTION_ENV: Final[str] = "DBT_INDIRECT_SELECTION"
 DBT_EMPTY_INDIRECT_SELECTION: Final[str] = "empty"
 
+# Threshold for switching to selector file to avoid CLI argument length limits
+# https://github.com/dagster-io/dagster/issues/16997
+_SELECTION_ARGS_THRESHOLD: Final[int] = 200
+
 DUPLICATE_ASSET_KEY_ERROR_MESSAGE = (
     "The following dbt resources are configured with identical Dagster asset keys."
     " Please ensure that each dbt resource generates a unique Dagster asset key."
@@ -168,7 +175,9 @@ def get_asset_keys_by_output_name_for_source(
         raise KeyError(f"Could not find a dbt source with name: {source_name}")
 
     return {
-        dagster_name_fn(value): dagster_dbt_translator.get_asset_spec(manifest, unique_id, None).key
+        dagster_name_fn(value): dagster_dbt_translator.get_asset_spec(
+            manifest, unique_id, dbt_project
+        ).key
         for unique_id, value in matching.items()
     }
 
@@ -442,6 +451,10 @@ def get_updated_cli_invocation_params_for_context(
             [assets_def]
         )
 
+        # Get project_dir from dbt_project if available
+        project_dir = Path(dbt_project.project_dir) if dbt_project else None
+        target_project = dbt_project
+
         selection_args, indirect_selection_override = get_subset_selection_for_context(
             context=context,
             manifest=manifest,
@@ -451,17 +464,49 @@ def get_updated_cli_invocation_params_for_context(
             dagster_dbt_translator=dagster_dbt_translator,
             current_dbt_indirect_selection_env=indirect_selection,
         )
+        if (
+            selection_args[0] == "--select"
+            and project_dir
+            and len(resources := selection_args[1].split(" ")) > _SELECTION_ARGS_THRESHOLD
+        ):
+            temp_project_dir = tempfile.mkdtemp()
+            shutil.copytree(project_dir, temp_project_dir, dirs_exist_ok=True)
+            selectors_path = Path(temp_project_dir) / "selectors.yml"
+
+            # Delete any existing selectors, we need to create our own
+            if selectors_path.exists():
+                selectors_path.unlink()
+
+            selector_name = f"dagster_run_{context.run_id}"
+            temp_selectors = {
+                "selectors": [
+                    {
+                        "name": selector_name,
+                        "definition": {"union": list(resources)},
+                    }
+                ]
+            }
+            selectors_path.write_text(yaml.safe_dump(temp_selectors))
+            logger.info(
+                f"DBT selection of {len(resources)} resources exceeds threshold of {_SELECTION_ARGS_THRESHOLD}. "
+                "This may exceed system argument length limits. "
+                f"Executing materialization against temporary copy of DBT project at {temp_project_dir} with ephemeral selector."
+            )
+            selection_args = ["--selector", selector_name]
+            target_project = replace(dbt_project, project_dir=temp_project_dir)
 
         indirect_selection = (
             indirect_selection_override if indirect_selection_override else indirect_selection
         )
+    else:
+        target_project = dbt_project
 
     return DbtCliInvocationPartialParams(
         manifest=manifest,
         dagster_dbt_translator=dagster_dbt_translator,
         selection_args=selection_args,
         indirect_selection=indirect_selection,
-        dbt_project=dbt_project,
+        dbt_project=target_project,
     )
 
 
@@ -604,25 +649,6 @@ def default_owners_from_dbt_resource_props(
     return [owner] if isinstance(owner, str) else owner
 
 
-def default_freshness_policy_fn(
-    dbt_resource_props: Mapping[str, Any],
-) -> Optional[LegacyFreshnessPolicy]:
-    dagster_metadata = dbt_resource_props.get("meta", {}).get("dagster", {})
-    freshness_policy_config = dagster_metadata.get("freshness_policy", {})
-
-    freshness_policy = (
-        LegacyFreshnessPolicy(
-            maximum_lag_minutes=float(freshness_policy_config["maximum_lag_minutes"]),
-            cron_schedule=freshness_policy_config.get("cron_schedule"),
-            cron_schedule_timezone=freshness_policy_config.get("cron_schedule_timezone"),
-        )
-        if freshness_policy_config
-        else None
-    )
-
-    return freshness_policy
-
-
 def default_auto_materialize_policy_fn(
     dbt_resource_props: Mapping[str, Any],
 ) -> Optional[AutoMaterializePolicy]:

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt/cloud/asset_defs.py

@@ -33,7 +33,6 @@ from dagster_dbt.asset_utils import (
     default_asset_key_fn,
     default_auto_materialize_policy_fn,
     default_description_fn,
-    default_freshness_policy_fn,
     default_group_from_dbt_resource_props,
     get_node,
 )
@@ -326,10 +325,6 @@ class DbtCloudCacheableAssetsDefinition(CacheableAssetsDefinition):
             def get_group_name(cls, dbt_resource_props):  # pyright: ignore[reportIncompatibleMethodOverride]
                 return self._node_info_to_group_fn(dbt_resource_props)
 
-            @classmethod
-            def get_freshness_policy(cls, dbt_resource_props):  # pyright: ignore[reportIncompatibleMethodOverride]
-                return self._node_info_to_freshness_policy_fn(dbt_resource_props)
-
             @classmethod
             def get_auto_materialize_policy(cls, dbt_resource_props):  # pyright: ignore[reportIncompatibleMethodOverride]
                 return self._node_info_to_auto_materialize_policy_fn(dbt_resource_props)
@@ -375,11 +370,6 @@ class DbtCloudCacheableAssetsDefinition(CacheableAssetsDefinition):
                     for spec in specs
                 },
             },
-            legacy_freshness_policies_by_output_name={
-                spec.key.to_python_identifier(): spec.legacy_freshness_policy
-                for spec in specs
-                if spec.legacy_freshness_policy
-            },
             auto_materialize_policies_by_output_name={
                 spec.key.to_python_identifier(): spec.auto_materialize_policy
                 for spec in specs
@@ -546,9 +536,6 @@ def load_assets_from_dbt_cloud_job(
     node_info_to_group_fn: Callable[
         [Mapping[str, Any]], Optional[str]
     ] = default_group_from_dbt_resource_props,
-    node_info_to_freshness_policy_fn: Callable[
-        [Mapping[str, Any]], Optional[LegacyFreshnessPolicy]
-    ] = default_freshness_policy_fn,
     node_info_to_auto_materialize_policy_fn: Callable[
         [Mapping[str, Any]], Optional[AutoMaterializePolicy]
     ] = default_auto_materialize_policy_fn,
@@ -570,13 +557,6 @@ def load_assets_from_dbt_cloud_job(
             dbt source -> AssetKey([source_name, table_name])
         node_info_to_group_fn (Dict[str, Any] -> Optional[str]): A function that takes a
             dictionary of dbt node info and returns the group that this node should be assigned to.
-        node_info_to_freshness_policy_fn (Dict[str, Any] -> Optional[FreshnessPolicy]): A function
-            that takes a dictionary of dbt node info and optionally returns a FreshnessPolicy that
-            should be applied to this node. By default, freshness policies will be created from
-            config applied to dbt models, i.e.:
-            `dagster_freshness_policy={"maximum_lag_minutes": 60, "cron_schedule": "0 9 * * *"}`
-            will result in that model being assigned
-            `FreshnessPolicy(maximum_lag_minutes=60, cron_schedule="0 9 * * *")`
         node_info_to_auto_materialize_policy_fn (Dict[str, Any] -> Optional[AutoMaterializePolicy]):
             A function that takes a dictionary of dbt node info and optionally returns a AutoMaterializePolicy
             that should be applied to this node. By default, AutoMaterializePolicies will be created from
@@ -631,7 +611,7 @@ def load_assets_from_dbt_cloud_job(
         job_id=job_id,
         node_info_to_asset_key=node_info_to_asset_key,
         node_info_to_group_fn=node_info_to_group_fn,
-        node_info_to_freshness_policy_fn=node_info_to_freshness_policy_fn,
+        node_info_to_freshness_policy_fn=lambda _: None,
         node_info_to_auto_materialize_policy_fn=node_info_to_auto_materialize_policy_fn,
         partitions_def=partitions_def,
         partition_key_to_vars_fn=partition_key_to_vars_fn,

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt/components/dbt_project/component.py

@@ -1,10 +1,9 @@
 import itertools
 import json
-import shutil
 from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from contextvars import ContextVar
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from functools import cached_property
 from pathlib import Path
 from typing import Annotated, Any, Literal, Optional, Union
@@ -40,6 +39,12 @@ from dagster_dbt.dagster_dbt_translator import (
 from dagster_dbt.dbt_manifest import validate_manifest
 from dagster_dbt.dbt_manifest_asset_selection import DbtManifestAssetSelection
 from dagster_dbt.dbt_project import DbtProject
+from dagster_dbt.dbt_project_manager import (
+    DbtProjectArgsManager,
+    DbtProjectManager,
+    NoopDbtProjectManager,
+    RemoteGitDbtProjectManager,
+)
 from dagster_dbt.utils import ASSET_RESOURCE_TYPES
 
 
@@ -63,29 +68,18 @@ class DbtProjectArgs(dg.Resolvable):
     state_path: Optional[str] = None
 
 
-def resolve_dbt_project(context: ResolutionContext, model) -> DbtProject:
-    if isinstance(model, str):
-        return DbtProject(
-            context.resolve_source_relative_path(
-                context.resolve_value(model, as_type=str),
-            )
-        )
-
-    args = DbtProjectArgs.resolve_from_model(context, model)
-
-    kwargs = {}  # use optionally splatted kwargs to avoid redefining default value
-    if args.target_path:
-        kwargs["target_path"] = args.target_path
+def resolve_dbt_project(context: ResolutionContext, model) -> DbtProjectManager:
+    if isinstance(model, RemoteGitDbtProjectManager.model()):
+        return RemoteGitDbtProjectManager.resolve_from_model(context, model)
 
-    return DbtProject(
-        project_dir=context.resolve_source_relative_path(args.project_dir),
-        target=args.target,
-        profiles_dir=args.profiles_dir,
-        state_path=args.state_path,
-        packaged_project_dir=args.packaged_project_dir,
-        profile=args.profile,
-        **kwargs,
+    args = (
+        DbtProjectArgs(project_dir=context.resolve_value(model, as_type=str))
+        if isinstance(model, str)
+        else DbtProjectArgs.resolve_from_model(context, model)
     )
+    # resolve the project_dir relative to where this component is defined
+    args = replace(args, project_dir=context.resolve_source_relative_path(args.project_dir))
+    return DbtProjectArgsManager(args)
 
 
 DbtMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
@@ -113,13 +107,25 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
 
     Scaffold a DbtProjectComponent definition by running `dg scaffold defs dagster_dbt.DbtProjectComponent --project-path path/to/your/existing/dbt_project`
     in the Dagster project directory.
+
+    Example:
+
+        .. code-block:: yaml
+
+            # defs.yaml
+
+            type: dagster_dbt.DbtProjectComponent
+            attributes:
+              project: "{{ project_root }}/path/to/dbt_project"
+              cli_args:
+                - build
     """
 
     project: Annotated[
-        DbtProject,
+        Union[DbtProject, DbtProjectManager],
         Resolver(
             resolve_dbt_project,
-            model_field_type=Union[str, DbtProjectArgs.model()],
+            model_field_type=Union[str, DbtProjectArgs.model(), RemoteGitDbtProjectManager.model()],
             description="The path to the dbt project or a mapping defining a DbtProject",
             examples=[
                 "{{ project_root }}/path/to/dbt_project",
@@ -212,8 +218,8 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
     @property
     def defs_state_config(self) -> DefsStateConfig:
         return DefsStateConfig(
-            key=f"{self.__class__.__name__}[{self.project.name}]",
-            type=DefsStateManagementType.LOCAL_FILESYSTEM,
+            key=f"{self.__class__.__name__}[{self._project_manager.defs_state_discriminator}]",
+            management_type=DefsStateManagementType.LOCAL_FILESYSTEM,
             refresh_if_dev=self.prepare_if_dev,
         )
 
@@ -225,9 +231,66 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
     def _base_translator(self) -> "DagsterDbtTranslator":
         return DagsterDbtTranslator(self.translation_settings)
 
+    def get_resource_props(self, manifest: Mapping[str, Any], unique_id: str) -> Mapping[str, Any]:
+        """Given a parsed manifest and a dbt unique_id, returns the dictionary of properties
+        for the corresponding dbt resource (e.g. model, seed, snapshot, source) as defined
+        in your dbt project. This can be used as a convenience method when overriding the
+        `get_asset_spec` method.
+
+        Args:
+            manifest (Mapping[str, Any]): The parsed manifest of the dbt project.
+            unique_id (str): The unique_id of the dbt resource.
+
+        Returns:
+            Mapping[str, Any]: The dictionary of properties for the corresponding dbt resource.
+
+        Examples:
+            .. code-block:: python
+
+                class CustomDbtProjectComponent(DbtProjectComponent):
+
+                    def get_asset_spec(self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]) -> dg.AssetSpec:
+                        base_spec = super().get_asset_spec(manifest, unique_id, project)
+                        resource_props = self.get_resource_props(manifest, unique_id)
+                        if resource_props["meta"].get("use_custom_group"):
+                            return base_spec.replace_attributes(group_name="custom_group")
+                        else:
+                            return base_spec
+        """
+        return get_node(manifest, unique_id)
+
+    @public
     def get_asset_spec(
         self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]
     ) -> dg.AssetSpec:
+        """Generates an AssetSpec for a given dbt node.
+
+        This method can be overridden in a subclass to customize how dbt nodes are converted
+        to Dagster asset specs. By default, it delegates to the configured DagsterDbtTranslator.
+
+        Args:
+            manifest: The dbt manifest dictionary containing information about all dbt nodes
+            unique_id: The unique identifier for the dbt node (e.g., "model.my_project.my_model")
+            project: The DbtProject object, if available
+
+        Returns:
+            An AssetSpec that represents the dbt node as a Dagster asset
+
+        Example:
+            Override this method to add custom tags to all dbt models:
+
+            .. code-block:: python
+
+                from dagster_dbt import DbtProjectComponent
+                import dagster as dg
+
+                class CustomDbtProjectComponent(DbtProjectComponent):
+                    def get_asset_spec(self, manifest, unique_id, project):
+                        base_spec = super().get_asset_spec(manifest, unique_id, project)
+                        return base_spec.replace_attributes(
+                            tags={**base_spec.tags, "custom_tag": "my_value"}
+                        )
+        """
         return self._base_translator.get_asset_spec(manifest, unique_id, project)
 
     def get_asset_check_spec(
@@ -240,37 +303,40 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
         return self._base_translator.get_asset_check_spec(asset_spec, manifest, unique_id, project)
 
     @cached_property
-    def cli_resource(self):
-        return DbtCliResource(self.project)
+    def _project_manager(self) -> DbtProjectManager:
+        if isinstance(self.project, DbtProject):
+            return NoopDbtProjectManager(self.project)
+        else:
+            return self.project
+
+    @cached_property
+    def dbt_project(self) -> DbtProject:
+        return self._project_manager.get_project(None)
 
     def get_asset_selection(
         self, select: str, exclude: str = DBT_DEFAULT_EXCLUDE
     ) -> DbtManifestAssetSelection:
         return DbtManifestAssetSelection.build(
-            manifest=self.project.manifest_path,
+            manifest=self.dbt_project.manifest_path,
             dagster_dbt_translator=self.translator,
             select=select,
             exclude=exclude,
         )
 
     def write_state_to_path(self, state_path: Path) -> None:
-        # compile the manifest
-        self.project.preparer.prepare(self.project)
-        # move the manifest to the correct path
-        shutil.copyfile(self.project.manifest_path, state_path)
+        self._project_manager.prepare(state_path)
 
     def build_defs_from_state(
         self, context: dg.ComponentLoadContext, state_path: Optional[Path]
     ) -> dg.Definitions:
-        if state_path is not None:
-            shutil.copyfile(state_path, self.project.manifest_path)
+        project = self._project_manager.get_project(state_path)
 
         res_ctx = context.resolution_context
 
         @dbt_assets(
-            manifest=self.project.manifest_path,
-            project=self.project,
-            name=self.op.name if self.op else self.project.name,
+            manifest=project.manifest_path,
+            project=project,
+            name=self.op.name if self.op else project.name,
             op_tags=self.op.tags if self.op else None,
             dagster_dbt_translator=self.translator,
             select=self.select,
@@ -279,7 +345,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
         )
         def _fn(context: dg.AssetExecutionContext):
             with _set_resolution_context(res_ctx):
-                yield from self.execute(context=context, dbt=self.cli_resource)
+                yield from self.execute(context=context, dbt=DbtCliResource(project))
 
         return dg.Definitions(assets=[_fn])
 
@@ -336,12 +402,39 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
             iterator = iterator.fetch_row_counts()
         return iterator
 
+    @public
     def execute(self, context: dg.AssetExecutionContext, dbt: DbtCliResource) -> Iterator:
+        """Executes the dbt command for the selected assets.
+
+        This method can be overridden in a subclass to customize the execution behavior,
+        such as adding custom logging, modifying CLI arguments, or handling events differently.
+
+        Args:
+            context: The asset execution context provided by Dagster
+            dbt: The DbtCliResource used to execute dbt commands
+
+        Yields:
+            Events from the dbt CLI execution (e.g., AssetMaterialization, AssetObservation)
+
+        Example:
+            Override this method to add custom logging before and after execution:
+
+            .. code-block:: python
+
+                from dagster_dbt import DbtProjectComponent
+                import dagster as dg
+
+                class CustomDbtProjectComponent(DbtProjectComponent):
+                    def execute(self, context, dbt):
+                        context.log.info("Starting custom dbt execution")
+                        yield from super().execute(context, dbt)
+                        context.log.info("Completed custom dbt execution")
+        """
         yield from self._get_dbt_event_iterator(context, dbt)
 
     @cached_property
     def _validated_manifest(self):
-        return validate_manifest(self.project.manifest_path)
+        return validate_manifest(self.dbt_project.manifest_path)
 
     @cached_property
     def _validated_translator(self):
@@ -364,7 +457,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
         return dagster_dbt_translator.get_asset_spec(
             manifest,
             next(iter(matching_model_ids)),
-            self.project,
+            self.dbt_project,
         ).key
 
 
@@ -396,4 +489,4 @@ def get_projects_from_dbt_component(components: Path) -> list[DbtProject]:
         of_type=DbtProjectComponent
     )
 
-    return [component.project for component in project_components]
+    return [component.dbt_project for component in project_components]

dagster_dbt-0.28.1/dagster_dbt/components/dbt_project/scaffolder.py

@@ -0,0 +1,65 @@
+import os
+from pathlib import Path
+from typing import Any, Optional
+
+import dagster._check as check
+from dagster._core.errors import DagsterInvalidInvocationError
+from dagster.components.component.component_scaffolder import Scaffolder
+from dagster.components.component_scaffolding import scaffold_component
+from dagster.components.scaffold.scaffold import ScaffoldRequest
+from pydantic import BaseModel, Field
+
+
+class DbtScaffoldParams(BaseModel):
+    init: bool = Field(default=False)
+    project_path: Optional[str] = None
+    git_url: Optional[str] = None
+
+
+class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
+    @classmethod
+    def get_scaffold_params(cls) -> type[DbtScaffoldParams]:
+        return DbtScaffoldParams
+
+    def _init_dbt_project(self) -> None:
+        try:
+            from dbt.cli.main import dbtRunner
+        except ImportError:
+            raise DagsterInvalidInvocationError(
+                "dbt-core is not installed. Please install dbt to scaffold this component."
+            )
+
+        dbtRunner().invoke(["init"])
+
+    def _git_url_params(self, git_url: str, project_path: Optional[str]) -> dict[str, Any]:
+        repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
+        return {"project": {"repo_url": git_url, **repo_relative_path}}
+
+    def scaffold(self, request: ScaffoldRequest[DbtScaffoldParams]) -> None:
+        project_root = request.project_root or Path(os.getcwd())
+        check.param_invariant(
+            not (request.params.init and (request.params.git_url or request.params.project_path)),
+            "init and git_url/project_path cannot be set at the same time.",
+        )
+        if request.params.init:
+            self._init_dbt_project()
+            subpaths = [
+                path for path in project_root.iterdir() if path.is_dir() and path.name != "logs"
+            ]
+            check.invariant(len(subpaths) == 1, "Expected exactly one subpath to be created.")
+            # this path should be relative to this directory
+            params = {"project": subpaths[0].name}
+        elif request.params.git_url is not None:
+            # project_path is the path to the dbt project within the git repository
+            project_path = request.params.project_path
+            repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
+            params = {"project": {"repo_url": request.params.git_url, **repo_relative_path}}
+        elif request.params.project_path is not None:
+            # project_path represents a local directory
+            project_root_tmpl = "{{ project_root }}"
+            rel_path = os.path.relpath(request.params.project_path, start=project_root)
+            path_str = f"{project_root_tmpl}/{rel_path}"
+            params = {"project": path_str}
+        else:
+            params = {"project": None}
+        scaffold_component(request, params)

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt/core/resource.py

@@ -428,7 +428,7 @@ class DbtCliResource(ConfigurableResource):
             adapter = get_adapter(config)
             manifest = ManifestLoader.load_macros(
                 config,
-                adapter.connections.set_query_header,  # type: ignore
+                adapter.connections.set_query_header,
                 base_macros_only=True,
             )
             adapter.set_macro_resolver(manifest)
@@ -627,8 +627,12 @@ class DbtCliResource(ConfigurableResource):
         dagster_dbt_translator = updated_params.dagster_dbt_translator
         selection_args = updated_params.selection_args
         indirect_selection = updated_params.indirect_selection
-
         target_path = target_path or self._get_unique_target_path(context=context)
+        project_dir = Path(
+            updated_params.dbt_project.project_dir
+            if updated_params.dbt_project
+            else self.project_dir
+        )
         env = {
             # Allow IO streaming when running in Windows.
             # Also, allow it to be overriden by the current environment.
@@ -659,7 +663,7 @@ class DbtCliResource(ConfigurableResource):
             **({"DBT_PROFILES_DIR": self.profiles_dir} if self.profiles_dir else {}),
             # The DBT_PROJECT_DIR environment variable is set to the path containing the dbt project
             # See https://docs.getdbt.com/reference/dbt_project.yml for more information.
-            **({"DBT_PROJECT_DIR": self.project_dir} if self.project_dir else {}),
+            "DBT_PROJECT_DIR": str(project_dir),
         }
 
         # set dbt indirect selection if needed to execute specific dbt tests due to asset check
@@ -683,14 +687,13 @@ class DbtCliResource(ConfigurableResource):
             *profile_args,
             *selection_args,
         ]
-        project_dir = Path(self.project_dir)
 
         if not target_path.is_absolute():
             target_path = project_dir.joinpath(target_path)
 
         # run dbt --version to get the dbt core version
         adapter: Optional[BaseAdapter] = None
-        with pushd(self.project_dir):
+        with pushd(str(project_dir)):
             # we do not need to initialize the adapter if we are using the fusion engine
             if self._cli_version.major < 2:
                 try:

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt/dagster_dbt_translator.py

@@ -12,7 +12,6 @@ from dagster import (
     AutoMaterializePolicy,
     AutomationCondition,
     DagsterInvalidDefinitionError,
-    LegacyFreshnessPolicy,
     PartitionMapping,
 )
 from dagster._annotations import beta, public
@@ -36,7 +35,6 @@ from dagster_dbt.asset_utils import (
     default_auto_materialize_policy_fn,
     default_code_version_fn,
     default_description_fn,
-    default_freshness_policy_fn,
     default_group_from_dbt_resource_props,
     default_metadata_from_dbt_resource_props,
     default_owners_from_dbt_resource_props,
@@ -96,6 +94,34 @@ class DagsterDbtTranslator:
 
         return self._settings
 
+    def get_resource_props(self, manifest: Mapping[str, Any], unique_id: str) -> Mapping[str, Any]:
+        """Given a parsed manifest and a dbt unique_id, returns the dictionary of properties
+        for the corresponding dbt resource (e.g. model, seed, snapshot, source) as defined
+        in your dbt project. This can be used as a convenience method when overriding the
+        `get_asset_spec` method.
+
+        Args:
+            manifest (Mapping[str, Any]): The parsed manifest of the dbt project.
+            unique_id (str): The unique_id of the dbt resource.
+
+        Returns:
+            Mapping[str, Any]: The dictionary of properties for the corresponding dbt resource.
+
+        Examples:
+            .. code-block:: python
+
+                class CustomDagsterDbtTranslator(DagsterDbtTranslator):
+
+                    def get_asset_spec(self, manifest: Mapping[str, Any], unique_id: str, project: Optional[DbtProject]) -> dg.AssetSpec:
+                        base_spec = super().get_asset_spec(manifest, unique_id, project)
+                        resource_props = self.get_resource_props(manifest, unique_id)
+                        if resource_props["meta"].get("use_custom_group"):
+                            return base_spec.replace_attributes(group_name="custom_group")
+                        else:
+                            return base_spec
+        """
+        return get_node(manifest, unique_id)
+
     def get_asset_spec(
         self,
         manifest: Mapping[str, Any],
@@ -115,7 +141,7 @@ class DagsterDbtTranslator:
             return self._resolved_specs[memo_id]
 
         group_props = {group["name"]: group for group in manifest.get("groups", {}).values()}
-        resource_props = get_node(manifest, unique_id)
+        resource_props = self.get_resource_props(manifest, unique_id)
 
         # calculate the dependencies for the asset
         upstream_ids = get_upstream_unique_ids(manifest, resource_props)
@@ -123,7 +149,7 @@ class DagsterDbtTranslator:
             AssetDep(
                 asset=self.get_asset_spec(manifest, upstream_id, project).key,
                 partition_mapping=self.get_partition_mapping(
-                    resource_props, get_node(manifest, upstream_id)
+                    resource_props, self.get_resource_props(manifest, upstream_id)
                 ),
             )
             for upstream_id in upstream_ids
@@ -158,7 +184,6 @@ class DagsterDbtTranslator:
             group_name=self.get_group_name(resource_props),
             code_version=self.get_code_version(resource_props),
             automation_condition=self.get_automation_condition(resource_props),
-            legacy_freshness_policy=self.get_freshness_policy(resource_props),
             owners=self.get_owners(owners_resource_props),
             tags=self.get_tags(resource_props),
             kinds={"dbt", manifest.get("metadata", {}).get("adapter_type", "dbt")},
@@ -493,60 +518,6 @@ class DagsterDbtTranslator:
         """
         return default_owners_from_dbt_resource_props(dbt_resource_props)
 
-    @public
-    @beta(emit_runtime_warning=False)
-    def get_freshness_policy(
-        self, dbt_resource_props: Mapping[str, Any]
-    ) -> Optional[LegacyFreshnessPolicy]:
-        """A function that takes a dictionary representing properties of a dbt resource, and
-        returns the Dagster :py:class:`dagster.FreshnessPolicy` for that resource.
-
-        Note that a dbt resource is unrelated to Dagster's resource concept, and simply represents
-        a model, seed, snapshot or source in a given dbt project. You can learn more about dbt
-        resources and the properties available in this dictionary here:
-        https://docs.getdbt.com/reference/artifacts/manifest-json#resource-details
-
-        This method can be overridden to provide a custom freshness policy for a dbt resource.
-
-        Args:
-            dbt_resource_props (Mapping[str, Any]): A dictionary representing the dbt resource.
-
-        Returns:
-            Optional[FreshnessPolicy]: A Dagster freshness policy.
-
-        Examples:
-            Set a custom freshness policy for all dbt resources:
-
-            .. code-block:: python
-
-                from typing import Any, Mapping
-
-                from dagster_dbt import DagsterDbtTranslator
-
-
-                class CustomDagsterDbtTranslator(DagsterDbtTranslator):
-                    def get_freshness_policy(self, dbt_resource_props: Mapping[str, Any]) -> Optional[FreshnessPolicy]:
-                        return FreshnessPolicy(maximum_lag_minutes=60)
-
-            Set a custom freshness policy for dbt resources with a specific tag:
-
-            .. code-block:: python
-
-                from typing import Any, Mapping
-
-                from dagster_dbt import DagsterDbtTranslator
-
-
-                class CustomDagsterDbtTranslator(DagsterDbtTranslator):
-                    def get_freshness_policy(self, dbt_resource_props: Mapping[str, Any]) -> Optional[FreshnessPolicy]:
-                        freshness_policy = None
-                        if "my_custom_tag" in dbt_resource_props.get("tags", []):
-                            freshness_policy = FreshnessPolicy(maximum_lag_minutes=60)
-
-                        return freshness_policy
-        """
-        return default_freshness_policy_fn(dbt_resource_props)
-
     @public
     @beta(emit_runtime_warning=False)
     def get_auto_materialize_policy(

dagster_dbt-0.28.1/dagster_dbt/dbt_project_manager.py

@@ -0,0 +1,170 @@
+import os
+import shutil
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated, Optional
+from urllib.parse import quote, urlparse, urlunparse
+
+from dagster._core.errors import DagsterInvalidDefinitionError
+from dagster.components.resolved.base import Resolvable
+from dagster.components.resolved.model import Resolver
+from git import Repo
+
+from dagster_dbt.dbt_project import DbtProject
+
+if TYPE_CHECKING:
+    from dagster_dbt.components.dbt_project.component import DbtProjectArgs
+
+
+class DbtProjectManager(ABC):
+    """Helper class that wraps a dbt project that may or may not be available on local disk at the time
+    it is instantiated. Provides methods for syncing the project to a local path and for instantiating the
+    final DbtProject object when ready.
+    """
+
+    @property
+    @abstractmethod
+    def defs_state_discriminator(self) -> str: ...
+
+    @abstractmethod
+    def sync(self, state_path: Path) -> None: ...
+
+    def _local_project_dir(self, state_path: Path) -> Path:
+        return state_path.parent / "project"
+
+    def prepare(self, state_path: Path) -> None:
+        """Syncs the project to the given local path and ensures the manifest is generated
+        and the dependencies are installed.
+        """
+        # ensure local dir is empty
+        local_dir = self._local_project_dir(state_path)
+        shutil.rmtree(local_dir, ignore_errors=True)
+        local_dir.mkdir()
+
+        # ensure project exists in the dir and is compiled
+        self.sync(state_path)
+
+        # indicate that project has been prepared
+        state_path.touch()
+
+    @abstractmethod
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject": ...
+
+
+@dataclass
+class NoopDbtProjectManager(DbtProjectManager):
+    """Wraps a DbtProject that has already been fully instantiated. Used for cases where a
+    user directly provides a DbtProject to the DbtProjectComponent.
+    """
+
+    project: "DbtProject"
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return self.project.name
+
+    def sync(self, state_path: Path) -> None:
+        pass
+
+    def get_project(self, state_path: Optional[Path]) -> DbtProject:
+        return self.project
+
+
+@dataclass
+class DbtProjectArgsManager(DbtProjectManager):
+    """Wraps DbtProjectArgs provided to the DbtProjectComponent. Avoids instantiating the DbtProject object
+    immediately as this would cause errors in cases where the project_dir has not yet been synced.
+    """
+
+    args: "DbtProjectArgs"
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return Path(self.args.project_dir).stem
+
+    def sync(self, state_path: Path) -> None:
+        # we prepare the project in the original project directory rather than the new one
+        # so that code that does not have access to the local state path can still access
+        # the manifest.json file.
+        project = self.get_project(None)
+        project.preparer.prepare(project)
+        shutil.copytree(
+            self.args.project_dir, self._local_project_dir(state_path), dirs_exist_ok=True
+        )
+
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject":
+        kwargs = asdict(self.args)
+
+        project_dir = self._local_project_dir(state_path) if state_path else self.args.project_dir
+        return DbtProject(
+            project_dir=project_dir,
+            # allow default values on DbtProject to take precedence
+            **{k: v for k, v in kwargs.items() if v is not None and k != "project_dir"},
+        )
+
+
+@dataclass
+class RemoteGitDbtProjectManager(DbtProjectManager, Resolvable):
+    """Wraps a remote git repository containing a dbt project."""
+
+    repo_url: str
+    repo_relative_path: Annotated[
+        str,
+        Resolver.default(
+            description="The relative path to the dbt project within the repository.",
+            examples=["dbt/my_dbt_project"],
+        ),
+    ] = "."
+    token: Annotated[
+        Optional[str],
+        Resolver.default(
+            description="Token for authenticating to the provided repository.",
+            examples=["'{{ env.GITHUB_TOKEN }}'"],
+        ),
+    ] = None
+    profile: Annotated[
+        Optional[str],
+        Resolver.default(description="The profile to use for the dbt project."),
+    ] = None
+    target: Annotated[
+        Optional[str],
+        Resolver.default(
+            description="The target to use for the dbt project.",
+            examples=["dev", "prod"],
+        ),
+    ] = None
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return self.repo_url
+
+    def _get_clone_url(self) -> str:
+        # in github action environments, the token will be available automatically as an env var
+        raw_token = self.token or os.getenv("GITHUB_TOKEN")
+        if raw_token is None:
+            return self.repo_url
+        else:
+            # insert the token into the url
+            token = quote(raw_token, safe="")
+            parts = urlparse(self.repo_url)
+            return urlunparse(parts._replace(netloc=f"{token}@{parts.netloc}"))
+
+    def sync(self, state_path: Path) -> None:
+        Repo.clone_from(self._get_clone_url(), self._local_project_dir(state_path), depth=1)
+        project = self.get_project(state_path)
+        project.preparer.prepare(project)
+
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject":
+        if state_path is None:
+            raise DagsterInvalidDefinitionError(
+                "Attempted to `get_project()` on a `RemoteGitDbtProjectWrapper` without a path. "
+                "This can happen when calling unsupported methods on the `DbtProjectComponent`."
+            )
+
+        local_dir = self._local_project_dir(state_path)
+        return DbtProject(
+            project_dir=local_dir / self.repo_relative_path,
+            profile=self.profile,
+            target=self.target,
+        )

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt/freshness_builder.py

@@ -6,7 +6,6 @@ from dagster import (
     AssetsDefinition,
     _check as check,
 )
-from dagster._annotations import beta
 from dagster._core.definitions.asset_checks.asset_check_factories.freshness_checks.last_update import (
     build_last_update_freshness_checks,
 )
@@ -27,13 +26,18 @@ from dagster._core.errors import DagsterInvariantViolationError
 if TYPE_CHECKING:
     from dagster import AssetKey
 
+from dagster._symbol_annotations.lifecycle import superseded
+
 from dagster_dbt.asset_utils import (
     get_asset_keys_to_resource_props,
     get_manifest_and_translator_from_dbt_assets,
 )
 
 
-@beta
+@superseded(
+    additional_warn_text="Create `FreshnessPolicy` objects for your dbt models by overriding `get_asset_spec` "
+    "in your `DagsterDbtTranslator`, or by updating the `translation` configuration of your `DbtProjectComponent` instead."
+)
 def build_freshness_checks_from_dbt_assets(
     *,
     dbt_assets: Sequence[AssetsDefinition],

dagster_dbt-0.28.1/dagster_dbt/version.py

@@ -0,0 +1 @@
+__version__ = "0.28.1"

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1/dagster_dbt.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.27.16
+Version: 0.28.1
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
@@ -15,8 +15,9 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.11.16
+Requires-Dist: dagster==1.12.1
 Requires-Dist: dbt-core<1.11,>=1.7
+Requires-Dist: gitpython
 Requires-Dist: Jinja2
 Requires-Dist: networkx
 Requires-Dist: orjson

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ dagster_dbt/dbt_core_version.py
 dagster_dbt/dbt_manifest.py
 dagster_dbt/dbt_manifest_asset_selection.py
 dagster_dbt/dbt_project.py
+dagster_dbt/dbt_project_manager.py
 dagster_dbt/dbt_version.py
 dagster_dbt/errors.py
 dagster_dbt/freshness_builder.py

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/dagster_dbt.egg-info/requires.txt

@@ -1,5 +1,6 @@
-dagster==1.11.16
+dagster==1.12.1
 dbt-core<1.11,>=1.7
+gitpython
 Jinja2
 networkx
 orjson

{dagster_dbt-0.27.16 → dagster_dbt-0.28.1}/setup.py

@@ -40,9 +40,10 @@ setup(
     include_package_data=True,
     python_requires=">=3.9,<3.14",
     install_requires=[
-        "dagster==1.11.16",
+        "dagster==1.12.1",
         # Follow the version support constraints for dbt Core: https://docs.getdbt.com/docs/dbt-versions/core
         f"dbt-core>=1.7,<{DBT_CORE_VERSION_UPPER_BOUND}",
+        "gitpython",
         "Jinja2",
         "networkx",
         "orjson",

dagster_dbt-0.27.16/dagster_dbt/components/dbt_project/scaffolder.py

@@ -1,50 +0,0 @@
-import os
-from pathlib import Path
-from typing import Optional
-
-import dagster._check as check
-from dagster._core.errors import DagsterInvalidInvocationError
-from dagster.components.component.component_scaffolder import Scaffolder
-from dagster.components.component_scaffolding import scaffold_component
-from dagster.components.scaffold.scaffold import ScaffoldRequest
-from pydantic import BaseModel, Field
-
-
-class DbtScaffoldParams(BaseModel):
-    init: bool = Field(default=False)
-    project_path: Optional[str] = None
-
-
-class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
-    @classmethod
-    def get_scaffold_params(cls) -> type[DbtScaffoldParams]:
-        return DbtScaffoldParams
-
-    def scaffold(self, request: ScaffoldRequest[DbtScaffoldParams]) -> None:
-        project_root = request.project_root or os.getcwd()
-        if request.params.project_path:
-            project_root_tmpl = "{{ project_root }}"
-            rel_path = os.path.relpath(request.params.project_path, start=project_root)
-            path_str = f"{project_root_tmpl}/{rel_path}"
-
-        elif request.params.init:
-            try:
-                from dbt.cli.main import dbtRunner
-            except ImportError:
-                raise DagsterInvalidInvocationError(
-                    "dbt-core is not installed. Please install dbt to scaffold this component."
-                )
-
-            dbtRunner().invoke(["init"])
-            subpaths = [
-                path
-                for path in Path(project_root).iterdir()
-                if path.is_dir() and path.name != "logs"
-            ]
-            check.invariant(len(subpaths) == 1, "Expected exactly one subpath to be created.")
-            # this path should be relative to this directory
-            path_str = subpaths[0].name
-        else:
-            path_str = None
-
-        scaffold_component(request, {"project": path_str})

dagster_dbt-0.27.16/dagster_dbt/version.py

@@ -1 +0,0 @@
-__version__ = "0.27.16"