dagster-dbt 0.28.0__py3-none-any.whl → 0.28.2__py3-none-any.whl

dagster_dbt/asset_decorator.py

@@ -1,5 +1,5 @@
-from collections.abc import Mapping
-from typing import Any, Callable, Optional
+from collections.abc import Callable, Mapping
+from typing import Any, Optional
 
 from dagster import (
     AssetsDefinition,

dagster_dbt/asset_utils.py

@@ -1,11 +1,14 @@
 import hashlib
 import os
+import shutil
+import tempfile
 import textwrap
 from collections import defaultdict
 from collections.abc import Iterable, Mapping, Sequence
 from pathlib import Path
 from typing import TYPE_CHECKING, AbstractSet, Annotated, Any, Final, Optional, Union  # noqa: UP035
 
+import yaml
 from dagster import (
     AssetCheckKey,
     AssetCheckSpec,
@@ -32,6 +35,7 @@ from dagster._core.definitions.metadata import TableMetadataSet
 from dagster._core.errors import DagsterInvalidPropertyError
 from dagster._core.types.dagster_type import Nothing
 from dagster._record import ImportFrom, record
+from dagster_shared.record import replace
 
 from dagster_dbt.dbt_project import DbtProject
 from dagster_dbt.metadata_set import DbtMetadataSet
@@ -55,6 +59,10 @@ DBT_DEFAULT_SELECTOR = ""
 DBT_INDIRECT_SELECTION_ENV: Final[str] = "DBT_INDIRECT_SELECTION"
 DBT_EMPTY_INDIRECT_SELECTION: Final[str] = "empty"
 
+# Threshold for switching to selector file to avoid CLI argument length limits
+# https://github.com/dagster-io/dagster/issues/16997
+_SELECTION_ARGS_THRESHOLD: Final[int] = 200
+
 DUPLICATE_ASSET_KEY_ERROR_MESSAGE = (
     "The following dbt resources are configured with identical Dagster asset keys."
     " Please ensure that each dbt resource generates a unique Dagster asset key."
@@ -443,6 +451,10 @@ def get_updated_cli_invocation_params_for_context(
             [assets_def]
         )
 
+        # Get project_dir from dbt_project if available
+        project_dir = Path(dbt_project.project_dir) if dbt_project else None
+        target_project = dbt_project
+
         selection_args, indirect_selection_override = get_subset_selection_for_context(
             context=context,
             manifest=manifest,
@@ -452,17 +464,49 @@ def get_updated_cli_invocation_params_for_context(
             dagster_dbt_translator=dagster_dbt_translator,
             current_dbt_indirect_selection_env=indirect_selection,
         )
+        if (
+            selection_args[0] == "--select"
+            and project_dir
+            and len(resources := selection_args[1].split(" ")) > _SELECTION_ARGS_THRESHOLD
+        ):
+            temp_project_dir = tempfile.mkdtemp()
+            shutil.copytree(project_dir, temp_project_dir, dirs_exist_ok=True)
+            selectors_path = Path(temp_project_dir) / "selectors.yml"
+
+            # Delete any existing selectors, we need to create our own
+            if selectors_path.exists():
+                selectors_path.unlink()
+
+            selector_name = f"dagster_run_{context.run_id}"
+            temp_selectors = {
+                "selectors": [
+                    {
+                        "name": selector_name,
+                        "definition": {"union": list(resources)},
+                    }
+                ]
+            }
+            selectors_path.write_text(yaml.safe_dump(temp_selectors))
+            logger.info(
+                f"DBT selection of {len(resources)} resources exceeds threshold of {_SELECTION_ARGS_THRESHOLD}. "
+                "This may exceed system argument length limits. "
+                f"Executing materialization against temporary copy of DBT project at {temp_project_dir} with ephemeral selector."
+            )
+            selection_args = ["--selector", selector_name]
+            target_project = replace(dbt_project, project_dir=temp_project_dir)
 
         indirect_selection = (
             indirect_selection_override if indirect_selection_override else indirect_selection
         )
+    else:
+        target_project = dbt_project
 
     return DbtCliInvocationPartialParams(
         manifest=manifest,
         dagster_dbt_translator=dagster_dbt_translator,
         selection_args=selection_args,
         indirect_selection=indirect_selection,
-        dbt_project=dbt_project,
+        dbt_project=target_project,
     )
 
 

dagster_dbt/cloud/asset_defs.py

@@ -1,9 +1,9 @@
 import json
 import shlex
 from argparse import ArgumentParser, Namespace
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from contextlib import suppress
-from typing import Any, Callable, Optional, Union, cast
+from typing import Any, Optional, Union, cast
 
 import dagster._check as check
 from dagster import (

dagster_dbt/cloud/utils.py

@@ -1,5 +1,5 @@
-from collections.abc import Iterator, Mapping, Sequence
-from typing import Any, Callable, Optional, Union
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from typing import Any, Optional, Union
 
 import dateutil
 from dagster import (

dagster_dbt/cloud_v2/asset_decorator.py

@@ -1,4 +1,5 @@
-from typing import Any, Callable, Optional
+from collections.abc import Callable
+from typing import Any, Optional
 
 from dagster import AssetsDefinition, multi_asset
 from dagster._annotations import public

dagster_dbt/compat.py

@@ -1,9 +1,8 @@
 import logging
 from enum import Enum
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeAlias
 
 from packaging import version
-from typing_extensions import TypeAlias
 
 # it's unclear exactly which dbt import adds a handler to the root logger, but something certainly does!
 # on this line, we keep track of the set of handlers that are on the root logger BEFORE any dbt imports

dagster_dbt/components/dbt_project/component.py

@@ -1,13 +1,12 @@
 import itertools
 import json
-import shutil
 from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from contextvars import ContextVar
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from functools import cached_property
 from pathlib import Path
-from typing import Annotated, Any, Literal, Optional, Union
+from typing import Annotated, Any, Literal, Optional, TypeAlias, Union
 
 import dagster as dg
 from dagster._annotations import public
@@ -26,10 +25,17 @@ from dagster.components.utils.translation import (
 )
 from dagster_shared import check
 from dagster_shared.serdes.objects.models.defs_state_info import DefsStateManagementType
-from typing_extensions import TypeAlias
 
-from dagster_dbt.asset_decorator import dbt_assets
-from dagster_dbt.asset_utils import DBT_DEFAULT_EXCLUDE, DBT_DEFAULT_SELECT, get_node
+from dagster_dbt.asset_utils import (
+    DAGSTER_DBT_EXCLUDE_METADATA_KEY,
+    DAGSTER_DBT_SELECT_METADATA_KEY,
+    DAGSTER_DBT_SELECTOR_METADATA_KEY,
+    DBT_DEFAULT_EXCLUDE,
+    DBT_DEFAULT_SELECT,
+    DBT_DEFAULT_SELECTOR,
+    build_dbt_specs,
+    get_node,
+)
 from dagster_dbt.components.dbt_project.scaffolder import DbtProjectComponentScaffolder
 from dagster_dbt.core.resource import DbtCliResource
 from dagster_dbt.dagster_dbt_translator import (
@@ -40,6 +46,12 @@ from dagster_dbt.dagster_dbt_translator import (
 from dagster_dbt.dbt_manifest import validate_manifest
 from dagster_dbt.dbt_manifest_asset_selection import DbtManifestAssetSelection
 from dagster_dbt.dbt_project import DbtProject
+from dagster_dbt.dbt_project_manager import (
+    DbtProjectArgsManager,
+    DbtProjectManager,
+    NoopDbtProjectManager,
+    RemoteGitDbtProjectManager,
+)
 from dagster_dbt.utils import ASSET_RESOURCE_TYPES
 
 
@@ -63,29 +75,18 @@ class DbtProjectArgs(dg.Resolvable):
     state_path: Optional[str] = None
 
 
-def resolve_dbt_project(context: ResolutionContext, model) -> DbtProject:
-    if isinstance(model, str):
-        return DbtProject(
-            context.resolve_source_relative_path(
-                context.resolve_value(model, as_type=str),
-            )
-        )
-
-    args = DbtProjectArgs.resolve_from_model(context, model)
-
-    kwargs = {}  # use optionally splatted kwargs to avoid redefining default value
-    if args.target_path:
-        kwargs["target_path"] = args.target_path
+def resolve_dbt_project(context: ResolutionContext, model) -> DbtProjectManager:
+    if isinstance(model, RemoteGitDbtProjectManager.model()):
+        return RemoteGitDbtProjectManager.resolve_from_model(context, model)
 
-    return DbtProject(
-        project_dir=context.resolve_source_relative_path(args.project_dir),
-        target=args.target,
-        profiles_dir=args.profiles_dir,
-        state_path=args.state_path,
-        packaged_project_dir=args.packaged_project_dir,
-        profile=args.profile,
-        **kwargs,
+    args = (
+        DbtProjectArgs(project_dir=context.resolve_value(model, as_type=str))
+        if isinstance(model, str)
+        else DbtProjectArgs.resolve_from_model(context, model)
     )
+    # resolve the project_dir relative to where this component is defined
+    args = replace(args, project_dir=context.resolve_source_relative_path(args.project_dir))
+    return DbtProjectArgsManager(args)
 
 
 DbtMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
@@ -128,10 +129,10 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
     """
 
     project: Annotated[
-        DbtProject,
+        Union[DbtProject, DbtProjectManager],
         Resolver(
             resolve_dbt_project,
-            model_field_type=Union[str, DbtProjectArgs.model()],
+            model_field_type=Union[str, DbtProjectArgs.model(), RemoteGitDbtProjectManager.model()],
             description="The path to the dbt project or a mapping defining a DbtProject",
             examples=[
                 "{{ project_root }}/path/to/dbt_project",
@@ -199,6 +200,13 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
             examples=["tag:skip_dagster"],
         ),
     ] = DBT_DEFAULT_EXCLUDE
+    selector: Annotated[
+        str,
+        Resolver.default(
+            description="The dbt selector for models in the project you want to include.",
+            examples=["custom_selector"],
+        ),
+    ] = DBT_DEFAULT_SELECTOR
     translation: Annotated[
         Optional[TranslationFn[Mapping[str, Any]]],
         TranslationFnResolver(template_vars_for_translation_fn=lambda data: {"node": data}),
@@ -224,11 +232,38 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
     @property
     def defs_state_config(self) -> DefsStateConfig:
         return DefsStateConfig(
-            key=f"{self.__class__.__name__}[{self.project.name}]",
+            key=f"DbtProjectComponent[{self._project_manager.defs_state_discriminator}]",
             management_type=DefsStateManagementType.LOCAL_FILESYSTEM,
             refresh_if_dev=self.prepare_if_dev,
         )
 
+    @property
+    def op_config_schema(self) -> Optional[type[dg.Config]]:
+        return None
+
+    @property
+    def config_cls(self) -> Optional[type[dg.Config]]:
+        """Internal property that returns the config schema for the op.
+
+        Delegates to op_config_schema for backwards compatibility and consistency
+        with other component types.
+        """
+        return self.op_config_schema
+
+    def _get_op_spec(self, project: DbtProject) -> OpSpec:
+        default = self.op or OpSpec(name=project.name)
+        # always inject required tags
+        return default.model_copy(
+            update=dict(
+                tags={
+                    **(default.tags or {}),
+                    **({DAGSTER_DBT_SELECT_METADATA_KEY: self.select} if self.select else {}),
+                    **({DAGSTER_DBT_EXCLUDE_METADATA_KEY: self.exclude} if self.exclude else {}),
+                    **({DAGSTER_DBT_SELECTOR_METADATA_KEY: self.selector} if self.selector else {}),
+                }
+            )
+        )
+
     @cached_property
     def translator(self) -> "DagsterDbtTranslator":
         return DbtProjectComponentTranslator(self, self.translation_settings)
@@ -309,46 +344,60 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
         return self._base_translator.get_asset_check_spec(asset_spec, manifest, unique_id, project)
 
     @cached_property
-    def cli_resource(self):
-        return DbtCliResource(self.project)
+    def _project_manager(self) -> DbtProjectManager:
+        if isinstance(self.project, DbtProject):
+            return NoopDbtProjectManager(self.project)
+        else:
+            return self.project
+
+    @cached_property
+    def dbt_project(self) -> DbtProject:
+        return self._project_manager.get_project(None)
 
     def get_asset_selection(
         self, select: str, exclude: str = DBT_DEFAULT_EXCLUDE
     ) -> DbtManifestAssetSelection:
         return DbtManifestAssetSelection.build(
-            manifest=self.project.manifest_path,
+            manifest=self.dbt_project.manifest_path,
             dagster_dbt_translator=self.translator,
             select=select,
             exclude=exclude,
         )
 
     def write_state_to_path(self, state_path: Path) -> None:
-        # compile the manifest
-        self.project.preparer.prepare(self.project)
-        # move the manifest to the correct path
-        shutil.copyfile(self.project.manifest_path, state_path)
+        self._project_manager.prepare(state_path)
 
     def build_defs_from_state(
         self, context: dg.ComponentLoadContext, state_path: Optional[Path]
     ) -> dg.Definitions:
-        if state_path is not None:
-            shutil.copyfile(state_path, self.project.manifest_path)
+        project = self._project_manager.get_project(state_path)
 
         res_ctx = context.resolution_context
 
-        @dbt_assets(
-            manifest=self.project.manifest_path,
-            project=self.project,
-            name=self.op.name if self.op else self.project.name,
-            op_tags=self.op.tags if self.op else None,
-            dagster_dbt_translator=self.translator,
+        asset_specs, check_specs = build_dbt_specs(
+            translator=validate_translator(self.translator),
+            manifest=validate_manifest(project.manifest_path),
             select=self.select,
             exclude=self.exclude,
-            backfill_policy=self.op.backfill_policy if self.op else None,
+            selector=self.selector,
+            project=project,
+            io_manager_key=None,
+        )
+        op_spec = self._get_op_spec(project)
+
+        @dg.multi_asset(
+            specs=asset_specs,
+            check_specs=check_specs,
+            can_subset=True,
+            name=op_spec.name,
+            op_tags=op_spec.tags,
+            backfill_policy=op_spec.backfill_policy,
+            pool=op_spec.pool,
+            config_schema=self.config_cls.to_fields_dict() if self.config_cls else None,
         )
         def _fn(context: dg.AssetExecutionContext):
             with _set_resolution_context(res_ctx):
-                yield from self.execute(context=context, dbt=self.cli_resource)
+                yield from self.execute(context=context, dbt=DbtCliResource(project))
 
         return dg.Definitions(assets=[_fn])
 
@@ -437,7 +486,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
 
     @cached_property
     def _validated_manifest(self):
-        return validate_manifest(self.project.manifest_path)
+        return validate_manifest(self.dbt_project.manifest_path)
 
     @cached_property
     def _validated_translator(self):
@@ -460,7 +509,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
         return dagster_dbt_translator.get_asset_spec(
             manifest,
             next(iter(matching_model_ids)),
-            self.project,
+            self.dbt_project,
         ).key
 
 
@@ -492,4 +541,4 @@ def get_projects_from_dbt_component(components: Path) -> list[DbtProject]:
         of_type=DbtProjectComponent
     )
 
-    return [component.project for component in project_components]
+    return [component.dbt_project for component in project_components]

dagster_dbt/components/dbt_project/scaffolder.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 import dagster._check as check
 from dagster._core.errors import DagsterInvalidInvocationError
@@ -13,6 +13,7 @@ from pydantic import BaseModel, Field
 class DbtScaffoldParams(BaseModel):
     init: bool = Field(default=False)
     project_path: Optional[str] = None
+    git_url: Optional[str] = None
 
 
 class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
@@ -20,31 +21,45 @@ class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
     def get_scaffold_params(cls) -> type[DbtScaffoldParams]:
         return DbtScaffoldParams
 
-    def scaffold(self, request: ScaffoldRequest[DbtScaffoldParams]) -> None:
-        project_root = request.project_root or os.getcwd()
-        if request.params.project_path:
-            project_root_tmpl = "{{ project_root }}"
-            rel_path = os.path.relpath(request.params.project_path, start=project_root)
-            path_str = f"{project_root_tmpl}/{rel_path}"
+    def _init_dbt_project(self) -> None:
+        try:
+            from dbt.cli.main import dbtRunner
+        except ImportError:
+            raise DagsterInvalidInvocationError(
+                "dbt-core is not installed. Please install dbt to scaffold this component."
+            )
+
+        dbtRunner().invoke(["init"])
 
-        elif request.params.init:
-            try:
-                from dbt.cli.main import dbtRunner
-            except ImportError:
-                raise DagsterInvalidInvocationError(
-                    "dbt-core is not installed. Please install dbt to scaffold this component."
-                )
+    def _git_url_params(self, git_url: str, project_path: Optional[str]) -> dict[str, Any]:
+        repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
+        return {"project": {"repo_url": git_url, **repo_relative_path}}
 
-            dbtRunner().invoke(["init"])
+    def scaffold(self, request: ScaffoldRequest[DbtScaffoldParams]) -> None:
+        project_root = request.project_root or Path(os.getcwd())
+        check.param_invariant(
+            not (request.params.init and (request.params.git_url or request.params.project_path)),
+            "init and git_url/project_path cannot be set at the same time.",
+        )
+        if request.params.init:
+            self._init_dbt_project()
             subpaths = [
-                path
-                for path in Path(project_root).iterdir()
-                if path.is_dir() and path.name != "logs"
+                path for path in project_root.iterdir() if path.is_dir() and path.name != "logs"
             ]
             check.invariant(len(subpaths) == 1, "Expected exactly one subpath to be created.")
             # this path should be relative to this directory
-            path_str = subpaths[0].name
+            params = {"project": subpaths[0].name}
+        elif request.params.git_url is not None:
+            # project_path is the path to the dbt project within the git repository
+            project_path = request.params.project_path
+            repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
+            params = {"project": {"repo_url": request.params.git_url, **repo_relative_path}}
+        elif request.params.project_path is not None:
+            # project_path represents a local directory
+            project_root_tmpl = "{{ project_root }}"
+            rel_path = os.path.relpath(request.params.project_path, start=project_root)
+            path_str = f"{project_root_tmpl}/{rel_path}"
+            params = {"project": path_str}
         else:
-            path_str = None
-
-        scaffold_component(request, {"project": path_str})
+            params = {"project": None}
+        scaffold_component(request, params)

dagster_dbt/core/dbt_cli_invocation.py

@@ -8,7 +8,7 @@ import sys
 from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from pathlib import Path
-from typing import Any, Final, NamedTuple, Optional, Union, cast
+from typing import Any, Final, Literal, NamedTuple, Optional, Union, cast
 
 import orjson
 from dagster import (
@@ -24,7 +24,6 @@ from dagster import (
 from dagster._annotations import public
 from dagster._core.errors import DagsterExecutionInterruptedError
 from packaging import version
-from typing_extensions import Literal
 
 from dagster_dbt.compat import BaseAdapter, BaseColumn, BaseRelation
 from dagster_dbt.core.dbt_cli_event import (

dagster_dbt/core/dbt_event_iterator.py

@@ -1,6 +1,6 @@
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from concurrent.futures import ThreadPoolExecutor
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
 
 from dagster import (
     AssetCheckResult,

dagster_dbt/core/resource.py

@@ -428,7 +428,7 @@ class DbtCliResource(ConfigurableResource):
             adapter = get_adapter(config)
             manifest = ManifestLoader.load_macros(
                 config,
-                adapter.connections.set_query_header,  # type: ignore
+                adapter.connections.set_query_header,
                 base_macros_only=True,
             )
             adapter.set_macro_resolver(manifest)
@@ -627,8 +627,12 @@ class DbtCliResource(ConfigurableResource):
         dagster_dbt_translator = updated_params.dagster_dbt_translator
         selection_args = updated_params.selection_args
         indirect_selection = updated_params.indirect_selection
-
         target_path = target_path or self._get_unique_target_path(context=context)
+        project_dir = Path(
+            updated_params.dbt_project.project_dir
+            if updated_params.dbt_project
+            else self.project_dir
+        )
         env = {
             # Allow IO streaming when running in Windows.
             # Also, allow it to be overriden by the current environment.
@@ -659,7 +663,7 @@ class DbtCliResource(ConfigurableResource):
             **({"DBT_PROFILES_DIR": self.profiles_dir} if self.profiles_dir else {}),
             # The DBT_PROJECT_DIR environment variable is set to the path containing the dbt project
             # See https://docs.getdbt.com/reference/dbt_project.yml for more information.
-            **({"DBT_PROJECT_DIR": self.project_dir} if self.project_dir else {}),
+            "DBT_PROJECT_DIR": str(project_dir),
         }
 
         # set dbt indirect selection if needed to execute specific dbt tests due to asset check
@@ -683,14 +687,13 @@ class DbtCliResource(ConfigurableResource):
             *profile_args,
             *selection_args,
         ]
-        project_dir = Path(self.project_dir)
 
         if not target_path.is_absolute():
             target_path = project_dir.joinpath(target_path)
 
         # run dbt --version to get the dbt core version
         adapter: Optional[BaseAdapter] = None
-        with pushd(self.project_dir):
+        with pushd(str(project_dir)):
             # we do not need to initialize the adapter if we are using the fusion engine
             if self._cli_version.major < 2:
                 try:

dagster_dbt/dbt_project_manager.py

@@ -0,0 +1,172 @@
+import os
+import shutil
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated, Optional
+from urllib.parse import quote, urlparse, urlunparse
+
+from dagster._core.errors import DagsterInvalidDefinitionError
+from dagster.components.resolved.base import Resolvable
+from dagster.components.resolved.model import Resolver
+
+from dagster_dbt.dbt_project import DbtProject
+
+if TYPE_CHECKING:
+    from dagster_dbt.components.dbt_project.component import DbtProjectArgs
+
+
+class DbtProjectManager(ABC):
+    """Helper class that wraps a dbt project that may or may not be available on local disk at the time
+    it is instantiated. Provides methods for syncing the project to a local path and for instantiating the
+    final DbtProject object when ready.
+    """
+
+    @property
+    @abstractmethod
+    def defs_state_discriminator(self) -> str: ...
+
+    @abstractmethod
+    def sync(self, state_path: Path) -> None: ...
+
+    def _local_project_dir(self, state_path: Path) -> Path:
+        return state_path.parent / "project"
+
+    def prepare(self, state_path: Path) -> None:
+        """Syncs the project to the given local path and ensures the manifest is generated
+        and the dependencies are installed.
+        """
+        # ensure local dir is empty
+        local_dir = self._local_project_dir(state_path)
+        shutil.rmtree(local_dir, ignore_errors=True)
+        local_dir.mkdir()
+
+        # ensure project exists in the dir and is compiled
+        self.sync(state_path)
+
+        # indicate that project has been prepared
+        state_path.touch()
+
+    @abstractmethod
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject": ...
+
+
+@dataclass
+class NoopDbtProjectManager(DbtProjectManager):
+    """Wraps a DbtProject that has already been fully instantiated. Used for cases where a
+    user directly provides a DbtProject to the DbtProjectComponent.
+    """
+
+    project: "DbtProject"
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return self.project.name
+
+    def sync(self, state_path: Path) -> None:
+        pass
+
+    def get_project(self, state_path: Optional[Path]) -> DbtProject:
+        return self.project
+
+
+@dataclass
+class DbtProjectArgsManager(DbtProjectManager):
+    """Wraps DbtProjectArgs provided to the DbtProjectComponent. Avoids instantiating the DbtProject object
+    immediately as this would cause errors in cases where the project_dir has not yet been synced.
+    """
+
+    args: "DbtProjectArgs"
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return Path(self.args.project_dir).stem
+
+    def sync(self, state_path: Path) -> None:
+        # we prepare the project in the original project directory rather than the new one
+        # so that code that does not have access to the local state path can still access
+        # the manifest.json file.
+        project = self.get_project(None)
+        project.preparer.prepare(project)
+        shutil.copytree(
+            self.args.project_dir, self._local_project_dir(state_path), dirs_exist_ok=True
+        )
+
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject":
+        kwargs = asdict(self.args)
+
+        project_dir = self._local_project_dir(state_path) if state_path else self.args.project_dir
+        return DbtProject(
+            project_dir=project_dir,
+            # allow default values on DbtProject to take precedence
+            **{k: v for k, v in kwargs.items() if v is not None and k != "project_dir"},
+        )
+
+
+@dataclass
+class RemoteGitDbtProjectManager(DbtProjectManager, Resolvable):
+    """Wraps a remote git repository containing a dbt project."""
+
+    repo_url: str
+    repo_relative_path: Annotated[
+        str,
+        Resolver.default(
+            description="The relative path to the dbt project within the repository.",
+            examples=["dbt/my_dbt_project"],
+        ),
+    ] = "."
+    token: Annotated[
+        Optional[str],
+        Resolver.default(
+            description="Token for authenticating to the provided repository.",
+            examples=["'{{ env.GITHUB_TOKEN }}'"],
+        ),
+    ] = None
+    profile: Annotated[
+        Optional[str],
+        Resolver.default(description="The profile to use for the dbt project."),
+    ] = None
+    target: Annotated[
+        Optional[str],
+        Resolver.default(
+            description="The target to use for the dbt project.",
+            examples=["dev", "prod"],
+        ),
+    ] = None
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return self.repo_url
+
+    def _get_clone_url(self) -> str:
+        # in github action environments, the token will be available automatically as an env var
+        raw_token = self.token or os.getenv("GITHUB_TOKEN")
+        if raw_token is None:
+            return self.repo_url
+        else:
+            # insert the token into the url
+            token = quote(raw_token, safe="")
+            parts = urlparse(self.repo_url)
+            return urlunparse(parts._replace(netloc=f"{token}@{parts.netloc}"))
+
+    def sync(self, state_path: Path) -> None:
+        # defer git import to avoid side-effects on import
+        from git import Repo
+
+        Repo.clone_from(self._get_clone_url(), self._local_project_dir(state_path), depth=1)
+        project = self.get_project(state_path)
+        project.preparer.prepare(project)
+
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject":
+        if state_path is None:
+            raise DagsterInvalidDefinitionError(
+                "Attempted to `get_project()` on a `RemoteGitDbtProjectWrapper` without a path. "
+                "This can happen when calling unsupported methods on the `DbtProjectComponent`."
+            )
+
+        local_dir = self._local_project_dir(state_path)
+        return DbtProject(
+            project_dir=local_dir / self.repo_relative_path,
+            profile=self.profile,
+            target=self.target,
+        )

dagster_dbt/include/pyproject.toml.jinja

@@ -3,7 +3,7 @@ name = "{{ project_name }}"
 version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
-requires-python = ">=3.9,<3.14"
+requires-python = ">=3.10,<3.14"
 dependencies = [
     "dagster",
     "dagster-cloud",

dagster_dbt/version.py

@@ -1 +1 @@
-__version__ = "0.28.0"
+__version__ = "0.28.2"

{dagster_dbt-0.28.0.dist-info → dagster_dbt-0.28.2.dist-info}/METADATA

@@ -1,22 +1,22 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.28.0
+Version: 0.28.2
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.14
+Requires-Python: >=3.10,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.12.0
+Requires-Dist: dagster==1.12.2
 Requires-Dist: dbt-core<1.11,>=1.7
+Requires-Dist: gitpython
 Requires-Dist: Jinja2
 Requires-Dist: networkx
 Requires-Dist: orjson

{dagster_dbt-0.28.0.dist-info → dagster_dbt-0.28.2.dist-info}/RECORD

@@ -1,31 +1,32 @@
 dagster_dbt/__init__.py,sha256=rlPCxCzovXNDiqXBUEcgNVHjZrXQXwcX3kwUhe_69C4,4708
-dagster_dbt/asset_decorator.py,sha256=8OqlypoTwKCs6dAxOuBAO32krbHBQRLVGe6uDzpTSXE,14892
+dagster_dbt/asset_decorator.py,sha256=OFFjhmDQO6dK8N3U7VQF7gergswkIS3eHnMRwAe2BZY,14892
 dagster_dbt/asset_specs.py,sha256=2EdWIhY2QZhtGXM7N-kkkeK3ClnGgYX7ayRi_X11cLg,2741
-dagster_dbt/asset_utils.py,sha256=bmUm9fCnSUVzGjBLwx14vUNOecizrLjfqlI2y8qxO2Q,44761
-dagster_dbt/compat.py,sha256=iiAez9pLskfbwLdj6LxIrFvoU-efBpy2BETXQyAsQvY,3638
+dagster_dbt/asset_utils.py,sha256=6M3i5_quRys2VCQEzPQwBz5MEl2zY2NJIVZ8r2i-3p4,46651
+dagster_dbt/compat.py,sha256=lqzGonzQE7Lb825yRqUaQPXeNQp8umAR4LqyD6COXdc,3609
 dagster_dbt/dagster_dbt_translator.py,sha256=skUpl7NxG7ysDfDOs1pSq1c9-ciByN7LrGvO0ZsC4W4,30274
 dagster_dbt/dbt_core_version.py,sha256=w1P62qDdbApXKv0XvUNr2p7FlwiW68csugsNaqmjNjM,38
 dagster_dbt/dbt_manifest.py,sha256=q10Qq1whh-dLfWtFbTYXYbqBVCf0oU8T1yRPyy9ASw0,1307
 dagster_dbt/dbt_manifest_asset_selection.py,sha256=KSEHcVdtfFZaEqSQDUsAx8H8BQe62jZxAOE_CLtwVlI,5072
 dagster_dbt/dbt_project.py,sha256=HtUuNYbKMAPWqTOPgPeJK4u6b1nKyqHrRNB8sYwOWVw,12203
+dagster_dbt/dbt_project_manager.py,sha256=FPHSmo4ep9jNDyRjB0bDR76dOrthgRN1u3CLMklpzS0,6005
 dagster_dbt/dbt_version.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_dbt/errors.py,sha256=a8xag0tjh8OQVkiL10_uwY4hkAgRCH6HtTGaZxIbXZI,1073
 dagster_dbt/freshness_builder.py,sha256=DHAC3AGWAwIA7psDf72R0t5p8NjxDytGU9g5xnca1hc,6395
 dagster_dbt/metadata_set.py,sha256=lqjASYoYeM_Ey6r8UsPUkRMwmuAIfFCFvkNm0xW5xTg,512
 dagster_dbt/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
 dagster_dbt/utils.py,sha256=gT6xO7buRolkhc2fa5ySUPfD1eXo3e6RJWZAKkM6yFo,8513
-dagster_dbt/version.py,sha256=MRQGtOXBhcDKeeNOL0LiB-cllo6kfd8_KGJOvaDp0XQ,23
+dagster_dbt/version.py,sha256=K-TM2fq9AmH_Dk8Cadam72wILDZ_6qftLHvY9P1Fc3I,23
 dagster_dbt/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_dbt/cli/app.py,sha256=9oBx85uzAkMtBdV39vNm5rxuAGPVYzzDUs6Ek-KL0XY,13516
 dagster_dbt/cloud/__init__.py,sha256=8WKaLuPl_pUG9Cv78GW782vrWQfqK8QtAWegkTxA9r4,441
-dagster_dbt/cloud/asset_defs.py,sha256=brrMeblPY-UUhAZ16WKOZ1vP7s9z1WZdHdK0SmSyq8o,29311
+dagster_dbt/cloud/asset_defs.py,sha256=4mdXd3tiGd9OPBBF6xvH81DZhcyGL9pAw3WH8dB7Qu8,29311
 dagster_dbt/cloud/cli.py,sha256=VnKzBjn-BPpjn4nPZm5xSrboAKpRhlCa-4IxsN1ROCo,4525
 dagster_dbt/cloud/ops.py,sha256=rsU4qPCRUUzeHRInZph7YEz_iynwu_KidO5vMAYHX5E,4615
 dagster_dbt/cloud/resources.py,sha256=AWW3VTBS6zrSKeDjMJovbnu54sKena93FoZle6ZSSq8,31747
 dagster_dbt/cloud/types.py,sha256=gsigGlKjR3gLePppiYX09S2vx-0PyPvH7AkzOphDSMA,2886
-dagster_dbt/cloud/utils.py,sha256=qmeEUKCw-KksKUIAxO7W1e5Em-C7hrCXU1PEKnGMk0o,7158
+dagster_dbt/cloud/utils.py,sha256=NnPxHp3r2cpJ0Vi1Gt4eBmBlNbAhdwaR7GUZRkFsLRc,7158
 dagster_dbt/cloud_v2/__init__.py,sha256=9NjPjqm8ytsKiogiW9cUpHgQFaht1q6g_3K7KSKtzsU,471
-dagster_dbt/cloud_v2/asset_decorator.py,sha256=KusLZ4LdY02coN9KYbQr2g4EgEVqWcvjOm98BvhRruk,3114
+dagster_dbt/cloud_v2/asset_decorator.py,sha256=wHL77K2NXvnU0PDEsqyqpiRX1yp1cuQiit1oxRpNY3s,3141
 dagster_dbt/cloud_v2/cli_invocation.py,sha256=EX2a0G7HIlOJBES-Lf3DIp0y1L1vGeiO3ms1a0F9tWc,2235
 dagster_dbt/cloud_v2/client.py,sha256=V7UsepwHteJiEhQ8XT-lzFV-xDlspzVDgMJDxd6FLEg,16674
 dagster_dbt/cloud_v2/resources.py,sha256=ENzllq2rJlD4Tb3_9JTsMVsoBjyKBs28543-L3lfxLU,16925
@@ -34,25 +35,25 @@ dagster_dbt/cloud_v2/sensor_builder.py,sha256=8mAm-1ZFvoiVVdxSkSQbMxg18aQUY5kazu
 dagster_dbt/cloud_v2/types.py,sha256=dI-NIguj582LwTTMXdY5r0U4-INDn9anNy-ciGuEc1s,4136
 dagster_dbt/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_dbt/components/dbt_project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dagster_dbt/components/dbt_project/component.py,sha256=DfAif7IKnzvJSY8RJDGUzwpugSUyzZNPs_pk6E7EmFY,18661
-dagster_dbt/components/dbt_project/scaffolder.py,sha256=AqwRnX3cFpvCSxEeEpyQj0Vei84WSWUi2BIMCtkxaDE,1893
+dagster_dbt/components/dbt_project/component.py,sha256=oVj1RGp27LBNWxdDaGhvEyw18fOZzUKN8IYwmV2PxGo,20503
+dagster_dbt/components/dbt_project/scaffolder.py,sha256=tbxvFfcvF5KBdHJmjCIxiujtHeo0iLMwnuVM6Ym_iQk,2914
 dagster_dbt/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_dbt/core/dbt_cli_event.py,sha256=Yqr-UuzFF7jIcI4KRAmiAhGTX9Ay0tIMN2NaJegsrJQ,25203
-dagster_dbt/core/dbt_cli_invocation.py,sha256=7ZrwAgivAtZGwrHLxfpmNan004IuEEdBSaKZeKp69Mg,17534
-dagster_dbt/core/dbt_event_iterator.py,sha256=s0gU6wIgOpmbo50309FHp9zTqlLkIWcgLkETmidxPRo,16953
-dagster_dbt/core/resource.py,sha256=6HhBBa8SRAJPhDtnHRNPTjdFPQvNYIdfAXQY3_LdqHk,30782
+dagster_dbt/core/dbt_cli_invocation.py,sha256=lJLoRvy630xUAEdnH76yl5PJ98-OsLx06l813XjzZOM,17505
+dagster_dbt/core/dbt_event_iterator.py,sha256=IL3QsVUDMfGk04SFTkrVEfbbYu745Kn8OyMQlrlHxIE,16953
+dagster_dbt/core/resource.py,sha256=1qIEneW2aZYeArF5w_QV4-Xmf7Kp9kpGaQaxUlL3ZZs,30851
 dagster_dbt/core/utils.py,sha256=ciXjLhFTNVTyDtVWP4Kjee0LWZkIuVJOltvlv0COzDo,577
 dagster_dbt/include/__init__.py,sha256=8ujr-ROlJ5x64POs-bH-0zfjZ2QHx-FgKUZAvAFbSs8,89
-dagster_dbt/include/pyproject.toml.jinja,sha256=h5Od0HP4c7gQLRNIDyDJLzTMpp5sc4qtrzOVM-R75IA,660
+dagster_dbt/include/pyproject.toml.jinja,sha256=_BbC3zmV6ajtGnDMesyut0Yl0Fc7bY-F887KTIKMm1o,661
 dagster_dbt/include/setup.py.jinja,sha256=N3NjYuXvWmaY8bj5GftVxVvImDUGC4YbFc_MdP1-quA,618
 dagster_dbt/include/scaffold/__init__.py.jinja,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dagster_dbt/include/scaffold/assets.py.jinja,sha256=JImqnDUP5ewy8RVti4IuXL70yJnoke54dMuZ3G8Wlx8,562
 dagster_dbt/include/scaffold/definitions.py.jinja,sha256=Hou7emwkEeh5YXTdqjYFrAc2SK-Q6MgTNsQOKA_Vy3s,364
 dagster_dbt/include/scaffold/project.py.jinja,sha256=YNtkT5Hq4VbGw-b7QcxdelhXsesIKORwVuBFGFdfeUs,432
 dagster_dbt/include/scaffold/schedules.py.jinja,sha256=Xua_VtPjYFc498A5uaBGQ36GwV1gqciO4P3D8Yt9M-Y,413
-dagster_dbt-0.28.0.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
-dagster_dbt-0.28.0.dist-info/METADATA,sha256=XUBRuIhSbaeG0yRgt_cf2xsRvAPz3Ae7r-k2k4oVpCk,1596
-dagster_dbt-0.28.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_dbt-0.28.0.dist-info/entry_points.txt,sha256=pbv0tVoTB7cByG-noE8rC6atvthh64qBaTo7PkQ9HbM,163
-dagster_dbt-0.28.0.dist-info/top_level.txt,sha256=hoOwFvw9OpJUN1azE6UVHcxMKqhUwR_BTN0Ay-iKUDA,12
-dagster_dbt-0.28.0.dist-info/RECORD,,
+dagster_dbt-0.28.2.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_dbt-0.28.2.dist-info/METADATA,sha256=k0MPai8vmyGPfpswuSgc3l5IDf6NItMeWsKcsy3lCpU,1572
+dagster_dbt-0.28.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_dbt-0.28.2.dist-info/entry_points.txt,sha256=pbv0tVoTB7cByG-noE8rC6atvthh64qBaTo7PkQ9HbM,163
+dagster_dbt-0.28.2.dist-info/top_level.txt,sha256=hoOwFvw9OpJUN1azE6UVHcxMKqhUwR_BTN0Ay-iKUDA,12
+dagster_dbt-0.28.2.dist-info/RECORD,,