dagster-dbt 0.28.0__tar.gz → 0.28.2__tar.gz

{dagster_dbt-0.28.0/dagster_dbt.egg-info → dagster_dbt-0.28.2}/PKG-INFO

@@ -1,22 +1,22 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.28.0
+Version: 0.28.2
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.14
+Requires-Python: >=3.10,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.12.0
+Requires-Dist: dagster==1.12.2
 Requires-Dist: dbt-core<1.11,>=1.7
+Requires-Dist: gitpython
 Requires-Dist: Jinja2
 Requires-Dist: networkx
 Requires-Dist: orjson

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/asset_decorator.py

@@ -1,5 +1,5 @@
-from collections.abc import Mapping
-from typing import Any, Callable, Optional
+from collections.abc import Callable, Mapping
+from typing import Any, Optional
 
 from dagster import (
     AssetsDefinition,

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/asset_utils.py

@@ -1,11 +1,14 @@
 import hashlib
 import os
+import shutil
+import tempfile
 import textwrap
 from collections import defaultdict
 from collections.abc import Iterable, Mapping, Sequence
 from pathlib import Path
 from typing import TYPE_CHECKING, AbstractSet, Annotated, Any, Final, Optional, Union  # noqa: UP035
 
+import yaml
 from dagster import (
     AssetCheckKey,
     AssetCheckSpec,
@@ -32,6 +35,7 @@ from dagster._core.definitions.metadata import TableMetadataSet
 from dagster._core.errors import DagsterInvalidPropertyError
 from dagster._core.types.dagster_type import Nothing
 from dagster._record import ImportFrom, record
+from dagster_shared.record import replace
 
 from dagster_dbt.dbt_project import DbtProject
 from dagster_dbt.metadata_set import DbtMetadataSet
@@ -55,6 +59,10 @@ DBT_DEFAULT_SELECTOR = ""
 DBT_INDIRECT_SELECTION_ENV: Final[str] = "DBT_INDIRECT_SELECTION"
 DBT_EMPTY_INDIRECT_SELECTION: Final[str] = "empty"
 
+# Threshold for switching to selector file to avoid CLI argument length limits
+# https://github.com/dagster-io/dagster/issues/16997
+_SELECTION_ARGS_THRESHOLD: Final[int] = 200
+
 DUPLICATE_ASSET_KEY_ERROR_MESSAGE = (
     "The following dbt resources are configured with identical Dagster asset keys."
     " Please ensure that each dbt resource generates a unique Dagster asset key."
@@ -443,6 +451,10 @@ def get_updated_cli_invocation_params_for_context(
             [assets_def]
         )
 
+        # Get project_dir from dbt_project if available
+        project_dir = Path(dbt_project.project_dir) if dbt_project else None
+        target_project = dbt_project
+
         selection_args, indirect_selection_override = get_subset_selection_for_context(
             context=context,
             manifest=manifest,
@@ -452,17 +464,49 @@ def get_updated_cli_invocation_params_for_context(
             dagster_dbt_translator=dagster_dbt_translator,
             current_dbt_indirect_selection_env=indirect_selection,
         )
+        if (
+            selection_args[0] == "--select"
+            and project_dir
+            and len(resources := selection_args[1].split(" ")) > _SELECTION_ARGS_THRESHOLD
+        ):
+            temp_project_dir = tempfile.mkdtemp()
+            shutil.copytree(project_dir, temp_project_dir, dirs_exist_ok=True)
+            selectors_path = Path(temp_project_dir) / "selectors.yml"
+
+            # Delete any existing selectors, we need to create our own
+            if selectors_path.exists():
+                selectors_path.unlink()
+
+            selector_name = f"dagster_run_{context.run_id}"
+            temp_selectors = {
+                "selectors": [
+                    {
+                        "name": selector_name,
+                        "definition": {"union": list(resources)},
+                    }
+                ]
+            }
+            selectors_path.write_text(yaml.safe_dump(temp_selectors))
+            logger.info(
+                f"DBT selection of {len(resources)} resources exceeds threshold of {_SELECTION_ARGS_THRESHOLD}. "
+                "This may exceed system argument length limits. "
+                f"Executing materialization against temporary copy of DBT project at {temp_project_dir} with ephemeral selector."
+            )
+            selection_args = ["--selector", selector_name]
+            target_project = replace(dbt_project, project_dir=temp_project_dir)
 
         indirect_selection = (
             indirect_selection_override if indirect_selection_override else indirect_selection
         )
+    else:
+        target_project = dbt_project
 
     return DbtCliInvocationPartialParams(
         manifest=manifest,
         dagster_dbt_translator=dagster_dbt_translator,
         selection_args=selection_args,
         indirect_selection=indirect_selection,
-        dbt_project=dbt_project,
+        dbt_project=target_project,
     )
 
 

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/cloud/asset_defs.py

@@ -1,9 +1,9 @@
 import json
 import shlex
 from argparse import ArgumentParser, Namespace
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from contextlib import suppress
-from typing import Any, Callable, Optional, Union, cast
+from typing import Any, Optional, Union, cast
 
 import dagster._check as check
 from dagster import (

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/cloud/utils.py

@@ -1,5 +1,5 @@
-from collections.abc import Iterator, Mapping, Sequence
-from typing import Any, Callable, Optional, Union
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from typing import Any, Optional, Union
 
 import dateutil
 from dagster import (

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/cloud_v2/asset_decorator.py

@@ -1,4 +1,5 @@
-from typing import Any, Callable, Optional
+from collections.abc import Callable
+from typing import Any, Optional
 
 from dagster import AssetsDefinition, multi_asset
 from dagster._annotations import public

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/compat.py

@@ -1,9 +1,8 @@
 import logging
 from enum import Enum
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeAlias
 
 from packaging import version
-from typing_extensions import TypeAlias
 
 # it's unclear exactly which dbt import adds a handler to the root logger, but something certainly does!
 # on this line, we keep track of the set of handlers that are on the root logger BEFORE any dbt imports

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/components/dbt_project/component.py

@@ -1,13 +1,12 @@
 import itertools
 import json
-import shutil
 from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from contextvars import ContextVar
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from functools import cached_property
 from pathlib import Path
-from typing import Annotated, Any, Literal, Optional, Union
+from typing import Annotated, Any, Literal, Optional, TypeAlias, Union
 
 import dagster as dg
 from dagster._annotations import public
@@ -26,10 +25,17 @@ from dagster.components.utils.translation import (
 )
 from dagster_shared import check
 from dagster_shared.serdes.objects.models.defs_state_info import DefsStateManagementType
-from typing_extensions import TypeAlias
 
-from dagster_dbt.asset_decorator import dbt_assets
-from dagster_dbt.asset_utils import DBT_DEFAULT_EXCLUDE, DBT_DEFAULT_SELECT, get_node
+from dagster_dbt.asset_utils import (
+    DAGSTER_DBT_EXCLUDE_METADATA_KEY,
+    DAGSTER_DBT_SELECT_METADATA_KEY,
+    DAGSTER_DBT_SELECTOR_METADATA_KEY,
+    DBT_DEFAULT_EXCLUDE,
+    DBT_DEFAULT_SELECT,
+    DBT_DEFAULT_SELECTOR,
+    build_dbt_specs,
+    get_node,
+)
 from dagster_dbt.components.dbt_project.scaffolder import DbtProjectComponentScaffolder
 from dagster_dbt.core.resource import DbtCliResource
 from dagster_dbt.dagster_dbt_translator import (
@@ -40,6 +46,12 @@ from dagster_dbt.dagster_dbt_translator import (
 from dagster_dbt.dbt_manifest import validate_manifest
 from dagster_dbt.dbt_manifest_asset_selection import DbtManifestAssetSelection
 from dagster_dbt.dbt_project import DbtProject
+from dagster_dbt.dbt_project_manager import (
+    DbtProjectArgsManager,
+    DbtProjectManager,
+    NoopDbtProjectManager,
+    RemoteGitDbtProjectManager,
+)
 from dagster_dbt.utils import ASSET_RESOURCE_TYPES
 
 
@@ -63,29 +75,18 @@ class DbtProjectArgs(dg.Resolvable):
     state_path: Optional[str] = None
 
 
-def resolve_dbt_project(context: ResolutionContext, model) -> DbtProject:
-    if isinstance(model, str):
-        return DbtProject(
-            context.resolve_source_relative_path(
-                context.resolve_value(model, as_type=str),
-            )
-        )
-
-    args = DbtProjectArgs.resolve_from_model(context, model)
-
-    kwargs = {}  # use optionally splatted kwargs to avoid redefining default value
-    if args.target_path:
-        kwargs["target_path"] = args.target_path
+def resolve_dbt_project(context: ResolutionContext, model) -> DbtProjectManager:
+    if isinstance(model, RemoteGitDbtProjectManager.model()):
+        return RemoteGitDbtProjectManager.resolve_from_model(context, model)
 
-    return DbtProject(
-        project_dir=context.resolve_source_relative_path(args.project_dir),
-        target=args.target,
-        profiles_dir=args.profiles_dir,
-        state_path=args.state_path,
-        packaged_project_dir=args.packaged_project_dir,
-        profile=args.profile,
-        **kwargs,
+    args = (
+        DbtProjectArgs(project_dir=context.resolve_value(model, as_type=str))
+        if isinstance(model, str)
+        else DbtProjectArgs.resolve_from_model(context, model)
     )
+    # resolve the project_dir relative to where this component is defined
+    args = replace(args, project_dir=context.resolve_source_relative_path(args.project_dir))
+    return DbtProjectArgsManager(args)
 
 
 DbtMetadataAddons: TypeAlias = Literal["column_metadata", "row_count"]
@@ -128,10 +129,10 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
     """
 
     project: Annotated[
-        DbtProject,
+        Union[DbtProject, DbtProjectManager],
         Resolver(
             resolve_dbt_project,
-            model_field_type=Union[str, DbtProjectArgs.model()],
+            model_field_type=Union[str, DbtProjectArgs.model(), RemoteGitDbtProjectManager.model()],
             description="The path to the dbt project or a mapping defining a DbtProject",
             examples=[
                 "{{ project_root }}/path/to/dbt_project",
@@ -199,6 +200,13 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
             examples=["tag:skip_dagster"],
         ),
     ] = DBT_DEFAULT_EXCLUDE
+    selector: Annotated[
+        str,
+        Resolver.default(
+            description="The dbt selector for models in the project you want to include.",
+            examples=["custom_selector"],
+        ),
+    ] = DBT_DEFAULT_SELECTOR
     translation: Annotated[
         Optional[TranslationFn[Mapping[str, Any]]],
         TranslationFnResolver(template_vars_for_translation_fn=lambda data: {"node": data}),
@@ -224,11 +232,38 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
     @property
     def defs_state_config(self) -> DefsStateConfig:
         return DefsStateConfig(
-            key=f"{self.__class__.__name__}[{self.project.name}]",
+            key=f"DbtProjectComponent[{self._project_manager.defs_state_discriminator}]",
             management_type=DefsStateManagementType.LOCAL_FILESYSTEM,
             refresh_if_dev=self.prepare_if_dev,
         )
 
+    @property
+    def op_config_schema(self) -> Optional[type[dg.Config]]:
+        return None
+
+    @property
+    def config_cls(self) -> Optional[type[dg.Config]]:
+        """Internal property that returns the config schema for the op.
+
+        Delegates to op_config_schema for backwards compatibility and consistency
+        with other component types.
+        """
+        return self.op_config_schema
+
+    def _get_op_spec(self, project: DbtProject) -> OpSpec:
+        default = self.op or OpSpec(name=project.name)
+        # always inject required tags
+        return default.model_copy(
+            update=dict(
+                tags={
+                    **(default.tags or {}),
+                    **({DAGSTER_DBT_SELECT_METADATA_KEY: self.select} if self.select else {}),
+                    **({DAGSTER_DBT_EXCLUDE_METADATA_KEY: self.exclude} if self.exclude else {}),
+                    **({DAGSTER_DBT_SELECTOR_METADATA_KEY: self.selector} if self.selector else {}),
+                }
+            )
+        )
+
     @cached_property
     def translator(self) -> "DagsterDbtTranslator":
         return DbtProjectComponentTranslator(self, self.translation_settings)
@@ -309,46 +344,60 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
         return self._base_translator.get_asset_check_spec(asset_spec, manifest, unique_id, project)
 
     @cached_property
-    def cli_resource(self):
-        return DbtCliResource(self.project)
+    def _project_manager(self) -> DbtProjectManager:
+        if isinstance(self.project, DbtProject):
+            return NoopDbtProjectManager(self.project)
+        else:
+            return self.project
+
+    @cached_property
+    def dbt_project(self) -> DbtProject:
+        return self._project_manager.get_project(None)
 
     def get_asset_selection(
         self, select: str, exclude: str = DBT_DEFAULT_EXCLUDE
     ) -> DbtManifestAssetSelection:
         return DbtManifestAssetSelection.build(
-            manifest=self.project.manifest_path,
+            manifest=self.dbt_project.manifest_path,
             dagster_dbt_translator=self.translator,
             select=select,
             exclude=exclude,
         )
 
     def write_state_to_path(self, state_path: Path) -> None:
-        # compile the manifest
-        self.project.preparer.prepare(self.project)
-        # move the manifest to the correct path
-        shutil.copyfile(self.project.manifest_path, state_path)
+        self._project_manager.prepare(state_path)
 
     def build_defs_from_state(
         self, context: dg.ComponentLoadContext, state_path: Optional[Path]
     ) -> dg.Definitions:
-        if state_path is not None:
-            shutil.copyfile(state_path, self.project.manifest_path)
+        project = self._project_manager.get_project(state_path)
 
         res_ctx = context.resolution_context
 
-        @dbt_assets(
-            manifest=self.project.manifest_path,
-            project=self.project,
-            name=self.op.name if self.op else self.project.name,
-            op_tags=self.op.tags if self.op else None,
-            dagster_dbt_translator=self.translator,
+        asset_specs, check_specs = build_dbt_specs(
+            translator=validate_translator(self.translator),
+            manifest=validate_manifest(project.manifest_path),
             select=self.select,
             exclude=self.exclude,
-            backfill_policy=self.op.backfill_policy if self.op else None,
+            selector=self.selector,
+            project=project,
+            io_manager_key=None,
+        )
+        op_spec = self._get_op_spec(project)
+
+        @dg.multi_asset(
+            specs=asset_specs,
+            check_specs=check_specs,
+            can_subset=True,
+            name=op_spec.name,
+            op_tags=op_spec.tags,
+            backfill_policy=op_spec.backfill_policy,
+            pool=op_spec.pool,
+            config_schema=self.config_cls.to_fields_dict() if self.config_cls else None,
         )
         def _fn(context: dg.AssetExecutionContext):
             with _set_resolution_context(res_ctx):
-                yield from self.execute(context=context, dbt=self.cli_resource)
+                yield from self.execute(context=context, dbt=DbtCliResource(project))
 
         return dg.Definitions(assets=[_fn])
 
@@ -437,7 +486,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
 
     @cached_property
     def _validated_manifest(self):
-        return validate_manifest(self.project.manifest_path)
+        return validate_manifest(self.dbt_project.manifest_path)
 
     @cached_property
     def _validated_translator(self):
@@ -460,7 +509,7 @@ class DbtProjectComponent(StateBackedComponent, dg.Resolvable):
         return dagster_dbt_translator.get_asset_spec(
             manifest,
             next(iter(matching_model_ids)),
-            self.project,
+            self.dbt_project,
         ).key
 
 
@@ -492,4 +541,4 @@ def get_projects_from_dbt_component(components: Path) -> list[DbtProject]:
         of_type=DbtProjectComponent
     )
 
-    return [component.project for component in project_components]
+    return [component.dbt_project for component in project_components]

dagster_dbt-0.28.2/dagster_dbt/components/dbt_project/scaffolder.py

@@ -0,0 +1,65 @@
+import os
+from pathlib import Path
+from typing import Any, Optional
+
+import dagster._check as check
+from dagster._core.errors import DagsterInvalidInvocationError
+from dagster.components.component.component_scaffolder import Scaffolder
+from dagster.components.component_scaffolding import scaffold_component
+from dagster.components.scaffold.scaffold import ScaffoldRequest
+from pydantic import BaseModel, Field
+
+
+class DbtScaffoldParams(BaseModel):
+    init: bool = Field(default=False)
+    project_path: Optional[str] = None
+    git_url: Optional[str] = None
+
+
+class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
+    @classmethod
+    def get_scaffold_params(cls) -> type[DbtScaffoldParams]:
+        return DbtScaffoldParams
+
+    def _init_dbt_project(self) -> None:
+        try:
+            from dbt.cli.main import dbtRunner
+        except ImportError:
+            raise DagsterInvalidInvocationError(
+                "dbt-core is not installed. Please install dbt to scaffold this component."
+            )
+
+        dbtRunner().invoke(["init"])
+
+    def _git_url_params(self, git_url: str, project_path: Optional[str]) -> dict[str, Any]:
+        repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
+        return {"project": {"repo_url": git_url, **repo_relative_path}}
+
+    def scaffold(self, request: ScaffoldRequest[DbtScaffoldParams]) -> None:
+        project_root = request.project_root or Path(os.getcwd())
+        check.param_invariant(
+            not (request.params.init and (request.params.git_url or request.params.project_path)),
+            "init and git_url/project_path cannot be set at the same time.",
+        )
+        if request.params.init:
+            self._init_dbt_project()
+            subpaths = [
+                path for path in project_root.iterdir() if path.is_dir() and path.name != "logs"
+            ]
+            check.invariant(len(subpaths) == 1, "Expected exactly one subpath to be created.")
+            # this path should be relative to this directory
+            params = {"project": subpaths[0].name}
+        elif request.params.git_url is not None:
+            # project_path is the path to the dbt project within the git repository
+            project_path = request.params.project_path
+            repo_relative_path = {"repo_relative_path": project_path} if project_path else {}
+            params = {"project": {"repo_url": request.params.git_url, **repo_relative_path}}
+        elif request.params.project_path is not None:
+            # project_path represents a local directory
+            project_root_tmpl = "{{ project_root }}"
+            rel_path = os.path.relpath(request.params.project_path, start=project_root)
+            path_str = f"{project_root_tmpl}/{rel_path}"
+            params = {"project": path_str}
+        else:
+            params = {"project": None}
+        scaffold_component(request, params)

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/core/dbt_cli_invocation.py

@@ -8,7 +8,7 @@ import sys
 from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from pathlib import Path
-from typing import Any, Final, NamedTuple, Optional, Union, cast
+from typing import Any, Final, Literal, NamedTuple, Optional, Union, cast
 
 import orjson
 from dagster import (
@@ -24,7 +24,6 @@ from dagster import (
 from dagster._annotations import public
 from dagster._core.errors import DagsterExecutionInterruptedError
 from packaging import version
-from typing_extensions import Literal
 
 from dagster_dbt.compat import BaseAdapter, BaseColumn, BaseRelation
 from dagster_dbt.core.dbt_cli_event import (

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/core/dbt_event_iterator.py

@@ -1,6 +1,6 @@
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from concurrent.futures import ThreadPoolExecutor
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
 
 from dagster import (
     AssetCheckResult,

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/core/resource.py

@@ -428,7 +428,7 @@ class DbtCliResource(ConfigurableResource):
             adapter = get_adapter(config)
             manifest = ManifestLoader.load_macros(
                 config,
-                adapter.connections.set_query_header,  # type: ignore
+                adapter.connections.set_query_header,
                 base_macros_only=True,
             )
             adapter.set_macro_resolver(manifest)
@@ -627,8 +627,12 @@ class DbtCliResource(ConfigurableResource):
         dagster_dbt_translator = updated_params.dagster_dbt_translator
         selection_args = updated_params.selection_args
         indirect_selection = updated_params.indirect_selection
-
         target_path = target_path or self._get_unique_target_path(context=context)
+        project_dir = Path(
+            updated_params.dbt_project.project_dir
+            if updated_params.dbt_project
+            else self.project_dir
+        )
         env = {
             # Allow IO streaming when running in Windows.
             # Also, allow it to be overriden by the current environment.
@@ -659,7 +663,7 @@ class DbtCliResource(ConfigurableResource):
             **({"DBT_PROFILES_DIR": self.profiles_dir} if self.profiles_dir else {}),
             # The DBT_PROJECT_DIR environment variable is set to the path containing the dbt project
             # See https://docs.getdbt.com/reference/dbt_project.yml for more information.
-            **({"DBT_PROJECT_DIR": self.project_dir} if self.project_dir else {}),
+            "DBT_PROJECT_DIR": str(project_dir),
         }
 
         # set dbt indirect selection if needed to execute specific dbt tests due to asset check
@@ -683,14 +687,13 @@ class DbtCliResource(ConfigurableResource):
             *profile_args,
             *selection_args,
         ]
-        project_dir = Path(self.project_dir)
 
         if not target_path.is_absolute():
             target_path = project_dir.joinpath(target_path)
 
         # run dbt --version to get the dbt core version
         adapter: Optional[BaseAdapter] = None
-        with pushd(self.project_dir):
+        with pushd(str(project_dir)):
             # we do not need to initialize the adapter if we are using the fusion engine
             if self._cli_version.major < 2:
                 try:

dagster_dbt-0.28.2/dagster_dbt/dbt_project_manager.py

@@ -0,0 +1,172 @@
+import os
+import shutil
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated, Optional
+from urllib.parse import quote, urlparse, urlunparse
+
+from dagster._core.errors import DagsterInvalidDefinitionError
+from dagster.components.resolved.base import Resolvable
+from dagster.components.resolved.model import Resolver
+
+from dagster_dbt.dbt_project import DbtProject
+
+if TYPE_CHECKING:
+    from dagster_dbt.components.dbt_project.component import DbtProjectArgs
+
+
+class DbtProjectManager(ABC):
+    """Helper class that wraps a dbt project that may or may not be available on local disk at the time
+    it is instantiated. Provides methods for syncing the project to a local path and for instantiating the
+    final DbtProject object when ready.
+    """
+
+    @property
+    @abstractmethod
+    def defs_state_discriminator(self) -> str: ...
+
+    @abstractmethod
+    def sync(self, state_path: Path) -> None: ...
+
+    def _local_project_dir(self, state_path: Path) -> Path:
+        return state_path.parent / "project"
+
+    def prepare(self, state_path: Path) -> None:
+        """Syncs the project to the given local path and ensures the manifest is generated
+        and the dependencies are installed.
+        """
+        # ensure local dir is empty
+        local_dir = self._local_project_dir(state_path)
+        shutil.rmtree(local_dir, ignore_errors=True)
+        local_dir.mkdir()
+
+        # ensure project exists in the dir and is compiled
+        self.sync(state_path)
+
+        # indicate that project has been prepared
+        state_path.touch()
+
+    @abstractmethod
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject": ...
+
+
+@dataclass
+class NoopDbtProjectManager(DbtProjectManager):
+    """Wraps a DbtProject that has already been fully instantiated. Used for cases where a
+    user directly provides a DbtProject to the DbtProjectComponent.
+    """
+
+    project: "DbtProject"
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return self.project.name
+
+    def sync(self, state_path: Path) -> None:
+        pass
+
+    def get_project(self, state_path: Optional[Path]) -> DbtProject:
+        return self.project
+
+
+@dataclass
+class DbtProjectArgsManager(DbtProjectManager):
+    """Wraps DbtProjectArgs provided to the DbtProjectComponent. Avoids instantiating the DbtProject object
+    immediately as this would cause errors in cases where the project_dir has not yet been synced.
+    """
+
+    args: "DbtProjectArgs"
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return Path(self.args.project_dir).stem
+
+    def sync(self, state_path: Path) -> None:
+        # we prepare the project in the original project directory rather than the new one
+        # so that code that does not have access to the local state path can still access
+        # the manifest.json file.
+        project = self.get_project(None)
+        project.preparer.prepare(project)
+        shutil.copytree(
+            self.args.project_dir, self._local_project_dir(state_path), dirs_exist_ok=True
+        )
+
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject":
+        kwargs = asdict(self.args)
+
+        project_dir = self._local_project_dir(state_path) if state_path else self.args.project_dir
+        return DbtProject(
+            project_dir=project_dir,
+            # allow default values on DbtProject to take precedence
+            **{k: v for k, v in kwargs.items() if v is not None and k != "project_dir"},
+        )
+
+
+@dataclass
+class RemoteGitDbtProjectManager(DbtProjectManager, Resolvable):
+    """Wraps a remote git repository containing a dbt project."""
+
+    repo_url: str
+    repo_relative_path: Annotated[
+        str,
+        Resolver.default(
+            description="The relative path to the dbt project within the repository.",
+            examples=["dbt/my_dbt_project"],
+        ),
+    ] = "."
+    token: Annotated[
+        Optional[str],
+        Resolver.default(
+            description="Token for authenticating to the provided repository.",
+            examples=["'{{ env.GITHUB_TOKEN }}'"],
+        ),
+    ] = None
+    profile: Annotated[
+        Optional[str],
+        Resolver.default(description="The profile to use for the dbt project."),
+    ] = None
+    target: Annotated[
+        Optional[str],
+        Resolver.default(
+            description="The target to use for the dbt project.",
+            examples=["dev", "prod"],
+        ),
+    ] = None
+
+    @property
+    def defs_state_discriminator(self) -> str:
+        return self.repo_url
+
+    def _get_clone_url(self) -> str:
+        # in github action environments, the token will be available automatically as an env var
+        raw_token = self.token or os.getenv("GITHUB_TOKEN")
+        if raw_token is None:
+            return self.repo_url
+        else:
+            # insert the token into the url
+            token = quote(raw_token, safe="")
+            parts = urlparse(self.repo_url)
+            return urlunparse(parts._replace(netloc=f"{token}@{parts.netloc}"))
+
+    def sync(self, state_path: Path) -> None:
+        # defer git import to avoid side-effects on import
+        from git import Repo
+
+        Repo.clone_from(self._get_clone_url(), self._local_project_dir(state_path), depth=1)
+        project = self.get_project(state_path)
+        project.preparer.prepare(project)
+
+    def get_project(self, state_path: Optional[Path]) -> "DbtProject":
+        if state_path is None:
+            raise DagsterInvalidDefinitionError(
+                "Attempted to `get_project()` on a `RemoteGitDbtProjectWrapper` without a path. "
+                "This can happen when calling unsupported methods on the `DbtProjectComponent`."
+            )
+
+        local_dir = self._local_project_dir(state_path)
+        return DbtProject(
+            project_dir=local_dir / self.repo_relative_path,
+            profile=self.profile,
+            target=self.target,
+        )

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt/include/pyproject.toml.jinja

@@ -3,7 +3,7 @@ name = "{{ project_name }}"
 version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
-requires-python = ">=3.9,<3.14"
+requires-python = ">=3.10,<3.14"
 dependencies = [
     "dagster",
     "dagster-cloud",

dagster_dbt-0.28.2/dagster_dbt/version.py

@@ -0,0 +1 @@
+__version__ = "0.28.2"

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2/dagster_dbt.egg-info}/PKG-INFO

@@ -1,22 +1,22 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.28.0
+Version: 0.28.2
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9,<3.14
+Requires-Python: >=3.10,<3.14
 License-File: LICENSE
-Requires-Dist: dagster==1.12.0
+Requires-Dist: dagster==1.12.2
 Requires-Dist: dbt-core<1.11,>=1.7
+Requires-Dist: gitpython
 Requires-Dist: Jinja2
 Requires-Dist: networkx
 Requires-Dist: orjson

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ dagster_dbt/dbt_core_version.py
 dagster_dbt/dbt_manifest.py
 dagster_dbt/dbt_manifest_asset_selection.py
 dagster_dbt/dbt_project.py
+dagster_dbt/dbt_project_manager.py
 dagster_dbt/dbt_version.py
 dagster_dbt/errors.py
 dagster_dbt/freshness_builder.py

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/dagster_dbt.egg-info/requires.txt

@@ -1,5 +1,6 @@
-dagster==1.12.0
+dagster==1.12.2
 dbt-core<1.11,>=1.7
+gitpython
 Jinja2
 networkx
 orjson

{dagster_dbt-0.28.0 → dagster_dbt-0.28.2}/setup.py

@@ -28,7 +28,6 @@ setup(
     description="A Dagster integration for dbt",
     url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt",
     classifiers=[
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
@@ -38,11 +37,12 @@ setup(
     ],
     packages=find_packages(exclude=["dagster_dbt_tests*"]),
     include_package_data=True,
-    python_requires=">=3.9,<3.14",
+    python_requires=">=3.10,<3.14",
     install_requires=[
-        "dagster==1.12.0",
+        "dagster==1.12.2",
         # Follow the version support constraints for dbt Core: https://docs.getdbt.com/docs/dbt-versions/core
         f"dbt-core>=1.7,<{DBT_CORE_VERSION_UPPER_BOUND}",
+        "gitpython",
         "Jinja2",
         "networkx",
         "orjson",

dagster_dbt-0.28.0/dagster_dbt/components/dbt_project/scaffolder.py

@@ -1,50 +0,0 @@
-import os
-from pathlib import Path
-from typing import Optional
-
-import dagster._check as check
-from dagster._core.errors import DagsterInvalidInvocationError
-from dagster.components.component.component_scaffolder import Scaffolder
-from dagster.components.component_scaffolding import scaffold_component
-from dagster.components.scaffold.scaffold import ScaffoldRequest
-from pydantic import BaseModel, Field
-
-
-class DbtScaffoldParams(BaseModel):
-    init: bool = Field(default=False)
-    project_path: Optional[str] = None
-
-
-class DbtProjectComponentScaffolder(Scaffolder[DbtScaffoldParams]):
-    @classmethod
-    def get_scaffold_params(cls) -> type[DbtScaffoldParams]:
-        return DbtScaffoldParams
-
-    def scaffold(self, request: ScaffoldRequest[DbtScaffoldParams]) -> None:
-        project_root = request.project_root or os.getcwd()
-        if request.params.project_path:
-            project_root_tmpl = "{{ project_root }}"
-            rel_path = os.path.relpath(request.params.project_path, start=project_root)
-            path_str = f"{project_root_tmpl}/{rel_path}"
-
-        elif request.params.init:
-            try:
-                from dbt.cli.main import dbtRunner
-            except ImportError:
-                raise DagsterInvalidInvocationError(
-                    "dbt-core is not installed. Please install dbt to scaffold this component."
-                )
-
-            dbtRunner().invoke(["init"])
-            subpaths = [
-                path
-                for path in Path(project_root).iterdir()
-                if path.is_dir() and path.name != "logs"
-            ]
-            check.invariant(len(subpaths) == 1, "Expected exactly one subpath to be created.")
-            # this path should be relative to this directory
-            path_str = subpaths[0].name
-        else:
-            path_str = None
-
-        scaffold_component(request, {"project": path_str})

dagster_dbt-0.28.0/dagster_dbt/version.py

@@ -1 +0,0 @@
-__version__ = "0.28.0"