dagster-dbt 0.23.3__py3-none-any.whl → 0.28.4__py3-none-any.whl

dagster_dbt/__init__.py

@@ -1,5 +1,6 @@
-from .asset_decorator import dbt_assets as dbt_assets
-from .asset_utils import (
+from dagster_dbt.asset_decorator import dbt_assets as dbt_assets
+from dagster_dbt.asset_specs import build_dbt_asset_specs as build_dbt_asset_specs
+from dagster_dbt.asset_utils import (
     build_dbt_asset_selection as build_dbt_asset_selection,
     build_schedule_from_dbt_selection as build_schedule_from_dbt_selection,
     default_group_from_dbt_resource_props as default_group_from_dbt_resource_props,
@@ -9,7 +10,7 @@ from .asset_utils import (
     get_asset_keys_by_output_name_for_source as get_asset_keys_by_output_name_for_source,
     group_from_dbt_resource_props_fallback_to_directory as group_from_dbt_resource_props_fallback_to_directory,
 )
-from .cloud import (
+from dagster_dbt.cloud import (
     DbtCloudClientResource as DbtCloudClientResource,
     DbtCloudOutput as DbtCloudOutput,
     DbtCloudResource as DbtCloudResource,
@@ -17,25 +18,39 @@ from .cloud import (
     dbt_cloud_run_op as dbt_cloud_run_op,
     load_assets_from_dbt_cloud_job as load_assets_from_dbt_cloud_job,
 )
-from .core import (
-    DbtCliEventMessage as DbtCliEventMessage,
-    DbtCliInvocation as DbtCliInvocation,
-    DbtCliResource as DbtCliResource,
+from dagster_dbt.cloud_v2 import (
+    DbtCloudCredentials as DbtCloudCredentials,
+    DbtCloudWorkspace as DbtCloudWorkspace,
+    build_dbt_cloud_polling_sensor as build_dbt_cloud_polling_sensor,
+    dbt_cloud_assets as dbt_cloud_assets,
+    load_dbt_cloud_asset_specs as load_dbt_cloud_asset_specs,
+    load_dbt_cloud_check_specs as load_dbt_cloud_check_specs,
 )
-from .dagster_dbt_translator import (
+from dagster_dbt.components.dbt_project.component import DbtProjectComponent as DbtProjectComponent
+from dagster_dbt.core.dbt_cli_event import DbtCliEventMessage as DbtCliEventMessage
+from dagster_dbt.core.dbt_cli_invocation import DbtCliInvocation as DbtCliInvocation
+from dagster_dbt.core.resource import DbtCliResource as DbtCliResource
+from dagster_dbt.dagster_dbt_translator import (
     DagsterDbtTranslator as DagsterDbtTranslator,
     DagsterDbtTranslatorSettings as DagsterDbtTranslatorSettings,
-    KeyPrefixDagsterDbtTranslator as KeyPrefixDagsterDbtTranslator,
 )
-from .dbt_manifest_asset_selection import DbtManifestAssetSelection as DbtManifestAssetSelection
-from .dbt_project import DbtProject as DbtProject
-from .errors import (
+from dagster_dbt.dbt_manifest_asset_selection import (
+    DbtManifestAssetSelection as DbtManifestAssetSelection,
+)
+from dagster_dbt.dbt_project import (
+    DagsterDbtProjectPreparer as DagsterDbtProjectPreparer,
+    DbtProject as DbtProject,
+    DbtProjectPreparer as DbtProjectPreparer,
+)
+from dagster_dbt.errors import (
     DagsterDbtCliRuntimeError as DagsterDbtCliRuntimeError,
-    DagsterDbtCliUnexpectedOutputError as DagsterDbtCliUnexpectedOutputError,
     DagsterDbtCloudJobInvariantViolationError as DagsterDbtCloudJobInvariantViolationError,
     DagsterDbtError as DagsterDbtError,
 )
-from .version import __version__ as __version__
+from dagster_dbt.freshness_builder import (
+    build_freshness_checks_from_dbt_assets as build_freshness_checks_from_dbt_assets,
+)
+from dagster_dbt.version import __version__ as __version__
 
 # isort: split
 
@@ -43,155 +58,41 @@ from .version import __version__ as __version__
 # ##### DYNAMIC IMPORTS
 # ########################
 import importlib
-from typing import TYPE_CHECKING, Any, Mapping, Sequence, Tuple
+from collections.abc import Mapping, Sequence
+from typing import Any, Final, Tuple  # noqa: F401, UP035
 
 from dagster._annotations import deprecated
-from dagster._core.libraries import DagsterLibraryRegistry
 from dagster._utils.warnings import deprecation_warning
-from typing_extensions import Final
+from dagster_shared.libraries import DagsterLibraryRegistry
+
+from dagster_dbt.compat import DBT_PYTHON_VERSION
 
 DagsterLibraryRegistry.register("dagster-dbt", __version__)
 
-if TYPE_CHECKING:
-    ##### EXAMPLE
-    # from dagster.some.module import (
-    #     Foo as Foo,
-    # )
-
-    # isort: split
-    ##### Deprecating DbtCliClientResource
-    from .core import (
-        DbtCliClientResource as DbtCliClientResource,
-        DbtCliOutput as DbtCliOutput,
-        dbt_cli_resource as dbt_cli_resource,
-    )
-    from .dbt_resource import DbtResource as DbtResource
-    from .errors import (
-        DagsterDbtCliFatalRuntimeError as DagsterDbtCliFatalRuntimeError,
-        DagsterDbtCliHandledRuntimeError as DagsterDbtCliHandledRuntimeError,
-        DagsterDbtCliOutputsNotFoundError as DagsterDbtCliOutputsNotFoundError,
-    )
-    from .types import DbtOutput as DbtOutput
-
-    ##### Deprecating dbt ops
-    # isort: split
-    from .ops import (
-        dbt_build_op as dbt_build_op,
-        dbt_compile_op as dbt_compile_op,
-        dbt_docs_generate_op as dbt_docs_generate_op,
-        dbt_ls_op as dbt_ls_op,
-        dbt_run_op as dbt_run_op,
-        dbt_seed_op as dbt_seed_op,
-        dbt_snapshot_op as dbt_snapshot_op,
-        dbt_test_op as dbt_test_op,
+if DBT_PYTHON_VERSION is not None:
+    DagsterLibraryRegistry.register(
+        "dbt-core", DBT_PYTHON_VERSION.base_version, is_dagster_package=False
     )
+else:
+    DagsterLibraryRegistry.register("dbt-fusion", "unknown", is_dagster_package=False)
 
-    ##### Deprecating load_assets_from_XXX
-    # isort: split
-    from .asset_defs import (
-        load_assets_from_dbt_manifest as load_assets_from_dbt_manifest,
-        load_assets_from_dbt_project as load_assets_from_dbt_project,
-    )
 
-_DEPRECATED: Final[Mapping[str, Tuple[str, str, str]]] = {
+_DEPRECATED: Final[Mapping[str, tuple[str, str, str]]] = {
     ##### EXAMPLE
     # "Foo": (
     #     "dagster.some.module",
     #     "1.1.0",  # breaking version
     #     "Use Bar instead.",
     # ),
-    **{
-        value: (
-            module,
-            "0.24.0",
-            additional_warn_text,
-        )
-        for value, module, additional_warn_text in [
-            (
-                "DbtCliClientResource",
-                "dagster_dbt.core",
-                "Use DbtCliResource instead",
-            ),
-            ("DbtCliOutput", "dagster_dbt.core", None),
-            (
-                "dbt_cli_resource",
-                "dagster_dbt.core",
-                "Use DbtCliResource instead",
-            ),
-            (
-                "DbtResource",
-                "dagster_dbt.dbt_resource",
-                "Use DbtCliResource instead",
-            ),
-            ("DagsterDbtCliFatalRuntimeError", "dagster_dbt.errors", None),
-            ("DagsterDbtCliHandledRuntimeError", "dagster_dbt.errors", None),
-            ("DagsterDbtCliOutputsNotFoundError", "dagster_dbt.errors", None),
-            ("DbtOutput", "dagster_dbt.types", None),
-        ]
-    },
-    **{
-        value: (
-            module,
-            "0.24.0",
-            (
-                "Use the @dbt_assets decorator, DbtCliResource, and DagsterDbtTranslator instead.\n\n"
-                "For examples on how to use @dbt_assets and DbtCliResource to execute commands like"
-                " `dbt run` or `dbt build` on your dbt project, see our API docs:"
-                " https://docs.dagster.io/_apidocs/libraries/dagster-dbt#dagster_dbt.dbt_assets.\n\n"
-                "For examples on how to customize your dbt assets using DagsterDbtTranslator"
-                " see the reference:"
-                " https://docs.dagster.io/integrations/dbt/reference#understanding-asset-definition-attributes"
-                f"{additional_text}"
-            ),
-        )
-        for value, module, additional_text in [
-            (
-                "load_assets_from_dbt_manifest",
-                "dagster_dbt.asset_defs",
-                "",
-            ),
-            (
-                "load_assets_from_dbt_project",
-                "dagster_dbt.asset_defs",
-                (
-                    ".\n\nTo generate a dbt manifest for @dbt_assets at run time using `dbt parse`,"
-                    " see the reference:"
-                    " https://docs.dagster.io/integrations/dbt/reference#loading-dbt-models-from-a-dbt-project"
-                ),
-            ),
-        ]
-    },
 }
 
-_DEPRECATED_WARNING: Final[Mapping[str, Tuple[str, str, str]]] = {
+_DEPRECATED_WARNING: Final[Mapping[str, tuple[str, str, str]]] = {
     ##### EXAMPLE
     # "Foo": (
     #     "dagster.some.module",
     #     "1.1.0",  # breaking version
     #     "Use Bar instead.",
     # ),
-    **{
-        value: (
-            module,
-            "0.24.0",
-            (
-                "Use the @op decorator and DbtCliResource instead.\n\n"
-                "For examples on how to use @op and DbtCliResource to execute commands like"
-                " `dbt run` or `dbt build`, see our API docs:"
-                " https://docs.dagster.io/_apidocs/libraries/dagster-dbt#dagster_dbt.DbtCliResource.cli"
-            ),
-        )
-        for value, module in [
-            ("dbt_build_op", "dagster_dbt.ops"),
-            ("dbt_compile_op", "dagster_dbt.ops"),
-            ("dbt_docs_generate_op", "dagster_dbt.ops"),
-            ("dbt_ls_op", "dagster_dbt.ops"),
-            ("dbt_run_op", "dagster_dbt.ops"),
-            ("dbt_seed_op", "dagster_dbt.ops"),
-            ("dbt_snapshot_op", "dagster_dbt.ops"),
-            ("dbt_test_op", "dagster_dbt.ops"),
-        ]
-    },
 }
 
 

dagster_dbt/asset_decorator.py

@@ -1,72 +1,48 @@
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    FrozenSet,
-    Mapping,
-    Optional,
-    Sequence,
-    Set,
-    Tuple,
-)
+from collections.abc import Callable, Mapping
+from typing import Any, Optional
 
 from dagster import (
-    AssetCheckKey,
-    AssetCheckSpec,
-    AssetDep,
-    AssetKey,
-    AssetOut,
     AssetsDefinition,
     BackfillPolicy,
     DagsterInvalidDefinitionError,
-    Nothing,
     PartitionsDefinition,
+    RetryPolicy,
     TimeWindowPartitionsDefinition,
     multi_asset,
 )
-from dagster._utils.warnings import (
-    experimental_warning,
-)
+from dagster._utils.warnings import suppress_dagster_warnings
 
-from .asset_utils import (
+from dagster_dbt.asset_utils import (
     DAGSTER_DBT_EXCLUDE_METADATA_KEY,
-    DAGSTER_DBT_MANIFEST_METADATA_KEY,
     DAGSTER_DBT_SELECT_METADATA_KEY,
-    DAGSTER_DBT_TRANSLATOR_METADATA_KEY,
-    default_asset_check_fn,
-    default_code_version_fn,
-    get_deps,
-    has_self_dependency,
-)
-from .dagster_dbt_translator import DagsterDbtTranslator, DbtManifestWrapper, validate_translator
-from .dbt_manifest import DbtManifestParam, validate_manifest
-from .utils import (
-    ASSET_RESOURCE_TYPES,
-    dagster_name_fn,
-    get_dbt_resource_props_by_dbt_unique_id_from_manifest,
-    select_unique_ids_from_manifest,
-)
-
-DUPLICATE_ASSET_KEY_ERROR_MESSAGE = (
-    "The following dbt resources are configured with identical Dagster asset keys."
-    " Please ensure that each dbt resource generates a unique Dagster asset key."
-    " See the reference for configuring Dagster asset keys for your dbt project:"
-    " https://docs.dagster.io/integrations/dbt/reference#customizing-asset-keys."
+    DAGSTER_DBT_SELECTOR_METADATA_KEY,
+    DBT_DEFAULT_EXCLUDE,
+    DBT_DEFAULT_SELECT,
+    DBT_DEFAULT_SELECTOR,
+    build_dbt_specs,
 )
+from dagster_dbt.dagster_dbt_translator import DagsterDbtTranslator, validate_translator
+from dagster_dbt.dbt_manifest import DbtManifestParam, validate_manifest
+from dagster_dbt.dbt_project import DbtProject
 
 
+@suppress_dagster_warnings
 def dbt_assets(
     *,
     manifest: DbtManifestParam,
-    select: str = "fqn:*",
-    exclude: Optional[str] = None,
+    select: str = DBT_DEFAULT_SELECT,
+    exclude: Optional[str] = DBT_DEFAULT_EXCLUDE,
+    selector: Optional[str] = DBT_DEFAULT_SELECTOR,
     name: Optional[str] = None,
     io_manager_key: Optional[str] = None,
     partitions_def: Optional[PartitionsDefinition] = None,
     dagster_dbt_translator: Optional[DagsterDbtTranslator] = None,
     backfill_policy: Optional[BackfillPolicy] = None,
     op_tags: Optional[Mapping[str, Any]] = None,
-    required_resource_keys: Optional[Set[str]] = None,
+    required_resource_keys: Optional[set[str]] = None,
+    project: Optional[DbtProject] = None,
+    retry_policy: Optional[RetryPolicy] = None,
+    pool: Optional[str] = None,
 ) -> Callable[[Callable[..., Any]], AssetsDefinition]:
     """Create a definition for how to compute a set of dbt resources, described by a manifest.json.
     When invoking dbt commands using :py:class:`~dagster_dbt.DbtCliResource`'s
@@ -82,6 +58,8 @@ def dbt_assets(
             to include. Defaults to ``fqn:*``.
         exclude (Optional[str]): A dbt selection string for the models in a project that you want
             to exclude. Defaults to "".
+        selector (Optional[str]): A dbt selector for the models in a project that you want to
+            include. Cannot be combined with select or exclude. Defaults to None.
         name (Optional[str]): The name of the op.
         io_manager_key (Optional[str]): The IO manager key that will be set on each of the returned
             assets. When other ops are downstream of the loaded assets, the IOManager specified
@@ -98,6 +76,12 @@ def dbt_assets(
             are not strings will be json encoded and must meet the criteria that
             `json.loads(json.dumps(value)) == value`.
         required_resource_keys (Optional[Set[str]]): Set of required resource handles.
+        project (Optional[DbtProject]): A DbtProject instance which provides a pointer to the dbt
+            project location and manifest. Not required, but needed to attach code references from
+            model code to Dagster assets.
+        retry_policy (Optional[RetryPolicy]): The retry policy for the op that computes the asset.
+        pool (Optional[str]): A string that identifies the concurrency pool that governs the dbt
+            assets' execution.
 
     Examples:
         Running ``dbt build`` for a dbt project:
@@ -329,26 +313,14 @@ def dbt_assets(
     dagster_dbt_translator = validate_translator(dagster_dbt_translator or DagsterDbtTranslator())
     manifest = validate_manifest(manifest)
 
-    unique_ids = select_unique_ids_from_manifest(
-        select=select, exclude=exclude or "", manifest_json=manifest
-    )
-    node_info_by_dbt_unique_id = get_dbt_resource_props_by_dbt_unique_id_from_manifest(manifest)
-    dbt_unique_id_deps = get_deps(
-        dbt_nodes=node_info_by_dbt_unique_id,
-        selected_unique_ids=unique_ids,
-        asset_resource_types=ASSET_RESOURCE_TYPES,
-    )
-    (
-        deps,
-        outs,
-        internal_asset_deps,
-        check_specs,
-    ) = get_dbt_multi_asset_args(
-        dbt_nodes=node_info_by_dbt_unique_id,
-        dbt_unique_id_deps=dbt_unique_id_deps,
-        io_manager_key=io_manager_key,
+    specs, check_specs = build_dbt_specs(
+        translator=dagster_dbt_translator,
         manifest=manifest,
-        dagster_dbt_translator=dagster_dbt_translator,
+        select=select,
+        exclude=exclude or DBT_DEFAULT_EXCLUDE,
+        selector=selector or DBT_DEFAULT_SELECTOR,
+        io_manager_key=io_manager_key,
+        project=project,
     )
 
     if op_tags and DAGSTER_DBT_SELECT_METADATA_KEY in op_tags:
@@ -363,9 +335,16 @@ def dbt_assets(
             " with op_tags"
         )
 
+    if op_tags and DAGSTER_DBT_SELECTOR_METADATA_KEY in op_tags:
+        raise DagsterInvalidDefinitionError(
+            f"To specify a dbt selector, use the 'selector' argument, not '{DAGSTER_DBT_SELECTOR_METADATA_KEY}'"
+            " with op_tags"
+        )
+
     resolved_op_tags = {
         **({DAGSTER_DBT_SELECT_METADATA_KEY: select} if select else {}),
         **({DAGSTER_DBT_EXCLUDE_METADATA_KEY: exclude} if exclude else {}),
+        **({DAGSTER_DBT_SELECTOR_METADATA_KEY: selector} if selector else {}),
         **(op_tags if op_tags else {}),
     }
 
@@ -377,175 +356,15 @@ def dbt_assets(
         backfill_policy = BackfillPolicy.single_run()
 
     return multi_asset(
-        outs=outs,
         name=name,
-        internal_asset_deps=internal_asset_deps,
-        deps=deps,
+        specs=specs,
+        check_specs=check_specs,
+        can_subset=True,
         required_resource_keys=required_resource_keys,
-        compute_kind="dbt",
         partitions_def=partitions_def,
-        can_subset=True,
         op_tags=resolved_op_tags,
-        check_specs=check_specs,
         backfill_policy=backfill_policy,
+        retry_policy=retry_policy,
+        pool=pool,
+        allow_arbitrary_check_specs=True,
     )
-
-
-def get_dbt_multi_asset_args(
-    dbt_nodes: Mapping[str, Any],
-    dbt_unique_id_deps: Mapping[str, FrozenSet[str]],
-    io_manager_key: Optional[str],
-    manifest: Mapping[str, Any],
-    dagster_dbt_translator: DagsterDbtTranslator,
-) -> Tuple[
-    Sequence[AssetDep],
-    Dict[str, AssetOut],
-    Dict[str, Set[AssetKey]],
-    Sequence[AssetCheckSpec],
-]:
-    deps: Set[AssetDep] = set()
-    outs: Dict[str, AssetOut] = {}
-    internal_asset_deps: Dict[str, Set[AssetKey]] = {}
-    check_specs_by_key: Dict[AssetCheckKey, AssetCheckSpec] = {}
-
-    dbt_unique_id_and_resource_types_by_asset_key: Dict[AssetKey, Tuple[Set[str], Set[str]]] = {}
-    dbt_group_resource_props_by_group_name: Dict[str, Dict[str, Any]] = {
-        dbt_group_resource_props["name"]: dbt_group_resource_props
-        for dbt_group_resource_props in manifest["groups"].values()
-    }
-
-    for unique_id, parent_unique_ids in dbt_unique_id_deps.items():
-        dbt_resource_props = dbt_nodes[unique_id]
-        dbt_group_resource_props = dbt_group_resource_props_by_group_name.get(
-            dbt_resource_props.get("group")
-        )
-
-        output_name = dagster_name_fn(dbt_resource_props)
-        asset_key = dagster_dbt_translator.get_asset_key(dbt_resource_props)
-
-        unique_ids_for_asset_key, resource_types_for_asset_key = (
-            dbt_unique_id_and_resource_types_by_asset_key.setdefault(asset_key, (set(), set()))
-        )
-        unique_ids_for_asset_key.add(unique_id)
-        resource_types_for_asset_key.add(dbt_resource_props["resource_type"])
-
-        outs[output_name] = AssetOut(
-            key=asset_key,
-            dagster_type=Nothing,
-            io_manager_key=io_manager_key,
-            description=dagster_dbt_translator.get_description(dbt_resource_props),
-            is_required=False,
-            metadata={
-                **dagster_dbt_translator.get_metadata(dbt_resource_props),
-                DAGSTER_DBT_MANIFEST_METADATA_KEY: DbtManifestWrapper(manifest=manifest),
-                DAGSTER_DBT_TRANSLATOR_METADATA_KEY: dagster_dbt_translator,
-            },
-            owners=dagster_dbt_translator.get_owners(
-                {
-                    **dbt_resource_props,
-                    **({"group": dbt_group_resource_props} if dbt_group_resource_props else {}),
-                }
-            ),
-            tags=dagster_dbt_translator.get_tags(dbt_resource_props),
-            group_name=dagster_dbt_translator.get_group_name(dbt_resource_props),
-            code_version=default_code_version_fn(dbt_resource_props),
-            freshness_policy=dagster_dbt_translator.get_freshness_policy(dbt_resource_props),
-            auto_materialize_policy=dagster_dbt_translator.get_auto_materialize_policy(
-                dbt_resource_props
-            ),
-        )
-
-        test_unique_ids = [
-            child_unique_id
-            for child_unique_id in manifest["child_map"][unique_id]
-            if child_unique_id.startswith("test")
-        ]
-        for test_unique_id in test_unique_ids:
-            check_spec = default_asset_check_fn(
-                manifest, dbt_nodes, dagster_dbt_translator, asset_key, test_unique_id
-            )
-            if check_spec:
-                check_specs_by_key[check_spec.key] = check_spec
-
-        # Translate parent unique ids to dependencies
-        output_internal_deps = internal_asset_deps.setdefault(output_name, set())
-        for parent_unique_id in parent_unique_ids:
-            dbt_parent_resource_props = dbt_nodes[parent_unique_id]
-            parent_asset_key = dagster_dbt_translator.get_asset_key(dbt_parent_resource_props)
-            parent_partition_mapping = dagster_dbt_translator.get_partition_mapping(
-                dbt_resource_props,
-                dbt_parent_resource_props=dbt_parent_resource_props,
-            )
-
-            parent_unique_ids_for_asset_key, parent_resource_types_for_asset_key = (
-                dbt_unique_id_and_resource_types_by_asset_key.setdefault(
-                    parent_asset_key, (set(), set())
-                )
-            )
-            parent_unique_ids_for_asset_key.add(parent_unique_id)
-            parent_resource_types_for_asset_key.add(dbt_parent_resource_props["resource_type"])
-
-            if parent_partition_mapping:
-                experimental_warning("DagsterDbtTranslator.get_partition_mapping")
-
-            # Add this parent as an internal dependency
-            output_internal_deps.add(parent_asset_key)
-
-            # Mark this parent as an input if it has no dependencies
-            if parent_unique_id not in dbt_unique_id_deps:
-                deps.add(
-                    AssetDep(
-                        asset=parent_asset_key,
-                        partition_mapping=parent_partition_mapping,
-                    )
-                )
-
-        self_partition_mapping = dagster_dbt_translator.get_partition_mapping(
-            dbt_resource_props,
-            dbt_parent_resource_props=dbt_resource_props,
-        )
-        if self_partition_mapping and has_self_dependency(dbt_resource_props):
-            experimental_warning("+meta.dagster.has_self_dependency")
-
-            deps.add(
-                AssetDep(
-                    asset=asset_key,
-                    partition_mapping=self_partition_mapping,
-                )
-            )
-            output_internal_deps.add(asset_key)
-
-    dbt_unique_ids_by_duplicate_asset_key = {
-        asset_key: sorted(unique_ids)
-        for asset_key, (
-            unique_ids,
-            resource_types,
-        ) in dbt_unique_id_and_resource_types_by_asset_key.items()
-        if len(unique_ids) != 1
-        and not (
-            resource_types == set(["source"])
-            and dagster_dbt_translator.settings.enable_duplicate_source_asset_keys
-        )
-    }
-    if dbt_unique_ids_by_duplicate_asset_key:
-        error_messages = []
-        for asset_key, unique_ids in dbt_unique_ids_by_duplicate_asset_key.items():
-            formatted_ids = []
-            for id in unique_ids:
-                unique_id_file_path = dbt_nodes[id]["original_file_path"]
-                formatted_ids.append(f"  - `{id}` ({unique_id_file_path})")
-
-            error_messages.append(
-                "\n".join(
-                    [
-                        f"The following dbt resources have the asset key `{asset_key.path}`:",
-                        *formatted_ids,
-                    ]
-                )
-            )
-
-        raise DagsterInvalidDefinitionError(
-            "\n\n".join([DUPLICATE_ASSET_KEY_ERROR_MESSAGE, *error_messages])
-        )
-
-    return list(deps), outs, internal_asset_deps, list(check_specs_by_key.values())

dagster_dbt/asset_specs.py

@@ -0,0 +1,65 @@
+from collections.abc import Sequence
+from typing import Optional
+
+from dagster import AssetSpec
+
+from dagster_dbt.asset_utils import (
+    DBT_DEFAULT_EXCLUDE,
+    DBT_DEFAULT_SELECT,
+    DBT_DEFAULT_SELECTOR,
+    build_dbt_specs,
+)
+from dagster_dbt.dagster_dbt_translator import DagsterDbtTranslator, validate_translator
+from dagster_dbt.dbt_manifest import DbtManifestParam, validate_manifest
+from dagster_dbt.dbt_project import DbtProject
+
+
+def build_dbt_asset_specs(
+    *,
+    manifest: DbtManifestParam,
+    dagster_dbt_translator: Optional[DagsterDbtTranslator] = None,
+    select: str = DBT_DEFAULT_SELECT,
+    exclude: Optional[str] = DBT_DEFAULT_EXCLUDE,
+    selector: Optional[str] = DBT_DEFAULT_SELECTOR,
+    project: Optional[DbtProject] = None,
+) -> Sequence[AssetSpec]:
+    """Build a list of asset specs from a set of dbt resources selected from a dbt manifest.
+
+    Args:
+        manifest (Union[Mapping[str, Any], str, Path]): The contents of a manifest.json file
+            or the path to a manifest.json file. A manifest.json contains a representation of a
+            dbt project (models, tests, macros, etc). We use this representation to create
+            corresponding Dagster asset specs.
+        dagster_dbt_translator (Optional[DagsterDbtTranslator]): Allows customizing how to map
+            dbt models, seeds, etc. to asset keys and asset metadata.
+        select (str): A dbt selection string for the models in a project that you want
+            to include. Defaults to ``fqn:*``.
+        exclude (Optional[str]): A dbt selection string for the models in a project that you want
+            to exclude. Defaults to "".
+        selector (Optional[str]): A dbt selector for the models in a project that you want
+            to include. Defaults to None.
+        project (Optional[DbtProject]): A DbtProject instance which provides a pointer to the dbt
+            project location and manifest. Not required, but needed to attach code references from
+            model code to Dagster assets.
+
+    Returns:
+        Sequence[AssetSpec]: A list of asset specs.
+    """
+    manifest = validate_manifest(manifest)
+    dagster_dbt_translator = validate_translator(dagster_dbt_translator or DagsterDbtTranslator())
+
+    specs, _ = build_dbt_specs(
+        manifest=manifest,
+        translator=dagster_dbt_translator,
+        select=select,
+        exclude=exclude or DBT_DEFAULT_EXCLUDE,
+        selector=selector or DBT_DEFAULT_SELECTOR,
+        io_manager_key=None,
+        project=project,
+    )
+
+    return [
+        # Allow specs to be represented as external assets by adhering to external asset invariants.
+        spec.replace_attributes(skippable=False, code_version=None)
+        for spec in specs
+    ]