dagster-dbt 0.23.3__py3-none-any.whl → 0.28.4__py3-none-any.whl

dagster_dbt/errors.py

@@ -1,98 +1,15 @@
-import warnings
 from abc import ABC
-from typing import Any, Mapping, Optional, Sequence
 
-from dagster import (
-    DagsterInvariantViolationError,
-    Failure,
-    MetadataValue,
-    _check as check,
-)
+from dagster import DagsterInvariantViolationError, Failure
 
 
 class DagsterDbtError(Failure, ABC):
     """The base exception of the ``dagster-dbt`` library."""
 
 
-class DagsterDbtCliUnexpectedOutputError(DagsterDbtError):
-    """Represents an error when parsing the output of a dbt CLI command."""
-
-    invalid_line_nos: Sequence[int]
-
-    def __init__(self, invalid_line_nos: Sequence[int]):
-        check.sequence_param(invalid_line_nos, "invalid_line_nos", int)
-        line_nos_str = ", ".join(map(str, invalid_line_nos))
-        description = f"dbt CLI emitted unexpected output on lines {line_nos_str}"
-        metadata = {
-            "Invalid CLI Output Line Numbers": MetadataValue.json({"line_nos": invalid_line_nos})
-        }
-        super().__init__(description, metadata=metadata)
-        self.invalid_line_nos = invalid_line_nos
-
-
 class DagsterDbtCliRuntimeError(DagsterDbtError, ABC):
     """Represents an error while executing a dbt CLI command."""
 
-    def __init__(
-        self,
-        description: str,
-        logs: Optional[Sequence[Mapping[str, Any]]] = None,
-        raw_output: Optional[str] = None,
-        messages: Optional[Sequence[str]] = None,
-    ):
-        if logs is not None:
-            warnings.warn(
-                "`logs` is a deprecated argument to DagsterDbtCliRuntimeError and will be discarded"
-            )
-        if raw_output is not None:
-            warnings.warn(
-                "`raw_output` is a deprecated argument to DagsterDbtCliRuntimeError and will be"
-                " discarded"
-            )
-        metadata = {"Parsed CLI Messages": "\n".join(messages or [])}
-        super().__init__(description, metadata=metadata)
-
-
-class DagsterDbtCliHandledRuntimeError(DagsterDbtCliRuntimeError):
-    """Represents a model error reported by the dbt CLI at runtime (return code 1)."""
-
-    def __init__(
-        self,
-        logs: Optional[Sequence[Mapping[str, Any]]] = None,
-        raw_output: Optional[str] = None,
-        messages: Optional[Sequence[str]] = None,
-    ):
-        super().__init__("Handled error in the dbt CLI (return code 1)", logs, raw_output, messages)
-
-
-class DagsterDbtCliFatalRuntimeError(DagsterDbtCliRuntimeError):
-    """Represents a fatal error in the dbt CLI (return code 2)."""
-
-    def __init__(
-        self,
-        logs: Optional[Sequence[Mapping[str, Any]]] = None,
-        raw_output: Optional[str] = None,
-        messages: Optional[Sequence[str]] = None,
-    ):
-        super().__init__(
-            "Fatal error in the dbt CLI (return code 2): " + " ".join(messages or []),
-            logs,
-            raw_output,
-            messages,
-        )
-
-
-class DagsterDbtCliOutputsNotFoundError(DagsterDbtError):
-    """Represents a problem in finding the ``target/run_results.json`` artifact when executing a dbt
-    CLI command.
-
-    For more details on ``target/run_results.json``, see
-    https://docs.getdbt.com/reference/dbt-artifacts#run_resultsjson.
-    """
-
-    def __init__(self, path: str):
-        super().__init__(f"Expected to find file at path {path}")
-
 
 class DagsterDbtCloudJobInvariantViolationError(DagsterDbtError, DagsterInvariantViolationError):
     """Represents an error when a dbt Cloud job is not supported by the ``dagster-dbt`` library."""
@@ -102,5 +19,13 @@ class DagsterDbtProjectNotFoundError(DagsterDbtError):
     """Error when the specified project directory can not be found."""
 
 
+class DagsterDbtProfilesDirectoryNotFoundError(DagsterDbtError):
+    """Error when the specified profiles directory can not be found."""
+
+
 class DagsterDbtManifestNotFoundError(DagsterDbtError):
     """Error when we expect manifest.json to generated already but it is absent."""
+
+
+class DagsterDbtProjectYmlFileNotFoundError(DagsterDbtError):
+    """Error when a dbt_project.yml file can not be found in the specified project directory."""

dagster_dbt/freshness_builder.py

@@ -0,0 +1,147 @@
+import datetime
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any
+
+from dagster import (
+    AssetsDefinition,
+    _check as check,
+)
+from dagster._core.definitions.asset_checks.asset_check_factories.freshness_checks.last_update import (
+    build_last_update_freshness_checks,
+)
+from dagster._core.definitions.asset_checks.asset_check_factories.freshness_checks.time_partition import (
+    build_time_partition_freshness_checks,
+)
+from dagster._core.definitions.asset_checks.asset_check_factories.utils import (
+    DEFAULT_FRESHNESS_SEVERITY,
+    asset_to_keys_iterable,
+    assets_to_keys,
+    ensure_no_duplicate_assets,
+)
+from dagster._core.definitions.asset_checks.asset_check_spec import AssetCheckSeverity
+from dagster._core.definitions.asset_checks.asset_checks_definition import AssetChecksDefinition
+from dagster._core.definitions.partitions.definition import TimeWindowPartitionsDefinition
+from dagster._core.errors import DagsterInvariantViolationError
+
+if TYPE_CHECKING:
+    from dagster import AssetKey
+
+from dagster._symbol_annotations.lifecycle import superseded
+
+from dagster_dbt.asset_utils import (
+    get_asset_keys_to_resource_props,
+    get_manifest_and_translator_from_dbt_assets,
+)
+
+
+@superseded(
+    additional_warn_text="Create `FreshnessPolicy` objects for your dbt models by overriding `get_asset_spec` "
+    "in your `DagsterDbtTranslator`, or by updating the `translation` configuration of your `DbtProjectComponent` instead."
+)
+def build_freshness_checks_from_dbt_assets(
+    *,
+    dbt_assets: Sequence[AssetsDefinition],
+) -> Sequence[AssetChecksDefinition]:
+    """Returns a sequence of freshness checks constructed from the provided dbt assets.
+
+    Freshness checks can be configured on a per-model basis in the model schema configuration.
+
+    For assets which are not partitioned based on time, the freshness check configuration mirrors
+    that of the :py:func:`build_last_update_freshness_checks` function. `lower_bound_delta` is provided in
+    terms of seconds, and `deadline_cron` is optional.
+
+    For time-partitioned assets, the freshness check configuration mirrors that of the
+    :py:func:`build_time_partition_freshness_checks` function.
+
+    Below is example of configuring a non-time-partitioned dbt asset with a freshness check.
+    This code would be placed in the schema.yml file for the dbt model.
+
+    .. code-block:: YAML
+
+        models:
+          - name: customers
+            ...
+            meta:
+              dagster:
+                freshness_check:
+                  lower_bound_delta_seconds: 86400 # 1 day
+                  deadline_cron: "0 0 * * *" # Optional
+                  severity: "WARN" # Optional, defaults to "WARN"
+
+    Below is an example of configuring a time-partitioned dbt asset with a freshness check.
+    This code would be placed in the schema.yml file for the dbt model.
+
+    .. code-block:: yaml
+
+        models:
+          - name: customers
+            ...
+            meta:
+              dagster:
+                freshness_check:
+                  deadline_cron: "0 0 * * *"
+                  severity: "WARN" # Optional, defaults to "WARN"
+
+    Args:
+        dbt_assets (Sequence[AssetsDefinition]): A sequence of dbt assets to construct freshness
+            checks from.
+
+    Returns:
+        Sequence[AssetChecksDefinition]: A sequence of asset checks definitions representing the
+            freshness checks for the provided dbt assets.
+    """
+    freshness_checks = []
+    dbt_assets = check.sequence_param(dbt_assets, "dbt_assets", AssetsDefinition)
+    ensure_no_duplicate_assets(dbt_assets)
+    asset_key_to_assets_def: dict[AssetKey, AssetsDefinition] = {}
+    asset_key_to_resource_props: Mapping[AssetKey, Mapping[str, Any]] = {}
+    for assets_def in dbt_assets:
+        manifest, translator, _ = get_manifest_and_translator_from_dbt_assets([assets_def])
+        asset_key_to_resource_props_for_def = get_asset_keys_to_resource_props(manifest, translator)
+        for asset_key in asset_to_keys_iterable(assets_def):
+            if asset_key not in asset_key_to_resource_props_for_def:
+                raise DagsterInvariantViolationError(
+                    f"Could not find dbt resource properties for asset key {asset_key.to_user_string()}."
+                )
+            asset_key_to_assets_def[asset_key] = assets_def
+            asset_key_to_resource_props[asset_key] = asset_key_to_resource_props_for_def[asset_key]
+    for asset_key in assets_to_keys(dbt_assets):
+        dbt_resource_props = asset_key_to_resource_props[asset_key]
+        assets_def = asset_key_to_assets_def[asset_key]
+        dagster_metadata = dbt_resource_props.get("meta", {}).get("dagster", {})
+        freshness_check_config = dagster_metadata.get("freshness_check", {})
+        if not freshness_check_config:
+            continue
+
+        severity_str = freshness_check_config.get("severity")
+        severity = AssetCheckSeverity(severity_str) if severity_str else DEFAULT_FRESHNESS_SEVERITY
+        if assets_def.partitions_def and isinstance(
+            assets_def.partitions_def, TimeWindowPartitionsDefinition
+        ):
+            freshness_checks.extend(
+                build_time_partition_freshness_checks(
+                    assets=[asset_key],
+                    deadline_cron=freshness_check_config.get("deadline_cron"),
+                    severity=severity,
+                )
+            )
+        else:
+            try:
+                lower_bound_delta_seconds_str = freshness_check_config["lower_bound_delta_seconds"]
+                lower_bound_seconds = int(lower_bound_delta_seconds_str)
+            except:
+                raise DagsterInvariantViolationError(
+                    "lower_bound_delta_seconds must be provided, and parseable as an integer "
+                    "in the freshness_check config for a non-time-partitioned asset."
+                )
+
+            freshness_checks.extend(
+                build_last_update_freshness_checks(
+                    assets=[translator.get_asset_key(dbt_resource_props)],  # pyright: ignore[reportPossiblyUnboundVariable]
+                    deadline_cron=freshness_check_config.get("deadline_cron"),
+                    lower_bound_delta=datetime.timedelta(seconds=lower_bound_seconds),
+                    severity=severity,
+                )
+            )
+
+    return freshness_checks

dagster_dbt/include/pyproject.toml.jinja

@@ -1,3 +1,24 @@
+[project]
+name = "{{ project_name }}"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10,<3.14"
+dependencies = [
+    "dagster",
+    "dagster-cloud",
+    "dagster-dbt",
+    "dbt-core<{{ dbt_core_version_upper_bound }}",
+    {%- for dbt_adapter in dbt_adapter_packages %}
+    "{{ dbt_adapter }}<{{ dbt_core_version_upper_bound }}",
+    {%- endfor %}
+]
+
+[project.optional-dependencies]
+dev = [
+    "dagster-webserver",
+]
+
 [build-system]
 requires = ["setuptools"]
 build-backend = "setuptools.build_meta"

dagster_dbt/include/scaffold/assets.py.jinja

@@ -1,17 +1,10 @@
 from dagster import AssetExecutionContext
 from dagster_dbt import DbtCliResource, dbt_assets
 
-{% if use_experimental_dbt_project -%}
 from .project import {{ dbt_project_name }}
-{% else -%}
-from .constants import dbt_manifest_path
-{% endif %}
 
-{% if use_experimental_dbt_project -%}
+
 @dbt_assets(manifest={{ dbt_project_name }}.manifest_path)
-{% else -%}
-@dbt_assets(manifest=dbt_manifest_path)
-{% endif -%}
 def {{ dbt_assets_name }}(context: AssetExecutionContext, dbt: DbtCliResource):
     {% if use_experimental_dbt_state -%}
     # When dbt state is available, pass it to the dbt invocation.

dagster_dbt/include/scaffold/definitions.py.jinja

@@ -1,28 +1,13 @@
-{% if use_experimental_dbt_project -%}
 from dagster import Definitions
 from dagster_dbt import DbtCliResource
-{% else -%}
-import os
-
-from dagster import Definitions
-from dagster_dbt import DbtCliResource
-{% endif %}
 from .assets import {{ dbt_assets_name }}
-{% if use_experimental_dbt_project -%}
 from .project import {{ dbt_project_name }}
-{% else -%}
-from .constants import dbt_project_dir
-{% endif -%}
 from .schedules import schedules
 
 defs = Definitions(
     assets=[{{ dbt_assets_name }}],
     schedules=schedules,
     resources={
-        {% if use_experimental_dbt_project -%}
         "dbt": DbtCliResource(project_dir={{ dbt_project_name }}),
-        {%- else -%}
-        "dbt": DbtCliResource(project_dir=os.fspath(dbt_project_dir)),
-        {%- endif %}
     },
 )

dagster_dbt/include/scaffold/project.py.jinja

@@ -9,3 +9,4 @@ from dagster_dbt import DbtProject
     state_path=Path("target", "state"),
     {%- endif %}
 )
+{{ dbt_project_name }}.prepare_if_dev()

dagster_dbt/include/setup.py.jinja

@@ -4,19 +4,18 @@ setup(
     name="{{ project_name }}",
     version="0.0.1",
     packages=find_packages(),
-    {%- if use_experimental_dbt_project %}
     package_data={
         "{{ project_name }}": [
             "dbt-project/**/*",
         ],
     },
-    {%- endif %}
     install_requires=[
         "dagster",
         "dagster-cloud",
         "dagster-dbt",
+        "dbt-core<{{ dbt_core_version_upper_bound }}",
         {%- for dbt_adapter in dbt_adapter_packages %}
-        "{{ dbt_adapter }}",
+        "{{ dbt_adapter }}<{{ dbt_core_version_upper_bound }}",
         {%- endfor %}
     ],
     extras_require={

dagster_dbt/metadata_set.py

@@ -0,0 +1,18 @@
+from typing import Optional
+
+from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet
+
+
+class DbtMetadataSet(NamespacedMetadataSet):
+    """Metadata entries that apply to dbt objects.
+
+    Args:
+        materialization_type (Optional[str]): The materialization type, like "table" or "view". See
+            https://docs.getdbt.com/docs/build/materializations.
+    """
+
+    materialization_type: Optional[str]
+
+    @classmethod
+    def namespace(cls) -> str:
+        return "dagster-dbt"