dagster-dbt 0.28.5__tar.gz → 0.28.7__tar.gz

{dagster_dbt-0.28.5/dagster_dbt.egg-info → dagster_dbt-0.28.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.28.5
+Version: 0.28.7
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
@@ -10,11 +10,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10,<3.14
+Requires-Python: >=3.10,<3.15
 License-File: LICENSE
-Requires-Dist: dagster==1.12.5
+Requires-Dist: dagster==1.12.7
 Requires-Dist: dbt-core<1.11,>=1.7
 Requires-Dist: gitpython
 Requires-Dist: Jinja2

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/asset_utils.py

@@ -572,6 +572,9 @@ def default_metadata_from_dbt_resource_props(
             dbt_resource_props.get("database"),
             dbt_resource_props.get("schema"),
             dbt_resource_props.get("alias"),
+            dbt_resource_props.get("name")
+            if dbt_resource_props.get("resource_type") == "source"
+            else None,
         ]
         if relation_part
     ]

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/cloud_v2/cli_invocation.py

@@ -1,3 +1,4 @@
+import sys
 from collections.abc import Iterator, Mapping, Sequence
 from typing import Any, Optional, Union
 
@@ -54,6 +55,12 @@ class DbtCloudCliInvocation:
         self, timeout: Optional[float] = None
     ) -> Iterator[Union[AssetCheckEvaluation, AssetCheckResult, AssetMaterialization, Output]]:
         run = self.run_handler.wait(timeout=timeout)
+
+        # Write dbt Cloud run logs to stdout
+        logs = self.run_handler.get_run_logs()
+        if logs:
+            sys.stdout.write(logs)
+
         if "run_results.json" in self.run_handler.list_run_artifacts():
             run_results = DbtCloudJobRunResults.from_run_results_json(
                 run_results_json=self.run_handler.get_run_results()

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/cloud_v2/client.py

@@ -92,7 +92,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
         data: Optional[Mapping[str, Any]] = None,
         params: Optional[Mapping[str, Any]] = None,
         session_attr: str = "_get_session",
-    ) -> Mapping[str, Any]:
+    ) -> requests.Response:
         url = f"{base_url}/{endpoint}" if endpoint else base_url
 
         num_retries = 0
@@ -107,7 +107,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
                     timeout=self.request_timeout,
                 )
                 response.raise_for_status()
-                return response.json()
+                return response
             except RequestException as e:
                 self._log.error(
                     f"Request to dbt Cloud API failed for url {url} with method {method} : {e}"
@@ -143,7 +143,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
         """
         if not description:
             description = "A job that runs dbt models, sources, and tests."
-        return self._make_request(
+        response = self._make_request(
             method="post",
             endpoint="jobs",
             base_url=self.api_v2_url,
@@ -155,7 +155,8 @@ class DbtCloudWorkspaceClient(DagsterModel):
                 "description": description,
                 "job_type": "other",
             },
-        )["data"]
+        )
+        return response.json()["data"]
 
     def list_jobs(
         self,
@@ -185,7 +186,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
                 "limit": DAGSTER_DBT_CLOUD_LIST_JOBS_INDIVIDUAL_REQUEST_LIMIT,
                 "offset": len(results),
             },
-        )["data"]:
+        ).json()["data"]:
             results.extend(jobs)
             if len(jobs) < DAGSTER_DBT_CLOUD_LIST_JOBS_INDIVIDUAL_REQUEST_LIMIT:
                 break
@@ -201,7 +202,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
             method="get",
             endpoint=f"jobs/{job_id}",
             base_url=self.api_v2_url,
-        )["data"]
+        ).json()["data"]
 
     def destroy_job(self, job_id: int) -> Mapping[str, Any]:
         """Destroys a given dbt Cloud job.
@@ -213,7 +214,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
             method="delete",
             endpoint=f"jobs/{job_id}",
             base_url=self.api_v2_url,
-        )["data"]
+        ).json()["data"]
 
     def trigger_job_run(
         self, job_id: int, steps_override: Optional[Sequence[str]] = None
@@ -237,7 +238,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
             data={"steps_override": steps_override, "cause": DAGSTER_ADHOC_TRIGGER_CAUSE}
             if steps_override
             else {"cause": DAGSTER_ADHOC_TRIGGER_CAUSE},
-        )["data"]
+        ).json()["data"]
 
     def get_runs_batch(
         self,
@@ -278,26 +279,35 @@ class DbtCloudWorkspaceClient(DagsterModel):
                 "finished_at__range": f"""["{finished_at_lower_bound.isoformat()}", "{finished_at_upper_bound.isoformat()}"]""",
                 "order_by": "finished_at",
             },
-        )
+        ).json()
         data = cast("Sequence[Mapping[str, Any]]", resp["data"])
         total_count = resp["extra"]["pagination"]["total_count"]
         return data, total_count
 
-    def get_run_details(self, run_id: int) -> Mapping[str, Any]:
+    def get_run_details(
+        self, run_id: int, include_related: Optional[Sequence[str]] = None
+    ) -> Mapping[str, Any]:
         """Retrieves the details of a given dbt Cloud Run.
 
         Args:
-            run_id (str): The dbt Cloud Run ID. You can retrieve this value from the
+            run_id (int): The dbt Cloud Run ID. You can retrieve this value from the
                 URL of the given run in the dbt Cloud UI.
+            include_related (Optional[Sequence[str]]): List of related fields to pull with the run.
+                Valid values are "trigger", "job", "debug_logs", and "run_steps".
 
         Returns:
             Dict[str, Any]: Parsed json data representing the API response.
         """
+        params = {}
+        if include_related:
+            params["include_related"] = ",".join(include_related)
+
         return self._make_request(
             method="get",
             endpoint=f"runs/{run_id}",
             base_url=self.api_v2_url,
-        )["data"]
+            params=params,
+        ).json()["data"]
 
     def poll_run(
         self,
@@ -352,21 +362,25 @@ class DbtCloudWorkspaceClient(DagsterModel):
                 endpoint=f"runs/{run_id}/artifacts",
                 base_url=self.api_v2_url,
                 session_attr="_get_artifact_session",
-            )["data"],
+            ).json()["data"],
         )
 
     def get_run_artifact(self, run_id: int, path: str) -> Mapping[str, Any]:
         """Retrieves an artifact at the given path for a given dbt Cloud Run.
 
+        Args:
+            run_id (int): The dbt Cloud Run ID.
+            path (str): The path to the artifact (e.g., "run_results.json", "manifest.json").
+
         Returns:
-            Dict[str, Any]: Parsed json data representing the API response.
+            Dict[str, Any]: Parsed json data representing the artifact.
         """
         return self._make_request(
             method="get",
             endpoint=f"runs/{run_id}/artifacts/{path}",
             base_url=self.api_v2_url,
             session_attr="_get_artifact_session",
-        )
+        ).json()
 
     def get_run_results_json(self, run_id: int) -> Mapping[str, Any]:
         """Retrieves the run_results.json artifact of a given dbt Cloud Run.
@@ -384,6 +398,65 @@ class DbtCloudWorkspaceClient(DagsterModel):
         """
         return self.get_run_artifact(run_id=run_id, path="manifest.json")
 
+    def get_run_logs(self, run_id: int, max_retries: int = 3, retry_delay: float = 2.0) -> str:
+        """Retrieves the stdout/stderr logs from a given dbt Cloud Run.
+
+        This method fetches logs from the run_steps field by calling get_run_details
+        with include_related=["run_steps"]. Each step contains a logs field with
+        the stdout/stderr output for that step.
+
+        Note: There can be a slight delay between when a run completes and when the logs
+        are fully populated in the API. This method will retry a few times if it detects
+        completed steps with empty logs.
+
+        Args:
+            run_id (int): The dbt Cloud Run ID.
+            max_retries (int): Maximum number of times to retry fetching logs if empty. Defaults to 3.
+            retry_delay (float): Time in seconds to wait between retries. Defaults to 2.0.
+
+        Returns:
+            str: The concatenated log text content from all run steps.
+        """
+        for attempt in range(max_retries):
+            run_details = self.get_run_details(run_id=run_id, include_related=["run_steps"])
+
+            logs_parts = []
+            run_steps = run_details.get("run_steps", [])
+            completed_steps_with_empty_logs = 0
+
+            for step in run_steps:
+                step_name = step.get("name", "Unknown Step")
+                step_logs = step.get("logs", "")
+                step_status = step.get("status_humanized", "unknown")
+
+                # Track completed steps with empty logs
+                if step_status == "Success" and not step_logs:
+                    completed_steps_with_empty_logs += 1
+
+                if step_logs:
+                    logs_parts.append(f"=== Step: {step_name} ===")
+                    logs_parts.append(step_logs)
+                    logs_parts.append("")  # Empty line between steps
+
+            # If we have completed steps with empty logs and retries left, wait and try again
+            if completed_steps_with_empty_logs > 0 and attempt < max_retries - 1:
+                self._log.warning(
+                    f"Found {completed_steps_with_empty_logs} completed steps with empty logs for run {run_id}. "
+                    f"Retrying in {retry_delay} seconds..."
+                )
+                time.sleep(retry_delay)
+                continue
+
+            # Either we got all logs or we're out of retries
+            if completed_steps_with_empty_logs > 0:
+                self._log.warning(
+                    f"Still missing logs for {completed_steps_with_empty_logs} completed steps after {max_retries} attempts"
+                )
+
+            return "\n".join(logs_parts) if logs_parts else ""
+
+        return ""
+
     def get_project_details(self, project_id: int) -> Mapping[str, Any]:
         """Retrieves the details of a given dbt Cloud Project.
 
@@ -398,7 +471,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
             method="get",
             endpoint=f"projects/{project_id}",
             base_url=self.api_v2_url,
-        )["data"]
+        ).json()["data"]
 
     def get_environment_details(self, environment_id: int) -> Mapping[str, Any]:
         """Retrieves the details of a given dbt Cloud Environment.
@@ -414,7 +487,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
             method="get",
             endpoint=f"environments/{environment_id}",
             base_url=self.api_v2_url,
-        )["data"]
+        ).json()["data"]
 
     def get_account_details(self) -> Mapping[str, Any]:
         """Retrieves the details of the account associated to the dbt Cloud workspace.
@@ -426,7 +499,7 @@ class DbtCloudWorkspaceClient(DagsterModel):
             method="get",
             endpoint=None,
             base_url=self.api_v2_url,
-        )["data"]
+        ).json()["data"]
 
     def verify_connection(self) -> None:
         """Verifies the connection to the dbt Cloud REST API."""

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/cloud_v2/run_handler.py

@@ -12,7 +12,9 @@ from dagster import (
     get_dagster_logger,
 )
 from dagster._record import record
+from dagster._time import get_current_timestamp
 from dateutil import parser
+from requests.exceptions import RequestException
 
 from dagster_dbt.asset_utils import build_dbt_specs, get_asset_check_key_for_test
 from dagster_dbt.cloud_v2.client import DbtCloudWorkspaceClient
@@ -61,8 +63,28 @@ class DbtCloudJobRunHandler:
     def list_run_artifacts(self) -> Sequence[str]:
         return self.client.list_run_artifacts(run_id=self.run_id)
 
+    def get_run_logs(self) -> Optional[str]:
+        """Retrieves the stdout/stderr logs from the completed dbt Cloud run.
+
+        This method fetches logs from the run_steps by calling get_run_details
+        with include_related=["run_steps"].
+
+        Returns:
+            Optional[str]: The concatenated log text content from all run steps,
+                or None if logs are not available.
+        """
+        try:
+            return self.client.get_run_logs(run_id=self.run_id)
+        except RequestException as e:
+            logger.warning(f"Failed to retrieve logs for run {self.run_id}: {e}")
+            return None
+
 
 def get_completed_at_timestamp(result: Mapping[str, Any]) -> float:
+    timing = result["timing"]
+    if len(timing) == 0:
+        # as a fallback, use the current timestamp
+        return get_current_timestamp()
     # result["timing"] is a list of events in run_results.json
     # For successful models and passing tests,
     # the last item of that list includes the timing details of the execution.

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/cloud_v2/sensor_builder.py

@@ -142,7 +142,7 @@ def sorted_asset_events(
     return [
         sorted_event[1]
         for sorted_event in sorted(
-            materializations_and_timestamps, key=lambda x: (x[0], topo_aks.index(x[1].asset_key))
+            materializations_and_timestamps, key=lambda x: (topo_aks.index(x[1].asset_key), x[0])
         )
     ]
 

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/dagster_dbt_translator.py

@@ -62,6 +62,8 @@ class DagsterDbtTranslatorSettings(Resolvable):
             rather than fully qualified name. Defaults to False.
         enable_source_tests_as_checks (bool): Whether to load dbt source tests as Dagster asset checks.
             Defaults to False. If False, asset observations will be emitted for source tests.
+        enable_source_metadata (bool): Whether to include metadata on AssetDep objects for dbt sources.
+            If set to True, enables the ability to remap upstream asset keys based on table name. Defaults to False.
     """
 
     enable_asset_checks: bool = True
@@ -69,6 +71,7 @@ class DagsterDbtTranslatorSettings(Resolvable):
     enable_code_references: bool = False
     enable_dbt_selection_by_name: bool = False
     enable_source_tests_as_checks: bool = False
+    enable_source_metadata: bool = False
 
 
 class DagsterDbtTranslator:
@@ -145,15 +148,23 @@ class DagsterDbtTranslator:
 
         # calculate the dependencies for the asset
         upstream_ids = get_upstream_unique_ids(manifest, resource_props)
-        deps = [
-            AssetDep(
-                asset=self.get_asset_spec(manifest, upstream_id, project).key,
-                partition_mapping=self.get_partition_mapping(
-                    resource_props, self.get_resource_props(manifest, upstream_id)
-                ),
+        deps: list[AssetDep] = []
+        for upstream_id in upstream_ids:
+            spec = self.get_asset_spec(manifest, upstream_id, project)
+            partition_mapping = self.get_partition_mapping(
+                resource_props, self.get_resource_props(manifest, upstream_id)
             )
-            for upstream_id in upstream_ids
-        ]
+
+            deps.append(
+                AssetDep(
+                    asset=spec.key,
+                    partition_mapping=partition_mapping,
+                    metadata=spec.metadata
+                    if self.settings.enable_source_metadata and upstream_id.startswith("source")
+                    else None,
+                )
+            )
+
         self_partition_mapping = self.get_partition_mapping(resource_props, resource_props)
         if self_partition_mapping and has_self_dependency(resource_props):
             deps.append(

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/dbt_project.py

@@ -132,6 +132,42 @@ class DagsterDbtProjectPreparer(DbtProjectPreparer):
             .wait()
         )
 
+        # Remove seed entries from partial_parse to force re-parsing at runtime.
+        # This ensures seeds get correct root_path based on current project location.
+        self._invalidate_seeds_in_partial_parse(project)
+
+    def _invalidate_seeds_in_partial_parse(self, project: "DbtProject") -> None:
+        """Remove seed entries from partial_parse.msgpack to force re-parsing.
+
+        Seeds contain root_path which is an absolute path from build time. When state
+        is generated in one environment (e.g., CI/CD) and used in another (e.g., deployed
+        container), the root_path points to the wrong location and seed loading fails.
+
+        By removing seed entries from the cache, dbt will re-parse them at runtime with
+        the correct current project path. Models keep their cached data for fast loading.
+        """
+        import msgpack
+
+        partial_parse_path = project.project_dir / project.target_path / "partial_parse.msgpack"
+        if not partial_parse_path.exists():
+            return
+
+        with open(partial_parse_path, "rb") as f:
+            data = msgpack.unpack(f, raw=False, strict_map_key=False)
+
+        # Remove seed nodes
+        seed_node_ids = [k for k in data.get("nodes", {}).keys() if k.startswith("seed.")]
+        for seed_id in seed_node_ids:
+            del data["nodes"][seed_id]
+
+        # Remove seed file entries (CSVs)
+        seed_file_ids = [k for k in data.get("files", {}).keys() if k.lower().endswith(".csv")]
+        for file_id in seed_file_ids:
+            del data["files"][file_id]
+
+        with open(partial_parse_path, "wb") as f:
+            msgpack.pack(data, f)
+
 
 @record_custom
 class DbtProject(IHaveNew):

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt/include/pyproject.toml.jinja

@@ -3,7 +3,7 @@ name = "{{ project_name }}"
 version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
-requires-python = ">=3.10,<3.14"
+requires-python = ">=3.10,<3.15"
 dependencies = [
     "dagster",
     "dagster-cloud",

dagster_dbt-0.28.7/dagster_dbt/version.py

@@ -0,0 +1 @@
+__version__ = "0.28.7"

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7/dagster_dbt.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-dbt
-Version: 0.28.5
+Version: 0.28.7
 Summary: A Dagster integration for dbt
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dbt
 Author: Dagster Labs
@@ -10,11 +10,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10,<3.14
+Requires-Python: >=3.10,<3.15
 License-File: LICENSE
-Requires-Dist: dagster==1.12.5
+Requires-Dist: dagster==1.12.7
 Requires-Dist: dbt-core<1.11,>=1.7
 Requires-Dist: gitpython
 Requires-Dist: Jinja2

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/dagster_dbt.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dagster==1.12.5
+dagster==1.12.7
 dbt-core<1.11,>=1.7
 gitpython
 Jinja2

{dagster_dbt-0.28.5 → dagster_dbt-0.28.7}/setup.py

@@ -32,14 +32,15 @@ setup(
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
     ],
     packages=find_packages(exclude=["dagster_dbt_tests*"]),
     include_package_data=True,
-    python_requires=">=3.10,<3.14",
+    python_requires=">=3.10,<3.15",
     install_requires=[
-        "dagster==1.12.5",
+        "dagster==1.12.7",
         # Follow the version support constraints for dbt Core: https://docs.getdbt.com/docs/dbt-versions/core
         f"dbt-core>=1.7,<{DBT_CORE_VERSION_UPPER_BOUND}",
         "gitpython",

dagster_dbt-0.28.5/dagster_dbt/version.py

@@ -1 +0,0 @@
-__version__ = "0.28.5"