dagster 1.11.14__tar.gz → 1.11.16__tar.gz

{dagster-1.11.14/dagster.egg-info → dagster-1.11.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster
-Version: 1.11.14
+Version: 1.11.16
 Summary: Dagster is an orchestration platform for the development, production, and observation of data assets.
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
@@ -60,8 +60,8 @@ Requires-Dist: universal_pathlib; python_version < "3.12"
 Requires-Dist: universal_pathlib>=0.2.0; python_version >= "3.12"
 Requires-Dist: rich
 Requires-Dist: filelock
-Requires-Dist: dagster-pipes==1.11.14
-Requires-Dist: dagster-shared==1.11.14
+Requires-Dist: dagster-pipes==1.11.16
+Requires-Dist: dagster-shared==1.11.16
 Requires-Dist: antlr4-python3-runtime
 Provides-Extra: docker
 Requires-Dist: docker; extra == "docker"
@@ -259,7 +259,7 @@ Join our community here:
 - 🌟 [Star us on GitHub](https://github.com/dagster-io/dagster)
 - 📥 [Subscribe to our Newsletter](https://dagster.io/newsletter-signup)
 - 🐦 [Follow us on Twitter](https://twitter.com/dagster)
-- 🕴️ [Follow us on LinkedIn](https://linkedin.com/showcase/dagster)
+- 🕴️ [Follow us on LinkedIn](https://www.linkedin.com/company/dagsterlabs/)
 - 📺 [Subscribe to our YouTube channel](https://www.youtube.com/@dagsterio)
 - 📚 [Read our blog posts](https://dagster.io/blog)
 - 👋 [Join us on Slack](https://dagster.io/slack)

{dagster-1.11.14 → dagster-1.11.16}/README.md

@@ -130,7 +130,7 @@ Join our community here:
 - 🌟 [Star us on GitHub](https://github.com/dagster-io/dagster)
 - 📥 [Subscribe to our Newsletter](https://dagster.io/newsletter-signup)
 - 🐦 [Follow us on Twitter](https://twitter.com/dagster)
-- 🕴️ [Follow us on LinkedIn](https://linkedin.com/showcase/dagster)
+- 🕴️ [Follow us on LinkedIn](https://www.linkedin.com/company/dagsterlabs/)
 - 📺 [Subscribe to our YouTube channel](https://www.youtube.com/@dagsterio)
 - 📚 [Read our blog posts](https://dagster.io/blog)
 - 👋 [Join us on Slack](https://dagster.io/slack)

dagster-1.11.16/dagster/_api/snapshot_repository.py

@@ -0,0 +1,88 @@
+import os
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from typing import TYPE_CHECKING
+
+import dagster._check as check
+from dagster._core.errors import DagsterUserCodeProcessError
+from dagster._core.remote_representation.external_data import RepositoryErrorSnap, RepositorySnap
+from dagster._serdes import deserialize_value
+from dagster._utils.error import SerializableErrorInfo
+
+if TYPE_CHECKING:
+    from dagster._core.remote_representation.code_location import CodeLocation
+    from dagster._core.snap import JobSnap
+    from dagster._grpc.client import DagsterGrpcClient
+
+
+def sync_get_external_repositories_data_grpc(
+    api_client: "DagsterGrpcClient", code_location: "CodeLocation", defer_snapshots: bool
+) -> Mapping[str, tuple[RepositorySnap, Mapping[str, "JobSnap"]]]:
+    from dagster._core.remote_origin import RemoteRepositoryOrigin
+    from dagster._core.remote_representation.code_location import CodeLocation
+    from dagster._core.remote_representation.external_data import (
+        extract_serialized_job_snap_from_serialized_job_data_snap,
+    )
+    from dagster._core.snap import JobSnap
+
+    check.inst_param(code_location, "code_location", CodeLocation)
+
+    repo_datas = {}
+    for repository_name in code_location.repository_names:  # type: ignore
+        repo_origin = RemoteRepositoryOrigin(
+            code_location.origin,
+            repository_name,
+        )
+        result = deserialize_value(
+            api_client.external_repository(
+                remote_repository_origin=RemoteRepositoryOrigin(
+                    code_location.origin,
+                    repository_name,
+                ),
+                defer_snapshots=defer_snapshots,
+            ),
+            (RepositorySnap, RepositoryErrorSnap),
+        )
+
+        if isinstance(result, RepositoryErrorSnap):
+            raise DagsterUserCodeProcessError.from_error_info(result.error)
+
+        job_snaps = {}
+
+        if not result.has_job_data():
+            job_refs = result.get_job_refs()
+            with ThreadPoolExecutor(
+                max_workers=int(os.getenv("DAGSTER_EXTERNAL_JOB_THREADPOOL_WORKERS", "4"))
+            ) as executor:
+                job_responses = list(
+                    executor.map(
+                        lambda job_ref: api_client.external_job(
+                            repo_origin,
+                            job_ref.name,
+                        ),
+                        job_refs,
+                    )
+                )
+
+            for i, job_ref in enumerate(job_refs):
+                job_response = job_responses[i]
+                if not job_response.serialized_job_data:
+                    error = (
+                        deserialize_value(job_response.serialized_error, SerializableErrorInfo)
+                        if job_response.serialized_error
+                        else "no captured error"
+                    )
+                    raise Exception(
+                        f"Error fetching job data for {job_ref.name} in code server:\n{error}"
+                    )
+                job_snap = deserialize_value(
+                    extract_serialized_job_snap_from_serialized_job_data_snap(
+                        job_response.serialized_job_data
+                    ),
+                    JobSnap,
+                )
+
+                job_snaps[job_ref.name] = job_snap
+
+        repo_datas[repository_name] = result, job_snaps
+    return repo_datas

{dagster-1.11.14 → dagster-1.11.16}/dagster/_cli/code_server.py

@@ -25,7 +25,9 @@ def get_default_proxy_server_heartbeat_timeout():
     return int(os.getenv("DAGSTER_PROXY_SERVER_HEARTBEAT_TIMEOUT", "30"))
 
 
-DEFAULT_HEARTBEAT_TIMEOUT = get_default_proxy_server_heartbeat_timeout()
+def get_default_grpc_proxy_heartbeat_ttl():
+    """Get the default heartbeat TTL for the gRPC proxy server."""
+    return int(os.getenv("DAGSTER_GRPC_PROXY_HEARTBEAT_TTL_SECONDS", "30"))
 
 
 @click.group(name="code-server")
@@ -162,9 +164,17 @@ def code_server_cli():
     "--heartbeat-timeout",
     type=click.INT,
     required=False,
-    default=DEFAULT_HEARTBEAT_TIMEOUT,
+    default=get_default_proxy_server_heartbeat_timeout(),
     help="How long to wait for a heartbeat from the caller before timing out. Only comes into play if --heartbeat is set. Defaults to 30 seconds.",
 )
+@click.option(
+    "--heartbeat-ttl",
+    type=click.INT,
+    required=False,
+    default=get_default_grpc_proxy_heartbeat_ttl(),
+    help="How long spawned API server processes will wait without receiving a heartbeat from clients before shutting down. This is separate from --heartbeat-timeout, which controls how long the proxy server itself waits for heartbeats from its caller. Defaults to 30 seconds.",
+    envvar="DAGSTER_GRPC_PROXY_HEARTBEAT_TTL_SECONDS",
+)
 @click.option(
     "--instance-ref",
     type=click.STRING,
@@ -194,7 +204,8 @@ def start_command(
     location_name: Optional[str],
     startup_timeout: int,
     heartbeat: bool,
-    heartbeat_timeout,
+    heartbeat_timeout: int,
+    heartbeat_ttl: int,
     instance_ref: Optional[str],
     defs_state_info: Optional[str],
     **other_opts,
@@ -266,6 +277,7 @@ def start_command(
         logger=logger,
         server_heartbeat=heartbeat,
         server_heartbeat_timeout=heartbeat_timeout,
+        heartbeat_ttl=heartbeat_ttl,
         defs_state_info=deserialize_value(defs_state_info, DefsStateInfo)
         if defs_state_info
         else None,

{dagster-1.11.14 → dagster-1.11.16}/dagster/_cli/workspace/cli_target.py

@@ -466,6 +466,7 @@ def _get_code_pointer_dict_from_python_pointer_opts(
         working_directory=working_directory,
         attribute=params.attribute,
         autoload_defs_module_name=params.autoload_defs_module_name,
+        resolve_lazy_defs=True,
     )
 
     # repository_name -> code_pointer

{dagster-1.11.14 → dagster-1.11.16}/dagster/_core/asset_graph_view/asset_graph_view.py

@@ -612,6 +612,54 @@ class AssetGraphView(LoadingContext):
         )
         return EntitySubset(self, key=key, value=_ValidatedEntitySubsetValue(value))
 
+    async def _compute_execution_failed_unpartitioned(self, key: AssetKey) -> bool:
+        from dagster._core.event_api import AssetRecordsFilter
+        from dagster._core.storage.dagster_run import DagsterRunStatus, RunRecord
+        from dagster._core.storage.event_log.base import AssetRecord
+        from dagster._utils.storage import get_materialization_chunk_size
+
+        planned_materialization_info = (
+            self.instance.event_log_storage.get_latest_planned_materialization_info(key)
+        )
+        if not planned_materialization_info:
+            # has never been planned
+            return False
+
+        planned_storage_id = planned_materialization_info.storage_id
+        planned_run_id = planned_materialization_info.run_id
+        run = await RunRecord.gen(self, planned_run_id)
+
+        # note that if the run did fail, it's still possible that the materialization was successful,
+        # hence the extra code below this conditional
+        if not run or run.dagster_run.status != DagsterRunStatus.FAILURE:
+            # latest run did not fail
+            return False
+
+        # performance optimization: we will generally have this record cached, and in most cases
+        # the most recent materialization will map to the most recent planned run
+        asset_record = await AssetRecord.gen(self, key)
+        asset_entry = asset_record.asset_entry if asset_record else None
+        latest_materialization = asset_entry.last_materialization_record if asset_entry else None
+        if latest_materialization and latest_materialization.run_id == planned_run_id:
+            return False
+
+        # look for any materializations for the latest planned run for cases where
+        # the run failed but the materialization was successful
+        has_more = True
+        cursor = None
+        while has_more:
+            result = self.instance.fetch_materializations(
+                AssetRecordsFilter(asset_key=key, after_storage_id=planned_storage_id),
+                limit=get_materialization_chunk_size(),
+                cursor=cursor,
+            )
+            has_more, cursor = result.has_more, result.cursor
+            if any(record.run_id == planned_run_id for record in result.records):
+                return False
+
+        # could not find any materializations for the latest planned run
+        return True
+
     async def _compute_execution_failed_asset_subset(self, key: AssetKey) -> EntitySubset[AssetKey]:
         from dagster._core.storage.partition_status_cache import AssetStatusCacheValue
 
@@ -623,8 +671,9 @@ class AssetGraphView(LoadingContext):
                 if cache_value
                 else self.get_empty_subset(key=key)
             )
-        value = self._queryer.get_failed_asset_subset(asset_key=key).value
-        return EntitySubset(self, key=key, value=_ValidatedEntitySubsetValue(value))
+        else:
+            value = await self._compute_execution_failed_unpartitioned(key)
+            return EntitySubset(self, key=key, value=_ValidatedEntitySubsetValue(value))
 
     async def _compute_missing_asset_subset(
         self, key: AssetKey, from_subset: EntitySubset

{dagster-1.11.14 → dagster-1.11.16}/dagster/_core/definitions/asset_health/asset_check_health.py

@@ -25,6 +25,9 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
     failing_checks: set[AssetCheckKey]
     warning_checks: set[AssetCheckKey]
     all_checks: set[AssetCheckKey]
+    latest_passing_check_timestamp: Optional[float] = None
+    latest_failing_check_timestamp: Optional[float] = None
+    latest_warning_check_timestamp: Optional[float] = None
 
     @classmethod
     def default(cls) -> "AssetCheckHealthState":
@@ -33,6 +36,9 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
             failing_checks=set(),
             warning_checks=set(),
             all_checks=set(),
+            latest_passing_check_timestamp=None,
+            latest_failing_check_timestamp=None,
+            latest_warning_check_timestamp=None,
         )
 
     @property
@@ -77,6 +83,10 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
         warning_checks = set()
         failing_checks = set()
 
+        latest_passing_check_timestamp = None
+        latest_failing_check_timestamp = None
+        latest_warning_check_timestamp = None
+
         check_records = await AssetCheckSummaryRecord.gen_many(
             loading_context,
             check_keys,
@@ -125,20 +135,55 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
                     and last_check_evaluation.severity == AssetCheckSeverity.WARN
                 ):
                     warning_checks.add(check_key)
+                    if check_record.last_completed_check_execution_record is not None and (
+                        latest_warning_check_timestamp is None
+                        or latest_warning_check_timestamp
+                        < check_record.last_completed_check_execution_record.create_timestamp
+                    ):
+                        latest_warning_check_timestamp = (
+                            check_record.last_completed_check_execution_record.create_timestamp
+                        )
                 else:
                     failing_checks.add(check_key)
+                    if check_record.last_completed_check_execution_record is not None and (
+                        latest_failing_check_timestamp is None
+                        or latest_failing_check_timestamp
+                        < check_record.last_completed_check_execution_record.create_timestamp
+                    ):
+                        latest_failing_check_timestamp = (
+                            check_record.last_completed_check_execution_record.create_timestamp
+                        )
             elif last_check_execution_status == AssetCheckExecutionResolvedStatus.EXECUTION_FAILED:
                 # EXECUTION_FAILED checks don't have an evaluation and we want to report them as failures
                 failing_checks.add(check_key)
+                if check_record.last_completed_check_execution_record is not None and (
+                    latest_failing_check_timestamp is None
+                    or latest_failing_check_timestamp
+                    < check_record.last_completed_check_execution_record.create_timestamp
+                ):
+                    latest_failing_check_timestamp = (
+                        check_record.last_completed_check_execution_record.create_timestamp
+                    )
             else:
                 # asset check passed
                 passing_checks.add(check_key)
+                if check_record.last_completed_check_execution_record is not None and (
+                    latest_passing_check_timestamp is None
+                    or latest_passing_check_timestamp
+                    < check_record.last_completed_check_execution_record.create_timestamp
+                ):
+                    latest_passing_check_timestamp = (
+                        check_record.last_completed_check_execution_record.create_timestamp
+                    )
 
         return AssetCheckHealthState(
             passing_checks=passing_checks,
             failing_checks=failing_checks,
             warning_checks=warning_checks,
             all_checks=check_keys,
+            latest_passing_check_timestamp=latest_passing_check_timestamp,
+            latest_failing_check_timestamp=latest_failing_check_timestamp,
+            latest_warning_check_timestamp=latest_warning_check_timestamp,
         )
 
     @classmethod

{dagster-1.11.14 → dagster-1.11.16}/dagster/_core/definitions/asset_health/asset_freshness_health.py

@@ -24,6 +24,7 @@ class AssetFreshnessHealthState(LoadableBy[AssetKey]):
     """Maintains the latest freshness state for the asset."""
 
     freshness_state: FreshnessState
+    updated_timestamp: Optional[float] = None
 
     @property
     def health_status(self) -> AssetHealthStatus:
@@ -49,9 +50,11 @@ class AssetFreshnessHealthState(LoadableBy[AssetKey]):
             # freshness policy has no evaluations yet
             return cls(
                 freshness_state=FreshnessState.UNKNOWN,
+                updated_timestamp=None,
             )
         return cls(
             freshness_state=freshness_state_record.freshness_state,
+            updated_timestamp=freshness_state_record.updated_at.timestamp(),
         )
 
     @classmethod

{dagster-1.11.14 → dagster-1.11.16}/dagster/_core/definitions/asset_health/asset_materialization_health.py

@@ -35,6 +35,7 @@ class MinimalAssetMaterializationHealthState(LoadableBy[AssetKey]):
     num_failed_partitions: int
     num_currently_materialized_partitions: int
     partitions_snap: Optional[PartitionsSnap]
+    latest_failed_to_materialize_timestamp: Optional[float] = None
 
     @property
     def health_status(self) -> AssetHealthStatus:
@@ -62,6 +63,7 @@ class MinimalAssetMaterializationHealthState(LoadableBy[AssetKey]):
             num_failed_partitions=asset_materialization_health_state.failed_subset.size,
             num_currently_materialized_partitions=asset_materialization_health_state.currently_materialized_subset.size,
             partitions_snap=asset_materialization_health_state.partitions_snap,
+            latest_failed_to_materialize_timestamp=asset_materialization_health_state.latest_failed_to_materialize_timestamp,
         )
 
     @classmethod
@@ -101,6 +103,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
     partitions_snap: Optional[PartitionsSnap]
     latest_terminal_run_id: Optional[str]
     latest_materialization_timestamp: Optional[float] = None
+    latest_failed_to_materialize_timestamp: Optional[float] = None
 
     @property
     def partitions_def(self) -> Optional[PartitionsDefinition]:
@@ -152,6 +155,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
 
             last_run_id = None
             latest_materialization_timestamp = None
+            latest_failed_to_materialize_timestamp = None
             if asset_record is not None:
                 entry = asset_record.asset_entry
                 latest_record = max(
@@ -167,6 +171,11 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                     if entry.last_materialization_record
                     else None
                 )
+                latest_failed_to_materialize_timestamp = (
+                    entry.last_failed_to_materialize_record.timestamp
+                    if entry.last_failed_to_materialize_record
+                    else None
+                )
 
             return cls(
                 materialized_subset=SerializableEntitySubset(
@@ -178,6 +187,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                 partitions_snap=PartitionsSnap.from_def(partitions_def),
                 latest_terminal_run_id=last_run_id,
                 latest_materialization_timestamp=latest_materialization_timestamp,
+                latest_failed_to_materialize_timestamp=latest_failed_to_materialize_timestamp,
             )
 
         if asset_record is None:
@@ -187,6 +197,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                 partitions_snap=None,
                 latest_terminal_run_id=None,
                 latest_materialization_timestamp=None,
+                latest_failed_to_materialize_timestamp=None,
             )
 
         asset_entry = asset_record.asset_entry
@@ -195,6 +206,11 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
             if asset_entry.last_materialization_record
             else None
         )
+        latest_failed_to_materialize_timestamp = (
+            asset_entry.last_failed_to_materialize_record.timestamp
+            if asset_entry.last_failed_to_materialize_record
+            else None
+        )
         if asset_entry.last_run_id is None:
             return AssetMaterializationHealthState(
                 materialized_subset=SerializableEntitySubset(key=asset_key, value=False),
@@ -202,6 +218,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                 partitions_snap=None,
                 latest_terminal_run_id=None,
                 latest_materialization_timestamp=latest_materialization_timestamp,
+                latest_failed_to_materialize_timestamp=latest_failed_to_materialize_timestamp,
             )
 
         has_ever_materialized = asset_entry.last_materialization is not None
@@ -218,6 +235,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
             partitions_snap=None,
             latest_terminal_run_id=latest_terminal_run_id,
             latest_materialization_timestamp=latest_materialization_timestamp,
+            latest_failed_to_materialize_timestamp=latest_failed_to_materialize_timestamp,
         )
 
     @classmethod

{dagster-1.11.14 → dagster-1.11.16}/dagster/_core/definitions/asset_key.py

@@ -195,7 +195,7 @@ def check_opt_coercible_to_asset_key_prefix_param(
 def key_prefix_from_coercible(key_prefix: CoercibleToAssetKeyPrefix) -> Sequence[str]:
     if isinstance(key_prefix, str):
         return [key_prefix]
-    elif isinstance(key_prefix, list):
+    elif isinstance(key_prefix, Sequence):
         return key_prefix
     else:
         check.failed(f"Unexpected type for key_prefix: {type(key_prefix)}")

{dagster-1.11.14 → dagster-1.11.16}/dagster/_core/definitions/assets/graph/remote_asset_graph.py

@@ -463,18 +463,27 @@ class RemoteAssetGraph(BaseAssetGraph[TRemoteAssetNode], ABC, Generic[TRemoteAss
         self,
     ) -> Mapping[AssetCheckKey, RemoteAssetCheckNode]: ...
 
+    # TODO this should just be what is returned from get()
+    def get_remote_asset_check_node(self, key: AssetCheckKey) -> RemoteAssetCheckNode:
+        return self.remote_asset_check_nodes_by_key[key]
+
+    def _get_asset_check_node_from_remote_asset_check_node(
+        self, remote_node: RemoteAssetCheckNode
+    ) -> AssetCheckNode:
+        return AssetCheckNode(
+            remote_node.asset_check.key,
+            remote_node.asset_check.additional_asset_keys,
+            remote_node.asset_check.blocking,
+            remote_node.asset_check.description,
+            remote_node.asset_check.automation_condition,
+            {},  # metadata not yet on AssetCheckNodeSnap
+        )
+
     ##### COMMON ASSET GRAPH INTERFACE
     @cached_property
     def _asset_check_nodes_by_key(self) -> Mapping[AssetCheckKey, AssetCheckNode]:  # pyright: ignore[reportIncompatibleVariableOverride]
         return {
-            k: AssetCheckNode(
-                k,
-                v.asset_check.additional_asset_keys,
-                v.asset_check.blocking,
-                v.asset_check.description,
-                v.asset_check.automation_condition,
-                {},  # metadata not yet on AssetCheckNodeSnap
-            )
+            k: self._get_asset_check_node_from_remote_asset_check_node(v)
             for k, v in self.remote_asset_check_nodes_by_key.items()
         }
 
@@ -484,7 +493,7 @@ class RemoteAssetGraph(BaseAssetGraph[TRemoteAssetNode], ABC, Generic[TRemoteAss
         if isinstance(entity_key, AssetKey):
             return self.get(entity_key).execution_set_entity_keys
         else:  # AssetCheckKey
-            return self.remote_asset_check_nodes_by_key[entity_key].execution_set_entity_keys
+            return self.get_remote_asset_check_node(entity_key).execution_set_entity_keys
 
     ##### REMOTE-SPECIFIC METHODS
 
@@ -684,7 +693,7 @@ class RemoteWorkspaceAssetGraph(RemoteAssetGraph[RemoteWorkspaceAssetNode]):
         if isinstance(key, AssetKey):
             return self.get(key).resolve_to_singular_repo_scoped_node().repository_handle
         else:
-            return self.remote_asset_check_nodes_by_key[key].handle
+            return self.get_remote_asset_check_node(key).handle
 
     def get_repo_scoped_node(
         self, key: EntityKey, repository_selector: "RepositorySelector"

{dagster-1.11.14 → dagster-1.11.16}/dagster/_core/definitions/decorators/repository_decorator.py

@@ -103,7 +103,7 @@ class _Repository:
 
         repository_definitions = fn()
         context = DefinitionsLoadContext.get()
-        defs_state_info = context.defs_state_info
+        defs_state_info = context.accessed_defs_state_info
 
         if context.load_type == DefinitionsLoadType.INITIALIZATION:
             reconstruction_metadata = context.get_pending_reconstruction_metadata()