dagster 1.11.14__tar.gz → 1.11.15__tar.gz

{dagster-1.11.14/dagster.egg-info → dagster-1.11.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster
-Version: 1.11.14
+Version: 1.11.15
 Summary: Dagster is an orchestration platform for the development, production, and observation of data assets.
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
@@ -60,8 +60,8 @@ Requires-Dist: universal_pathlib; python_version < "3.12"
 Requires-Dist: universal_pathlib>=0.2.0; python_version >= "3.12"
 Requires-Dist: rich
 Requires-Dist: filelock
-Requires-Dist: dagster-pipes==1.11.14
-Requires-Dist: dagster-shared==1.11.14
+Requires-Dist: dagster-pipes==1.11.15
+Requires-Dist: dagster-shared==1.11.15
 Requires-Dist: antlr4-python3-runtime
 Provides-Extra: docker
 Requires-Dist: docker; extra == "docker"
@@ -259,7 +259,7 @@ Join our community here:
 - 🌟 [Star us on GitHub](https://github.com/dagster-io/dagster)
 - 📥 [Subscribe to our Newsletter](https://dagster.io/newsletter-signup)
 - 🐦 [Follow us on Twitter](https://twitter.com/dagster)
-- 🕴️ [Follow us on LinkedIn](https://linkedin.com/showcase/dagster)
+- 🕴️ [Follow us on LinkedIn](https://www.linkedin.com/company/dagsterlabs/)
 - 📺 [Subscribe to our YouTube channel](https://www.youtube.com/@dagsterio)
 - 📚 [Read our blog posts](https://dagster.io/blog)
 - 👋 [Join us on Slack](https://dagster.io/slack)

{dagster-1.11.14 → dagster-1.11.15}/README.md

@@ -130,7 +130,7 @@ Join our community here:
 - 🌟 [Star us on GitHub](https://github.com/dagster-io/dagster)
 - 📥 [Subscribe to our Newsletter](https://dagster.io/newsletter-signup)
 - 🐦 [Follow us on Twitter](https://twitter.com/dagster)
-- 🕴️ [Follow us on LinkedIn](https://linkedin.com/showcase/dagster)
+- 🕴️ [Follow us on LinkedIn](https://www.linkedin.com/company/dagsterlabs/)
 - 📺 [Subscribe to our YouTube channel](https://www.youtube.com/@dagsterio)
 - 📚 [Read our blog posts](https://dagster.io/blog)
 - 👋 [Join us on Slack](https://dagster.io/slack)

{dagster-1.11.14 → dagster-1.11.15}/dagster/_api/snapshot_repository.py

@@ -11,7 +11,7 @@ if TYPE_CHECKING:
     from dagster._grpc.client import DagsterGrpcClient
 
 
-def sync_get_streaming_external_repositories_data_grpc(
+def sync_get_external_repositories_data_grpc(
     api_client: "DagsterGrpcClient", code_location: "CodeLocation"
 ) -> Mapping[str, RepositorySnap]:
     from dagster._core.remote_origin import RemoteRepositoryOrigin
@@ -21,21 +21,12 @@ def sync_get_streaming_external_repositories_data_grpc(
 
     repo_datas = {}
     for repository_name in code_location.repository_names:  # type: ignore
-        external_repository_chunks = list(
-            api_client.streaming_external_repository(
+        result = deserialize_value(
+            api_client.external_repository(
                 remote_repository_origin=RemoteRepositoryOrigin(
                     code_location.origin,
                     repository_name,
                 )
-            )
-        )
-
-        result = deserialize_value(
-            "".join(
-                [
-                    chunk["serialized_external_repository_chunk"]
-                    for chunk in external_repository_chunks
-                ]
             ),
             (RepositorySnap, RepositoryErrorSnap),
         )
@@ -47,7 +38,7 @@ def sync_get_streaming_external_repositories_data_grpc(
     return repo_datas
 
 
-async def gen_streaming_external_repositories_data_grpc(
+async def gen_external_repositories_data_grpc(
     api_client: "DagsterGrpcClient", code_location: "CodeLocation"
 ) -> Mapping[str, RepositorySnap]:
     from dagster._core.remote_origin import RemoteRepositoryOrigin
@@ -57,22 +48,12 @@ async def gen_streaming_external_repositories_data_grpc(
 
     repo_datas = {}
     for repository_name in code_location.repository_names:  # type: ignore
-        external_repository_chunks = [
-            chunk
-            async for chunk in api_client.gen_streaming_external_repository(
+        result = deserialize_value(
+            await api_client.gen_external_repository(
                 remote_repository_origin=RemoteRepositoryOrigin(
                     code_location.origin,
                     repository_name,
                 )
-            )
-        ]
-
-        result = deserialize_value(
-            "".join(
-                [
-                    chunk["serialized_external_repository_chunk"]
-                    for chunk in external_repository_chunks
-                ]
             ),
             (RepositorySnap, RepositoryErrorSnap),
         )

{dagster-1.11.14 → dagster-1.11.15}/dagster/_cli/workspace/cli_target.py

@@ -466,6 +466,7 @@ def _get_code_pointer_dict_from_python_pointer_opts(
         working_directory=working_directory,
         attribute=params.attribute,
         autoload_defs_module_name=params.autoload_defs_module_name,
+        resolve_lazy_defs=True,
     )
 
     # repository_name -> code_pointer

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/asset_graph_view/asset_graph_view.py

@@ -612,6 +612,54 @@ class AssetGraphView(LoadingContext):
         )
         return EntitySubset(self, key=key, value=_ValidatedEntitySubsetValue(value))
 
+    async def _compute_execution_failed_unpartitioned(self, key: AssetKey) -> bool:
+        from dagster._core.event_api import AssetRecordsFilter
+        from dagster._core.storage.dagster_run import DagsterRunStatus, RunRecord
+        from dagster._core.storage.event_log.base import AssetRecord
+        from dagster._utils.storage import get_materialization_chunk_size
+
+        planned_materialization_info = (
+            self.instance.event_log_storage.get_latest_planned_materialization_info(key)
+        )
+        if not planned_materialization_info:
+            # has never been planned
+            return False
+
+        planned_storage_id = planned_materialization_info.storage_id
+        planned_run_id = planned_materialization_info.run_id
+        run = await RunRecord.gen(self, planned_run_id)
+
+        # note that if the run did fail, it's still possible that the materialization was successful,
+        # hence the extra code below this conditional
+        if not run or run.dagster_run.status != DagsterRunStatus.FAILURE:
+            # latest run did not fail
+            return False
+
+        # performance optimization: we will generally have this record cached, and in most cases
+        # the most recent materialization will map to the most recent planned run
+        asset_record = await AssetRecord.gen(self, key)
+        asset_entry = asset_record.asset_entry if asset_record else None
+        latest_materialization = asset_entry.last_materialization_record if asset_entry else None
+        if latest_materialization and latest_materialization.run_id == planned_run_id:
+            return False
+
+        # look for any materializations for the latest planned run for cases where
+        # the run failed but the materialization was successful
+        has_more = True
+        cursor = None
+        while has_more:
+            result = self.instance.fetch_materializations(
+                AssetRecordsFilter(asset_key=key, after_storage_id=planned_storage_id),
+                limit=get_materialization_chunk_size(),
+                cursor=cursor,
+            )
+            has_more, cursor = result.has_more, result.cursor
+            if any(record.run_id == planned_run_id for record in result.records):
+                return False
+
+        # could not find any materializations for the latest planned run
+        return True
+
     async def _compute_execution_failed_asset_subset(self, key: AssetKey) -> EntitySubset[AssetKey]:
         from dagster._core.storage.partition_status_cache import AssetStatusCacheValue
 
@@ -623,8 +671,9 @@ class AssetGraphView(LoadingContext):
                 if cache_value
                 else self.get_empty_subset(key=key)
             )
-        value = self._queryer.get_failed_asset_subset(asset_key=key).value
-        return EntitySubset(self, key=key, value=_ValidatedEntitySubsetValue(value))
+        else:
+            value = await self._compute_execution_failed_unpartitioned(key)
+            return EntitySubset(self, key=key, value=_ValidatedEntitySubsetValue(value))
 
     async def _compute_missing_asset_subset(
         self, key: AssetKey, from_subset: EntitySubset

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/asset_health/asset_check_health.py

@@ -25,6 +25,9 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
     failing_checks: set[AssetCheckKey]
     warning_checks: set[AssetCheckKey]
     all_checks: set[AssetCheckKey]
+    latest_passing_check_timestamp: Optional[float] = None
+    latest_failing_check_timestamp: Optional[float] = None
+    latest_warning_check_timestamp: Optional[float] = None
 
     @classmethod
     def default(cls) -> "AssetCheckHealthState":
@@ -33,6 +36,9 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
             failing_checks=set(),
             warning_checks=set(),
             all_checks=set(),
+            latest_passing_check_timestamp=None,
+            latest_failing_check_timestamp=None,
+            latest_warning_check_timestamp=None,
         )
 
     @property
@@ -77,6 +83,10 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
         warning_checks = set()
         failing_checks = set()
 
+        latest_passing_check_timestamp = None
+        latest_failing_check_timestamp = None
+        latest_warning_check_timestamp = None
+
         check_records = await AssetCheckSummaryRecord.gen_many(
             loading_context,
             check_keys,
@@ -125,20 +135,55 @@ class AssetCheckHealthState(LoadableBy[AssetKey]):
                     and last_check_evaluation.severity == AssetCheckSeverity.WARN
                 ):
                     warning_checks.add(check_key)
+                    if check_record.last_completed_check_execution_record is not None and (
+                        latest_warning_check_timestamp is None
+                        or latest_warning_check_timestamp
+                        < check_record.last_completed_check_execution_record.create_timestamp
+                    ):
+                        latest_warning_check_timestamp = (
+                            check_record.last_completed_check_execution_record.create_timestamp
+                        )
                 else:
                     failing_checks.add(check_key)
+                    if check_record.last_completed_check_execution_record is not None and (
+                        latest_failing_check_timestamp is None
+                        or latest_failing_check_timestamp
+                        < check_record.last_completed_check_execution_record.create_timestamp
+                    ):
+                        latest_failing_check_timestamp = (
+                            check_record.last_completed_check_execution_record.create_timestamp
+                        )
             elif last_check_execution_status == AssetCheckExecutionResolvedStatus.EXECUTION_FAILED:
                 # EXECUTION_FAILED checks don't have an evaluation and we want to report them as failures
                 failing_checks.add(check_key)
+                if check_record.last_completed_check_execution_record is not None and (
+                    latest_failing_check_timestamp is None
+                    or latest_failing_check_timestamp
+                    < check_record.last_completed_check_execution_record.create_timestamp
+                ):
+                    latest_failing_check_timestamp = (
+                        check_record.last_completed_check_execution_record.create_timestamp
+                    )
             else:
                 # asset check passed
                 passing_checks.add(check_key)
+                if check_record.last_completed_check_execution_record is not None and (
+                    latest_passing_check_timestamp is None
+                    or latest_passing_check_timestamp
+                    < check_record.last_completed_check_execution_record.create_timestamp
+                ):
+                    latest_passing_check_timestamp = (
+                        check_record.last_completed_check_execution_record.create_timestamp
+                    )
 
         return AssetCheckHealthState(
             passing_checks=passing_checks,
             failing_checks=failing_checks,
             warning_checks=warning_checks,
             all_checks=check_keys,
+            latest_passing_check_timestamp=latest_passing_check_timestamp,
+            latest_failing_check_timestamp=latest_failing_check_timestamp,
+            latest_warning_check_timestamp=latest_warning_check_timestamp,
         )
 
     @classmethod

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/asset_health/asset_freshness_health.py

@@ -24,6 +24,7 @@ class AssetFreshnessHealthState(LoadableBy[AssetKey]):
     """Maintains the latest freshness state for the asset."""
 
     freshness_state: FreshnessState
+    updated_timestamp: Optional[float] = None
 
     @property
     def health_status(self) -> AssetHealthStatus:
@@ -49,9 +50,11 @@ class AssetFreshnessHealthState(LoadableBy[AssetKey]):
             # freshness policy has no evaluations yet
             return cls(
                 freshness_state=FreshnessState.UNKNOWN,
+                updated_timestamp=None,
             )
         return cls(
             freshness_state=freshness_state_record.freshness_state,
+            updated_timestamp=freshness_state_record.updated_at.timestamp(),
         )
 
     @classmethod

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/asset_health/asset_materialization_health.py

@@ -35,6 +35,7 @@ class MinimalAssetMaterializationHealthState(LoadableBy[AssetKey]):
     num_failed_partitions: int
     num_currently_materialized_partitions: int
     partitions_snap: Optional[PartitionsSnap]
+    latest_failed_to_materialize_timestamp: Optional[float] = None
 
     @property
     def health_status(self) -> AssetHealthStatus:
@@ -62,6 +63,7 @@ class MinimalAssetMaterializationHealthState(LoadableBy[AssetKey]):
             num_failed_partitions=asset_materialization_health_state.failed_subset.size,
             num_currently_materialized_partitions=asset_materialization_health_state.currently_materialized_subset.size,
             partitions_snap=asset_materialization_health_state.partitions_snap,
+            latest_failed_to_materialize_timestamp=asset_materialization_health_state.latest_failed_to_materialize_timestamp,
         )
 
     @classmethod
@@ -101,6 +103,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
     partitions_snap: Optional[PartitionsSnap]
     latest_terminal_run_id: Optional[str]
     latest_materialization_timestamp: Optional[float] = None
+    latest_failed_to_materialize_timestamp: Optional[float] = None
 
     @property
     def partitions_def(self) -> Optional[PartitionsDefinition]:
@@ -152,6 +155,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
 
             last_run_id = None
             latest_materialization_timestamp = None
+            latest_failed_to_materialize_timestamp = None
             if asset_record is not None:
                 entry = asset_record.asset_entry
                 latest_record = max(
@@ -167,6 +171,11 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                     if entry.last_materialization_record
                     else None
                 )
+                latest_failed_to_materialize_timestamp = (
+                    entry.last_failed_to_materialize_record.timestamp
+                    if entry.last_failed_to_materialize_record
+                    else None
+                )
 
             return cls(
                 materialized_subset=SerializableEntitySubset(
@@ -178,6 +187,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                 partitions_snap=PartitionsSnap.from_def(partitions_def),
                 latest_terminal_run_id=last_run_id,
                 latest_materialization_timestamp=latest_materialization_timestamp,
+                latest_failed_to_materialize_timestamp=latest_failed_to_materialize_timestamp,
             )
 
         if asset_record is None:
@@ -187,6 +197,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                 partitions_snap=None,
                 latest_terminal_run_id=None,
                 latest_materialization_timestamp=None,
+                latest_failed_to_materialize_timestamp=None,
             )
 
         asset_entry = asset_record.asset_entry
@@ -195,6 +206,11 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
             if asset_entry.last_materialization_record
             else None
         )
+        latest_failed_to_materialize_timestamp = (
+            asset_entry.last_failed_to_materialize_record.timestamp
+            if asset_entry.last_failed_to_materialize_record
+            else None
+        )
         if asset_entry.last_run_id is None:
             return AssetMaterializationHealthState(
                 materialized_subset=SerializableEntitySubset(key=asset_key, value=False),
@@ -202,6 +218,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
                 partitions_snap=None,
                 latest_terminal_run_id=None,
                 latest_materialization_timestamp=latest_materialization_timestamp,
+                latest_failed_to_materialize_timestamp=latest_failed_to_materialize_timestamp,
             )
 
         has_ever_materialized = asset_entry.last_materialization is not None
@@ -218,6 +235,7 @@ class AssetMaterializationHealthState(LoadableBy[AssetKey]):
             partitions_snap=None,
             latest_terminal_run_id=latest_terminal_run_id,
             latest_materialization_timestamp=latest_materialization_timestamp,
+            latest_failed_to_materialize_timestamp=latest_failed_to_materialize_timestamp,
         )
 
     @classmethod

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/asset_key.py

@@ -195,7 +195,7 @@ def check_opt_coercible_to_asset_key_prefix_param(
 def key_prefix_from_coercible(key_prefix: CoercibleToAssetKeyPrefix) -> Sequence[str]:
     if isinstance(key_prefix, str):
         return [key_prefix]
-    elif isinstance(key_prefix, list):
+    elif isinstance(key_prefix, Sequence):
         return key_prefix
     else:
         check.failed(f"Unexpected type for key_prefix: {type(key_prefix)}")

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/assets/graph/remote_asset_graph.py

@@ -463,18 +463,27 @@ class RemoteAssetGraph(BaseAssetGraph[TRemoteAssetNode], ABC, Generic[TRemoteAss
         self,
     ) -> Mapping[AssetCheckKey, RemoteAssetCheckNode]: ...
 
+    # TODO this should just be what is returned from get()
+    def get_remote_asset_check_node(self, key: AssetCheckKey) -> RemoteAssetCheckNode:
+        return self.remote_asset_check_nodes_by_key[key]
+
+    def _get_asset_check_node_from_remote_asset_check_node(
+        self, remote_node: RemoteAssetCheckNode
+    ) -> AssetCheckNode:
+        return AssetCheckNode(
+            remote_node.asset_check.key,
+            remote_node.asset_check.additional_asset_keys,
+            remote_node.asset_check.blocking,
+            remote_node.asset_check.description,
+            remote_node.asset_check.automation_condition,
+            {},  # metadata not yet on AssetCheckNodeSnap
+        )
+
     ##### COMMON ASSET GRAPH INTERFACE
     @cached_property
     def _asset_check_nodes_by_key(self) -> Mapping[AssetCheckKey, AssetCheckNode]:  # pyright: ignore[reportIncompatibleVariableOverride]
         return {
-            k: AssetCheckNode(
-                k,
-                v.asset_check.additional_asset_keys,
-                v.asset_check.blocking,
-                v.asset_check.description,
-                v.asset_check.automation_condition,
-                {},  # metadata not yet on AssetCheckNodeSnap
-            )
+            k: self._get_asset_check_node_from_remote_asset_check_node(v)
             for k, v in self.remote_asset_check_nodes_by_key.items()
         }
 
@@ -484,7 +493,7 @@ class RemoteAssetGraph(BaseAssetGraph[TRemoteAssetNode], ABC, Generic[TRemoteAss
         if isinstance(entity_key, AssetKey):
             return self.get(entity_key).execution_set_entity_keys
         else:  # AssetCheckKey
-            return self.remote_asset_check_nodes_by_key[entity_key].execution_set_entity_keys
+            return self.get_remote_asset_check_node(entity_key).execution_set_entity_keys
 
     ##### REMOTE-SPECIFIC METHODS
 
@@ -684,7 +693,7 @@ class RemoteWorkspaceAssetGraph(RemoteAssetGraph[RemoteWorkspaceAssetNode]):
         if isinstance(key, AssetKey):
             return self.get(key).resolve_to_singular_repo_scoped_node().repository_handle
         else:
-            return self.remote_asset_check_nodes_by_key[key].handle
+            return self.get_remote_asset_check_node(key).handle
 
     def get_repo_scoped_node(
         self, key: EntityKey, repository_selector: "RepositorySelector"

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/decorators/repository_decorator.py

@@ -103,7 +103,7 @@ class _Repository:
 
         repository_definitions = fn()
         context = DefinitionsLoadContext.get()
-        defs_state_info = context.defs_state_info
+        defs_state_info = context.accessed_defs_state_info
 
         if context.load_type == DefinitionsLoadType.INITIALIZATION:
             reconstruction_metadata = context.get_pending_reconstruction_metadata()

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/definitions_load_context.py

@@ -7,7 +7,13 @@ from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, Optional, TypeVar, cast
 
-from dagster_shared.serdes.objects.models.defs_state_info import DefsStateInfo
+from dagster_shared.serdes.objects.models.defs_state_info import (
+    CODE_SERVER_STATE_VERSION,
+    LOCAL_STATE_VERSION,
+    DefsKeyStateInfo,
+    DefsStateInfo,
+    DefsStateManagementType,
+)
 from dagster_shared.serdes.serdes import PackableValue, deserialize_value, serialize_value
 
 from dagster._core.definitions.assets.definition.cacheable_assets_definition import (
@@ -17,8 +23,13 @@ from dagster._core.definitions.definitions_class import Definitions
 from dagster._core.definitions.metadata.metadata_value import (
     CodeLocationReconstructionMetadataValue,
 )
-from dagster._core.errors import DagsterInvalidInvocationError, DagsterInvariantViolationError
+from dagster._core.errors import DagsterInvariantViolationError
 from dagster._core.storage.defs_state.base import DefsStateStorage
+from dagster.components.utils.defs_state import DefsStateConfig
+from dagster.components.utils.project_paths import (
+    get_code_server_metadata_key,
+    get_local_state_path,
+)
 
 if TYPE_CHECKING:
     from dagster._core.definitions.repository_definition import RepositoryLoadData
@@ -62,6 +73,11 @@ class DefinitionsLoadContext:
         self._pending_reconstruction_metadata = {}
 
         defs_state_info = repository_load_data.defs_state_info if repository_load_data else None
+
+        # keep track of the keys that have been accessed during the load process
+        self._accessed_defs_state_keys = (
+            set() if defs_state_info is None else set(defs_state_info.info_mapping.keys())
+        )
         if load_type == DefinitionsLoadType.INITIALIZATION and defs_state_info is None:
             # defs_state_info is passed in during INITIALIZATION if explicit state versions
             # are provided via CLI arguments, otherwise we use the latest available state info
@@ -98,6 +114,13 @@ class DefinitionsLoadContext:
     def add_to_pending_reconstruction_metadata(self, key: str, metadata: Any) -> None:
         self._pending_reconstruction_metadata[key] = metadata
 
+    def add_code_server_defs_state_info(self, key: str, metadata: Any) -> None:
+        """Marks state that was stored during the code server initialization process."""
+        self._defs_state_info = DefsStateInfo.add_version(
+            self._defs_state_info, key, CODE_SERVER_STATE_VERSION
+        )
+        self.add_to_pending_reconstruction_metadata(get_code_server_metadata_key(key), metadata)
+
     def get_pending_reconstruction_metadata(self) -> Mapping[str, Any]:
         return self._pending_reconstruction_metadata
 
@@ -136,35 +159,84 @@ class DefinitionsLoadContext:
         )
 
     @property
-    def defs_state_info(self) -> Optional[DefsStateInfo]:
-        return self._defs_state_info
+    def accessed_defs_state_info(self) -> Optional[DefsStateInfo]:
+        return (
+            self._defs_state_info.for_keys(self._accessed_defs_state_keys)
+            if self._defs_state_info
+            else None
+        )
 
-    def _get_defs_state_version(self, key: str) -> Optional[str]:
+    def _mark_defs_key_accessed(self, key: str) -> None:
+        self._accessed_defs_state_keys.add(key)
+
+    def _get_defs_key_state_info(self, key: str) -> Optional[DefsKeyStateInfo]:
         """Ensures that if we attempt to access a key that doesn't exist, we mark it as None."""
+        self._mark_defs_key_accessed(key)
         current_info = self._defs_state_info or DefsStateInfo.empty()
-        version = current_info.get_version(key)
-        if version is None:
+        key_info = current_info.info_mapping.get(key)
+        if key_info is None:
             self._defs_state_info = DefsStateInfo.add_version(current_info, key, None)
-        return version
+        return key_info
+
+    def _get_defs_state_from_reconstruction_metadata(self, key: str) -> str:
+        metadata_key = get_code_server_metadata_key(key)
+        if self.load_type == DefinitionsLoadType.RECONSTRUCTION:
+            return self.reconstruction_metadata[metadata_key]
+        else:
+            return self._pending_reconstruction_metadata[metadata_key]
+
+    def add_defs_state_info(
+        self, key: str, version: str, create_timestamp: Optional[float] = None
+    ) -> None:
+        self._mark_defs_key_accessed(key)
+        self._defs_state_info = DefsStateInfo.add_version(
+            self._defs_state_info, key, version, create_timestamp
+        )
 
     @contextmanager
-    def temp_state_path(self, key: str) -> Iterator[Optional[Path]]:
-        """Context manager that creates a temporary path to hold local state for a component."""
-        state_storage = DefsStateStorage.get()
-        if state_storage is None:
-            raise DagsterInvalidInvocationError(
-                "Attempted to get a temp state path without a StateStorage in context. "
-                "This is likely the result of an internal framework error."
-            )
+    def state_path(
+        self, config: DefsStateConfig, state_storage: DefsStateStorage, project_root: Path
+    ) -> Iterator[Optional[Path]]:
+        """Context manager that creates a temporary path to hold local state for a component.
+
+        Args:
+            config: The state configuration for the component.
+            state_storage: The state storage instance.
+            project_root: The root directory of the project.
+        """
         # if no state has ever been written for this key, we return None to indicate that no state is available
-        version = self._get_defs_state_version(key)
-        if version is None:
-            yield None
-            return
-        with tempfile.TemporaryDirectory() as temp_dir:
-            state_path = Path(temp_dir) / key
-            state_storage.download_state_to_path(key, version, state_path)
+        key = config.key
+        key_info = self._get_defs_key_state_info(key)
+
+        if config.type == DefsStateManagementType.LOCAL_FILESYSTEM:
+            # it is possible for local state to exist without the defs_state_storage being aware
+            # of it if the state was added during docker build
+            state_path = get_local_state_path(key, project_root)
+            if not state_path.exists():
+                yield None
+                return
+            self.add_defs_state_info(key, LOCAL_STATE_VERSION, state_path.stat().st_ctime)
             yield state_path
+        elif config.type == DefsStateManagementType.VERSIONED_STATE_STORAGE:
+            key_info = self._get_defs_key_state_info(key)
+            # this implies that no state has been stored since the management type was changed
+            if key_info is None or key_info.management_type != config.type:
+                yield None
+                return
+            # grab state for storage
+            with tempfile.TemporaryDirectory() as temp_dir:
+                state_path = Path(temp_dir) / "state"
+                state_storage.download_state_to_path(key, key_info.version, state_path)
+                yield state_path
+        elif config.type == DefsStateManagementType.LEGACY_CODE_SERVER_SNAPSHOTS:
+            # state is stored in the reconstruction metadata
+            with tempfile.TemporaryDirectory() as temp_dir:
+                state_path = Path(temp_dir) / "state"
+                state = self._get_defs_state_from_reconstruction_metadata(key)
+                state_path.write_text(state)
+                yield state_path
+        else:
+            raise DagsterInvariantViolationError(f"Invalid management type: {config.type}")
 
 
 TState = TypeVar("TState", bound=PackableValue)

{dagster-1.11.14 → dagster-1.11.15}/dagster/_core/definitions/job_definition.py

@@ -446,7 +446,7 @@ class JobDefinition(IHasInternalInit):
     @property
     def run_config_schema(self) -> "RunConfigSchema":
         if self._run_config_schema is None:
-            self._run_config_schema = _create_run_config_schema(self, self.required_resource_keys)
+            self._run_config_schema = _create_run_config_schema(self)
         return self._run_config_schema
 
     @public
@@ -1431,7 +1431,6 @@ def _build_all_node_defs(node_defs: Sequence[NodeDefinition]) -> Mapping[str, No
 
 def _create_run_config_schema(
     job_def: JobDefinition,
-    required_resources: AbstractSet[str],
 ) -> "RunConfigSchema":
     from dagster._core.definitions.run_config import (
         RunConfigSchemaCreationData,
@@ -1439,12 +1438,19 @@ def _create_run_config_schema(
         define_run_config_schema_type,
     )
     from dagster._core.definitions.run_config_schema import RunConfigSchema
+    from dagster._core.remote_representation.code_location import is_implicit_asset_job_name
 
-    # When executing with a subset job, include the missing nodes
+    # When executing with a subset job that is not an implicit asset job, include the missing nodes
     # from the original job as ignored to allow execution with
     # run config that is valid for the original
     ignored_nodes: Sequence[Node] = []
-    if job_def.is_subset:
+
+    if job_def.is_subset and is_implicit_asset_job_name(job_def.name):
+        included_resource_defs = job_def.get_required_resource_defs()
+    else:
+        included_resource_defs = job_def.resource_defs
+
+    if job_def.is_subset and not is_implicit_asset_job_name(job_def.name):
         if isinstance(job_def.graph, SubselectedGraphDefinition):  # op selection provided
             ignored_nodes = job_def.graph.get_top_level_omitted_nodes()
         elif job_def.asset_selection_data:
@@ -1465,10 +1471,10 @@ def _create_run_config_schema(
             graph_def=job_def.graph,
             dependency_structure=job_def.graph.dependency_structure,
             executor_def=job_def.executor_def,
-            resource_defs=job_def.resource_defs,
+            resource_defs=included_resource_defs,
             logger_defs=job_def.loggers,
             ignored_nodes=ignored_nodes,
-            required_resources=required_resources,
+            required_resources=job_def.required_resource_keys,
             direct_inputs=job_def.input_values,
             asset_layer=job_def.asset_layer,
         )