furu 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

furu/core/furu.py

@@ -5,17 +5,34 @@ import os
 import signal
 import socket
 import sys
+import threading
 import time
 import traceback
 from abc import ABC, abstractmethod
 from pathlib import Path
 from types import FrameType
-from typing import Any, Callable, ClassVar, Self, TypedDict, TypeVar, cast, overload
+from typing import (
+    AbstractSet,
+    Any,
+    Callable,
+    ClassVar,
+    Hashable,
+    Mapping,
+    Protocol,
+    Self,
+    Sequence,
+    TypedDict,
+    TypeAlias,
+    TypeVar,
+    cast,
+)
 
 import chz
 import submitit
 from typing_extensions import dataclass_transform
 
+from chz.field import Field as ChzField
+
 from ..adapters import SubmititAdapter
 from ..adapters.submitit import SubmititJob
 from ..config import FURU_CONFIG
@@ -23,6 +40,7 @@ from ..errors import (
     MISSING,
     FuruComputeError,
     FuruLockNotAcquired,
+    FuruValidationError,
     FuruWaitTimeout,
 )
 from ..runtime import current_holder
@@ -41,7 +59,6 @@ from ..storage import (
 from ..storage.state import (
     _FuruState,
     _OwnerDict,
-    _StateAttemptFailed,
     _StateAttemptQueued,
     _StateAttemptRunning,
     _StateResultAbsent,
@@ -174,9 +191,39 @@ class Furu[T](ABC):
         raise NotImplementedError(f"{self.__class__.__name__}._load() not implemented")
 
     def _validate(self: Self) -> bool:
-        """Validate that result is complete and correct (override if needed)."""
+        """
+        Validate that result is complete and correct (override if needed).
+
+        Return False or raise FuruValidationError to mark artifacts as invalid.
+        """
         return True
 
+    def _dependencies(self: Self) -> "DependencySpec | None":
+        """Return extra dependencies not captured by fields."""
+        return None
+
+    def _executor_spec_key(self: Self) -> str:
+        return "default"
+
+    def _get_dependencies(self: Self, *, recursive: bool = True) -> list["Furu"]:
+        """Collect Furu dependencies from fields and `_dependencies()`."""
+        seen = {self._furu_hash}
+        dependencies: list[Furu] = []
+        _collect_dependencies(self, dependencies, seen, recursive=recursive)
+        return dependencies
+
+    def _dependency_hashes(self: Self) -> list[str]:
+        dependencies = _direct_dependencies(self)
+        if not dependencies:
+            return []
+
+        digests: set[str] = set()
+        for dependency in dependencies:
+            if dependency is self:
+                raise ValueError("Furu dependencies cannot include self")
+            digests.add(dependency._furu_hash)
+        return sorted(digests)
+
     def _invalidate_cached_success(self: Self, directory: Path, *, reason: str) -> None:
         logger = get_logger()
         logger.warning(
@@ -199,6 +246,27 @@ class Furu[T](ABC):
             directory, {"type": "result_invalidated", "reason": reason, "at": now}
         )
 
+    def _prepare_executor_rerun(self: Self, directory: Path) -> None:
+        if not self._always_rerun():
+            return
+        if not directory.exists():
+            return
+        migration = self._alias_record(directory)
+        if migration is not None and self._alias_is_active(directory, migration):
+            self._maybe_detach_alias(
+                directory=directory,
+                record=migration,
+                reason="always_rerun",
+            )
+        state = StateManager.read_state(directory)
+        if isinstance(state.result, _StateResultSuccess):
+            self._invalidate_cached_success(directory, reason="always_rerun enabled")
+
+    @property
+    def furu_hash(self: Self) -> str:
+        """Return the stable content hash for this Furu object."""
+        return self._furu_hash
+
     @property
     def _furu_hash(self: Self) -> str:
         """Compute hash of this object's content for storage identification."""
@@ -251,6 +319,33 @@ class Furu[T](ABC):
         """Log a message to the current holder's `furu.log`."""
         return log(message, level=level)
 
+    def _exists_quiet(self: Self) -> bool:
+        directory = self._base_furu_dir()
+        state = self.get_state(directory)
+
+        if not isinstance(state.result, _StateResultSuccess):
+            return False
+        try:
+            return self._validate()
+        except FuruValidationError as exc:
+            logger = get_logger()
+            logger.warning(
+                "exists %s -> false (validate invalid for %s: %s)",
+                directory,
+                f"{self.__class__.__name__}({self._furu_hash})",
+                exc,
+            )
+            return False
+        except Exception as exc:
+            logger = get_logger()
+            logger.exception(
+                "exists %s -> false (validate crashed for %s: %s)",
+                directory,
+                f"{self.__class__.__name__}({self._furu_hash})",
+                exc,
+            )
+            return False
+
     def exists(self: Self) -> bool:
         """Check if result exists and is valid."""
         logger = get_logger()
@@ -274,31 +369,93 @@ class Furu[T](ABC):
         """Get migration record for this object."""
         return MigrationManager.read_migration(self._base_furu_dir())
 
-    @overload
-    def load_or_create(self, executor: submitit.Executor) -> T | submitit.Job[T]: ...
-
-    @overload
-    def load_or_create(self, executor: None = None) -> T: ...
-
-    def load_or_create(
-        self: Self,
-        executor: submitit.Executor | None = None,
-    ) -> T | submitit.Job[T]:
+    def get(self: Self, *, force: bool = False) -> T:
         """
         Load result if it exists, computing if necessary.
 
         Args:
-            executor: Optional executor for batch submission (e.g., submitit.Executor)
+            force: Allow computation inside executor contexts if the spec matches.
 
         Returns:
-            Result if wait=True, job handle if wait=False, or None if already exists
+            Loaded or computed result.
 
         Raises:
             FuruComputeError: If computation fails with detailed error information
         """
+        from furu.execution.context import EXEC_CONTEXT
+        from furu.errors import (
+            FuruExecutionError,
+            FuruMissingArtifact,
+            FuruSpecMismatch,
+        )
+
+        ctx = EXEC_CONTEXT.get()
+        if ctx.mode == "executor":
+            directory = self._base_furu_dir()
+            if force:
+                if (
+                    ctx.current_node_hash is None
+                    or self._furu_hash != ctx.current_node_hash
+                ):
+                    raise FuruExecutionError(
+                        "force=True not allowed: only the current node may compute in executor mode. "
+                        f"current_node_hash={ctx.current_node_hash!r} "
+                        f"obj={self.__class__.__name__}({self._furu_hash})",
+                        hints=[
+                            "Declare this object as a dependency instead of calling dep.get(force=True).",
+                            "Inside executor mode, use get(force=True) only on the node being executed.",
+                        ],
+                    )
+                self._prepare_executor_rerun(directory)
+
+            exists_ok = self._exists_quiet()
+            if exists_ok and not (force and self._always_rerun()):
+                return self._load()
+
+            if force and not exists_ok:
+                state = self.get_state(directory)
+                if isinstance(state.result, _StateResultSuccess):
+                    self._invalidate_cached_success(
+                        directory, reason="_validate returned false (executor)"
+                    )
+
+            if not force:
+                raise FuruMissingArtifact(
+                    "Missing artifact "
+                    f"{self.__class__.__name__}({self._furu_hash}) in executor mode. "
+                    f"Requested by {ctx.current_node_hash}. Declare it as a dependency."
+                )
+
+            required = self._executor_spec_key()
+            if ctx.spec_key is None or required != ctx.spec_key:
+                raise FuruSpecMismatch(
+                    "force=True not allowed: "
+                    f"required={required!r} != worker={ctx.spec_key!r} (v1 exact match)"
+                )
+
+            status, created_here, result = self._run_locally(
+                start_time=time.time(),
+                allow_failed=FURU_CONFIG.retry_failed,
+                executor_mode=True,
+            )
+            if status == "success":
+                if created_here:
+                    return cast(T, result)
+                return self._load()
+
+            raise self._build_failed_state_error(
+                self._base_furu_dir(),
+                None,
+                message="Computation previously failed",
+            )
+
+        return self._get_impl_interactive(force=force)
+
+    def _get_impl_interactive(self: Self, *, force: bool) -> T:
         logger = get_logger()
         parent_holder = current_holder()
         has_parent = parent_holder is not None and parent_holder is not self
+        retry_failed_effective = FURU_CONFIG.retry_failed
         if has_parent:
             logger.debug(
                 "dep: begin %s %s %s",
@@ -380,7 +537,16 @@ class Furu[T](ABC):
 
                 state0 = StateManager.read_state(directory)
 
-                needs_reconcile = True
+                if (
+                    isinstance(state0.result, _StateResultFailed)
+                    and not retry_failed_effective
+                ):
+                    raise self._build_failed_state_error(
+                        directory,
+                        state0,
+                        message="Computation previously failed",
+                    )
+
                 if isinstance(state0.result, _StateResultSuccess):
                     # Double check logic if we fell through to here (e.g. race condition or invalidation above)
                     if self._always_rerun():
@@ -395,9 +561,6 @@ class Furu[T](ABC):
                                     directory, reason="_validate returned false"
                                 )
                                 state0 = StateManager.read_state(directory)
-                            else:
-                                # Valid success found, skip reconcile
-                                needs_reconcile = False
                         except Exception as e:
                             self._invalidate_cached_success(
                                 directory,
@@ -405,11 +568,6 @@ class Furu[T](ABC):
                             )
                             state0 = StateManager.read_state(directory)
 
-                if needs_reconcile and executor is not None:
-                    adapter0 = SubmititAdapter(executor)
-                    self._reconcile(directory, adapter=adapter0)
-                    state0 = StateManager.read_state(directory)
-
                 attempt0 = state0.attempt
                 if isinstance(state0.result, _StateResultSuccess):
                     decision = "success->load"
@@ -429,7 +587,7 @@ class Furu[T](ABC):
                 if decision != "success->load":
                     write_separator()
                     logger.debug(
-                        "load_or_create %s %s %s (%s)",
+                        "get %s %s %s (%s)",
                         self.__class__.__name__,
                         self._furu_hash,
                         directory,
@@ -449,7 +607,7 @@ class Furu[T](ABC):
                         # failures even when we suppressed the cache-hit header line.
                         write_separator()
                         logger.error(
-                            "load_or_create %s %s (load failed)",
+                            "get %s %s (load failed)",
                             self.__class__.__name__,
                             self._furu_hash,
                         )
@@ -459,54 +617,30 @@ class Furu[T](ABC):
                             e,
                         ) from e
 
-                # Synchronous execution
-                if executor is None:
-                    status, created_here, result = self._run_locally(
-                        start_time=start_time
-                    )
-                    if status == "success":
-                        ok = True
-                        if created_here:
-                            logger.debug(
-                                "load_or_create: %s created -> return",
-                                self.__class__.__name__,
-                            )
-                            return cast(T, result)
+                status, created_here, result = self._run_locally(
+                    start_time=start_time,
+                    allow_failed=retry_failed_effective,
+                    executor_mode=False,
+                )
+                if status == "success":
+                    ok = True
+                    if created_here:
                         logger.debug(
-                            "load_or_create: %s success -> _load()",
+                            "get: %s created -> return",
                             self.__class__.__name__,
                         )
-                        return self._load()
-
-                    state = StateManager.read_state(directory)
-                    attempt = state.attempt
-                    message = (
-                        attempt.error.message
-                        if isinstance(attempt, _StateAttemptFailed)
-                        else None
-                    )
-                    suffix = (
-                        f": {message}" if isinstance(message, str) and message else ""
-                    )
-                    raise FuruComputeError(
-                        f"Computation {status}{suffix}",
-                        StateManager.get_state_path(directory),
+                        return cast(T, result)
+                    logger.debug(
+                        "get: %s success -> _load()",
+                        self.__class__.__name__,
                     )
+                    return self._load()
 
-                # Asynchronous execution with submitit
-                (submitit_folder := self._base_furu_dir() / "submitit").mkdir(
-                    exist_ok=True, parents=True
-                )
-                executor.folder = submitit_folder
-                adapter = SubmititAdapter(executor)
-
-                logger.debug(
-                    "load_or_create: %s -> submitit submit_once()",
-                    self.__class__.__name__,
+                raise self._build_failed_state_error(
+                    directory,
+                    None,
+                    message="Computation previously failed",
                 )
-                job = self._submit_once(adapter, directory, None)
-                ok = True
-                return cast(submitit.Job[T], job)
         finally:
             if has_parent:
                 logger.debug(
@@ -517,7 +651,7 @@ class Furu[T](ABC):
                 )
 
     def _log_console_start(self, action_color: str) -> None:
-        """Log the start of load_or_create to console with caller info."""
+        """Log the start of get to console with caller info."""
         logger = get_logger()
         frame = sys._getframe(1)
 
@@ -537,7 +671,7 @@ class Furu[T](ABC):
                 frame = frame.f_back
 
         logger.info(
-            "load_or_create %s %s",
+            "get %s %s",
             self.__class__.__name__,
             self._furu_hash,
             extra={
@@ -547,12 +681,50 @@ class Furu[T](ABC):
             },
         )
 
+    def _add_exception_breadcrumbs(self, exc: BaseException, directory: Path) -> None:
+        if not hasattr(exc, "add_note"):
+            return
+        note = f"Furu dir: {directory}"
+        exc.add_note(note)
+
+    @staticmethod
+    def _failed_state_hints() -> list[str]:
+        return [
+            "To retry this failed artifact: set FURU_RETRY_FAILED=1 or call get() again.",
+            "To inspect details: open the furu dir shown above.",
+        ]
+
+    def _build_failed_state_error(
+        self,
+        directory: Path,
+        state: _FuruState | None,
+        *,
+        message: str,
+    ) -> FuruComputeError:
+        current_state = state or StateManager.read_state(directory)
+        attempt = current_state.attempt
+        error = getattr(attempt, "error", None) if attempt is not None else None
+        return FuruComputeError(
+            message,
+            StateManager.get_state_path(directory),
+            recorded_error_type=getattr(error, "type", None),
+            recorded_error_message=getattr(error, "message", None),
+            recorded_traceback=getattr(error, "traceback", None),
+            hints=self._failed_state_hints(),
+        )
+
+    def _effective_max_wait_time_sec(self) -> float | None:
+        if FURU_CONFIG.max_wait_time_sec is not None:
+            return FURU_CONFIG.max_wait_time_sec
+        return self._max_wait_time_sec
+
     def _check_timeout(self, start_time: float) -> None:
         """Check if operation has timed out."""
-        if self._max_wait_time_sec is not None:
-            if time.time() - start_time > self._max_wait_time_sec:
+        max_wait_time = self._effective_max_wait_time_sec()
+        if max_wait_time is not None:
+            if time.time() - start_time > max_wait_time:
                 raise FuruWaitTimeout(
-                    f"Furu operation timed out after {self._max_wait_time_sec} seconds."
+                    f"Furu operation timed out after {max_wait_time} seconds."
                 )
 
     def _is_migrated_state(self, directory: Path) -> bool:
@@ -613,8 +785,10 @@ class Furu[T](ABC):
         event: dict[str, str | int] = {
             "type": "migration_overwrite",
             "policy": record.policy,
-            "from": f"{record.from_namespace}:{record.from_hash}",
-            "to": f"{record.to_namespace}:{record.to_hash}",
+            "from_namespace": record.from_namespace,
+            "from_hash": record.from_hash,
+            "to_namespace": record.to_namespace,
+            "to_hash": record.to_hash,
             "reason": reason,
         }
         StateManager.append_event(directory, event.copy())
@@ -625,6 +799,8 @@ class Furu[T](ABC):
         adapter: SubmititAdapter,
         directory: Path,
         on_job_id: Callable[[str], None] | None,
+        *,
+        allow_failed: bool,
     ) -> SubmititJob | None:
         """Submit job once without waiting (fire-and-forget mode)."""
         logger = get_logger()
@@ -693,7 +869,7 @@ class Furu[T](ABC):
                 scheduler={},
             )
 
-            job = adapter.submit(lambda: self._worker_entry())
+            job = adapter.submit(lambda: self._worker_entry(allow_failed=allow_failed))
 
             # Save job handle and watch for job ID
             adapter.pickle_job(job, directory)
@@ -729,102 +905,205 @@ class Furu[T](ABC):
         finally:
             StateManager.release_lock(lock_fd, lock_path)
 
-    def _worker_entry(self: Self) -> None:
+    def _worker_entry(self: Self, *, allow_failed: bool | None = None) -> None:
         """Entry point for worker process (called by submitit or locally)."""
         with enter_holder(self):
             logger = get_logger()
-            directory = self._base_furu_dir()
-            directory.mkdir(parents=True, exist_ok=True)
-
-            env_info = self._collect_submitit_env()
-
-            try:
-                with compute_lock(
-                    directory,
+            # Ensure executor semantics apply to *all* work in the worker, not
+            # just `_create()`. This prevents accidental dependency computation
+            # (e.g., from within `_validate()` or metadata hooks).
+            from furu.execution.context import EXEC_CONTEXT, ExecContext
+
+            exec_token = EXEC_CONTEXT.set(
+                ExecContext(
+                    mode="executor",
+                    spec_key=self._executor_spec_key(),
                     backend="submitit",
-                    lease_duration_sec=FURU_CONFIG.lease_duration_sec,
-                    heartbeat_interval_sec=FURU_CONFIG.heartbeat_interval_sec,
-                    owner={
-                        "pid": os.getpid(),
-                        "host": socket.gethostname(),
-                        "user": getpass.getuser(),
-                        "command": " ".join(sys.argv) if sys.argv else "<unknown>",
-                    },
-                    scheduler={
-                        "backend": env_info.get("backend"),
-                        "job_id": env_info.get("slurm_job_id"),
-                    },
-                    max_wait_time_sec=None,  # Workers wait indefinitely
-                    poll_interval_sec=FURU_CONFIG.poll_interval,
-                    wait_log_every_sec=FURU_CONFIG.wait_log_every_sec,
-                    reconcile_fn=lambda d: self._reconcile(d),
-                ) as ctx:
-                    # Refresh metadata (now safe - attempt is already recorded)
-                    metadata = MetadataManager.create_metadata(
-                        self, directory, ignore_diff=FURU_CONFIG.ignore_git_diff
-                    )
-                    MetadataManager.write_metadata(metadata, directory)
+                    current_node_hash=self._furu_hash,
+                )
+            )
+            try:
+                directory = self._base_furu_dir()
+                directory.mkdir(parents=True, exist_ok=True)
+                always_rerun = self._always_rerun()
+                needs_success_invalidation = False
+                if not always_rerun:
+                    exists_ok = self._exists_quiet()
+                    if not exists_ok:
+                        state = self.get_state(directory)
+                        if isinstance(state.result, _StateResultSuccess):
+                            needs_success_invalidation = True
+
+                env_info = self._collect_submitit_env()
+                allow_failed_effective = (
+                    allow_failed
+                    if allow_failed is not None
+                    else FURU_CONFIG.retry_failed
+                )
+                allow_success = always_rerun or needs_success_invalidation
 
-                    # Set up signal handlers
-                    self._setup_signal_handlers(
-                        directory, ctx.stop_heartbeat, attempt_id=ctx.attempt_id
-                    )
+                try:
+                    with compute_lock(
+                        directory,
+                        backend="submitit",
+                        lease_duration_sec=FURU_CONFIG.lease_duration_sec,
+                        heartbeat_interval_sec=FURU_CONFIG.heartbeat_interval_sec,
+                        owner={
+                            "pid": os.getpid(),
+                            "host": socket.gethostname(),
+                            "user": getpass.getuser(),
+                            "command": " ".join(sys.argv) if sys.argv else "<unknown>",
+                        },
+                        scheduler={
+                            "backend": env_info.get("backend"),
+                            "job_id": env_info.get("slurm_job_id"),
+                        },
+                        max_wait_time_sec=None,  # Workers wait indefinitely
+                        poll_interval_sec=FURU_CONFIG.poll_interval,
+                        wait_log_every_sec=FURU_CONFIG.wait_log_every_sec,
+                        reconcile_fn=lambda d: self._reconcile(d),
+                        allow_failed=allow_failed_effective,
+                        allow_success=allow_success,
+                    ) as ctx:
+                        self._prepare_executor_rerun(directory)
+                        if not always_rerun:
+                            exists_ok = self._exists_quiet()
+                            if not exists_ok:
+                                state = self.get_state(directory)
+                                if isinstance(state.result, _StateResultSuccess):
+                                    self._invalidate_cached_success(
+                                        directory,
+                                        reason="_validate returned false (worker)",
+                                    )
+
+                        stage = "metadata"
+                        try:
+                            # Refresh metadata (now safe - attempt is already recorded)
+                            metadata = MetadataManager.create_metadata(
+                                self,
+                                directory,
+                                ignore_diff=FURU_CONFIG.ignore_git_diff,
+                            )
+                            MetadataManager.write_metadata(metadata, directory)
 
-                    try:
-                        # Run computation
-                        logger.debug(
-                            "_create: begin %s %s %s",
-                            self.__class__.__name__,
-                            self._furu_hash,
-                            directory,
-                        )
-                        self._create()
-                        logger.debug(
-                            "_create: ok %s %s %s",
-                            self.__class__.__name__,
-                            self._furu_hash,
-                            directory,
-                        )
-                        StateManager.write_success_marker(
-                            directory, attempt_id=ctx.attempt_id
-                        )
-                        StateManager.finish_attempt_success(
-                            directory, attempt_id=ctx.attempt_id
-                        )
-                        logger.info(
-                            "_create ok %s %s",
-                            self.__class__.__name__,
-                            self._furu_hash,
-                            extra={"furu_console_only": True},
-                        )
-                    except Exception as e:
-                        logger.error(
-                            "_create failed %s %s %s",
-                            self.__class__.__name__,
-                            self._furu_hash,
-                            directory,
-                            extra={"furu_file_only": True},
-                        )
-                        logger.error(
-                            "%s", format_traceback(e), extra={"furu_file_only": True}
-                        )
+                            # Set up signal handlers
+                            stage = "signal handler setup"
+                            self._setup_signal_handlers(
+                                directory,
+                                ctx.stop_heartbeat,
+                                attempt_id=ctx.attempt_id,
+                            )
 
-                        tb = "".join(
-                            traceback.format_exception(type(e), e, e.__traceback__)
-                        )
-                        StateManager.finish_attempt_failed(
-                            directory,
-                            attempt_id=ctx.attempt_id,
-                            error={
-                                "type": type(e).__name__,
-                                "message": str(e),
-                                "traceback": tb,
-                            },
+                            stage = "_create"
+                            # Run computation
+                            logger.debug(
+                                "_create: begin %s %s %s",
+                                self.__class__.__name__,
+                                self._furu_hash,
+                                directory,
+                            )
+                            self._create()
+                            logger.debug(
+                                "_create: ok %s %s %s",
+                                self.__class__.__name__,
+                                self._furu_hash,
+                                directory,
+                            )
+                            StateManager.write_success_marker(
+                                directory, attempt_id=ctx.attempt_id
+                            )
+                            StateManager.finish_attempt_success(
+                                directory, attempt_id=ctx.attempt_id
+                            )
+                            logger.info(
+                                "_create ok %s %s",
+                                self.__class__.__name__,
+                                self._furu_hash,
+                                extra={"furu_console_only": True},
+                            )
+                        except Exception as e:
+                            if stage == "_create":
+                                logger.error(
+                                    "_create failed %s %s %s",
+                                    self.__class__.__name__,
+                                    self._furu_hash,
+                                    directory,
+                                    extra={"furu_file_only": True},
+                                )
+                            else:
+                                logger.error(
+                                    "attempt failed (%s) %s %s %s",
+                                    stage,
+                                    self.__class__.__name__,
+                                    self._furu_hash,
+                                    directory,
+                                    extra={"furu_file_only": True},
+                                )
+                            logger.error(
+                                "%s",
+                                format_traceback(e),
+                                extra={"furu_file_only": True},
+                            )
+
+                            tb = "".join(
+                                traceback.format_exception(type(e), e, e.__traceback__)
+                            )
+                            StateManager.finish_attempt_failed(
+                                directory,
+                                attempt_id=ctx.attempt_id,
+                                error={
+                                    "type": type(e).__name__,
+                                    "message": str(e),
+                                    "traceback": tb,
+                                },
+                            )
+                            self._add_exception_breadcrumbs(e, directory)
+                            if stage != "_create":
+                                message = (
+                                    "Failed to create metadata"
+                                    if stage == "metadata"
+                                    else "Failed to set up signal handlers"
+                                )
+                                raise FuruComputeError(
+                                    message,
+                                    StateManager.get_state_path(directory),
+                                    e,
+                                ) from e
+                            raise
+                except FuruLockNotAcquired as exc:
+                    # Experiment already completed; succeed if success, fail if failed.
+                    state = StateManager.read_state(directory)
+                    state_path = StateManager.get_state_path(directory)
+                    attempt = state.attempt
+                    attempt_info = "no active attempt"
+                    if attempt is not None:
+                        attempt_info = (
+                            f"attempt {attempt.id} status {attempt.status} "
+                            f"backend {attempt.backend}"
                         )
-                        raise
-            except FuruLockNotAcquired:
-                # Experiment already completed (success or failed), nothing to do
-                return
+                    hints = [
+                        f"Furu hash: {self._furu_hash}",
+                        f"Directory: {directory}",
+                        f"State file: {state_path}",
+                        f"Attempt: {attempt_info}",
+                    ]
+                    if isinstance(state.result, _StateResultSuccess):
+                        return
+                    if isinstance(state.result, _StateResultFailed):
+                        if allow_failed_effective:
+                            return
+                        raise FuruComputeError(
+                            "Worker refused to run: experiment already failed",
+                            state_path,
+                            exc,
+                            hints=hints,
+                        ) from exc
+                    raise FuruLockNotAcquired(
+                        "Worker refused to run: experiment already running elsewhere",
+                        hints=hints,
+                    ) from exc
+            finally:
+                EXEC_CONTEXT.reset(exec_token)
 
     def _collect_submitit_env(self: Self) -> _SubmititEnvInfo:
         """Collect submitit/slurm environment information."""
@@ -850,16 +1129,23 @@ class Furu[T](ABC):
 
         return info
 
-    def _run_locally(self: Self, start_time: float) -> tuple[str, bool, T | None]:
+    def _run_locally(
+        self: Self,
+        start_time: float,
+        *,
+        allow_failed: bool,
+        executor_mode: bool = False,
+    ) -> tuple[str, bool, T | None]:
         """Run computation locally, returning (status, created_here, result)."""
         logger = get_logger()
         directory = self._base_furu_dir()
 
         # Calculate remaining time for the lock wait
         max_wait: float | None = None
-        if self._max_wait_time_sec is not None:
+        max_wait_time = self._effective_max_wait_time_sec()
+        if max_wait_time is not None:
             elapsed = time.time() - start_time
-            max_wait = max(0.0, self._max_wait_time_sec - elapsed)
+            max_wait = max(0.0, max_wait_time - elapsed)
 
         try:
             with compute_lock(
@@ -878,26 +1164,23 @@ class Furu[T](ABC):
                 poll_interval_sec=FURU_CONFIG.poll_interval,
                 wait_log_every_sec=FURU_CONFIG.wait_log_every_sec,
                 reconcile_fn=lambda d: self._reconcile(d),
+                allow_failed=allow_failed,
             ) as ctx:
-                # Create metadata (now safe - attempt is already recorded)
+                stage = "metadata"
                 try:
+                    # Create metadata (now safe - attempt is already recorded)
                     metadata = MetadataManager.create_metadata(
                         self, directory, ignore_diff=FURU_CONFIG.ignore_git_diff
                     )
                     MetadataManager.write_metadata(metadata, directory)
-                except Exception as e:
-                    raise FuruComputeError(
-                        "Failed to create metadata",
-                        StateManager.get_state_path(directory),
-                        e,
-                    ) from e
-
-                # Set up preemption handler
-                self._setup_signal_handlers(
-                    directory, ctx.stop_heartbeat, attempt_id=ctx.attempt_id
-                )
 
-                try:
+                    # Set up preemption handler
+                    stage = "signal handler setup"
+                    self._setup_signal_handlers(
+                        directory, ctx.stop_heartbeat, attempt_id=ctx.attempt_id
+                    )
+
+                    stage = "_create"
                     # Run the computation
                     logger.debug(
                         "_create: begin %s %s %s",
@@ -905,7 +1188,23 @@ class Furu[T](ABC):
                         self._furu_hash,
                         directory,
                     )
-                    result = self._create()
+                    token = None
+                    if executor_mode:
+                        from furu.execution.context import EXEC_CONTEXT, ExecContext
+
+                        token = EXEC_CONTEXT.set(
+                            ExecContext(
+                                mode="executor",
+                                spec_key=self._executor_spec_key(),
+                                backend="local",
+                                current_node_hash=self._furu_hash,
+                            )
+                        )
+                    try:
+                        result = self._create()
+                    finally:
+                        if token is not None:
+                            EXEC_CONTEXT.reset(token)
                     logger.debug(
                         "_create: ok %s %s %s",
                         self.__class__.__name__,
@@ -926,13 +1225,23 @@ class Furu[T](ABC):
                     )
                     return "success", True, result
                 except Exception as e:
-                    logger.error(
-                        "_create failed %s %s %s",
-                        self.__class__.__name__,
-                        self._furu_hash,
-                        directory,
-                        extra={"furu_file_only": True},
-                    )
+                    if stage == "_create":
+                        logger.error(
+                            "_create failed %s %s %s",
+                            self.__class__.__name__,
+                            self._furu_hash,
+                            directory,
+                            extra={"furu_file_only": True},
+                        )
+                    else:
+                        logger.error(
+                            "attempt failed (%s) %s %s %s",
+                            stage,
+                            self.__class__.__name__,
+                            self._furu_hash,
+                            directory,
+                            extra={"furu_file_only": True},
+                        )
                     logger.error(
                         "%s", format_traceback(e), extra={"furu_file_only": True}
                     )
@@ -950,6 +1259,18 @@ class Furu[T](ABC):
                             "traceback": tb,
                         },
                     )
+                    self._add_exception_breadcrumbs(e, directory)
+                    if stage != "_create":
+                        message = (
+                            "Failed to create metadata"
+                            if stage == "metadata"
+                            else "Failed to set up signal handlers"
+                        )
+                        raise FuruComputeError(
+                            message,
+                            StateManager.get_state_path(directory),
+                            e,
+                        ) from e
                     raise
         except FuruLockNotAcquired:
             # Lock couldn't be acquired because experiment already completed
@@ -981,6 +1302,8 @@ class Furu[T](ABC):
         attempt_id: str,
     ) -> None:
         """Set up signal handlers for graceful preemption."""
+        if threading.current_thread() is not threading.main_thread():
+            return
 
         def handle_signal(signum: int, frame: FrameType | None) -> None:
             try:
@@ -998,4 +1321,201 @@ class Furu[T](ABC):
             signal.signal(sig, handle_signal)
 
 
+class DependencyChzSpec(Protocol):
+    __chz_fields__: dict[str, ChzField]
+
+
+DependencySequence: TypeAlias = Sequence[Furu]
+DependencySet: TypeAlias = AbstractSet[Furu]
+DependencyMapping: TypeAlias = Mapping[str, Furu]
+DependencyCollection: TypeAlias = DependencySequence | DependencySet | DependencyMapping
+DependencyValue: TypeAlias = Furu | DependencyCollection
+DependencySpec: TypeAlias = DependencyValue | DependencyChzSpec
+DependencyLeaf: TypeAlias = str | int | float | bool | None | Path | bytes
+DependencyScanValue: TypeAlias = (
+    DependencyLeaf
+    | Furu
+    | Mapping[Hashable, "DependencyScanValue"]
+    | Sequence["DependencyScanValue"]
+    | AbstractSet["DependencyScanValue"]
+    | DependencyChzSpec
+)
+
+
+def _collect_dependencies(
+    obj: Furu,
+    dependencies: list[Furu],
+    seen: set[str],
+    *,
+    recursive: bool,
+) -> None:
+    for dependency in _direct_dependencies(obj):
+        digest = dependency._furu_hash
+        if digest in seen:
+            continue
+        seen.add(digest)
+        dependencies.append(dependency)
+        if recursive:
+            _collect_dependencies(
+                dependency,
+                dependencies,
+                seen,
+                recursive=recursive,
+            )
+
+
+def _direct_dependencies(obj: Furu) -> list[Furu]:
+    dependencies: list[Furu] = []
+    for field in chz.chz_fields(obj).values():
+        value = cast(DependencyScanValue, getattr(obj, field.logical_name))
+        dependencies.extend(_collect_dependencies_from_value(value))
+    extra = obj._dependencies()
+    if extra is not None:
+        dependencies.extend(_collect_dependencies_from_spec(extra, path="dependencies"))
+    return dependencies
+
+
+def _collect_dependencies_from_value(value: DependencyScanValue) -> list[Furu]:
+    dependencies: list[Furu] = []
+    if isinstance(value, Furu):
+        dependencies.append(value)
+        return dependencies
+    if isinstance(value, dict):
+        mapping = cast(Mapping[Hashable, DependencyScanValue], value)
+        for item in mapping.values():
+            dependencies.extend(_collect_dependencies_from_value(item))
+        return dependencies
+    if isinstance(value, (list, tuple)):
+        sequence = cast(Sequence[DependencyScanValue], value)
+        for item in sequence:
+            dependencies.extend(_collect_dependencies_from_value(item))
+        return dependencies
+    if isinstance(value, (set, frozenset)):
+        items = _sorted_dependency_set(cast(AbstractSet[DependencyScanValue], value))
+        for item in items:
+            dependencies.extend(_collect_dependencies_from_value(item))
+        return dependencies
+    if chz.is_chz(value):
+        for field in chz.chz_fields(value).values():
+            field_value = cast(DependencyScanValue, getattr(value, field.logical_name))
+            dependencies.extend(_collect_dependencies_from_value(field_value))
+    return dependencies
+
+
+def _collect_dependencies_from_spec(value: DependencySpec, path: str) -> list[Furu]:
+    if isinstance(value, Furu):
+        return [value]
+    if isinstance(value, dict):
+        return _collect_dependencies_from_mapping(
+            cast(Mapping[Hashable, DependencyValue], value),
+            path,
+        )
+    if isinstance(value, (list, tuple)):
+        return _collect_dependencies_from_sequence(
+            cast(Sequence[DependencyValue], value),
+            path,
+        )
+    if isinstance(value, (set, frozenset)):
+        return _collect_dependencies_from_set(
+            cast(AbstractSet[DependencyValue], value),
+            path,
+        )
+    if chz.is_chz(value):
+        dependencies: list[Furu] = []
+        for field in chz.chz_fields(value).values():
+            field_value = getattr(value, field.logical_name)
+            field_path = f"{path}.{field.logical_name}"
+            dependencies.extend(
+                _collect_dependencies_from_value_spec(field_value, field_path)
+            )
+        return dependencies
+    raise _dependency_type_error(path, value)
+
+
+def _collect_dependencies_from_value_spec(
+    value: DependencyValue,
+    path: str,
+) -> list[Furu]:
+    if isinstance(value, Furu):
+        return [value]
+    if isinstance(value, dict):
+        return _collect_dependencies_from_mapping(
+            cast(Mapping[Hashable, DependencyValue], value),
+            path,
+        )
+    if isinstance(value, (list, tuple)):
+        return _collect_dependencies_from_sequence(
+            cast(Sequence[DependencyValue], value),
+            path,
+        )
+    if isinstance(value, (set, frozenset)):
+        return _collect_dependencies_from_set(
+            cast(AbstractSet[DependencyValue], value),
+            path,
+        )
+    raise _dependency_type_error(path, value)
+
+
+def _collect_dependencies_from_mapping(
+    mapping: Mapping[Hashable, DependencyValue],
+    path: str,
+) -> list[Furu]:
+    dependencies: list[Furu] = []
+    for key, item in mapping.items():
+        if not isinstance(item, Furu):
+            raise _dependency_type_error(f"{path}[{key!r}]", item)
+        dependencies.append(item)
+    return dependencies
+
+
+def _collect_dependencies_from_sequence(
+    sequence: Sequence[DependencyValue],
+    path: str,
+) -> list[Furu]:
+    dependencies: list[Furu] = []
+    for index, item in enumerate(sequence):
+        if not isinstance(item, Furu):
+            raise _dependency_type_error(f"{path}[{index}]", item)
+        dependencies.append(item)
+    return dependencies
+
+
+def _collect_dependencies_from_set(
+    values: AbstractSet[DependencyValue],
+    path: str,
+) -> list[Furu]:
+    dependencies: list[Furu] = []
+    ordered = sorted(
+        list(cast(AbstractSet[DependencyScanValue], values)),
+        key=_dependency_sort_key,
+    )
+    for index, item in enumerate(ordered):
+        if not isinstance(item, Furu):
+            raise _dependency_type_error(f"{path}[{index}]", item)
+        dependencies.append(item)
+    return dependencies
+
+
+def _sorted_dependency_set(
+    values: AbstractSet[DependencyScanValue],
+) -> list[DependencyScanValue]:
+    return sorted(list(values), key=_dependency_sort_key)
+
+
+def _dependency_sort_key(value: DependencyScanValue) -> tuple[int, str]:
+    if isinstance(value, Furu):
+        return (0, value._furu_hash)
+    return (1, f"{type(value).__name__}:{value!r}")
+
+
+def _dependency_type_error(
+    path: str,
+    value: DependencySpec | DependencyValue | DependencyScanValue,
+) -> TypeError:
+    return TypeError(
+        f"{path} must be a Furu instance or a collection of Furu instances; "
+        f"got {type(value).__name__}"
+    )
+
+
 _H = TypeVar("_H", bound=Furu, covariant=True)