experimaestro 2.0.0a8__py3-none-any.whl → 2.0.0b8__py3-none-any.whl

experimaestro/settings.py

@@ -5,6 +5,7 @@ from functools import lru_cache
 from pathlib import Path
 from typing import Dict, Optional, List
 import logging
+import fnmatch
 
 
 @dataclass
@@ -38,6 +39,12 @@ class WorkspaceSettings:
     alt_workspaces: List[str] = field(default_factory=list)
     """Alternative workspaces to find jobs or experiments"""
 
+    max_retries: int = 3
+    """Maximum number of retries for resumable tasks that timeout (default: 3)"""
+
+    triggers: List[str] = field(default_factory=list)
+    """Glob patterns to automatically select this workspace based on experiment ID"""
+
     def __post_init__(self):
         self.path = self.path.expanduser().resolve()
 
@@ -84,9 +91,21 @@ def get_workspace(id: Optional[str] = None) -> Optional[WorkspaceSettings]:
 
 
 def find_workspace(
-    *, workspace: Optional[str] = None, workdir: Optional[Path] = None
+    *,
+    workspace: Optional[str] = None,
+    workdir: Optional[Path] = None,
+    experiment_id: Optional[str] = None,
 ) -> WorkspaceSettings:
-    """Find workspace"""
+    """Find workspace
+
+    Args:
+        workspace: Explicit workspace ID to use
+        workdir: Explicit working directory path
+        experiment_id: Experiment ID to match against workspace triggers
+
+    Returns:
+        WorkspaceSettings object
+    """
     workdir = Path(workdir) if workdir else None
 
     if workspace:
@@ -103,8 +122,28 @@ def find_workspace(
         logging.info("Using workdir %s", workdir)
         ws_env = WorkspaceSettings("", workdir)
     else:
-        ws_env = get_workspace()
-        assert ws_env is not None, "No workdir or workspace defined, and no default"
-        logging.info("Using default workspace %s", ws_env.id)
+        # Try to match experiment_id against workspace triggers
+        matched_workspace = None
+        if experiment_id:
+            workspaces = get_settings().workspaces
+            for ws in workspaces:
+                for trigger in ws.triggers:
+                    if fnmatch.fnmatch(experiment_id, trigger):
+                        matched_workspace = ws
+                        logging.info(
+                            "Auto-selected workspace %s (matched trigger '%s')",
+                            ws.id,
+                            trigger,
+                        )
+                        break
+                if matched_workspace:
+                    break
+
+        if matched_workspace:
+            ws_env = matched_workspace
+        else:
+            ws_env = get_workspace()
+            assert ws_env is not None, "No workdir or workspace defined, and no default"
+            logging.info("Using default workspace %s", ws_env.id)
 
     return ws_env

experimaestro/sphinx/__init__.py

@@ -113,12 +113,12 @@ class ConfigDocumenter(ClassDocumenter):
     @staticmethod
     def formatDefault(value) -> str:
         if isinstance(value, Config):
-            objecttype = value.__xpmtype__.objecttype
+            value_type = value.__xpmtype__.value_type
             params = ", ".join(
                 [f"{key}={value}" for key, value in value.__xpm__.values.items()]
             )
             # It would be possible to do better... if not
-            return f"{objecttype.__module__}.{objecttype.__qualname__}({params})"
+            return f"{value_type.__module__}.{value_type.__qualname__}({params})"
 
         return str(value)
 
@@ -176,7 +176,7 @@ class ConfigDocumenter(ClassDocumenter):
             self.add_line("   " + _("Bases: %s") % ", ".join(base_classes), sourcename)
 
         # Adds return type if different
-        if xpminfo.returntype != xpminfo.objecttype:
+        if xpminfo.returntype != xpminfo.value_type:
             self.add_line("", sourcename)
             self.add_line(
                 "   " + _("Submit type: %s") % restify(xpminfo.returntype), sourcename

experimaestro/taskglobals.py

@@ -3,6 +3,23 @@ from pathlib import Path
 from typing import Optional
 
 
+class LauncherInformation:
+    """Minimal launcher information available during task execution.
+
+    This is a lightweight class used to query launcher-specific information
+    (like remaining time) during task execution. It's set by the generated
+    Python script that runs the task.
+    """
+
+    def remaining_time(self) -> Optional[float]:
+        """Returns the remaining time in seconds before the job times out.
+
+        Returns:
+            The remaining time in seconds, or None if no time limit.
+        """
+        return None
+
+
 class Env:
     _instance = None
 
@@ -12,6 +29,9 @@ class Env:
     # The current task path
     taskpath: Optional[Path] = None
 
+    # Launcher information (only set when running a task)
+    launcher_info: Optional[LauncherInformation] = None
+
     # Set to True when multi-processing when
     # in slave mode:
     # - no progress report

experimaestro/tests/conftest.py

@@ -19,6 +19,86 @@ def xpmdirectory(tmp_path_factory):
         shutil.rmtree(workdir)
 
 
+@pytest.fixture(scope="function", autouse=True)
+def reset_scheduler():
+    """Reset scheduler state between tests to avoid state leakage with singleton pattern"""
+    from experimaestro.scheduler.base import Scheduler
+    from experimaestro.server import Server
+
+    # Get the singleton instance if it exists
+    if Scheduler._instance is not None:
+        scheduler = Scheduler._instance
+        # Clear job registrations but keep scheduler running
+        logging.debug(
+            f"FIXTURE: Clearing scheduler before test - jobs count: {len(scheduler.jobs)}"
+        )
+        # Clear experiment references from all jobs
+        for job in scheduler.jobs.values():
+            job.experiments.clear()
+        scheduler.jobs.clear()
+        scheduler.waitingjobs.clear()
+        scheduler.experiments.clear()
+        # Clear state provider experiment providers to avoid stale references
+        if (
+            hasattr(scheduler, "state_provider")
+            and scheduler.state_provider is not None
+        ):
+            # Close all experiment providers
+            for provider in scheduler.state_provider.experiment_providers.values():
+                provider.close()
+            scheduler.state_provider.experiment_providers.clear()
+            logging.debug("FIXTURE: Cleared state provider experiment providers")
+
+        # Also clear listeners to prevent stale listeners
+        scheduler.clear_listeners()
+
+        # Re-add state_provider as listener if it exists
+        if (
+            hasattr(scheduler, "state_provider")
+            and scheduler.state_provider is not None
+        ):
+            scheduler.addlistener(scheduler.state_provider)
+
+    # Reset server instance too
+    if Server._instance is not None:
+        logging.debug("FIXTURE: Clearing server instance")
+        Server._instance = None
+
+    yield
+
+    # Cleanup after test - clear again
+    if Scheduler._instance is not None:
+        scheduler = Scheduler._instance
+        logging.debug(
+            f"FIXTURE: Clearing scheduler after test - jobs count: {len(scheduler.jobs)}"
+        )
+        # Clear experiment references from all jobs
+        for job in scheduler.jobs.values():
+            job.experiments.clear()
+        scheduler.jobs.clear()
+        scheduler.waitingjobs.clear()
+        scheduler.experiments.clear()
+        # Clear state provider experiment providers
+        if (
+            hasattr(scheduler, "state_provider")
+            and scheduler.state_provider is not None
+        ):
+            for provider in scheduler.state_provider.experiment_providers.values():
+                provider.close()
+            scheduler.state_provider.experiment_providers.clear()
+        scheduler.clear_listeners()
+        # Re-add state_provider as listener if it exists
+        if (
+            hasattr(scheduler, "state_provider")
+            and scheduler.state_provider is not None
+        ):
+            scheduler.addlistener(scheduler.state_provider)
+
+    # Reset server after test
+    if Server._instance is not None:
+        Server._instance = None
+
+
 # Sets a flag
 def pytest_configure(config):
     import sys

experimaestro/tests/core/test_generics.py

@@ -3,7 +3,7 @@
 from typing import Generic, Optional, TypeVar
 
 import pytest
-from experimaestro import Config, Param
+from experimaestro import field, Config, Param
 from experimaestro.core.arguments import Argument
 from experimaestro.core.types import TypeVarType
 
@@ -162,7 +162,7 @@ class TreeGenericConfig(Config, Generic[T]):
 class TagTreeGenericConfig(TreeGenericConfig[T], Generic[T]):
     """A tagged version of TreeGenericConfig to test recursive generics"""
 
-    tag: Param[str] = "default"
+    tag: Param[str] = field(ignore_default="default")
 
 
 def test_core_generics_recursive():

experimaestro/tests/identifier_stability.json

@@ -0,0 +1,45 @@
+{
+  "bool_false": "bb61efa2769d20e6665fd63911d8a1e2fcdd2af22ff1e6c860d2b26ab7b04ab2",
+  "bool_true": "e718f2e3a3cc5b6b816a9645f587d3009efb08642bd22db45c8c288b78ff11f4",
+  "cycle_simple": "a73ef01b1c3e4e0187aee95eda96d1c069fd4757ad0137ac66adbf3a9502673f",
+  "default_override": "90951821af9c0d84b3f300fadfab63387bbfad6d1982dfbaa5b4d7ebbbfcf800",
+  "default_with_default": "0203eb7eb6a13e3c4592c9366f76a5f53dd2c5211c576547873184af86558bc3",
+  "dict_empty": "d2c32c9305431266e4ab1f5a70face4cee13b02a01af4ba0a6046fb254971b5f",
+  "dict_multiple": "01994d7bc212a73ea9d80332bf460922ca786a9d4ab8d8f444b3673901c75c99",
+  "dict_nested_empty": "77ebb66bcfe1c24c166dd80ceaae5840c1729f5c435a9d4ae040e8285b9beca7",
+  "dict_nested_multiple": "4476cd6934c5cc4a63cce1594cbbed622d0e70f6291b6d8cb4092d7b2612bb15",
+  "dict_nested_single": "373a3e409042029439fdbcb679b4e6388242901772d273ddefafd5663f9e57e4",
+  "dict_single": "dd879ad5038694c95134926ab3524696437f6ec96e52341a4e8c8fd44a1c2ae2",
+  "enum_value_a": "96c98d4683658b0a8e62d67abcb32c918506f8e455685680ee50f7b174e91366",
+  "enum_value_b": "433e23f7e2ee01a850bd97da097cf158873cb7465d3faa86218d86c0f7c38834",
+  "enum_value_c": "5881adaa535d2c3c842e4f163fbabbe5f537bc18f48a5bc2e9887ad9b3deb00b",
+  "float_negative": "67a86cea76bc90be4ec052e8c2f08829fb989e64a1d419541e68485dff85dba1",
+  "float_simple": "f445ab15c80965e89436c9e58a369fb969f216f392e4cbb19846830731f9a1e4",
+  "float_zero": "35879908ca1652ea07a4b49f0c44e9aa2ca264bedf14974dd9a765b1fafd1921",
+  "instance_different_values": "e175afb36163f56078beb516cf7489238b491571519e25c2b5ff68afbeccc643",
+  "instance_separate": "6d6274a5b541f60833e5d15a556a1b8adfaaa6dd0970a09a57849edd7a0c6fdd",
+  "instance_shared": "d9a76235da634b81b7559d561322263698852fa2012005781569154d7ad3cfc5",
+  "int_negative": "4e2ad6ee44e1c9429b56900aea7ba81a6b746a38678d2e29a89805bfb32b9153",
+  "int_positive": "2c57a590b8bf1bb5283a54d971814e196f395269f2973096dc277dbc24218962",
+  "int_zero": "2696ea881e0f601d4ad75679560e0e3305fa2f15552952d88ac87df4cc6f9f49",
+  "list_empty": "457140939f4e4ea43c5cfa4e05f4db0ed82891f0b550e23bfedf1510aea94d0c",
+  "list_multiple": "14575fd83be49b8f23d43d54fab90768ea0c296a829eeaa1b5a312f8322fb2ef",
+  "list_nested_empty": "fa30a32619931b4048a9f9854d82975e955c48017cd72b474344fa6b5a9c9bbe",
+  "list_nested_multiple": "4cc6e0d3d4ac32209334b8667d6b18f37cc5fd1677309eaeec89e7862d98ec5f",
+  "list_nested_single": "160a7e361e3482536479beaf8250f3107436c59e60ca5da573548da60e4b9bcf",
+  "list_single": "d33a881039f9a79cb7d688e547acdc79092b86b9e05fcb65faebbebfb38b3067",
+  "nested_multi": "8a1a37250d6f90caa549b02f8899dcba51ce01e5f6f511896b6c00b9c4a714a0",
+  "nested_simple": "a52569bc853a4cacbc044ba3082bbc10670f9174279e283168115828de979be1",
+  "option_override": "e98d969a3a309b2a43bab46c0fce6a6ea3c8c337a63ecb78c3705b8c281927b5",
+  "option_with_default": "e98d969a3a309b2a43bab46c0fce6a6ea3c8c337a63ecb78c3705b8c281927b5",
+  "str_empty": "bfdf01b69cbed525f27d0a9c1ba1043086ae5705fbc4027f5cf899a394e38bca",
+  "str_simple": "ad42020604bcc3c36bbcef8843de7b7c3af80b4198a109b7768d65bc2f788b1a",
+  "str_unicode": "26972840e4f5f71b2303902e0247aaf1e27d8a14ab6495c433d1f95c32dd40e8",
+  "task_simple": "1ff8ca42cdc94959e1b0c3c019ef4ab1f45b30a4309cc9fef3e42f4ea7da3e86",
+  "task_submitted": "834fae0fffb762b20064e8c648221dab99e81ba6f00622219fccce1bd0a18a17",
+  "task_using_output": "6c51d8124133038482472973a439d785b7ce53e46bac096e047e0e6cf1fc104e",
+  "task_with_config": "b26f8a8f7b1b9f6bda7e9c7e334071097b377ac48caec9d7da7fe98bc8c97c84",
+  "task_with_init": "285697abd5eaef36264f640ef790880f076daea4aff1814f1a518aa014ba4b0d",
+  "task_with_multiple_init": "d0e8610e1312d9a3398b839c691e16c741d4520823763067465a3ddab63acb30",
+  "task_with_output": "9cbaadb16fc6168286703afe35805108c4600abd05380fe56160f50e20b3cbb6"
+}

experimaestro/tests/launchers/bin/sacct

@@ -13,9 +13,13 @@ fi
 find "$XPM_SLURM_DIR/jobs" -name "*.start" | while read name; do
     jobid=${name%.start}
     sf="$jobid.status"
+    timeout_marker="$jobid.timeout"
     if test -f "$sf"; then
         exitcode="$(cat $sf)"
-        if test "$exitcode" == 0; then
+        # Check for timeout marker file
+        if test -f "$timeout_marker"; then
+            status=TIMEOUT
+        elif test "$exitcode" == 0; then
             status=COMPLETED
         else
             status=FAILED
@@ -25,4 +29,4 @@ find "$XPM_SLURM_DIR/jobs" -name "*.start" | while read name; do
     fi
 
     echo "$(basename $jobid)|${status}|$(cat ${jobid}.start)|$(cat ${jobid}.start)|"
-done
+done

experimaestro/tests/launchers/bin/sbatch

@@ -65,12 +65,14 @@ done < "$1"
 
 cd "$chdir"
 echo "Starting $@ ${args[@]} > $stdout 2> $stderr" >&2
+# Get job ID before forking
+JOBID="$$"
 (
     export PATH="${CURDIR}/bin:$PATH"
+    export SLURM_JOB_ID="$JOBID"
     eval "$@" "${args[@]}"
-    echo $? > "$XPM_SLURM_DIR/jobs/$$.status"
+    echo $? > "$XPM_SLURM_DIR/jobs/$JOBID.status"
 ) > $stdout 2> $stderr &
-JOBID="$$"
 date > "$XPM_SLURM_DIR/jobs/$JOBID.start"
 disown
 

experimaestro/tests/launchers/test_slurm.py

@@ -6,6 +6,8 @@ from experimaestro.connectors.local import LocalConnector
 from experimaestro.launchers.slurm import (
     SlurmLauncher,
 )
+from experimaestro import field, ResumableTask, Param
+from experimaestro.scheduler import JobState
 import shutil
 import pytest
 from .common import waitFromSpec, takeback
@@ -84,3 +86,81 @@ def test_slurm_takeback(slurmlauncher, tmp_path):
     datapath = tmp_path / "data"
 
     takeback(slurmlauncher, datapath, txp1, txp2)
+
+
+class SlurmResumableTask(ResumableTask):
+    """ResumableTask that simulates timeout on first N attempts for SLURM testing"""
+
+    checkpoint: Param[Path]
+    timeout_count: Param[int] = field(ignore_default=2)
+    slurm_jobs_dir: Param[Path]  # Path to mock SLURM jobs directory
+    output_file: Param[Path] = field(ignore_default=None)
+
+    def execute(self):
+        import os
+
+        # Read current attempt count from checkpoint
+        attempt = 1
+        if self.checkpoint.exists():
+            attempt = int(self.checkpoint.read_text()) + 1
+
+        print(f"SlurmResumableTask attempt #{attempt}")
+
+        # Write updated attempt count
+        self.checkpoint.write_text(str(attempt))
+
+        # Simulate timeout for first timeout_count attempts
+        if attempt <= self.timeout_count:
+            print(f"Simulating SLURM TIMEOUT on attempt {attempt}")
+            # Create timeout marker file for mock SLURM
+            # The marker needs to be named <jobid>.timeout in the SLURM jobs directory
+            # Use SLURM_JOB_ID environment variable (set by mock sbatch, like real SLURM)
+            job_id = os.environ.get("SLURM_JOB_ID")
+            if job_id:
+                timeout_marker = self.slurm_jobs_dir / f"{job_id}.timeout"
+                timeout_marker.write_text(f"timeout on attempt {attempt}")
+            # Exit with error to trigger SLURM timeout handling
+            raise RuntimeError(f"Simulated timeout on attempt {attempt}")
+
+        # Success - task completed
+        print(f"Task completed successfully on attempt {attempt}")
+        if self.output_file:
+            self.output_file.write_text(f"Completed after {attempt} attempts")
+
+
+@pytest.mark.timeout(30)
+def test_slurm_resumable_task(tmp_path: Path, slurmlauncher: SlurmLauncher):
+    """Test that ResumableTask retries and resumes after SLURM timeouts"""
+    with TemporaryExperiment("slurm-resumable", workdir=tmp_path / "xp", maxwait=25):
+        checkpoint = tmp_path / "checkpoint.txt"
+        output_file = tmp_path / "output.txt"
+
+        # Get the SLURM jobs directory from the launcher's binpath
+        slurm_jobs_dir = slurmlauncher.binpath.parent / "slurm" / "jobs"
+
+        # Submit task with max_retries to allow multiple timeout retries
+        task = SlurmResumableTask.C(
+            checkpoint=checkpoint,
+            timeout_count=2,  # Timeout on first 2 attempts
+            slurm_jobs_dir=slurm_jobs_dir,
+            output_file=output_file,
+        ).submit(launcher=slurmlauncher, max_retries=5)
+
+        # Wait for the task to complete
+        state = task.__xpm__.job.wait()
+
+        # Verify task completed successfully after retries
+        assert state == JobState.DONE, f"Task did not complete successfully: {state}"
+        assert (
+            task.__xpm__.job.retry_count == 2
+        ), f"Expected 2 retries, got {task.__xpm__.job.retry_count}"
+
+        # Verify checkpoint shows 3 attempts (2 timeouts + 1 success)
+        assert checkpoint.exists(), "Checkpoint file was not created"
+        assert (
+            int(checkpoint.read_text()) == 3
+        ), f"Expected 3 attempts, got {checkpoint.read_text()}"
+
+        # Verify output file was created on success
+        assert output_file.exists(), "Output file was not created"
+        assert "Completed after 3 attempts" in output_file.read_text()

experimaestro/tests/tasks/test_dynamic.py

@@ -0,0 +1,231 @@
+# Test for future task outputs handling
+# https://github.com/experimaestro/experimaestro-python/issues/90
+
+from functools import partial
+import json
+import logging
+from pathlib import Path
+import sys
+import time
+from experimaestro import (
+    Config,
+    Param,
+    Task,
+    ResumableTask,
+    DependentMarker,
+    LightweightTask,
+    field,
+    PathGenerator,
+)
+from experimaestro.core.arguments import Meta
+from experimaestro.tests.utils import TemporaryDirectory, TemporaryExperiment
+
+
+class Model(Config):
+    pass
+
+
+class Checkpoint(Config):
+    step: Param[int]
+    model: Param[Model]
+
+
+class CheckpointLoader(LightweightTask):
+    checkpoint: Param[Checkpoint]
+
+    def execute(self):
+        pass
+
+
+class Evaluate(Task):
+    model: Param[Model]
+
+    def execute(self):
+        pass
+
+
+class Validation(Config):
+    model: Param[Model]
+
+    def checkpoint(self, dep: DependentMarker, *, step: int) -> Checkpoint:
+        return dep(Checkpoint.C(model=self.model, step=step))
+
+    def compute(self, step: int):
+        self.register_task_output(self.checkpoint, step=step)
+
+
+class Learn(ResumableTask):
+    model: Param[Model]
+    validation: Param[Validation]
+
+    # Control files for synchronization with tests
+    max_step_file: Meta[Path] = field(default_factory=PathGenerator("max_step"))
+    state_file: Meta[Path] = field(default_factory=PathGenerator("state.json"))
+
+    def execute(self):
+        start_step = 0
+
+        if self.state_file.exists():
+            with self.state_file.open("r") as f:
+                state = json.load(f)
+                start_step = state.get("last_step", 0)
+                logging.info("Resuming from step %d", start_step)
+
+        # Wait for max_step_file to know how far to go
+        while not self.max_step_file.is_file():
+            time.sleep(0.1)
+
+        with self.max_step_file.open("r") as f:
+            max_step = int(f.read().strip())
+        self.max_step_file.unlink()
+
+        # Use absolute value for step comparison
+        # Negative max_step means: produce up to |max_step| then crash (simulate interruption)
+        # Positive max_step means: produce up to max_step then complete normally
+        abs_max = abs(max_step)
+
+        for step in [15, 30, 45]:
+            if step <= start_step:
+                logging.info("Skipping already processed step %d", step)
+                continue
+
+            if step > abs_max:
+                # We're past the limit, stop here
+                break
+
+            self.validation.compute(step)
+
+            # Save state after each checkpoint
+            with self.state_file.open("w") as f:
+                json.dump({"last_step": step}, f)
+
+            # If max_step is negative (e.g. -15), simulate exit after producing |max_step|
+            if max_step < 0 and step >= abs_max:
+                logging.warning("Simulating interruption after step %d", step)
+                sys.exit(1)
+
+
+def evaluate(evaluations, checkpoint: Checkpoint):
+    logging.warning("Evaluating checkpoint %s", checkpoint)
+    task = Evaluate.C(model=checkpoint.model)
+    checkpoint_loader = CheckpointLoader.C(checkpoint=checkpoint)
+    evaluations.append(task.submit(init_tasks=[checkpoint_loader]))
+
+
+def test_task_dynamic_simple():
+    """Test that dynamic task outputs trigger callbacks
+
+    This test verifies that callbacks are guaranteed to complete before
+    the experiment context exits. The callback waits for jobs to complete
+    before submitting evaluations, which validates that the synchronization
+    logic correctly waits for all callbacks to finish.
+    """
+    import asyncio
+
+    evaluations = []
+    xp_ref = [None]  # To access xp from callback
+
+    def collect_checkpoint(checkpoint: Checkpoint):
+        """Callback that waits for jobs to complete before evaluating
+
+        This simulates a real-world scenario where the callback needs to wait
+        for the triggering task to complete before it can proceed (e.g., to
+        read outputs from the task's directory).
+        """
+        logging.info("Received checkpoint %s, waiting for jobs to complete", checkpoint)
+        xp = xp_ref[0]
+
+        # Wait for unfinished jobs to become 0 (all tasks completed)
+        async def wait_for_jobs_done():
+            async with xp.scheduler.exitCondition:
+                while xp.unfinishedJobs > 0:
+                    await xp.scheduler.exitCondition.wait()
+
+        asyncio.run_coroutine_threadsafe(
+            wait_for_jobs_done(), xp.scheduler.loop
+        ).result()
+
+        # Now submit evaluation
+        logging.info("Jobs done, submitting evaluation for checkpoint %s", checkpoint)
+        evaluate(evaluations, checkpoint)
+
+    with TemporaryDirectory() as workdir:
+        with TemporaryExperiment("dynamic", maxwait=10, workdir=workdir) as xp:
+            xp_ref[0] = xp
+            model = Model.C()
+            validation = Validation.C(model=model)
+            learn = Learn.C(model=model, validation=validation)
+            learn.watch_output(validation.checkpoint, collect_checkpoint)
+
+            learn.submit()
+
+            # Allow the task to run up to step 30
+            learn.max_step_file.parent.mkdir(parents=True, exist_ok=True)
+            with learn.max_step_file.open("w") as f:
+                f.write("30")
+
+            logging.info("Experiment will wait for completion...")
+
+        assert len(evaluations) == 2, f"Expected 2 evaluations, got {len(evaluations)}"
+
+
+def test_task_dynamic_replay():
+    """Test that dynamic outputs are replayed when a task is restarted
+
+    Scenario:
+    1. First run: task produces checkpoint for step 15, then exits (simulated timeout)
+    2. Second run: task should replay the step 15 checkpoint and produce new ones
+    """
+    with TemporaryDirectory() as workdir:
+        # First run: produce one checkpoint then exit
+        evaluations_run1 = []
+        try:
+            with TemporaryExperiment("dynamic_replay", maxwait=5, workdir=workdir):
+                model = Model.C()
+                validation = Validation.C(model=model)
+                learn = Learn.C(model=model, validation=validation)
+                learn.watch_output(
+                    validation.checkpoint, partial(evaluate, evaluations_run1)
+                )
+
+                learn.submit()
+
+                # Allow task to produce step 15 checkpoint, then simulate crash
+                # Negative value means: produce up to |value| then exit with error
+                learn.max_step_file.parent.mkdir(parents=True, exist_ok=True)
+                with learn.max_step_file.open("w") as f:
+                    f.write("-15")
+
+        except Exception as e:
+            # Expected: the task will fail when trying to go past max_step
+            logging.info("First run ended (expected): %s", e)
+
+        # First run should have produced at least one evaluation (for step 15)
+        assert (
+            len(evaluations_run1) == 1
+        ), f"Run 1: Expected 1 evaluation, got {len(evaluations_run1)}"
+
+        # Second run: restart and continue
+        evaluations_run2 = []
+        with TemporaryExperiment("dynamic_replay", maxwait=30, workdir=workdir):
+            model = Model.C()
+            validation = Validation.C(model=model)
+            learn = Learn.C(model=model, validation=validation)
+            learn.watch_output(
+                validation.checkpoint, partial(evaluate, evaluations_run2)
+            )
+
+            learn.submit()
+
+            # Allow task to run to completion (step 45)
+            learn.max_step_file.parent.mkdir(parents=True, exist_ok=True)
+            with learn.max_step_file.open("w") as f:
+                f.write("45")
+
+        # Second run should have:
+        # - Replayed the step 15 checkpoint (from first run)
+        # - Produced step 30 and 45 checkpoints
+        # Total: 3 evaluations (but step 15 was replayed, not re-produced)
+        assert (
+            len(evaluations_run2) == 3
+        ), f"Run 2: Expected 3 evaluations, got {len(evaluations_run2)}"