experimaestro 2.0.0b8__py3-none-any.whl → 2.0.0b17__py3-none-any.whl

experimaestro/tests/test_stray_jobs.py

@@ -0,0 +1,261 @@
+"""Tests for stray job detection.
+
+A stray job is a running job that is not associated with any active experiment.
+This happens when an experiment plan changes (e.g., same experiment ID is relaunched
+with different parameters).
+"""
+
+import time
+from pathlib import Path
+import tempfile
+import logging
+
+from experimaestro import (
+    Task,
+    Param,
+    Meta,
+    field,
+    PathGenerator,
+    experiment,
+    GracefulExperimentExit,
+)
+from experimaestro.scheduler.workspace import RunMode
+from experimaestro.scheduler import JobState
+from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider
+
+logger = logging.getLogger(__name__)
+
+
+class ControllableTask(Task):
+    """A task that can be controlled via files for testing purposes."""
+
+    value: Param[int]
+    touch: Meta[Path] = field(default_factory=PathGenerator("touch"))
+    wait: Meta[Path] = field(default_factory=PathGenerator("wait"))
+
+    def execute(self):
+        # Signal that the task has started
+        with open(self.touch, "w") as out:
+            out.write("started")
+
+        # Wait for external signal to continue
+        while not self.wait.is_file():
+            time.sleep(0.1)
+
+
+MAX_WAIT_ITERATIONS = 50  # 5 seconds max wait
+
+
+def test_stray_job_detection():
+    """Test that a running job becomes stray when experiment is relaunched with different params.
+
+    Scenario:
+    1. Start experiment with task(value=1)
+    2. Wait for task to start running
+    3. Exit experiment without waiting (using GracefulExperimentExit)
+    4. Start same experiment (same ID) with task(value=2)
+    5. The first job should now be detected as stray
+    6. Signal first task to finish
+    7. Verify stray detection works correctly
+    """
+    with tempfile.TemporaryDirectory(prefix="xpm_stray_test_") as workdir:
+        workdir_path = Path(workdir)
+        experiment_id = "stray_test"
+
+        # Phase 1: Start experiment with value=1
+        with experiment(workdir_path, experiment_id) as _:
+            task1 = ControllableTask.C(value=1)
+            # First do a dry run to get the file paths
+            task1.submit(run_mode=RunMode.DRY_RUN)
+            touch_path = task1.touch
+            wait_path = task1.wait
+
+            # Now submit for real
+            task1 = ControllableTask.C(value=1)
+            task1.submit()
+
+            # Wait for task to start
+            counter = 0
+            while not touch_path.exists():
+                time.sleep(0.1)
+                counter += 1
+                if counter >= MAX_WAIT_ITERATIONS:
+                    raise AssertionError("Timeout waiting for task1 to start")
+
+            raise GracefulExperimentExit()
+
+        # Phase 2: Start same experiment with value=2
+        try:
+            with experiment(workdir_path, experiment_id) as _:
+                task2 = ControllableTask.C(value=2)
+                task2.submit()
+
+                # Give the scheduler a moment to process
+                time.sleep(0.5)
+
+                # Check for stray jobs using a fresh state provider
+                provider = WorkspaceStateProvider(workdir_path)
+
+                stray_jobs = provider.get_stray_jobs()
+
+                # At least the first task should be stray (running but not in current experiment)
+                # Note: task2 might also appear as stray temporarily because status.json
+                # hasn't been flushed yet
+                assert len(stray_jobs) >= 1, (
+                    f"Expected at least 1 stray job, found {len(stray_jobs)}"
+                )
+
+                # Verify that at least one stray job is actually running
+                running_stray = [j for j in stray_jobs if j.state == JobState.RUNNING]
+                assert len(running_stray) >= 1, (
+                    f"Expected at least 1 running stray job, found {len(running_stray)}"
+                )
+
+                # Signal first task to finish
+                with open(wait_path, "w") as f:
+                    f.write("done")
+
+                # Give it a moment to finish
+                time.sleep(0.5)
+
+                # After task1 finishes, it should no longer be stray (it's not running)
+                # task1 should no longer be stray (it finished)
+                # task2 might still appear as stray if status hasn't been flushed
+                # So we just verify that the count decreased by at least 1
+                # (or at least, that task1's specific job is no longer in the list)
+
+                # Exit gracefully to not wait for task2
+                raise GracefulExperimentExit()
+        finally:
+            # Clean up: signal task2 to finish if it started
+            with experiment(workdir_path, experiment_id, run_mode=RunMode.DRY_RUN):
+                task2_dry = ControllableTask.C(value=2)
+                task2_dry.submit(run_mode=RunMode.DRY_RUN)
+                task2_wait = task2_dry.wait
+
+            if task2_wait and task2_wait.exists() is False:
+                # task2 might have started
+                task2_touch = task2_dry.touch
+                if task2_touch.exists():
+                    with open(task2_wait, "w") as f:
+                        f.write("done")
+                    time.sleep(0.5)
+
+
+def test_running_state_detection():
+    """Test that running jobs are correctly detected from PID files."""
+    with tempfile.TemporaryDirectory(prefix="xpm_running_test_") as workdir:
+        workdir_path = Path(workdir)
+        experiment_id = "running_test"
+
+        with experiment(workdir_path, experiment_id) as _:
+            task = ControllableTask.C(value=1)
+            task.submit(run_mode=RunMode.DRY_RUN)
+            touch_path = task.touch
+            wait_path = task.wait
+
+            task = ControllableTask.C(value=1)
+            task.submit()
+
+            # Wait for task to start
+            counter = 0
+            while not touch_path.exists():
+                time.sleep(0.1)
+                counter += 1
+                if counter >= MAX_WAIT_ITERATIONS:
+                    raise AssertionError("Timeout waiting for task to start")
+
+            # Create a fresh provider to check the running state
+            provider = WorkspaceStateProvider(workdir_path)
+
+            # Get all jobs on disk
+            jobs_base = workdir_path / "jobs"
+            job_paths = list(jobs_base.glob("*/*"))
+            assert len(job_paths) == 1, (
+                f"Expected 1 job on disk, found {len(job_paths)}"
+            )
+
+            # Check that the job is detected as running
+            job_path = job_paths[0]
+            task_id = job_path.parent.name
+            job_id = job_path.name
+
+            mock_job = provider._create_mock_job_from_path(job_path, task_id, job_id)
+            assert mock_job.state == JobState.RUNNING, (
+                f"Expected job state RUNNING, got {mock_job.state}"
+            )
+
+            # Signal task to finish
+            with open(wait_path, "w") as f:
+                f.write("done")
+
+
+def test_completed_job_not_stray():
+    """Test that completed jobs are not detected as stray.
+
+    Scenario:
+    1. Start experiment, run and complete a job
+    2. Start a new run of the same experiment with a different job
+    3. The first job should be orphan (not in the new run) but NOT stray (not running)
+    """
+    with tempfile.TemporaryDirectory(prefix="xpm_completed_test_") as workdir:
+        workdir_path = Path(workdir)
+        experiment_id = "completed_test"
+
+        # Phase 1: Create and complete a job
+        with experiment(workdir_path, experiment_id) as _:
+            task = ControllableTask.C(value=1)
+            task.submit(run_mode=RunMode.DRY_RUN)
+            touch_path = task.touch
+            wait_path = task.wait
+
+            task = ControllableTask.C(value=1)
+            task.submit()
+
+            # Wait for task to start
+            counter = 0
+            while not touch_path.exists():
+                time.sleep(0.1)
+                counter += 1
+                if counter >= MAX_WAIT_ITERATIONS:
+                    raise AssertionError("Timeout waiting for task to start")
+
+            # Signal to finish immediately
+            with open(wait_path, "w") as f:
+                f.write("done")
+
+        # Phase 2: Start a new run with a different job
+        # This makes the old job an orphan (not in current run)
+        with experiment(workdir_path, experiment_id) as _:
+            task2 = ControllableTask.C(value=2)
+            task2.submit(run_mode=RunMode.DRY_RUN)
+            touch_path2 = task2.touch
+            wait_path2 = task2.wait
+
+            task2 = ControllableTask.C(value=2)
+            task2.submit()
+
+            # Wait for task2 to start
+            counter = 0
+            while not touch_path2.exists():
+                time.sleep(0.1)
+                counter += 1
+                if counter >= MAX_WAIT_ITERATIONS:
+                    raise AssertionError("Timeout waiting for task2 to start")
+
+            # Now check that the first job is NOT stray (it's completed)
+            provider = WorkspaceStateProvider(workdir_path)
+
+            stray_jobs = provider.get_stray_jobs()
+
+            # Task1 is not running, so it should not be stray
+            # Task2 might appear as stray because status.json hasn't been flushed
+            # Filter to check that no DONE jobs are stray
+            done_stray = [j for j in stray_jobs if j.state == JobState.DONE]
+            assert len(done_stray) == 0, (
+                f"Expected 0 completed stray jobs, found {len(done_stray)}"
+            )
+
+            # Signal task2 to finish
+            with open(wait_path2, "w") as f:
+                f.write("done")

experimaestro/tests/test_tasks.py

@@ -132,14 +132,13 @@ def test_restart(terminate):
 def test_submitted_twice():
     """Check that a job cannot be submitted twice within the same experiment"""
     with TemporaryExperiment("duplicate", maxwait=20):
-
         task1 = SimpleTask.C(x=1)
         o1 = task1.submit()
 
         task2 = SimpleTask.C(x=1)
         o2 = task2.submit()
 
-        print(o1)
+        print(o1)  # noqa: T201
         assert o1.task is not o2.task
         assert task1.__xpm__.job is task2.__xpm__.job, f"{id(task1)} != {id(task2)}"
 

experimaestro/tests/test_token_locking.py

@@ -9,6 +9,7 @@ import pytest
 import tempfile
 from pathlib import Path
 import time
+import hashlib
 
 from experimaestro.tokens import CounterToken
 from experimaestro.locking import LockError
@@ -16,22 +17,54 @@ from experimaestro.locking import LockError
 pytestmark = pytest.mark.anyio
 
 
+class MockIdentifier:
+    """Mock identifier with hex() method."""
+
+    def __init__(self, value: str):
+        self._hex = hashlib.sha256(value.encode()).hexdigest()
+
+    def hex(self):
+        return self._hex
+
+
+class MockXPM:
+    """Mock __xpm__ object."""
+
+    def __init__(self, name: str):
+        self.identifier = type("Identifier", (), {"main": MockIdentifier(name)})()
+
+
+class MockConfig:
+    """Mock config object."""
+
+    def __init__(self, name: str):
+        self.__xpm__ = MockXPM(name)
+
+
+def create_mock_job(name: str, tmpdir: str):
+    """Create a mock job with all required attributes."""
+
+    class MockJob:
+        task_id = "mock-task"
+        config = MockConfig(name)
+
+        @property
+        def identifier(self):
+            return f"mock-job-{name}"
+
+        @property
+        def basepath(self):
+            return Path(tmpdir) / name
+
+    return MockJob()
+
+
 async def test_token_acquire_release():
     """Test basic token acquire and release"""
     with tempfile.TemporaryDirectory() as tmpdir:
         token = CounterToken("test-basic", Path(tmpdir) / "token", count=1)
 
-        # Create a mock job target
-        class MockJob:
-            @property
-            def identifier(self):
-                return "mock-job-1"
-
-            @property
-            def basepath(self):
-                return Path(tmpdir) / "job1"
-
-        job = MockJob()
+        job = create_mock_job("1", tmpdir)
 
         # Create dependency
         dep = token.dependency(1)
@@ -52,20 +85,8 @@ async def test_token_blocking():
     with tempfile.TemporaryDirectory() as tmpdir:
         token = CounterToken("test-blocking", Path(tmpdir) / "token", count=1)
 
-        class MockJob:
-            def __init__(self, name):
-                self.name = name
-
-            @property
-            def identifier(self):
-                return f"mock-job-{self.name}"
-
-            @property
-            def basepath(self):
-                return Path(tmpdir) / self.name
-
-        job1 = MockJob("1")
-        job2 = MockJob("2")
+        job1 = create_mock_job("1", tmpdir)
+        job2 = create_mock_job("2", tmpdir)
 
         dep1 = token.dependency(1)
         dep1.target = job1
@@ -99,20 +120,8 @@ async def test_token_notification():
     with tempfile.TemporaryDirectory() as tmpdir:
         token = CounterToken("test-notify", Path(tmpdir) / "token", count=1)
 
-        class MockJob:
-            def __init__(self, name):
-                self.name = name
-
-            @property
-            def identifier(self):
-                return f"mock-job-{self.name}"
-
-            @property
-            def basepath(self):
-                return Path(tmpdir) / self.name
-
-        job1 = MockJob("1")
-        job2 = MockJob("2")
+        job1 = create_mock_job("1", tmpdir)
+        job2 = create_mock_job("2", tmpdir)
 
         dep1 = token.dependency(1)
         dep1.target = job1
@@ -151,20 +160,8 @@ async def test_token_multiple_waiting():
     with tempfile.TemporaryDirectory() as tmpdir:
         token = CounterToken("test-multiple", Path(tmpdir) / "token", count=1)
 
-        class MockJob:
-            def __init__(self, name):
-                self.name = name
-
-            @property
-            def identifier(self):
-                return f"mock-job-{self.name}"
-
-            @property
-            def basepath(self):
-                return Path(tmpdir) / self.name
-
         # Acquire the token
-        job1 = MockJob("1")
+        job1 = create_mock_job("1", tmpdir)
         dep1 = token.dependency(1)
         dep1.target = job1
         lock1 = await dep1.aio_lock(timeout=0.5)
@@ -173,7 +170,7 @@ async def test_token_multiple_waiting():
         acquired_order = []
 
         async def acquire_task(name):
-            job = MockJob(name)
+            job = create_mock_job(name, tmpdir)
             dep = token.dependency(1)
             dep.target = job
             lock = await dep.aio_lock(timeout=10.0)
@@ -206,20 +203,8 @@ async def test_token_timeout_zero():
     with tempfile.TemporaryDirectory() as tmpdir:
         token = CounterToken("test-timeout-zero", Path(tmpdir) / "token", count=1)
 
-        class MockJob:
-            def __init__(self, name):
-                self.name = name
-
-            @property
-            def identifier(self):
-                return f"mock-job-{self.name}"
-
-            @property
-            def basepath(self):
-                return Path(tmpdir) / self.name
-
-        job1 = MockJob("1")
-        job2 = MockJob("2")
+        job1 = create_mock_job("1", tmpdir)
+        job2 = create_mock_job("2", tmpdir)
 
         dep1 = token.dependency(1)
         dep1.target = job1

experimaestro/tests/test_tokens.py

@@ -8,7 +8,7 @@ from pathlib import Path
 
 import subprocess
 from experimaestro import Task, Param
-from experimaestro.tokens import CounterToken, TokenFile
+from experimaestro.tokens import CounterToken, TokenLockFile
 from experimaestro.scheduler import JobState
 from .utils import (
     TemporaryExperiment,
@@ -101,7 +101,7 @@ def test_token_cleanup():
         # Just lock directly (but without process)
         # The absence of process should be detected right away
         logging.info("Lock without process")
-        TokenFile.create(dependency)
+        TokenLockFile.from_dependency(dependency)
         task2 = dummy_task.C(x=2)
         task2.add_dependencies(token.dependency(1)).submit()
         xp.wait()
@@ -111,7 +111,7 @@ def test_token_cleanup():
         job = dependency.target
         with fasteners.InterProcessLock(job.lockpath):
             logging.info("Creating dependency %s", dependency)
-            TokenFile.create(dependency)
+            TokenLockFile.from_dependency(dependency)
             lockingpath = job.path / "testtoken.signal"
             command = [
                 sys.executable,
@@ -145,7 +145,7 @@ def test_token_monitor():
         )
         return task
 
-    with TemporaryExperiment("tokens1", maxwait=20, port=0) as xp1:
+    with TemporaryExperiment("tokens1", maxwait=20) as xp1:
         # Use the same workspace for both experiments
         with TemporaryExperiment(
             "tokens2", workdir=xp1.workspace.path, maxwait=20
@@ -203,8 +203,7 @@ def test_token_reschedule():
 
                 # Create the locking path
                 logging.info(
-                    "Both processes are ready:"
-                    "allowing tasks to finish by writing in %s",
+                    "Both processes are ready:allowing tasks to finish by writing in %s",
                     lockingpath,
                 )
                 lockingpath.write_text("Let's go")