experimaestro 2.0.0a8__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/tests/test_tags.py

@@ -1,5 +1,6 @@
 from typing import Dict
 from pathlib import Path
+import logging
 from experimaestro import (
     tag,
     LightweightTask,
@@ -118,3 +119,138 @@ def test_objects_tags():
     a = A.C(x=tag(1))
     a.__xpm__.seal(context)
     assert a.__xpm__.tags() == {"x": 1}
+
+
+def test_conflicting_tags_warning(caplog):
+    """Test that conflicting tag values produce a warning"""
+
+    class Inner(Config):
+        value: Param[int]
+
+    class Outer(Config):
+        inner: Param[Inner]
+        x: Param[int]
+
+    # Create inner config with tag "mytag" = 1
+    inner = Inner.C(value=10).tag("mytag", 1)
+
+    # Create outer config with same tag "mytag" = 2 (conflicting)
+    outer = Outer.C(inner=inner, x=5).tag("mytag", 2)
+
+    # Getting tags should warn about conflict
+    with caplog.at_level(logging.WARNING):
+        tags = outer.tags()
+
+    # The warning should mention the conflicting tag
+    assert any("mytag" in record.message for record in caplog.records)
+    assert any("conflicting" in record.message.lower() for record in caplog.records)
+
+    # The last value should win
+    assert tags["mytag"] == 2
+
+
+def test_same_tag_same_value_no_warning(caplog):
+    """Test that same tag with same value does not produce a warning"""
+
+    class Inner(Config):
+        value: Param[int]
+
+    class Outer(Config):
+        inner: Param[Inner]
+
+    # Create inner config with tag "mytag" = 1
+    inner = Inner.C(value=10).tag("mytag", 1)
+
+    # Create outer config with same tag "mytag" = 1 (same value)
+    outer = Outer.C(inner=inner).tag("mytag", 1)
+
+    # Getting tags should NOT warn (same value)
+    with caplog.at_level(logging.WARNING):
+        tags = outer.tags()
+
+    # No warning for same values
+    assert not any("mytag" in record.message for record in caplog.records)
+    assert tags["mytag"] == 1
+
+
+def test_tag_source_tracking():
+    """Test that tag source locations are tracked"""
+
+    class MyConfig(Config):
+        x: Param[int]
+
+    config = MyConfig.C(x=tag(5))
+
+    # Check that tags have source info stored internally
+    assert "x" in config.__xpm__._tags
+    value, source = config.__xpm__._tags["x"]
+    assert value == 5
+    # Source should contain file path and line number
+    assert ":" in source
+    assert "test_tags.py" in source
+
+
+def test_tag_method_source_tracking():
+    """Test that tag() method also tracks source location"""
+
+    class MyConfig(Config):
+        x: Param[int]
+
+    config = MyConfig.C(x=5)
+    config.tag("mytag", "myvalue")
+
+    # Check that tag has source info
+    assert "mytag" in config.__xpm__._tags
+    value, source = config.__xpm__._tags["mytag"]
+    assert value == "myvalue"
+    assert ":" in source
+    assert "test_tags.py" in source
+
+
+def test_tag_via_setattr():
+    """Test that config.key = tag(value) works and tracks source"""
+
+    class MyConfig(Config):
+        x: Param[int]
+
+    config = MyConfig.C(x=5)
+    config.x = tag(10)
+
+    # Check that tag was set correctly
+    assert config.tags() == {"x": 10}
+    assert config.x == 10
+
+    # Check that source is tracked
+    value, source = config.__xpm__._tags["x"]
+    assert value == 10
+    assert "test_tags.py" in source
+
+
+def test_tag_setattr_conflict_warning(caplog):
+    """Test that setting conflicting tag via setattr produces warning"""
+
+    class Inner(Config):
+        value: Param[int]
+
+    class Outer(Config):
+        inner: Param[Inner]
+        x: Param[int]
+
+    # Create with tag via constructor
+    inner = Inner.C(value=tag(1))
+
+    # Create outer with same tag name
+    outer = Outer.C(inner=inner, x=5)
+    outer.x = tag(2)  # Set tag on x
+
+    # Add a conflicting value tag
+    outer.tag("value", 99)
+
+    # Getting tags should warn about conflict
+    with caplog.at_level(logging.WARNING):
+        tags = outer.tags()
+
+    # The warning should mention the conflicting tag
+    assert any("value" in record.message for record in caplog.records)
+    assert tags["value"] == 99  # Last value wins
+    assert tags["x"] == 2

experimaestro/tests/test_tasks.py

@@ -1,11 +1,20 @@
 # --- Task and types definitions
 
+import sys
+import time
 from pathlib import Path
 import pytest
 import logging
-from experimaestro import Config, deprecate, Task, Param
+from experimaestro import (
+    Config,
+    Task,
+    Param,
+    ResumableTask,
+    Meta,
+    field,
+    PathGenerator,
+)
 from experimaestro.scheduler.workspace import RunMode
-from experimaestro.tools.jobs import fix_deprecated
 from experimaestro.scheduler import FailedExperiment, JobState
 from experimaestro import SubmitHook, Job, Launcher, LightweightTask
 
@@ -144,125 +153,6 @@ def test_configcache():
     assert task.__xpm__.job.wait() == JobState.DONE
 
 
-# ---- Deprecation
-
-
-class NewConfig(Config):
-    __xpmid__ = "new"
-
-
-@deprecate
-class DeprecatedConfig(NewConfig):
-    __xpmid__ = "deprecated"
-
-
-class OldConfig(NewConfig):
-    __xpmid__ = "deprecated"
-
-
-class TaskWithDeprecated(Task):
-    p: Param[NewConfig]
-
-    def execute(self):
-        pass
-
-
-def checknewpaths(task_new, task_old_path):
-    task_new_path = task_new.__xpm__.job.path  # type: Path
-
-    assert task_new_path.exists(), f"New path {task_new_path} should exist"
-    assert task_new_path.is_symlink(), f"New path {task_new_path} should be a symlink"
-
-    assert task_new_path.resolve() == task_old_path
-
-
-def test_tasks_deprecated_inner():
-    """Test that when submitting the task, the computed identifier is the one of
-    the new class"""
-    with TemporaryExperiment("deprecated", maxwait=0) as xp:
-        # --- Check that paths are really different first
-        task_new = TaskWithDeprecated.C(p=NewConfig.C()).submit(
-            run_mode=RunMode.DRY_RUN
-        )
-        task_old = TaskWithDeprecated.C(p=OldConfig.C()).submit(
-            run_mode=RunMode.DRY_RUN
-        )
-        task_deprecated = TaskWithDeprecated.C(p=DeprecatedConfig.C()).submit(
-            run_mode=RunMode.DRY_RUN
-        )
-
-        logging.debug("New task ID: %s", task_new.__xpm__.identifier.all.hex())
-        logging.debug("Old task ID: %s", task_old.__xpm__.identifier.all.hex())
-        logging.debug(
-            "Old task (with deprecated flag): %s",
-            task_deprecated.__xpm__.identifier.all.hex(),
-        )
-        assert (
-            task_new.stdout() != task_old.stdout()
-        ), "Old and new path should be different"
-
-        assert (
-            task_new.stdout() == task_deprecated.stdout()
-        ), "Deprecated path should be the same as non deprecated"
-
-        # --- Now check that automatic linking is performed
-
-        # Run old task with deprecated configuration
-        task_old = TaskWithDeprecated.C(p=OldConfig.C()).submit()
-        task_old.wait()
-        task_old_path = task_old.stdout().parent
-
-        # Fix deprecated
-        OldConfig.__xpmtype__.deprecate()
-        fix_deprecated(xp.workspace.path, True, False)
-
-        checknewpaths(task_new, task_old_path)
-
-
-class NewTask(Task):
-    x: Param[int]
-
-    def execute(self):
-        pass
-
-
-class OldTask(NewTask):
-    __xpmid__ = "deprecated"
-
-
-@deprecate
-class DeprecatedTask(NewTask):
-    __xpmid__ = "deprecated"
-
-
-def test_tasks_deprecated():
-    """Test that when submitting the task, the computed identifier is the one of
-    the new class"""
-    with TemporaryExperiment("deprecated", maxwait=20) as xp:
-        # Check that paths are really different first
-        task_new = NewTask.C(x=1).submit(run_mode=RunMode.DRY_RUN)
-        task_old = OldTask.C(x=1).submit(run_mode=RunMode.DRY_RUN)
-        task_deprecated = DeprecatedTask.C(x=1).submit(run_mode=RunMode.DRY_RUN)
-
-        assert (
-            task_new.stdout() != task_old.stdout()
-        ), "Old and new path should be different"
-        assert (
-            task_new.stdout() == task_deprecated.stdout()
-        ), "Deprecated path should be the same as non deprecated"
-
-        # OK, now check that automatic linking is performed
-        task_old = OldTask.C(x=1).submit()
-        task_old.wait()
-        task_old_path = task_old.stdout().parent
-
-        # Fix deprecated
-        OldTask.__xpmtype__.deprecate()
-        fix_deprecated(xp.workspace.path, True, False)
-
-        checknewpaths(task_new, task_old_path)
-
-
 class needs_java(SubmitHook):
     def __init__(self, version: int):
         self.version = version
@@ -318,3 +208,99 @@ def test_task_lightweight_init():
             MyLightweightTask.C(x=x).submit(init_tasks=[lwtask]).__xpm__.job.wait()
             == JobState.DONE
         ), "Init tasks should be executed"
+
+
+# --- Test for resumable task resubmission
+
+
+class ControllableResumableTask(ResumableTask):
+    """A resumable task that can be controlled via files"""
+
+    control_file: Meta[Path] = field(default_factory=PathGenerator("control"))
+
+    def execute(self):
+        # Wait for control file
+        while not self.control_file.is_file():
+            time.sleep(0.1)
+
+        # Read control: "fail" to exit with error, "complete" to succeed
+        action = self.control_file.read_text().strip()
+        self.control_file.unlink()
+
+        if action == "fail":
+            sys.exit(1)
+
+
+def test_resumable_task_resubmit():
+    """Test resubmitting a failed ResumableTask within the same experiment"""
+    with TemporaryExperiment("resumable_resubmit", maxwait=30):
+        task1 = ControllableResumableTask.C()
+        task1.submit()
+
+        # Tell task to fail
+        task1.control_file.parent.mkdir(parents=True, exist_ok=True)
+        task1.control_file.write_text("fail")
+
+        # Wait for the job to fail
+        job = task1.__xpm__.job
+        assert job.wait() == JobState.ERROR, "Job should have failed"
+
+        # Resubmit by creating a new instance with same parameters
+        task2 = ControllableResumableTask.C()
+        task2.submit()
+
+        # Tell task to complete
+        task2.control_file.write_text("complete")
+
+        # Wait for the resubmitted job to complete
+        assert task2.__xpm__.job.wait() == JobState.DONE
+
+
+def test_resumable_task_resubmit_across_experiments():
+    """Test resubmitting a failed ResumableTask across two experiment instances"""
+    with TemporaryDirectory(prefix="xpm", suffix="resubmit_across") as workdir:
+        # First experiment: task fails
+        try:
+            with TemporaryExperiment("resubmit_across", maxwait=10, workdir=workdir):
+                task1 = ControllableResumableTask.C()
+                task1.submit()
+
+                # Tell task to fail
+                task1.control_file.parent.mkdir(parents=True, exist_ok=True)
+                task1.control_file.write_text("fail")
+        except Exception as e:
+            logging.info("First experiment ended (expected): %s", e)
+
+        # Second experiment: task completes
+        with TemporaryExperiment("resubmit_across", maxwait=30, workdir=workdir):
+            task2 = ControllableResumableTask.C()
+            task2.submit()
+
+            # Tell task to complete
+            task2.control_file.write_text("complete")
+
+            # Wait for the resubmitted job to complete
+            assert task2.__xpm__.job.wait() == JobState.DONE
+
+
+def test_task_resubmit_across_experiments():
+    """Test resubmitting a completed task across two experiment instances"""
+    with TemporaryDirectory(prefix="xpm", suffix="resubmit_across") as workdir:
+        # First experiment: task completes
+        with TemporaryExperiment("resubmit_across", maxwait=30, workdir=workdir):
+            task1 = ControllableResumableTask.C()
+            task1.submit()
+
+            # Tell task to complete
+            task1.control_file.parent.mkdir(parents=True, exist_ok=True)
+            task1.control_file.write_text("complete")
+
+            assert task1.__xpm__.job.wait() == JobState.DONE
+
+        # Second experiment: resubmit completed task (uses same workdir)
+        with TemporaryExperiment("resubmit_across", maxwait=30, workdir=workdir):
+            task2 = ControllableResumableTask.C()
+            task2.submit()
+
+            # Task should recognize it's already done
+            assert task2.__xpm__.job.wait() == JobState.DONE

experimaestro/tests/test_token_locking.py

@@ -0,0 +1,252 @@
+"""Unit tests for token locking mechanism
+
+Tests the CounterToken condition variable-based synchronization
+without requiring full scheduler integration.
+"""
+
+import asyncio
+import pytest
+import tempfile
+from pathlib import Path
+import time
+
+from experimaestro.tokens import CounterToken
+from experimaestro.locking import LockError
+
+pytestmark = pytest.mark.anyio
+
+
+async def test_token_acquire_release():
+    """Test basic token acquire and release"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-basic", Path(tmpdir) / "token", count=1)
+
+        # Create a mock job target
+        class MockJob:
+            @property
+            def identifier(self):
+                return "mock-job-1"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / "job1"
+
+        job = MockJob()
+
+        # Create dependency
+        dep = token.dependency(1)
+        dep.target = job
+
+        # Should be able to acquire
+        lock = await dep.aio_lock(timeout=1.0)
+        assert lock is not None
+        assert token.available == 0
+
+        # Release
+        lock.release()
+        assert token.available == 1
+
+
+async def test_token_blocking():
+    """Test that acquiring blocks when no tokens available"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-blocking", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        job1 = MockJob("1")
+        job2 = MockJob("2")
+
+        dep1 = token.dependency(1)
+        dep1.target = job1
+
+        dep2 = token.dependency(1)
+        dep2.target = job2
+
+        # Acquire with first dependency
+        lock1 = await dep1.aio_lock(timeout=0.5)
+        assert token.available == 0
+
+        # Second acquire should timeout
+        start = time.time()
+        with pytest.raises(LockError, match="Timeout"):
+            await dep2.aio_lock(timeout=0.5)
+        elapsed = time.time() - start
+        assert 0.4 < elapsed < 0.7  # Should timeout around 0.5s
+
+        # Release first lock
+        lock1.release()
+        assert token.available == 1
+
+        # Now second should succeed
+        lock2 = await dep2.aio_lock(timeout=0.5)
+        assert lock2 is not None
+        lock2.release()
+
+
+async def test_token_notification():
+    """Test that condition notification wakes up waiting tasks"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-notify", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        job1 = MockJob("1")
+        job2 = MockJob("2")
+
+        dep1 = token.dependency(1)
+        dep1.target = job1
+
+        dep2 = token.dependency(1)
+        dep2.target = job2
+
+        # Acquire with first dependency
+        lock1 = await dep1.aio_lock(timeout=0.5)
+
+        # Start second acquisition in background
+        async def acquire_second():
+            lock = await dep2.aio_lock(timeout=5.0)  # Long timeout
+            return lock
+
+        task = asyncio.create_task(acquire_second())
+
+        # Give it time to start waiting
+        await asyncio.sleep(0.1)
+
+        # Release first lock - should notify waiting task
+        start = time.time()
+        lock1.release()
+
+        # Second task should complete quickly (not timeout)
+        lock2 = await task
+        elapsed = time.time() - start
+
+        assert lock2 is not None
+        assert elapsed < 1.0  # Should wake up immediately, not wait 5s
+        lock2.release()
+
+
+async def test_token_multiple_waiting():
+    """Test multiple tasks waiting for tokens"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-multiple", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        # Acquire the token
+        job1 = MockJob("1")
+        dep1 = token.dependency(1)
+        dep1.target = job1
+        lock1 = await dep1.aio_lock(timeout=0.5)
+
+        # Start multiple waiting tasks
+        acquired_order = []
+
+        async def acquire_task(name):
+            job = MockJob(name)
+            dep = token.dependency(1)
+            dep.target = job
+            lock = await dep.aio_lock(timeout=10.0)
+            acquired_order.append(name)
+            await asyncio.sleep(0.05)  # Hold briefly
+            lock.release()
+
+        tasks = [
+            asyncio.create_task(acquire_task("2")),
+            asyncio.create_task(acquire_task("3")),
+            asyncio.create_task(acquire_task("4")),
+        ]
+
+        # Give tasks time to start waiting
+        await asyncio.sleep(0.1)
+
+        # Release first lock
+        lock1.release()
+
+        # Wait for all tasks to complete
+        await asyncio.gather(*tasks)
+
+        # All tasks should have acquired the lock
+        assert len(acquired_order) == 3
+        assert set(acquired_order) == {"2", "3", "4"}
+
+
+async def test_token_timeout_zero():
+    """Test that timeout=0 waits indefinitely"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-timeout-zero", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        job1 = MockJob("1")
+        job2 = MockJob("2")
+
+        dep1 = token.dependency(1)
+        dep1.target = job1
+
+        dep2 = token.dependency(1)
+        dep2.target = job2
+
+        # Acquire with first
+        lock1 = await dep1.aio_lock(timeout=0.5)
+
+        # Start waiting with timeout=0 (infinite)
+        async def acquire_infinite():
+            return await dep2.aio_lock(timeout=0)  # Should wait forever
+
+        task = asyncio.create_task(acquire_infinite())
+
+        # Give it time to start waiting
+        await asyncio.sleep(0.1)
+
+        # Wait a bit more - task should still be waiting
+        await asyncio.sleep(0.5)
+        assert not task.done()
+
+        # Release first lock
+        lock1.release()
+
+        # Now task should complete
+        lock2 = await asyncio.wait_for(task, timeout=2.0)
+        assert lock2 is not None
+        lock2.release()

experimaestro/tests/test_tokens.py

@@ -59,9 +59,12 @@ def token_experiment(xp, token, ntasks=3):
         assert (times[i - 1] > times[i]) or (times[i] > times[i - 1])
 
 
-@pytest.mark.xfail(strict=True)
+@pytest.mark.xfail(
+    strict=False,
+    reason="Timing-dependent: tasks may run sequentially even without token",
+)
 def test_token_fail():
-    """Simple token test: should fail without token"""
+    """Simple token test: should fail without token (but may pass due to timing)"""
     with TemporaryExperiment("tokens", maxwait=20) as xp:
         token_experiment(xp, None)
 
@@ -71,6 +74,7 @@ def test_token_ok():
     with TemporaryExperiment("tokens", maxwait=20) as xp:
         token = CounterToken("token-ok", xp.workdir / "token", 1)
         token_experiment(xp, token)
+
     logging.info("Finished token_ok test")
 
 
@@ -129,7 +133,7 @@ def test_token_cleanup():
 
 
 def test_token_monitor():
-    """Two different schedulers (within the same process)
+    """Two different experiments (within the same process and workspace)
 
     Test the ability of the token to monitor the filesystem
     """
@@ -141,18 +145,18 @@ def test_token_monitor():
         )
         return task
 
-    with TemporaryExperiment("tokens1", maxwait=20, port=0) as xp1, TemporaryExperiment(
-        "tokens2", maxwait=20
-    ) as xp2:
-        path = xp1.workspace.path / "test_token.file"
-        task1 = run(xp1, 1, path)
-        task2 = run(xp2, 2, path)
+    with TemporaryExperiment("tokens1", maxwait=20, port=0) as xp1:
+        # Use the same workspace for both experiments
+        with TemporaryExperiment(
+            "tokens2", workdir=xp1.workspace.path, maxwait=20
+        ) as xp2:
+            path = xp1.workspace.path / "test_token.file"
+            task1 = run(xp1, 1, path)
+            task2 = run(xp2, 2, path)
 
-        time.sleep(0.5)
-        path.write_text("Hello world")
+            time.sleep(0.5)
+            path.write_text("Hello world")
 
-        xp1.wait()
-        xp2.wait()
         time1 = get_times(task1)
         time2 = get_times(task2)