experimaestro 1.11.1__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/tests/test_token_locking.py

@@ -0,0 +1,252 @@
+"""Unit tests for token locking mechanism
+
+Tests the CounterToken condition variable-based synchronization
+without requiring full scheduler integration.
+"""
+
+import asyncio
+import pytest
+import tempfile
+from pathlib import Path
+import time
+
+from experimaestro.tokens import CounterToken
+from experimaestro.locking import LockError
+
+pytestmark = pytest.mark.anyio
+
+
+async def test_token_acquire_release():
+    """Test basic token acquire and release"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-basic", Path(tmpdir) / "token", count=1)
+
+        # Create a mock job target
+        class MockJob:
+            @property
+            def identifier(self):
+                return "mock-job-1"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / "job1"
+
+        job = MockJob()
+
+        # Create dependency
+        dep = token.dependency(1)
+        dep.target = job
+
+        # Should be able to acquire
+        lock = await dep.aio_lock(timeout=1.0)
+        assert lock is not None
+        assert token.available == 0
+
+        # Release
+        lock.release()
+        assert token.available == 1
+
+
+async def test_token_blocking():
+    """Test that acquiring blocks when no tokens available"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-blocking", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        job1 = MockJob("1")
+        job2 = MockJob("2")
+
+        dep1 = token.dependency(1)
+        dep1.target = job1
+
+        dep2 = token.dependency(1)
+        dep2.target = job2
+
+        # Acquire with first dependency
+        lock1 = await dep1.aio_lock(timeout=0.5)
+        assert token.available == 0
+
+        # Second acquire should timeout
+        start = time.time()
+        with pytest.raises(LockError, match="Timeout"):
+            await dep2.aio_lock(timeout=0.5)
+        elapsed = time.time() - start
+        assert 0.4 < elapsed < 0.7  # Should timeout around 0.5s
+
+        # Release first lock
+        lock1.release()
+        assert token.available == 1
+
+        # Now second should succeed
+        lock2 = await dep2.aio_lock(timeout=0.5)
+        assert lock2 is not None
+        lock2.release()
+
+
+async def test_token_notification():
+    """Test that condition notification wakes up waiting tasks"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-notify", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        job1 = MockJob("1")
+        job2 = MockJob("2")
+
+        dep1 = token.dependency(1)
+        dep1.target = job1
+
+        dep2 = token.dependency(1)
+        dep2.target = job2
+
+        # Acquire with first dependency
+        lock1 = await dep1.aio_lock(timeout=0.5)
+
+        # Start second acquisition in background
+        async def acquire_second():
+            lock = await dep2.aio_lock(timeout=5.0)  # Long timeout
+            return lock
+
+        task = asyncio.create_task(acquire_second())
+
+        # Give it time to start waiting
+        await asyncio.sleep(0.1)
+
+        # Release first lock - should notify waiting task
+        start = time.time()
+        lock1.release()
+
+        # Second task should complete quickly (not timeout)
+        lock2 = await task
+        elapsed = time.time() - start
+
+        assert lock2 is not None
+        assert elapsed < 1.0  # Should wake up immediately, not wait 5s
+        lock2.release()
+
+
+async def test_token_multiple_waiting():
+    """Test multiple tasks waiting for tokens"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-multiple", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        # Acquire the token
+        job1 = MockJob("1")
+        dep1 = token.dependency(1)
+        dep1.target = job1
+        lock1 = await dep1.aio_lock(timeout=0.5)
+
+        # Start multiple waiting tasks
+        acquired_order = []
+
+        async def acquire_task(name):
+            job = MockJob(name)
+            dep = token.dependency(1)
+            dep.target = job
+            lock = await dep.aio_lock(timeout=10.0)
+            acquired_order.append(name)
+            await asyncio.sleep(0.05)  # Hold briefly
+            lock.release()
+
+        tasks = [
+            asyncio.create_task(acquire_task("2")),
+            asyncio.create_task(acquire_task("3")),
+            asyncio.create_task(acquire_task("4")),
+        ]
+
+        # Give tasks time to start waiting
+        await asyncio.sleep(0.1)
+
+        # Release first lock
+        lock1.release()
+
+        # Wait for all tasks to complete
+        await asyncio.gather(*tasks)
+
+        # All tasks should have acquired the lock
+        assert len(acquired_order) == 3
+        assert set(acquired_order) == {"2", "3", "4"}
+
+
+async def test_token_timeout_zero():
+    """Test that timeout=0 waits indefinitely"""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        token = CounterToken("test-timeout-zero", Path(tmpdir) / "token", count=1)
+
+        class MockJob:
+            def __init__(self, name):
+                self.name = name
+
+            @property
+            def identifier(self):
+                return f"mock-job-{self.name}"
+
+            @property
+            def basepath(self):
+                return Path(tmpdir) / self.name
+
+        job1 = MockJob("1")
+        job2 = MockJob("2")
+
+        dep1 = token.dependency(1)
+        dep1.target = job1
+
+        dep2 = token.dependency(1)
+        dep2.target = job2
+
+        # Acquire with first
+        lock1 = await dep1.aio_lock(timeout=0.5)
+
+        # Start waiting with timeout=0 (infinite)
+        async def acquire_infinite():
+            return await dep2.aio_lock(timeout=0)  # Should wait forever
+
+        task = asyncio.create_task(acquire_infinite())
+
+        # Give it time to start waiting
+        await asyncio.sleep(0.1)
+
+        # Wait a bit more - task should still be waiting
+        await asyncio.sleep(0.5)
+        assert not task.done()
+
+        # Release first lock
+        lock1.release()
+
+        # Now task should complete
+        lock2 = await asyncio.wait_for(task, timeout=2.0)
+        assert lock2 is not None
+        lock2.release()

experimaestro/tests/test_tokens.py

@@ -32,7 +32,7 @@ def token_experiment(xp, token, ntasks=3):
 
     tasks = []
     for it in range(ntasks):
-        task = TokenTask(path=path, x=it)
+        task = TokenTask.C(path=path, x=it)
         if token:
             task.add_dependencies(token.dependency(1))
         tasks.append(task.submit())
@@ -59,9 +59,12 @@ def token_experiment(xp, token, ntasks=3):
         assert (times[i - 1] > times[i]) or (times[i] > times[i - 1])
 
 
-@pytest.mark.xfail(strict=True)
+@pytest.mark.xfail(
+    strict=False,
+    reason="Timing-dependent: tasks may run sequentially even without token",
+)
 def test_token_fail():
-    """Simple token test: should fail without token"""
+    """Simple token test: should fail without token (but may pass due to timing)"""
     with TemporaryExperiment("tokens", maxwait=20) as xp:
         token_experiment(xp, None)
 
@@ -71,6 +74,7 @@ def test_token_ok():
     with TemporaryExperiment("tokens", maxwait=20) as xp:
         token = CounterToken("token-ok", xp.workdir / "token", 1)
         token_experiment(xp, token)
+
     logging.info("Finished token_ok test")
 
 
@@ -86,7 +90,7 @@ def test_token_cleanup():
     with TemporaryExperiment("token_cleanup", maxwait=20) as xp:
         token = CounterToken("token-cleanup", xp.workdir / "token-cleanup", 1)
 
-        task = dummy_task(x=1)
+        task = dummy_task.C(x=1)
         dependency = token.dependency(1)
         task.add_dependencies(dependency)
         # Just to create the directory
@@ -98,7 +102,7 @@ def test_token_cleanup():
         # The absence of process should be detected right away
         logging.info("Lock without process")
         TokenFile.create(dependency)
-        task2 = dummy_task(x=2)
+        task2 = dummy_task.C(x=2)
         task2.add_dependencies(token.dependency(1)).submit()
         xp.wait()
 
@@ -118,7 +122,7 @@ def test_token_cleanup():
             p1 = subprocess.Popen(command)
             job.pidpath.write_text(json.dumps({"pid": p1.pid, "type": "local"}))
 
-            task3 = dummy_task(x=3)
+            task3 = dummy_task.C(x=3)
             task3.add_dependencies(token.dependency(1)).submit()
 
             # Ends the script "waitforfile.py"
@@ -129,28 +133,30 @@ def test_token_cleanup():
 
 
 def test_token_monitor():
-    """Two different schedulers (within the same process)
+    """Two different experiments (within the same process and workspace)
 
     Test the ability of the token to monitor the filesystem
     """
 
     def run(xp, x, path):
         token = xp.workspace.connector.createtoken("test-token-monitor", 1)
-        task = TokenTask(path=path, x=x).add_dependencies(token.dependency(1)).submit()
+        task = (
+            TokenTask.C(path=path, x=x).add_dependencies(token.dependency(1)).submit()
+        )
         return task
 
-    with TemporaryExperiment("tokens1", maxwait=20, port=0) as xp1, TemporaryExperiment(
-        "tokens2", maxwait=20
-    ) as xp2:
-        path = xp1.workspace.path / "test_token.file"
-        task1 = run(xp1, 1, path)
-        task2 = run(xp2, 2, path)
+    with TemporaryExperiment("tokens1", maxwait=20, port=0) as xp1:
+        # Use the same workspace for both experiments
+        with TemporaryExperiment(
+            "tokens2", workdir=xp1.workspace.path, maxwait=20
+        ) as xp2:
+            path = xp1.workspace.path / "test_token.file"
+            task1 = run(xp1, 1, path)
+            task2 = run(xp2, 2, path)
 
-        time.sleep(0.5)
-        path.write_text("Hello world")
+            time.sleep(0.5)
+            path.write_text("Hello world")
 
-        xp1.wait()
-        xp2.wait()
         time1 = get_times(task1)
         time2 = get_times(task2)
 
@@ -241,7 +247,7 @@ def test_token_process():
 
 def restart_function(xp):
     token = CounterToken("restart-token", xp.workdir / "token", 1)
-    token(1, restart.Restart()).submit()
+    token(1, restart.Restart.C()).submit()
 
 
 @pytest.mark.parametrize("terminate", restart.TERMINATES_FUNC)

experimaestro/tests/test_types.py

@@ -1,7 +1,7 @@
 # --- Task and types definitions
 
 import logging
-from experimaestro import Config, Param
+from experimaestro import Config, Param, field
 from typing import Union
 
 import pytest
@@ -26,17 +26,17 @@ def test_multiple_inheritance():
 
     for C in (C1, C2):
         logging.info("Testing %s", C)
-        ctype = C.__xpmtype__
+        ctype = C.__getxpmtype__()
         assert issubclass(C, A)
         assert issubclass(C, B)
         assert issubclass(C, B1)
 
-        assert ctype.objecttype == C.__xpmtype__.objecttype
+        assert ctype.value_type == C.__getxpmtype__().value_type
 
-        assert issubclass(C.__xpmtype__.objecttype, B1.__xpmtype__.basetype)
-        assert issubclass(C.__xpmtype__.objecttype, B.__xpmtype__.basetype)
-        assert issubclass(C.__xpmtype__.objecttype, A.__xpmtype__.basetype)
-        assert not issubclass(C.__xpmtype__.objecttype, ConfigMixin)
+        assert issubclass(C.__getxpmtype__().value_type, B1.__getxpmtype__().value_type)
+        assert issubclass(C.__getxpmtype__().value_type, B.__getxpmtype__().value_type)
+        assert issubclass(C.__getxpmtype__().value_type, A.__getxpmtype__().value_type)
+        assert not issubclass(C.__getxpmtype__().value_type, ConfigMixin)
 
 
 def test_missing_hierarchy():
@@ -49,7 +49,7 @@ def test_missing_hierarchy():
     class B(A1):
         pass
 
-    B.__xpmtype__
+    B.__getxpmtype__()
 
     assert issubclass(B, A)
     assert issubclass(B, A1)
@@ -59,7 +59,129 @@ def test_types_union():
     class A(Config):
         x: Param[Union[int, str]]
 
-    A(x=1)
-    A(x="hello")
+    A.C(x=1)
+    A.C(x="hello")
     with pytest.raises(ValueError):
-        A(x=[])
+        A.C(x=[])
+
+
+def test_override_warning_without_flag(caplog):
+    """Test that overriding a parameter without overrides=True produces a warning"""
+
+    class Parent(Config):
+        value: Param[int]
+
+    with caplog.at_level(logging.WARNING, logger="xpm"):
+        # Child overrides value without overrides=True
+        class Child(Parent):
+            value: Param[int]
+
+        # Force initialization to trigger the warning
+        Child.__getxpmtype__().arguments
+
+    assert "overrides parent parameter" in caplog.text
+    assert "Child" in caplog.text
+    assert "value" in caplog.text
+
+
+def test_override_no_warning_with_flag(caplog):
+    """Test that overriding with overrides=True suppresses the warning"""
+
+    class Parent(Config):
+        value: Param[int]
+
+    with caplog.at_level(logging.WARNING, logger="xpm"):
+        # Child overrides value with overrides=True
+        class Child(Parent):
+            value: Param[int] = field(overrides=True)
+
+        # Force initialization
+        Child.__getxpmtype__().arguments
+
+    # No warning should be issued
+    assert "overrides parent parameter" not in caplog.text
+
+
+def test_override_type_check_subtype_config():
+    """Test that overriding Config type with subtype is allowed"""
+
+    class BaseValue(Config):
+        x: Param[int]
+
+    class DerivedValue(BaseValue):
+        y: Param[int]
+
+    class Parent(Config):
+        value: Param[BaseValue]
+
+    # Should succeed - DerivedValue is subtype of BaseValue
+    class Child(Parent):
+        value: Param[DerivedValue] = field(overrides=True)
+
+    Child.__getxpmtype__().arguments
+
+
+def test_override_type_check_incompatible_config():
+    """Test that overriding Config type with incompatible type raises error"""
+
+    class ValueA(Config):
+        x: Param[int]
+
+    class ValueB(Config):
+        y: Param[int]
+
+    class Parent(Config):
+        value: Param[ValueA]
+
+    # Should fail - ValueB is not a subtype of ValueA
+    with pytest.raises(TypeError, match="is not a subtype"):
+
+        class Child(Parent):
+            value: Param[ValueB] = field(overrides=True)
+
+        Child.__getxpmtype__().arguments
+
+
+def test_override_type_check_primitive_incompatible():
+    """Test that overriding primitive type with incompatible type raises error"""
+
+    class Parent(Config):
+        value: Param[int]
+
+    # Should fail - str is not a subtype of int
+    with pytest.raises(TypeError, match="is not compatible"):
+
+        class Child(Parent):
+            value: Param[str] = field(overrides=True)
+
+        Child.__getxpmtype__().arguments
+
+
+def test_override_type_check_same_type():
+    """Test that overriding with the same type is allowed"""
+
+    class Parent(Config):
+        value: Param[int]
+
+    # Should succeed - same type
+    class Child(Parent):
+        value: Param[int] = field(overrides=True)
+
+    Child.__getxpmtype__().arguments
+
+
+def test_no_override_warning_for_new_param(caplog):
+    """Test that defining a new parameter doesn't produce a warning"""
+
+    class Parent(Config):
+        x: Param[int]
+
+    with caplog.at_level(logging.WARNING, logger="xpm"):
+        # Child defines a new parameter y, doesn't override x
+        class Child(Parent):
+            y: Param[int]
+
+        Child.__getxpmtype__().arguments
+
+    # No warning should be issued for new parameter
+    assert "overrides parent parameter" not in caplog.text

experimaestro/tests/test_validation.py

@@ -36,22 +36,22 @@ class C(Config):
 
 
 def test_validation_simple():
-    expect_validate(A(value=1))
+    expect_validate(A.C(value=1))
 
 
 def test_validation_missing():
-    expect_notvalidate(A())
+    expect_notvalidate(A.C())
 
 
 def test_validation_simple_nested():
-    b = B()
-    b.a = A(value=1)
+    b = B.C()
+    b.a = A.C(value=1)
     expect_validate(b)
 
 
 def test_validation_missing_nested():
-    b = B()
-    b.a = A()
+    b = B.C()
+    b.a = A.C()
     expect_notvalidate(b)
 
 
@@ -68,11 +68,11 @@ def test_validation_type():
         __xpmid__ = valns.type.c
 
     with pytest.raises(ValueError):
-        C(a=B())
+        C.C(a=B.C())
 
     with pytest.raises(ValueError):
-        c = C()
-        c.a = B()
+        c = C.C()
+        c.a = B.C()
 
 
 def test_validation_subtype():
@@ -86,7 +86,7 @@ def test_validation_subtype():
         __xpmid__ = valns.subtype.b
         a: Param[A]
 
-    expect_validate(B(a=A1()))
+    expect_validate(B.C(a=A1.C()))
 
 
 def test_validation_path_generator():
@@ -96,7 +96,7 @@ def test_validation_path_generator():
         __xpmid__ = valns.path.a
         value: Meta[Path] = field(default_factory=PathGenerator("file.txt"))
 
-    a = A()
+    a = A.C()
     a.__xpm__.validate()
     with TemporaryExperiment("constant") as xp:
         jobcontext = Job(a)
@@ -116,7 +116,7 @@ def test_validation_constant():
         __xpmid__ = valns.constant.a
         value: Constant[int] = 1
 
-    a = A()
+    a = A.C()
     a.__xpm__.validate()
     with TemporaryExperiment("constant"):
         joba = Job(a)
@@ -133,7 +133,7 @@ class Child(Parent):
 
 
 def test_validation_child():
-    expect_validate(Child(x=1))
+    expect_validate(Child.C(x=1))
 
 
 # --- Path argument checks
@@ -144,7 +144,7 @@ class PathParent(Config):
 
 
 def test_validation_path_option():
-    c = PathParent()
+    c = PathParent.C()
     expect_validate(c)
 
 
@@ -157,7 +157,7 @@ def test_validation_seal():
     class A(Config):
         a: Param[int]
 
-    a = A(a=2)
+    a = A.C(a=2)
     a.__xpm__.seal(EmptyContext())
 
     with pytest.raises(AttributeError):
@@ -174,10 +174,10 @@ def test_validation_validation_enum():
     class EnumConfig(Config):
         a: Param[EnumParam]
 
-    expect_validate(EnumConfig(a=EnumParam.FIRST))
+    expect_validate(EnumConfig.C(a=EnumParam.FIRST))
 
     try:
-        EnumConfig(a=1)
+        EnumConfig.C(a=1)
         assert False, "Enum value should be rejected"
     except AssertionError:
         pass
@@ -199,7 +199,7 @@ class TaskConfigConsumer(Config):
 
 
 def test_validation_taskargument():
-    x = taskconfig()
+    x = taskconfig.C()
     with TemporaryExperiment("fake"):
         x.submit(run_mode=RunMode.DRY_RUN)
-        expect_validate(TaskConfigConsumer(x=x))
+        expect_validate(TaskConfigConsumer.C(x=x))