experimaestro 1.11.1__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/tests/test_tags.py

@@ -1,5 +1,6 @@
 from typing import Dict
 from pathlib import Path
+import logging
 from experimaestro import (
     tag,
     LightweightTask,
@@ -21,20 +22,20 @@ class Config2(Config):
 
 
 def test_tag():
-    c = Config1(x=5)
+    c = Config1.C(x=5)
     c.tag("x", 5)
     assert c.tags() == {"x": 5}
 
 
 def test_taggedvalue():
-    c = Config1(x=tag(5))
+    c = Config1.C(x=tag(5))
     assert c.tags() == {"x": 5}
 
 
 def test_tagcontain():
     """Test that tags are not propagated to the upper configurations"""
-    c1 = Config1(x=5)
-    c2 = Config2(c=c1, x=tag(3)).tag("out", 1)
+    c1 = Config1.C(x=5)
+    c2 = Config2.C(c=c1, x=tag(3)).tag("out", 1)
     assert c1.tags() == {}
     assert c2.tags() == {"x": 3, "out": 1}
 
@@ -50,17 +51,17 @@ def test_inneroutput():
     class Evaluate(Task):
         task: Param[MyTask]
 
-    output = Output().tag("hello", "world")
-    task = MyTask(outputs={}, mainoutput=output)
+    output = Output.C().tag("hello", "world")
+    task = MyTask.C(outputs={}, mainoutput=output)
     task.submit(run_mode=RunMode.DRY_RUN)
     assert output.tags() == {"hello": "world"}
 
-    output = Output().tag("hello", "world")
-    task = MyTask(outputs={"a": output}, mainoutput=Output())
+    output = Output.C().tag("hello", "world")
+    task = MyTask.C(outputs={"a": output}, mainoutput=Output.C())
     task.submit(run_mode=RunMode.DRY_RUN)
     assert output.tags() == {"hello": "world"}
 
-    evaluate = Evaluate(task=task).submit(run_mode=RunMode.DRY_RUN)
+    evaluate = Evaluate.C(task=task).submit(run_mode=RunMode.DRY_RUN)
     assert evaluate.__xpm__.tags() == {"hello": "world"}
 
 
@@ -80,21 +81,21 @@ def test_tags_init_tasks():
         x: Param[MyConfig]
 
         def task_outputs(self, dep) -> MyConfig:
-            return dep(MyConfig())
+            return dep(MyConfig.C())
 
-    init_task = InitTask().tag("hello", "world")
-    task = MyTask()
+    init_task = InitTask.C().tag("hello", "world")
+    task = MyTask.C()
     result = task.submit(run_mode=RunMode.DRY_RUN, init_tasks=[init_task])
     assert result.tags() == {"hello": "world"}
 
-    other_task = TaskWithOutput(x=MyConfig().tag("hello", "world"))
+    other_task = TaskWithOutput.C(x=MyConfig.C().tag("hello", "world"))
     assert other_task.tags() == {"hello": "world"}
 
     result = other_task.submit(run_mode=RunMode.DRY_RUN)
     assert isinstance(result, MyConfig)
     assert result.tags() == {"hello": "world"}
 
-    result = MyTask().submit(run_mode=RunMode.DRY_RUN, init_tasks=[result])
+    result = MyTask.C().submit(run_mode=RunMode.DRY_RUN, init_tasks=[result])
     assert result.tags() == {"hello": "world"}
 
 
@@ -115,6 +116,141 @@ def test_objects_tags():
         x: Param[int]
 
     context = DirectoryContext(Path("/__fakepath__"))
-    a = A(x=tag(1))
+    a = A.C(x=tag(1))
     a.__xpm__.seal(context)
     assert a.__xpm__.tags() == {"x": 1}
+
+
+def test_conflicting_tags_warning(caplog):
+    """Test that conflicting tag values produce a warning"""
+
+    class Inner(Config):
+        value: Param[int]
+
+    class Outer(Config):
+        inner: Param[Inner]
+        x: Param[int]
+
+    # Create inner config with tag "mytag" = 1
+    inner = Inner.C(value=10).tag("mytag", 1)
+
+    # Create outer config with same tag "mytag" = 2 (conflicting)
+    outer = Outer.C(inner=inner, x=5).tag("mytag", 2)
+
+    # Getting tags should warn about conflict
+    with caplog.at_level(logging.WARNING):
+        tags = outer.tags()
+
+    # The warning should mention the conflicting tag
+    assert any("mytag" in record.message for record in caplog.records)
+    assert any("conflicting" in record.message.lower() for record in caplog.records)
+
+    # The last value should win
+    assert tags["mytag"] == 2
+
+
+def test_same_tag_same_value_no_warning(caplog):
+    """Test that same tag with same value does not produce a warning"""
+
+    class Inner(Config):
+        value: Param[int]
+
+    class Outer(Config):
+        inner: Param[Inner]
+
+    # Create inner config with tag "mytag" = 1
+    inner = Inner.C(value=10).tag("mytag", 1)
+
+    # Create outer config with same tag "mytag" = 1 (same value)
+    outer = Outer.C(inner=inner).tag("mytag", 1)
+
+    # Getting tags should NOT warn (same value)
+    with caplog.at_level(logging.WARNING):
+        tags = outer.tags()
+
+    # No warning for same values
+    assert not any("mytag" in record.message for record in caplog.records)
+    assert tags["mytag"] == 1
+
+
+def test_tag_source_tracking():
+    """Test that tag source locations are tracked"""
+
+    class MyConfig(Config):
+        x: Param[int]
+
+    config = MyConfig.C(x=tag(5))
+
+    # Check that tags have source info stored internally
+    assert "x" in config.__xpm__._tags
+    value, source = config.__xpm__._tags["x"]
+    assert value == 5
+    # Source should contain file path and line number
+    assert ":" in source
+    assert "test_tags.py" in source
+
+
+def test_tag_method_source_tracking():
+    """Test that tag() method also tracks source location"""
+
+    class MyConfig(Config):
+        x: Param[int]
+
+    config = MyConfig.C(x=5)
+    config.tag("mytag", "myvalue")
+
+    # Check that tag has source info
+    assert "mytag" in config.__xpm__._tags
+    value, source = config.__xpm__._tags["mytag"]
+    assert value == "myvalue"
+    assert ":" in source
+    assert "test_tags.py" in source
+
+
+def test_tag_via_setattr():
+    """Test that config.key = tag(value) works and tracks source"""
+
+    class MyConfig(Config):
+        x: Param[int]
+
+    config = MyConfig.C(x=5)
+    config.x = tag(10)
+
+    # Check that tag was set correctly
+    assert config.tags() == {"x": 10}
+    assert config.x == 10
+
+    # Check that source is tracked
+    value, source = config.__xpm__._tags["x"]
+    assert value == 10
+    assert "test_tags.py" in source
+
+
+def test_tag_setattr_conflict_warning(caplog):
+    """Test that setting conflicting tag via setattr produces warning"""
+
+    class Inner(Config):
+        value: Param[int]
+
+    class Outer(Config):
+        inner: Param[Inner]
+        x: Param[int]
+
+    # Create with tag via constructor
+    inner = Inner.C(value=tag(1))
+
+    # Create outer with same tag name
+    outer = Outer.C(inner=inner, x=5)
+    outer.x = tag(2)  # Set tag on x
+
+    # Add a conflicting value tag
+    outer.tag("value", 99)
+
+    # Getting tags should warn about conflict
+    with caplog.at_level(logging.WARNING):
+        tags = outer.tags()
+
+    # The warning should mention the conflicting tag
+    assert any("value" in record.message for record in caplog.records)
+    assert tags["value"] == 99  # Last value wins
+    assert tags["x"] == 2

experimaestro/tests/test_tasks.py

@@ -1,11 +1,20 @@
 # --- Task and types definitions
 
+import sys
+import time
 from pathlib import Path
 import pytest
 import logging
-from experimaestro import Config, deprecate, Task, Param
+from experimaestro import (
+    Config,
+    Task,
+    Param,
+    ResumableTask,
+    Meta,
+    field,
+    PathGenerator,
+)
 from experimaestro.scheduler.workspace import RunMode
-from experimaestro.tools.jobs import fix_deprecated
 from experimaestro.scheduler import FailedExperiment, JobState
 from experimaestro import SubmitHook, Job, Launcher, LightweightTask
 
@@ -29,8 +38,8 @@ from .definitions_types import IntegerTask, FloatTask
 
 def test_task_types():
     with TemporaryExperiment("simple"):
-        assert IntegerTask(value=5).submit().__xpm__.job.wait() == JobState.DONE
-        assert FloatTask(value=5.1).submit().__xpm__.job.wait() == JobState.DONE
+        assert IntegerTask.C(value=5).submit().__xpm__.job.wait() == JobState.DONE
+        assert FloatTask.C(value=5.1).submit().__xpm__.job.wait() == JobState.DONE
 
 
 def test_simple_task():
@@ -38,11 +47,11 @@ def test_simple_task():
         assert isinstance(workdir, Path)
         with TemporaryExperiment("helloworld", workdir=workdir, maxwait=20):
             # Submit the tasks
-            hello = Say(word="hello").submit()
-            world = Say(word="world").submit()
+            hello = Say.C(word="hello").submit()
+            world = Say.C(word="world").submit()
 
             # Concat will depend on the two first tasks
-            concat = Concat(strings=[hello, world]).submit()
+            concat = Concat.C(strings=[hello, world]).submit()
 
         assert concat.__xpm__.job.state == JobState.DONE
         assert Path(concat.stdout()).read_text() == "HELLO WORLD\n"
@@ -51,16 +60,16 @@ def test_simple_task():
 def test_not_submitted():
     """A not submitted task should not be accepted as an argument"""
     with TemporaryExperiment("helloworld", maxwait=2):
-        hello = Say(word="hello")
+        hello = Say.C(word="hello")
         with pytest.raises(ValueError):
-            Concat(strings=[hello])
+            Concat.C(strings=[hello])
 
 
 def test_fail_simple():
     """Failing task... should fail"""
     with pytest.raises(FailedExperiment):
         with TemporaryExperiment("failing", maxwait=20):
-            fail = Fail().submit()
+            fail = Fail.C().submit()
             fail.touch()
 
 
@@ -70,8 +79,8 @@ def test_foreign_type():
         # Submit the tasks
         from .tasks.foreign import ForeignClassB2
 
-        b = ForeignClassB2(x=1, y=2)
-        a = ForeignTaskA(b=b).submit()
+        b = ForeignClassB2.C(x=1, y=2)
+        a = ForeignTaskA.C(b=b).submit()
 
         assert a.__xpm__.job.wait() == JobState.DONE
         assert a.stdout().read_text().strip() == "1"
@@ -81,8 +90,8 @@ def test_fail_dep():
     """Failing task... should cancel dependent"""
     with pytest.raises(FailedExperiment):
         with TemporaryExperiment("failingdep"):
-            fail = Fail().submit()
-            dep = FailConsumer(fail=fail).submit()
+            fail = Fail.C().submit()
+            dep = FailConsumer.C(fail=fail).submit()
             fail.touch()
 
     assert fail.__xpm__.job.state == JobState.ERROR
@@ -92,14 +101,14 @@ def test_fail_dep():
 def test_unknown_attribute():
     """No check when setting attributes while executing"""
     with TemporaryExperiment("unknown"):
-        method = SetUnknown().submit()
+        method = SetUnknown.C().submit()
 
     assert method.__xpm__.job.wait() == JobState.DONE
 
 
 def test_function():
     with TemporaryExperiment("function"):
-        method = Method(a=1).submit()
+        method = Method.C(a=1).submit()
 
     assert method.__xpm__.job.wait() == JobState.DONE
 
@@ -111,7 +120,7 @@ def test_done():
 
 
 def restart_function(xp):
-    restart.Restart().submit()
+    restart.Restart.C().submit()
 
 
 @pytest.mark.parametrize("terminate", restart.TERMINATES_FUNC)
@@ -123,135 +132,27 @@ def test_restart(terminate):
 def test_submitted_twice():
     """Check that a job cannot be submitted twice within the same experiment"""
     with TemporaryExperiment("duplicate", maxwait=20):
-        task1 = SimpleTask(x=1).submit()
-        task2 = SimpleTask(x=1).submit()
-        assert task1 is task2, f"{id(task1)} != {id(task2)}"
+
+        task1 = SimpleTask.C(x=1)
+        o1 = task1.submit()
+
+        task2 = SimpleTask.C(x=1)
+        o2 = task2.submit()
+
+        print(o1)
+        assert o1.task is not o2.task
+        assert task1.__xpm__.job is task2.__xpm__.job, f"{id(task1)} != {id(task2)}"
 
 
 def test_configcache():
     """Test a configuration cache"""
 
     with TemporaryExperiment("configcache", maxwait=20):
-        task = CacheConfigTask(data=CacheConfig()).submit()
+        task = CacheConfigTask.C(data=CacheConfig.C()).submit()
 
     assert task.__xpm__.job.wait() == JobState.DONE
 
 
-# ---- Deprecation
-
-
-class NewConfig(Config):
-    __xpmid__ = "new"
-
-
-@deprecate
-class DeprecatedConfig(NewConfig):
-    __xpmid__ = "deprecated"
-
-
-class OldConfig(NewConfig):
-    __xpmid__ = "deprecated"
-
-
-class TaskWithDeprecated(Task):
-    p: Param[NewConfig]
-
-    def execute(self):
-        pass
-
-
-def checknewpaths(task_new, task_old_path):
-    task_new_path = task_new.__xpm__.job.path  # type: Path
-
-    assert task_new_path.exists(), f"New path {task_new_path} should exist"
-    assert task_new_path.is_symlink(), f"New path {task_new_path} should be a symlink"
-
-    assert task_new_path.resolve() == task_old_path
-
-
-def test_tasks_deprecated_inner():
-    """Test that when submitting the task, the computed identifier is the one of
-    the new class"""
-    with TemporaryExperiment("deprecated", maxwait=0) as xp:
-        # --- Check that paths are really different first
-        task_new = TaskWithDeprecated(p=NewConfig()).submit(run_mode=RunMode.DRY_RUN)
-        task_old = TaskWithDeprecated(p=OldConfig()).submit(run_mode=RunMode.DRY_RUN)
-        task_deprecated = TaskWithDeprecated(p=DeprecatedConfig()).submit(
-            run_mode=RunMode.DRY_RUN
-        )
-
-        logging.debug("New task ID: %s", task_new.__xpm__.identifier.all.hex())
-        logging.debug("Old task ID: %s", task_old.__xpm__.identifier.all.hex())
-        logging.debug(
-            "Old task (with deprecated flag): %s",
-            task_deprecated.__xpm__.identifier.all.hex(),
-        )
-        assert (
-            task_new.stdout() != task_old.stdout()
-        ), "Old and new path should be different"
-
-        assert (
-            task_new.stdout() == task_deprecated.stdout()
-        ), "Deprecated path should be the same as non deprecated"
-
-        # --- Now check that automatic linking is performed
-
-        # Run old task with deprecated configuration
-        task_old = TaskWithDeprecated(p=OldConfig()).submit()
-        task_old.wait()
-        task_old_path = task_old.stdout().parent
-
-        # Fix deprecated
-        OldConfig.__xpmtype__.deprecate()
-        fix_deprecated(xp.workspace.path, True, False)
-
-        checknewpaths(task_new, task_old_path)
-
-
-class NewTask(Task):
-    x: Param[int]
-
-    def execute(self):
-        pass
-
-
-class OldTask(NewTask):
-    __xpmid__ = "deprecated"
-
-
-@deprecate
-class DeprecatedTask(NewTask):
-    __xpmid__ = "deprecated"
-
-
-def test_tasks_deprecated():
-    """Test that when submitting the task, the computed identifier is the one of
-    the new class"""
-    with TemporaryExperiment("deprecated", maxwait=20) as xp:
-        # Check that paths are really different first
-        task_new = NewTask(x=1).submit(run_mode=RunMode.DRY_RUN)
-        task_old = OldTask(x=1).submit(run_mode=RunMode.DRY_RUN)
-        task_deprecated = DeprecatedTask(x=1).submit(run_mode=RunMode.DRY_RUN)
-
-        assert (
-            task_new.stdout() != task_old.stdout()
-        ), "Old and new path should be different"
-        assert (
-            task_new.stdout() == task_deprecated.stdout()
-        ), "Deprecated path should be the same as non deprecated"
-
-        # OK, now check that automatic linking is performed
-        task_old = OldTask(x=1).submit()
-        task_old.wait()
-        task_old_path = task_old.stdout().parent
-
-        # Fix deprecated
-        OldTask.__xpmtype__.deprecate()
-        fix_deprecated(xp.workspace.path, True, False)
-
-        checknewpaths(task_new, task_old_path)
-
-
 class needs_java(SubmitHook):
     def __init__(self, version: int):
         self.version = version
@@ -270,7 +171,7 @@ class HookedTask(Task):
 
 
 def test_task_submit_hook():
-    result = HookedTask().submit(run_mode=RunMode.DRY_RUN)
+    result = HookedTask.C().submit(run_mode=RunMode.DRY_RUN)
     assert (
         result.__xpm__.task.__xpm__.job.environ.get("JAVA_HOME", None)
         == "THE_JAVA_HOME"
@@ -299,31 +200,107 @@ class MyLightweightTask(Task):
         assert self.x.data == 1
 
 
-def test_task_lightweight():
-    with TemporaryExperiment("lightweight", maxwait=20):
-        x = LightweightConfig()
-        lwtask = LightweightTask(x=x)
-        assert (
-            MyLightweightTask(x=x).add_pretasks(lwtask).submit().__xpm__.job.wait()
-            == JobState.DONE
-        ), "Pre-tasks should be executed"
-
-        x_2 = LightweightConfig()
-        lwtask_2 = LightweightTask(x=x)
-        assert (
-            MyLightweightTask(x=x_2.add_pretasks(lwtask_2))
-            .add_pretasks(lwtask_2)
-            .submit()
-            .__xpm__.job.wait()
-            == JobState.DONE
-        ), "Pre-tasks should be run just once"
-
-
 def test_task_lightweight_init():
     with TemporaryExperiment("lightweight_init", maxwait=20):
-        x = LightweightConfig()
-        lwtask = LightweightTask(x=x)
+        x = LightweightConfig.C()
+        lwtask = LightweightTask.C(x=x)
         assert (
-            MyLightweightTask(x=x).submit(init_tasks=[lwtask]).__xpm__.job.wait()
+            MyLightweightTask.C(x=x).submit(init_tasks=[lwtask]).__xpm__.job.wait()
             == JobState.DONE
         ), "Init tasks should be executed"
+
+
+# --- Test for resumable task resubmission
+
+
+class ControllableResumableTask(ResumableTask):
+    """A resumable task that can be controlled via files"""
+
+    control_file: Meta[Path] = field(default_factory=PathGenerator("control"))
+
+    def execute(self):
+        # Wait for control file
+        while not self.control_file.is_file():
+            time.sleep(0.1)
+
+        # Read control: "fail" to exit with error, "complete" to succeed
+        action = self.control_file.read_text().strip()
+        self.control_file.unlink()
+
+        if action == "fail":
+            sys.exit(1)
+
+
+def test_resumable_task_resubmit():
+    """Test resubmitting a failed ResumableTask within the same experiment"""
+    with TemporaryExperiment("resumable_resubmit", maxwait=30):
+        task1 = ControllableResumableTask.C()
+        task1.submit()
+
+        # Tell task to fail
+        task1.control_file.parent.mkdir(parents=True, exist_ok=True)
+        task1.control_file.write_text("fail")
+
+        # Wait for the job to fail
+        job = task1.__xpm__.job
+        assert job.wait() == JobState.ERROR, "Job should have failed"
+
+        # Resubmit by creating a new instance with same parameters
+        task2 = ControllableResumableTask.C()
+        task2.submit()
+
+        # Tell task to complete
+        task2.control_file.write_text("complete")
+
+        # Wait for the resubmitted job to complete
+        assert task2.__xpm__.job.wait() == JobState.DONE
+
+
+def test_resumable_task_resubmit_across_experiments():
+    """Test resubmitting a failed ResumableTask across two experiment instances"""
+    with TemporaryDirectory(prefix="xpm", suffix="resubmit_across") as workdir:
+        # First experiment: task fails
+        try:
+            with TemporaryExperiment("resubmit_across", maxwait=10, workdir=workdir):
+                task1 = ControllableResumableTask.C()
+                task1.submit()
+
+                # Tell task to fail
+                task1.control_file.parent.mkdir(parents=True, exist_ok=True)
+                task1.control_file.write_text("fail")
+        except Exception as e:
+            logging.info("First experiment ended (expected): %s", e)
+
+        # Second experiment: task completes
+        with TemporaryExperiment("resubmit_across", maxwait=30, workdir=workdir):
+            task2 = ControllableResumableTask.C()
+            task2.submit()
+
+            # Tell task to complete
+            task2.control_file.write_text("complete")
+
+            # Wait for the resubmitted job to complete
+            assert task2.__xpm__.job.wait() == JobState.DONE
+
+
+def test_task_resubmit_across_experiments():
+    """Test resubmitting a completed task across two experiment instances"""
+    with TemporaryDirectory(prefix="xpm", suffix="resubmit_across") as workdir:
+        # First experiment: task completes
+        with TemporaryExperiment("resubmit_across", maxwait=30, workdir=workdir):
+            task1 = ControllableResumableTask.C()
+            task1.submit()
+
+            # Tell task to complete
+            task1.control_file.parent.mkdir(parents=True, exist_ok=True)
+            task1.control_file.write_text("complete")
+
+            assert task1.__xpm__.job.wait() == JobState.DONE
+
+        # Second experiment: resubmit completed task (uses same workdir)
+        with TemporaryExperiment("resubmit_across", maxwait=30, workdir=workdir):
+            task2 = ControllableResumableTask.C()
+            task2.submit()
+
+            # Task should recognize it's already done
+            assert task2.__xpm__.job.wait() == JobState.DONE