experimaestro 2.0.0a8__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/tests/test_objects.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 import pytest
-from experimaestro import Config, Task, Annotated, copyconfig, default
+from experimaestro import Config, Task, Annotated, copyconfig, field
 from experimaestro.core.arguments import Param
 from experimaestro.core.objects import ConfigMixin
 from experimaestro.generators import pathgenerator
@@ -16,17 +16,17 @@ def xp():
 
 
 class A(Config):
-    x: Param[int] = 3
+    x: Param[int] = field(ignore_default=3)
 
 
 def test_object_default():
     """Test plain default value"""
-    a = A()
+    a = A.C()
     assert a.x == 3
 
 
 class B(Config):
-    a: Param[A] = A.C(x=3)
+    a: Param[A] = field(ignore_default=A.C(x=3))
 
 
 class C(B):
@@ -38,7 +38,7 @@ class D(B, A):
 
 
 class DefaultAnnotationConfig(Config):
-    a: Annotated[A, default(A.C(x=3))]
+    a: Param[A] = field(default=A.C(x=3))
 
 
 def test_object_config_default():
@@ -84,3 +84,310 @@ def test_copyconfig(xp):
 
     assert copy_b.x == b.x
     assert "path" not in copy_b.__xpm__.values
+
+
+# --- Composition operator tests (GH #33) ---
+
+
+class CompositionA(Config):
+    x: Param[int]
+
+
+class CompositionSubA(CompositionA):
+    """Subclass of CompositionA"""
+
+    y: Param[int] = field(ignore_default=0)
+
+
+class CompositionB(Config):
+    a: Param[CompositionA]
+
+
+class CompositionC(Config):
+    """Config with two parameters of same type - should be ambiguous"""
+
+    a1: Param[CompositionA]
+    a2: Param[CompositionA]
+
+
+class CompositionD(Config):
+    """Config with no matching parameter"""
+
+    x: Param[int]
+
+
+class CompositionE(Config):
+    """Config with two parameters, one subclass of the other"""
+
+    base: Param[CompositionA]
+    sub: Param[CompositionSubA]
+
+
+def test_composition_operator():
+    """Test that B() @ A(x=1) is equivalent to B(a=A(x=1))"""
+    a = CompositionA.C(x=42)
+    b = CompositionB.C() @ a
+
+    assert b.a is a
+    assert b.a.x == 42
+
+
+def test_composition_operator_chained():
+    """Test chaining composition operators
+
+    Chaining A @ B @ C adds both B and C to A (same outer config).
+    For nested structures, use parentheses: A @ (B @ C)
+    """
+
+    class MultiParam(Config):
+        a: Param[CompositionA]
+        b: Param[CompositionB]
+
+    # Chaining adds multiple configs to same outer config
+    result = MultiParam.C() @ CompositionA.C(x=10) @ CompositionB.C()
+
+    assert result.a.x == 10
+    assert result.b is not None
+
+
+def test_composition_operator_nested():
+    """Test nested composition with parentheses"""
+
+    class Outer(Config):
+        b: Param[CompositionB]
+
+    # For nested structures, use parentheses
+    result = Outer.C() @ (CompositionB.C() @ CompositionA.C(x=10))
+
+    assert result.b.a.x == 10
+
+
+def test_composition_operator_ambiguous():
+    """Test that ambiguous composition raises ValueError"""
+    a = CompositionA.C(x=1)
+
+    with pytest.raises(ValueError, match="Ambiguous"):
+        CompositionC.C() @ a
+
+
+def test_composition_operator_no_match():
+    """Test that composition with no matching param raises ValueError"""
+    a = CompositionA.C(x=1)
+
+    with pytest.raises(ValueError, match="No parameter"):
+        CompositionD.C() @ a
+
+
+def test_composition_operator_subclass():
+    """Test composition works with subclasses"""
+    sub_a = CompositionSubA.C(x=5, y=10)
+    b = CompositionB.C() @ sub_a
+
+    assert b.a is sub_a
+    assert b.a.x == 5
+
+
+def test_composition_operator_subclass_hierarchy():
+    """Test composition when two params have subclass relationship
+
+    When CompositionSubA is passed, both 'base' (CompositionA) and 'sub'
+    (CompositionSubA) match. This should be ambiguous since both accept it.
+    """
+    sub_a = CompositionSubA.C(x=1, y=2)
+
+    # SubA matches both base (CompositionA) and sub (CompositionSubA)
+    with pytest.raises(ValueError, match="Ambiguous"):
+        CompositionE.C() @ sub_a
+
+
+def test_composition_operator_exact_match():
+    """Test composition when base class instance matches only base param"""
+    # CompositionA matches only 'base', not 'sub' (which requires SubA)
+    a = CompositionA.C(x=1)
+    e = CompositionE.C() @ a
+
+    assert e.base is a
+    assert e.base.x == 1
+
+
+# --- Value class decorator tests (GH #99) ---
+
+# Test 1: Basic value class registration
+
+
+class ValueBasicModel(Config):
+    x: Param[int] = field(ignore_default=1)
+
+
+@ValueBasicModel.value_class()
+class ValueBasicModelImpl(ValueBasicModel):
+    def compute(self):
+        return self.x * 2
+
+
+# Test 2: Subclass without explicit value class
+
+
+class ValueInheritBase(Config):
+    x: Param[int] = field(ignore_default=1)
+
+
+@ValueInheritBase.value_class()
+class ValueInheritBaseImpl(ValueInheritBase):
+    pass
+
+
+class ValueInheritSubNoExplicit(ValueInheritBase):
+    """Subclass without explicit value class"""
+
+    y: Param[int] = field(ignore_default=2)
+
+
+# Test 3: Value class with proper inheritance
+
+
+class ValueInheritParent(Config):
+    x: Param[int] = field(ignore_default=1)
+
+
+@ValueInheritParent.value_class()
+class ValueInheritParentImpl(ValueInheritParent):
+    def compute(self):
+        return self.x * 2
+
+
+class ValueInheritChild(ValueInheritParent):
+    y: Param[int] = field(ignore_default=2)
+
+
+@ValueInheritChild.value_class()
+class ValueInheritChildImpl(ValueInheritChild, ValueInheritParentImpl):
+    def compute_both(self):
+        return self.x + self.y
+
+
+# Test 4: Skip intermediate class (A -> B -> C, only A and C have value classes)
+
+
+class ValueSkipBase(Config):
+    x: Param[int] = field(ignore_default=1)
+
+
+@ValueSkipBase.value_class()
+class ValueSkipBaseImpl(ValueSkipBase):
+    def compute(self):
+        return self.x * 2
+
+
+class ValueSkipIntermediate(ValueSkipBase):
+    """Intermediate class without explicit value class"""
+
+    y: Param[int] = field(ignore_default=2)
+
+
+class ValueSkipDeep(ValueSkipIntermediate):
+    """Deep subclass with value class"""
+
+    z: Param[int] = field(ignore_default=3)
+
+
+@ValueSkipDeep.value_class()
+class ValueSkipDeepImpl(ValueSkipDeep, ValueSkipBaseImpl):
+    def compute_all(self):
+        return self.x + self.y + self.z
+
+
+# --- Value class tests ---
+
+
+def test_value_decorator_basic():
+    """Test basic value class registration"""
+    # XPMValue should return the registered value class
+    assert ValueBasicModel.XPMValue is ValueBasicModelImpl
+
+    # Creating an instance should use the value class
+    config = ValueBasicModel.C(x=5)
+    instance = config.instance()
+
+    assert isinstance(instance, ValueBasicModelImpl)
+    assert instance.x == 5
+    assert instance.compute() == 10
+
+
+def test_value_decorator_inheritance_no_explicit():
+    """Test that subclass without value class uses config class as value"""
+    # SubModel has no explicit value class, XPMValue returns the config class
+    assert ValueInheritSubNoExplicit.XPMValue is ValueInheritSubNoExplicit
+
+    config = ValueInheritSubNoExplicit.C(x=3, y=4)
+    instance = config.instance()
+
+    # Instance is created from the config class (no explicit value type)
+    assert isinstance(instance, ValueInheritSubNoExplicit)
+    assert instance.x == 3
+    assert instance.y == 4
+
+
+def test_value_decorator_inheritance_with_explicit():
+    """Test value class with proper inheritance from parent value class"""
+    assert ValueInheritChild.XPMValue is ValueInheritChildImpl
+
+    config = ValueInheritChild.C(x=3, y=4)
+    instance = config.instance()
+
+    assert isinstance(instance, ValueInheritChildImpl)
+    assert isinstance(instance, ValueInheritParentImpl)
+    assert instance.x == 3
+    assert instance.y == 4
+    assert instance.compute() == 6  # From parent value class
+    assert instance.compute_both() == 7  # From this value class
+
+
+def test_value_decorator_must_be_subclass():
+    """Test that value class must be subclass of config"""
+
+    class LocalModel(Config):
+        x: Param[int]
+
+    class OtherConfig(Config):
+        z: Param[int]
+
+    with pytest.raises(TypeError, match="must be a subclass of"):
+
+        @LocalModel.value_class()
+        class InvalidValue(OtherConfig):  # Not a subclass of LocalModel
+            pass
+
+
+def test_value_decorator_must_inherit_parent_value():
+    """Test that value class must inherit from parent value class"""
+
+    class LocalBase(Config):
+        x: Param[int] = field(ignore_default=1)
+
+    @LocalBase.value_class()
+    class LocalBaseImpl(LocalBase):
+        pass
+
+    class LocalChild(LocalBase):
+        y: Param[int] = field(ignore_default=2)
+
+    with pytest.raises(TypeError, match="must be a subclass of.*parent value class"):
+
+        @LocalChild.value_class()
+        class InvalidChildValue(LocalChild):  # Missing LocalBaseImpl inheritance
+            pass
+
+
+def test_value_decorator_skip_intermediate():
+    """Test value class when intermediate class has no value class"""
+    # ValueSkipBase has impl, ValueSkipIntermediate has none, ValueSkipDeep has impl
+    assert ValueSkipDeep.XPMValue is ValueSkipDeepImpl
+
+    config = ValueSkipDeep.C(x=1, y=2, z=3)
+    instance = config.instance()
+
+    assert isinstance(instance, ValueSkipDeepImpl)
+    assert isinstance(instance, ValueSkipBaseImpl)
+    assert instance.compute() == 2  # From ValueSkipBaseImpl
+    assert instance.compute_all() == 6  # From ValueSkipDeepImpl

experimaestro/tests/test_outputs.py

@@ -1,11 +1,11 @@
 """Test for task outputs"""
 
-from experimaestro import Config, Task, Param
+from experimaestro import field, Config, Task, Param
 from experimaestro.scheduler.workspace import RunMode
 
 
 class B(Config):
-    x: Param[int] = 1
+    x: Param[int] = field(ignore_default=1)
 
 
 class A(Config):

experimaestro/tests/test_param.py

@@ -5,7 +5,6 @@ Test annotation handling for configurations and tasks
 
 # Annotation specific tests
 
-import sys
 from pathlib import Path
 from typing import Dict, Optional, List
 from experimaestro.core.context import SerializationContext
@@ -17,7 +16,6 @@ from experimaestro import (
     Constant,
     Param,
     Task,
-    default,
     Meta,
     Config,
     pathgenerator,
@@ -80,8 +78,8 @@ def test_type_hinting():
         __xpmid__ = "annotations.class_variable.config"
 
         x: Param[int]
-        y: Param[float] = 2.3
-        y2: Annotated[float, default(2.3)]
+        y: Param[float] = field(ignore_default=2.3)
+        y2: Param[float] = field(ignore_default=2.3)
         z: Param[Optional[float]]
         t: Param[List[float]]
         w: Param[int]
@@ -180,7 +178,7 @@ def test_config_class():
 
 def test_constant():
     class A(Config):
-        x: Constant[int] = 2
+        x: Constant[int] = field(ignore_default=2)
 
     a = A.C()
     assert a.x == 2, "Constant value not set"
@@ -214,7 +212,7 @@ def test_inheritance():
         x: Param[int]
 
     class B(A):
-        y: Param[int] = 3
+        y: Param[int] = field(ignore_default=3)
 
     b = B.C()
     b.x = 2
@@ -227,7 +225,7 @@ def test_redefined_param():
         x: Param[int]
 
     class B(Config):
-        x: Param[int] = 3
+        x: Param[int] = field(ignore_default=3)
 
     atx = A.C.__getxpmtype__().getArgument("x")
     btx = B.C.__getxpmtype__().getArgument("x")
@@ -284,7 +282,7 @@ def test_default_mismatch():
     """Test mismatch between default and type"""
 
     class A(Config):
-        x: Param[int] = 0.2
+        x: Param[int] = field(ignore_default=0.2)
 
     with pytest.raises(TypeError):
         A.__getxpmtype__().getArgument("x")
@@ -297,7 +295,7 @@ def test_param_default_set():
     """Test that the default setting is well set"""
 
     class A0(Config):
-        x: Param[int] = 2
+        x: Param[int] = field(ignore_default=2)
 
     assert A0.C().instance().x == 2
     assert A0.C(x=3).instance().x == 3
@@ -336,6 +334,4 @@ def test_help():
     assert xpmtype.description.strip() == "Long description of A."
     assert xpmtype.arguments["y"].help == "Parameter y"
 
-    # Only python >= 3.9
-    if sys.version_info.major == 3 and sys.version_info.minor > 8:
-        assert xpmtype.arguments["x"].help == "Parameter x"
+    assert xpmtype.arguments["x"].help == "Parameter x"

experimaestro/tests/test_partial_paths.py

@@ -0,0 +1,231 @@
+"""Integration tests for partial paths and cleanup"""
+
+from pathlib import Path
+from experimaestro import (
+    Task,
+    Param,
+    Meta,
+    field,
+    PathGenerator,
+    subparameters,
+    param_group,
+)
+from experimaestro.scheduler import JobState
+
+from .utils import TemporaryExperiment, TemporaryDirectory
+
+
+# Define parameter groups
+iter_group = param_group("iter")
+
+
+class TaskWithPartial(Task):
+    """Task that uses subparameters for partial paths"""
+
+    # Define a subparameters set
+    checkpoints = subparameters(exclude_groups=[iter_group])
+
+    # Parameter in iter_group - excluded from partial identifier
+    max_iter: Param[int] = field(groups=[iter_group])
+
+    # Parameter not in any group - included in partial identifier
+    learning_rate: Param[float]
+
+    # Path generated using the partial identifier
+    checkpoint_path: Meta[Path] = field(
+        default_factory=PathGenerator("checkpoint", partial=checkpoints)
+    )
+
+    def execute(self):
+        # Create the checkpoint directory and a marker file
+        self.checkpoint_path.mkdir(parents=True, exist_ok=True)
+        (self.checkpoint_path / "model.pt").write_text("checkpoint data")
+
+
+def test_partial_path_created():
+    """Test that partial paths are correctly created during task execution"""
+    with TemporaryDirectory(prefix="xpm", suffix="partial") as workdir:
+        with TemporaryExperiment("partial_test", workdir=workdir, maxwait=30):
+            task = TaskWithPartial.C(max_iter=100, learning_rate=0.1).submit()
+
+        assert task.__xpm__.job.state == JobState.DONE
+
+        # Verify the partial path was created
+        assert task.checkpoint_path.exists()
+        assert (task.checkpoint_path / "model.pt").exists()
+
+        # Verify the path is in the partials directory
+        partials_path = workdir / "partials"
+        assert partials_path.exists()
+
+        # The checkpoint_path should be under partials/TASK_ID/checkpoints/PARTIAL_ID/
+        # Use resolve() to handle symlinks like /var -> /private/var on macOS
+        assert task.checkpoint_path.resolve().is_relative_to(partials_path.resolve())
+
+
+def test_partial_path_shared_across_tasks():
+    """Test that tasks with same non-excluded params share partial paths"""
+    with TemporaryDirectory(prefix="xpm", suffix="partial_shared") as workdir:
+        with TemporaryExperiment("partial_shared", workdir=workdir, maxwait=30):
+            # Submit two tasks with different max_iter but same learning_rate
+            task1 = TaskWithPartial.C(max_iter=100, learning_rate=0.1).submit()
+            task2 = TaskWithPartial.C(max_iter=200, learning_rate=0.1).submit()
+
+        assert task1.__xpm__.job.state == JobState.DONE
+        assert task2.__xpm__.job.state == JobState.DONE
+
+        # They should share the same partial path
+        assert task1.checkpoint_path == task2.checkpoint_path
+
+
+def test_partial_path_different_for_different_params():
+    """Test that tasks with different non-excluded params have different partial paths"""
+    with TemporaryDirectory(prefix="xpm", suffix="partial_diff") as workdir:
+        with TemporaryExperiment("partial_diff", workdir=workdir, maxwait=30):
+            # Submit two tasks with different learning_rate
+            task1 = TaskWithPartial.C(max_iter=100, learning_rate=0.1).submit()
+            task2 = TaskWithPartial.C(max_iter=100, learning_rate=0.2).submit()
+
+        assert task1.__xpm__.job.state == JobState.DONE
+        assert task2.__xpm__.job.state == JobState.DONE
+
+        # They should have different partial paths
+        assert task1.checkpoint_path != task2.checkpoint_path
+
+
+def test_partial_registered_in_database():
+    """Test that partials are registered in the database when jobs are submitted"""
+    from experimaestro.scheduler.state_provider import WorkspaceStateProvider
+    from experimaestro.scheduler.state_db import PartialModel, JobPartialModel
+
+    with TemporaryDirectory(prefix="xpm", suffix="partial_db") as workdir:
+        with TemporaryExperiment("partial_db", workdir=workdir, maxwait=30) as xp:
+            task = TaskWithPartial.C(max_iter=100, learning_rate=0.1).submit()
+
+        assert task.__xpm__.job.state == JobState.DONE
+
+        # Get the state provider and check database
+        # Note: Must use read_only=False since the experiment left a singleton
+        # with read_only=False that hasn't been closed yet
+        provider = WorkspaceStateProvider.get_instance(workdir, read_only=False)
+
+        try:
+            with provider.workspace_db.bind_ctx([PartialModel, JobPartialModel]):
+                # Check that partial is registered
+                partials = list(PartialModel.select())
+                assert len(partials) == 1
+                assert partials[0].subparameters_name == "checkpoints"
+
+                # Check that job is linked to partial
+                job_partials = list(JobPartialModel.select())
+                assert len(job_partials) == 1
+                assert job_partials[0].partial_id == partials[0].partial_id
+                assert job_partials[0].experiment_id == xp.workdir.name
+        finally:
+            provider.close()
+
+
+def test_orphan_partial_cleanup():
+    """Test that orphan partials are cleaned up when jobs are deleted"""
+    from experimaestro.scheduler.state_provider import WorkspaceStateProvider
+    from experimaestro.scheduler.state_db import PartialModel, JobPartialModel
+
+    with TemporaryDirectory(prefix="xpm", suffix="partial_cleanup") as workdir:
+        with TemporaryExperiment("partial_cleanup", workdir=workdir, maxwait=30) as xp:
+            task = TaskWithPartial.C(max_iter=100, learning_rate=0.1).submit()
+
+        assert task.__xpm__.job.state == JobState.DONE
+        checkpoint_path = task.checkpoint_path
+
+        # Verify partial path exists
+        assert checkpoint_path.exists()
+
+        # Get the state provider
+        provider = WorkspaceStateProvider.get_instance(workdir, read_only=False)
+
+        try:
+            # Delete the job
+            with provider.workspace_db.bind_ctx([PartialModel, JobPartialModel]):
+                job_partials = list(JobPartialModel.select())
+                assert len(job_partials) == 1
+
+            # Delete job (this also removes job-partial link)
+            provider.delete_job(
+                task.__xpm__.job.identifier,
+                xp.workdir.name,
+                xp.run_id,
+            )
+
+            # Now the partial should be orphaned
+            orphans = provider.get_orphan_partials()
+            assert len(orphans) == 1
+
+            # Cleanup orphan partials
+            deleted = provider.cleanup_orphan_partials(perform=True)
+            assert len(deleted) == 1
+
+            # Verify partial directory is deleted
+            assert not checkpoint_path.exists()
+
+            # Verify partial is removed from database
+            with provider.workspace_db.bind_ctx([PartialModel]):
+                partials = list(PartialModel.select())
+                assert len(partials) == 0
+        finally:
+            provider.close()
+
+
+def test_shared_partial_not_orphaned():
+    """Test that partials shared by multiple jobs are not orphaned until all jobs deleted"""
+    from experimaestro.scheduler.state_provider import WorkspaceStateProvider
+
+    with TemporaryDirectory(prefix="xpm", suffix="partial_shared_cleanup") as workdir:
+        with TemporaryExperiment(
+            "partial_shared_cleanup", workdir=workdir, maxwait=30
+        ) as xp:
+            # Submit two tasks with same learning_rate (same partial)
+            task1 = TaskWithPartial.C(max_iter=100, learning_rate=0.1).submit()
+            task2 = TaskWithPartial.C(max_iter=200, learning_rate=0.1).submit()
+
+        assert task1.__xpm__.job.state == JobState.DONE
+        assert task2.__xpm__.job.state == JobState.DONE
+
+        # They share the same partial path
+        checkpoint_path = task1.checkpoint_path
+        assert checkpoint_path == task2.checkpoint_path
+        assert checkpoint_path.exists()
+
+        provider = WorkspaceStateProvider.get_instance(workdir, read_only=False)
+
+        try:
+            # Delete first job
+            provider.delete_job(
+                task1.__xpm__.job.identifier,
+                xp.workdir.name,
+                xp.run_id,
+            )
+
+            # Partial should NOT be orphaned (still used by task2)
+            orphans = provider.get_orphan_partials()
+            assert len(orphans) == 0
+
+            # Partial directory should still exist
+            assert checkpoint_path.exists()
+
+            # Delete second job
+            provider.delete_job(
+                task2.__xpm__.job.identifier,
+                xp.workdir.name,
+                xp.run_id,
+            )
+
+            # Now partial should be orphaned
+            orphans = provider.get_orphan_partials()
+            assert len(orphans) == 1
+
+            # Cleanup
+            deleted = provider.cleanup_orphan_partials(perform=True)
+            assert len(deleted) == 1
+            assert not checkpoint_path.exists()
+        finally:
+            provider.close()