experimaestro 1.8.0rc5__py3-none-any.whl → 1.8.0rc7__py3-none-any.whl

experimaestro/core/objects/config_utils.py

@@ -0,0 +1,58 @@
+from contextlib import contextmanager
+from typing import Any, Dict, Set
+
+
+def getqualattr(module, qualname):
+    """Get a qualified attributed value"""
+    cls = module
+    for part in qualname.split("."):
+        cls = getattr(cls, part)
+    return cls
+
+
+@contextmanager
+def add_to_path(p):
+    """Temporarily add a path to sys.path"""
+    import sys
+
+    old_path = sys.path
+    sys.path = sys.path[:]
+    sys.path.insert(0, p)
+    try:
+        yield
+    finally:
+        sys.path = old_path
+
+
+class ObjectStore:
+    def __init__(self):
+        self.store: Dict[int, Any] = {}
+        self.constructed: Set[int] = set()
+
+    def set_constructed(self, identifier: int):
+        self.constructed.add(identifier)
+
+    def is_constructed(self, identifier: int):
+        return identifier in self.constructed
+
+    def retrieve(self, identifier: int):
+        return self.store.get(identifier, None)
+
+    def add_stub(self, identifier: int, stub: Any):
+        self.store[identifier] = stub
+
+
+class SealedError(Exception):
+    """Exception when trying to modify a sealed configuration"""
+
+    pass
+
+
+class TaggedValue:
+    def __init__(self, value):
+        self.value = value
+
+
+class classproperty(property):
+    def __get__(self, owner_self, owner_cls):
+        return self.fget(owner_cls)

experimaestro/core/objects/config_walk.py

@@ -0,0 +1,150 @@
+from pathlib import Path
+from enum import Enum
+from typing import (
+    Any,
+    Dict,
+    Tuple,
+)
+from contextlib import contextmanager
+
+
+class ConfigWalkContext:
+    """Context when generating values in configurations"""
+
+    @property
+    def path(self):
+        """Returns the path of the job directory"""
+        raise NotImplementedError()
+
+    def __init__(self):
+        self._configpath = None
+
+    @property
+    def task(self):
+        return None
+
+    def currentpath(self) -> Path:
+        """Returns the configuration folder"""
+        if self._configpath:
+            return self.path / self._configpath
+        return self.path
+
+    @contextmanager
+    def push(self, key: str):
+        """Push a new key to contextualize paths"""
+        p = self._configpath
+        try:
+            self._configpath = (Path("out") if p is None else p) / key
+            yield key
+        finally:
+            self._configpath = p
+
+
+class ConfigWalk:
+    """Allows to perform an operation on all nested configurations"""
+
+    def __init__(self, context: ConfigWalkContext = None, recurse_task=False):
+        """
+
+        :param recurse_task: Recurse into linked tasks
+        :param context: The context, by default only tracks the position in the
+            config tree
+        """
+        self.recurse_task = recurse_task
+        self.context = ConfigWalkContext() if context is None else context
+
+        # Stores already visited nodes
+        self.visited = {}
+
+    def preprocess(self, config) -> Tuple[bool, Any]:
+        """Returns a tuple boolean/value
+
+        The boolean value is used to stop the processing if False.
+        The value is returned
+        """
+        return True, None
+
+    def postprocess(self, stub, config, values: Dict[str, Any]):
+        return stub
+
+    def list(self, i: int):
+        return self.context.push(str(i))
+
+    def map(self, k: str):
+        return self.context.push(k)
+
+    def stub(self, config):
+        return config
+
+    def __call__(self, x):
+        from experimaestro.core.objects import Config
+        from experimaestro.core.objects import ConfigInformation  # noqa: F401
+
+        if isinstance(x, Config):
+            info = x.__xpm__  # type: ConfigInformation
+
+            # Avoid loops
+            xid = id(x)
+            if xid in self.visited:
+                return self.visited[xid]
+
+            # Get a stub
+            stub = self.stub(x)
+            self.visited[xid] = stub
+
+            # Pre-process
+            flag, value = self.preprocess(x)
+
+            if not flag:
+                # Stop processing and returns value
+                return value
+
+            # Process all the arguments
+            result = {}
+            for arg, v in info.xpmvalues():
+                if v is not None:
+                    with self.map(arg.name):
+                        result[arg.name] = self(v)
+                else:
+                    result[arg.name] = None
+
+            # Deals with pre-tasks
+            if info.pre_tasks:
+                with self.map("__pre_tasks__"):
+                    self(info.pre_tasks)
+
+            if info.init_tasks:
+                with self.map("__init_tasks__"):
+                    self(info.init_tasks)
+
+            # Process task if different
+            if (
+                x.__xpm__.task is not None
+                and self.recurse_task
+                and x.__xpm__.task is not x
+            ):
+                self(x.__xpm__.task)
+
+            processed = self.postprocess(stub, x, result)
+            self.visited[xid] = processed
+            return processed
+
+        if isinstance(x, list):
+            result = []
+            for i, sv in enumerate(x):
+                with self.list(i):
+                    result.append(self(sv))
+            return result
+
+        if isinstance(x, dict):
+            result = {}
+            for key, value in x.items():
+                assert isinstance(key, (str, float, int))
+                with self.map(key):
+                    result[key] = self(value)
+            return result
+
+        if isinstance(x, (float, int, str, Path, Enum)):
+            return x
+
+        raise NotImplementedError(f"Cannot handle a value of type {type(x)}")

experimaestro/core/objects.pyi

@@ -41,40 +41,8 @@ from typing_extensions import Self
 
 TConfig = TypeVar("TConfig", bound="Config")
 
-class Identifier:
-    main: Incomplete
-    sub: Incomplete
-    def __init__(self, main: bytes, sub: Optional[bytes] = ...) -> None: ...
-    def all(self): ...
-    def state_dict(self): ...
-    @staticmethod
-    def from_state_dict(data: Union[Dict[str, str], str]): ...
-
-def is_ignored(value): ...
-def remove_meta(value): ...
-
 class ObjectStore: ...
 
-class HashComputer:
-    OBJECT_ID: bytes
-    INT_ID: bytes
-    FLOAT_ID: bytes
-    STR_ID: bytes
-    PATH_ID: bytes
-    NAME_ID: bytes
-    NONE_ID: bytes
-    LIST_ID: bytes
-    TASK_ID: bytes
-    DICT_ID: bytes
-    ENUM_ID: bytes
-    def __init__(self) -> None: ...
-    def identifier(self) -> Identifier: ...
-    def update(self, value, myself: bool = ...): ...
-
-def updatedependencies(
-    dependencies, value: Config, path: List[str], taskids: Set[int]
-): ...
-
 class TaggedValue:
     value: Incomplete
     def __init__(self, value) -> None: ...
@@ -122,7 +90,7 @@ class ConfigInformation:
     job: Job
     dependencies: Incomplete
     watched_outputs: List[WatchedOutput]
-    def __init__(self, pyobject: TypeConfig) -> None: ...
+    def __init__(self, pyobject: ConfigMixin) -> None: ...
     def set_meta(self, value: Optional[bool]): ...
     @property
     def meta(self): ...
@@ -162,7 +130,7 @@ class ConfigInformation:
         definitions: List[Dict],
         as_instance: bool = ...,
         save_directory: Optional[Path] = ...,
-    ) -> TypeConfig: ...
+    ) -> ConfigMixin: ...
     @overload
     @staticmethod
     def fromParameters(
@@ -183,7 +151,7 @@ class ConfigInformation:
 def clone(v): ...
 def cache(fn, name: str): ...
 
-class TypeConfig:
+class ConfigMixin:
     __xpmtype__: ObjectType
     __xpm__: Incomplete
     def __init__(self, **kwargs) -> None: ...
@@ -217,8 +185,8 @@ class Config:
     __use_xpmobject__: ClassVar[bool]
 
     XPMValue: Type[Self]
-    XPMConfig: Union[Type[Self], Type[TypeConfig[Self]]]
-    C: Union[Type[Self], Type[TypeConfig[Self]]]
+    XPMConfig: Union[Type[Self], Type[ConfigMixin[Self]]]
+    C: Union[Type[Self], Type[ConfigMixin[Self]]]
 
     @classmethod
     def __getxpmtype__(cls) -> ObjectType: ...
@@ -252,6 +220,6 @@ class Task(LightweightTask):
 def copyconfig(config_or_output: TConfig, **kwargs) -> TConfig: ...
 def setmeta(config: TConfig, flag: bool) -> TConfig: ...
 
-class TypeConfig(Generic[T]):
+class ConfigMixin(Generic[T]):
     def __validate__(self):
         pass

experimaestro/core/serialization.py

@@ -17,7 +17,7 @@ def json_object(context: SerializationContext, value: Any, objects=[]):
 
     if isinstance(value, Config):
         value.__xpm__.__get_objects__(objects, context)
-    elif isinstance(value, list):
+    elif isinstance(value, (list, tuple)):
         for el in value:
             ConfigInformation.__collect_objects__(el, objects, context)
     elif isinstance(value, dict):

experimaestro/core/types.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 import inspect
 import sys
-from typing import Set, Union, Dict, Iterator, List, get_args, get_origin
+from typing import Set, TypeVar, Union, Dict, Iterator, List, get_args, get_origin
 from collections import ChainMap
 from pathlib import Path
 import typing
@@ -130,10 +130,13 @@ class Type:
         if union_t := typingutils.get_union(key):
             return UnionType([Type.fromType(t) for t in union_t])
 
-        # Takes care of generics
+        # Takes care of generics, like List[int], not List
         if get_origin(key):
             return GenericType(key)
 
+        if isinstance(key, TypeVar):
+            return TypeVarType(key)
+
         raise Exception("No type found for %s", key)
 
 
@@ -214,7 +217,7 @@ class ObjectType(Type):
         identifier: Union[str, Identifier] = None,
     ):
         """Creates a type"""
-        from .objects import Config, TypeConfig
+        from .objects import Config, ConfigMixin
 
         # Task related attributes
         self.taskcommandfactory = None
@@ -268,7 +271,7 @@ class ObjectType(Type):
             s.__getxpmtype__().configtype
             for s in tp.__bases__
             if issubclass(s, Config) and (s is not Config)
-        ) or (TypeConfig,)
+        ) or (ConfigMixin,)
 
         *tp_qual, tp_name = self.basetype.__qualname__.split(".")
         self.configtype = type(
@@ -597,6 +600,23 @@ class AnyType(Type):
         return value
 
 
+class TypeVarType(Type):
+    def __init__(self, typevar: TypeVar):
+        self.typevar = typevar
+
+    def name(self):
+        return str(self.typevar)
+
+    def validate(self, value):
+        return value
+
+    def __str__(self):
+        return f"TypeVar({self.typevar})"
+
+    def __repr__(self):
+        return f"TypeVar({self.typevar})"
+
+
 Any = AnyType()
 
 
@@ -698,6 +718,10 @@ class GenericType(Type):
     def __repr__(self):
         return repr(self.type)
 
+    def identifier(self):
+        """Returns the identifier of the type"""
+        return Identifier(f"{self.origin}.{self.type}")
+
     def validate(self, value):
         # Now, let's check generics...
         mros = typingutils.generic_mro(type(value))

experimaestro/notifications.py

@@ -109,6 +109,7 @@ class Reporter(threading.Thread):
         return any(level.modified(self) for level in self.levels)
 
     def check_urls(self):
+        """Check whether we have new schedulers to notify"""
         mtime = os.path.getmtime(self.path)
         if mtime > self.lastcheck:
             for f in self.path.iterdir():

experimaestro/scheduler/base.py

@@ -515,6 +515,7 @@ class Scheduler:
             return other
 
         job._future = asyncio.run_coroutine_threadsafe(self.aio_submit(job), self.loop)
+        return None
 
     def prepare(self, job: Job):
         """Prepares the job for running"""

experimaestro/tests/core/test_generics.py

@@ -0,0 +1,206 @@
+"""Tests for the use of generics in configurations"""
+
+from typing import Generic, Optional, TypeVar
+
+import pytest
+from experimaestro import Config, Param
+from experimaestro.core.arguments import Argument
+from experimaestro.core.types import TypeVarType
+
+T = TypeVar("T")
+
+
+class SimpleConfig(Config):
+    pass
+
+
+class SimpleConfigChild(SimpleConfig):
+    pass
+
+
+class SimpleGenericConfig(Config, Generic[T]):
+    x: Param[T]
+
+
+class SimpleGenericConfigChild(SimpleGenericConfig, Generic[T]):
+    """A child class of SimpleGenericConfig that also uses generics"""
+
+    pass
+
+
+def test_core_generics_typevar():
+    a = SimpleGenericConfig.C(x=1)
+
+    x_arg = a.__xpmtype__.arguments["x"]
+
+    # Check correct interpretation of typevar
+    assert type(x_arg) is Argument
+    assert isinstance(x_arg.type, TypeVarType)
+    assert x_arg.type.typevar == T
+
+    assert isinstance(a.x, int)
+
+
+def test_core_generics_simple():
+    a = SimpleGenericConfig.C(x=2)
+
+    # OK
+    a.x = 3
+
+    # Fails: changing generics is not allowed
+    with pytest.raises(TypeError):
+        a.x = "a string"
+
+    # typevar bindings are local to the instance,
+    # so we can create a new instance with a different type
+    SimpleGenericConfig.C(x="a string")
+
+
+class DoubleGenericConfig(Config, Generic[T]):
+    x: Param[T]
+    y: Param[T]
+
+
+def test_core_generics_double():
+    # OK
+    DoubleGenericConfig.C(x=1, y=1)
+
+    # Fails
+    with pytest.raises(TypeError):
+        DoubleGenericConfig.C(x=1, y="a")
+
+    a = DoubleGenericConfig.C(x=1, y=1)
+    a.y = 2
+    with pytest.raises(TypeError):
+        a.x = "b"
+
+
+def test_core_generics_double_rebind():
+    a = DoubleGenericConfig.C(x=1, y=1)
+    # Rebinding to a different type should not work
+    with pytest.raises(TypeError):
+        a.x, a.y = "some", "string"
+
+
+def test_core_generics_double_plus():
+    # Testing with inheritance
+    # We allow subclasses of the typevar binding
+    # We also allow generalizing up the typevar binding
+    # This means that we can use a super class of the typevar binding
+
+    # Works
+    a = DoubleGenericConfig.C(x=SimpleConfigChild.C())
+    a.y = SimpleConfig.C()
+
+    # Works also
+    b = DoubleGenericConfig.C(x=SimpleConfig.C())
+    b.y = SimpleConfigChild.C()
+
+    a.x = SimpleConfigChild.C()
+
+    with pytest.raises(TypeError):
+        a.x = "a string"
+
+
+def test_core_generics_double_type_escalation():
+    a = DoubleGenericConfig.C(x=SimpleConfigChild.C())
+    a.y = SimpleConfigChild.C()
+    # T is now bound to SimpleConfigChild
+
+    a.y = SimpleConfig.C()
+    # T is now bound to SimpleConfig
+
+    a.y = object()
+    # T is now bound to object, which is a super class of SimpleConfigChild
+
+    # This is allowed, since we are not changing the typevar binding
+    a.x = "a string"
+
+    a.y = dict()
+    # This is allowed, since we are not changing the typevar binding
+
+
+def test_core_generics_double_deep_bind():
+    # Since we are deep binding the typevar T to a specific type,
+    # we should not be able to have coherent *local-only* type bindings
+    # The type bindings are transient
+
+    with pytest.raises(TypeError):
+        DoubleGenericConfig.C(
+            x=DoubleGenericConfig.C(x=1, y=2), y=DoubleGenericConfig.C(x=3, y=4)
+        )
+
+
+class NestedConfig(Config, Generic[T]):
+    x: Param[DoubleGenericConfig[T]]
+    y: Param[SimpleGenericConfig[T]]
+
+
+def test_core_generics_nested():
+    # OK
+    NestedConfig.C(x=DoubleGenericConfig.C(x=1, y=1), y=SimpleGenericConfig.C(x=2))
+
+    # Not OK
+    with pytest.raises(TypeError):
+        NestedConfig.C(
+            x=DoubleGenericConfig.C(x=1, y=1), y=SimpleGenericConfig.C(x="b")
+        )
+
+    with pytest.raises(TypeError):
+        a = NestedConfig.C(
+            x=DoubleGenericConfig.C(x=1, y=1), y=SimpleGenericConfig.C(x=1)
+        )
+        a.x.x = "a string"
+
+
+class TreeGenericConfig(Config, Generic[T]):
+    x: Param[T]
+    left: Optional["TreeGenericConfig[T]"] = None
+    right: Optional["TreeGenericConfig[T]"] = None
+
+
+class TagTreeGenericConfig(TreeGenericConfig[T], Generic[T]):
+    """A tagged version of TreeGenericConfig to test recursive generics"""
+
+    tag: Param[str] = "default"
+
+
+def test_core_generics_recursive():
+    a = TreeGenericConfig.C(x=1)
+    a.left = TreeGenericConfig.C(x=2)
+    a.right = TreeGenericConfig.C(x=3)
+
+    with pytest.raises(TypeError):
+        a.left.x = "a string"
+
+    # OK to use a child class
+    a.left = TagTreeGenericConfig.C(x=4, tag="left")
+
+    with pytest.raises(TypeError):
+        a.left.x = "a string"
+
+
+def test_core_generics_recursive_child():
+    # Testing with a child class on the generic value
+    a = TreeGenericConfig.C(x=SimpleConfig.C())
+    a.left = TreeGenericConfig.C(x=SimpleConfig.C())
+    a.right = TreeGenericConfig.C(x=SimpleConfig.C())
+
+    a.left.x = SimpleConfigChild.C()
+
+    with pytest.raises(TypeError):
+        a.left.x = "a string"
+
+
+U = TypeVar("U", bound=SimpleConfigChild)
+
+
+class BoundGenericConfig(Config, Generic[U]):
+    x: Param[U]
+
+
+def test_core_generics_bound_typevar():
+    a = BoundGenericConfig.C(x=SimpleConfigChild.C())
+    assert isinstance(a.x, SimpleConfigChild)
+    with pytest.raises(TypeError):
+        a.x = SimpleConfig.C()

experimaestro/tests/restart.py

@@ -27,6 +27,8 @@ TERMINATES_FUNC = [terminate]
 if is_posix():
     TERMINATES_FUNC.append(sigint)
 
+MAX_RESTART_WAIT = 50  # 5 seconds
+
 
 class Restart(Task):
     touch: Meta[Path] = field(default_factory=PathGenerator("touch"))
@@ -81,7 +83,7 @@ def restart(terminate: Callable, experiment):
         while not task.touch.is_file():
             time.sleep(0.1)
             counter += 1
-            if counter >= 20:
+            if counter >= MAX_RESTART_WAIT:
                 terminate(xpmprocess)
                 assert False, "Timeout waiting for task to be executed"
 

experimaestro/tests/test_instance.py

@@ -1,6 +1,6 @@
 from typing import Optional
 from experimaestro import Param, Config
-from experimaestro.core.objects import TypeConfig
+from experimaestro.core.objects import ConfigMixin
 from experimaestro.core.serializers import SerializationLWTask
 
 
@@ -21,10 +21,10 @@ def test_simple_instance():
     b = B(a=a)
     b = b.instance()
 
-    assert not isinstance(b, TypeConfig)
+    assert not isinstance(b, ConfigMixin)
     assert isinstance(b, B.__xpmtype__.objecttype)
 
-    assert not isinstance(b.a, TypeConfig)
+    assert not isinstance(b.a, ConfigMixin)
     assert isinstance(b.a, A1.__xpmtype__.objecttype)
     assert isinstance(b.a, A.__xpmtype__.basetype)
 

experimaestro/tests/test_objects.py

@@ -1,9 +1,10 @@
+import logging
 from pathlib import Path
 
 import pytest
 from experimaestro import Config, Task, Annotated, copyconfig, default
 from experimaestro.core.arguments import Param
-from experimaestro.core.objects import TypeConfig
+from experimaestro.core.objects import ConfigMixin
 from experimaestro.core.types import XPMValue
 from experimaestro.generators import pathgenerator
 from experimaestro.scheduler.workspace import RunMode
@@ -65,9 +66,9 @@ def test_hierarchy():
     assert issubclass(B, Config)
     assert issubclass(C, Config)
 
-    assert not issubclass(OA, TypeConfig)
-    assert not issubclass(OB, TypeConfig)
-    assert not issubclass(OC, TypeConfig)
+    assert not issubclass(OA, ConfigMixin)
+    assert not issubclass(OB, ConfigMixin)
+    assert not issubclass(OC, ConfigMixin)
 
     assert issubclass(C, B)
 
@@ -91,3 +92,18 @@ def test_copyconfig(xp):
 
     assert copy_b.x == b.x
     assert "path" not in copy_b.__xpm__.values
+
+
+def test_direct_config_warns(caplog):
+    """Test that using a building Config directly raises a warning"""
+    message = "Config.__new__ is deprecated"
+
+    with caplog.at_level(logging.WARNING):
+        A(x=3)
+        assert message in caplog.text
+
+    caplog.clear()
+
+    with caplog.at_level(logging.WARNING):
+        A.C(x=3)
+        assert message not in caplog.text