experimaestro 1.6.1__py3-none-any.whl → 1.7.0rc0__py3-none-any.whl

experimaestro/tests/test_instance.py

@@ -1,21 +1,18 @@
 from typing import Optional
-from experimaestro import config, Param, Config
+from experimaestro import Param, Config
 from experimaestro.core.objects import TypeConfig
 from experimaestro.core.serializers import SerializationLWTask
 
 
-@config()
-class A:
+class A(Config):
     x: Param[int] = 1
 
 
-@config()
 class A1(A):
     pass
 
 
-@config()
-class B:
+class B(Config):
     a: Param[A]
 
 

experimaestro/tests/test_param.py

@@ -13,14 +13,16 @@ from experimaestro.core.types import DictType, IntType, StrType
 from enum import Enum
 import pytest
 from experimaestro import (
-    config,
     Option,
     Constant,
     Param,
     Task,
     default,
+    Meta,
     Config,
     pathgenerator,
+    PathGenerator,
+    field,
     Annotated,
 )
 import experimaestro.core.types as types
@@ -29,25 +31,24 @@ from experimaestro.xpmutils import DirectoryContext
 # --- Test manual name for configuration
 
 
-@config("annotations.b")
-class B:
+class B(Config):
+    __xpmid__ = "annotations.b"
     pass
 
 
 def test_fullname():
-    assert str(B.__xpmtype__.identifier) == "annotations.b"
+    assert str(B.__getxpmtype__().identifier) == "annotations.b"
 
 
 # --- Automatic name for configuration
 
 
-@config()
-class A:
+class A(Config):
     pass
 
 
 def test_noname():
-    assert str(A.__xpmtype__.identifier) == "experimaestro.tests.test_param.a"
+    assert str(A.__getxpmtype__().identifier) == "experimaestro.tests.test_param.a"
 
 
 def serializeCycle(config: Config):
@@ -69,8 +70,7 @@ def ArgumentValue(default=None, *, help=""):
 def test_type_hinting():
     """Test for type hinting"""
 
-    @config()
-    class MyConfig:
+    class MyConfig(Config):
         """A configuration
 
         Attributes:
@@ -89,7 +89,7 @@ def test_type_hinting():
         path: Annotated[Path, pathgenerator("world")]
         option: Option[str]
 
-    ot = MyConfig.__xpmtype__
+    ot = MyConfig.__getxpmtype__()
 
     # Check required parameter
     arg_x = ot.getArgument("x")
@@ -139,7 +139,7 @@ def test_type_hinting():
 
 def test_generatedpath():
     class A(Config):
-        path: Annotated[Path, pathgenerator("test.txt")]
+        path: Meta[Path] = field(default_factory=PathGenerator("test.txt"))
 
     class B(Config):
         a: Param[A]
@@ -179,8 +179,7 @@ def test_config_class():
 
 
 def test_constant():
-    @config()
-    class A:
+    class A(Config):
         x: Constant[int] = 2
 
     a = A()
@@ -224,16 +223,14 @@ def test_inheritance():
 
 
 def test_redefined_param():
-    @config()
-    class A:
+    class A(Config):
         x: Param[int]
 
-    @config()
-    class B:
+    class B(Config):
         x: Param[int] = 3
 
-    atx = A.C.__xpmtype__.getArgument("x")
-    btx = B.C.__xpmtype__.getArgument("x")
+    atx = A.C.__getxpmtype__().getArgument("x")
+    btx = B.C.__getxpmtype__().getArgument("x")
 
     assert atx.required
 
@@ -258,18 +255,39 @@ def test_param_dict():
         A(x={"wrong": 1.2})
 
 
+# --- Default
+
+
+class ConfigWithDefault(Config):
+    x: Param[int] = field(default=1)
+
+
+def test_param_default():
+    assert ConfigWithDefault().x == 1
+
+
+class ConfigWithDefaultFactory(Config):
+    x: Param[int] = field(default_factory=lambda: 1)
+
+
+def test_param_default_factory():
+    value = ConfigWithDefaultFactory()
+    context = DirectoryContext(Path("/__fakepath__"))
+    value.__xpm__.seal(context)
+    assert value.x == 1
+
+
 # --- Task annotations
 
 
 def test_default_mismatch():
     """Test mismatch between default and type"""
 
-    @config()
-    class A:
+    class A(Config):
         x: Param[int] = 0.2
 
     with pytest.raises(TypeError):
-        A.__xpmtype__.getArgument("x")
+        A.__getxpmtype__().getArgument("x")
 
 
 # --- Handling help annotations

experimaestro/tests/test_tags.py

@@ -3,8 +3,6 @@ from pathlib import Path
 from experimaestro import (
     tag,
     LightweightTask,
-    config,
-    argument,
     Config,
     Task,
     Param,
@@ -13,17 +11,13 @@ from experimaestro.scheduler.workspace import RunMode
 from experimaestro.xpmutils import DirectoryContext
 
 
-@argument("x", type=int)
-@config()
-class Config1:
-    pass
+class Config1(Config):
+    x: Param[int]
 
 
-@argument("x", type=int)
-@argument("c", type=Config1)
-@config()
-class Config2:
-    pass
+class Config2(Config):
+    x: Param[int]
+    c: Param[Config1]
 
 
 def test_tag():

experimaestro/tests/test_tokens.py

@@ -7,7 +7,7 @@ import time
 from pathlib import Path
 
 import subprocess
-from experimaestro import Task, param
+from experimaestro import Task, Param
 from experimaestro.tokens import CounterToken, TokenFile
 from experimaestro.scheduler import JobState
 from .utils import (
@@ -74,8 +74,9 @@ def test_token_ok():
     logging.info("Finished token_ok test")
 
 
-@param("x", type=int)
 class dummy_task(Task):
+    x: Param[int]
+
     def execute(self):
         pass
 

experimaestro/tests/test_types.py

@@ -1,31 +1,26 @@
 # --- Task and types definitions
 
 import logging
-from experimaestro import Config, config
-from experimaestro.core.objects import TypeConfig
+from experimaestro import Config, Param
+from typing import Union
 
-from .utils import TemporaryExperiment
-from experimaestro.scheduler import JobState
+import pytest
+from experimaestro.core.objects import TypeConfig
 
 
 def test_multiple_inheritance():
-    @config()
-    class A:
+    class A(Config):
         pass
 
-    @config()
-    class B:
+    class B(Config):
         pass
 
-    @config()
     class B1(B):
         pass
 
-    @config()
     class C1(B1, A):
         pass
 
-    @config()
     class C2(A, B1):
         pass
 
@@ -45,14 +40,12 @@ def test_multiple_inheritance():
 
 
 def test_missing_hierarchy():
-    @config()
-    class A:
+    class A(Config):
         pass
 
     class A1(A):
         pass
 
-    @config()
     class B(A1):
         pass
 
@@ -60,3 +53,13 @@ def test_missing_hierarchy():
 
     assert issubclass(B, A)
     assert issubclass(B, A1)
+
+
+def test_types_union():
+    class A(Config):
+        x: Param[Union[int, str]]
+
+    A(x=1)
+    A(x="hello")
+    with pytest.raises(ValueError):
+        A(x=[])

experimaestro/tests/test_validation.py

@@ -2,18 +2,9 @@
 
 import pytest
 from pathlib import Path
-from experimaestro import (
-    config,
-    Task,
-    Identifier,
-    argument,
-    pathoption,
-    ConstantParam,
-    Param,
-    Config,
-)
+from experimaestro import Task, field, Identifier, Constant, Param, Config, Meta
 from enum import Enum
-import experimaestro.core.types as types
+from experimaestro.generators import PathGenerator
 from experimaestro.scheduler import Job, JobContext
 from experimaestro.scheduler.workspace import RunMode
 from .utils import TemporaryExperiment
@@ -31,57 +22,50 @@ def expect_notvalidate(value):
         value.__xpm__.validate()
 
 
-@argument("value", type=int)
-@config()
-class A:
-    pass
+class A(Config):
+    value: Param[int]
 
 
-@argument("a", type=A)
-@config()
-class B:
-    pass
+class B(Config):
+    a: Param[A]
 
 
-@pathoption("path", "outdir")
-@config()
-class C:
+class C(Config):
+    path: Meta[Path] = field(default_factory=PathGenerator("outdir"))
     pass
 
 
-def test_simple():
+def test_validation_simple():
     expect_validate(A(value=1))
 
 
-def test_missing():
+def test_validation_missing():
     expect_notvalidate(A())
 
 
-def test_simple_nested():
+def test_validation_simple_nested():
     b = B()
     b.a = A(value=1)
     expect_validate(b)
 
 
-def test_missing_nested():
+def test_validation_missing_nested():
     b = B()
     b.a = A()
     expect_notvalidate(b)
 
 
-def test_type():
-    @config(valns.type.a)
-    class A:
+def test_validation_type():
+    class A(Config):
+        __xpmid__ = valns.type.a
         pass
 
-    @config(valns.type.b)
-    class B:
-        pass
+    class B(Config):
+        __xpmid__ = valns.type.b
 
-    @argument("a", A)
-    @config(valns.type.c)
-    class C:
-        pass
+    class C(Config):
+        a: Param[A]
+        __xpmid__ = valns.type.c
 
     with pytest.raises(ValueError):
         C(a=B())
@@ -91,30 +75,26 @@ def test_type():
         c.a = B()
 
 
-def test_subtype():
-    @config(valns.subtype.a)
-    class A:
-        pass
+def test_validation_subtype():
+    class A(Config):
+        __xpmid__ = valns.subtype.a
 
-    @config(valns.subtype.a1)
     class A1(A):
-        pass
+        __xpmid__ = valns.subtype.a1
 
-    @argument("a", A)
-    @config(valns.subtype.b)
-    class B:
-        pass
+    class B(Config):
+        __xpmid__ = valns.subtype.b
+        a: Param[A]
 
     expect_validate(B(a=A1()))
 
 
-def test_path():
-    """Test of @pathoption"""
+def test_validation_path_generator():
+    """Test of path generator"""
 
-    @pathoption("value", "file.txt")
-    @config(valns.path.a)
-    class A:
-        pass
+    class A(Config):
+        __xpmid__ = valns.path.a
+        value: Meta[Path] = field(default_factory=PathGenerator("file.txt"))
 
     a = A()
     a.__xpm__.validate()
@@ -129,13 +109,12 @@ def test_path():
         assert a.value.parents[3] == xp.workspace.path
 
 
-def test_constant():
-    """Test of @ConstantParam"""
+def test_validation_constant():
+    """Test of constant"""
 
-    @ConstantParam("value", 1)
-    @config(valns.constant.a)
-    class A:
-        pass
+    class A(Config):
+        __xpmid__ = valns.constant.a
+        value: Constant[int] = 1
 
     a = A()
     a.__xpm__.validate()
@@ -145,31 +124,26 @@ def test_constant():
         assert a.value == 1
 
 
-@argument("x", type=int)
-@config()
-class Parent:
-    pass
+class Parent(Config):
+    x: Param[int]
 
 
-@config()
 class Child(Parent):
     pass
 
 
-def test_child():
+def test_validation_child():
     expect_validate(Child(x=1))
 
 
 # --- Path argument checks
 
 
-@pathoption("x", "x")
-@config()
-class PathParent:
-    pass
+class PathParent(Config):
+    x: Meta[Path] = field(default_factory=PathGenerator("x"))
 
 
-def test_path_option():
+def test_validation_path_option():
     c = PathParent()
     expect_validate(c)
 
@@ -177,28 +151,11 @@ def test_path_option():
 # --- Default value
 
 
-@pytest.mark.parametrize(
-    "value,apitype",
-    [(1.5, types.FloatType), (1, types.IntType), (False, types.BoolType)],
-)
-def test_default(value, apitype):
-    @argument("default", default=value)
-    @config(valns.default[str(type(value))])
-    class Default:
-        pass
-
-    value = Default()
-    expect_validate(value)
-    assert Default.__xpmtype__.arguments["default"].type.__class__ == apitype
-
-
-def test_seal():
+def test_validation_seal():
     """Test value sealing"""
 
-    @argument("a", int)
-    @config()
-    class A:
-        pass
+    class A(Config):
+        a: Param[int]
 
     a = A(a=2)
     a.__xpm__.seal(EmptyContext())
@@ -207,7 +164,7 @@ def test_seal():
         a.a = 1
 
 
-def test_validation_enum():
+def test_validation_validation_enum():
     """Path arguments should be ignored"""
 
     class EnumParam(Enum):
@@ -241,7 +198,7 @@ class TaskConfigConsumer(Config):
     x: Param[TaskParentConfig]
 
 
-def test_taskargument():
+def test_validation_taskargument():
     x = taskconfig()
     with TemporaryExperiment("fake"):
         x.submit(run_mode=RunMode.DRY_RUN)

experimaestro/tokens.py

@@ -59,6 +59,8 @@ class CounterTokenLock(Lock):
 
 
 class CounterTokenDependency(Dependency):
+    """A dependency onto a token"""
+
     def __init__(self, token: "CounterToken", count: int):
         super().__init__(token)
         self._token = token
@@ -66,6 +68,7 @@ class CounterTokenDependency(Dependency):
 
     @property
     def name(self):
+        """The (file) name for this dependency, when taken"""
         return f"{self.target.identifier}.token"
 
     def status(self) -> DependencyStatus:
@@ -165,7 +168,7 @@ class CounterToken(Token, FileSystemEventHandler):
     - TIMESTAMP.token contains (1) the number of tokens (2) the job URI
     """
 
-    """Maps paths to instances"""
+    """Maps token keys to CounterToken instances"""
     TOKENS: Dict[str, "CounterToken"] = {}
 
     @staticmethod
@@ -193,7 +196,7 @@ class CounterToken(Token, FileSystemEventHandler):
         """[summary]
 
         Arguments:
-            path {Path} -- The file path of the token file
+            path {Path} -- The file path of the token directory
             count {int} -- Number of tokens (overrides previous definitions)
             force --   If the token has already been created, force to write the maximum
                        number of tokens
@@ -298,6 +301,9 @@ class CounterToken(Token, FileSystemEventHandler):
                         tokenfile = TokenFile(path)
                         tokenfile.watch()
                         self.cache[path.name] = tokenfile
+        except FileNotFoundError:
+            # We did not find the token file... just ignore
+            pass
         except Exception:
             logger.exception("Uncaught exception in on_modified handler")
             raise
@@ -336,13 +342,18 @@ class CounterToken(Token, FileSystemEventHandler):
                         for dependency in dependents:
                             dependency.check()
 
+            # A modified dependency not in cache
             elif path.name.endswith(".token") and path.name not in self.cache:
                 with self.lock:
                     if path.name not in self.cache:
                         logger.debug("Token file not in cache %s", path.name)
-                        tokenfile = TokenFile(path)
-                        tokenfile.watch()
-                        self.cache[path.name] = tokenfile
+                        try:
+                            tokenfile = TokenFile(path)
+                            tokenfile.watch()
+                            self.cache[path.name] = tokenfile
+                        except FileNotFoundError:
+                            # Well, the file did not exist anymore...
+                            pass
         except Exception:
             logger.exception("Uncaught exception in on_modified handler")
             raise

experimaestro/typingutils.py

@@ -26,6 +26,13 @@ def isgenericalias(typehint):
     return isinstance(typehint, GenericAlias)
 
 
+def get_union(typehint):
+    """Return the list of types of a union (or the type itself if it is not an union)"""
+    if isgenericalias(typehint) and typehint.__origin__ == typing.Union:
+        return typehint.__args__
+    return None
+
+
 def get_optional(typehint):
     if isgenericalias(typehint) and typehint.__origin__ == typing.Union:
         if len(typehint.__args__) == 2:

experimaestro/utils/asyncio.py

@@ -10,8 +10,12 @@ def asyncThreadcheck(name, func, *args, **kwargs) -> asyncio.Future:
 
     def dowait():
         logging.debug("Running %s", func)
-        result = func(*args, **kwargs)
-        logging.debug("Got result from %s", func)
+        try:
+            result = func(*args, **kwargs)
+            logging.debug("Got result from %s", func)
+        except Exception:
+            logging.exception("Got an error in the thread")
+            raise
         loop.call_soon_threadsafe(future.set_result, result)
 
     # Start thread

experimaestro/utils/resources.py

@@ -24,15 +24,19 @@ class ResourcePathWrapper(PathLike):
         parents = [s.name for s in reversed(self.path.parents)][1:]
         return ".".join(parents)
 
-    @property
+    @cached_property
     def name(self):
         return self.path.name
 
     def is_file(self):
-        return resources.is_resource(self.package, self.name)
+        return any(
+            traversable.name == self.name and traversable.is_file()
+            for traversable in resources.files(self.package).iterdir()
+        )
 
     def __fspath__(self):
-        return resources.path(self.package, self.name).__fspath__()
+        """Return the file system path representation of the object"""
+        return resources.as_file(resources.files(self.package) / self.name)
 
     @contextmanager
     def open(self, *args, **kwargs):

{experimaestro-1.6.1.dist-info → experimaestro-1.7.0rc0.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.6.1
+Version: 1.7.0rc0
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager
 Author: Benjamin Piwowarski
 Author-email: benjamin@piwowarski.fr
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -14,7 +14,6 @@ Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -35,7 +34,7 @@ Requires-Dist: huggingface-hub (>0.17)
 Requires-Dist: humanfriendly (>=10,<11)
 Requires-Dist: marshmallow (>=3.20,<4.0)
 Requires-Dist: omegaconf (>=2.3,<3.0)
-Requires-Dist: psutil (>=5.9.5,<6.0.0)
+Requires-Dist: psutil (>=7)
 Requires-Dist: pyparsing (>=3.1,<4.0)
 Requires-Dist: pytools (>=2023.1.1,<2024.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)