experimaestro 1.6.1__py3-none-any.whl → 1.15.2__py3-none-any.whl

experimaestro/tests/test_validation.py

@@ -2,18 +2,9 @@
 
 import pytest
 from pathlib import Path
-from experimaestro import (
-    config,
-    Task,
-    Identifier,
-    argument,
-    pathoption,
-    ConstantParam,
-    Param,
-    Config,
-)
+from experimaestro import Task, field, Identifier, Constant, Param, Config, Meta
 from enum import Enum
-import experimaestro.core.types as types
+from experimaestro.generators import PathGenerator
 from experimaestro.scheduler import Job, JobContext
 from experimaestro.scheduler.workspace import RunMode
 from .utils import TemporaryExperiment
@@ -31,57 +22,50 @@ def expect_notvalidate(value):
         value.__xpm__.validate()
 
 
-@argument("value", type=int)
-@config()
-class A:
-    pass
+class A(Config):
+    value: Param[int]
 
 
-@argument("a", type=A)
-@config()
-class B:
-    pass
+class B(Config):
+    a: Param[A]
 
 
-@pathoption("path", "outdir")
-@config()
-class C:
+class C(Config):
+    path: Meta[Path] = field(default_factory=PathGenerator("outdir"))
     pass
 
 
-def test_simple():
+def test_validation_simple():
     expect_validate(A(value=1))
 
 
-def test_missing():
+def test_validation_missing():
     expect_notvalidate(A())
 
 
-def test_simple_nested():
+def test_validation_simple_nested():
     b = B()
     b.a = A(value=1)
     expect_validate(b)
 
 
-def test_missing_nested():
+def test_validation_missing_nested():
     b = B()
     b.a = A()
     expect_notvalidate(b)
 
 
-def test_type():
-    @config(valns.type.a)
-    class A:
+def test_validation_type():
+    class A(Config):
+        __xpmid__ = valns.type.a
         pass
 
-    @config(valns.type.b)
-    class B:
-        pass
+    class B(Config):
+        __xpmid__ = valns.type.b
 
-    @argument("a", A)
-    @config(valns.type.c)
-    class C:
-        pass
+    class C(Config):
+        a: Param[A]
+        __xpmid__ = valns.type.c
 
     with pytest.raises(ValueError):
         C(a=B())
@@ -91,30 +75,26 @@ def test_type():
         c.a = B()
 
 
-def test_subtype():
-    @config(valns.subtype.a)
-    class A:
-        pass
+def test_validation_subtype():
+    class A(Config):
+        __xpmid__ = valns.subtype.a
 
-    @config(valns.subtype.a1)
     class A1(A):
-        pass
+        __xpmid__ = valns.subtype.a1
 
-    @argument("a", A)
-    @config(valns.subtype.b)
-    class B:
-        pass
+    class B(Config):
+        __xpmid__ = valns.subtype.b
+        a: Param[A]
 
     expect_validate(B(a=A1()))
 
 
-def test_path():
-    """Test of @pathoption"""
+def test_validation_path_generator():
+    """Test of path generator"""
 
-    @pathoption("value", "file.txt")
-    @config(valns.path.a)
-    class A:
-        pass
+    class A(Config):
+        __xpmid__ = valns.path.a
+        value: Meta[Path] = field(default_factory=PathGenerator("file.txt"))
 
     a = A()
     a.__xpm__.validate()
@@ -129,13 +109,12 @@ def test_path():
         assert a.value.parents[3] == xp.workspace.path
 
 
-def test_constant():
-    """Test of @ConstantParam"""
+def test_validation_constant():
+    """Test of constant"""
 
-    @ConstantParam("value", 1)
-    @config(valns.constant.a)
-    class A:
-        pass
+    class A(Config):
+        __xpmid__ = valns.constant.a
+        value: Constant[int] = 1
 
     a = A()
     a.__xpm__.validate()
@@ -145,31 +124,26 @@ def test_constant():
         assert a.value == 1
 
 
-@argument("x", type=int)
-@config()
-class Parent:
-    pass
+class Parent(Config):
+    x: Param[int]
 
 
-@config()
 class Child(Parent):
     pass
 
 
-def test_child():
+def test_validation_child():
     expect_validate(Child(x=1))
 
 
 # --- Path argument checks
 
 
-@pathoption("x", "x")
-@config()
-class PathParent:
-    pass
+class PathParent(Config):
+    x: Meta[Path] = field(default_factory=PathGenerator("x"))
 
 
-def test_path_option():
+def test_validation_path_option():
     c = PathParent()
     expect_validate(c)
 
@@ -177,28 +151,11 @@ def test_path_option():
 # --- Default value
 
 
-@pytest.mark.parametrize(
-    "value,apitype",
-    [(1.5, types.FloatType), (1, types.IntType), (False, types.BoolType)],
-)
-def test_default(value, apitype):
-    @argument("default", default=value)
-    @config(valns.default[str(type(value))])
-    class Default:
-        pass
-
-    value = Default()
-    expect_validate(value)
-    assert Default.__xpmtype__.arguments["default"].type.__class__ == apitype
-
-
-def test_seal():
+def test_validation_seal():
     """Test value sealing"""
 
-    @argument("a", int)
-    @config()
-    class A:
-        pass
+    class A(Config):
+        a: Param[int]
 
     a = A(a=2)
     a.__xpm__.seal(EmptyContext())
@@ -207,7 +164,7 @@ def test_seal():
         a.a = 1
 
 
-def test_validation_enum():
+def test_validation_validation_enum():
     """Path arguments should be ignored"""
 
     class EnumParam(Enum):
@@ -241,7 +198,7 @@ class TaskConfigConsumer(Config):
     x: Param[TaskParentConfig]
 
 
-def test_taskargument():
+def test_validation_taskargument():
     x = taskconfig()
     with TemporaryExperiment("fake"):
         x.submit(run_mode=RunMode.DRY_RUN)

experimaestro/tokens.py

@@ -59,6 +59,8 @@ class CounterTokenLock(Lock):
 
 
 class CounterTokenDependency(Dependency):
+    """A dependency onto a token"""
+
     def __init__(self, token: "CounterToken", count: int):
         super().__init__(token)
         self._token = token
@@ -66,6 +68,7 @@ class CounterTokenDependency(Dependency):
 
     @property
     def name(self):
+        """The (file) name for this dependency, when taken"""
         return f"{self.target.identifier}.token"
 
     def status(self) -> DependencyStatus:
@@ -165,7 +168,7 @@ class CounterToken(Token, FileSystemEventHandler):
     - TIMESTAMP.token contains (1) the number of tokens (2) the job URI
     """
 
-    """Maps paths to instances"""
+    """Maps token keys to CounterToken instances"""
     TOKENS: Dict[str, "CounterToken"] = {}
 
     @staticmethod
@@ -193,7 +196,7 @@ class CounterToken(Token, FileSystemEventHandler):
         """[summary]
 
         Arguments:
-            path {Path} -- The file path of the token file
+            path {Path} -- The file path of the token directory
             count {int} -- Number of tokens (overrides previous definitions)
             force --   If the token has already been created, force to write the maximum
                        number of tokens
@@ -298,6 +301,9 @@ class CounterToken(Token, FileSystemEventHandler):
                         tokenfile = TokenFile(path)
                         tokenfile.watch()
                         self.cache[path.name] = tokenfile
+        except FileNotFoundError:
+            # We did not find the token file... just ignore
+            pass
         except Exception:
             logger.exception("Uncaught exception in on_modified handler")
             raise
@@ -336,13 +342,18 @@ class CounterToken(Token, FileSystemEventHandler):
                         for dependency in dependents:
                             dependency.check()
 
+            # A modified dependency not in cache
             elif path.name.endswith(".token") and path.name not in self.cache:
                 with self.lock:
                     if path.name not in self.cache:
                         logger.debug("Token file not in cache %s", path.name)
-                        tokenfile = TokenFile(path)
-                        tokenfile.watch()
-                        self.cache[path.name] = tokenfile
+                        try:
+                            tokenfile = TokenFile(path)
+                            tokenfile.watch()
+                            self.cache[path.name] = tokenfile
+                        except FileNotFoundError:
+                            # Well, the file did not exist anymore...
+                            pass
         except Exception:
             logger.exception("Uncaught exception in on_modified handler")
             raise

experimaestro/typingutils.py

@@ -8,14 +8,7 @@ if sys.version_info.major == 3:
     else:
         from typing import _collect_parameters
 
-    if sys.version_info.minor < 9:
-        from typing_extensions import (
-            _AnnotatedAlias as AnnotatedAlias,
-            get_args,
-            get_origin,
-        )
-    else:
-        from typing import _AnnotatedAlias as AnnotatedAlias, get_args, get_origin
+    from typing import _AnnotatedAlias as AnnotatedAlias, get_args, get_origin
 
 GenericAlias = typing._GenericAlias
 
@@ -26,6 +19,13 @@ def isgenericalias(typehint):
     return isinstance(typehint, GenericAlias)
 
 
+def get_union(typehint):
+    """Return the list of types of a union (or the type itself if it is not an union)"""
+    if isgenericalias(typehint) and typehint.__origin__ == typing.Union:
+        return typehint.__args__
+    return None
+
+
 def get_optional(typehint):
     if isgenericalias(typehint) and typehint.__origin__ == typing.Union:
         if len(typehint.__args__) == 2:

experimaestro/utils/asyncio.py

@@ -10,8 +10,12 @@ def asyncThreadcheck(name, func, *args, **kwargs) -> asyncio.Future:
 
     def dowait():
         logging.debug("Running %s", func)
-        result = func(*args, **kwargs)
-        logging.debug("Got result from %s", func)
+        try:
+            result = func(*args, **kwargs)
+            logging.debug("Got result from %s", func)
+        except Exception:
+            logging.exception("Got an error in the thread")
+            raise
         loop.call_soon_threadsafe(future.set_result, result)
 
     # Start thread

experimaestro/utils/multiprocessing.py

@@ -0,0 +1,44 @@
+import logging
+import multiprocessing as mp
+import time
+import os
+import signal
+import threading
+
+
+def delayed_shutdown(delay=60, *, exit_code=1, grace_period=5):
+    """After *delay*'s try a graceful stop, then SIGKILL anything left.
+
+    :param delay: Delay in seconds before killing
+    :param grace_period: Delay in seconds before force-killing a child process
+    :param exit_code: The exit code to use
+    """
+
+    def _killer():
+        time.sleep(delay)
+
+        logging.info("Stall dectected – killing all subprocesses")
+
+        # 1️⃣ Try graceful termination
+        for p in mp.active_children():
+            # sends SIGTERM / TerminateProcess
+            p.terminate()
+
+        alive = mp.active_children()
+        deadline = time.time() + grace_period
+        while alive and time.time() < deadline:
+            alive = [p for p in alive if p.is_alive()]
+            time.sleep(0.1)
+
+        # 2️⃣ Anything still alive? Nuke it.
+        for p in alive:
+            try:
+                os.kill(p.pid, signal.SIGKILL)
+            except OSError:
+                pass
+
+        # 3️⃣ Finally kill the parent
+        os.kill(os.getpid(), signal.SIGKILL)
+
+    # Start the thread (non blocking)
+    threading.Thread(target=_killer, daemon=True).start()

experimaestro/utils/resources.py

@@ -24,15 +24,19 @@ class ResourcePathWrapper(PathLike):
         parents = [s.name for s in reversed(self.path.parents)][1:]
         return ".".join(parents)
 
-    @property
+    @cached_property
     def name(self):
         return self.path.name
 
     def is_file(self):
-        return resources.is_resource(self.package, self.name)
+        return any(
+            traversable.name == self.name and traversable.is_file()
+            for traversable in resources.files(self.package).iterdir()
+        )
 
     def __fspath__(self):
-        return resources.path(self.package, self.name).__fspath__()
+        """Return the file system path representation of the object"""
+        return resources.as_file(resources.files(self.package) / self.name)
 
     @contextmanager
     def open(self, *args, **kwargs):

{experimaestro-1.6.1.dist-info → experimaestro-1.15.2.dist-info}/METADATA

@@ -1,53 +1,46 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: experimaestro
-Version: 1.6.1
+Version: 1.15.2
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
+License-File: LICENSE
 Keywords: experiment manager
 Author: Benjamin Piwowarski
 Author-email: benjamin@piwowarski.fr
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: arpeggio (>=2,<3)
-Requires-Dist: attrs (>=23.1.0,<24.0.0)
+Requires-Dist: attrs (>=23.1.0,<24)
 Requires-Dist: click (>=8)
 Requires-Dist: decorator (>=5,<6)
-Requires-Dist: docstring-parser (>=0.15,<0.16)
-Requires-Dist: fasteners (>=0.19,<0.20)
-Requires-Dist: flask (>=2.3,<3.0)
-Requires-Dist: flask-socketio (>=5.3,<6.0)
-Requires-Dist: gevent (>=23.9,<24.0)
-Requires-Dist: gevent-websocket (>=0.10,<0.11)
+Requires-Dist: docstring-parser (>=0.15,<1)
+Requires-Dist: fasteners (>=0.19,<1)
+Requires-Dist: flask (>=2.3)
+Requires-Dist: flask-socketio (>=5.3)
+Requires-Dist: gevent (>=25)
+Requires-Dist: gevent-websocket (>=0.10)
 Requires-Dist: huggingface-hub (>0.17)
-Requires-Dist: humanfriendly (>=10,<11)
-Requires-Dist: marshmallow (>=3.20,<4.0)
-Requires-Dist: omegaconf (>=2.3,<3.0)
-Requires-Dist: psutil (>=5.9.5,<6.0.0)
-Requires-Dist: pyparsing (>=3.1,<4.0)
-Requires-Dist: pytools (>=2023.1.1,<2024.0.0)
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
-Requires-Dist: requests (>=2.31,<3.0)
+Requires-Dist: humanfriendly (>=10)
+Requires-Dist: marshmallow (>=3.20,<4)
+Requires-Dist: mkdocs (>=1.5,<2)
+Requires-Dist: omegaconf (>=2.3,<3)
+Requires-Dist: psutil (>=7,<8)
+Requires-Dist: pyparsing (>=3.1,<4)
+Requires-Dist: pytools (>=2023.1.1,<2024)
+Requires-Dist: pyyaml (>=6.0.1,<7)
+Requires-Dist: requests (>=2.31,<3)
 Requires-Dist: rpyc (>=5,<7)
-Requires-Dist: sortedcontainers (>=2.4,<3.0)
-Requires-Dist: termcolor (>=2.3)
-Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Requires-Dist: sortedcontainers (>=2.4,<3)
+Requires-Dist: termcolor (>=2.3,<3)
+Requires-Dist: tqdm (>=4.66.1,<5)
 Requires-Dist: typing-extensions (>=4.2) ; python_version < "3.12"
-Requires-Dist: watchdog (>=2,<3)
-Project-URL: Documentation, https://experimaestro-python.readthedocs.io/
-Project-URL: Repository, https://github.com/experimaestro/experimaestro-python
+Requires-Dist: watchdog (>=2)
 Description-Content-Type: text/markdown
 
 [![PyPI version](https://badge.fury.io/py/experimaestro.svg)](https://badge.fury.io/py/experimaestro)
@@ -145,11 +138,11 @@ def cli(port, workdir, sleeptime):
     # Sets the working directory and the name of the xp
     with experiment(workdir, "helloworld", port=port) as xp:
         # Submit the tasks
-        hello = Say(word="hello", sleeptime=sleeptime).submit()
-        world = Say(word="world", sleeptime=sleeptime).submit()
+        hello = Say.C(word="hello", sleeptime=sleeptime).submit()
+        world = Say.C(word="world", sleeptime=sleeptime).submit()
 
         # Concat will depend on the two first tasks
-        Concat(strings=[hello, world], sleeptime=sleeptime).tag("y", 1).submit()
+        Concat.C(strings=[hello, world], sleeptime=sleeptime).tag("y", 1).submit()
 
 
 if __name__ == "__main__":