experimaestro 1.6.2__tar.gz → 1.7.0rc1__tar.gz

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.6.2
+Version: 1.7.0rc1
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager
 Author: Benjamin Piwowarski
 Author-email: benjamin@piwowarski.fr
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -14,7 +14,6 @@ Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -35,7 +34,7 @@ Requires-Dist: huggingface-hub (>0.17)
 Requires-Dist: humanfriendly (>=10,<11)
 Requires-Dist: marshmallow (>=3.20,<4.0)
 Requires-Dist: omegaconf (>=2.3,<3.0)
-Requires-Dist: psutil (>=5.9.5,<6.0.0)
+Requires-Dist: psutil (>=7)
 Requires-Dist: pyparsing (>=3.1,<4.0)
 Requires-Dist: pytools (>=2023.1.1,<2024.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/pyproject.toml

@@ -17,9 +17,10 @@ classifiers = [
 include = [
     "src/experimaestro/server/data/*",
     "src/experimaestro/sphinx/static/experimaestro.css",
-    "src/experimaestro/mkdocs/style.css"
+    "src/experimaestro/mkdocs/style.css",
+    { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
 ]
-version = "1.6.2"
+version = "1.7.0-rc1"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -28,21 +29,23 @@ enable = false
 vcs = "git"
 style = "semver"
 dirty = true
-format-jinja = "{% if distance == 0 %}{{ base }}{% else %}{{ base }}+{{ distance }}.{{ commit }}{% if dirty %}.dirty{% endif %}{% endif %}"
-
+format-jinja = """{%- set pre = [] -%}{%- set metadata = [] -%}
+{%- if revision is not none -%}{{ pre.append("rc" + revision|string) or "" }}{%- endif -%}
+{%- if distance > 0 -%}{{ metadata.append(distance|string) or "" }}{%- endif -%}
+{{ serialize_semver(base, pre, metadata)}}"""
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
 build-backend = "poetry_dynamic_versioning.backend"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 click = ">=8"
 omegaconf = "^2.3"
 typing-extensions = {version = ">=4.2", markers = "python_version < \"3.12\""}
 attrs = "^23.1.0"
 fasteners = "^0.19"
 pyyaml = "^6.0.1"
-psutil = "^5.9.5"
+psutil = ">=7"
 pytools = "^2023.1.1"
 tqdm = "^4.66.1"
 docstring-parser = "^0.15"
@@ -119,7 +122,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.6.2"
+version = "1.7.0rc0"
 changelog_start_rev = "0.15.0"
 tag_format = "v$version"
 update_changelog_on_bump = true

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/__init__.py

@@ -36,15 +36,17 @@ from .core.arguments import (
     DataPath,
     Annotated,
     Constant,
+    field,
     # Annotations helpers
     help,
     default,
 )
-from .generators import pathgenerator
+from .generators import pathgenerator, PathGenerator
 from .core.objects import (
     Config,
     copyconfig,
     setmeta,
+    DependentMarker,
     Task,
     LightweightTask,
     ObjectStore,

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/annotations.py

@@ -8,7 +8,7 @@ import experimaestro.core.objects as objects
 import experimaestro.core.types as types
 from experimaestro.generators import PathGenerator
 
-from .core.arguments import Argument as CoreArgument
+from .core.arguments import Argument as CoreArgument, field
 from .core.objects import Config
 from .core.types import Any, Identifier, TypeProxy, Type, ObjectType
 from .utils import logger
@@ -134,12 +134,22 @@ class param:
         self.type = Type.fromType(type) if type else None
         self.help = help
         self.ignored = ignored
-        self.default = default
         self.required = required
-        self.generator = None
         self.checker = checker
         self.constant = constant
 
+        self.generator = None
+        self.default = None
+
+        # Set default or generator
+        if isinstance(default, field):
+            if default.default is not None:
+                self.default = default
+            elif default.default_factory is not None:
+                self.generator = default.default_factory
+        else:
+            self.default = default
+
     def __call__(self, tp):
         # Don't annotate in task mode
         tp.__getxpmtype__().addAnnotation(self)

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/cli/filter.py

@@ -3,7 +3,7 @@ import pyparsing as pp
 from pathlib import Path
 import json
 from experimaestro.compat import cached_property
-import regex
+import re
 from experimaestro.scheduler import JobState
 
 
@@ -87,7 +87,7 @@ class NotInExpr(BaseInExpr):
 class RegexExpr:
     def __init__(self, tokens):
         self.var, expr = tokens
-        self.regex = regex.compile(expr)
+        self.regex = re.compile(expr)
 
     def __repr__(self):
         return f"""REGEX[{self.varname}, {self.value}]"""
@@ -103,7 +103,7 @@ class RegexExpr:
         if not value:
             return False
 
-        return self.regex.match(value)
+        return self.re.match(value)
 
 
 class ConstantString:

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/cli/jobs.py

@@ -69,7 +69,7 @@ def process(
         if (p / "jobs.bak").is_dir():
             cprint(f"  Experiment {p.name} has not finished yet", "red")
             if (not perform) and (kill or clean):
-                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                cprint("  Preventing kill/clean (use --perform if you want to)", "yellow")
                 kill = False
                 clean = False
 

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/connectors/__init__.py

@@ -9,7 +9,8 @@ This module contains :
 """
 
 import enum
-from typing import Any, Dict, Mapping, Type, Union
+import logging
+from typing import Any, Dict, Mapping, Type, Union, Optional
 from pathlib import Path
 from experimaestro.utils import logger
 from experimaestro.locking import Lock
@@ -86,12 +87,12 @@ class Process:
     @staticmethod
     def fromDefinition(connector: "Connector", definition: Dict[str, Any]) -> "Process":
         """Retrieves a process from a serialized definition"""
-        handler = Process.handler(definition["type"])
+        handler_type = definition["type"]
+        handler = Process.handler(handler_type)
+        assert handler is not None, f"No handler of type {handler_type}"
         try:
             return handler.fromspec(connector, definition)
         except Exception as e:
-            import logging
-
             logging.exception("Could not retrieve job from specification")
             raise e
 
@@ -101,7 +102,11 @@ class Process:
         if Process.HANDLERS is None:
             Process.HANDLERS = {}
             for ep in pkg_resources.iter_entry_points(group="experimaestro.process"):
-                Process.HANDLERS[ep.name] = ep.load()
+                logging.debug("Adding process handler for type %s", ep.name)
+                handler = ep.load()
+                Process.HANDLERS[ep.name] = handler
+                if handler is None:
+                    logging.error("Handler of type %s is null", ep.name)
 
         return Process.HANDLERS.get(key, None)
 
@@ -117,10 +122,14 @@ class Process:
         """True is the process is truly running (I/O)"""
         return (await self.aio_state()).running
 
-    async def aio_code(self):
-        """Returns a future containing the returned code"""
+    async def aio_code(self) -> Optional[int]:
+        """Returns a future containing the returned code
+
+        Returns None if the process has already finished – and no information is
+        known about the process.
+        """
         code = await asyncThreadcheck("aio_code", self.wait)
-        logger.debug("Got for return code %s: %s", self, code)
+        logger.debug("Got return code %s for %s", code, self)
         return code
 
     def kill(self):

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/connectors/local.py

@@ -2,6 +2,7 @@
 """
 
 import subprocess
+from typing import Optional
 from pathlib import Path, WindowsPath, PosixPath
 import os
 import threading
@@ -29,11 +30,13 @@ class PsutilProcess(Process):
     def __init__(self, pid: int):
         self._process = psutil.Process(pid)
 
-    def wait(self) -> int:
+    def wait(self) -> Optional[int]:
         logger.debug("Waiting (psutil) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (psutil) for process with PID %s", self._process.pid
+            "Finished to wait (psutil) for process with PID %s: code %s",
+            self._process.pid,
+            code,
         )
         return code
 
@@ -57,7 +60,9 @@ class LocalProcess(Process):
         logger.debug("Waiting (python) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (python) for process with PID %s", self._process.pid
+            "Finished to wait (python) for process with PID %s: %s",
+            self._process.pid,
+            code,
         )
         return code
 

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/core/arguments.py

@@ -1,6 +1,6 @@
 """Management of the arguments (params, options, etc) associated with the XPM objects"""
 
-from typing import Optional, TypeVar, TYPE_CHECKING
+from typing import Optional, TypeVar, TYPE_CHECKING, Callable, Any
 from experimaestro.typingutils import get_optional
 from pathlib import Path
 import sys
@@ -75,11 +75,22 @@ class Argument:
         self.constant = constant
         self.ignored = self.type.ignore if ignored is None else ignored
         self.required = required
-        self.default = default
-        self.generator = generator
         self.objecttype = None
         self.is_data = is_data
 
+        self.generator = generator
+        self.default = None
+
+        if default is not None:
+            assert self.generator is None, "generator and default are exclusive options"
+            if isinstance(default, field):
+                if default.default is not None:
+                    self.default = default.default
+                elif default.default_factory is not None:
+                    self.generator = default.default_factory
+            else:
+                self.default = default
+
         assert (
             not self.constant or self.default is not None
         ), "Cannot be constant without default"
@@ -170,6 +181,18 @@ DataPath = Annotated[Path, dataHint]
 """Annotates a path that should be kept to restore an object to its state"""
 
 
+class field:
+    """Extra information for a given experimaestro field (param or meta)"""
+
+    def __init__(self, *, default: Any = None, default_factory: Callable = None):
+        assert not (
+            (default is not None) and (default_factory is not None)
+        ), "default and default_factory are mutually exclusive options"
+
+        self.default_factory = default_factory
+        self.default = default
+
+
 class help(TypeAnnotation):
     def __init__(self, text: str):
         self.text = text

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/core/objects.py

@@ -3,6 +3,10 @@
 from functools import cached_property
 import json
 
+from attr import define
+
+from experimaestro import taskglobals
+
 try:
     from types import NoneType
 except Exception:
@@ -21,6 +25,7 @@ import inspect
 import importlib
 from typing import (
     Any,
+    Callable,
     ClassVar,
     Dict,
     Iterator,
@@ -136,6 +141,8 @@ class ConfigPath:
 
 hash_logger = logging.getLogger("xpm.hash")
 
+DependentMarker = Callable[["Config"], None]
+
 
 class HashComputer:
     """This class is in charge of computing a config/task identifier"""
@@ -555,6 +562,24 @@ class ObjectStore:
         self.store[identifier] = stub
 
 
+@define()
+class WatchedOutput:
+    #: The enclosing job
+    job: "Job"
+
+    #: The configuration containing the watched output
+    config: "ConfigInformation"
+
+    #: The watched output (name)
+    method_name: str
+
+    #: The watched output method (called with the JSON event)
+    method: Callable
+
+    #: The callback to call (with the output of the previous method)
+    callback: Callable
+
+
 class ConfigInformation:
     """Holds experimaestro information for a config (or task) instance"""
 
@@ -595,6 +620,9 @@ class ConfigInformation:
         # Initialization tasks
         self.init_tasks: List["LightweightTask"] = []
 
+        # Watched outputs
+        self.watched_outputs: List[WatchedOutput] = []
+
         # Cached information
 
         self._full_identifier = None
@@ -724,9 +752,16 @@ class ConfigInformation:
                 # Generate values
                 for k, argument in config.__xpmtype__.arguments.items():
                     if argument.generator:
-                        config.__xpm__.set(
-                            k, argument.generator(self.context, config), bypass=True
-                        )
+                        sig = inspect.signature(argument.generator)
+                        if len(sig.parameters) == 2:
+                            value = argument.generator(self.context, config)
+                        elif len(sig.parameters) == 0:
+                            value = argument.generator()
+                        else:
+                            assert (
+                                False
+                            ), "generator has either two parameters (context and config), or none"
+                        config.__xpm__.set(k, value, bypass=True)
 
                 config.__xpm__._sealed = True
 
@@ -899,6 +934,19 @@ class ConfigInformation:
         # Now, seal the object
         self.seal(context)
 
+    def watch_output(self, method, callback):
+        """Watch the task output linked with a given method
+
+        :param method: The method to watch
+        :param callback: The callback
+        """
+        watched = WatchedOutput(
+            self, method.__self__, method.__name__, method, callback
+        )
+        self.watched_outputs.append(watched)
+        if self.job:
+            self.job.watch_output(watched)
+
     def submit(
         self,
         workspace: "Workspace",
@@ -1000,7 +1048,7 @@ class ConfigInformation:
 
                 print(file=sys.stderr)  # noqa: T201
 
-        # Handle an output configuration
+        # Handle an output configuration # FIXME: remove
         def mark_output(config: "Config"):
             """Sets a dependency on the job"""
             assert not isinstance(config, Task), "Cannot set a dependency on a task"
@@ -1011,12 +1059,18 @@ class ConfigInformation:
         self.task = self.pyobject
 
         if hasattr(self.pyobject, "task_outputs"):
-            self._taskoutput = self.pyobject.task_outputs(mark_output)
+            self._taskoutput = self.pyobject.task_outputs(self.mark_output)
         else:
             self._taskoutput = self.task = self.pyobject
 
         return self._taskoutput
 
+    def mark_output(self, config: "Config"):
+        """Sets a dependency on the job"""
+        assert not isinstance(config, Task), "Cannot set a dependency on a task"
+        config.__xpm__.task = self.pyobject
+        return config
+
     # --- Serialization
 
     @staticmethod
@@ -1345,7 +1399,9 @@ class ConfigInformation:
                     mod = importlib.import_module(module_name)
                 except ModuleNotFoundError:
                     # More hints on the nature of the error
-                    logging.warning("(1) Either the python path is wrong – %s", ":".join(sys.path))
+                    logging.warning(
+                        "(1) Either the python path is wrong – %s", ":".join(sys.path)
+                    )
                     logging.warning("(2) There is not __init__.py in your module")
                     raise
 
@@ -1790,6 +1846,10 @@ class classproperty(property):
 class Config:
     """Base type for all objects in python interface"""
 
+    __xpmid__: ClassVar[Optional[str]]
+    """Optional configuration ID, mostly useful when moving a class to another
+    package to avoid changes in computed task identifiers"""
+
     __xpmtype__: ClassVar[ObjectType]
     """The object type holds all the information about a specific subclass
     experimaestro metadata"""
@@ -1914,6 +1974,22 @@ class Config:
         """Access pre-tasks"""
         raise AssertionError("Pre-tasks can be accessed only during configuration")
 
+    def register_task_output(self, method, *args, **kwargs):
+        # Determine the path for this...
+        path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        data = json.dumps(
+            {
+                "key": f"{self.__xpmidentifier__}/{method.__name__}",
+                "args": args,
+                "kwargs": kwargs,
+            }
+        )
+        with path.open("at") as fp:
+            fp.writelines([data, "\n"])
+            fp.flush()
+
 
 class LightweightTask(Config):
     """A task that can be run before or after a real task to modify its behaviour"""
@@ -1931,6 +2007,14 @@ class Task(LightweightTask):
     def submit(self):
         raise AssertionError("This method can only be used during configuration")
 
+    def watch_output(self, method, callback):
+        """Sets up a callback
+
+        :param method: a method within a configuration
+        :param callback: the callback
+        """
+        self.__xpm__.watch_output(method, callback)
+
 
 # --- Utility functions
 

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/core/objects.pyi

@@ -1,4 +1,5 @@
 from abc import ABC
+from attrs import define
 import typing_extensions
 
 from experimaestro.core.types import ObjectType
@@ -108,6 +109,10 @@ class ConfigWalk(ConfigProcessing):
     def map(self, k: str): ...
 
 def getqualattr(module, qualname): ...
+@define(frozen=True)
+class WatchedOutput:
+    config: "Config"
+    method_name: str
 
 class ConfigInformation:
     LOADING: bool
@@ -116,6 +121,7 @@ class ConfigInformation:
     values: Dict[str, Any]
     job: Job
     dependencies: Incomplete
+    watched_outputs: List[WatchedOutput]
     def __init__(self, pyobject: TypeConfig) -> None: ...
     def set_meta(self, value: Optional[bool]): ...
     @property
@@ -231,7 +237,7 @@ class LightweightTask(Config):
     def execute(self) -> None: ...
 
 class Task(LightweightTask):
-    __tags__: Dict[str, str]
+    # __tags__: Dict[str, str]
 
     def submit(
         self,

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/core/types.py

@@ -26,6 +26,7 @@ if typing.TYPE_CHECKING:
 
 class Identifier:
     def __init__(self, name: str):
+        assert isinstance(name, str)
         self.name = name
 
     def __hash__(self):
@@ -126,6 +127,9 @@ class Type:
         if t:
             return DictType(Type.fromType(t[0]), Type.fromType(t[1]))
 
+        if union_t := typingutils.get_union(key):
+            return UnionType([Type.fromType(t) for t in union_t])
+
         # Takes care of generics
         if get_origin(key):
             return GenericType(key)
@@ -223,7 +227,7 @@ class ObjectType(Type):
             __xpmid__ = getattr(tp, "__xpmid__")
             if isinstance(__xpmid__, Identifier):
                 identifier = __xpmid__
-            if inspect.ismethod(__xpmid__):
+            elif inspect.ismethod(__xpmid__):
                 identifier = Identifier(__xpmid__())
             elif "__xpmid__" in tp.__dict__:
                 identifier = Identifier(__xpmid__)
@@ -576,7 +580,7 @@ class PathType(Type):
             return Path(value.get("$value"))
 
         if not isinstance(value, (str, Path)):
-            raise TypeError("value is not a pathlike value")
+            raise TypeError(f"value is not a pathlike value ({type(value)})")
         return Path(value)
 
     @property
@@ -631,6 +635,32 @@ class EnumType(Type):
         return f"Enum({self.type})"
 
 
+class UnionType(Type):
+    def __init__(self, types: List[Type]):
+        self.types = types
+
+    def name(self):
+        return "Union[" + ", ".join(t.name() for t in self.types) + "]"
+
+    def __str__(self):
+        return "[" + " | ".join(t.name() for t in self.types) + " ]"
+
+    def __repr__(self):
+        return str(self)
+
+    def validate(self, value):
+        for subtype in self.types:
+            try:
+                return subtype.validate(value)
+            except ValueError:
+                pass
+            except TypeError:
+                pass
+
+        if not isinstance(value, dict):
+            raise ValueError(f"value is not within the types {self}")
+
+
 class DictType(Type):
     def __init__(self, keytype: Type, valuetype: Type):
         self.keytype = keytype
@@ -675,6 +705,7 @@ class GenericType(Type):
             (mro for mro in mros if (get_origin(mro) or mro) is self.origin), None
         )
         target = get_origin(self.type) or self.type
+
         if matching is None:
             raise ValueError(
                 f"{type(value)} is not of type {target} ({type(value).__mro__})"

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/generators.py

@@ -1,11 +1,12 @@
 import inspect
 from pathlib import Path
+from abc import ABC, abstractmethod
 from typing import Callable, Union
 from experimaestro.core.arguments import ArgumentOptions, TypeAnnotation
 from experimaestro.core.objects import ConfigWalkContext, Config
 
 
-class Generator:
+class Generator(ABC):
     """Base class for all generators"""
 
     def isoutput(self):
@@ -13,6 +14,10 @@ class Generator:
         path within the job folder)"""
         return False
 
+    @abstractmethod
+    def __call__(self, context: ConfigWalkContext, config: Config):
+        ...
+
 
 class PathGenerator(Generator):
     """Generates a path"""

{experimaestro-1.6.2 → experimaestro-1.7.0rc1}/src/experimaestro/ipc.py

@@ -7,6 +7,7 @@ import sys
 import logging
 from .utils import logger
 from watchdog.observers import Observer
+from watchdog.observers.api import ObservedWatch
 from watchdog.events import FileSystemEventHandler
 
 
@@ -20,7 +21,9 @@ class IPCom:
         self.observer.start()
         self.pid = os.getpid()
 
-    def fswatch(self, watcher: FileSystemEventHandler, path: Path, recursive=False):
+    def fswatch(
+        self, watcher: FileSystemEventHandler, path: Path, recursive=False
+    ) -> ObservedWatch:
         if not self.observer.is_alive():
             logging.error("Observer is not alive")