experimaestro 1.6.1__py3-none-any.whl → 1.7.0__py3-none-any.whl

experimaestro/__init__.py

@@ -36,15 +36,17 @@ from .core.arguments import (
     DataPath,
     Annotated,
     Constant,
+    field,
     # Annotations helpers
     help,
     default,
 )
-from .generators import pathgenerator
+from .generators import pathgenerator, PathGenerator
 from .core.objects import (
     Config,
     copyconfig,
     setmeta,
+    DependentMarker,
     Task,
     LightweightTask,
     ObjectStore,

experimaestro/annotations.py

@@ -8,7 +8,7 @@ import experimaestro.core.objects as objects
 import experimaestro.core.types as types
 from experimaestro.generators import PathGenerator
 
-from .core.arguments import Argument as CoreArgument
+from .core.arguments import Argument as CoreArgument, field
 from .core.objects import Config
 from .core.types import Any, Identifier, TypeProxy, Type, ObjectType
 from .utils import logger
@@ -134,12 +134,22 @@ class param:
         self.type = Type.fromType(type) if type else None
         self.help = help
         self.ignored = ignored
-        self.default = default
         self.required = required
-        self.generator = None
         self.checker = checker
         self.constant = constant
 
+        self.generator = None
+        self.default = None
+
+        # Set default or generator
+        if isinstance(default, field):
+            if default.default is not None:
+                self.default = default
+            elif default.default_factory is not None:
+                self.generator = default.default_factory
+        else:
+            self.default = default
+
     def __call__(self, tp):
         # Don't annotate in task mode
         tp.__getxpmtype__().addAnnotation(self)

experimaestro/cli/filter.py

@@ -3,7 +3,7 @@ import pyparsing as pp
 from pathlib import Path
 import json
 from experimaestro.compat import cached_property
-import regex
+import re
 from experimaestro.scheduler import JobState
 
 
@@ -87,7 +87,7 @@ class NotInExpr(BaseInExpr):
 class RegexExpr:
     def __init__(self, tokens):
         self.var, expr = tokens
-        self.regex = regex.compile(expr)
+        self.regex = re.compile(expr)
 
     def __repr__(self):
         return f"""REGEX[{self.varname}, {self.value}]"""
@@ -103,7 +103,7 @@ class RegexExpr:
         if not value:
             return False
 
-        return self.regex.match(value)
+        return self.re.match(value)
 
 
 class ConstantString:

experimaestro/cli/jobs.py

@@ -69,7 +69,7 @@ def process(
         if (p / "jobs.bak").is_dir():
             cprint(f"  Experiment {p.name} has not finished yet", "red")
             if (not perform) and (kill or clean):
-                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                cprint("  Preventing kill/clean (use --perform if you want to)", "yellow")
                 kill = False
                 clean = False
 

experimaestro/commandline.py

@@ -276,12 +276,6 @@ class CommandLineJob(Job):
 
         scriptbuilder = self.launcher.scriptbuilder()
         self.path.mkdir(parents=True, exist_ok=True)
-        donepath = self.donepath
-
-        # Check again if done (now that we have locked)
-        if not overwrite and donepath.is_file():
-            logger.info("Job %s is already done", self)
-            return JobState.DONE
 
         # Now we can write the script
         scriptbuilder.lockfiles.append(self.lockpath)
@@ -293,15 +287,17 @@ class CommandLineJob(Job):
         if self._process:
             return self._process
 
+        # Prepare the files to be run
         scriptPath = self.prepare()
 
+        # OK, now starts the process
         logger.info("Starting job %s", self.jobpath)
         processbuilder = self.launcher.processbuilder()
         processbuilder.environ = self.environ
         processbuilder.command.append(self.launcher.connector.resolve(scriptPath))
         processbuilder.stderr = Redirect.file(self.stderr)
         processbuilder.stdout = Redirect.file(self.stdout)
-        self._process = processbuilder.start()
+        self._process = processbuilder.start(True)
 
         with self.pidpath.open("w") as fp:
             json.dump(self._process.tospec(), fp)

experimaestro/connectors/__init__.py

@@ -9,7 +9,8 @@ This module contains :
 """
 
 import enum
-from typing import Any, Dict, Mapping, Type, Union
+import logging
+from typing import Any, Dict, Mapping, Type, Union, Optional
 from pathlib import Path
 from experimaestro.utils import logger
 from experimaestro.locking import Lock
@@ -86,12 +87,12 @@ class Process:
     @staticmethod
     def fromDefinition(connector: "Connector", definition: Dict[str, Any]) -> "Process":
         """Retrieves a process from a serialized definition"""
-        handler = Process.handler(definition["type"])
+        handler_type = definition["type"]
+        handler = Process.handler(handler_type)
+        assert handler is not None, f"No handler of type {handler_type}"
         try:
             return handler.fromspec(connector, definition)
         except Exception as e:
-            import logging
-
             logging.exception("Could not retrieve job from specification")
             raise e
 
@@ -101,7 +102,11 @@ class Process:
         if Process.HANDLERS is None:
             Process.HANDLERS = {}
             for ep in pkg_resources.iter_entry_points(group="experimaestro.process"):
-                Process.HANDLERS[ep.name] = ep.load()
+                logging.debug("Adding process handler for type %s", ep.name)
+                handler = ep.load()
+                Process.HANDLERS[ep.name] = handler
+                if handler is None:
+                    logging.error("Handler of type %s is null", ep.name)
 
         return Process.HANDLERS.get(key, None)
 
@@ -117,10 +122,14 @@ class Process:
         """True is the process is truly running (I/O)"""
         return (await self.aio_state()).running
 
-    async def aio_code(self):
-        """Returns a future containing the returned code"""
+    async def aio_code(self) -> Optional[int]:
+        """Returns a future containing the returned code
+
+        Returns None if the process has already finished – and no information is
+        known about the process.
+        """
         code = await asyncThreadcheck("aio_code", self.wait)
-        logger.debug("Got for return code %s: %s", self, code)
+        logger.debug("Got return code %s for %s", code, self)
         return code
 
     def kill(self):
@@ -145,8 +154,11 @@ class ProcessBuilder:
         self.environ: Mapping[str, str] = {}
         self.command = []
 
-    def start(self) -> Process:
-        """Start the process"""
+    def start(self, task_mode: bool = False) -> Process:
+        """Start the process
+
+        :param task_mode: True if the process is a job script
+        """
         raise NotImplementedError("Method not implemented in %s" % self.__class__)
 
 

experimaestro/connectors/local.py

@@ -1,7 +1,7 @@
-"""All classes related to localhost management
-"""
+"""All classes related to localhost management"""
 
 import subprocess
+from typing import Optional
 from pathlib import Path, WindowsPath, PosixPath
 import os
 import threading
@@ -29,11 +29,13 @@ class PsutilProcess(Process):
     def __init__(self, pid: int):
         self._process = psutil.Process(pid)
 
-    def wait(self) -> int:
+    def wait(self) -> Optional[int]:
         logger.debug("Waiting (psutil) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (psutil) for process with PID %s", self._process.pid
+            "Finished to wait (psutil) for process with PID %s: code %s",
+            self._process.pid,
+            code,
         )
         return code
 
@@ -57,7 +59,9 @@ class LocalProcess(Process):
         logger.debug("Waiting (python) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (python) for process with PID %s", self._process.pid
+            "Finished to wait (python) for process with PID %s: %s",
+            self._process.pid,
+            code,
         )
         return code
 
@@ -102,8 +106,11 @@ def getstream(redirect: Redirect, write: bool):
 
 
 class LocalProcessBuilder(ProcessBuilder):
-    def start(self):
-        """Start the process"""
+    def start(self, task_mode=False):
+        """Start the process
+
+        :param task_mode: just ignored
+        """
         stdin = getstream(self.stdin, False)
         stdout = getstream(self.stdout, True)
         stderr = getstream(self.stderr, True)
@@ -194,7 +201,9 @@ class LocalConnector(Connector):
         return LocalProcessBuilder()
 
     def resolve(self, path: Path, basepath: Path = None) -> str:
-        assert isinstance(path, PosixPath) or isinstance(path, WindowsPath)
+        assert isinstance(path, PosixPath) or isinstance(
+            path, WindowsPath
+        ), f"Unrecognized path {type(path)}"
         if not basepath:
             return str(path.absolute())
         try:

experimaestro/connectors/ssh.py

@@ -200,7 +200,7 @@ class SshProcessBuilder(ProcessBuilder):
         super().__init__()
         self.connector = connector
 
-    def start(self):
+    def start(self, task_mode: bool = False):
         """Start the process"""
 
         trans = str.maketrans({'"': r"\"", "$": r"\$"})

experimaestro/core/arguments.py

@@ -1,6 +1,6 @@
 """Management of the arguments (params, options, etc) associated with the XPM objects"""
 
-from typing import Optional, TypeVar, TYPE_CHECKING
+from typing import Optional, TypeVar, TYPE_CHECKING, Callable, Any
 from experimaestro.typingutils import get_optional
 from pathlib import Path
 import sys
@@ -75,11 +75,22 @@ class Argument:
         self.constant = constant
         self.ignored = self.type.ignore if ignored is None else ignored
         self.required = required
-        self.default = default
-        self.generator = generator
         self.objecttype = None
         self.is_data = is_data
 
+        self.generator = generator
+        self.default = None
+
+        if default is not None:
+            assert self.generator is None, "generator and default are exclusive options"
+            if isinstance(default, field):
+                if default.default is not None:
+                    self.default = default.default
+                elif default.default_factory is not None:
+                    self.generator = default.default_factory
+            else:
+                self.default = default
+
         assert (
             not self.constant or self.default is not None
         ), "Cannot be constant without default"
@@ -170,6 +181,18 @@ DataPath = Annotated[Path, dataHint]
 """Annotates a path that should be kept to restore an object to its state"""
 
 
+class field:
+    """Extra information for a given experimaestro field (param or meta)"""
+
+    def __init__(self, *, default: Any = None, default_factory: Callable = None):
+        assert not (
+            (default is not None) and (default_factory is not None)
+        ), "default and default_factory are mutually exclusive options"
+
+        self.default_factory = default_factory
+        self.default = default
+
+
 class help(TypeAnnotation):
     def __init__(self, text: str):
         self.text = text

experimaestro/core/objects.py

@@ -3,6 +3,10 @@
 from functools import cached_property
 import json
 
+from attr import define
+
+from experimaestro import taskglobals
+
 try:
     from types import NoneType
 except Exception:
@@ -21,6 +25,7 @@ import inspect
 import importlib
 from typing import (
     Any,
+    Callable,
     ClassVar,
     Dict,
     Iterator,
@@ -136,6 +141,8 @@ class ConfigPath:
 
 hash_logger = logging.getLogger("xpm.hash")
 
+DependentMarker = Callable[["Config"], None]
+
 
 class HashComputer:
     """This class is in charge of computing a config/task identifier"""
@@ -555,6 +562,24 @@ class ObjectStore:
         self.store[identifier] = stub
 
 
+@define()
+class WatchedOutput:
+    #: The enclosing job
+    job: "Job"
+
+    #: The configuration containing the watched output
+    config: "ConfigInformation"
+
+    #: The watched output (name)
+    method_name: str
+
+    #: The watched output method (called with the JSON event)
+    method: Callable
+
+    #: The callback to call (with the output of the previous method)
+    callback: Callable
+
+
 class ConfigInformation:
     """Holds experimaestro information for a config (or task) instance"""
 
@@ -595,6 +620,9 @@ class ConfigInformation:
         # Initialization tasks
         self.init_tasks: List["LightweightTask"] = []
 
+        # Watched outputs
+        self.watched_outputs: List[WatchedOutput] = []
+
         # Cached information
 
         self._full_identifier = None
@@ -724,9 +752,16 @@ class ConfigInformation:
                 # Generate values
                 for k, argument in config.__xpmtype__.arguments.items():
                     if argument.generator:
-                        config.__xpm__.set(
-                            k, argument.generator(self.context, config), bypass=True
-                        )
+                        sig = inspect.signature(argument.generator)
+                        if len(sig.parameters) == 2:
+                            value = argument.generator(self.context, config)
+                        elif len(sig.parameters) == 0:
+                            value = argument.generator()
+                        else:
+                            assert (
+                                False
+                            ), "generator has either two parameters (context and config), or none"
+                        config.__xpm__.set(k, value, bypass=True)
 
                 config.__xpm__._sealed = True
 
@@ -899,6 +934,19 @@ class ConfigInformation:
         # Now, seal the object
         self.seal(context)
 
+    def watch_output(self, method, callback):
+        """Watch the task output linked with a given method
+
+        :param method: The method to watch
+        :param callback: The callback
+        """
+        watched = WatchedOutput(
+            self, method.__self__, method.__name__, method, callback
+        )
+        self.watched_outputs.append(watched)
+        if self.job:
+            self.job.watch_output(watched)
+
     def submit(
         self,
         workspace: "Workspace",
@@ -1000,7 +1048,7 @@ class ConfigInformation:
 
                 print(file=sys.stderr)  # noqa: T201
 
-        # Handle an output configuration
+        # Handle an output configuration # FIXME: remove
         def mark_output(config: "Config"):
             """Sets a dependency on the job"""
             assert not isinstance(config, Task), "Cannot set a dependency on a task"
@@ -1011,12 +1059,18 @@ class ConfigInformation:
         self.task = self.pyobject
 
         if hasattr(self.pyobject, "task_outputs"):
-            self._taskoutput = self.pyobject.task_outputs(mark_output)
+            self._taskoutput = self.pyobject.task_outputs(self.mark_output)
         else:
             self._taskoutput = self.task = self.pyobject
 
         return self._taskoutput
 
+    def mark_output(self, config: "Config"):
+        """Sets a dependency on the job"""
+        assert not isinstance(config, Task), "Cannot set a dependency on a task"
+        config.__xpm__.task = self.pyobject
+        return config
+
     # --- Serialization
 
     @staticmethod
@@ -1345,7 +1399,9 @@ class ConfigInformation:
                     mod = importlib.import_module(module_name)
                 except ModuleNotFoundError:
                     # More hints on the nature of the error
-                    logging.warning("(1) Either the python path is wrong – %s", ":".join(sys.path))
+                    logging.warning(
+                        "(1) Either the python path is wrong – %s", ":".join(sys.path)
+                    )
                     logging.warning("(2) There is not __init__.py in your module")
                     raise
 
@@ -1790,6 +1846,10 @@ class classproperty(property):
 class Config:
     """Base type for all objects in python interface"""
 
+    __xpmid__: ClassVar[Optional[str]]
+    """Optional configuration ID, mostly useful when moving a class to another
+    package to avoid changes in computed task identifiers"""
+
     __xpmtype__: ClassVar[ObjectType]
     """The object type holds all the information about a specific subclass
     experimaestro metadata"""
@@ -1914,6 +1974,22 @@ class Config:
         """Access pre-tasks"""
         raise AssertionError("Pre-tasks can be accessed only during configuration")
 
+    def register_task_output(self, method, *args, **kwargs):
+        # Determine the path for this...
+        path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        data = json.dumps(
+            {
+                "key": f"{self.__xpmidentifier__}/{method.__name__}",
+                "args": args,
+                "kwargs": kwargs,
+            }
+        )
+        with path.open("at") as fp:
+            fp.writelines([data, "\n"])
+            fp.flush()
+
 
 class LightweightTask(Config):
     """A task that can be run before or after a real task to modify its behaviour"""
@@ -1931,6 +2007,14 @@ class Task(LightweightTask):
     def submit(self):
         raise AssertionError("This method can only be used during configuration")
 
+    def watch_output(self, method, callback):
+        """Sets up a callback
+
+        :param method: a method within a configuration
+        :param callback: the callback
+        """
+        self.__xpm__.watch_output(method, callback)
+
 
 # --- Utility functions
 

experimaestro/core/objects.pyi

@@ -1,4 +1,5 @@
 from abc import ABC
+from attrs import define
 import typing_extensions
 
 from experimaestro.core.types import ObjectType
@@ -108,6 +109,10 @@ class ConfigWalk(ConfigProcessing):
     def map(self, k: str): ...
 
 def getqualattr(module, qualname): ...
+@define(frozen=True)
+class WatchedOutput:
+    config: "Config"
+    method_name: str
 
 class ConfigInformation:
     LOADING: bool
@@ -116,6 +121,7 @@ class ConfigInformation:
     values: Dict[str, Any]
     job: Job
     dependencies: Incomplete
+    watched_outputs: List[WatchedOutput]
     def __init__(self, pyobject: TypeConfig) -> None: ...
     def set_meta(self, value: Optional[bool]): ...
     @property
@@ -231,7 +237,7 @@ class LightweightTask(Config):
     def execute(self) -> None: ...
 
 class Task(LightweightTask):
-    __tags__: Dict[str, str]
+    # __tags__: Dict[str, str]
 
     def submit(
         self,

experimaestro/core/types.py

@@ -26,6 +26,7 @@ if typing.TYPE_CHECKING:
 
 class Identifier:
     def __init__(self, name: str):
+        assert isinstance(name, str)
         self.name = name
 
     def __hash__(self):
@@ -126,6 +127,9 @@ class Type:
         if t:
             return DictType(Type.fromType(t[0]), Type.fromType(t[1]))
 
+        if union_t := typingutils.get_union(key):
+            return UnionType([Type.fromType(t) for t in union_t])
+
         # Takes care of generics
         if get_origin(key):
             return GenericType(key)
@@ -223,7 +227,7 @@ class ObjectType(Type):
             __xpmid__ = getattr(tp, "__xpmid__")
             if isinstance(__xpmid__, Identifier):
                 identifier = __xpmid__
-            if inspect.ismethod(__xpmid__):
+            elif inspect.ismethod(__xpmid__):
                 identifier = Identifier(__xpmid__())
             elif "__xpmid__" in tp.__dict__:
                 identifier = Identifier(__xpmid__)
@@ -576,7 +580,7 @@ class PathType(Type):
             return Path(value.get("$value"))
 
         if not isinstance(value, (str, Path)):
-            raise TypeError("value is not a pathlike value")
+            raise TypeError(f"value is not a pathlike value ({type(value)})")
         return Path(value)
 
     @property
@@ -631,6 +635,32 @@ class EnumType(Type):
         return f"Enum({self.type})"
 
 
+class UnionType(Type):
+    def __init__(self, types: List[Type]):
+        self.types = types
+
+    def name(self):
+        return "Union[" + ", ".join(t.name() for t in self.types) + "]"
+
+    def __str__(self):
+        return "[" + " | ".join(t.name() for t in self.types) + " ]"
+
+    def __repr__(self):
+        return str(self)
+
+    def validate(self, value):
+        for subtype in self.types:
+            try:
+                return subtype.validate(value)
+            except ValueError:
+                pass
+            except TypeError:
+                pass
+
+        if not isinstance(value, dict):
+            raise ValueError(f"value is not within the types {self}")
+
+
 class DictType(Type):
     def __init__(self, keytype: Type, valuetype: Type):
         self.keytype = keytype
@@ -675,6 +705,7 @@ class GenericType(Type):
             (mro for mro in mros if (get_origin(mro) or mro) is self.origin), None
         )
         target = get_origin(self.type) or self.type
+
         if matching is None:
             raise ValueError(
                 f"{type(value)} is not of type {target} ({type(value).__mro__})"