experimaestro 1.6.1__py3-none-any.whl → 1.7.0rc0__py3-none-any.whl

experimaestro/__init__.py

@@ -36,15 +36,17 @@ from .core.arguments import (
     DataPath,
     Annotated,
     Constant,
+    field,
     # Annotations helpers
     help,
     default,
 )
-from .generators import pathgenerator
+from .generators import pathgenerator, PathGenerator
 from .core.objects import (
     Config,
     copyconfig,
     setmeta,
+    DependentMarker,
     Task,
     LightweightTask,
     ObjectStore,

experimaestro/annotations.py

@@ -8,7 +8,7 @@ import experimaestro.core.objects as objects
 import experimaestro.core.types as types
 from experimaestro.generators import PathGenerator
 
-from .core.arguments import Argument as CoreArgument
+from .core.arguments import Argument as CoreArgument, field
 from .core.objects import Config
 from .core.types import Any, Identifier, TypeProxy, Type, ObjectType
 from .utils import logger
@@ -134,12 +134,22 @@ class param:
         self.type = Type.fromType(type) if type else None
         self.help = help
         self.ignored = ignored
-        self.default = default
         self.required = required
-        self.generator = None
         self.checker = checker
         self.constant = constant
 
+        self.generator = None
+        self.default = None
+
+        # Set default or generator
+        if isinstance(default, field):
+            if default.default is not None:
+                self.default = default
+            elif default.default_factory is not None:
+                self.generator = default.default_factory
+        else:
+            self.default = default
+
     def __call__(self, tp):
         # Don't annotate in task mode
         tp.__getxpmtype__().addAnnotation(self)

experimaestro/cli/filter.py

@@ -3,7 +3,7 @@ import pyparsing as pp
 from pathlib import Path
 import json
 from experimaestro.compat import cached_property
-import regex
+import re
 from experimaestro.scheduler import JobState
 
 
@@ -87,7 +87,7 @@ class NotInExpr(BaseInExpr):
 class RegexExpr:
     def __init__(self, tokens):
         self.var, expr = tokens
-        self.regex = regex.compile(expr)
+        self.regex = re.compile(expr)
 
     def __repr__(self):
         return f"""REGEX[{self.varname}, {self.value}]"""
@@ -103,7 +103,7 @@ class RegexExpr:
         if not value:
             return False
 
-        return self.regex.match(value)
+        return self.re.match(value)
 
 
 class ConstantString:

experimaestro/cli/jobs.py

@@ -69,7 +69,7 @@ def process(
         if (p / "jobs.bak").is_dir():
             cprint(f"  Experiment {p.name} has not finished yet", "red")
             if (not perform) and (kill or clean):
-                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                cprint("  Preventing kill/clean (use --perform if you want to)", "yellow")
                 kill = False
                 clean = False
 

experimaestro/connectors/__init__.py

@@ -9,7 +9,8 @@ This module contains :
 """
 
 import enum
-from typing import Any, Dict, Mapping, Type, Union
+import logging
+from typing import Any, Dict, Mapping, Type, Union, Optional
 from pathlib import Path
 from experimaestro.utils import logger
 from experimaestro.locking import Lock
@@ -86,12 +87,12 @@ class Process:
     @staticmethod
     def fromDefinition(connector: "Connector", definition: Dict[str, Any]) -> "Process":
         """Retrieves a process from a serialized definition"""
-        handler = Process.handler(definition["type"])
+        handler_type = definition["type"]
+        handler = Process.handler(handler_type)
+        assert handler is not None, f"No handler of type {handler_type}"
         try:
             return handler.fromspec(connector, definition)
         except Exception as e:
-            import logging
-
             logging.exception("Could not retrieve job from specification")
             raise e
 
@@ -101,7 +102,11 @@ class Process:
         if Process.HANDLERS is None:
             Process.HANDLERS = {}
             for ep in pkg_resources.iter_entry_points(group="experimaestro.process"):
-                Process.HANDLERS[ep.name] = ep.load()
+                logging.debug("Adding process handler for type %s", ep.name)
+                handler = ep.load()
+                Process.HANDLERS[ep.name] = handler
+                if handler is None:
+                    logging.error("Handler of type %s is null", ep.name)
 
         return Process.HANDLERS.get(key, None)
 
@@ -117,10 +122,14 @@ class Process:
         """True is the process is truly running (I/O)"""
         return (await self.aio_state()).running
 
-    async def aio_code(self):
-        """Returns a future containing the returned code"""
+    async def aio_code(self) -> Optional[int]:
+        """Returns a future containing the returned code
+
+        Returns None if the process has already finished – and no information is
+        known about the process.
+        """
         code = await asyncThreadcheck("aio_code", self.wait)
-        logger.debug("Got for return code %s: %s", self, code)
+        logger.debug("Got return code %s for %s", code, self)
         return code
 
     def kill(self):

experimaestro/connectors/local.py

@@ -2,6 +2,7 @@
 """
 
 import subprocess
+from typing import Optional
 from pathlib import Path, WindowsPath, PosixPath
 import os
 import threading
@@ -29,11 +30,13 @@ class PsutilProcess(Process):
     def __init__(self, pid: int):
         self._process = psutil.Process(pid)
 
-    def wait(self) -> int:
+    def wait(self) -> Optional[int]:
         logger.debug("Waiting (psutil) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (psutil) for process with PID %s", self._process.pid
+            "Finished to wait (psutil) for process with PID %s: code %s",
+            self._process.pid,
+            code,
         )
         return code
 
@@ -57,7 +60,9 @@ class LocalProcess(Process):
         logger.debug("Waiting (python) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (python) for process with PID %s", self._process.pid
+            "Finished to wait (python) for process with PID %s: %s",
+            self._process.pid,
+            code,
         )
         return code
 

experimaestro/core/arguments.py

@@ -1,6 +1,6 @@
 """Management of the arguments (params, options, etc) associated with the XPM objects"""
 
-from typing import Optional, TypeVar, TYPE_CHECKING
+from typing import Optional, TypeVar, TYPE_CHECKING, Callable, Any
 from experimaestro.typingutils import get_optional
 from pathlib import Path
 import sys
@@ -75,11 +75,22 @@ class Argument:
         self.constant = constant
         self.ignored = self.type.ignore if ignored is None else ignored
         self.required = required
-        self.default = default
-        self.generator = generator
         self.objecttype = None
         self.is_data = is_data
 
+        self.generator = generator
+        self.default = None
+
+        if default is not None:
+            assert self.generator is None, "generator and default are exclusive options"
+            if isinstance(default, field):
+                if default.default is not None:
+                    self.default = default.default
+                elif default.default_factory is not None:
+                    self.generator = default.default_factory
+            else:
+                self.default = default
+
         assert (
             not self.constant or self.default is not None
         ), "Cannot be constant without default"
@@ -170,6 +181,18 @@ DataPath = Annotated[Path, dataHint]
 """Annotates a path that should be kept to restore an object to its state"""
 
 
+class field:
+    """Extra information for a given experimaestro field (param or meta)"""
+
+    def __init__(self, *, default: Any = None, default_factory: Callable = None):
+        assert not (
+            (default is not None) and (default_factory is not None)
+        ), "default and default_factory are mutually exclusive options"
+
+        self.default_factory = default_factory
+        self.default = default
+
+
 class help(TypeAnnotation):
     def __init__(self, text: str):
         self.text = text

experimaestro/core/objects.py

@@ -3,6 +3,10 @@
 from functools import cached_property
 import json
 
+from attr import define
+
+from experimaestro import taskglobals
+
 try:
     from types import NoneType
 except Exception:
@@ -21,6 +25,7 @@ import inspect
 import importlib
 from typing import (
     Any,
+    Callable,
     ClassVar,
     Dict,
     Iterator,
@@ -136,6 +141,8 @@ class ConfigPath:
 
 hash_logger = logging.getLogger("xpm.hash")
 
+DependentMarker = Callable[["Config"], None]
+
 
 class HashComputer:
     """This class is in charge of computing a config/task identifier"""
@@ -555,6 +562,24 @@ class ObjectStore:
         self.store[identifier] = stub
 
 
+@define()
+class WatchedOutput:
+    #: The enclosing job
+    job: "Job"
+
+    #: The configuration containing the watched output
+    config: "ConfigInformation"
+
+    #: The watched output (name)
+    method_name: str
+
+    #: The watched output method (called with the JSON event)
+    method: Callable
+
+    #: The callback to call (with the output of the previous method)
+    callback: Callable
+
+
 class ConfigInformation:
     """Holds experimaestro information for a config (or task) instance"""
 
@@ -595,6 +620,9 @@ class ConfigInformation:
         # Initialization tasks
         self.init_tasks: List["LightweightTask"] = []
 
+        # Watched outputs
+        self.watched_outputs: List[WatchedOutput] = []
+
         # Cached information
 
         self._full_identifier = None
@@ -724,9 +752,16 @@ class ConfigInformation:
                 # Generate values
                 for k, argument in config.__xpmtype__.arguments.items():
                     if argument.generator:
-                        config.__xpm__.set(
-                            k, argument.generator(self.context, config), bypass=True
-                        )
+                        sig = inspect.signature(argument.generator)
+                        if len(sig.parameters) == 2:
+                            value = argument.generator(self.context, config)
+                        elif len(sig.parameters) == 0:
+                            value = argument.generator()
+                        else:
+                            assert (
+                                False
+                            ), "generator has either two parameters (context and config), or none"
+                        config.__xpm__.set(k, value, bypass=True)
 
                 config.__xpm__._sealed = True
 
@@ -899,6 +934,19 @@ class ConfigInformation:
         # Now, seal the object
         self.seal(context)
 
+    def watch_output(self, method, callback):
+        """Watch the task output linked with a given method
+
+        :param method: The method to watch
+        :param callback: The callback
+        """
+        watched = WatchedOutput(
+            self, method.__self__, method.__name__, method, callback
+        )
+        self.watched_outputs.append(watched)
+        if self.job:
+            self.job.watch_output(watched)
+
     def submit(
         self,
         workspace: "Workspace",
@@ -1000,7 +1048,7 @@ class ConfigInformation:
 
                 print(file=sys.stderr)  # noqa: T201
 
-        # Handle an output configuration
+        # Handle an output configuration # FIXME: remove
         def mark_output(config: "Config"):
             """Sets a dependency on the job"""
             assert not isinstance(config, Task), "Cannot set a dependency on a task"
@@ -1011,12 +1059,18 @@ class ConfigInformation:
         self.task = self.pyobject
 
         if hasattr(self.pyobject, "task_outputs"):
-            self._taskoutput = self.pyobject.task_outputs(mark_output)
+            self._taskoutput = self.pyobject.task_outputs(self.mark_output)
         else:
             self._taskoutput = self.task = self.pyobject
 
         return self._taskoutput
 
+    def mark_output(self, config: "Config"):
+        """Sets a dependency on the job"""
+        assert not isinstance(config, Task), "Cannot set a dependency on a task"
+        config.__xpm__.task = self.pyobject
+        return config
+
     # --- Serialization
 
     @staticmethod
@@ -1345,7 +1399,9 @@ class ConfigInformation:
                     mod = importlib.import_module(module_name)
                 except ModuleNotFoundError:
                     # More hints on the nature of the error
-                    logging.warning("(1) Either the python path is wrong – %s", ":".join(sys.path))
+                    logging.warning(
+                        "(1) Either the python path is wrong – %s", ":".join(sys.path)
+                    )
                     logging.warning("(2) There is not __init__.py in your module")
                     raise
 
@@ -1790,6 +1846,10 @@ class classproperty(property):
 class Config:
     """Base type for all objects in python interface"""
 
+    __xpmid__: ClassVar[Optional[str]]
+    """Optional configuration ID, mostly useful when moving a class to another
+    package to avoid changes in computed task identifiers"""
+
     __xpmtype__: ClassVar[ObjectType]
     """The object type holds all the information about a specific subclass
     experimaestro metadata"""
@@ -1914,6 +1974,22 @@ class Config:
         """Access pre-tasks"""
         raise AssertionError("Pre-tasks can be accessed only during configuration")
 
+    def register_task_output(self, method, *args, **kwargs):
+        # Determine the path for this...
+        path = taskglobals.Env.instance().xpm_path / "task-outputs.jsonl"
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        data = json.dumps(
+            {
+                "key": f"{self.__xpmidentifier__}/{method.__name__}",
+                "args": args,
+                "kwargs": kwargs,
+            }
+        )
+        with path.open("at") as fp:
+            fp.writelines([data, "\n"])
+            fp.flush()
+
 
 class LightweightTask(Config):
     """A task that can be run before or after a real task to modify its behaviour"""
@@ -1931,6 +2007,14 @@ class Task(LightweightTask):
     def submit(self):
         raise AssertionError("This method can only be used during configuration")
 
+    def watch_output(self, method, callback):
+        """Sets up a callback
+
+        :param method: a method within a configuration
+        :param callback: the callback
+        """
+        self.__xpm__.watch_output(method, callback)
+
 
 # --- Utility functions
 

experimaestro/core/objects.pyi

@@ -1,4 +1,5 @@
 from abc import ABC
+from attrs import define
 import typing_extensions
 
 from experimaestro.core.types import ObjectType
@@ -108,6 +109,10 @@ class ConfigWalk(ConfigProcessing):
     def map(self, k: str): ...
 
 def getqualattr(module, qualname): ...
+@define(frozen=True)
+class WatchedOutput:
+    config: "Config"
+    method_name: str
 
 class ConfigInformation:
     LOADING: bool
@@ -116,6 +121,7 @@ class ConfigInformation:
     values: Dict[str, Any]
     job: Job
     dependencies: Incomplete
+    watched_outputs: List[WatchedOutput]
     def __init__(self, pyobject: TypeConfig) -> None: ...
     def set_meta(self, value: Optional[bool]): ...
     @property
@@ -231,7 +237,7 @@ class LightweightTask(Config):
     def execute(self) -> None: ...
 
 class Task(LightweightTask):
-    __tags__: Dict[str, str]
+    # __tags__: Dict[str, str]
 
     def submit(
         self,

experimaestro/core/types.py

@@ -26,6 +26,7 @@ if typing.TYPE_CHECKING:
 
 class Identifier:
     def __init__(self, name: str):
+        assert isinstance(name, str)
         self.name = name
 
     def __hash__(self):
@@ -126,6 +127,9 @@ class Type:
         if t:
             return DictType(Type.fromType(t[0]), Type.fromType(t[1]))
 
+        if union_t := typingutils.get_union(key):
+            return UnionType([Type.fromType(t) for t in union_t])
+
         # Takes care of generics
         if get_origin(key):
             return GenericType(key)
@@ -223,7 +227,7 @@ class ObjectType(Type):
             __xpmid__ = getattr(tp, "__xpmid__")
             if isinstance(__xpmid__, Identifier):
                 identifier = __xpmid__
-            if inspect.ismethod(__xpmid__):
+            elif inspect.ismethod(__xpmid__):
                 identifier = Identifier(__xpmid__())
             elif "__xpmid__" in tp.__dict__:
                 identifier = Identifier(__xpmid__)
@@ -576,7 +580,7 @@ class PathType(Type):
             return Path(value.get("$value"))
 
         if not isinstance(value, (str, Path)):
-            raise TypeError("value is not a pathlike value")
+            raise TypeError(f"value is not a pathlike value ({type(value)})")
         return Path(value)
 
     @property
@@ -631,6 +635,32 @@ class EnumType(Type):
         return f"Enum({self.type})"
 
 
+class UnionType(Type):
+    def __init__(self, types: List[Type]):
+        self.types = types
+
+    def name(self):
+        return "Union[" + ", ".join(t.name() for t in self.types) + "]"
+
+    def __str__(self):
+        return "[" + " | ".join(t.name() for t in self.types) + " ]"
+
+    def __repr__(self):
+        return str(self)
+
+    def validate(self, value):
+        for subtype in self.types:
+            try:
+                return subtype.validate(value)
+            except ValueError:
+                pass
+            except TypeError:
+                pass
+
+        if not isinstance(value, dict):
+            raise ValueError(f"value is not within the types {self}")
+
+
 class DictType(Type):
     def __init__(self, keytype: Type, valuetype: Type):
         self.keytype = keytype
@@ -675,6 +705,7 @@ class GenericType(Type):
             (mro for mro in mros if (get_origin(mro) or mro) is self.origin), None
         )
         target = get_origin(self.type) or self.type
+
         if matching is None:
             raise ValueError(
                 f"{type(value)} is not of type {target} ({type(value).__mro__})"

experimaestro/experiments/cli.py

@@ -59,7 +59,7 @@ class ExperimentCallable(Protocol):
 class ConfigurationLoader:
     def __init__(self):
         self.yamls = []
-        self.pythonpath = set()
+        self.python_path = set()
 
     def load(self, yaml_file: Path):
         """Loads a YAML file, and parents one if they exist"""
@@ -76,9 +76,9 @@ class ConfigurationLoader:
         for path in _data.get("pythonpath", []):
             path = Path(path)
             if path.is_absolute():
-                self.pythonpath.add(path.resolve())
+                self.python_path.add(path.resolve())
             else:
-                self.pythonpath.add((yaml_file.parent / path).resolve())
+                self.python_path.add((yaml_file.parent / path).resolve())
 
 
 @click.option("--debug", is_flag=True, help="Print debug information")
@@ -181,7 +181,7 @@ def experiments_cli(  # noqa: C901
         configuration.merge_with(OmegaConf.from_dotlist(extra_conf))
 
     # --- Get the XP file
-    pythonpath = list(conf_loader.pythonpath)
+    python_path = list(conf_loader.python_path)
     if module_name is None:
         module_name = configuration.get("module", None)
 
@@ -192,9 +192,9 @@ def experiments_cli(  # noqa: C901
                 not module_name
             ), "Module name and experiment file are mutually exclusive options"
             xp_file = Path(xp_file)
-            if not pythonpath:
-                pythonpath.append(xp_file.parent)
-            logging.info("Using python path: %s", ", ".join(str(s) for s in pythonpath))
+            if not python_path:
+                python_path.append(xp_file.parent)
+            logging.info("Using python path: %s", ", ".join(str(s) for s in python_path))
 
     assert (
         module_name or xp_file
@@ -209,7 +209,7 @@ def experiments_cli(  # noqa: C901
     # --- Finds the "run" function
 
     # Modifies the Python path
-    for path in pythonpath:
+    for path in python_path:
         sys.path.append(str(path))
 
     if xp_file:
@@ -226,7 +226,11 @@ def experiments_cli(  # noqa: C901
             )
     else:
         # Module
-        mod = importlib.import_module(module_name)
+        try:
+            mod = importlib.import_module(module_name)
+        except ModuleNotFoundError as e:
+            logging.error("Module not found: %s with python path %s", e, sys.path)
+            raise
 
     helper = getattr(mod, "run", None)
 
@@ -265,10 +269,11 @@ def experiments_cli(  # noqa: C901
 
     # Define the workspace
     ws_env = find_workspace(workdir=workdir, workspace=workspace)
+    
     workdir = ws_env.path
 
     logging.info("Using working directory %s", str(workdir.resolve()))
-
+    
     # --- Runs the experiment
     with experiment(
         ws_env, configuration.id, host=host, port=port, run_mode=run_mode
@@ -278,6 +283,9 @@ def experiments_cli(  # noqa: C901
         for key, value in env:
             xp.setenv(key, value)
 
+        # Sets the python path
+        xp.workspace.python_path.extend(python_path)
+
         try:
             # Run the experiment
             helper.xp = xp

experimaestro/generators.py

@@ -1,11 +1,12 @@
 import inspect
 from pathlib import Path
+from abc import ABC, abstractmethod
 from typing import Callable, Union
 from experimaestro.core.arguments import ArgumentOptions, TypeAnnotation
 from experimaestro.core.objects import ConfigWalkContext, Config
 
 
-class Generator:
+class Generator(ABC):
     """Base class for all generators"""
 
     def isoutput(self):
@@ -13,6 +14,10 @@ class Generator:
         path within the job folder)"""
         return False
 
+    @abstractmethod
+    def __call__(self, context: ConfigWalkContext, config: Config):
+        ...
+
 
 class PathGenerator(Generator):
     """Generates a path"""

experimaestro/ipc.py

@@ -7,6 +7,7 @@ import sys
 import logging
 from .utils import logger
 from watchdog.observers import Observer
+from watchdog.observers.api import ObservedWatch
 from watchdog.events import FileSystemEventHandler
 
 
@@ -20,7 +21,9 @@ class IPCom:
         self.observer.start()
         self.pid = os.getpid()
 
-    def fswatch(self, watcher: FileSystemEventHandler, path: Path, recursive=False):
+    def fswatch(
+        self, watcher: FileSystemEventHandler, path: Path, recursive=False
+    ) -> ObservedWatch:
         if not self.observer.is_alive():
             logging.error("Observer is not alive")
 

experimaestro/launcherfinder/registry.py

@@ -78,13 +78,16 @@ class LauncherRegistry:
 
             from importlib import util
 
-            spec = util.spec_from_file_location("xpm_launchers_conf", launchers_py)
-            module = util.module_from_spec(spec)
-            spec.loader.exec_module(module)
+            with launchers_py.__fspath__() as fp:
+                spec = util.spec_from_file_location("xpm_launchers_conf", fp)
+                module = util.module_from_spec(spec)
+                spec.loader.exec_module(module)
 
             self.find_launcher_fn = getattr(module, "find_launcher", None)
             if self.find_launcher_fn is None:
-                logger.warn("No find_launcher() function was found in %s", launchers_py)
+                logger.warning(
+                    "No find_launcher() function was found in %s", launchers_py
+                )
 
         # Read the configuration file
         self.connectors = load_yaml(