experimaestro 1.6.1__py3-none-any.whl → 1.15.2__py3-none-any.whl

experimaestro/__init__.py

@@ -27,7 +27,15 @@ from .annotations import (
     # Method
     config_only,
 )
-from .core.serialization import load, save, state_dict, from_state_dict, from_task_dir
+from .core.serialization import (
+    load,
+    save,
+    state_dict,
+    from_state_dict,
+    from_task_dir,
+    serialize,
+    deserialize,
+)
 from .core.arguments import (
     # Types
     Param,
@@ -36,15 +44,17 @@ from .core.arguments import (
     DataPath,
     Annotated,
     Constant,
+    field,
     # Annotations helpers
     help,
     default,
 )
-from .generators import pathgenerator
+from .generators import pathgenerator, PathGenerator
 from .core.objects import (
     Config,
     copyconfig,
     setmeta,
+    DependentMarker,
     Task,
     LightweightTask,
     ObjectStore,
@@ -53,8 +63,9 @@ from .core.context import SerializationContext
 from .core.serializers import SerializationLWTask, PathSerializationLWTask
 from .core.types import Any, SubmitHook
 from .launchers import Launcher
-from .scheduler.workspace import Workspace, RunMode
 from .scheduler import Scheduler, experiment, FailedExperiment
+from .scheduler.workspace import Workspace, RunMode
+from .scheduler.state import get_experiment
 from .notifications import progress, tqdm
 from .checkers import Choices
 from .xpmutils import DirectoryContext

experimaestro/annotations.py

@@ -8,7 +8,7 @@ import experimaestro.core.objects as objects
 import experimaestro.core.types as types
 from experimaestro.generators import PathGenerator
 
-from .core.arguments import Argument as CoreArgument
+from .core.arguments import Argument as CoreArgument, field
 from .core.objects import Config
 from .core.types import Any, Identifier, TypeProxy, Type, ObjectType
 from .utils import logger
@@ -134,12 +134,22 @@ class param:
         self.type = Type.fromType(type) if type else None
         self.help = help
         self.ignored = ignored
-        self.default = default
         self.required = required
-        self.generator = None
         self.checker = checker
         self.constant = constant
 
+        self.generator = None
+        self.default = None
+
+        # Set default or generator
+        if isinstance(default, field):
+            if default.default is not None:
+                self.default = default
+            elif default.default_factory is not None:
+                self.generator = default.default_factory
+        else:
+            self.default = default
+
     def __call__(self, tp):
         # Don't annotate in task mode
         tp.__getxpmtype__().addAnnotation(self)

experimaestro/cli/filter.py

@@ -1,20 +1,27 @@
+import asyncio
+import logging
 from typing import Any, Callable, Dict, List, Optional
 import pyparsing as pp
 from pathlib import Path
 import json
 from experimaestro.compat import cached_property
-import regex
+import re
 from experimaestro.scheduler import JobState
 
 
 class JobInformation:
-    def __init__(self, path: Path, scriptname: str):
+    def __init__(self, path: Path, scriptname: str, check: bool = False):
         self.path = path
         self.scriptname = scriptname
+        self.check = check
 
     @cached_property
     def params(self):
-        return json.loads((self.path / "params.json").read_text())
+        try:
+            return json.loads((self.path / "params.json").read_text())
+        except Exception:
+            logging.warning("Could not load params.json in %s", self.path)
+            return {"tags": {}}
 
     @cached_property
     def tags(self) -> List[str]:
@@ -27,6 +34,13 @@ class JobInformation:
         if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
         if (self.path / f"{self.scriptname}.pid").is_file():
+            if self.check:
+                if process := self.getprocess():
+                    state = asyncio.run(process.aio_state(0))
+                    if state is None or state.finished:
+                        return JobState.ERROR
+                else:
+                    return JobState.ERROR
             return JobState.RUNNING
         else:
             return None
@@ -87,7 +101,7 @@ class NotInExpr(BaseInExpr):
 class RegexExpr:
     def __init__(self, tokens):
         self.var, expr = tokens
-        self.regex = regex.compile(expr)
+        self.regex = re.compile(expr)
 
     def __repr__(self):
         return f"""REGEX[{self.varname}, {self.value}]"""
@@ -103,7 +117,7 @@ class RegexExpr:
         if not value:
             return False
 
-        return self.regex.match(value)
+        return self.re.match(value)
 
 
 class ConstantString:

experimaestro/cli/jobs.py

@@ -1,4 +1,5 @@
 # flake8: noqa: T201
+import asyncio
 import subprocess
 from typing import Optional
 from shutil import rmtree
@@ -49,6 +50,7 @@ def process(
     filter="",
     perform=False,
     fullpath=False,
+    check=False,
 ):
     from .filter import createFilter, JobInformation
     from experimaestro.scheduler import JobState
@@ -63,13 +65,14 @@ def process(
         for job in p.glob("jobs/*/*"):
             job_path = job.resolve()
             if job_path.is_dir():
-                *_, scriptname = job_path.parent.name.rsplit(".", 1)
-                job2xp.setdefault(scriptname, set()).add(p.name)
+                job2xp.setdefault(job_path.name, set()).add(p.name)
 
         if (p / "jobs.bak").is_dir():
             cprint(f"  Experiment {p.name} has not finished yet", "red")
             if (not perform) and (kill or clean):
-                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                cprint(
+                    "  Preventing kill/clean (use --perform if you want to)", "yellow"
+                )
                 kill = False
                 clean = False
 
@@ -79,11 +82,11 @@ def process(
         p = job.resolve()
         if p.is_dir():
             *_, scriptname = p.parent.name.rsplit(".", 1)
-            xps = job2xp.get(scriptname, set())
+            xps = job2xp.get(job.name, set())
             if experiment and experiment not in xps:
                 continue
 
-            info = JobInformation(p, scriptname)
+            info = JobInformation(p, scriptname, check=check)
             job_str = (
                 (str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}")
                 + " "
@@ -151,6 +154,7 @@ def process(
 @click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
+@click.option("--no-check", is_flag=True, help="Check that running jobs")
 @jobs.command()
 @click.pass_context
 def list(
@@ -160,6 +164,7 @@ def list(
     tags: bool,
     ready: bool,
     fullpath: bool,
+    no_check: bool,
 ):
     process(
         ctx.obj.workspace,
@@ -168,6 +173,7 @@ def list(
         tags=tags,
         ready=ready,
         fullpath=fullpath,
+        check=not no_check,
     )
 
 
@@ -187,6 +193,7 @@ def kill(
     ready: bool,
     fullpath: bool,
     perform: bool,
+    check: bool,
 ):
     process(
         ctx.obj.workspace,

experimaestro/commandline.py

@@ -276,12 +276,6 @@ class CommandLineJob(Job):
 
         scriptbuilder = self.launcher.scriptbuilder()
         self.path.mkdir(parents=True, exist_ok=True)
-        donepath = self.donepath
-
-        # Check again if done (now that we have locked)
-        if not overwrite and donepath.is_file():
-            logger.info("Job %s is already done", self)
-            return JobState.DONE
 
         # Now we can write the script
         scriptbuilder.lockfiles.append(self.lockpath)
@@ -293,15 +287,17 @@ class CommandLineJob(Job):
         if self._process:
             return self._process
 
+        # Prepare the files to be run
         scriptPath = self.prepare()
 
+        # OK, now starts the process
         logger.info("Starting job %s", self.jobpath)
         processbuilder = self.launcher.processbuilder()
         processbuilder.environ = self.environ
         processbuilder.command.append(self.launcher.connector.resolve(scriptPath))
         processbuilder.stderr = Redirect.file(self.stderr)
         processbuilder.stdout = Redirect.file(self.stdout)
-        self._process = processbuilder.start()
+        self._process = processbuilder.start(True)
 
         with self.pidpath.open("w") as fp:
             json.dump(self._process.tospec(), fp)

experimaestro/connectors/__init__.py

@@ -9,7 +9,8 @@ This module contains :
 """
 
 import enum
-from typing import Any, Dict, Mapping, Type, Union
+import logging
+from typing import Any, Dict, Mapping, Type, Union, Optional
 from pathlib import Path
 from experimaestro.utils import logger
 from experimaestro.locking import Lock
@@ -86,12 +87,12 @@ class Process:
     @staticmethod
     def fromDefinition(connector: "Connector", definition: Dict[str, Any]) -> "Process":
         """Retrieves a process from a serialized definition"""
-        handler = Process.handler(definition["type"])
+        handler_type = definition["type"]
+        handler = Process.handler(handler_type)
+        assert handler is not None, f"No handler of type {handler_type}"
         try:
             return handler.fromspec(connector, definition)
         except Exception as e:
-            import logging
-
             logging.exception("Could not retrieve job from specification")
             raise e
 
@@ -101,7 +102,11 @@ class Process:
         if Process.HANDLERS is None:
             Process.HANDLERS = {}
             for ep in pkg_resources.iter_entry_points(group="experimaestro.process"):
-                Process.HANDLERS[ep.name] = ep.load()
+                logging.debug("Adding process handler for type %s", ep.name)
+                handler = ep.load()
+                Process.HANDLERS[ep.name] = handler
+                if handler is None:
+                    logging.error("Handler of type %s is null", ep.name)
 
         return Process.HANDLERS.get(key, None)
 
@@ -109,18 +114,25 @@ class Process:
         """Wait until the process finishes and returns the error code"""
         raise NotImplementedError(f"Not implemented: {self.__class__}.wait")
 
-    async def aio_state(self) -> ProcessState:
-        """Returns the job state"""
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
+        """Returns the job state
+
+        :param timeout: maximum waiting time for a refresh
+        """
         raise NotImplementedError(f"Not implemented: {self.__class__}.aio_state")
 
     async def aio_isrunning(self):
         """True is the process is truly running (I/O)"""
         return (await self.aio_state()).running
 
-    async def aio_code(self):
-        """Returns a future containing the returned code"""
+    async def aio_code(self) -> Optional[int]:
+        """Returns a future containing the returned code
+
+        Returns None if the process has already finished – and no information is
+        known about the process.
+        """
         code = await asyncThreadcheck("aio_code", self.wait)
-        logger.debug("Got for return code %s: %s", self, code)
+        logger.debug("Got return code %s for %s", code, self)
         return code
 
     def kill(self):
@@ -145,8 +157,11 @@ class ProcessBuilder:
         self.environ: Mapping[str, str] = {}
         self.command = []
 
-    def start(self) -> Process:
-        """Start the process"""
+    def start(self, task_mode: bool = False) -> Process:
+        """Start the process
+
+        :param task_mode: True if the process is a job script
+        """
         raise NotImplementedError("Method not implemented in %s" % self.__class__)
 
 

experimaestro/connectors/local.py

@@ -1,7 +1,7 @@
-"""All classes related to localhost management
-"""
+"""All classes related to localhost management"""
 
 import subprocess
+from typing import Optional
 from pathlib import Path, WindowsPath, PosixPath
 import os
 import threading
@@ -29,15 +29,17 @@ class PsutilProcess(Process):
     def __init__(self, pid: int):
         self._process = psutil.Process(pid)
 
-    def wait(self) -> int:
+    def wait(self) -> Optional[int]:
         logger.debug("Waiting (psutil) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (psutil) for process with PID %s", self._process.pid
+            "Finished to wait (psutil) for process with PID %s: code %s",
+            self._process.pid,
+            code,
         )
         return code
 
-    async def aio_state(self):
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
         if self._process.is_running():
             return ProcessState.RUNNING
         return ProcessState.FINISHED
@@ -57,11 +59,13 @@ class LocalProcess(Process):
         logger.debug("Waiting (python) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (python) for process with PID %s", self._process.pid
+            "Finished to wait (python) for process with PID %s: %s",
+            self._process.pid,
+            code,
         )
         return code
 
-    async def aio_state(self):
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
         code = self._process.poll()
         if code is None:
             return ProcessState.RUNNING
@@ -102,8 +106,11 @@ def getstream(redirect: Redirect, write: bool):
 
 
 class LocalProcessBuilder(ProcessBuilder):
-    def start(self):
-        """Start the process"""
+    def start(self, task_mode=False):
+        """Start the process
+
+        :param task_mode: just ignored
+        """
         stdin = getstream(self.stdin, False)
         stdout = getstream(self.stdout, True)
         stderr = getstream(self.stderr, True)
@@ -194,7 +201,9 @@ class LocalConnector(Connector):
         return LocalProcessBuilder()
 
     def resolve(self, path: Path, basepath: Path = None) -> str:
-        assert isinstance(path, PosixPath) or isinstance(path, WindowsPath)
+        assert isinstance(path, PosixPath) or isinstance(
+            path, WindowsPath
+        ), f"Unrecognized path {type(path)}"
         if not basepath:
             return str(path.absolute())
         try:

experimaestro/connectors/ssh.py

@@ -200,7 +200,7 @@ class SshProcessBuilder(ProcessBuilder):
         super().__init__()
         self.connector = connector
 
-    def start(self):
+    def start(self, task_mode: bool = False):
         """Start the process"""
 
         trans = str.maketrans({'"': r"\"", "$": r"\$"})

experimaestro/core/arguments.py

@@ -1,6 +1,6 @@
 """Management of the arguments (params, options, etc) associated with the XPM objects"""
 
-from typing import Optional, TypeVar, TYPE_CHECKING
+from typing import Optional, TypeVar, TYPE_CHECKING, Callable, Any
 from experimaestro.typingutils import get_optional
 from pathlib import Path
 import sys
@@ -75,11 +75,24 @@ class Argument:
         self.constant = constant
         self.ignored = self.type.ignore if ignored is None else ignored
         self.required = required
-        self.default = default
-        self.generator = generator
         self.objecttype = None
         self.is_data = is_data
 
+        self.generator = generator
+        self.default = None
+        self.ignore_generated = False
+
+        if default is not None:
+            assert self.generator is None, "generator and default are exclusive options"
+            if isinstance(default, field):
+                self.ignore_generated = default.ignore_generated
+                if default.default is not None:
+                    self.default = default.default
+                elif default.default_factory is not None:
+                    self.generator = default.default_factory
+            else:
+                self.default = default
+
         assert (
             not self.constant or self.default is not None
         ), "Cannot be constant without default"
@@ -170,6 +183,25 @@ DataPath = Annotated[Path, dataHint]
 """Annotates a path that should be kept to restore an object to its state"""
 
 
+class field:
+    """Extra information for a given experimaestro field (param or meta)"""
+
+    def __init__(
+        self,
+        *,
+        default: Any = None,
+        default_factory: Callable = None,
+        ignore_generated=False,
+    ):
+        assert not (
+            (default is not None) and (default_factory is not None)
+        ), "default and default_factory are mutually exclusive options"
+
+        self.default_factory = default_factory
+        self.default = default
+        self.ignore_generated = ignore_generated
+
+
 class help(TypeAnnotation):
     def __init__(self, text: str):
         self.text = text

experimaestro/core/callbacks.py

@@ -0,0 +1,52 @@
+from collections import defaultdict
+import threading
+from typing import Callable, ClassVar, Optional
+from experimaestro.core.objects import ConfigInformation
+from experimaestro.scheduler import Listener, Job, JobState, experiment
+
+
+class TaskEventListener(Listener):
+    INSTANCE: ClassVar[Optional["TaskEventListener"]] = None
+    """The general instance"""
+
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.experiments: set[int] = set()
+
+        self._on_completed: defaultdict[int, Callable] = defaultdict(list)
+
+    @staticmethod
+    def connect(xp: experiment):
+        _self = TaskEventListener.instance()
+        with _self.lock:
+            if id(xp) not in _self.experiments:
+                _self.experiments.add(id(xp))
+                xp.scheduler.addlistener(_self)
+
+    @staticmethod
+    def instance():
+        if TaskEventListener.INSTANCE is None:
+            TaskEventListener.INSTANCE = TaskEventListener()
+
+        return TaskEventListener.INSTANCE
+
+    def job_state(self, job: Job):
+        if job.state == JobState.DONE:
+            with self.lock:
+                for callback in self._on_completed.get(id(job.config.__xpm__), []):
+                    callback()
+
+    @staticmethod
+    def on_completed(
+        config_information: ConfigInformation, callback: Callable[[], None]
+    ):
+        instance = TaskEventListener.instance()
+
+        with instance.lock:
+            instance._on_completed[id(config_information)].append(callback)
+
+            if (
+                config_information.job is not None
+                and config_information.job == JobState.DONE
+            ):
+                callback()

experimaestro/core/context.py

@@ -1,22 +1,21 @@
 from contextlib import contextmanager
 from pathlib import Path
+
+try:
+    from pathlib import UnsupportedOperation
+except ImportError:
+    UnsupportedOperation = OSError
 import shutil
 from typing import List, Optional, Protocol, Set, Union
 import os
-import sys
-
-has_hardlink_to = sys.version_info.major == 3 and sys.version_info.minor >= 10
 
 
 def shallow_copy(src_path: Path, dest_path: Path):
     """Copy a directory or file, trying to use hard links if possible"""
     if src_path.is_file():
         try:
-            if has_hardlink_to:
-                dest_path.hardlink_to(src_path)
-            else:
-                dest_path.link_to(src_path)
-        except OSError:
+            dest_path.hardlink_to(src_path)
+        except (NotImplementedError, UnsupportedOperation, OSError):
             shutil.copy(src_path, dest_path)
     else:
         if dest_path.exists():
@@ -74,7 +73,7 @@ class SerializationContext:
 
 
 class SerializedPathLoader(Protocol):
-    def __call__(path: Union[Path, str, SerializedPath]) -> Path:
+    def __call__(self, path: Union[Path, str, SerializedPath]) -> Path:
         """Get a filesystem path from a relative path
 
         :param path: The relative path