experimaestro 1.7.0rc1__tar.gz → 1.7.0rc3__tar.gz

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.7.0rc1
+Version: 1.7.0rc3
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/pyproject.toml

@@ -20,7 +20,7 @@ include = [
     "src/experimaestro/mkdocs/style.css",
     { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
 ]
-version = "1.7.0-rc1"
+version = "1.7.0-rc3"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -122,7 +122,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.7.0rc0"
+version = "1.7.0rc3"
 changelog_start_rev = "0.15.0"
 tag_format = "v$version"
 update_changelog_on_bump = true

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/commandline.py

@@ -276,12 +276,6 @@ class CommandLineJob(Job):
 
         scriptbuilder = self.launcher.scriptbuilder()
         self.path.mkdir(parents=True, exist_ok=True)
-        donepath = self.donepath
-
-        # Check again if done (now that we have locked)
-        if not overwrite and donepath.is_file():
-            logger.info("Job %s is already done", self)
-            return JobState.DONE
 
         # Now we can write the script
         scriptbuilder.lockfiles.append(self.lockpath)
@@ -293,15 +287,17 @@ class CommandLineJob(Job):
         if self._process:
             return self._process
 
+        # Prepare the files to be run
         scriptPath = self.prepare()
 
+        # OK, now starts the process
         logger.info("Starting job %s", self.jobpath)
         processbuilder = self.launcher.processbuilder()
         processbuilder.environ = self.environ
         processbuilder.command.append(self.launcher.connector.resolve(scriptPath))
         processbuilder.stderr = Redirect.file(self.stderr)
         processbuilder.stdout = Redirect.file(self.stdout)
-        self._process = processbuilder.start()
+        self._process = processbuilder.start(True)
 
         with self.pidpath.open("w") as fp:
             json.dump(self._process.tospec(), fp)

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/connectors/__init__.py

@@ -154,8 +154,11 @@ class ProcessBuilder:
         self.environ: Mapping[str, str] = {}
         self.command = []
 
-    def start(self) -> Process:
-        """Start the process"""
+    def start(self, task_mode: bool = False) -> Process:
+        """Start the process
+
+        :param task_mode: True if the process is a job script
+        """
         raise NotImplementedError("Method not implemented in %s" % self.__class__)
 
 

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/connectors/local.py

@@ -1,5 +1,4 @@
-"""All classes related to localhost management
-"""
+"""All classes related to localhost management"""
 
 import subprocess
 from typing import Optional
@@ -107,8 +106,11 @@ def getstream(redirect: Redirect, write: bool):
 
 
 class LocalProcessBuilder(ProcessBuilder):
-    def start(self):
-        """Start the process"""
+    def start(self, task_mode=False):
+        """Start the process
+
+        :param task_mode: just ignored
+        """
         stdin = getstream(self.stdin, False)
         stdout = getstream(self.stdout, True)
         stderr = getstream(self.stderr, True)
@@ -199,7 +201,9 @@ class LocalConnector(Connector):
         return LocalProcessBuilder()
 
     def resolve(self, path: Path, basepath: Path = None) -> str:
-        assert isinstance(path, PosixPath) or isinstance(path, WindowsPath)
+        assert isinstance(path, PosixPath) or isinstance(
+            path, WindowsPath
+        ), f"Unrecognized path {type(path)}"
         if not basepath:
             return str(path.absolute())
         try:

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/connectors/ssh.py

@@ -200,7 +200,7 @@ class SshProcessBuilder(ProcessBuilder):
         super().__init__()
         self.connector = connector
 
-    def start(self):
+    def start(self, task_mode: bool = False):
         """Start the process"""
 
         trans = str.maketrans({'"': r"\"", "$": r"\$"})

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/experiments/cli.py

@@ -194,7 +194,11 @@ def experiments_cli(  # noqa: C901
             xp_file = Path(xp_file)
             if not python_path:
                 python_path.append(xp_file.parent)
-            logging.info("Using python path: %s", ", ".join(str(s) for s in python_path))
+            logging.info(
+                "Using python path: %s", ", ".join(str(s) for s in python_path)
+            )
+    else:
+        xp_file = Path(xp_file)
 
     assert (
         module_name or xp_file
@@ -269,11 +273,11 @@ def experiments_cli(  # noqa: C901
 
     # Define the workspace
     ws_env = find_workspace(workdir=workdir, workspace=workspace)
-    
+
     workdir = ws_env.path
 
     logging.info("Using working directory %s", str(workdir.resolve()))
-    
+
     # --- Runs the experiment
     with experiment(
         ws_env, configuration.id, host=host, port=port, run_mode=run_mode

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/launcherfinder/registry.py

@@ -8,7 +8,6 @@ import typing
 from omegaconf import DictConfig, OmegaConf, SCMode
 import pkg_resources
 from experimaestro.utils import logger
-
 from .base import ConnectorConfiguration, TokenConfiguration
 from .specs import HostRequirement
 
@@ -158,9 +157,14 @@ class LauncherRegistry:
                 specs.append(spec)
 
         # Use launcher function
+        from experimaestro.launchers import Launcher
+
         if self.find_launcher_fn is not None:
             for spec in specs:
                 if launcher := self.find_launcher_fn(spec, tags):
+                    assert isinstance(
+                        launcher, Launcher
+                    ), "f{self.find_launcher_fn} did not return a Launcher but {type(launcher)}"
                     return launcher
 
         return None

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/launchers/slurm/base.py

@@ -11,6 +11,7 @@ from typing import (
 )
 from experimaestro.connectors.local import LocalConnector
 import re
+from shlex import quote as shquote
 from contextlib import contextmanager
 from dataclasses import dataclass
 from experimaestro.launcherfinder.registry import (
@@ -235,15 +236,15 @@ class SlurmProcessBuilder(ProcessBuilder):
         super().__init__()
         self.launcher = launcher
 
-    def start(self) -> BatchSlurmProcess:
+    def start(self, task_mode: bool = False) -> BatchSlurmProcess:
         """Start the process"""
         builder = self.launcher.connector.processbuilder()
-        builder.workingDirectory = self.workingDirectory
         builder.environ = self.launcher.launcherenv
         builder.detach = False
 
         if not self.detach:
             # Simplest case: we wait for the output
+            builder.workingDirectory = self.workingDirectory
             builder.command = [f"{self.launcher.binpath}/srun"]
             builder.command.extend(self.launcher.options.args())
             builder.command.extend(self.command)
@@ -255,11 +256,17 @@ class SlurmProcessBuilder(ProcessBuilder):
             return builder.start()
 
         builder.command = [f"{self.launcher.binpath}/sbatch", "--parsable"]
-        builder.command.extend(self.launcher.options.args())
 
-        addstream(builder.command, "-e", self.stderr)
-        addstream(builder.command, "-o", self.stdout)
-        addstream(builder.command, "-i", self.stdin)
+        if not task_mode:
+            # Use command line parameters when not running a task
+            builder.command.extend(self.launcher.options.args())
+
+            if self.workingDirectory:
+                workdir = self.launcher.connector.resolve(self.workingDirectory)
+                builder.command.append(f"--chdir={workdir}")
+            addstream(builder.command, "-e", self.stderr)
+            addstream(builder.command, "-o", self.stdout)
+            addstream(builder.command, "-i", self.stdin)
 
         builder.command.extend(self.command)
         logger.info(
@@ -427,12 +434,43 @@ class SlurmLauncher(Launcher):
 
         We assume *nix, but should be changed to PythonScriptBuilder when working
         """
-        builder = PythonScriptBuilder()
-        builder.processtype = "slurm"
-        return builder
+        return SlurmScriptBuilder(self)
 
     def processbuilder(self) -> SlurmProcessBuilder:
         """Returns the process builder for this launcher
 
         By default, returns the associated connector builder"""
         return SlurmProcessBuilder(self)
+
+
+class SlurmScriptBuilder(PythonScriptBuilder):
+    def __init__(self, launcher: SlurmLauncher, pythonpath=None):
+        super().__init__(pythonpath)
+        self.launcher = launcher
+        self.processtype = "slurm"
+
+    def write(self, job):
+        py_path = super().write(job)
+        main_path = py_path.parent
+
+        def relpath(path: Path):
+            return shquote(self.launcher.connector.resolve(path, main_path))
+
+        # Writes the sbatch shell script containing all the options
+        sh_path = job.jobpath / ("%s.sh" % job.name)
+        with sh_path.open("wt") as out:
+            out.write("""#!/bin/sh\n\n""")
+
+            workdir = self.launcher.connector.resolve(main_path)
+            out.write(f"#SBATCH --chdir={shquote(workdir)}\n")
+            out.write(f"""#SBATCH --error={relpath(job.stderr)}\n""")
+            out.write(f"""#SBATCH --output={relpath(job.stdout)}\n""")
+
+            for arg in self.launcher.options.args():
+                out.write(f"""#SBATCH {arg}\n""")
+
+            # We finish by the call to srun
+            out.write(f"""\nsrun ./{relpath(py_path)}\n\n""")
+
+        self.launcher.connector.setExecutable(sh_path, True)
+        return sh_path

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/scheduler/base.py

@@ -740,6 +740,7 @@ class Scheduler:
                     code = await process.aio_code()
                     logger.debug("Got return code %s for %s", code, job)
 
+                    # Check the file if there is no return code
                     if code is None:
                         # Case where we cannot retrieve the code right away
                         if job.donepath.is_file():
@@ -916,6 +917,7 @@ class experiment:
 
         async def awaitcompletion():
             assert self.central is not None
+            logger.debug("Waiting to exit scheduler...")
             async with self.central.exitCondition:
                 while True:
                     if self.exitMode:
@@ -1011,6 +1013,7 @@ class experiment:
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
+        logger.debug("Exiting scheduler context")
         # If no exception and normal run mode, remove old "jobs"
         if self.workspace.run_mode == RunMode.NORMAL:
             if exc_type is None and self.jobsbakpath.is_dir():
@@ -1051,6 +1054,7 @@ class experiment:
             # Put back old experiment as current one
             experiment.CURRENT = self.old_experiment
             if self.server:
+                logger.info("Stopping web server")
                 self.server.stop()
 
     async def update_task_output_count(self, delta: int):

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/scheduler/dynamic_outputs.py

@@ -135,7 +135,7 @@ class TaskOutputsWorker(threading.Thread):
     """This worker process dynamic output queue for one experiment"""
 
     def __init__(self, xp: experiment):
-        super().__init__(name="task outputs worker")
+        super().__init__(name="task outputs worker", daemon=True)
         self.queue = queue.Queue()
         self.xp = xp
 

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/scriptbuilder.py

@@ -51,6 +51,8 @@ class PythonScriptBuilder:
         self.lockfiles: List[Path] = []
         self.notificationURL: Optional[str] = None
         self.command: Optional[AbstractCommand] = None
+
+        # This is used to serialize the full process identifier on disk
         self.processtype = "local"
 
     def write(self, job: CommandLineJob):
@@ -63,7 +65,7 @@ class PythonScriptBuilder:
             job {CommandLineJob} -- [description]
 
         Returns:
-            [type] -- [description]
+            str -- The script path on disk
         """
         assert isinstance(
             job, CommandLineJob

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/sphinx/__init__.py

@@ -10,19 +10,16 @@ from docutils import nodes
 from sphinx.application import Sphinx
 from sphinx import addnodes
 from sphinx.ext.autodoc import ClassDocumenter, Documenter, restify
-from sphinx.locale import _, __
+from sphinx.locale import _
 from sphinx.util import inspect, logging
 from sphinx.domains.python import (
     PyClasslike,
     PyAttribute,
-    PyObject,
-    directives,
-    desc_signature,
-    _parse_annotation,
+    directives,  # noqa: F401
 )
+from sphinx.addnodes import desc_signature
 from sphinx.util.typing import OptionSpec
 from docutils.statemachine import StringList
-import logging
 import re
 
 from experimaestro import Config, Task
@@ -97,9 +94,6 @@ class ConfigDocumenter(ClassDocumenter):
         can_document = inspect.isclass(member) and issubclass(member, Config)
         return can_document
 
-    def add_directive_header(self, sig: str) -> None:
-        super().add_directive_header(sig)
-
     def get_object_members(self, want_all: bool):  # -> Tuple[bool, ObjectMembers]:
         r = super().get_object_members(want_all)
         return r
@@ -157,7 +151,7 @@ class ConfigDocumenter(ClassDocumenter):
 
         # Our specific code
         if issubclass(self.object, Task):
-            self.add_line(f"   :task:", sourcename)
+            self.add_line("   :task:", sourcename)
 
         # add inheritance info, if wanted
         if not self.doc_as_attr and self.options.show_inheritance:
@@ -196,7 +190,6 @@ class ConfigDocumenter(ClassDocumenter):
     def add_content(
         self, more_content: Optional[StringList], no_docstring: bool = False
     ) -> None:
-
         xpminfo = getxpminfo(self.object)
         source_name = self.get_sourcename()
 
@@ -214,9 +207,9 @@ class ConfigDocumenter(ClassDocumenter):
                     source_name,
                 )
             if argument.generator:
-                self.add_line(f"   :generated:", source_name)
+                self.add_line("   :generated:", source_name)
             elif argument.constant:
-                self.add_line(f"   :constant:", source_name)
+                self.add_line("   :constant:", source_name)
 
             # self.add_line("", source_name)
             if argument.help:

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/tests/launchers/bin/sbatch

@@ -2,8 +2,11 @@
 
 # slurm test suite
 
+pwd 1>&2
+CURDIR="$(realpath "$(dirname "$0")"/..)"
+
 # Where we store the jobs
-XPM_SLURM_DIR="$(realpath "$(dirname "$0")"/..)/slurm"
+XPM_SLURM_DIR="${CURDIR}/slurm"
 if ! test -d "$XPM_SLURM_DIR"; then
     echo "Directory $XPM_SLURM_DIR does not exist" 1>&2
     exit 1
@@ -51,8 +54,19 @@ while true; do
     esac
 done
 
-echo "Starting $@ ${args[@]} > $stdout" >&2
+chdir=$(pwd)
+while IFS= read -r line; do
+    case "$line" in
+        "#SBATCH --output="*)stdout=${line#*#SBATCH --output=};;
+        "#SBATCH --error="*) stderr=${line#*#SBATCH --error=};;
+        "#SBATCH --chdir="*) chdir=${line#*#SBATCH --chdir=};;
+    esac
+done < "$1"
+
+echo "Starting $@ ${args[@]} > $stdout 2> $stderr" >&2
 (
+    if test "$chdir"; then cd "$chdir"; fi
+    export PATH="${CURDIR}/bin:$PATH"
     eval "$@" "${args[@]}"
     echo $? > "$XPM_SLURM_DIR/jobs/$$.status"
 ) > $stdout 2> $stderr &

experimaestro-1.7.0rc3/src/experimaestro/tests/launchers/bin/srun

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Just run the command
+
+eval "$@"

{experimaestro-1.7.0rc1 → experimaestro-1.7.0rc3}/src/experimaestro/tests/launchers/common.py

@@ -80,10 +80,14 @@ def takeback(launcher, datapath, txp1, txp2):
     waiting = datapath / "waiting"
 
     with txp1:
-        WaitUntilTouched(touching=touching, waiting=waiting).submit(launcher=launcher)
+        task: WaitUntilTouched = WaitUntilTouched(
+            touching=touching, waiting=waiting
+        ).submit(launcher=launcher)
 
         logger.debug("Waiting for task to create 'touching' file")
         while not touching.is_file():
+            if task.__xpm__.job.state.finished():
+                raise Exception("Job has finished... too early")
             time.sleep(0.01)
 
         with txp2: