experimaestro 1.7.0rc2__tar.gz → 1.7.1__tar.gz

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.7.0rc2
+Version: 1.7.1
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager
@@ -34,7 +34,6 @@ Requires-Dist: huggingface-hub (>0.17)
 Requires-Dist: humanfriendly (>=10,<11)
 Requires-Dist: marshmallow (>=3.20,<4.0)
 Requires-Dist: omegaconf (>=2.3,<3.0)
-Requires-Dist: protobuf (>5)
 Requires-Dist: psutil (>=7)
 Requires-Dist: pyparsing (>=3.1,<4.0)
 Requires-Dist: pytools (>=2023.1.1,<2024.0.0)

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/pyproject.toml

@@ -20,7 +20,7 @@ include = [
     "src/experimaestro/mkdocs/style.css",
     { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
 ]
-version = "1.7.0-rc2"
+version = "1.7.1"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -64,7 +64,6 @@ watchdog = "^2"
 marshmallow = "^3.20"
 decorator = "^5"
 rpyc = ">=5,<7"
-protobuf = ">5" 
 
 [tool.poetry.group.ssh]
 optional = true
@@ -123,7 +122,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.7.0rc0"
-changelog_start_rev = "0.15.0"
-tag_format = "v$version"
-update_changelog_on_bump = true
+version = "1.7.1"
+changelog_start_rev = "v1.0.0"
+tag_format = "v$major.$minor.$patch$prerelease"
+# update_changelog_on_bump = true

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/commandline.py

@@ -276,12 +276,6 @@ class CommandLineJob(Job):
 
         scriptbuilder = self.launcher.scriptbuilder()
         self.path.mkdir(parents=True, exist_ok=True)
-        donepath = self.donepath
-
-        # Check again if done (now that we have locked)
-        if not overwrite and donepath.is_file():
-            logger.info("Job %s is already done", self)
-            return JobState.DONE
 
         # Now we can write the script
         scriptbuilder.lockfiles.append(self.lockpath)
@@ -293,15 +287,17 @@ class CommandLineJob(Job):
         if self._process:
             return self._process
 
+        # Prepare the files to be run
         scriptPath = self.prepare()
 
+        # OK, now starts the process
         logger.info("Starting job %s", self.jobpath)
         processbuilder = self.launcher.processbuilder()
         processbuilder.environ = self.environ
         processbuilder.command.append(self.launcher.connector.resolve(scriptPath))
         processbuilder.stderr = Redirect.file(self.stderr)
         processbuilder.stdout = Redirect.file(self.stdout)
-        self._process = processbuilder.start()
+        self._process = processbuilder.start(True)
 
         with self.pidpath.open("w") as fp:
             json.dump(self._process.tospec(), fp)

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/connectors/__init__.py

@@ -154,8 +154,11 @@ class ProcessBuilder:
         self.environ: Mapping[str, str] = {}
         self.command = []
 
-    def start(self) -> Process:
-        """Start the process"""
+    def start(self, task_mode: bool = False) -> Process:
+        """Start the process
+
+        :param task_mode: True if the process is a job script
+        """
         raise NotImplementedError("Method not implemented in %s" % self.__class__)
 
 

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/connectors/local.py

@@ -1,5 +1,4 @@
-"""All classes related to localhost management
-"""
+"""All classes related to localhost management"""
 
 import subprocess
 from typing import Optional
@@ -107,8 +106,11 @@ def getstream(redirect: Redirect, write: bool):
 
 
 class LocalProcessBuilder(ProcessBuilder):
-    def start(self):
-        """Start the process"""
+    def start(self, task_mode=False):
+        """Start the process
+
+        :param task_mode: just ignored
+        """
         stdin = getstream(self.stdin, False)
         stdout = getstream(self.stdout, True)
         stderr = getstream(self.stderr, True)
@@ -199,7 +201,9 @@ class LocalConnector(Connector):
         return LocalProcessBuilder()
 
     def resolve(self, path: Path, basepath: Path = None) -> str:
-        assert isinstance(path, PosixPath) or isinstance(path, WindowsPath)
+        assert isinstance(path, PosixPath) or isinstance(
+            path, WindowsPath
+        ), f"Unrecognized path {type(path)}"
         if not basepath:
             return str(path.absolute())
         try:

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/connectors/ssh.py

@@ -200,7 +200,7 @@ class SshProcessBuilder(ProcessBuilder):
         super().__init__()
         self.connector = connector
 
-    def start(self):
+    def start(self, task_mode: bool = False):
         """Start the process"""
 
         trans = str.maketrans({'"': r"\"", "$": r"\$"})

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/experiments/cli.py

@@ -1,4 +1,3 @@
-import imp
 import importlib
 import inspect
 import json
@@ -68,6 +67,12 @@ class ConfigurationLoader:
 
         with yaml_file.open("rt") as fp:
             _data = yaml.full_load(fp)
+
+        if "file" in _data:
+            path = Path(_data["file"])
+            if not path.is_absolute():
+                _data["file"] = str((yaml_file.parent / path).resolve())
+
         if parent := _data.get("parent", None):
             self.load(yaml_file.parent / parent)
 
@@ -193,7 +198,7 @@ def experiments_cli(  # noqa: C901
             ), "Module name and experiment file are mutually exclusive options"
             xp_file = Path(xp_file)
             if not python_path:
-                python_path.append(xp_file.parent)
+                python_path.append(xp_file.parent.absolute())
             logging.info(
                 "Using python path: %s", ", ".join(str(s) for s in python_path)
             )
@@ -220,14 +225,12 @@ def experiments_cli(  # noqa: C901
         if not xp_file.exists() and xp_file.suffix != ".py":
             xp_file = xp_file.with_suffix(".py")
         xp_file: Path = Path(yaml_file).parent / xp_file
-        with open(xp_file) as src:
-            module_name = xp_file.with_suffix("").name
-            mod = imp.load_module(
-                module_name,
-                src,
-                str(xp_file.absolute()),
-                (".py", "r", imp.PY_SOURCE),
-            )
+        module_name = xp_file.with_suffix("").name
+        spec = importlib.util.spec_from_file_location(
+            module_name, str(xp_file.absolute())
+        )
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
     else:
         # Module
         try:

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/launcherfinder/registry.py

@@ -8,7 +8,6 @@ import typing
 from omegaconf import DictConfig, OmegaConf, SCMode
 import pkg_resources
 from experimaestro.utils import logger
-
 from .base import ConnectorConfiguration, TokenConfiguration
 from .specs import HostRequirement
 
@@ -158,9 +157,14 @@ class LauncherRegistry:
                 specs.append(spec)
 
         # Use launcher function
+        from experimaestro.launchers import Launcher
+
         if self.find_launcher_fn is not None:
             for spec in specs:
                 if launcher := self.find_launcher_fn(spec, tags):
+                    assert isinstance(
+                        launcher, Launcher
+                    ), "f{self.find_launcher_fn} did not return a Launcher but {type(launcher)}"
                     return launcher
 
         return None

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/launchers/slurm/base.py

@@ -11,6 +11,7 @@ from typing import (
 )
 from experimaestro.connectors.local import LocalConnector
 import re
+from shlex import quote as shquote
 from contextlib import contextmanager
 from dataclasses import dataclass
 from experimaestro.launcherfinder.registry import (
@@ -235,15 +236,15 @@ class SlurmProcessBuilder(ProcessBuilder):
         super().__init__()
         self.launcher = launcher
 
-    def start(self) -> BatchSlurmProcess:
+    def start(self, task_mode: bool = False) -> BatchSlurmProcess:
         """Start the process"""
         builder = self.launcher.connector.processbuilder()
-        builder.workingDirectory = self.workingDirectory
         builder.environ = self.launcher.launcherenv
         builder.detach = False
 
         if not self.detach:
             # Simplest case: we wait for the output
+            builder.workingDirectory = self.workingDirectory
             builder.command = [f"{self.launcher.binpath}/srun"]
             builder.command.extend(self.launcher.options.args())
             builder.command.extend(self.command)
@@ -255,11 +256,17 @@ class SlurmProcessBuilder(ProcessBuilder):
             return builder.start()
 
         builder.command = [f"{self.launcher.binpath}/sbatch", "--parsable"]
-        builder.command.extend(self.launcher.options.args())
 
-        addstream(builder.command, "-e", self.stderr)
-        addstream(builder.command, "-o", self.stdout)
-        addstream(builder.command, "-i", self.stdin)
+        if not task_mode:
+            # Use command line parameters when not running a task
+            builder.command.extend(self.launcher.options.args())
+
+            if self.workingDirectory:
+                workdir = self.launcher.connector.resolve(self.workingDirectory)
+                builder.command.append(f"--chdir={workdir}")
+            addstream(builder.command, "-e", self.stderr)
+            addstream(builder.command, "-o", self.stdout)
+            addstream(builder.command, "-i", self.stdin)
 
         builder.command.extend(self.command)
         logger.info(
@@ -427,12 +434,43 @@ class SlurmLauncher(Launcher):
 
         We assume *nix, but should be changed to PythonScriptBuilder when working
         """
-        builder = PythonScriptBuilder()
-        builder.processtype = "slurm"
-        return builder
+        return SlurmScriptBuilder(self)
 
     def processbuilder(self) -> SlurmProcessBuilder:
         """Returns the process builder for this launcher
 
         By default, returns the associated connector builder"""
         return SlurmProcessBuilder(self)
+
+
+class SlurmScriptBuilder(PythonScriptBuilder):
+    def __init__(self, launcher: SlurmLauncher, pythonpath=None):
+        super().__init__(pythonpath)
+        self.launcher = launcher
+        self.processtype = "slurm"
+
+    def write(self, job):
+        py_path = super().write(job)
+        main_path = py_path.parent
+
+        def relpath(path: Path):
+            return shquote(self.launcher.connector.resolve(path, main_path))
+
+        # Writes the sbatch shell script containing all the options
+        sh_path = job.jobpath / ("%s.sh" % job.name)
+        with sh_path.open("wt") as out:
+            out.write("""#!/bin/sh\n\n""")
+
+            workdir = self.launcher.connector.resolve(main_path)
+            out.write(f"#SBATCH --chdir={shquote(workdir)}\n")
+            out.write(f"""#SBATCH --error={relpath(job.stderr)}\n""")
+            out.write(f"""#SBATCH --output={relpath(job.stdout)}\n""")
+
+            for arg in self.launcher.options.args():
+                out.write(f"""#SBATCH {arg}\n""")
+
+            # We finish by the call to srun
+            out.write(f"""\nsrun ./{relpath(py_path)}\n\n""")
+
+        self.launcher.connector.setExecutable(sh_path, True)
+        return sh_path

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/notifications.py

@@ -12,16 +12,15 @@ from tqdm.auto import tqdm as std_tqdm
 
 from .utils import logger
 from experimaestro.taskglobals import Env as TaskEnv
-from experimaestro.notifications_pb2 import LevelInformation
 
 # --- Progress and other notifications
 
 T = TypeVar("T")
 
+
 @dataclass
 class LevelInformation:
     level: int
-    
     desc: Optional[str]
     progress: float
 
@@ -214,7 +213,6 @@ class Reporter(threading.Thread):
 
                 self.cv.notify_all()
 
-    #: The reporter instance
     INSTANCE: ClassVar[Optional["Reporter"]] = None
 
     @staticmethod

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/scheduler/base.py

@@ -740,6 +740,7 @@ class Scheduler:
                     code = await process.aio_code()
                     logger.debug("Got return code %s for %s", code, job)
 
+                    # Check the file if there is no return code
                     if code is None:
                         # Case where we cannot retrieve the code right away
                         if job.donepath.is_file():
@@ -916,6 +917,7 @@ class experiment:
 
         async def awaitcompletion():
             assert self.central is not None
+            logger.debug("Waiting to exit scheduler...")
             async with self.central.exitCondition:
                 while True:
                     if self.exitMode:
@@ -1011,6 +1013,7 @@ class experiment:
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
+        logger.debug("Exiting scheduler context")
         # If no exception and normal run mode, remove old "jobs"
         if self.workspace.run_mode == RunMode.NORMAL:
             if exc_type is None and self.jobsbakpath.is_dir():
@@ -1051,6 +1054,7 @@ class experiment:
             # Put back old experiment as current one
             experiment.CURRENT = self.old_experiment
             if self.server:
+                logger.info("Stopping web server")
                 self.server.stop()
 
     async def update_task_output_count(self, delta: int):

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/scheduler/dynamic_outputs.py

@@ -135,7 +135,7 @@ class TaskOutputsWorker(threading.Thread):
     """This worker process dynamic output queue for one experiment"""
 
     def __init__(self, xp: experiment):
-        super().__init__(name="task outputs worker")
+        super().__init__(name="task outputs worker", daemon=True)
         self.queue = queue.Queue()
         self.xp = xp
 

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/scriptbuilder.py

@@ -51,6 +51,8 @@ class PythonScriptBuilder:
         self.lockfiles: List[Path] = []
         self.notificationURL: Optional[str] = None
         self.command: Optional[AbstractCommand] = None
+
+        # This is used to serialize the full process identifier on disk
         self.processtype = "local"
 
     def write(self, job: CommandLineJob):
@@ -63,7 +65,7 @@ class PythonScriptBuilder:
             job {CommandLineJob} -- [description]
 
         Returns:
-            [type] -- [description]
+            str -- The script path on disk
         """
         assert isinstance(
             job, CommandLineJob

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/sphinx/__init__.py

@@ -10,19 +10,13 @@ from docutils import nodes
 from sphinx.application import Sphinx
 from sphinx import addnodes
 from sphinx.ext.autodoc import ClassDocumenter, Documenter, restify
-from sphinx.locale import _, __
+from sphinx.locale import _
 from sphinx.util import inspect, logging
-from sphinx.domains.python import (
-    PyClasslike,
-    PyAttribute,
-    PyObject,
-    directives,
-    desc_signature,
-    _parse_annotation,
-)
+from sphinx.domains.python import PyClasslike, PyAttribute, directives
+from sphinx.domains.python import PyObject  # noqa: F401
+from sphinx.addnodes import desc_signature
 from sphinx.util.typing import OptionSpec
 from docutils.statemachine import StringList
-import logging
 import re
 
 from experimaestro import Config, Task
@@ -97,9 +91,6 @@ class ConfigDocumenter(ClassDocumenter):
         can_document = inspect.isclass(member) and issubclass(member, Config)
         return can_document
 
-    def add_directive_header(self, sig: str) -> None:
-        super().add_directive_header(sig)
-
     def get_object_members(self, want_all: bool):  # -> Tuple[bool, ObjectMembers]:
         r = super().get_object_members(want_all)
         return r
@@ -157,7 +148,7 @@ class ConfigDocumenter(ClassDocumenter):
 
         # Our specific code
         if issubclass(self.object, Task):
-            self.add_line(f"   :task:", sourcename)
+            self.add_line("   :task:", sourcename)
 
         # add inheritance info, if wanted
         if not self.doc_as_attr and self.options.show_inheritance:
@@ -196,7 +187,6 @@ class ConfigDocumenter(ClassDocumenter):
     def add_content(
         self, more_content: Optional[StringList], no_docstring: bool = False
     ) -> None:
-
         xpminfo = getxpminfo(self.object)
         source_name = self.get_sourcename()
 
@@ -214,9 +204,9 @@ class ConfigDocumenter(ClassDocumenter):
                     source_name,
                 )
             if argument.generator:
-                self.add_line(f"   :generated:", source_name)
+                self.add_line("   :generated:", source_name)
             elif argument.constant:
-                self.add_line(f"   :constant:", source_name)
+                self.add_line("   :constant:", source_name)
 
             # self.add_line("", source_name)
             if argument.help:

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/tests/launchers/bin/sbatch

@@ -2,8 +2,11 @@
 
 # slurm test suite
 
+pwd 1>&2
+CURDIR="$(realpath "$(dirname "$0")"/..)"
+
 # Where we store the jobs
-XPM_SLURM_DIR="$(realpath "$(dirname "$0")"/..)/slurm"
+XPM_SLURM_DIR="${CURDIR}/slurm"
 if ! test -d "$XPM_SLURM_DIR"; then
     echo "Directory $XPM_SLURM_DIR does not exist" 1>&2
     exit 1
@@ -51,8 +54,19 @@ while true; do
     esac
 done
 
-echo "Starting $@ ${args[@]} > $stdout" >&2
+chdir=$(pwd)
+while IFS= read -r line; do
+    case "$line" in
+        "#SBATCH --output="*)stdout=${line#*#SBATCH --output=};;
+        "#SBATCH --error="*) stderr=${line#*#SBATCH --error=};;
+        "#SBATCH --chdir="*) chdir=${line#*#SBATCH --chdir=};;
+    esac
+done < "$1"
+
+cd "$chdir"
+echo "Starting $@ ${args[@]} > $stdout 2> $stderr" >&2
 (
+    export PATH="${CURDIR}/bin:$PATH"
     eval "$@" "${args[@]}"
     echo $? > "$XPM_SLURM_DIR/jobs/$$.status"
 ) > $stdout 2> $stderr &

experimaestro-1.7.1/src/experimaestro/tests/launchers/bin/srun

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Just run the command
+
+eval "$@"

{experimaestro-1.7.0rc2 → experimaestro-1.7.1}/src/experimaestro/tests/launchers/common.py

@@ -80,10 +80,14 @@ def takeback(launcher, datapath, txp1, txp2):
     waiting = datapath / "waiting"
 
     with txp1:
-        WaitUntilTouched(touching=touching, waiting=waiting).submit(launcher=launcher)
+        task: WaitUntilTouched = WaitUntilTouched(
+            touching=touching, waiting=waiting
+        ).submit(launcher=launcher)
 
         logger.debug("Waiting for task to create 'touching' file")
         while not touching.is_file():
+            if task.__xpm__.job.state.finished():
+                raise Exception("Job has finished... too early")
             time.sleep(0.01)
 
         with txp2:

experimaestro-1.7.0rc2/src/experimaestro/notifications.proto

@@ -1,17 +0,0 @@
-syntax = "proto3";
-import "google/protobuf/any.proto";
-
-
-message Envelope {
-    google.protobuf.Any payload = 1;
-}
-
-message LevelInformation {
-    int32 level = 1;
-
-    optional string desc = 2; // Optional in proto3 is handled by default unless you want to use 'optional'
-    float progress = 3;
-
-    float previous_progress = 4;
-    string previous_desc = 5;
-}

experimaestro-1.7.0rc2/src/experimaestro/notifications_pb2.py

@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# NO CHECKED-IN PROTOBUF GENCODE
-# source: src/experimaestro/notifications.proto
-# Protobuf Python Version: 5.29.3
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import runtime_version as _runtime_version
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-_runtime_version.ValidateProtobufRuntimeVersion(
-    _runtime_version.Domain.PUBLIC,
-    5,
-    29,
-    3,
-    '',
-    'src/experimaestro/notifications.proto'
-)
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%src/experimaestro/notifications.proto\x1a\x19google/protobuf/any.proto\"1\n\x08\x45nvelope\x12%\n\x07payload\x18\x01 \x01(\x0b\x32\x14.google.protobuf.Any\"\x81\x01\n\x10LevelInformation\x12\r\n\x05level\x18\x01 \x01(\x05\x12\x11\n\x04\x64\x65sc\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x08progress\x18\x03 \x01(\x02\x12\x19\n\x11previous_progress\x18\x04 \x01(\x02\x12\x15\n\rprevious_desc\x18\x05 \x01(\tB\x07\n\x05_descb\x06proto3')
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'src.experimaestro.notifications_pb2', _globals)
-if not _descriptor._USE_C_DESCRIPTORS:
-  DESCRIPTOR._loaded_options = None
-  _globals['_ENVELOPE']._serialized_start=68
-  _globals['_ENVELOPE']._serialized_end=117
-  _globals['_LEVELINFORMATION']._serialized_start=120
-  _globals['_LEVELINFORMATION']._serialized_end=249
-# @@protoc_insertion_point(module_scope)