experimaestro 1.7.0rc2__py3-none-any.whl → 1.7.0rc3__py3-none-any.whl

experimaestro/commandline.py

@@ -276,12 +276,6 @@ class CommandLineJob(Job):
 
         scriptbuilder = self.launcher.scriptbuilder()
         self.path.mkdir(parents=True, exist_ok=True)
-        donepath = self.donepath
-
-        # Check again if done (now that we have locked)
-        if not overwrite and donepath.is_file():
-            logger.info("Job %s is already done", self)
-            return JobState.DONE
 
         # Now we can write the script
         scriptbuilder.lockfiles.append(self.lockpath)
@@ -293,15 +287,17 @@ class CommandLineJob(Job):
         if self._process:
             return self._process
 
+        # Prepare the files to be run
         scriptPath = self.prepare()
 
+        # OK, now starts the process
         logger.info("Starting job %s", self.jobpath)
         processbuilder = self.launcher.processbuilder()
         processbuilder.environ = self.environ
         processbuilder.command.append(self.launcher.connector.resolve(scriptPath))
         processbuilder.stderr = Redirect.file(self.stderr)
         processbuilder.stdout = Redirect.file(self.stdout)
-        self._process = processbuilder.start()
+        self._process = processbuilder.start(True)
 
         with self.pidpath.open("w") as fp:
             json.dump(self._process.tospec(), fp)

experimaestro/connectors/__init__.py

@@ -154,8 +154,11 @@ class ProcessBuilder:
         self.environ: Mapping[str, str] = {}
         self.command = []
 
-    def start(self) -> Process:
-        """Start the process"""
+    def start(self, task_mode: bool = False) -> Process:
+        """Start the process
+
+        :param task_mode: True if the process is a job script
+        """
         raise NotImplementedError("Method not implemented in %s" % self.__class__)
 
 

experimaestro/connectors/local.py

@@ -1,5 +1,4 @@
-"""All classes related to localhost management
-"""
+"""All classes related to localhost management"""
 
 import subprocess
 from typing import Optional
@@ -107,8 +106,11 @@ def getstream(redirect: Redirect, write: bool):
 
 
 class LocalProcessBuilder(ProcessBuilder):
-    def start(self):
-        """Start the process"""
+    def start(self, task_mode=False):
+        """Start the process
+
+        :param task_mode: just ignored
+        """
         stdin = getstream(self.stdin, False)
         stdout = getstream(self.stdout, True)
         stderr = getstream(self.stderr, True)
@@ -199,7 +201,9 @@ class LocalConnector(Connector):
         return LocalProcessBuilder()
 
     def resolve(self, path: Path, basepath: Path = None) -> str:
-        assert isinstance(path, PosixPath) or isinstance(path, WindowsPath)
+        assert isinstance(path, PosixPath) or isinstance(
+            path, WindowsPath
+        ), f"Unrecognized path {type(path)}"
         if not basepath:
             return str(path.absolute())
         try:

experimaestro/connectors/ssh.py

@@ -200,7 +200,7 @@ class SshProcessBuilder(ProcessBuilder):
         super().__init__()
         self.connector = connector
 
-    def start(self):
+    def start(self, task_mode: bool = False):
         """Start the process"""
 
         trans = str.maketrans({'"': r"\"", "$": r"\$"})

experimaestro/launcherfinder/registry.py

@@ -8,7 +8,6 @@ import typing
 from omegaconf import DictConfig, OmegaConf, SCMode
 import pkg_resources
 from experimaestro.utils import logger
-
 from .base import ConnectorConfiguration, TokenConfiguration
 from .specs import HostRequirement
 
@@ -158,9 +157,14 @@ class LauncherRegistry:
                 specs.append(spec)
 
         # Use launcher function
+        from experimaestro.launchers import Launcher
+
         if self.find_launcher_fn is not None:
             for spec in specs:
                 if launcher := self.find_launcher_fn(spec, tags):
+                    assert isinstance(
+                        launcher, Launcher
+                    ), "f{self.find_launcher_fn} did not return a Launcher but {type(launcher)}"
                     return launcher
 
         return None

experimaestro/launchers/slurm/base.py

@@ -11,6 +11,7 @@ from typing import (
 )
 from experimaestro.connectors.local import LocalConnector
 import re
+from shlex import quote as shquote
 from contextlib import contextmanager
 from dataclasses import dataclass
 from experimaestro.launcherfinder.registry import (
@@ -235,15 +236,15 @@ class SlurmProcessBuilder(ProcessBuilder):
         super().__init__()
         self.launcher = launcher
 
-    def start(self) -> BatchSlurmProcess:
+    def start(self, task_mode: bool = False) -> BatchSlurmProcess:
         """Start the process"""
         builder = self.launcher.connector.processbuilder()
-        builder.workingDirectory = self.workingDirectory
         builder.environ = self.launcher.launcherenv
         builder.detach = False
 
         if not self.detach:
             # Simplest case: we wait for the output
+            builder.workingDirectory = self.workingDirectory
             builder.command = [f"{self.launcher.binpath}/srun"]
             builder.command.extend(self.launcher.options.args())
             builder.command.extend(self.command)
@@ -255,11 +256,17 @@ class SlurmProcessBuilder(ProcessBuilder):
             return builder.start()
 
         builder.command = [f"{self.launcher.binpath}/sbatch", "--parsable"]
-        builder.command.extend(self.launcher.options.args())
 
-        addstream(builder.command, "-e", self.stderr)
-        addstream(builder.command, "-o", self.stdout)
-        addstream(builder.command, "-i", self.stdin)
+        if not task_mode:
+            # Use command line parameters when not running a task
+            builder.command.extend(self.launcher.options.args())
+
+            if self.workingDirectory:
+                workdir = self.launcher.connector.resolve(self.workingDirectory)
+                builder.command.append(f"--chdir={workdir}")
+            addstream(builder.command, "-e", self.stderr)
+            addstream(builder.command, "-o", self.stdout)
+            addstream(builder.command, "-i", self.stdin)
 
         builder.command.extend(self.command)
         logger.info(
@@ -427,12 +434,43 @@ class SlurmLauncher(Launcher):
 
         We assume *nix, but should be changed to PythonScriptBuilder when working
         """
-        builder = PythonScriptBuilder()
-        builder.processtype = "slurm"
-        return builder
+        return SlurmScriptBuilder(self)
 
     def processbuilder(self) -> SlurmProcessBuilder:
         """Returns the process builder for this launcher
 
         By default, returns the associated connector builder"""
         return SlurmProcessBuilder(self)
+
+
+class SlurmScriptBuilder(PythonScriptBuilder):
+    def __init__(self, launcher: SlurmLauncher, pythonpath=None):
+        super().__init__(pythonpath)
+        self.launcher = launcher
+        self.processtype = "slurm"
+
+    def write(self, job):
+        py_path = super().write(job)
+        main_path = py_path.parent
+
+        def relpath(path: Path):
+            return shquote(self.launcher.connector.resolve(path, main_path))
+
+        # Writes the sbatch shell script containing all the options
+        sh_path = job.jobpath / ("%s.sh" % job.name)
+        with sh_path.open("wt") as out:
+            out.write("""#!/bin/sh\n\n""")
+
+            workdir = self.launcher.connector.resolve(main_path)
+            out.write(f"#SBATCH --chdir={shquote(workdir)}\n")
+            out.write(f"""#SBATCH --error={relpath(job.stderr)}\n""")
+            out.write(f"""#SBATCH --output={relpath(job.stdout)}\n""")
+
+            for arg in self.launcher.options.args():
+                out.write(f"""#SBATCH {arg}\n""")
+
+            # We finish by the call to srun
+            out.write(f"""\nsrun ./{relpath(py_path)}\n\n""")
+
+        self.launcher.connector.setExecutable(sh_path, True)
+        return sh_path

experimaestro/notifications.py

@@ -12,16 +12,15 @@ from tqdm.auto import tqdm as std_tqdm
 
 from .utils import logger
 from experimaestro.taskglobals import Env as TaskEnv
-from experimaestro.notifications_pb2 import LevelInformation
 
 # --- Progress and other notifications
 
 T = TypeVar("T")
 
+
 @dataclass
 class LevelInformation:
     level: int
-    
     desc: Optional[str]
     progress: float
 
@@ -214,7 +213,6 @@ class Reporter(threading.Thread):
 
                 self.cv.notify_all()
 
-    #: The reporter instance
     INSTANCE: ClassVar[Optional["Reporter"]] = None
 
     @staticmethod

experimaestro/scheduler/base.py

@@ -740,6 +740,7 @@ class Scheduler:
                     code = await process.aio_code()
                     logger.debug("Got return code %s for %s", code, job)
 
+                    # Check the file if there is no return code
                     if code is None:
                         # Case where we cannot retrieve the code right away
                         if job.donepath.is_file():
@@ -916,6 +917,7 @@ class experiment:
 
         async def awaitcompletion():
             assert self.central is not None
+            logger.debug("Waiting to exit scheduler...")
             async with self.central.exitCondition:
                 while True:
                     if self.exitMode:
@@ -1011,6 +1013,7 @@ class experiment:
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
+        logger.debug("Exiting scheduler context")
         # If no exception and normal run mode, remove old "jobs"
         if self.workspace.run_mode == RunMode.NORMAL:
             if exc_type is None and self.jobsbakpath.is_dir():
@@ -1051,6 +1054,7 @@ class experiment:
             # Put back old experiment as current one
             experiment.CURRENT = self.old_experiment
             if self.server:
+                logger.info("Stopping web server")
                 self.server.stop()
 
     async def update_task_output_count(self, delta: int):

experimaestro/scheduler/dynamic_outputs.py

@@ -135,7 +135,7 @@ class TaskOutputsWorker(threading.Thread):
     """This worker process dynamic output queue for one experiment"""
 
     def __init__(self, xp: experiment):
-        super().__init__(name="task outputs worker")
+        super().__init__(name="task outputs worker", daemon=True)
         self.queue = queue.Queue()
         self.xp = xp
 

experimaestro/scriptbuilder.py

@@ -51,6 +51,8 @@ class PythonScriptBuilder:
         self.lockfiles: List[Path] = []
         self.notificationURL: Optional[str] = None
         self.command: Optional[AbstractCommand] = None
+
+        # This is used to serialize the full process identifier on disk
         self.processtype = "local"
 
     def write(self, job: CommandLineJob):
@@ -63,7 +65,7 @@ class PythonScriptBuilder:
             job {CommandLineJob} -- [description]
 
         Returns:
-            [type] -- [description]
+            str -- The script path on disk
         """
         assert isinstance(
             job, CommandLineJob

experimaestro/sphinx/__init__.py

@@ -10,19 +10,16 @@ from docutils import nodes
 from sphinx.application import Sphinx
 from sphinx import addnodes
 from sphinx.ext.autodoc import ClassDocumenter, Documenter, restify
-from sphinx.locale import _, __
+from sphinx.locale import _
 from sphinx.util import inspect, logging
 from sphinx.domains.python import (
     PyClasslike,
     PyAttribute,
-    PyObject,
-    directives,
-    desc_signature,
-    _parse_annotation,
+    directives,  # noqa: F401
 )
+from sphinx.addnodes import desc_signature
 from sphinx.util.typing import OptionSpec
 from docutils.statemachine import StringList
-import logging
 import re
 
 from experimaestro import Config, Task
@@ -97,9 +94,6 @@ class ConfigDocumenter(ClassDocumenter):
         can_document = inspect.isclass(member) and issubclass(member, Config)
         return can_document
 
-    def add_directive_header(self, sig: str) -> None:
-        super().add_directive_header(sig)
-
     def get_object_members(self, want_all: bool):  # -> Tuple[bool, ObjectMembers]:
         r = super().get_object_members(want_all)
         return r
@@ -157,7 +151,7 @@ class ConfigDocumenter(ClassDocumenter):
 
         # Our specific code
         if issubclass(self.object, Task):
-            self.add_line(f"   :task:", sourcename)
+            self.add_line("   :task:", sourcename)
 
         # add inheritance info, if wanted
         if not self.doc_as_attr and self.options.show_inheritance:
@@ -196,7 +190,6 @@ class ConfigDocumenter(ClassDocumenter):
     def add_content(
         self, more_content: Optional[StringList], no_docstring: bool = False
     ) -> None:
-
         xpminfo = getxpminfo(self.object)
         source_name = self.get_sourcename()
 
@@ -214,9 +207,9 @@ class ConfigDocumenter(ClassDocumenter):
                     source_name,
                 )
             if argument.generator:
-                self.add_line(f"   :generated:", source_name)
+                self.add_line("   :generated:", source_name)
             elif argument.constant:
-                self.add_line(f"   :constant:", source_name)
+                self.add_line("   :constant:", source_name)
 
             # self.add_line("", source_name)
             if argument.help:

experimaestro/tests/launchers/bin/sbatch

@@ -2,8 +2,11 @@
 
 # slurm test suite
 
+pwd 1>&2
+CURDIR="$(realpath "$(dirname "$0")"/..)"
+
 # Where we store the jobs
-XPM_SLURM_DIR="$(realpath "$(dirname "$0")"/..)/slurm"
+XPM_SLURM_DIR="${CURDIR}/slurm"
 if ! test -d "$XPM_SLURM_DIR"; then
     echo "Directory $XPM_SLURM_DIR does not exist" 1>&2
     exit 1
@@ -51,8 +54,19 @@ while true; do
     esac
 done
 
-echo "Starting $@ ${args[@]} > $stdout" >&2
+chdir=$(pwd)
+while IFS= read -r line; do
+    case "$line" in
+        "#SBATCH --output="*)stdout=${line#*#SBATCH --output=};;
+        "#SBATCH --error="*) stderr=${line#*#SBATCH --error=};;
+        "#SBATCH --chdir="*) chdir=${line#*#SBATCH --chdir=};;
+    esac
+done < "$1"
+
+echo "Starting $@ ${args[@]} > $stdout 2> $stderr" >&2
 (
+    if test "$chdir"; then cd "$chdir"; fi
+    export PATH="${CURDIR}/bin:$PATH"
     eval "$@" "${args[@]}"
     echo $? > "$XPM_SLURM_DIR/jobs/$$.status"
 ) > $stdout 2> $stderr &

experimaestro/tests/launchers/bin/srun

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Just run the command
+
+eval "$@"

experimaestro/tests/launchers/common.py

@@ -80,10 +80,14 @@ def takeback(launcher, datapath, txp1, txp2):
     waiting = datapath / "waiting"
 
     with txp1:
-        WaitUntilTouched(touching=touching, waiting=waiting).submit(launcher=launcher)
+        task: WaitUntilTouched = WaitUntilTouched(
+            touching=touching, waiting=waiting
+        ).submit(launcher=launcher)
 
         logger.debug("Waiting for task to create 'touching' file")
         while not touching.is_file():
+            if task.__xpm__.job.state.finished():
+                raise Exception("Job has finished... too early")
             time.sleep(0.01)
 
         with txp2:

{experimaestro-1.7.0rc2.dist-info → experimaestro-1.7.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.7.0rc2
+Version: 1.7.0rc3
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager
@@ -34,7 +34,6 @@ Requires-Dist: huggingface-hub (>0.17)
 Requires-Dist: humanfriendly (>=10,<11)
 Requires-Dist: marshmallow (>=3.20,<4.0)
 Requires-Dist: omegaconf (>=2.3,<3.0)
-Requires-Dist: protobuf (>5)
 Requires-Dist: psutil (>=7)
 Requires-Dist: pyparsing (>=3.1,<4.0)
 Requires-Dist: pytools (>=2023.1.1,<2024.0.0)

{experimaestro-1.7.0rc2.dist-info → experimaestro-1.7.0rc3.dist-info}/RECORD

@@ -6,11 +6,11 @@ experimaestro/cli/__init__.py,sha256=mzc-qqTFtZnFwCQl7IiwlYXEx08kLGwdntWayCerZ6E
 experimaestro/cli/filter.py,sha256=sdP651rnvMfzdwRdv0AiG1BqK8Z6Mia8l-D-qatGB04,5781
 experimaestro/cli/jobs.py,sha256=QJNno0r6aKylol9jsGRgEPRmehEIyl7pv7CZk-DfvJg,7771
 experimaestro/click.py,sha256=6BkeQHEgcxaxzq3xEvEEzwzuBj5-dkfrpOGikuA8L00,1377
-experimaestro/commandline.py,sha256=NS1ubme8DTJtDS2uWwdHLQiZsl6TSK1LkNxu39c3-cw,9463
+experimaestro/commandline.py,sha256=MJIJcfppGCjNA8ozxXUzbUSeDOlTExuzhxryGx3_lIo,9314
 experimaestro/compat.py,sha256=dQqE2ZNHLM2wtdfp7fBRYMfC33qNehVf9J6FGRBUQhs,171
-experimaestro/connectors/__init__.py,sha256=wXVUGWAK7f222KzwXUEP-2LOGdJP43iSXSygGye9EO8,5929
-experimaestro/connectors/local.py,sha256=5X22G1hg7EMAxWmtsk2wQ8Q6Cp4I7OtzzZ0Cu0fgRnk,5929
-experimaestro/connectors/ssh.py,sha256=P6XfCdC4mQTsFzgI-dEdx6AdVhwd0T6VrQpPmNpmiPM,8575
+experimaestro/connectors/__init__.py,sha256=Pe8CZ73IedTIb51-zbdMMk-JFFkZ0t1cOL6QuvdSJjg,6026
+experimaestro/connectors/local.py,sha256=348akOw69t7LgiTBMPG5McCg821I8qfR5GvYNU14sv4,6051
+experimaestro/connectors/ssh.py,sha256=5giqvv1y0QQKF-GI0IFUzI_Z5H8Bj9EuL_Szpvk899Q,8600
 experimaestro/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/core/arguments.py,sha256=7hpkU1f8LJ7JL8kQaD514h9CFSfMotYLsVfMsMmdpWk,6487
 experimaestro/core/context.py,sha256=Q8_ngiHRBZ0laavXRJNiDvdCprrnROVTWaHfrwMdlG4,2638
@@ -30,13 +30,13 @@ experimaestro/ipc.py,sha256=Xn3tYME83jLEB0nFak3DwEIhpL5IRZpCl3jirBF_jl4,1570
 experimaestro/launcherfinder/__init__.py,sha256=qRUDyv3B9UsAM8Q31mRrZrTZox0AptwdmOY4f2K-TUo,279
 experimaestro/launcherfinder/base.py,sha256=q47SsF_cXdo5O6ZhFKn5385WVFcx8Wd-BcEpd6tRpbs,515
 experimaestro/launcherfinder/parser.py,sha256=pYbfEJw7osnqZWm7fkVhQawhpNU8dLU_6vEjtXdc8E8,2279
-experimaestro/launcherfinder/registry.py,sha256=5KKElGd9aS1gegx-A6JVXL-LACk02wFVQxeUIEbZfUc,6105
+experimaestro/launcherfinder/registry.py,sha256=4Ix86rrooRucOlKtiCufmdvrebT-FRrzpqQu0qHpJJg,6337
 experimaestro/launcherfinder/specs.py,sha256=G8za6mEmkVxuZY_ab3OhWJIpONpcBMO_iXeB30sUbhI,6448
 experimaestro/launchers/__init__.py,sha256=lXn544sgJExr6uirILWzAXu_IfmfyqFZOt4OzRnjHXg,2525
 experimaestro/launchers/direct.py,sha256=JZh6WOPnO6ED_xlOs8pL4MRFmnRhmXzpVxTl-ByaD2A,258
 experimaestro/launchers/oar.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/launchers/slurm/__init__.py,sha256=R1Zwd4phZaXV8FwCYhzfB44n0V4cf-hBQzOc6NkFQ0s,41
-experimaestro/launchers/slurm/base.py,sha256=syIC8Frfz-kWhpYkmPTfMjrskx7_bzvmRMQJ40lkF50,14185
+experimaestro/launchers/slurm/base.py,sha256=s6rE_wQBmFuW_Ee75F4iG2ktpWHdKY_SXQsZs3qObB8,15678
 experimaestro/locking.py,sha256=hPT-LuDGZTijpbme8O0kEoB9a3WjdVzI2h31OT44UxE,1477
 experimaestro/mkdocs/__init__.py,sha256=u3AT-uBu3PqyZZXBr6U_ffioEoSZngDdw85005DbyDA,34
 experimaestro/mkdocs/annotations.py,sha256=qpDw8lzrxpsOShXcpcP_LAeR3UhiIXAybG8UvS64-OU,263
@@ -44,19 +44,17 @@ experimaestro/mkdocs/base.py,sha256=SwLh9s7BZfrTAZdBaealSqVeLAroDSwLLMOHmLCxMPQ,
 experimaestro/mkdocs/metaloader.py,sha256=qCqnTWhlgxql-oe46E8AbvYdoM311-lQh-msmPnbllQ,1481
 experimaestro/mkdocs/style.css,sha256=42kJ6Ozq_n4Iw5UfJ4-nO1u-HN3ELvV7Vhvj1Xkn7rQ,66
 experimaestro/mypy.py,sha256=M39VFuDrab-ymlCDIF5jys9oKpTwnuBPzb1T8Un5J3s,285
-experimaestro/notifications.proto,sha256=0bDb3DYeuP_69rUsBJxMA_2m-P9HznQampw7YGovLUU,363
-experimaestro/notifications.py,sha256=j5BbUkQ9JkRvAJCH60MtrOPbaZ1iWTafc-6bFKYrod8,9368
-experimaestro/notifications_pb2.py,sha256=nkUsLXGOXtKIxdASePIFPgLEH6DuGXUKVX1TpxjMS4M,1847
+experimaestro/notifications.py,sha256=UOcCai8wPGzAI_Zu1-KUr9a8Ns1eHUym3gwjUHXUtSE,9274
 experimaestro/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/rpyc.py,sha256=ZRKol-3tVoeoUITLNFenLF4dhWBLW_FvSV_GvsypmeI,3605
 experimaestro/run.py,sha256=58ZlIZ2dQ7a0un2iGiyHJhK14zc18BnpEFDis7OyTPA,5222
 experimaestro/scheduler/__init__.py,sha256=ERmmOxz_9mUkIuccNbzUa5Y6gVLLVDdyc4cCxbCCUbY,20
-experimaestro/scheduler/base.py,sha256=qlduyqawKoO0FFcRMtaoPpp_MdNbkb2VB9HQX6l02Rg,35146
+experimaestro/scheduler/base.py,sha256=R40hKI_2cpyZ8F4GTwEulTWczPAaGYi1UaaELh9etGM,35368
 experimaestro/scheduler/dependencies.py,sha256=n9XegwrmjayOIxt3xhuTEBVEBGSq4oeVdzz-FviDGXo,1994
-experimaestro/scheduler/dynamic_outputs.py,sha256=WGnPmSM5feNoKm4L72SmjzbzO8A0POQaY7bfVf6QlzQ,5719
+experimaestro/scheduler/dynamic_outputs.py,sha256=yYPL98I0nSgDjlE3Sk9dtvovh2PZ6rUDnKjDNnAg1dc,5732
 experimaestro/scheduler/services.py,sha256=aCKkNZMULlceabqf-kOs_-C7KPINnjU3Q-I00o5x6iY,2189
 experimaestro/scheduler/workspace.py,sha256=KNdxPBwUk7gO8h2utMCrlIVKB-f2Ylqg_IxLc4okp_8,2320
-experimaestro/scriptbuilder.py,sha256=nLtwZG136mcji3UhbUDNw4Wj8nbHcj1XysZ2f0_A8eQ,4947
+experimaestro/scriptbuilder.py,sha256=6GKUkgixLqSEy41sNr-_HNcrjKb8uxaoQ65DywRYsC0,5027
 experimaestro/server/__init__.py,sha256=F2bzLf2q29Haj2OIbPA26r5WVbaipBNylIozg-As758,10854
 experimaestro/server/data/016b4a6cdced82ab3aa1.ttf,sha256=AD8RVBhWpkmmyCNcYmbIk2IkxdYJ5RRC2iTcVVbRT78,189684
 experimaestro/server/data/0c35d18bf06992036b69.woff2,sha256=gmX2R4Y5fWuDLRygqv3xSa2E5ydZ__qfcnLpGg-wFdE,128352
@@ -89,9 +87,8 @@ experimaestro/server/data/index.js.map,sha256=cv7KF28Uq5dy7Ux3yRnoSVztrOFVid359f
 experimaestro/server/data/login.html,sha256=4dvhSOn6DHp_tbmzqIKrqq2uAo0sAUbgLVD0lTnPp4s,511
 experimaestro/server/data/manifest.json,sha256=EpzHQZzrGh9c1Kf63nrqvI33H1cm0nLYfdh5lDm8ijI,318
 experimaestro/settings.py,sha256=U6gTVBL5Z4Rk0_7BAVoavVJKN2sQNRpspE-601Elfys,3170
-experimaestro/sphinx/__init__.py,sha256=heovvtwbYToZM-b6HNi4pJdBoo_97usdEawhMGSK3bk,9560
+experimaestro/sphinx/__init__.py,sha256=xac4NmlxvciVONXfr3CChHH7AxZeAuQoC1MYL4Hwu3g,9440
 experimaestro/sphinx/static/experimaestro.css,sha256=0rEgt1LoDdD-a_R5rVfWZ19zD1gR-1L7q3f4UibIB58,294
-experimaestro/streaming_state.py,sha256=NEneDTRWRYfvhqButJCg8rzmVb5coy-sTazljub7umQ,3927
 experimaestro/taskglobals.py,sha256=Lp0bqobVLndR7fOtF9qPI7utTKQXXwTdVN6l5Av9Dc4,660
 experimaestro/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/tests/conftest.py,sha256=CtC6TvUS9sbgSc3pZYyTyEvfilnNGPpOUJvi-jn6twI,695
@@ -101,9 +98,10 @@ experimaestro/tests/connectors/utils.py,sha256=9sM3Pwy2nRfSr7pwQoOoSCDhBrEcDsEEl
 experimaestro/tests/definitions_types.py,sha256=nMoQxZxhTJAYV87Ce9F2dAITxXGHf7Uw5j-MKsZ3LmQ,399
 experimaestro/tests/launchers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/tests/launchers/bin/sacct,sha256=9mmRAYCE4RBSBOf7aanhFw9hzujOUlcw3OJmZJ3K-Wc,639
-experimaestro/tests/launchers/bin/sbatch,sha256=NMjBRvZ-TrpZCSLfMAirR5Vf1-IQFPNNNaDpIw56vPE,1350
+experimaestro/tests/launchers/bin/sbatch,sha256=MGzCDw6ywc973nMViPijwLzJcAW-ghJynMJA00h9AoQ,1746
+experimaestro/tests/launchers/bin/srun,sha256=3oE3qq0UFpVtTvXfR1kH3tovFYX74fp1Fk-o8zgsaJA,47
 experimaestro/tests/launchers/bin/test.py,sha256=MbxdAd2Sf7T-Hj3ldmrtngbQuBdNOkXjMcICJTf39wI,477
-experimaestro/tests/launchers/common.py,sha256=ykTMdCLWsf6jmKzfqPoRMeHn93GCNSi755yOS5CYT_I,2832
+experimaestro/tests/launchers/common.py,sha256=E09KhrQfNQNZbaeVkXBgHk2906DvaCEU7CqEQvLbZTE,2994
 experimaestro/tests/launchers/config_slurm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/tests/launchers/config_slurm/launchers.py,sha256=DohwQVv1eXWfDpAYFg7KJekEm7q7G-lMou2lPg-PKOk,838
 experimaestro/tests/launchers/test_local.py,sha256=4oGgWH2YgkEm-Muu6s4cwlgriXtYr5xAd72DVoIw_Yk,717
@@ -112,8 +110,6 @@ experimaestro/tests/restart.py,sha256=wivq06i3SoxL_cUKti9J_o3MykynmLic6czuCZSHc_
 experimaestro/tests/restart_main.py,sha256=iAFzw0H1q9Aq7t6TrSAj236QBnYU52qx0VF-2dz6tx4,295
 experimaestro/tests/scripts/notifyandwait.py,sha256=3BLXLI5XgP3BnKf6sQ56oKoMVqR80VMHmploJ3JO6Ko,407
 experimaestro/tests/scripts/waitforfile.py,sha256=EDT-TuFi2fU_qt51K5EmAxjw_OnJKkBW7UCfhrtDbVw,120
-experimaestro/tests/state_streamer.proto,sha256=KopoTOt0VpRKbZ_ed9boQyaWB7oNOlDDKhqX-cj9pZs,121
-experimaestro/tests/state_streamer_pb2.py,sha256=5mbmirhDMAOU09Kx9Wo_YrKF1PK6Ebr2TNBELQ2dfMQ,1541
 experimaestro/tests/task_tokens.py,sha256=vgqUa-S_YC2Id9pGOSv40qFTwq1WGZkFhr556Z5o678,477
 experimaestro/tests/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 experimaestro/tests/tasks/all.py,sha256=OMkHsWZkErCmTajiNO7hNhvnk9eKzJC-VatWgabWlsI,1955
@@ -131,7 +127,6 @@ experimaestro/tests/test_progress.py,sha256=wtIGQzlV3ldd_wMng11LinVESchW-1J954mC
 experimaestro/tests/test_serializers.py,sha256=xSCezAM9yH_Ix1wr7j0au9SyBv9DtZ7b0zs2-Ynt-VM,2338
 experimaestro/tests/test_snippets.py,sha256=rojnyDjtmAMnSuDUj6Bv9XEgdP8oQf2nVc132JF8vsM,3081
 experimaestro/tests/test_ssh.py,sha256=KS1NWltiXrJBSStY9d4mwrexeqgNGWmhxuAU_WLQDAU,1449
-experimaestro/tests/test_state_streamer.py,sha256=T1_Mz-dkHEcpPjCGcUsat6s4cOizxHt1so4bw7VkypM,609
 experimaestro/tests/test_tags.py,sha256=v_8_7LuEHfY_gfa0JRCUkmgJh8h6RMya_nd5NcPAJPw,2852
 experimaestro/tests/test_tasks.py,sha256=bUSB_UT1MTN2P_RPHd4AT5NK-DFsgCVeFKSiXu3bEz8,9429
 experimaestro/tests/test_tokens.py,sha256=U3nKBL1KftWhmk3dQZZLQd-MLJL_SscMjI3zMieMHfc,7823
@@ -152,8 +147,8 @@ experimaestro/utils/jupyter.py,sha256=JcEo2yQK7x3Cr1tNl5FqGMZOICxCv9DwMvL5xsWdQP
 experimaestro/utils/resources.py,sha256=j-nvsTFwmgENMoVGOD2Ap-UD3WU85WkI0IgeSszMCX4,1328
 experimaestro/utils/settings.py,sha256=jpFMqF0DLL4_P1xGal0zVR5cOrdD8O0Y2IOYvnRgN3k,793
 experimaestro/xpmutils.py,sha256=S21eMbDYsHfvmZ1HmKpq5Pz5O-1HnCLYxKbyTBbASyQ,638
-experimaestro-1.7.0rc2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-experimaestro-1.7.0rc2.dist-info/METADATA,sha256=61zmKYFsyGyj3HXvJATsPb3CFO3dn2ZXlR66pegSjic,6191
-experimaestro-1.7.0rc2.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-experimaestro-1.7.0rc2.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
-experimaestro-1.7.0rc2.dist-info/RECORD,,
+experimaestro-1.7.0rc3.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+experimaestro-1.7.0rc3.dist-info/METADATA,sha256=-fuyq-zQ7nqyCb4NJ0fbTWp5YTZKgFDk6o26noSDrJQ,6162
+experimaestro-1.7.0rc3.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+experimaestro-1.7.0rc3.dist-info/entry_points.txt,sha256=TppTNiz5qm5xm1fhAcdLKdCLMrlL-eQggtCrCI00D9c,446
+experimaestro-1.7.0rc3.dist-info/RECORD,,

experimaestro/notifications.proto

@@ -1,17 +0,0 @@
-syntax = "proto3";
-import "google/protobuf/any.proto";
-
-
-message Envelope {
-    google.protobuf.Any payload = 1;
-}
-
-message LevelInformation {
-    int32 level = 1;
-
-    optional string desc = 2; // Optional in proto3 is handled by default unless you want to use 'optional'
-    float progress = 3;
-
-    float previous_progress = 4;
-    string previous_desc = 5;
-}

experimaestro/notifications_pb2.py

@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# NO CHECKED-IN PROTOBUF GENCODE
-# source: src/experimaestro/notifications.proto
-# Protobuf Python Version: 5.29.3
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import runtime_version as _runtime_version
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-_runtime_version.ValidateProtobufRuntimeVersion(
-    _runtime_version.Domain.PUBLIC,
-    5,
-    29,
-    3,
-    '',
-    'src/experimaestro/notifications.proto'
-)
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%src/experimaestro/notifications.proto\x1a\x19google/protobuf/any.proto\"1\n\x08\x45nvelope\x12%\n\x07payload\x18\x01 \x01(\x0b\x32\x14.google.protobuf.Any\"\x81\x01\n\x10LevelInformation\x12\r\n\x05level\x18\x01 \x01(\x05\x12\x11\n\x04\x64\x65sc\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x10\n\x08progress\x18\x03 \x01(\x02\x12\x19\n\x11previous_progress\x18\x04 \x01(\x02\x12\x15\n\rprevious_desc\x18\x05 \x01(\tB\x07\n\x05_descb\x06proto3')
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'src.experimaestro.notifications_pb2', _globals)
-if not _descriptor._USE_C_DESCRIPTORS:
-  DESCRIPTOR._loaded_options = None
-  _globals['_ENVELOPE']._serialized_start=68
-  _globals['_ENVELOPE']._serialized_end=117
-  _globals['_LEVELINFORMATION']._serialized_start=120
-  _globals['_LEVELINFORMATION']._serialized_end=249
-# @@protoc_insertion_point(module_scope)

experimaestro/streaming_state.py

@@ -1,124 +0,0 @@
-"""Handles streaming states
-
-Allows to track a global state via a file
-
-- partial states can be output
-- every K bytes, a global state is output
-
-The reader can then recover quickly by seeking the last global state output.
-
-"""
-
-import struct
-from io import BufferedWriter, BufferedReader, SEEK_END
-from typing import Type, Generator, Union
-
-from google.protobuf.message import Message
-
-# Constants
-PARTIAL_UPDATE = 1
-GLOBAL_STATE = 2
-HEADER_FORMAT = ">BI"  # 1-byte type tag, 4-byte message size
-HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
-MAGIC_NUMBER = b"\xab\xcd\xef\x00"
-MAGIC_SIZE = len(MAGIC_NUMBER)
-
-
-class StreamingStateWriter:
-    def __init__(
-        self,
-        file: BufferedWriter,
-        partial_cls: Type[Message],
-        snapshot_cls: Type[Message],
-        chunk_size: int = 8192,
-    ):
-        self.file = file
-        self.partial_cls = partial_cls
-        self.snapshot_cls = snapshot_cls
-        self.chunk_size = chunk_size
-        self.bytes_since_last_snapshot = 0
-
-    def write_partial(self, message: Message):
-        assert isinstance(message, self.partial_cls)
-        self._write_message(PARTIAL_UPDATE, message)
-
-    def write_snapshot_if_needed(self, message: Message):
-        assert isinstance(message, self.snapshot_cls)
-        if self.bytes_since_last_snapshot >= self.chunk_size:
-            self._write_message(GLOBAL_STATE, message, with_magic=True)
-            self.bytes_since_last_snapshot = 0
-
-    def _write_message(self, type_tag: int, message: Message, with_magic: bool = False):
-        data = message.SerializeToString()
-        header = struct.pack(HEADER_FORMAT, type_tag, len(data))
-        if with_magic:
-            self.file.write(MAGIC_NUMBER)
-            self.bytes_since_last_snapshot += MAGIC_SIZE
-        self.file.write(header + data)
-        self.file.flush()
-        self.bytes_since_last_snapshot += HEADER_SIZE + len(data)
-
-
-class StreamingStateReader:
-    def __init__(
-        self,
-        file: BufferedReader,
-        partial_cls: Type[Message],
-        snapshot_cls: Type[Message],
-    ):
-        self.file = file
-        self.partial_cls = partial_cls
-        self.snapshot_cls = snapshot_cls
-
-    def __iter__(self) -> Generator[Union[Message, Message], None, None]:
-        while True:
-            peek = self.file.peek(MAGIC_SIZE)[:MAGIC_SIZE]
-            if peek == MAGIC_NUMBER:
-                self.file.read(MAGIC_SIZE)
-
-            header = self.file.read(HEADER_SIZE)
-            if len(header) < HEADER_SIZE:
-                break
-
-            type_tag, length = struct.unpack(HEADER_FORMAT, header)
-            payload = self.file.read(length)
-
-            if type_tag == PARTIAL_UPDATE:
-                msg = self.partial_cls()
-            elif type_tag == GLOBAL_STATE:
-                msg = self.snapshot_cls()
-            else:
-                raise ValueError(f"Unknown type tag: {type_tag}")
-
-            msg.ParseFromString(payload)
-            yield msg
-
-    def seek_last_snapshot(self) -> Union[Message, None]:
-        self.file.seek(0, SEEK_END)
-        file_size = self.file.tell()
-
-        window = 4096
-        pos = file_size
-
-        while pos > 0:
-            read_size = min(window, pos)
-            pos -= read_size
-            self.file.seek(pos)
-            data = self.file.read(read_size)
-
-            idx = data.rfind(MAGIC_NUMBER)
-            if idx != -1:
-                snapshot_pos = pos + idx + MAGIC_SIZE
-                self.file.seek(snapshot_pos)
-                header = self.file.read(HEADER_SIZE)
-                if len(header) < HEADER_SIZE:
-                    break
-                type_tag, length = struct.unpack(HEADER_FORMAT, header)
-                if type_tag != GLOBAL_STATE:
-                    continue
-                payload = self.file.read(length)
-                msg = self.snapshot_cls()
-                msg.ParseFromString(payload)
-                return msg
-
-        return None

experimaestro/tests/state_streamer.proto

@@ -1,9 +0,0 @@
-syntax = "proto3";
-
-message PartialUpdate {
-    int32 updated_field = 1;
-}
-
-message GlobalState {
-    int32 field = 1;
-}

experimaestro/tests/state_streamer_pb2.py

@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# NO CHECKED-IN PROTOBUF GENCODE
-# source: src/experimaestro/tests/state_streamer.proto
-# Protobuf Python Version: 5.29.3
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import runtime_version as _runtime_version
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-_runtime_version.ValidateProtobufRuntimeVersion(
-    _runtime_version.Domain.PUBLIC,
-    5,
-    29,
-    3,
-    '',
-    'src/experimaestro/tests/state_streamer.proto'
-)
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,src/experimaestro/tests/state_streamer.proto\"&\n\rPartialUpdate\x12\x15\n\rupdated_field\x18\x01 \x01(\x05\"\x1c\n\x0bGlobalState\x12\r\n\x05\x66ield\x18\x01 \x01(\x05\x62\x06proto3')
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'src.experimaestro.tests.state_streamer_pb2', _globals)
-if not _descriptor._USE_C_DESCRIPTORS:
-  DESCRIPTOR._loaded_options = None
-  _globals['_PARTIALUPDATE']._serialized_start=48
-  _globals['_PARTIALUPDATE']._serialized_end=86
-  _globals['_GLOBALSTATE']._serialized_start=88
-  _globals['_GLOBALSTATE']._serialized_end=116
-# @@protoc_insertion_point(module_scope)

experimaestro/tests/test_state_streamer.py

@@ -1,18 +0,0 @@
-from .state_streamer_pb2 import PartialUpdate, GlobalState
-
-# Writing
-with open("log.bin", "wb") as f:
-    writer = StreamingStateWriter(f, PartialUpdate, GlobalState)
-    for i in range(1000):
-        writer.write_partial(PartialUpdate(updated_field=i))
-        if i % 100 == 0:
-            writer.write_snapshot_if_needed(GlobalState(field=i))
-
-# Reading
-with open("log.bin", "rb") as f:
-    reader = StreamingStateReader(f, PartialUpdate, GlobalState)
-    last_snapshot = reader.seek_last_snapshot()
-    print("Last Snapshot:", last_snapshot)
-
-    for msg in reader:
-        print(type(msg).__name__, msg)