experimaestro 1.8.9__tar.gz → 1.9.1__tar.gz

{experimaestro-1.8.9 → experimaestro-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.8.9
+Version: 1.9.1
 Summary: "Experimaestro is a computer science experiment manager"
 License: GPL-3
 Keywords: experiment manager

{experimaestro-1.8.9 → experimaestro-1.9.1}/pyproject.toml

@@ -20,7 +20,7 @@ include = [
     "src/experimaestro/mkdocs/style.css",
     { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
 ]
-version = "1.8.9"
+version = "1.9.1"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -122,7 +122,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.8.9"
+version = "1.9.1"
 changelog_start_rev = "v1.0.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 # update_changelog_on_bump = true

{experimaestro-1.8.9 → experimaestro-1.9.1}/src/experimaestro/cli/filter.py

@@ -1,3 +1,4 @@
+import asyncio
 from typing import Any, Callable, Dict, List, Optional
 import pyparsing as pp
 from pathlib import Path
@@ -8,9 +9,10 @@ from experimaestro.scheduler import JobState
 
 
 class JobInformation:
-    def __init__(self, path: Path, scriptname: str):
+    def __init__(self, path: Path, scriptname: str, check: bool = False):
         self.path = path
         self.scriptname = scriptname
+        self.check = check
 
     @cached_property
     def params(self):
@@ -27,6 +29,13 @@ class JobInformation:
         if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
         if (self.path / f"{self.scriptname}.pid").is_file():
+            if self.check:
+                if process := self.getprocess():
+                    state = asyncio.run(process.aio_state(0))
+                    if state is None or state.finished:
+                        return JobState.ERROR
+                else:
+                    return JobState.ERROR
             return JobState.RUNNING
         else:
             return None

{experimaestro-1.8.9 → experimaestro-1.9.1}/src/experimaestro/cli/jobs.py

@@ -1,4 +1,5 @@
 # flake8: noqa: T201
+import asyncio
 import subprocess
 from typing import Optional
 from shutil import rmtree
@@ -49,6 +50,7 @@ def process(
     filter="",
     perform=False,
     fullpath=False,
+    check=False,
 ):
     from .filter import createFilter, JobInformation
     from experimaestro.scheduler import JobState
@@ -63,13 +65,14 @@ def process(
         for job in p.glob("jobs/*/*"):
             job_path = job.resolve()
             if job_path.is_dir():
-                *_, scriptname = job_path.parent.name.rsplit(".", 1)
-                job2xp.setdefault(scriptname, set()).add(p.name)
+                job2xp.setdefault(job_path.name, set()).add(p.name)
 
         if (p / "jobs.bak").is_dir():
             cprint(f"  Experiment {p.name} has not finished yet", "red")
             if (not perform) and (kill or clean):
-                cprint("  Preventing kill/clean (use --perform if you want to)", "yellow")
+                cprint(
+                    "  Preventing kill/clean (use --perform if you want to)", "yellow"
+                )
                 kill = False
                 clean = False
 
@@ -79,11 +82,11 @@ def process(
         p = job.resolve()
         if p.is_dir():
             *_, scriptname = p.parent.name.rsplit(".", 1)
-            xps = job2xp.get(scriptname, set())
+            xps = job2xp.get(job.name, set())
             if experiment and experiment not in xps:
                 continue
 
-            info = JobInformation(p, scriptname)
+            info = JobInformation(p, scriptname, check=check)
             job_str = (
                 (str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}")
                 + " "
@@ -151,6 +154,7 @@ def process(
 @click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
+@click.option("--no-check", is_flag=True, help="Check that running jobs")
 @jobs.command()
 @click.pass_context
 def list(
@@ -160,6 +164,7 @@ def list(
     tags: bool,
     ready: bool,
     fullpath: bool,
+    no_check: bool,
 ):
     process(
         ctx.obj.workspace,
@@ -168,6 +173,7 @@ def list(
         tags=tags,
         ready=ready,
         fullpath=fullpath,
+        check=not no_check,
     )
 
 
@@ -187,6 +193,7 @@ def kill(
     ready: bool,
     fullpath: bool,
     perform: bool,
+    check: bool,
 ):
     process(
         ctx.obj.workspace,

{experimaestro-1.8.9 → experimaestro-1.9.1}/src/experimaestro/connectors/__init__.py

@@ -114,8 +114,11 @@ class Process:
         """Wait until the process finishes and returns the error code"""
         raise NotImplementedError(f"Not implemented: {self.__class__}.wait")
 
-    async def aio_state(self) -> ProcessState:
-        """Returns the job state"""
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
+        """Returns the job state
+
+        :param timeout: maximum waiting time for a refresh
+        """
         raise NotImplementedError(f"Not implemented: {self.__class__}.aio_state")
 
     async def aio_isrunning(self):

{experimaestro-1.8.9 → experimaestro-1.9.1}/src/experimaestro/connectors/local.py

@@ -39,7 +39,7 @@ class PsutilProcess(Process):
         )
         return code
 
-    async def aio_state(self):
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
         if self._process.is_running():
             return ProcessState.RUNNING
         return ProcessState.FINISHED
@@ -65,7 +65,7 @@ class LocalProcess(Process):
         )
         return code
 
-    async def aio_state(self):
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
         code = self._process.poll()
         if code is None:
             return ProcessState.RUNNING

{experimaestro-1.8.9 → experimaestro-1.9.1}/src/experimaestro/launchers/slurm/base.py

@@ -87,7 +87,7 @@ class SlurmProcessWatcher(threading.Thread):
     WATCHERS: Dict[Tuple[Tuple[str, Any]], "SlurmProcessWatcher"] = {}
 
     def __init__(self, launcher: "SlurmLauncher"):
-        super().__init__()
+        super().__init__(daemon=True)
         self.launcher = launcher
         self.count = 1
         self.jobs: Dict[str, SlurmJobState] = {}
@@ -184,10 +184,10 @@ class BatchSlurmProcess(Process):
                 if state and state.finished():
                     return 0 if state.slurm_state == "COMPLETED" else 1
 
-    async def aio_state(self):
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
         def check():
             with SlurmProcessWatcher.get(self.launcher) as watcher:
-                jobinfo = watcher.getjob(self.jobid)
+                jobinfo = watcher.getjob(self.jobid, timeout=timeout)
                 return jobinfo.state if jobinfo else ProcessState.SCHEDULED
 
         return await asyncThreadcheck("slurm.aio_isrunning", check)
@@ -212,7 +212,7 @@ class BatchSlurmProcess(Process):
 
         # Checks that the process is running
         with SlurmProcessWatcher.get(launcher) as watcher:
-            logger.info("Checking SLURM job %s", process.jobid)
+            logger.debug("Checking SLURM job %s", process.jobid)
             jobinfo = watcher.getjob(process.jobid, timeout=0.1)
             if jobinfo and jobinfo.state.running:
                 logger.debug(