experimaestro 1.5.6__tar.gz → 1.5.7__tar.gz

{experimaestro-1.5.6 → experimaestro-1.5.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 1.5.6
+Version: 1.5.7
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 License: GPL-3

{experimaestro-1.5.6 → experimaestro-1.5.7}/pyproject.toml

@@ -19,7 +19,7 @@ include = [
     "src/experimaestro/sphinx/static/experimaestro.css",
     "src/experimaestro/mkdocs/style.css"
 ]
-version = "1.5.6"
+version = "1.5.7"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -123,7 +123,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.5.6"
+version = "1.5.7"
 changelog_start_rev = "0.15.0"
 tag_format = "v$version"
 update_changelog_on_bump = true

experimaestro-1.5.7/src/experimaestro/__main__.py

@@ -0,0 +1,12 @@
+from experimaestro.cli import cli
+
+# flake8: noqa: F401
+import experimaestro.cli.jobs
+
+
+def main():
+    cli(obj=None)
+
+
+if __name__ == "__main__":
+    main()

experimaestro-1.5.6/src/experimaestro/__main__.py → experimaestro-1.5.7/src/experimaestro/cli/__init__.py

@@ -1,6 +1,5 @@
 # flake8: noqa: T201
 import sys
-from types import ModuleType
 from typing import Set, Optional
 import pkg_resources
 from itertools import chain
@@ -10,11 +9,12 @@ import logging
 from functools import cached_property, update_wrapper
 from pathlib import Path
 import subprocess
-from termcolor import colored, cprint
+from termcolor import cprint
 
 import experimaestro
 from experimaestro.experiments.cli import experiments_cli
 import experimaestro.launcherfinder.registry as launcher_registry
+from experimaestro.settings import find_workspace
 
 # --- Command line main options
 logging.basicConfig(level=logging.INFO)
@@ -175,121 +175,6 @@ def diff(path: Path):
     check(".", job, new_job, set())
 
 
-@click.argument("path", type=Path, callback=check_xp_path)
-@click.option("--experiment", default=None, help="Restrict to this experiment")
-@click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
-@click.option("--filter", default="", help="Filter expression")
-@click.option(
-    "--force",
-    is_flag=True,
-    help="Force operation even if experiment has not been completed yet",
-)
-@click.option("--kill", is_flag=True, help="Kill filtered tasks (when/before running)")
-@click.option("--clean", is_flag=True, help="Remove finished tasks directories")
-@cli.command()
-def jobs(
-    path: Path,
-    experiment: str,
-    filter: str,
-    tags: bool,
-    ready: bool,
-    kill: bool,
-    clean: bool,
-    force: bool,
-):
-    """Job control: list, kill and clean
-
-    The job filter is a boolean expression where tags (alphanumeric)
-    and special job information (@state for job state, @name for job full
-    name) can be compared to a given value (using '~' for regex matching,
-    '=', 'not in', or 'in')
-
-    For instance,
-
-    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
-
-    selects jobs where the tag model is "bm25", the tag mode is either
-    "a" or "b", and the state is running.
-
-    """
-    for p in (path / "xp").glob("*"):
-        if experiment and p.name != experiment:
-            continue
-
-        from .filter import createFilter, JobInformation
-        from experimaestro.scheduler import JobState
-
-        _filter = createFilter(filter) if filter else lambda x: True
-
-        print(f"* Experiment {p.name}")
-        if (p / "jobs.bak").is_dir():
-            cprint("  Experiment has not finished yet", "red")
-            if not force and (kill or clean):
-                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
-                kill = False
-                clean = False
-        print()
-
-        for job in p.glob("jobs/*/*"):
-            info = None
-            p = job.resolve()
-            if p.is_dir():
-                *_, scriptname = p.parent.name.rsplit(".", 1)
-
-                info = JobInformation(p, scriptname)
-                if filter:
-                    if not _filter(info):
-                        continue
-
-                if info.state is None:
-                    print(
-                        colored(f"NODIR   {job.parent.name}/{job.name}", "red"), end=""
-                    )
-                elif info.state.running():
-                    if kill:
-                        process = info.getprocess()
-                        print("KILLING", process)
-                        process.kill()
-                    print(
-                        colored(
-                            f"{info.state.name:8}{job.parent.name}/{job.name}", "yellow"
-                        ),
-                        end="",
-                    )
-                elif info.state == JobState.DONE:
-                    print(
-                        colored(f"DONE    {job.parent.name}/{job.name}", "green"),
-                        end="",
-                    )
-                elif info.state == JobState.ERROR:
-                    print(
-                        colored(f"FAIL    {job.parent.name}/{job.name}", "red"), end=""
-                    )
-                else:
-                    print(
-                        colored(
-                            f"{info.state.name:8}{job.parent.name}/{job.name}", "red"
-                        ),
-                        end="",
-                    )
-
-            else:
-                if not ready:
-                    continue
-                print(colored(f"READY {job.parent.name}/{job.name}", "yellow"), end="")
-
-            if tags:
-                print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
-            else:
-                print()
-
-            if clean and info.state and info.state.finished():
-                cprint("Cleaning...", "red")
-                rmtree(p)
-        print()
-
-
 @click.option("--show-all", is_flag=True, help="Show even not orphans")
 @click.option(
     "--ignore-old", is_flag=True, help="Ignore old jobs for unfinished experiments"
@@ -407,11 +292,14 @@ cli.add_command(Launchers("tokens", help="Token specific commands"))
 
 
 @cli.group()
-@click.argument("workdir", type=Path, callback=check_xp_path)
+@click.option("--workdir", type=Path, default=None)
+@click.option("--workspace", type=str, default=None)
 @click.pass_context
-def experiments(ctx, workdir):
+def experiments(ctx, workdir, workspace):
     """Manage experiments"""
-    ctx.obj = workdir
+    ws = find_workspace(workdir=workdir, workspace=workspace)
+    path = check_xp_path(None, None, ws.path)
+    ctx.obj = path
 
 
 @experiments.command()
@@ -422,11 +310,3 @@ def list(workdir: Path):
             cprint(f"[unfinished] {p.name}", "yellow")
         else:
             cprint(p.name, "cyan")
-
-
-def main():
-    cli(obj=None)
-
-
-if __name__ == "__main__":
-    main()

{experimaestro-1.5.6/src/experimaestro → experimaestro-1.5.7/src/experimaestro/cli}/filter.py

@@ -22,12 +22,12 @@ class JobInformation:
 
     @cached_property
     def state(self) -> Optional[JobState]:
-        if (self.path / f"{self.scriptname}.pid").is_file():
-            return JobState.RUNNING
-        elif (self.path / f"{self.scriptname}.done").is_file():
+        if (self.path / f"{self.scriptname}.done").is_file():
             return JobState.DONE
-        elif (self.path / f"{self.scriptname}.failed").is_file():
+        if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
+        if (self.path / f"{self.scriptname}.pid").is_file():
+            return JobState.RUNNING
         else:
             return None
 

experimaestro-1.5.7/src/experimaestro/cli/jobs.py

@@ -0,0 +1,245 @@
+# flake8: noqa: T201
+import subprocess
+from typing import Optional
+from shutil import rmtree
+import click
+from pathlib import Path
+from termcolor import colored, cprint
+
+from experimaestro.settings import find_workspace
+from . import check_xp_path, cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def jobs(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Job control: list, kill and clean
+
+    The job filter is a boolean expression where tags (alphanumeric)
+    and special job information (@state for job state, @name for job full
+    name) can be compared to a given value (using '~' for regex matching,
+    '=', 'not in', or 'in')
+
+    For instance,
+
+    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
+
+    selects jobs where the tag model is "bm25", the tag mode is either
+    "a" or "b", and the state is running.
+
+    """
+    ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+    check_xp_path(ctx, None, ws.path)
+
+
+def process(
+    workspace,
+    *,
+    experiment="",
+    tags="",
+    ready=False,
+    clean=False,
+    kill=False,
+    filter="",
+    perform=False,
+    fullpath=False,
+):
+    path = workspace.path
+    for p in (path / "xp").glob("*"):
+        if experiment and p.name != experiment:
+            continue
+
+        from .filter import createFilter, JobInformation
+        from experimaestro.scheduler import JobState
+
+        _filter = createFilter(filter) if filter else lambda x: True
+
+        print(f"* Experiment {p.name}")
+        if (p / "jobs.bak").is_dir():
+            cprint("  Experiment has not finished yet", "red")
+            if not perform and (kill or clean):
+                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                kill = False
+                clean = False
+        print()
+
+        for job in p.glob("jobs/*/*"):
+            info = None
+            p = job.resolve()
+            if p.is_dir():
+                *_, scriptname = p.parent.name.rsplit(".", 1)
+
+                info = JobInformation(p, scriptname)
+                job_path = (
+                    str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}"
+                )
+
+                if filter:
+                    if not _filter(info):
+                        continue
+
+                if info.state is None:
+                    print(colored(f"NODIR   {job_path}", "red"), end="")
+                elif info.state.running():
+                    if kill:
+                        if perform:
+                            process = info.getprocess()
+                            print("KILLING", process)
+                            process.kill()
+                        else:
+                            print("KILLING (not performing)", process)
+                    print(
+                        colored(f"{info.state.name:8}{job_path}", "yellow"),
+                        end="",
+                    )
+                elif info.state == JobState.DONE:
+                    print(
+                        colored(f"DONE    {job_path}", "green"),
+                        end="",
+                    )
+                elif info.state == JobState.ERROR:
+                    print(colored(f"FAIL    {job_path}", "red"), end="")
+                else:
+                    print(
+                        colored(f"{info.state.name:8}{job_path}", "red"),
+                        end="",
+                    )
+
+            else:
+                if not ready:
+                    continue
+                print(colored(f"READY {job_path}", "yellow"), end="")
+
+            if tags:
+                print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
+            else:
+                print()
+
+            if clean and info.state and info.state.finished():
+                if perform:
+                    cprint("Cleaning...", "red")
+                    rmtree(p)
+                else:
+                    cprint("Cleaning... (not performed)", "red")
+        print()
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def list(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the killing")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def kill(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        kill=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the cleaning")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def clean(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        clean=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.argument("jobid", type=str)
+@click.option(
+    "--follow", "-f", help="Use tail instead of less to follow changes", is_flag=True
+)
+@click.option("--std", help="Follow stdout instead of stderr", is_flag=True)
+@jobs.command()
+@click.pass_context
+def log(ctx, jobid: str, follow: bool, std: bool):
+    task_name, task_hash = jobid.split("/")
+    _, name = task_name.rsplit(".", 1)
+    path = (
+        ctx.obj.workspace.path
+        / "jobs"
+        / task_name
+        / task_hash
+        / f"""{name}.{'out' if std else 'err'}"""
+    )
+    if follow:
+        subprocess.run(["tail", "-f", path])
+    else:
+        subprocess.run(["less", "-r", path])
+
+
+@click.argument("jobid", type=str)
+@jobs.command()
+@click.pass_context
+def path(ctx, jobid: str):
+    task_name, task_hash = jobid.split("/")
+    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    print(path)

{experimaestro-1.5.6 → experimaestro-1.5.7}/src/experimaestro/core/objects.py

@@ -916,14 +916,14 @@ class ConfigInformation:
 
         # --- Submit the job
 
+        # Sets the init tasks
+        self.init_tasks = init_tasks
+
         # Creates a new job
         self.job = self.xpmtype.task(
             self.pyobject, launcher=launcher, workspace=workspace, run_mode=run_mode
         )
 
-        # Sets the init tasks
-        self.init_tasks = init_tasks
-
         # Validate the object
         job_context = JobContext(self.job)
         self.validate_and_seal(job_context)
@@ -979,6 +979,9 @@ class ConfigInformation:
                 elif self.job.failedpath.is_file():
                     color = "light_red"
                     cprint(f"[failed] {s}", color, file=sys.stderr)
+                elif self.job.pidpath.is_file():
+                    color = "blue"
+                    cprint(f"[running] {s}", color, file=sys.stderr)
                 else:
                     color = "light_blue"
                     cprint(f"[not run] {s}", color, file=sys.stderr)

{experimaestro-1.5.6 → experimaestro-1.5.7}/src/experimaestro/experiments/cli.py

@@ -11,7 +11,7 @@ import yaml
 from experimaestro import LauncherRegistry, RunMode, experiment
 from experimaestro.experiments.configuration import ConfigurationBase
 from experimaestro.exceptions import HandledException
-from experimaestro.settings import get_workspace
+from experimaestro.settings import find_workspace
 from omegaconf import OmegaConf, SCMode
 from termcolor import cprint
 
@@ -231,27 +231,8 @@ def experiments_cli(  # noqa: C901
     )
 
     # Define the workspace
-    workdir = Path(workdir) if workdir else None
-
-    if workspace:
-        ws_env = get_workspace(workspace)
-        if ws_env is None:
-            raise RuntimeError("No workspace named %s", workspace)
-
-        logging.info("Using workspace %s", ws_env.id)
-        if workdir:
-            # Overrides working directory
-            logging.info(" override working directory: %s", workdir)
-            ws_env.path = workdir
-        else:
-            workdir = ws_env.path
-    elif workdir:
-        logging.info("Using workdir %s", workdir)
-        ws_env = workdir
-    else:
-        ws_env = get_workspace()
-        assert ws_env is not None, "No workdir or workspace defined, and no default"
-        logging.info("Using default workspace %s", ws_env.id)
+    ws_env = find_workspace(workdir=workdir, workspace=workspace)
+    workdir = ws_env.path
 
     logging.info("Using working directory %s", str(workdir.resolve()))
 

{experimaestro-1.5.6 → experimaestro-1.5.7}/src/experimaestro/launchers/slurm/base.py

@@ -262,7 +262,9 @@ class SlurmProcessBuilder(ProcessBuilder):
         addstream(builder.command, "-i", self.stdin)
 
         builder.command.extend(self.command)
-        logger.info("slurm sbatch command: %s", builder.command)
+        logger.info(
+            "slurm sbatch command: %s", " ".join(f'"{s}"' for s in builder.command)
+        )
         handler = OutputCaptureHandler()
         builder.stdout = Redirect.pipe(handler)
         builder.stderr = Redirect.inherit()

{experimaestro-1.5.6 → experimaestro-1.5.7}/src/experimaestro/notifications.py

@@ -44,6 +44,12 @@ class LevelInformation:
         return f"[{self.level}] {self.desc} {int(self.progress*1000)/10}%"
 
 
+class ListenerInformation:
+    def __init__(self, url: str):
+        self.url = url
+        self.error_count = 0
+
+
 class Reporter(threading.Thread):
     NOTIFICATION_FOLDER = ".notifications"
 
@@ -59,7 +65,7 @@ class Reporter(threading.Thread):
         super().__init__(daemon=True)
         self.path = path / Reporter.NOTIFICATION_FOLDER
         self.path.mkdir(exist_ok=True)
-        self.urls: Dict[str, str] = {}
+        self.urls: Dict[str, ListenerInformation] = {}
 
         # Last check of notification URLs
         self.lastcheck = 0
@@ -80,7 +86,7 @@ class Reporter(threading.Thread):
             self.cv.notifyAll()
 
     @staticmethod
-    def isfatal_httperror(e: Exception) -> bool:
+    def isfatal_httperror(e: Exception, info: ListenerInformation) -> bool:
         """Returns True if this HTTP error indicates that the server won't recover"""
         if isinstance(e, HTTPError):
             if e.code >= 400 and e.code < 500:
@@ -90,6 +96,13 @@ class Reporter(threading.Thread):
                 return True
             if isinstance(e.reason, socket.gaierror) and e.reason.errno == -2:
                 return True
+            if isinstance(e.reason, TimeoutError):
+                info.error_count += 1
+
+            # Too many errors
+            if info.error_count > 3:
+                logger.info("Too many errors with %s", info.error_count)
+                return True
 
         return False
 
@@ -100,8 +113,8 @@ class Reporter(threading.Thread):
         mtime = os.path.getmtime(self.path)
         if mtime > self.lastcheck:
             for f in self.path.iterdir():
-                self.urls[f.name] = f.read_text().strip()
-                logger.info("Added new notification URL: %s", self.urls[f.name])
+                self.urls[f.name] = ListenerInformation(f.read_text().strip())
+                logger.info("Added new notification URL: %s", self.urls[f.name].url)
                 f.unlink()
 
             self.lastcheck = os.path.getmtime(self.path)
@@ -128,7 +141,9 @@ class Reporter(threading.Thread):
                         params = level.report()
 
                         # Go over all URLs
-                        for key, baseurl in self.urls.items():
+                        for key, info in self.urls.items():
+                            baseurl = info.url
+
                             url = "{}/progress?{}".format(
                                 baseurl, urllib.parse.urlencode(params)
                             )
@@ -147,7 +162,7 @@ class Reporter(threading.Thread):
                                     url,
                                     e,
                                 )
-                                if Reporter.isfatal_httperror(e):
+                                if Reporter.isfatal_httperror(e, info):
                                     toremove.append(key)
 
                 # Removes unvalid URLs
@@ -165,7 +180,8 @@ class Reporter(threading.Thread):
             self.check_urls()
             if self.urls:
                 # Go over all URLs
-                for key, baseurl in self.urls.items():
+                for key, info in self.urls.items():
+                    baseurl = info.url
                     url = "{}?status=eoj".format(baseurl)
                     try:
                         with urlopen(url) as _:
@@ -243,7 +259,7 @@ class xpm_tqdm(std_tqdm):
 
     def update(self, n=1):
         result = super().update(n)
-        if self.total is not None:
+        if self.total is not None and self.total > 0:
             progress(self.n / self.total, level=self.pos, console=False)
         return result
 

{experimaestro-1.5.6 → experimaestro-1.5.7}/src/experimaestro/run.py

@@ -155,7 +155,6 @@ class TaskRunner:
                 self.donepath.touch()
 
                 # ... and finish the exit process
-                logger.info("This is the end (TODO: remove this line)")
                 raise
             else:
                 self.handle_error(e.code, None)

{experimaestro-1.5.6 → experimaestro-1.5.7}/src/experimaestro/scheduler/base.py

@@ -803,11 +803,6 @@ class experiment:
             else None
         )
 
-        # Copy environment variable from main (but do not
-        # override)
-        for key, value in settings.env.items():
-            self.setenv(key, value, override=False)
-
         if os.environ.get("XPM_ENABLEFAULTHANDLER", "0") == "1":
             import faulthandler