experimaestro 1.5.3__tar.gz → 1.6.0__tar.gz

{experimaestro-1.5.3 → experimaestro-1.6.0}/PKG-INFO

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: experimaestro
-Version: 1.5.3
+Version: 1.6.0
 Summary: "Experimaestro is a computer science experiment manager"
-Home-page: https://github.com/experimaestro/experimaestro-python
 License: GPL-3
 Keywords: experiment manager
 Author: Benjamin Piwowarski
@@ -20,13 +19,13 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: arpeggio (>=2,<3)
 Requires-Dist: attrs (>=23.1.0,<24.0.0)
 Requires-Dist: click (>=8)
 Requires-Dist: decorator (>=5,<6)
 Requires-Dist: docstring-parser (>=0.15,<0.16)
-Requires-Dist: fabric (>=3,<4)
 Requires-Dist: fasteners (>=0.19,<0.20)
 Requires-Dist: flask (>=2.3,<3.0)
 Requires-Dist: flask-socketio (>=5.3,<6.0)
@@ -41,7 +40,7 @@ Requires-Dist: pyparsing (>=3.1,<4.0)
 Requires-Dist: pytools (>=2023.1.1,<2024.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: requests (>=2.31,<3.0)
-Requires-Dist: rpyc (>=5,<6)
+Requires-Dist: rpyc (>=5,<7)
 Requires-Dist: sortedcontainers (>=2.4,<3.0)
 Requires-Dist: termcolor (>=2.3)
 Requires-Dist: tqdm (>=4.66.1,<5.0.0)

{experimaestro-1.5.3 → experimaestro-1.6.0}/pyproject.toml

@@ -19,7 +19,7 @@ include = [
     "src/experimaestro/sphinx/static/experimaestro.css",
     "src/experimaestro/mkdocs/style.css"
 ]
-version = "1.5.3"
+version = "1.6.0"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -59,15 +59,15 @@ flask-socketio = "^5.3"
 arpeggio = "^2"
 watchdog = "^2"
 marshmallow = "^3.20"
-fabric = "^3"
 decorator = "^5"
-rpyc = "^5"
+rpyc = ">=5,<7"
 
 [tool.poetry.group.ssh]
 optional = true
 
 [tool.poetry.group.ssh.dependencies]
 paramiko = "^3.3"
+fabric = "^3"
 
 [tool.poetry.group.dev]
 optional = true
@@ -90,10 +90,6 @@ slurm = "experimaestro.launchers.slurm:BatchSlurmProcess"
 local = "experimaestro.connectors.local:LocalConnector"
 ssh = "experimaestro.connectors.ssh:SshConnector"
 
-[tool.poetry.plugins."experimaestro.launchers"]
-unix = "experimaestro.launchers.direct:DirectLauncher"
-slurm = "experimaestro.launchers.slurm:SlurmLauncher"
-
 [tool.poetry.plugins."experimaestro.tokens"]
 unix = "experimaestro.tokens:CounterToken"
 
@@ -123,7 +119,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.5.3"
+version = "1.6.0"
 changelog_start_rev = "0.15.0"
 tag_format = "v$version"
 update_changelog_on_bump = true

{experimaestro-1.5.3 → experimaestro-1.6.0}/src/experimaestro/__init__.py

@@ -53,7 +53,6 @@ from .core.context import SerializationContext
 from .core.serializers import SerializationLWTask, PathSerializationLWTask
 from .core.types import Any, SubmitHook
 from .launchers import Launcher
-from .scheduler.environment import Environment
 from .scheduler.workspace import Workspace, RunMode
 from .scheduler import Scheduler, experiment, FailedExperiment
 from .notifications import progress, tqdm

experimaestro-1.6.0/src/experimaestro/__main__.py

@@ -0,0 +1,12 @@
+from experimaestro.cli import cli
+
+# flake8: noqa: F401
+import experimaestro.cli.jobs
+
+
+def main():
+    cli(obj=None)
+
+
+if __name__ == "__main__":
+    main()

experimaestro-1.5.3/src/experimaestro/__main__.py → experimaestro-1.6.0/src/experimaestro/cli/__init__.py

@@ -1,6 +1,5 @@
 # flake8: noqa: T201
 import sys
-from types import ModuleType
 from typing import Set, Optional
 import pkg_resources
 from itertools import chain
@@ -10,11 +9,12 @@ import logging
 from functools import cached_property, update_wrapper
 from pathlib import Path
 import subprocess
-from termcolor import colored, cprint
+from termcolor import cprint
 
 import experimaestro
 from experimaestro.experiments.cli import experiments_cli
 import experimaestro.launcherfinder.registry as launcher_registry
+from experimaestro.settings import find_workspace
 
 # --- Command line main options
 logging.basicConfig(level=logging.INFO)
@@ -175,121 +175,6 @@ def diff(path: Path):
     check(".", job, new_job, set())
 
 
-@click.argument("path", type=Path, callback=check_xp_path)
-@click.option("--experiment", default=None, help="Restrict to this experiment")
-@click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
-@click.option("--filter", default="", help="Filter expression")
-@click.option(
-    "--force",
-    is_flag=True,
-    help="Force operation even if experiment has not been completed yet",
-)
-@click.option("--kill", is_flag=True, help="Kill filtered tasks (when/before running)")
-@click.option("--clean", is_flag=True, help="Remove finished tasks directories")
-@cli.command()
-def jobs(
-    path: Path,
-    experiment: str,
-    filter: str,
-    tags: bool,
-    ready: bool,
-    kill: bool,
-    clean: bool,
-    force: bool,
-):
-    """Job control: list, kill and clean
-
-    The job filter is a boolean expression where tags (alphanumeric)
-    and special job information (@state for job state, @name for job full
-    name) can be compared to a given value (using '~' for regex matching,
-    '=', 'not in', or 'in')
-
-    For instance,
-
-    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
-
-    selects jobs where the tag model is "bm25", the tag mode is either
-    "a" or "b", and the state is running.
-
-    """
-    for p in (path / "xp").glob("*"):
-        if experiment and p.name != experiment:
-            continue
-
-        from .filter import createFilter, JobInformation
-        from experimaestro.scheduler import JobState
-
-        _filter = createFilter(filter) if filter else lambda x: True
-
-        print(f"* Experiment {p.name}")
-        if (p / "jobs.bak").is_dir():
-            cprint("  Experiment has not finished yet", "red")
-            if not force and (kill or clean):
-                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
-                kill = False
-                clean = False
-        print()
-
-        for job in p.glob("jobs/*/*"):
-            info = None
-            p = job.resolve()
-            if p.is_dir():
-                *_, scriptname = p.parent.name.rsplit(".", 1)
-
-                info = JobInformation(p, scriptname)
-                if filter:
-                    if not _filter(info):
-                        continue
-
-                if info.state is None:
-                    print(
-                        colored(f"NODIR   {job.parent.name}/{job.name}", "red"), end=""
-                    )
-                elif info.state.running():
-                    if kill:
-                        process = info.getprocess()
-                        print("KILLING", process)
-                        process.kill()
-                    print(
-                        colored(
-                            f"{info.state.name:8}{job.parent.name}/{job.name}", "yellow"
-                        ),
-                        end="",
-                    )
-                elif info.state == JobState.DONE:
-                    print(
-                        colored(f"DONE    {job.parent.name}/{job.name}", "green"),
-                        end="",
-                    )
-                elif info.state == JobState.ERROR:
-                    print(
-                        colored(f"FAIL    {job.parent.name}/{job.name}", "red"), end=""
-                    )
-                else:
-                    print(
-                        colored(
-                            f"{info.state.name:8}{job.parent.name}/{job.name}", "red"
-                        ),
-                        end="",
-                    )
-
-            else:
-                if not ready:
-                    continue
-                print(colored(f"READY {job.parent.name}/{job.name}", "yellow"), end="")
-
-            if tags:
-                print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
-            else:
-                print()
-
-            if clean and info.state and info.state.finished():
-                cprint("Cleaning...", "red")
-                rmtree(p)
-        print()
-
-
 @click.option("--show-all", is_flag=True, help="Show even not orphans")
 @click.option(
     "--ignore-old", is_flag=True, help="Ignore old jobs for unfinished experiments"
@@ -407,11 +292,14 @@ cli.add_command(Launchers("tokens", help="Token specific commands"))
 
 
 @cli.group()
-@click.argument("workdir", type=Path, callback=check_xp_path)
+@click.option("--workdir", type=Path, default=None)
+@click.option("--workspace", type=str, default=None)
 @click.pass_context
-def experiments(ctx, workdir):
+def experiments(ctx, workdir, workspace):
     """Manage experiments"""
-    ctx.obj = workdir
+    ws = find_workspace(workdir=workdir, workspace=workspace)
+    path = check_xp_path(None, None, ws.path)
+    ctx.obj = path
 
 
 @experiments.command()
@@ -422,11 +310,3 @@ def list(workdir: Path):
             cprint(f"[unfinished] {p.name}", "yellow")
         else:
             cprint(p.name, "cyan")
-
-
-def main():
-    cli(obj=None)
-
-
-if __name__ == "__main__":
-    main()

{experimaestro-1.5.3/src/experimaestro → experimaestro-1.6.0/src/experimaestro/cli}/filter.py

@@ -22,12 +22,12 @@ class JobInformation:
 
     @cached_property
     def state(self) -> Optional[JobState]:
-        if (self.path / f"{self.scriptname}.pid").is_file():
-            return JobState.RUNNING
-        elif (self.path / f"{self.scriptname}.done").is_file():
+        if (self.path / f"{self.scriptname}.done").is_file():
             return JobState.DONE
-        elif (self.path / f"{self.scriptname}.failed").is_file():
+        if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
+        if (self.path / f"{self.scriptname}.pid").is_file():
+            return JobState.RUNNING
         else:
             return None
 

experimaestro-1.6.0/src/experimaestro/cli/jobs.py

@@ -0,0 +1,261 @@
+# flake8: noqa: T201
+import subprocess
+from typing import Optional
+from shutil import rmtree
+import click
+from pathlib import Path
+from termcolor import colored, cprint
+
+from experimaestro.settings import find_workspace
+from . import check_xp_path, cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def jobs(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Job control: list, kill and clean
+
+    The job filter is a boolean expression where tags (alphanumeric)
+    and special job information (@state for job state, @name for job full
+    name) can be compared to a given value (using '~' for regex matching,
+    '=', 'not in', or 'in')
+
+    For instance,
+
+    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
+
+    selects jobs where the tag model is "bm25", the tag mode is either
+    "a" or "b", and the state is running.
+
+    """
+    ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+    check_xp_path(ctx, None, ws.path)
+
+
+def process(
+    workspace,
+    *,
+    experiment="",
+    tags="",
+    ready=False,
+    clean=False,
+    kill=False,
+    filter="",
+    perform=False,
+    fullpath=False,
+):
+    from .filter import createFilter, JobInformation
+    from experimaestro.scheduler import JobState
+
+    _filter = createFilter(filter) if filter else lambda x: True
+
+    # Get all jobs from experiments
+    job2xp = {}
+
+    path = workspace.path
+    for p in (path / "xp").glob("*"):
+        for job in p.glob("jobs/*/*"):
+            job_path = job.resolve()
+            if job_path.is_dir():
+                *_, scriptname = job_path.parent.name.rsplit(".", 1)
+                job2xp.setdefault(scriptname, set()).add(p.name)
+
+        if (p / "jobs.bak").is_dir():
+            cprint(f"  Experiment {p.name} has not finished yet", "red")
+            if (not perform) and (kill or clean):
+                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                kill = False
+                clean = False
+
+    # Now, process jobs
+    for job in path.glob("jobs/*/*"):
+        info = None
+        p = job.resolve()
+        if p.is_dir():
+            *_, scriptname = p.parent.name.rsplit(".", 1)
+            xps = job2xp.get(scriptname, set())
+            if experiment and experiment not in xps:
+                continue
+
+            info = JobInformation(p, scriptname)
+            job_str = (
+                (str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}")
+                + " "
+                + ",".join(xps)
+            )
+
+            if filter:
+                if not _filter(info):
+                    continue
+
+            if info.state is None:
+                print(colored(f"NODIR   {job_str}", "red"), end="")
+            elif info.state.running():
+                if kill:
+                    if perform:
+                        process = info.getprocess()
+                        if process is None:
+                            cprint(
+                                "internal error – no process could be retrieved",
+                                "red",
+                            )
+                        else:
+                            cprint(f"KILLING {process}", "light_red")
+                            process.kill()
+                    else:
+                        print("KILLING (not performing)", process)
+                print(
+                    colored(f"{info.state.name:8}{job_str}", "yellow"),
+                    end="",
+                )
+            elif info.state == JobState.DONE:
+                print(
+                    colored(f"DONE    {job_str}", "green"),
+                    end="",
+                )
+            elif info.state == JobState.ERROR:
+                print(colored(f"FAIL    {job_str}", "red"), end="")
+            else:
+                print(
+                    colored(f"{info.state.name:8}{job_str}", "red"),
+                    end="",
+                )
+
+        else:
+            if not ready:
+                continue
+            print(colored(f"READY {job_path}", "yellow"), end="")
+
+        if tags:
+            print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
+        else:
+            print()
+
+        if clean and info.state and info.state.finished():
+            if perform:
+                cprint("Cleaning...", "red")
+                rmtree(p)
+            else:
+                cprint("Cleaning... (not performed)", "red")
+    print()
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def list(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the killing")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def kill(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        kill=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the cleaning")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def clean(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        clean=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.argument("jobid", type=str)
+@click.option(
+    "--follow", "-f", help="Use tail instead of less to follow changes", is_flag=True
+)
+@click.option("--std", help="Follow stdout instead of stderr", is_flag=True)
+@jobs.command()
+@click.pass_context
+def log(ctx, jobid: str, follow: bool, std: bool):
+    task_name, task_hash = jobid.split("/")
+    _, name = task_name.rsplit(".", 1)
+    path = (
+        ctx.obj.workspace.path
+        / "jobs"
+        / task_name
+        / task_hash
+        / f"""{name}.{'out' if std else 'err'}"""
+    )
+    if follow:
+        subprocess.run(["tail", "-f", path])
+    else:
+        subprocess.run(["less", "-r", path])
+
+
+@click.argument("jobid", type=str)
+@jobs.command()
+@click.pass_context
+def path(ctx, jobid: str):
+    task_name, task_hash = jobid.split("/")
+    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    print(path)

{experimaestro-1.5.3 → experimaestro-1.6.0}/src/experimaestro/click.py

@@ -1,6 +1,4 @@
 import click
-from experimaestro import Environment
-from experimaestro.run import parse_commandline
 
 """Defines the task command line argument prefix for experimaestro-handled command lines"""
 
@@ -45,36 +43,3 @@ class forwardoption(metaclass=forwardoptionMetaclass):
     def __getattr__(self, key):
         """Access to a class field"""
         return forwardoption([key])
-
-
-def environment(name: str):
-    def annotate(f):
-        def callback_env(ctx, name, value):
-            if value:
-                assert name not in ctx.params, "Environment has already been set"
-            else:
-                return ctx.params.get(name, None)
-            return Environment.get(value)
-
-        def callback(ctx, param, value):
-            if value:
-                if name not in ctx.params:
-                    ctx.params[name] = Environment()
-                ctx.params[name].workdir = value
-
-        f = click.option(
-            f"--{name}-workdir",
-            type=str,
-            callback=callback,
-            expose_value=False,
-            help="Experimaestro environment",
-        )(f)
-        f = click.option(
-            f"--{name}",
-            type=str,
-            callback=callback_env,
-            help="Experimaestro environment",
-        )(f)
-        return f
-
-    return annotate

{experimaestro-1.5.3 → experimaestro-1.6.0}/src/experimaestro/connectors/ssh.py

@@ -1,12 +1,15 @@
+try:
+    from pathlib import Path, _posix_flavour
+except ImportError:
+    # Avoids problem with python 3.12 where this module does not work
+    # anyways
+    _posix_flavour = None
+
 from dataclasses import dataclass
-from pathlib import Path, _posix_flavour
 import io
 import os
 import re
-from experimaestro.launcherfinder import LauncherRegistry, YAMLDataClass
-from fabric import Connection
-from invoke import Promise
-import invoke.exceptions
+from experimaestro.launcherfinder import LauncherRegistry
 from urllib.parse import urlparse
 from itertools import chain
 from . import Connector
@@ -19,6 +22,22 @@ from . import (
 from experimaestro.locking import Lock
 from experimaestro.tokens import Token
 
+try:
+    from fabric import Connection
+    from invoke import Promise
+    from invoke.exceptions import Failure
+except Exception:
+    # Just define placeholders
+    class Connection:
+        pass
+
+    class Promise:
+        pass
+
+    class Failure(Exception):
+        pass
+
+
 # Might be wise to switch to https://github.com/marian-code/ssh-utilities
 
 
@@ -132,7 +151,7 @@ class SshPath(Path):
 
 
 @dataclass
-class SshConfiguration(YAMLDataClass):
+class SshConfiguration:
     hostname: str
 
     def create(self, registry: LauncherRegistry):
@@ -172,7 +191,7 @@ class SshProcess(Process):
     def wait(self) -> int:
         try:
             self.promise.join()
-        except invoke.exceptions.Failure:
+        except Failure:
             raise