experimaestro 1.5.4__py3-none-any.whl → 1.6.0__py3-none-any.whl

experimaestro/cli/__init__.py

@@ -0,0 +1,312 @@
+# flake8: noqa: T201
+import sys
+from typing import Set, Optional
+import pkg_resources
+from itertools import chain
+from shutil import rmtree
+import click
+import logging
+from functools import cached_property, update_wrapper
+from pathlib import Path
+import subprocess
+from termcolor import cprint
+
+import experimaestro
+from experimaestro.experiments.cli import experiments_cli
+import experimaestro.launcherfinder.registry as launcher_registry
+from experimaestro.settings import find_workspace
+
+# --- Command line main options
+logging.basicConfig(level=logging.INFO)
+
+
+def pass_cfg(f):
+    """Pass configuration information"""
+
+    @click.pass_context
+    def new_func(ctx, *args, **kwargs):
+        return ctx.invoke(f, ctx.obj, *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+def check_xp_path(ctx, self, path: Path):
+    if not (path / ".__experimaestro__").is_file():
+        cprint(f"{path} is not an experimaestro working directory", "red")
+        for path in path.parents:
+            if (path / ".__experimaestro__").is_file():
+                cprint(f"{path} could be the folder you want", "green")
+                if click.confirm("Do you want to use this folder?"):
+                    return path
+        sys.exit(1)
+
+    return path
+
+
+class RunConfig:
+    def __init__(self):
+        self.traceback = False
+
+
+def pass_cfg(f):
+    """Pass configuration information"""
+
+    @click.pass_context
+    def new_func(ctx, *args, **kwargs):
+        return ctx.invoke(f, ctx.obj, *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+@click.group()
+@click.option("--quiet", is_flag=True, help="Be quiet")
+@click.option("--debug", is_flag=True, help="Be even more verbose (implies traceback)")
+@click.option(
+    "--traceback", is_flag=True, help="Display traceback if an exception occurs"
+)
+@click.pass_context
+def cli(ctx, quiet, debug, traceback):
+    if quiet:
+        logging.getLogger().setLevel(logging.WARN)
+    elif debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    ctx.obj = RunConfig()
+    ctx.obj.traceback = traceback
+
+
+# Adds the run-experiment command
+cli.add_command(experiments_cli, "run-experiment")
+
+
+@cli.command(help="Get version")
+def version():
+    print(experimaestro.__version__)
+
+
+@click.argument("parameters", type=Path)
+@cli.command(context_settings={"allow_extra_args": True})
+def run(parameters):
+    """Run a task"""
+
+    from experimaestro.run import run as do_run
+
+    do_run(parameters)
+
+
+@click.argument("path2", type=Path)
+@click.argument("path1", type=Path)
+@cli.command(context_settings={"allow_extra_args": True})
+def parameters_difference(path1, path2):
+    """Compute the difference between two configurations"""
+
+    from experimaestro.tools.diff import diff
+
+    diff(path1, path2)
+
+
+@click.option(
+    "--clean", is_flag=True, help="Remove the socket file and its enclosing directory"
+)
+@click.argument("unix-path", type=Path)
+@cli.command()
+def rpyc_server(unix_path, clean):
+    """Start an rPyC server"""
+    from experimaestro.rpyc import start_server
+
+    start_server(unix_path, clean=clean)
+
+
+@cli.group()
+def deprecated():
+    """Manage identifier changes"""
+    pass
+
+
+@click.option("--fix", is_flag=True, help="Generate links to new IDs")
+@click.option("--cleanup", is_flag=True, help="Remove symbolic links and move folders")
+@click.argument("path", type=Path, callback=check_xp_path)
+@deprecated.command(name="list")
+def deprecated_list(path: Path, fix: bool, cleanup: bool):
+    """List deprecated jobs"""
+    from experimaestro.tools.jobs import fix_deprecated
+
+    if cleanup and not fix:
+        logging.warning("Ignoring --cleanup since we are not fixing old IDs")
+    fix_deprecated(path, fix, cleanup)
+
+
+@click.argument("path", type=Path, callback=check_xp_path)
+@deprecated.command()
+def diff(path: Path):
+    """Show the reason of the identifier change for a job"""
+    from experimaestro.tools.jobs import load_job
+    from experimaestro import Config
+    from experimaestro.core.objects import ConfigWalkContext
+
+    _, job = load_job(path / "params.json", discard_id=False)
+    _, new_job = load_job(path / "params.json")
+
+    def check(path: str, value, new_value, done: Set[int]):
+        if isinstance(value, Config):
+            if id(value) in done:
+                return
+            done.add(id(value))
+
+            old_id = value.__xpm__.identifier.all.hex()
+            new_id = new_value.__xpm__.identifier.all.hex()
+
+            if new_id != old_id:
+                print(f"{path} differ: {new_id} vs {old_id}")
+
+                for arg in value.__xpmtype__.arguments.values():
+                    arg_value = getattr(value, arg.name)
+                    arg_newvalue = getattr(new_value, arg.name)
+                    check(f"{path}/{arg.name}", arg_value, arg_newvalue, done)
+
+        elif isinstance(value, list):
+            for ix, (array_value, array_newvalue) in enumerate(zip(value, new_value)):
+                check(f"{path}.{ix}", array_value, array_newvalue, done)
+
+        elif isinstance(value, dict):
+            for key, dict_value in value.items():
+                check(f"{path}.{key}", dict_value, new_value[key], done)
+
+    check(".", job, new_job, set())
+
+
+@click.option("--show-all", is_flag=True, help="Show even not orphans")
+@click.option(
+    "--ignore-old", is_flag=True, help="Ignore old jobs for unfinished experiments"
+)
+@click.option("--clean", is_flag=True, help="Prune the orphan folders")
+@click.option("--size", is_flag=True, help="Show size of each folder")
+@click.argument("path", type=Path, callback=check_xp_path)
+@cli.command()
+def orphans(path: Path, clean: bool, size: bool, show_all: bool, ignore_old: bool):
+    """Check for tasks that are not part of an experimental plan"""
+
+    jobspath = path / "jobs"
+
+    def getjobs(path: Path):
+        return ((str(p.relative_to(path)), p) for p in path.glob("*/*") if p.is_dir())
+
+    def show(key: str, prefix=""):
+        if size:
+            print(
+                prefix,
+                subprocess.check_output(["du", "-hs", key], cwd=jobspath)
+                .decode("utf-8")
+                .strip(),
+                sep=None,
+            )
+        else:
+            print(prefix, key, sep=None)
+
+    for p in (path / "xp").glob("*/jobs.bak"):
+        logging.warning("Experiment %s has not completed successfully", p.parent.name)
+
+    # Retrieve the jobs within expedriments (jobs and jobs.bak folder within experiments)
+    xpjobs = set()
+    if ignore_old:
+        paths = (path / "xp").glob("*/jobs")
+    else:
+        paths = chain((path / "xp").glob("*/jobs"), (path / "xp").glob("*/jobs.bak"))
+
+    for p in paths:
+        if p.is_dir():
+            for relpath, path in getjobs(p):
+                xpjobs.add(relpath)
+
+    # Now, look at stored jobs
+    found = 0
+    for key, jobpath in getjobs(jobspath):
+        if key not in xpjobs:
+            show(key)
+            if clean:
+                logging.info("Removing data in %s", jobpath)
+                rmtree(jobpath)
+        else:
+            if show_all:
+                show(key, prefix="[not orphan] ")
+            found += 1
+
+    print(f"{found} jobs are not orphans")
+
+
+def arg_split(ctx, param, value):
+    # split columns by ',' and remove whitespace
+    return set(c.strip() for c in value.split(","))
+
+
+@click.option("--skip", default=set(), callback=arg_split)
+@click.argument("package", type=str)
+@click.argument("objects", type=Path)
+@cli.command()
+def check_documentation(objects, package, skip):
+    """Check that all the configuration and tasks are documented within a
+    package, relying on the sphinx objects.inv file"""
+    from experimaestro.tools.documentation import documented_from_objects, undocumented
+
+    documented = documented_from_objects(objects)
+    errors, configs = undocumented([package], documented, skip)
+    for config in configs:
+        cprint(f"{config.__module__}.{config.__qualname__}", "red")
+
+    if errors > 0 or configs:
+        sys.exit(1)
+
+
+@click.option("--config", type=Path, help="Show size of each folder")
+@click.argument("spec", type=str)
+@cli.command()
+def find_launchers(config: Optional[Path], spec: str):
+    """Find launchers matching a specification"""
+    if config is not None:
+        launcher_registry.LauncherRegistry.set_config_dir(config)
+
+    print(launcher_registry.find_launcher(spec))
+
+
+class Launchers(click.MultiCommand):
+    """Connectors commands"""
+
+    @cached_property
+    def commands(self):
+        map = {}
+        for ep in pkg_resources.iter_entry_points(f"experimaestro.{self.name}"):
+            if get_cli := getattr(ep.load(), "get_cli", None):
+                map[ep.name] = get_cli()
+        return map
+
+    def list_commands(self, ctx):
+        return self.commands.keys()
+
+    def get_command(self, ctx, name):
+        return self.commands[name]
+
+
+cli.add_command(Launchers("launchers", help="Launcher specific commands"))
+cli.add_command(Launchers("connectors", help="Connector specific commands"))
+cli.add_command(Launchers("tokens", help="Token specific commands"))
+
+
+@cli.group()
+@click.option("--workdir", type=Path, default=None)
+@click.option("--workspace", type=str, default=None)
+@click.pass_context
+def experiments(ctx, workdir, workspace):
+    """Manage experiments"""
+    ws = find_workspace(workdir=workdir, workspace=workspace)
+    path = check_xp_path(None, None, ws.path)
+    ctx.obj = path
+
+
+@experiments.command()
+@pass_cfg
+def list(workdir: Path):
+    for p in (workdir / "xp").iterdir():
+        if (p / "jobs.bak").exists():
+            cprint(f"[unfinished] {p.name}", "yellow")
+        else:
+            cprint(p.name, "cyan")

experimaestro/{filter.py → cli/filter.py}

@@ -22,12 +22,12 @@ class JobInformation:
 
     @cached_property
     def state(self) -> Optional[JobState]:
-        if (self.path / f"{self.scriptname}.pid").is_file():
-            return JobState.RUNNING
-        elif (self.path / f"{self.scriptname}.done").is_file():
+        if (self.path / f"{self.scriptname}.done").is_file():
             return JobState.DONE
-        elif (self.path / f"{self.scriptname}.failed").is_file():
+        if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
+        if (self.path / f"{self.scriptname}.pid").is_file():
+            return JobState.RUNNING
         else:
             return None
 

experimaestro/cli/jobs.py

@@ -0,0 +1,261 @@
+# flake8: noqa: T201
+import subprocess
+from typing import Optional
+from shutil import rmtree
+import click
+from pathlib import Path
+from termcolor import colored, cprint
+
+from experimaestro.settings import find_workspace
+from . import check_xp_path, cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def jobs(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Job control: list, kill and clean
+
+    The job filter is a boolean expression where tags (alphanumeric)
+    and special job information (@state for job state, @name for job full
+    name) can be compared to a given value (using '~' for regex matching,
+    '=', 'not in', or 'in')
+
+    For instance,
+
+    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
+
+    selects jobs where the tag model is "bm25", the tag mode is either
+    "a" or "b", and the state is running.
+
+    """
+    ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+    check_xp_path(ctx, None, ws.path)
+
+
+def process(
+    workspace,
+    *,
+    experiment="",
+    tags="",
+    ready=False,
+    clean=False,
+    kill=False,
+    filter="",
+    perform=False,
+    fullpath=False,
+):
+    from .filter import createFilter, JobInformation
+    from experimaestro.scheduler import JobState
+
+    _filter = createFilter(filter) if filter else lambda x: True
+
+    # Get all jobs from experiments
+    job2xp = {}
+
+    path = workspace.path
+    for p in (path / "xp").glob("*"):
+        for job in p.glob("jobs/*/*"):
+            job_path = job.resolve()
+            if job_path.is_dir():
+                *_, scriptname = job_path.parent.name.rsplit(".", 1)
+                job2xp.setdefault(scriptname, set()).add(p.name)
+
+        if (p / "jobs.bak").is_dir():
+            cprint(f"  Experiment {p.name} has not finished yet", "red")
+            if (not perform) and (kill or clean):
+                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                kill = False
+                clean = False
+
+    # Now, process jobs
+    for job in path.glob("jobs/*/*"):
+        info = None
+        p = job.resolve()
+        if p.is_dir():
+            *_, scriptname = p.parent.name.rsplit(".", 1)
+            xps = job2xp.get(scriptname, set())
+            if experiment and experiment not in xps:
+                continue
+
+            info = JobInformation(p, scriptname)
+            job_str = (
+                (str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}")
+                + " "
+                + ",".join(xps)
+            )
+
+            if filter:
+                if not _filter(info):
+                    continue
+
+            if info.state is None:
+                print(colored(f"NODIR   {job_str}", "red"), end="")
+            elif info.state.running():
+                if kill:
+                    if perform:
+                        process = info.getprocess()
+                        if process is None:
+                            cprint(
+                                "internal error – no process could be retrieved",
+                                "red",
+                            )
+                        else:
+                            cprint(f"KILLING {process}", "light_red")
+                            process.kill()
+                    else:
+                        print("KILLING (not performing)", process)
+                print(
+                    colored(f"{info.state.name:8}{job_str}", "yellow"),
+                    end="",
+                )
+            elif info.state == JobState.DONE:
+                print(
+                    colored(f"DONE    {job_str}", "green"),
+                    end="",
+                )
+            elif info.state == JobState.ERROR:
+                print(colored(f"FAIL    {job_str}", "red"), end="")
+            else:
+                print(
+                    colored(f"{info.state.name:8}{job_str}", "red"),
+                    end="",
+                )
+
+        else:
+            if not ready:
+                continue
+            print(colored(f"READY {job_path}", "yellow"), end="")
+
+        if tags:
+            print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
+        else:
+            print()
+
+        if clean and info.state and info.state.finished():
+            if perform:
+                cprint("Cleaning...", "red")
+                rmtree(p)
+            else:
+                cprint("Cleaning... (not performed)", "red")
+    print()
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def list(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the killing")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def kill(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        kill=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the cleaning")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def clean(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        clean=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.argument("jobid", type=str)
+@click.option(
+    "--follow", "-f", help="Use tail instead of less to follow changes", is_flag=True
+)
+@click.option("--std", help="Follow stdout instead of stderr", is_flag=True)
+@jobs.command()
+@click.pass_context
+def log(ctx, jobid: str, follow: bool, std: bool):
+    task_name, task_hash = jobid.split("/")
+    _, name = task_name.rsplit(".", 1)
+    path = (
+        ctx.obj.workspace.path
+        / "jobs"
+        / task_name
+        / task_hash
+        / f"""{name}.{'out' if std else 'err'}"""
+    )
+    if follow:
+        subprocess.run(["tail", "-f", path])
+    else:
+        subprocess.run(["less", "-r", path])
+
+
+@click.argument("jobid", type=str)
+@jobs.command()
+@click.pass_context
+def path(ctx, jobid: str):
+    task_name, task_hash = jobid.split("/")
+    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    print(path)

experimaestro/click.py

@@ -1,6 +1,4 @@
 import click
-from experimaestro import Environment
-from experimaestro.run import parse_commandline
 
 """Defines the task command line argument prefix for experimaestro-handled command lines"""
 
@@ -45,36 +43,3 @@ class forwardoption(metaclass=forwardoptionMetaclass):
     def __getattr__(self, key):
         """Access to a class field"""
         return forwardoption([key])
-
-
-def environment(name: str):
-    def annotate(f):
-        def callback_env(ctx, name, value):
-            if value:
-                assert name not in ctx.params, "Environment has already been set"
-            else:
-                return ctx.params.get(name, None)
-            return Environment.get(value)
-
-        def callback(ctx, param, value):
-            if value:
-                if name not in ctx.params:
-                    ctx.params[name] = Environment()
-                ctx.params[name].workdir = value
-
-        f = click.option(
-            f"--{name}-workdir",
-            type=str,
-            callback=callback,
-            expose_value=False,
-            help="Experimaestro environment",
-        )(f)
-        f = click.option(
-            f"--{name}",
-            type=str,
-            callback=callback_env,
-            help="Experimaestro environment",
-        )(f)
-        return f
-
-    return annotate

experimaestro/connectors/ssh.py

@@ -1,12 +1,15 @@
+try:
+    from pathlib import Path, _posix_flavour
+except ImportError:
+    # Avoids problem with python 3.12 where this module does not work
+    # anyways
+    _posix_flavour = None
+
 from dataclasses import dataclass
-from pathlib import Path, _posix_flavour
 import io
 import os
 import re
-from experimaestro.launcherfinder import LauncherRegistry, YAMLDataClass
-from fabric import Connection
-from invoke import Promise
-import invoke.exceptions
+from experimaestro.launcherfinder import LauncherRegistry
 from urllib.parse import urlparse
 from itertools import chain
 from . import Connector
@@ -19,6 +22,22 @@ from . import (
 from experimaestro.locking import Lock
 from experimaestro.tokens import Token
 
+try:
+    from fabric import Connection
+    from invoke import Promise
+    from invoke.exceptions import Failure
+except Exception:
+    # Just define placeholders
+    class Connection:
+        pass
+
+    class Promise:
+        pass
+
+    class Failure(Exception):
+        pass
+
+
 # Might be wise to switch to https://github.com/marian-code/ssh-utilities
 
 
@@ -132,7 +151,7 @@ class SshPath(Path):
 
 
 @dataclass
-class SshConfiguration(YAMLDataClass):
+class SshConfiguration:
     hostname: str
 
     def create(self, registry: LauncherRegistry):
@@ -172,7 +191,7 @@ class SshProcess(Process):
     def wait(self) -> int:
         try:
             self.promise.join()
-        except invoke.exceptions.Failure:
+        except Failure:
             raise