experimaestro 1.5.1__py3-none-any.whl → 2.0.0a8__py3-none-any.whl

experimaestro/cli/__init__.py

@@ -0,0 +1,311 @@
+# flake8: noqa: T201
+import sys
+from typing import Set, Optional
+import pkg_resources
+from itertools import chain
+from shutil import rmtree
+import click
+import logging
+from functools import cached_property, update_wrapper
+from pathlib import Path
+import subprocess
+from termcolor import cprint
+
+import experimaestro
+from experimaestro.experiments.cli import experiments_cli
+import experimaestro.launcherfinder.registry as launcher_registry
+from experimaestro.settings import find_workspace
+
+# --- Command line main options
+logging.basicConfig(level=logging.INFO)
+
+
+def check_xp_path(ctx, self, path: Path):
+    if not (path / ".__experimaestro__").is_file():
+        cprint(f"{path} is not an experimaestro working directory", "red")
+        for path in path.parents:
+            if (path / ".__experimaestro__").is_file():
+                cprint(f"{path} could be the folder you want", "green")
+                if click.confirm("Do you want to use this folder?"):
+                    return path
+        sys.exit(1)
+
+    return path
+
+
+class RunConfig:
+    def __init__(self):
+        self.traceback = False
+
+
+def pass_cfg(f):
+    """Pass configuration information"""
+
+    @click.pass_context
+    def new_func(ctx, *args, **kwargs):
+        return ctx.invoke(f, ctx.obj, *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+@click.group()
+@click.option("--quiet", is_flag=True, help="Be quiet")
+@click.option("--debug", is_flag=True, help="Be even more verbose (implies traceback)")
+@click.option(
+    "--traceback", is_flag=True, help="Display traceback if an exception occurs"
+)
+@click.pass_context
+def cli(ctx, quiet, debug, traceback):
+    if quiet:
+        logging.getLogger().setLevel(logging.WARN)
+    elif debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    ctx.obj = RunConfig()
+    ctx.obj.traceback = traceback
+
+
+# Adds the run-experiment command
+cli.add_command(experiments_cli, "run-experiment")
+
+
+@cli.command(help="Get version")
+def version():
+    print(experimaestro.__version__)
+
+
+@click.argument("parameters", type=Path)
+@cli.command(context_settings={"allow_extra_args": True})
+def run(parameters):
+    """Run a task"""
+
+    from experimaestro.run import run as do_run
+
+    do_run(parameters)
+
+
+@click.argument("path2", type=Path)
+@click.argument("path1", type=Path)
+@cli.command(context_settings={"allow_extra_args": True})
+def parameters_difference(path1, path2):
+    """Compute the difference between two configurations"""
+
+    from experimaestro.tools.diff import diff
+
+    diff(path1, path2)
+
+
+@click.option(
+    "--clean", is_flag=True, help="Remove the socket file and its enclosing directory"
+)
+@click.argument("unix-path", type=Path)
+@cli.command()
+def rpyc_server(unix_path, clean):
+    """Start an rPyC server"""
+    from experimaestro.rpyc import start_server
+
+    start_server(unix_path, clean=clean)
+
+
+@cli.group()
+def deprecated():
+    """Manage identifier changes"""
+    pass
+
+
+@click.option("--fix", is_flag=True, help="Generate links to new IDs")
+@click.option("--cleanup", is_flag=True, help="Remove symbolic links and move folders")
+@click.argument("path", type=Path, callback=check_xp_path)
+@deprecated.command(name="list")
+def deprecated_list(path: Path, fix: bool, cleanup: bool):
+    """List deprecated jobs"""
+    from experimaestro.tools.jobs import fix_deprecated
+
+    if cleanup and not fix:
+        logging.warning("Ignoring --cleanup since we are not fixing old IDs")
+    fix_deprecated(path, fix, cleanup)
+
+
+@click.argument("path", type=Path, callback=check_xp_path)
+@deprecated.command()
+def diff(path: Path):
+    """Show the reason of the identifier change for a job"""
+    from experimaestro.tools.jobs import load_job
+    from experimaestro import Config
+
+    _, job = load_job(path / "params.json", discard_id=False)
+    _, new_job = load_job(path / "params.json")
+
+    def check(path: str, value, new_value, done: Set[int]):
+        if isinstance(value, Config):
+            if id(value) in done:
+                return
+            done.add(id(value))
+
+            old_id = value.__xpm__.identifier.all.hex()
+            new_id = new_value.__xpm__.identifier.all.hex()
+
+            if new_id != old_id:
+                print(f"{path} differ: {new_id} vs {old_id}")
+
+                for arg in value.__xpmtype__.arguments.values():
+                    arg_value = getattr(value, arg.name)
+                    arg_newvalue = getattr(new_value, arg.name)
+                    check(f"{path}/{arg.name}", arg_value, arg_newvalue, done)
+
+        elif isinstance(value, list):
+            for ix, (array_value, array_newvalue) in enumerate(zip(value, new_value)):
+                check(f"{path}.{ix}", array_value, array_newvalue, done)
+
+        elif isinstance(value, dict):
+            for key, dict_value in value.items():
+                check(f"{path}.{key}", dict_value, new_value[key], done)
+
+    check(".", job, new_job, set())
+
+
+@click.option("--show-all", is_flag=True, help="Show even not orphans")
+@click.option(
+    "--ignore-old", is_flag=True, help="Ignore old jobs for unfinished experiments"
+)
+@click.option("--clean", is_flag=True, help="Prune the orphan folders")
+@click.option("--size", is_flag=True, help="Show size of each folder")
+@click.argument("path", type=Path, callback=check_xp_path)
+@cli.command()
+def orphans(path: Path, clean: bool, size: bool, show_all: bool, ignore_old: bool):
+    """Check for tasks that are not part of an experimental plan"""
+
+    jobspath = path / "jobs"
+
+    def getjobs(path: Path):
+        return ((str(p.relative_to(path)), p) for p in path.glob("*/*") if p.is_dir())
+
+    def show(key: str, prefix=""):
+        if size:
+            print(
+                prefix,
+                subprocess.check_output(["du", "-hs", key], cwd=jobspath)
+                .decode("utf-8")
+                .strip(),
+                sep=None,
+            )
+        else:
+            print(prefix, key, sep=None)
+
+    for p in (path / "xp").glob("*/jobs.bak"):
+        logging.warning("Experiment %s has not completed successfully", p.parent.name)
+
+    # Retrieve the jobs within expedriments (jobs and jobs.bak folder within experiments)
+    xpjobs = set()
+    if ignore_old:
+        paths = (path / "xp").glob("*/jobs")
+    else:
+        paths = chain((path / "xp").glob("*/jobs"), (path / "xp").glob("*/jobs.bak"))
+
+    for p in paths:
+        if p.is_dir():
+            for relpath, path in getjobs(p):
+                xpjobs.add(relpath)
+
+    # Now, look at stored jobs
+    found = 0
+    for key, jobpath in getjobs(jobspath):
+        if key not in xpjobs:
+            show(key)
+            if clean:
+                logging.info("Removing data in %s", jobpath)
+                rmtree(jobpath)
+        else:
+            if show_all:
+                show(key, prefix="[not orphan] ")
+            found += 1
+
+    print(f"{found} jobs are not orphans")
+
+
+def arg_split(ctx, param, value):
+    # split columns by ',' and remove whitespace
+    return set(c.strip() for c in value.split(","))
+
+
+@click.option("--skip", default=set(), callback=arg_split)
+@click.argument("package", type=str)
+@click.argument("objects", type=Path)
+@cli.command()
+def check_documentation(objects, package, skip):
+    """Check that all the configuration and tasks are documented within a
+    package, relying on the sphinx objects.inv file"""
+    from experimaestro.tools.documentation import documented_from_objects, undocumented
+
+    documented = documented_from_objects(objects)
+    errors, configs = undocumented([package], documented, skip)
+    for config in configs:
+        cprint(f"{config.__module__}.{config.__qualname__}", "red")
+
+    if errors > 0 or configs:
+        sys.exit(1)
+
+
+@click.option("--config", type=Path, help="Show size of each folder")
+@click.argument("spec", type=str)
+@cli.command()
+def find_launchers(config: Optional[Path], spec: str):
+    """Find launchers matching a specification"""
+    if config is not None:
+        launcher_registry.LauncherRegistry.set_config_dir(config)
+
+    print(launcher_registry.find_launcher(spec))
+
+
+class Launchers(click.MultiCommand):
+    """Connectors commands"""
+
+    @cached_property
+    def commands(self):
+        map = {}
+        for ep in pkg_resources.iter_entry_points(f"experimaestro.{self.name}"):
+            if get_cli := getattr(ep.load(), "get_cli", None):
+                map[ep.name] = get_cli()
+        return map
+
+    def list_commands(self, ctx):
+        return self.commands.keys()
+
+    def get_command(self, ctx, name):
+        return self.commands[name]
+
+
+cli.add_command(Launchers("launchers", help="Launcher specific commands"))
+cli.add_command(Launchers("connectors", help="Connector specific commands"))
+cli.add_command(Launchers("tokens", help="Token specific commands"))
+
+# Import and add progress commands
+from .progress import progress as progress_cli
+
+cli.add_command(progress_cli)
+
+# Import and add jobs commands
+from .jobs import jobs as jobs_cli
+
+cli.add_command(jobs_cli)
+
+
+@cli.group()
+@click.option("--workdir", type=Path, default=None)
+@click.option("--workspace", type=str, default=None)
+@click.pass_context
+def experiments(ctx, workdir, workspace):
+    """Manage experiments"""
+    ws = find_workspace(workdir=workdir, workspace=workspace)
+    path = check_xp_path(None, None, ws.path)
+    ctx.obj = path
+
+
+@experiments.command()
+@pass_cfg
+def list(workdir: Path):
+    for p in (workdir / "xp").iterdir():
+        if (p / "jobs.bak").exists():
+            cprint(f"[unfinished] {p.name}", "yellow")
+        else:
+            cprint(p.name, "cyan")

experimaestro/{filter.py → cli/filter.py}

@@ -1,20 +1,27 @@
+import asyncio
+import logging
 from typing import Any, Callable, Dict, List, Optional
 import pyparsing as pp
 from pathlib import Path
 import json
 from experimaestro.compat import cached_property
-import regex
+import re
 from experimaestro.scheduler import JobState
 
 
 class JobInformation:
-    def __init__(self, path: Path, scriptname: str):
+    def __init__(self, path: Path, scriptname: str, check: bool = False):
         self.path = path
         self.scriptname = scriptname
+        self.check = check
 
     @cached_property
     def params(self):
-        return json.loads((self.path / "params.json").read_text())
+        try:
+            return json.loads((self.path / "params.json").read_text())
+        except Exception:
+            logging.warning("Could not load params.json in %s", self.path)
+            return {"tags": {}}
 
     @cached_property
     def tags(self) -> List[str]:
@@ -22,12 +29,19 @@ class JobInformation:
 
     @cached_property
     def state(self) -> Optional[JobState]:
-        if (self.path / f"{self.scriptname}.pid").is_file():
-            return JobState.RUNNING
-        elif (self.path / f"{self.scriptname}.done").is_file():
+        if (self.path / f"{self.scriptname}.done").is_file():
             return JobState.DONE
-        elif (self.path / f"{self.scriptname}.failed").is_file():
+        if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
+        if (self.path / f"{self.scriptname}.pid").is_file():
+            if self.check:
+                if process := self.getprocess():
+                    state = asyncio.run(process.aio_state(0))
+                    if state is None or state.finished:
+                        return JobState.ERROR
+                else:
+                    return JobState.ERROR
+            return JobState.RUNNING
         else:
             return None
 
@@ -87,7 +101,7 @@ class NotInExpr(BaseInExpr):
 class RegexExpr:
     def __init__(self, tokens):
         self.var, expr = tokens
-        self.regex = regex.compile(expr)
+        self.regex = re.compile(expr)
 
     def __repr__(self):
         return f"""REGEX[{self.varname}, {self.value}]"""
@@ -103,7 +117,7 @@ class RegexExpr:
         if not value:
             return False
 
-        return self.regex.match(value)
+        return self.re.match(value)
 
 
 class ConstantString:

experimaestro/cli/jobs.py

@@ -0,0 +1,268 @@
+# flake8: noqa: T201
+import asyncio
+import subprocess
+from typing import Optional
+from shutil import rmtree
+import click
+from pathlib import Path
+from termcolor import colored, cprint
+
+from experimaestro.settings import find_workspace
+from . import check_xp_path, cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def jobs(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Job control: list, kill and clean
+
+    The job filter is a boolean expression where tags (alphanumeric)
+    and special job information (@state for job state, @name for job full
+    name) can be compared to a given value (using '~' for regex matching,
+    '=', 'not in', or 'in')
+
+    For instance,
+
+    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
+
+    selects jobs where the tag model is "bm25", the tag mode is either
+    "a" or "b", and the state is running.
+
+    """
+    ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+    check_xp_path(ctx, None, ws.path)
+
+
+def process(
+    workspace,
+    *,
+    experiment="",
+    tags="",
+    ready=False,
+    clean=False,
+    kill=False,
+    filter="",
+    perform=False,
+    fullpath=False,
+    check=False,
+):
+    from .filter import createFilter, JobInformation
+    from experimaestro.scheduler import JobState
+
+    _filter = createFilter(filter) if filter else lambda x: True
+
+    # Get all jobs from experiments
+    job2xp = {}
+
+    path = workspace.path
+    for p in (path / "xp").glob("*"):
+        for job in p.glob("jobs/*/*"):
+            job_path = job.resolve()
+            if job_path.is_dir():
+                job2xp.setdefault(job_path.name, set()).add(p.name)
+
+        if (p / "jobs.bak").is_dir():
+            cprint(f"  Experiment {p.name} has not finished yet", "red")
+            if (not perform) and (kill or clean):
+                cprint(
+                    "  Preventing kill/clean (use --perform if you want to)", "yellow"
+                )
+                kill = False
+                clean = False
+
+    # Now, process jobs
+    for job in path.glob("jobs/*/*"):
+        info = None
+        p = job.resolve()
+        if p.is_dir():
+            *_, scriptname = p.parent.name.rsplit(".", 1)
+            xps = job2xp.get(job.name, set())
+            if experiment and experiment not in xps:
+                continue
+
+            info = JobInformation(p, scriptname, check=check)
+            job_str = (
+                (str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}")
+                + " "
+                + ",".join(xps)
+            )
+
+            if filter:
+                if not _filter(info):
+                    continue
+
+            if info.state is None:
+                print(colored(f"NODIR   {job_str}", "red"), end="")
+            elif info.state.running():
+                if kill:
+                    if perform:
+                        process = info.getprocess()
+                        if process is None:
+                            cprint(
+                                "internal error – no process could be retrieved",
+                                "red",
+                            )
+                        else:
+                            cprint(f"KILLING {process}", "light_red")
+                            process.kill()
+                    else:
+                        print("KILLING (not performing)", process)
+                print(
+                    colored(f"{info.state.name:8}{job_str}", "yellow"),
+                    end="",
+                )
+            elif info.state == JobState.DONE:
+                print(
+                    colored(f"DONE    {job_str}", "green"),
+                    end="",
+                )
+            elif info.state == JobState.ERROR:
+                print(colored(f"FAIL    {job_str}", "red"), end="")
+            else:
+                print(
+                    colored(f"{info.state.name:8}{job_str}", "red"),
+                    end="",
+                )
+
+        else:
+            if not ready:
+                continue
+            print(colored(f"READY {job_path}", "yellow"), end="")
+
+        if tags:
+            print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
+        else:
+            print()
+
+        if clean and info.state and info.state.finished():
+            if perform:
+                cprint("Cleaning...", "red")
+                rmtree(p)
+            else:
+                cprint("Cleaning... (not performed)", "red")
+    print()
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@click.option("--no-check", is_flag=True, help="Check that running jobs")
+@jobs.command()
+@click.pass_context
+def list(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    no_check: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        fullpath=fullpath,
+        check=not no_check,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the killing")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def kill(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+    check: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        kill=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the cleaning")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def clean(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        clean=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.argument("jobid", type=str)
+@click.option(
+    "--follow", "-f", help="Use tail instead of less to follow changes", is_flag=True
+)
+@click.option("--std", help="Follow stdout instead of stderr", is_flag=True)
+@jobs.command()
+@click.pass_context
+def log(ctx, jobid: str, follow: bool, std: bool):
+    task_name, task_hash = jobid.split("/")
+    _, name = task_name.rsplit(".", 1)
+    path = (
+        ctx.obj.workspace.path
+        / "jobs"
+        / task_name
+        / task_hash
+        / f"""{name}.{'out' if std else 'err'}"""
+    )
+    if follow:
+        subprocess.run(["tail", "-f", path])
+    else:
+        subprocess.run(["less", "-r", path])
+
+
+@click.argument("jobid", type=str)
+@jobs.command()
+@click.pass_context
+def path(ctx, jobid: str):
+    task_name, task_hash = jobid.split("/")
+    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    print(path)