experimaestro 1.5.6__py3-none-any.whl → 1.5.7__py3-none-any.whl

experimaestro/cli/__init__.py

@@ -0,0 +1,312 @@
+# flake8: noqa: T201
+import sys
+from typing import Set, Optional
+import pkg_resources
+from itertools import chain
+from shutil import rmtree
+import click
+import logging
+from functools import cached_property, update_wrapper
+from pathlib import Path
+import subprocess
+from termcolor import cprint
+
+import experimaestro
+from experimaestro.experiments.cli import experiments_cli
+import experimaestro.launcherfinder.registry as launcher_registry
+from experimaestro.settings import find_workspace
+
+# --- Command line main options
+logging.basicConfig(level=logging.INFO)
+
+
+def pass_cfg(f):
+    """Pass configuration information"""
+
+    @click.pass_context
+    def new_func(ctx, *args, **kwargs):
+        return ctx.invoke(f, ctx.obj, *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+def check_xp_path(ctx, self, path: Path):
+    if not (path / ".__experimaestro__").is_file():
+        cprint(f"{path} is not an experimaestro working directory", "red")
+        for path in path.parents:
+            if (path / ".__experimaestro__").is_file():
+                cprint(f"{path} could be the folder you want", "green")
+                if click.confirm("Do you want to use this folder?"):
+                    return path
+        sys.exit(1)
+
+    return path
+
+
+class RunConfig:
+    def __init__(self):
+        self.traceback = False
+
+
+def pass_cfg(f):
+    """Pass configuration information"""
+
+    @click.pass_context
+    def new_func(ctx, *args, **kwargs):
+        return ctx.invoke(f, ctx.obj, *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+@click.group()
+@click.option("--quiet", is_flag=True, help="Be quiet")
+@click.option("--debug", is_flag=True, help="Be even more verbose (implies traceback)")
+@click.option(
+    "--traceback", is_flag=True, help="Display traceback if an exception occurs"
+)
+@click.pass_context
+def cli(ctx, quiet, debug, traceback):
+    if quiet:
+        logging.getLogger().setLevel(logging.WARN)
+    elif debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    ctx.obj = RunConfig()
+    ctx.obj.traceback = traceback
+
+
+# Adds the run-experiment command
+cli.add_command(experiments_cli, "run-experiment")
+
+
+@cli.command(help="Get version")
+def version():
+    print(experimaestro.__version__)
+
+
+@click.argument("parameters", type=Path)
+@cli.command(context_settings={"allow_extra_args": True})
+def run(parameters):
+    """Run a task"""
+
+    from experimaestro.run import run as do_run
+
+    do_run(parameters)
+
+
+@click.argument("path2", type=Path)
+@click.argument("path1", type=Path)
+@cli.command(context_settings={"allow_extra_args": True})
+def parameters_difference(path1, path2):
+    """Compute the difference between two configurations"""
+
+    from experimaestro.tools.diff import diff
+
+    diff(path1, path2)
+
+
+@click.option(
+    "--clean", is_flag=True, help="Remove the socket file and its enclosing directory"
+)
+@click.argument("unix-path", type=Path)
+@cli.command()
+def rpyc_server(unix_path, clean):
+    """Start an rPyC server"""
+    from experimaestro.rpyc import start_server
+
+    start_server(unix_path, clean=clean)
+
+
+@cli.group()
+def deprecated():
+    """Manage identifier changes"""
+    pass
+
+
+@click.option("--fix", is_flag=True, help="Generate links to new IDs")
+@click.option("--cleanup", is_flag=True, help="Remove symbolic links and move folders")
+@click.argument("path", type=Path, callback=check_xp_path)
+@deprecated.command(name="list")
+def deprecated_list(path: Path, fix: bool, cleanup: bool):
+    """List deprecated jobs"""
+    from experimaestro.tools.jobs import fix_deprecated
+
+    if cleanup and not fix:
+        logging.warning("Ignoring --cleanup since we are not fixing old IDs")
+    fix_deprecated(path, fix, cleanup)
+
+
+@click.argument("path", type=Path, callback=check_xp_path)
+@deprecated.command()
+def diff(path: Path):
+    """Show the reason of the identifier change for a job"""
+    from experimaestro.tools.jobs import load_job
+    from experimaestro import Config
+    from experimaestro.core.objects import ConfigWalkContext
+
+    _, job = load_job(path / "params.json", discard_id=False)
+    _, new_job = load_job(path / "params.json")
+
+    def check(path: str, value, new_value, done: Set[int]):
+        if isinstance(value, Config):
+            if id(value) in done:
+                return
+            done.add(id(value))
+
+            old_id = value.__xpm__.identifier.all.hex()
+            new_id = new_value.__xpm__.identifier.all.hex()
+
+            if new_id != old_id:
+                print(f"{path} differ: {new_id} vs {old_id}")
+
+                for arg in value.__xpmtype__.arguments.values():
+                    arg_value = getattr(value, arg.name)
+                    arg_newvalue = getattr(new_value, arg.name)
+                    check(f"{path}/{arg.name}", arg_value, arg_newvalue, done)
+
+        elif isinstance(value, list):
+            for ix, (array_value, array_newvalue) in enumerate(zip(value, new_value)):
+                check(f"{path}.{ix}", array_value, array_newvalue, done)
+
+        elif isinstance(value, dict):
+            for key, dict_value in value.items():
+                check(f"{path}.{key}", dict_value, new_value[key], done)
+
+    check(".", job, new_job, set())
+
+
+@click.option("--show-all", is_flag=True, help="Show even not orphans")
+@click.option(
+    "--ignore-old", is_flag=True, help="Ignore old jobs for unfinished experiments"
+)
+@click.option("--clean", is_flag=True, help="Prune the orphan folders")
+@click.option("--size", is_flag=True, help="Show size of each folder")
+@click.argument("path", type=Path, callback=check_xp_path)
+@cli.command()
+def orphans(path: Path, clean: bool, size: bool, show_all: bool, ignore_old: bool):
+    """Check for tasks that are not part of an experimental plan"""
+
+    jobspath = path / "jobs"
+
+    def getjobs(path: Path):
+        return ((str(p.relative_to(path)), p) for p in path.glob("*/*") if p.is_dir())
+
+    def show(key: str, prefix=""):
+        if size:
+            print(
+                prefix,
+                subprocess.check_output(["du", "-hs", key], cwd=jobspath)
+                .decode("utf-8")
+                .strip(),
+                sep=None,
+            )
+        else:
+            print(prefix, key, sep=None)
+
+    for p in (path / "xp").glob("*/jobs.bak"):
+        logging.warning("Experiment %s has not completed successfully", p.parent.name)
+
+    # Retrieve the jobs within expedriments (jobs and jobs.bak folder within experiments)
+    xpjobs = set()
+    if ignore_old:
+        paths = (path / "xp").glob("*/jobs")
+    else:
+        paths = chain((path / "xp").glob("*/jobs"), (path / "xp").glob("*/jobs.bak"))
+
+    for p in paths:
+        if p.is_dir():
+            for relpath, path in getjobs(p):
+                xpjobs.add(relpath)
+
+    # Now, look at stored jobs
+    found = 0
+    for key, jobpath in getjobs(jobspath):
+        if key not in xpjobs:
+            show(key)
+            if clean:
+                logging.info("Removing data in %s", jobpath)
+                rmtree(jobpath)
+        else:
+            if show_all:
+                show(key, prefix="[not orphan] ")
+            found += 1
+
+    print(f"{found} jobs are not orphans")
+
+
+def arg_split(ctx, param, value):
+    # split columns by ',' and remove whitespace
+    return set(c.strip() for c in value.split(","))
+
+
+@click.option("--skip", default=set(), callback=arg_split)
+@click.argument("package", type=str)
+@click.argument("objects", type=Path)
+@cli.command()
+def check_documentation(objects, package, skip):
+    """Check that all the configuration and tasks are documented within a
+    package, relying on the sphinx objects.inv file"""
+    from experimaestro.tools.documentation import documented_from_objects, undocumented
+
+    documented = documented_from_objects(objects)
+    errors, configs = undocumented([package], documented, skip)
+    for config in configs:
+        cprint(f"{config.__module__}.{config.__qualname__}", "red")
+
+    if errors > 0 or configs:
+        sys.exit(1)
+
+
+@click.option("--config", type=Path, help="Show size of each folder")
+@click.argument("spec", type=str)
+@cli.command()
+def find_launchers(config: Optional[Path], spec: str):
+    """Find launchers matching a specification"""
+    if config is not None:
+        launcher_registry.LauncherRegistry.set_config_dir(config)
+
+    print(launcher_registry.find_launcher(spec))
+
+
+class Launchers(click.MultiCommand):
+    """Connectors commands"""
+
+    @cached_property
+    def commands(self):
+        map = {}
+        for ep in pkg_resources.iter_entry_points(f"experimaestro.{self.name}"):
+            if get_cli := getattr(ep.load(), "get_cli", None):
+                map[ep.name] = get_cli()
+        return map
+
+    def list_commands(self, ctx):
+        return self.commands.keys()
+
+    def get_command(self, ctx, name):
+        return self.commands[name]
+
+
+cli.add_command(Launchers("launchers", help="Launcher specific commands"))
+cli.add_command(Launchers("connectors", help="Connector specific commands"))
+cli.add_command(Launchers("tokens", help="Token specific commands"))
+
+
+@cli.group()
+@click.option("--workdir", type=Path, default=None)
+@click.option("--workspace", type=str, default=None)
+@click.pass_context
+def experiments(ctx, workdir, workspace):
+    """Manage experiments"""
+    ws = find_workspace(workdir=workdir, workspace=workspace)
+    path = check_xp_path(None, None, ws.path)
+    ctx.obj = path
+
+
+@experiments.command()
+@pass_cfg
+def list(workdir: Path):
+    for p in (workdir / "xp").iterdir():
+        if (p / "jobs.bak").exists():
+            cprint(f"[unfinished] {p.name}", "yellow")
+        else:
+            cprint(p.name, "cyan")

experimaestro/{filter.py → cli/filter.py}

@@ -22,12 +22,12 @@ class JobInformation:
 
     @cached_property
     def state(self) -> Optional[JobState]:
-        if (self.path / f"{self.scriptname}.pid").is_file():
-            return JobState.RUNNING
-        elif (self.path / f"{self.scriptname}.done").is_file():
+        if (self.path / f"{self.scriptname}.done").is_file():
             return JobState.DONE
-        elif (self.path / f"{self.scriptname}.failed").is_file():
+        if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
+        if (self.path / f"{self.scriptname}.pid").is_file():
+            return JobState.RUNNING
         else:
             return None
 

experimaestro/cli/jobs.py

@@ -0,0 +1,245 @@
+# flake8: noqa: T201
+import subprocess
+from typing import Optional
+from shutil import rmtree
+import click
+from pathlib import Path
+from termcolor import colored, cprint
+
+from experimaestro.settings import find_workspace
+from . import check_xp_path, cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def jobs(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Job control: list, kill and clean
+
+    The job filter is a boolean expression where tags (alphanumeric)
+    and special job information (@state for job state, @name for job full
+    name) can be compared to a given value (using '~' for regex matching,
+    '=', 'not in', or 'in')
+
+    For instance,
+
+    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
+
+    selects jobs where the tag model is "bm25", the tag mode is either
+    "a" or "b", and the state is running.
+
+    """
+    ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+    check_xp_path(ctx, None, ws.path)
+
+
+def process(
+    workspace,
+    *,
+    experiment="",
+    tags="",
+    ready=False,
+    clean=False,
+    kill=False,
+    filter="",
+    perform=False,
+    fullpath=False,
+):
+    path = workspace.path
+    for p in (path / "xp").glob("*"):
+        if experiment and p.name != experiment:
+            continue
+
+        from .filter import createFilter, JobInformation
+        from experimaestro.scheduler import JobState
+
+        _filter = createFilter(filter) if filter else lambda x: True
+
+        print(f"* Experiment {p.name}")
+        if (p / "jobs.bak").is_dir():
+            cprint("  Experiment has not finished yet", "red")
+            if not perform and (kill or clean):
+                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                kill = False
+                clean = False
+        print()
+
+        for job in p.glob("jobs/*/*"):
+            info = None
+            p = job.resolve()
+            if p.is_dir():
+                *_, scriptname = p.parent.name.rsplit(".", 1)
+
+                info = JobInformation(p, scriptname)
+                job_path = (
+                    str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}"
+                )
+
+                if filter:
+                    if not _filter(info):
+                        continue
+
+                if info.state is None:
+                    print(colored(f"NODIR   {job_path}", "red"), end="")
+                elif info.state.running():
+                    if kill:
+                        if perform:
+                            process = info.getprocess()
+                            print("KILLING", process)
+                            process.kill()
+                        else:
+                            print("KILLING (not performing)", process)
+                    print(
+                        colored(f"{info.state.name:8}{job_path}", "yellow"),
+                        end="",
+                    )
+                elif info.state == JobState.DONE:
+                    print(
+                        colored(f"DONE    {job_path}", "green"),
+                        end="",
+                    )
+                elif info.state == JobState.ERROR:
+                    print(colored(f"FAIL    {job_path}", "red"), end="")
+                else:
+                    print(
+                        colored(f"{info.state.name:8}{job_path}", "red"),
+                        end="",
+                    )
+
+            else:
+                if not ready:
+                    continue
+                print(colored(f"READY {job_path}", "yellow"), end="")
+
+            if tags:
+                print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
+            else:
+                print()
+
+            if clean and info.state and info.state.finished():
+                if perform:
+                    cprint("Cleaning...", "red")
+                    rmtree(p)
+                else:
+                    cprint("Cleaning... (not performed)", "red")
+        print()
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def list(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the killing")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def kill(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        kill=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the cleaning")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def clean(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        clean=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.argument("jobid", type=str)
+@click.option(
+    "--follow", "-f", help="Use tail instead of less to follow changes", is_flag=True
+)
+@click.option("--std", help="Follow stdout instead of stderr", is_flag=True)
+@jobs.command()
+@click.pass_context
+def log(ctx, jobid: str, follow: bool, std: bool):
+    task_name, task_hash = jobid.split("/")
+    _, name = task_name.rsplit(".", 1)
+    path = (
+        ctx.obj.workspace.path
+        / "jobs"
+        / task_name
+        / task_hash
+        / f"""{name}.{'out' if std else 'err'}"""
+    )
+    if follow:
+        subprocess.run(["tail", "-f", path])
+    else:
+        subprocess.run(["less", "-r", path])
+
+
+@click.argument("jobid", type=str)
+@jobs.command()
+@click.pass_context
+def path(ctx, jobid: str):
+    task_name, task_hash = jobid.split("/")
+    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    print(path)

experimaestro/core/objects.py

@@ -916,14 +916,14 @@ class ConfigInformation:
 
         # --- Submit the job
 
+        # Sets the init tasks
+        self.init_tasks = init_tasks
+
         # Creates a new job
         self.job = self.xpmtype.task(
             self.pyobject, launcher=launcher, workspace=workspace, run_mode=run_mode
         )
 
-        # Sets the init tasks
-        self.init_tasks = init_tasks
-
         # Validate the object
         job_context = JobContext(self.job)
         self.validate_and_seal(job_context)
@@ -979,6 +979,9 @@ class ConfigInformation:
                 elif self.job.failedpath.is_file():
                     color = "light_red"
                     cprint(f"[failed] {s}", color, file=sys.stderr)
+                elif self.job.pidpath.is_file():
+                    color = "blue"
+                    cprint(f"[running] {s}", color, file=sys.stderr)
                 else:
                     color = "light_blue"
                     cprint(f"[not run] {s}", color, file=sys.stderr)

experimaestro/experiments/cli.py

@@ -11,7 +11,7 @@ import yaml
 from experimaestro import LauncherRegistry, RunMode, experiment
 from experimaestro.experiments.configuration import ConfigurationBase
 from experimaestro.exceptions import HandledException
-from experimaestro.settings import get_workspace
+from experimaestro.settings import find_workspace
 from omegaconf import OmegaConf, SCMode
 from termcolor import cprint
 
@@ -231,27 +231,8 @@ def experiments_cli(  # noqa: C901
     )
 
     # Define the workspace
-    workdir = Path(workdir) if workdir else None
-
-    if workspace:
-        ws_env = get_workspace(workspace)
-        if ws_env is None:
-            raise RuntimeError("No workspace named %s", workspace)
-
-        logging.info("Using workspace %s", ws_env.id)
-        if workdir:
-            # Overrides working directory
-            logging.info(" override working directory: %s", workdir)
-            ws_env.path = workdir
-        else:
-            workdir = ws_env.path
-    elif workdir:
-        logging.info("Using workdir %s", workdir)
-        ws_env = workdir
-    else:
-        ws_env = get_workspace()
-        assert ws_env is not None, "No workdir or workspace defined, and no default"
-        logging.info("Using default workspace %s", ws_env.id)
+    ws_env = find_workspace(workdir=workdir, workspace=workspace)
+    workdir = ws_env.path
 
     logging.info("Using working directory %s", str(workdir.resolve()))
 

experimaestro/launchers/slurm/base.py

@@ -262,7 +262,9 @@ class SlurmProcessBuilder(ProcessBuilder):
         addstream(builder.command, "-i", self.stdin)
 
         builder.command.extend(self.command)
-        logger.info("slurm sbatch command: %s", builder.command)
+        logger.info(
+            "slurm sbatch command: %s", " ".join(f'"{s}"' for s in builder.command)
+        )
         handler = OutputCaptureHandler()
         builder.stdout = Redirect.pipe(handler)
         builder.stderr = Redirect.inherit()