experimaestro 1.5.6__tar.gz → 1.5.8__tar.gz

{experimaestro-1.5.6 → experimaestro-1.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: experimaestro
-Version: 1.5.6
+Version: 1.5.8
 Summary: "Experimaestro is a computer science experiment manager"
 Home-page: https://github.com/experimaestro/experimaestro-python
 License: GPL-3

{experimaestro-1.5.6 → experimaestro-1.5.8}/pyproject.toml

@@ -19,7 +19,7 @@ include = [
     "src/experimaestro/sphinx/static/experimaestro.css",
     "src/experimaestro/mkdocs/style.css"
 ]
-version = "1.5.6"
+version = "1.5.8"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
 
@@ -90,10 +90,6 @@ slurm = "experimaestro.launchers.slurm:BatchSlurmProcess"
 local = "experimaestro.connectors.local:LocalConnector"
 ssh = "experimaestro.connectors.ssh:SshConnector"
 
-[tool.poetry.plugins."experimaestro.launchers"]
-unix = "experimaestro.launchers.direct:DirectLauncher"
-slurm = "experimaestro.launchers.slurm:SlurmLauncher"
-
 [tool.poetry.plugins."experimaestro.tokens"]
 unix = "experimaestro.tokens:CounterToken"
 
@@ -123,7 +119,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.5.6"
+version = "1.5.8"
 changelog_start_rev = "0.15.0"
 tag_format = "v$version"
 update_changelog_on_bump = true

experimaestro-1.5.8/src/experimaestro/__main__.py

@@ -0,0 +1,12 @@
+from experimaestro.cli import cli
+
+# flake8: noqa: F401
+import experimaestro.cli.jobs
+
+
+def main():
+    cli(obj=None)
+
+
+if __name__ == "__main__":
+    main()

experimaestro-1.5.6/src/experimaestro/__main__.py → experimaestro-1.5.8/src/experimaestro/cli/__init__.py

@@ -1,6 +1,5 @@
 # flake8: noqa: T201
 import sys
-from types import ModuleType
 from typing import Set, Optional
 import pkg_resources
 from itertools import chain
@@ -10,11 +9,12 @@ import logging
 from functools import cached_property, update_wrapper
 from pathlib import Path
 import subprocess
-from termcolor import colored, cprint
+from termcolor import cprint
 
 import experimaestro
 from experimaestro.experiments.cli import experiments_cli
 import experimaestro.launcherfinder.registry as launcher_registry
+from experimaestro.settings import find_workspace
 
 # --- Command line main options
 logging.basicConfig(level=logging.INFO)
@@ -175,121 +175,6 @@ def diff(path: Path):
     check(".", job, new_job, set())
 
 
-@click.argument("path", type=Path, callback=check_xp_path)
-@click.option("--experiment", default=None, help="Restrict to this experiment")
-@click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
-@click.option("--filter", default="", help="Filter expression")
-@click.option(
-    "--force",
-    is_flag=True,
-    help="Force operation even if experiment has not been completed yet",
-)
-@click.option("--kill", is_flag=True, help="Kill filtered tasks (when/before running)")
-@click.option("--clean", is_flag=True, help="Remove finished tasks directories")
-@cli.command()
-def jobs(
-    path: Path,
-    experiment: str,
-    filter: str,
-    tags: bool,
-    ready: bool,
-    kill: bool,
-    clean: bool,
-    force: bool,
-):
-    """Job control: list, kill and clean
-
-    The job filter is a boolean expression where tags (alphanumeric)
-    and special job information (@state for job state, @name for job full
-    name) can be compared to a given value (using '~' for regex matching,
-    '=', 'not in', or 'in')
-
-    For instance,
-
-    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
-
-    selects jobs where the tag model is "bm25", the tag mode is either
-    "a" or "b", and the state is running.
-
-    """
-    for p in (path / "xp").glob("*"):
-        if experiment and p.name != experiment:
-            continue
-
-        from .filter import createFilter, JobInformation
-        from experimaestro.scheduler import JobState
-
-        _filter = createFilter(filter) if filter else lambda x: True
-
-        print(f"* Experiment {p.name}")
-        if (p / "jobs.bak").is_dir():
-            cprint("  Experiment has not finished yet", "red")
-            if not force and (kill or clean):
-                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
-                kill = False
-                clean = False
-        print()
-
-        for job in p.glob("jobs/*/*"):
-            info = None
-            p = job.resolve()
-            if p.is_dir():
-                *_, scriptname = p.parent.name.rsplit(".", 1)
-
-                info = JobInformation(p, scriptname)
-                if filter:
-                    if not _filter(info):
-                        continue
-
-                if info.state is None:
-                    print(
-                        colored(f"NODIR   {job.parent.name}/{job.name}", "red"), end=""
-                    )
-                elif info.state.running():
-                    if kill:
-                        process = info.getprocess()
-                        print("KILLING", process)
-                        process.kill()
-                    print(
-                        colored(
-                            f"{info.state.name:8}{job.parent.name}/{job.name}", "yellow"
-                        ),
-                        end="",
-                    )
-                elif info.state == JobState.DONE:
-                    print(
-                        colored(f"DONE    {job.parent.name}/{job.name}", "green"),
-                        end="",
-                    )
-                elif info.state == JobState.ERROR:
-                    print(
-                        colored(f"FAIL    {job.parent.name}/{job.name}", "red"), end=""
-                    )
-                else:
-                    print(
-                        colored(
-                            f"{info.state.name:8}{job.parent.name}/{job.name}", "red"
-                        ),
-                        end="",
-                    )
-
-            else:
-                if not ready:
-                    continue
-                print(colored(f"READY {job.parent.name}/{job.name}", "yellow"), end="")
-
-            if tags:
-                print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
-            else:
-                print()
-
-            if clean and info.state and info.state.finished():
-                cprint("Cleaning...", "red")
-                rmtree(p)
-        print()
-
-
 @click.option("--show-all", is_flag=True, help="Show even not orphans")
 @click.option(
     "--ignore-old", is_flag=True, help="Ignore old jobs for unfinished experiments"
@@ -407,11 +292,14 @@ cli.add_command(Launchers("tokens", help="Token specific commands"))
 
 
 @cli.group()
-@click.argument("workdir", type=Path, callback=check_xp_path)
+@click.option("--workdir", type=Path, default=None)
+@click.option("--workspace", type=str, default=None)
 @click.pass_context
-def experiments(ctx, workdir):
+def experiments(ctx, workdir, workspace):
     """Manage experiments"""
-    ctx.obj = workdir
+    ws = find_workspace(workdir=workdir, workspace=workspace)
+    path = check_xp_path(None, None, ws.path)
+    ctx.obj = path
 
 
 @experiments.command()
@@ -422,11 +310,3 @@ def list(workdir: Path):
             cprint(f"[unfinished] {p.name}", "yellow")
         else:
             cprint(p.name, "cyan")
-
-
-def main():
-    cli(obj=None)
-
-
-if __name__ == "__main__":
-    main()

{experimaestro-1.5.6/src/experimaestro → experimaestro-1.5.8/src/experimaestro/cli}/filter.py

@@ -22,12 +22,12 @@ class JobInformation:
 
     @cached_property
     def state(self) -> Optional[JobState]:
-        if (self.path / f"{self.scriptname}.pid").is_file():
-            return JobState.RUNNING
-        elif (self.path / f"{self.scriptname}.done").is_file():
+        if (self.path / f"{self.scriptname}.done").is_file():
             return JobState.DONE
-        elif (self.path / f"{self.scriptname}.failed").is_file():
+        if (self.path / f"{self.scriptname}.failed").is_file():
             return JobState.ERROR
+        if (self.path / f"{self.scriptname}.pid").is_file():
+            return JobState.RUNNING
         else:
             return None
 

experimaestro-1.5.8/src/experimaestro/cli/jobs.py

@@ -0,0 +1,251 @@
+# flake8: noqa: T201
+import subprocess
+from typing import Optional
+from shutil import rmtree
+import click
+from pathlib import Path
+from termcolor import colored, cprint
+
+from experimaestro.settings import find_workspace
+from . import check_xp_path, cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def jobs(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Job control: list, kill and clean
+
+    The job filter is a boolean expression where tags (alphanumeric)
+    and special job information (@state for job state, @name for job full
+    name) can be compared to a given value (using '~' for regex matching,
+    '=', 'not in', or 'in')
+
+    For instance,
+
+    model = "bm25" and mode in ["a", b"] and @state = "RUNNING"
+
+    selects jobs where the tag model is "bm25", the tag mode is either
+    "a" or "b", and the state is running.
+
+    """
+    ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+    check_xp_path(ctx, None, ws.path)
+
+
+def process(
+    workspace,
+    *,
+    experiment="",
+    tags="",
+    ready=False,
+    clean=False,
+    kill=False,
+    filter="",
+    perform=False,
+    fullpath=False,
+):
+    path = workspace.path
+    for p in (path / "xp").glob("*"):
+        if experiment and p.name != experiment:
+            continue
+
+        from .filter import createFilter, JobInformation
+        from experimaestro.scheduler import JobState
+
+        _filter = createFilter(filter) if filter else lambda x: True
+
+        print(f"* Experiment {p.name}")
+        if (p / "jobs.bak").is_dir():
+            cprint("  Experiment has not finished yet", "red")
+            if not perform and (kill or clean):
+                cprint("  Preventing kill/clean (use --force if you want to)", "yellow")
+                kill = False
+                clean = False
+        print()
+
+        for job in p.glob("jobs/*/*"):
+            info = None
+            p = job.resolve()
+            if p.is_dir():
+                *_, scriptname = p.parent.name.rsplit(".", 1)
+
+                info = JobInformation(p, scriptname)
+                job_path = (
+                    str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}"
+                )
+
+                if filter:
+                    if not _filter(info):
+                        continue
+
+                if info.state is None:
+                    print(colored(f"NODIR   {job_path}", "red"), end="")
+                elif info.state.running():
+                    if kill:
+                        if perform:
+                            process = info.getprocess()
+                            if process is None:
+                                cprint(
+                                    "internal error – no process could be retrieved",
+                                    "red",
+                                )
+                            else:
+                                cprint(f"KILLING {process}", "light_red")
+                                process.kill()
+                        else:
+                            print("KILLING (not performing)", process)
+                    print(
+                        colored(f"{info.state.name:8}{job_path}", "yellow"),
+                        end="",
+                    )
+                elif info.state == JobState.DONE:
+                    print(
+                        colored(f"DONE    {job_path}", "green"),
+                        end="",
+                    )
+                elif info.state == JobState.ERROR:
+                    print(colored(f"FAIL    {job_path}", "red"), end="")
+                else:
+                    print(
+                        colored(f"{info.state.name:8}{job_path}", "red"),
+                        end="",
+                    )
+
+            else:
+                if not ready:
+                    continue
+                print(colored(f"READY {job_path}", "yellow"), end="")
+
+            if tags:
+                print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
+            else:
+                print()
+
+            if clean and info.state and info.state.finished():
+                if perform:
+                    cprint("Cleaning...", "red")
+                    rmtree(p)
+                else:
+                    cprint("Cleaning... (not performed)", "red")
+        print()
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def list(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the killing")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def kill(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        kill=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.option("--experiment", default=None, help="Restrict to this experiment")
+@click.option("--tags", is_flag=True, help="Show tags")
+@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
+@click.option("--filter", default="", help="Filter expression")
+@click.option("--perform", is_flag=True, help="Really perform the cleaning")
+@click.option("--fullpath", is_flag=True, help="Prints full paths")
+@jobs.command()
+@click.pass_context
+def clean(
+    ctx,
+    experiment: str,
+    filter: str,
+    tags: bool,
+    ready: bool,
+    fullpath: bool,
+    perform: bool,
+):
+    process(
+        ctx.obj.workspace,
+        experiment=experiment,
+        filter=filter,
+        tags=tags,
+        ready=ready,
+        clean=True,
+        perform=perform,
+        fullpath=fullpath,
+    )
+
+
+@click.argument("jobid", type=str)
+@click.option(
+    "--follow", "-f", help="Use tail instead of less to follow changes", is_flag=True
+)
+@click.option("--std", help="Follow stdout instead of stderr", is_flag=True)
+@jobs.command()
+@click.pass_context
+def log(ctx, jobid: str, follow: bool, std: bool):
+    task_name, task_hash = jobid.split("/")
+    _, name = task_name.rsplit(".", 1)
+    path = (
+        ctx.obj.workspace.path
+        / "jobs"
+        / task_name
+        / task_hash
+        / f"""{name}.{'out' if std else 'err'}"""
+    )
+    if follow:
+        subprocess.run(["tail", "-f", path])
+    else:
+        subprocess.run(["less", "-r", path])
+
+
+@click.argument("jobid", type=str)
+@jobs.command()
+@click.pass_context
+def path(ctx, jobid: str):
+    task_name, task_hash = jobid.split("/")
+    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    print(path)

{experimaestro-1.5.6 → experimaestro-1.5.8}/src/experimaestro/connectors/ssh.py

@@ -3,7 +3,7 @@ from pathlib import Path, _posix_flavour
 import io
 import os
 import re
-from experimaestro.launcherfinder import LauncherRegistry, YAMLDataClass
+from experimaestro.launcherfinder import LauncherRegistry
 from fabric import Connection
 from invoke import Promise
 import invoke.exceptions
@@ -132,7 +132,7 @@ class SshPath(Path):
 
 
 @dataclass
-class SshConfiguration(YAMLDataClass):
+class SshConfiguration:
     hostname: str
 
     def create(self, registry: LauncherRegistry):

{experimaestro-1.5.6 → experimaestro-1.5.8}/src/experimaestro/core/objects.py

@@ -916,14 +916,14 @@ class ConfigInformation:
 
         # --- Submit the job
 
+        # Sets the init tasks
+        self.init_tasks = init_tasks
+
         # Creates a new job
         self.job = self.xpmtype.task(
             self.pyobject, launcher=launcher, workspace=workspace, run_mode=run_mode
         )
 
-        # Sets the init tasks
-        self.init_tasks = init_tasks
-
         # Validate the object
         job_context = JobContext(self.job)
         self.validate_and_seal(job_context)
@@ -979,6 +979,9 @@ class ConfigInformation:
                 elif self.job.failedpath.is_file():
                     color = "light_red"
                     cprint(f"[failed] {s}", color, file=sys.stderr)
+                elif self.job.pidpath.is_file():
+                    color = "blue"
+                    cprint(f"[running] {s}", color, file=sys.stderr)
                 else:
                     color = "light_blue"
                     cprint(f"[not run] {s}", color, file=sys.stderr)

{experimaestro-1.5.6 → experimaestro-1.5.8}/src/experimaestro/core/types.py

@@ -325,12 +325,17 @@ class ObjectType(Type):
 
         # Get the module
         module = inspect.getmodule(self.originaltype)
-        if getattr(module, "__file__", None) is None:
+        self._module = module.__name__
+        self._package = module.__package__
+
+        if self._module and self._package:
             self._file = None
         else:
             self._file = Path(inspect.getfile(self.originaltype)).absolute()
-        self._module = module.__name__
-        self._package = module.__package__
+
+        assert (
+            self._module and self._package
+        ) or self._file, f"Could not detect module/file for {self.originaltype}"
 
         # The class of the object
 

{experimaestro-1.5.6 → experimaestro-1.5.8}/src/experimaestro/experiments/cli.py

@@ -1,26 +1,28 @@
+import imp
 import inspect
 import json
 import logging
 import sys
 from pathlib import Path
-from typing import Any, List, Optional, Protocol, Tuple, Dict
+from typing import Any, List, Optional, Protocol, Tuple
 
 import click
 import omegaconf
 import yaml
-from experimaestro import LauncherRegistry, RunMode, experiment
-from experimaestro.experiments.configuration import ConfigurationBase
-from experimaestro.exceptions import HandledException
-from experimaestro.settings import get_workspace
 from omegaconf import OmegaConf, SCMode
 from termcolor import cprint
 
+from experimaestro import LauncherRegistry, RunMode, experiment
+from experimaestro.exceptions import HandledException
+from experimaestro.experiments.configuration import ConfigurationBase
+from experimaestro.settings import find_workspace
+
 
 class ExperimentHelper:
     """Helper for experiments"""
 
-    # The experiment
     xp: experiment
+    """The experiment object"""
 
     #: Run function
     callable: "ExperimentCallable"
@@ -175,28 +177,23 @@ def experiments_cli(  # noqa: C901
     xp_file = Path(xp_file)
     if not xp_file.exists() and xp_file.suffix != ".py":
         xp_file = xp_file.with_suffix(".py")
-    xp_file = Path(yaml_file).parent / xp_file
-
-    with open(xp_file, "r") as f:
-        source = f.read()
-    if sys.version_info < (3, 9):
-        the__file__ = str(xp_file)
-    else:
-        the__file__ = str(xp_file.absolute())
-
-    code = compile(source, filename=the__file__, mode="exec")
-    _locals: Dict[str, Any] = {}
+    xp_file: Path = Path(yaml_file).parent / xp_file
 
-    sys.path.append(str(xp_file.parent.absolute()))
+    # --- Finds the "run" function
     try:
-        exec(code, _locals, _locals)
+        sys.path.append(str(xp_file.parent.absolute()))
+        with open(xp_file) as src:
+            module_name = xp_file.with_suffix("").name
+            mod = imp.load_module(
+                module_name, src, str(xp_file.absolute()), (".py", "r", imp.PY_SOURCE)
+            )
+            helper = getattr(mod, "run", None)
     finally:
         sys.path.pop()
 
     # --- ... and runs it
-    helper = _locals.get("run", None)
     if helper is None:
-        raise ValueError(f"Could not find run function in {the__file__}")
+        raise ValueError(f"Could not find run function in {xp_file}")
 
     if not isinstance(helper, ExperimentHelper):
         helper = ExperimentHelper(helper)
@@ -231,27 +228,8 @@ def experiments_cli(  # noqa: C901
     )
 
     # Define the workspace
-    workdir = Path(workdir) if workdir else None
-
-    if workspace:
-        ws_env = get_workspace(workspace)
-        if ws_env is None:
-            raise RuntimeError("No workspace named %s", workspace)
-
-        logging.info("Using workspace %s", ws_env.id)
-        if workdir:
-            # Overrides working directory
-            logging.info(" override working directory: %s", workdir)
-            ws_env.path = workdir
-        else:
-            workdir = ws_env.path
-    elif workdir:
-        logging.info("Using workdir %s", workdir)
-        ws_env = workdir
-    else:
-        ws_env = get_workspace()
-        assert ws_env is not None, "No workdir or workspace defined, and no default"
-        logging.info("Using default workspace %s", ws_env.id)
+    ws_env = find_workspace(workdir=workdir, workspace=workspace)
+    workdir = ws_env.path
 
     logging.info("Using working directory %s", str(workdir.resolve()))
 

{experimaestro-1.5.6 → experimaestro-1.5.8}/src/experimaestro/launcherfinder/__init__.py

@@ -10,5 +10,5 @@ from .specs import (
     HostRequirement,
     MatchRequirement,
 )
-from .registry import find_launcher, LauncherRegistry, YAMLDataClass
+from .registry import find_launcher, LauncherRegistry
 from .parser import parse

experimaestro-1.5.8/src/experimaestro/launcherfinder/base.py

@@ -0,0 +1,17 @@
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from experimaestro.connectors import Connector
+    from experimaestro.tokens import Token
+    from .registry import LauncherRegistry
+
+
+class ConnectorConfiguration:
+    def create(self, registry: "LauncherRegistry") -> "Connector":
+        raise NotImplementedError(f"For {self.__class__}")
+
+
+class TokenConfiguration:
+    def create(self, registry: "LauncherRegistry", identifier: str) -> "Token":
+        raise NotImplementedError(f"For {self.__class__}")