experimaestro 1.5.1__py3-none-any.whl → 2.0.0a8__py3-none-any.whl

experimaestro/cli/progress.py

@@ -0,0 +1,269 @@
+"""Simplified CLI commands for managing and viewing progress files"""
+
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Dict
+
+import click
+from termcolor import colored
+
+try:
+    from tqdm import tqdm
+
+    TQDM_AVAILABLE = True
+except ImportError:
+    TQDM_AVAILABLE = False
+
+from experimaestro.progress import ProgressEntry, ProgressFileReader
+from experimaestro.settings import find_workspace
+from . import cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def progress(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Progress tracking commands"""
+    ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+
+
+def format_timestamp(timestamp: float) -> str:
+    """Format timestamp for display"""
+    dt = datetime.fromtimestamp(timestamp)
+    return dt.strftime("%Y-%m-%d %H:%M:%S")
+
+
+@click.argument("jobid", type=str)
+@progress.command()
+@click.pass_context
+def show(ctx, jobid: str):
+    """Show current progress state (default command)
+
+    JOBID format: task_name/task_hash
+    """
+    try:
+        task_name, task_hash = jobid.split("/")
+    except ValueError:
+        raise click.ClickException("JOBID must be in format task_name/task_hash")
+
+    workspace = ctx.obj.workspace
+    task_path = workspace.path / "jobs" / task_name / task_hash
+
+    if not task_path.exists():
+        raise click.ClickException(f"Job directory not found: {task_path}")
+
+    reader = ProgressFileReader(task_path)
+    current_progress = reader.get_current_progress()
+
+    if not current_progress:
+        click.echo("No progress information available")
+        return
+
+    # Filter out EOJ markers
+    current_progress = {k: v for k, v in current_progress.items() if k != -1}
+
+    if not current_progress:
+        click.echo("No progress information available")
+        return
+
+    click.echo(f"Progress for job {jobid}")
+    click.echo("=" * 80)
+
+    # Show simple text-based progress for each level
+    for level in sorted(current_progress.keys()):
+        entry = current_progress[level]
+        indent = "  " * level
+        progress_pct = f"{entry.progress * 100:5.1f}%"
+        desc = entry.desc or f"Level {level}"
+        timestamp = format_timestamp(entry.timestamp)
+
+        color = "green" if entry.progress >= 1.0 else "yellow"
+        click.echo(colored(f"{indent}L{level}: {progress_pct} - {desc}", color))
+        click.echo(colored(f"{indent}     Last updated: {timestamp}", "cyan"))
+
+
+def create_progress_bar(
+    level: int,
+    desc: str,
+    progress: float = 0.0,
+) -> tqdm:
+    """Create a properly aligned progress bar like dashboard style"""
+    if level > 0:
+        indent = "   " * (level - 1) + "└─ "
+    else:
+        indent = ""
+    label = f"{indent}L{level}"
+
+    colors = ["blue", "yellow", "magenta", "cyan", "white"]
+    bar_color = colors[level % len(colors)]
+
+    unit = desc[:50] if desc else f"Level {level}"
+    ncols = 100
+    wbar = 50
+
+    return tqdm(
+        total=100,
+        desc=label,
+        position=level,
+        leave=True,
+        bar_format=f"{{desc}}: {{percentage:3.0f}}%|{{bar:{wbar - len(indent)}}}| {{unit}}",  # noqa: F541
+        ncols=ncols,  # Adjust width based on level
+        unit=unit,
+        colour=bar_color,
+        initial=progress * 100,
+    )
+
+
+def _update_progress_display(
+    reader: ProgressFileReader, progress_bars: Dict[int, tqdm]
+) -> bool:
+    """Update the tqdm progress bars in dashboard style"""
+    current_state: Dict[int, ProgressEntry] = {
+        k: v for k, v in reader.get_current_state().items() if k != -1
+    }
+
+    if not current_state:
+        click.echo("No progress information available yet...")
+        return False
+
+    # Update existing bars and create new ones
+    for _level, entry in current_state.items():
+        progress_val = entry.progress * 100
+        desc = entry.desc or f"Level {entry.level}"
+
+        if entry.level not in progress_bars:
+            progress_bars[entry.level] = create_progress_bar(
+                entry.level, desc, progress_val
+            )
+
+        bar = progress_bars[entry.level]
+        bar.unit = desc[:50]
+        bar.n = progress_val
+
+        bar.refresh()
+
+    # Remove bars for levels that no longer exist
+    levels_to_remove = set(progress_bars.keys()) - set(current_state.keys())
+    for level in levels_to_remove:
+        progress_bars[level].close()
+        del progress_bars[level]
+
+    return True
+
+
+@click.argument("jobid", type=str)
+@click.option("--refresh-rate", "-r", default=0.5, help="Refresh rate in seconds")
+@progress.command()
+@click.pass_context
+def live(ctx, jobid: str, refresh_rate: float):
+    """Show live progress with tqdm-style bars
+
+    JOBID format: task_name/task_hash
+    """
+    if not TQDM_AVAILABLE:
+        click.echo("tqdm is not available. Install with: pip install tqdm")
+        click.echo("Falling back to basic display...")
+        ctx.invoke(show, jobid=jobid)
+        return
+
+    try:
+        task_name, task_hash = jobid.split("/")
+    except ValueError:
+        raise click.ClickException("JOBID must be in format task_name/task_hash")
+
+    workspace = ctx.obj.workspace
+    task_path = workspace.path / "jobs" / task_name / task_hash
+
+    if not task_path.exists():
+        raise click.ClickException(f"Job directory not found: {task_path}")
+
+    reader = ProgressFileReader(task_path)
+    progress_bars: Dict[int, tqdm] = {}
+
+    def cleanup_bars():
+        """Clean up all progress bars"""
+        for bar in progress_bars.values():
+            bar.close()
+        progress_bars.clear()
+
+    click.echo(f"Live progress for job {jobid}")
+    click.echo("Press Ctrl+C to stop")
+    click.echo("=" * 80)
+
+    try:
+        if not _update_progress_display(reader, progress_bars):
+            click.echo("No progress information available yet...")
+
+        while True:
+            time.sleep(refresh_rate)
+
+            if not _update_progress_display(reader, progress_bars):
+                # Check if job is complete
+                if reader.is_done():
+                    click.echo("\nJob completed!")
+                    break
+
+            # Check if all progress bars are at 100%
+            if progress_bars and all(bar.n >= 100 for bar in progress_bars.values()):
+                cleanup_bars()
+                click.echo("\nAll progress completed!")
+                break
+
+    except KeyboardInterrupt:
+        click.echo("\nStopped monitoring progress")
+    finally:
+        cleanup_bars()
+
+
+@progress.command(name="list")
+@click.pass_context
+def list_jobs(ctx):
+    """List all jobs with progress information"""
+    ws = ctx.obj.workspace
+    jobs_path = ws.path / "jobs"
+
+    if not jobs_path.exists():
+        click.echo("No jobs directory found")
+        return
+
+    for task_dir in jobs_path.iterdir():
+        if not task_dir.is_dir():
+            continue
+
+        for job_dir in task_dir.iterdir():
+            if not job_dir.is_dir():
+                continue
+
+            progress_dir = job_dir / ".experimaestro"
+            if not progress_dir.exists():
+                continue
+
+            # Check if there are progress files
+            progress_files = list(progress_dir.glob("progress-*.jsonl"))
+            if not progress_files:
+                continue
+
+            job_id = f"{task_dir.name}/{job_dir.name}"
+            reader = ProgressFileReader(job_dir)
+            current_state = reader.get_current_state()
+
+            # if current_progress:
+            if current_state:
+                # Get overall progress (level 0)
+                level_0 = current_state.get(0)
+                if level_0:
+                    color = "green" if level_0.progress >= 1.0 else "yellow"
+                    desc = f"{level_0.desc}" if level_0.desc else ""
+                    progress_pct = f"{level_0.progress * 100:5.1f}%"
+                    click.echo(colored(f"{job_id:50} - {progress_pct} - {desc}", color))
+
+                else:
+                    click.echo(f"{job_id:50} No level 0 progress")
+            else:
+                click.echo(f"{job_id:50} No progress data")

experimaestro/click.py

@@ -1,6 +1,4 @@
 import click
-from experimaestro import Environment
-from experimaestro.run import parse_commandline
 
 """Defines the task command line argument prefix for experimaestro-handled command lines"""
 
@@ -45,36 +43,3 @@ class forwardoption(metaclass=forwardoptionMetaclass):
     def __getattr__(self, key):
         """Access to a class field"""
         return forwardoption([key])
-
-
-def environment(name: str):
-    def annotate(f):
-        def callback_env(ctx, name, value):
-            if value:
-                assert name not in ctx.params, "Environment has already been set"
-            else:
-                return ctx.params.get(name, None)
-            return Environment.get(value)
-
-        def callback(ctx, param, value):
-            if value:
-                if name not in ctx.params:
-                    ctx.params[name] = Environment()
-                ctx.params[name].workdir = value
-
-        f = click.option(
-            f"--{name}-workdir",
-            type=str,
-            callback=callback,
-            expose_value=False,
-            help="Experimaestro environment",
-        )(f)
-        f = click.option(
-            f"--{name}",
-            type=str,
-            callback=callback_env,
-            help="Experimaestro environment",
-        )(f)
-        return f
-
-    return annotate

experimaestro/commandline.py

@@ -276,12 +276,6 @@ class CommandLineJob(Job):
 
         scriptbuilder = self.launcher.scriptbuilder()
         self.path.mkdir(parents=True, exist_ok=True)
-        donepath = self.donepath
-
-        # Check again if done (now that we have locked)
-        if not overwrite and donepath.is_file():
-            logger.info("Job %s is already done", self)
-            return JobState.DONE
 
         # Now we can write the script
         scriptbuilder.lockfiles.append(self.lockpath)
@@ -293,15 +287,17 @@ class CommandLineJob(Job):
         if self._process:
             return self._process
 
+        # Prepare the files to be run
         scriptPath = self.prepare()
 
+        # OK, now starts the process
         logger.info("Starting job %s", self.jobpath)
         processbuilder = self.launcher.processbuilder()
         processbuilder.environ = self.environ
         processbuilder.command.append(self.launcher.connector.resolve(scriptPath))
         processbuilder.stderr = Redirect.file(self.stderr)
         processbuilder.stdout = Redirect.file(self.stdout)
-        self._process = processbuilder.start()
+        self._process = processbuilder.start(True)
 
         with self.pidpath.open("w") as fp:
             json.dump(self._process.tospec(), fp)

experimaestro/connectors/__init__.py

@@ -9,13 +9,14 @@ This module contains :
 """
 
 import enum
-from typing import Any, Dict, Mapping, Type, Union
+import logging
+from typing import Any, Dict, Mapping, Type, Union, Optional
 from pathlib import Path
 from experimaestro.utils import logger
 from experimaestro.locking import Lock
 from experimaestro.tokens import Token
 from experimaestro.utils.asyncio import asyncThreadcheck
-import pkg_resources
+from importlib.metadata import entry_points
 
 
 class RedirectType(enum.Enum):
@@ -86,12 +87,12 @@ class Process:
     @staticmethod
     def fromDefinition(connector: "Connector", definition: Dict[str, Any]) -> "Process":
         """Retrieves a process from a serialized definition"""
-        handler = Process.handler(definition["type"])
+        handler_type = definition["type"]
+        handler = Process.handler(handler_type)
+        assert handler is not None, f"No handler of type {handler_type}"
         try:
             return handler.fromspec(connector, definition)
         except Exception as e:
-            import logging
-
             logging.exception("Could not retrieve job from specification")
             raise e
 
@@ -100,8 +101,12 @@ class Process:
         """Get a handler"""
         if Process.HANDLERS is None:
             Process.HANDLERS = {}
-            for ep in pkg_resources.iter_entry_points(group="experimaestro.process"):
-                Process.HANDLERS[ep.name] = ep.load()
+            for ep in entry_points(group="experimaestro.process"):
+                logging.debug("Adding process handler for type %s", ep.name)
+                handler = ep.load()
+                Process.HANDLERS[ep.name] = handler
+                if handler is None:
+                    logging.error("Handler of type %s is null", ep.name)
 
         return Process.HANDLERS.get(key, None)
 
@@ -109,18 +114,25 @@ class Process:
         """Wait until the process finishes and returns the error code"""
         raise NotImplementedError(f"Not implemented: {self.__class__}.wait")
 
-    async def aio_state(self) -> ProcessState:
-        """Returns the job state"""
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
+        """Returns the job state
+
+        :param timeout: maximum waiting time for a refresh
+        """
         raise NotImplementedError(f"Not implemented: {self.__class__}.aio_state")
 
     async def aio_isrunning(self):
         """True is the process is truly running (I/O)"""
         return (await self.aio_state()).running
 
-    async def aio_code(self):
-        """Returns a future containing the returned code"""
+    async def aio_code(self) -> Optional[int]:
+        """Returns a future containing the returned code
+
+        Returns None if the process has already finished – and no information is
+        known about the process.
+        """
         code = await asyncThreadcheck("aio_code", self.wait)
-        logger.debug("Got for return code %s: %s", self, code)
+        logger.debug("Got return code %s for %s", code, self)
         return code
 
     def kill(self):
@@ -145,8 +157,11 @@ class ProcessBuilder:
         self.environ: Mapping[str, str] = {}
         self.command = []
 
-    def start(self) -> Process:
-        """Start the process"""
+    def start(self, task_mode: bool = False) -> Process:
+        """Start the process
+
+        :param task_mode: True if the process is a job script
+        """
         raise NotImplementedError("Method not implemented in %s" % self.__class__)
 
 

experimaestro/connectors/local.py

@@ -1,7 +1,7 @@
-"""All classes related to localhost management
-"""
+"""All classes related to localhost management"""
 
 import subprocess
+from typing import Optional
 from pathlib import Path, WindowsPath, PosixPath
 import os
 import threading
@@ -29,15 +29,17 @@ class PsutilProcess(Process):
     def __init__(self, pid: int):
         self._process = psutil.Process(pid)
 
-    def wait(self) -> int:
+    def wait(self) -> Optional[int]:
         logger.debug("Waiting (psutil) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (psutil) for process with PID %s", self._process.pid
+            "Finished to wait (psutil) for process with PID %s: code %s",
+            self._process.pid,
+            code,
         )
         return code
 
-    async def aio_state(self):
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
         if self._process.is_running():
             return ProcessState.RUNNING
         return ProcessState.FINISHED
@@ -57,11 +59,13 @@ class LocalProcess(Process):
         logger.debug("Waiting (python) for process with PID %s", self._process.pid)
         code = self._process.wait()
         logger.debug(
-            "Finished to wait (python) for process with PID %s", self._process.pid
+            "Finished to wait (python) for process with PID %s: %s",
+            self._process.pid,
+            code,
         )
         return code
 
-    async def aio_state(self):
+    async def aio_state(self, timeout: float | None = None) -> ProcessState:
         code = self._process.poll()
         if code is None:
             return ProcessState.RUNNING
@@ -102,8 +106,11 @@ def getstream(redirect: Redirect, write: bool):
 
 
 class LocalProcessBuilder(ProcessBuilder):
-    def start(self):
-        """Start the process"""
+    def start(self, task_mode=False):
+        """Start the process
+
+        :param task_mode: just ignored
+        """
         stdin = getstream(self.stdin, False)
         stdout = getstream(self.stdout, True)
         stderr = getstream(self.stderr, True)
@@ -194,7 +201,9 @@ class LocalConnector(Connector):
         return LocalProcessBuilder()
 
     def resolve(self, path: Path, basepath: Path = None) -> str:
-        assert isinstance(path, PosixPath) or isinstance(path, WindowsPath)
+        assert isinstance(path, PosixPath) or isinstance(
+            path, WindowsPath
+        ), f"Unrecognized path {type(path)}"
         if not basepath:
             return str(path.absolute())
         try:

experimaestro/connectors/ssh.py

@@ -1,12 +1,15 @@
+try:
+    from pathlib import Path, _posix_flavour
+except ImportError:
+    # Avoids problem with python 3.12 where this module does not work
+    # anyways
+    _posix_flavour = None
+
 from dataclasses import dataclass
-from pathlib import Path, _posix_flavour
 import io
 import os
 import re
-from experimaestro.launcherfinder import LauncherRegistry, YAMLDataClass
-from fabric import Connection
-from invoke import Promise
-import invoke.exceptions
+from experimaestro.launcherfinder import LauncherRegistry
 from urllib.parse import urlparse
 from itertools import chain
 from . import Connector
@@ -19,6 +22,22 @@ from . import (
 from experimaestro.locking import Lock
 from experimaestro.tokens import Token
 
+try:
+    from fabric import Connection
+    from invoke import Promise
+    from invoke.exceptions import Failure
+except Exception:
+    # Just define placeholders
+    class Connection:
+        pass
+
+    class Promise:
+        pass
+
+    class Failure(Exception):
+        pass
+
+
 # Might be wise to switch to https://github.com/marian-code/ssh-utilities
 
 
@@ -132,7 +151,7 @@ class SshPath(Path):
 
 
 @dataclass
-class SshConfiguration(YAMLDataClass):
+class SshConfiguration:
     hostname: str
 
     def create(self, registry: LauncherRegistry):
@@ -172,7 +191,7 @@ class SshProcess(Process):
     def wait(self) -> int:
         try:
             self.promise.join()
-        except invoke.exceptions.Failure:
+        except Failure:
             raise
 
 
@@ -181,7 +200,7 @@ class SshProcessBuilder(ProcessBuilder):
         super().__init__()
         self.connector = connector
 
-    def start(self):
+    def start(self, task_mode: bool = False):
         """Start the process"""
 
         trans = str.maketrans({'"': r"\"", "$": r"\$"})

experimaestro/core/arguments.py

@@ -1,6 +1,6 @@
 """Management of the arguments (params, options, etc) associated with the XPM objects"""
 
-from typing import Optional, TypeVar, TYPE_CHECKING
+from typing import Optional, TypeVar, TYPE_CHECKING, Callable, Any
 from experimaestro.typingutils import get_optional
 from pathlib import Path
 import sys
@@ -75,11 +75,25 @@ class Argument:
         self.constant = constant
         self.ignored = self.type.ignore if ignored is None else ignored
         self.required = required
-        self.default = default
-        self.generator = generator
         self.objecttype = None
         self.is_data = is_data
 
+        self.generator = generator
+        self.default = None
+        self.ignore_generated = False
+
+        if default is not None:
+            assert self.generator is None, "generator and default are exclusive options"
+            if isinstance(default, field):
+                self.ignore_generated = default.ignore_generated
+
+                if default.default is not None:
+                    self.default = default.default
+                elif default.default_factory is not None:
+                    self.generator = default.default_factory
+            else:
+                self.default = default
+
         assert (
             not self.constant or self.default is not None
         ), "Cannot be constant without default"
@@ -170,6 +184,34 @@ DataPath = Annotated[Path, dataHint]
 """Annotates a path that should be kept to restore an object to its state"""
 
 
+class field:
+    """Extra information for a given experimaestro field (param or meta)"""
+
+    def __init__(
+        self,
+        *,
+        default: Any = None,
+        default_factory: Callable = None,
+        ignore_generated=False,
+    ):
+        """Gives some extra per-field information
+
+        :param default: a default value, defaults to None
+        :param default_factory: a default factory for values, defaults to None
+        :param ignore_generated: True if the value is hidden – it won't be accessible in
+            tasks, defaults to False. The interest of hidden is to add a
+            configuration field that changes the identifier, but will not be
+            used.
+        """
+        assert not (
+            (default is not None) and (default_factory is not None)
+        ), "default and default_factory are mutually exclusive options"
+
+        self.default_factory = default_factory
+        self.default = default
+        self.ignore_generated = ignore_generated
+
+
 class help(TypeAnnotation):
     def __init__(self, text: str):
         self.text = text