experimaestro 1.11.1__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/cli/jobs.py

@@ -1,8 +1,6 @@
 # flake8: noqa: T201
-import asyncio
 import subprocess
 from typing import Optional
-from shutil import rmtree
 import click
 from pathlib import Path
 from termcolor import colored, cprint
@@ -34,6 +32,9 @@ def jobs(
     selects jobs where the tag model is "bm25", the tag mode is either
     "a" or "b", and the state is running.
 
+    Note: Jobs are read from the workspace database. If jobs are missing,
+    run 'experimaestro experiments sync' to synchronize the database
+    with the filesystem.
     """
     ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
     check_xp_path(ctx, None, ws.path)
@@ -44,117 +45,117 @@ def process(
     *,
     experiment="",
     tags="",
-    ready=False,
     clean=False,
     kill=False,
     filter="",
     perform=False,
     fullpath=False,
-    check=False,
+    count=0,
 ):
-    from .filter import createFilter, JobInformation
+    """Process jobs from the workspace database
+
+    Args:
+        workspace: Workspace settings
+        experiment: Filter by experiment ID
+        tags: Show tags in output
+        clean: Clean finished jobs
+        kill: Kill running jobs
+        filter: Filter expression
+        perform: Actually perform kill/clean (dry run if False)
+        fullpath: Show full paths instead of short names
+        count: Limit output to N most recent jobs (0 = no limit)
+    """
+    from .filter import createFilter
+    from experimaestro.scheduler.state_provider import WorkspaceStateProvider
     from experimaestro.scheduler import JobState
 
-    _filter = createFilter(filter) if filter else lambda x: True
-
-    # Get all jobs from experiments
-    job2xp = {}
-
-    path = workspace.path
-    for p in (path / "xp").glob("*"):
-        for job in p.glob("jobs/*/*"):
-            job_path = job.resolve()
-            if job_path.is_dir():
-                job2xp.setdefault(job_path.name, set()).add(p.name)
-
-        if (p / "jobs.bak").is_dir():
-            cprint(f"  Experiment {p.name} has not finished yet", "red")
-            if (not perform) and (kill or clean):
-                cprint(
-                    "  Preventing kill/clean (use --perform if you want to)", "yellow"
-                )
-                kill = False
-                clean = False
-
-    # Now, process jobs
-    for job in path.glob("jobs/*/*"):
-        info = None
-        p = job.resolve()
-        if p.is_dir():
-            *_, scriptname = p.parent.name.rsplit(".", 1)
-            xps = job2xp.get(job.name, set())
-            if experiment and experiment not in xps:
-                continue
-
-            info = JobInformation(p, scriptname, check=check)
-            job_str = (
-                (str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}")
-                + " "
-                + ",".join(xps)
-            )
-
-            if filter:
-                if not _filter(info):
-                    continue
-
-            if info.state is None:
-                print(colored(f"NODIR   {job_str}", "red"), end="")
-            elif info.state.running():
+    _filter = createFilter(filter) if filter else None
+
+    # Get state provider (write mode for kill/clean operations)
+    read_only = not (kill or clean)
+    provider = WorkspaceStateProvider.get_instance(workspace.path, read_only=read_only)
+
+    try:
+        # Get all jobs from the database
+        all_jobs = provider.get_all_jobs()
+
+        # Filter by experiment if specified
+        if experiment:
+            all_jobs = [j for j in all_jobs if j.experiment_id == experiment]
+
+        # Apply filter expression
+        if _filter:
+            all_jobs = [j for j in all_jobs if _filter(j)]
+
+        # Sort by submission time (most recent first)
+        # Jobs without submittime go to the end
+        all_jobs.sort(key=lambda j: j.submittime or 0, reverse=True)
+
+        # Limit to N most recent jobs if count is specified
+        if count > 0:
+            all_jobs = all_jobs[:count]
+
+        if not all_jobs:
+            cprint("No jobs found.", "yellow")
+            return
+
+        # Process each job
+        for job in all_jobs:
+            job_str = str(job.path) if fullpath else f"{job.task_id}/{job.identifier}"
+
+            # Add experiment info
+            if job.experiment_id:
+                job_str += f" [{job.experiment_id}]"
+
+            if job.state is None or job.state == JobState.UNSCHEDULED:
+                print(colored(f"UNSCHED {job_str}", "red"), end="")
+            elif job.state.running():
                 if kill:
                     if perform:
-                        process = info.getprocess()
-                        if process is None:
-                            cprint(
-                                "internal error – no process could be retrieved",
-                                "red",
-                            )
+                        if provider.kill_job(job, perform=True):
+                            cprint(f"KILLED  {job_str}", "light_red")
                         else:
-                            cprint(f"KILLING {process}", "light_red")
-                            process.kill()
+                            cprint(f"KILL FAILED {job_str}", "red")
                     else:
-                        print("KILLING (not performing)", process)
-                print(
-                    colored(f"{info.state.name:8}{job_str}", "yellow"),
-                    end="",
-                )
-            elif info.state == JobState.DONE:
-                print(
-                    colored(f"DONE    {job_str}", "green"),
-                    end="",
-                )
-            elif info.state == JobState.ERROR:
+                        cprint(f"KILLING {job_str} (dry run)", "yellow")
+                else:
+                    print(colored(f"{job.state.name:8}{job_str}", "yellow"), end="")
+            elif job.state == JobState.DONE:
+                print(colored(f"DONE    {job_str}", "green"), end="")
+            elif job.state == JobState.ERROR:
                 print(colored(f"FAIL    {job_str}", "red"), end="")
             else:
-                print(
-                    colored(f"{info.state.name:8}{job_str}", "red"),
-                    end="",
-                )
+                print(colored(f"{job.state.name:8}{job_str}", "red"), end="")
 
-        else:
-            if not ready:
-                continue
-            print(colored(f"READY {job_path}", "yellow"), end="")
+            # Show tags if requested
+            if tags and job.tags:
+                print(f""" {" ".join(f"{k}={v}" for k, v in job.tags.items())}""")
+            elif not (kill and perform):
+                print()
 
-        if tags:
-            print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
-        else:
-            print()
+            # Clean finished jobs
+            if clean and job.state and job.state.finished():
+                if perform:
+                    if provider.clean_job(job, perform=True):
+                        cprint("  Cleaned", "red")
+                    else:
+                        cprint("  Clean failed", "red")
+                else:
+                    cprint("  Would clean (dry run)", "yellow")
 
-        if clean and info.state and info.state.finished():
-            if perform:
-                cprint("Cleaning...", "red")
-                rmtree(p)
-            else:
-                cprint("Cleaning... (not performed)", "red")
-    print()
+        print()
+
+    finally:
+        # Close provider if we created it for write mode
+        if not read_only:
+            provider.close()
 
 
 @click.option("--experiment", default=None, help="Restrict to this experiment")
 @click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
-@click.option("--no-check", is_flag=True, help="Check that running jobs")
+@click.option("--count", "-c", default=0, type=int, help="Limit to N most recent jobs")
 @jobs.command()
 @click.pass_context
 def list(
@@ -162,24 +163,22 @@ def list(
     experiment: str,
     filter: str,
     tags: bool,
-    ready: bool,
     fullpath: bool,
-    no_check: bool,
+    count: int,
 ):
+    """List all jobs in the workspace (sorted by submission date, most recent first)"""
     process(
         ctx.obj.workspace,
         experiment=experiment,
         filter=filter,
         tags=tags,
-        ready=ready,
         fullpath=fullpath,
-        check=not no_check,
+        count=count,
     )
 
 
 @click.option("--experiment", default=None, help="Restrict to this experiment")
 @click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--perform", is_flag=True, help="Really perform the killing")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
@@ -190,17 +189,15 @@ def kill(
     experiment: str,
     filter: str,
     tags: bool,
-    ready: bool,
     fullpath: bool,
     perform: bool,
-    check: bool,
 ):
+    """Kill running jobs"""
     process(
         ctx.obj.workspace,
         experiment=experiment,
         filter=filter,
         tags=tags,
-        ready=ready,
         kill=True,
         perform=perform,
         fullpath=fullpath,
@@ -209,7 +206,6 @@ def kill(
 
 @click.option("--experiment", default=None, help="Restrict to this experiment")
 @click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--perform", is_flag=True, help="Really perform the cleaning")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
@@ -220,16 +216,15 @@ def clean(
     experiment: str,
     filter: str,
     tags: bool,
-    ready: bool,
     fullpath: bool,
     perform: bool,
 ):
+    """Clean finished jobs (delete directories and DB entries)"""
     process(
         ctx.obj.workspace,
         experiment=experiment,
         filter=filter,
         tags=tags,
-        ready=ready,
         clean=True,
         perform=perform,
         fullpath=fullpath,
@@ -244,25 +239,81 @@ def clean(
 @jobs.command()
 @click.pass_context
 def log(ctx, jobid: str, follow: bool, std: bool):
+    """View job log (stderr by default, stdout with --std)
+
+    JOBID format: task.name/hash (e.g., mymodule.MyTask/abc123)
+    """
     task_name, task_hash = jobid.split("/")
     _, name = task_name.rsplit(".", 1)
-    path = (
+    log_path = (
         ctx.obj.workspace.path
         / "jobs"
         / task_name
         / task_hash
         / f"""{name}.{'out' if std else 'err'}"""
     )
+    if not log_path.exists():
+        cprint(f"Log file not found: {log_path}", "red")
+        return
     if follow:
-        subprocess.run(["tail", "-f", path])
+        subprocess.run(["tail", "-f", log_path])
     else:
-        subprocess.run(["less", "-r", path])
+        subprocess.run(["less", "-r", log_path])
 
 
 @click.argument("jobid", type=str)
 @jobs.command()
 @click.pass_context
 def path(ctx, jobid: str):
+    """Print the path to a job directory
+
+    JOBID format: task.name/hash (e.g., mymodule.MyTask/abc123)
+    """
     task_name, task_hash = jobid.split("/")
-    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
-    print(path)
+    job_path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    if not job_path.exists():
+        cprint(f"Job directory not found: {job_path}", "red")
+        return
+    print(job_path)
+
+
+@click.option("--perform", is_flag=True, help="Actually delete orphan partials")
+@jobs.command("cleanup-partials")
+@click.pass_context
+def cleanup_partials(ctx, perform: bool):
+    """Clean up orphan partial directories
+
+    Partial directories are shared checkpoint locations created by
+    subparameters. When all jobs using a partial are deleted, the
+    partial becomes orphaned and can be cleaned up.
+
+    This command finds all orphan partials and deletes them (or shows
+    what would be deleted in dry-run mode).
+    """
+    from experimaestro.scheduler.state_provider import WorkspaceStateProvider
+
+    provider = WorkspaceStateProvider.get_instance(
+        ctx.obj.workspace.path, read_only=not perform
+    )
+
+    try:
+        orphan_paths = provider.cleanup_orphan_partials(perform=perform)
+
+        if not orphan_paths:
+            cprint("No orphan partials found.", "green")
+            return
+
+        if perform:
+            cprint(f"Cleaned {len(orphan_paths)} orphan partial(s):", "green")
+        else:
+            cprint(f"Found {len(orphan_paths)} orphan partial(s) (dry run):", "yellow")
+
+        for path in orphan_paths:
+            if perform:
+                print(colored(f"  Deleted: {path}", "red"))
+            else:
+                print(colored(f"  Would delete: {path}", "yellow"))
+
+    finally:
+        if perform:
+            provider.close()

experimaestro/cli/progress.py

@@ -0,0 +1,269 @@
+"""Simplified CLI commands for managing and viewing progress files"""
+
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Dict
+
+import click
+from termcolor import colored
+
+try:
+    from tqdm import tqdm
+
+    TQDM_AVAILABLE = True
+except ImportError:
+    TQDM_AVAILABLE = False
+
+from experimaestro.progress import ProgressEntry, ProgressFileReader
+from experimaestro.settings import find_workspace
+from . import cli
+
+
+@click.option("--workspace", default="", help="Experimaestro workspace")
+@click.option("--workdir", type=Path, default=None)
+@cli.group()
+@click.pass_context
+def progress(
+    ctx,
+    workdir: Optional[Path],
+    workspace: Optional[str],
+):
+    """Progress tracking commands"""
+    ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
+
+
+def format_timestamp(timestamp: float) -> str:
+    """Format timestamp for display"""
+    dt = datetime.fromtimestamp(timestamp)
+    return dt.strftime("%Y-%m-%d %H:%M:%S")
+
+
+@click.argument("jobid", type=str)
+@progress.command()
+@click.pass_context
+def show(ctx, jobid: str):
+    """Show current progress state (default command)
+
+    JOBID format: task_name/task_hash
+    """
+    try:
+        task_name, task_hash = jobid.split("/")
+    except ValueError:
+        raise click.ClickException("JOBID must be in format task_name/task_hash")
+
+    workspace = ctx.obj.workspace
+    task_path = workspace.path / "jobs" / task_name / task_hash
+
+    if not task_path.exists():
+        raise click.ClickException(f"Job directory not found: {task_path}")
+
+    reader = ProgressFileReader(task_path)
+    current_progress = reader.get_current_progress()
+
+    if not current_progress:
+        click.echo("No progress information available")
+        return
+
+    # Filter out EOJ markers
+    current_progress = {k: v for k, v in current_progress.items() if k != -1}
+
+    if not current_progress:
+        click.echo("No progress information available")
+        return
+
+    click.echo(f"Progress for job {jobid}")
+    click.echo("=" * 80)
+
+    # Show simple text-based progress for each level
+    for level in sorted(current_progress.keys()):
+        entry = current_progress[level]
+        indent = "  " * level
+        progress_pct = f"{entry.progress * 100:5.1f}%"
+        desc = entry.desc or f"Level {level}"
+        timestamp = format_timestamp(entry.timestamp)
+
+        color = "green" if entry.progress >= 1.0 else "yellow"
+        click.echo(colored(f"{indent}L{level}: {progress_pct} - {desc}", color))
+        click.echo(colored(f"{indent}     Last updated: {timestamp}", "cyan"))
+
+
+def create_progress_bar(
+    level: int,
+    desc: str,
+    progress: float = 0.0,
+) -> tqdm:
+    """Create a properly aligned progress bar like dashboard style"""
+    if level > 0:
+        indent = "   " * (level - 1) + "└─ "
+    else:
+        indent = ""
+    label = f"{indent}L{level}"
+
+    colors = ["blue", "yellow", "magenta", "cyan", "white"]
+    bar_color = colors[level % len(colors)]
+
+    unit = desc[:50] if desc else f"Level {level}"
+    ncols = 100
+    wbar = 50
+
+    return tqdm(
+        total=100,
+        desc=label,
+        position=level,
+        leave=True,
+        bar_format=f"{{desc}}: {{percentage:3.0f}}%|{{bar:{wbar - len(indent)}}}| {{unit}}",  # noqa: F541
+        ncols=ncols,  # Adjust width based on level
+        unit=unit,
+        colour=bar_color,
+        initial=progress * 100,
+    )
+
+
+def _update_progress_display(
+    reader: ProgressFileReader, progress_bars: Dict[int, tqdm]
+) -> bool:
+    """Update the tqdm progress bars in dashboard style"""
+    current_state: Dict[int, ProgressEntry] = {
+        k: v for k, v in reader.get_current_state().items() if k != -1
+    }
+
+    if not current_state:
+        click.echo("No progress information available yet...")
+        return False
+
+    # Update existing bars and create new ones
+    for _level, entry in current_state.items():
+        progress_val = entry.progress * 100
+        desc = entry.desc or f"Level {entry.level}"
+
+        if entry.level not in progress_bars:
+            progress_bars[entry.level] = create_progress_bar(
+                entry.level, desc, progress_val
+            )
+
+        bar = progress_bars[entry.level]
+        bar.unit = desc[:50]
+        bar.n = progress_val
+
+        bar.refresh()
+
+    # Remove bars for levels that no longer exist
+    levels_to_remove = set(progress_bars.keys()) - set(current_state.keys())
+    for level in levels_to_remove:
+        progress_bars[level].close()
+        del progress_bars[level]
+
+    return True
+
+
+@click.argument("jobid", type=str)
+@click.option("--refresh-rate", "-r", default=0.5, help="Refresh rate in seconds")
+@progress.command()
+@click.pass_context
+def live(ctx, jobid: str, refresh_rate: float):
+    """Show live progress with tqdm-style bars
+
+    JOBID format: task_name/task_hash
+    """
+    if not TQDM_AVAILABLE:
+        click.echo("tqdm is not available. Install with: pip install tqdm")
+        click.echo("Falling back to basic display...")
+        ctx.invoke(show, jobid=jobid)
+        return
+
+    try:
+        task_name, task_hash = jobid.split("/")
+    except ValueError:
+        raise click.ClickException("JOBID must be in format task_name/task_hash")
+
+    workspace = ctx.obj.workspace
+    task_path = workspace.path / "jobs" / task_name / task_hash
+
+    if not task_path.exists():
+        raise click.ClickException(f"Job directory not found: {task_path}")
+
+    reader = ProgressFileReader(task_path)
+    progress_bars: Dict[int, tqdm] = {}
+
+    def cleanup_bars():
+        """Clean up all progress bars"""
+        for bar in progress_bars.values():
+            bar.close()
+        progress_bars.clear()
+
+    click.echo(f"Live progress for job {jobid}")
+    click.echo("Press Ctrl+C to stop")
+    click.echo("=" * 80)
+
+    try:
+        if not _update_progress_display(reader, progress_bars):
+            click.echo("No progress information available yet...")
+
+        while True:
+            time.sleep(refresh_rate)
+
+            if not _update_progress_display(reader, progress_bars):
+                # Check if job is complete
+                if reader.is_done():
+                    click.echo("\nJob completed!")
+                    break
+
+            # Check if all progress bars are at 100%
+            if progress_bars and all(bar.n >= 100 for bar in progress_bars.values()):
+                cleanup_bars()
+                click.echo("\nAll progress completed!")
+                break
+
+    except KeyboardInterrupt:
+        click.echo("\nStopped monitoring progress")
+    finally:
+        cleanup_bars()
+
+
+@progress.command(name="list")
+@click.pass_context
+def list_jobs(ctx):
+    """List all jobs with progress information"""
+    ws = ctx.obj.workspace
+    jobs_path = ws.path / "jobs"
+
+    if not jobs_path.exists():
+        click.echo("No jobs directory found")
+        return
+
+    for task_dir in jobs_path.iterdir():
+        if not task_dir.is_dir():
+            continue
+
+        for job_dir in task_dir.iterdir():
+            if not job_dir.is_dir():
+                continue
+
+            progress_dir = job_dir / ".experimaestro"
+            if not progress_dir.exists():
+                continue
+
+            # Check if there are progress files
+            progress_files = list(progress_dir.glob("progress-*.jsonl"))
+            if not progress_files:
+                continue
+
+            job_id = f"{task_dir.name}/{job_dir.name}"
+            reader = ProgressFileReader(job_dir)
+            current_state = reader.get_current_state()
+
+            # if current_progress:
+            if current_state:
+                # Get overall progress (level 0)
+                level_0 = current_state.get(0)
+                if level_0:
+                    color = "green" if level_0.progress >= 1.0 else "yellow"
+                    desc = f"{level_0.desc}" if level_0.desc else ""
+                    progress_pct = f"{level_0.progress * 100:5.1f}%"
+                    click.echo(colored(f"{job_id:50} - {progress_pct} - {desc}", color))
+
+                else:
+                    click.echo(f"{job_id:50} No level 0 progress")
+            else:
+                click.echo(f"{job_id:50} No progress data")