experimaestro 2.0.0a8__py3-none-any.whl → 2.0.0b8__py3-none-any.whl

experimaestro/cli/jobs.py

@@ -1,8 +1,6 @@
 # flake8: noqa: T201
-import asyncio
 import subprocess
 from typing import Optional
-from shutil import rmtree
 import click
 from pathlib import Path
 from termcolor import colored, cprint
@@ -34,6 +32,9 @@ def jobs(
     selects jobs where the tag model is "bm25", the tag mode is either
     "a" or "b", and the state is running.
 
+    Note: Jobs are read from the workspace database. If jobs are missing,
+    run 'experimaestro experiments sync' to synchronize the database
+    with the filesystem.
     """
     ws = ctx.obj.workspace = find_workspace(workdir=workdir, workspace=workspace)
     check_xp_path(ctx, None, ws.path)
@@ -44,117 +45,117 @@ def process(
     *,
     experiment="",
     tags="",
-    ready=False,
     clean=False,
     kill=False,
     filter="",
     perform=False,
     fullpath=False,
-    check=False,
+    count=0,
 ):
-    from .filter import createFilter, JobInformation
+    """Process jobs from the workspace database
+
+    Args:
+        workspace: Workspace settings
+        experiment: Filter by experiment ID
+        tags: Show tags in output
+        clean: Clean finished jobs
+        kill: Kill running jobs
+        filter: Filter expression
+        perform: Actually perform kill/clean (dry run if False)
+        fullpath: Show full paths instead of short names
+        count: Limit output to N most recent jobs (0 = no limit)
+    """
+    from .filter import createFilter
+    from experimaestro.scheduler.state_provider import WorkspaceStateProvider
     from experimaestro.scheduler import JobState
 
-    _filter = createFilter(filter) if filter else lambda x: True
-
-    # Get all jobs from experiments
-    job2xp = {}
-
-    path = workspace.path
-    for p in (path / "xp").glob("*"):
-        for job in p.glob("jobs/*/*"):
-            job_path = job.resolve()
-            if job_path.is_dir():
-                job2xp.setdefault(job_path.name, set()).add(p.name)
-
-        if (p / "jobs.bak").is_dir():
-            cprint(f"  Experiment {p.name} has not finished yet", "red")
-            if (not perform) and (kill or clean):
-                cprint(
-                    "  Preventing kill/clean (use --perform if you want to)", "yellow"
-                )
-                kill = False
-                clean = False
-
-    # Now, process jobs
-    for job in path.glob("jobs/*/*"):
-        info = None
-        p = job.resolve()
-        if p.is_dir():
-            *_, scriptname = p.parent.name.rsplit(".", 1)
-            xps = job2xp.get(job.name, set())
-            if experiment and experiment not in xps:
-                continue
-
-            info = JobInformation(p, scriptname, check=check)
-            job_str = (
-                (str(job.resolve()) if fullpath else f"{job.parent.name}/{job.name}")
-                + " "
-                + ",".join(xps)
-            )
-
-            if filter:
-                if not _filter(info):
-                    continue
-
-            if info.state is None:
-                print(colored(f"NODIR   {job_str}", "red"), end="")
-            elif info.state.running():
+    _filter = createFilter(filter) if filter else None
+
+    # Get state provider (write mode for kill/clean operations)
+    read_only = not (kill or clean)
+    provider = WorkspaceStateProvider.get_instance(workspace.path, read_only=read_only)
+
+    try:
+        # Get all jobs from the database
+        all_jobs = provider.get_all_jobs()
+
+        # Filter by experiment if specified
+        if experiment:
+            all_jobs = [j for j in all_jobs if j.experiment_id == experiment]
+
+        # Apply filter expression
+        if _filter:
+            all_jobs = [j for j in all_jobs if _filter(j)]
+
+        # Sort by submission time (most recent first)
+        # Jobs without submittime go to the end
+        all_jobs.sort(key=lambda j: j.submittime or 0, reverse=True)
+
+        # Limit to N most recent jobs if count is specified
+        if count > 0:
+            all_jobs = all_jobs[:count]
+
+        if not all_jobs:
+            cprint("No jobs found.", "yellow")
+            return
+
+        # Process each job
+        for job in all_jobs:
+            job_str = str(job.path) if fullpath else f"{job.task_id}/{job.identifier}"
+
+            # Add experiment info
+            if job.experiment_id:
+                job_str += f" [{job.experiment_id}]"
+
+            if job.state is None or job.state == JobState.UNSCHEDULED:
+                print(colored(f"UNSCHED {job_str}", "red"), end="")
+            elif job.state.running():
                 if kill:
                     if perform:
-                        process = info.getprocess()
-                        if process is None:
-                            cprint(
-                                "internal error – no process could be retrieved",
-                                "red",
-                            )
+                        if provider.kill_job(job, perform=True):
+                            cprint(f"KILLED  {job_str}", "light_red")
                         else:
-                            cprint(f"KILLING {process}", "light_red")
-                            process.kill()
+                            cprint(f"KILL FAILED {job_str}", "red")
                     else:
-                        print("KILLING (not performing)", process)
-                print(
-                    colored(f"{info.state.name:8}{job_str}", "yellow"),
-                    end="",
-                )
-            elif info.state == JobState.DONE:
-                print(
-                    colored(f"DONE    {job_str}", "green"),
-                    end="",
-                )
-            elif info.state == JobState.ERROR:
+                        cprint(f"KILLING {job_str} (dry run)", "yellow")
+                else:
+                    print(colored(f"{job.state.name:8}{job_str}", "yellow"), end="")
+            elif job.state == JobState.DONE:
+                print(colored(f"DONE    {job_str}", "green"), end="")
+            elif job.state == JobState.ERROR:
                 print(colored(f"FAIL    {job_str}", "red"), end="")
             else:
-                print(
-                    colored(f"{info.state.name:8}{job_str}", "red"),
-                    end="",
-                )
+                print(colored(f"{job.state.name:8}{job_str}", "red"), end="")
 
-        else:
-            if not ready:
-                continue
-            print(colored(f"READY {job_path}", "yellow"), end="")
+            # Show tags if requested
+            if tags and job.tags:
+                print(f""" {" ".join(f"{k}={v}" for k, v in job.tags.items())}""")
+            elif not (kill and perform):
+                print()
 
-        if tags:
-            print(f""" {" ".join(f"{k}={v}" for k, v in info.tags.items())}""")
-        else:
-            print()
+            # Clean finished jobs
+            if clean and job.state and job.state.finished():
+                if perform:
+                    if provider.clean_job(job, perform=True):
+                        cprint("  Cleaned", "red")
+                    else:
+                        cprint("  Clean failed", "red")
+                else:
+                    cprint("  Would clean (dry run)", "yellow")
 
-        if clean and info.state and info.state.finished():
-            if perform:
-                cprint("Cleaning...", "red")
-                rmtree(p)
-            else:
-                cprint("Cleaning... (not performed)", "red")
-    print()
+        print()
+
+    finally:
+        # Close provider if we created it for write mode
+        if not read_only:
+            provider.close()
 
 
 @click.option("--experiment", default=None, help="Restrict to this experiment")
 @click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
-@click.option("--no-check", is_flag=True, help="Check that running jobs")
+@click.option("--count", "-c", default=0, type=int, help="Limit to N most recent jobs")
 @jobs.command()
 @click.pass_context
 def list(
@@ -162,24 +163,22 @@ def list(
     experiment: str,
     filter: str,
     tags: bool,
-    ready: bool,
     fullpath: bool,
-    no_check: bool,
+    count: int,
 ):
+    """List all jobs in the workspace (sorted by submission date, most recent first)"""
     process(
         ctx.obj.workspace,
         experiment=experiment,
         filter=filter,
         tags=tags,
-        ready=ready,
         fullpath=fullpath,
-        check=not no_check,
+        count=count,
     )
 
 
 @click.option("--experiment", default=None, help="Restrict to this experiment")
 @click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--perform", is_flag=True, help="Really perform the killing")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
@@ -190,17 +189,15 @@ def kill(
     experiment: str,
     filter: str,
     tags: bool,
-    ready: bool,
     fullpath: bool,
     perform: bool,
-    check: bool,
 ):
+    """Kill running jobs"""
     process(
         ctx.obj.workspace,
         experiment=experiment,
         filter=filter,
         tags=tags,
-        ready=ready,
         kill=True,
         perform=perform,
         fullpath=fullpath,
@@ -209,7 +206,6 @@ def kill(
 
 @click.option("--experiment", default=None, help="Restrict to this experiment")
 @click.option("--tags", is_flag=True, help="Show tags")
-@click.option("--ready", is_flag=True, help="Include tasks which are not yet scheduled")
 @click.option("--filter", default="", help="Filter expression")
 @click.option("--perform", is_flag=True, help="Really perform the cleaning")
 @click.option("--fullpath", is_flag=True, help="Prints full paths")
@@ -220,16 +216,15 @@ def clean(
     experiment: str,
     filter: str,
     tags: bool,
-    ready: bool,
     fullpath: bool,
     perform: bool,
 ):
+    """Clean finished jobs (delete directories and DB entries)"""
     process(
         ctx.obj.workspace,
         experiment=experiment,
         filter=filter,
         tags=tags,
-        ready=ready,
         clean=True,
         perform=perform,
         fullpath=fullpath,
@@ -244,25 +239,81 @@ def clean(
 @jobs.command()
 @click.pass_context
 def log(ctx, jobid: str, follow: bool, std: bool):
+    """View job log (stderr by default, stdout with --std)
+
+    JOBID format: task.name/hash (e.g., mymodule.MyTask/abc123)
+    """
     task_name, task_hash = jobid.split("/")
     _, name = task_name.rsplit(".", 1)
-    path = (
+    log_path = (
         ctx.obj.workspace.path
         / "jobs"
         / task_name
         / task_hash
         / f"""{name}.{'out' if std else 'err'}"""
     )
+    if not log_path.exists():
+        cprint(f"Log file not found: {log_path}", "red")
+        return
     if follow:
-        subprocess.run(["tail", "-f", path])
+        subprocess.run(["tail", "-f", log_path])
     else:
-        subprocess.run(["less", "-r", path])
+        subprocess.run(["less", "-r", log_path])
 
 
 @click.argument("jobid", type=str)
 @jobs.command()
 @click.pass_context
 def path(ctx, jobid: str):
+    """Print the path to a job directory
+
+    JOBID format: task.name/hash (e.g., mymodule.MyTask/abc123)
+    """
     task_name, task_hash = jobid.split("/")
-    path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
-    print(path)
+    job_path = ctx.obj.workspace.path / "jobs" / task_name / task_hash
+    if not job_path.exists():
+        cprint(f"Job directory not found: {job_path}", "red")
+        return
+    print(job_path)
+
+
+@click.option("--perform", is_flag=True, help="Actually delete orphan partials")
+@jobs.command("cleanup-partials")
+@click.pass_context
+def cleanup_partials(ctx, perform: bool):
+    """Clean up orphan partial directories
+
+    Partial directories are shared checkpoint locations created by
+    subparameters. When all jobs using a partial are deleted, the
+    partial becomes orphaned and can be cleaned up.
+
+    This command finds all orphan partials and deletes them (or shows
+    what would be deleted in dry-run mode).
+    """
+    from experimaestro.scheduler.state_provider import WorkspaceStateProvider
+
+    provider = WorkspaceStateProvider.get_instance(
+        ctx.obj.workspace.path, read_only=not perform
+    )
+
+    try:
+        orphan_paths = provider.cleanup_orphan_partials(perform=perform)
+
+        if not orphan_paths:
+            cprint("No orphan partials found.", "green")
+            return
+
+        if perform:
+            cprint(f"Cleaned {len(orphan_paths)} orphan partial(s):", "green")
+        else:
+            cprint(f"Found {len(orphan_paths)} orphan partial(s) (dry run):", "yellow")
+
+        for path in orphan_paths:
+            if perform:
+                print(colored(f"  Deleted: {path}", "red"))
+            else:
+                print(colored(f"  Would delete: {path}", "yellow"))
+
+    finally:
+        if perform:
+            provider.close()

experimaestro/cli/refactor.py

@@ -0,0 +1,249 @@
+"""Refactoring commands for experimaestro codebase patterns"""
+
+import ast
+import re
+import sys
+from pathlib import Path
+from typing import Iterator
+
+import click
+from termcolor import cprint
+
+
+class DefaultValueFinder(ast.NodeVisitor):
+    """AST visitor to find class definitions with Param/Meta/Option annotations"""
+
+    def __init__(self, source_lines: list[str]):
+        self.source_lines = source_lines
+        self.findings: list[dict] = []
+        self.current_class: str | None = None
+
+    def visit_ClassDef(self, node: ast.ClassDef):
+        old_class = self.current_class
+        self.current_class = node.name
+
+        # Check if this class might be a Config/Task (has annotations)
+        for item in node.body:
+            if isinstance(item, ast.AnnAssign):
+                self._check_annotation(item, node)
+
+        self.generic_visit(node)
+        self.current_class = old_class
+
+    def _check_annotation(self, node: ast.AnnAssign, class_node: ast.ClassDef):
+        """Check if an annotated assignment uses Param/Meta/Option with bare default"""
+        if not isinstance(node.target, ast.Name):
+            return
+
+        param_name = node.target.id
+
+        # Check if annotation is Param[...], Meta[...], or Option[...]
+        annotation = node.annotation
+        is_param_type = False
+
+        if isinstance(annotation, ast.Subscript):
+            if isinstance(annotation.value, ast.Name):
+                if annotation.value.id in ("Param", "Meta", "Option"):
+                    is_param_type = True
+
+        if not is_param_type:
+            return
+
+        # Check if there's a default value
+        if node.value is None:
+            return
+
+        # Check if the default is already wrapped in field()
+        if isinstance(node.value, ast.Call):
+            if isinstance(node.value.func, ast.Name):
+                if node.value.func.id == "field":
+                    return  # Already using field()
+
+        # Found a bare default value
+        self.findings.append(
+            {
+                "class_name": self.current_class,
+                "param_name": param_name,
+                "line": node.lineno,
+                "col_offset": node.col_offset,
+                "end_line": node.end_lineno,
+                "end_col_offset": node.end_col_offset,
+                "value_line": node.value.lineno,
+                "value_col": node.value.col_offset,
+                "value_end_line": node.value.end_lineno,
+                "value_end_col": node.value.end_col_offset,
+            }
+        )
+
+
+def find_bare_defaults(file_path: Path) -> list[dict]:
+    """Find all bare default values in a Python file"""
+    try:
+        source = file_path.read_text()
+        tree = ast.parse(source)
+    except (SyntaxError, UnicodeDecodeError):
+        return []
+
+    source_lines = source.splitlines()
+    finder = DefaultValueFinder(source_lines)
+    finder.visit(tree)
+    return finder.findings
+
+
+def refactor_file(file_path: Path, perform: bool) -> int:
+    """Refactor a single file, returns number of changes made/found"""
+    findings = find_bare_defaults(file_path)
+    if not findings:
+        return 0
+
+    source = file_path.read_text()
+    source_lines = source.splitlines(keepends=True)
+
+    # Sort findings by line number in reverse order (to not mess up offsets)
+    findings.sort(key=lambda f: (f["line"], f["col_offset"]), reverse=True)
+
+    changes_made = 0
+    for finding in findings:
+        class_name = finding["class_name"]
+        param_name = finding["param_name"]
+        line_num = finding["line"]
+
+        # Get the line content
+        line_idx = line_num - 1
+        if line_idx >= len(source_lines):
+            continue
+
+        line = source_lines[line_idx]
+
+        # Try to find and replace the pattern on this line
+        # Pattern: `param_name: Param[...] = value` -> `param_name: Param[...] = field(ignore_default=value)`
+        # We need to be careful with multi-line values
+
+        # Simple case: value is on the same line
+        if finding["value_line"] == finding["value_end_line"] == line_num:
+            # Extract the value part
+            value_start = finding["value_col"]
+            value_end = finding["value_end_col"]
+
+            # Get the original value string
+            original_value = line[value_start:value_end]
+
+            # Create the replacement
+            new_value = f"field(ignore_default={original_value})"
+
+            # Replace in the line
+            new_line = line[:value_start] + new_value + line[value_end:]
+            source_lines[line_idx] = new_line
+
+            if perform:
+                cprint(
+                    f"  {file_path}:{line_num}: {class_name}.{param_name} = {original_value} "
+                    f"-> field(ignore_default={original_value})",
+                    "green",
+                )
+            else:
+                cprint(
+                    f"  {file_path}:{line_num}: {class_name}.{param_name} = {original_value} "
+                    f"-> field(ignore_default={original_value})",
+                    "yellow",
+                )
+
+            changes_made += 1
+        else:
+            # Multi-line value - more complex handling needed
+            # For now, just report it
+            cprint(
+                f"  {file_path}:{line_num}: {class_name}.{param_name} has multi-line default "
+                f"(manual fix required)",
+                "red",
+            )
+            changes_made += 1
+
+    if perform and changes_made > 0:
+        # Check if we need to add 'field' import
+        new_source = "".join(source_lines)
+
+        # Simple check for field import
+        if "from experimaestro" in new_source or "import experimaestro" in new_source:
+            # Check if field is already imported
+            if not re.search(
+                r"from\s+experimaestro[^\n]*\bfield\b", new_source
+            ) and not re.search(
+                r"from\s+experimaestro\.core\.arguments[^\n]*\bfield\b", new_source
+            ):
+                # Try to add field to existing import
+                new_source = re.sub(
+                    r"(from\s+experimaestro\s+import\s+)([^\n]+)",
+                    r"\1field, \2",
+                    new_source,
+                    count=1,
+                )
+
+        file_path.write_text(new_source)
+
+    return changes_made
+
+
+def find_python_files(path: Path) -> Iterator[Path]:
+    """Find all Python files in a directory"""
+    if path.is_file():
+        if path.suffix == ".py":
+            yield path
+    else:
+        for py_file in path.rglob("*.py"):
+            # Skip common directories
+            parts = py_file.parts
+            if any(
+                p in parts
+                for p in ("__pycache__", ".git", ".venv", "venv", "node_modules")
+            ):
+                continue
+            yield py_file
+
+
+@click.group()
+def refactor():
+    """Refactor codebase patterns"""
+    pass
+
+
+@refactor.command(name="default-values")
+@click.option(
+    "--perform",
+    is_flag=True,
+    help="Perform the refactoring (default is dry-run)",
+)
+@click.argument("path", type=click.Path(exists=True, path_type=Path), default=".")
+def default_values(path: Path, perform: bool):
+    """Fix ambiguous default values in configuration files.
+
+    Converts `x: Param[int] = 23` to `x: Param[int] = field(ignore_default=23)`
+    to make the behavior explicit.
+
+    By default runs in dry-run mode. Use --perform to apply changes.
+    """
+    if not perform:
+        cprint("DRY RUN MODE: No changes will be written", "yellow")
+        cprint("Use --perform to apply changes\n", "yellow")
+
+    total_changes = 0
+    files_with_changes = 0
+
+    for py_file in find_python_files(path):
+        changes = refactor_file(py_file, perform)
+        if changes > 0:
+            total_changes += changes
+            files_with_changes += 1
+
+    if total_changes == 0:
+        cprint("\nNo bare default values found.", "green")
+    else:
+        action = "Fixed" if perform else "Found"
+        cprint(
+            f"\n{action} {total_changes} bare default value(s) in {files_with_changes} file(s).",
+            "green" if perform else "yellow",
+        )
+        if not perform:
+            cprint("Run with --perform to apply changes.", "yellow")
+
+    sys.exit(0 if perform or total_changes == 0 else 1)

experimaestro/click.py

@@ -37,7 +37,6 @@ class forwardoption(metaclass=forwardoptionMetaclass):
         name = "--%s" % (option_name or argument.name.replace("_", "-"))
         default = kwargs["default"] if "default" in kwargs else argument.default
 
-        # TODO: set the type of the option when not a simple type
         return click.option(name, help=argument.help or "", default=default)
 
     def __getattr__(self, key):