experimaestro 2.0.0b4__py3-none-any.whl → 2.0.0b17__py3-none-any.whl

experimaestro/__init__.py

@@ -40,11 +40,11 @@ from .core.arguments import (
     help,
 )
 from .generators import pathgenerator, PathGenerator
-from .core.subparameters import (
-    subparameters,
+from .core.partial import (
+    partial,
     param_group,
     ParameterGroup,
-    Subparameters,
+    Partial,
 )
 from .core.objects import (
     Config,
@@ -61,16 +61,23 @@ from .core.context import SerializationContext
 from .core.serializers import SerializationLWTask, PathSerializationLWTask
 from .core.types import Any, SubmitHook
 from .launchers import Launcher
-from .scheduler import Scheduler, experiment, FailedExperiment
+from .scheduler import (
+    Scheduler,
+    experiment,
+    FailedExperiment,
+    DirtyGitError,
+    GracefulExperimentExit,
+)
 from .exceptions import GracefulTimeout
 from .scheduler.workspace import Workspace, RunMode
-from .scheduler.state import get_experiment
+from .scheduler.transient import TransientMode
 from .notifications import progress, tqdm
 from .checkers import Choices
 from .xpmutils import DirectoryContext
 from .mkdocs.annotations import documentation
 from .scheduler.base import Job
 from .launcherfinder.registry import LauncherRegistry
+from .experiments.configuration import DirtyGitAction
 
 
 def set_launcher(launcher: Launcher):

experimaestro/cli/__init__.py

@@ -1,7 +1,6 @@
 # flake8: noqa: T201
 import sys
 from typing import Set, Optional
-from itertools import chain
 from shutil import rmtree
 import click
 import logging
@@ -164,27 +163,23 @@ def diff(path: Path):
     check(".", job, new_job, set())
 
 
-@click.option("--show-all", is_flag=True, help="Show even not orphans")
-@click.option(
-    "--ignore-old", is_flag=True, help="Ignore old jobs for unfinished experiments"
-)
 @click.option("--clean", is_flag=True, help="Prune the orphan folders")
 @click.option("--size", is_flag=True, help="Show size of each folder")
 @click.argument("path", type=Path, callback=check_xp_path)
 @cli.command()
-def orphans(path: Path, clean: bool, size: bool, show_all: bool, ignore_old: bool):
-    """Check for tasks that are not part of an experimental plan"""
+def orphans(path: Path, clean: bool, size: bool):
+    """Check for tasks that are not part of an experimental plan
 
-    jobspath = path / "jobs"
-
-    def getjobs(path: Path):
-        return ((str(p.relative_to(path)), p) for p in path.glob("*/*") if p.is_dir())
+    Uses the same orphan detection as the TUI (WorkspaceStateProvider.get_orphan_jobs).
+    """
+    from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider
 
-    def show(key: str, prefix=""):
+    def show(job, prefix=""):
+        key = f"{job.task_id}/{job.identifier}"
         if size:
             print(
                 prefix,
-                subprocess.check_output(["du", "-hs", key], cwd=jobspath)
+                subprocess.check_output(["du", "-hs", str(job.path)])
                 .decode("utf-8")
                 .strip(),
                 sep=None,
@@ -192,35 +187,21 @@ def orphans(path: Path, clean: bool, size: bool, show_all: bool, ignore_old: boo
         else:
             print(prefix, key, sep=None)
 
-    for p in (path / "xp").glob("*/jobs.bak"):
-        logging.warning("Experiment %s has not completed successfully", p.parent.name)
-
-    # Retrieve the jobs within expedriments (jobs and jobs.bak folder within experiments)
-    xpjobs = set()
-    if ignore_old:
-        paths = (path / "xp").glob("*/jobs")
-    else:
-        paths = chain((path / "xp").glob("*/jobs"), (path / "xp").glob("*/jobs.bak"))
-
-    for p in paths:
-        if p.is_dir():
-            for relpath, path in getjobs(p):
-                xpjobs.add(relpath)
-
-    # Now, look at stored jobs
-    found = 0
-    for key, jobpath in getjobs(jobspath):
-        if key not in xpjobs:
-            show(key)
-            if clean:
-                logging.info("Removing data in %s", jobpath)
-                rmtree(jobpath)
-        else:
-            if show_all:
-                show(key, prefix="[not orphan] ")
-            found += 1
+    # Use WorkspaceStateProvider.get_orphan_jobs() - same as TUI
+    provider = WorkspaceStateProvider.get_instance(path)
+    orphan_jobs = provider.get_orphan_jobs()
+
+    if not orphan_jobs:
+        print("No orphan jobs found.")
+        return
 
-    print(f"{found} jobs are not orphans")
+    print(f"Found {len(orphan_jobs)} orphan job(s):")
+    for job in orphan_jobs:
+        show(job)
+        if clean:
+            logging.info("Removing data in %s", job.path)
+            if job.path and job.path.exists():
+                rmtree(job.path)
 
 
 def arg_split(ctx, param, value):
@@ -279,22 +260,140 @@ cli.add_command(Launchers("launchers", help="Launcher specific commands"))
 cli.add_command(Launchers("connectors", help="Connector specific commands"))
 cli.add_command(Launchers("tokens", help="Token specific commands"))
 
-# Import and add progress commands
-from .progress import progress as progress_cli
-
-cli.add_command(progress_cli)
-
 # Import and add jobs commands
-from .jobs import jobs as jobs_cli
+from .jobs import jobs as jobs_cli  # noqa: E402
 
 cli.add_command(jobs_cli)
 
 # Import and add refactor commands
-from .refactor import refactor as refactor_cli
+from .refactor import refactor as refactor_cli  # noqa: E402
 
 cli.add_command(refactor_cli)
 
 
+@cli.group()
+def migrate():
+    """Migration commands for experimaestro workspace upgrades"""
+    pass
+
+
+@migrate.command("v1-to-v2")
+@click.argument("workdir", type=Path, callback=check_xp_path)
+@click.option(
+    "--dry-run", is_flag=True, help="Show what would be done without making changes"
+)
+@click.option(
+    "--keep-old", is_flag=True, help="Keep the old xp directory after migration"
+)
+def migrate_v1_to_v2(workdir: Path, dry_run: bool, keep_old: bool):
+    """Migrate workspace from v1 (xp/) to v2 (experiments/) layout
+
+    This command migrates experiment directories from the old layout:
+        workdir/xp/{experiment-id}/
+    to the new layout:
+        workdir/experiments/{experiment-id}/{run-id}/
+
+    Each old experiment directory becomes a single run directory with the
+    run ID based on its modification time.
+    """
+    from datetime import datetime
+
+    old_xp_dir = workdir / "xp"
+    new_experiments_dir = workdir / "experiments"
+
+    if not old_xp_dir.exists():
+        cprint(f"No old 'xp' directory found at {old_xp_dir}", "yellow")
+        return
+
+    # List all experiments in the old directory
+    old_experiments = [d for d in old_xp_dir.iterdir() if d.is_dir()]
+
+    if not old_experiments:
+        cprint("No experiments found in xp/ directory", "yellow")
+        return
+
+    cprint(f"Found {len(old_experiments)} experiment(s) to migrate:", "cyan")
+    for exp_dir in old_experiments:
+        cprint(f"  - {exp_dir.name}", "white")
+
+    if dry_run:
+        cprint("\nDRY RUN MODE - showing what would be done:", "yellow")
+
+    migrated = 0
+    for exp_dir in old_experiments:
+        exp_id = exp_dir.name
+
+        # Generate run_id from directory modification time
+        mtime = exp_dir.stat().st_mtime
+        mtime_dt = datetime.fromtimestamp(mtime)
+        run_id = mtime_dt.strftime("%Y%m%d_%H%M%S")
+
+        # Target path
+        new_exp_base = new_experiments_dir / exp_id
+        new_run_dir = new_exp_base / run_id
+
+        # Handle collision
+        suffix = 1
+        while new_run_dir.exists():
+            run_id = f"{mtime_dt.strftime('%Y%m%d_%H%M%S')}.{suffix}"
+            new_run_dir = new_exp_base / run_id
+            suffix += 1
+
+        if dry_run:
+            cprint(f"  {exp_dir} -> {new_run_dir}", "white")
+        else:
+            # Create the parent directory
+            new_exp_base.mkdir(parents=True, exist_ok=True)
+
+            # Move the experiment directory
+            import shutil
+
+            try:
+                shutil.move(str(exp_dir), str(new_run_dir))
+                cprint(
+                    f"  Migrated: {exp_id} -> {new_run_dir.relative_to(workdir)}",
+                    "green",
+                )
+                migrated += 1
+            except Exception as e:
+                cprint(f"  Failed to migrate {exp_id}: {e}", "red")
+
+    if not dry_run:
+        cprint(f"\nMigrated {migrated}/{len(old_experiments)} experiment(s)", "cyan")
+
+        # Handle old xp directory
+        remaining = list(old_xp_dir.iterdir())
+        if remaining:
+            if keep_old:
+                # Keep remaining files, rename directory
+                renamed_xp_dir = workdir / "xp_MIGRATED_TO_V2"
+                old_xp_dir.rename(renamed_xp_dir)
+                cprint(
+                    f"Renamed 'xp' -> 'xp_MIGRATED_TO_V2' ({len(remaining)} item(s))",
+                    "yellow",
+                )
+            else:
+                cprint(
+                    f"'xp' directory still contains {len(remaining)} item(s), not removing",
+                    "yellow",
+                )
+                cprint("Remove manually or use --keep-old to rename", "yellow")
+                return
+        else:
+            # Empty directory - remove it
+            old_xp_dir.rmdir()
+            cprint("Removed empty 'xp' directory", "green")
+
+        # Create a broken symlink to prevent v1 from recreating xp/
+        # v1 will find the symlink but fail when trying to use it
+        broken_link = workdir / "xp"
+        if not broken_link.exists() and not broken_link.is_symlink():
+            broken_link.symlink_to("/experimaestro_v2_migrated_workspace_do_not_use_v1")
+            cprint(
+                "Created broken 'xp' symlink to prevent experimaestro v1 usage", "green"
+            )
+
+
 @cli.group()
 @click.option("--workdir", type=Path, default=None)
 @click.option("--workspace", type=str, default=None)
@@ -309,11 +408,100 @@ def experiments(ctx, workdir, workspace):
 @experiments.command()
 @pass_cfg
 def list(workdir: Path):
-    for p in (workdir / "xp").iterdir():
-        if (p / "jobs.bak").exists():
-            cprint(f"[unfinished] {p.name}", "yellow")
+    """List experiments in the workspace"""
+    from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider
+
+    # Get experiments from state provider for detailed info
+    state_provider = WorkspaceStateProvider.get_instance(workdir)
+    experiments_list = state_provider.get_experiments()
+
+    # Build lookup by experiment_id
+    exp_info = {exp.experiment_id: exp for exp in experiments_list}
+
+    # New layout: experiments/{exp-id}/{run-id}/
+    experiments_dir = workdir / "experiments"
+    if not experiments_dir.exists():
+        cprint("No experiments found", "yellow")
+        return
+
+    for exp_dir in experiments_dir.iterdir():
+        if not exp_dir.is_dir():
+            continue
+
+        exp_id = exp_dir.name
+        exp = exp_info.get(exp_id)
+
+        # Build display string
+        display_parts = [exp_id]
+
+        # Add current run_id if available
+        if exp and getattr(exp, "current_run_id", None):
+            display_parts.append(f"[run: {exp.current_run_id}]")
+
+        # Add hostname if available
+        if exp and getattr(exp, "hostname", None):
+            display_parts.append(f"[{exp.hostname}]")
+
+        # Add job stats if available
+        if exp:
+            display_parts.append(f"({exp.finished_jobs}/{exp.total_jobs} jobs)")
+
+        display_str = " ".join(display_parts)
+        cprint(display_str, "cyan")
+
+
+def _run_monitor_ui(
+    state_provider, workdir: Path, console: bool, port: int, title: str = ""
+):
+    """Shared code for running monitor UI (TUI or web)
+
+    Args:
+        state_provider: StateProvider instance (local or remote)
+        workdir: Local workspace/cache directory
+        console: If True, use TUI; otherwise use web UI
+        port: Port for web server
+        title: Optional title for status messages
+    """
+    try:
+        if console:
+            # Use Textual TUI
+            from experimaestro.tui import ExperimentTUI
+
+            app = ExperimentTUI(
+                workdir, state_provider=state_provider, watch=True, show_logs=True
+            )
+            app.run()
         else:
-            cprint(p.name, "cyan")
+            # Use React web server
+            from experimaestro.webui import WebUIServer
+
+            if title:
+                cprint(
+                    f"Starting experiment monitor for {title} on http://localhost:{port}",
+                    "green",
+                )
+            else:
+                cprint(
+                    f"Starting experiment monitor on http://localhost:{port}", "green"
+                )
+            cprint("Press Ctrl+C to stop", "yellow")
+
+            settings = ServerSettings()
+            settings.port = port
+            server = WebUIServer.instance(settings, state_provider=state_provider)
+            server.start()
+
+            try:
+                import time
+
+                while True:
+                    time.sleep(1)
+            except KeyboardInterrupt:
+                pass
+    finally:
+        cprint("\nShutting down...", "yellow")
+        if state_provider:
+            state_provider.close()
 
 
 @experiments.command()
@@ -322,115 +510,186 @@ def list(workdir: Path):
     "--port", type=int, default=12345, help="Port for web server (default: 12345)"
 )
 @click.option(
-    "--sync", is_flag=True, help="Force sync from disk before starting monitor"
+    "--watcher",
+    type=click.Choice(["auto", "polling", "inotify", "fsevents", "kqueue", "windows"]),
+    default="auto",
+    help="Filesystem watcher type (auto=platform default, polling=network mounts)",
+)
+@click.option(
+    "--polling-interval",
+    type=float,
+    default=1.0,
+    help="Polling interval in seconds (only for --watcher=polling)",
+)
+@click.option(
+    "--sync",
+    is_flag=True,
+    hidden=True,
+    help="Deprecated: no longer needed (filesystem state is always current)",
 )
 @pass_cfg
-def monitor(workdir: Path, console: bool, port: int, sync: bool):
-    """Monitor experiments with web UI or console TUI"""
-    # Force sync from disk if requested
+def monitor(
+    workdir: Path,
+    console: bool,
+    port: int,
+    watcher: str,
+    polling_interval: float,
+    sync: bool,
+):
+    """Monitor local experiments with web UI or console TUI"""
+    # --sync is deprecated (kept for backwards compatibility)
     if sync:
-        from experimaestro.scheduler.state_sync import sync_workspace_from_disk
+        cprint(
+            "Note: --sync is deprecated and no longer needed "
+            "(filesystem state is always current)",
+            "yellow",
+        )
 
-        cprint("Syncing workspace from disk...", "yellow")
-        sync_workspace_from_disk(workdir, write_mode=True, force=True)
-        cprint("Sync complete", "green")
+    # Configure filesystem watcher type
+    from experimaestro.ipc import IPCom, WatcherType
 
-    if console:
-        # Use Textual TUI
-        from experimaestro.tui import ExperimentTUI
+    if watcher != "auto":
+        IPCom.set_watcher_type(WatcherType(watcher), polling_interval)
+    elif polling_interval != 1.0:
+        IPCom.set_watcher_type(WatcherType.POLLING, polling_interval)
 
-        app = ExperimentTUI(workdir, watch=True)
-        app.run()
-    else:
-        # Use React web server
-        from experimaestro.scheduler.state_provider import WorkspaceStateProvider
-        from experimaestro.server import Server
+    from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider
 
-        cprint(f"Starting experiment monitor on http://localhost:{port}", "green")
-        cprint("Press Ctrl+C to stop", "yellow")
+    state_provider = WorkspaceStateProvider.get_instance(workdir)
 
-        state_provider = WorkspaceStateProvider.get_instance(
-            workdir,
-            sync_on_start=not sync,  # Skip auto-sync if we just did a forced one
-        )
-        settings = ServerSettings()
-        settings.port = port
-        server = Server.instance(settings, state_provider=state_provider)
-        server.start()
+    _run_monitor_ui(state_provider, workdir, console, port)
 
-        try:
-            import time
 
-            while True:
-                time.sleep(1)
-        except KeyboardInterrupt:
-            cprint("\nShutting down...", "yellow")
-            state_provider.close()
+@experiments.command("ssh-monitor")
+@click.argument("host", type=str)
+@click.argument("remote_workdir", type=str)
+@click.option("--console", is_flag=True, help="Use console TUI instead of web UI")
+@click.option(
+    "--port", type=int, default=12345, help="Port for web server (default: 12345)"
+)
+@click.option(
+    "--watcher",
+    type=click.Choice(["auto", "polling", "inotify", "fsevents", "kqueue", "windows"]),
+    default="auto",
+    help="Filesystem watcher type (auto=platform default, polling=network mounts)",
+)
+@click.option(
+    "--polling-interval",
+    type=float,
+    default=1.0,
+    help="Polling interval in seconds (only for --watcher=polling)",
+)
+@click.option(
+    "--remote-xpm",
+    type=str,
+    default=None,
+    help="Path to experimaestro on remote host (default: use 'uv tool run')",
+)
+@click.option(
+    "--ssh-option",
+    "-o",
+    multiple=True,
+    help="Additional SSH options (can be repeated, e.g., -o '-p 2222')",
+)
+def ssh_monitor(
+    host: str,
+    remote_workdir: str,
+    console: bool,
+    port: int,
+    watcher: str,
+    polling_interval: float,
+    remote_xpm: str,
+    ssh_option: tuple,
+):
+    """Monitor experiments on a remote server via SSH
+
+    HOST is the SSH host (e.g., user@server)
+    REMOTE_WORKDIR is the workspace path on the remote server
+
+    Examples:
+        experimaestro experiments ssh-monitor myserver /path/to/workspace
+        experimaestro experiments ssh-monitor user@host /workspace --console
+        experimaestro experiments ssh-monitor host /workspace --remote-xpm /opt/xpm/bin/experimaestro
+    """
+    # Configure filesystem watcher type
+    from experimaestro.ipc import IPCom, WatcherType
+
+    if watcher != "auto":
+        IPCom.set_watcher_type(WatcherType(watcher), polling_interval)
+    elif polling_interval != 1.0:
+        IPCom.set_watcher_type(WatcherType.POLLING, polling_interval)
+
+    from experimaestro.scheduler.remote.client import SSHStateProviderClient
+
+    cprint(f"Connecting to {host}...", "yellow")
+    state_provider = SSHStateProviderClient(
+        host=host,
+        remote_workspace=remote_workdir,
+        ssh_options=list(ssh_option) if ssh_option else None,
+        remote_xpm_path=remote_xpm,
+    )
+    try:
+        state_provider.connect()
+        cprint(f"Connected to {host}", "green")
+    except Exception as e:
+        cprint(f"Failed to connect: {e}", "red")
+        raise click.Abort()
+
+    _run_monitor_ui(
+        state_provider,
+        state_provider.local_cache_dir,
+        console,
+        port,
+        title=host,
+    )
+
+
+@experiments.command("monitor-server")
+@pass_cfg
+def monitor_server(workdir: Path):
+    """Start monitoring server for SSH connections (JSON-RPC over stdio)
+
+    This command is intended to be run over SSH to provide remote monitoring.
+    Communication is via JSON-RPC over stdin/stdout.
+
+    Example:
+        ssh host 'experimaestro experiments --workdir /path monitor-server'
+    """
+    from experimaestro.scheduler.remote.server import SSHStateProviderServer
+
+    server = SSHStateProviderServer(workdir)
+    try:
+        server.start()
+    except KeyboardInterrupt:
+        server.stop()
 
 
 @experiments.command()
 @click.option(
     "--dry-run",
     is_flag=True,
-    help="Don't write to database, only show what would be synced",
+    help="[DEPRECATED] No longer needed with filesystem-based state tracking",
 )
 @click.option(
     "--force",
     is_flag=True,
-    help="Force sync even if recently synced (bypasses time throttling)",
+    help="[DEPRECATED] No longer needed with filesystem-based state tracking",
 )
 @click.option(
     "--no-wait",
     is_flag=True,
-    help="Don't wait for lock, fail immediately if unavailable",
+    help="[DEPRECATED] No longer needed with filesystem-based state tracking",
 )
 @pass_cfg
 def sync(workdir: Path, dry_run: bool, force: bool, no_wait: bool):
-    """Synchronize workspace database from disk state
+    """[DEPRECATED] Synchronize workspace database from disk state
 
-    Scans experiment directories and job marker files to update the workspace
-    database. Uses exclusive locking to prevent conflicts with running experiments.
+    This command is deprecated. With the new filesystem-based state tracking,
+    state is read directly from status.json and events files. No synchronization
+    is needed.
     """
-    from experimaestro.scheduler.state_sync import sync_workspace_from_disk
-    from experimaestro.scheduler.workspace import Workspace
-    from experimaestro.settings import Settings
-
-    # Get settings and workspace settings
-    settings = Settings.instance()
-    ws_settings = find_workspace(workdir=workdir)
-
-    # Create workspace instance (manages database lifecycle)
-    workspace = Workspace(
-        settings=settings,
-        workspace_settings=ws_settings,
-        sync_on_init=False,  # Don't sync on init since we're explicitly syncing
+    cprint(
+        "Warning: 'sync' command is deprecated. "
+        "State is now tracked via filesystem (status.json) - no sync needed.",
+        "yellow",
     )
-
-    try:
-        # Enter workspace context to initialize database
-        with workspace:
-            cprint(f"Syncing workspace: {workspace.path}", "cyan")
-            if dry_run:
-                cprint("DRY RUN MODE: No changes will be written", "yellow")
-            if force:
-                cprint("FORCE MODE: Bypassing time throttling", "yellow")
-
-            # Run sync
-            sync_workspace_from_disk(
-                workspace=workspace,
-                write_mode=not dry_run,
-                force=force,
-                blocking=not no_wait,
-            )
-
-            cprint("Sync completed successfully", "green")
-
-    except RuntimeError as e:
-        cprint(f"Sync failed: {e}", "red")
-        sys.exit(1)
-    except Exception as e:
-        cprint(f"Unexpected error during sync: {e}", "red")
-        import traceback
-
-        traceback.print_exc()
-        sys.exit(1)