experimaestro 1.11.1__py3-none-any.whl → 2.0.0b4__py3-none-any.whl

experimaestro/cli/__init__.py

@@ -1,7 +1,6 @@
 # flake8: noqa: T201
 import sys
 from typing import Set, Optional
-import pkg_resources
 from itertools import chain
 from shutil import rmtree
 import click
@@ -10,26 +9,17 @@ from functools import cached_property, update_wrapper
 from pathlib import Path
 import subprocess
 from termcolor import cprint
+from importlib.metadata import entry_points
 
 import experimaestro
 from experimaestro.experiments.cli import experiments_cli
 import experimaestro.launcherfinder.registry as launcher_registry
-from experimaestro.settings import find_workspace
+from experimaestro.settings import ServerSettings, find_workspace
 
 # --- Command line main options
 logging.basicConfig(level=logging.INFO)
 
 
-def pass_cfg(f):
-    """Pass configuration information"""
-
-    @click.pass_context
-    def new_func(ctx, *args, **kwargs):
-        return ctx.invoke(f, ctx.obj, *args, **kwargs)
-
-    return update_wrapper(new_func, f)
-
-
 def check_xp_path(ctx, self, path: Path):
     if not (path / ".__experimaestro__").is_file():
         cprint(f"{path} is not an experimaestro working directory", "red")
@@ -142,7 +132,6 @@ def diff(path: Path):
     """Show the reason of the identifier change for a job"""
     from experimaestro.tools.jobs import load_job
     from experimaestro import Config
-    from experimaestro.core.objects import ConfigWalkContext
 
     _, job = load_job(path / "params.json", discard_id=False)
     _, new_job = load_job(path / "params.json")
@@ -268,13 +257,13 @@ def find_launchers(config: Optional[Path], spec: str):
     print(launcher_registry.find_launcher(spec))
 
 
-class Launchers(click.MultiCommand):
-    """Connectors commands"""
+class Launchers(click.Group):
+    """Dynamic command group for entry point discovery"""
 
     @cached_property
     def commands(self):
         map = {}
-        for ep in pkg_resources.iter_entry_points(f"experimaestro.{self.name}"):
+        for ep in entry_points(group=f"experimaestro.{self.name}"):
             if get_cli := getattr(ep.load(), "get_cli", None):
                 map[ep.name] = get_cli()
         return map
@@ -290,6 +279,21 @@ cli.add_command(Launchers("launchers", help="Launcher specific commands"))
 cli.add_command(Launchers("connectors", help="Connector specific commands"))
 cli.add_command(Launchers("tokens", help="Token specific commands"))
 
+# Import and add progress commands
+from .progress import progress as progress_cli
+
+cli.add_command(progress_cli)
+
+# Import and add jobs commands
+from .jobs import jobs as jobs_cli
+
+cli.add_command(jobs_cli)
+
+# Import and add refactor commands
+from .refactor import refactor as refactor_cli
+
+cli.add_command(refactor_cli)
+
 
 @cli.group()
 @click.option("--workdir", type=Path, default=None)
@@ -310,3 +314,123 @@ def list(workdir: Path):
             cprint(f"[unfinished] {p.name}", "yellow")
         else:
             cprint(p.name, "cyan")
+
+
+@experiments.command()
+@click.option("--console", is_flag=True, help="Use console TUI instead of web UI")
+@click.option(
+    "--port", type=int, default=12345, help="Port for web server (default: 12345)"
+)
+@click.option(
+    "--sync", is_flag=True, help="Force sync from disk before starting monitor"
+)
+@pass_cfg
+def monitor(workdir: Path, console: bool, port: int, sync: bool):
+    """Monitor experiments with web UI or console TUI"""
+    # Force sync from disk if requested
+    if sync:
+        from experimaestro.scheduler.state_sync import sync_workspace_from_disk
+
+        cprint("Syncing workspace from disk...", "yellow")
+        sync_workspace_from_disk(workdir, write_mode=True, force=True)
+        cprint("Sync complete", "green")
+
+    if console:
+        # Use Textual TUI
+        from experimaestro.tui import ExperimentTUI
+
+        app = ExperimentTUI(workdir, watch=True)
+        app.run()
+    else:
+        # Use React web server
+        from experimaestro.scheduler.state_provider import WorkspaceStateProvider
+        from experimaestro.server import Server
+
+        cprint(f"Starting experiment monitor on http://localhost:{port}", "green")
+        cprint("Press Ctrl+C to stop", "yellow")
+
+        state_provider = WorkspaceStateProvider.get_instance(
+            workdir,
+            sync_on_start=not sync,  # Skip auto-sync if we just did a forced one
+        )
+        settings = ServerSettings()
+        settings.port = port
+        server = Server.instance(settings, state_provider=state_provider)
+        server.start()
+
+        try:
+            import time
+
+            while True:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            cprint("\nShutting down...", "yellow")
+            state_provider.close()
+
+
+@experiments.command()
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Don't write to database, only show what would be synced",
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Force sync even if recently synced (bypasses time throttling)",
+)
+@click.option(
+    "--no-wait",
+    is_flag=True,
+    help="Don't wait for lock, fail immediately if unavailable",
+)
+@pass_cfg
+def sync(workdir: Path, dry_run: bool, force: bool, no_wait: bool):
+    """Synchronize workspace database from disk state
+
+    Scans experiment directories and job marker files to update the workspace
+    database. Uses exclusive locking to prevent conflicts with running experiments.
+    """
+    from experimaestro.scheduler.state_sync import sync_workspace_from_disk
+    from experimaestro.scheduler.workspace import Workspace
+    from experimaestro.settings import Settings
+
+    # Get settings and workspace settings
+    settings = Settings.instance()
+    ws_settings = find_workspace(workdir=workdir)
+
+    # Create workspace instance (manages database lifecycle)
+    workspace = Workspace(
+        settings=settings,
+        workspace_settings=ws_settings,
+        sync_on_init=False,  # Don't sync on init since we're explicitly syncing
+    )
+
+    try:
+        # Enter workspace context to initialize database
+        with workspace:
+            cprint(f"Syncing workspace: {workspace.path}", "cyan")
+            if dry_run:
+                cprint("DRY RUN MODE: No changes will be written", "yellow")
+            if force:
+                cprint("FORCE MODE: Bypassing time throttling", "yellow")
+
+            # Run sync
+            sync_workspace_from_disk(
+                workspace=workspace,
+                write_mode=not dry_run,
+                force=force,
+                blocking=not no_wait,
+            )
+
+            cprint("Sync completed successfully", "green")
+
+    except RuntimeError as e:
+        cprint(f"Sync failed: {e}", "red")
+        sys.exit(1)
+    except Exception as e:
+        cprint(f"Unexpected error during sync: {e}", "red")
+        import traceback
+
+        traceback.print_exc()
+        sys.exit(1)

experimaestro/cli/filter.py

@@ -1,57 +1,15 @@
-import asyncio
-import logging
-from typing import Any, Callable, Dict, List, Optional
-import pyparsing as pp
-from pathlib import Path
-import json
-from experimaestro.compat import cached_property
+"""Filter expressions for job queries
+
+This module provides a filter expression parser for querying jobs by state,
+tags, and other attributes.
+"""
+
 import re
-from experimaestro.scheduler import JobState
-
-
-class JobInformation:
-    def __init__(self, path: Path, scriptname: str, check: bool = False):
-        self.path = path
-        self.scriptname = scriptname
-        self.check = check
-
-    @cached_property
-    def params(self):
-        try:
-            return json.loads((self.path / "params.json").read_text())
-        except Exception:
-            logging.warning("Could not load params.json in %s", self.path)
-            return {"tags": {}}
-
-    @cached_property
-    def tags(self) -> List[str]:
-        return self.params["tags"]
-
-    @cached_property
-    def state(self) -> Optional[JobState]:
-        if (self.path / f"{self.scriptname}.done").is_file():
-            return JobState.DONE
-        if (self.path / f"{self.scriptname}.failed").is_file():
-            return JobState.ERROR
-        if (self.path / f"{self.scriptname}.pid").is_file():
-            if self.check:
-                if process := self.getprocess():
-                    state = asyncio.run(process.aio_state(0))
-                    if state is None or state.finished:
-                        return JobState.ERROR
-                else:
-                    return JobState.ERROR
-            return JobState.RUNNING
-        else:
-            return None
-
-    def getprocess(self):
-        from experimaestro.connectors import Process
-        from experimaestro.connectors.local import LocalConnector
-
-        connector = LocalConnector.instance()
-        pinfo = json.loads((self.path / f"{self.scriptname}.pid").read_text())
-        return Process.fromDefinition(connector, pinfo)
+from typing import Callable, TYPE_CHECKING
+import pyparsing as pp
+
+if TYPE_CHECKING:
+    from experimaestro.scheduler.state_provider import MockJob
 
 
 # --- classes for processing
@@ -61,14 +19,14 @@ class VarExpr:
     def __init__(self, values):
         (self.varname,) = values
 
-    def get(self, info: JobInformation):
+    def get(self, job: "MockJob"):
         if self.varname == "@state":
-            return info.state.name if info.state else None
+            return job.state.name if job.state else None
 
         if self.varname == "@name":
-            return str(info.path.parent.name)
+            return str(job.path.parent.name)
 
-        return info.tags.get(self.varname, None)
+        return job.tags.get(self.varname, None)
 
     def __repr__(self):
         return f"""VAR<{self.varname}>"""
@@ -81,8 +39,8 @@ class BaseInExpr:
 
 
 class InExpr(BaseInExpr):
-    def filter(self, information: JobInformation):
-        value = self.var.get(information)
+    def filter(self, job: "MockJob"):
+        value = self.var.get(job)
         return value in self.values
 
     def __repr__(self):
@@ -90,8 +48,8 @@ class InExpr(BaseInExpr):
 
 
 class NotInExpr(BaseInExpr):
-    def filter(self, information: JobInformation):
-        value = self.var.get(information)
+    def filter(self, job: "MockJob"):
+        value = self.var.get(job)
         return value not in self.values
 
     def __repr__(self):
@@ -106,25 +64,25 @@ class RegexExpr:
     def __repr__(self):
         return f"""REGEX[{self.varname}, {self.value}]"""
 
-    def matches(self, manager, publication):
+    def matches(self, _manager, publication):
         if self.varname == "tag":
             return self.value in publication.tags
 
         raise AssertionError()
 
-    def filter(self, information: JobInformation):
-        value = self.var.get(information)
+    def filter(self, job: "MockJob"):
+        value = self.var.get(job)
         if not value:
             return False
 
-        return self.re.match(value)
+        return self.regex.match(value)
 
 
 class ConstantString:
     def __init__(self, tokens):
         (self.value,) = tokens
 
-    def get(self, information: JobInformation):
+    def get(self, _job: "MockJob"):
         return self.value
 
     def __repr__(self):
@@ -138,8 +96,8 @@ class EqExpr:
     def __repr__(self):
         return f"""EQ[{self.var1}, {self.var2}]"""
 
-    def filter(self, information: JobInformation):
-        return self.var1.get(information) == self.var2.get(information)
+    def filter(self, job: "MockJob"):
+        return self.var1.get(job) == self.var2.get(job)
 
 
 class LogicExpr:
@@ -149,11 +107,11 @@ class LogicExpr:
         self.operator, self.y = tokens
         self.x = None
 
-    def filter(self, information: JobInformation):
+    def filter(self, job: "MockJob"):
         if self.operator == "and":
-            return self.y.filter(information) and self.x.filter(information)
+            return self.y.filter(job) and self.x.filter(job)
 
-        return self.y.filter(information) or self.x.filter(information)
+        return self.y.filter(job) or self.x.filter(job)
 
     @staticmethod
     def summary(tokens):
@@ -187,7 +145,10 @@ quotedString = pp.QuotedString('"', unquoteResults=True) | pp.QuotedString(
     "'", unquoteResults=True
 )
 
-var = l("@state") | l("@name") | pp.Word(pp.alphas)
+# Tag names can contain letters, digits, underscores, and hyphens
+# First character must be a letter, rest can include digits, underscores, hyphens
+tag_name = pp.Word(pp.alphas, pp.alphanums + "_-")
+var = l("@state") | l("@name") | tag_name
 var.setParseAction(VarExpr)
 
 regexExpr = var + tilde + quotedString
@@ -220,7 +181,14 @@ filterExpr = (
 expr = (matchExpr + pp.Optional(pipe + filterExpr)).setParseAction(LogicExpr.generator)
 
 
-def createFilter(query: str) -> Callable[[Dict[str, Any]], bool]:
-    """Returns a filter object given a query"""
+def createFilter(query: str) -> Callable[["MockJob"], bool]:
+    """Returns a filter function given a query string
+
+    Args:
+        query: Filter expression (e.g., '@state = "DONE" and model = "bm25"')
+
+    Returns:
+        A callable that takes a MockJob and returns True if it matches
+    """
     (r,) = logicExpr.parseString(query, parseAll=True)
     return r.filter