labnotes 0.0.24__tar.gz → 0.0.26__tar.gz

{labnotes-0.0.24 → labnotes-0.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.24
+Version: 0.0.26
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.24 → labnotes-0.0.26}/src/labnotes/context.md

@@ -87,10 +87,13 @@ Runs can take a long time. Start `run` in the background, then check on it with
 
 In a terminal, `run` shows a live table of runs (status and time) and stays
 open when they finish. Runs not started yet show grayed as `pending`, and as
-`skipped` once stopped; `q` quits. Elsewhere, or with `--no-tui`, it prints a
+`skipped` once stopped. Move the cursor to a pending run and press `n` to run it
+next. `d` deselects a pending run so it won't run; `d` again puts it back at
+the end of the queue. `+` and `-` change how many run at once; lowering it lets running runs
+finish; `q` quits. Elsewhere, or with `--no-tui`, it prints a
 line per event instead.
 
-- `--concurrency N` runs N at a time (default 1).
+- `--concurrency N` runs N at a time to start with (default 1).
 - `-e PATH` reads another file instead of `experiment.toml`.
 - Ctrl-C stops the running scripts and records their end. A second Ctrl-C
   kills them. Closing the terminal or sending SIGTERM stops them the same way,

{labnotes-0.0.24 → labnotes-0.0.26}/src/labnotes/models.py

@@ -6,7 +6,17 @@ from typing import Any
 from labnotes.default_counts import DEFAULT_COUNTS
 
 Report = Callable[[dict[str, object]], None]
-Listen = Callable[[Callable[[], None]], None]
+
+
+@dataclass(frozen=True)
+class Controls:
+    interrupt: Callable[[], None]
+    run_next: Callable[[str], None]
+    set_concurrency: Callable[[int], None]
+    toggle: Callable[[str], None]
+
+
+Listen = Callable[[Controls], None]
 Start = Callable[[Report, Listen], Coroutine[Any, Any, list[dict[str, object]]]]
 
 

{labnotes-0.0.24 → labnotes-0.0.26}/src/labnotes/runner/describe_event.py

@@ -12,6 +12,14 @@ def describe_event(event: dict[str, object]) -> str:
         case "ended":
             outcome = event["error"] or f"exited {event['exit_code']}"
             return f"{event['run']} {outcome}"
+        case "queued":
+            return f"{event['run']} runs next."
+        case "deselected":
+            return f"{event['run']} deselected."
+        case "reselected":
+            return f"{event['run']} back in the queue."
+        case "concurrency":
+            return f"{event['concurrency']} at a time."
         case "stopping":
             return "Stopping runs. Ctrl-C again to kill them."
         case _:

labnotes-0.0.26/src/labnotes/runner/run_experiment.py

@@ -0,0 +1,138 @@
+import asyncio
+import contextlib
+import json
+import os
+import signal
+
+from labnotes.models import (
+    Controls,
+    ExperimentConfig,
+    LabnotesConfig,
+    Listen,
+    Report,
+)
+from labnotes.runner.execute_run import execute_run
+from labnotes.runner.new_run_id import new_run_id
+from labnotes.runner.param_grid import param_grid
+from labnotes.runner.plan_runs import plan_runs
+from labnotes.runner.run_folder import run_folder
+from labnotes.tables.list_experiment_runs import list_experiment_runs
+from labnotes.tables.open_tables import open_tables
+
+Entry = tuple[str, dict[str, object]]
+
+
+async def run_experiment(
+    config: LabnotesConfig,
+    name: str,
+    experiment: ExperimentConfig,
+    concurrency: int,
+    report: Report,
+    listen: Listen,
+) -> list[dict[str, object]]:
+    existing = await list_experiment_runs(open_tables(config), name, experiment.counts)
+    finished = [json.loads(str(run["params"])) for run in existing if run["counted"]]
+    planned: set[str] = set()
+
+    def taken(run: str) -> bool:
+        return (
+            run in planned or run_folder(config.root, config.runs, name, run).exists()
+        )
+
+    plan: list[Entry] = []
+    for params in plan_runs(
+        param_grid(experiment.params), experiment.repeats, finished
+    ):
+        run = new_run_id(taken)
+        planned.add(run)
+        plan.append((run, params))
+    runs = [{"run": run, "params": params} for run, params in plan]
+    report({"event": "planned", "runs": runs, "concurrency": concurrency})
+    running: set[asyncio.subprocess.Process] = set()
+    results: list[dict[str, object]] = []
+    stopping = asyncio.Event()
+
+    def signal_runs(signal_number: int) -> None:
+        for process in running:
+            with contextlib.suppress(ProcessLookupError):
+                os.killpg(process.pid, signal_number)
+
+    def interrupt() -> None:
+        plan.clear()
+        if not stopping.is_set():
+            stopping.set()
+            report({"event": "stopping"})
+            signal_runs(signal.SIGTERM)
+        elif running:
+            report({"event": "killing"})
+            signal_runs(signal.SIGKILL)
+
+    held: list[Entry] = []
+
+    def find(entries: list[Entry], run: str) -> Entry | None:
+        return next((entry for entry in entries if entry[0] == run), None)
+
+    def arrange(event: str, run: str) -> None:
+        pending = [id_ for id_, _ in plan]
+        deselected = [id_ for id_, _ in held]
+        report({"event": event, "run": run, "pending": pending, "held": deselected})
+
+    def run_next(run: str) -> None:
+        entry = find(plan, run)
+        if entry is not None:
+            plan.remove(entry)
+            plan.insert(0, entry)
+            arrange("queued", run)
+
+    def toggle(run: str) -> None:
+        if stopping.is_set():
+            return
+        if (entry := find(plan, run)) is not None:
+            plan.remove(entry)
+            held.append(entry)
+            arrange("deselected", run)
+        elif (entry := find(held, run)) is not None:
+            held.remove(entry)
+            plan.append(entry)
+            arrange("reselected", run)
+            fill()
+
+    limit = concurrency
+    live: set[asyncio.Task[None]] = set()
+    started: list[asyncio.Task[None]] = []
+    idle = asyncio.Event()
+
+    def fill() -> None:
+        while plan and len(live) < limit:
+            task = asyncio.create_task(start(*plan.pop(0)))
+            task.add_done_callback(end)
+            live.add(task)
+            started.append(task)
+        if not live:
+            idle.set()
+
+    def end(task: asyncio.Task[None]) -> None:
+        live.remove(task)
+        fill()
+
+    def set_concurrency(count: int) -> None:
+        nonlocal limit
+        limit = max(1, count)
+        report({"event": "concurrency", "concurrency": limit})
+        fill()
+
+    async def start(run: str, params: dict[str, object]) -> None:
+        folder = run_folder(config.root, config.runs, name, run)
+        folder.mkdir(parents=True)
+        report({"event": "started", "run": run, "params": params})
+        result = await execute_run(
+            config.root, folder, experiment.command, params, running, stopping
+        )
+        report({"event": "ended", "run": run, **result})
+        results.append(result)
+
+    listen(Controls(interrupt, run_next, set_concurrency, toggle))
+    fill()
+    await idle.wait()
+    await asyncio.gather(*started)
+    return results

{labnotes-0.0.24 → labnotes-0.0.26}/src/labnotes/runner/run_headless.py

@@ -1,17 +1,17 @@
 import asyncio
 import signal
-from collections.abc import Callable
 
 import click
 
-from labnotes.models import Start
+from labnotes.models import Controls, Start
 from labnotes.runner.describe_event import describe_event
 from labnotes.runner.stop_then_kill import stop_then_kill
 
 
 def run_headless(start: Start) -> list[dict[str, object]]:
-    def listen(interrupt: Callable[[], None]) -> None:
-        asyncio.get_running_loop().add_signal_handler(signal.SIGINT, interrupt)
-        stop_then_kill(interrupt, signal.SIGTERM, signal.SIGHUP)
+    def listen(controls: Controls) -> None:
+        loop = asyncio.get_running_loop()
+        loop.add_signal_handler(signal.SIGINT, controls.interrupt)
+        stop_then_kill(controls.interrupt, signal.SIGTERM, signal.SIGHUP)
 
     return asyncio.run(start(lambda event: click.echo(describe_event(event)), listen))

{labnotes-0.0.24 → labnotes-0.0.26}/src/labnotes/tui/run_app.py

@@ -1,12 +1,11 @@
 import signal
 import time
-from collections.abc import Callable
 
 from textual.app import App, ComposeResult
 from textual.binding import Binding
 from textual.widgets import DataTable, Footer, Header, Static
 
-from labnotes.models import Start
+from labnotes.models import Controls, Start
 from labnotes.runner.describe_event import describe_event
 from labnotes.runner.stop_then_kill import stop_then_kill
 from labnotes.runner.param_args import param_args
@@ -19,6 +18,10 @@ COLUMNS = ("run", "params", "status", "time")
 class RunApp(App[None]):
     BINDINGS = [
         Binding("ctrl+c", "interrupt", "Stop runs"),
+        Binding("n", "run_next", "Run next"),
+        Binding("d", "toggle_run", "Deselect"),
+        Binding("plus,equals_sign", "concurrency(1)", "More at once", key_display="+"),
+        Binding("minus", "concurrency(-1)", "Fewer at once", key_display="-"),
         Binding("q", "quit", "Quit"),
     ]
 
@@ -27,14 +30,17 @@ class RunApp(App[None]):
         self.start = start
         self.title = title
         self.results: list[dict[str, object]] = []
-        self.interrupt: Callable[[], None] = lambda: None
+        self.controls = Controls(
+            lambda: None, lambda run: None, lambda count: None, lambda run: None
+        )
+        self.concurrency = 1
         self.done = False
         self.started: dict[str, float] = {}
         self.pending: set[str] = set()
 
     def compose(self) -> ComposeResult:
         yield Header()
-        yield DataTable(id="runs", cursor_type="none")
+        yield DataTable(id="runs", cursor_type="row")
         yield Static(id="status")
         yield Footer()
 
@@ -54,8 +60,8 @@ class RunApp(App[None]):
         passed = len(self.results) - failed
         self.show(f"{passed} passed, {failed} failed. q to quit.")
 
-    def listen(self, interrupt: Callable[[], None]) -> None:
-        self.interrupt = interrupt
+    def listen(self, controls: Controls) -> None:
+        self.controls = controls
         stop_then_kill(self.action_interrupt, signal.SIGTERM, signal.SIGHUP)
 
     def show_event(self, event: dict[str, object]) -> None:
@@ -68,6 +74,7 @@ class RunApp(App[None]):
                     cells = (planned["run"], args, "pending", "")
                     table.add_row(*map(dim, cells), key=planned["run"])
                     self.pending.add(planned["run"])
+                self.show_concurrency(event)
                 self.show(describe_event(event))
             case "started":
                 self.pending.discard(run)
@@ -81,6 +88,23 @@ class RunApp(App[None]):
                 table.update_cell(run, "status", outcome, update_width=True)
                 self.tick()
                 del self.started[run]
+            case "queued":
+                self.reorder(event)
+                table.move_cursor(row=table.get_row_index(run))
+                self.show(describe_event(event))
+            case "deselected":
+                self.pending.discard(run)
+                table.update_cell(run, "status", dim("deselected"))
+                self.reorder(event)
+                self.show(describe_event(event))
+            case "reselected":
+                self.pending.add(run)
+                table.update_cell(run, "status", dim("pending"))
+                self.reorder(event)
+                self.show(describe_event(event))
+            case "concurrency":
+                self.show_concurrency(event)
+                self.show(describe_event(event))
             case "stopping":
                 for pending in self.pending:
                     table.update_cell(pending, "status", dim("skipped"))
@@ -88,6 +112,18 @@ class RunApp(App[None]):
             case _:
                 self.show(describe_event(event))
 
+    def reorder(self, event: dict[str, object]) -> None:
+        table = self.query_one(DataTable)
+        waiting: list[str] = [*event["pending"], *event["held"]]  # type: ignore[misc]
+        for run in waiting:
+            row = table.get_row(run)
+            table.remove_row(run)
+            table.add_row(*row, key=run)
+
+    def show_concurrency(self, event: dict[str, object]) -> None:
+        self.concurrency = int(event["concurrency"])  # type: ignore[call-overload]
+        self.sub_title = f"{self.concurrency} at a time"
+
     def tick(self) -> None:
         table = self.query_one(DataTable)
         for run, began in self.started.items():
@@ -102,10 +138,28 @@ class RunApp(App[None]):
         if self.done:
             self.exit()
         else:
-            self.interrupt()
+            self.controls.interrupt()
 
     async def action_quit(self) -> None:
         if self.done:
             self.exit()
         else:
             self.show("Runs are still going. Ctrl-C stops them.")
+
+    def cursor_run(self) -> str | None:
+        table = self.query_one(DataTable)
+        if not table.row_count:
+            return None
+        cell = table.coordinate_to_cell_key(table.cursor_coordinate)
+        return str(cell.row_key.value)
+
+    def action_run_next(self) -> None:
+        if (run := self.cursor_run()) is not None:
+            self.controls.run_next(run)
+
+    def action_toggle_run(self) -> None:
+        if (run := self.cursor_run()) is not None:
+            self.controls.toggle(run)
+
+    def action_concurrency(self, change: int) -> None:
+        self.controls.set_concurrency(self.concurrency + change)

labnotes-0.0.24/src/labnotes/runner/run_experiment.py

@@ -1,76 +0,0 @@
-import asyncio
-import contextlib
-import json
-import os
-import signal
-
-from labnotes.models import ExperimentConfig, LabnotesConfig, Listen, Report
-from labnotes.runner.execute_run import execute_run
-from labnotes.runner.new_run_id import new_run_id
-from labnotes.runner.param_grid import param_grid
-from labnotes.runner.plan_runs import plan_runs
-from labnotes.runner.run_folder import run_folder
-from labnotes.tables.list_experiment_runs import list_experiment_runs
-from labnotes.tables.open_tables import open_tables
-
-
-async def run_experiment(
-    config: LabnotesConfig,
-    name: str,
-    experiment: ExperimentConfig,
-    concurrency: int,
-    report: Report,
-    listen: Listen,
-) -> list[dict[str, object]]:
-    existing = await list_experiment_runs(open_tables(config), name, experiment.counts)
-    finished = [json.loads(str(run["params"])) for run in existing if run["counted"]]
-    planned: set[str] = set()
-
-    def taken(run: str) -> bool:
-        return (
-            run in planned or run_folder(config.root, config.runs, name, run).exists()
-        )
-
-    plan: list[tuple[str, dict[str, object]]] = []
-    for params in plan_runs(
-        param_grid(experiment.params), experiment.repeats, finished
-    ):
-        run = new_run_id(taken)
-        planned.add(run)
-        plan.append((run, params))
-    runs = [{"run": run, "params": params} for run, params in plan]
-    report({"event": "planned", "runs": runs, "concurrency": concurrency})
-    running: set[asyncio.subprocess.Process] = set()
-    results: list[dict[str, object]] = []
-    stopping = asyncio.Event()
-
-    def signal_runs(signal_number: int) -> None:
-        for process in running:
-            with contextlib.suppress(ProcessLookupError):
-                os.killpg(process.pid, signal_number)
-
-    def interrupt() -> None:
-        plan.clear()
-        if not stopping.is_set():
-            stopping.set()
-            report({"event": "stopping"})
-            signal_runs(signal.SIGTERM)
-        elif running:
-            report({"event": "killing"})
-            signal_runs(signal.SIGKILL)
-
-    async def work() -> None:
-        while plan:
-            run, params = plan.pop(0)
-            folder = run_folder(config.root, config.runs, name, run)
-            folder.mkdir(parents=True)
-            report({"event": "started", "run": run, "params": params})
-            result = await execute_run(
-                config.root, folder, experiment.command, params, running, stopping
-            )
-            report({"event": "ended", "run": run, **result})
-            results.append(result)
-
-    listen(interrupt)
-    await asyncio.gather(*(work() for _ in range(concurrency)))
-    return results