labnotes 0.2.13__tar.gz → 0.2.14__tar.gz

{labnotes-0.2.13 → labnotes-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.2.13
+Version: 0.2.14
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.2.13 → labnotes-0.2.14}/src/labnotes/experiments.py

@@ -57,6 +57,7 @@ class PassthroughCommand(click.Command):
     help="With run: how many runs at once. Overrides experiment.toml and"
     " .labnotes.toml; 1 if none sets it.",
 )
+@click.option("--run-id", help="With run: restart only this failed run by id.")
 @click.option(
     "-e",
     "--experiment-config",
@@ -114,6 +115,7 @@ def experiments(
     action: str | None,
     output_format: str | None,
     concurrency: int | None,
+    run_id: str | None,
     experiment_config: Path | None,
     timeout: int | None,
     setup: str | None,
@@ -150,6 +152,10 @@ def experiments(
             click.echo(f"Created {path / 'experiment.toml'}")
             return
         case str(), "run":
+            if run_id is not None and (ctx.args or passthrough is not None):
+                raise click.UsageError(
+                    "--run-id uses the run's recorded params and -- ARGS."
+                )
             try:
                 only = parse_subset(ctx.args)
             except ValueError as error:
@@ -165,6 +171,7 @@ def experiments(
                 setup,
                 teardown,
                 ctx.meta.get(COMMAND_LINE),
+                run_id,
             )
             try:
                 if sys.stdout.isatty() and sys.stderr.isatty():
@@ -185,6 +192,8 @@ def experiments(
             return
     if passthrough is not None:
         raise click.UsageError("-- ARGS needs run.")
+    if run_id is not None:
+        raise click.UsageError("--run-id needs run.")
     if ctx.args:
         raise click.UsageError(f"Unexpected {ctx.args[0]}. --PARAM=VALUE needs run.")
     if action not in (None, "runs"):

{labnotes-0.2.13 → labnotes-0.2.14}/src/labnotes/runner/describe_event.py

@@ -20,6 +20,8 @@ def describe_event(event: dict[str, object]) -> str:
             return f"Stopping {event['run']}."
         case "killing run":
             return f"Killing {event['run']}."
+        case "held":
+            return f"{event['run']} held by another run."
         case "retried" if event["retry"] == event["run"]:
             return f"{event['run']} retried in its folder."
         case "retried":

{labnotes-0.2.13 → labnotes-0.2.14}/src/labnotes/runner/run_experiment.py

@@ -16,7 +16,7 @@ from labnotes.models import (
 )
 from labnotes.retries import RETRY_EXIT_CODE, RETRY_SECONDS
 from labnotes.runner.check_hook_placeholders import check_hook_placeholders
-from labnotes.runner.claim_run import claim_run
+from labnotes.runner.claim_run import LOCK_FILE, claim_run
 from labnotes.runner.execute_run import execute_run
 from labnotes.runner.fill_experiment_env import fill_experiment_env
 from labnotes.runner.fill_dirs import fill_dirs
@@ -29,6 +29,7 @@ from labnotes.runner.placeholders import PLACEHOLDERS
 from labnotes.runner.plan_runs import plan_runs
 from labnotes.runner.project_snapshot import ProjectSnapshot
 from labnotes.runner.run_folder import run_folder
+from labnotes.runner.run_locked import run_locked
 from labnotes.runner.signal_tree import signal_tree
 from labnotes.state_path import STATE_ENV
 from labnotes.tables.experiment_run_ids import experiment_run_ids
@@ -53,22 +54,41 @@ async def run_experiment(
     environment: dict[str, str] | None = None,
     snapshot: ProjectSnapshot | None = None,
     command_line: str | None = None,
+    run_id: str | None = None,
 ) -> list[dict[str, object]]:
     command_root = snapshot.root if snapshot is not None else config.root
     inherited = dict(os.environ) if environment is None else environment
     tables = open_tables(config)
     existing = await experiment_run_ids(tables, name)
     counted = await list_experiment_runs(tables, name, experiment.counts)
-    finished = [json.loads(str(r["params"])) for r in counted if r["counted"]]
+    held = [
+        r
+        for r in counted
+        if not r["counted"]
+        and r.get("folder") is not None
+        and (config.root / str(r["folder"]) / LOCK_FILE).exists()
+        and run_locked(config.root / str(r["folder"]))
+    ]
+    held_ids = {str(r["id"]) for r in held}
+    finished = [
+        json.loads(str(r["params"]))
+        for r in counted
+        if r["counted"] or str(r.get("id")) in held_ids
+    ]
     planned: set[str] = set()
 
     def taken(run: str) -> bool:
         return run in planned or run in existing
 
-    grid = [
-        cell for cell in param_grid(experiment.params) if in_subset(cell, only or {})
-    ]
-    cells = plan_runs(grid, experiment.repeats, finished)
+    if run_id is None:
+        grid = [
+            cell
+            for cell in param_grid(experiment.params)
+            if in_subset(cell, only or {})
+        ]
+        cells = plan_runs(grid, experiment.repeats, finished)
+    else:
+        cells = []
 
     locks: dict[str, int] = {}
 
@@ -81,8 +101,7 @@ async def run_experiment(
         return True
 
     def release(run: str) -> None:
-        if run in locks:
-            os.close(locks.pop(run))
+        os.close(locks.pop(run))
 
     resumable: list[Entry] = []
     if experiment.continues:
@@ -91,6 +110,30 @@ async def run_experiment(
             for row in await list_resumable_runs(tables, name, experiment.counts)
         ]
     plan: list[Entry] = []
+    target_folder: Path | None = None
+    if run_id is not None:
+        selected = next((row for row in counted if row["id"] == run_id), None)
+        if selected is None:
+            raise ValueError(f"No run {run_id} in {name}.")
+        if selected["ended"] is None or (
+            selected["exit_code"] == 0 and not selected["error"]
+        ):
+            raise ValueError(f"Run {run_id} has not failed.")
+        source = config.root / str(selected["folder"])
+        record = json.loads((source / "run.json").read_text())
+        args = record.get("args", [])
+        params = json.loads(str(selected["params"]))
+        lock = claim_run(source)
+        if lock is None:
+            raise ValueError(f"Run {run_id} is held by another run.")
+        locks[run_id] = lock
+        if experiment.continues:
+            target_folder = source
+            plan.append((run_id, params))
+        else:
+            new_id = new_stamped_run_id(taken)
+            planned.add(new_id)
+            plan.append((new_id, params))
     for params in cells:
         resumed = next(
             (
@@ -126,6 +169,8 @@ async def run_experiment(
             "concurrency": concurrency,
         }
     )
+    for row in held:
+        report({"event": "held", "run": str(row["id"])})
     running: dict[str, set[asyncio.subprocess.Process]] = {}
     trees: dict[str, set[psutil.Process]] = {}
     stopped: set[str] = set()
@@ -246,10 +291,11 @@ async def run_experiment(
         fill()
 
     async def start(run: str, params: dict[str, object]) -> None:
-        folder = run_folder(config.root, config.runs, name, run)
+        folder = target_folder or run_folder(config.root, config.runs, name, run)
         folder.mkdir(parents=True, exist_ok=True)
-        if experiment.continues:
-            claim(run)
+        if not claim(run):
+            report({"event": "held", "run": run})
+            return
         if snapshot is not None and snapshot.sha is not None:
             (folder / "git-sha.txt").write_text(snapshot.sha + "\n")
             (folder / "git-diff.patch").write_bytes(snapshot.diff or b"")

{labnotes-0.2.13 → labnotes-0.2.14}/src/labnotes/runner/start_experiment.py

@@ -2,7 +2,6 @@ from pathlib import Path
 
 from labnotes.models import LabnotesConfig, Listen, Report, Start
 from labnotes.runner.finalize_lost_runs import finalize_lost_runs
-from labnotes.runner.hold_experiment import hold_experiment
 from labnotes.runner.prepare_experiment import prepare_experiment
 from labnotes.runner.project_snapshot import project_snapshot
 from labnotes.runner.run_experiment import run_experiment
@@ -19,13 +18,14 @@ def start_experiment(
     setup: str | None,
     teardown: str | None,
     command_line: str | None = None,
+    run_id: str | None = None,
 ) -> Start:
     async def start(report: Report, listen: Listen) -> list[dict[str, object]]:
         experiment, limit = await prepare_experiment(
             config, name, experiment_config, concurrency, only, args, setup, teardown
         )
         await finalize_lost_runs(config)
-        with project_snapshot(config.root) as snapshot, hold_experiment(config, name):
+        with project_snapshot(config.root) as snapshot:
             return await run_experiment(
                 config,
                 name,
@@ -38,6 +38,7 @@ def start_experiment(
                 args=args,
                 snapshot=snapshot,
                 command_line=command_line,
+                run_id=run_id,
             )
 
     return start

{labnotes-0.2.13 → labnotes-0.2.14}/src/labnotes/tui/runs_browser.py

@@ -1,6 +1,9 @@
+import asyncio
+import os
+import sys
 from collections.abc import Mapping
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 from textual.app import ComposeResult
 from textual.binding import Binding
@@ -8,6 +11,8 @@ from textual.widget import Widget
 from textual.widgets import DataTable, Static
 
 from labnotes.runs_view_command import runs_view_command
+from labnotes.models import LabnotesConfig
+from labnotes.runner.claim_run import claim_run
 from labnotes.tui.fill_table import fill_table
 from labnotes.tui.run_output import RunOutput
 from labnotes.tui.run_panel import RunPanel
@@ -30,6 +35,7 @@ class RunsBrowser(Widget):
     BINDINGS = [
         Binding("enter", "open_row", "Open", priority=True),
         Binding("escape", "close_run", "Back"),
+        Binding("r", "restart_run", "Restart", priority=True),
     ]
 
     def __init__(self) -> None:
@@ -38,6 +44,7 @@ class RunsBrowser(Widget):
         self.view: RunsView | None = None
         self.experiment = Experiment(Path(), "", ())
         self.viewing: str | None = None
+        self.restart_status = ""
 
     def compose(self) -> ComposeResult:
         yield RunsTable(id="runs", cursor_type="row")
@@ -48,6 +55,8 @@ class RunsBrowser(Widget):
             return self.viewing is None
         if action == "close_run":
             return self.viewing is not None
+        if action == "restart_run":
+            return self.viewing is None
         return True
 
     def on_labnotes_changed(self, message: Changed) -> None:
@@ -103,11 +112,62 @@ class RunsBrowser(Widget):
         self.show_status()
 
     def show_status(self) -> None:
-        text = "" if self.view is None else self.view.status()
+        text = self.restart_status or ("" if self.view is None else self.view.status())
         status = self.query_one("#view-status", Static)
         status.update(text)
         status.display = bool(text) and self.viewing is None
 
+    def action_restart_run(self) -> None:
+        table = self.query_one(DataTable)
+        if not table.row_count:
+            return
+        folder = str(
+            table.coordinate_to_cell_key(table.cursor_coordinate).row_key.value
+        )
+        row = next(row for row in self.experiment.runs if str(row["folder"]) == folder)
+        run_id = str(row["id"])
+        if row.get("ended") is None or (
+            row.get("exit_code") == 0 and not row.get("error")
+        ):
+            self.restart_status = f"{run_id} has not failed."
+            self.show_status()
+            return
+        lock = claim_run(self.experiment.root / folder)
+        if lock is None:
+            self.restart_status = f"{run_id} is held by another run."
+            self.show_status()
+            return
+        os.close(lock)
+        self.restart_status = f"Restarting {run_id}..."
+        self.show_status()
+        self.run_worker(self.restart(run_id), exit_on_error=False)
+
+    async def restart(self, run_id: str) -> None:
+        config = cast(LabnotesConfig, getattr(self.app, "config"))
+        process = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-m",
+            "labnotes",
+            "-c",
+            str(config.path),
+            "experiments",
+            self.experiment.name,
+            "run",
+            "--run-id",
+            run_id,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        if process.returncode == 0:
+            self.restart_status = (
+                f"Restarted {run_id}. Select the run to view its output."
+            )
+        else:
+            detail = stderr.decode().strip() or stdout.decode().strip()
+            self.restart_status = f"Could not restart {run_id}: {detail}"
+        self.show_status()
+
     async def action_open_row(self) -> None:
         table = self.query_one(DataTable)
         if not table.row_count:

{labnotes-0.2.13 → labnotes-0.2.14}/testing-conventions.toml

@@ -19,7 +19,7 @@ reason = "WorkerState members are singletons, so is equals =="
 [[python.exempt]]
 path = "src/labnotes/runner/run_experiment.py"
 rules = ["mutation"]
-lines = [190, 297]
+lines = [235, 343]
 reason = "a retry is only scheduled below the budget and each one adds one, so tried never passes retries: >= equals ==; both sides are small ints, which CPython caches, so is equals =="
 
 [[python.exempt]]
@@ -31,7 +31,7 @@ reason = "exit_code is non-negative here; when bit 7 is set, - 128 equals ^ 128,
 [[python.exempt]]
 path = "src/labnotes/experiments.py"
 rules = ["mutation"]
-lines = [120]
+lines = [213]
 reason = "click hands back the \"json\" choice literal itself, so is not equals !="
 
 [[python.exempt]]