labnotes 0.0.103__tar.gz → 0.0.104__tar.gz

{labnotes-0.0.103 → labnotes-0.0.104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.103
+Version: 0.0.104
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/context.md

@@ -92,14 +92,28 @@ A script run by labnotes should:
 - take each param as a `--name value` flag, parsing lists and tables from JSON;
 - take its output folder as an argument, passed in `command` as `{run_dir}`, and write everything it produces there;
 - exit non-zero when it fails, so the run counts as failed and runs again next time;
+- exit 75 (`EX_TEMPFAIL`) only when the run couldn't do its work for a reason outside the experiment, like no Docker network or no capacity, so labnotes retries it (see below). labnotes never retries any other exit on its own;
 - stop any background process it starts before it exits; the run ends when `command` exits.
 - flush its output as it goes (`print(..., flush=True)`, or `PYTHONUNBUFFERED=1`), so the live view shows it as it happens.
 
+A run that exits 75 is retried on its own, with the same params, in a new run
+folder whose `run.json` holds `"retry_of": "<run id>"`. The first retry waits 30
+seconds, and each one after waits twice as long. `retries` in `experiment.toml`
+sets how many (0 turns them off), then `retries` under `[experiments]` in
+`.labnotes.toml`, then 2. The failed run keeps its folder, logs and exit code,
+and the default `counts` rule skips it, so a retry fills its place rather than
+adding a repeat. When the retries run out, the last run keeps exit 75: it never
+got to test its params, so treat it as blocked, not as a result. Stopping the
+runs, or `x` on a run waiting to retry, cancels the retry. A run that was stopped never retries. The live view
+shows a waiting retry as `retry in 30s` and the end as `gave up (exit 75)`;
+`run status --json` reports `retrying`, `retried`, `gave up` and
+`retry cancelled` events.
+
 Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs --format json`; a run with no `ended` is still going.
 
 In a terminal, `run` shows a live table of runs (status and time) and stays
 open when they finish. Failed runs show their status in red; `f` hides or shows them, and `r` on one
-runs it again under a new id. `j` and `k` move the cursor, `g` and `G` jump to the
+runs it again under a new id, with `retry_of` set. `j` and `k` move the cursor, `g` and `G` jump to the
 first and last row. Runs not started yet show grayed as `pending`, and as
 `skipped` once stopped. Move the cursor to a pending run and press `n` to run it
 next. `m` grabs a pending run; `j` and `k` move it through the queue, Enter or `m`
@@ -140,7 +154,8 @@ going. Elsewhere, it prints a line per event instead.
   run to a place in the queue counted from 0,
   `set_concurrency` to a number, `interrupt` or `finish`.
 - A second `run` of the same experiment fails while the first is going.
-- It exits 0 even if runs fail; the summary line counts failures. If it can't
+- It exits 0 even if runs fail; the summary line counts failures. A run that
+  was retried counts as retried, not failed. If it can't
   start or reach the daemon, it prints the cause on one line and exits 1.
 
 `uvx labnotes server` serves the experiments, their markdown files and their runs as a local site, read from the same `.labnotes.toml`. An experiment's page shows its `experiment.toml` on one line, then a Runs tab with a table of its runs and a Notebooks tab with one tab per markdown file. Clicking a run opens its `stdout.log` and `stderr.log` together, followed as they're written until the run ends.

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/experiments.py

@@ -79,8 +79,15 @@ def experiments(
                     results = run_headless(start)
             except ValueError as error:
                 raise click.ClickException(str(error)) from error
-            failed = sum(result["exit_code"] != 0 for result in results)
-            click.echo(f"{len(results) - failed} passed, {failed} failed.")
+            superseded = {r["retry_of"] for r in results if "retry_of" in r}
+            retried = sum(result.get("run") in superseded for result in results)
+            failed = sum(
+                result["exit_code"] != 0 and result.get("run") not in superseded
+                for result in results
+            )
+            again = f" {retried} retried." if retried else ""
+            passed = len(results) - failed - retried
+            click.echo(f"{passed} passed, {failed} failed.{again}")
             return
     if action not in (None, "runs"):
         raise click.UsageError(f"Unknown action {action}. Use runs or run.")

labnotes-0.0.104/src/labnotes/is_non_negative_int.py

@@ -0,0 +1,2 @@
+def is_non_negative_int(value: object) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool) and value >= 0

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/models.py

@@ -5,6 +5,7 @@ from typing import Any, Protocol
 
 from labnotes.default_counts import DEFAULT_COUNTS
 from labnotes.modes import DEFAULT_MODE
+from labnotes.retries import DEFAULT_RETRIES
 
 Report = Callable[[dict[str, object]], None]
 
@@ -47,6 +48,7 @@ class LabnotesConfig:
     runs: str
     mode: str = DEFAULT_MODE
     concurrency: int = 1
+    retries: int = DEFAULT_RETRIES
     # Set when the config's folder is itself the one experiment.
     experiment: str | None = None
     # [experiment] in .labnotes.toml, under each experiment.toml.
@@ -62,3 +64,4 @@ class ExperimentConfig:
     counts: str = DEFAULT_COUNTS
     mode: str = DEFAULT_MODE
     concurrency: int = 1
+    retries: int = DEFAULT_RETRIES

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/read_config.py

@@ -2,10 +2,12 @@ import tomllib
 from pathlib import Path
 
 from labnotes.models import LabnotesConfig
+from labnotes.is_non_negative_int import is_non_negative_int
 from labnotes.is_positive_int import is_positive_int
 from labnotes.modes import DEFAULT_MODE, MODES
 from labnotes.runner.read_experiment_config import SETTINGS
 from labnotes.read_path import DEFAULT_PATHS, read_path
+from labnotes.retries import DEFAULT_RETRIES
 
 CONFIG_FILE = ".labnotes.toml"
 
@@ -45,12 +47,16 @@ def read_config(path: Path) -> LabnotesConfig:
         raise ValueError(
             f"{path}: [experiments] concurrency must be a positive integer"
         )
+    retries = experiments.get("retries", DEFAULT_RETRIES)
+    if not is_non_negative_int(retries):
+        raise ValueError(f"{path}: [experiments] retries must be 0 or more")
     return LabnotesConfig(
         path.resolve(),
         root,
         **paths,
         mode=mode,
         concurrency=concurrency,
+        retries=retries,
         experiment=root.name if single else None,
         defaults=defaults,
     )

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/read_path.py

@@ -16,7 +16,7 @@ PLACEHOLDERS = {
 # experiments is missing: one experiment has no folder to point at.
 SINGLE_PLACEHOLDERS: dict[str, set[str]] = {"files": set(), "runs": {"run"}}
 # Keys other readers own, so the path check lets them through.
-OTHER_KEYS = {"experiments": {"mode", "concurrency"}}
+OTHER_KEYS = {"experiments": {"mode", "concurrency", "retries"}}
 PLACEHOLDER = re.compile(r"\{([^}]*)\}")
 SINGLE = "experiment.toml sits next to .labnotes.toml, so this folder is the experiment"
 

labnotes-0.0.104/src/labnotes/retries.py

@@ -0,0 +1,4 @@
+# EX_TEMPFAIL from sysexits.h: the only exit labnotes retries on its own.
+RETRY_EXIT_CODE = 75
+DEFAULT_RETRIES = 2
+RETRY_SECONDS = 30.0

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/runner/describe_event.py

@@ -1,4 +1,6 @@
+from labnotes.retries import RETRY_EXIT_CODE
 from labnotes.runner.param_args import param_args
+from labnotes.tui.format_elapsed import format_elapsed
 
 
 def describe_event(event: dict[str, object]) -> str:
@@ -28,6 +30,15 @@ def describe_event(event: dict[str, object]) -> str:
             return f"Killing {event['run']}."
         case "retried":
             return f"{event['run']} retried as {event['retry']}."
+        case "retrying":
+            wait = format_elapsed(event["seconds"])  # type: ignore[arg-type]
+            attempt = f"{event['attempt']} of {event['retries']}"
+            return f"{event['run']} retries in {wait} (retry {attempt})."
+        case "gave up":
+            tried = f"after {event['retries']} retries"
+            return f"{event['run']} gave up (exit {RETRY_EXIT_CODE}) {tried}."
+        case "retry cancelled":
+            return f"{event['run']} won't retry."
         case "idle":
             return "No runs left."
         case "concurrency":

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/runner/execute_run.py

@@ -23,6 +23,7 @@ async def execute_run(
     running: set[asyncio.subprocess.Process],
     stopping: asyncio.Event,
     tree: set[psutil.Process],
+    retry_of: str | None = None,
     timeout: float | None = None,
     stop: Callable[[], None] = lambda: None,
 ) -> dict[str, object]:
@@ -33,6 +34,8 @@ async def execute_run(
         "exit_code": None,
         "error": None,
     }
+    if retry_of is not None:
+        run["retry_of"] = retry_of
     write_run_json(folder, run)
     with (
         (folder / "stdout.log").open("wb") as stdout,

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/runner/prepare_experiment.py

@@ -26,6 +26,7 @@ async def prepare_experiment(
         source or "",
         config.mode,
         config.concurrency,
+        config.retries,
         config.defaults,
     )
     return experiment, experiment.concurrency if concurrency is None else concurrency

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/runner/read_experiment_config.py

@@ -6,10 +6,20 @@ from typing import Any
 from labnotes.compose_experiment import compose_experiment
 from labnotes.default_counts import DEFAULT_COUNTS
 from labnotes.models import ExperimentConfig
+from labnotes.is_non_negative_int import is_non_negative_int
 from labnotes.is_positive_int import is_positive_int
 from labnotes.modes import MODES
 
-SETTINGS = {"name", "command", "repeats", "params", "counts", "mode", "concurrency"}
+SETTINGS = {
+    "name",
+    "command",
+    "repeats",
+    "params",
+    "counts",
+    "mode",
+    "concurrency",
+    "retries",
+}
 
 
 def read_experiment_config(
@@ -17,6 +27,7 @@ def read_experiment_config(
     source: str,
     mode: str,
     concurrency: int,
+    retries: int,
     defaults: dict[str, object] | None = None,
 ) -> ExperimentConfig:
     try:
@@ -55,4 +66,9 @@ def read_experiment_config(
     concurrency = values.get("concurrency", concurrency)
     if not is_positive_int(concurrency):
         raise ValueError(f"{path}: concurrency must be a positive integer")
-    return ExperimentConfig(command, repeats, params, name, counts, mode, concurrency)
+    retries = values.get("retries", retries)
+    if not is_non_negative_int(retries):
+        raise ValueError(f"{path}: retries must be 0 or more")
+    return ExperimentConfig(
+        command, repeats, params, name, counts, mode, concurrency, retries
+    )

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/runner/run_experiment.py

@@ -11,6 +11,7 @@ from labnotes.models import (
     Listen,
     Report,
 )
+from labnotes.retries import RETRY_EXIT_CODE, RETRY_SECONDS
 from labnotes.runner.execute_run import execute_run
 from labnotes.runner.new_param_run_id import new_param_run_id
 from labnotes.runner.new_stamped_run_id import new_stamped_run_id
@@ -33,6 +34,7 @@ async def run_experiment(
     report: Report,
     listen: Listen,
     timeout: float | None = None,
+    retry_seconds: float = RETRY_SECONDS,
 ) -> list[dict[str, object]]:
     tables = open_tables(config)
     existing = await experiment_run_ids(tables, name)
@@ -94,11 +96,19 @@ async def run_experiment(
             stopping.set()
             report({"event": "stopping"})
             signal_runs(signal.SIGTERM)
+            if waiting:
+                for run in list(waiting):
+                    cancel_retry(run)
+                fill()
         elif any(running.values()):
             report({"event": "killing"})
             signal_runs(signal.SIGKILL)
 
     def kill(run: str) -> None:
+        if run in waiting:
+            cancel_retry(run)
+            fill()
+            return
         if not running.get(run):
             return
         if run in stopped:
@@ -147,16 +157,54 @@ async def run_experiment(
             fill()
 
     ended: dict[str, dict[str, object]] = {}
+    parents: dict[str, str] = {}
+    attempts: dict[str, int] = {}
+    waiting: dict[str, asyncio.TimerHandle] = {}
 
-    def retry(run: str) -> None:
-        if stopping.is_set() or run not in ended:
-            return
+    def requeue(run: str, attempt: int) -> None:
         again = new_id(ended[run])
         planned.add(again)
+        parents[again] = run
+        attempts[again] = attempt
         plan.append((again, ended[run]))
         report({"event": "retried", "run": run, "retry": again, "params": ended[run]})
         fill()
 
+    def retry(run: str) -> None:
+        if stopping.is_set() or run not in ended:
+            return
+        if run in waiting:
+            waiting.pop(run).cancel()
+            requeue(run, attempts.get(run, 0) + 1)
+        else:
+            requeue(run, 0)
+
+    def retry_later(run: str) -> None:
+        tried = attempts.get(run, 0)
+        if tried >= experiment.retries:
+            report({"event": "gave up", "run": run, "retries": tried})
+            return
+        seconds = retry_seconds * 2**tried
+
+        def due() -> None:
+            del waiting[run]
+            requeue(run, tried + 1)
+
+        waiting[run] = asyncio.get_running_loop().call_later(seconds, due)
+        report(
+            {
+                "event": "retrying",
+                "run": run,
+                "attempt": tried + 1,
+                "retries": experiment.retries,
+                "seconds": seconds,
+            }
+        )
+
+    def cancel_retry(run: str) -> None:
+        waiting.pop(run).cancel()
+        report({"event": "retry cancelled", "run": run})
+
     limit = concurrency
     live: set[asyncio.Task[None]] = set()
     started: list[asyncio.Task[None]] = []
@@ -169,7 +217,7 @@ async def run_experiment(
             task.add_done_callback(end)
             live.add(task)
             started.append(task)
-        if not live:
+        if not (live or waiting):
             report({"event": "idle"})
             if finishing:
                 done.set()
@@ -177,7 +225,7 @@ async def run_experiment(
     def finish() -> None:
         nonlocal finishing
         finishing = True
-        if not (live or plan):
+        if not (live or plan or waiting):
             done.set()
 
     def end(task: asyncio.Task[None]) -> None:
@@ -204,6 +252,7 @@ async def run_experiment(
             running[run],
             stopping,
             trees[run],
+            parents.get(run),
             timeout=timeout,
             stop=lambda: kill(run),
         )
@@ -211,6 +260,10 @@ async def run_experiment(
         report({"event": "ended", "run": run, **result})
         ended[run] = params
         results.append(result)
+        if result["exit_code"] == RETRY_EXIT_CODE and not (
+            stopping.is_set() or run in stopped
+        ):
+            retry_later(run)
 
     listen(
         Controls(

{labnotes-0.0.103 → labnotes-0.0.104}/src/labnotes/tui/run_app.py

@@ -9,6 +9,7 @@ from textual.binding import Binding
 from textual.widgets import DataTable, Footer, Header, Input, Static
 
 from labnotes.models import Controls, Start
+from labnotes.retries import RETRY_EXIT_CODE
 from labnotes.runner.describe_event import describe_event
 from labnotes.runner.stop_then_kill import GRACE_SECONDS, stop_then_kill
 from labnotes.runner.param_args import param_args
@@ -75,6 +76,8 @@ class RunApp(App[None]):
         self.cells: dict[str, list[object]] = {}
         self.filter = ""
         self.failed: set[str] = set()
+        self.retried: set[str] = set()
+        self.waiting: dict[str, float] = {}
         self.hiding = False
         self.folders: dict[str, Path] = {}
         self.viewing: str | None = None
@@ -178,7 +181,24 @@ class RunApp(App[None]):
                 self.set_cell(run, "status", dim(PENDING))
                 self.reorder(event, table.cursor_row)
                 self.show(describe_event(event))
+            case "retrying":
+                self.waiting[run] = time.monotonic() + float(event["seconds"])  # type: ignore[arg-type]
+                self.tick()
+                self.show(describe_event(event))
+            case "gave up":
+                gave_up = f"{FAILED} gave up (exit {RETRY_EXIT_CODE})"
+                self.set_cell(run, "status", Text(gave_up, style="red"))
+                self.show(describe_event(event))
+            case "retry cancelled":
+                del self.waiting[run]
+                outcome = f"{FAILED} exited {RETRY_EXIT_CODE}"
+                self.set_cell(run, "status", Text(outcome, style="red"))
+                self.show(describe_event(event))
             case "retried":
+                if self.waiting.pop(run, None) is not None:
+                    outcome = f"{FAILED} exited {RETRY_EXIT_CODE}, retried"
+                    self.set_cell(run, "status", Text(outcome, style="red"))
+                self.retried.add(run)
                 again = str(event["retry"])
                 args = " ".join(param_args(event["params"]))  # type: ignore[arg-type]
                 self.cells[again] = [
@@ -200,9 +220,11 @@ class RunApp(App[None]):
                 self.show(describe_event(event))
 
     def show_summary(self) -> None:
-        failed = len(self.failed)
-        passed = len(self.ended) - failed
-        self.show(f"{passed} passed, {failed} failed. Ctrl-C to quit.")
+        retried = len(self.retried)
+        failed = len(self.failed - self.retried)
+        passed = len(self.ended) - failed - retried
+        again = f" {retried} retried." if retried else ""
+        self.show(f"{passed} passed, {failed} failed.{again} Ctrl-C to quit.")
 
     def reposition(self, event: dict[str, object], run: str) -> None:
         table = self.query_one(DataTable)
@@ -246,8 +268,13 @@ class RunApp(App[None]):
         self.query_one("#concurrency", Static).update(f"{self.concurrency} at a time")
 
     def tick(self) -> None:
+        now = time.monotonic()
         for run, began in self.started.items():
-            self.set_cell(run, "time", format_elapsed(time.monotonic() - began))
+            self.set_cell(run, "time", format_elapsed(now - began))
+        for run, due in self.waiting.items():
+            wait = format_elapsed(max(0, due - now))
+            outcome = f"{FAILED} exited {RETRY_EXIT_CODE}, retry in {wait}"
+            self.set_cell(run, "status", Text(outcome, style="yellow"))
 
     def spin(self) -> None:
         self.frame = (self.frame + 1) % len(SPINNER)

{labnotes-0.0.103 → labnotes-0.0.104}/testing-conventions.toml

@@ -63,3 +63,9 @@ path = "src/labnotes/tui/experiments_screen.py"
 rules = ["mutation"]
 lines = [36]
 reason = "WorkerState members are singletons, so is equals =="
+
+[[python.exempt]]
+path = "src/labnotes/runner/run_experiment.py"
+rules = ["mutation"]
+lines = [184, 263]
+reason = "a retry is only scheduled below the budget and each one adds one, so tried never passes retries: >= equals ==; both sides are small ints, which CPython caches, so is equals =="