labnotes 0.2.18__tar.gz → 0.2.20__tar.gz

{labnotes-0.2.18 → labnotes-0.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.2.18
+Version: 0.2.20
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/daemon/server.py

@@ -61,6 +61,8 @@ class ExperimentDaemon:
             cast(str | None, options.get("teardown")),
             cast(str | None, options.get("command_line")),
             cast(str | None, options.get("run_id")),
+            cast(int | None, options.get("retries")),
+            bool(options.get("replace_capped")),
         )
         path = str(selected.run_state.path)
         invocation = Invocation()

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/experiments.py

@@ -60,7 +60,23 @@ class PassthroughCommand(click.Command):
     help="With run: how many runs at once. Overrides experiment.toml and"
     " .labnotes.toml; 1 if none sets it.",
 )
-@click.option("--run-id", help="With run: restart only this failed run by id.")
+@click.option(
+    "--run-id",
+    help="With run: restart only this failed run by id, even if out of retries.",
+)
+@click.option(
+    "--retries",
+    type=click.IntRange(min=0),
+    help="With run: how many times a failed run is tried again, on exit 75 or,"
+    " with continues, by a later run. Overrides experiment.toml and"
+    " .labnotes.toml; 3 if none sets it.",
+)
+@click.option(
+    "--replace-capped",
+    is_flag=True,
+    help="With run: start a fresh run for a cell whose run is out of retries."
+    " Unset, that cell stays short.",
+)
 @click.option(
     "--headless",
     is_flag=True,
@@ -124,6 +140,8 @@ def experiments(
     output_format: str | None,
     concurrency: int | None,
     run_id: str | None,
+    retries: int | None,
+    replace_capped: bool,
     headless: bool,
     experiment_config: Path | None,
     timeout: int | None,
@@ -191,6 +209,8 @@ def experiments(
                                 "teardown": teardown,
                                 "command_line": ctx.meta.get(COMMAND_LINE),
                                 "run_id": run_id,
+                                "retries": retries,
+                                "replace_capped": replace_capped,
                             },
                         },
                     )
@@ -220,6 +240,8 @@ def experiments(
                             teardown,
                             ctx.meta.get(COMMAND_LINE),
                             run_id,
+                            retries,
+                            replace_capped,
                         )
                     )
             except ValueError as error:
@@ -239,6 +261,8 @@ def experiments(
                 output_format is not None
                 or concurrency is not None
                 or run_id is not None
+                or retries is not None
+                or replace_capped
                 or experiment_config is not None
                 or timeout is not None
                 or setup is not None
@@ -261,6 +285,10 @@ def experiments(
         raise click.UsageError("-- ARGS needs run.")
     if run_id is not None:
         raise click.UsageError("--run-id needs run.")
+    if retries is not None:
+        raise click.UsageError("--retries needs run.")
+    if replace_capped:
+        raise click.UsageError("--replace-capped needs run.")
     if ctx.args:
         raise click.UsageError(f"Unexpected {ctx.args[0]}. --PARAM=VALUE needs run.")
     if action not in (None, "runs"):

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/models.py

@@ -65,3 +65,5 @@ class ExperimentConfig:
     teardown: str | None = None
     env: dict[str, str] = field(default_factory=dict)
     continues: bool = False
+    # A run option, not a config key: lets a run that used up `retries` free its slot.
+    replace_capped: bool = False

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/retries.py

@@ -1,4 +1,4 @@
 # EX_TEMPFAIL from sysexits.h: the only exit labnotes retries on its own.
 RETRY_EXIT_CODE = 75
-DEFAULT_RETRIES = 2
+DEFAULT_RETRIES = 3
 RETRY_SECONDS = 30.0

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/runner/describe_event.py

@@ -33,6 +33,10 @@ def describe_event(event: dict[str, object]) -> str:
         case "gave up":
             tried = f"after {event['retries']} retries"
             return f"{event['run']} gave up (exit {RETRY_EXIT_CODE}) {tried}."
+        case "capped":
+            cells = len(event["runs"])  # type: ignore[arg-type]
+            noun = "cell" if cells == 1 else "cells"
+            return f"{cells} {noun} gave up after {event['retries']} retries."
         case "retry cancelled":
             return f"{event['run']} won't retry."
         case "idle":

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/runner/prepare_experiment.py

@@ -17,6 +17,8 @@ async def prepare_experiment(
     args: list[str] | None = None,
     setup: str | None = None,
     teardown: str | None = None,
+    retries: int | None = None,
+    replace_capped: bool = False,
 ) -> tuple[ExperimentConfig, int]:
     if config.experiment not in (None, name):
         raise ValueError(
@@ -39,6 +41,10 @@ async def prepare_experiment(
         experiment = dataclasses.replace(experiment, setup=setup)
     if teardown is not None:
         experiment = dataclasses.replace(experiment, teardown=teardown)
+    if retries is not None:
+        experiment = dataclasses.replace(experiment, retries=retries)
+    if replace_capped:
+        experiment = dataclasses.replace(experiment, replace_capped=True)
     if only:
         check_subset(experiment.params, only)
     if args:

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/runner/run_experiment.py

@@ -84,6 +84,17 @@ async def run_experiment(
         for r in counted
         if r["counted"] or str(r.get("id")) in held_ids
     ]
+    resumable: list[Entry] = []
+    capped: list[Entry] = []
+    tries: dict[str, int] = {}
+    if experiment.continues:
+        for row in await list_resumable_runs(tables, name, experiment.counts):
+            entry = (str(row["id"]), json.loads(str(row["params"])))
+            tries[entry[0]] = int(str(row["invocations"]))
+            over = tries[entry[0]] > experiment.retries
+            (capped if over else resumable).append(entry)
+    if not experiment.replace_capped:
+        finished += [params for _, params in capped]
     planned: set[str] = set()
 
     def taken(run: str) -> bool:
@@ -112,12 +123,6 @@ async def run_experiment(
     def release(run: str) -> None:
         os.close(locks.pop(run))
 
-    resumable: list[Entry] = []
-    if experiment.continues:
-        resumable = [
-            (str(row["id"]), json.loads(str(row["params"])))
-            for row in await list_resumable_runs(tables, name, experiment.counts)
-        ]
     plan: list[Entry] = []
     target_folder: Path | None = None
     if run_id is not None:
@@ -155,6 +160,8 @@ async def run_experiment(
         run = new_stamped_run_id(taken) if resumed is None else resumed[0]
         planned.add(run)
         plan.append((run, params))
+    # A resumed run's earlier invocations spend the same retries as exit 75 does.
+    earlier = {run: tries[run] for run, _ in plan if run in tries and run != run_id}
     hooks = {
         "setup": experiment.setup,
         "teardown": experiment.teardown,
@@ -180,6 +187,9 @@ async def run_experiment(
     )
     for row in held:
         emit({"event": "held", "run": str(row["id"])})
+    gave_up = [run for run, params in capped if in_subset(params, only or {})]
+    if gave_up and run_id is None and not experiment.replace_capped:
+        emit({"event": "capped", "runs": gave_up, "retries": experiment.retries})
     running: dict[str, set[asyncio.subprocess.Process]] = {}
     trees: dict[str, set[psutil.Process]] = {}
     stopped: set[str] = set()
@@ -240,11 +250,11 @@ async def run_experiment(
         fill()
 
     def retry_later(run: str) -> None:
-        tried = attempts.get(run, 0)
+        tried = attempts.get(run, earlier.get(run, 0))
         if tried >= experiment.retries:
             emit({"event": "gave up", "run": run, "retries": tried})
             return
-        seconds = retry_seconds * 2**tried
+        seconds = retry_seconds * 2 ** (tried - earlier.get(run, 0))
 
         def due() -> None:
             del waiting[run]

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/runner/start_experiment.py

@@ -32,12 +32,23 @@ def start_experiment(
     teardown: str | None,
     command_line: str | None = None,
     run_id: str | None = None,
+    retries: int | None = None,
+    replace_capped: bool = False,
 ) -> ExperimentStart:
     state = RunState.create(config, name)
 
     async def start(report: Report, listen: Listen) -> list[dict[str, object]]:
         experiment, limit = await prepare_experiment(
-            config, name, experiment_config, concurrency, only, args, setup, teardown
+            config,
+            name,
+            experiment_config,
+            concurrency,
+            only,
+            args,
+            setup,
+            teardown,
+            retries,
+            replace_capped,
         )
         await finalize_lost_runs(config)
         with project_snapshot(config.root) as snapshot:

{labnotes-0.2.18 → labnotes-0.2.20}/src/labnotes/tables/list_resumable_runs.py

@@ -8,8 +8,13 @@ async def list_resumable_runs(
     tables: DirSQL, name: str, counts: str
 ) -> list[dict[str, object]]:
     # Ended badly: a nonzero or missing exit code, not merely uncounted.
+    # Invocations stopped by hand don't spend retries.
     return await tables.query(
-        f"SELECT id, params FROM runs WHERE experiment = {quote_string(name)}"
+        "SELECT id, params, CASE WHEN json_type(run, '$.invocations') = 'array'"
+        " THEN (SELECT count(*) FROM json_each(run, '$.invocations')"
+        " WHERE json_extract(value, '$.ended_by') IS NOT 'stop')"
+        " ELSE json_extract(run, '$.ended_by') IS NOT 'stop' END AS invocations"
+        f" FROM runs WHERE experiment = {quote_string(name)}"
         f" AND NOT coalesce(({counts}), false) AND NOT {UNFINISHED}"
         " AND (exit_code IS NULL OR exit_code <> 0)"
         " AND NOT coalesce(run ->> 'excluded', false) ORDER BY created, id"

labnotes-0.2.20/tests/perf/conftest.py

@@ -0,0 +1,17 @@
+import faulthandler
+from collections.abc import Iterator
+
+import pytest
+
+TIMEOUT_SECONDS = 120
+
+
+@pytest.fixture(autouse=True)
+def hard_timeout() -> Iterator[None]:
+    # A frozen event loop never reaches an assertion. The watchdog thread
+    # still fires, dumps every stack, and exits.
+    faulthandler.dump_traceback_later(TIMEOUT_SECONDS, exit=True)
+    try:
+        yield
+    finally:
+        faulthandler.cancel_dump_traceback_later()