labnotes 0.0.102__tar.gz → 0.0.103__tar.gz

{labnotes-0.0.102 → labnotes-0.0.103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.102
+Version: 0.0.103
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/context.md

@@ -117,6 +117,10 @@ going. Elsewhere, it prints a line per event instead.
   in `experiment.toml` sets it, then `concurrency` under `[experiments]` in
   `.labnotes.toml`, then 1.
 - `-e PATH` reads another file instead of `experiment.toml`.
+- `--timeout SECONDS` stops any one run still going after that many seconds:
+  SIGTERM, then SIGKILL 5 seconds later. Its
+  `run.json` gets `"error": "timed out after ..."` next to its exit code.
+  Unset, runs have no time limit.
 - `run` hands its runs to the project's labnotes daemon, starting it if
   needed, and shows or waits for them. The daemon listens on a socket in
   `/tmp/labnotes-<uid>/` and exits once nothing is running and no client has

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/daemon/daemon_app.py

@@ -102,6 +102,7 @@ def daemon_app(config: LabnotesConfig) -> Starlette:
                     concurrency,
                     lambda event: record({**event, "experiment": name}),
                     listen,
+                    body.get("timeout"),  # type: ignore[arg-type]
                 )
             # Clients hear why the session ended; the daemon keeps serving.
             except Exception as failure:  # noqa: BLE001

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/daemon/remote_start.py

@@ -15,6 +15,7 @@ def remote_start(
     name: str,
     experiment_config: Path | None,
     concurrency: int | None,
+    timeout: float | None = None,
 ) -> Start:
     async def start(report: Report, listen: Listen) -> list[dict[str, object]]:
         try:
@@ -44,6 +45,7 @@ def remote_start(
                     if experiment_config is None
                     else str(experiment_config.resolve()),
                     "concurrency": concurrency,
+                    "timeout": timeout,
                 },
             )
             if reply.status_code != 202:

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/experiments.py

@@ -40,6 +40,13 @@ from labnotes.tui.run_tui import run_tui
     type=click.Path(exists=True, dir_okay=False, path_type=Path),
     help="With run: read this file instead of the experiment's experiment.toml.",
 )
+@click.option(
+    "--timeout",
+    type=click.IntRange(min=1),
+    metavar="SECONDS",
+    help="With run: stop any run still going after this many seconds."
+    " Unset, runs have no time limit.",
+)
 @click.pass_context
 def experiments(
     ctx: click.Context,
@@ -48,6 +55,7 @@ def experiments(
     output_format: str | None,
     concurrency: int | None,
     experiment_config: Path | None,
+    timeout: int | None,
 ) -> None:
     config = load_config(ctx)
     if config is None:
@@ -63,7 +71,7 @@ def experiments(
             click.echo(f"Created {path / 'experiment.toml'}")
             return
         case str(), "run":
-            start = remote_start(config, name, experiment_config, concurrency)
+            start = remote_start(config, name, experiment_config, concurrency, timeout)
             try:
                 if sys.stdout.isatty() and sys.stderr.isatty():
                     results = run_tui(start, name)

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/load_run.py

@@ -10,7 +10,7 @@ async def load_run(
 ) -> dict[str, object] | None:
     rows = await tables.query(
         "SELECT runs.experiment, experiments.display_name, runs.id, runs.folder,"
-        " runs.created, runs.ended, runs.exit_code, runs.params"
+        " runs.created, runs.ended, runs.exit_code, runs.error, runs.params"
         " FROM runs JOIN experiments ON experiments.name = runs.experiment"
         f" WHERE runs.experiment = {quote_string(experiment)}"
         f" AND runs.id = {quote_string(run)}"

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/runner/execute_run.py

@@ -1,15 +1,18 @@
 import asyncio
 import os
 import time
+from collections.abc import Callable
 from pathlib import Path
 
 import psutil
 
 from labnotes.log_path import LOG_ENV
 from labnotes.runner.shell_command import shell_command
+from labnotes.runner.stop_then_kill import GRACE_SECONDS
 from labnotes.runner.wait_for_tree import wait_for_tree
 from labnotes.runner.write_run_json import write_run_json
 from labnotes.tables.iso_time import iso_time
+from labnotes.tui.format_elapsed import format_elapsed
 
 
 async def execute_run(
@@ -20,6 +23,8 @@ async def execute_run(
     running: set[asyncio.subprocess.Process],
     stopping: asyncio.Event,
     tree: set[psutil.Process],
+    timeout: float | None = None,
+    stop: Callable[[], None] = lambda: None,
 ) -> dict[str, object]:
     run: dict[str, object] = {
         "params": params,
@@ -50,14 +55,27 @@ async def execute_run(
             run["error"] = str(error)
         else:
             running.add(process)
+            exited = asyncio.ensure_future(process.wait())
+            kill_later = None
             try:
-                run["exit_code"] = await process.wait()
+                if (
+                    timeout is not None
+                    and not (await asyncio.wait({exited}, timeout=timeout))[0]
+                ):
+                    run["error"] = f"timed out after {format_elapsed(timeout)}"
+                    stop()
+                    kill_later = asyncio.get_running_loop().call_later(
+                        GRACE_SECONDS, stop
+                    )
+                run["exit_code"] = await exited
                 # The shell can exit on SIGTERM before what it started. Once
                 # stopping, the run stays killable until everything signalled
                 # is gone. A normal exit ends the run even if helpers linger.
-                if stopping.is_set():
+                if stopping.is_set() or kill_later is not None:
                     await wait_for_tree(tree)
             finally:
+                if kill_later is not None:
+                    kill_later.cancel()
                 running.discard(process)
     run["ended"] = iso_time(time.time())
     write_run_json(folder, run)

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/runner/run_experiment.py

@@ -32,6 +32,7 @@ async def run_experiment(
     concurrency: int,
     report: Report,
     listen: Listen,
+    timeout: float | None = None,
 ) -> list[dict[str, object]]:
     tables = open_tables(config)
     existing = await experiment_run_ids(tables, name)
@@ -203,6 +204,8 @@ async def run_experiment(
             running[run],
             stopping,
             trees[run],
+            timeout=timeout,
+            stop=lambda: kill(run),
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})

{labnotes-0.0.102 → labnotes-0.0.103}/src/labnotes/templates/run.html

@@ -5,7 +5,7 @@
 <section class="intro">
   <h1 class="mono">{{ run.id }}</h1>
   <p class="config-summary">
-    {% if run.ended is none %}<span class="chip run-state">running</span>{% elif run.exit_code is none %}<span class="chip run-state bad">ended</span>{% else %}<span class="chip run-state {{ 'good' if run.exit_code == 0 else 'bad' }}">exit {{ run.exit_code }}</span>{% endif %}
+    {% if run.ended is none %}<span class="chip run-state">running</span>{% elif run.error %}<span class="chip run-state bad">{{ run.error }}</span>{% elif run.exit_code is none %}<span class="chip run-state bad">ended</span>{% else %}<span class="chip run-state {{ 'good' if run.exit_code == 0 else 'bad' }}">exit {{ run.exit_code }}</span>{% endif %}
     {% if run.created %}<span class="setting">started <code>{{ run.created }}</code></span>{% endif %}
     {% for key, value in run.params.items() %}<span class="setting">{{ key }} <code>{{ value | value }}</code></span>{% endfor %}
   </p>