labnotes 0.0.69__tar.gz → 0.0.70__tar.gz

{labnotes-0.0.69 → labnotes-0.0.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.69
+Version: 0.0.70
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>
@@ -9,10 +9,12 @@ Requires-Python: >=3.12
 Requires-Dist: click>=8.5.0
 Requires-Dist: dirsql>=0.4.110
 Requires-Dist: haikunator>=2.1.0
+Requires-Dist: httpx2>=2.13.1
 Requires-Dist: ipykernel>=7.4.0
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: jupyter-client>=8.10.0
 Requires-Dist: markdown-it-py>=4.2.0
+Requires-Dist: psutil>=7.2.2
 Requires-Dist: rich>=14
 Requires-Dist: starlette>=1.8.0
 Requires-Dist: textual>=8.2.8

{labnotes-0.0.69 → labnotes-0.0.70}/pyproject.toml

@@ -14,10 +14,12 @@ dependencies = [
     "click>=8.5.0",
     "dirsql>=0.4.110",
     "haikunator>=2.1.0",
+    "httpx2>=2.13.1",
     "ipykernel>=7.4.0",
     "jinja2>=3.1.6",
     "jupyter-client>=8.10.0",
     "markdown-it-py>=4.2.0",
+    "psutil>=7.2.2",
     "rich>=14",
     "starlette>=1.8.0",
     "textual>=8.2.8",
@@ -33,11 +35,11 @@ labnotes = "labnotes.cli:main"
 [dependency-groups]
 dev = [
     "curtaincall>=0.3.0",
-    "httpx2>=2.13.1",
     "mypy>=2.4.0",
     "playwright>=1.64.0",
     "pytest>=8.0",
     "ruff==0.8.4",
+    "types-psutil>=7.2.2.20260906",
 ]
 
 [tool.hatch.version]

labnotes-0.0.70/src/labnotes/__main__.py

@@ -0,0 +1,3 @@
+from labnotes.cli import main
+
+main(prog_name="labnotes")

{labnotes-0.0.69 → labnotes-0.0.70}/src/labnotes/cli.py

@@ -3,10 +3,12 @@ from pathlib import Path
 import click
 
 from labnotes.context_guide import context_guide
+from labnotes.daemon_command import daemon_command
 from labnotes.experiments import experiments
 from labnotes.init import init
 from labnotes.load_config import load_config
 from labnotes.read_config import CONFIG_FILE
+from labnotes.run_command import run_command
 from labnotes.server import server
 
 
@@ -37,6 +39,8 @@ def context(ctx: click.Context) -> None:
     click.echo(context_guide(load_config(ctx), Path.cwd()), nl=False)
 
 
+main.add_command(daemon_command)
 main.add_command(experiments)
 main.add_command(init)
+main.add_command(run_command)
 main.add_command(server)

{labnotes-0.0.69 → labnotes-0.0.70}/src/labnotes/context.md

@@ -116,7 +116,15 @@ line per event instead.
   kills them. Closing the terminal or sending SIGTERM stops them the same way,
   then kills any still going 5 seconds later. Once the runs are done, either
   one closes the live view.
-- A second `run` of the same experiment fails while the first holds the lock.
+- Outside a terminal, `run` hands its runs to the project's labnotes daemon,
+  starting it if needed, and waits for them. The daemon listens on
+  `.labnotes.sock` in the project folder and exits once nothing is running and
+  no client has been connected for 15 seconds. Runs belong to the daemon, so
+  killing the waiting `run` outright leaves them going. Ctrl-C or SIGTERM to it
+  still stops them.
+- `labnotes run stop` stops every run the daemon has going: SIGTERM, then
+  SIGKILL 5 seconds later.
+- A second `run` of the same experiment fails while the first is going.
 - It exits 0 even if runs fail; the summary line counts failures.
 
 `uvx labnotes server` serves the experiments, their markdown files and their runs as a local site, read from the same `.labnotes.toml`. An experiment's page shows its `experiment.toml` on one line, then a Runs tab with a table of its runs and a Notebooks tab with one tab per markdown file.

labnotes-0.0.70/src/labnotes/daemon/daemon_app.py

@@ -0,0 +1,145 @@
+import asyncio
+import contextlib
+import dataclasses
+import json
+from collections.abc import AsyncIterator
+from pathlib import Path
+
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response, StreamingResponse
+from starlette.routing import Route
+
+from labnotes.models import Controls, LabnotesConfig
+from labnotes.runner.hold_folder import hold_folder
+from labnotes.runner.prepare_experiment import prepare_experiment
+from labnotes.runner.run_experiment import run_experiment
+from labnotes.runner.stop_then_kill import GRACE_SECONDS
+
+Event = dict[str, object]
+ACTIONS = frozenset(field.name for field in dataclasses.fields(Controls))
+
+
+def daemon_app(config: LabnotesConfig) -> Starlette:
+    sessions: dict[str, Controls | None] = {}
+    clients: set[asyncio.Queue[Event]] = set()
+    tasks: set[asyncio.Task[str | None]] = set()
+
+    def broadcast(event: Event) -> None:
+        for queue in clients:
+            queue.put_nowait(event)
+
+    def refuse(status: int, error: str) -> Response:
+        return JSONResponse({"error": error}, status_code=status)
+
+    async def events(request: Request) -> Response:
+        queue: asyncio.Queue[Event] = asyncio.Queue()
+
+        async def stream() -> AsyncIterator[str]:
+            clients.add(queue)
+            try:
+                event: Event = {"event": "connected"}
+                while True:
+                    yield f"data: {json.dumps(event, default=str)}\n\n"
+                    event = await queue.get()
+            finally:
+                clients.discard(queue)
+
+        return StreamingResponse(stream(), media_type="text/event-stream")
+
+    async def start(name: str, body: dict[str, object]) -> Response:
+        path = body.get("experiment_config")
+        try:
+            experiment, concurrency, runs = await prepare_experiment(
+                config,
+                name,
+                None if path is None else Path(str(path)),
+                body.get("concurrency"),  # type: ignore[arg-type]
+            )
+        except ValueError as error:
+            return refuse(400, str(error))
+        lock = contextlib.ExitStack()
+        try:
+            lock.enter_context(hold_folder(runs))
+        except BlockingIOError:
+            return refuse(409, f"{name} is already running ({runs} is held).")
+        ready = asyncio.Event()
+
+        def listen(controls: Controls) -> None:
+            sessions[name] = controls
+            ready.set()
+
+        async def session() -> str | None:
+            error = None
+            with lock:
+                try:
+                    await run_experiment(
+                        config,
+                        name,
+                        experiment,
+                        concurrency,
+                        lambda event: broadcast({**event, "experiment": name}),
+                        listen,
+                    )
+                # Clients hear why the session ended; the daemon keeps serving.
+                except Exception as failure:  # noqa: BLE001
+                    error = str(failure)
+            del sessions[name]
+            broadcast({"event": "done", "experiment": name, "error": error})
+            return error
+
+        task = asyncio.create_task(session())
+        tasks.add(task)
+        task.add_done_callback(tasks.discard)
+        waiting = asyncio.create_task(ready.wait())
+        await asyncio.wait({task, waiting}, return_when=asyncio.FIRST_COMPLETED)
+        waiting.cancel()
+        if not ready.is_set():
+            return refuse(500, str(task.result()))
+        return JSONResponse({}, status_code=202)
+
+    async def run(request: Request) -> Response:
+        name = request.path_params["name"]
+        body = await request.json()
+        if name in sessions:
+            return refuse(409, f"{name} is already running.")
+        sessions[name] = None
+        response = await start(name, body)
+        if response.status_code != 202:
+            sessions.pop(name, None)
+        return response
+
+    async def control(request: Request) -> Response:
+        name, action = request.path_params["name"], request.path_params["action"]
+        body = await request.json()
+        controls = sessions.get(name)
+        if action not in ACTIONS or controls is None:
+            return refuse(404, f"No {action} for {name}.")
+        getattr(controls, action)(*body["args"])
+        return JSONResponse({})
+
+    def stop_all() -> None:
+        stopping = [controls for controls in sessions.values() if controls]
+
+        def interrupt() -> None:
+            for controls in stopping:
+                controls.interrupt()
+
+        interrupt()
+        asyncio.get_running_loop().call_later(GRACE_SECONDS, interrupt)
+
+    async def stop(request: Request) -> Response:
+        stop_all()
+        return JSONResponse({})
+
+    app = Starlette(
+        routes=[
+            Route("/events", events),
+            Route("/stop", stop, methods=["POST"]),
+            Route("/experiments/{name}/run", run, methods=["POST"]),
+            Route("/experiments/{name}/{action}", control, methods=["POST"]),
+        ]
+    )
+    app.state.idle = lambda: not clients and not sessions
+    app.state.stop = stop_all
+    return app

labnotes-0.0.70/src/labnotes/daemon/daemon_client.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+import httpx2
+
+from labnotes.daemon.socket_path import socket_path
+
+
+def daemon_client(root: Path) -> httpx2.AsyncClient:
+    transport = httpx2.AsyncHTTPTransport(uds=str(socket_path(root)))
+    return httpx2.AsyncClient(
+        transport=transport, base_url="http://labnotes", timeout=None
+    )

labnotes-0.0.70/src/labnotes/daemon/daemon_running.py

@@ -0,0 +1,14 @@
+import asyncio
+from pathlib import Path
+
+from labnotes.daemon.socket_path import socket_path
+
+
+async def daemon_running(root: Path) -> bool:
+    try:
+        _, writer = await asyncio.open_unix_connection(socket_path(root))
+    except (FileNotFoundError, ConnectionRefusedError):
+        return False
+    writer.close()
+    await writer.wait_closed()
+    return True

labnotes-0.0.70/src/labnotes/daemon/daemon_server.py

@@ -0,0 +1,12 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+import uvicorn
+
+
+class DaemonServer(uvicorn.Server):
+    # A signal stops the runs through the daemon's own handlers; uvicorn's
+    # would shut the server down under them.
+    @contextmanager
+    def capture_signals(self) -> Iterator[None]:
+        yield

labnotes-0.0.70/src/labnotes/daemon/ensure_daemon.py

@@ -0,0 +1,26 @@
+import asyncio
+import subprocess
+import sys
+
+from labnotes.daemon.daemon_running import daemon_running
+from labnotes.models import LabnotesConfig
+
+START_SECONDS = 10
+
+
+async def ensure_daemon(config: LabnotesConfig) -> None:
+    if await daemon_running(config.root):
+        return
+    subprocess.Popen(
+        [sys.executable, "-m", "labnotes", "-c", str(config.path.resolve()), "daemon"],
+        cwd=config.root,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True,
+    )
+    for _ in range(START_SECONDS * 10):
+        await asyncio.sleep(0.1)
+        if await daemon_running(config.root):
+            return
+    raise ValueError(f"The labnotes daemon did not start in {START_SECONDS}s.")

labnotes-0.0.70/src/labnotes/daemon/exit_when_idle.py

@@ -0,0 +1,15 @@
+import asyncio
+from collections.abc import Callable
+
+import uvicorn
+
+# Long enough for a TUI restart or a mod reload to reconnect first.
+IDLE_EXIT_SECONDS = 15
+
+
+async def exit_when_idle(server: uvicorn.Server, idle: Callable[[], bool]) -> None:
+    quiet = 0
+    while quiet < IDLE_EXIT_SECONDS:
+        await asyncio.sleep(1)
+        quiet = quiet + 1 if idle() else 0
+    server.should_exit = True

labnotes-0.0.70/src/labnotes/daemon/remote_start.py

@@ -0,0 +1,69 @@
+import asyncio
+import dataclasses
+from collections.abc import Callable
+from pathlib import Path
+
+import httpx2
+
+from labnotes.daemon.daemon_client import daemon_client
+from labnotes.daemon.ensure_daemon import ensure_daemon
+from labnotes.models import Controls, LabnotesConfig, Listen, Report, Start
+
+
+def remote_start(
+    config: LabnotesConfig,
+    name: str,
+    experiment_config: Path | None,
+    concurrency: int | None,
+) -> Start:
+    async def start(report: Report, listen: Listen) -> list[dict[str, object]]:
+        await ensure_daemon(config)
+        sending: set[asyncio.Task[httpx2.Response]] = set()
+        results: list[dict[str, object]] = []
+        async with (
+            daemon_client(config.root) as client,
+            client.stream("GET", "/events") as response,
+        ):
+            events = aiter(httpx2.EventSource(response))
+            await anext(events)
+            reply = await client.post(
+                f"/experiments/{name}/run",
+                json={
+                    "experiment_config": None
+                    if experiment_config is None
+                    else str(experiment_config.resolve()),
+                    "concurrency": concurrency,
+                },
+            )
+            if reply.status_code != 202:
+                raise ValueError(reply.json()["error"])
+
+            def send(action: str) -> Callable[..., None]:
+                def post(*args: object) -> None:
+                    task = asyncio.create_task(
+                        client.post(
+                            f"/experiments/{name}/{action}", json={"args": list(args)}
+                        )
+                    )
+                    sending.add(task)
+                    task.add_done_callback(sending.discard)
+
+                return post
+
+            listen(Controls(*(send(f.name) for f in dataclasses.fields(Controls))))
+            async for message in events:
+                event: dict[str, object] = message.json()  # type: ignore[assignment]
+                if event.pop("experiment", None) != name:
+                    continue
+                match event:
+                    case {"event": "done", "error": None}:
+                        await asyncio.gather(*sending)
+                        return results
+                    case {"event": "done", "error": error}:
+                        raise ValueError(f"{name} stopped: {error}")
+                    case {"event": "ended"}:
+                        results.append(event)
+                report(event)
+        raise ValueError("Lost the labnotes daemon.")
+
+    return start

labnotes-0.0.70/src/labnotes/daemon/run_daemon.py

@@ -0,0 +1,27 @@
+import asyncio
+import signal
+
+import uvicorn
+
+from labnotes.daemon.daemon_app import daemon_app
+from labnotes.daemon.daemon_server import DaemonServer
+from labnotes.daemon.exit_when_idle import exit_when_idle
+from labnotes.daemon.socket_path import socket_path
+from labnotes.models import LabnotesConfig
+
+
+async def run_daemon(config: LabnotesConfig) -> None:
+    path = socket_path(config.root)
+    app = daemon_app(config)
+    server = DaemonServer(
+        uvicorn.Config(app, uds=str(path), lifespan="off", log_level="warning")
+    )
+    loop = asyncio.get_running_loop()
+    for signal_number in (signal.SIGTERM, signal.SIGHUP):
+        loop.add_signal_handler(signal_number, app.state.stop)
+    watcher = asyncio.create_task(exit_when_idle(server, app.state.idle))
+    try:
+        await server.serve()
+    finally:
+        watcher.cancel()
+        path.unlink(missing_ok=True)

labnotes-0.0.70/src/labnotes/daemon/serve_daemon.py

@@ -0,0 +1,16 @@
+import asyncio
+import contextlib
+
+from labnotes.daemon.run_daemon import run_daemon
+from labnotes.models import LabnotesConfig
+from labnotes.runner.hold_folder import hold_folder
+
+
+def serve_daemon(config: LabnotesConfig) -> None:
+    lock = contextlib.ExitStack()
+    try:
+        lock.enter_context(hold_folder(config.root))
+    except BlockingIOError:
+        return  # Another daemon already serves this project.
+    with lock:
+        asyncio.run(run_daemon(config))

labnotes-0.0.70/src/labnotes/daemon/socket_path.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+SOCKET_NAME = ".labnotes.sock"
+
+
+def socket_path(root: Path) -> Path:
+    return root / SOCKET_NAME

labnotes-0.0.70/src/labnotes/daemon/stop_runs.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+from labnotes.daemon.daemon_client import daemon_client
+from labnotes.daemon.daemon_running import daemon_running
+
+
+async def stop_runs(root: Path) -> bool:
+    if not await daemon_running(root):
+        return False
+    async with daemon_client(root) as client:
+        (await client.post("/stop")).raise_for_status()
+    return True

labnotes-0.0.70/src/labnotes/daemon_command.py

@@ -0,0 +1,14 @@
+import click
+
+from labnotes.daemon.serve_daemon import serve_daemon
+from labnotes.load_config import load_config
+from labnotes.read_config import CONFIG_FILE
+
+
+@click.command(name="daemon", hidden=True)
+@click.pass_context
+def daemon_command(ctx: click.Context) -> None:
+    config = load_config(ctx)
+    if config is None:
+        raise click.ClickException(f"No {CONFIG_FILE} here.")
+    serve_daemon(config)

{labnotes-0.0.69 → labnotes-0.0.70}/src/labnotes/experiments.py

@@ -5,6 +5,7 @@ from pathlib import Path
 import click
 
 from labnotes.create_experiment import create_experiment
+from labnotes.daemon.remote_start import remote_start
 from labnotes.load_config import load_config
 from labnotes.print_rows import print_rows
 from labnotes.read_config import CONFIG_FILE
@@ -62,12 +63,15 @@ def experiments(
             click.echo(f"Created {path / 'experiment.toml'}")
             return
         case str(), "run":
-            live = sys.stdout.isatty() and sys.stderr.isatty()
-            drive = run_tui if live else lambda start, title: run_headless(start)
             try:
-                results = start_experiment(
-                    config, name, experiment_config, concurrency, drive
-                )
+                if sys.stdout.isatty() and sys.stderr.isatty():
+                    results = start_experiment(
+                        config, name, experiment_config, concurrency, run_tui
+                    )
+                else:
+                    results = run_headless(
+                        remote_start(config, name, experiment_config, concurrency)
+                    )
             except ValueError as error:
                 raise click.ClickException(str(error)) from error
             failed = sum(result["exit_code"] != 0 for result in results)

labnotes-0.0.70/src/labnotes/run_command.py

@@ -0,0 +1,26 @@
+import asyncio
+
+import click
+
+from labnotes.daemon.stop_runs import stop_runs
+from labnotes.load_config import load_config
+from labnotes.read_config import CONFIG_FILE
+
+
+@click.group(name="run", help="Control the runs going in this project.")
+def run_command() -> None:
+    pass
+
+
+@run_command.command(
+    help="Stop every run. Scripts get SIGTERM, then SIGKILL 5 seconds later."
+)
+@click.pass_context
+def stop(ctx: click.Context) -> None:
+    config = load_config(ctx)
+    if config is None:
+        raise click.ClickException(
+            f"No {CONFIG_FILE} here. Run labnotes init, or pass -c PATH."
+        )
+    stopping = asyncio.run(stop_runs(config.root))
+    click.echo("Stopping runs." if stopping else "Nothing is running.")

{labnotes-0.0.69 → labnotes-0.0.70}/src/labnotes/runner/execute_run.py

@@ -2,8 +2,10 @@ import asyncio
 import time
 from pathlib import Path
 
+import psutil
+
 from labnotes.runner.shell_command import shell_command
-from labnotes.runner.wait_for_group import wait_for_group
+from labnotes.runner.wait_for_tree import wait_for_tree
 from labnotes.runner.write_run_json import write_run_json
 from labnotes.tables.iso_time import iso_time
 
@@ -15,6 +17,7 @@ async def execute_run(
     params: dict[str, object],
     running: set[asyncio.subprocess.Process],
     stopping: asyncio.Event,
+    tree: set[psutil.Process],
 ) -> dict[str, object]:
     run: dict[str, object] = {
         "params": params,
@@ -34,9 +37,6 @@ async def execute_run(
                 cwd=root,
                 stdout=stdout,
                 stderr=stderr,
-                # Own process group, so Ctrl-C reaches labnotes first and the
-                # runner decides how each run stops.
-                start_new_session=True,
             )
         except OSError as error:
             run["error"] = str(error)
@@ -45,10 +45,10 @@ async def execute_run(
             try:
                 run["exit_code"] = await process.wait()
                 # The shell can exit on SIGTERM before what it started. Once
-                # stopping, the run stays killable until its whole group is
-                # gone. A normal exit ends the run even if helpers linger.
+                # stopping, the run stays killable until everything signalled
+                # is gone. A normal exit ends the run even if helpers linger.
                 if stopping.is_set():
-                    await wait_for_group(process.pid)
+                    await wait_for_tree(tree)
             finally:
                 running.discard(process)
     run["ended"] = iso_time(time.time())

labnotes-0.0.70/src/labnotes/runner/prepare_experiment.py

@@ -0,0 +1,30 @@
+from pathlib import Path
+
+from labnotes.models import ExperimentConfig, LabnotesConfig
+from labnotes.runner.read_experiment_config import read_experiment_config
+from labnotes.runner.runs_parent import runs_parent
+from labnotes.tables.file_text import file_text
+
+
+async def prepare_experiment(
+    config: LabnotesConfig,
+    name: str,
+    experiment_config: Path | None,
+    concurrency: int | None,
+) -> tuple[ExperimentConfig, int, Path]:
+    if experiment_config is None:
+        folder = config.experiments.replace("{experiment}", name)
+        experiment_config = config.root / folder / "experiment.toml"
+    source = await file_text(config.root, experiment_config)
+    if source is None:
+        raise ValueError(f"No experiment.toml for {name} at {experiment_config}.")
+    experiment = read_experiment_config(
+        experiment_config, source, config.mode, config.concurrency
+    )
+    runs = runs_parent(config.root, config.runs, name)
+    runs.mkdir(parents=True, exist_ok=True)
+    return (
+        experiment,
+        experiment.concurrency if concurrency is None else concurrency,
+        runs,
+    )

{labnotes-0.0.69 → labnotes-0.0.70}/src/labnotes/runner/run_experiment.py

@@ -1,9 +1,9 @@
 import asyncio
-import contextlib
 import json
-import os
 import signal
 
+import psutil
+
 from labnotes.models import (
     Controls,
     ExperimentConfig,
@@ -17,6 +17,7 @@ from labnotes.runner.new_stamped_run_id import new_stamped_run_id
 from labnotes.runner.param_grid import param_grid
 from labnotes.runner.plan_runs import plan_runs
 from labnotes.runner.run_folder import run_folder
+from labnotes.runner.signal_tree import signal_tree
 from labnotes.tables.experiment_run_ids import experiment_run_ids
 from labnotes.tables.list_experiment_runs import list_experiment_runs
 from labnotes.tables.open_tables import open_tables
@@ -70,14 +71,14 @@ async def run_experiment(
         }
     )
     running: dict[str, set[asyncio.subprocess.Process]] = {}
+    trees: dict[str, set[psutil.Process]] = {}
     stopped: set[str] = set()
     results: list[dict[str, object]] = []
     stopping = asyncio.Event()
 
     def signal_run(run: str, signal_number: int) -> None:
         for process in running[run]:
-            with contextlib.suppress(ProcessLookupError):
-                os.killpg(process.pid, signal_number)
+            signal_tree(process.pid, signal_number, trees[run])
 
     def signal_runs(signal_number: int) -> None:
         for run in running:
@@ -183,6 +184,7 @@ async def run_experiment(
         folder.mkdir(parents=True)
         report({"event": "started", "run": run, "params": params, "folder": folder})
         running[run] = set()
+        trees[run] = set()
         result = await execute_run(
             config.root,
             folder,
@@ -190,8 +192,9 @@ async def run_experiment(
             params,
             running[run],
             stopping,
+            trees[run],
         )
-        del running[run]
+        del running[run], trees[run]
         report({"event": "ended", "run": run, **result})
         ended[run] = params
         results.append(result)