labnotes 0.2.16__tar.gz → 0.2.18__tar.gz

{labnotes-0.2.16 → labnotes-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.2.16
+Version: 0.2.18
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.2.16 → labnotes-0.2.18}/src/labnotes/cli.py

@@ -4,6 +4,7 @@ import click
 
 from labnotes.command_line_group import CommandLineGroup
 from labnotes.context_guide import context_guide
+from labnotes.daemon_command import daemon
 from labnotes.experiments import experiments
 from labnotes.init import init
 from labnotes.load_config import load_config
@@ -42,3 +43,4 @@ def context(ctx: click.Context) -> None:
 main.add_command(experiments)
 main.add_command(init)
 main.add_command(server)
+main.add_command(daemon)

{labnotes-0.2.16 → labnotes-0.2.18}/src/labnotes/create_app.py

@@ -63,7 +63,7 @@ def create_app(config: LabnotesConfig) -> Starlette:
             await kernel.stop()
 
     async def index(request: Request) -> Response:
-        experiments = await list_experiments(request.app.state.tables)
+        experiments = await list_experiments(request.app.state.tables, config)
         return templates.TemplateResponse(
             request,
             "index.html",

labnotes-0.2.18/src/labnotes/daemon/__init__.py

@@ -0,0 +1,3 @@
+"""One local scheduler process per experiment and labnotes version."""
+
+__all__: list[str] = []

labnotes-0.2.18/src/labnotes/daemon/client.py

@@ -0,0 +1,35 @@
+"""Read the socket record and confirm its daemon owns the lock."""
+
+import fcntl
+import os
+from pathlib import Path
+
+from labnotes.daemon.socket_path import lock_path, socket_record
+from labnotes.models import LabnotesConfig
+
+
+def daemon_socket(config: LabnotesConfig, name: str) -> Path | None:
+    """Return a socket only while its daemon owns the version-keyed lock."""
+    record = socket_record(config, name)
+    try:
+        path = Path(record.read_text().strip())
+    except FileNotFoundError:
+        return None
+    lock = lock_path(config, name)
+    try:
+        descriptor = os.open(lock, os.O_RDONLY)
+    except FileNotFoundError:
+        path.unlink(missing_ok=True)
+        record.unlink(missing_ok=True)
+        return None
+    try:
+        try:
+            fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            return path
+        fcntl.flock(descriptor, fcntl.LOCK_UN)
+    finally:
+        os.close(descriptor)
+    path.unlink(missing_ok=True)
+    record.unlink(missing_ok=True)
+    return None

labnotes-0.2.18/src/labnotes/daemon/command.py

@@ -0,0 +1,32 @@
+"""Exchange one JSON command and one reply with a live daemon."""
+
+import json
+import socket
+
+from labnotes.daemon.client import daemon_socket
+from labnotes.daemon.socket_path import socket_record
+from labnotes.models import LabnotesConfig
+
+
+def command(
+    config: LabnotesConfig, name: str, message: dict[str, object]
+) -> dict[str, object]:
+    """Send a request and read exactly one reply from the live daemon."""
+    path = daemon_socket(config, name)
+    if path is None:
+        raise ValueError(f"No daemon is running for {name}.")
+    try:
+        with socket.socket(socket.AF_UNIX) as client:
+            client.settimeout(10)
+            client.connect(str(path))
+            client.sendall((json.dumps(message) + "\n").encode())
+            with client.makefile("r") as replies:
+                line = replies.readline()
+    except OSError as error:
+        path.unlink(missing_ok=True)
+        socket_record(config, name).unlink(missing_ok=True)
+        raise ValueError(f"No daemon is running for {name}.") from error
+    if not line:
+        raise ValueError(f"Daemon for {name} closed without a reply.")
+    reply: dict[str, object] = json.loads(line)
+    return reply

labnotes-0.2.18/src/labnotes/daemon/ensure.py

@@ -0,0 +1,46 @@
+"""Start a detached experiment daemon when no live owner exists."""
+
+import asyncio
+import subprocess
+import sys
+
+from labnotes.daemon.client import daemon_socket
+from labnotes.daemon.socket_path import discovery_folder
+from labnotes.models import LabnotesConfig
+
+START_SECONDS = 10
+
+
+async def ensure_daemon(config: LabnotesConfig, name: str) -> None:
+    """Return once a detached daemon holds the experiment's socket."""
+    if daemon_socket(config, name) is not None:
+        return
+    folder = discovery_folder(config, name)
+    folder.mkdir(parents=True, exist_ok=True)
+    errors = folder / "daemon.stderr"
+    errors.touch(mode=0o600, exist_ok=True)
+    with errors.open("ab") as stream:
+        process = subprocess.Popen(
+            [
+                sys.executable,
+                "-m",
+                "labnotes",
+                "-c",
+                str(config.path.resolve()),
+                "daemon",
+                name,
+            ],
+            cwd=config.root,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+            stderr=stream,
+            start_new_session=True,
+        )
+    for _ in range(START_SECONDS * 10):
+        if daemon_socket(config, name) is not None:
+            return
+        if (status := process.poll()) is not None and status != 0:
+            detail = errors.read_text().strip()
+            raise ValueError(detail or f"Daemon for {name} exited during startup.")
+        await asyncio.sleep(0.1)
+    raise ValueError(f"Daemon for {name} did not start in {START_SECONDS}s.")

labnotes-0.2.18/src/labnotes/daemon/marker.py

@@ -0,0 +1,27 @@
+"""Read the running-cell count from run folders for a live daemon."""
+
+from dirsql import DirSQL
+
+from labnotes.daemon.client import daemon_socket
+from labnotes.models import LabnotesConfig
+from labnotes.runner.claim_run import LOCK_FILE
+from labnotes.runner.run_locked import run_locked
+from labnotes.sql_quote import quote_string
+
+
+async def daemon_marker(
+    tables: DirSQL, config: LabnotesConfig, name: str
+) -> dict[str, object]:
+    """Mark a live owner and count its unfinished cells from disk-backed rows."""
+    if daemon_socket(config, name) is None:
+        return {"daemon": False, "running_cells": 0}
+    rows = await tables.query(
+        "SELECT folder FROM runs"
+        f" WHERE experiment = {quote_string(name)} AND ended IS NULL"
+    )
+    running = 0
+    for row in rows:
+        folder = config.root / str(row["folder"])
+        if (folder / LOCK_FILE).exists() and run_locked(folder):
+            running += 1
+    return {"daemon": True, "running_cells": running}

labnotes-0.2.18/src/labnotes/daemon/server.py

@@ -0,0 +1,237 @@
+"""Own experiment invocations behind a private request/response socket."""
+
+import asyncio
+import fcntl
+import json
+import os
+import signal
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+from labnotes.daemon.socket_path import lock_path, socket_path, socket_record
+from labnotes.models import Controls, LabnotesConfig
+from labnotes.runner.start_experiment import start_experiment
+
+IDLE_SECONDS = 5.0
+START_WAIT_SECONDS = 30.0
+
+
+@dataclass
+class Invocation:
+    """Keep controls with the one invocation that produced a journal."""
+
+    task: asyncio.Task[None] | None = None
+    controls: Controls | None = None
+    stop_requested: bool = False
+
+
+class ExperimentDaemon:
+    """Serve commands while cells run or a client remains attached."""
+
+    def __init__(self, config: LabnotesConfig, name: str) -> None:
+        self.config = config
+        self.name = name
+        self.active: dict[str, Invocation] = {}
+        self.clients = 0
+        self.closing = False
+        self.last_busy = time.monotonic()
+
+    def stop_all(self) -> None:
+        """Ask every invocation to stop, including one still preparing."""
+        for invocation in self.active.values():
+            invocation.stop_requested = True
+            if invocation.controls is not None:
+                invocation.controls.interrupt()
+
+    async def start(self, options: dict[str, object]) -> dict[str, object]:
+        """Start one independent invocation and return its journal path."""
+        selected = start_experiment(
+            self.config,
+            self.name,
+            Path(str(options["experiment_config"]))
+            if options.get("experiment_config") is not None
+            else None,
+            cast(int | None, options.get("concurrency")),
+            cast(int | None, options.get("timeout")),
+            cast(dict[str, list[str]] | None, options.get("only")),
+            cast(list[str] | None, options.get("args")),
+            cast(str | None, options.get("setup")),
+            cast(str | None, options.get("teardown")),
+            cast(str | None, options.get("command_line")),
+            cast(str | None, options.get("run_id")),
+        )
+        path = str(selected.run_state.path)
+        invocation = Invocation()
+        self.active[path] = invocation
+        ready: asyncio.Future[None] = asyncio.get_running_loop().create_future()
+
+        def listen(controls: Controls) -> None:
+            invocation.controls = controls
+            controls.finish()
+            if invocation.stop_requested:
+                controls.interrupt()
+            if not ready.done():
+                ready.set_result(None)
+
+        async def run() -> None:
+            try:
+                await selected(lambda event: None, listen)
+            except Exception as error:
+                if not ready.done():
+                    ready.set_exception(error)
+                else:
+                    selected.run_state.record(
+                        {"event": "daemon error", "error": str(error)}
+                    )
+            finally:
+                self.active.pop(path, None)
+
+        task = asyncio.create_task(run())
+        invocation.task = task
+        try:
+            await asyncio.wait_for(ready, timeout=START_WAIT_SECONDS)
+        except TimeoutError:
+            task.cancel()
+            await asyncio.gather(task, return_exceptions=True)
+            return {"ok": False, "error": "Experiment preparation timed out"}
+        except ValueError as error:
+            return {"ok": False, "error": str(error)}
+        return {"ok": True, "journal": path}
+
+    async def dispatch(self, message: dict[str, object]) -> dict[str, object]:
+        """Apply one command and provide one success or error reply."""
+        action = message.get("command")
+        if action == "ping":
+            return {"ok": True}
+        if action in ("start", "restart"):
+            options = message.get("options", {})
+            if not isinstance(options, dict):
+                return {"ok": False, "error": "options must be an object"}
+            if action == "restart" and "run_id" not in options:
+                return {"ok": False, "error": "restart needs run_id"}
+            return await self.start(options)
+        if action == "stop_all":
+            self.stop_all()
+            return {"ok": True}
+        if action == "stop_one":
+            run = message.get("run")
+            if not isinstance(run, str):
+                return {"ok": False, "error": "stop_one needs a run id"}
+            for invocation in self.active.values():
+                if invocation.controls is not None:
+                    invocation.controls.kill(run)
+            return {"ok": True}
+        if action == "set_concurrency":
+            count = message.get("count")
+            if not isinstance(count, int) or count < 1:
+                return {"ok": False, "error": "count must be positive"}
+            journal = message.get("journal")
+            targets = (
+                [self.active[str(journal)]]
+                if journal is not None and str(journal) in self.active
+                else list(self.active.values())
+                if journal is None
+                else []
+            )
+            if not targets:
+                return {"ok": False, "error": "No active invocation"}
+            for invocation in targets:
+                if invocation.controls is not None:
+                    invocation.controls.set_concurrency(count)
+            return {"ok": True}
+        if action == "attach":
+            return {"ok": True}
+        return {"ok": False, "error": f"Unknown command {action}"}
+
+    async def handle(
+        self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+    ) -> None:
+        """Hold an attach connection open without streaming events to it."""
+        self.clients += 1
+        try:
+            line = await reader.readline()
+            if not line:
+                return
+            message = json.loads(line)
+            if not isinstance(message, dict):
+                reply = {"ok": False, "error": "Request must be an object"}
+            else:
+                try:
+                    reply = await self.dispatch(message)
+                except Exception as error:
+                    # A request always gets one error reply, even on internal failures.
+                    reply = {"ok": False, "error": str(error)}
+            writer.write((json.dumps(reply) + "\n").encode())
+            await writer.drain()
+            if isinstance(message, dict) and message.get("command") == "attach":
+                await reader.read()
+        except (json.JSONDecodeError, asyncio.LimitOverrunError):
+            writer.write(b'{"ok":false,"error":"Invalid request"}\n')
+            await writer.drain()
+        finally:
+            self.clients -= 1
+            writer.close()
+            await writer.wait_closed()
+
+    async def wait_until_idle(self) -> None:
+        """Leave a short window for another TUI after the last cell ends."""
+        while True:
+            if self.closing:
+                return
+            if self.active or self.clients:
+                self.last_busy = time.monotonic()
+            elif time.monotonic() - self.last_busy >= IDLE_SECONDS:
+                return
+            await asyncio.sleep(0.2)
+
+
+async def serve_daemon(config: LabnotesConfig, name: str) -> None:
+    """Hold the experiment/version lock for one Unix server lifetime."""
+    lock = lock_path(config, name)
+    lock.parent.mkdir(parents=True, exist_ok=True)
+    lock.touch(mode=0o600, exist_ok=True)
+    descriptor = os.open(lock, os.O_RDONLY)
+    try:
+        for _ in range(10):
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                await asyncio.sleep(0.1)
+            else:
+                break
+        else:
+            return
+        path = socket_path(config, name)
+        path.unlink(missing_ok=True)
+        daemon = ExperimentDaemon(config, name)
+        server = await asyncio.start_unix_server(daemon.handle, path=str(path))
+        path.chmod(0o600)
+        record = socket_record(config, name)
+        pending = record.with_suffix(".socket.tmp")
+        pending.write_text(str(path), encoding="utf-8")
+        pending.chmod(0o600)
+        pending.replace(record)
+        loop = asyncio.get_running_loop()
+
+        def close() -> None:
+            daemon.closing = True
+            daemon.stop_all()
+
+        loop.add_signal_handler(signal.SIGTERM, close)
+        loop.add_signal_handler(signal.SIGINT, close)
+        loop.add_signal_handler(signal.SIGHUP, lambda: None)
+        try:
+            await daemon.wait_until_idle()
+        finally:
+            server.close()
+            await server.wait_closed()
+            daemon.stop_all()
+            await asyncio.gather(
+                *(item.task for item in daemon.active.values() if item.task is not None)
+            )
+            record.unlink(missing_ok=True)
+            path.unlink(missing_ok=True)
+    finally:
+        os.close(descriptor)

labnotes-0.2.18/src/labnotes/daemon/socket_path.py

@@ -0,0 +1,51 @@
+"""Locate the private Unix socket and its on-disk discovery record."""
+
+import hashlib
+import os
+import tempfile
+from importlib.metadata import version
+from pathlib import Path
+
+from labnotes.models import LabnotesConfig
+from labnotes.runner.run_state import journal_folder
+from labnotes.runner.runs_parent import runs_parent
+
+
+def daemon_key(config: LabnotesConfig, name: str) -> str:
+    """Key an experiment by its resolved path and installed package version."""
+    folder = config.root / config.experiments.replace("{experiment}", name)
+    identity = f"{folder.resolve()}\0{version('labnotes')}"
+    return hashlib.sha256(identity.encode()).hexdigest()[:16]
+
+
+def runtime_folder() -> Path:
+    """Keep socket names short even when the experiment path is deep."""
+    runtime = os.environ.get("XDG_RUNTIME_DIR")
+    folder = (
+        Path(runtime) / "labnotes"
+        if runtime
+        else Path(tempfile.gettempdir()) / f"labnotes-{os.getuid()}"
+    )
+    folder.mkdir(mode=0o700, parents=True, exist_ok=True)
+    folder.chmod(0o700)
+    return folder
+
+
+def socket_path(config: LabnotesConfig, name: str) -> Path:
+    """Return the version-keyed bind path outside the experiment tree."""
+    return runtime_folder() / f"{daemon_key(config, name)}.sock"
+
+
+def discovery_folder(config: LabnotesConfig, name: str) -> Path:
+    """Find daemon metadata alongside the experiment's run journals."""
+    return journal_folder(runs_parent(config.root, config.runs, name))
+
+
+def socket_record(config: LabnotesConfig, name: str) -> Path:
+    """Point clients to the current socket without guessing runtime settings."""
+    return discovery_folder(config, name) / f"{daemon_key(config, name)}.socket"
+
+
+def lock_path(config: LabnotesConfig, name: str) -> Path:
+    """Serialize one daemon per experiment and package version."""
+    return discovery_folder(config, name) / f"{daemon_key(config, name)}.lock"

labnotes-0.2.18/src/labnotes/daemon_command.py

@@ -0,0 +1,20 @@
+"""Internal CLI entry point for a detached experiment daemon."""
+
+import asyncio
+
+import click
+
+from labnotes.daemon.server import serve_daemon
+from labnotes.load_config import load_config
+from labnotes.read_config import CONFIG_FILE
+
+
+@click.command(hidden=True)
+@click.argument("name")
+@click.pass_context
+def daemon(ctx: click.Context, name: str) -> None:
+    """Serve one experiment from a detached process."""
+    config = load_config(ctx)
+    if config is None:
+        raise click.ClickException(f"No {CONFIG_FILE} here.")
+    asyncio.run(serve_daemon(config, name))

{labnotes-0.2.16 → labnotes-0.2.18}/src/labnotes/experiments.py

@@ -6,7 +6,10 @@ import click
 
 from labnotes.command_line_group import COMMAND_LINE
 from labnotes.create_experiment import create_experiment
+from labnotes.daemon.command import command as daemon_command
+from labnotes.daemon.ensure import ensure_daemon
 from labnotes.runner.start_experiment import start_experiment
+from labnotes.runner.run_state import RunState
 from labnotes.utils.format_bytes import format_bytes
 from labnotes.load_config import load_config
 from labnotes.parse_params import parse_params
@@ -43,7 +46,7 @@ class PassthroughCommand(click.Command):
     context_settings={"ignore_unknown_options": True, "allow_extra_args": True},
 )
 @click.argument("name", required=False)
-@click.argument("action", required=False, metavar="[runs|run]")
+@click.argument("action", required=False, metavar="[runs|run|stop]")
 @click.option(
     "--format",
     "output_format",
@@ -58,6 +61,11 @@ class PassthroughCommand(click.Command):
     " .labnotes.toml; 1 if none sets it.",
 )
 @click.option("--run-id", help="With run: restart only this failed run by id.")
+@click.option(
+    "--headless",
+    is_flag=True,
+    help="With run: keep the scheduler in this foreground process.",
+)
 @click.option(
     "-e",
     "--experiment-config",
@@ -116,6 +124,7 @@ def experiments(
     output_format: str | None,
     concurrency: int | None,
     run_id: str | None,
+    headless: bool,
     experiment_config: Path | None,
     timeout: int | None,
     setup: str | None,
@@ -141,6 +150,8 @@ def experiments(
         for flag, value in given.items():
             if value is not None:
                 raise click.UsageError(f"{flag} needs new.")
+    if headless and action != "run":
+        raise click.UsageError("--headless needs run.")
     match name, action:
         case "new", _:
             try:
@@ -160,24 +171,57 @@ def experiments(
                 only = parse_subset(ctx.args)
             except ValueError as error:
                 raise click.UsageError(str(error)) from error
-            start = start_experiment(
-                config,
-                name,
-                experiment_config,
-                concurrency,
-                timeout,
-                only,
-                passthrough or None,
-                setup,
-                teardown,
-                ctx.meta.get(COMMAND_LINE),
-                run_id,
-            )
             try:
-                if sys.stdout.isatty() and sys.stderr.isatty():
-                    results = run_tui(start, name, config, experiment_config)
+                if not headless and sys.stdout.isatty() and sys.stderr.isatty():
+                    asyncio.run(ensure_daemon(config, name))
+                    reply = daemon_command(
+                        config,
+                        name,
+                        {
+                            "command": "restart" if run_id else "start",
+                            "options": {
+                                "experiment_config": str(experiment_config)
+                                if experiment_config
+                                else None,
+                                "concurrency": concurrency,
+                                "timeout": timeout,
+                                "only": only,
+                                "args": passthrough or None,
+                                "setup": setup,
+                                "teardown": teardown,
+                                "command_line": ctx.meta.get(COMMAND_LINE),
+                                "run_id": run_id,
+                            },
+                        },
+                    )
+                    if not reply.get("ok"):
+                        raise ValueError(
+                            str(reply.get("error", "Daemon rejected start"))
+                        )
+                    run_tui(
+                        None,
+                        name,
+                        config,
+                        experiment_config,
+                        daemon_journal=RunState(Path(str(reply["journal"]))),
+                    )
+                    return
                 else:
-                    results = run_headless(start)
+                    results = run_headless(
+                        start_experiment(
+                            config,
+                            name,
+                            experiment_config,
+                            concurrency,
+                            timeout,
+                            only,
+                            passthrough or None,
+                            setup,
+                            teardown,
+                            ctx.meta.get(COMMAND_LINE),
+                            run_id,
+                        )
+                    )
             except ValueError as error:
                 raise click.ClickException(str(error)) from error
             superseded = {r["retry_of"] for r in results if "retry_of" in r}
@@ -190,6 +234,29 @@ def experiments(
             passed = len(results) - failed - retried
             click.echo(f"{passed} passed, {failed} failed.{again}")
             return
+        case str(), "stop":
+            if (
+                output_format is not None
+                or concurrency is not None
+                or run_id is not None
+                or experiment_config is not None
+                or timeout is not None
+                or setup is not None
+                or teardown is not None
+                or passthrough is not None
+                or ctx.args
+            ):
+                raise click.UsageError("stop takes no run or listing options.")
+            try:
+                reply = daemon_command(config, name, {"command": "stop_all"})
+            except ValueError as error:
+                raise click.ClickException(str(error)) from error
+            if not reply.get("ok"):
+                raise click.ClickException(
+                    str(reply.get("error", "Daemon rejected stop"))
+                )
+            click.echo(f"Asked daemon for {name} to stop all runs.")
+            return
     if passthrough is not None:
         raise click.UsageError("-- ARGS needs run.")
     if run_id is not None:
@@ -197,7 +264,7 @@ def experiments(
     if ctx.args:
         raise click.UsageError(f"Unexpected {ctx.args[0]}. --PARAM=VALUE needs run.")
     if action not in (None, "runs"):
-        raise click.UsageError(f"Unknown action {action}. Use runs or run.")
+        raise click.UsageError(f"Unknown action {action}. Use runs, run or stop.")
     if output_format is None and sys.stdout.isatty() and sys.stderr.isatty():
         try:
             browse_tui(config, name)