labnotes 0.0.77__tar.gz → 0.0.79__tar.gz

{labnotes-0.0.77 → labnotes-0.0.79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.77
+Version: 0.0.79
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.77 → labnotes-0.0.79}/src/labnotes/context.md

@@ -126,7 +126,10 @@ going. Elsewhere, it prints a line per event instead.
   far plus the latest line each running run logged. `-f` keeps following, and
   waits for a daemon when none is up. `--json` prints one JSON object per line.
   Following doesn't keep the daemon up.
+- `labnotes run control NAME ACTION [ARG]` sends one live-view control to a
+  running experiment: `kill`, `retry`, `run_next` or `toggle` a run,
+  `set_concurrency` to a number, `interrupt` or `finish`.
 - A second `run` of the same experiment fails while the first is going.
 - It exits 0 even if runs fail; the summary line counts failures.
 
-`uvx labnotes server` serves the experiments, their markdown files and their runs as a local site, read from the same `.labnotes.toml`. An experiment's page shows its `experiment.toml` on one line, then a Runs tab with a table of its runs and a Notebooks tab with one tab per markdown file.
+`uvx labnotes server` serves the experiments, their markdown files and their runs as a local site, read from the same `.labnotes.toml`. An experiment's page shows its `experiment.toml` on one line, then a Runs tab with a table of its runs and a Notebooks tab with one tab per markdown file. Clicking a run opens its `stdout.log` and `stderr.log` together, followed as they're written until the run ends.

{labnotes-0.0.77 → labnotes-0.0.79}/src/labnotes/create_app.py

@@ -28,11 +28,13 @@ from labnotes.format_started import format_started
 from labnotes.format_value import format_value
 from labnotes.list_experiments import list_experiments
 from labnotes.load_experiment import load_experiment
+from labnotes.load_run import load_run
 from labnotes.models import LabnotesConfig
 from labnotes.note_path import note_path
 from labnotes.notebook_kernel import NotebookKernel
 from labnotes.python_blocks import python_blocks
 from labnotes.render_markdown import render_markdown
+from labnotes.run_output import run_output
 from labnotes.save_conflict_copy import save_conflict_copy
 from labnotes.save_note import save_note
 from labnotes.saved_outputs import saved_outputs
@@ -84,6 +86,32 @@ def create_app(config: LabnotesConfig) -> Starlette:
             request, "experiment.html", {"experiment": loaded}
         )
 
+    async def find_run(request: Request) -> dict[str, object]:
+        found = await load_run(
+            request.app.state.tables,
+            request.path_params["name"],
+            request.path_params["run"],
+        )
+        if found is None:
+            raise HTTPException(404, "No such run")
+        return found
+
+    async def run(request: Request) -> Response:
+        found = await find_run(request)
+        return templates.TemplateResponse(request, "run.html", {"run": found})
+
+    async def output(request: Request) -> Response:
+        await find_run(request)
+        return StreamingResponse(
+            run_output(
+                request.app.state.tables,
+                config.root,
+                request.path_params["name"],
+                request.path_params["run"],
+            ),
+            media_type="text/event-stream",
+        )
+
     async def read_note(request: Request) -> dict[str, str]:
         name = request.path_params["name"]
         note = request.path_params["note"]
@@ -191,6 +219,8 @@ def create_app(config: LabnotesConfig) -> Starlette:
             Route("/", index),
             Route("/experiments/{name}", experiment),
             Route("/experiments/{name}/changes", changes),
+            Route("/experiments/{name}/runs/{run:path}", run),
+            Route("/experiments/{name}/output/{run:path}", output),
             Route("/experiments/{name}/notes", post_note, methods=["POST"]),
             Route("/experiments/{name}/notes/{note:path}", get_note),
             Route("/experiments/{name}/notes/{note:path}", put_note, methods=["PUT"]),

labnotes-0.0.79/src/labnotes/daemon/control_run.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+
+from labnotes.daemon.daemon_client import daemon_client
+from labnotes.daemon.daemon_running import daemon_running
+
+
+async def control_run(
+    root: Path, name: str, action: str, args: list[object]
+) -> str | None:
+    if not await daemon_running(root):
+        return "Nothing is running."
+    async with daemon_client(root) as client:
+        reply = await client.post(f"/experiments/{name}/{action}", json={"args": args})
+    if reply.is_success:
+        return None
+    return str(reply.json()["error"])

labnotes-0.0.79/src/labnotes/load_run.py

@@ -0,0 +1,21 @@
+import json
+
+from dirsql import DirSQL
+
+from labnotes.sql_quote import quote_string
+
+
+async def load_run(
+    tables: DirSQL, experiment: str, run: str
+) -> dict[str, object] | None:
+    rows = await tables.query(
+        "SELECT runs.experiment, experiments.display_name, runs.id, runs.folder,"
+        " runs.created, runs.ended, runs.exit_code, runs.params"
+        " FROM runs JOIN experiments ON experiments.name = runs.experiment"
+        f" WHERE runs.experiment = {quote_string(experiment)}"
+        f" AND runs.id = {quote_string(run)}"
+    )
+    if not rows:
+        return None
+    [row] = rows
+    return {**row, "params": json.loads(str(row["params"]))}

{labnotes-0.0.77 → labnotes-0.0.79}/src/labnotes/run_command.py

@@ -6,6 +6,7 @@ from labnotes.daemon.stop_runs import stop_runs
 from labnotes.load_config import load_config
 from labnotes.read_config import CONFIG_FILE
 from labnotes.run_status import run_status
+from labnotes.run_control import run_control
 
 
 @click.group(name="run", help="Control the runs going in this project.")
@@ -28,3 +29,4 @@ def stop(ctx: click.Context) -> None:
 
 
 run_command.add_command(run_status)
+run_command.add_command(run_control)

labnotes-0.0.79/src/labnotes/run_control.py

@@ -0,0 +1,46 @@
+import asyncio
+import typing
+
+import click
+
+from labnotes.daemon.control_run import control_run
+from labnotes.load_config import load_config
+from labnotes.models import Controls
+from labnotes.read_config import CONFIG_FILE
+
+PARAMS = {
+    action: typing.get_args(hint)[0]
+    for action, hint in typing.get_type_hints(Controls).items()
+}
+
+
+@click.command(
+    name="control",
+    help=(
+        "Send one control to an experiment's runs, as the live view's keys do. "
+        "kill, retry, run_next and toggle take a run; set_concurrency a number; "
+        "interrupt and finish nothing."
+    ),
+)
+@click.argument("name")
+@click.argument("action", type=click.Choice(list(PARAMS)))
+@click.argument("args", nargs=-1)
+@click.pass_context
+def run_control(
+    ctx: click.Context, name: str, action: str, args: tuple[str, ...]
+) -> None:
+    config = load_config(ctx)
+    if config is None:
+        raise click.ClickException(
+            f"No {CONFIG_FILE} here. Run labnotes init, or pass -c PATH."
+        )
+    kinds = PARAMS[action]
+    if len(args) != len(kinds):
+        raise click.UsageError(f"{action} takes {len(kinds)} argument(s).")
+    try:
+        values = [kind(args[i]) for i, kind in enumerate(kinds)]
+    except ValueError:
+        raise click.UsageError(f"{action} takes a number.") from None
+    refused = asyncio.run(control_run(config.root, name, action, values))
+    if refused is not None:
+        raise click.ClickException(refused)

labnotes-0.0.79/src/labnotes/run_output.py

@@ -0,0 +1,66 @@
+import codecs
+import json
+from collections.abc import AsyncIterator, Iterable, Iterator
+from pathlib import Path
+
+from dirsql import DirSQL
+
+from labnotes.load_run import load_run
+from labnotes.tables.read_log import STREAMS
+
+
+async def run_output(
+    tables: DirSQL, root: Path, experiment: str, run_id: str
+) -> AsyncIterator[str]:
+    # Watch before reading, so nothing written in between is missed.
+    events = tables.watch()
+    run = await load_run(tables, experiment, run_id)
+    if run is None:
+        return
+    folder = root / str(run["folder"])
+    offsets = dict.fromkeys(STREAMS, 0)
+    decoders = {
+        stream: codecs.getincrementaldecoder("utf-8")("replace") for stream in STREAMS
+    }
+    last: str | None = None
+    mid_line = False
+
+    def read(streams: Iterable[str]) -> Iterator[str]:
+        nonlocal last, mid_line
+        for stream in streams:
+            path = folder / f"{stream}.log"
+            if not path.exists():
+                continue
+            with path.open("rb") as log:
+                log.seek(offsets[stream])
+                data = log.read()
+            offsets[stream] += len(data)
+            text = decoders[stream].decode(data)
+            if not text:
+                continue
+            if mid_line and stream != last:
+                text = "\n" + text
+            last, mid_line = stream, not text.endswith("\n")
+            yield f"data: {json.dumps({'stream': stream, 'text': text})}\n\n"
+
+    for message in read(STREAMS):
+        yield message
+    exit_code = run["exit_code"]
+    if run["ended"] is None:
+        async for event in events:
+            row = event.row
+            if row is None or row["experiment"] != experiment:
+                continue
+            if event.table == "logs" and row["run"] == run_id:
+                for message in read([row["stream"]]):
+                    yield message
+            elif (
+                event.table == "runs"
+                and row["id"] == run_id
+                and row["ended"] is not None
+            ):
+                exit_code = row["exit_code"]
+                break
+        for message in read(STREAMS):
+            yield message
+    yield f"event: end\ndata: {json.dumps(exit_code)}\n\n"

{labnotes-0.0.77 → labnotes-0.0.79}/src/labnotes/static/labnotes.css

@@ -295,6 +295,19 @@ h3 {
 .group-toggle[aria-expanded="true"]::before { content: "▾"; }
 .group:not(.open) > :not(.group-row) { display: none; }
 .group.open .run-row td:nth-child(2) { padding-left: 2rem; }
+.run-row { cursor: pointer; }
+.run-link { text-decoration: none; }
+
+/* Run output */
+.run-state.good { color: var(--good); }
+.run-state.bad { color: var(--bad); }
+.run-log { background: var(--surface); border: 1px solid var(--rule); border-radius: 6px; }
+.run-log header { display: flex; gap: 1rem; padding: 0.4rem 1.25rem; border-bottom: 1px solid var(--rule); font-family: var(--mono); font-size: 0.75rem; color: var(--ink-3); }
+.run-log header span::before { content: "■ "; }
+.run-output { margin: 0; padding: 1rem 1.25rem; max-height: 70vh; overflow: auto; font-size: 0.82rem; white-space: pre-wrap; overflow-wrap: anywhere; }
+.run-output:empty::before { content: "No output yet."; color: var(--ink-3); }
+.run-output .stderr, .run-log header .stderr::before { color: var(--bad); }
+.run-log header .stdout::before { color: var(--ink-2); }
 
 /* Notes */
 .note {

labnotes-0.0.79/src/labnotes/static/row-link.js

@@ -0,0 +1,8 @@
+// Safari ignores position: relative on <tr>, so a stretched ::after link covers the whole page.
+// Delegated, so rows the runs filter swaps in stay clickable.
+document.addEventListener("click", (event) => {
+  const link = event.target.closest(".ledger tbody tr, .run-row")?.querySelector(".title-link, .run-link");
+  if (!link || event.target.closest("a") || getSelection().toString()) return;
+  if (event.metaKey || event.ctrlKey) window.open(link.href, "_blank");
+  else location.href = link.href;
+});

labnotes-0.0.79/src/labnotes/static/run-output.js

@@ -0,0 +1,24 @@
+const output = document.querySelector(".run-output");
+const state = document.querySelector(".run-state");
+const source = new EventSource(output.dataset.url);
+
+// The server resends the whole log on each connection, so a reconnect starts clean.
+source.addEventListener("open", () => output.replaceChildren());
+
+source.addEventListener("message", (event) => {
+  const { stream, text } = JSON.parse(event.data);
+  const following = output.scrollTop + output.clientHeight >= output.scrollHeight - 4;
+  const chunk = document.createElement("span");
+  chunk.className = stream;
+  chunk.textContent = text;
+  output.append(chunk);
+  if (following) output.scrollTop = output.scrollHeight;
+});
+
+source.addEventListener("end", (event) => {
+  source.close();
+  const code = JSON.parse(event.data);
+  state.textContent = code === null ? "ended" : `exit ${code}`;
+  state.classList.add(code === 0 ? "good" : "bad");
+  output.dataset.ended = "";
+});

{labnotes-0.0.77 → labnotes-0.0.79}/src/labnotes/tables/open_tables.py

@@ -6,6 +6,7 @@ from labnotes.models import LabnotesConfig
 from labnotes.tables.path_glob import path_glob
 from labnotes.tables.read_experiment import read_experiment
 from labnotes.tables.read_file import read_file
+from labnotes.tables.read_log import read_log
 from labnotes.tables.read_run import read_run
 
 # experiments is a view, so files and runs can't declare a foreign key to it.
@@ -40,6 +41,13 @@ CREATE TABLE runs (
   PRIMARY KEY (experiment, id)
 )
 """
+# Sizes only: the server reads log text from disk, the row change is the signal.
+LOGS = """
+CREATE TABLE logs (
+  experiment TEXT, run TEXT, stream TEXT, size INTEGER,
+  PRIMARY KEY (experiment, run, stream)
+)
+"""
 
 
 def open_tables(config: LabnotesConfig) -> DirSQL:
@@ -59,6 +67,12 @@ def open_tables(config: LabnotesConfig) -> DirSQL:
                 glob=path_glob(f"{config.runs}/run.json"),
                 on_file=partial(read_run, root, config.runs),
             ),
+            Table(
+                name="logs",
+                ddl=LOGS,
+                glob=path_glob(f"{config.runs}/std*.log"),
+                on_file=partial(read_log, root, config.runs),
+            ),
             Table(
                 name="experiment_configs",
                 ddl=EXPERIMENTS,

labnotes-0.0.79/src/labnotes/tables/read_log.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from labnotes.tables.path_pattern import path_pattern
+
+STREAMS = ("stdout", "stderr")
+
+
+def read_log(root: Path, template: str, path: str) -> list[dict[str, object]]:
+    relative = Path(path).relative_to(root).as_posix()
+    found = path_pattern(f"{template}/{{stream}}.log").fullmatch(relative)
+    if found is None or found["stream"] not in STREAMS:
+        return []
+    return [
+        {
+            "experiment": found["experiment"],
+            "run": found["run"],
+            "stream": found["stream"],
+            "size": Path(path).stat().st_size,
+        }
+    ]

{labnotes-0.0.77 → labnotes-0.0.79}/src/labnotes/templates/experiment.html

@@ -83,7 +83,7 @@
     {% macro run_rows(run) %}
       <tr class="run-row">
         <td><span class="status {{ 'good' if run.labnotes_completed else 'bad' }}" title="{{ 'complete' if run.labnotes_completed else 'not complete' }}">{{ '✓' if run.labnotes_completed else '✕' }}</span></td>
-        {% for column in e.run_columns %}{% if column == 'id' and nested %}<td class="mono">{{ run.id.partition('/')[2] or run.id }}</td><td></td>{% else %}<td class="{{ 'mono' if loop.first }}{{ 'num' if column == 'duration' }}">{{ run[column] | duration if column == 'duration' else (run[column] | started if column == 'started' else run[column] | value) }}</td>{% endif %}{% endfor %}
+        {% for column in e.run_columns %}{% if column == 'id' %}<td class="mono"><a class="run-link" href="{{ url_for('run', name=e.name, run=run.id) }}">{{ run.id.partition('/')[2] or run.id if nested else run.id }}</a></td>{% if nested %}<td></td>{% endif %}{% else %}<td class="{{ 'mono' if loop.first }}{{ 'num' if column == 'duration' }}">{{ run[column] | duration if column == 'duration' else (run[column] | started if column == 'started' else run[column] | value) }}</td>{% endif %}{% endfor %}
       </tr>
     {% endmacro %}
     {% if nested %}
@@ -117,4 +117,5 @@
 {% endif %}
 <script src="{{ asset('experiment.js') }}" defer></script>
 <script src="{{ asset('local-time.js') }}" defer></script>
+<script src="{{ asset('row-link.js') }}" defer></script>
 {% endblock %}

labnotes-0.0.79/src/labnotes/templates/run.html

@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+{% block title %}{{ run.id }} · {{ run.display_name }} · labnotes{% endblock %}
+{% block crumbs %}<span class="crumb"><a href="{{ url_for('index') }}">Experiments</a></span><span class="crumb"><a href="{{ url_for('experiment', name=run.experiment) }}">{{ run.display_name }}</a></span><span class="crumb mono">{{ run.id }}</span>{% endblock %}
+{% block main %}
+<section class="intro">
+  <h1 class="mono">{{ run.id }}</h1>
+  <p class="config-summary">
+    {% if run.ended is none %}<span class="chip run-state">running</span>{% elif run.exit_code is none %}<span class="chip run-state bad">ended</span>{% else %}<span class="chip run-state {{ 'good' if run.exit_code == 0 else 'bad' }}">exit {{ run.exit_code }}</span>{% endif %}
+    {% if run.created %}<span class="setting">started <code>{{ run.created }}</code></span>{% endif %}
+    {% for key, value in run.params.items() %}<span class="setting">{{ key }} <code>{{ value | value }}</code></span>{% endfor %}
+  </p>
+</section>
+
+<section class="run-log">
+  <header><span class="stdout">stdout</span><span class="stderr">stderr</span></header>
+  <pre class="run-output" data-url="{{ url_for('output', name=run.experiment, run=run.id) }}"></pre>
+</section>
+<script src="{{ asset('run-output.js') }}" defer></script>
+{% endblock %}

{labnotes-0.0.77 → labnotes-0.0.79}/testing-conventions.toml

@@ -39,3 +39,9 @@ path = "src/labnotes/daemon/remote_start.py"
 rules = ["mutation"]
 lines = [39]
 reason = "the daemon answers a run with 202, 400, 409 or 500, so every status but 202 is above it: != equals >"
+
+[[python.exempt]]
+path = "src/labnotes/run_control.py"
+rules = ["mutation"]
+lines = [38]
+reason = "argument counts are small ints, which CPython caches, so is not equals !="

labnotes-0.0.77/src/labnotes/static/row-link.js

@@ -1,9 +0,0 @@
-// Safari ignores position: relative on <tr>, so a stretched ::after link covers the whole page.
-for (const row of document.querySelectorAll(".ledger tbody tr")) {
-  const link = row.querySelector(".title-link");
-  row.addEventListener("click", (event) => {
-    if (event.target.closest("a") || getSelection().toString()) return;
-    if (event.metaKey || event.ctrlKey) window.open(link.href, "_blank");
-    else location.href = link.href;
-  });
-}