labnotes 0.0.105__tar.gz → 0.0.107__tar.gz

{labnotes-0.0.105 → labnotes-0.0.107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.105
+Version: 0.0.107
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/context.md

@@ -75,8 +75,11 @@ script so it writes its outputs there.
 Each param is appended as `--name value`, with the name as written (use
 `source-language` for a `--source-language` flag), booleans as 0 or 1, and lists
 and tables as JSON. labnotes writes `run.json` (params, created, ended, exit
-code), `stdout.log` and `stderr.log` to the
-run folder.
+code, `ended_by`, `signal`), `stdout.log` and `stderr.log` to the
+run folder. `ended_by` is `"timeout"`, `"stop"` (labnotes stopped it),
+`"signal"` (a signal labnotes didn't send, like the out-of-memory killer) or
+null (the script exited itself). `signal` is the signal number read from the
+exit code (negative, or 128 + N from the shell), else null.
 A run counts when it exited 0 and its `run.json` doesn't hold
 `"excluded": true`, so excluding a run by hand means adding that key; the next
 `run` replaces it. Other keys you add to `run.json` are kept and show in `run`.
@@ -109,9 +112,10 @@ shows a waiting retry as `retry in 30s` and the end as `gave up (exit 75)`;
 `run status --json` reports `retrying`, `retried`, `gave up` and
 `retry cancelled` events.
 
-Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs --format json`; a run with no `ended` is still going.
+Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs --format json`; a run with no `ended` is still going. Each run lists `size`, the bytes in its run folder, and a running run lists `memory`, the resident bytes of its command and everything it started.
 
-In a terminal, `run` shows a live table of runs (status and time) and stays
+In a terminal, `run` shows a live table of runs (status, time, memory and
+folder size) and stays
 open when they finish. Failed runs show their status in red; `f` hides or shows them, and `r` on one
 runs it again under a new id, with `retry_of` set. `j` and `k` move the cursor, `g` and `G` jump to the
 first and last row. Runs not started yet show grayed as `pending`, and as
@@ -133,7 +137,8 @@ going. Elsewhere, it prints a line per event instead.
 - `-e PATH` reads another file instead of `experiment.toml`.
 - `--timeout SECONDS` stops any one run still going after that many seconds:
   SIGTERM, then SIGKILL 5 seconds later. Its
-  `run.json` gets `"error": "timed out after ..."` next to its exit code.
+  `run.json` gets `"error": "timed out after ..."` and `"ended_by": "timeout"`
+  next to its exit code.
   Unset, runs have no time limit.
 - `run` hands its runs to the project's labnotes daemon, starting it if
   needed, and shows or waits for them. The daemon listens on a socket in
@@ -146,7 +151,8 @@ going. Elsewhere, it prints a line per event instead.
 - `labnotes run stop` stops every run the daemon has going: SIGTERM, then
   SIGKILL 5 seconds later.
 - `labnotes run status` prints what the daemon has going, a line per event so
-  far plus the latest line each running run logged. `-f` keeps following, and
+  far plus the latest line each running run logged and what each running run
+  uses (`usage` events with `--json`). `-f` keeps following, and
   waits for a daemon when none is up. `--json` prints one JSON object per line.
   Following doesn't keep the daemon up.
 - `labnotes run control NAME ACTION [ARG]` sends one live-view control to a

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/daemon/describe_status.py

@@ -1,3 +1,4 @@
+from labnotes.utils.format_bytes import format_bytes
 from labnotes.runner.describe_event import describe_event
 
 
@@ -5,6 +6,10 @@ def describe_status(event: dict[str, object]) -> str:
     match event:
         case {"event": "output"}:
             said = f"{event['run']}: {event['line']}"
+        case {"event": "usage"}:
+            memory = format_bytes(event["memory"]) or "?"  # type: ignore[arg-type]
+            size = format_bytes(event["size"]) or "?"  # type: ignore[arg-type]
+            said = f"{event['run']} uses {memory} of memory, {size} on disk."
         case {"event": "done", "error": None}:
             said = "Done."
         case {"event": "done"}:

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/daemon/watch_daemon.py

@@ -6,9 +6,11 @@ import httpx2
 
 from labnotes.daemon.daemon_client import daemon_client
 from labnotes.daemon.run_logs import RunLogs
+from labnotes.utils.run_usage import run_usage
 
 Event = dict[str, object]
 TAIL_SECONDS = 1.0
+USAGE_SECONDS = 30.0
 
 
 async def watch_daemon(root: Path, follow: bool, emit: Callable[[Event], None]) -> None:
@@ -23,7 +25,21 @@ async def watch_daemon(root: Path, follow: bool, emit: Callable[[Event], None])
             await asyncio.sleep(TAIL_SECONDS)
             tail()
 
+    async def sample() -> None:
+        for (experiment, run), folder in list(logs.folders.items()):
+            if (experiment, run) not in logs.folders:
+                continue
+            usage = await run_usage(folder, True)
+            if (experiment, run) in logs.folders:
+                emit({"event": "usage", "experiment": experiment, "run": run, **usage})
+
+    async def keep_sampling() -> None:
+        while True:
+            await asyncio.sleep(USAGE_SECONDS)
+            await sample()
+
     tailing = asyncio.create_task(keep_tailing()) if follow else None
+    sampling = None
     try:
         async with (
             daemon_client(root) as client,
@@ -36,8 +52,10 @@ async def watch_daemon(root: Path, follow: bool, emit: Callable[[Event], None])
                 match event:
                     case {"event": "replayed"}:
                         tail()
+                        await sample()
                         if not follow:
                             return
+                        sampling = asyncio.create_task(keep_sampling())
                         continue
                     case {"event": "started", "experiment": str(name), "run": str(run)}:
                         logs.add(name, run, Path(str(event["folder"])))
@@ -48,5 +66,6 @@ async def watch_daemon(root: Path, follow: bool, emit: Callable[[Event], None])
     except httpx2.TransportError:
         pass
     finally:
-        if tailing is not None:
-            tailing.cancel()
+        for task in (tailing, sampling):
+            if task is not None:
+                task.cancel()

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/experiments.py

@@ -6,6 +6,7 @@ import click
 
 from labnotes.create_experiment import create_experiment
 from labnotes.daemon.remote_start import remote_start
+from labnotes.utils.format_bytes import format_bytes
 from labnotes.load_config import load_config
 from labnotes.print_rows import print_rows
 from labnotes.read_config import CONFIG_FILE
@@ -14,6 +15,8 @@ from labnotes.tables.experiment_rows import experiment_rows
 from labnotes.tui.browse_tui import browse_tui
 from labnotes.tui.run_tui import run_tui
 
+BYTES = ("memory", "size")
+
 
 @click.command(
     help="List experiments, show one, list its runs, or run it."
@@ -103,4 +106,9 @@ def experiments(
         raise click.ClickException(str(error)) from error
     if rows is None:
         raise click.ClickException(f"No experiment named {name}.")
+    if output_format != "json":
+        rows = [
+            row | {key: format_bytes(row[key]) for key in BYTES if key in row}  # type: ignore[arg-type]
+            for row in rows
+        ]
     print_rows(rows, output_format or "table")

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/runner/execute_run.py

@@ -1,4 +1,5 @@
 import asyncio
+import contextlib
 import os
 import time
 from collections.abc import Callable
@@ -7,6 +8,7 @@ from pathlib import Path
 import psutil
 
 from labnotes.log_path import LOG_ENV
+from labnotes.runner.exit_signal import exit_signal
 from labnotes.runner.shell_command import shell_command
 from labnotes.runner.stop_then_kill import GRACE_SECONDS
 from labnotes.runner.wait_for_tree import wait_for_tree
@@ -26,6 +28,7 @@ async def execute_run(
     retry_of: str | None = None,
     timeout: float | None = None,
     stop: Callable[[], None] = lambda: None,
+    stopped: Callable[[], bool] = lambda: False,
 ) -> dict[str, object]:
     run: dict[str, object] = {
         "params": params,
@@ -33,6 +36,8 @@ async def execute_run(
         "ended": None,
         "exit_code": None,
         "error": None,
+        "ended_by": None,
+        "signal": None,
     }
     if retry_of is not None:
         run["retry_of"] = retry_of
@@ -58,6 +63,10 @@ async def execute_run(
             run["error"] = str(error)
         else:
             running.add(process)
+            run["pid"] = process.pid
+            with contextlib.suppress(psutil.Error):
+                run["pid_created"] = psutil.Process(process.pid).create_time()
+            write_run_json(folder, run)
             exited = asyncio.ensure_future(process.wait())
             kill_later = None
             try:
@@ -70,7 +79,14 @@ async def execute_run(
                     kill_later = asyncio.get_running_loop().call_later(
                         GRACE_SECONDS, stop
                     )
-                run["exit_code"] = await exited
+                run["exit_code"] = code = await exited
+                run["signal"] = exit_signal(code)
+                if kill_later is not None:
+                    run["ended_by"] = "timeout"
+                elif stopping.is_set() or stopped():
+                    run["ended_by"] = "stop"
+                elif run["signal"] is not None:
+                    run["ended_by"] = "signal"
                 # The shell can exit on SIGTERM before what it started. Once
                 # stopping, the run stays killable until everything signalled
                 # is gone. A normal exit ends the run even if helpers linger.

labnotes-0.0.107/src/labnotes/runner/exit_signal.py

@@ -0,0 +1,11 @@
+import signal
+
+
+def exit_signal(exit_code: int | None) -> int | None:
+    if exit_code is None:
+        return None
+    if exit_code < 0:
+        return -exit_code
+    # A shell reports a child killed by signal N as exit 128 + N.
+    number = exit_code - 128
+    return number if 0 < number < signal.NSIG else None

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/runner/run_experiment.py

@@ -255,6 +255,7 @@ async def run_experiment(
             parents.get(run),
             timeout=timeout,
             stop=lambda: kill(run),
+            stopped=lambda: run in stopped,
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/tables/experiment_rows.py

@@ -1,6 +1,7 @@
 import json
 
 from labnotes.models import LabnotesConfig
+from labnotes.utils.run_usage import run_usage
 from labnotes.summarize_config import summarize_config
 from labnotes.tables.list_experiment_runs import list_experiment_runs
 from labnotes.tables.open_tables import open_tables
@@ -21,7 +22,11 @@ async def experiment_rows(
             summary = summarize_config(
                 json.loads(str(stored)) if stored else None, config.mode
             )
-            return await list_experiment_runs(
+            runs = await list_experiment_runs(
                 tables, name, str(summary["completed_run"])
             )
+            for run in runs:
+                going = all(run[key] is None for key in ("ended", "exit_code", "error"))
+                run |= await run_usage(config.root / str(run["folder"]), going)
+            return runs
     return rows

labnotes-0.0.107/src/labnotes/tui/config.py

@@ -0,0 +1 @@
+RUN_COLUMNS = ("run", "params", "status", "time", "memory", "size")

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/tui/run_app.py

@@ -1,15 +1,19 @@
 import asyncio
 import signal
 import time
+from collections.abc import Sequence
 from pathlib import Path
 
 from rich.text import Text
+from textual import work
 from textual.app import App, ComposeResult
 from textual.binding import Binding
 from textual.widgets import DataTable, Footer, Header, Input, Static
 
+from labnotes.utils.format_bytes import format_bytes
 from labnotes.models import Controls, Start
 from labnotes.retries import RETRY_EXIT_CODE
+from labnotes.utils.run_usage import run_usage
 from labnotes.runner.describe_event import describe_event
 from labnotes.runner.stop_then_kill import GRACE_SECONDS, stop_then_kill
 from labnotes.runner.param_args import param_args
@@ -20,7 +24,6 @@ from labnotes.tui.log_blocks import LogBlocks
 from labnotes.tui.run_output import RunOutput
 from labnotes.tui.runs_table import RunsTable
 
-COLUMNS = ("run", "params", "status", "time")
 PENDING, PASSED, FAILED = "○", "✓", "✗"
 SPINNER = "|/-\\"
 SKIPPED, DESELECTED = "–", "⊘"
@@ -50,8 +53,9 @@ class RunApp(App[None]):
         Binding("minus", "concurrency(-1)", "Fewer at once", key_display="-"),
     ]
 
-    def __init__(self, start: Start, title: str) -> None:
+    def __init__(self, start: Start, title: str, columns: Sequence[str]) -> None:
         super().__init__()
+        self.columns = tuple(columns)
         self.start = start
         self.title = title
         self.results: list[dict[str, object]] = []
@@ -73,7 +77,7 @@ class RunApp(App[None]):
         self.frame = 0
         self.started: dict[str, float] = {}
         self.pending: set[str] = set()
-        self.cells: dict[str, list[object]] = {}
+        self.cells: dict[str, dict[str, object]] = {}
         self.filter = ""
         self.failed: set[str] = set()
         self.retried: set[str] = set()
@@ -83,6 +87,8 @@ class RunApp(App[None]):
         self.viewing: str | None = None
         self.grabbed: str | None = None
         self.grabbed_from: list[str] = []
+        self.sampling = False
+        self.unmeasured: dict[str, None] = {}
 
     def compose(self) -> ComposeResult:
         yield Header()
@@ -95,7 +101,7 @@ class RunApp(App[None]):
 
     def on_mount(self) -> None:
         table = self.query_one(DataTable)
-        for column in COLUMNS:
+        for column in self.columns:
             table.add_column(column, key=column)
         self.query_one(Input).display = False
         self.query_one(RunOutput).display = False
@@ -127,8 +133,9 @@ class RunApp(App[None]):
                 self.title = str(event["title"])
                 for planned in event["runs"]:  # type: ignore[attr-defined]
                     args = " ".join(param_args(planned["params"]))
-                    cells = (planned["run"].rpartition("/")[2], args, PENDING, "")
-                    self.cells[planned["run"]] = [*map(dim, cells)]
+                    self.cells[planned["run"]] = self.waiting_cells(
+                        planned["run"], args
+                    )
                     self.pending.add(planned["run"])
                 self.redraw(table.cursor_row)
                 self.show_concurrency(event)
@@ -145,13 +152,13 @@ class RunApp(App[None]):
                 if run == self.next_run:
                     self.show("")
                 args = " ".join(param_args(event["params"]))  # type: ignore[arg-type]
-                cells = (
-                    run.rpartition("/")[2],
-                    args,
-                    SPINNER[self.frame],
-                    format_elapsed(0),
-                )
-                for column, cell in zip(COLUMNS, cells):
+                shown = {
+                    "run": run.rpartition("/")[2],
+                    "params": args,
+                    "status": SPINNER[self.frame],
+                    "time": format_elapsed(0),
+                }
+                for column, cell in shown.items():
                     self.set_cell(run, column, cell)
             case "ended":
                 outcome = PASSED
@@ -166,6 +173,8 @@ class RunApp(App[None]):
                 self.tick()
                 del self.started[run]
                 self.redraw(table.cursor_row)
+                self.unmeasured[run] = None
+                self.measure()
             case "queued":
                 self.reposition(event, run)
                 self.next_run = run
@@ -201,9 +210,7 @@ class RunApp(App[None]):
                 self.retried.add(run)
                 again = str(event["retry"])
                 args = " ".join(param_args(event["params"]))  # type: ignore[arg-type]
-                self.cells[again] = [
-                    *map(dim, (again.rpartition("/")[2], args, PENDING, ""))
-                ]
+                self.cells[again] = self.waiting_cells(again, args)
                 self.pending.add(again)
                 self.redraw(table.cursor_row)
                 self.show(describe_event(event))
@@ -244,14 +251,20 @@ class RunApp(App[None]):
         table.clear()
         for run, cells in self.cells.items():
             hidden = self.hiding and run in self.failed
-            if self.filter in str(cells[1]) and not hidden:
-                table.add_row(*cells, key=run)
+            if self.filter in str(cells.get("params", "")) and not hidden:
+                table.add_row(*(cells.get(c, "") for c in self.columns), key=run)
         table.move_cursor(row=row)
 
+    def waiting_cells(self, run: str, args: str) -> dict[str, object]:
+        shown = {"run": run.rpartition("/")[2], "params": args, "status": PENDING}
+        return {
+            column: dim(shown.get(column, "")) for column in {*self.columns, *shown}
+        }
+
     def set_cell(self, run: str, column: str, cell: object) -> None:
-        self.cells[run][COLUMNS.index(column)] = cell
+        self.cells[run][column] = cell
         table = self.query_one(DataTable)
-        if run in table.rows:
+        if run in table.rows and column in self.columns:
             table.update_cell(run, column, cell, update_width=True)
 
     def check_action(self, action: str, parameters: tuple[object, ...]) -> bool:
@@ -275,6 +288,31 @@ class RunApp(App[None]):
             wait = format_elapsed(max(0, due - now))
             outcome = f"{FAILED} exited {RETRY_EXIT_CODE}, retry in {wait}"
             self.set_cell(run, "status", Text(outcome, style="yellow"))
+        if self.started:
+            self.measure()
+
+    def measure(self) -> None:
+        if not self.sampling:
+            self.sampling = True
+            self.sample()
+
+    @work
+    async def sample(self) -> None:
+        try:
+            for run in list(self.started):
+                usage = await run_usage(self.folders[run], True)
+                if run in self.started:
+                    self.show_usage(run, usage)
+            while self.unmeasured:
+                run = next(iter(self.unmeasured))
+                del self.unmeasured[run]
+                self.show_usage(run, await run_usage(self.folders[run], False))
+        finally:
+            self.sampling = False
+
+    def show_usage(self, run: str, usage: dict[str, int | None]) -> None:
+        self.set_cell(run, "memory", format_bytes(usage["memory"]))
+        self.set_cell(run, "size", format_bytes(usage["size"]))
 
     def spin(self) -> None:
         self.frame = (self.frame + 1) % len(SPINNER)

{labnotes-0.0.105 → labnotes-0.0.107}/src/labnotes/tui/run_tui.py

@@ -1,9 +1,10 @@
 from labnotes.models import Start
+from labnotes.tui.config import RUN_COLUMNS
 from labnotes.tui.run_app import RunApp
 
 
 def run_tui(start: Start, title: str) -> list[dict[str, object]]:
-    app = RunApp(start, title)
+    app = RunApp(start, title, RUN_COLUMNS)
     app.run()
     if app.error is not None:
         raise app.error

labnotes-0.0.107/src/labnotes/utils/format_bytes.py

@@ -0,0 +1,5 @@
+from rich.filesize import decimal
+
+
+def format_bytes(count: int | None) -> str:
+    return "" if count is None else decimal(count)

labnotes-0.0.107/src/labnotes/utils/run_usage.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+
+from dirsql import DirSQL
+
+from labnotes.utils.tree_rss import tree_rss
+
+
+def field(name: str) -> str:
+    return f"(SELECT content ->> '{name}' FROM './run.json' WHERE json_valid(content))"
+
+
+USAGE = (
+    "SELECT (SELECT SUM(size) FROM './**') AS size,"
+    f" {field('pid')} AS pid, {field('pid_created')} AS created,"
+    f" coalesce({field('ended')}, {field('exit_code')}, {field('error')}) AS done"
+)
+
+
+async def run_usage(folder: Path, running: bool) -> dict[str, int | None]:
+    # Run folders are often gitignored (DVC, large outputs); size them anyway.
+    [row] = await DirSQL(str(folder), tables=[], no_ignore=True).query(USAGE)
+    pid, created = row["pid"], row["created"]
+    memory = None
+    if (
+        running
+        and row["done"] is None
+        and isinstance(pid, int)
+        and isinstance(created, int | float)
+    ):
+        memory = tree_rss(pid, created)
+    return {"memory": memory, "size": row["size"]}

labnotes-0.0.107/src/labnotes/utils/tree_rss.py

@@ -0,0 +1,27 @@
+import contextlib
+
+import psutil
+
+# run.json round-trips create_time through JSON, and psutil derives it from
+# boot time, so allow a little slack when matching.
+SAME_START_SECONDS = 0.01
+
+
+def tree_rss(pid: int, created: float) -> int | None:
+    try:
+        root = psutil.Process(pid)
+        if abs(root.create_time() - created) > SAME_START_SECONDS:
+            return None
+    except (psutil.NoSuchProcess, psutil.AccessDenied):
+        return None
+    try:
+        tree = [root, *root.children(recursive=True)]
+    except psutil.NoSuchProcess:
+        return None
+    except psutil.AccessDenied:
+        tree = [root]
+    sizes = []
+    for process in tree:
+        with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
+            sizes.append(process.memory_info().rss)
+    return sum(sizes) if sizes else None

{labnotes-0.0.105 → labnotes-0.0.107}/testing-conventions.toml

@@ -69,3 +69,15 @@ path = "src/labnotes/runner/run_experiment.py"
 rules = ["mutation"]
 lines = [184, 263]
 reason = "a retry is only scheduled below the budget and each one adds one, so tried never passes retries: >= equals ==; both sides are small ints, which CPython caches, so is equals =="
+
+[[python.exempt]]
+path = "src/labnotes/runner/exit_signal.py"
+rules = ["mutation"]
+lines = [10]
+reason = "exit_code is non-negative here; when bit 7 is set, - 128 equals ^ 128, and when it is clear ^ 128 lands above NSIG like a non-positive difference"
+
+[[python.exempt]]
+path = "src/labnotes/experiments.py"
+rules = ["mutation"]
+lines = [109]
+reason = "click hands back the \"json\" choice literal itself, so is not equals !="