labnotes 0.0.110__tar.gz → 0.0.112__tar.gz

{labnotes-0.0.110 → labnotes-0.0.112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.110
+Version: 0.0.112
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/context.md

@@ -25,8 +25,8 @@ When `experiment.toml` sits next to `.labnotes.toml`, the root is the one experi
 - `files`: `experiment`, `filename`, `path`, `ctime`, `mtime`.
 - `runs`: `experiment`, `id`, `folder`, `created`, `ended`, `exit_code`,
   `error`, `stdout`, `stderr`, `params` as JSON, `meta` as JSON (null without
-  a `meta` command), and `run`, the whole `run.json` as JSON, read from each run
-  folder's `run.json`.
+  a `meta` command), `command`, and `run`, the whole `run.json` as JSON, read
+  from each run folder's `run.json`.
 
 Keys come from folder names, so renaming a folder renames its rows on the next
 read. Paths are relative to the root, and times are UTC ISO 8601.
@@ -75,9 +75,11 @@ inside it (`runs/20261020-000429-brave-fox/quiet-river`); the run id is that pat
 script so it writes its outputs there.
 Each param is appended as `--name value`, with the name as written (use
 `source-language` for a `--source-language` flag), booleans as 0 or 1, and lists
-and tables as JSON. labnotes writes `run.json` (params, created, ended, exit
-code, `ended_by`, `signal`), `stdout.log` and `stderr.log` to the
-run folder. `ended_by` is `"timeout"`, `"stop"` (labnotes stopped it),
+and tables as JSON. labnotes writes `run.json` (params, `command`, created,
+ended, exit code, `ended_by`, `signal`), `stdout.log` and `stderr.log` to the
+run folder. `command` is the shell command the run executed, with `{run_dir}`
+and the params filled in, written before the run starts, so a killed run has it
+and a later edit to `experiment.toml` doesn't hide what a run did. `ended_by` is `"timeout"`, `"stop"` (labnotes stopped it),
 `"signal"` (a signal labnotes didn't send, like the out-of-memory killer),
 `"lost"` (its process died unseen; the daemon marks these when it starts, with
 `ended` set and `exit_code` null) or null (the script exited itself). `signal` is the signal number read from the
@@ -121,6 +123,13 @@ shows a waiting retry as `retry in 30s` and the end as `gave up (exit 75)`;
 
 Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs --format json`; a run with no `ended` is still going. Each run lists `size`, the bytes in its run folder, and a running run lists `memory`, the resident bytes of its command and everything it started.
 
+While a run goes, labnotes appends a memory sample to `memory.jsonl` in its run
+folder every 5 seconds: one JSON line, `{"time": <unix seconds>, "rss": <bytes>,
+"processes": <count>}`, for the command and everything it started. It keeps
+every sample, so a run killed for memory still shows the climb.
+`memory_interval` in `experiment.toml` (or `[experiment]` in `.labnotes.toml`)
+sets the seconds between samples; 0 turns it off.
+
 In a terminal, `run` shows a live table of runs (status, time, memory and
 folder size) and stays
 open when they finish. Failed runs show their status in red; `f` hides or shows them, and `r` on one

labnotes-0.0.112/src/labnotes/is_non_negative_number.py

@@ -0,0 +1,10 @@
+import math
+
+
+def is_non_negative_number(value: object) -> bool:
+    return (
+        isinstance(value, int | float)
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+        and value >= 0
+    )

labnotes-0.0.112/src/labnotes/memory_interval.py

@@ -0,0 +1,2 @@
+# Seconds between memory samples, unless experiment.toml sets `memory_interval`.
+DEFAULT_MEMORY_INTERVAL = 5.0

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/models.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import Any, Protocol
 
 from labnotes.default_counts import DEFAULT_COUNTS
+from labnotes.memory_interval import DEFAULT_MEMORY_INTERVAL
 from labnotes.modes import DEFAULT_MODE
 from labnotes.retries import DEFAULT_RETRIES
 
@@ -66,3 +67,4 @@ class ExperimentConfig:
     concurrency: int = 1
     retries: int = DEFAULT_RETRIES
     meta: str | None = None
+    memory_interval: float = DEFAULT_MEMORY_INTERVAL

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/runner/execute_run.py

@@ -9,6 +9,7 @@ import psutil
 
 from labnotes.log_path import LOG_ENV
 from labnotes.runner.exit_signal import exit_signal
+from labnotes.runner.record_memory import record_memory
 from labnotes.runner.shell_command import shell_command
 from labnotes.runner.stop_then_kill import GRACE_SECONDS
 from labnotes.runner.wait_for_tree import wait_for_tree
@@ -30,9 +31,13 @@ async def execute_run(
     stop: Callable[[], None] = lambda: None,
     stopped: Callable[[], bool] = lambda: False,
     meta: dict[str, object] | None = None,
+    *,
+    memory_interval: float,
 ) -> dict[str, object]:
+    resolved = shell_command(command, folder, params)
     run: dict[str, object] = {
         "params": params,
+        "command": resolved,
         "created": iso_time(time.time()),
         "ended": None,
         "exit_code": None,
@@ -51,7 +56,7 @@ async def execute_run(
     ):
         try:
             process = await asyncio.create_subprocess_shell(
-                shell_command(command, folder, params),
+                resolved,
                 cwd=root,
                 env={
                     "FORCE_COLOR": "1",
@@ -67,9 +72,17 @@ async def execute_run(
         else:
             running.add(process)
             run["pid"] = process.pid
+            created = None
             with contextlib.suppress(psutil.Error):
-                run["pid_created"] = psutil.Process(process.pid).create_time()
+                run["pid_created"] = created = psutil.Process(process.pid).create_time()
             write_run_json(folder, run)
+            sampler = None
+            if memory_interval > 0 and created is not None:
+                sampler = asyncio.create_task(
+                    record_memory(
+                        folder / "memory.jsonl", process.pid, created, memory_interval
+                    )
+                )
             exited = asyncio.ensure_future(process.wait())
             kill_later = None
             try:
@@ -98,6 +111,9 @@ async def execute_run(
             finally:
                 if kill_later is not None:
                     kill_later.cancel()
+                if sampler is not None:
+                    sampler.cancel()
+                    await asyncio.wait({sampler})
                 running.discard(process)
     run["ended"] = iso_time(time.time())
     write_run_json(folder, run)

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/runner/read_experiment_config.py

@@ -7,6 +7,8 @@ from labnotes.compose_experiment import compose_experiment
 from labnotes.default_counts import DEFAULT_COUNTS
 from labnotes.models import ExperimentConfig
 from labnotes.is_non_negative_int import is_non_negative_int
+from labnotes.is_non_negative_number import is_non_negative_number
+from labnotes.memory_interval import DEFAULT_MEMORY_INTERVAL
 from labnotes.is_positive_int import is_positive_int
 from labnotes.modes import MODES
 
@@ -20,6 +22,7 @@ SETTINGS = {
     "concurrency",
     "retries",
     "meta",
+    "memory_interval",
 }
 
 
@@ -73,6 +76,18 @@ def read_experiment_config(
     meta = values.get("meta")
     if meta is not None and (not isinstance(meta, str) or not meta.strip()):
         raise ValueError(f"{path}: meta must be a string")
+    memory_interval = values.get("memory_interval", DEFAULT_MEMORY_INTERVAL)
+    if not is_non_negative_number(memory_interval):
+        raise ValueError(f"{path}: memory_interval must be 0 or more seconds")
     return ExperimentConfig(
-        command, repeats, params, name, counts, mode, concurrency, retries, meta
+        command,
+        repeats,
+        params,
+        name,
+        counts,
+        mode,
+        concurrency,
+        retries,
+        meta,
+        memory_interval,
     )

labnotes-0.0.112/src/labnotes/runner/record_memory.py

@@ -0,0 +1,18 @@
+import asyncio
+import json
+import time
+from pathlib import Path
+
+from labnotes.utils.tree_usage import tree_usage
+
+
+async def record_memory(path: Path, pid: int, created: float, interval: float) -> None:
+    with path.open("a", encoding="utf-8") as samples:
+        while True:
+            usage = tree_usage(pid, created)
+            if usage is not None:
+                rss, processes = usage
+                sample = {"time": time.time(), "rss": rss, "processes": processes}
+                samples.write(json.dumps(sample) + "\n")
+                samples.flush()
+            await asyncio.sleep(interval)

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/runner/run_experiment.py

@@ -261,6 +261,7 @@ async def run_experiment(
             stop=lambda: kill(run),
             stopped=lambda: run in stopped,
             meta=meta,
+            memory_interval=experiment.memory_interval,
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/tables/open_tables.py

@@ -10,6 +10,7 @@ from labnotes.tables.read_blocks import read_blocks
 from labnotes.tables.read_experiment import read_experiment
 from labnotes.tables.read_file import read_file
 from labnotes.tables.read_log import read_log
+from labnotes.tables.read_memory import read_memory
 from labnotes.tables.read_run import read_run
 
 FILES = """
@@ -22,7 +23,7 @@ RUNS = """
 CREATE TABLE runs (
   experiment TEXT, id TEXT, folder TEXT, created TEXT, ended TEXT,
   exit_code INTEGER, error TEXT,
-  stdout TEXT, stderr TEXT, params TEXT, meta TEXT, run TEXT,
+  stdout TEXT, stderr TEXT, params TEXT, meta TEXT, command TEXT, run TEXT,
   PRIMARY KEY (experiment, id)
 )
 """
@@ -39,6 +40,12 @@ CREATE TABLE blocks (
   PRIMARY KEY (experiment, run)
 )
 """
+MEMORY = """
+CREATE TABLE memory (
+  experiment TEXT, run TEXT, line INTEGER, time REAL, rss INTEGER, processes INTEGER,
+  PRIMARY KEY (experiment, run, line)
+)
+"""
 
 
 def open_tables(config: LabnotesConfig) -> DirSQL:
@@ -78,6 +85,14 @@ def open_tables(config: LabnotesConfig) -> DirSQL:
                     read_blocks, root, config.runs, experiment=config.experiment
                 ),
             ),
+            Table(
+                name="memory",
+                ddl=MEMORY,
+                glob=path_glob(f"{config.runs}/memory.jsonl"),
+                on_file=partial(
+                    read_memory, root, config.runs, experiment=config.experiment
+                ),
+            ),
             Table(
                 name="experiment_configs",
                 ddl=experiments_ddl(config),

labnotes-0.0.112/src/labnotes/tables/read_memory.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+
+from labnotes.tables.path_pattern import path_pattern
+
+
+def read_memory(
+    root: Path, template: str, path: str, experiment: str | None = None
+) -> list[dict[str, object]]:
+    relative = Path(path).relative_to(root).as_posix()
+    found = path_pattern(f"{template}/memory.jsonl").fullmatch(relative)
+    if found is None:
+        return []
+    rows: list[dict[str, object]] = []
+    lines = Path(path).read_text(encoding="utf-8").splitlines()
+    for line, text in enumerate(lines):
+        # A run killed mid-write leaves a cut-off last line.
+        try:
+            sample = json.loads(text)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(sample, dict):
+            continue
+        rows.append(
+            {
+                "experiment": experiment or found["experiment"],
+                "run": found["run"],
+                "line": line,
+                "time": sample.get("time"),
+                "rss": sample.get("rss"),
+                "processes": sample.get("processes"),
+            }
+        )
+    return rows

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/tables/read_run.py

@@ -26,6 +26,7 @@ def read_run(
             "stderr": f"{folder}/stderr.log",
             "params": json.dumps(run.get("params", {})),
             "meta": None if (meta := run.get("meta")) is None else json.dumps(meta),
+            "command": run.get("command"),
             "run": json.dumps(run),
         }
     ]

{labnotes-0.0.110 → labnotes-0.0.112}/src/labnotes/utils/process_alive.py

@@ -1,6 +1,6 @@
 import psutil
 
-from labnotes.utils.tree_rss import SAME_START_SECONDS
+from labnotes.utils.tree_usage import SAME_START_SECONDS
 
 
 def process_alive(pid: int, created: float | None) -> bool:

labnotes-0.0.112/src/labnotes/utils/tree_rss.py

@@ -0,0 +1,6 @@
+from labnotes.utils.tree_usage import tree_usage
+
+
+def tree_rss(pid: int, created: float) -> int | None:
+    usage = tree_usage(pid, created)
+    return None if usage is None else usage[0]

labnotes-0.0.110/src/labnotes/utils/tree_rss.py → labnotes-0.0.112/src/labnotes/utils/tree_usage.py

@@ -7,7 +7,7 @@ import psutil
 SAME_START_SECONDS = 0.01
 
 
-def tree_rss(pid: int, created: float) -> int | None:
+def tree_usage(pid: int, created: float) -> tuple[int, int] | None:
     try:
         root = psutil.Process(pid)
         if abs(root.create_time() - created) > SAME_START_SECONDS:
@@ -24,4 +24,4 @@ def tree_rss(pid: int, created: float) -> int | None:
     for process in tree:
         with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
             sizes.append(process.memory_info().rss)
-    return sum(sizes) if sizes else None
+    return (sum(sizes), len(sizes)) if sizes else None