labnotes 0.0.111__tar.gz → 0.0.112__tar.gz

{labnotes-0.0.111 → labnotes-0.0.112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.111
+Version: 0.0.112
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.111 → labnotes-0.0.112}/src/labnotes/context.md

@@ -123,6 +123,13 @@ shows a waiting retry as `retry in 30s` and the end as `gave up (exit 75)`;
 
 Runs can take a long time. Start `run` in the background, then check on it with `labnotes experiments foo runs --format json`; a run with no `ended` is still going. Each run lists `size`, the bytes in its run folder, and a running run lists `memory`, the resident bytes of its command and everything it started.
 
+While a run goes, labnotes appends a memory sample to `memory.jsonl` in its run
+folder every 5 seconds: one JSON line, `{"time": <unix seconds>, "rss": <bytes>,
+"processes": <count>}`, for the command and everything it started. It keeps
+every sample, so a run killed for memory still shows the climb.
+`memory_interval` in `experiment.toml` (or `[experiment]` in `.labnotes.toml`)
+sets the seconds between samples; 0 turns it off.
+
 In a terminal, `run` shows a live table of runs (status, time, memory and
 folder size) and stays
 open when they finish. Failed runs show their status in red; `f` hides or shows them, and `r` on one

labnotes-0.0.112/src/labnotes/is_non_negative_number.py

@@ -0,0 +1,10 @@
+import math
+
+
+def is_non_negative_number(value: object) -> bool:
+    return (
+        isinstance(value, int | float)
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+        and value >= 0
+    )

labnotes-0.0.112/src/labnotes/memory_interval.py

@@ -0,0 +1,2 @@
+# Seconds between memory samples, unless experiment.toml sets `memory_interval`.
+DEFAULT_MEMORY_INTERVAL = 5.0

{labnotes-0.0.111 → labnotes-0.0.112}/src/labnotes/models.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import Any, Protocol
 
 from labnotes.default_counts import DEFAULT_COUNTS
+from labnotes.memory_interval import DEFAULT_MEMORY_INTERVAL
 from labnotes.modes import DEFAULT_MODE
 from labnotes.retries import DEFAULT_RETRIES
 
@@ -66,3 +67,4 @@ class ExperimentConfig:
     concurrency: int = 1
     retries: int = DEFAULT_RETRIES
     meta: str | None = None
+    memory_interval: float = DEFAULT_MEMORY_INTERVAL

{labnotes-0.0.111 → labnotes-0.0.112}/src/labnotes/runner/execute_run.py

@@ -9,6 +9,7 @@ import psutil
 
 from labnotes.log_path import LOG_ENV
 from labnotes.runner.exit_signal import exit_signal
+from labnotes.runner.record_memory import record_memory
 from labnotes.runner.shell_command import shell_command
 from labnotes.runner.stop_then_kill import GRACE_SECONDS
 from labnotes.runner.wait_for_tree import wait_for_tree
@@ -30,6 +31,8 @@ async def execute_run(
     stop: Callable[[], None] = lambda: None,
     stopped: Callable[[], bool] = lambda: False,
     meta: dict[str, object] | None = None,
+    *,
+    memory_interval: float,
 ) -> dict[str, object]:
     resolved = shell_command(command, folder, params)
     run: dict[str, object] = {
@@ -69,9 +72,17 @@ async def execute_run(
         else:
             running.add(process)
             run["pid"] = process.pid
+            created = None
             with contextlib.suppress(psutil.Error):
-                run["pid_created"] = psutil.Process(process.pid).create_time()
+                run["pid_created"] = created = psutil.Process(process.pid).create_time()
             write_run_json(folder, run)
+            sampler = None
+            if memory_interval > 0 and created is not None:
+                sampler = asyncio.create_task(
+                    record_memory(
+                        folder / "memory.jsonl", process.pid, created, memory_interval
+                    )
+                )
             exited = asyncio.ensure_future(process.wait())
             kill_later = None
             try:
@@ -100,6 +111,9 @@ async def execute_run(
             finally:
                 if kill_later is not None:
                     kill_later.cancel()
+                if sampler is not None:
+                    sampler.cancel()
+                    await asyncio.wait({sampler})
                 running.discard(process)
     run["ended"] = iso_time(time.time())
     write_run_json(folder, run)

{labnotes-0.0.111 → labnotes-0.0.112}/src/labnotes/runner/read_experiment_config.py

@@ -7,6 +7,8 @@ from labnotes.compose_experiment import compose_experiment
 from labnotes.default_counts import DEFAULT_COUNTS
 from labnotes.models import ExperimentConfig
 from labnotes.is_non_negative_int import is_non_negative_int
+from labnotes.is_non_negative_number import is_non_negative_number
+from labnotes.memory_interval import DEFAULT_MEMORY_INTERVAL
 from labnotes.is_positive_int import is_positive_int
 from labnotes.modes import MODES
 
@@ -20,6 +22,7 @@ SETTINGS = {
     "concurrency",
     "retries",
     "meta",
+    "memory_interval",
 }
 
 
@@ -73,6 +76,18 @@ def read_experiment_config(
     meta = values.get("meta")
     if meta is not None and (not isinstance(meta, str) or not meta.strip()):
         raise ValueError(f"{path}: meta must be a string")
+    memory_interval = values.get("memory_interval", DEFAULT_MEMORY_INTERVAL)
+    if not is_non_negative_number(memory_interval):
+        raise ValueError(f"{path}: memory_interval must be 0 or more seconds")
     return ExperimentConfig(
-        command, repeats, params, name, counts, mode, concurrency, retries, meta
+        command,
+        repeats,
+        params,
+        name,
+        counts,
+        mode,
+        concurrency,
+        retries,
+        meta,
+        memory_interval,
     )

labnotes-0.0.112/src/labnotes/runner/record_memory.py

@@ -0,0 +1,18 @@
+import asyncio
+import json
+import time
+from pathlib import Path
+
+from labnotes.utils.tree_usage import tree_usage
+
+
+async def record_memory(path: Path, pid: int, created: float, interval: float) -> None:
+    with path.open("a", encoding="utf-8") as samples:
+        while True:
+            usage = tree_usage(pid, created)
+            if usage is not None:
+                rss, processes = usage
+                sample = {"time": time.time(), "rss": rss, "processes": processes}
+                samples.write(json.dumps(sample) + "\n")
+                samples.flush()
+            await asyncio.sleep(interval)

{labnotes-0.0.111 → labnotes-0.0.112}/src/labnotes/runner/run_experiment.py

@@ -261,6 +261,7 @@ async def run_experiment(
             stop=lambda: kill(run),
             stopped=lambda: run in stopped,
             meta=meta,
+            memory_interval=experiment.memory_interval,
         )
         del running[run], trees[run]
         report({"event": "ended", "run": run, **result})

{labnotes-0.0.111 → labnotes-0.0.112}/src/labnotes/tables/open_tables.py

@@ -10,6 +10,7 @@ from labnotes.tables.read_blocks import read_blocks
 from labnotes.tables.read_experiment import read_experiment
 from labnotes.tables.read_file import read_file
 from labnotes.tables.read_log import read_log
+from labnotes.tables.read_memory import read_memory
 from labnotes.tables.read_run import read_run
 
 FILES = """
@@ -39,6 +40,12 @@ CREATE TABLE blocks (
   PRIMARY KEY (experiment, run)
 )
 """
+MEMORY = """
+CREATE TABLE memory (
+  experiment TEXT, run TEXT, line INTEGER, time REAL, rss INTEGER, processes INTEGER,
+  PRIMARY KEY (experiment, run, line)
+)
+"""
 
 
 def open_tables(config: LabnotesConfig) -> DirSQL:
@@ -78,6 +85,14 @@ def open_tables(config: LabnotesConfig) -> DirSQL:
                     read_blocks, root, config.runs, experiment=config.experiment
                 ),
             ),
+            Table(
+                name="memory",
+                ddl=MEMORY,
+                glob=path_glob(f"{config.runs}/memory.jsonl"),
+                on_file=partial(
+                    read_memory, root, config.runs, experiment=config.experiment
+                ),
+            ),
             Table(
                 name="experiment_configs",
                 ddl=experiments_ddl(config),

labnotes-0.0.112/src/labnotes/tables/read_memory.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+
+from labnotes.tables.path_pattern import path_pattern
+
+
+def read_memory(
+    root: Path, template: str, path: str, experiment: str | None = None
+) -> list[dict[str, object]]:
+    relative = Path(path).relative_to(root).as_posix()
+    found = path_pattern(f"{template}/memory.jsonl").fullmatch(relative)
+    if found is None:
+        return []
+    rows: list[dict[str, object]] = []
+    lines = Path(path).read_text(encoding="utf-8").splitlines()
+    for line, text in enumerate(lines):
+        # A run killed mid-write leaves a cut-off last line.
+        try:
+            sample = json.loads(text)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(sample, dict):
+            continue
+        rows.append(
+            {
+                "experiment": experiment or found["experiment"],
+                "run": found["run"],
+                "line": line,
+                "time": sample.get("time"),
+                "rss": sample.get("rss"),
+                "processes": sample.get("processes"),
+            }
+        )
+    return rows

{labnotes-0.0.111 → labnotes-0.0.112}/src/labnotes/utils/process_alive.py

@@ -1,6 +1,6 @@
 import psutil
 
-from labnotes.utils.tree_rss import SAME_START_SECONDS
+from labnotes.utils.tree_usage import SAME_START_SECONDS
 
 
 def process_alive(pid: int, created: float | None) -> bool:

labnotes-0.0.112/src/labnotes/utils/tree_rss.py

@@ -0,0 +1,6 @@
+from labnotes.utils.tree_usage import tree_usage
+
+
+def tree_rss(pid: int, created: float) -> int | None:
+    usage = tree_usage(pid, created)
+    return None if usage is None else usage[0]

labnotes-0.0.111/src/labnotes/utils/tree_rss.py → labnotes-0.0.112/src/labnotes/utils/tree_usage.py

@@ -7,7 +7,7 @@ import psutil
 SAME_START_SECONDS = 0.01
 
 
-def tree_rss(pid: int, created: float) -> int | None:
+def tree_usage(pid: int, created: float) -> tuple[int, int] | None:
     try:
         root = psutil.Process(pid)
         if abs(root.create_time() - created) > SAME_START_SECONDS:
@@ -24,4 +24,4 @@ def tree_rss(pid: int, created: float) -> int | None:
     for process in tree:
         with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
             sizes.append(process.memory_info().rss)
-    return sum(sizes) if sizes else None
+    return (sum(sizes), len(sizes)) if sizes else None