labnotes 0.0.126__tar.gz → 0.0.128__tar.gz

{labnotes-0.0.126 → labnotes-0.0.128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.126
+Version: 0.0.128
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.126 → labnotes-0.0.128}/src/labnotes/context.md

@@ -72,12 +72,15 @@ Each run gets a new run folder. In `matrix` mode it is named for the UTC time it
 its folder the same way. One that plans more gets one folder named that way, and its runs nest
 inside it (`runs/20261020-000429-brave-fox/quiet-river`); the run id is that path under `runs/`. Either way, folder listings sort by start. `command` runs in a shell from the root, with
 `{run_dir}` replaced by the run folder's absolute path. Pass it to the
-script so it writes its outputs there.
+script so it writes its outputs there. `{run_id}` becomes the run id, and
+`{invocation_id}` an id every run of one `run` command shares: the run id when
+it plans one run, else the adhoc invocation folder's name, or in `matrix` mode a
+stamped id that names no folder. All three are shell-quoted.
 Each param is appended as `--name value`, with the name as written (use
 `source-language` for a `--source-language` flag), booleans as 0 or 1, and lists
 and tables as JSON. labnotes writes `run.json` (params, `command`, created,
 ended, exit code, `ended_by`, `signal`), `stdout.log` and `stderr.log` to the
-run folder. `command` is the shell command the run executed, with `{run_dir}`
+run folder. `command` is the shell command the run executed, with the placeholders
 and the params filled in, written before the run starts, so a killed run has it
 and a later edit to `experiment.toml` doesn't hide what a run did. `ended_by` is `"timeout"`, `"stop"` (labnotes stopped it),
 `"signal"` (a signal labnotes didn't send, like the out-of-memory killer),
@@ -93,7 +96,9 @@ the value from the start, even if the code changed while they waited.
 commands run once per `run` from the root: setup before the first run starts,
 teardown after the last one ends, even if runs failed, were stopped, or setup
 failed (so teardown must cope with a partial setup). A failing setup starts no
-run; a failing teardown fails the invocation.
+run; a failing teardown fails the invocation. Both get `{invocation_id}`, and
+`{run_id}` only when one run is planned; otherwise `{run_id}` there fails the
+invocation before anything starts.
 A run counts when it exited 0 and its `run.json` doesn't hold
 `"excluded": true`, so excluding a run by hand means adding that key; the next
 `run` replaces it. Other keys you add to `run.json` are kept and show in `run`.
@@ -130,7 +135,10 @@ Runs can take a long time. Start `run` in the background, then check on it with
 
 While a run goes, labnotes appends a memory sample to `memory.jsonl` in its run
 folder every 5 seconds: one JSON line, `{"time": <unix seconds>, "rss": <bytes>,
-"processes": <count>}`, for the command and everything it started. It keeps
+"processes": <count>}`, for the command's process tree and its labeled Docker
+containers. `LABNOTES_RUN` is the absolute run folder path; scripts label containers
+with `docker run --label "labnotes.run=$LABNOTES_RUN" ...` to count their memory.
+`processes` counts host processes in the run's tree, not containers. It keeps
 every sample, so a run killed for memory still shows the climb.
 `memory_interval` in `experiment.toml` (or `[experiment]` in `.labnotes.toml`)
 sets the seconds between samples; 0 turns it off.

{labnotes-0.0.126 → labnotes-0.0.128}/src/labnotes/log_path.py

@@ -1,8 +1,8 @@
 import os
 from pathlib import Path
 
-# The one env contract between runner and script; config is otherwise never env.
 LOG_ENV = "LABNOTES_LOG"
+RUN_ENV = "LABNOTES_RUN"
 
 
 def log_path() -> Path | None:

labnotes-0.0.128/src/labnotes/runner/cgroup_memory.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+def cgroup_memory(path: Path) -> int:
+    current = int((path / "memory.current").read_text())
+    try:
+        stats = (path / "memory.stat").read_text().splitlines()
+    except OSError:
+        return current
+    for line in stats:
+        name, _, value = line.partition(" ")
+        if name == "inactive_file":
+            try:
+                return max(0, current - int(value))
+            except ValueError:
+                return current
+    return current

labnotes-0.0.128/src/labnotes/runner/container_cgroup.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+CGROUP_ROOT = Path("/sys/fs/cgroup")
+PROC_ROOT = Path("/proc")
+
+
+def container_cgroup(pid: int) -> Path | None:
+    try:
+        lines = (PROC_ROOT / str(pid) / "cgroup").read_text().splitlines()
+    except OSError:
+        return None
+    for line in lines:
+        if line.startswith("0::"):
+            return CGROUP_ROOT / line.removeprefix("0::").lstrip("/")
+    return None

labnotes-0.0.128/src/labnotes/runner/container_memory.py

@@ -0,0 +1,100 @@
+import asyncio
+import shutil
+import time
+from pathlib import Path
+
+from labnotes.runner.cgroup_memory import cgroup_memory
+from labnotes.runner.container_cgroup import container_cgroup
+from labnotes.runner.docker_output import docker_output
+
+LABEL = "labnotes.run"
+
+
+class ContainerMemory:
+    """Track this run's Docker cgroups without starting a CLI process per sample."""
+
+    def __init__(self, run_id: str) -> None:
+        self.run_id = run_id
+        self.cgroups: dict[str, Path] = {}
+        self.events: asyncio.subprocess.Process | None = None
+        self.watcher: asyncio.Task[None] | None = None
+
+    async def start(self) -> None:
+        if shutil.which("docker") is None:
+            return
+        since = str(int(time.time()))
+        ids = await asyncio.to_thread(
+            docker_output,
+            "ps",
+            "--quiet",
+            "--no-trunc",
+            "--filter",
+            f"label={LABEL}={self.run_id}",
+        )
+        if ids is None:
+            return
+        for container_id in ids.splitlines():
+            await self._add(container_id)
+        try:
+            self.events = await asyncio.create_subprocess_exec(
+                "docker",
+                "events",
+                "--since",
+                since,
+                "--filter",
+                "type=container",
+                "--filter",
+                f"label={LABEL}={self.run_id}",
+                "--format",
+                "{{.Actor.ID}} {{.Action}}",
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.DEVNULL,
+            )
+        except OSError:
+            return
+        self.watcher = asyncio.create_task(self._watch())
+
+    async def _add(self, container_id: str) -> None:
+        raw_pid = await asyncio.to_thread(
+            docker_output, "inspect", "--format", "{{.State.Pid}}", container_id
+        )
+        if raw_pid is None:
+            return
+        try:
+            pid = int(raw_pid.strip())
+        except ValueError:
+            return
+        if pid > 0 and (path := container_cgroup(pid)) is not None:
+            self.cgroups[container_id] = path
+
+    async def _watch(self) -> None:
+        assert self.events is not None
+        assert self.events.stdout is not None
+        while line := await self.events.stdout.readline():
+            container_id, _, action = line.decode().strip().partition(" ")
+            if action == "start":
+                await self._add(container_id)
+            elif action in {"die", "destroy", "stop"}:
+                self.cgroups.pop(container_id, None)
+
+    def usage(self) -> int:
+        total = 0
+        for path in self.cgroups.values():
+            try:
+                total += cgroup_memory(path)
+            except (OSError, ValueError):
+                pass
+        return total
+
+    async def close(self) -> None:
+        if self.watcher is not None:
+            self.watcher.cancel()
+            await asyncio.gather(self.watcher, return_exceptions=True)
+        if self.events is not None:
+            if self.events.returncode is None:
+                self.events.terminate()
+            try:
+                await asyncio.wait_for(self.events.wait(), timeout=1)
+            except TimeoutError:
+                self.events.kill()
+                await self.events.wait()

labnotes-0.0.128/src/labnotes/runner/docker_output.py

@@ -0,0 +1,11 @@
+import subprocess
+
+
+def docker_output(*args: str) -> str | None:
+    try:
+        result = subprocess.run(
+            ["docker", *args], capture_output=True, text=True, timeout=3, check=False
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    return result.stdout if result.returncode == 0 else None

{labnotes-0.0.126 → labnotes-0.0.128}/src/labnotes/runner/execute_run.py

@@ -7,7 +7,7 @@ from pathlib import Path
 
 import psutil
 
-from labnotes.log_path import LOG_ENV
+from labnotes.log_path import LOG_ENV, RUN_ENV
 from labnotes.runner.exit_signal import exit_signal
 from labnotes.runner.record_memory import record_memory
 from labnotes.runner.shell_command import shell_command
@@ -66,6 +66,7 @@ async def execute_run(
                     "CLICOLOR_FORCE": "1",
                     **os.environ,
                     LOG_ENV: str(folder / "log.jsonl"),
+                    RUN_ENV: str(folder.resolve()),
                 },
                 stdout=stdout,
                 stderr=stderr,

labnotes-0.0.128/src/labnotes/runner/fill_ids.py

@@ -0,0 +1,8 @@
+import shlex
+from collections.abc import Mapping
+
+
+def fill_ids(command: str, ids: Mapping[str, str]) -> str:
+    for name, value in ids.items():
+        command = command.replace(f"{{{name}}}", shlex.quote(value))
+    return command

labnotes-0.0.128/src/labnotes/runner/hook_ids.py

@@ -0,0 +1,14 @@
+def hook_ids(
+    hooks: dict[str, str | None], invocation: str, runs: list[str]
+) -> dict[str, str]:
+    match runs:
+        case [run]:
+            return {"run_id": run, "invocation_id": invocation}
+    for hook, command in hooks.items():
+        if command is not None and "{run_id}" in command:
+            raise ValueError(
+                f"{hook} command {command} uses {{run_id}}, but this invocation plans "
+                f"{len(runs)} runs. {{run_id}} needs exactly one; "
+                "use {invocation_id} instead."
+            )
+    return {"invocation_id": invocation}

labnotes-0.0.128/src/labnotes/runner/record_memory.py

@@ -0,0 +1,28 @@
+import asyncio
+import json
+import time
+from pathlib import Path
+
+from labnotes.runner.container_memory import ContainerMemory
+from labnotes.utils.tree_usage import tree_usage
+
+
+async def record_memory(path: Path, pid: int, created: float, interval: float) -> None:
+    containers = ContainerMemory(str(path.parent.resolve()))
+    try:
+        await containers.start()
+        with path.open("a", encoding="utf-8") as samples:
+            while True:
+                usage = tree_usage(pid, created)
+                if usage is not None:
+                    rss, processes = usage
+                    sample = {
+                        "time": time.time(),
+                        "rss": rss + containers.usage(),
+                        "processes": processes,
+                    }
+                    samples.write(json.dumps(sample) + "\n")
+                    samples.flush()
+                await asyncio.sleep(interval)
+    finally:
+        await containers.close()

{labnotes-0.0.126 → labnotes-0.0.128}/src/labnotes/runner/run_experiment.py

@@ -13,6 +13,8 @@ from labnotes.models import (
 )
 from labnotes.retries import RETRY_EXIT_CODE, RETRY_SECONDS
 from labnotes.runner.execute_run import execute_run
+from labnotes.runner.fill_ids import fill_ids
+from labnotes.runner.hook_ids import hook_ids
 from labnotes.runner.in_subset import in_subset
 from labnotes.runner.invocation_hook import invocation_hook
 from labnotes.runner.invocation_meta import invocation_meta
@@ -77,6 +79,14 @@ async def run_experiment(
         run = new_id(params)
         planned.add(run)
         plan.append((run, params))
+    match plan:
+        case [(invocation, _)]:
+            pass
+        case _:
+            invocation = group or new_stamped_run_id(taken)
+    planned.add(invocation)
+    hooks = {"setup": experiment.setup, "teardown": experiment.teardown}
+    ids = hook_ids(hooks, invocation, [run for run, _ in plan])
     runs = [{"run": run, "params": params} for run, params in plan]
     report(
         {
@@ -212,7 +222,7 @@ async def run_experiment(
         result = await execute_run(
             config.root,
             folder,
-            experiment.command,
+            fill_ids(experiment.command, {"run_id": run, "invocation_id": invocation}),
             params,
             running[run],
             stopping,
@@ -237,14 +247,16 @@ async def run_experiment(
     async def teardown() -> None:
         if experiment.teardown is not None:
             report({"event": "tearing down"})
-            await invocation_hook(config.root, "teardown", experiment.teardown)
+            await invocation_hook(
+                config.root, "teardown", fill_ids(experiment.teardown, ids)
+            )
 
     listen(Controls(interrupt, set_concurrency, kill, finish))
     # Like vitest's globalSetup: a failed setup still gets its teardown.
     try:
         if experiment.setup is not None:
             report({"event": "setting up"})
-            await invocation_hook(config.root, "setup", experiment.setup)
+            await invocation_hook(config.root, "setup", fill_ids(experiment.setup, ids))
         fill()
         await done.wait()
         await asyncio.gather(*started)

labnotes-0.0.126/src/labnotes/runner/record_memory.py

@@ -1,18 +0,0 @@
-import asyncio
-import json
-import time
-from pathlib import Path
-
-from labnotes.utils.tree_usage import tree_usage
-
-
-async def record_memory(path: Path, pid: int, created: float, interval: float) -> None:
-    with path.open("a", encoding="utf-8") as samples:
-        while True:
-            usage = tree_usage(pid, created)
-            if usage is not None:
-                rss, processes = usage
-                sample = {"time": time.time(), "rss": rss, "processes": processes}
-                samples.write(json.dumps(sample) + "\n")
-                samples.flush()
-            await asyncio.sleep(interval)