labnotes 0.0.126__tar.gz → 0.0.127__tar.gz

{labnotes-0.0.126 → labnotes-0.0.127}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: labnotes
-Version: 0.0.126
+Version: 0.0.127
 Summary: Browse experiment designs, notes, and run data from a folder.
 Project-URL: Repository, https://github.com/thekevinscott/labnotes
 Author-email: Kevin Scott <me@thekevinscott.com>

{labnotes-0.0.126 → labnotes-0.0.127}/src/labnotes/context.md

@@ -130,7 +130,10 @@ Runs can take a long time. Start `run` in the background, then check on it with
 
 While a run goes, labnotes appends a memory sample to `memory.jsonl` in its run
 folder every 5 seconds: one JSON line, `{"time": <unix seconds>, "rss": <bytes>,
-"processes": <count>}`, for the command and everything it started. It keeps
+"processes": <count>}`, for the command's process tree and its labeled Docker
+containers. `LABNOTES_RUN` is the absolute run folder path; scripts label containers
+with `docker run --label "labnotes.run=$LABNOTES_RUN" ...` to count their memory.
+`processes` counts host processes in the run's tree, not containers. It keeps
 every sample, so a run killed for memory still shows the climb.
 `memory_interval` in `experiment.toml` (or `[experiment]` in `.labnotes.toml`)
 sets the seconds between samples; 0 turns it off.

{labnotes-0.0.126 → labnotes-0.0.127}/src/labnotes/log_path.py

@@ -1,8 +1,8 @@
 import os
 from pathlib import Path
 
-# The one env contract between runner and script; config is otherwise never env.
 LOG_ENV = "LABNOTES_LOG"
+RUN_ENV = "LABNOTES_RUN"
 
 
 def log_path() -> Path | None:

labnotes-0.0.127/src/labnotes/runner/cgroup_memory.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+def cgroup_memory(path: Path) -> int:
+    current = int((path / "memory.current").read_text())
+    try:
+        stats = (path / "memory.stat").read_text().splitlines()
+    except OSError:
+        return current
+    for line in stats:
+        name, _, value = line.partition(" ")
+        if name == "inactive_file":
+            try:
+                return max(0, current - int(value))
+            except ValueError:
+                return current
+    return current

labnotes-0.0.127/src/labnotes/runner/container_cgroup.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+CGROUP_ROOT = Path("/sys/fs/cgroup")
+PROC_ROOT = Path("/proc")
+
+
+def container_cgroup(pid: int) -> Path | None:
+    try:
+        lines = (PROC_ROOT / str(pid) / "cgroup").read_text().splitlines()
+    except OSError:
+        return None
+    for line in lines:
+        if line.startswith("0::"):
+            return CGROUP_ROOT / line.removeprefix("0::").lstrip("/")
+    return None

labnotes-0.0.127/src/labnotes/runner/container_memory.py

@@ -0,0 +1,100 @@
+import asyncio
+import shutil
+import time
+from pathlib import Path
+
+from labnotes.runner.cgroup_memory import cgroup_memory
+from labnotes.runner.container_cgroup import container_cgroup
+from labnotes.runner.docker_output import docker_output
+
+LABEL = "labnotes.run"
+
+
+class ContainerMemory:
+    """Track this run's Docker cgroups without starting a CLI process per sample."""
+
+    def __init__(self, run_id: str) -> None:
+        self.run_id = run_id
+        self.cgroups: dict[str, Path] = {}
+        self.events: asyncio.subprocess.Process | None = None
+        self.watcher: asyncio.Task[None] | None = None
+
+    async def start(self) -> None:
+        if shutil.which("docker") is None:
+            return
+        since = str(int(time.time()))
+        ids = await asyncio.to_thread(
+            docker_output,
+            "ps",
+            "--quiet",
+            "--no-trunc",
+            "--filter",
+            f"label={LABEL}={self.run_id}",
+        )
+        if ids is None:
+            return
+        for container_id in ids.splitlines():
+            await self._add(container_id)
+        try:
+            self.events = await asyncio.create_subprocess_exec(
+                "docker",
+                "events",
+                "--since",
+                since,
+                "--filter",
+                "type=container",
+                "--filter",
+                f"label={LABEL}={self.run_id}",
+                "--format",
+                "{{.Actor.ID}} {{.Action}}",
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.DEVNULL,
+            )
+        except OSError:
+            return
+        self.watcher = asyncio.create_task(self._watch())
+
+    async def _add(self, container_id: str) -> None:
+        raw_pid = await asyncio.to_thread(
+            docker_output, "inspect", "--format", "{{.State.Pid}}", container_id
+        )
+        if raw_pid is None:
+            return
+        try:
+            pid = int(raw_pid.strip())
+        except ValueError:
+            return
+        if pid > 0 and (path := container_cgroup(pid)) is not None:
+            self.cgroups[container_id] = path
+
+    async def _watch(self) -> None:
+        assert self.events is not None
+        assert self.events.stdout is not None
+        while line := await self.events.stdout.readline():
+            container_id, _, action = line.decode().strip().partition(" ")
+            if action == "start":
+                await self._add(container_id)
+            elif action in {"die", "destroy", "stop"}:
+                self.cgroups.pop(container_id, None)
+
+    def usage(self) -> int:
+        total = 0
+        for path in self.cgroups.values():
+            try:
+                total += cgroup_memory(path)
+            except (OSError, ValueError):
+                pass
+        return total
+
+    async def close(self) -> None:
+        if self.watcher is not None:
+            self.watcher.cancel()
+            await asyncio.gather(self.watcher, return_exceptions=True)
+        if self.events is not None:
+            if self.events.returncode is None:
+                self.events.terminate()
+            try:
+                await asyncio.wait_for(self.events.wait(), timeout=1)
+            except TimeoutError:
+                self.events.kill()
+                await self.events.wait()

labnotes-0.0.127/src/labnotes/runner/docker_output.py

@@ -0,0 +1,11 @@
+import subprocess
+
+
+def docker_output(*args: str) -> str | None:
+    try:
+        result = subprocess.run(
+            ["docker", *args], capture_output=True, text=True, timeout=3, check=False
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    return result.stdout if result.returncode == 0 else None

{labnotes-0.0.126 → labnotes-0.0.127}/src/labnotes/runner/execute_run.py

@@ -7,7 +7,7 @@ from pathlib import Path
 
 import psutil
 
-from labnotes.log_path import LOG_ENV
+from labnotes.log_path import LOG_ENV, RUN_ENV
 from labnotes.runner.exit_signal import exit_signal
 from labnotes.runner.record_memory import record_memory
 from labnotes.runner.shell_command import shell_command
@@ -66,6 +66,7 @@ async def execute_run(
                     "CLICOLOR_FORCE": "1",
                     **os.environ,
                     LOG_ENV: str(folder / "log.jsonl"),
+                    RUN_ENV: str(folder.resolve()),
                 },
                 stdout=stdout,
                 stderr=stderr,

labnotes-0.0.127/src/labnotes/runner/record_memory.py

@@ -0,0 +1,28 @@
+import asyncio
+import json
+import time
+from pathlib import Path
+
+from labnotes.runner.container_memory import ContainerMemory
+from labnotes.utils.tree_usage import tree_usage
+
+
+async def record_memory(path: Path, pid: int, created: float, interval: float) -> None:
+    containers = ContainerMemory(str(path.parent.resolve()))
+    try:
+        await containers.start()
+        with path.open("a", encoding="utf-8") as samples:
+            while True:
+                usage = tree_usage(pid, created)
+                if usage is not None:
+                    rss, processes = usage
+                    sample = {
+                        "time": time.time(),
+                        "rss": rss + containers.usage(),
+                        "processes": processes,
+                    }
+                    samples.write(json.dumps(sample) + "\n")
+                    samples.flush()
+                await asyncio.sleep(interval)
+    finally:
+        await containers.close()

labnotes-0.0.126/src/labnotes/runner/record_memory.py

@@ -1,18 +0,0 @@
-import asyncio
-import json
-import time
-from pathlib import Path
-
-from labnotes.utils.tree_usage import tree_usage
-
-
-async def record_memory(path: Path, pid: int, created: float, interval: float) -> None:
-    with path.open("a", encoding="utf-8") as samples:
-        while True:
-            usage = tree_usage(pid, created)
-            if usage is not None:
-                rss, processes = usage
-                sample = {"time": time.time(), "rss": rss, "processes": processes}
-                samples.write(json.dumps(sample) + "\n")
-                samples.flush()
-            await asyncio.sleep(interval)