acumen 0.0.1.dev0__py3-none-any.whl

acumen/logs.py

@@ -0,0 +1,255 @@
+"""Live agent logs for the meta-agents.
+
+``draft``/``improve``/``tasks``/``ship`` each spin up one long, autonomous agent whose transcript
+today is written into a throwaway ``CLAUDE_CONFIG_DIR`` and ``rmtree``'d on return — a black box.
+:class:`LiveLog` makes each run observable:
+
+* **A live JSONL**, appended and flushed *one normalized event per SDK message, during the run*,
+  at ``logs/acumen-<command>-<datetime>.jsonl``. This is the contract for a master agent watching
+  a run by **reading the file** on its own schedule — never a flood of the command's stdout into
+  its context ("print, don't read"). The feed is deliberately compact: it does **not** inline huge
+  tool results (file dumps, command output), only their status and size.
+* **A terminal mirror** when ``--stream`` is on (off by default): each turn's assistant text and a
+  one-line-per-tool-call summary printed live for a human watching.
+* **An HTML log** rendered at the end from the SDK-native transcript (:func:`finalize`), via the
+  same ``claude-code-log`` path the benchmark runner uses. The rich post-hoc artifact; the JSONL
+  is the lightweight live feed.
+
+The class is stateful (it correlates tool results back to the tool that produced them) but has no
+agent dependency, so it is unit-testable on a synthetic message sequence — like ``cli._Progress``.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from datetime import datetime
+from pathlib import Path
+from typing import Any, TextIO
+
+from claude_agent_sdk import (
+    AssistantMessage,
+    ResultMessage,
+    SystemMessage,
+    TextBlock,
+    ThinkingBlock,
+    ToolResultBlock,
+    ToolUseBlock,
+    UserMessage,
+)
+
+from acumen.transcript import locate_transcript, render_transcript
+
+#: tool_input keys, in priority order, whose value best summarizes a tool call in one line.
+_SUMMARY_KEYS = ("command", "file_path", "path", "notebook_path", "pattern", "url", "query", "description", "prompt")
+
+#: Thinking blocks can be huge; the live feed keeps only a short preview of them.
+_THINKING_CAP = 240
+#: How long a one-line tool-input summary may get before it is truncated.
+_SUMMARY_CAP = 160
+
+
+def _truncate(text: str, cap: int) -> str:
+    text = text.strip()
+    return text if len(text) <= cap else text[:cap] + "…"
+
+
+def _tool_summary(tool_input: Any) -> str:
+    """A one-line summary of a tool call's input — the salient argument, whitespace-collapsed."""
+    if not isinstance(tool_input, dict):
+        return ""
+    for key in _SUMMARY_KEYS:
+        value = tool_input.get(key)
+        if isinstance(value, str) and value.strip():
+            return _truncate(" ".join(value.split()), _SUMMARY_CAP)
+    parts = [f"{key}={tool_input[key]!r}" for key in list(tool_input)[:3]]
+    return _truncate(", ".join(parts), _SUMMARY_CAP)
+
+
+def _content_len(content: Any) -> int:
+    """Character length of a tool result's content — recorded instead of inlining the content."""
+    if isinstance(content, str):
+        return len(content)
+    if isinstance(content, list):
+        total = 0
+        for block in content:
+            if isinstance(block, dict):
+                total += len(str(block.get("text", "")))
+            else:
+                total += len(str(block))
+        return total
+    return 0
+
+
+class LiveLog:
+    """A live, append-only JSONL log of one meta-agent run, with optional terminal streaming.
+
+    Open one on ``logs/acumen-<command>-<datetime>.jsonl`` with :meth:`open`, feed every SDK
+    message to :meth:`append` inside the ``async for`` loop, and call :meth:`finalize` once the
+    run is done (before the throwaway config dir is deleted) to render the HTML log. Close it to
+    flush and release the file.
+    """
+
+    def __init__(
+        self,
+        jsonl_path: Path,
+        *,
+        stream: bool = False,
+        echo: Callable[[str], None] = print,
+        clock: Callable[[], str] | None = None,
+    ) -> None:
+        self.jsonl_path = Path(jsonl_path)
+        self.html_path = self.jsonl_path.with_suffix(".html")
+        self.stream = stream
+        self.html_rendered = False
+        self.n_events = 0
+        self._echo = echo
+        self._clock = clock or (lambda: datetime.now().isoformat(timespec="seconds"))
+        self._tool_names: dict[str, str] = {}
+        self.jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+        self._fh: TextIO = self.jsonl_path.open("w", encoding="utf-8")
+
+    @classmethod
+    def open(
+        cls,
+        log_dir: Path,
+        command: str,
+        *,
+        stream: bool = False,
+        now: datetime | None = None,
+        echo: Callable[[str], None] = print,
+    ) -> LiveLog:
+        """Open a log for ``command`` under ``log_dir`` with the fixed filename pattern.
+
+        The filename is ``acumen-<command>-<YYYYMMDD-HHMMSS>.jsonl``; the directory is the only
+        thing a caller overrides.
+        """
+        stamp = (now or datetime.now()).strftime("%Y%m%d-%H%M%S")
+        return cls(Path(log_dir) / f"acumen-{command}-{stamp}.jsonl", stream=stream, echo=echo)
+
+    def append(self, message: Any) -> None:
+        """Normalize one SDK message to a compact event, append it (flushed), and maybe echo it."""
+        event = self._normalize(message)
+        if event is None:
+            return
+        record = {"ts": self._clock(), **event}
+        self._fh.write(json.dumps(record) + "\n")
+        self._fh.flush()
+        self.n_events += 1
+        if self.stream:
+            for line in self._render_lines(record):
+                self._echo(line)
+
+    def finalize(self, *, config_dir: Path, work_dir: Path, result: ResultMessage | None) -> bool:
+        """Locate the run's SDK-native transcript and render the HTML log from it.
+
+        Call before the throwaway config dir is removed — the native transcript still lives under
+        it at this point. A no-op that returns ``False`` if the run produced no result.
+        """
+        if result is None:
+            return False
+        native = locate_transcript(config_dir, work_dir, result.session_id)
+        if native is None or not native.is_file():
+            return False
+        self.html_rendered = render_transcript(native, self.html_path)
+        return self.html_rendered
+
+    def close(self) -> None:
+        """Flush and release the underlying JSONL file. Idempotent."""
+        if not self._fh.closed:
+            self._fh.close()
+
+    def __enter__(self) -> LiveLog:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    # ── Normalization ──────────────────────────────────────────────────────────────────
+
+    def _normalize(self, message: Any) -> dict[str, Any] | None:
+        if isinstance(message, AssistantMessage):
+            return self._assistant_event(message)
+        if isinstance(message, UserMessage):
+            return self._user_event(message)
+        if isinstance(message, SystemMessage):
+            return {"type": "system", "subtype": message.subtype}
+        if isinstance(message, ResultMessage):
+            return {
+                "type": "result",
+                "is_error": bool(message.is_error),
+                "subtype": message.subtype,
+                "turns": message.num_turns,
+                "cost_usd": round(message.total_cost_usd or 0.0, 4),
+                "duration_s": round((message.duration_ms or 0) / 1000, 2),
+                "session_id": message.session_id,
+            }
+        return None
+
+    def _assistant_event(self, message: AssistantMessage) -> dict[str, Any]:
+        text_parts: list[str] = []
+        thinking_parts: list[str] = []
+        tools: list[dict[str, Any]] = []
+        for block in message.content:
+            if isinstance(block, TextBlock):
+                if block.text.strip():
+                    text_parts.append(block.text)
+            elif isinstance(block, ThinkingBlock):
+                thinking_parts.append(block.thinking)
+            elif isinstance(block, ToolUseBlock):
+                self._tool_names[block.id] = block.name
+                tools.append({"id": block.id, "name": block.name, "summary": _tool_summary(block.input)})
+        event: dict[str, Any] = {"type": "assistant"}
+        if message.model:
+            event["model"] = message.model
+        text = "\n".join(text_parts).strip()
+        if text:
+            event["text"] = text
+        if thinking_parts:
+            event["thinking"] = _truncate("\n".join(thinking_parts), _THINKING_CAP)
+        if tools:
+            event["tools"] = tools
+        return event
+
+    def _user_event(self, message: UserMessage) -> dict[str, Any] | None:
+        content = message.content
+        if not isinstance(content, list):
+            return None  # a plain user turn (our own prompt) — nothing worth an event
+        results: list[dict[str, Any]] = []
+        for block in content:
+            if isinstance(block, ToolResultBlock):
+                results.append(
+                    {
+                        "tool_use_id": block.tool_use_id,
+                        "name": self._tool_names.get(block.tool_use_id),
+                        "status": "error" if block.is_error else "ok",
+                        "chars": _content_len(block.content),
+                    }
+                )
+        return {"type": "tool_result", "results": results} if results else None
+
+    # ── Terminal mirror (--stream) ─────────────────────────────────────────────────────
+
+    def _render_lines(self, event: dict[str, Any]) -> list[str]:
+        kind = event.get("type")
+        lines: list[str] = []
+        if kind == "assistant":
+            if event.get("thinking"):
+                lines.append(f"  · thinking: {event['thinking']}")
+            if event.get("text"):
+                lines.append(str(event["text"]).strip())
+            for tool in event.get("tools", []):
+                summary = tool.get("summary") or ""
+                lines.append(f"  → {tool['name']}: {summary}".rstrip())
+        elif kind == "tool_result":
+            for res in event["results"]:
+                mark = "✗" if res["status"] == "error" else "✓"
+                name = res.get("name") or "tool"
+                lines.append(f"  {mark} {name} ({res['chars']} chars)")
+        elif kind == "result":
+            mark = "✗ error" if event["is_error"] else "● done"
+            lines.append(
+                f"{mark}: {event['subtype']} · {event['turns']} turns · "
+                f"${event['cost_usd']:.2f} · {event['duration_s']}s"
+            )
+        return lines

acumen/paths.py

@@ -0,0 +1,164 @@
+"""Run-directory layout — the single source of truth for path construction.
+
+The layout is ``runs/{arm}/{split}/{model}/{task_id}/rep_{n}/`` and is consumed by the
+runner, the resume check, the improver's allow/deny list, and the report. Anything that
+needs to know where a run lives asks this module.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+Split = Literal["train", "test"]
+
+SPLITS: tuple[Split, ...] = ("train", "test")
+
+NOSKILL_ARM = "noskill"
+
+ANSWER_FILE = "answer.md"
+SCRIPT_FILE = "script.py"
+TRANSCRIPT_JSONL = "transcript.jsonl"
+TRANSCRIPT_HTML = "transcript.html"
+RESULT_FILE = "result.json"
+
+#: Every file a completed run leaf is expected to contain.
+RUN_FILES = (ANSWER_FILE, SCRIPT_FILE, TRANSCRIPT_JSONL, TRANSCRIPT_HTML, RESULT_FILE)
+
+_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+")
+_ARM_RE = re.compile(r"^skill_v(\d+)$")
+_REP_RE = re.compile(r"^rep_(\d+)$")
+
+
+class PathError(ValueError):
+    """Raised when a path cannot be constructed or parsed."""
+
+
+def slugify(text: str) -> str:
+    """Make ``text`` safe to use as a single path component.
+
+    Parameters
+    ----------
+    text
+        Arbitrary text, e.g. a model name such as ``anthropic/claude-opus-5``.
+
+    Returns
+    -------
+    A string containing only ``[A-Za-z0-9._-]``.
+    """
+    slug = _UNSAFE.sub("-", text).strip("-")
+    if not slug or slug in {".", ".."}:
+        raise PathError(f"{text!r} has no filesystem-safe representation")
+    return slug
+
+
+def is_safe_component(text: str) -> bool:
+    """Return whether ``text`` is already usable as a path component verbatim."""
+    return bool(text) and text not in {".", ".."} and not _UNSAFE.search(text)
+
+
+def arm_name(skill: str | None) -> str:
+    """Return the run-directory arm name for a skill version.
+
+    Parameters
+    ----------
+    skill
+        A skill version such as ``"v1"``, or ``None`` for the baseline arm.
+
+    Returns
+    -------
+    ``"noskill"`` for the baseline, otherwise ``"skill_v1"``-style.
+    """
+    if skill is None:
+        return NOSKILL_ARM
+    version = skill[1:] if skill.startswith("v") else skill
+    if not version.isdigit():
+        raise PathError(f"skill version must look like 'v1', got {skill!r}")
+    return f"skill_v{int(version)}"
+
+
+def skill_from_arm(arm: str) -> str | None:
+    """Invert :func:`arm_name` — return the skill version an arm refers to."""
+    if arm == NOSKILL_ARM:
+        return None
+    match = _ARM_RE.match(arm)
+    if not match:
+        raise PathError(f"not an arm name: {arm!r}")
+    return f"v{int(match.group(1))}"
+
+
+@dataclass(frozen=True)
+class RunKey:
+    """Identifies exactly one benchmark run — one leaf of the run tree."""
+
+    arm: str
+    split: Split
+    model: str
+    task_id: str
+    rep: int
+
+    @property
+    def skill(self) -> str | None:
+        """The skill version this run exercises, or ``None`` for the baseline."""
+        return skill_from_arm(self.arm)
+
+
+def run_dir(runs_root: Path, key: RunKey) -> Path:
+    """Build the directory for a single run.
+
+    Parameters
+    ----------
+    runs_root
+        The ``runs/`` root directory.
+    key
+        Which run to locate.
+
+    Returns
+    -------
+    ``runs_root/{arm}/{split}/{model}/{task_id}/rep_{n}``.
+    """
+    if key.split not in SPLITS:
+        raise PathError(f"split must be one of {SPLITS}, got {key.split!r}")
+    if key.rep < 1:
+        raise PathError(f"rep must be >= 1, got {key.rep}")
+    if not is_safe_component(key.task_id):
+        raise PathError(f"task id {key.task_id!r} is not filesystem-safe")
+    return runs_root / key.arm / key.split / slugify(key.model) / key.task_id / f"rep_{key.rep}"
+
+
+def parse_run_dir(runs_root: Path, path: Path) -> RunKey:
+    """Recover a :class:`RunKey` from a run directory.
+
+    The model is returned in its slugified form, which is what the path carries; the
+    canonical model name lives in ``result.json``.
+    """
+    try:
+        rel = path.resolve().relative_to(runs_root.resolve())
+    except ValueError as err:
+        raise PathError(f"{path} is not under {runs_root}") from err
+    parts = rel.parts
+    if len(parts) != 5:
+        raise PathError(f"expected {runs_root}/arm/split/model/task/rep_n, got {rel}")
+    arm, split, model, task_id, rep = parts
+    skill_from_arm(arm)  # validates the arm name
+    if split not in SPLITS:
+        raise PathError(f"split must be one of {SPLITS}, got {split!r}")
+    match = _REP_RE.match(rep)
+    if not match:
+        raise PathError(f"expected 'rep_<n>', got {rep!r}")
+    return RunKey(arm=arm, split=split, model=model, task_id=task_id, rep=int(match.group(1)))  # type: ignore[arg-type]
+
+
+def is_complete(directory: Path) -> bool:
+    """Return whether a run directory holds a complete run.
+
+    A readable ``result.json`` is what marks a run done — this is the check that makes
+    resuming a pass work.
+    """
+    result = directory / RESULT_FILE
+    try:
+        return result.is_file() and result.stat().st_size > 0
+    except OSError:
+        return False

acumen/procs.py

@@ -0,0 +1,195 @@
+"""Killing the processes an agent leaves behind.
+
+An agent is a ``claude`` CLI subprocess, and that CLI spawns processes of its own: every
+Bash tool call is a shell, and a shell can start a pytest, a fit that runs for an hour, or
+something backgrounded with ``&``. The SDK terminates the CLI and nothing else — so does a
+crash, a cap breach, or a Ctrl-C — and the descendants survive. They keep burning CPU
+alongside the runs still going, and they keep writing into a directory that is about to be
+deleted.
+
+The parent link cannot find them, on any platform. On Unix an orphan is reparented to init,
+so the tree that identified it is gone; on Windows it is not reparented at all and its
+recorded parent id dangles, free to be reused by an unrelated process. What survives in
+both cases is the environment: :func:`label_env` stamps every agent with :data:`RUN_MARKER`,
+set to that run's private temp directory, and a process inherits its environment from
+whatever started it. So the marker identifies a descendant even after the parent is gone,
+the process has renamed itself, or it has changed directory. :func:`reap` finds everything
+carrying a given run's marker and kills it; the sandbox and the meta-agent staging dirs call
+it on the way out, before their directory is removed, on every exit path including
+cancellation.
+
+The marker is what makes this exact rather than a guess at which paths a run touches. The
+benchmark sandboxes do put their throwaway ``HOME`` and ``TMPDIR`` under the same directory,
+but ``ship``'s agent deliberately runs in the operator's real environment and real checkout,
+where no such path appears — one explicit variable covers both.
+
+Reading *another* process's environment is the one step with no portable stdlib spelling:
+it is a file read on Linux, a ``KERN_PROCARGS2`` sysctl on macOS, and a ``ReadProcessMemory``
+walk of the PEB on Windows. ``psutil`` implements all three natively, which is why it is a
+dependency. :func:`supported` reports whether the running platform is one of them.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from pathlib import Path
+
+import psutil
+
+#: Environment variable naming the run a process belongs to, set to the run's temp directory.
+#: Stamped on every agent by :func:`label_env` and inherited by everything the agent spawns.
+#: Uppercase deliberately: Windows environment keys are case-insensitive and psutil upper-cases
+#: them when reading a process block, so an all-caps name is the one spelling that round-trips
+#: identically on every platform.
+RUN_MARKER = "ACUMEN_RUN"
+
+#: How long to let a signalled process exit before escalating to a hard kill. An upper bound,
+#: not a fixed pause — the wait ends the moment the processes are gone, which is the common
+#: case. It still needs to stay modest: the callers are synchronous teardown paths inside
+#: async runs, so this blocks the event loop and every other run on it.
+GRACE_S = 2.0
+
+#: Raised by psutil when a process is gone, is another user's, or is unreadable for a
+#: platform-specific reason (on Windows a 32-bit interpreter cannot read a 64-bit process's
+#: environment). Every one of these means "not identifiable as ours", never "abort the scan".
+_UNREADABLE = (psutil.Error, OSError, ValueError)
+
+
+def supported() -> bool:
+    """Whether orphan reaping works on this platform.
+
+    True on Linux, macOS and Windows. False only where psutil cannot read another process's
+    environment, in which case :func:`reap` does nothing and orphans are left alone.
+    """
+    return hasattr(psutil.Process, "environ")
+
+
+def label_env(env: dict[str, str], holder: Path) -> dict[str, str]:
+    """Stamp an agent's environment with the run it belongs to, and return it.
+
+    Call this on the mapping handed to the SDK, with the same ``holder`` later passed to
+    :func:`reap`. Without the stamp a run's orphans are unidentifiable and :func:`reap` has
+    nothing to match on.
+
+    Parameters
+    ----------
+    env
+        The agent's environment, modified in place.
+    holder
+        The run's private temp directory, which is unique per run and so serves as its id.
+
+    Returns
+    -------
+    The same mapping, for chaining onto the call that built it.
+    """
+    env[RUN_MARKER] = str(holder)
+    return env
+
+
+def _protected() -> set[int]:
+    """Return acumen's own pid and every ancestor of it.
+
+    A belt-and-braces guard. Matching is on a variable acumen itself never has in its
+    environment, so these should never come up as candidates — but the cost of a false
+    positive here is killing the process doing the killing, or the operator's shell.
+    """
+    me = psutil.Process()
+    protected = {me.pid}
+    try:
+        protected.update(parent.pid for parent in me.parents())
+    except _UNREADABLE:  # pragma: no cover - an ancestor exiting mid-walk
+        pass
+    return protected
+
+
+def _labelled(proc: psutil.Process, holder: Path) -> bool:
+    """Whether ``proc`` belongs to the run rooted at ``holder``.
+
+    The environment psutil reports is the one the process was *executed* with, which is why
+    the marker still identifies a descendant whose parent has since died.
+
+    ``cwd`` is checked as a second signal, for the rare process started with a wiped
+    environment that is nonetheless running inside the run's directory. Path comparison is
+    case-insensitive on Windows, which :class:`~pathlib.Path` already handles.
+    """
+    try:
+        if proc.environ().get(RUN_MARKER) == str(holder):
+            return True
+    except _UNREADABLE:
+        pass
+    try:
+        cwd = proc.cwd()
+    except _UNREADABLE:
+        return False
+    return bool(cwd) and Path(cwd).is_relative_to(holder)
+
+
+def _survivor_procs(holder: Path) -> Iterator[psutil.Process]:
+    """Yield live processes descended from the agent that ran in ``holder``."""
+    if not supported():
+        return
+    protected = _protected()
+    for proc in psutil.process_iter():
+        # Processes come and go mid-scan; one that vanished is simply not a survivor.
+        if proc.pid not in protected and _labelled(proc, holder):
+            yield proc
+
+
+def survivors(holder: Path) -> list[int]:
+    """Return the pids of every live process descended from the agent that ran in ``holder``.
+
+    Parameters
+    ----------
+    holder
+        The run's temp directory, the same one passed to :func:`label_env`.
+
+    Returns
+    -------
+    The matching pids, excluding acumen and its ancestors. Empty on an unsupported platform.
+    """
+    return [proc.pid for proc in _survivor_procs(holder)]
+
+
+def _quietly(action: Callable[[], None]) -> None:
+    """Run a kill that may lose its race with the process exiting on its own."""
+    try:
+        action()
+    except _UNREADABLE:
+        pass
+
+
+def reap(holder: Path, *, grace: float = GRACE_S) -> list[int]:
+    """Kill everything the agent that ran in ``holder`` left behind.
+
+    Asks politely first (``SIGTERM``; on Windows, where there is no such signal, psutil
+    terminates outright), then kills whatever is still standing after ``grace``. The second
+    round re-scans rather than reusing the first round's list, so a process spawned while
+    the first signals were landing is caught too.
+
+    Safe to call when nothing leaked, when the agent exited cleanly, and on a platform
+    psutil cannot read — in each case it does nothing and returns an empty list.
+
+    Parameters
+    ----------
+    holder
+        The run's temp directory, as passed to :func:`survivors`.
+    grace
+        Seconds to wait before the hard kill. Blocks the caller, but only until the
+        signalled processes are actually gone.
+
+    Returns
+    -------
+    The pids that were signalled, for logging. An empty list means nothing was orphaned.
+    """
+    doomed = list(_survivor_procs(holder))
+    if not doomed:
+        return []
+    for proc in doomed:
+        _quietly(proc.terminate)
+    # Waiting on a process that is our own direct child consumes its exit status, which
+    # would otherwise be the SDK's to collect. Harmless here: callers reap only once the
+    # SDK has closed its transport, so the agent process has already been waited on.
+    psutil.wait_procs(doomed, timeout=grace)
+    for proc in _survivor_procs(holder):
+        _quietly(proc.kill)
+    return [proc.pid for proc in doomed]