tumwater 0.4.0 → 0.4.2

package/dist/src/agent/run-spawn.js

@@ -0,0 +1,226 @@
+/** The build-and-spawn stage of the agent runner (plans/agent-backends.md "The seam"): the
+ * part of a run before the stream pipeline — the `tumwater_run` raw-log marker, the stream
+ * parser, the run marker, backend-bin resolution, the notes shim, the run spec, the backend's
+ * optional `prepare`, the session-ledger record, the native-to-canonical translator, the two
+ * incremental decoders, and the child spawn. It returns the `SpawnedRun` bundle the coordinator
+ * (src/agent/run-agent.ts) binds its watchdogs and settlement wiring to, and passes to
+ * createAgentStream (src/agent/run-stream.ts).
+ * Part 1/2 of the runAgent split: the streaming pipeline (part 2/2) lives in run-stream.ts.
+ * This module imports only run-agent's type surface, so
+ * there is no runtime import cycle. */
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import { StringDecoder } from "node:string_decoder";
+import { ensureParentDir, rotateIfLarge } from "../files/files.js";
+import { terminateChild } from "../process/process.js";
+import { makeRunMarker } from "../process/run-marker.js";
+import { PiStreamParser } from "../pi/pi-stream.js";
+import { recordSession, sessionRecordFor } from "./session-ledger.js";
+import { errorMessage } from "../text/text.js";
+import { warnEventBestEffort, TUMWATER_SOURCE } from "../events/events.js";
+import { seamImageInput } from "./image-input.js";
+import { ensureNotesShim } from "./notes-shim.js";
+import { nearestTumwaterRoot } from "../paths.js";
+import { piModelsPath, readPiProviders } from "../pi/pi-models.js";
+export function openAppendLog(file, maxBytes) {
+    ensureParentDir(file);
+    rotateIfLarge(file, maxBytes);
+    const log = { stream: fs.createWriteStream(file, { flags: "a" }), broken: false };
+    log.stream.on("error", () => {
+        log.broken = true;
+    });
+    return log;
+}
+/** Invoke the backend and spawn the child from the pre-spawn pieces, returning the bundle. */
+function buildAndSpawn(backend, opts, pre) {
+    const { runSpec, parser, runMarker, resolved, blind, runTmp } = pre;
+    const invocation = backend.invocation(runSpec);
+    // The id the pre-spawn ledger record carries: a backend that mints one ahead of the run
+    // (claude, a codex resume), else the session name (pi, a fresh codex run — its id is born in
+    // the stream). The stream-mint pass later compares against it so a codex run appends its
+    // streamed thread id exactly once, and only when it differs from this.
+    const preSpawnSessionId = invocation.sessionId ?? opts.sessionName;
+    // Record the run in the session ledger before its spawn, so a later resume can pick the newest
+    // session of its own kind instead of whatever file is newest in the dir. The id is the backend's
+    // own when it mints one ahead of the run (claude's UUID, or the resumed record's id); pi mints
+    // none and keeps the session name.
+    recordSession(opts.sessionDir, sessionRecordFor(backend.id, preSpawnSessionId, opts));
+    const translator = backend.translator();
+    // Decode stdout incrementally instead of per chunk: a raw Buffer.toString("utf8") replaces any
+    // multi-byte character whose bytes straddle two 'data' events with U+FFFD, corrupting that
+    // line's text (commit subjects, summaries, transcripts). StringDecoder holds back the incomplete
+    // trailing bytes until the next chunk completes them.
+    const decoder = new StringDecoder("utf8");
+    // stderr gets its own incremental decoder for the same reason: the failure message and the
+    // stderr-keyed matchers (crash, stream-severed, config) read this buffer, so a character whose
+    // bytes straddle two 'data' events must not become U+FFFD.
+    const stderrDecoder = new StringDecoder("utf8");
+    const child = spawn(resolved.bin, invocation.args, {
+        cwd: opts.cwd,
+        // A backend that feeds the prompt on stdin (every registered one) gets a pipe the runner
+        // writes and ends below; one that returns no stdin keeps stdio[0] "ignore".
+        stdio: [invocation.stdin !== undefined ? "pipe" : "ignore", "pipe", "pipe"],
+        // pi sets its process title at startup, and so does every npm its tool calls run: on macOS
+        // each would register with LaunchServices and leak a launchservicesd port. Detached so pi
+        // leads its own process group: terminateChild signals the group, so a run's tool-call
+        // grandchildren — killed or finished — die with it instead of leaking to launchd.
+        detached: true,
+        // The run's mark, inherited by every tool call and its descendants: pi 0.87.1's bash tool
+        // spawns each command detached in its OWN group, so the group sweep below cannot reach what
+        // a tool call backgrounds (BUGS.md 2026-09-30) — the mark is the only attribution that
+        // follows a reparented orphan. Appended to any inherited mark so a nested harness under test
+        // keeps the outer run's.
+        env: { ...invocation.env, TMPDIR: runTmp },
+    });
+    if (invocation.stdin !== undefined) {
+        // A backend that dies before draining its stdin — an immediately-failing startup such as an
+        // unknown flag, an unresumable session, or a not-logged-in exit — leaves this write nothing
+        // to feed. Once the prompt exceeds the OS pipe buffer the write then fails EPIPE, and an
+        // unhandled 'error' event on the stdin stream is thrown from the event loop and takes the
+        // whole process (the whole team) down. The child's own exit status and stderr already report
+        // why the run died, so swallowing the failed write adds no silence the caller needs to break;
+        // git.ts's patchId carries the same guard for a patch-id that dies before reading its input.
+        child.stdin?.on("error", () => { });
+        child.stdin?.end(invocation.stdin);
+    }
+    // Armed now that the child exists, so the parser's parse-time onBlindRead (fires during feed,
+    // always after spawn) can reach it.
+    blind.terminate = () => terminateChild(child);
+    return {
+        child,
+        resolved,
+        runSpec,
+        invocation,
+        parser,
+        translator,
+        decoder,
+        stderrDecoder,
+        runMarker,
+        preSpawnSessionId,
+        blindRead: () => blind.read,
+    };
+}
+/** Build and spawn one non-interactive agent run, from the marker write through the child spawn,
+ * and return the bundle the run's stream pipeline, watchdogs and settlement wiring consume. The
+ * caller's setup (opened logs, the run's private TMPDIR) is passed in; a synchronous throw here
+ * propagates to the caller, which rejects the run's promise. The backend's optional `prepare` is
+ * awaited here and warned about (never thrown away) exactly as the coordinator once did.
+ *
+ * The return type is a union on purpose: a backend without a `prepare` step (pi, claude,
+ * opencode) builds and spawns synchronously and returns the bundle DIRECTLY, so the coordinator
+ * does not `await` it and its watchdogs (a setInterval registration) start in the same
+ * synchronous step — the ordering the logical-clock watchdog tests pin, which advance the fake
+ * clock immediately after the run promise is created. A backend WITH a `prepare` (codex)
+ * resolves its trust first and returns a promise that resolves to the bundle after the await. */
+export function spawnPiRun(backend, logs, runTmp, opts) {
+    const rawLog = logs.raw;
+    // The first blind image read an authoring run made (plans/image-input.md "Detecting a blind
+    // read"): the path, plus an indirection to the child so the parser's parse-time callback
+    // (which fires during feed, after spawn) can kill it. Unset for a gate run, which only records
+    // the read. Declared before the parser so its onBlindRead closure (which assigns them at parse
+    // time, after spawn) can reference them.
+    const blind = {};
+    // The marker precedes this run's first event in file order — written to the same stream
+    // stdout lines flow through, before spawn, so ordering is exact by construction; on a failed
+    // spawn finish() still ends the stream and flushes it. Exactly one per run: the kind, agent
+    // and start time are always there, the label only when the caller set one (`history
+    // <role> <n> --transcript` selects a run's segments by this ts).
+    {
+        const marker = {
+            type: "tumwater_run",
+            kind: opts.kind,
+            agent: backend.id,
+            ts: Date.now(),
+        };
+        if (opts.label)
+            marker.label = opts.label;
+        rawLog.stream.write(JSON.stringify(marker) + "\n");
+    }
+    const parser = new PiStreamParser(opts.onToolCallStart, {
+        // The seam's effective image input (src/agent/image-input.ts): a text-only seam makes every
+        // image read blind even when pi adds no note — the parser checks against its own note rule.
+        imageInput: seamImageInput(opts.config, readPiProviders(piModelsPath())),
+        onBlindRead: (path) => {
+            // Kill an authoring run at once, before the agent goes on working from a picture it never
+            // saw (terminateChild is what the watchdogs use). Gate runs (reviewer, resolver) only
+            // record the read and keep going.
+            if (opts.kind === "author") {
+                blind.read = path;
+                blind.terminate?.();
+            }
+        },
+    });
+    // The run's cross-group attribution mark, minted before the spawn so the environment can carry
+    // it and the exit sweep can name it.
+    const runMarker = makeRunMarker();
+    const resolved = backend.resolveBin(opts.config);
+    // Install the notes shim under the team root's `.tumwater/bin` so every backend's run finds
+    // `tumwater-notes` first on PATH (notes-shim.ts). A failure to write it logs one warning and
+    // the run goes on without it — notes would then be unwritable by the command, but the run
+    // itself is not blocked.
+    let binDir;
+    try {
+        // Install the shim only under a real team root — the nearest ancestor holding a `.tumwater`.
+        // A session dir with no such ancestor (a lab script, a smoke run, a prompt-lab replay) must
+        // never mint one: nearestTumwaterRoot is undefined then, and the run goes on without the
+        // shim, exactly as if the install had failed — writing `<fallback>/.tumwater` from a run
+        // created a phantom harness root in a shared directory and broke every other suite on the
+        // host (BUGS.md 2026-10-18).
+        const root = nearestTumwaterRoot(opts.sessionDir);
+        if (root !== undefined)
+            binDir = ensureNotesShim(root);
+    }
+    catch {
+        opts.onAgentWarning?.("could not install the tumwater-notes shim; this run will not find tumwater-notes on PATH");
+    }
+    const runSpec = {
+        cwd: opts.cwd,
+        prompt: opts.prompt,
+        config: opts.config,
+        sessionDir: opts.sessionDir,
+        sessionName: opts.sessionName,
+        continueSession: opts.continueSession,
+        kind: opts.kind,
+        agentBin: resolved.bin,
+        runMarker,
+        notesPath: opts.notesPath,
+        ...(binDir !== undefined ? { binDir } : {}),
+        ...(opts.maxBudgetUsd !== undefined ? { maxBudgetUsd: opts.maxBudgetUsd } : {}),
+    };
+    // A backend's asynchronous pre-run work (the codex write guard resolving its trust by spawning
+    // `app-server`, plans/codex-write-guard.md "Trust (parts 4–5)"): awaited just before the
+    // synchronous invocation, so trust is settled before invocation reads it. It never throws a run
+    // away — a throw is a warm-up failure, warned about on the run's own root (skipped with no root)
+    // and the run goes on; a codex trust lookup failure already records `off` and warns from prepare
+    // itself, so this catch is the guard for a backend whose prepare throws unexpectedly. The call
+    // itself is also guarded: a synchronous `prepare` throw would otherwise escape and (on the
+    // asynchronous path) leave the run's promise pending, so it is warned-and-continued like an
+    // async rejection.
+    const warnRoot = nearestTumwaterRoot(opts.sessionDir);
+    let prepared;
+    try {
+        prepared = backend.prepare?.(runSpec);
+    }
+    catch (error) {
+        if (warnRoot !== undefined) {
+            warnEventBestEffort(warnRoot, TUMWATER_SOURCE, `agent prepare failed, run continues: ${errorMessage(error)}`);
+        }
+    }
+    if (prepared === undefined) {
+        // No `prepare` step: build and spawn synchronously and hand the bundle straight back, so
+        // the coordinator's watchdog start lands in the same synchronous step (see the doc comment).
+        return buildAndSpawn(backend, opts, { runSpec, parser, runMarker, resolved, blind, runTmp });
+    }
+    // Only a backend with a `prepare` step (codex) awaits it, then builds and spawns.
+    return (async () => {
+        try {
+            await prepared;
+        }
+        catch (error) {
+            if (warnRoot !== undefined) {
+                warnEventBestEffort(warnRoot, TUMWATER_SOURCE, `agent prepare failed, run continues: ${errorMessage(error)}`);
+            }
+        }
+        return buildAndSpawn(backend, opts, { runSpec, parser, runMarker, resolved, blind, runTmp });
+    })();
+}

package/dist/src/agent/run-stream.js

@@ -0,0 +1,185 @@
+/** The streaming pipeline of the backend agent runner (plans/agent-backends.md "The seam"):
+ * everything the run does live with the child's stdout/stderr after spawn — the native-to-
+ * canonical stdout line splitting with its over-long-line discard, the stderr window and its
+ * fallback-clone warning matcher, the stream-minted session's second ledger record, and the
+ * settle/flush that resolves the run's promise. It is the part of the runAgent split (part
+ * 2/2) that the coordinator (src/agent/run-agent.ts) delegates to behind the `AgentStream`
+ * interface: run-agent wires the child's `data` events to `feedStdout`/`feedStderr`, binds the
+ * settlement state's stream getters, and keeps the watchdogs, abort wiring and the
+ * `error`/`exit`/`close` listeners itself. This module imports only run-agent's, run-spawn's
+ * and run-settle's type surfaces, so there is no runtime import cycle. */
+import { removeTree } from "../files/files.js";
+import { recordSession, sessionRecordFor } from "./session-ledger.js";
+/** How many trailing stderr characters are carried into the next chunk's warning match, so a
+ * warning split across two `data` events is still seen. The warnings are short lines, so a few
+ * hundred characters always cover a straddle. */
+const STDERR_MATCH_TAIL = 512;
+/** The largest native stdout line the runner will buffer. A complete native event is one JSON
+ * object; a line this long is already at the default whole-log cap (`logMaxBytes`, 16 MiB), so
+ * continuing to buffer it can only be a malformed stream. Past it `feedCanonical` drops the
+ * buffered line and skips input to the next newline, so a backend that streams bytes with no
+ * newline — each byte resetting the quiet watchdog — cannot grow the buffer until the run
+ * timeout (4 h by default) exhausts the harness's memory. */
+export const MAX_NATIVE_LINE_CHARS = 16 * 1024 * 1024;
+/** Build the streaming pipeline for one spawned run. The coordinator passes the bundle, its
+ * watchdogs, the abort wiring, the two logs the flush-at-settle writes, the run's TMPDIR it
+ * removes at settle, and the promise's `resolve` — everything the moved closures captured
+ * before the refactor — and this call returns the single `AgentStream` the coordinator binds
+ * the child's `data` events and the settlement state to. */
+export function createAgentStream(input) {
+    const { opts, backend, spawned, wd, onAbort, rawLog, nativeLog, runTmp, resolve } = input;
+    const { parser, translator, decoder, stderrDecoder, preSpawnSessionId } = spawned;
+    // The stream's mutable state (its own per-run locals, shared by exactly the closures below):
+    // the stderr window and its warning matcher tail, the settled guard, and the translator's
+    // native-line buffer.
+    let stderr = "";
+    // The fallback-clone warning arrives at the START of a run, while `stderr` keeps only the
+    // last 64 KiB; match each chunk (with a short carried tail for a straddle) so a chatty run
+    // that pushes the warning out of that window still surfaces the mispriced id.
+    let fallbackClone;
+    let stderrTail = "";
+    let settled = false;
+    // The translator's native line buffer: feed the decoder's text through it and only complete
+    // lines reach the translator, whose canonical output feeds the parser (and the raw log).
+    // `nativeOverflow` marks a buffered line that passed MAX_NATIVE_LINE_CHARS without a newline:
+    // its remainder is skipped to the next newline so it can never be mistaken for an event.
+    let nativeTail = "";
+    let nativeOverflow = false;
+    const onLine = (line) => rawLog.stream.write(line + "\n");
+    // The translator's session id, once the stream has named it (codex's `thread.started`).
+    // The pre-spawn record for a fresh codex run carries the session name; the id it actually
+    // ran with is born in the stream, so the first time this returns one the pre-spawn record
+    // lacked, append a second ledger record carrying it (same kind, name, cwd and model) — the
+    // record a later `exec resume` resumes (design §4 "Sessions").
+    let streamMintedSession = false;
+    const emitNativeLine = (line) => {
+        if (!nativeLog.broken)
+            nativeLog.stream.write(line + "\n");
+        for (const canonical of translator.translate(line))
+            parser.feed(canonical + "\n", onLine);
+        const sid = translator.sessionId?.();
+        if (sid !== undefined && sid !== preSpawnSessionId && !streamMintedSession) {
+            streamMintedSession = true;
+            recordSession(opts.sessionDir, sessionRecordFor(backend.id, sid, opts));
+        }
+    };
+    // Split native stdout into complete lines, translate each, and feed the canonical lines to
+    // the parser (which also hands them to onLine for the raw log). A trailing partial native
+    // line stays buffered exactly as the parser's own buffer once held it. On the final flush
+    // (`final`), a remaining tail is emitted as one line: a backend may close after flushing a
+    // complete JSON event but not its newline terminator, and dropping that tail loses the
+    // run's authoritative result (cost, final text). A genuinely truncated fragment parses as
+    // no data (parseJsonObject → []), so this can only recover the last event.
+    const feedCanonical = (text, final = false) => {
+        // A buffered overflow: discard this line's remainder up to (and including) its newline, so
+        // the stream resynchronizes on the next real event line instead of emitting a truncated one.
+        if (nativeOverflow) {
+            const nl = text.indexOf("\n");
+            if (nl < 0)
+                return;
+            nativeOverflow = false;
+            text = text.slice(nl + 1);
+        }
+        nativeTail += text;
+        let start = 0;
+        for (;;) {
+            const nl = nativeTail.indexOf("\n", start);
+            if (nl < 0)
+                break;
+            emitNativeLine(nativeTail.slice(start, nl));
+            start = nl + 1;
+        }
+        if (start > 0)
+            nativeTail = nativeTail.slice(start);
+        if (nativeTail.length > MAX_NATIVE_LINE_CHARS) {
+            // No newline within the cap: drop the line and warn. On the final flush there is no
+            // remainder to skip, so only the buffer is cleared.
+            nativeTail = "";
+            if (!final) {
+                nativeOverflow = true;
+                opts.onAgentWarning?.(`native output line exceeded ${MAX_NATIVE_LINE_CHARS / (1024 * 1024)} MiB without a newline and was discarded`);
+            }
+        }
+        else if (final && nativeTail !== "") {
+            emitNativeLine(nativeTail);
+            nativeTail = "";
+        }
+    };
+    const feedStdout = (chunk) => {
+        wd.noteByte();
+        feedCanonical(decoder.write(chunk));
+    };
+    const feedStderr = (chunk) => {
+        // stderr is rare and meaningful (crash traces, warnings): treat it as progress.
+        wd.noteStderr();
+        const text = stderrDecoder.write(chunk);
+        fallbackClone ??= backend.matchStderrWarning?.(text, stderrTail);
+        stderrTail = (stderrTail + text).slice(-STDERR_MATCH_TAIL);
+        stderr += text;
+        if (stderr.length > 64 * 1024)
+            stderr = stderr.slice(-64 * 1024);
+    };
+    const finish = (result) => {
+        if (settled)
+            return;
+        settled = true;
+        // A blind image read (plans/image-input.md "Detecting a blind read"): surface it to the
+        // caller once, at settle, with the action its kind earned — the authoring run was killed
+        // ("fail"), a gate run only recorded it ("logged"). The caller logs the image_blind event
+        // with its own role/run context.
+        if (parser.imageBlind) {
+            opts.onBlindRead?.(parser.imageBlind.path, opts.kind === "author" ? "fail" : "logged");
+        }
+        // Best-effort: an unremovable scratch dir (read-only subdir, a straggler still writing) must
+        // not throw out of the close handler before the run's result resolves.
+        try {
+            removeTree(runTmp);
+        }
+        catch {
+            /* left for the OS temp sweep */
+        }
+        wd.stop();
+        opts.signal?.removeEventListener("abort", onAbort);
+        // Surface the translator's per-run warnings once, after all native lines were fed.
+        for (const warning of translator.warnings?.() ?? [])
+            opts.onAgentWarning?.(warning);
+        // Resolve only once the logs have flushed: writes complete on libuv's threadpool, so
+        // resolving before 'finish' can leave the tail of <role>.pi.jsonl (or the native log)
+        // unwritten — a run that dies right after runAgent settles (an abort during a restart
+        // drain, a supervisor swap) loses its last lines. end()'s callback fires on 'finish';
+        // if a stream errors instead it never does, so settle from 'error' too — a broken log
+        // degrades to a lost log, never a stuck run.
+        let resolved = false;
+        const settle = () => {
+            if (!resolved) {
+                resolved = true;
+                resolve(result);
+            }
+        };
+        let pending = 0;
+        const flushed = () => {
+            pending -= 1;
+            if (pending === 0)
+                settle();
+        };
+        for (const log of [rawLog, nativeLog]) {
+            log.stream.on("error", settle);
+            if (!log.broken) {
+                pending += 1;
+                log.stream.end(flushed);
+            }
+        }
+        if (pending === 0)
+            settle();
+    };
+    return {
+        feedStdout,
+        feedStderr,
+        onLine,
+        feedCanonical,
+        finish,
+        settled: () => settled,
+        stderr: () => stderr,
+        fallbackClone: () => fallbackClone,
+    };
+}

package/dist/src/agent/session-ledger.js

@@ -38,6 +38,22 @@ export function recordSession(dir, record) {
         // Best effort: a lost ledger line costs resumability, not the run.
     }
 }
+/** The one run's ledger record, minus the id the two recording sites mint differently — the
+ * pre-spawn id in run-spawn.ts, or the stream-minted session id in run-stream.ts. One place
+ * for the record's non-id shape (agent, kind, name, cwd, the optional model and the stamp), so
+ * the two appends cannot drift on the model spread or the required fields. `agent` is the
+ * owning backend's id; the run options supply the rest. */
+export function sessionRecordFor(agent, id, opts) {
+    return {
+        agent,
+        id,
+        kind: opts.kind,
+        name: opts.sessionName,
+        cwd: opts.cwd,
+        ...(opts.config.model !== undefined ? { model: opts.config.model } : {}),
+        at: Date.now(),
+    };
+}
 /** True when `dir` holds at least one session file a backend could resume by directory. This is
  * the backend-neutral filesystem check behind pi's `--continue` and the fallback for an
  * unregistered backend. The ledger file shares the dir and is a `.jsonl` too, so it is

package/dist/src/agent/write-guard.js

@@ -1,5 +1,6 @@
 /** The write guard on a run's file-editing tools: pi's `edit`/`write` (src/pi-extension/
- * write-guard.ts), claude's `Edit`/`Write` (src/agent/claude/write-guard-hook.ts) and OpenCode's
+ * write-guard.ts), claude's `Edit`/`Write` (src/agent/claude/claude-write-guard-hook.ts) and
+ * OpenCode's
  * `edit` permission (src/agent/opencode/write-guard-plugin/index.ts) refuse a write that would
  * land under the team root but outside the run's own checkout.
  *
@@ -124,8 +125,9 @@ export function strayWriteRefusal(target, runDir, resolveFrom = runDir) {
         + (instead ?? "Leave it alone and work only inside the worktree.");
 }
 /** The stdin-driven guard-hook driver shared by the claude and codex PreToolUse hooks
- * (src/agent/claude/write-guard-hook.ts, src/agent/codex/write-guard-hook.ts). Each hook runs as
- * `node write-guard-hook.js` with the backend's hook payload JSON on stdin; this reads it in one
+ * (src/agent/claude/claude-write-guard-hook.ts, src/agent/codex/codex-write-guard-hook.ts). Each
+ * hook runs as `node claude-write-guard-hook.js` / `node codex-write-guard-hook.js` with the
+ * backend's hook payload JSON on stdin; this reads it in one
  * place, asks `computeRefusal` for a refusal string (null lets the edit/patch run), and prints the
  * `deny` decision only when there is a refusal. Any failure — unreadable stdin, malformed JSON, a
  * throw — reads as "no refusal" (null), so a broken hook cannot stop every edit of a run; the