pi-durable-subagents 1.0.16 → 1.0.18

package/dist/orchestrator/executor/index.js

@@ -7,12 +7,13 @@
 // session-corrupt{call,line}: a malformed native session line was skipped (once per line, E4).
 // fence-failed{exec,error} is documented in sweep.ts. The orchestrator ledger owns hold/release{pool,slot,exec}
 // and mem{available,admitted,exec} (every admission; a repeated refusal at most every 30 s per call). All transitions are serialized before publication.
-import { mkdir, open, readdir, readFile, rm, unlink, writeFile } from "node:fs/promises";
+import { mkdir, open, readdir, readFile, realpath, rm, unlink, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
-import { join } from "node:path";
+import { delimiter, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { CT, JT, attentionEntries, isEntry } from "../../types.js";
-import { callDir, callInbox, callSession, outboxRoot } from "../../paths.js";
+import { binDir, callDir, callInbox, callSession, journalPath, outboxRoot } from "../../paths.js";
+import { readJournalSnapshot } from "../../kernel/journal.js";
 import { Outbox } from "../../kernel/mailbox.js";
 import { contentHash, forwardRid } from "../../kernel/ids.js";
 import { capacity, seal } from "../../kernel/guards.js";
@@ -25,10 +26,11 @@ import { hibernation, openQuestion } from "./hibernate.js";
 import { emptyLedger, foldLedger, holdings as holdingsOf, settingsOf } from "../ledger.js";
 import { evidence, fatalProviderError, quotaExhausted, refusedByProvider, forgetSession, readSessionState, receiptId, sessionModel } from "./session.js";
 import { activeTotal } from "./time.js";
+import { entriesOf, trackedState, usageIds } from "./indexes.js";
 import { observeExecution } from "./observe.js";
 import { availableMemory } from "./memory.js";
 import { reached, sessionUsage, totalUsage } from "./usage.js";
-import { recordFenceFailure, resolveFenceAttention, serialContainment, skipLostCandidate, sweepExecutions } from "./sweep.js";
+import { indexSweep, recordFenceFailure, resolveFenceAttention, serialContainment, skipLostCandidate, sweepExecutions } from "./sweep.js";
 import { gateRetired } from "./effects/gate.js";
 import { WorktreeIndex, worktreeCalls, worktreeLabel, worktreePair, worktreeRoots } from "./worktree.js";
 const MEM_RECORD_MS = 30000;
@@ -44,7 +46,7 @@ const extension = fileURLToPath(new URL(`../../agent/extension.${import.meta.url
 const entriesFor = (journal, call) => journal.entries().filter(e => e.call === call);
 const current = (journal, call) => entriesFor(journal, call).findLast(e => e.type === JT.exec)?.exec;
 const sealed = (journal, call) => entriesFor(journal, call).find(e => e.type === JT.sealed)?.result;
-const has = (journal, type, exec) => journal.entries().some(e => e.type === type && e.exec === exec);
+const has = (journal, type, exec) => entriesOf(journal, type, exec).length > 0;
 /** The rid of the model request a follow-up naming a model makes (P12): derived, so withdrawing or replacing the
  *  follow-up withdraws its model request too. */
 export const modelRid = (rid) => contentHash([rid, "model"]);
@@ -147,17 +149,19 @@ export default function createExecutor(ledgers, options = {}) {
         wake();
     }
     function tracked(journal, exec) {
-        return journal.entries().filter(e => e.type === "tracked" && e.exec === exec).map(e => ({ pid: Number(e.pid), ppid: 0, start: String(e.start) }));
+        return trackedState(journal, exec).rows;
     }
+    /** Persist every newly seen identity, tagged or not: a process that later clears its tag and leaves the tree must
+     *  still be fenced after an orchestrator restart. */
     async function track(journal, exec, found) {
         const known = tracked(journal, exec);
-        found ??= (await containment.scan(new Map([[exec, known]]))).get(exec) ?? [];
+        found ??= (await containment.scan(new Map([[exec, known]]), { maxAgeMs: config.k?.trackerMs ?? 1000 })).get(exec) ?? [];
         await serial(async () => {
-            const ids = new Set(tracked(journal, exec).map(p => `${p.pid}:${p.start}`));
+            const { ids } = trackedState(journal, exec);
             for (const p of found)
                 if (p.start && !ids.has(`${p.pid}:${p.start}`)) {
                     await journal.append("tracked", { exec, pid: p.pid, start: p.start });
-                    ids.add(`${p.pid}:${p.start}`);
+                    trackedState(journal, exec);
                 }
         });
         return found;
@@ -339,18 +343,15 @@ export default function createExecutor(ledgers, options = {}) {
     }
     async function recordUsage(t, values) {
         await serial(async () => {
-            const recorded = new Set(t.journal.entries().filter(e => e.type === "usage" && e.call === t.callId).map(e => e.id));
             for (const u of values)
-                if (!recorded.has(u.id)) {
+                if (!usageIds(t.journal, t.callId).has(u.id))
                     await t.journal.append("usage", { call: t.callId, ...u });
-                    recorded.add(u.id);
-                }
             await workflowReached(t);
         });
     }
     async function workflowReached(t) {
-        const hit = reached(totalUsage(t.journal.entries()), t.workflowBudget), id = `budget:${t.wid}`;
-        if (hit && !t.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === id))
+        const hit = reached(totalUsage(entriesOf(t.journal, "usage")), t.workflowBudget), id = `budget:${t.wid}`;
+        if (hit && !entriesOf(t.journal, JT.attention).some(e => isEntry(e, JT.attention) && e.item.id === id))
             await t.journal.append(JT.attention, { item: { id, rev: 1, kind: "budget", text: "Workflow budget reached", wid: t.wid } });
         return hit;
     }
@@ -436,7 +437,9 @@ export default function createExecutor(ledgers, options = {}) {
             const timer = setInterval(() => { checking = checking.then(check); }, config.k?.trackerMs ?? 1000);
             try {
                 await check();
-                result = await effects.beforeSeal(a.ticket, exec, result, { signal: a.controller.signal });
+                await launchGate(a, exec, "gate");
+                if (!interrupted(a))
+                    result = await effects.beforeSeal(a.ticket, exec, result, { signal: a.controller.signal });
                 await check();
             }
             catch (error) {
@@ -445,6 +448,7 @@ export default function createExecutor(ledgers, options = {}) {
             finally {
                 clearInterval(timer);
                 await checking;
+                a.live = undefined;
             }
         }
         const value = await serial(async () => {
@@ -481,9 +485,114 @@ export default function createExecutor(ledgers, options = {}) {
     }
     const stopped = (a) => a.stopped || entriesFor(a.ticket.journal, a.ticket.callId).some(e => e.type === "stop-intent");
     const interrupted = (a) => closed || a.suspended || a.retired || stopped(a);
+    // Restart (RestartBody): while one is decided no execution passes this gate, so the executions it finds live are all
+    // there are; one that passes is live until its fence (or the end of its gates). Synchronous from the check to the mark.
+    let paused = false;
+    const unpaused = new Set();
+    async function launchGate(a, exec, phase) {
+        while (paused && !interrupted(a)) {
+            await new Promise(resolve => {
+                const timer = setTimeout(done, config.k?.trackerMs ?? 1000);
+                function done() { clearTimeout(timer); unpaused.delete(done); resolve(); }
+                unpaused.add(done);
+                a.wake = done;
+            });
+        }
+        if (!interrupted(a))
+            a.live = { exec, since: Date.now(), phase };
+    }
+    // Writer lock: one call that writes in a worktree runs there at a time. Ownership is durable (writer-hold/-release in
+    // the orchestrator ledger) and lasts from the first admission to the call's seal or retirement, across executions,
+    // hibernation and orchestrator restarts. Waiters are admitted in the order they first waited.
+    const writerWaits = new Map();
+    /** A waiter recovered from the journal keeps its place this long for its call to come back after a restart. */
+    const RECOVERED_WAIT_MS = 60_000;
+    async function writerRoot(t, cwd) {
+        if (t.spec.isolation === "worktree")
+            return;
+        const tools = t.spec.tools ?? t.agent.tools;
+        if (!(t.spec.writer ?? (!tools || tools.some(n => n === "edit" || n === "write"))))
+            return;
+        return await roots(cwd, "x") ?? await realpath(cwd).catch(() => cwd);
+    }
+    /** Whether every execution of `call` was fenced: until then its processes may still write. */
+    function writerFenced(entries, call) {
+        const fenced = new Set(entries.filter(e => e.type === JT.fenced).map(e => String(e.exec)));
+        return entries.every(e => e.type !== JT.exec || callOf(String(e.exec)) !== call || fenced.has(String(e.exec)));
+    }
+    /** An owner whose call ended (sealed, retired, revised away, its workflow done or pruned) and whose executions were
+     *  all fenced no longer holds the lock. */
+    function writerEnded(call) {
+        if (active.has(call))
+            return false;
+        const { wid } = address(call), rev = Number(/^.*@(\d+)\//.exec(call)?.[1]);
+        const journal = journals.get(wid), entries = journal && !journal.closed ? journal.entries() : readJournalSnapshot(journalPath(home, wid));
+        if (!entries.length)
+            return true;
+        if (!writerFenced(entries, call))
+            return false;
+        const revised = entries.findLastIndex(e => e.type === "revised");
+        if (revised >= 0 && Number(entries[revised].revision) > rev)
+            return true;
+        return entries.some((e, i) => (e.type === JT.sealed || e.type === "retired") && e.call === call || i > revised && e.type === JT.done);
+    }
+    /** Serial sections only: whether `a` may write in `root` now; records the wait (durably, for status) when not. */
+    async function writerAdmit(a, root) {
+        if ((settings().writerLock ?? "queue") === "off")
+            return true;
+        const t = a.ticket, call = t.callId;
+        let owner = folded().writers.get(root)?.call;
+        if (owner && owner !== call && writerEnded(owner)) {
+            await orch.append("writer-release", { root, call: owner, ended: true });
+            owner = undefined;
+        }
+        if (owner === call)
+            return true;
+        const mine = writerWaits.get(call) ?? { root, since: t.journal.entries().find(e => e.type === "writer-wait" && e.call === call)?.ts ?? Date.now() };
+        writerWaits.set(call, mine);
+        delete mine.recovered;
+        const waiting = (c, w) => active.has(c) ? !interrupted(active.get(c))
+            : w.recovered !== undefined && Date.now() - w.recovered < RECOVERED_WAIT_MS && !writerEnded(c);
+        const earlier = [...writerWaits].filter(([c, w]) => c !== call && w.root === root && (w.since < mine.since || w.since === mine.since && c < call) && waiting(c, w)).map(([c]) => c);
+        const holder = owner ?? earlier[0];
+        if (!holder) {
+            await orch.append("writer-hold", { root, call });
+            writerWaits.delete(call);
+            const waited = t.journal.entries().findLast(e => (e.type === "writer-wait" || e.type === "writer-acquired") && e.call === call);
+            if (waited?.type === "writer-wait") {
+                await t.journal.append("writer-acquired", { call, root });
+                const item = attentionEntries(t.journal.entries()).find(x => x.item.id === `writer:${call}`)?.item;
+                if (item && !t.journal.entries().some(e => e.type === JT.attentionResolved && e.id === item.id && e.rev === item.rev))
+                    await t.journal.append(JT.attentionResolved, { id: item.id, rev: item.rev, resolution: "acquired" });
+            }
+            return true;
+        }
+        const last = t.journal.entries().findLast(e => (e.type === "writer-wait" || e.type === "writer-acquired") && e.call === call);
+        if (mine.holder !== holder && !(last?.type === "writer-wait" && last.holder === holder && last.root === root)) {
+            await t.journal.append("writer-wait", { call, root, holder, ...(owner ? {} : { queued: true }) });
+        }
+        mine.holder = holder;
+        const id = `writer:${call}`;
+        if (!t.journal.entries().some(e => isEntry(e, JT.attention) && e.item.id === id)) {
+            const h = address(holder);
+            const text = `${worktreeLabel(call)} waits for the writer lock of ${root}: ${h.wid}/${h.key} ${owner ? "holds it" : "waits for it first"}. ` +
+                `It starts when that call ends. To run it now: stop one of them, run it with writer:false (when it does not write there) or isolation:"worktree"`;
+            await t.journal.append(JT.attention, { item: { id, rev: 1, kind: "conflict", text, wid: t.wid, call } });
+        }
+        return false;
+    }
+    async function writerRelease(call) {
+        await serial(async () => {
+            writerWaits.delete(call);
+            for (const [root, owner] of folded().writers)
+                if (owner.call === call)
+                    await orch.append("writer-release", { root, call });
+        });
+        wake();
+    }
     /** Wait for a slot. The candidates are decided again on every attempt, inside the serial section that also applies
      *  config.json changes: a queued call follows a reloaded defaultModel, pool or limit, never a mix of two versions. */
-    async function acquire(a, exec, decide) {
+    async function acquire(a, exec, decide, writer) {
         let continuation = false;
         for (;;) {
             let signal;
@@ -491,6 +600,8 @@ export default function createExecutor(ledgers, options = {}) {
             const chosen = await serial(async () => {
                 if (interrupted(a) || await workflowReached(a.ticket))
                     return;
+                if (writer && !await writerAdmit(a, writer))
+                    return;
                 const decision = await decide(), models = decision.candidates, pool = decision.pool;
                 continuation = decision.continuation;
                 for (const model of models) {
@@ -737,7 +848,7 @@ export default function createExecutor(ledgers, options = {}) {
         const t = a.ticket, journal = t.journal;
         const session = callSession(home, t.wid, t.key, t.gen), dir = callDir(home, t.wid, t.key, t.gen);
         const makeResult = (status, output = "", error) => buildCallResult({ key: t.key, gen: t.gen, status, output, ...(error ? { error } : {}) });
-        let cwd = t.cwd;
+        let cwd = t.cwd, writer;
         if (sealed(journal, t.callId)) {
             const result = sealed(journal, t.callId);
             await effects.afterSeal(t, result);
@@ -840,7 +951,7 @@ export default function createExecutor(ledgers, options = {}) {
             }
             let decision;
             try {
-                decision = await acquire(a, exec, () => launchModel(t, entries, previous));
+                decision = await acquire(a, exec, () => launchModel(t, entries, previous), writer ??= await writerRoot(t, cwd));
             }
             catch (error) {
                 await fence(journal, exec, { park: a });
@@ -883,6 +994,7 @@ export default function createExecutor(ledgers, options = {}) {
                 await sender.send(t.callId, "continue", { message: continueMessage(dangling) }, undefined, { rid: contentHash([exec, "dispatch"]) });
             else
                 await sender.send(t.callId, "task", { message: t.opening?.message ?? t.spec.task }, undefined, { rid: contentHash([exec, "dispatch"]) });
+            await launchGate(a, exec, "child");
             if (interrupted(a)) {
                 await fence(journal, exec, { park: a });
                 return finish(journal, t.callId, exec, makeResult("stopped"));
@@ -890,10 +1002,11 @@ export default function createExecutor(ledgers, options = {}) {
             let child;
             try {
                 child = await containment.spawn({ exec, command: "pi", args: [...buildPiArgs(t.agent, t.spec, { sessionPath: session, systemPromptPath: prompt, continuation: decision.continuation, controlTools: t.spec.schema === undefined ? ["ask"] : ["ask", "report"], ...(model.id ? { model } : {}) }), "-e", extension], cwd,
-                    env: { DSA_HOME: home, DSA_EXEC: exec, DSA_CALL: t.callId, DSA_INBOX: inbox(t.callId), DSA_JOURNAL: journal.path, ...(t.spec.schema !== undefined ? { DSA_SCHEMA: schema } : {}), ...(t.spec.budget ? { DSA_BUDGET: JSON.stringify(t.spec.budget) } : {}), ...(model.provider && model.id ? { DSA_MODEL: `${model.provider}/${model.id}` } : {}) } });
+                    env: { DSA_HOME: home, PATH: [binDir(home), process.env.PATH].filter(Boolean).join(delimiter), DSA_EXEC: exec, DSA_CALL: t.callId, DSA_INBOX: inbox(t.callId), DSA_JOURNAL: journal.path, ...(t.spec.schema !== undefined ? { DSA_SCHEMA: schema } : {}), ...(t.spec.budget ? { DSA_BUDGET: JSON.stringify(t.spec.budget) } : {}), ...(model.provider && model.id ? { DSA_MODEL: `${model.provider}/${model.id}` } : {}) } });
             }
             catch (error) {
                 await fence(journal, exec, { park: a });
+                a.live = undefined;
                 return finish(journal, t.callId, exec, makeResult("failed", "", `Spawn failed: ${String(error)}`));
             }
             try {
@@ -921,6 +1034,7 @@ export default function createExecutor(ledgers, options = {}) {
             }
             finally {
                 await fence(journal, exec, { child, park: a });
+                a.live = undefined;
             }
         }
     }
@@ -954,8 +1068,14 @@ export default function createExecutor(ledgers, options = {}) {
                 try {
                     if (exec && await fence(ticket.journal, exec))
                         await release(exec);
+                    // The writer lock outlives executions and restarts; it ends with the call (or its retirement).
+                    // Not before every execution is fenced: a later sweep proves it, and the next writer's admission releases it.
+                    if ((sealed(ticket.journal, ticket.callId) || a.retired || ticket.journal.entries().some(e => e.type === "retired" && e.call === ticket.callId))
+                        && writerFenced(ticket.journal.entries(), ticket.callId))
+                        await writerRelease(ticket.callId);
                 }
                 finally {
+                    writerWaits.delete(ticket.callId);
                     active.delete(ticket.callId);
                     collected.delete(ticket.callId);
                     for (const e of inUse.keys())
@@ -1150,6 +1270,7 @@ export default function createExecutor(ledgers, options = {}) {
         },
         async recover(wid, journal) {
             journals.set(wid, journal);
+            indexSweep(journal);
             await serial(() => resolveWorktrees());
             await effects.recover(journal);
             for (const e of journal.entries().filter(e => e.type === JT.exec)) {
@@ -1158,6 +1279,17 @@ export default function createExecutor(ledgers, options = {}) {
                 if (await fence(journal, exec))
                     await release(exec);
             }
+            // Writers that waited before the restart keep their order while their calls come back.
+            for (const e of journal.entries()) {
+                const call = String(e.call);
+                if (e.type === "writer-wait" && !writerWaits.has(call))
+                    writerWaits.set(call, { root: String(e.root), since: e.ts, recovered: Date.now() });
+                else if (e.type === "writer-acquired" || e.type === JT.sealed || e.type === "retired") {
+                    const w = writerWaits.get(call);
+                    if (w?.recovered !== undefined)
+                        writerWaits.delete(call);
+                }
+            }
             const calls = new Set(journal.entries().filter(e => e.type === JT.exec).map(e => String(e.call)));
             for (const call of calls) {
                 const session = await readCall(journal, call), values = sessionUsage(session, call);
@@ -1214,6 +1346,12 @@ export default function createExecutor(ledgers, options = {}) {
             }
         },
         busy: () => active.size > 0,
+        quiesce() {
+            paused = true;
+            const live = [...active.values()].filter(a => a.live).map(a => ({ wid: a.ticket.wid, key: a.ticket.key, gen: a.ticket.gen, callId: a.ticket.callId, ...a.live }));
+            return { live, resume: () => { paused = false; for (const fn of [...unpaused])
+                    fn(); } };
+        },
         suspend(only) {
             if (only) { // scoped (one session's quit, one workflow): other workflows keep running and dispatching
                 const targets = [...active.values()].filter(a => only(a.ticket.wid));

package/dist/orchestrator/executor/indexes.js

@@ -0,0 +1,63 @@
+const indexes = new WeakMap();
+export function entriesOf(journal, type, exec) {
+    let index = indexes.get(journal);
+    if (!index) {
+        index = { count: 0, types: new Map(), execs: new Map() };
+        indexes.set(journal, index);
+    }
+    const entries = journal.entries();
+    while (index.count < entries.length) {
+        const e = entries[index.count++], bucket = index.types.get(e.type) ?? [];
+        if (!index.types.has(e.type))
+            index.types.set(e.type, bucket);
+        bucket.push(e);
+        if (typeof e.exec === "string") {
+            const byExec = index.execs.get(e.type) ?? new Map();
+            if (!index.execs.has(e.type))
+                index.execs.set(e.type, byExec);
+            const owned = byExec.get(e.exec) ?? [];
+            if (!byExec.has(e.exec))
+                byExec.set(e.exec, owned);
+            owned.push(e);
+        }
+    }
+    return (exec === undefined ? index.types.get(type) : index.execs.get(type)?.get(exec)) ?? [];
+}
+const identities = new WeakMap();
+export function trackedState(journal, exec) {
+    let byExec = identities.get(journal);
+    if (!byExec) {
+        byExec = new Map();
+        identities.set(journal, byExec);
+    }
+    let state = byExec.get(exec);
+    if (!state) {
+        state = { count: 0, rows: [], ids: new Set() };
+        byExec.set(exec, state);
+    }
+    const entries = entriesOf(journal, "tracked", exec);
+    while (state.count < entries.length) {
+        const e = entries[state.count++], pid = Number(e.pid), start = String(e.start), id = `${pid}:${start}`;
+        if (!state.ids.has(id)) {
+            state.ids.add(id);
+            state.rows.push({ pid, ppid: 0, start });
+        }
+    }
+    return state;
+}
+const usage = new WeakMap();
+export function usageIds(journal, call) {
+    let state = usage.get(journal);
+    if (!state) {
+        state = { count: 0, calls: new Map() };
+        usage.set(journal, state);
+    }
+    const entries = entriesOf(journal, "usage");
+    while (state.count < entries.length) {
+        const e = entries[state.count++], key = String(e.call), ids = state.calls.get(key) ?? new Set();
+        if (!state.calls.has(key))
+            state.calls.set(key, ids);
+        ids.add(String(e.id));
+    }
+    return state.calls.get(call) ?? new Set();
+}

package/dist/orchestrator/executor/observe.js

@@ -9,8 +9,10 @@ import { JT, attentionEntries } from "../../types.js";
 import { callDir, callSession } from "../../paths.js";
 import { monotoneTime } from "../../kernel/guards.js";
 import { readSession } from "./session.js";
+import { entriesOf } from "./indexes.js";
 import { ActiveTime, activeTotal } from "./time.js";
 import { observation, reached, sessionUsage, totalUsage } from "./usage.js";
+export const sessionChanged = (name) => name === null || String(name) === "session.jsonl" || String(name) === "inbox";
 /** A tool call's command for a status line: bash's command, else its arguments, on one line and clipped. */
 export function toolCommand(args) {
     const a = args && typeof args === "object" ? args : {};
@@ -23,6 +25,8 @@ export async function observeExecution(d) {
     const { home, config, ticket: t, exec, child, serial } = d;
     const session = callSession(home, t.wid, t.key, t.gen), clock = new ActiveTime(), started = clock.last;
     let progress = started, providerError;
+    // Diagnostic metadata only: checkpoint the latest received stream update, never its content.
+    let lastStream;
     const tools = new Set();
     // The tool calls running now, for the stall text: a silent long command and a stuck call read differently.
     const running = new Map();
@@ -31,14 +35,14 @@ export async function observeExecution(d) {
     let signal;
     const boundary = new Promise(resolve => { signal = resolve; d.setWake(resolve); });
     let failure, pending = Promise.resolve(), ending = false;
-    const has = (type) => t.journal.entries().some(e => e.type === type && e.exec === exec);
+    const has = (type) => entriesOf(t.journal, type, exec).length > 0;
     const enqueue = (fn) => { if (!ending)
         pending = pending.then(fn).catch(error => { failure = error; signal(); }); };
     const saveTime = () => serial(async () => {
-        const previous = t.journal.entries().findLast(e => e.type === "time" && e.exec === exec);
+        const previous = entriesOf(t.journal, "time", exec).at(-1);
         if (!monotoneTime({ active: Number(previous?.active ?? 0) }, { active: clock.active }))
             throw new Error("Nonmonotone active time");
-        await t.journal.append("time", { exec, active: clock.active });
+        await t.journal.append("time", { exec, active: clock.active, ...(lastStream ? { lastStream } : {}) });
     });
     const limits = async () => {
         if (t.spec.timeoutMs !== undefined && prior + clock.active >= t.spec.timeoutMs) {
@@ -46,18 +50,23 @@ export async function observeExecution(d) {
                 await t.journal.append("timeout-intent", { exec, call: t.callId }); });
             signal();
         }
-        if (reached(totalUsage(t.journal.entries(), t.callId), t.spec.budget))
+        if (reached(totalUsage(entriesOf(t.journal, "usage"), t.callId), t.spec.budget))
             signal();
     };
+    const openAlert = (id) => {
+        const last = attentionEntries(entriesOf(t.journal, JT.attention), id).at(-1)?.item;
+        return last && !entriesOf(t.journal, JT.attentionResolved).some(e => e.id === id && e.rev === last.rev);
+    };
+    const progressDue = () => !clock.asking && !tools.size && performance.now() - progress >= (config.k?.progressMs ?? 600000);
     const stall = () => serial(async () => {
         const id = `stall:${t.callId}`;
-        const items = attentionEntries(t.journal.entries(), id), last = items.at(-1)?.item;
-        const open = last && !t.journal.entries().some(e => e.type === JT.attentionResolved && e.id === id && e.rev === last.rev);
+        const items = attentionEntries(entriesOf(t.journal, JT.attention), id), last = items.at(-1)?.item;
+        const open = last && !entriesOf(t.journal, JT.attentionResolved).some(e => e.id === id && e.rev === last.rev);
         const fresh = items.at(-1)?.exec === exec ? clock.last > Number(items.at(-1)?.horizon) : clock.last > started;
         if (open && fresh)
             await t.journal.append(JT.attentionResolved, { id, rev: last.rev, resolution: "activity" });
-        else if (!open && !clock.asking && performance.now() - clock.last >= (config.k?.stallMs ?? 600000))
-            await t.journal.append(JT.attention, { exec, horizon: clock.last, item: { id, rev: (last?.rev ?? 0) + 1, kind: "stall", text: `${t.wid}/${t.key}: no execution activity for ${Math.floor((performance.now() - clock.last) / 60000)}m` + runningText(), wid: t.wid, call: t.callId } });
+        else if (!open && !clock.asking && !progressDue() && !openAlert(`noprogress:${t.callId}`) && performance.now() - clock.last >= (config.k?.stallMs ?? 600000))
+            await t.journal.append(JT.attention, { exec, horizon: clock.last, ...(lastStream ? { lastStream } : {}), item: { id, rev: (last?.rev ?? 0) + 1, kind: "stall", text: `${t.wid}/${t.key}: no execution activity observed for ${Math.floor((performance.now() - clock.last) / 60000)}m` + runningText(), wid: t.wid, call: t.callId } });
     });
     /** "; running bash `make matrix` for 14m (no output or CPU use seen)": a silent long command, not a stuck model. */
     const runningText = () => {
@@ -68,15 +77,15 @@ export async function observeExecution(d) {
     };
     const noProgress = () => serial(async () => {
         const id = `noprogress:${t.callId}`;
-        const items = attentionEntries(t.journal.entries(), id), last = items.at(-1)?.item;
-        const open = last && !t.journal.entries().some(e => e.type === JT.attentionResolved && e.id === id && e.rev === last.rev);
+        const items = attentionEntries(entriesOf(t.journal, JT.attention), id), last = items.at(-1)?.item;
+        const open = last && !entriesOf(t.journal, JT.attentionResolved).some(e => e.id === id && e.rev === last.rev);
         const fresh = items.at(-1)?.exec === exec ? progress > Number(items.at(-1)?.horizon) : progress > started;
         if (open && fresh)
             await t.journal.append(JT.attentionResolved, { id, rev: last.rev, resolution: "progress" });
-        else if (!open && !clock.asking && !tools.size && performance.now() - progress >= (config.k?.progressMs ?? 600000)) {
-            const text = `${t.wid}/${t.key}: running but no progress for ${Math.floor((performance.now() - progress) / 60000)}m (no output tokens or tool results)` +
+        else if (!open && progressDue() && !openAlert(`stall:${t.callId}`)) {
+            const text = `${t.wid}/${t.key}: no new output or tool progress received for ${Math.floor((performance.now() - progress) / 60000)}m; the model may still be processing` +
                 (providerError ? `; last provider error: ${providerError.slice(0, 200)}` : "");
-            await t.journal.append(JT.attention, { exec, horizon: progress, item: { id, rev: (last?.rev ?? 0) + 1, kind: "stall", text, wid: t.wid, call: t.callId } });
+            await t.journal.append(JT.attention, { exec, horizon: progress, ...(lastStream ? { lastStream } : {}), item: { id, rev: (last?.rev ?? 0) + 1, kind: "stall", text, wid: t.wid, call: t.callId } });
         }
     });
     child.stdin.on("error", () => { });
@@ -105,6 +114,10 @@ export async function observeExecution(d) {
         // P18: Apply RPC boundaries at receipt, before any in-flight scan can resume.
         // Durable observations remain queued; clock transitions never wait on I/O.
         clock.event(event, performance.now());
+        if (event.type === "message_update") {
+            const type = event.assistantMessageEvent?.type;
+            lastStream = { type: typeof type === "string" && /^(thinking|text|toolcall)_(start|delta|end)$/.test(type) ? type : "message_update", receivedAt: Date.now() };
+        }
         const message = event.message;
         const error = event.type === "auto_retry_start" ? event.errorMessage : event.type === "auto_retry_end" ? event.finalError :
             event.type === "message_end" && message?.stopReason === "error" ? message.errorMessage : undefined;
@@ -122,6 +135,9 @@ export async function observeExecution(d) {
         if (event.type === "message_update" || ["tool_execution_start", "tool_execution_update", "tool_execution_end"].includes(String(event.type)) ||
             event.type === "message_end" && message?.stopReason !== "error" && (message?.usage?.output ?? 0) > 0)
             progress = performance.now();
+        // Stream deltas already updated receipt-time clocks; periodic checks bound decision latency.
+        if (event.type === "message_update" || event.type === "tool_execution_update")
+            return;
         enqueue(async () => {
             const args = event.args;
             if (event.type === "tool_execution_start" && (event.toolName === "edit" || event.toolName === "write") && typeof args?.path === "string")
@@ -146,10 +162,13 @@ export async function observeExecution(d) {
             }
         });
     });
-    let scanning = false, failingSince;
-    const scan = () => {
-        if (scanning)
+    let scanning = false, sessionDirty = false, failingSince;
+    const scan = (processes = true) => {
+        if (scanning) {
+            if (!processes)
+                sessionDirty = true;
             return;
+        }
         scanning = true;
         enqueue(async () => {
             try {
@@ -158,8 +177,10 @@ export async function observeExecution(d) {
                 // a rerun). Only a scan that keeps failing for a minute ends the observation.
                 // Measured in elapsed time: the scan period is fixed at start while k.trackerMs can be reloaded.
                 try {
-                    clock.scan(await d.track());
-                    failingSince = undefined;
+                    if (processes) {
+                        clock.scan(await d.track());
+                        failingSince = undefined;
+                    }
                 }
                 catch (error) {
                     failingSince ??= performance.now();
@@ -187,11 +208,20 @@ export async function observeExecution(d) {
             }
             finally {
                 scanning = false;
+                if (sessionDirty) {
+                    sessionDirty = false;
+                    scan(false);
+                }
             }
         });
     };
-    const timer = setInterval(scan, config.k?.trackerMs ?? 1000);
-    const watcher = watch(callDir(home, t.wid, t.key, t.gen), scan);
+    const timer = setInterval(() => scan(), Math.min(config.k?.trackerMs ?? 1000, 1000));
+    let notification;
+    const watcher = watch(callDir(home, t.wid, t.key, t.gen), (_event, name) => {
+        if (!sessionChanged(name) || notification)
+            return;
+        notification = setTimeout(() => { notification = undefined; scan(false); }, 10);
+    });
     watcher.on("error", () => { });
     try {
         if (d.interrupted())
@@ -207,6 +237,7 @@ export async function observeExecution(d) {
     }
     finally {
         watcher.close();
+        clearTimeout(notification);
         clearInterval(timer);
         d.setWake(() => { });
         ending = true;

package/dist/orchestrator/executor/sweep.js

@@ -1,10 +1,12 @@
 import { JT, attentionEntries, isEntry } from "../../types.js";
 import { Containment as PlatformContainment } from "../../platform/containment.js";
 import { ProcessTable } from "../../platform/proctable.js";
-/** C2, F4: A containment whose process-table snapshots never overlap (ProcessTable.list is not reentrant). */
-export function serialContainment(table = new ProcessTable()) {
+/** C2, F4: Real observers share snapshots; injected tables retain serialized access. */
+export function serialContainment(table) {
+    if (!table || table instanceof ProcessTable)
+        return new PlatformContainment(table);
     let queue = Promise.resolve();
-    return new PlatformContainment({ list(known) { const next = queue.then(() => table.list(known)); queue = next.catch(() => { }); return next; } });
+    return new PlatformContainment({ list(known, options) { const next = queue.then(() => table.list(known, options)); queue = next.catch(() => { }); return next; } });
 }
 /** K7: Three consecutive losses for a candidate within its pool suspend it for ten minutes. */
 export async function skipLostCandidate(journal, orch, exec) {
@@ -58,36 +60,51 @@ export async function fenceAttentionResolved(journal, id) {
     if (item && !journal.entries().some(e => e.type === JT.attentionResolved && e.id === `fence:${id}` && e.rev === item.rev))
         await journal.append(JT.attentionResolved, { id: `fence:${id}`, rev: item.rev, resolution: "fenced" });
 }
+const sweeps = new WeakMap();
+/** A1: Fold old history during recovery, so the first sweep does not allocate it on the timer path. */
+export function indexSweep(journal) {
+    let state = sweeps.get(journal);
+    if (!state) {
+        state = { count: 0, fenced: new Set(), retired: new Set(), failed: new Set(), gated: new Set(), tracked: new Map(), execs: [], gates: [] };
+        sweeps.set(journal, state);
+    }
+    const { fenced, retired, failed, gated, tracked, execs, gates } = state, entries = journal.entries();
+    while (state.count < entries.length) {
+        const e = entries[state.count++];
+        if (e.type === JT.exec)
+            execs.push(e);
+        else if (e.type === "gate-intent")
+            gates.push(e);
+        else if (e.type === JT.fenced)
+            fenced.add(e.exec);
+        else if (e.type === "retired")
+            retired.add(e.call);
+        else if (e.type === "fence-failed")
+            failed.add(e.exec);
+        else if (e.type === "gate")
+            gated.add(e.id);
+        else if (e.type === "tracked" || e.type === "gate-tracked") {
+            const id = String(e.type === "tracked" ? e.exec : e.id), list = tracked.get(id) ?? [];
+            if (!tracked.has(id))
+                tracked.set(id, list);
+            list.push(e.type === "tracked" ? { pid: Number(e.pid), ppid: 0, start: String(e.start) } : e.process);
+        }
+    }
+    return state;
+}
 /** P23, F1, F2: Every K1 re-fence retired identities and retry failed fences (executions and gates without an outcome)
  *  with one shared scan; a failing identity is reported through `failed` and never stops the sweep of the others. */
 export async function sweepExecutions(journals, containment, hooks = {}) {
     const targets = [];
     for (const journal of journals) {
-        const fenced = new Set(), retired = new Set(), failed = new Set(), gated = new Set(), tracked = new Map();
-        const entries = journal.entries();
-        for (const e of entries) {
-            if (e.type === JT.fenced)
-                fenced.add(e.exec);
-            else if (e.type === "retired")
-                retired.add(e.call);
-            else if (e.type === "fence-failed")
-                failed.add(e.exec);
-            else if (e.type === "gate")
-                gated.add(e.id);
-            else if (e.type === "tracked")
-                tracked.set(String(e.exec), [...tracked.get(String(e.exec)) ?? [], { pid: Number(e.pid), ppid: 0, start: String(e.start) }]);
-            else if (e.type === "gate-tracked")
-                tracked.set(String(e.id), [...tracked.get(String(e.id)) ?? [], e.process]);
-        }
+        const { fenced, retired, failed, gated, tracked, execs, gates } = indexSweep(journal);
         // A gate whose recovery fence failed has no outcome yet; it is retried here until its processes are gone.
-        for (const e of entries) {
+        for (const e of gates) {
             const id = String(e.id);
             if (e.type === "gate-intent" && !gated.has(id) && (failed.has(id) || hooks.failing?.(id)))
                 targets.push({ journal, exec: id, call: String(e.call), fenced: false, failed: true, tracked: tracked.get(id) ?? [] });
         }
-        for (const e of entries) {
-            if (e.type !== JT.exec)
-                continue;
+        for (const e of execs) {
             const exec = String(e.exec), isFailed = failed.has(exec) || !!hooks.failing?.(exec);
             if (fenced.has(exec) || retired.has(e.call) || isFailed)
                 targets.push({ journal, exec, call: String(e.call), fenced: fenced.has(exec), failed: isFailed, tracked: tracked.get(exec) ?? [] });