zcode-acp-server 0.44.0 → 0.44.2

package/dist/handlers/session.js

@@ -724,14 +724,14 @@ function isMarttyClient(server) {
  * session/load and would double-render a resume replay.
  */
 async function replayResumeHistory(server, cx, acpSid, zcodeSid) {
-    // Plain fetch (callers only invoke this after a resume flight settled the
-    // store — see resumePreservingModel) INSIDE the batch: the guard must be
-    // held across the session/messages RPC. Fetched outside, a prompt landing
-    // in that window dispatches live turn updates through the lock-free fast
-    // path (enqueueSessionSend) and the TUI renders the new turn ABOVE the
-    // history that arrives afterwards.
+    // Guarded fetch (a resume flight settles the store, but a capped flight
+    // leaves hydration running — fetchMessagesForReplay re-settles then) INSIDE
+    // the batch: the guard must be held across the session/messages RPC.
+    // Fetched outside, a prompt landing in that window dispatches live turn
+    // updates through the lock-free fast path (enqueueSessionSend) and the TUI
+    // renders the new turn ABOVE the history that arrives afterwards.
     await withReplayBatch(acpSid, async () => {
-        const messages = await fetchMessages(server, zcodeSid);
+        const messages = await fetchMessagesForReplay(server, zcodeSid);
         if (messages.length === 0)
             return;
         const slice = sliceTail(messages, MARTTY_RESUME_TAIL);
@@ -838,8 +838,9 @@ export async function resumeSession(server, params, cx) {
     // first: microtasks (the SDK's response send) drain before immediates.
     if (isMarttyClient(server)) {
         setImmediate(() => {
-            // Plain fetch, no settle: any resume flight already settled the store
-            // before this session/resume returned (the settle rides the flight).
+            // The replay's fetch is marker-guarded (see fetchMessagesForReplay): a
+            // resume flight settles the store before this session/resume returned,
+            // and a capped flight leaves the marker for the replay to re-settle.
             replayResumeHistory(server, cx, acpSid, zcodeSid).catch((e) => {
                 warn(`session/resume: TUI history replay failed (non-fatal): ` +
                     `${e instanceof Error ? e.message : String(e)}`);
@@ -949,7 +950,7 @@ export async function resumeIntoSession(server, cx, acpSid, zcodeTarget) {
     // Fetch inside the batch (see replayResumeHistory): the guard must cover
     // the history RPC, or a concurrent prompt renders above the replay.
     await withReplayBatch(acpSid, async () => {
-        const history = settledHistory ?? (await fetchMessages(server, zcodeTarget));
+        const history = settledHistory ?? (await fetchMessagesForReplay(server, zcodeTarget));
         if (history.length > 0)
             server.markSessionActive(acpSid);
         const slice = fullSlice(history);
@@ -1069,7 +1070,8 @@ export async function loadSession(server, params, cx, opts = {}) {
     // Named `history` — a local `messages` would shadow the i18n `messages()`
     // helper (TDZ crash from a catch block). A resume flight's SETTLED snapshot
     // is the mid-hydration guard; only an already-live session (no flight) falls
-    // back to a plain read — its store is stable.
+    // back to a plain read — which fetchMessagesForReplay upgrades to a settle
+    // poll when the last flight capped out mid-hydration.
     let slice;
     if (opts.replayHistory === false) {
         // Boot-resume interception (session/new): the terminal TUI client does not
@@ -1077,7 +1079,7 @@ export async function loadSession(server, params, cx, opts = {}) {
         // and blocks on the response until the replay finishes — so skip the
         // dispatch. The differ baseline below still runs, so the next turn's
         // completion diff does not re-emit the historical messages.
-        const history = settledHistory ?? (await fetchMessages(server, zcodeSid));
+        const history = settledHistory ?? (await fetchMessagesForReplay(server, zcodeSid));
         // History on disk = real interaction (covers untitled sessions resumed from
         // a previous bridge lifetime) — make the session discoverable remotely.
         if (history.length > 0)
@@ -1090,7 +1092,7 @@ export async function loadSession(server, params, cx, opts = {}) {
         // a prompt landing while the history RPC is in flight must queue behind
         // the batch, not dispatch above the replayed history.
         slice = await withReplayBatch(acpSid, async () => {
-            const history = settledHistory ?? (await fetchMessages(server, zcodeSid));
+            const history = settledHistory ?? (await fetchMessagesForReplay(server, zcodeSid));
             if (history.length > 0)
                 server.markSessionActive(acpSid);
             const batchSlice = limit === null ? fullSlice(history) : sliceTail(history, limit);
@@ -1236,7 +1238,9 @@ export async function runOneTurn(server, opts) {
     // Per-session ProjectionDiffer (persists across turns). The baseline mark_seen
     // prevents the differ from re-emitting history at turn completion.
     const differ = getOrCreateDiffer(server, zcodeSid);
-    const baselineMsgs = await fetchMessages(server, zcodeSid);
+    // Marker-guarded read: a prefix baseline during armed hydration would make
+    // the completion diff re-emit the messages hydration later appends.
+    const baselineMsgs = await fetchMessagesForReplay(server, zcodeSid);
     differ.markSeen(baselineMsgs);
     // Subscribe BEFORE send so we don't lose early turn.completed on short turns.
     // subscribe() throws on failure, surfacing the backend's real error (reader
@@ -1280,6 +1284,9 @@ export async function runOneTurn(server, opts) {
     // above. Kept as a binding only so the call fits the Promise-returning shape.
     void snapshot;
     backend.registerEventListener(zcodeSid, listener);
+    // Armed (not run) on a successful end_turn — the finally starts the detached
+    // compaction AFTER this turn's cleanup (see runAutoCompactDetached).
+    let autoCompactDue = false;
     try {
         // Transient turn failures (e.g. provider network blips surfaced as
         // turn.failed with cause code model_request_failed) are retried by
@@ -1305,7 +1312,7 @@ export async function runOneTurn(server, opts) {
                 // reconcile the differ baseline so the retried turn's new messages
                 // aren't treated as already-seen, surface a retry hint, then back off.
                 if (turn.cancelled) {
-                    stopBackendTurn(server, zcodeSid, turn.foregroundExecutionId);
+                    stopBackendTurn(server, zcodeSid, turn);
                     return { stopReason: "cancelled" };
                 }
                 differ.markSeen(await fetchMessages(server, zcodeSid));
@@ -1318,9 +1325,13 @@ export async function runOneTurn(server, opts) {
             }
             const chunkMsgId = randomUUID();
             // Drain gate: a recent cancel/preempt means the backend side needs
-            // settling before the send — see drainBackendAfterCancel.
+            // settling before the send — see drainBackendAfterCancel. EXEMPT while a
+            // detached auto-compact runs: it legitimately holds the prompt lock, and
+            // the gate's close escalation would tear down the resident runtime and
+            // kill the compaction — the busy-retry below waits it out instead.
             const cancelledRecently = server.lastCancelledAt.get(zcodeSid) !== undefined &&
-                Date.now() - server.lastCancelledAt.get(zcodeSid) < DRAIN_WINDOW_MS;
+                Date.now() - server.lastCancelledAt.get(zcodeSid) < DRAIN_WINDOW_MS &&
+                !server.autoCompactInFlight.has(zcodeSid);
             if (cancelledRecently) {
                 const drained = await drainBackendAfterCancel(server, {
                     acpSid,
@@ -1352,9 +1363,13 @@ export async function runOneTurn(server, opts) {
                 : { sessionId: zcodeSid, content: sendText };
             const sendT0 = Date.now();
             let sendAttempt = 0;
+            // One-shot "waiting for compaction" notice — a minutes-long busy window
+            // with no explanation reads as a hang (the very thing that made users
+            // ESC out of the old blocking auto-compact).
+            let compactWaitNotified = false;
             while (true) {
                 if (turn.cancelled) {
-                    stopBackendTurn(server, zcodeSid, turn.foregroundExecutionId);
+                    stopBackendTurn(server, zcodeSid, turn);
                     return { stopReason: "cancelled" };
                 }
                 sendAttempt++;
@@ -1368,15 +1383,19 @@ export async function runOneTurn(server, opts) {
                 if (expectBusy) {
                     await sleep(SEND_RETRY_INTERVAL_MS);
                     if (turn.cancelled) {
-                        stopBackendTurn(server, zcodeSid, turn.foregroundExecutionId);
+                        stopBackendTurn(server, zcodeSid, turn);
                         return { stopReason: "cancelled" };
                     }
                 }
                 const sendResp = await backend.request(server.nextId(), "session/send", sendParams, 15000);
                 if (!sendResp.error) {
                     const accepted = (sendResp.result ?? {});
-                    if (accepted.accepted)
+                    if (accepted.accepted) {
+                        // From this moment the turn may own a running generation —
+                        // stopBackendTurn's compaction guard no longer spares it.
+                        turn.sendAccepted = true;
                         break; // backend took it → turn starts
+                    }
                     throw new Error("zcode send not accepted");
                 }
                 const sendErrCode = sendResp.error.code;
@@ -1405,8 +1424,20 @@ export async function runOneTurn(server, opts) {
                 const NOTIFY_TURN_STALE_MS = 600_000;
                 const notifySince = server.notifyTurnActiveSince.get(zcodeSid);
                 const notifyActive = notifySince !== undefined && Date.now() - notifySince < NOTIFY_TURN_STALE_MS;
-                const busyBudget = notifyActive ? NOTIFY_TURN_BUSY_TIMEOUT_MS : SEND_RETRY_TIMEOUT_MS;
+                // A detached auto-compact is the same shape: a real model turn holding
+                // the lock for minutes. Extend the budget past its full worst case
+                // (settle cap 300s + startup + probe gaps) and tell the user once.
+                // Both flags can be true at once (a background notify turn AND a
+                // compaction queued behind it) — take the LARGER budget, never the
+                // smaller: the smaller fails the user's prompt mid-wait.
+                const COMPACT_BUSY_TIMEOUT_MS = 360_000;
+                const compacting = server.autoCompactInFlight.has(zcodeSid);
+                const busyBudget = Math.max(notifyActive ? NOTIFY_TURN_BUSY_TIMEOUT_MS : 0, compacting ? COMPACT_BUSY_TIMEOUT_MS : 0, SEND_RETRY_TIMEOUT_MS);
                 const budget = isBusy ? busyBudget : WARMUP_RETRY_TIMEOUT_MS;
+                if (isBusy && compacting && !compactWaitNotified) {
+                    compactWaitNotified = true;
+                    await sendTextChunk(cx, acpSid, messages().autoCompactWaiting, randomUUID());
+                }
                 if (Date.now() - sendT0 > budget) {
                     throw new Error(`zcode send failed: ${isBusy ? "backend still busy" : "send keeps being rejected"} after ${Math.round(budget / 1000)}s (${sendResp.error.message ?? ""})`);
                 }
@@ -1430,19 +1461,21 @@ export async function runOneTurn(server, opts) {
                     Date.now() - (server.lastCancelledAt.get(zcodeSid) ?? 0) < CANCEL_RESIDUE_WINDOW_MS;
                 const result = await runEventTurn(server, listener, monitor, differ, cx, acpSid, chunkMsgId, turn, gateArmed);
                 // Auto-compact: if context usage exceeds the threshold, compact before
-                // returning so the next prompt has room. Configured via
+                // the NEXT prompt needs the room. Configured via
                 // ZCODE_ACP_AUTO_COMPACT_THRESHOLD (absolute token count; 0/unset =
                 // disabled). Only on end_turn — cancelled/max_turn_requests skips
                 // compaction, as does a stall-recovered end_turn (the completion was
                 // inferred by the stall heuristic, not confirmed by turn.completed —
                 // compressing an in-flight task's would destroy the work).
-                // Best-effort: failures are logged inside maybeAutoCompact, never thrown.
-                if (opts.autoCompact !== false &&
-                    result.stopReason === "end_turn" &&
-                    !turn.stallRecovered) {
-                    const { maybeAutoCompact } = await import("../config/auto-compact.js");
-                    await maybeAutoCompact(server, cx, acpSid, zcodeSid);
-                }
+                // ARMED here, RUN detached from the finally: awaiting it inside the
+                // turn kept the finished turn in pendingTurns for the whole
+                // compaction, so any cancel/follow-up preempt killed the compaction's
+                // internal turn (stop + drain close-escalation) while waitForTurnIdle
+                // still reported a false "✓ compressed". Detached, the response
+                // returns at once and the running indicator settles before the
+                // compaction too.
+                autoCompactDue =
+                    opts.autoCompact !== false && result.stopReason === "end_turn" && !turn.stallRecovered;
                 return result;
             }
             catch (e) {
@@ -1541,6 +1574,20 @@ export async function runOneTurn(server, opts) {
         // (preempt): the preempting turn's own running:true must survive.
         const stillBusy = [...server.pendingTurns.values()].some((t) => t.zcodeSid === zcodeSid);
         await emitTurnState(stillBusy);
+        // Start the armed auto-compact ONLY now, after deregistration and the
+        // running:false turnState: from here on a cancel or follow-up prompt finds
+        // no turn to preempt (the compaction is untouchable housekeeping), and a
+        // prompt racing it waits on the busy-retry instead of killing it.
+        if (autoCompactDue) {
+            try {
+                const { runAutoCompactDetached } = await import("../config/auto-compact.js");
+                runAutoCompactDetached(server, cx, acpSid, zcodeSid);
+            }
+            catch (e) {
+                // Never let the arming failure mask the completed turn's result.
+                warn(`auto-compact arming failed: ${e instanceof Error ? e.message : String(e)}`);
+            }
+        }
     }
 }
 /** One prompt round: subscribe-before-send, run the event-driven turn loop. */
@@ -1771,7 +1818,7 @@ export async function cancel(server, params) {
             matched = true;
             turn.cancelled = true;
             if (!turn.stopSent) {
-                stopBackendTurn(server, zcodeSid, turn.foregroundExecutionId);
+                stopBackendTurn(server, zcodeSid, turn);
                 turn.stopSent = true;
             }
             // Record cancel time so a prompt arriving in the backend's ~20s
@@ -1878,7 +1925,20 @@ export function isBackendLostRequestError(e) {
  * backend's prompt lock releases when ITS finalisation completes — that,
  * not any bridge-side signal, is what the next prompt's send-retry waits on.
  */
-function stopBackendTurn(server, zcodeSid, foregroundExecutionId) {
+function stopBackendTurn(server, zcodeSid, turn) {
+    const foregroundExecutionId = turn.foregroundExecutionId;
+    // A turn whose send was NEVER accepted owns no generation of its own; while
+    // our detached compaction runs, the only thing foreground on this session
+    // is the compaction's internal turn — an unguarded stop here (ESPECIALLY
+    // the v4 stop, which "targets whatever is currently foreground") would
+    // kill it. ESC on a prompt parked in the compaction wait is exactly how
+    // that happens. Once the send WAS accepted the turn may own a generation
+    // (turn.started — and with it the execution id — can lag, or never arrive
+    // on a deaf stream), so the stop pair always fires for accepted turns.
+    if (!foregroundExecutionId && !turn.sendAccepted && server.autoCompactInFlight.has(zcodeSid)) {
+        log(`  [stop] turn never started and a detached compaction is running for ${zcodeSid} — skipping the stop pair`);
+        return;
+    }
     try {
         server.ensureBackend().send("session/stop", { sessionId: zcodeSid });
     }
@@ -1971,7 +2031,7 @@ export async function drainBackendAfterCancel(server, deps) {
     let escalated = false;
     while (Date.now() - drainT0 < DRAIN_TIMEOUT_MS) {
         if (turn.cancelled) {
-            stopBackendTurn(server, zcodeSid, turn.foregroundExecutionId);
+            stopBackendTurn(server, zcodeSid, turn);
             return "cancelled";
         }
         const proj = await monitor.pollOnce();
@@ -2078,7 +2138,7 @@ export function preemptInFlightTurn(server, zcodeSid, selfRequestId) {
             continue;
         turn.cancelled = true; // signal the old turn to stop its retry loops
         if (!turn.stopSent) {
-            stopBackendTurn(server, zcodeSid, turn.foregroundExecutionId);
+            stopBackendTurn(server, zcodeSid, turn);
             turn.stopSent = true;
         }
         // Record cancel time so the prompt()'s send-retry can use the recovery
@@ -2345,8 +2405,13 @@ async function resumePreservingModel(server, zcParams) {
 }
 /** Settle-poll gap after a fresh resume (see fetchMessagesSettled). */
 const RESUME_SETTLE_GAP_MS = 300;
-/** Cap for the settle poll — past this the largest snapshot seen wins. */
-const RESUME_SETTLE_CAP_MS = 2000;
+/**
+ * Cap for the settle poll — past this the largest snapshot seen wins. Large
+ * enough for the full two-stable-read plateau on slow readers (big sessions'
+ * reads take seconds each — 3–5 reads must fit, observed 2.4–6.3s/read on a
+ * 7413-message session), while still bounding a genuinely stuck hydration.
+ */
+const RESUME_SETTLE_CAP_MS = 30_000;
 /** Non-growing reads required to call the store settled (plateau guard). */
 const RESUME_SETTLE_STABLE_READS = 2;
 /**
@@ -2357,12 +2422,23 @@ const RESUME_SETTLE_STABLE_READS = 2;
  * is fine once hydration finished; big sessions hydrate slowly, hence
  * "often but not always"). Poll until the count has stopped growing for TWO
  * consecutive reads (a single equal pair can be a >gap plateau inside a slow
- * hydration), capped; on the cap the largest snapshot seen wins. Only fresh
- * resumes pay the extra round-trips — an already-live session's store is
- * stable.
+ * hydration), capped; on the cap the largest snapshot seen wins. A capped
+ * exit records the session in server.hydrationUnsettled so later replay
+ * reads re-settle (fetchMessagesForReplay); a stable exit clears it.
+ *
+ * Re-settles (a marker left by a capped settle) fast-path on the WATERMARK:
+ * one non-growing read that reaches the largest length any settle has ever
+ * observed counts as caught-up. Demanding the full two-read plateau again
+ * meant slow-reading sessions NEVER cleared the marker — every load re-paid
+ * a capped settle (observed 2026-09-20: 22–56s loads on a 7413-message
+ * session). Watermarks reset on backend respawn (ensureBackend).
  */
 export async function fetchMessagesSettled(server, zcodeSid) {
     let messages = await fetchMessages(server, zcodeSid);
+    // The catch-up anchor: what a previous settle already saw of this store.
+    const entryWatermark = server.hydrationWatermark.get(zcodeSid) ?? 0;
+    if (messages.length > entryWatermark)
+        server.hydrationWatermark.set(zcodeSid, messages.length);
     let stable = 0;
     const deadline = Date.now() + RESUME_SETTLE_CAP_MS;
     for (;;) {
@@ -2370,17 +2446,45 @@ export async function fetchMessagesSettled(server, zcodeSid) {
         const next = await fetchMessages(server, zcodeSid);
         // Shrinking should not happen; never trade down either way.
         if (next.length <= messages.length) {
-            if (++stable >= RESUME_SETTLE_STABLE_READS)
+            // A failed read comes back EMPTY (fetchMessages swallows errors) — never
+            // let it vouch for stability while hydration may still be running.
+            const errored = next.length === 0 && messages.length > 0;
+            // Watermark fast-path: the store reached everything any settle has ever
+            // seen AND this read confirms it is not growing — caught up.
+            const reachedWatermark = entryWatermark > 0 && messages.length >= entryWatermark;
+            if (!errored && (reachedWatermark || ++stable >= RESUME_SETTLE_STABLE_READS)) {
+                server.hydrationUnsettled.delete(zcodeSid);
+                server.hydrationWatermark.set(zcodeSid, Math.max(messages.length, entryWatermark));
                 return messages;
+            }
         }
         else {
             stable = 0;
             messages = next;
+            if (messages.length > (server.hydrationWatermark.get(zcodeSid) ?? 0)) {
+                server.hydrationWatermark.set(zcodeSid, messages.length);
+            }
         }
-        if (Date.now() >= deadline)
+        if (Date.now() >= deadline) {
+            server.hydrationUnsettled.add(zcodeSid);
+            warn(`settle capped at ${RESUME_SETTLE_CAP_MS}ms for ${zcodeSid} still growing ` +
+                `(${messages.length} messages) — replaying the largest snapshot, ` +
+                `next load re-settles`);
             return messages;
+        }
     }
 }
+/**
+ * History read for replay paths: plain unless the last settle for this
+ * session capped out mid-hydration — then re-settle first (the marker is
+ * maintained by fetchMessagesSettled). Stable sessions pay zero extra reads;
+ * a still-hydrating session gets the settle poll instead of a prefix.
+ */
+async function fetchMessagesForReplay(server, zcodeSid) {
+    if (!server.hydrationUnsettled.has(zcodeSid))
+        return fetchMessages(server, zcodeSid);
+    return fetchMessagesSettled(server, zcodeSid);
+}
 /**
  * After a faithful resume the session keeps its own last model; when that
  * model no longer belongs to an enabled provider in config.json (deleted or
@@ -2629,7 +2733,7 @@ export async function runEventTurn(server, listener, monitor, differ, cx, acpSid
             else if (turn.cancelled) {
                 // Preserve the pre-existing bounded cancel behaviour. A stuck prompt
                 // lock after stop must not keep a user-cancelled turn alive forever.
-                stopBackendTurn(server, turn.zcodeSid, turn.foregroundExecutionId);
+                stopBackendTurn(server, turn.zcodeSid, turn);
                 return turnResult(translator, "max_turn_requests");
             }
             else {
@@ -2670,7 +2774,7 @@ export async function runEventTurn(server, listener, monitor, differ, cx, acpSid
                         return turnResult(translator, "end_turn");
                     }
                     log(`  [stall] watermark frozen ${Math.round(frozenMs / 1000)}s with no output; stopping backend turn`);
-                    stopBackendTurn(server, turn.zcodeSid, turn.foregroundExecutionId);
+                    stopBackendTurn(server, turn.zcodeSid, turn);
                     return turnResult(translator, "max_turn_requests");
                 }
             }
@@ -2703,7 +2807,7 @@ export async function runEventTurn(server, listener, monitor, differ, cx, acpSid
             // to no turn listener, and the next turn's turn-attribution gate
             // discards any residue that slipped into the queue meanwhile.
             if (!turn.stopSent) {
-                stopBackendTurn(server, turn.zcodeSid, turn.foregroundExecutionId);
+                stopBackendTurn(server, turn.zcodeSid, turn);
                 turn.stopSent = true;
             }
             return turnResult(translator, "cancelled");
@@ -2778,7 +2882,7 @@ export async function runEventTurn(server, listener, monitor, differ, cx, acpSid
                             }
                             else if (!emittedOutput) {
                                 // No text and no output → suspected failure.
-                                stopBackendTurn(server, turn.zcodeSid, turn.foregroundExecutionId);
+                                stopBackendTurn(server, turn.zcodeSid, turn);
                                 throw new RequestError(-32603, "turn produced no output");
                             }
                         }
@@ -2976,7 +3080,7 @@ export async function runEventTurn(server, listener, monitor, differ, cx, acpSid
             }
             if (translator.turnFailed) {
                 // Best-effort stop in case the failed turn left a residual lock.
-                stopBackendTurn(server, turn.zcodeSid, turn.foregroundExecutionId);
+                stopBackendTurn(server, turn.zcodeSid, turn);
                 // Throw a TurnFailedError carrying the structured error so the caller
                 // (prompt's retry loop) can classify transient vs fatal. The error
                 // message is formatted for display when it ultimately reaches the user.