quadwork 2.1.0 → 2.2.1

package/server/index.js

@@ -11,6 +11,7 @@ const routes = require("./routes");
 const fileChat = require("./file-chat");
 const { dispatchToAgentPTY, cleanupSession: cleanupPtyDispatcher } = require("./pty-dispatcher");
 const { runAcMigration } = require("./migrate-ac");
+const selfHeal = require("./self-heal");
 
 const net = require("net");
 const config = readConfig();
@@ -502,6 +503,39 @@ async function spawnAgentPty(project, agent, opts = {}) {
       if (session.scrollback.length > SCROLLBACK_SIZE) {
         session.scrollback = session.scrollback.slice(-SCROLLBACK_SIZE);
       }
+
+      // #797: observe-only self-heal detector. Wrapped in its own try/catch so
+      // a detector bug can never break the PTY → xterm pipeline; the chunk is
+      // never consumed or altered (the WS viewer forwards it independently).
+      try {
+        selfHeal.observeChunk(key, data, {
+          now: Date.now(),
+          recovering: !!session._autoRecovering,
+          onRestart: () => {
+            session._autoRecovering = true;
+            console.log(`[self-heal] ${key}: thinking-block 400 detected — restarting session`);
+            // #825: NO clearSelfHeal here — the breaker window must persist
+            // across auto-restarts so repeated trips can pause auto-recovery.
+            restartAgentSession(key, { reason: "thinking-block-400" })
+              .then((result) => {
+                if (result && result.ok) {
+                  emitSystemMessage(project, `${agent} auto-restarted (recovered from thinking-block API error)`);
+                }
+              })
+              .catch((err) => console.error(`[self-heal] ${key}: auto-restart failed:`, err.message))
+              .finally(() => {
+                const s = agentSessions.get(key);
+                if (s) s._autoRecovering = false;
+              });
+          },
+          onBreaker: (message) => {
+            console.log(`[self-heal] ${key}: ${message}`);
+            emitSystemMessage(project, message);
+          },
+        });
+      } catch (err) {
+        console.error(`[self-heal] ${key}: detector error (ignored):`, err.message);
+      }
     });
 
     term.onExit(({ exitCode }) => {
@@ -525,7 +559,16 @@ async function spawnAgentPty(project, agent, opts = {}) {
   }
 }
 
-async function stopAgentSession(key) {
+async function stopAgentSession(key, { clearSelfHeal = false } = {}) {
+  // #825 (#797 follow-up): a MANUAL stop/restart/reset must reset the per-agent
+  // self-heal circuit-breaker window, so a fresh operator-driven session isn't
+  // suppressed by a stale "paused" state from a prior trip. This is gated:
+  // the auto-restart path (restartAgentSession with reason "thinking-block-400")
+  // leaves clearSelfHeal=false, because clearing on every auto-restart would
+  // reset countInWindow each time and defeat the #797 breaker entirely. Cleared
+  // before the session lookup so a manual stop resets the window even when no
+  // live session remains (e.g. the agent already exited).
+  if (clearSelfHeal) selfHeal.clearState(key);
   const session = agentSessions.get(key);
   if (!session) {
     agentSessions.set(key, { projectId: null, agentId: null, term: null, viewers: new Set(), viewerDims: new Map(), lastDims: null, state: "stopped", error: null });
@@ -596,7 +639,7 @@ app.post("/api/agents/:project/reset", async (req, res) => {
     for (const agentId of allAgentIds) {
       const s = agentSessions.get(`${projectId}/${agentId}`);
       if (s) s._suppressLifecycleMsg = true;
-      await stopAgentSession(`${projectId}/${agentId}`);
+      await stopAgentSession(`${projectId}/${agentId}`, { clearSelfHeal: true }); // #825: manual reset resets the self-heal window
     }
 
     // Respawn all agents with fresh MCP tokens
@@ -635,7 +678,7 @@ app.post("/api/full-reset", async (_req, res) => {
     console.log("[full-reset] stopping all agent sessions...");
     const sessionKeys = [...agentSessions.keys()];
     for (const key of sessionKeys) {
-      await stopAgentSession(key);
+      await stopAgentSession(key, { clearSelfHeal: true }); // #825: manual full-reset resets the self-heal window
     }
 
     console.log("[full-reset] stopping Butler...");
@@ -707,23 +750,35 @@ app.post("/api/agents/:project/:agent/start", async (req, res) => {
 app.post("/api/agents/:project/:agent/stop", async (req, res) => {
   const { project, agent } = req.params;
   const key = `${project}/${agent}`;
-  await stopAgentSession(key);
+  await stopAgentSession(key, { clearSelfHeal: true }); // #825: manual stop resets the self-heal window
   res.json({ ok: true, state: "stopped" });
 });
 
 // --- Lifecycle: restart ---
 
-app.post("/api/agents/:project/:agent/restart", async (req, res) => {
-  const { project, agent } = req.params;
-  const key = `${project}/${agent}`;
+// #797: shared restart sequence, used by both the manual restart route and
+// the self-heal detector. Exactly the prior route body — no new lifecycle
+// logic. The `reason` is informational (logged by callers).
+async function restartAgentSession(key, { reason, clearSelfHeal = false } = {}) {
+  const [project, agent] = key.split("/");
+  console.log(`[restart] ${key}: restarting session (reason: ${reason || "unspecified"})`);
 
   // #241: must await deregister before respawn so the slot frees and
   // the fresh register lands at slot 1 instead of head-2.
   const existing = agentSessions.get(key);
   if (existing) existing._suppressLifecycleMsg = true;
-  await stopAgentSession(key);
+  // #825: forward clearSelfHeal — a manual restart resets the breaker window;
+  // the self-heal auto-restart does NOT (preserves the #797 circuit breaker).
+  await stopAgentSession(key, { clearSelfHeal });
+
+  return spawnAgentPty(project, agent, { suppressLifecycleMsg: true });
+}
+
+app.post("/api/agents/:project/:agent/restart", async (req, res) => {
+  const { project, agent } = req.params;
+  const key = `${project}/${agent}`;
 
-  const result = await spawnAgentPty(project, agent, { suppressLifecycleMsg: true });
+  const result = await restartAgentSession(key, { reason: "manual", clearSelfHeal: true }); // #825
   if (result.ok) {
     emitSystemMessage(project, `${agent} restarted`);
     res.json({ ok: true, state: "running", pid: result.pid });
@@ -972,10 +1027,11 @@ app.get("/api/butler/status", (_req, res) => {
 const triggers = new Map();
 
 const DEFAULT_MESSAGE = `@head @re1 @re2 @dev — Queue check.
-Head: Merge any PR with both approvals, assign next from queue.
+Discovery: read GITHUB.md (or GET /api/github-parsed) for issue/PR state instead of running gh. If GITHUB.md is absent or stale (>2 cycles / _stale), do ONE direct gh read to confirm. GITHUB.md may lag — confirm with a direct gh read before any merge/review decision.
+Head: Merge any PR with both current-revision approvals, assign next from queue.
 Dev: Work on assigned ticket or address review feedback.
-RE1/RE2: Review open PRs. If Dev pushed fixes, re-review. Post verdict on PR AND notify here.
-ALL: Communicate via this chat by tagging agents. Your terminal is NOT visible.`;
+RE1/RE2: Review ONLY PRs you were @mentioned on in this chat (not all open PRs). If Dev pushed fixes, re-review. Post verdict on PR AND notify here.
+ALL: If nothing is assigned or pending for you, no-op quietly. Communicate via this chat by tagging agents. Your terminal is NOT visible.`;
 
 // #518: server-side bridge lifecycle helpers. Stop and start Telegram +
 // Discord bridges so they respond to batch transitions even when the
@@ -1070,8 +1126,16 @@ async function sendTriggerMessage(projectId) {
       );
       if (bpRes.ok) {
         const bp = await bpRes.json();
-        if (bp && bp.complete) {
-          console.log(`[auto-trigger] ${projectId}: batch complete, auto-stopped`);
+        // #810: gate auto-stop on completeConfirmed (two distinct successful
+        // fetch cycles), NOT a single transient/stale `complete`.
+        // #864: also auto-stop on an explicit operator clear (`liveActiveBatchCleared`).
+        // The preserved snapshot may keep `items` non-empty / `complete` mixed, so
+        // `completeConfirmed` alone won't fire when items don't all resolve as
+        // merged/closed (e.g. a duplicate unmerged PR). The cleared flag is the
+        // operator's intent and overrides those signals for lifecycle purposes.
+        const clearedByOperator = !!(bp && bp.liveActiveBatchCleared);
+        if (bp && (bp.completeConfirmed || clearedByOperator)) {
+          console.log(`[auto-trigger] ${projectId}: batch ${clearedByOperator ? "cleared by operator" : "complete (confirmed)"}, auto-stopped`);
           stopTrigger(projectId);
           // Also stop caffeinate if no other triggers remain running
           // (#441 companion fix). caffeinateProcess is global (not
@@ -1663,12 +1727,24 @@ async function autoStopPollingTick() {
       if (!res.ok) continue;
       const bp = await res.json();
       const hasItems = bp.items && bp.items.length > 0;
+      // #810: gate auto-stop on completeConfirmed (two distinct successful fetch
+      // cycles), not a single transient/stale `complete`. Track prev on the
+      // confirmed value so the bridge-stop transition guard fires on it.
+      // #864: an explicit operator clear (`liveActiveBatchCleared`) ALSO triggers
+      // the stop path, so trigger + bridges shut down when Head sets the Active
+      // Batch section to empty even if the preserved snapshot's items don't all
+      // resolve as merged/closed (e.g. a duplicate unmerged PR keeps the items
+      // in `in_review`). The cleared flag is the operator's intent.
+      const confirmed = !!bp.completeConfirmed;
+      const clearedByOperator = !!bp.liveActiveBatchCleared;
+      const shouldStop = confirmed || clearedByOperator;
       const prev = _bridgeBatchPrev.get(project.id);
-      _bridgeBatchPrev.set(project.id, { complete: bp.complete, hasItems });
+      _bridgeBatchPrev.set(project.id, { complete: shouldStop, hasItems });
 
-      if (bp && bp.complete) {
+      if (bp && shouldStop) {
         if (hasTriggerAuto) {
-          console.log(`[auto-trigger] ${project.id}: batch complete, auto-stopped (poller)`);
+          const reason = clearedByOperator ? "cleared by operator" : "complete (confirmed)";
+          console.log(`[auto-trigger] ${project.id}: batch ${reason}, auto-stopped (poller)`);
           stopTrigger(project.id);
           if (caffeinateProcess.process && triggers.size === 0) {
             try { caffeinateProcess.process.kill("SIGTERM"); } catch {}
@@ -1684,7 +1760,9 @@ async function autoStopPollingTick() {
       }
 
       // #518: detect batch-start transition → auto-start bridges
-      if (hasBridgeAuto && hasItems && !bp.complete) {
+      // #864: do NOT auto-start on a cleared queue even though hasItems may be
+      // true from the preserved snapshot — the operator's clear is a stop signal.
+      if (hasBridgeAuto && hasItems && !bp.complete && !clearedByOperator) {
         const isNewBatch = !prev || prev.complete || !prev.hasItems;
         if (isNewBatch) {
           await autoStartBridges(project.id, project, qwPort);
@@ -1842,6 +1920,16 @@ server.listen(PORT, "127.0.0.1", async () => {
 
   runStartupMigrations(startupCfg);
 
+  // #856: Auto-reseed worktree AGENTS.md when the package version changes.
+  // Per-project completion state lives in ~/.quadwork/reseed-state.json, so
+  // projects deferred mid-batch stay pending and retry on the next startup.
+  // Failures here MUST NOT block server boot.
+  try {
+    await routes.autoReseedOnStartup(startupCfg);
+  } catch (err) {
+    console.error(`[reseed] auto-reseed failed: ${err.message}`);
+  }
+
   if (startupCfg.butler && startupCfg.butler.enabled && startupCfg.butler.auto_start) {
     const result = spawnButlerPty();
     if (result.ok) console.log(`[butler] auto-started (PID: ${result.pid})`);