quadwork 1.11.1 → 1.12.1

package/server/__tests__/rate-limit-handling.test.js

@@ -0,0 +1,168 @@
+/**
+ * #554 — Verify rate-limit-aware caching and backoff in server/routes.js.
+ *
+ * Tests verify:
+ * 1. Rate limit state variables exist and are initialised
+ * 2. adaptiveTTL returns extended TTLs when rate is low/critical
+ * 3. cachedGhEndpoint serves stale data with _rateLimited flag
+ * 4. GitHub endpoints use the cached helper
+ * 5. /api/github/rate-limit endpoint is registered
+ * 6. Batch progress handler has rate-limit guard
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const ROUTES_PATH = path.join(__dirname, "..", "routes.js");
+const src = fs.readFileSync(ROUTES_PATH, "utf-8");
+
+// ---------------------------------------------------------------------------
+// 1. Rate limit state and constants
+// ---------------------------------------------------------------------------
+
+describe("#554 rate limit infrastructure (code analysis)", () => {
+  test("_rateLimit state object is defined with expected fields", () => {
+    expect(src).toContain("const _rateLimit = {");
+    expect(src).toContain("remaining:");
+    expect(src).toContain("resetAt:");
+  });
+
+  test("RATE_LIMIT_LOW_THRESHOLD and RATE_LIMIT_CRITICAL are defined", () => {
+    expect(src).toMatch(/RATE_LIMIT_LOW_THRESHOLD\s*=\s*\d+/);
+    expect(src).toMatch(/RATE_LIMIT_CRITICAL\s*=\s*\d+/);
+  });
+
+  test("refreshRateLimit calls gh api rate_limit", () => {
+    expect(src).toContain("gh");
+    expect(src).toContain("api");
+    expect(src).toContain("rate_limit");
+  });
+
+  test("startRateLimitPolling is called at module load", () => {
+    expect(src).toContain("startRateLimitPolling()");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// 2. Adaptive TTL
+// ---------------------------------------------------------------------------
+
+describe("#554 adaptive TTL logic", () => {
+  test("adaptiveTTL function is defined", () => {
+    expect(src).toContain("function adaptiveTTL(baseTTL)");
+  });
+
+  test("adaptiveTTL returns Infinity when critically rate-limited", () => {
+    expect(src).toMatch(/isRateLimited\(\).*Infinity/s);
+  });
+
+  test("adaptiveTTL extends TTL when rate is low", () => {
+    expect(src).toMatch(/isRateLow\(\).*120[_,]?000/s);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// 3. Cached endpoint helper
+// ---------------------------------------------------------------------------
+
+describe("#554 cachedGhEndpoint helper", () => {
+  test("cachedGhEndpoint function is defined", () => {
+    expect(src).toContain("function cachedGhEndpoint(");
+  });
+
+  test("serves stale data with _rateLimited flag when critical", () => {
+    const fnStart = src.indexOf("function cachedGhEndpoint(");
+    const fnBody = src.slice(fnStart, fnStart + 800);
+    expect(fnBody).toContain("_rateLimited");
+    expect(fnBody).toContain("_stale");
+  });
+
+  test("uses adaptiveTTL for cache checks", () => {
+    const fnStart = src.indexOf("function cachedGhEndpoint(");
+    const fnBody = src.slice(fnStart, fnStart + 600);
+    expect(fnBody).toContain("adaptiveTTL");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// 4. GitHub endpoints use cached helper
+// ---------------------------------------------------------------------------
+
+describe("#554 GitHub endpoints use cachedGhEndpoint", () => {
+  test("/api/github/issues uses cachedGhEndpoint", () => {
+    const section = src.slice(
+      src.indexOf('"/api/github/issues"'),
+      src.indexOf('"/api/github/issues"') + 300,
+    );
+    expect(section).toContain("cachedGhEndpoint");
+  });
+
+  test("/api/github/prs uses cachedGhEndpoint", () => {
+    const section = src.slice(
+      src.indexOf('"/api/github/prs"'),
+      src.indexOf('"/api/github/prs"') + 300,
+    );
+    expect(section).toContain("cachedGhEndpoint");
+  });
+
+  test("/api/github/closed-issues uses cachedGhEndpoint", () => {
+    const section = src.slice(
+      src.indexOf('"/api/github/closed-issues"'),
+      src.indexOf('"/api/github/closed-issues"') + 500,
+    );
+    expect(section).toContain("cachedGhEndpoint");
+  });
+
+  test("/api/github/merged-prs uses cachedGhEndpoint", () => {
+    const section = src.slice(
+      src.indexOf('"/api/github/merged-prs"'),
+      src.indexOf('"/api/github/merged-prs"') + 500,
+    );
+    expect(section).toContain("cachedGhEndpoint");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// 5. Rate limit API endpoint
+// ---------------------------------------------------------------------------
+
+describe("#554 /api/github/rate-limit endpoint", () => {
+  test("endpoint is registered", () => {
+    expect(src).toContain('"/api/github/rate-limit"');
+  });
+
+  test("returns remaining, limit, resetInMinutes, low, critical fields", () => {
+    const idx = src.indexOf('"/api/github/rate-limit"');
+    const section = src.slice(idx, idx + 500);
+    expect(section).toContain("remaining");
+    expect(section).toContain("limit");
+    expect(section).toContain("resetInMinutes");
+    expect(section).toContain("low");
+    expect(section).toContain("critical");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// 6. Batch progress rate limit guard
+// ---------------------------------------------------------------------------
+
+describe("#554 batch progress rate limit awareness", () => {
+  test("batch progress handler checks isRateLimited before gh calls", () => {
+    const batchStart = src.indexOf('"/api/batch-progress"');
+    const batchSection = src.slice(batchStart, batchStart + 600);
+    expect(batchSection).toContain("isRateLimited()");
+    expect(batchSection).toContain("_rateLimited");
+  });
+
+  test("batch progress uses adaptiveTTL for cache", () => {
+    const batchStart = src.indexOf('"/api/batch-progress"');
+    const batchSection = src.slice(batchStart, batchStart + 400);
+    expect(batchSection).toContain("adaptiveTTL");
+  });
+
+  test("projects endpoint uses adaptiveTTL for cache", () => {
+    const projStart = src.indexOf('"/api/projects"');
+    const projSection = src.slice(projStart, projStart + 400);
+    expect(projSection).toContain("adaptiveTTL");
+  });
+});

package/server/agentchattr-registry.js

@@ -162,9 +162,26 @@ function stopHeartbeat(handle) {
   if (handle) clearInterval(handle);
 }
 
+/**
+ * Register an agent with retries and exponential backoff.
+ * Returns {name, token, slot} on success, null after all attempts fail.
+ * Uses registerAgent internally so registerAgent.lastError is populated.
+ */
+async function registerAgentWithRetry(serverPort, base, label = null, { force = false, attempts = 3, delayMs = 2000 } = {}) {
+  for (let i = 0; i < attempts; i++) {
+    const result = await registerAgent(serverPort, base, label, { force });
+    if (result) return result;
+    if (i < attempts - 1) {
+      await new Promise((res) => setTimeout(res, delayMs * Math.pow(2, i)));
+    }
+  }
+  return null;
+}
+
 module.exports = {
   waitForAgentChattrReady,
   registerAgent,
+  registerAgentWithRetry,
   deregisterAgent,
   startHeartbeat,
   stopHeartbeat,

package/server/config.js

@@ -55,7 +55,9 @@ function sanitizeOperatorName(value) {
   return truncated;
 }
 
-// Migration: rename old agent keys to new ones
+// Migration: rename old agent keys to new ones.
+// Keep this map — it migrates pre-v1.8 configs on startup so existing
+// installs transition to the canonical head/dev/re1/re2 slugs.
 const AGENT_KEY_MAP = { t1: "head", t2a: "re1", t2b: "re2", t3: "dev", reviewer1: "re1", reviewer2: "re2" };
 
 function migrateAgentKeys(config) {

package/server/index.js

@@ -13,7 +13,7 @@ const {
   patchAgentchattrConfigForTelegramBridge,
   projectAgentchattrConfigPath,
 } = routes;
-const { waitForAgentChattrReady, registerAgent, deregisterAgent, startHeartbeat, stopHeartbeat } = require("./agentchattr-registry");
+const { waitForAgentChattrReady, registerAgent, registerAgentWithRetry, deregisterAgent, startHeartbeat, stopHeartbeat } = require("./agentchattr-registry");
 const { patchAgentchattrCss } = require("./install-agentchattr");
 const { startQueueWatcher, stopQueueWatcher } = require("./queue-watcher");
 
@@ -356,7 +356,15 @@ async function buildAgentArgs(projectId, agentId) {
       // write that into the per-agent MCP config file.
       const chattrInfo = resolveProjectChattr(projectId);
       acServerPort = Number(new URL(chattrInfo.url).port) || 8300;
-      await waitForAgentChattrReady(acServerPort);
+      // #565: extend timeout to 30s — first setup may need AC to install
+      // (git clone + venv + pip install) before it can bind a port.
+      const acReady = await waitForAgentChattrReady(acServerPort, 30000);
+      if (!acReady) {
+        console.warn(`[#565] Agent ${agentId}: AC not reachable on port ${acServerPort} after 30s. Spawning without chat integration.`);
+        // #565: preserve acServerPort and acInjectMode so deferred
+        // recovery in spawnAgentPty can retry registration later.
+        return { args, acRegistrationName: null, acServerPort, acRegistrationToken: null, acInjectMode: injectMode, acMcpHttpPort: mcpHttpPort || null };
+      }
       // #242: best-effort deregister any stale registration of the
       // canonical name (left over by a crashed previous QuadWork
       // session) so the fresh register lands at slot 1 instead of
@@ -370,16 +378,18 @@ async function buildAgentArgs(projectId, agentId) {
         clearPersistedAgentToken(projectId, agentId);
       }
       // #478: force-replace so AC expires any ghost slots for this base
-      const registration = await registerAgent(acServerPort, agentId, agentCfg.display_name || null, { force: true });
+      // #565: retry with backoff and degrade gracefully if AC is not ready
+      const registration = await registerAgentWithRetry(acServerPort, agentId, agentCfg.display_name || null, { force: true });
       if (!registration) {
-        throw new Error(`Failed to register ${agentId}: ${registerAgent.lastError}`);
+        console.warn(`[#565] Agent ${agentId}: AC registration failed after retries (${registerAgent.lastError}). Spawning without chat integration.`);
+      } else {
+        acRegistrationName = registration.name;
+        acRegistrationToken = registration.token;
+        writePersistedAgentToken(projectId, agentId, registration.token);
+        const mcpConfigPath = writeMcpConfigFile(projectId, agentId, mcpHttpPort, registration.token);
+        const flag = agentCfg.mcp_flag || "--mcp-config";
+        args.push(flag, mcpConfigPath);
       }
-      acRegistrationName = registration.name;
-      acRegistrationToken = registration.token;
-      writePersistedAgentToken(projectId, agentId, registration.token);
-      const mcpConfigPath = writeMcpConfigFile(projectId, agentId, mcpHttpPort, registration.token);
-      const flag = agentCfg.mcp_flag || "--mcp-config";
-      args.push(flag, mcpConfigPath);
     } else if (injectMode === "proxy_flag") {
       // Codex: register with AgentChattr first (#240) so the proxy
       // injects a real per-agent token, not the global session token.
@@ -387,7 +397,14 @@ async function buildAgentArgs(projectId, agentId) {
       // projects without a per-project agentchattr_url still work.
       const chattrInfo = resolveProjectChattr(projectId);
       acServerPort = Number(new URL(chattrInfo.url).port) || 8300;
-      await waitForAgentChattrReady(acServerPort);
+      // #565: extend timeout to 30s for first-setup scenario
+      const acReady = await waitForAgentChattrReady(acServerPort, 30000);
+      if (!acReady) {
+        console.warn(`[#565] Agent ${agentId}: AC not reachable on port ${acServerPort} after 30s. Spawning without chat integration.`);
+        // #565: preserve acServerPort and acInjectMode so deferred
+        // recovery in spawnAgentPty can retry registration later.
+        return { args, acRegistrationName: null, acServerPort, acRegistrationToken: null, acInjectMode: injectMode, acMcpHttpPort: mcpHttpPort || null };
+      }
       // #242: best-effort deregister stale canonical name first using
       // the persisted bearer token from a previous session.
       const stalePersistedToken = readPersistedAgentToken(projectId, agentId);
@@ -396,17 +413,19 @@ async function buildAgentArgs(projectId, agentId) {
         clearPersistedAgentToken(projectId, agentId);
       }
       // #478: force-replace so AC expires any ghost slots for this base
-      const registration = await registerAgent(acServerPort, agentId, agentCfg.display_name || null, { force: true });
+      // #565: retry with backoff and degrade gracefully if AC is not ready
+      const registration = await registerAgentWithRetry(acServerPort, agentId, agentCfg.display_name || null, { force: true });
       if (!registration) {
-        throw new Error(`Failed to register ${agentId}: ${registerAgent.lastError}`);
-      }
-      acRegistrationName = registration.name;
-      acRegistrationToken = registration.token;
-      writePersistedAgentToken(projectId, agentId, registration.token);
-      const upstreamUrl = `http://127.0.0.1:${mcpHttpPort}`;
-      const proxyUrl = await startMcpProxy(projectId, agentId, upstreamUrl, registration.token);
-      if (proxyUrl) {
-        args.push("-c", `mcp_servers.agentchattr.url="${proxyUrl}"`);
+        console.warn(`[#565] Agent ${agentId}: AC registration failed after retries (${registerAgent.lastError}). Spawning without chat integration.`);
+      } else {
+        acRegistrationName = registration.name;
+        acRegistrationToken = registration.token;
+        writePersistedAgentToken(projectId, agentId, registration.token);
+        const upstreamUrl = `http://127.0.0.1:${mcpHttpPort}`;
+        const proxyUrl = await startMcpProxy(projectId, agentId, upstreamUrl, registration.token);
+        if (proxyUrl) {
+          args.push("-c", `mcp_servers.agentchattr.url="${proxyUrl}"`);
+        }
       }
     }
   }
@@ -526,11 +545,14 @@ async function spawnAgentPty(project, agent) {
   if (!cwd) return { ok: false, error: `Unknown agent: ${key}` };
 
   const command = resolveAgentCommand(project, agent) || (process.env.SHELL || "/bin/zsh");
-  const built = await buildAgentArgs(project, agent);
-  const args = built.args;
   const extraEnv = buildAgentEnv(project, agent);
 
   try {
+    // #565: buildAgentArgs is inside try-catch so registration failures
+    // cannot crash the server as an unhandled rejection.
+    const built = await buildAgentArgs(project, agent);
+    const args = built.args;
+
     const term = pty.spawn(command, args, {
       name: "xterm-256color",
       cols: 120,
@@ -611,6 +633,25 @@ async function spawnAgentPty(project, agent) {
       }
     }
 
+    // #565: deferred restart — if the agent spawned without AC
+    // registration (AC wasn't ready or registration failed), wait for
+    // AC to come up then stop + respawn the agent so it gets the full
+    // MCP CLI args (--mcp-config / -c mcp_servers...url) that can only
+    // be set at process launch time.
+    if (!session.acRegistrationName && session.acServerPort && session.acInjectMode) {
+      const deferredRestart = async () => {
+        const ready = await waitForAgentChattrReady(session.acServerPort, 60000);
+        if (!ready) return;
+        // Guard: agent may have been stopped manually while we waited.
+        const current = agentSessions.get(key);
+        if (!current || !current.term || current.state !== "running") return;
+        console.log(`[#565] Agent ${agent}: AC is now reachable — restarting agent to gain chat integration.`);
+        await stopAgentSession(key);
+        await spawnAgentPty(project, agent);
+      };
+      deferredRestart().catch(() => {});
+    }
+
     term.onExit(({ exitCode }) => {
       const current = agentSessions.get(key);
       if (current && current.term === term) {