switchroom 0.19.41 → 0.19.43

package/telegram-plugin/tests/retry-grammy-http-error.test.ts

@@ -0,0 +1,404 @@
+/**
+ * Outcome test for the transient-network retry class, driven through the REAL
+ * seam that carries the bug: a real grammy `Bot` whose injected `fetch` throws
+ * a realistic transport error, wrapped by the real `createRetryApiCall` loop.
+ *
+ * Why not a hand-rolled `throw new Error('fetch failed')`: grammy never lets
+ * that shape reach the retry loop. `Api.callMethod` catches every fetch
+ * rejection and rethrows `toHttpError(method, sensitiveLogs, err)`
+ * (grammy 1.44.0, out/core/client.js:58 → out/core/error.js:76-83), which
+ * produces an `HttpError` whose message is `Network request for '<method>'
+ * failed!` — the original `fetch failed` / `ECONNRESET` text is DROPPED unless
+ * `client.sensitiveLogs` is on (it is off by default and we do not enable it,
+ * because it leaks the bot token URL into logs).
+ *
+ * So the only honest test of "does a transport blip get retried" has to go
+ * through grammy's own wrapping. Faking the error shape is what let the dead
+ * branch ship green (#4117).
+ */
+
+import { describe, it, expect, vi, afterEach } from 'vitest'
+import { Bot, HttpError, GrammyError } from 'grammy'
+import {
+  createRetryApiCall,
+  willRetryTelegramFailure,
+  isTransientNetworkMessage,
+  isTransientTransportError,
+  type RetryObserver,
+} from '../retry-api-call.js'
+import { installTgPostLogger } from '../shared/bot-runtime.js'
+import { detectGatewayFindings, scanAgent } from '../../src/fleet-health/detect.js'
+
+afterEach(() => {
+  vi.restoreAllMocks()
+})
+
+const TOKEN = '111111:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+
+function okResponse(result: unknown): Response {
+  return new Response(JSON.stringify({ ok: true, result }), {
+    status: 200,
+    headers: { 'content-type': 'application/json' },
+  })
+}
+
+/**
+ * A real grammy Bot whose transport fails `failures` times with `err`, then
+ * succeeds. Returns the bot plus the live call count.
+ */
+function makeBot(err: unknown, failures: number) {
+  const calls = { n: 0 }
+  const bot = new Bot(TOKEN, {
+    botInfo: {
+      id: 1,
+      is_bot: true,
+      first_name: 'test',
+      username: 'test_bot',
+      can_join_groups: true,
+      can_read_all_group_messages: false,
+      supports_inline_queries: false,
+      can_connect_to_business: false,
+      has_main_web_app: false,
+    },
+    client: {
+      fetch: (async () => {
+        calls.n += 1
+        if (calls.n <= failures) throw err
+        return okResponse({ message_id: 42, date: 0, chat: { id: 7, type: 'private' } })
+      }) as unknown as typeof fetch,
+    },
+  })
+  return { bot, calls }
+}
+
+describe('grammy transport failures reach the retry loop as HttpError', () => {
+  it('grammy rewrites the transport message so the substring predicate cannot see it', async () => {
+    const { bot } = makeBot(new TypeError('fetch failed'), 99)
+
+    const caught = await bot.api.sendMessage(7, 'hi').then(
+      () => null,
+      (e: unknown) => e,
+    )
+
+    // The shape the retry loop actually receives.
+    expect(caught).toBeInstanceOf(HttpError)
+    expect(caught).not.toBeInstanceOf(GrammyError)
+    expect((caught as HttpError).message).toBe("Network request for 'sendMessage' failed!")
+
+    // ...and the original transport text is gone, so the substring classifier
+    // used by the retry loop returns false on it. This is the defect.
+    expect(isTransientNetworkMessage((caught as HttpError).message)).toBe(false)
+
+    // The cause is preserved on `.error`, which is where the class lives.
+    expect((caught as HttpError).error).toBeInstanceOf(TypeError)
+  })
+})
+
+describe('retryApiCall retries a real grammy transport failure', () => {
+  for (const [label, cause] of [
+    ['fetch failed (undici)', new TypeError('fetch failed')],
+    ['ECONNRESET', Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' })],
+    ['ETIMEDOUT', Object.assign(new Error('connect ETIMEDOUT'), { code: 'ETIMEDOUT' })],
+    ['ENOTFOUND', Object.assign(new Error('getaddrinfo ENOTFOUND api.telegram.org'), {
+      code: 'ENOTFOUND',
+    })],
+  ] as const) {
+    it(`delivers the message after a transient ${label}`, async () => {
+      const { bot, calls } = makeBot(cause, 1)
+      const retries: string[] = []
+      const observer: RetryObserver = {
+        onRetry: ({ reason }) => {
+          retries.push(reason)
+        },
+      }
+      const retryApiCall = createRetryApiCall({
+        sleep: async () => {},
+        observer,
+      })
+
+      // OUTCOME: the send actually lands, and the loop actually retried.
+      const res = await retryApiCall(() => bot.api.sendMessage(7, 'hi'), {
+        verb: 'sendMessage',
+      })
+
+      expect(res).toMatchObject({ message_id: 42 })
+      expect(calls.n).toBe(2)
+      expect(retries).toEqual(['network'])
+    })
+  }
+
+  it('does NOT retry a caller-initiated abort — shutdown must not stall', async () => {
+    const aborted = Object.assign(new Error('The operation was aborted'), {
+      name: 'AbortError',
+      code: 'ABORT_ERR',
+    })
+    const { bot, calls } = makeBot(aborted, 99)
+    const sleeps: number[] = []
+    const retryApiCall = createRetryApiCall({
+      sleep: async (ms: number) => {
+        sleeps.push(ms)
+      },
+    })
+
+    await expect(
+      retryApiCall(() => bot.api.sendMessage(7, 'hi'), { verb: 'sendMessage' }),
+    ).rejects.toBeInstanceOf(HttpError)
+
+    // Terminal on the first attempt, and no backoff was slept.
+    expect(calls.n).toBe(1)
+    expect(sleeps).toEqual([])
+  })
+
+  it('DOES retry a grammy request timeout', async () => {
+    const timedOut = new Error("Request to 'sendMessage' timed out after 500 seconds")
+    const { bot, calls } = makeBot(timedOut, 1)
+    const retryApiCall = createRetryApiCall({ sleep: async () => {} })
+
+    const res = await retryApiCall(() => bot.api.sendMessage(7, 'hi'), {
+      verb: 'sendMessage',
+    })
+
+    expect(res).toMatchObject({ message_id: 42 })
+    expect(calls.n).toBe(2)
+  })
+
+  it('gives up after maxRetries when the transport never recovers', async () => {
+    const { bot, calls } = makeBot(new TypeError('fetch failed'), 99)
+    const retries: string[] = []
+    const giveUps: number[] = []
+    const retryApiCall = createRetryApiCall({
+      maxRetries: 3,
+      sleep: async () => {},
+      observer: {
+        onRetry: ({ reason }) => retries.push(reason),
+        onGiveUp: ({ attempts }) => giveUps.push(attempts),
+      },
+    })
+
+    await expect(
+      retryApiCall(() => bot.api.sendMessage(7, 'hi'), { verb: 'sendMessage' }),
+    ).rejects.toBeInstanceOf(HttpError)
+
+    expect(calls.n).toBe(3)
+    expect(retries).toEqual(['network', 'network'])
+    expect(giveUps).toEqual([3])
+  })
+
+  it('does NOT retry a real GrammyError 403 that arrives over the same seam', async () => {
+    const calls = { n: 0 }
+    const bot = new Bot(TOKEN, {
+      botInfo: {
+        id: 1,
+        is_bot: true,
+        first_name: 'test',
+        username: 'test_bot',
+        can_join_groups: true,
+        can_read_all_group_messages: false,
+        supports_inline_queries: false,
+        can_connect_to_business: false,
+        has_main_web_app: false,
+      },
+      client: {
+        fetch: (async () => {
+          calls.n += 1
+          return new Response(
+            JSON.stringify({
+              ok: false,
+              error_code: 403,
+              description: 'Forbidden: bot was blocked by the user',
+            }),
+            { status: 403, headers: { 'content-type': 'application/json' } },
+          )
+        }) as unknown as typeof fetch,
+      },
+    })
+
+    const retryApiCall = createRetryApiCall({ sleep: async () => {} })
+
+    await expect(
+      retryApiCall(() => bot.api.sendMessage(7, 'hi'), { verb: 'sendMessage' }),
+    ).rejects.toBeInstanceOf(GrammyError)
+
+    // Terminal on the first attempt — a 403 must never be retried.
+    expect(calls.n).toBe(1)
+  })
+})
+
+describe('the log tier and the retry policy agree about a transport blip', () => {
+  /**
+   * Full production composition: retry policy → tg-post transformer → real
+   * grammy client → a fetch that blips once. Returns the gateway log lines
+   * shaped exactly as `gateway-supervisor.log` carries them, so the REAL
+   * fleet-health detector can be run over them.
+   */
+  async function gatewayLinesForBlip(failures: number): Promise<{
+    lines: string[]
+    threw: boolean
+  }> {
+    const written: string[] = []
+    const spy = vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => {
+      written.push(String(chunk))
+      return true
+    }) as unknown as typeof process.stderr.write)
+    let threw = false
+    try {
+      const { bot } = makeBot(new TypeError('fetch failed'), failures)
+      installTgPostLogger(bot)
+      const retryApiCall = createRetryApiCall({ sleep: async () => {} })
+      // `sendRichMessage` is the user-facing reply verb the fleet-health
+      // `reply-delivery-failure` signature is scoped to (detect.ts:169) — using
+      // any other method would make the detector assertions below vacuous.
+      await retryApiCall(() =>
+        bot.api.sendRichMessage(4242, { markdown: 'the answer' }),
+      ).catch(() => {
+        threw = true
+      })
+    } finally {
+      spy.mockRestore()
+    }
+    const lines = written
+      .filter((l) => l.startsWith('tg-post '))
+      .map((l) => `2026-08-01T12:00:00Z gateway: ${l.trimEnd()}`)
+    return { lines, threw }
+  }
+
+  it('blip-then-ok: the reply lands, the blip logs status=retry, nothing escalates', async () => {
+    const { lines, threw } = await gatewayLinesForBlip(1)
+
+    // Ground truth: the reply WAS delivered. Without this the rest is vacuous.
+    expect(threw).toBe(false)
+    expect(lines).toHaveLength(2)
+    expect(lines[1]).toContain('status=ok')
+
+    // The blip is on the log, honestly labelled as an ATTEMPT — not an outcome.
+    expect(lines[0]).toContain('status=retry')
+    expect(lines[0]).toContain('err=HttpError')
+
+    // THE outcome: fleet-health raises nothing for a delivered reply.
+    const joined = lines.join('\n')
+    expect(detectGatewayFindings('alpha', joined).gw_hits['reply-delivery-failure']).toBe(0)
+    expect(scanAgent('alpha', '', joined).escalate).toBe(false)
+  })
+
+  it('transport down for every attempt: exactly one delivery-failure alert', async () => {
+    const { lines, threw } = await gatewayLinesForBlip(99)
+
+    expect(threw).toBe(true)
+    // Three attempts: two survivable, the last one terminal.
+    expect(lines).toHaveLength(3)
+    expect(lines.filter((l) => l.includes('status=retry'))).toHaveLength(2)
+    expect(lines.filter((l) => l.includes('status=err'))).toHaveLength(1)
+
+    // One alert for one lost answer — not one per attempt, not zero.
+    const joined = lines.join('\n')
+    expect(detectGatewayFindings('alpha', joined).gw_hits['reply-delivery-failure']).toBe(1)
+    expect(scanAgent('alpha', '', joined).escalate).toBe(true)
+  })
+})
+
+describe('isTransientTransportError keeps the non-retryable classes out', () => {
+  it('excludes local resource exhaustion even when grammy wrapped it', () => {
+    const enospc = Object.assign(new Error('ENOSPC: no space left on device'), {
+      code: 'ENOSPC',
+    })
+    expect(isTransientTransportError(enospc)).toBe(false)
+    // #2923: retrying a local disk failure in a tight loop is what tripped the
+    // per-bot flood ban. Wrapping it in an HttpError must not make it retryable.
+    expect(
+      isTransientTransportError(new HttpError("Network request for 'sendMessage' failed!", enospc)),
+    ).toBe(false)
+  })
+
+  it('excludes every GrammyError (Telegram answered — that IS the outcome)', () => {
+    for (const code of [400, 403, 404]) {
+      const err = new GrammyError(
+        'Call to sendMessage failed!',
+        { ok: false, error_code: code, description: 'nope' },
+        'sendMessage',
+        {},
+      )
+      expect(isTransientTransportError(err)).toBe(false)
+    }
+  })
+
+  it('excludes a caller-initiated abort, but NOT a grammy request timeout', () => {
+    // `bot.stop()` / an explicit AbortSignal aborts the fetch. Retrying that
+    // three times just re-aborts and adds backoff sleeps to shutdown.
+    const aborted = Object.assign(new Error('The operation was aborted'), {
+      name: 'AbortError',
+      code: 'ABORT_ERR',
+    })
+    expect(
+      isTransientTransportError(
+        new HttpError("Network request for 'sendMessage' failed!", aborted),
+      ),
+    ).toBe(false)
+
+    // grammy's OWN timeout is a different object: `createTimeout` rejects the
+    // race with a plain Error BEFORE aborting the controller
+    // (grammy 1.44.0, out/core/client.js:168-178). That IS transient.
+    expect(
+      isTransientTransportError(
+        new HttpError(
+          "Network request for 'sendMessage' failed!",
+          new Error("Request to 'sendMessage' timed out after 500 seconds"),
+        ),
+      ),
+    ).toBe(true)
+  })
+
+  it('excludes an unrelated programming error', () => {
+    expect(isTransientTransportError(new TypeError('x is not a function'))).toBe(false)
+    expect(isTransientTransportError(undefined)).toBe(false)
+  })
+
+  it('accepts a transport error preserved through standard cause-chaining', () => {
+    expect(
+      isTransientTransportError(
+        new Error('send failed', { cause: new TypeError('fetch failed') }),
+      ),
+    ).toBe(true)
+  })
+})
+
+describe('willRetryTelegramFailure agrees with the loop for HttpError', () => {
+  it('reports a mid-policy grammy transport failure as retryable', async () => {
+    const { bot } = makeBot(new TypeError('fetch failed'), 99)
+    const err = (await bot.api.sendMessage(7, 'hi').catch((e: unknown) => e)) as HttpError
+
+    // Same fact the loop uses: attempt 0 of 3 ⇒ another attempt follows.
+    expect(
+      willRetryTelegramFailure(
+        { errorCode: null, message: err.message, cause: err },
+        { attempt: 0, maxRetries: 3, maxFloodSleepMs: 120_000 },
+      ),
+    ).toBe(true)
+  })
+
+  it('reports the LAST attempt as terminal', async () => {
+    const { bot } = makeBot(new TypeError('fetch failed'), 99)
+    const err = (await bot.api.sendMessage(7, 'hi').catch((e: unknown) => e)) as HttpError
+
+    expect(
+      willRetryTelegramFailure(
+        { errorCode: null, message: err.message, cause: err },
+        { attempt: 2, maxRetries: 3, maxFloodSleepMs: 120_000 },
+      ),
+    ).toBe(false)
+  })
+
+  it('reports a real GrammyError 403 as terminal', async () => {
+    const err = new GrammyError(
+      'Call to sendMessage failed!',
+      { ok: false, error_code: 403, description: 'Forbidden' },
+      'sendMessage',
+      {},
+    )
+    expect(
+      willRetryTelegramFailure(
+        { errorCode: err.error_code, message: err.message, cause: err },
+        { attempt: 0, maxRetries: 3, maxFloodSleepMs: 120_000 },
+      ),
+    ).toBe(false)
+  })
+})

package/telegram-plugin/tests/rollout-narration-edit-socket.test.ts

@@ -0,0 +1,170 @@
+/**
+ * #4065 — the SOCKET-LEVEL proof that hostd actually RECEIVES the
+ * `rollout_status_edited` reply over the real transport.
+ *
+ * Why this file exists
+ * --------------------
+ * Every other test for this feature fakes the seam that carries the bug:
+ * `rollout-status-edit.test.ts` drives `handleRolloutStatusEdit` against an
+ * in-process `{ send }` sink, and `rollout-narrator.test.ts` drives the
+ * narrator against a fake relay. Both were green while the feature was
+ * COMPLETELY INERT in production, because the relay's `edit()` half-closed the
+ * socket (`client.end()`) right after writing the request, and the gateway's
+ * IPC server is `Bun.listen`, which tears the connection down on the peer FIN
+ * (~5ms) instead of keeping it readable. The real reply — which only arrives
+ * AFTER a Telegram API round-trip, i.e. hundreds of ms later — was written to a
+ * dead socket (`socket.write` returning -1, unchecked by `IpcClientImpl.send`)
+ * and never reached hostd. hostd then always resolved
+ * `{ok:false, gone:false, reason:"no edit reply"}`, which by design never
+ * re-posts: identical behaviour to no fix at all.
+ *
+ * So this test wires the REAL `createIpcServer` (Bun.listen) to the REAL
+ * `SocketRolloutNarrationRelay` over a real unix socket, and makes the reply
+ * land only AFTER a delay at least as long as a Telegram round-trip. It fails
+ * on the half-close implementation and passes on the hold-open one.
+ *
+ * Runs under bun (`Bun.listen` is a bun built-in) — vitest-excluded, named by
+ * telegram-plugin/scripts/bun-test-ci.sh via the `tests/` target.
+ */
+import { describe, it, expect, afterEach } from "vitest";
+import { mkdtempSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { createIpcServer, type IpcServer, type IpcClient } from "../gateway/ipc-server.js";
+import { handleRolloutStatusEdit } from "../gateway/rollout-status-edit.js";
+import type { RolloutStatusEditMessage } from "../gateway/ipc-protocol.js";
+import { SocketRolloutNarrationRelay } from "../../src/host-control/rollout-narration-relay.js";
+
+/**
+ * The load-bearing constant. A real `editMessageText` costs a network
+ * round-trip to api.telegram.org — on the order of 100-500ms, and more under
+ * `robustApiCall`'s 429 retry policy. The reply CANNOT be delivered
+ * synchronously with the request, which is exactly why a half-closed socket
+ * loses it. Anything comfortably above one event-loop turn reproduces it; 300ms
+ * is a conservative stand-in for the round-trip while keeping the suite fast.
+ */
+const TELEGRAM_ROUND_TRIP_MS = 300;
+
+function tmpSocket(): string {
+  const dir = mkdtempSync(join(tmpdir(), "rollout-edit-sock-"));
+  return join(dir, "gateway.sock");
+}
+
+function wait(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+/**
+ * Stand up the real IPC server with the real edit handler behind a delayed
+ * `editMessage`, and return a real relay pointed at it.
+ */
+function harness(editMessage: () => Promise<unknown>): {
+  server: IpcServer;
+  relay: SocketRolloutNarrationRelay;
+} {
+  const socketPath = tmpSocket();
+  const server = createIpcServer({
+    socketPath,
+    onClientRegistered: () => {},
+    onClientDisconnected: () => {},
+    onToolCall: async () => ({ type: "tool_call_result", id: "x", success: true }),
+    onSessionEvent: () => {},
+    onPermissionRequest: () => {},
+    onHeartbeat: () => {},
+    onScheduleRestart: () => {},
+    onRolloutStatusEdit: (client: IpcClient, msg: RolloutStatusEditMessage) =>
+      handleRolloutStatusEdit(
+        {
+          selfAgentName: "carrie",
+          operatorChatId: () => "12345",
+          editMessage,
+          log: () => {},
+        },
+        client,
+        msg,
+      ),
+  });
+  const relay = new SocketRolloutNarrationRelay({
+    resolveGatewaySocket: () => socketPath,
+    editReplyTimeoutMs: 5000,
+  });
+  return { server, relay };
+}
+
+describe("rollout narration edit — real socket, delayed reply (#4065)", () => {
+  const servers: IpcServer[] = [];
+
+  afterEach(async () => {
+    for (const s of servers) await s.close();
+    servers.length = 0;
+  });
+
+  it("receives an ok reply that arrives a Telegram round-trip after the request", async () => {
+    const { server, relay } = harness(async () => {
+      await wait(TELEGRAM_ROUND_TRIP_MS);
+      return { message_id: 77 };
+    });
+    servers.push(server);
+    await wait(50);
+
+    const started = Date.now();
+    const outcome = await relay.edit({
+      requestId: "ro-ok-1",
+      agentName: "carrie",
+      messageId: 77,
+      text: "phase 2/4 — restarting carrie",
+    });
+
+    // The outcome must be the gateway's real answer, not the connection-closed
+    // fallback the half-close produced.
+    expect(outcome).toEqual({ ok: true });
+    // And it must genuinely have waited for the round-trip, i.e. the socket
+    // stayed alive across it rather than resolving early on close.
+    expect(Date.now() - started).toBeGreaterThanOrEqual(TELEGRAM_ROUND_TRIP_MS);
+  });
+
+  it("receives a gone:true classification that arrives after the round-trip", async () => {
+    const { server, relay } = harness(async () => {
+      await wait(TELEGRAM_ROUND_TRIP_MS);
+      throw new Error("Bad Request: message to edit not found");
+    });
+    servers.push(server);
+    await wait(50);
+
+    const outcome = await relay.edit({
+      requestId: "ro-gone-1",
+      agentName: "carrie",
+      messageId: 77,
+      text: "phase 3/4 — verifying carrie",
+    });
+
+    // This is the whole point of #4065: only an explicit `gone` may trigger the
+    // narrator's one-shot re-post. A lost reply degrades to gone:false and the
+    // operator keeps staring at a frozen card.
+    expect(outcome.ok).toBe(false);
+    expect(outcome.gone).toBe(true);
+    expect(outcome.reason).toMatch(/message to edit not found/);
+  });
+
+  it("receives a transient classification (gone:false) after the round-trip", async () => {
+    const { server, relay } = harness(async () => {
+      await wait(TELEGRAM_ROUND_TRIP_MS);
+      throw new Error("Bad Gateway");
+    });
+    servers.push(server);
+    await wait(50);
+
+    const outcome = await relay.edit({
+      requestId: "ro-transient-1",
+      agentName: "carrie",
+      messageId: 77,
+      text: "phase 4/4 — carrie back up",
+    });
+
+    expect(outcome.ok).toBe(false);
+    expect(outcome.gone).toBe(false);
+    // A REAL gateway answer, distinguishable from the "no edit reply" the
+    // dropped-reply bug produced.
+    expect(outcome.reason).toMatch(/Bad Gateway/);
+  });
+});