switchroom 0.18.10 → 0.18.12

package/telegram-plugin/tests/retry-api-call.test.ts

@@ -15,8 +15,14 @@ import {
   createSwallowingRetryApiCall,
   retryWithThreadFallback,
   isHtmlParseRejectError,
+  isMessageTooLongError,
+  isPhotoDimensionRejectError,
   isLocalResourceError,
+  isFloodWaitActiveError,
   LOCAL_RESOURCE_EXHAUSTED,
+  FLOOD_WAIT_ACTIVE,
+  DEFAULT_MAX_FLOOD_SLEEP_MS,
+  type FloodWaitActiveError,
   type RetryObserver,
 } from '../retry-api-call.js'
 import { errors, makeGrammyError } from './fake-bot-api.js'
@@ -571,3 +577,395 @@ describe('#2923 — LOCAL resource exhaustion is NOT retried (avoids flood ban)'
     expect(seen).toEqual([42])
   })
 })
+
+describe('#3084 — a long flood ban fails FAST instead of sleeping for hours', () => {
+  /**
+   * Deterministic fake clock: `sleep` never waits, it just advances a counter.
+   * Nothing here touches a real timer, so "did we try to sleep 4.6 hours?" is
+   * an assertion on the fake clock, not a stopwatch.
+   */
+  function fakeClock() {
+    let elapsedMs = 0
+    const slept: number[] = []
+    return {
+      get elapsedMs() {
+        return elapsedMs
+      },
+      slept,
+      sleep: async (ms: number) => {
+        slept.push(ms)
+        elapsedMs += ms
+      },
+    }
+  }
+
+  // The real value overlord's gateway logged on 2026-07-11.
+  const REAL_BAN_SEC = 16_739
+
+  it('does NOT sleep a 4.6-hour retry_after (clock never advances past the ceiling)', async () => {
+    const clock = fakeClock()
+    let calls = 0
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: clock.sleep })
+
+    await expect(
+      retry(async () => {
+        calls++
+        throw errors.floodWait(REAL_BAN_SEC)
+      }),
+    ).rejects.toThrow(FLOOD_WAIT_ACTIVE)
+
+    expect(calls).toBe(1) // failed fast on the first 429 — no retry storm
+    expect(clock.slept).toEqual([]) // never even attempted the sleep
+    expect(clock.elapsedMs).toBe(0)
+    expect(clock.elapsedMs).toBeLessThanOrEqual(DEFAULT_MAX_FLOOD_SLEEP_MS)
+  })
+
+  it('carries retry_after / untilTs / original on the thrown marker', async () => {
+    const clock = fakeClock()
+    const before = Date.now()
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: clock.sleep })
+
+    const err = await retry(async () => {
+      throw errors.floodWait(REAL_BAN_SEC)
+    }).catch((e: unknown) => e)
+
+    expect(isFloodWaitActiveError(err)).toBe(true)
+    const flood = err as FloodWaitActiveError
+    expect(flood.retryAfterSec).toBe(REAL_BAN_SEC)
+    expect(flood.untilTs).toBeGreaterThanOrEqual(before + REAL_BAN_SEC * 1000)
+    expect(flood.original).toBeInstanceOf(GrammyError)
+  })
+
+  it('STILL records the flood window via onFloodWait before throwing (#2923 breaker)', async () => {
+    const clock = fakeClock()
+    const seen: number[] = []
+    const order: string[] = []
+    const retry = createRetryApiCall({
+      maxRetries: 3,
+      sleep: clock.sleep,
+      onFloodWait: (s) => {
+        seen.push(s)
+        order.push('recorded')
+      },
+      observer: {
+        onGiveUp: () => {
+          order.push('gave_up')
+        },
+      },
+    })
+
+    await expect(
+      retry(async () => {
+        throw errors.floodWait(REAL_BAN_SEC)
+      }),
+    ).rejects.toThrow(FLOOD_WAIT_ACTIVE)
+
+    // The breaker MUST learn about the window — that is what suppresses the
+    // restart boot-card that would otherwise extend the ban.
+    expect(seen).toEqual([REAL_BAN_SEC])
+    expect(order).toEqual(['recorded', 'gave_up'])
+  })
+
+  it('a throwing onFloodWait hook still yields the marker, not the hook error', async () => {
+    const clock = fakeClock()
+    const retry = createRetryApiCall({
+      maxRetries: 3,
+      sleep: clock.sleep,
+      onFloodWait: () => {
+        throw new Error('disk full while persisting flood state')
+      },
+    })
+    await expect(
+      retry(async () => {
+        throw errors.floodWait(REAL_BAN_SEC)
+      }),
+    ).rejects.toThrow(FLOOD_WAIT_ACTIVE)
+  })
+
+  it('a SHORT retry_after under the ceiling still sleeps-and-retries and succeeds', async () => {
+    const clock = fakeClock()
+    let n = 0
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: clock.sleep })
+
+    // 30s — the everyday rate-limit blip. Behaviour must be unchanged.
+    const out = await retry(async () => {
+      if (n++ === 0) throw errors.floodWait(30)
+      return 'ok'
+    })
+
+    expect(out).toBe('ok')
+    expect(n).toBe(2)
+    expect(clock.slept).toEqual([30_000])
+  })
+
+  it('the historic 28s and 75s waits are still slept through (no regression)', async () => {
+    for (const sec of [28, 75]) {
+      const clock = fakeClock()
+      let n = 0
+      const retry = createRetryApiCall({ maxRetries: 3, sleep: clock.sleep })
+      const out = await retry(async () => {
+        if (n++ === 0) throw errors.floodWait(sec)
+        return 'ok'
+      })
+      expect(out).toBe('ok')
+      expect(clock.slept).toEqual([sec * 1000])
+    }
+  })
+
+  it('sleeps exactly AT the ceiling, throws just above it', async () => {
+    const atCeiling = DEFAULT_MAX_FLOOD_SLEEP_MS / 1000
+
+    const clockAt = fakeClock()
+    let n = 0
+    const retryAt = createRetryApiCall({ maxRetries: 3, sleep: clockAt.sleep })
+    expect(
+      await retryAt(async () => {
+        if (n++ === 0) throw errors.floodWait(atCeiling)
+        return 'ok'
+      }),
+    ).toBe('ok')
+    expect(clockAt.slept).toEqual([DEFAULT_MAX_FLOOD_SLEEP_MS])
+
+    const clockOver = fakeClock()
+    const retryOver = createRetryApiCall({ maxRetries: 3, sleep: clockOver.sleep })
+    await expect(
+      retryOver(async () => {
+        throw errors.floodWait(atCeiling + 1)
+      }),
+    ).rejects.toThrow(FLOOD_WAIT_ACTIVE)
+    expect(clockOver.slept).toEqual([])
+  })
+
+  it('honours a caller-supplied maxFloodSleepMs ceiling', async () => {
+    const clock = fakeClock()
+    const retry = createRetryApiCall({
+      maxRetries: 3,
+      sleep: clock.sleep,
+      maxFloodSleepMs: 5_000,
+    })
+    // 30s would sleep under the DEFAULT ceiling; with a 5s ceiling it throws.
+    await expect(
+      retry(async () => {
+        throw errors.floodWait(30)
+      }),
+    ).rejects.toThrow(FLOOD_WAIT_ACTIVE)
+    expect(clock.slept).toEqual([])
+  })
+
+  it('maxRetries semantics unchanged for sustained SHORT floods', async () => {
+    const clock = fakeClock()
+    let calls = 0
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: clock.sleep })
+
+    await expect(
+      retry(async () => {
+        calls++
+        throw errors.floodWait(10)
+      }),
+    ).rejects.toThrow('retryApiCall: max retries exceeded')
+
+    expect(calls).toBe(3)
+    expect(clock.slept).toEqual([10_000, 10_000, 10_000])
+  })
+
+  it('the swallowing wrapper absorbs the marker (no unhandled rejection)', async () => {
+    const clock = fakeClock()
+    const lines: string[] = []
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: clock.sleep })
+    const swallow = createSwallowingRetryApiCall(retry, (l) => lines.push(l))
+
+    const out = await swallow(
+      async () => {
+        throw errors.floodWait(REAL_BAN_SEC)
+      },
+      { verb: 'sendMessage' },
+    )
+
+    expect(out).toBeUndefined()
+    expect(lines.join('')).toContain(FLOOD_WAIT_ACTIVE)
+  })
+})
+
+describe('#3084 — the fast throw must not become a re-drive amplifier', () => {
+  const REAL_BAN_SEC = 16_739
+
+  /**
+   * EXACT duck-type predicate used by the card surfaces to arm their 429
+   * cooldown — copied verbatim from `worker-activity-feed.ts`
+   * (`extractRetryAfterSecs`) and `issues-card.ts`. They do NOT use
+   * `instanceof GrammyError`. If the marker doesn't satisfy this, they
+   * classify it 'transient', arm no cooldown, and their 5-6s heartbeats
+   * re-drive the send into the open ban.
+   */
+  function extractRetryAfterSecs(err: unknown): number | null {
+    if (err == null || typeof err !== 'object') return null
+    const e = err as { error_code?: unknown; parameters?: { retry_after?: unknown } }
+    if (e.error_code !== 429) return null
+    const ra = e.parameters?.retry_after
+    if (typeof ra === 'number' && Number.isFinite(ra) && ra > 0) return ra
+    return null
+  }
+
+  it('the marker satisfies the card surfaces\' 429 cooldown duck-type', async () => {
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: async () => {} })
+    const err = await retry(async () => {
+      throw errors.floodWait(REAL_BAN_SEC)
+    }).catch((e: unknown) => e)
+
+    // worker-activity-feed / issues-card would now arm a FULL-window cooldown
+    // instead of treating this as a generic 'transient' error.
+    expect(extractRetryAfterSecs(err)).toBe(REAL_BAN_SEC)
+  })
+
+  it('the marker is still NOT a GrammyError (resend-fallback probes must not match)', async () => {
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: async () => {} })
+    const err = await retry(async () => {
+      throw errors.floodWait(REAL_BAN_SEC)
+    }).catch((e: unknown) => e)
+
+    expect(err).not.toBeInstanceOf(GrammyError)
+    expect(isHtmlParseRejectError(err)).toBe(false)
+    expect(isMessageTooLongError(err)).toBe(false)
+    expect(isPhotoDimensionRejectError(err)).toBe(false)
+  })
+
+  it('a card heartbeat ticking across the whole 4.6h ban issues ONE Bot API call', async () => {
+    // The regression that must be impossible. Before the pre-call gate: the
+    // marker throws in ~1ms, the 6s heartbeat re-drives, and we fire ~2,760
+    // fresh requests into the still-open per-bot ban — feeding the exact flood
+    // counter #2923 exists to starve. Strictly worse than the original hang.
+    let now = 1_000_000
+    let state: { untilTs: number } | null = null
+    let apiCalls = 0
+
+    const retry = createRetryApiCall({
+      maxRetries: 3,
+      sleep: async () => {},
+      // The #2923 breaker: record the window...
+      onFloodWait: (sec) => {
+        state = { untilTs: now + sec * 1000 }
+      },
+      // ...and the #3084 gate: refuse to send into it.
+      floodWaitRemainingMs: () => (state ? Math.max(0, state.untilTs - now) : 0),
+    })
+
+    const HEARTBEAT_MS = 6_000
+    const ticks = Math.ceil((REAL_BAN_SEC * 1000) / HEARTBEAT_MS) // ~2,790
+    let markerThrows = 0
+
+    for (let i = 0; i < ticks; i++) {
+      // A worker-feed style heartbeat: best-effort edit, swallow failures,
+      // and (deliberately, to model the WORST case) arm no cooldown of its own.
+      await retry(async () => {
+        apiCalls++
+        throw errors.floodWait(REAL_BAN_SEC)
+      }).catch((e: unknown) => {
+        if (isFloodWaitActiveError(e)) markerThrows++
+      })
+      now += HEARTBEAT_MS
+    }
+
+    expect(ticks).toBeGreaterThan(2_700) // we really did simulate the full ban
+    expect(markerThrows).toBe(ticks) // every tick got a clean degraded state
+    // …and the Bot API was touched a HANDFUL of times, not once per tick.
+    // Without the pre-call gate this is ~2,790 requests fired into an open
+    // per-bot ban. With it: the first tick (which discovers the ban) plus the
+    // few tail ticks that run after the simulated window has elapsed and
+    // legitimately re-probe. Bounded and small is the whole contract.
+    expect(apiCalls).toBeLessThanOrEqual(5)
+    expect(apiCalls).toBeGreaterThanOrEqual(1)
+    expect(apiCalls / ticks).toBeLessThan(0.01) // <1% of ticks reach the API
+  })
+
+  it('resumes issuing calls once the window expires (the gate is not a latch)', async () => {
+    let now = 1_000_000
+    let state: { untilTs: number } | null = null
+    let apiCalls = 0
+
+    const retry = createRetryApiCall({
+      maxRetries: 3,
+      sleep: async () => {},
+      onFloodWait: (sec) => {
+        state = { untilTs: now + sec * 1000 }
+      },
+      floodWaitRemainingMs: () => (state ? Math.max(0, state.untilTs - now) : 0),
+    })
+
+    await expect(
+      retry(async () => {
+        apiCalls++
+        throw errors.floodWait(300)
+      }),
+    ).rejects.toThrow(FLOOD_WAIT_ACTIVE)
+    expect(apiCalls).toBe(1)
+
+    // Mid-window: gated, no call.
+    now += 100_000
+    await expect(retry(async () => 'ok')).rejects.toThrow(FLOOD_WAIT_ACTIVE)
+    expect(apiCalls).toBe(1)
+
+    // Window expired: the very next call goes through and succeeds.
+    now += 300_000
+    expect(
+      await retry(async () => {
+        apiCalls++
+        return 'ok'
+      }),
+    ).toBe('ok')
+    expect(apiCalls).toBe(2)
+  })
+
+  it('a SHORT open window is not short-circuited (sleep-and-retry still rides it out)', async () => {
+    // A 30s window is under the ceiling: the normal path may still succeed
+    // (the server may have cleared it), so we must NOT gate it.
+    let apiCalls = 0
+    let n = 0
+    const retry = createRetryApiCall({
+      maxRetries: 3,
+      sleep: async () => {},
+      floodWaitRemainingMs: () => 30_000,
+    })
+    const out = await retry(async () => {
+      apiCalls++
+      if (n++ === 0) throw errors.floodWait(30)
+      return 'ok'
+    })
+    expect(out).toBe('ok')
+    expect(apiCalls).toBe(2)
+  })
+
+  it('FAILS OPEN — a throwing/garbage flood probe never silences the bot', async () => {
+    const throwing = createRetryApiCall({
+      maxRetries: 3,
+      sleep: async () => {},
+      floodWaitRemainingMs: () => {
+        throw new Error('EACCES: unreadable flood-wait.json')
+      },
+    })
+    expect(await throwing(async () => 'ok')).toBe('ok')
+
+    // Garbage from a corrupt marker file must be treated as "no window".
+    // Infinity in particular would otherwise mute the bot forever.
+    for (const garbage of [NaN, -1, 0, Infinity]) {
+      const retry = createRetryApiCall({
+        maxRetries: 3,
+        sleep: async () => {},
+        floodWaitRemainingMs: () => garbage,
+      })
+      expect(await retry(async () => 'ok')).toBe('ok')
+    }
+  })
+
+  it('with no flood probe wired at all, behaviour is exactly as before', async () => {
+    let apiCalls = 0
+    let n = 0
+    const retry = createRetryApiCall({ maxRetries: 3, sleep: async () => {} })
+    const out = await retry(async () => {
+      apiCalls++
+      if (n++ === 0) throw errors.floodWait(30)
+      return 'ok'
+    })
+    expect(out).toBe('ok')
+    expect(apiCalls).toBe(2)
+  })
+})

package/telegram-plugin/tests/session-model-file.test.ts

@@ -15,6 +15,8 @@ import {
   readSessionModelFileRaw,
   restoreSessionModelFileRaw,
   clearSessionModelFile,
+  clearSessionModelBootAttempts,
+  SESSION_MODEL_BOOT_ATTEMPTS_FILE,
   writeRelaunchModelIntent,
   clearRelaunchModelIntent,
   readConfiguredDefaultModel,
@@ -229,3 +231,51 @@ describe('intentForRestartReason is keep-for-everything (#3039)', () => {
     }
   })
 })
+
+describe('clearSessionModelBootAttempts (#3043 item 2: bridge-register clears the crashloop counter)', () => {
+  const bootAttemptsPath = () => join(dir, SESSION_MODEL_BOOT_ATTEMPTS_FILE)
+
+  // Faithful model of start.sh.hbs "Override crashloop self-heal": each boot
+  // within 150s of the previous stamp bumps the count; a stale stamp resets to
+  // 1. Reproduced here so the accumulate-vs-reset OUTCOME is asserted, not just
+  // the delete call.
+  function simulateBootStamp(nowSec: number): number {
+    let count = 0
+    let prev = 0
+    if (existsSync(bootAttemptsPath())) {
+      const [c, p] = readFileSync(bootAttemptsPath(), 'utf8').trim().split(/\s+/)
+      count = Number(c) || 0
+      prev = Number(p) || 0
+    }
+    count = nowSec - prev < 150 ? count + 1 : 1
+    writeFileSync(bootAttemptsPath(), `${count} ${nowSec}\n`)
+    return count
+  }
+
+  it('WITHOUT a bridge register, three fast boots accumulate toward the 3-strike clear', () => {
+    // Three operator hand-bounces, each <150s apart, with no register between.
+    expect(simulateBootStamp(1000)).toBe(1)
+    expect(simulateBootStamp(1010)).toBe(2)
+    expect(simulateBootStamp(1020)).toBe(3) // start.sh would now clear a HEALTHY override — the false positive
+  })
+
+  it('a bridge register between boots resets the counter, so a healthy agent never reaches 3', () => {
+    expect(simulateBootStamp(1000)).toBe(1)
+    // Boot 1 came all the way up and the bridge registered → gateway clears it.
+    clearSessionModelBootAttempts(dir)
+    expect(existsSync(bootAttemptsPath())).toBe(false)
+
+    // Next fast boot starts fresh at 1 (no accumulation), and register clears again.
+    expect(simulateBootStamp(1010)).toBe(1)
+    clearSessionModelBootAttempts(dir)
+    expect(simulateBootStamp(1020)).toBe(1)
+    // The healthy agent's counter never climbs to the 3-strike clear.
+    const [count] = readFileSync(bootAttemptsPath(), 'utf8').trim().split(/\s+/)
+    expect(Number(count)).toBeLessThan(3)
+  })
+
+  it('is best-effort — no throw when the counter file is absent', () => {
+    expect(existsSync(bootAttemptsPath())).toBe(false)
+    expect(() => clearSessionModelBootAttempts(dir)).not.toThrow()
+  })
+})

package/telegram-plugin/tests/status-pin-boot-recovery.test.ts

@@ -232,18 +232,43 @@ describe("status-pin boot cleanup is mutex-gated (structural)", () => {
     );
   });
 
-  it("every statusPinBootCleanup() call site follows the won-lock check", () => {
-    // Find each *invocation* (with parens), excluding the declaration.
+  it("every boot pin-cleanup call site follows the won-lock check", () => {
+    // #3026: boot cleanup is now invoked via the mutex-gated orchestrator
+    // runBootPinCleanupAndDmSweep(), which awaits statusPinBootCleanup()
+    // (+ the other reapers) and then runs the DM stale-pin sweep. The
+    // invariant is unchanged: cleanup must only run after the lock is won.
     const lines = gatewaySrc.split("\n");
     const lockIdx = lines.findIndex((l) => l.includes("acquireStartupLock({"));
     expect(lockIdx).toBeGreaterThan(-1);
 
+    // Every `void runBootPinCleanupAndDmSweep()` invocation must come AFTER
+    // the acquireStartupLock outcome is available (the boot block the winner
+    // — or the link()-unsupported fallback — reaches).
+    const callIdxs = lines
+      .map((l, i) => ({ l, i }))
+      .filter(
+        ({ l }) =>
+          /void runBootPinCleanupAndDmSweep\(\)/.test(l) &&
+          !l.trimStart().startsWith("//"),
+      )
+      .map(({ i }) => i);
+    expect(callIdxs.length).toBeGreaterThan(0);
+    for (const idx of callIdxs) {
+      expect(idx).toBeGreaterThan(lockIdx);
+    }
+
+    // statusPinBootCleanup() itself must only ever be invoked from INSIDE
+    // that orchestrator — never at a bare post-import call site that a
+    // losing double-boot could reach.
     const declIdx = lines.findIndex((l) =>
       /async function statusPinBootCleanup/.test(l),
     );
     expect(declIdx).toBeGreaterThan(-1);
-
-    const callIdxs = lines
+    const orchestratorIdx = lines.findIndex((l) =>
+      /async function runBootPinCleanupAndDmSweep/.test(l),
+    );
+    expect(orchestratorIdx).toBeGreaterThan(-1);
+    const invocationIdxs = lines
       .map((l, i) => ({ l, i }))
       .filter(
         ({ l, i }) =>
@@ -253,25 +278,21 @@ describe("status-pin boot cleanup is mutex-gated (structural)", () => {
           !l.includes("statusPinBootCleanup() is deliberately NOT"),
       )
       .map(({ i }) => i);
-
-    // There must be at least one real call, and every real call must come
-    // AFTER the acquireStartupLock outcome is available (i.e. inside the boot
-    // block that only the winner reaches).
-    expect(callIdxs.length).toBeGreaterThan(0);
-    for (const idx of callIdxs) {
-      expect(idx).toBeGreaterThan(lockIdx);
+    expect(invocationIdxs.length).toBeGreaterThan(0);
+    for (const idx of invocationIdxs) {
+      expect(idx).toBeGreaterThan(orchestratorIdx);
     }
   });
 
   it("the blocked (losing) boot exits before reaching cleanup", () => {
     // In the mutex block, `outcome.status === 'blocked'` must lead to
-    // process.exit(1) BEFORE any statusPinBootCleanup() call in that block —
-    // so a loser never touches the shared store.
+    // process.exit(1) BEFORE the winner's runBootPinCleanupAndDmSweep() call
+    // in that block — so a loser never touches the shared store.
     const blockedIdx = gatewaySrc.indexOf("outcome.status === 'blocked'");
     expect(blockedIdx).toBeGreaterThan(-1);
     const afterBlocked = gatewaySrc.slice(blockedIdx);
     const exitIdx = afterBlocked.indexOf("process.exit(1)");
-    const cleanupIdx = afterBlocked.indexOf("void statusPinBootCleanup()");
+    const cleanupIdx = afterBlocked.indexOf("void runBootPinCleanupAndDmSweep()");
     expect(exitIdx).toBeGreaterThan(-1);
     expect(cleanupIdx).toBeGreaterThan(-1);
     // The exit for the blocked branch appears before the winner's cleanup call.