switchroom 0.16.46 → 0.17.0

package/telegram-plugin/gateway/busy-key-reaper.ts

@@ -0,0 +1,113 @@
+/**
+ * #2787 Mechanism B — the `claudeBusyKeys` orphan-reaper lifecycle, extracted
+ * so the gateway glue that mutates the busy-key set stays in lockstep with its
+ * shadow insertion-timestamp map and is unit-testable WITHOUT importing
+ * `gateway.ts` (which boots the bot + IPC listener on module load and so cannot
+ * run inside a unit test). The gateway keeps ownership of the two containers
+ * (`claudeBusyKeys: Set`, `claudeBusyKeySince: Map`) and calls these helpers at
+ * every mark / reap site; `disconnect-flush.ts` clears them at bridge death.
+ * The test harness (`tests/busy-key-reaper.test.ts`) wires the SAME containers,
+ * the SAME delivery queue, and the real `flushOnAgentDisconnect` to reproduce
+ * the disconnect → reconnect → re-mark and slow-vs-orphan delivery sequences.
+ *
+ * Two invariants this module enforces:
+ *
+ *  1. LOCKSTEP. The timestamp map shadows actual SET MEMBERSHIP, not its own
+ *     prior presence. `markBusyKeyLockstep` stamps a fresh timestamp whenever a
+ *     key transitions idle→busy (`claudeBusyKeys.has(key) === false`). Gating on
+ *     the set — not on `since.has` — is load-bearing: a disconnect flush clears
+ *     `claudeBusyKeys` directly, so if the shadow map ever lagged, a `!since.has`
+ *     guard would decline to re-stamp on the reconnect→re-mark and leave a
+ *     stale, >TTL-old timestamp on a freshly-marked key — which the reaper would
+ *     then reap out from under a live delivery.
+ *
+ *  2. SLOW-DELIVERY SAFETY (the #1922 hazard). `busy` is marked EAGERLY at
+ *     delivery, and the gateway→claude enqueue-ack lag can be up to ~5 MINUTES
+ *     under load (#1922). During that eager-mark→enqueue window the gateway's
+ *     `currentTurn` is null yet the turn is real-and-merely-slow. A bare
+ *     time-grace reaper cannot tell "slow" from "orphaned" and would reap a
+ *     genuine delivery, re-opening the idle-drain gate while claude is about to
+ *     process the very inbound whose key it just reaped (duplicate / concurrent
+ *     delivery on the CORE inbound path). So the reap is PROOF-GATED: a key is
+ *     reaped ONLY when it has NO entry in the delivery-confirm queue. A
+ *     slow-but-real inbound is tracked there from delivery until claude's
+ *     `enqueue` ack (via the never-drop re-deliver loop), so any key still
+ *     awaiting its turn is present and skipped — no matter how slow. The time
+ *     grace is retained as defense-in-depth, defaulting well above the observed
+ *     ack-lag tail. Only a busy-marked key with NO pending delivery AND age past
+ *     the grace is a true orphan (its turn acked and should have cleared busy at
+ *     turn_end, or it was a steer/interrupt inbound excluded from tracking that
+ *     amends a now-absent turn) — reaping those can never clobber a slow one.
+ */
+
+/** Lockstep mark: add to the busy set and, on the idle→busy transition only,
+ *  stamp the insertion time. Keyed on set membership so a re-mark after a
+ *  disconnect flush (which cleared the set directly) always re-stamps fresh. */
+export function markBusyKeyLockstep(
+  keys: Set<string>,
+  since: Map<string, number>,
+  key: string,
+  now: number,
+): void {
+  const wasBusy = keys.has(key)
+  keys.add(key)
+  if (!wasBusy) since.set(key, now)
+}
+
+/** Lockstep clear: delete from both containers. */
+export function clearBusyKeyLockstep(
+  keys: Set<string>,
+  since: Map<string, number>,
+  key: string,
+): void {
+  keys.delete(key)
+  since.delete(key)
+}
+
+export interface ReapOptions {
+  /** Grace before a busy-marked key with no pending delivery is a true orphan. */
+  ttlMs: number
+  /** True iff a delivery-confirm entry is still tracked for this key — i.e. a
+   *  slow-but-real inbound awaiting its enqueue ack. Such keys are NEVER reaped,
+   *  regardless of age (the #1922 slow-delivery guarantee). */
+  hasPendingDelivery: (key: string) => boolean
+  /** Optional line logger for each reaped key (observability). */
+  log?: (msg: string) => void
+}
+
+/**
+ * Reap orphaned busy markers. Call ONLY once the gateway has asserted no turn is
+ * in flight (`currentTurn == null`). Returns the keys reaped (for the caller's
+ * metrics / assertions). Also prunes shadow-map entries whose key already left
+ * the set, so the map can't grow unbounded.
+ */
+export function reapOrphanBusyKeys(
+  keys: Set<string>,
+  since: Map<string, number>,
+  now: number,
+  opts: ReapOptions,
+): string[] {
+  const reaped: string[] = []
+  for (const key of [...since.keys()]) {
+    if (!keys.has(key)) {
+      since.delete(key)
+      continue
+    }
+    // Proof gate: a key with a pending delivery is a slow-but-real inbound still
+    // awaiting its enqueue ack (the never-drop re-deliver loop owns it) — NEVER
+    // an orphan, regardless of age. Skip so a multi-minute (#1922) delivery
+    // can't be reaped out from under claude.
+    if (opts.hasPendingDelivery(key)) continue
+    const stampedAt = since.get(key) ?? now
+    if (now - stampedAt >= opts.ttlMs) {
+      keys.delete(key)
+      since.delete(key)
+      reaped.push(key)
+      opts.log?.(
+        `telegram gateway: reaped orphan busy key=${key} (no turn in flight, ` +
+          `no pending delivery, >${opts.ttlMs}ms) — unwedging idle-drain`,
+      )
+    }
+  }
+  return reaped
+}

package/telegram-plugin/gateway/disconnect-flush.ts

@@ -42,6 +42,12 @@ export interface DisconnectFlushDeps<Ctrl extends { finalize: (reason?: 'done' |
    *  activeTurnStartedAt — the bridge just died, every turn it
    *  was handed is dead by definition. */
   claudeBusyKeys: Set<string>
+  /** #2787: insertion-timestamp map for `claudeBusyKeys`, backing the orphan
+   *  reaper. MUST stay in lockstep with `claudeBusyKeys` at every clear site —
+   *  a stale timestamp surviving a clear makes a later re-marked key look
+   *  >TTL-old and get reaped against a live delivery. Deleted alongside every
+   *  `claudeBusyKeys.delete`/`.clear` below. */
+  claudeBusyKeySince: Map<string, number>
 
   /** Open draft-stream handles keyed by chat:thread:replyId. */
   activeDraftStreams: Map<string, Stream>
@@ -82,6 +88,7 @@ export function flushOnAgentDisconnect<
     activeReactionMsgIds,
     activeTurnStartedAt,
     claudeBusyKeys,
+    claudeBusyKeySince,
     activeDraftStreams,
     clearActiveReactions,
     disposeProgressDriver,
@@ -109,6 +116,7 @@ export function flushOnAgentDisconnect<
     activeReactionMsgIds.delete(key)
     activeTurnStartedAt.delete(key)
     claudeBusyKeys.delete(key)
+    claudeBusyKeySince.delete(key) // #2787: keep orphan-TTL map in lockstep
   }
   clearActiveReactions()
 
@@ -129,6 +137,7 @@ export function flushOnAgentDisconnect<
       activeTurnStartedAt.delete(k)
       activeReactionMsgIds.delete(k)
       claudeBusyKeys.delete(k)
+      claudeBusyKeySince.delete(k) // #2787: keep orphan-TTL map in lockstep
     }
     log(
       `telegram gateway: disconnect-flush swept ${danglingKeys.length} dangling turn key(s) ` +
@@ -156,6 +165,8 @@ export function flushOnAgentDisconnect<
     const orphanCount = claudeBusyKeys.size
     const orphanKeys = [...claudeBusyKeys]
     claudeBusyKeys.clear()
+    // #2787: keep the orphan-TTL map in lockstep with the set it shadows.
+    for (const k of orphanKeys) claudeBusyKeySince.delete(k)
     log(
       `telegram gateway: disconnect-flush cleared ${orphanCount} orphan claudeBusyKeys ` +
       `entr${orphanCount === 1 ? 'y' : 'ies'} (synthetic-inbound deliveries that never turn_ended)` +

package/telegram-plugin/gateway/escalation-bridge-gate.ts

@@ -0,0 +1,46 @@
+/**
+ * escalation-bridge-gate.ts — the bridge-alive gate for the obligation sweep's
+ * ESCALATE branch, extracted from obligationSweep so the "do NOT direct-send the
+ * 'I may have missed this' nudge while the Telegram bridge is down" decision
+ * (#2788 Gap A) is EXECUTABLE in a pure unit test.
+ *
+ * The bug (#2788 Gap A): the obligation sweep keeps running while the Telegram
+ * bridge is disconnected. Unlike the represent branch (which goes through the
+ * bridge → buffer and is naturally stranded behind a dead bridge), the escalate
+ * branch DIRECT-SENDS via `bot.api.sendMessage`, bypassing the bridge. So during
+ * a transient bridge flap the user could receive a false "I may have missed this"
+ * even though the real reply is simply queued behind the outage and will land the
+ * moment the bridge recovers.
+ *
+ * The fix is a deferral, not a drop: obligations already survive bridge death
+ * (they live in the durable ledger and are re-evaluated every sweep), so gating
+ * the escalate SEND on bridge-alive and returning without closing leaves the
+ * obligation OPEN. The next sweep after the bridge reconnects re-drives it — the
+ * nudge is deferred until it can reflect a real, post-recovery miss. This adds NO
+ * unbounded liveness dependency: the entire gateway's inbound delivery already
+ * rests on the bridge eventually reconnecting / the process restarting.
+ *
+ * PURE — no Telegram, no IPC; the gateway injects `bridgeAlive` (read from
+ * `ipcServer.getClient(agent)?.isAlive()`).
+ */
+
+export interface EscalationBridgeGateDeps {
+  /**
+   * True when this agent's bridge sidecar is currently registered and alive on
+   * the IPC server — i.e. a direct escalate send can actually reflect delivery
+   * reality rather than racing a queued-behind-a-dead-bridge reply.
+   */
+  bridgeAlive: boolean
+}
+
+/**
+ * Decide whether the escalate branch should DEFER (skip this sweep, leave the
+ * obligation OPEN) because the bridge is down.
+ *
+ * Returns true ⇒ defer: do not send the nudge, do not close — the durable
+ * obligation is re-driven on a later sweep once the bridge recovers.
+ * Returns false ⇒ the bridge is alive; proceed with escalation.
+ */
+export function shouldDeferEscalationForBridge(deps: EscalationBridgeGateDeps): boolean {
+  return !deps.bridgeAlive
+}

package/telegram-plugin/gateway/gate-parity-probe.ts

@@ -0,0 +1,102 @@
+/**
+ * Gate-parity drift probe — scaffolding for the inbound-delivery
+ * state-machine cutover (#2794, RFC PR3b→PR4).
+ *
+ * ## Why this exists
+ *
+ * The inbound-delivery pipeline is mid-migration and TRIPLE-maintained:
+ *
+ *   1. the pure state machine (`inbound-delivery-machine.ts`) — the model,
+ *   2. the shadow (`inbound-delivery-machine-shadow.ts`) — module-scope
+ *      machine state advanced by `shadowEmit`, now AUTHORITATIVE for the
+ *      turn-in-flight gate via `isMachineInTurn()`,
+ *   3. the legacy imperative `claudeBusyKeys` set — still fully maintained
+ *      in parallel and read as the kill-switch fallback.
+ *
+ * Per #2794 the standing risk is DRIFT: because the same turn-lifecycle
+ * fact ("is a turn in flight?") is tracked in two live places, a fix in
+ * one can silently diverge from the other. The machine is authoritative
+ * today, but `claudeBusyKeys` is still wired and still read on the
+ * kill-switch path — so the two MUST NOT drift in the dangerous direction.
+ *
+ * ## What "agree" means (and the ONE intended divergence)
+ *
+ * The machine and `claudeBusyKeys` are designed to agree on every
+ * WELL-FORMED schedule (every turnStart has a matching turnEnd). They are
+ * deliberately allowed to diverge in exactly ONE direction on a MALFORMED
+ * schedule — an orphaned turnStart (turn B opens before turn A's turnEnd
+ * ever lands, the gymbro/clerk 5-min dangle of 2026-05-28):
+ *
+ *   - `busykeys_dangle`  — machine reads idle (self-healed via TTL tick or
+ *                          single-activeTurn reopen) while `claudeBusyKeys`
+ *                          still holds an orphan key. This is EXPECTED and
+ *                          GOOD: it is precisely the wedge the machine was
+ *                          made authoritative to kill. Not a drift alarm.
+ *
+ *   - `machine_over_holds` — machine reads in-flight while `claudeBusyKeys`
+ *                            is empty. This is the DANGEROUS direction: the
+ *                            now-authoritative gate would hold closed while
+ *                            the imperative view says idle — a NEW wedge
+ *                            class the cutover must never introduce. This
+ *                            is the drift #2794 warns about; surface it.
+ *
+ * This module is a pure classifier plus a log-only runtime probe. It
+ * performs NO I/O of its own beyond an optional injected log sink and
+ * changes NO delivery behaviour — it only observes. Deleting it once the
+ * `claudeBusyKeys` shadow is removed in PR4 is a no-op for behaviour.
+ */
+
+export type GateParityDivergence = 'none' | 'busykeys_dangle' | 'machine_over_holds'
+
+/**
+ * Classify the relationship between the machine's authoritative
+ * turn-in-flight read and the legacy imperative `claudeBusyKeys` size.
+ * Pure — no side effects.
+ */
+export function gateParityDivergence(
+  machineInTurn: boolean,
+  busyKeysSize: number,
+): GateParityDivergence {
+  const busy = busyKeysSize > 0
+  if (machineInTurn === busy) return 'none'
+  // machine idle, busyKeys non-empty → the orphan dangle the machine heals.
+  if (!machineInTurn && busy) return 'busykeys_dangle'
+  // machine in-flight, busyKeys empty → the dangerous over-hold.
+  return 'machine_over_holds'
+}
+
+/**
+ * True only for divergences that indicate a real cutover regression
+ * (the machine holding a gate the imperative shadow believes is open).
+ * The benign `busykeys_dangle` — the very wedge the machine fixes — is
+ * NOT flagged, so this probe has zero false positives on the known-good
+ * self-heal path.
+ */
+export function isDangerousGateDivergence(d: GateParityDivergence): boolean {
+  return d === 'machine_over_holds'
+}
+
+/**
+ * Log-only runtime drift canary. Call at the authoritative gate read.
+ * Emits a single grep-friendly `gw-trace gate-drift` line ONLY on the
+ * dangerous over-hold direction. Returns the machine value UNCHANGED so
+ * it can wrap the gate read without altering behaviour:
+ *
+ *   return probeGateParity(isMachineInTurn(), claudeBusyKeys.size)
+ *
+ * @param log  optional sink (default stderr) — test hook.
+ */
+export function probeGateParity(
+  machineInTurn: boolean,
+  busyKeysSize: number,
+  log: (line: string) => void = (line) => process.stderr.write(line),
+): boolean {
+  const d = gateParityDivergence(machineInTurn, busyKeysSize)
+  if (isDangerousGateDivergence(d)) {
+    log(
+      `gw-trace gate-drift kind=${d} machineInTurn=${machineInTurn} ` +
+        `busyKeys=${busyKeysSize} note=machine-authoritative-gate-holds-while-imperative-idle\n`,
+    )
+  }
+  return machineInTurn
+}