switchroom 0.15.45 → 0.16.5

package/telegram-plugin/quota-watch.ts

@@ -30,6 +30,13 @@
  * IPC call (cheap). `probeQuota` is only called on state-change (when
  * we're going to send a message anyway) to get fresh numbers for the
  * notification body. On no-change polls, only `listState` is called.
+ *
+ * #2495 Change 3 — the transition-to-alarm probe is `forceLive` (bypasses
+ * the broker's probe-on-open TTL), so the DECISION to alarm is corroborated
+ * by a TRUE live probe of the affected account, not a possibly-stale cache
+ * read. The re-evaluation with fresh numbers can suppress an alarm whose
+ * stale-snapshot transition no longer holds. Steady state stays cheap: a
+ * no-change poll never probes. Cost is one live probe per transition edge.
  */
 
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from "fs";
@@ -175,6 +182,51 @@ export type QuotaWatchDecision =
     }
   | { kind: "skip"; accountLabel: string; reason: string };
 
+/**
+ * #2495 BLOCKER fix — the corroboration probe result, as the gateway's
+ * runQuotaWatch sees it from `brokerClient.probeQuota(..., forceLive=true)`.
+ * Structurally a subset of `ProbeQuotaEntry` (src/auth/broker/client.ts): a
+ * `result` discriminated on `ok`, plus a `served` tag the broker stamps to
+ * say HOW the result was sourced.
+ *
+ * The trap this guards: under `forceLive`, when the upstream live probe FAILS
+ * and the broker holds a prior snapshot, it returns `cachedSnapshotToResult`
+ * — `result.ok === true` but `served === "cache"` (server.ts opProbeQuota).
+ * A naive `result.ok` check then treats that stale cache read as a live
+ * corroboration, fires the alarm, and stamps the false "Live-probe
+ * corroborated (#2495)" footnote. The acceptance criterion is the opposite:
+ * an alarm must be backed by a LIVE probe, not a stale cache read.
+ */
+export type CorroborationProbe = {
+  result: { ok: true } | { ok: false };
+  /**
+   * How the result was sourced. `"live"` = fresh upstream probe (genuine
+   * corroboration). `"cache"` = served from the durable cache (TTL-hit or
+   * probe-failure fallback) — NOT corroboration. Absent on legacy responses,
+   * which we treat as NOT corroborated (fail-closed: never claim a live
+   * corroboration we can't prove).
+   */
+  served?: "live" | "cache";
+};
+
+/**
+ * #2495 BLOCKER fix — decide whether a forceLive corroboration probe counts
+ * as a genuine LIVE corroboration of the alarm.
+ *
+ * Genuine corroboration requires BOTH `result.ok` AND `served === "live"`.
+ * A result that is `ok:true` but `served:"cache"` (the failed-probe
+ * cache-fallback) is treated EXACTLY like a probe failure: it is NOT
+ * corroboration, so the caller must DEFER — leave watch state untouched and
+ * re-evaluate next tick when a true live probe can be obtained. A missing
+ * entry (`undefined`) is likewise not corroboration.
+ *
+ * Pure + total so it can be unit-tested at the seam without standing up the
+ * broker or the gateway loop.
+ */
+export function isLiveCorroboration(entry: CorroborationProbe | undefined): boolean {
+  return entry?.result.ok === true && entry.served === "live";
+}
+
 /**
  * Evaluate one account's quota state against its last-notified health.
  *
@@ -224,7 +276,11 @@ export function evaluateQuotaWatchAccount(args: {
     return { kind: "skip", accountLabel: label, reason: "stale-snapshot" };
   }
 
-  const currentHealth = classifyHealth(snap);
+  // #2494 Bug A — classify against THIS tick's clock so the refill
+  // normalization uses the same `now` the rest of the decision does (the
+  // default `new Date()` would diverge from a frozen test clock / a replayed
+  // tick and mis-zero a still-future reset window).
+  const currentHealth = classifyHealth(snap, new Date(now));
 
   // Unknown (probe failed) or blocked — skip entirely.
   if (currentHealth === "unknown" || currentHealth === "blocked") {
@@ -324,22 +380,58 @@ export type FleetAllExhaustedDecision =
  * cases the trigger-based interactive all-blocked card misses: a quiet period
  * (no agent happens to 429 into the wall) and the consumer/cron paths.
  *
- * Authoritative source: the broker's per-account `exhausted` flag (set by
- * mark-exhausted via failover + the consumer sensor), NOT probe-derived health
- * — so there is no probe-failure false-alarm. Requires at least one account;
- * an empty fleet never alerts.
+ * Source: the broker's per-account `exhausted` flag (set by mark-exhausted via
+ * failover + the consumer sensor). That flag is NOT purely live — `isAccountBlocked`
+ * (src/auth/broker/account-eligibility.ts) falls back to the persisted
+ * `exhausted_until` mark whenever there is no fresh live snapshot. During a
+ * broker-unreachable / probe-timeout blackout, short-lived auto-fallback marks
+ * can make `every(a.exhausted)` momentarily true with ZERO live corroboration
+ * (#2478, klanker 2026-06-20). So the `entered` alert requires POSITIVE LIVE
+ * CORROBORATION: an account counts toward "all exhausted" only when its
+ * `exhausted` flag is backed by a FRESH live snapshot (last_quota.capturedAt
+ * within `maxStaleMs`). If ANY account's exhaustion rests solely on a
+ * stale/absent-probe mark we are
+ * probe-blind and return `skip: "probe-blind"` — no false fleet alert. The
+ * guarantee is "no false alarm off stale marks during a probe blackout", NOT
+ * blanket probe-failure immunity. The `recovered` transition is unguarded so a
+ * legitimately-fired alert is never stranded. Requires at least one account; an
+ * empty fleet never alerts.
  */
 export function evaluateFleetAllExhausted(args: {
-  accounts: Array<{ label: string; exhausted: boolean; exhausted_until?: number }>;
+  accounts: Array<{
+    label: string;
+    exhausted: boolean;
+    exhausted_until?: number;
+    /** Most-recent live probe snapshot, used to corroborate `exhausted`. */
+    last_quota?: {
+      capturedAt: number;
+      overageDisabledReason?: string | null;
+    } | null;
+  }>;
   prev: QuotaWatchAccountState;
   now: number;
+  /** Staleness ceiling for "fresh probe"; 0 disables the gate (legacy callers/tests). */
+  tuning?: Pick<QuotaWatchTuning, "maxStaleMs">;
 }): FleetAllExhaustedDecision {
   const { accounts, prev, now } = args;
+  const maxStaleMs = args.tuning?.maxStaleMs ?? 0;
   const allExhausted = accounts.length > 0 && accounts.every((a) => a.exhausted);
   // "throttling" doubles as the "currently alerting all-exhausted" marker.
   const wasAlerting = prev.lastNotifiedHealth === "throttling";
 
   if (allExhausted && !wasAlerting) {
+    // Probe-blind guard (#2478): only fire `entered` if EVERY account's
+    // exhaustion is backed by live evidence — a fresh snapshot. An account
+    // exhausted solely on a stale/absent mark means we have no live
+    // corroboration → skip rather than false-alarm.
+    if (maxStaleMs > 0) {
+      const allLiveCorroborated = accounts.every((a) =>
+        exhaustionLiveCorroborated(a, now, maxStaleMs),
+      );
+      if (!allLiveCorroborated) {
+        return { kind: "skip", reason: "probe-blind" };
+      }
+    }
     return {
       kind: "notify",
       message: buildAllExhaustedMessage(accounts, now),
@@ -358,6 +450,42 @@ export function evaluateFleetAllExhausted(args: {
   return { kind: "skip", reason: allExhausted ? "still-all-exhausted" : "not-all-exhausted" };
 }
 
+/**
+ * Is an account's `exhausted` flag backed by live evidence (#2478)?
+ *
+ * True when the most-recent live probe is FRESH (`capturedAt` within
+ * `maxStaleMs`) — that fresh probe is what set/upholds the broker's blocked
+ * verdict. False when there is no `last_quota` at all, or the snapshot is
+ * stale: the `exhausted` flag then rests solely on a persisted mark with no
+ * live backing, which is exactly the probe-blind condition that false-fires
+ * the fleet alert.
+ *
+ * NOTE: `out_of_credits` is NOT treated as corroboration here. Per
+ * fix/out-of-credits-serve-block, out_of_credits is INFORMATIONAL — it is
+ * not exhaustion in its own right at any util. Corroboration requires a
+ * genuinely fresh quota snapshot (real 429 / util-wall path).
+ *
+ * Mirrors `snapshotFresh` in src/auth/broker/account-eligibility.ts (the
+ * serving-side authority); kept as a local check so the decision layer
+ * carries no broker dependency.
+ */
+function exhaustionLiveCorroborated(
+  account: {
+    last_quota?: { capturedAt: number; overageDisabledReason?: string | null } | null;
+  },
+  now: number,
+  maxStaleMs: number,
+): boolean {
+  const lq = account.last_quota;
+  if (!lq) return false;
+  // Mirror `snapshotFresh`'s clock-skew guard: a future-dated `capturedAt`
+  // makes `now - capturedAt` negative and would slip past the staleness gate,
+  // so a skewed snapshot reads as fresh. Reject snapshots dated more than the
+  // broker's 60_000 ms tolerance ahead of `now` (matches the inline literal in
+  // `snapshotFresh`, src/auth/broker/account-eligibility.ts).
+  return now - lq.capturedAt <= maxStaleMs && lq.capturedAt <= now + 60_000;
+}
+
 function buildAllExhaustedMessage(
   accounts: Array<{ label: string; exhausted_until?: number }>,
   now: number,
@@ -420,7 +548,7 @@ function buildThrottlingMessage(agentName: string, snap: AccountSnapshot): strin
     `Binding window: ${winLabel}${resetStr}`,
     `${activeNote}${altNote}`,
     ``,
-    `<i>Threshold: ${THROTTLING_THRESHOLD_PCT}% on either window. Source: broker quota cache.</i>`,
+    `<i>Threshold: ${THROTTLING_THRESHOLD_PCT}% on either window. Live-probe corroborated (#2495).</i>`,
     `<i>Run /auth for full fleet status or /usage for the active account.</i>`,
   ]
     .join("\n")

package/telegram-plugin/registry/turns-schema.test.ts

@@ -23,6 +23,30 @@ import {
   getTurnByKey,
 } from './turns-schema.js'
 
+// ---------------------------------------------------------------------------
+// Concurrency PRAGMAs — applySchema must arm busy_timeout so concurrent
+// writers (the subagent-tracker hooks + the gateway watcher) wait-and-retry
+// instead of failing with SQLITE_BUSY ("database is locked").
+// ---------------------------------------------------------------------------
+
+describe('registry concurrency PRAGMAs', () => {
+  it('arms busy_timeout (5000ms) on every opened connection', () => {
+    const db = openTurnsDbInMemory()
+    const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number }
+    expect(row.timeout).toBe(5000)
+    db.close()
+  })
+
+  it('uses WAL journal mode for concurrent readers', () => {
+    const db = openTurnsDbInMemory()
+    const row = db.prepare('PRAGMA journal_mode').get() as { journal_mode: string }
+    // `:memory:` reports 'memory'; a file DB reports 'wal'. Either way the
+    // exec ran without error — the file-path open (openTurnsDb) yields 'wal'.
+    expect(['wal', 'memory']).toContain(String(row.journal_mode).toLowerCase())
+    db.close()
+  })
+})
+
 // ---------------------------------------------------------------------------
 // Test 1 — empty DB
 // ---------------------------------------------------------------------------

package/telegram-plugin/registry/turns-schema.ts

@@ -172,6 +172,15 @@ const PHASE2_MIGRATIONS = [
 function applySchema(db: SqliteDatabase): void {
   db.exec('PRAGMA journal_mode = WAL')
   db.exec('PRAGMA synchronous = NORMAL')
+  // Concurrency: multiple writers contend on this registry (the PreToolUse
+  // subagent-tracker hook, the gateway's subagent-watcher backfill, the turns
+  // writer) — especially when several sub-agents dispatch at once. Without a
+  // busy_timeout, bun:sqlite/better-sqlite3 default to 0ms and the second
+  // contending write fails IMMEDIATELY with SQLITE_BUSY ("database is locked"),
+  // which the watcher swallows → jsonl_agent_id / parent_turn_key left NULL →
+  // worker card mis-routes to the operator DM + false silent-stall synthesis.
+  // 5s of wait-and-retry serializes the contenders instead of dropping writes.
+  db.exec('PRAGMA busy_timeout = 5000')
   db.exec(SCHEMA_SQL)
   // Run migrations. SQLite doesn't support "ADD COLUMN IF NOT EXISTS", so
   // we swallow the "duplicate column" error to stay idempotent on

package/telegram-plugin/runtime-metrics.ts

@@ -77,6 +77,19 @@ export type RuntimeMetricEvent =
       fallback_kind: 'working' | 'thinking'
       silence_ms: number
     }
+  /**
+   * #2527 — mid-turn liveness floor decision. `decision: 'fire'` when the
+   * quiet "still on it" beat was sent; otherwise the machine-readable skip
+   * reason for a declined forced ("Status?") poke. `forced` distinguishes
+   * the timer beat from a user-asked one.
+   */
+  | {
+      kind: 'mid_turn_floor'
+      key: string
+      silence_ms: number
+      forced: boolean
+      decision: string
+    }
   /**
    * #1445 cross-turn pending-async ambient lifecycle. `started` fires
    * when a turn ends with a captured anchor AND a pending Agent/Task/

package/telegram-plugin/session-tail.ts

@@ -98,7 +98,17 @@ export type SessionEvent =
   // the lazily-flushed transcript. The draft-mirror drives off THIS, not
   // the flush-gated `tool_use`, so activity streams deterministically.
   | { kind: 'tool_label'; toolUseId: string; label: string; toolName: string }
-  | { kind: 'text'; text: string }
+  // `blockIndex` = index of this text block in the assistant message's
+  // content[] — load-bearing: it keys the returned Map so callers emit
+  // events in source order. `lastInMessage` = true iff no tool_use block
+  // follows it in the SAME message. NOTE: `lastInMessage` is a PROJECTION
+  // ARTIFACT only — the current reducer-side narrative-dedup gate
+  // (narrative-dedup.ts) decides draft-then-send vs working-narration by
+  // LOOKAHEAD (the next tool_use / turn_end), NOT by reading this flag. It
+  // is retained as a stable projection output (pinned by the kernel test)
+  // and reserved for a future staging-skip optimization; do not assume the
+  // gate keys on it.
+  | { kind: 'text'; text: string; blockIndex: number; lastInMessage: boolean }
   | { kind: 'tool_result'; toolUseId: string; toolName: string | null; isError?: boolean; errorText?: string }
   | { kind: 'turn_end'; durationMs: number }
   // Multi-agent: sub-agent-scoped events. agentId is the sub-agent JSONL
@@ -106,8 +116,12 @@ export type SessionEvent =
   // as parent events; the reducer fans them out to per-sub-agent state.
   | { kind: 'sub_agent_started'; agentId: string; firstPromptText: string; subagentType?: string }
   | { kind: 'sub_agent_tool_use'; agentId: string; toolUseId: string | null; toolName: string; input?: Record<string, unknown>; precomputedLabel?: string }
-  | { kind: 'sub_agent_text'; agentId: string; text: string }
-  | { kind: 'sub_agent_narrative'; agentId: string; text: string }
+  // Same shared contract as the main-agent `text` kind — see its doc above
+  // (including the `lastInMessage` projection-artifact note). The wire-kind
+  // stays distinct (the gateway/watcher split is load-bearing) but the
+  // payload + `lastInMessage` derivation are identical so ONE shared dedup
+  // gate handles both tiers.
+  | { kind: 'sub_agent_text'; agentId: string; text: string; blockIndex: number; lastInMessage: boolean }
   | { kind: 'sub_agent_tool_result'; agentId: string; toolUseId: string; isError?: boolean; errorText?: string }
   | { kind: 'sub_agent_turn_end'; agentId: string }
   | { kind: 'sub_agent_nested_spawn'; agentId: string }
@@ -182,6 +196,49 @@ function extractToolResultErrorText(content: unknown): string {
   return ''
 }
 
+/**
+ * THE single text→narrative projection primitive. Both projectTranscriptLine
+ * and projectSubagentLine derive their text events through this helper so
+ * main-agent, sub-agent, worker, and every other execution shape inherit
+ * identical text-block semantics from ONE place: empty/whitespace blocks are
+ * dropped, and each surviving block carries its `blockIndex` plus the
+ * `lastInMessage` signal (no tool_use follows it in this message). NOTE:
+ * `lastInMessage` is a projection artifact — the reducer-side dedup gate
+ * decides SHOW/SUPPRESS by lookahead, not by reading this flag (see the
+ * SessionEvent `text` doc); it is reserved for a future staging-skip
+ * optimization.
+ *
+ * `make` adapts the shared payload into the tier-specific wire kind
+ * (`text` vs `sub_agent_text`); the contract — what counts as a text block,
+ * how `lastInMessage` is computed — lives here, not in the callers.
+ *
+ * Returns a `Map<blockIndex, SessionEvent>` keyed by the text block's source
+ * index, NOT a flat list. This is the load-bearing design choice: the callers
+ * must emit thinking / tool_use / text events in SOURCE ORDER (the reducer
+ * pairs a preamble to the immediately-next tool_use), so they iterate
+ * `content` once and, at each text position, emit the precomputed event from
+ * this map. The kernel owns the contract; the caller owns only the ordering.
+ */
+export function projectAssistantTextBlocks(
+  content: Array<Record<string, unknown>>,
+  make: (text: string, blockIndex: number, lastInMessage: boolean) => SessionEvent,
+): Map<number, SessionEvent> {
+  const out = new Map<number, SessionEvent>()
+  // Precompute the index of the last tool_use so each text block knows
+  // whether a tool_use follows it in THIS message (the draft-then-send signal).
+  let lastToolUseIdx = -1
+  content.forEach((c, i) => {
+    if (c.type === 'tool_use') lastToolUseIdx = i
+  })
+  content.forEach((c, i) => {
+    if (c.type !== 'text') return
+    const text = (c.text as string | undefined) ?? ''
+    if (text.trim().length === 0) return // drop empty/whitespace-only blocks
+    out.set(i, make(text, i, i > lastToolUseIdx))
+  })
+  return out
+}
+
 /**
  * Project a single transcript line into a SessionEvent (or null if it's
  * uninteresting noise). Caller is responsible for the JSON parse — if a
@@ -218,7 +275,16 @@ export function projectTranscriptLine(line: string): SessionEvent[] {
     const content = message?.content as Array<Record<string, unknown>> | undefined
     if (!Array.isArray(content)) return []
     const events: SessionEvent[] = []
-    for (const c of content) {
+    // Text→narrative projection comes from the ONE shared kernel
+    // (projectAssistantTextBlocks): it owns the empty-drop + blockIndex +
+    // lastInMessage contract. We emit its events at their source positions
+    // so thinking / tool_use / text stay in source order (the reducer pairs
+    // a preamble to the immediately-next tool_use).
+    const textEvents = projectAssistantTextBlocks(
+      content,
+      (text, blockIndex, lastInMessage): SessionEvent => ({ kind: 'text', text, blockIndex, lastInMessage }),
+    )
+    content.forEach((c, i) => {
       const ct = c.type as string | undefined
       if (ct === 'thinking') {
         events.push({ kind: 'thinking' })
@@ -237,10 +303,10 @@ export function projectTranscriptLine(line: string): SessionEvent[] {
           input: input && typeof input === 'object' ? input : undefined,
         })
       } else if (ct === 'text') {
-        const text = (c.text as string | undefined) ?? ''
-        events.push({ kind: 'text', text })
+        const ev = textEvents.get(i)
+        if (ev != null) events.push(ev)
       }
-    }
+    })
     return events
   }
 
@@ -357,7 +423,25 @@ export function projectSubagentLine(
     const content = message?.content as Array<Record<string, unknown>> | undefined
     if (!Array.isArray(content)) return []
     const events: SessionEvent[] = []
-    for (const c of content) {
+    // Text→narrative projection comes from the SAME shared kernel as the
+    // main agent (projectAssistantTextBlocks): one source for the empty-drop
+    // + blockIndex + lastInMessage contract. The `make` adapter only changes
+    // the wire kind to `sub_agent_text`. A nested Agent/Task tool_use still
+    // counts as a tool_use that follows a preceding text block — handled by
+    // the kernel — so a sub-agent preamble before a nested spawn is correctly
+    // NOT `lastInMessage`. We emit at source positions so text + tool_use
+    // stay in source order (the reducer pairs preamble → next tool_use).
+    const textEvents = projectAssistantTextBlocks(
+      content,
+      (text, blockIndex, lastInMessage): SessionEvent => ({
+        kind: 'sub_agent_text',
+        agentId,
+        text,
+        blockIndex,
+        lastInMessage,
+      }),
+    )
+    content.forEach((c, i) => {
       const ct = c.type as string | undefined
       if (ct === 'tool_use') {
         const name = (c.name as string | undefined) ?? ''
@@ -386,10 +470,11 @@ export function projectSubagentLine(
         // in the SAME assistant message must be emitted in source order
         // so the reducer consumes the preamble on the immediately-next
         // tool_use and sibling tool_uses fall back to filename/pattern.
-        const text = (c.text as string | undefined) ?? ''
-        events.push({ kind: 'sub_agent_text', agentId, text })
+        // The event itself comes from the shared kernel (textEvents above).
+        const ev = textEvents.get(i)
+        if (ev != null) events.push(ev)
       }
-    }
+    })
     // Authoritative early terminal: a background `Agent` worker's JSONL on
     // claude ≥2.1.156 never writes the `system/turn_duration` line below, so
     // the watcher used to only learn the worker finished via the ~5-min