switchroom 0.18.17 → 0.18.18

package/telegram-plugin/gateway/linear-activity.ts

@@ -297,11 +297,20 @@ export async function emitLinearAgentActivity(
   return { content: [{ type: 'text', text: `Linear ${type} emitted on session ${sessionId}` }] }
 }
 
+/** The exact HTML-comment carrying a capture's dedup key. Both the embed
+ *  (captureDedupMarker) and the dedup lookup match on this precise string, so a
+ *  re-capture is confirmed by exact-key equality, not fuzzy search relevance.
+ *  The trailing ` -->` and leading `: ` anchor the key so `abc` never matches
+ *  `abcd`. */
+export function captureDedupComment(dedupKey: string): string {
+  return `<!-- switchroom-capture: ${dedupKey} -->`
+}
+
 /** Hidden marker appended to a captured issue's description so a re-capture of
  *  the same Telegram message can be detected (dedup backstop; the gateway-side
  *  seen-set is the primary, race-free guard). */
 export function captureDedupMarker(dedupKey: string): string {
-  return `\n\n<!-- switchroom-capture: ${dedupKey} -->`
+  return `\n\n${captureDedupComment(dedupKey)}`
 }
 
 /**
@@ -390,19 +399,26 @@ export async function createLinearIssue(
   }
 
   // Dedup backstop: search for a prior capture of the same Telegram message.
+  // `searchIssues` is a relevance-ranked full-text search, so its top hit for a
+  // key can be an unrelated issue that merely shares tokens. We therefore only
+  // treat a result as a dedup match when its description carries the EXACT
+  // capture marker for this key, never the raw top hit.
   if (dedupKey) {
+    const marker = captureDedupComment(dedupKey)
     const search = await gql(
-      'query($term: String!) { searchIssues(term: $term) { nodes { id url title } } }',
+      'query($term: String!) { searchIssues(term: $term, first: 25) { nodes { id url title description } } }',
       { term: dedupKey },
     )
     if (search.ok) {
-      const hit = (search.data?.searchIssues?.nodes ?? [])[0] as { url?: string } | undefined
+      const nodes = (search.data?.searchIssues?.nodes ?? []) as Array<{ url?: string; description?: string }>
+      const hit = nodes.find((n) => typeof n.description === 'string' && n.description.includes(marker))
       if (hit?.url) {
         log(`telegram gateway: linear_create_issue: dedup hit key=${dedupKey} agent=${agent}\n`)
         return { content: [{ type: 'text', text: `Already filed: ${hit.url}` }] }
       }
     }
-    // a failed search is non-fatal — fall through to create (gateway seen-set is primary).
+    // a failed search or no exact-marker match is non-fatal — fall through to
+    // create (gateway seen-set is the primary, race-free guard).
   }
 
   // Resolve the team.

package/telegram-plugin/gateway/premium-recovery-wiring.ts

@@ -0,0 +1,122 @@
+/**
+ * premium-recovery-wiring.ts — the gateway GLUE for the "premium model
+ * recovered" ping, behind injected deps so the never-storm orchestration is
+ * unit-testable without importing the whole gateway (mirrors
+ * tier-downgrade-wiring.ts).
+ *
+ * The pure recovery predicate (`decidePremiumRecovery`), the honest wording +
+ * button (`renderPremiumRecoveryPing`), and the fleet-dedup key
+ * (`premiumRecoveryClaimKey`) all live in premium-recovery.ts. This module owns
+ * the ORDER-SENSITIVE side effects the gateway performs off that verdict — the
+ * P0 at-most-once guarantee a review cares about:
+ *
+ *   - the marker is READ first; a missing/corrupt marker is a no-op (no send);
+ *   - the pure decision gates the send: `fire === false` → NO clear, NO send
+ *     (the marker survives for a later tick once the tier actually recovers);
+ *   - a fleet-wide `claim-notification` gates against a bounce / concurrent
+ *     tick; on `!granted` the marker is cleared (so it can't linger and
+ *     re-attempt every tick) and we bail with NO send;
+ *   - on a GRANTED claim the marker is cleared BEFORE the send (never-storm:
+ *     at-most-once is the hard requirement — a transient send fault is covered
+ *     by the caller's retry policy, a double-send is not recoverable);
+ *   - EXACTLY ONE send per recorded chat, each carrying the session-scoped
+ *     `mdl:alias:<premium>` switch-back button (the SAME apply path as the model
+ *     menu — it does not bypass handleModelMenuCallback).
+ *
+ * Behaviour is a faithful extraction of the former inline
+ * `maybePremiumRecoveryPing`; the ordering above is preserved exactly.
+ */
+
+import {
+  renderPremiumRecoveryPing,
+  premiumRecoveryClaimKey,
+  type PremiumRecoveryPing,
+} from '../premium-recovery.js'
+import { MODEL_CALLBACK_ALIAS } from './model-command.js'
+
+/** The one-tap switch-back keyboard (single callback button). */
+export interface PremiumRecoveryKeyboard {
+  inline_keyboard: Array<Array<{ text: string; callback_data: string }>>
+}
+
+/** The minimal marker view the ping orchestration needs. */
+export interface PremiumRecoveryMarkerView {
+  /** The dropped premium `/model` token to offer switching back to. */
+  premiumModel: string
+  /** Chat ids to ping (the downgrade notice's allowFrom); may be empty → fallback. */
+  chats: string[]
+}
+
+export interface PremiumRecoveryPingDeps {
+  /** Bind-mounted agent state dir, or null when unresolvable (→ no-op). */
+  getAgentDir: () => string | null
+  /** Read the parsed `.premium-recovery` marker, or null when absent/corrupt. */
+  readMarker: (agentDir: string) => PremiumRecoveryMarkerView | null
+  /** Delete the marker (consume-once). Best-effort. */
+  clearMarker: (agentDir: string) => void
+  /** The agent name (for the fleet-dedup claim key). */
+  getAgent: () => string
+  /** The pure recovery predicate, bound to THIS tick's live accounts. Returns
+   *  whether to fire (the marker is already known present when this is called). */
+  decide: () => boolean
+  /** Fleet-wide at-most-once claim for this premium token. Returns granted
+   *  (fail-open: true on any broker error → degrades to at-least-once). */
+  claimNotification: (claimKey: string) => Promise<boolean>
+  /** Fallback chats when the marker records none (parity with the inline path). */
+  fallbackChats: () => string[]
+  /** Send the recovery ping (with the switch-back keyboard) to ONE chat. */
+  sendToChat: (
+    chatId: string,
+    ping: PremiumRecoveryPing,
+    keyboard: PremiumRecoveryKeyboard,
+  ) => void
+  /** Structured logger (stderr in prod, captured in test). */
+  log: (msg: string) => void
+}
+
+/**
+ * Run the premium-recovery ping glue. No-op unless a marker is pending, the
+ * pure decision says fire, and the fleet claim is granted — then clears the
+ * marker BEFORE fanning out exactly one send per chat. Async because the claim
+ * is a broker round-trip; the caller `.catch`es (a convenience ping is never
+ * allowed to throw the quota-watch tick).
+ */
+export async function runPremiumRecoveryPing(deps: PremiumRecoveryPingDeps): Promise<void> {
+  const agentDir = deps.getAgentDir()
+  if (!agentDir) return
+  const marker = deps.readMarker(agentDir)
+  if (marker == null) return
+  // Pure decision gate: on `false` leave the marker in place (a later tick fires
+  // once the tier recovers) — no clear, no claim, no send.
+  if (!deps.decide()) return
+  const agent = deps.getAgent()
+  // Fleet-wide dedup: a fresh gateway boot or a second tick inside the window
+  // must not re-send. Fail-open — a convenience ping degrades to at-least-once,
+  // never lost.
+  const granted = await deps.claimNotification(
+    premiumRecoveryClaimKey(agent, marker.premiumModel),
+  )
+  if (!granted) {
+    // Another gateway already owns this recovery ping — consume our marker so it
+    // can't linger and re-attempt every tick, and bail (NO send).
+    deps.clearMarker(agentDir)
+    return
+  }
+  // Consume the marker BEFORE sending (never-storm: at-most-once is the hard
+  // requirement for this ping; a transient send fault is covered by the
+  // caller's retry policy, whereas a double-send is not recoverable).
+  deps.clearMarker(agentDir)
+  const ping = renderPremiumRecoveryPing(marker.premiumModel)
+  const keyboard: PremiumRecoveryKeyboard = {
+    inline_keyboard: [
+      [{ text: ping.buttonText, callback_data: `${MODEL_CALLBACK_ALIAS}${marker.premiumModel}` }],
+    ],
+  }
+  const chats = marker.chats.length > 0 ? marker.chats : deps.fallbackChats()
+  for (const chatId of chats) {
+    deps.sendToChat(chatId, ping, keyboard)
+  }
+  deps.log(
+    `[premium-recovery] ${marker.premiumModel} servable again — ping sent agent=${agent} chats=${chats.length}`,
+  )
+}

package/telegram-plugin/gateway/session-model-file.ts

@@ -230,3 +230,106 @@ export function clearSessionEffortFile(agentDir: string): void {
     /* best-effort */
   }
 }
+
+// ─── Consume-once premium-recovery marker (tier-downgrade companion) ─────────
+//
+// A DURABLE marker the tier-downgrade writes when it walls a premium `/model`
+// selection fleet-wide. It records the DROPPED premium token + the chats to
+// notify, and survives the downgrade self-restart (start.sh never touches this
+// name — unlike `.session-model`, which is consumed at boot). The gateway's
+// `runQuotaWatch` tick reads it, and when the broker's `list-state` shows the
+// premium tier servable again (deterministic per-account eligibility), fires
+// EXACTLY ONE "available again" ping with a one-tap switch-back button, then
+// clears the marker (at-most-once). Also cleared the instant the user re-issues
+// `/model <premium>` manually (no stale ping). Shape-gated like the carriers
+// above — the model token passes the same MODEL_ARG_RE gate.
+
+export const PREMIUM_RECOVERY_FILE = '.premium-recovery'
+
+export interface PremiumRecoveryRecord {
+  /** The dropped premium `/model` token (e.g. `fable`) to offer switching back to. */
+  premiumModel: string
+  /** Telegram chat ids to ping on recovery (the downgrade notice's allowFrom). */
+  chats: string[]
+  ts: number
+}
+
+/**
+ * Parse `.premium-recovery` content. Null on corrupt JSON, a missing/wrong-typed
+ * field, a model token that fails the MODEL_ARG_RE shape gate, or a `chats`
+ * array that is not a non-empty list of non-empty strings.
+ */
+export function parsePremiumRecovery(text: string): PremiumRecoveryRecord | null {
+  try {
+    const raw = JSON.parse(text) as Partial<PremiumRecoveryRecord>
+    if (
+      typeof raw.premiumModel !== 'string' ||
+      !isValidModelArg(raw.premiumModel) ||
+      typeof raw.ts !== 'number' ||
+      !Array.isArray(raw.chats) ||
+      raw.chats.length === 0 ||
+      !raw.chats.every((c) => typeof c === 'string' && c.length > 0)
+    ) {
+      return null
+    }
+    return { premiumModel: raw.premiumModel, chats: raw.chats, ts: raw.ts }
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Write the premium-recovery marker. Throws on a non-canonical token (parity
+ * with the carriers) or an empty chat list — a marker with nowhere to ping is a
+ * bug, not a silent no-op.
+ */
+export function writePremiumRecoveryFile(
+  agentDir: string,
+  premiumModel: string,
+  chats: string[],
+): void {
+  if (!isValidModelArg(premiumModel)) {
+    throw new Error(`refusing to persist non-canonical premium-recovery token: ${JSON.stringify(premiumModel)}`)
+  }
+  const clean = chats.filter((c) => typeof c === 'string' && c.length > 0)
+  if (clean.length === 0) {
+    throw new Error('refusing to persist premium-recovery marker with no chats to notify')
+  }
+  atomicWrite(
+    join(agentDir, PREMIUM_RECOVERY_FILE),
+    `${JSON.stringify({ premiumModel, chats: clean, ts: Date.now() })}\n`,
+  )
+}
+
+/**
+ * Parsed premium-recovery marker, or null when absent/corrupt. A present-but-
+ * corrupt marker (null parse of a file that exists) is SWEPT on read: neither
+ * the ping path nor `clearPremiumRecoveryOnManualSwitch` can act on a null
+ * parse, so a garbled file would otherwise linger forever. Deleting it here is
+ * best-effort hygiene — the read still returns null either way.
+ */
+export function readPremiumRecoveryFile(agentDir: string): PremiumRecoveryRecord | null {
+  let raw: string
+  try {
+    raw = readFileSync(join(agentDir, PREMIUM_RECOVERY_FILE), 'utf8')
+  } catch {
+    return null // absent / unreadable — nothing on disk to sweep.
+  }
+  const parsed = parsePremiumRecovery(raw)
+  if (parsed == null) {
+    // Corrupt marker present on disk — garbage-collect it so it can't wedge the
+    // recovery path (best-effort; a failed unlink still returns null).
+    clearPremiumRecoveryFile(agentDir)
+    return null
+  }
+  return parsed
+}
+
+/** Delete the premium-recovery marker (consumed on ping / manual re-issue). Best-effort. */
+export function clearPremiumRecoveryFile(agentDir: string): void {
+  try {
+    rmSync(join(agentDir, PREMIUM_RECOVERY_FILE), { force: true })
+  } catch {
+    /* best-effort */
+  }
+}

package/telegram-plugin/gateway/tier-downgrade-wiring.ts

@@ -0,0 +1,121 @@
+/**
+ * tier-downgrade-wiring.ts — the gateway GLUE for the model-tier downgrade,
+ * behind injected deps so the orchestration is unit-testable without importing
+ * the whole gateway (mirrors throttle-tier-wiring.ts).
+ *
+ * The pure decision (`decideTierDowngrade`), the give-up/suppress routing
+ * (`planTierDowngrade`), and the user-facing wording (`renderTierDowngradeNotice`)
+ * all live in tier-downgrade.ts. This module owns the ORDER-SENSITIVE side
+ * effects the gateway performs off those verdicts — and the invariants a review
+ * cares about:
+ *
+ *   - the resume gate is PEEKed (not armed) first, so the fallible carrier write
+ *     happens before the single-flight latch is committed;
+ *   - the consume-once `.session-model` carrier is written BEFORE the arm; a
+ *     throwing write aborts the downgrade and leaves NO armed latch;
+ *   - the latch is armed only AFTER a successful carrier write;
+ *   - the durable premium-recovery marker write is best-effort (a failure never
+ *     aborts the downgrade — the recovery ping simply doesn't arm);
+ *   - the broadcast notice is the HONEST resume-on-default text (never a
+ *     revert-to-premium promise) and is sent only on a real downgrade;
+ *   - a `skip-inflight` resume-gate verdict returns `restart-pending` and emits
+ *     NO notice and NO restart (the concurrent turn's armed restart resumes).
+ */
+
+import {
+  decideTierDowngrade,
+  planTierDowngrade,
+  type ResumeGateVerdict,
+} from '../tier-downgrade.js'
+
+export type TierDowngradeOutcome = 'downgraded' | 'restart-pending' | 'skip'
+
+export interface TierDowngradeRunnerDeps {
+  /** Bind-mounted agent state dir, or null when unresolvable (→ skip). */
+  getAgentDir: () => string | null
+  /** Resolved configured-default token; '' / null → unresolved (never downgrade blind). */
+  getConfiguredDefault: () => string | null
+  /** Live session `/model` override, or null on the plain configured default. */
+  getSessionOverride: () => string | null
+  /** Model canonicalizer (the gateway passes resolveMainModel). */
+  resolve: (token: string) => string
+  /** PEEK the resume gate WITHOUT arming it. */
+  peekResumeGate: () => ResumeGateVerdict
+  /** Write the consume-once `.session-model` carrier for the default. May throw. */
+  writeCarrier: (agentDir: string, toModel: string, configuredDefault: string) => void
+  /** Commit the single-flight arm — called ONLY after a successful carrier write. */
+  armResumeGate: () => void
+  /** Persist the durable premium-recovery marker (best-effort; may throw — caught). */
+  writeRecoveryMarker: (agentDir: string, premiumModel: string) => void
+  /** Broadcast the honest downgrade notice to the operator chats. */
+  broadcastNotice: (markdown: string) => void
+  /** Fire the resume self-restart for `agent`. */
+  selfRestart: (agent: string) => void
+  /** The agent whose session self-restarts (SWITCHROOM_AGENT_NAME ?? triggerAgent). */
+  selfAgent: (triggerAgent: string) => string
+  /** Structured logger (stderr in prod, captured in test). */
+  log: (msg: string) => void
+}
+
+/**
+ * Run the tier-downgrade glue. Returns:
+ *   'downgraded'      — carrier written, latch armed, notice broadcast, restart fired.
+ *   'restart-pending' — a resume restart is already armed (concurrent turn); no
+ *                       notice, no restart (its restart replays the dead turn).
+ *   'skip'            — not applicable / carrier write failed; caller falls
+ *                       through to its all-blocked give-up card.
+ */
+export function runTierDowngrade(
+  triggerAgent: string,
+  deps: TierDowngradeRunnerDeps,
+): TierDowngradeOutcome {
+  const agentDir = deps.getAgentDir()
+  if (!agentDir) return 'skip'
+  const configuredDefault = deps.getConfiguredDefault() ?? ''
+  const decision = decideTierDowngrade({
+    sessionOverride: deps.getSessionOverride(),
+    configuredDefault,
+    resolve: deps.resolve,
+  })
+  // PEEK (no arm): the fallible carrier write must precede the latch commit.
+  const gateVerdict = deps.peekResumeGate()
+  const plan = planTierDowngrade(decision, gateVerdict, triggerAgent)
+  if (plan.kind === 'skip') {
+    if (decision.action === 'downgrade') {
+      deps.log(`[tier-downgrade] restart suppressed (${gateVerdict}) agent=${triggerAgent}`)
+    }
+    return 'skip'
+  }
+  if (plan.kind === 'suppress') {
+    deps.log(
+      `[tier-downgrade] give-up suppressed — a resume restart is already armed agent=${triggerAgent}`,
+    )
+    return 'restart-pending'
+  }
+  // plan.kind === 'downgrade'. Carrier BEFORE arm: a throwing write must not
+  // leave an armed latch with no pending restart.
+  try {
+    deps.writeCarrier(agentDir, plan.toModel, configuredDefault)
+  } catch (err) {
+    deps.log(
+      `[tier-downgrade] failed to write session-model carrier — aborting downgrade: ${(err as Error)?.message ?? err}`,
+    )
+    return 'skip'
+  }
+  deps.armResumeGate()
+  deps.log(
+    `[tier-downgrade] downgrading ${plan.fromModel} → ${plan.toModel} and resuming via self-restart agent=${triggerAgent}`,
+  )
+  // Durable premium-recovery marker — best-effort (survives the restart; the
+  // downgrade is the critical path and must not abort on a marker failure).
+  try {
+    deps.writeRecoveryMarker(agentDir, plan.fromModel)
+  } catch (err) {
+    deps.log(
+      `[tier-downgrade] premium-recovery marker write failed (non-fatal): ${(err as Error)?.message ?? err}`,
+    )
+  }
+  deps.broadcastNotice(plan.notice)
+  deps.selfRestart(deps.selfAgent(triggerAgent))
+  return 'downgraded'
+}

package/telegram-plugin/gateway/unhandled-rejection-policy.ts

@@ -12,7 +12,7 @@
 
 import { GrammyError, HttpError } from 'grammy'
 
-import { FLOOD_WAIT_ACTIVE } from '../retry-api-call.js'
+import { FLOOD_WAIT_ACTIVE, LOCAL_RESOURCE_EXHAUSTED } from '../retry-api-call.js'
 
 export type RejectionAction = 'shutdown' | 'log_only'
 
@@ -78,6 +78,19 @@ export function classifyRejection(
   // exact amplification the #2923 circuit breaker exists to stop.
   if (err instanceof Error && err.message === FLOOD_WAIT_ACTIVE) return 'log_only'
 
+  // LOCAL_RESOURCE_EXHAUSTED (#3099, sibling of FLOOD_WAIT_ACTIVE above):
+  // retry-api-call throws this plain Error marker when a send fails on a LOCAL
+  // disk/memory exhaustion (ENOSPC/EDQUOT/EIO/ENOMEM) rather than retrying it
+  // (#2923) — retrying a local-resource failure in a tight loop is what tripped
+  // the per-bot flood ban in the first place. A leaked one (a fire-and-forget
+  // send that wasn't wrapped in swallowingApiCall) must NOT crash the gateway:
+  // the box is ALREADY out of disk/memory, and a crash→restart drives a fresh
+  // round of boot-time sends and staging writes at a resource that is already
+  // exhausted — the exact amplification the #2923 marker exists to avoid. The
+  // degraded-state marker already carries this signal; a crash loop is the
+  // wrong way to surface a full disk. Same log_only posture as its sibling.
+  if (err instanceof Error && err.message === LOCAL_RESOURCE_EXHAUSTED) return 'log_only'
+
   if (!isGrammy) return 'shutdown'
 
   const e = err as { error_code?: number; description?: string }