switchroom 0.18.17 → 0.18.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/agent-scheduler/index.js +13 -0
  2. package/dist/auth-broker/index.js +13 -0
  3. package/dist/cli/notion-write-pretool.mjs +13 -0
  4. package/dist/cli/switchroom.js +605 -479
  5. package/dist/host-control/main.js +17 -1
  6. package/dist/vault/approvals/kernel-server.js +13 -0
  7. package/dist/vault/broker/server.js +13 -0
  8. package/package.json +1 -1
  9. package/telegram-plugin/bridge/bridge.ts +7 -1
  10. package/telegram-plugin/dist/bridge/bridge.js +26 -1
  11. package/telegram-plugin/dist/gateway/gateway.js +1401 -431
  12. package/telegram-plugin/dist/server.js +26 -1
  13. package/telegram-plugin/fleet-fallback-resume.ts +26 -3
  14. package/telegram-plugin/gateway/approval-hold.ts +49 -0
  15. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
  16. package/telegram-plugin/gateway/gateway.ts +362 -71
  17. package/telegram-plugin/gateway/linear-activity.ts +20 -4
  18. package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
  19. package/telegram-plugin/gateway/session-model-file.ts +103 -0
  20. package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
  21. package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
  22. package/telegram-plugin/llm-error-present.ts +436 -0
  23. package/telegram-plugin/operator-events.ts +7 -1
  24. package/telegram-plugin/permission-title.ts +172 -10
  25. package/telegram-plugin/premium-recovery.ts +101 -0
  26. package/telegram-plugin/raw-error-scrub.ts +73 -0
  27. package/telegram-plugin/retry-api-call.ts +8 -2
  28. package/telegram-plugin/send-gate-degraded.test.ts +152 -1
  29. package/telegram-plugin/send-gate-observability.test.ts +140 -0
  30. package/telegram-plugin/send-gate-observability.ts +65 -20
  31. package/telegram-plugin/send-gate.test.ts +143 -1
  32. package/telegram-plugin/send-gate.ts +212 -19
  33. package/telegram-plugin/session-tail.ts +16 -0
  34. package/telegram-plugin/shared/local-time.ts +69 -0
  35. package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
  36. package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
  37. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
  38. package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
  39. package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
  40. package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
  41. package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
  42. package/telegram-plugin/tests/permission-title.test.ts +167 -4
  43. package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
  44. package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
  45. package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
  46. package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
  47. package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
  48. package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
  49. package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
  50. package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
  51. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
  52. package/telegram-plugin/tier-downgrade.ts +198 -0
  53. package/telegram-plugin/tool-activity-summary.ts +99 -0
  54. package/telegram-plugin/worker-activity-feed.ts +509 -409
@@ -20509,6 +20509,19 @@ var init_schema = __esm(() => {
20509
20509
  approval_timeout_minutes: exports_external.number().int().nonnegative().optional().describe("Operator approval-card lifetime (minutes) for the tool-use 'Allow " + "once' card and the vault grant decision wait. After this long with " + "no operator tap, the card auto-denies (a TIMEOUT, not a denial \u2014 the " + "agent is told not to retry). Default 60. hostd-gated verbs " + "(mcp__hostd__*) keep their own longer window; the hostd " + "config-propose card is not governed by this key."),
20510
20510
  sub_agent_tick_interval_ms: exports_external.number().int().nonnegative().optional().describe("Heartbeat tick interval (ms) for sub-agent rendering. Forces a " + "re-render of the elapsed-time counter while sub-agents are running, " + "even during silent stretches between tool calls. Default 10000 (10 s). " + "Set to 0 to disable the elapsed-ticker path."),
20511
20511
  edit_budget_threshold: exports_external.number().int().nonnegative().optional().describe("Telegram API edit budget per minute before the progress-card driver " + "falls back to a slower coalesce window. When a chat accumulates more " + "than this many card edits in the trailing 60 s, the driver switches " + "to a wider coalesce interval until the rate drops back. Default 18. " + "Increase if your gateway frequently bumps the Telegram edit-rate ceiling " + "with many parallel sub-agents; decrease for a more conservative buffer."),
20512
+ send_gate: exports_external.object({
20513
+ enabled: exports_external.boolean().optional().describe("Master switch for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts) \u2014 the token-bucket scheduler every " + "Bot API call transits so per-surface throttles can't add up past a " + "flood ceiling. ON by default. Precedence: the operator break-glass " + "env var SWITCHROOM_TELEGRAM_SEND_GATE (0/false/off/no) ALWAYS wins " + "when explicitly set; this key only decides when that env var is " + "unset. Omit to keep the gate on."),
20514
+ global_per_sec: exports_external.number().positive().optional().describe("Global bucket sustained rate (Bot API calls/sec across ALL chats). " + "Default 25 (headroom under Telegram's ~30/s). Must be > 0 \u2014 a zero " + "or negative rate would wedge all outbound sends. Omit to keep 25."),
20515
+ global_burst: exports_external.number().int().positive().describe("Global bucket burst capacity. Default 4; worst-case 1s window " + "admits global_burst + global_per_sec = 29 < 30, a real margin under " + "the ceiling. Must be an integer >= 1 (a 0 capacity never admits a " + "token and wedges sends). Omit to keep 4.").optional(),
20516
+ per_chat_per_sec: exports_external.number().positive().optional().describe("Per-chat sustained rate (calls/sec to a single chat). Default 1. " + "Must be > 0 (zero/negative wedges that chat). Omit to keep 1."),
20517
+ per_chat_burst: exports_external.number().int().positive().optional().describe("Per-chat burst capacity. Default 3. Must be an integer >= 1 " + "(0 wedges the chat's bucket). Omit to keep 3."),
20518
+ per_group_per_min: exports_external.number().positive().optional().describe("Per-group sustained rate (calls/min to a single group/supergroup). " + "Default 18 (headroom under Telegram's ~20/min group ceiling). Must " + "be > 0. Omit to keep 18."),
20519
+ per_group_burst: exports_external.number().int().positive().optional().describe("Per-group burst capacity. Default 2. Must be an integer >= 1 " + "(0 wedges the group's bucket). Omit to keep 2."),
20520
+ edit_floor_ms: exports_external.number().int().nonnegative().optional().describe("Minimum ms between successive edits of the SAME message_id " + "(last-write-wins coalescing enforces this floor). Default 1500 " + "(Telegram's ~1 edit/sec/message practical ceiling). 0 disables the " + "floor. Must be an integer >= 0. Omit to keep 1500.")
20521
+ }).optional().describe("Tunable rate limits for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts). Every key is optional and defaults to " + "the send gate's built-in value, so omitting the whole block reproduces " + "today's exact behaviour \u2014 this is pure operator tuning, no default is " + "changed. Cascades from defaults.channels.telegram.send_gate."),
20522
+ worker_feed: exports_external.object({
20523
+ max_rows: exports_external.number().int().positive().optional().describe("Max live-worker rows rendered in the COMBINED worker-activity feed " + "(2+ background workers in one chat/thread coalesce into ONE message; " + "telegram-plugin/worker-activity-feed.ts) before a compact " + "'+M more working\u2026' spill line \u2014 keeps the coalesced body compact and " + "legible (and under the rich-message wire ceiling). Default 8. A " + "single-worker chat renders the full \uD83D\uDEE0 Worker card and ignores this. " + "Must be an integer >= 1. Omit to keep 8.")
20524
+ }).optional().describe("Tuning for the coalesced worker-activity feed " + "(telegram-plugin/worker-activity-feed.ts). Cascades from " + "defaults.channels.telegram.worker_feed."),
20512
20525
  stickers: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Sticker aliases for the `send_sticker` MCP tool (#576). Maps a " + "short alias name (e.g. 'happy', 'thinking') to a Telegram file_id. " + "Operator-curated \u2014 capture file_ids from inbound stickers the user " + "sends and add them here. The agent calls send_sticker(chat_id, " + "alias='happy') and the gateway resolves to the file_id at send " + "time. Aliases enable persona-flavored expressiveness without " + "exposing raw file_ids in the agent prompt. Personal-assistant / " + "health-coach personas benefit; coding agents typically don't " + "configure any."),
20513
20526
  voice_in: exports_external.object({
20514
20527
  enabled: exports_external.boolean().optional().describe("Master switch for voice-message transcription."),
@@ -40238,9 +40251,45 @@ function renderActivityFeedWithNested(lines, childLines, final = false, liveSuff
40238
40251
  stepCount
40239
40252
  });
40240
40253
  }
40254
+ var COMBINED_ROW_DESC_MAX = 72;
40255
+ function renderCombinedWorkerFeed(rows, opts) {
40256
+ if (rows.length === 0)
40257
+ return null;
40258
+ const maxRows = Math.max(1, Math.floor(opts.maxRows));
40259
+ const rowLines = (r) => {
40260
+ const desc = escapeMarkdown(truncate(stripMarkdown(r.description).replace(/\s+/g, " ").trim() || "background task", COMBINED_ROW_DESC_MAX));
40261
+ const toolWord = r.toolCount === 1 ? "tool" : "tools";
40262
+ const modelLabel = formatModelLabel(r.model);
40263
+ const modelPart = modelLabel != null ? ` \u00b7 ${escapeMarkdown(modelLabel)}` : "";
40264
+ const header = `**${desc}** _\u00b7 ${formatFeedElapsed(r.elapsedMs)} \u00b7 ${r.toolCount} ${toolWord}${modelPart}_`;
40265
+ const stepClean = stripMarkdown(r.currentStep).replace(/\s+/g, " ").trim();
40266
+ const step = stepClean.length > 0 ? `\u2192 _${escapeMarkdown(truncate(stepClean, STATUS_LINE_MAX))}_` : `\u2192 _starting\u2026_`;
40267
+ return [header, step];
40268
+ };
40269
+ const compose = (visibleCount) => {
40270
+ const shown = rows.slice(0, visibleCount);
40271
+ const hidden = rows.length - shown.length;
40272
+ const out = [`\uD83D\uDEE0 **Workers** \u00b7 _${rows.length} running_`];
40273
+ for (const r of shown) {
40274
+ const [h, s] = rowLines(r);
40275
+ out.push(h, s);
40276
+ }
40277
+ if (hidden > 0)
40278
+ out.push(`_+${hidden} more working\u2026_`);
40279
+ return stackCardLines(out);
40280
+ };
40281
+ let visible = Math.min(rows.length, maxRows);
40282
+ let body = compose(visible);
40283
+ while (body.length > STATUS_CARD_CHAR_BUDGET && visible > 1) {
40284
+ visible -= 1;
40285
+ body = compose(visible);
40286
+ }
40287
+ return body;
40288
+ }
40241
40289
 
40242
40290
  // retry-api-call.ts
40243
40291
  var import_grammy = __toESM(require_mod2(), 1);
40292
+ var LOCAL_RESOURCE_EXHAUSTED = "LOCAL_RESOURCE_EXHAUSTED";
40244
40293
  var FLOOD_WAIT_ACTIVE = "FLOOD_WAIT_ACTIVE";
40245
40294
  function makeFloodWaitActiveError(retryAfterSec, untilTs, original) {
40246
40295
  return Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
@@ -40371,13 +40420,16 @@ function createWorkerActivityFeed(opts) {
40371
40420
  const minEditInterval = opts.minEditIntervalMs ?? 2500;
40372
40421
  const firstPaintMin = opts.firstPaintMinMs ?? 8000;
40373
40422
  const heartbeatTickMs = opts.heartbeatTickMs ?? 6000;
40423
+ const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8));
40424
+ const reconcilePinFn = opts.reconcilePin ?? (() => {});
40374
40425
  const setIntervalFn = opts.setInterval ?? ((cb, ms) => {
40375
40426
  const t = setInterval(cb, ms);
40376
40427
  t.unref?.();
40377
40428
  return t;
40378
40429
  });
40379
40430
  const clearIntervalFn = opts.clearInterval ?? ((handle) => clearInterval(handle));
40380
- const handles = new Map;
40431
+ const groups = new Map;
40432
+ const agentIndex = new Map;
40381
40433
  const finalized = new Set;
40382
40434
  const FINALIZED_CAP = 256;
40383
40435
  function markFinalized(agentId) {
@@ -40391,267 +40443,356 @@ function createWorkerActivityFeed(opts) {
40391
40443
  }
40392
40444
  }
40393
40445
  let heartbeatTimer = null;
40394
- function sendOptsFor(h) {
40446
+ function feedKeyOf(chatId, threadId) {
40447
+ return `${chatId} ${threadId ?? ""}`;
40448
+ }
40449
+ function groupOfAgent(agentId) {
40450
+ const key = agentIndex.get(agentId);
40451
+ return key != null ? groups.get(key) : undefined;
40452
+ }
40453
+ function sendOptsFor(g) {
40395
40454
  return {
40396
40455
  disable_web_page_preview: true,
40397
40456
  disable_notification: true,
40398
- ...h.threadId != null ? { message_thread_id: h.threadId } : {}
40457
+ ...g.threadId != null ? { message_thread_id: g.threadId } : {}
40399
40458
  };
40400
40459
  }
40401
- function noteRateLimited(h, err, label) {
40460
+ function noteRateLimited(g, err, label) {
40402
40461
  const retryAfter = extractRetryAfterSecs(err);
40403
40462
  if (retryAfter == null)
40404
40463
  return;
40405
- h.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS;
40464
+ g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS;
40406
40465
  log(`worker-feed: ${label} 429 \u2014 backing off ${retryAfter}s`);
40407
40466
  }
40408
- function parkIfFloodWindowOpen(h) {
40467
+ function parkIfFloodWindowOpen(g) {
40409
40468
  const remaining = floodWaitRemainingMs();
40410
40469
  if (remaining <= 0)
40411
40470
  return false;
40412
40471
  const until = nowFn() + remaining + COOLDOWN_JITTER_MS;
40413
- if (until > h.cooldownUntil)
40414
- h.cooldownUntil = until;
40472
+ if (until > g.cooldownUntil)
40473
+ g.cooldownUntil = until;
40415
40474
  return true;
40416
40475
  }
40417
- function accumulateNarrative(h, view) {
40476
+ function accumulateNarrative(row, view) {
40418
40477
  const line = view.latestSummary.trim();
40419
40478
  if (line.length === 0)
40420
40479
  return;
40421
- if (h.narrative.includes(line))
40480
+ if (row.narrative.includes(line))
40422
40481
  return;
40423
- h.narrative.push(line);
40424
- h.stepStartedAtMs = nowFn();
40425
- if (h.narrative.length > STATUS_ROLLING_LINES) {
40426
- h.narrative.splice(0, h.narrative.length - STATUS_ROLLING_LINES);
40427
- }
40428
- }
40429
- async function doUpdate(h, view, liveSuffix = "") {
40430
- accumulateNarrative(h, view);
40431
- const merged = { ...view, narrativeLines: [...h.narrative] };
40432
- h.lastView = merged;
40433
- if (h.dispatchAtMs == null)
40434
- h.dispatchAtMs = nowFn() - view.elapsedMs;
40435
- if (nowFn() < h.cooldownUntil)
40482
+ row.narrative.push(line);
40483
+ row.stepStartedAtMs = nowFn();
40484
+ if (row.narrative.length > STATUS_ROLLING_LINES) {
40485
+ row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES);
40486
+ }
40487
+ }
40488
+ function liveElapsed(row, now) {
40489
+ const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0;
40490
+ return Math.max(base, row.lastView?.elapsedMs ?? 0);
40491
+ }
40492
+ function runningRows(g) {
40493
+ return [...g.workers.values()].filter((w) => w.state === "running" && w.lastView != null).sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0));
40494
+ }
40495
+ function renderGroupBody(g, now, terminalRecap, heartbeat) {
40496
+ const running = runningRows(g);
40497
+ if (running.length === 0) {
40498
+ if (terminalRecap == null)
40499
+ return null;
40500
+ return renderWorkerActivity(terminalRecap);
40501
+ }
40502
+ const elapsedFor = (r) => heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now);
40503
+ if (running.length === 1) {
40504
+ const r = running[0];
40505
+ const view = {
40506
+ ...r.lastView,
40507
+ elapsedMs: elapsedFor(r),
40508
+ narrativeLines: [...r.narrative]
40509
+ };
40510
+ let liveSuffix = "";
40511
+ if (heartbeat) {
40512
+ const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now);
40513
+ liveSuffix = formatStepSuffix(stepElapsed);
40514
+ }
40515
+ return renderWorkerActivity(view, liveSuffix);
40516
+ }
40517
+ const rows = running.map((r) => {
40518
+ const v = r.lastView;
40519
+ const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary;
40520
+ return {
40521
+ description: v.description,
40522
+ elapsedMs: elapsedFor(r),
40523
+ toolCount: v.toolCount,
40524
+ currentStep,
40525
+ model: v.model
40526
+ };
40527
+ });
40528
+ return renderCombinedWorkerFeed(rows, { maxRows });
40529
+ }
40530
+ function removeWorker(g, agentId) {
40531
+ g.workers.delete(agentId);
40532
+ agentIndex.delete(agentId);
40533
+ if (g.workers.size === 0)
40534
+ groups.delete(g.feedKey);
40535
+ }
40536
+ function syncPin(g) {
40537
+ const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null;
40538
+ reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId });
40539
+ }
40540
+ async function doRender(g, opts2 = {}) {
40541
+ const now = nowFn();
40542
+ const isTerminal = opts2.terminalRecap != null;
40543
+ const settleTerminal = () => {
40544
+ g.pendingFinalize = null;
40545
+ if (opts2.finishingAgentId != null)
40546
+ removeWorker(g, opts2.finishingAgentId);
40547
+ syncPin(g);
40548
+ };
40549
+ if (now < g.cooldownUntil) {
40550
+ if (isTerminal && opts2.terminalRecap != null)
40551
+ g.pendingFinalize = opts2.terminalRecap;
40552
+ return;
40553
+ }
40554
+ if (parkIfFloodWindowOpen(g)) {
40555
+ if (isTerminal && opts2.terminalRecap != null)
40556
+ g.pendingFinalize = opts2.terminalRecap;
40436
40557
  return;
40437
- if (parkIfFloodWindowOpen(h))
40558
+ }
40559
+ const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false);
40560
+ if (body == null) {
40561
+ if (isTerminal)
40562
+ settleTerminal();
40438
40563
  return;
40439
- const body = renderWorkerActivity(merged, liveSuffix);
40440
- if (h.messageId == null) {
40441
- if (view.elapsedMs < firstPaintMin)
40564
+ }
40565
+ if (g.messageId == null) {
40566
+ const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)));
40567
+ if (isTerminal) {
40568
+ settleTerminal();
40569
+ return;
40570
+ }
40571
+ if (maxElapsed < firstPaintMin)
40442
40572
  return;
40443
40573
  try {
40444
- const sent = await opts.bot.sendMessage(h.chatId, body, sendOptsFor(h));
40574
+ const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g));
40445
40575
  if (sent == null || typeof sent.message_id !== "number") {
40446
- parkIfFloodWindowOpen(h);
40447
- log(`worker-feed: first paint shed by send gate agent=${h.agentId} \u2014 not delivered`);
40576
+ parkIfFloodWindowOpen(g);
40577
+ log(`worker-feed: first paint shed by send gate feed=${g.feedKey} \u2014 not delivered`);
40448
40578
  return;
40449
40579
  }
40450
- h.messageId = sent.message_id;
40451
- h.lastBody = body;
40452
- h.lastEditAt = nowFn();
40453
- log(`worker-feed: paint agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} bytes=${body.length}`);
40580
+ g.messageId = sent.message_id;
40581
+ g.lastBody = body;
40582
+ g.lastEditAt = now;
40583
+ syncPin(g);
40584
+ log(`worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
40454
40585
  } catch (err) {
40455
- noteRateLimited(h, err, "send");
40586
+ noteRateLimited(g, err, "send");
40456
40587
  log(`worker-feed: send failed: ${err.message}`);
40457
40588
  }
40458
40589
  return;
40459
40590
  }
40460
- if (body === h.lastBody)
40461
- return;
40462
- if (nowFn() - h.lastEditAt < minEditInterval)
40463
- return;
40464
- try {
40465
- const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
40466
- if (isSendGateShed(res)) {
40467
- parkIfFloodWindowOpen(h);
40468
- return;
40469
- }
40470
- h.lastBody = body;
40471
- h.lastEditAt = nowFn();
40472
- log(`worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} bytes=${body.length}`);
40473
- } catch (err) {
40474
- const outcome = classifyEditError(err);
40475
- if (outcome === "rate_limited") {
40476
- noteRateLimited(h, err, "edit");
40477
- return;
40478
- }
40479
- if (outcome === "not_modified") {
40480
- h.lastBody = body;
40481
- h.lastEditAt = nowFn();
40482
- return;
40483
- }
40484
- if (outcome === "gone") {
40485
- h.messageId = null;
40486
- h.lastBody = null;
40487
- return;
40488
- }
40489
- log(`worker-feed: edit transient error agent=${h.agentId}: ${err.message}`);
40490
- }
40491
- }
40492
- async function doFinish(h, view) {
40493
- h.finished = true;
40494
- markFinalized(h.agentId);
40495
- if (h.messageId == null) {
40496
- h.pendingFinish = null;
40497
- return;
40498
- }
40499
- if (parkIfFloodWindowOpen(h) || nowFn() < h.cooldownUntil) {
40500
- h.pendingFinish = view;
40591
+ if (body === g.lastBody) {
40592
+ if (isTerminal)
40593
+ settleTerminal();
40501
40594
  return;
40502
40595
  }
40503
- const body = renderWorkerActivity({ ...view, narrativeLines: h.narrative });
40504
- if (body === h.lastBody) {
40505
- h.pendingFinish = null;
40596
+ if (!opts2.force && now - g.lastEditAt < minEditInterval)
40506
40597
  return;
40507
- }
40508
40598
  try {
40509
- const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
40599
+ const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g));
40510
40600
  if (isSendGateShed(res)) {
40511
- parkIfFloodWindowOpen(h);
40512
- h.pendingFinish = view;
40601
+ parkIfFloodWindowOpen(g);
40602
+ if (isTerminal && opts2.terminalRecap != null)
40603
+ g.pendingFinalize = opts2.terminalRecap;
40513
40604
  return;
40514
40605
  }
40515
- h.lastBody = body;
40516
- h.lastEditAt = nowFn();
40517
- h.pendingFinish = null;
40518
- log(`worker-feed: finish agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} state=${view.state} bytes=${body.length}`);
40606
+ g.lastBody = body;
40607
+ g.lastEditAt = now;
40608
+ if (isTerminal) {
40609
+ log(`worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? "-"} ` + `msgId=${g.messageId} agent=${opts2.finishingAgentId ?? "-"} ` + `state=${opts2.terminalRecap?.state ?? "done"} bytes=${body.length}`);
40610
+ } else {
40611
+ log(`worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
40612
+ }
40613
+ if (isTerminal)
40614
+ settleTerminal();
40519
40615
  } catch (err) {
40520
40616
  const outcome = classifyEditError(err);
40521
40617
  if (outcome === "rate_limited") {
40522
- noteRateLimited(h, err, "finish");
40523
- h.pendingFinish = view;
40618
+ noteRateLimited(g, err, isTerminal ? "finish" : "edit");
40619
+ if (isTerminal && opts2.terminalRecap != null)
40620
+ g.pendingFinalize = opts2.terminalRecap;
40524
40621
  return;
40525
40622
  }
40526
40623
  if (outcome === "not_modified") {
40527
- h.lastBody = body;
40528
- h.lastEditAt = nowFn();
40529
- h.pendingFinish = null;
40624
+ g.lastBody = body;
40625
+ g.lastEditAt = now;
40626
+ if (isTerminal)
40627
+ settleTerminal();
40530
40628
  return;
40531
40629
  }
40532
40630
  if (outcome === "gone") {
40533
- h.pendingFinish = null;
40631
+ g.messageId = null;
40632
+ g.lastBody = null;
40633
+ if (isTerminal)
40634
+ settleTerminal();
40635
+ else
40636
+ syncPin(g);
40534
40637
  return;
40535
40638
  }
40536
- h.pendingFinish = view;
40537
- log(`worker-feed: finish transient error agent=${h.agentId}: ${err.message}`);
40639
+ if (isTerminal && opts2.terminalRecap != null)
40640
+ g.pendingFinalize = opts2.terminalRecap;
40641
+ log(`worker-feed: edit transient error feed=${g.feedKey}: ${err.message}`);
40538
40642
  }
40539
40643
  }
40540
40644
  function heartbeatTick() {
40541
40645
  const now = nowFn();
40542
- for (const h of handles.values()) {
40543
- if (!handles.has(h.agentId))
40544
- continue;
40545
- if (h.pendingFinish != null && now >= h.cooldownUntil) {
40546
- const view2 = h.pendingFinish;
40547
- h.chain = h.chain.then(() => doFinish(h, view2)).catch((err) => {
40548
- log(`worker-feed: heartbeat finalize re-drive error ${h.agentId}: ${err.message}`);
40549
- }).finally(() => {
40550
- if (handles.get(h.agentId)?.pendingFinish == null) {
40551
- handles.delete(h.agentId);
40552
- }
40646
+ for (const g of [...groups.values()]) {
40647
+ if (g.pendingFinalize != null && now >= g.cooldownUntil) {
40648
+ const recap = g.pendingFinalize;
40649
+ const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId;
40650
+ g.chain = g.chain.then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId })).catch((err) => {
40651
+ log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${err.message}`);
40553
40652
  });
40554
40653
  continue;
40555
40654
  }
40556
- if (h.lastView == null)
40557
- continue;
40558
- if (h.lastView.state !== "running")
40655
+ if (now < g.cooldownUntil)
40559
40656
  continue;
40560
- if (now < h.cooldownUntil)
40657
+ const running = runningRows(g);
40658
+ if (running.length === 0)
40561
40659
  continue;
40562
- const liveElapsed = h.dispatchAtMs != null ? now - h.dispatchAtMs : h.lastView.elapsedMs;
40563
- if (h.messageId == null) {
40564
- if (liveElapsed < firstPaintMin)
40660
+ if (g.messageId == null) {
40661
+ const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)));
40662
+ if (maxElapsed < firstPaintMin)
40565
40663
  continue;
40566
- const view2 = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) };
40567
- h.chain = h.chain.then(() => doUpdate(h, view2)).catch((err) => {
40568
- log(`worker-feed: heartbeat first-paint chain error ${h.agentId}: ${err.message}`);
40664
+ g.chain = g.chain.then(() => doRender(g, {})).catch((err) => {
40665
+ log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${err.message}`);
40569
40666
  });
40570
40667
  continue;
40571
40668
  }
40572
- if (now - h.lastEditAt < minEditInterval)
40669
+ if (now - g.lastEditAt < minEditInterval)
40573
40670
  continue;
40574
- const stale = now - h.lastEditAt >= heartbeatTickMs;
40671
+ const stale = now - g.lastEditAt >= heartbeatTickMs;
40575
40672
  if (!stale)
40576
40673
  continue;
40577
- const stepElapsed = h.stepStartedAtMs != null ? now - h.stepStartedAtMs : liveElapsed;
40578
- const liveSuffix = formatStepSuffix(stepElapsed);
40579
- const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) };
40580
- h.chain = h.chain.then(() => doUpdate(h, view, liveSuffix)).catch((err) => {
40581
- log(`worker-feed: heartbeat chain error ${h.agentId}: ${err.message}`);
40674
+ g.chain = g.chain.then(() => doRender(g, { heartbeat: true })).catch((err) => {
40675
+ log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${err.message}`);
40582
40676
  });
40583
40677
  }
40584
40678
  }
40585
40679
  heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs);
40586
40680
  return {
40587
40681
  has(agentId) {
40588
- return handles.get(agentId)?.messageId != null;
40682
+ const g = groupOfAgent(agentId);
40683
+ return g != null && g.messageId != null && g.workers.has(agentId);
40589
40684
  },
40590
40685
  messageIdOf(agentId) {
40591
- return handles.get(agentId)?.messageId ?? null;
40686
+ return groupOfAgent(agentId)?.messageId ?? null;
40687
+ },
40688
+ hasRunningInFeed(feedKey) {
40689
+ const g = groups.get(feedKey);
40690
+ return g != null && g.workers.size > 0;
40592
40691
  },
40593
40692
  get size() {
40594
- return handles.size;
40693
+ let n = 0;
40694
+ for (const g of groups.values())
40695
+ n += g.workers.size;
40696
+ return n;
40595
40697
  },
40596
40698
  update(agentId, chatId, view, threadId) {
40597
40699
  if (chatId.length === 0)
40598
40700
  return Promise.resolve();
40599
40701
  if (finalized.has(agentId))
40600
40702
  return Promise.resolve();
40601
- const existing = handles.get(agentId);
40602
- if (existing?.finished === true)
40703
+ const existingRow = groupOfAgent(agentId)?.workers.get(agentId);
40704
+ if (existingRow?.finished === true)
40603
40705
  return Promise.resolve();
40604
- let h = existing;
40605
- if (h == null) {
40606
- h = {
40607
- agentId,
40706
+ const feedKey = feedKeyOf(chatId, threadId);
40707
+ let g = groups.get(feedKey);
40708
+ if (g == null) {
40709
+ g = {
40710
+ feedKey,
40608
40711
  chatId,
40609
40712
  threadId,
40610
40713
  messageId: null,
40611
40714
  lastBody: null,
40612
40715
  lastEditAt: 0,
40613
40716
  cooldownUntil: 0,
40614
- narrative: [],
40615
40717
  chain: Promise.resolve(),
40718
+ workers: new Map,
40719
+ pendingFinalize: null
40720
+ };
40721
+ groups.set(feedKey, g);
40722
+ }
40723
+ let row = g.workers.get(agentId);
40724
+ if (row == null) {
40725
+ row = {
40726
+ agentId,
40727
+ narrative: [],
40616
40728
  lastView: null,
40617
- dispatchAtMs: null,
40618
- stepStartedAtMs: null,
40729
+ state: "running",
40619
40730
  finished: false,
40620
- pendingFinish: null
40731
+ dispatchAtMs: null,
40732
+ stepStartedAtMs: null
40621
40733
  };
40622
- handles.set(agentId, h);
40623
- }
40624
- const handle = h;
40625
- handle.chain = handle.chain.then(() => doUpdate(handle, view)).catch((err) => {
40734
+ g.workers.set(agentId, row);
40735
+ agentIndex.set(agentId, feedKey);
40736
+ }
40737
+ accumulateNarrative(row, view);
40738
+ row.state = "running";
40739
+ row.lastView = { ...view, narrativeLines: [...row.narrative] };
40740
+ if (row.dispatchAtMs == null)
40741
+ row.dispatchAtMs = nowFn() - view.elapsedMs;
40742
+ const group = g;
40743
+ group.chain = group.chain.then(() => doRender(group)).catch((err) => {
40626
40744
  log(`worker-feed: update chain error ${agentId}: ${err.message}`);
40627
40745
  });
40628
- return handle.chain;
40746
+ return group.chain;
40629
40747
  },
40630
40748
  finish(agentId, view) {
40631
- const h = handles.get(agentId);
40632
- if (h == null)
40749
+ const g = groupOfAgent(agentId);
40750
+ const row = g?.workers.get(agentId);
40751
+ if (g == null || row == null) {
40752
+ markFinalized(agentId);
40633
40753
  return Promise.resolve();
40634
- h.chain = h.chain.then(() => doFinish(h, view)).catch((err) => {
40754
+ }
40755
+ row.finished = true;
40756
+ row.state = view.state === "failed" ? "failed" : "done";
40757
+ markFinalized(agentId);
40758
+ const group = g;
40759
+ group.chain = group.chain.then(() => {
40760
+ const others = runningRows(group).filter((w) => w.agentId !== agentId);
40761
+ if (others.length > 0) {
40762
+ removeWorker(group, agentId);
40763
+ syncPin(group);
40764
+ return doRender(group, { force: true });
40765
+ }
40766
+ const recap = { ...view, narrativeLines: [...row.narrative] };
40767
+ return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId });
40768
+ }).catch((err) => {
40635
40769
  log(`worker-feed: finish chain error ${agentId}: ${err.message}`);
40636
- }).finally(() => {
40637
- if (handles.get(agentId)?.pendingFinish == null) {
40638
- handles.delete(agentId);
40639
- }
40640
40770
  });
40641
- return h.chain;
40771
+ return group.chain;
40642
40772
  },
40643
40773
  drop(agentId) {
40644
40774
  markFinalized(agentId);
40645
- handles.delete(agentId);
40775
+ const g = groupOfAgent(agentId);
40776
+ if (g == null)
40777
+ return;
40778
+ const hadMessage = g.messageId != null;
40779
+ removeWorker(g, agentId);
40780
+ if (hadMessage)
40781
+ syncPin(g);
40782
+ if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
40783
+ g.chain = g.chain.then(() => doRender(g, { force: true })).catch((err) => {
40784
+ log(`worker-feed: drop re-render error ${agentId}: ${err.message}`);
40785
+ });
40786
+ }
40646
40787
  },
40647
40788
  resurrect(agentId) {
40648
40789
  const wasFinalized = finalized.delete(agentId);
40649
- const h = handles.get(agentId);
40650
- if (h != null) {
40651
- h.finished = false;
40652
- h.pendingFinish = null;
40790
+ const row = groupOfAgent(agentId)?.workers.get(agentId);
40791
+ if (row != null) {
40792
+ row.finished = false;
40793
+ row.state = "running";
40653
40794
  }
40654
- if (wasFinalized || h != null) {
40795
+ if (wasFinalized || row != null) {
40655
40796
  log(`worker-feed: resurrect agent=${agentId} \u2014 cleared finalized gate; card will repaint on next running cue`);
40656
40797
  }
40657
40798
  },
@@ -44582,6 +44723,14 @@ function holdReasonFor(err) {
44582
44723
  function isHeldUndeliverable2(pend) {
44583
44724
  return pend.undeliverable != null;
44584
44725
  }
44726
+ function applyDeliveredHoldReset(entry, now) {
44727
+ if (entry.undeliverable == null)
44728
+ return false;
44729
+ entry.undeliverable = null;
44730
+ entry.redeliveryFailures = 0;
44731
+ entry.startedAt = now;
44732
+ return true;
44733
+ }
44585
44734
  var HELD_CARD_REDELIVERY_CAP = 3;
44586
44735
  function selectHeldForRedelivery(entries, opts) {
44587
44736
  const empty = { send: [], deferred: [] };
@@ -55213,7 +55362,7 @@ function isLocalResourceError(err) {
55213
55362
  const msg = err instanceof Error ? err.message : String(err ?? "");
55214
55363
  return /\b(ENOSPC|EDQUOT|EIO|ENOMEM)\b/.test(msg) || /no space left on device/i.test(msg) || /disk quota exceeded/i.test(msg);
55215
55364
  }
55216
- var LOCAL_RESOURCE_EXHAUSTED = "LOCAL_RESOURCE_EXHAUSTED";
55365
+ var LOCAL_RESOURCE_EXHAUSTED2 = "LOCAL_RESOURCE_EXHAUSTED";
55217
55366
  var GIVE_UP_MESSAGE = "retryApiCall: max retries exceeded";
55218
55367
  var FLOOD_WAIT_ACTIVE2 = "FLOOD_WAIT_ACTIVE";
55219
55368
  function makeFloodWaitActiveError2(retryAfterSec, untilTs, original) {
@@ -55269,13 +55418,13 @@ function createRetryApiCall2(config = {}) {
55269
55418
  log?.(`telegram gateway: LOCAL resource exhaustion (${err.code ?? "disk/mem"}) \u2014 ` + `not retrying the send (would feed a flood ban); surfacing degraded state
55270
55419
  `);
55271
55420
  observer?.onGiveUp?.({ attempts: attempt + 1, error: err });
55272
- throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED), { original: err });
55421
+ throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED2), { original: err });
55273
55422
  }
55274
55423
  if (isGrammyErr && err.error_code === 429) {
55275
55424
  const retryAfter = Number(err.parameters?.retry_after ?? 5);
55276
55425
  const delayMs = retryAfter * 1000;
55277
55426
  try {
55278
- onFloodWait?.(retryAfter);
55427
+ onFloodWait?.(retryAfter, opts);
55279
55428
  } catch {}
55280
55429
  if (delayMs > maxFloodSleepMs) {
55281
55430
  log?.(`telegram gateway: 429 flood ban of ${retryAfter}s exceeds the ` + `${Math.round(maxFloodSleepMs / 1000)}s in-process sleep ceiling \u2014 ` + `not sleeping it; surfacing degraded state
@@ -55444,16 +55593,25 @@ function stableStringify(value) {
55444
55593
  });
55445
55594
  }
55446
55595
  var GROUP_TYPES2 = new Set(["group", "supergroup"]);
55596
+ var SEND_GATE_DEFAULTS = {
55597
+ globalPerSec: 25,
55598
+ globalBurst: 4,
55599
+ perChatPerSec: 1,
55600
+ perChatBurst: 3,
55601
+ perGroupPerMin: 18,
55602
+ perGroupBurst: 2,
55603
+ editFloorMs: 1500
55604
+ };
55447
55605
  function createSendGate(config) {
55448
55606
  const enabled2 = config.enabled;
55449
55607
  const clock = config.clock ?? systemClock;
55450
- const globalPerSec = config.globalPerSec ?? 25;
55451
- const globalBurst = config.globalBurst ?? 4;
55452
- const perChatPerSec = config.perChatPerSec ?? 1;
55453
- const perChatBurst = config.perChatBurst ?? 3;
55454
- const perGroupPerMin = config.perGroupPerMin ?? 18;
55455
- const perGroupBurst = config.perGroupBurst ?? 2;
55456
- const editFloorMs = config.editFloorMs ?? 1500;
55608
+ const globalPerSec = config.globalPerSec ?? SEND_GATE_DEFAULTS.globalPerSec;
55609
+ const globalBurst = config.globalBurst ?? SEND_GATE_DEFAULTS.globalBurst;
55610
+ const perChatPerSec = config.perChatPerSec ?? SEND_GATE_DEFAULTS.perChatPerSec;
55611
+ const perChatBurst = config.perChatBurst ?? SEND_GATE_DEFAULTS.perChatBurst;
55612
+ const perGroupPerMin = config.perGroupPerMin ?? SEND_GATE_DEFAULTS.perGroupPerMin;
55613
+ const perGroupBurst = config.perGroupBurst ?? SEND_GATE_DEFAULTS.perGroupBurst;
55614
+ const editFloorMs = config.editFloorMs ?? SEND_GATE_DEFAULTS.editFloorMs;
55457
55615
  const messageStateTtlMs = config.messageStateTtlMs ?? 60000;
55458
55616
  const maxMessageStates = config.maxMessageStates ?? 5000;
55459
55617
  const usefulTtlMs = config.usefulTtlMs ?? 120000;
@@ -55461,6 +55619,7 @@ function createSendGate(config) {
55461
55619
  const criticalJitterMaxMs = config.criticalJitterMaxMs ?? 250;
55462
55620
  const jitter = config.jitter ?? Math.random;
55463
55621
  const onWindowOpen = config.onWindowOpen;
55622
+ const conservativeGlobalFloodScope = config.conservativeGlobalFloodScope ?? false;
55464
55623
  const counters = {
55465
55624
  sent: 0,
55466
55625
  queued: 0,
@@ -55531,8 +55690,11 @@ function createSendGate(config) {
55531
55690
  applyWindow(scopeKey, untilTs, true);
55532
55691
  }
55533
55692
  function openScopedWindowsForOpts(opts, untilTs) {
55534
- applyWindow("global", untilTs, true);
55535
- if (opts?.chat_id) {
55693
+ const hasChatScope = opts?.chat_id != null && opts.chat_id !== "";
55694
+ if (!hasChatScope || conservativeGlobalFloodScope) {
55695
+ applyWindow("global", untilTs, true);
55696
+ }
55697
+ if (hasChatScope) {
55536
55698
  applyWindow(`chat:${opts.chat_id}`, untilTs, true);
55537
55699
  if (opts.chatType && GROUP_TYPES2.has(opts.chatType)) {
55538
55700
  applyWindow(`group:${opts.chat_id}`, untilTs, true);
@@ -55834,14 +55996,114 @@ function createSendGate(config) {
55834
55996
  }
55835
55997
  };
55836
55998
  }
55837
- return { gate, openFloodWindow, stats };
55999
+ function openScopedFloodWindows(opts, untilTs) {
56000
+ if (!enabled2)
56001
+ return;
56002
+ openScopedWindowsForOpts(opts, untilTs);
56003
+ }
56004
+ return { gate, openFloodWindow, openScopedFloodWindows, stats };
55838
56005
  }
55839
- function sendGateEnabledFromEnv(env = process.env) {
55840
- const v = env.SWITCHROOM_TELEGRAM_SEND_GATE;
55841
- if (v == null)
55842
- return true;
56006
+ function isOffValue(v) {
55843
56007
  const t = v.trim().toLowerCase();
55844
- return !(t === "0" || t === "false" || t === "off" || t === "no");
56008
+ return t === "0" || t === "false" || t === "off" || t === "no";
56009
+ }
56010
+ function parsePositiveNumber(raw) {
56011
+ if (raw == null || raw.trim() === "")
56012
+ return;
56013
+ const n = Number(raw);
56014
+ return Number.isFinite(n) && n > 0 ? n : undefined;
56015
+ }
56016
+ function parsePositiveInt(raw) {
56017
+ if (raw == null || raw.trim() === "")
56018
+ return;
56019
+ const n = Number(raw);
56020
+ return Number.isInteger(n) && n > 0 ? n : undefined;
56021
+ }
56022
+ function parseNonNegativeInt(raw) {
56023
+ if (raw == null || raw.trim() === "")
56024
+ return;
56025
+ const n = Number(raw);
56026
+ return Number.isInteger(n) && n >= 0 ? n : undefined;
56027
+ }
56028
+ function parseBoolFlag(raw) {
56029
+ if (raw == null || raw.trim() === "")
56030
+ return;
56031
+ const t = raw.trim().toLowerCase();
56032
+ if (t === "1" || t === "true" || t === "on" || t === "yes")
56033
+ return true;
56034
+ if (isOffValue(t))
56035
+ return false;
56036
+ return;
56037
+ }
56038
+ function sendGateConfigFromEnv(env = process.env) {
56039
+ const killSwitch = env.SWITCHROOM_TELEGRAM_SEND_GATE;
56040
+ let enabled2;
56041
+ if (killSwitch != null && killSwitch.trim() !== "") {
56042
+ enabled2 = !isOffValue(killSwitch);
56043
+ } else {
56044
+ const cfg = env.SWITCHROOM_TG_SEND_GATE_ENABLED;
56045
+ enabled2 = cfg != null && cfg.trim() !== "" ? !isOffValue(cfg) : true;
56046
+ }
56047
+ const out = { enabled: enabled2 };
56048
+ const globalPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_PER_SEC);
56049
+ if (globalPerSec !== undefined)
56050
+ out.globalPerSec = globalPerSec;
56051
+ const globalBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_BURST);
56052
+ if (globalBurst !== undefined)
56053
+ out.globalBurst = globalBurst;
56054
+ const perChatPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_PER_SEC);
56055
+ if (perChatPerSec !== undefined)
56056
+ out.perChatPerSec = perChatPerSec;
56057
+ const perChatBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_BURST);
56058
+ if (perChatBurst !== undefined)
56059
+ out.perChatBurst = perChatBurst;
56060
+ const perGroupPerMin = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_PER_MIN);
56061
+ if (perGroupPerMin !== undefined)
56062
+ out.perGroupPerMin = perGroupPerMin;
56063
+ const perGroupBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_BURST);
56064
+ if (perGroupBurst !== undefined)
56065
+ out.perGroupBurst = perGroupBurst;
56066
+ const editFloorMs = parseNonNegativeInt(env.SWITCHROOM_TG_SEND_GATE_EDIT_FLOOR_MS);
56067
+ if (editFloorMs !== undefined)
56068
+ out.editFloorMs = editFloorMs;
56069
+ const conservativeGlobal = parseBoolFlag(env.SWITCHROOM_TG_SEND_GATE_CONSERVATIVE_GLOBAL);
56070
+ if (conservativeGlobal !== undefined)
56071
+ out.conservativeGlobalFloodScope = conservativeGlobal;
56072
+ return out;
56073
+ }
56074
+
56075
+ // shared/local-time.ts
56076
+ function fmtLocalClock(ms, tz) {
56077
+ return new Intl.DateTimeFormat("en-US", {
56078
+ timeZone: tz,
56079
+ hour: "numeric",
56080
+ minute: "2-digit",
56081
+ hour12: true
56082
+ }).format(new Date(ms)).replace(/\s([AP])M$/, (_m, p) => `${p.toLowerCase()}m`);
56083
+ }
56084
+ function fmtLocalDate(ms, tz) {
56085
+ return new Intl.DateTimeFormat("en-GB", {
56086
+ timeZone: tz,
56087
+ day: "numeric",
56088
+ month: "short"
56089
+ }).format(new Date(ms));
56090
+ }
56091
+ function localDay(ms, tz) {
56092
+ return new Intl.DateTimeFormat("en-CA", {
56093
+ timeZone: tz,
56094
+ year: "numeric",
56095
+ month: "2-digit",
56096
+ day: "2-digit"
56097
+ }).format(new Date(ms));
56098
+ }
56099
+ function tzAbbrev(ms, tz) {
56100
+ const at = new Date(ms);
56101
+ for (const loc of ["en-US", "en-AU", "en-GB"]) {
56102
+ const v = new Intl.DateTimeFormat(loc, { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value;
56103
+ if (v && !/^(?:GMT|UTC)/i.test(v))
56104
+ return v;
56105
+ }
56106
+ return new Intl.DateTimeFormat("en-US", { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value ?? tz;
55845
56107
  }
55846
56108
 
55847
56109
  // send-gate-observability.ts
@@ -55878,8 +56140,18 @@ function createStatsLogger(config) {
55878
56140
  function isAlertableScope(scopeKey) {
55879
56141
  return scopeKey === "global" || scopeKey.startsWith("chat:") || scopeKey.startsWith("group:");
55880
56142
  }
55881
- function fmtTs(ms) {
55882
- return new Date(ms).toISOString().replace(/\.\d{3}/, "");
56143
+ function fmtLocalStamp(ms, tz, refMs) {
56144
+ const clock = fmtLocalClock(ms, tz);
56145
+ const sameDay = refMs != null && localDay(ms, tz) === localDay(refMs, tz);
56146
+ const body = sameDay ? clock : `${fmtLocalDate(ms, tz)} ${clock}`;
56147
+ return `${body} ${tzAbbrev(ms, tz)}`;
56148
+ }
56149
+ function fmtLocalRange(startMs, endMs, tz) {
56150
+ const sameDay = localDay(startMs, tz) === localDay(endMs, tz);
56151
+ if (sameDay) {
56152
+ return `${fmtLocalClock(startMs, tz)} to ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
56153
+ }
56154
+ return `${fmtLocalDate(startMs, tz)} ${fmtLocalClock(startMs, tz)} to ` + `${fmtLocalDate(endMs, tz)} ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
55883
56155
  }
55884
56156
  function fmtDur(ms) {
55885
56157
  const s = Math.round(ms / 1000);
@@ -55891,6 +56163,7 @@ function fmtDur(ms) {
55891
56163
  }
55892
56164
  function createFloodWindowObserver(config) {
55893
56165
  const alertThresholdMs = config.alertThresholdMs ?? 60000;
56166
+ const tz = config.tz ?? "UTC";
55894
56167
  let lastSeen = new Map;
55895
56168
  let firstTick = true;
55896
56169
  const owedOnClose = new Set;
@@ -55911,14 +56184,14 @@ function createFloodWindowObserver(config) {
55911
56184
  }
55912
56185
  function openAlertText(w, now) {
55913
56186
  const openFor = fmtDur(now - w.observedAt);
55914
- return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${fmtTs(w.untilTs)} UTC. ` + `Outbound to that scope is being suppressed.`;
56187
+ return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${fmtLocalStamp(w.untilTs, tz, now)}. ` + `Outbound to that scope is being suppressed.`;
55915
56188
  }
55916
56189
  function closeAlertText(recs) {
55917
56190
  const observedAt = Math.min(...recs.map((r) => r.observedAt));
55918
56191
  const untilTs = Math.max(...recs.map((r) => r.untilTs));
55919
56192
  const scopes = recs.map((r) => `\`${r.scopeKey}\``).join(", ");
55920
56193
  const plural = recs.length > 1 ? "s" : "";
55921
- return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${fmtTs(observedAt)} to ${fmtTs(untilTs)} UTC ` + `(~${fmtDur(untilTs - observedAt)}). Some outbound messages during that ` + `window were suppressed.`;
56194
+ return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${fmtLocalRange(observedAt, untilTs, tz)} ` + `(~${fmtDur(untilTs - observedAt)}). Some outbound messages during that ` + `window were suppressed.`;
55922
56195
  }
55923
56196
  async function tick() {
55924
56197
  const s = config.stats();
@@ -63787,18 +64060,28 @@ function createFleetFallbackResumeGate(opts = {}) {
63787
64060
  const maxAgeMs = opts.maxAgeMs ?? DEFAULT_RESUME_MAX_AGE_MS;
63788
64061
  const singleFlightMs = opts.singleFlightMs ?? DEFAULT_RESUME_SINGLE_FLIGHT_MS;
63789
64062
  let lastResumedAtMs = Number.NEGATIVE_INFINITY;
63790
- function decide(failedTurnStartedAtMs) {
64063
+ function peek(failedTurnStartedAtMs) {
63791
64064
  const now = nowFn();
63792
64065
  if (now - lastResumedAtMs < singleFlightMs)
63793
64066
  return "skip-inflight";
63794
64067
  if (failedTurnStartedAtMs != null && now - failedTurnStartedAtMs > maxAgeMs) {
63795
64068
  return "skip-stale";
63796
64069
  }
63797
- lastResumedAtMs = now;
63798
64070
  return "resume";
63799
64071
  }
64072
+ function arm() {
64073
+ lastResumedAtMs = nowFn();
64074
+ }
64075
+ function decide(failedTurnStartedAtMs) {
64076
+ const verdict = peek(failedTurnStartedAtMs);
64077
+ if (verdict === "resume")
64078
+ arm();
64079
+ return verdict;
64080
+ }
63800
64081
  return {
63801
64082
  decide,
64083
+ peek,
64084
+ arm,
63802
64085
  reset() {
63803
64086
  lastResumedAtMs = Number.NEGATIVE_INFINITY;
63804
64087
  },
@@ -64500,10 +64783,30 @@ function autoClassifyMidTurnInbound(i) {
64500
64783
  return { decision: "queue", reason: "cross_topic", sameTopic: false };
64501
64784
  return recent && i.msSinceLastAgentOutput <= i.topicSteerWindowMs ? { decision: "steer", reason: "same_topic_recent", sameTopic: true } : { decision: "queue", reason: "same_topic_stale", sameTopic: true };
64502
64785
  }
64786
+ // raw-error-scrub.ts
64787
+ function stripRawErrorBytes(raw) {
64788
+ if (typeof raw !== "string" || raw.length === 0)
64789
+ return "";
64790
+ let s = raw;
64791
+ s = s.replace(/API Error:?\s*\d*\s*/gi, " ");
64792
+ s = s.replace(/\bb'[^']*'/g, " ");
64793
+ s = s.replace(/\bb"[^"]*"/g, " ");
64794
+ s = s.replace(/[\u00b7\-\s]*\{[\s\S]*?["']type["']\s*:\s*["']error["'][\s\S]*$/i, " ");
64795
+ s = s.replace(/^\s*\{[\s\S]*\}\s*$/g, " ");
64796
+ s = s.replace(/\s+/g, " ").replace(/[\u00b7:\-\s]+$/g, "").replace(/^[\u00b7:\-\s]+/g, "").trim();
64797
+ return s;
64798
+ }
64799
+ function extractRequestId(raw) {
64800
+ if (typeof raw !== "string" || raw.length === 0)
64801
+ return;
64802
+ const m = raw.match(/["']request[_-]?id["']\s*:\s*["']([A-Za-z0-9._-]+)["']/i) ?? raw.match(/\brequest[_-]?id[=:]\s*([A-Za-z0-9._-]+)/i);
64803
+ return m ? m[1] : undefined;
64804
+ }
64805
+
64503
64806
  // operator-events.ts
64504
64807
  function renderOperatorEvent(ev) {
64505
64808
  const agent = escapeMarkdown(ev.agent);
64506
- const detail = escapeMarkdown(ev.detail);
64809
+ const detail = escapeMarkdown(stripRawErrorBytes(ev.detail));
64507
64810
  switch (ev.kind) {
64508
64811
  case "credentials-expired":
64509
64812
  return {
@@ -64720,6 +65023,13 @@ var transientUpstreamSignals = [
64720
65023
  "would exceed your account\u2019s rate limit",
64721
65024
  "would exceed your account's rate limit"
64722
65025
  ];
65026
+ function isTransientUpstreamSignal(text4) {
65027
+ if (typeof text4 !== "string" || text4.length === 0)
65028
+ return false;
65029
+ const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
65030
+ const lower = sample.toLowerCase();
65031
+ return transientUpstreamSignals.some((s) => lower.includes(s));
65032
+ }
64723
65033
  var litellmProxyLocal429Signals = [
64724
65034
  "deployment over user-defined ratelimit",
64725
65035
  "model rate limit exceeded. tpm limit",
@@ -64740,6 +65050,56 @@ function isLitellmProxyLocal429(text4) {
64740
65050
  return true;
64741
65051
  return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
64742
65052
  }
65053
+ function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
65054
+ const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
65055
+ if (typeof text4 !== "string" || text4.length === 0)
65056
+ return empty2;
65057
+ const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
65058
+ const lower = sample.toLowerCase();
65059
+ let limitType = null;
65060
+ let limit = null;
65061
+ const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
65062
+ if (eqLimit) {
65063
+ limitType = eqLimit[1];
65064
+ limit = Number(eqLimit[2]);
65065
+ }
65066
+ if (limitType == null) {
65067
+ const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
65068
+ if (v3Type)
65069
+ limitType = v3Type[1];
65070
+ }
65071
+ if (limit == null) {
65072
+ const v3Limit = lower.match(/current limit:\s*(\d+)/);
65073
+ if (v3Limit)
65074
+ limit = Number(v3Limit[1]);
65075
+ }
65076
+ let currentUsage = null;
65077
+ const usage = lower.match(/current usage=(\d+)/);
65078
+ if (usage)
65079
+ currentUsage = Number(usage[1]);
65080
+ let resetAtMs = null;
65081
+ const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
65082
+ if (resetsAt) {
65083
+ const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
65084
+ if (!Number.isNaN(d.getTime()))
65085
+ resetAtMs = d.getTime();
65086
+ }
65087
+ if (resetAtMs == null) {
65088
+ const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
65089
+ if (tryAgain) {
65090
+ const secs = Number(tryAgain[1]);
65091
+ if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
65092
+ resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
65093
+ }
65094
+ }
65095
+ }
65096
+ return {
65097
+ limitType,
65098
+ limit: limit != null && Number.isFinite(limit) ? limit : null,
65099
+ currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
65100
+ resetAtMs
65101
+ };
65102
+ }
64743
65103
  function detectModelUnavailable(stderr) {
64744
65104
  if (typeof stderr !== "string" || stderr.length === 0)
64745
65105
  return null;
@@ -64933,54 +65293,358 @@ function parseRelativeDuration(s) {
64933
65293
  }
64934
65294
  return matched && total > 0 ? total : null;
64935
65295
  }
64936
- function formatModelUnavailableCard(detection, agent, opts = {}) {
65296
+
65297
+ // throttle-tier.ts
65298
+ init_card_format();
65299
+ init_quota_check();
65300
+ var accountScopedThrottleSignals = [
65301
+ "would exceed your account's rate limit",
65302
+ "would exceed your account\u2019s rate limit",
65303
+ "not your account"
65304
+ ];
65305
+ function isAccountScopedThrottle(text4) {
65306
+ if (typeof text4 !== "string" || text4.length === 0)
65307
+ return false;
65308
+ const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
65309
+ const lower = sample.toLowerCase();
65310
+ return accountScopedThrottleSignals.some((s) => lower.includes(s));
65311
+ }
65312
+ function classify429Detail(text4) {
65313
+ if (isAccountScopedThrottle(text4))
65314
+ return "account-scoped";
65315
+ if (isLitellmProxyLocal429(text4))
65316
+ return "litellm-local";
65317
+ return "generic-transient";
65318
+ }
65319
+ var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
65320
+ var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
65321
+ function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS) {
65322
+ const last = prev.lastSentAtMsByAccount[account] ?? 0;
65323
+ if (now - last >= cooldownMs) {
65324
+ return {
65325
+ send: true,
65326
+ next: {
65327
+ lastSentAtMsByAccount: {
65328
+ ...prev.lastSentAtMsByAccount,
65329
+ [account]: now
65330
+ }
65331
+ }
65332
+ };
65333
+ }
65334
+ return { send: false, next: prev };
65335
+ }
65336
+ function renderThrottleNotice(opts) {
64937
65337
  const now = opts.now ?? new Date;
64938
- const slotPart = opts.slot ? ` (slot **${escapeMarkdown(opts.slot)}**)` : "";
64939
- const reason = formatReason(detection, now);
65338
+ const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
65339
+ const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
64940
65340
  const lines = [
64941
- `\u26a0\ufe0f **Model unavailable** on agent **${escapeMarkdown(agent)}**${slotPart}`,
64942
- `Reason: ${reason}`,
64943
- ""
65341
+ `\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
65342
+ `This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
65343
+ `_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
64944
65344
  ];
64945
- if (opts.autoFallbackInFlight) {
64946
- lines.push("_Auto-failover in progress \u2014 see the announcement below._");
64947
- } else {
64948
- lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
64949
- }
64950
65345
  return lines.join(`
64951
65346
  `);
64952
65347
  }
64953
- function formatReason(d, now) {
64954
- const reset2 = d.resetAt ? ` (${formatResetRelative(d.resetAt, now)})` : "";
64955
- switch (d.kind) {
64956
- case "quota_exhausted":
64957
- return `quota exhausted${reset2}`;
64958
- case "overload":
64959
- return `model overloaded${reset2}`;
64960
- case "rate_limited":
64961
- return `account rate-limited${reset2}`;
64962
- case "network":
64963
- return "network unreachable";
65348
+ function renderThrottleEscalationNotice(opts) {
65349
+ const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
65350
+ const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
65351
+ const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
65352
+ return `${head}
65353
+ ${tail}`;
65354
+ }
65355
+ // operator-events.ts
65356
+ function classifyClaudeError(raw) {
65357
+ try {
65358
+ return classifyInner(raw);
65359
+ } catch {
65360
+ return "unknown-4xx";
64964
65361
  }
64965
65362
  }
64966
- function resolveModelUnavailableFromOperatorEvent(ev) {
64967
- const detail = typeof ev.detail === "string" ? ev.detail : "";
64968
- if (ev.kind === "quota-exhausted") {
64969
- return detectModelUnavailable(detail) ?? { kind: "quota_exhausted", raw: detail };
65363
+ function classifyInner(raw) {
65364
+ if (raw == null)
65365
+ return "unknown-4xx";
65366
+ const obj = typeof raw === "object" ? raw : {};
65367
+ const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
65368
+ const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
65369
+ const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
65370
+ const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
65371
+ const sdkCode = extractString(obj, "error_code") ?? "";
65372
+ if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
65373
+ const msg = message.toLowerCase();
65374
+ if (msg.includes("expired") || msg.includes("refresh")) {
65375
+ return "credentials-expired";
65376
+ }
65377
+ return "credentials-invalid";
64970
65378
  }
64971
- if (ev.kind === "rate-limited") {
64972
- const detected = detectModelUnavailable(detail);
64973
- return detected?.kind === "quota_exhausted" ? detected : null;
65379
+ if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
65380
+ return "credentials-invalid";
64974
65381
  }
64975
- if (ev.kind === "unknown-5xx") {
64976
- return detectModelUnavailable(detail) ?? { kind: "overload", raw: detail };
65382
+ if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
65383
+ return "credit-exhausted";
64977
65384
  }
64978
- return detectModelUnavailable(detail);
65385
+ if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
65386
+ return "rate-limited";
65387
+ }
65388
+ if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
65389
+ return "rate-limited";
65390
+ }
65391
+ if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
65392
+ return "agent-crashed";
65393
+ }
65394
+ if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
65395
+ return "agent-restarted-unexpectedly";
65396
+ }
65397
+ if (status != null) {
65398
+ if (status >= 400 && status < 500)
65399
+ return "unknown-4xx";
65400
+ if (status >= 500 && status < 600)
65401
+ return "unknown-5xx";
65402
+ }
65403
+ return "unknown-4xx";
65404
+ }
65405
+ function extractString(obj, key) {
65406
+ const v = obj[key];
65407
+ return typeof v === "string" && v.length > 0 ? v : null;
65408
+ }
65409
+ function extractNumber(obj, key) {
65410
+ const v = obj[key];
65411
+ return typeof v === "number" ? v : null;
65412
+ }
65413
+ function getNestedObj(obj, key) {
65414
+ const v = obj[key];
65415
+ return typeof v === "object" && v != null ? v : {};
64979
65416
  }
65417
+ var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
65418
+ var cooldownMap2 = new Map;
64980
65419
 
64981
- // throttle-tier.ts
64982
- init_card_format();
64983
- init_quota_check();
65420
+ // llm-error-present.ts
65421
+ function extractModel(raw) {
65422
+ const m = raw.match(/["']?model["']?\s*[=:]\s*["']?((?:claude|sr)[A-Za-z0-9._-]+)/i);
65423
+ return m ? m[1] : undefined;
65424
+ }
65425
+ var TRANSIENT_KINDS = new Set([
65426
+ "rate_limit",
65427
+ "overload_529",
65428
+ "transient"
65429
+ ]);
65430
+ function isActionableKind(kind) {
65431
+ return kind === "auth" || kind === "quota_wall";
65432
+ }
65433
+ function parseLlmError(raw, retryState) {
65434
+ const text4 = typeof raw === "string" ? raw : "";
65435
+ const requestId = extractRequestId(text4);
65436
+ const model = extractModel(text4);
65437
+ const resetAt = parseResetTime(text4);
65438
+ const retryAfterMs = resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined;
65439
+ const { kind, source } = classifyKindAndSource(text4);
65440
+ let autoRetrying = false;
65441
+ let terminal = true;
65442
+ if (TRANSIENT_KINDS.has(kind)) {
65443
+ const { retryAttempt, maxRetries } = retryState ?? { retryAttempt: null, maxRetries: null };
65444
+ if (retryAttempt != null && maxRetries != null) {
65445
+ autoRetrying = retryAttempt < maxRetries;
65446
+ terminal = retryAttempt >= maxRetries;
65447
+ } else {
65448
+ autoRetrying = false;
65449
+ terminal = true;
65450
+ }
65451
+ }
65452
+ return {
65453
+ kind,
65454
+ coreText: buildCoreText(kind, source),
65455
+ ...resetAt != null ? { resetAt } : {},
65456
+ ...retryAfterMs != null ? { retryAfterMs } : {},
65457
+ ...model != null ? { model } : {},
65458
+ ...requestId != null ? { requestId } : {},
65459
+ source,
65460
+ autoRetrying,
65461
+ terminal
65462
+ };
65463
+ }
65464
+ function classifyKindAndSource(text4) {
65465
+ const lower = text4.toLowerCase();
65466
+ const claudeKind = classifyClaudeError({ message: text4, type: text4 });
65467
+ if (claudeKind === "credentials-expired" || claudeKind === "credentials-invalid") {
65468
+ return { kind: "auth", source: "anthropic" };
65469
+ }
65470
+ if (isLitellmProxyLocal429(text4)) {
65471
+ return { kind: "rate_limit", source: "litellm-local" };
65472
+ }
65473
+ const mu = detectModelUnavailable(text4);
65474
+ if (mu != null) {
65475
+ if (mu.kind === "quota_exhausted")
65476
+ return { kind: "quota_wall", source: "anthropic" };
65477
+ if (mu.kind === "network")
65478
+ return { kind: "transient", source: "network" };
65479
+ if (lower.includes("529") || lower.includes("overloaded")) {
65480
+ return { kind: "overload_529", source: "anthropic" };
65481
+ }
65482
+ const c = classify429Detail(text4);
65483
+ return {
65484
+ kind: "rate_limit",
65485
+ source: c === "litellm-local" ? "litellm-local" : "anthropic"
65486
+ };
65487
+ }
65488
+ if (claudeKind === "credit-exhausted") {
65489
+ return { kind: "quota_wall", source: "anthropic" };
65490
+ }
65491
+ if (claudeKind === "rate-limited") {
65492
+ return { kind: "rate_limit", source: "anthropic" };
65493
+ }
65494
+ if (claudeKind === "unknown-5xx") {
65495
+ return { kind: "overload_529", source: "anthropic" };
65496
+ }
65497
+ return { kind: "unknown", source: "anthropic" };
65498
+ }
65499
+ function buildCoreText(kind, source) {
65500
+ switch (kind) {
65501
+ case "rate_limit":
65502
+ return source === "litellm-local" ? "Hit the local proxy rate limit \u2014 retrying automatically." : "Rate limited by Anthropic \u2014 retrying automatically.";
65503
+ case "overload_529":
65504
+ return "Anthropic is overloaded (529) \u2014 retrying automatically.";
65505
+ case "quota_wall":
65506
+ return "Usage limit reached on this Claude subscription.";
65507
+ case "auth":
65508
+ return "Claude login needs re-authentication.";
65509
+ case "transient":
65510
+ return source === "network" ? "Couldn't reach Anthropic (network) \u2014 retrying automatically." : "A temporary upstream hiccup \u2014 retrying automatically.";
65511
+ case "unknown":
65512
+ return "The model returned an error.";
65513
+ }
65514
+ }
65515
+ function formatResetLocal(resetAt, tz, now = new Date) {
65516
+ if (resetAt == null)
65517
+ return "";
65518
+ const ms = resetAt.getTime();
65519
+ if (!Number.isFinite(ms))
65520
+ return "";
65521
+ const clock = fmtLocalClock(ms, tz);
65522
+ const abbrev = tzAbbrev(ms, tz);
65523
+ const rel = formatRelativeTail(ms - now.getTime());
65524
+ return rel ? `clears ~${clock} ${abbrev} (${rel})` : `clears ~${clock} ${abbrev}`;
65525
+ }
65526
+ function formatRelativeTail(deltaMs) {
65527
+ if (deltaMs <= 0)
65528
+ return "~now";
65529
+ const totalMin = Math.round(deltaMs / 60000);
65530
+ if (totalMin < 1)
65531
+ return "~in <1m";
65532
+ if (totalMin < 60)
65533
+ return `~in ${totalMin}m`;
65534
+ const hours = Math.floor(totalMin / 60);
65535
+ const mins = totalMin % 60;
65536
+ if (hours < 24)
65537
+ return mins > 0 ? `~in ${hours}h ${mins}m` : `~in ${hours}h`;
65538
+ const days = Math.floor(hours / 24);
65539
+ const remH = hours % 24;
65540
+ return remH > 0 ? `~in ${days}d ${remH}h` : `~in ${days}d`;
65541
+ }
65542
+ function renderLlmError(parsed, agent, tz, now = new Date) {
65543
+ const safeAgent = escapeAgent(agent);
65544
+ const emoji = kindEmoji(parsed.kind);
65545
+ const lines = [`${emoji} ${parsed.coreText} (**${safeAgent}**)`];
65546
+ const resetLine = formatResetLocal(parsed.resetAt, tz, now);
65547
+ if (resetLine)
65548
+ lines.push(`_${resetLine}_`);
65549
+ if (parsed.model)
65550
+ lines.push(`_model: ${escapeAgent(parsed.model)}_`);
65551
+ const text4 = lines.join(`
65552
+ `);
65553
+ switch (parsed.kind) {
65554
+ case "auth":
65555
+ return {
65556
+ text: text4,
65557
+ keyboard: {
65558
+ inline_keyboard: [
65559
+ [
65560
+ { text: "\uD83D\uDD10 Reauth now", callback_data: `op:reauth:${encodeURIComponent(agent)}` },
65561
+ { text: "\u274c Dismiss", callback_data: `op:dismiss:${encodeURIComponent(agent)}` }
65562
+ ]
65563
+ ]
65564
+ }
65565
+ };
65566
+ case "quota_wall":
65567
+ return {
65568
+ text: text4,
65569
+ keyboard: {
65570
+ inline_keyboard: [
65571
+ [{ text: "\u23f3 Wait", callback_data: `op:dismiss:${encodeURIComponent(agent)}` }]
65572
+ ]
65573
+ }
65574
+ };
65575
+ default:
65576
+ return { text: text4 };
65577
+ }
65578
+ }
65579
+ function kindEmoji(kind) {
65580
+ switch (kind) {
65581
+ case "rate_limit":
65582
+ return "\uD83D\uDEA6";
65583
+ case "overload_529":
65584
+ return "\uD83D\uDD25";
65585
+ case "quota_wall":
65586
+ return "\u26a0\ufe0f";
65587
+ case "auth":
65588
+ return "\uD83D\uDD11";
65589
+ case "transient":
65590
+ return "\uD83C\uDF10";
65591
+ case "unknown":
65592
+ return "\u26a0\ufe0f";
65593
+ }
65594
+ }
65595
+ function escapeAgent(s) {
65596
+ return s.replace(/([_*`\[\]])/g, "\\$1");
65597
+ }
65598
+ var ERROR_COLLAPSE_WINDOW_MS = 60000;
65599
+
65600
+ class ErrorPresenceGate {
65601
+ claims = new Map;
65602
+ keyFor(parsed, agent, now) {
65603
+ if (parsed.requestId)
65604
+ return `rid:${parsed.requestId}`;
65605
+ const bucket = Math.floor(now / ERROR_COLLAPSE_WINDOW_MS);
65606
+ return `${parsed.kind}:${agent}:${bucket}`;
65607
+ }
65608
+ claim(key, now = Date.now()) {
65609
+ this.prune(now);
65610
+ const existing = this.claims.get(key);
65611
+ if (existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS) {
65612
+ return false;
65613
+ }
65614
+ this.claims.set(key, now);
65615
+ return true;
65616
+ }
65617
+ isClaimed(key, now = Date.now()) {
65618
+ const existing = this.claims.get(key);
65619
+ return existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS;
65620
+ }
65621
+ prune(now) {
65622
+ for (const [k, at] of this.claims) {
65623
+ if (now - at >= ERROR_COLLAPSE_WINDOW_MS)
65624
+ this.claims.delete(k);
65625
+ }
65626
+ }
65627
+ reset() {
65628
+ this.claims.clear();
65629
+ }
65630
+ }
65631
+ var errorPresenceGate = new ErrorPresenceGate;
65632
+ function decideErrorSurface(parsed, agent, opts = { claim: true }) {
65633
+ const now = opts.now ?? Date.now();
65634
+ const gate = opts.gate ?? errorPresenceGate;
65635
+ if (isActionableKind(parsed.kind)) {
65636
+ if (opts.claim)
65637
+ gate.claim(gate.keyFor(parsed, agent, now), now);
65638
+ return "render";
65639
+ }
65640
+ if (parsed.autoRetrying && !parsed.terminal)
65641
+ return "suppress";
65642
+ const key = gate.keyFor(parsed, agent, now);
65643
+ if (opts.claim) {
65644
+ return gate.claim(key, now) ? "render" : "suppress";
65645
+ }
65646
+ return gate.isClaimed(key, now) ? "suppress" : "render";
65647
+ }
64984
65648
 
64985
65649
  // model-unavailable.ts
64986
65650
  init_quota_check();
@@ -64995,13 +65659,6 @@ var transientUpstreamSignals2 = [
64995
65659
  "would exceed your account\u2019s rate limit",
64996
65660
  "would exceed your account's rate limit"
64997
65661
  ];
64998
- function isTransientUpstreamSignal(text4) {
64999
- if (typeof text4 !== "string" || text4.length === 0)
65000
- return false;
65001
- const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
65002
- const lower = sample.toLowerCase();
65003
- return transientUpstreamSignals2.some((s) => lower.includes(s));
65004
- }
65005
65662
  var litellmProxyLocal429Signals2 = [
65006
65663
  "deployment over user-defined ratelimit",
65007
65664
  "model rate limit exceeded. tpm limit",
@@ -65022,55 +65679,76 @@ function isLitellmProxyLocal4292(text4) {
65022
65679
  return true;
65023
65680
  return litellmV3LimiterSignalPair2.every((s) => lower.includes(s));
65024
65681
  }
65025
- function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
65026
- const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
65027
- if (typeof text4 !== "string" || text4.length === 0)
65028
- return empty2;
65029
- const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
65682
+ function detectModelUnavailable2(stderr) {
65683
+ if (typeof stderr !== "string" || stderr.length === 0)
65684
+ return null;
65685
+ const sample = stderr.length > 16384 ? stderr.slice(0, 16384) : stderr;
65030
65686
  const lower = sample.toLowerCase();
65031
- let limitType = null;
65032
- let limit = null;
65033
- const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
65034
- if (eqLimit) {
65035
- limitType = eqLimit[1];
65036
- limit = Number(eqLimit[2]);
65687
+ if (transientUpstreamSignals2.some((s) => lower.includes(s))) {
65688
+ const resetAt = parseResetTime2(sample);
65689
+ return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
65037
65690
  }
65038
- if (limitType == null) {
65039
- const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
65040
- if (v3Type)
65041
- limitType = v3Type[1];
65691
+ if (isLitellmProxyLocal4292(sample)) {
65692
+ const resetAt = parseResetTime2(sample);
65693
+ return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
65042
65694
  }
65043
- if (limit == null) {
65044
- const v3Limit = lower.match(/current limit:\s*(\d+)/);
65045
- if (v3Limit)
65046
- limit = Number(v3Limit[1]);
65695
+ const quotaSignals = [
65696
+ "out of extra usage",
65697
+ "extra usage",
65698
+ "credit_balance_too_low",
65699
+ "credit balance too low",
65700
+ "usage limit",
65701
+ "usage_limit",
65702
+ "quota exhausted",
65703
+ "quota_exhausted",
65704
+ "plan limit",
65705
+ "subscription limit",
65706
+ "hit your limit",
65707
+ "hit the limit",
65708
+ "session limit",
65709
+ "session cap"
65710
+ ];
65711
+ if (quotaSignals.some((s) => lower.includes(s))) {
65712
+ const resetAt = parseResetTime2(sample);
65713
+ return resetAt !== undefined ? { kind: "quota_exhausted", resetAt, raw: stderr } : { kind: "quota_exhausted", raw: stderr };
65047
65714
  }
65048
- let currentUsage = null;
65049
- const usage = lower.match(/current usage=(\d+)/);
65050
- if (usage)
65051
- currentUsage = Number(usage[1]);
65052
- let resetAtMs = null;
65053
- const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
65054
- if (resetsAt) {
65055
- const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
65056
- if (!Number.isNaN(d.getTime()))
65057
- resetAtMs = d.getTime();
65715
+ const overloadSignals = [
65716
+ "overloaded_error",
65717
+ "overloaded",
65718
+ "rate_limit_error",
65719
+ "rate limit",
65720
+ "rate-limited",
65721
+ "http 429",
65722
+ '"status":429',
65723
+ "status: 429",
65724
+ " 429 ",
65725
+ "503 service",
65726
+ "service unavailable",
65727
+ '"status":529',
65728
+ "http 529",
65729
+ " 529 "
65730
+ ];
65731
+ if (overloadSignals.some((s) => lower.includes(s))) {
65732
+ const resetAt = parseResetTime2(sample);
65733
+ return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
65058
65734
  }
65059
- if (resetAtMs == null) {
65060
- const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
65061
- if (tryAgain) {
65062
- const secs = Number(tryAgain[1]);
65063
- if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
65064
- resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
65065
- }
65066
- }
65735
+ const networkSignals = [
65736
+ "econnrefused",
65737
+ "econnreset",
65738
+ "etimedout",
65739
+ "enotfound",
65740
+ "eai_again",
65741
+ "fetch failed",
65742
+ "network error",
65743
+ "socket hang up",
65744
+ "request timed out",
65745
+ "connection refused",
65746
+ "getaddrinfo"
65747
+ ];
65748
+ if (networkSignals.some((s) => lower.includes(s))) {
65749
+ return { kind: "network", raw: stderr };
65067
65750
  }
65068
- return {
65069
- limitType,
65070
- limit: limit != null && Number.isFinite(limit) ? limit : null,
65071
- currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
65072
- resetAtMs
65073
- };
65751
+ return null;
65074
65752
  }
65075
65753
  function parseResetTime2(text4, parseTimeNow = new Date) {
65076
65754
  const lower = text4.toLowerCase();
@@ -65194,31 +65872,77 @@ function parseRelativeDuration2(s) {
65194
65872
  }
65195
65873
  return matched && total > 0 ? total : null;
65196
65874
  }
65875
+ function formatModelUnavailableCard(detection, agent, opts = {}) {
65876
+ const now = opts.now ?? new Date;
65877
+ const slotPart = opts.slot ? ` (slot **${escapeMarkdown(opts.slot)}**)` : "";
65878
+ const reason = formatReason(detection, now);
65879
+ const lines = [
65880
+ `\u26a0\ufe0f **Model unavailable** on agent **${escapeMarkdown(agent)}**${slotPart}`,
65881
+ `Reason: ${reason}`,
65882
+ ""
65883
+ ];
65884
+ if (opts.autoFallbackInFlight) {
65885
+ lines.push("_Auto-failover in progress \u2014 see the announcement below._");
65886
+ } else {
65887
+ lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
65888
+ }
65889
+ return lines.join(`
65890
+ `);
65891
+ }
65892
+ function formatReason(d, now) {
65893
+ const reset2 = d.resetAt ? ` (${formatResetRelative(d.resetAt, now)})` : "";
65894
+ switch (d.kind) {
65895
+ case "quota_exhausted":
65896
+ return `quota exhausted${reset2}`;
65897
+ case "overload":
65898
+ return `model overloaded${reset2}`;
65899
+ case "rate_limited":
65900
+ return `account rate-limited${reset2}`;
65901
+ case "network":
65902
+ return "network unreachable";
65903
+ }
65904
+ }
65905
+ function resolveModelUnavailableFromOperatorEvent(ev) {
65906
+ const detail = typeof ev.detail === "string" ? ev.detail : "";
65907
+ if (ev.kind === "quota-exhausted") {
65908
+ return detectModelUnavailable2(detail) ?? { kind: "quota_exhausted", raw: detail };
65909
+ }
65910
+ if (ev.kind === "rate-limited") {
65911
+ const detected = detectModelUnavailable2(detail);
65912
+ return detected?.kind === "quota_exhausted" ? detected : null;
65913
+ }
65914
+ if (ev.kind === "unknown-5xx") {
65915
+ return detectModelUnavailable2(detail) ?? { kind: "overload", raw: detail };
65916
+ }
65917
+ return detectModelUnavailable2(detail);
65918
+ }
65197
65919
 
65198
65920
  // throttle-tier.ts
65199
- var accountScopedThrottleSignals = [
65921
+ init_card_format();
65922
+ init_quota_check();
65923
+ var accountScopedThrottleSignals2 = [
65200
65924
  "would exceed your account's rate limit",
65201
65925
  "would exceed your account\u2019s rate limit",
65202
65926
  "not your account"
65203
65927
  ];
65204
- function isAccountScopedThrottle(text4) {
65928
+ function isAccountScopedThrottle2(text4) {
65205
65929
  if (typeof text4 !== "string" || text4.length === 0)
65206
65930
  return false;
65207
65931
  const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
65208
65932
  const lower = sample.toLowerCase();
65209
- return accountScopedThrottleSignals.some((s) => lower.includes(s));
65933
+ return accountScopedThrottleSignals2.some((s) => lower.includes(s));
65210
65934
  }
65211
- function classify429Detail(text4) {
65212
- if (isAccountScopedThrottle(text4))
65935
+ function classify429Detail2(text4) {
65936
+ if (isAccountScopedThrottle2(text4))
65213
65937
  return "account-scoped";
65214
- if (isLitellmProxyLocal4292(text4))
65938
+ if (isLitellmProxyLocal429(text4))
65215
65939
  return "litellm-local";
65216
65940
  return "generic-transient";
65217
65941
  }
65218
65942
  function build429ClassifiedMetric(opts) {
65219
65943
  const detail = typeof opts.detail === "string" ? opts.detail : "";
65220
65944
  const litellm = parseLitellmLimitDetail(detail, new Date(opts.now));
65221
- const anthropicResetMs = parseResetTime2(detail, new Date(opts.now))?.getTime() ?? null;
65945
+ const anthropicResetMs = parseResetTime(detail, new Date(opts.now))?.getTime() ?? null;
65222
65946
  const resetAtMs = anthropicResetMs ?? litellm.resetAtMs;
65223
65947
  return {
65224
65948
  kind: "rate_limit_429_classified",
@@ -65232,20 +65956,20 @@ function build429ClassifiedMetric(opts) {
65232
65956
  current_usage: litellm.currentUsage
65233
65957
  };
65234
65958
  }
65235
- var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
65959
+ var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
65236
65960
  var THROTTLE_DEFAULT_WAIT_MS = 60000;
65237
65961
  function throttleRetryInPlaceMaxMs(env = process.env) {
65238
65962
  const raw = env.SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS;
65239
65963
  if (raw == null || raw === "")
65240
- return THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT;
65964
+ return THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
65241
65965
  const n = Number(raw);
65242
- return Number.isFinite(n) && n > 0 ? n : THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT;
65966
+ return Number.isFinite(n) && n > 0 ? n : THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
65243
65967
  }
65244
65968
  function decideThrottleTier(opts) {
65245
65969
  const { detail, now, thresholdMs } = opts;
65246
- if (!isAccountScopedThrottle(detail))
65970
+ if (!isAccountScopedThrottle2(detail))
65247
65971
  return { action: "none" };
65248
- const resetAt = parseResetTime2(detail, new Date(now));
65972
+ const resetAt = parseResetTime(detail, new Date(now));
65249
65973
  const resetAtMs = resetAt?.getTime();
65250
65974
  if (resetAtMs == null || !Number.isFinite(resetAtMs) || resetAtMs <= now) {
65251
65975
  return {
@@ -65259,47 +65983,7 @@ function decideThrottleTier(opts) {
65259
65983
  }
65260
65984
  return { action: "failover", resetAtMs };
65261
65985
  }
65262
- var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
65263
-
65264
- // throttle-tier.ts
65265
- init_card_format();
65266
- init_quota_check();
65267
- var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
65268
65986
  var THROTTLE_NOTICE_COOLDOWN_MS2 = 10 * 60000;
65269
- function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS2) {
65270
- const last = prev.lastSentAtMsByAccount[account] ?? 0;
65271
- if (now - last >= cooldownMs) {
65272
- return {
65273
- send: true,
65274
- next: {
65275
- lastSentAtMsByAccount: {
65276
- ...prev.lastSentAtMsByAccount,
65277
- [account]: now
65278
- }
65279
- }
65280
- };
65281
- }
65282
- return { send: false, next: prev };
65283
- }
65284
- function renderThrottleNotice(opts) {
65285
- const now = opts.now ?? new Date;
65286
- const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
65287
- const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
65288
- const lines = [
65289
- `\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
65290
- `This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
65291
- `_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
65292
- ];
65293
- return lines.join(`
65294
- `);
65295
- }
65296
- function renderThrottleEscalationNotice(opts) {
65297
- const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
65298
- const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
65299
- const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
65300
- return `${head}
65301
- ${tail}`;
65302
- }
65303
65987
 
65304
65988
  // gateway/throttle-tier-wiring.ts
65305
65989
  var THROTTLE_RETRY_NUDGE_SLACK_MS = 5000;
@@ -65320,7 +66004,7 @@ function createThrottleTierRunner(deps) {
65320
66004
  let granted = true;
65321
66005
  if (client3 && account) {
65322
66006
  try {
65323
- granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`, THROTTLE_NOTICE_COOLDOWN_MS2)).granted;
66007
+ granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`, THROTTLE_NOTICE_COOLDOWN_MS)).granted;
65324
66008
  } catch {
65325
66009
  granted = true;
65326
66010
  }
@@ -69406,6 +70090,7 @@ var MODEL_ARG_RE2 = /^[A-Za-z0-9][A-Za-z0-9._/\[\]-]{0,99}$/;
69406
70090
  function isValidModelArg2(arg) {
69407
70091
  return MODEL_ARG_RE2.test(arg);
69408
70092
  }
70093
+ var MODEL_CALLBACK_ALIAS2 = "mdl:alias:";
69409
70094
 
69410
70095
  // gateway/session-model-file.ts
69411
70096
  var SESSION_MODEL_FILE = ".session-model";
@@ -69472,6 +70157,172 @@ function clearSessionEffortFile(agentDir) {
69472
70157
  rmSync4(join29(agentDir, SESSION_EFFORT_FILE), { force: true });
69473
70158
  } catch {}
69474
70159
  }
70160
+ var PREMIUM_RECOVERY_FILE = ".premium-recovery";
70161
+ function parsePremiumRecovery(text4) {
70162
+ try {
70163
+ const raw = JSON.parse(text4);
70164
+ if (typeof raw.premiumModel !== "string" || !isValidModelArg2(raw.premiumModel) || typeof raw.ts !== "number" || !Array.isArray(raw.chats) || raw.chats.length === 0 || !raw.chats.every((c) => typeof c === "string" && c.length > 0)) {
70165
+ return null;
70166
+ }
70167
+ return { premiumModel: raw.premiumModel, chats: raw.chats, ts: raw.ts };
70168
+ } catch {
70169
+ return null;
70170
+ }
70171
+ }
70172
+ function writePremiumRecoveryFile(agentDir, premiumModel, chats) {
70173
+ if (!isValidModelArg2(premiumModel)) {
70174
+ throw new Error(`refusing to persist non-canonical premium-recovery token: ${JSON.stringify(premiumModel)}`);
70175
+ }
70176
+ const clean = chats.filter((c) => typeof c === "string" && c.length > 0);
70177
+ if (clean.length === 0) {
70178
+ throw new Error("refusing to persist premium-recovery marker with no chats to notify");
70179
+ }
70180
+ atomicWrite(join29(agentDir, PREMIUM_RECOVERY_FILE), `${JSON.stringify({ premiumModel, chats: clean, ts: Date.now() })}
70181
+ `);
70182
+ }
70183
+ function readPremiumRecoveryFile(agentDir) {
70184
+ let raw;
70185
+ try {
70186
+ raw = readFileSync25(join29(agentDir, PREMIUM_RECOVERY_FILE), "utf8");
70187
+ } catch {
70188
+ return null;
70189
+ }
70190
+ const parsed = parsePremiumRecovery(raw);
70191
+ if (parsed == null) {
70192
+ clearPremiumRecoveryFile(agentDir);
70193
+ return null;
70194
+ }
70195
+ return parsed;
70196
+ }
70197
+ function clearPremiumRecoveryFile(agentDir) {
70198
+ try {
70199
+ rmSync4(join29(agentDir, PREMIUM_RECOVERY_FILE), { force: true });
70200
+ } catch {}
70201
+ }
70202
+
70203
+ // tier-downgrade.ts
70204
+ function decideTierDowngrade(input) {
70205
+ const configured = typeof input.configuredDefault === "string" ? input.configuredDefault.trim() : "";
70206
+ if (configured.length === 0) {
70207
+ return { action: "skip", reason: "unresolved" };
70208
+ }
70209
+ const override = input.sessionOverride;
70210
+ if (override == null || override.length === 0) {
70211
+ return { action: "skip", reason: "on-default" };
70212
+ }
70213
+ if (input.resolve(override) === input.resolve(configured)) {
70214
+ return { action: "skip", reason: "on-default" };
70215
+ }
70216
+ return { action: "downgrade", toModel: configured, fromModel: override };
70217
+ }
70218
+ function renderTierDowngradeNotice(fromModel, toModel, agent) {
70219
+ return `\u2935\ufe0f **Downgrading model to keep going** on agent **${agent}**
70220
+ ` + `\`${fromModel}\` is overloaded across every account right now, so this turn is ` + `resuming on the default \`${toModel}\` to keep working. ` + `Re-issue \`/model ${fromModel}\` once it frees up \u2014 it won't switch back on its own.`;
70221
+ }
70222
+ function planTierDowngrade(decision, gateVerdict, agent) {
70223
+ if (decision.action !== "downgrade")
70224
+ return { kind: "skip" };
70225
+ if (gateVerdict === "skip-inflight")
70226
+ return { kind: "suppress" };
70227
+ if (gateVerdict === "skip-stale")
70228
+ return { kind: "skip" };
70229
+ return {
70230
+ kind: "downgrade",
70231
+ toModel: decision.toModel,
70232
+ fromModel: decision.fromModel,
70233
+ notice: renderTierDowngradeNotice(decision.fromModel, decision.toModel, agent)
70234
+ };
70235
+ }
70236
+
70237
+ // gateway/tier-downgrade-wiring.ts
70238
+ function runTierDowngrade(triggerAgent, deps) {
70239
+ const agentDir = deps.getAgentDir();
70240
+ if (!agentDir)
70241
+ return "skip";
70242
+ const configuredDefault = deps.getConfiguredDefault() ?? "";
70243
+ const decision = decideTierDowngrade({
70244
+ sessionOverride: deps.getSessionOverride(),
70245
+ configuredDefault,
70246
+ resolve: deps.resolve
70247
+ });
70248
+ const gateVerdict = deps.peekResumeGate();
70249
+ const plan = planTierDowngrade(decision, gateVerdict, triggerAgent);
70250
+ if (plan.kind === "skip") {
70251
+ if (decision.action === "downgrade") {
70252
+ deps.log(`[tier-downgrade] restart suppressed (${gateVerdict}) agent=${triggerAgent}`);
70253
+ }
70254
+ return "skip";
70255
+ }
70256
+ if (plan.kind === "suppress") {
70257
+ deps.log(`[tier-downgrade] give-up suppressed \u2014 a resume restart is already armed agent=${triggerAgent}`);
70258
+ return "restart-pending";
70259
+ }
70260
+ try {
70261
+ deps.writeCarrier(agentDir, plan.toModel, configuredDefault);
70262
+ } catch (err) {
70263
+ deps.log(`[tier-downgrade] failed to write session-model carrier \u2014 aborting downgrade: ${err?.message ?? err}`);
70264
+ return "skip";
70265
+ }
70266
+ deps.armResumeGate();
70267
+ deps.log(`[tier-downgrade] downgrading ${plan.fromModel} \u2192 ${plan.toModel} and resuming via self-restart agent=${triggerAgent}`);
70268
+ try {
70269
+ deps.writeRecoveryMarker(agentDir, plan.fromModel);
70270
+ } catch (err) {
70271
+ deps.log(`[tier-downgrade] premium-recovery marker write failed (non-fatal): ${err?.message ?? err}`);
70272
+ }
70273
+ deps.broadcastNotice(plan.notice);
70274
+ deps.selfRestart(deps.selfAgent(triggerAgent));
70275
+ return "downgraded";
70276
+ }
70277
+
70278
+ // premium-recovery.ts
70279
+ function renderPremiumRecoveryPing(premiumModel) {
70280
+ return {
70281
+ text: `\u2705 \`${premiumModel}\` is available again \u2014 tap to switch back to it for this session.`,
70282
+ buttonText: `Switch to ${premiumModel}`
70283
+ };
70284
+ }
70285
+ function premiumRecoveryClaimKey(agent, premiumModel) {
70286
+ return `premium-recovery:${agent}:${premiumModel}`;
70287
+ }
70288
+
70289
+ // gateway/premium-recovery-wiring.ts
70290
+ async function runPremiumRecoveryPing(deps) {
70291
+ const agentDir = deps.getAgentDir();
70292
+ if (!agentDir)
70293
+ return;
70294
+ const marker = deps.readMarker(agentDir);
70295
+ if (marker == null)
70296
+ return;
70297
+ if (!deps.decide())
70298
+ return;
70299
+ const agent = deps.getAgent();
70300
+ const granted = await deps.claimNotification(premiumRecoveryClaimKey(agent, marker.premiumModel));
70301
+ if (!granted) {
70302
+ deps.clearMarker(agentDir);
70303
+ return;
70304
+ }
70305
+ deps.clearMarker(agentDir);
70306
+ const ping = renderPremiumRecoveryPing(marker.premiumModel);
70307
+ const keyboard = {
70308
+ inline_keyboard: [
70309
+ [{ text: ping.buttonText, callback_data: `${MODEL_CALLBACK_ALIAS2}${marker.premiumModel}` }]
70310
+ ]
70311
+ };
70312
+ const chats = marker.chats.length > 0 ? marker.chats : deps.fallbackChats();
70313
+ for (const chatId of chats) {
70314
+ deps.sendToChat(chatId, ping, keyboard);
70315
+ }
70316
+ deps.log(`[premium-recovery] ${marker.premiumModel} servable again \u2014 ping sent agent=${agent} chats=${chats.length}`);
70317
+ }
70318
+
70319
+ // premium-recovery.ts
70320
+ function decidePremiumRecovery(opts) {
70321
+ if (!opts.hasMarker)
70322
+ return { fire: false, reason: "no-marker" };
70323
+ const servable = opts.accounts.some((a) => !a.exhausted && !a.premiumWalled);
70324
+ return servable ? { fire: true, reason: "recovered" } : { fire: false, reason: "still-walled" };
70325
+ }
69475
70326
 
69476
70327
  // ../src/agents/model-picker.ts
69477
70328
  var HEADER_RE = /Select model/;
@@ -77113,71 +77964,6 @@ import {
77113
77964
  } from "fs";
77114
77965
  import { join as join39 } from "path";
77115
77966
 
77116
- // operator-events.ts
77117
- function classifyClaudeError(raw) {
77118
- try {
77119
- return classifyInner(raw);
77120
- } catch {
77121
- return "unknown-4xx";
77122
- }
77123
- }
77124
- function classifyInner(raw) {
77125
- if (raw == null)
77126
- return "unknown-4xx";
77127
- const obj = typeof raw === "object" ? raw : {};
77128
- const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
77129
- const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
77130
- const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
77131
- const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
77132
- const sdkCode = extractString(obj, "error_code") ?? "";
77133
- if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
77134
- const msg = message.toLowerCase();
77135
- if (msg.includes("expired") || msg.includes("refresh")) {
77136
- return "credentials-expired";
77137
- }
77138
- return "credentials-invalid";
77139
- }
77140
- if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
77141
- return "credentials-invalid";
77142
- }
77143
- if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
77144
- return "credit-exhausted";
77145
- }
77146
- if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
77147
- return "rate-limited";
77148
- }
77149
- if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
77150
- return "rate-limited";
77151
- }
77152
- if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
77153
- return "agent-crashed";
77154
- }
77155
- if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
77156
- return "agent-restarted-unexpectedly";
77157
- }
77158
- if (status != null) {
77159
- if (status >= 400 && status < 500)
77160
- return "unknown-4xx";
77161
- if (status >= 500 && status < 600)
77162
- return "unknown-5xx";
77163
- }
77164
- return "unknown-4xx";
77165
- }
77166
- function extractString(obj, key) {
77167
- const v = obj[key];
77168
- return typeof v === "string" && v.length > 0 ? v : null;
77169
- }
77170
- function extractNumber(obj, key) {
77171
- const v = obj[key];
77172
- return typeof v === "number" ? v : null;
77173
- }
77174
- function getNestedObj(obj, key) {
77175
- const v = obj[key];
77176
- return typeof v === "object" && v != null ? v : {};
77177
- }
77178
- var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
77179
- var cooldownMap2 = new Map;
77180
-
77181
77967
  // session-tail.ts
77182
77968
  function sanitizeCwdToProjectName(cwd) {
77183
77969
  return cwd.replace(/[^a-zA-Z0-9]/g, "-");
@@ -77362,7 +78148,7 @@ function detectErrorInTranscriptLine(line) {
77362
78148
  const errStr = typeof obj.error === "string" ? obj.error : "";
77363
78149
  const text4 = extractAssistantText(obj);
77364
78150
  const kind2 = status === 429 ? isTransientUpstreamSignal(`${text4}
77365
- ${errStr}`) || isLitellmProxyLocal4292(`${text4}
78151
+ ${errStr}`) || isLitellmProxyLocal429(`${text4}
77366
78152
  ${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text4 });
77367
78153
  return {
77368
78154
  kind: kind2,
@@ -79671,6 +80457,29 @@ var RESOURCE_KEYS = ["path", "endpoint", "url", "resource", "route"];
79671
80457
  var ARG_SUMMARY_MAX_KEYS = 4;
79672
80458
  var ARG_VALUE_MAX = 40;
79673
80459
  var ARG_SUMMARY_LINE_MAX = 180;
80460
+ var CONTEXT_NOISE_KEYS = new Set([
80461
+ "reason",
80462
+ "why",
80463
+ "chat_id",
80464
+ "message_id",
80465
+ "message_thread_id",
80466
+ "thread_id",
80467
+ "origin_turn_id",
80468
+ "reply_to",
80469
+ "quote",
80470
+ "quote_text",
80471
+ "format",
80472
+ "parse_mode",
80473
+ "disable_web_page_preview",
80474
+ "disable_notification",
80475
+ "protect_content",
80476
+ "single_use",
80477
+ "ack_text",
80478
+ "inline_keyboard",
80479
+ "file_id"
80480
+ ]);
80481
+ var SENSITIVE_VALUE_KEY_RE = /token|secret|password|passwd|key|auth|dsn|conn|url|credential|cookie|session/i;
80482
+ var NON_HTTP_DSN_RE = /\b(?:postgres(?:ql)?|mysql|mariadb|redis|rediss|mongodb(?:\+srv)?|amqp|amqps):\/\/\S*@\S+/gi;
79674
80483
  var MCP_TOOL_DESCRIPTIONS = {
79675
80484
  "mcp__agent-config__config_get": "Read its own merged config",
79676
80485
  "mcp__agent-config__cron_list": "List its own scheduled tasks",
@@ -79721,8 +80530,12 @@ function formatPermissionCardBody(opts) {
79721
80530
  }
79722
80531
  const callerReason = callerSuppliedReason(opts.inputPreview);
79723
80532
  const rawWhy = (callerReason ?? "").replace(/\s+/g, " ").trim();
79724
- const truncatedWhy = rawWhy.length > DESCRIPTION_LINE_MAX ? rawWhy.slice(0, DESCRIPTION_LINE_MAX - 1) + "\u2026" : rawWhy;
79725
- lines.push(truncatedWhy.length > 0 ? `why: _${escapeTgHtml(truncatedWhy)}_` : `why: _not provided_`);
80533
+ if (rawWhy.length > 0) {
80534
+ const truncatedWhy = rawWhy.length > DESCRIPTION_LINE_MAX ? rawWhy.slice(0, DESCRIPTION_LINE_MAX - 1) + "\u2026" : rawWhy;
80535
+ lines.push(`why: _${escapeTgHtml(truncatedWhy)}_`);
80536
+ } else {
80537
+ lines.push(`context: _${escapeTgHtml(synthesizeContext(opts.toolName, opts.inputPreview))}_`);
80538
+ }
79726
80539
  const argSummary = mcpArgSummary(opts.toolName, opts.inputPreview);
79727
80540
  if (argSummary) {
79728
80541
  lines.push(`\u21b3 _${escapeTgHtml(argSummary)}_`);
@@ -79996,6 +80809,50 @@ function callerSuppliedReason(inputPreview) {
79996
80809
  }
79997
80810
  return null;
79998
80811
  }
80812
+ function synthesizeContext(toolName, inputPreview) {
80813
+ const input = parseInput2(inputPreview);
80814
+ const summary = input ? salientInputSummary(input) : null;
80815
+ const base = summary ?? naturalAction(toolName, inputPreview);
80816
+ const turnRef = input ? readString2(input, "origin_turn_id") : null;
80817
+ return turnRef ? `${base} \u00b7 turn ${shortTurnRef(turnRef)}` : base;
80818
+ }
80819
+ function shortTurnRef(turnId) {
80820
+ const t = turnId.trim();
80821
+ return t.length <= 10 ? t : `\u2026${t.slice(-8)}`;
80822
+ }
80823
+ function salientInputSummary(input) {
80824
+ const parts = [];
80825
+ for (const [key, value] of Object.entries(input)) {
80826
+ if (CONTEXT_NOISE_KEYS.has(key))
80827
+ continue;
80828
+ if (value == null)
80829
+ continue;
80830
+ if (parts.length >= ARG_SUMMARY_MAX_KEYS) {
80831
+ parts.push("\u2026");
80832
+ break;
80833
+ }
80834
+ if (typeof value === "object") {
80835
+ parts.push(key);
80836
+ continue;
80837
+ }
80838
+ const shown = truncate6(redactSalientValue(key, String(value)), ARG_VALUE_MAX);
80839
+ if (shown.length === 0)
80840
+ continue;
80841
+ parts.push(`${key}: ${shown}`);
80842
+ }
80843
+ if (parts.length === 0)
80844
+ return null;
80845
+ const joined = parts.join(", ");
80846
+ return joined.length > ARG_SUMMARY_LINE_MAX ? joined.slice(0, ARG_SUMMARY_LINE_MAX - 1) + "\u2026" : joined;
80847
+ }
80848
+ function redactSalientValue(key, value) {
80849
+ if (SENSITIVE_VALUE_KEY_RE.test(key))
80850
+ return REDACTED_MARKER;
80851
+ const dsnScrubbed = value.replace(NON_HTTP_DSN_RE, REDACTED_MARKER);
80852
+ const scrubbed = redact(`${key}=${dsnScrubbed}`);
80853
+ const prefix = `${key}=`;
80854
+ return scrubbed.startsWith(prefix) ? scrubbed.slice(prefix.length) : scrubbed;
80855
+ }
79999
80856
  function extractReasonFromRaw(raw) {
80000
80857
  const m = /"(?:reason|why)"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(raw);
80001
80858
  if (!m)
@@ -81224,10 +82081,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
81224
82081
  }
81225
82082
 
81226
82083
  // ../src/build-info.ts
81227
- var VERSION = "0.18.17";
81228
- var COMMIT_SHA = "372a4968";
81229
- var COMMIT_DATE = "2026-07-12T09:18:05Z";
81230
- var LATEST_PR = 3190;
82084
+ var VERSION = "0.18.18";
82085
+ var COMMIT_SHA = "a256b3b1";
82086
+ var COMMIT_DATE = "2026-07-13T03:10:38Z";
82087
+ var LATEST_PR = 3210;
81231
82088
  var COMMITS_AHEAD_OF_TAG = 0;
81232
82089
 
81233
82090
  // gateway/boot-version.ts
@@ -81280,6 +82137,8 @@ function classifyRejection(err, opts = {}) {
81280
82137
  return "log_only";
81281
82138
  if (err instanceof Error && err.message === FLOOD_WAIT_ACTIVE)
81282
82139
  return "log_only";
82140
+ if (err instanceof Error && err.message === LOCAL_RESOURCE_EXHAUSTED)
82141
+ return "log_only";
81283
82142
  if (!isGrammy)
81284
82143
  return "shutdown";
81285
82144
  const e = err;
@@ -81820,10 +82679,13 @@ async function emitLinearAgentActivity(args, deps = {}) {
81820
82679
  `);
81821
82680
  return { content: [{ type: "text", text: `Linear ${type} emitted on session ${sessionId}` }] };
81822
82681
  }
82682
+ function captureDedupComment(dedupKey) {
82683
+ return `<!-- switchroom-capture: ${dedupKey} -->`;
82684
+ }
81823
82685
  function captureDedupMarker(dedupKey) {
81824
82686
  return `
81825
82687
 
81826
- <!-- switchroom-capture: ${dedupKey} -->`;
82688
+ ${captureDedupComment(dedupKey)}`;
81827
82689
  }
81828
82690
  async function createLinearIssue(args, deps = {}) {
81829
82691
  const log = deps.log ?? ((s) => process.stderr.write(s));
@@ -81872,9 +82734,11 @@ async function createLinearIssue(args, deps = {}) {
81872
82734
  return { ok: true, data: json.data };
81873
82735
  };
81874
82736
  if (dedupKey) {
81875
- const search2 = await gql("query($term: String!) { searchIssues(term: $term) { nodes { id url title } } }", { term: dedupKey });
82737
+ const marker = captureDedupComment(dedupKey);
82738
+ const search2 = await gql("query($term: String!) { searchIssues(term: $term, first: 25) { nodes { id url title description } } }", { term: dedupKey });
81876
82739
  if (search2.ok) {
81877
- const hit = (search2.data?.searchIssues?.nodes ?? [])[0];
82740
+ const nodes = search2.data?.searchIssues?.nodes ?? [];
82741
+ const hit = nodes.find((n) => typeof n.description === "string" && n.description.includes(marker));
81878
82742
  if (hit?.url) {
81879
82743
  log(`telegram gateway: linear_create_issue: dedup hit key=${dedupKey} agent=${agent}
81880
82744
  `);
@@ -82681,8 +83545,15 @@ function buildBridgeDeadIdleNoticeInbound(args) {
82681
83545
  }
82682
83546
  };
82683
83547
  }
82684
- function isRealBridgeIdentity(agentName3) {
82685
- return agentName3 != null && agentName3.length > 0 && !isCronIdentity2(agentName3);
83548
+ function isRealBridgeIdentity(agentName3, selfAgentName) {
83549
+ if (agentName3 == null || agentName3.length === 0)
83550
+ return false;
83551
+ if (isCronIdentity2(agentName3))
83552
+ return false;
83553
+ if (selfAgentName != null && selfAgentName.length > 0) {
83554
+ return agentName3 === selfAgentName;
83555
+ }
83556
+ return true;
82686
83557
  }
82687
83558
  function createBridgeDeadWatchdog(opts) {
82688
83559
  const setTimer = opts.setTimer ?? ((fn, ms) => {
@@ -82696,6 +83567,7 @@ function createBridgeDeadWatchdog(opts) {
82696
83567
  const readCrashTail = opts.readCrashTail ?? ((p, t) => readFreshCrashLogTail(p, { nowMs: t }));
82697
83568
  const priorStreak = opts.priorStreak ?? 0;
82698
83569
  const maxConsecutive = opts.maxConsecutive ?? MAX_CONSECUTIVE_ESCALATIONS;
83570
+ const selfAgentName = opts.selfAgentName;
82699
83571
  let timer3 = null;
82700
83572
  let bridgeRegistered = false;
82701
83573
  let bridgeEverRegistered = false;
@@ -82773,14 +83645,14 @@ function createBridgeDeadWatchdog(opts) {
82773
83645
  return {
82774
83646
  arm: armInternal,
82775
83647
  noteBridgeRegistered: (agentName3) => {
82776
- if (!isRealBridgeIdentity(agentName3))
83648
+ if (!isRealBridgeIdentity(agentName3, selfAgentName))
82777
83649
  return;
82778
83650
  bridgeRegistered = true;
82779
83651
  bridgeEverRegistered = true;
82780
83652
  cancel();
82781
83653
  },
82782
83654
  noteBridgeDisconnected: (agentName3) => {
82783
- if (!isRealBridgeIdentity(agentName3))
83655
+ if (!isRealBridgeIdentity(agentName3, selfAgentName))
82784
83656
  return;
82785
83657
  bridgeRegistered = false;
82786
83658
  armInternal();
@@ -84887,8 +85759,9 @@ var PHOTO_EXTS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp"]);
84887
85759
  var FLOOD_STATE_PATH = floodStatePath(STATE_DIR);
84888
85760
  var FLOOD_WINDOWS_PATH = floodWindowsPath(STATE_DIR);
84889
85761
  var recordFloodWindow = makeFloodWindowRecorder(FLOOD_WINDOWS_PATH);
85762
+ var sendGateConfig = sendGateConfigFromEnv();
84890
85763
  var sendGate = createSendGate({
84891
- enabled: sendGateEnabledFromEnv(),
85764
+ ...sendGateConfig,
84892
85765
  initialWindows: loadInitialFloodWindows(FLOOD_STATE_PATH, FLOOD_WINDOWS_PATH, Date.now()),
84893
85766
  bootRamp: {},
84894
85767
  onWindowOpen: (scopeKey, untilTs) => recordFloodWindow(scopeKey, untilTs)
@@ -84896,10 +85769,10 @@ var sendGate = createSendGate({
84896
85769
  var probeFloodWaitRemainingMs = makeFloodWaitProbe2(FLOOD_STATE_PATH);
84897
85770
  var rawRobustApiCall = createRetryApiCall2({
84898
85771
  log: (line) => process.stderr.write(line),
84899
- onFloodWait: (retryAfterSec) => {
85772
+ onFloodWait: (retryAfterSec, opts) => {
84900
85773
  makeFloodWaitRecorder2(FLOOD_STATE_PATH)(retryAfterSec);
84901
85774
  try {
84902
- sendGate.openFloodWindow("global", Date.now() + Math.max(0, retryAfterSec) * 1000);
85775
+ sendGate.openScopedFloodWindows(opts, Date.now() + Math.max(0, retryAfterSec) * 1000);
84903
85776
  } catch {}
84904
85777
  },
84905
85778
  floodWaitRemainingMs: probeFloodWaitRemainingMs
@@ -84935,6 +85808,7 @@ var sendGateStatsLogger = createStatsLogger({
84935
85808
  var floodWindowObserver = createFloodWindowObserver({
84936
85809
  clock: { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) },
84937
85810
  log: (line) => process.stderr.write(line),
85811
+ tz: process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC",
84938
85812
  stats: () => sendGate.stats(),
84939
85813
  readWindows: (now) => readFloodWindows(FLOOD_WINDOWS_PATH, now),
84940
85814
  markAlerted: (scopeKey, alertedAt) => markFloodWindowAlerted(FLOOD_WINDOWS_PATH, scopeKey, alertedAt, Date.now()),
@@ -84949,7 +85823,7 @@ var floodWindowObserver = createFloodWindowObserver({
84949
85823
  await robustApiCall(() => bot.api.sendRichMessage(operator, richMessage2(text5), {}), { chat_id: String(operator), verb: "send-gate-flood-alert", priorityClass: "critical" });
84950
85824
  }
84951
85825
  });
84952
- if (sendGateEnabledFromEnv()) {
85826
+ if (sendGateConfig.enabled) {
84953
85827
  const observeTimer = setInterval(() => {
84954
85828
  try {
84955
85829
  sendGateStatsLogger.tick();
@@ -85419,7 +86293,8 @@ function isAutoFallbackCooldownActive(_agentName, now) {
85419
86293
  async function editCardExpired(chatId, messageId, body) {
85420
86294
  if (messageId == null)
85421
86295
  return;
85422
- await lockedBot.api.editMessageText(chatId, messageId, richMessage2(body), { reply_markup: { inline_keyboard: [] } }).catch(() => {});
86296
+ const text5 = richMessage2(body);
86297
+ await robustApiCall(() => lockedBot.api.editMessageText(chatId, messageId, text5, { reply_markup: { inline_keyboard: [] } }), { chat_id: chatId, verb: "card-expired.strip", priorityClass: "cosmetic", messageId, editPayload: body }).catch(() => {});
85423
86298
  }
85424
86299
  function recordMissedApproval(opts) {
85425
86300
  if (!MISSED_APPROVAL_REOFFER_ENABLED)
@@ -85646,10 +86521,7 @@ function postPermissionCard(requestId, pend) {
85646
86521
  if (live && sent && typeof sent.message_id === "number") {
85647
86522
  const landedThreadId = sent.message_thread_id ?? undefined;
85648
86523
  live.cards.push({ chatId, messageId: sent.message_id, threadId: landedThreadId });
85649
- if (live.undeliverable != null) {
85650
- live.undeliverable = null;
85651
- live.redeliveryFailures = 0;
85652
- live.startedAt = Date.now();
86524
+ if (applyDeliveredHoldReset(live, Date.now())) {
85653
86525
  reconcileBlockedApprovals();
85654
86526
  process.stderr.write(`telegram gateway: permission-card RE-DELIVERED request=${requestId} tool=${live.tool_name} chat=${chatId} \u2014 the operator can answer now; ` + `TTL clock restarted (they had zero seconds while the channel was shut)
85655
86527
  `);
@@ -85899,7 +86771,7 @@ function emitGatewayOperatorEvent(event) {
85899
86771
  let throttleEscalation = null;
85900
86772
  let escalationFired = false;
85901
86773
  let rateLimitedCooldownConsulted = false;
85902
- const rateLimit429Classification = kind === "rate-limited" ? classify429Detail(event.detail) : null;
86774
+ const rateLimit429Classification = kind === "rate-limited" ? classify429Detail2(event.detail) : null;
85903
86775
  if (rateLimit429Classification != null && rateLimit429Classification !== "account-scoped") {
85904
86776
  emitRuntimeMetric(build429ClassifiedMetric({
85905
86777
  agent,
@@ -85991,6 +86863,18 @@ function emitGatewayOperatorEvent(event) {
85991
86863
  const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime());
85992
86864
  fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt);
85993
86865
  }
86866
+ } else if (kind === "rate-limited" || kind === "unknown-5xx") {
86867
+ const parsed = parseLlmError(event.detail);
86868
+ const now = Date.now();
86869
+ if (decideErrorSurface(parsed, agent, { claim: true, now }) === "suppress") {
86870
+ process.stderr.write(`telegram gateway: operator-event collapsed (error-presence-gate) agent=${agent} kind=${kind}
86871
+ `);
86872
+ return;
86873
+ }
86874
+ const tz = process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC";
86875
+ const r = renderLlmError(parsed, agent, tz, new Date(now));
86876
+ renderedText = r.text;
86877
+ renderedKeyboard = r.keyboard;
85994
86878
  } else {
85995
86879
  try {
85996
86880
  const r = renderOperatorEvent(event);
@@ -86287,6 +87171,10 @@ async function runMidSessionCardReaper() {
86287
87171
  const reaps = decideWorkerPinReaps({
86288
87172
  pins: candidates,
86289
87173
  statusOf: (agentId) => {
87174
+ if (agentId.startsWith("group:")) {
87175
+ const feedKey = agentId.slice("group:".length);
87176
+ return workerActivityFeed?.hasRunningInFeed(feedKey) ? "running" : "terminal";
87177
+ }
86290
87178
  if (turnsDb == null)
86291
87179
  return "unknown";
86292
87180
  try {
@@ -86386,24 +87274,6 @@ async function reconcileStatusPinInner(pinKey, chatId, desired) {
86386
87274
  statusPinPinnedAt.set(pinKey, Date.now());
86387
87275
  }
86388
87276
  }
86389
- function reconcileWorkerPin(agentId, chatId, running) {
86390
- if (!PIN_STATUS_WHILE_WORKING)
86391
- return;
86392
- const key = `wk:${agentId}`;
86393
- if (!running) {
86394
- const unpinChat = chatId ?? statusPinChatIds.get(key);
86395
- if (unpinChat == null)
86396
- return;
86397
- reconcileStatusPin(key, unpinChat, { pinned: false });
86398
- return;
86399
- }
86400
- if (chatId == null)
86401
- return;
86402
- const messageId = workerActivityFeed?.messageIdOf(agentId) ?? null;
86403
- if (messageId == null)
86404
- return;
86405
- reconcileStatusPin(key, chatId, { pinned: true, messageId });
86406
- }
86407
87277
  async function unpinAllStatusPins() {
86408
87278
  const keys = [...statusPinState.keys()];
86409
87279
  for (const key of keys) {
@@ -87036,7 +87906,8 @@ var bridgeDeadWatchdog = createBridgeDeadWatchdog({
87036
87906
  markerPath: join54(STATE_DIR, "bridge-dead-escalation.json"),
87037
87907
  log: (line) => process.stderr.write(`${line}
87038
87908
  `),
87039
- priorStreak: bridgeDeadPriorStreak
87909
+ priorStreak: bridgeDeadPriorStreak,
87910
+ selfAgentName: process.env.SWITCHROOM_AGENT_NAME ?? ""
87040
87911
  });
87041
87912
  if (BRIDGE_DEAD_ESCALATION_ENABLED) {
87042
87913
  bridgeDeadWatchdog.arm();
@@ -93190,11 +94061,13 @@ function recordTypedModelSwitch(reply, requestedModelArg, _deps) {
93190
94061
  if (!reply.selectedModel)
93191
94062
  return "";
93192
94063
  sessionModelSource.setOverride(reply.selectedModel);
94064
+ clearPremiumRecoveryOnManualSwitch(reply.selectedModel);
93193
94065
  return "";
93194
94066
  }
93195
94067
  function recordModelMenuSideEffects(outcome, modelDeps, cbChatId, cbThreadId, prevSessionModel) {
93196
94068
  if (outcome.selectedModel) {
93197
94069
  sessionModelSource.setOverride(outcome.selectedModel);
94070
+ clearPremiumRecoveryOnManualSwitch(outcome.selectedModel);
93198
94071
  }
93199
94072
  if (outcome.clearedDefault) {
93200
94073
  const smDir = resolveAgentDirFromEnv();
@@ -94093,6 +94966,80 @@ function broadcastFleetFallbackFailure(triggerAgent, reason) {
94093
94966
  swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(html), {}), { chat_id, verb: "fleet-fallback:failure-notify" });
94094
94967
  }
94095
94968
  }
94969
+ function broadcastTierNotice(markdown) {
94970
+ const access = loadAccess();
94971
+ if (access.allowFrom.length === 0)
94972
+ return;
94973
+ for (const chat_id of access.allowFrom) {
94974
+ swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(markdown), { disable_notification: true }), { chat_id: String(chat_id), verb: "tier-downgrade:notify" });
94975
+ }
94976
+ }
94977
+ function maybeTierDowngrade(triggerAgent) {
94978
+ return runTierDowngrade(triggerAgent, {
94979
+ getAgentDir: () => resolveAgentDirFromEnv() ?? null,
94980
+ getConfiguredDefault: () => {
94981
+ const dir = resolveAgentDirFromEnv();
94982
+ if (!dir)
94983
+ return null;
94984
+ return resolveMainModel(readConfiguredDefaultModel(dir) ?? undefined);
94985
+ },
94986
+ getSessionOverride: () => sessionModelSource.getOverride(),
94987
+ resolve: (t) => resolveMainModel(t),
94988
+ peekResumeGate: () => fleetFallbackResumeGate.peek(newestActiveTurnStartedAtMs()),
94989
+ writeCarrier: (dir, toModel, cfg) => writeSessionModelFile(dir, toModel, cfg),
94990
+ armResumeGate: () => fleetFallbackResumeGate.arm(),
94991
+ writeRecoveryMarker: (dir, premiumModel) => {
94992
+ const chats = loadAccess().allowFrom.map((c) => String(c));
94993
+ if (chats.length > 0)
94994
+ writePremiumRecoveryFile(dir, premiumModel, chats);
94995
+ },
94996
+ broadcastNotice: (md) => broadcastTierNotice(md),
94997
+ selfRestart: (agent) => triggerSelfRestart(agent, "tier-downgrade-resume"),
94998
+ selfAgent: (t) => process.env.SWITCHROOM_AGENT_NAME ?? t,
94999
+ log: (msg) => process.stderr.write(`telegram gateway: ${msg}
95000
+ `)
95001
+ });
95002
+ }
95003
+ async function maybePremiumRecoveryPing(brokerClient, accounts) {
95004
+ return runPremiumRecoveryPing({
95005
+ getAgentDir: () => resolveAgentDirFromEnv() ?? null,
95006
+ readMarker: (dir) => readPremiumRecoveryFile(dir),
95007
+ clearMarker: (dir) => clearPremiumRecoveryFile(dir),
95008
+ getAgent: () => getMyAgentName(),
95009
+ decide: () => decidePremiumRecovery({
95010
+ hasMarker: true,
95011
+ accounts: accounts.map((a) => ({
95012
+ exhausted: a.exhausted,
95013
+ premiumWalled: a.premium_walled === true
95014
+ }))
95015
+ }).fire,
95016
+ claimNotification: (key) => claimQuotaNotification(brokerClient, key),
95017
+ fallbackChats: () => loadAccess().allowFrom.map((c) => String(c)),
95018
+ sendToChat: (chat_id, ping, keyboard) => {
95019
+ swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(ping.text), {
95020
+ disable_notification: true,
95021
+ reply_markup: keyboard
95022
+ }), { chat_id: String(chat_id), verb: "premium-recovery:notify" });
95023
+ },
95024
+ log: (msg) => process.stderr.write(`telegram gateway: ${msg}
95025
+ `)
95026
+ });
95027
+ }
95028
+ function clearPremiumRecoveryOnManualSwitch(appliedToken) {
95029
+ if (appliedToken == null || appliedToken.length === 0)
95030
+ return;
95031
+ const agentDir = resolveAgentDirFromEnv();
95032
+ if (!agentDir)
95033
+ return;
95034
+ const marker = readPremiumRecoveryFile(agentDir);
95035
+ if (marker == null)
95036
+ return;
95037
+ if (resolveMainModel(appliedToken) === resolveMainModel(marker.premiumModel)) {
95038
+ clearPremiumRecoveryFile(agentDir);
95039
+ process.stderr.write(`telegram gateway: [premium-recovery] marker cleared \u2014 user manually re-issued /model ${marker.premiumModel}
95040
+ `);
95041
+ }
95042
+ }
94096
95043
  async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, trigger) {
94097
95044
  try {
94098
95045
  const client3 = await getAuthBrokerClient2(triggerAgent);
@@ -94126,6 +95073,10 @@ async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, tri
94126
95073
  if (outcome.kind === "switched") {
94127
95074
  fallbackAllBlockedNoticeState = { lastSentAtMs: 0 };
94128
95075
  } else if (outcome.kind === "all-blocked") {
95076
+ const tier = maybeTierDowngrade(triggerAgent);
95077
+ if (tier === "downgraded" || tier === "restart-pending") {
95078
+ return false;
95079
+ }
94129
95080
  const verdict = evaluateAllBlockedNotice(fallbackAllBlockedNoticeState, Date.now());
94130
95081
  if (!verdict.send) {
94131
95082
  process.stderr.write(`telegram gateway: [fleet-fallback] all-blocked card suppressed (cooldown) agent=${triggerAgent}
@@ -94235,6 +95186,10 @@ async function runQuotaWatch(opts = {}) {
94235
95186
  if (!listStateData.accounts || listStateData.accounts.length === 0) {
94236
95187
  return;
94237
95188
  }
95189
+ await maybePremiumRecoveryPing(brokerClient, listStateData.accounts).catch((err) => {
95190
+ process.stderr.write(`telegram gateway: [premium-recovery] ping check failed (non-fatal): ${err?.message ?? err}
95191
+ `);
95192
+ });
94238
95193
  const snapshots = buildSnapshotsFromCachedState2(listStateData);
94239
95194
  let watchState = loadQuotaWatchState(stateDir);
94240
95195
  const now = Date.now();
@@ -97336,6 +98291,10 @@ var didOneTimeSetup = false;
97336
98291
  const watcherAgentDir = resolveAgentDirFromEnv();
97337
98292
  if (watcherAgentDir != null) {
97338
98293
  const workerFeedEnabled = isWorkerActivityFeedEnabled(process.env.SWITCHROOM_WORKER_ACTIVITY_FEED);
98294
+ const workerFeedMaxRows = (() => {
98295
+ const raw = Number(process.env.SWITCHROOM_TG_WORKER_FEED_MAX_ROWS);
98296
+ return Number.isInteger(raw) && raw > 0 ? raw : undefined;
98297
+ })();
97339
98298
  const foregroundNestingEnabled = process.env.SWITCHROOM_FOREGROUND_SUBAGENT_NESTING !== "0";
97340
98299
  const orphanStatusEnabled = isOrphanSubagentStatusEnabled(process.env.SWITCHROOM_ORPHAN_SUBAGENT_STATUS);
97341
98300
  workerActivityFeed?.stop();
@@ -97354,6 +98313,20 @@ var didOneTimeSetup = false;
97354
98313
  })
97355
98314
  },
97356
98315
  floodWaitRemainingMs: probeFloodWaitRemainingMs,
98316
+ maxRows: workerFeedMaxRows,
98317
+ reconcilePin: ({ feedKey, chatId, messageId }) => {
98318
+ if (!PIN_STATUS_WHILE_WORKING)
98319
+ return;
98320
+ const key = `wk:group:${feedKey}`;
98321
+ if (messageId != null) {
98322
+ reconcileStatusPin(key, chatId, { pinned: true, messageId });
98323
+ } else {
98324
+ const unpinChat = chatId || statusPinChatIds.get(key);
98325
+ if (unpinChat != null && unpinChat.length > 0) {
98326
+ reconcileStatusPin(key, unpinChat, { pinned: false });
98327
+ }
98328
+ }
98329
+ },
97357
98330
  log: (msg) => process.stderr.write(`telegram gateway: ${msg}
97358
98331
  `)
97359
98332
  });
@@ -97419,7 +98392,6 @@ var didOneTimeSetup = false;
97419
98392
  state: outcome === "failed" ? "failed" : "done",
97420
98393
  model: dispatch.feedModel ?? undefined
97421
98394
  });
97422
- reconcileWorkerPin(agentId, null, false);
97423
98395
  }
97424
98396
  return;
97425
98397
  }
@@ -97467,7 +98439,6 @@ var didOneTimeSetup = false;
97467
98439
  state: outcome === "failed" ? "failed" : "done",
97468
98440
  model: dispatch.feedModel ?? undefined
97469
98441
  });
97470
- reconcileWorkerPin(agentId, null, false);
97471
98442
  }
97472
98443
  return;
97473
98444
  }
@@ -97481,7 +98452,6 @@ var didOneTimeSetup = false;
97481
98452
  state: outcome === "failed" ? "failed" : "done",
97482
98453
  model: dispatch.feedModel ?? undefined
97483
98454
  });
97484
- reconcileWorkerPin(agentId, null, false);
97485
98455
  }
97486
98456
  const handbackOrigin = resolveSubagentOriginChat(agentId);
97487
98457
  const decision = decideSubagentHandback({
@@ -97555,7 +98525,7 @@ var didOneTimeSetup = false;
97555
98525
  elapsedMs,
97556
98526
  state: "running",
97557
98527
  model: feedModel
97558
- }, wk.threadId)?.then(() => reconcileWorkerPin(agentId, wkChat, true));
98528
+ }, wk.threadId);
97559
98529
  return;
97560
98530
  }
97561
98531
  if (surface !== "nest")
@@ -97614,7 +98584,7 @@ var didOneTimeSetup = false;
97614
98584
  elapsedMs,
97615
98585
  state: "running",
97616
98586
  model: feedModel
97617
- }, wk.threadId)?.then(() => reconcileWorkerPin(agentId, wkChat, true));
98587
+ }, wk.threadId);
97618
98588
  return;
97619
98589
  }
97620
98590
  const progressOrigin = resolveSubagentOriginChat(agentId);