switchroom 0.18.17 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1401 -431
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +362 -71
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -20509,6 +20509,19 @@ var init_schema = __esm(() => {
|
|
|
20509
20509
|
approval_timeout_minutes: exports_external.number().int().nonnegative().optional().describe("Operator approval-card lifetime (minutes) for the tool-use 'Allow " + "once' card and the vault grant decision wait. After this long with " + "no operator tap, the card auto-denies (a TIMEOUT, not a denial \u2014 the " + "agent is told not to retry). Default 60. hostd-gated verbs " + "(mcp__hostd__*) keep their own longer window; the hostd " + "config-propose card is not governed by this key."),
|
|
20510
20510
|
sub_agent_tick_interval_ms: exports_external.number().int().nonnegative().optional().describe("Heartbeat tick interval (ms) for sub-agent rendering. Forces a " + "re-render of the elapsed-time counter while sub-agents are running, " + "even during silent stretches between tool calls. Default 10000 (10 s). " + "Set to 0 to disable the elapsed-ticker path."),
|
|
20511
20511
|
edit_budget_threshold: exports_external.number().int().nonnegative().optional().describe("Telegram API edit budget per minute before the progress-card driver " + "falls back to a slower coalesce window. When a chat accumulates more " + "than this many card edits in the trailing 60 s, the driver switches " + "to a wider coalesce interval until the rate drops back. Default 18. " + "Increase if your gateway frequently bumps the Telegram edit-rate ceiling " + "with many parallel sub-agents; decrease for a more conservative buffer."),
|
|
20512
|
+
send_gate: exports_external.object({
|
|
20513
|
+
enabled: exports_external.boolean().optional().describe("Master switch for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts) \u2014 the token-bucket scheduler every " + "Bot API call transits so per-surface throttles can't add up past a " + "flood ceiling. ON by default. Precedence: the operator break-glass " + "env var SWITCHROOM_TELEGRAM_SEND_GATE (0/false/off/no) ALWAYS wins " + "when explicitly set; this key only decides when that env var is " + "unset. Omit to keep the gate on."),
|
|
20514
|
+
global_per_sec: exports_external.number().positive().optional().describe("Global bucket sustained rate (Bot API calls/sec across ALL chats). " + "Default 25 (headroom under Telegram's ~30/s). Must be > 0 \u2014 a zero " + "or negative rate would wedge all outbound sends. Omit to keep 25."),
|
|
20515
|
+
global_burst: exports_external.number().int().positive().describe("Global bucket burst capacity. Default 4; worst-case 1s window " + "admits global_burst + global_per_sec = 29 < 30, a real margin under " + "the ceiling. Must be an integer >= 1 (a 0 capacity never admits a " + "token and wedges sends). Omit to keep 4.").optional(),
|
|
20516
|
+
per_chat_per_sec: exports_external.number().positive().optional().describe("Per-chat sustained rate (calls/sec to a single chat). Default 1. " + "Must be > 0 (zero/negative wedges that chat). Omit to keep 1."),
|
|
20517
|
+
per_chat_burst: exports_external.number().int().positive().optional().describe("Per-chat burst capacity. Default 3. Must be an integer >= 1 " + "(0 wedges the chat's bucket). Omit to keep 3."),
|
|
20518
|
+
per_group_per_min: exports_external.number().positive().optional().describe("Per-group sustained rate (calls/min to a single group/supergroup). " + "Default 18 (headroom under Telegram's ~20/min group ceiling). Must " + "be > 0. Omit to keep 18."),
|
|
20519
|
+
per_group_burst: exports_external.number().int().positive().optional().describe("Per-group burst capacity. Default 2. Must be an integer >= 1 " + "(0 wedges the group's bucket). Omit to keep 2."),
|
|
20520
|
+
edit_floor_ms: exports_external.number().int().nonnegative().optional().describe("Minimum ms between successive edits of the SAME message_id " + "(last-write-wins coalescing enforces this floor). Default 1500 " + "(Telegram's ~1 edit/sec/message practical ceiling). 0 disables the " + "floor. Must be an integer >= 0. Omit to keep 1500.")
|
|
20521
|
+
}).optional().describe("Tunable rate limits for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts). Every key is optional and defaults to " + "the send gate's built-in value, so omitting the whole block reproduces " + "today's exact behaviour \u2014 this is pure operator tuning, no default is " + "changed. Cascades from defaults.channels.telegram.send_gate."),
|
|
20522
|
+
worker_feed: exports_external.object({
|
|
20523
|
+
max_rows: exports_external.number().int().positive().optional().describe("Max live-worker rows rendered in the COMBINED worker-activity feed " + "(2+ background workers in one chat/thread coalesce into ONE message; " + "telegram-plugin/worker-activity-feed.ts) before a compact " + "'+M more working\u2026' spill line \u2014 keeps the coalesced body compact and " + "legible (and under the rich-message wire ceiling). Default 8. A " + "single-worker chat renders the full \uD83D\uDEE0 Worker card and ignores this. " + "Must be an integer >= 1. Omit to keep 8.")
|
|
20524
|
+
}).optional().describe("Tuning for the coalesced worker-activity feed " + "(telegram-plugin/worker-activity-feed.ts). Cascades from " + "defaults.channels.telegram.worker_feed."),
|
|
20512
20525
|
stickers: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Sticker aliases for the `send_sticker` MCP tool (#576). Maps a " + "short alias name (e.g. 'happy', 'thinking') to a Telegram file_id. " + "Operator-curated \u2014 capture file_ids from inbound stickers the user " + "sends and add them here. The agent calls send_sticker(chat_id, " + "alias='happy') and the gateway resolves to the file_id at send " + "time. Aliases enable persona-flavored expressiveness without " + "exposing raw file_ids in the agent prompt. Personal-assistant / " + "health-coach personas benefit; coding agents typically don't " + "configure any."),
|
|
20513
20526
|
voice_in: exports_external.object({
|
|
20514
20527
|
enabled: exports_external.boolean().optional().describe("Master switch for voice-message transcription."),
|
|
@@ -40238,9 +40251,45 @@ function renderActivityFeedWithNested(lines, childLines, final = false, liveSuff
|
|
|
40238
40251
|
stepCount
|
|
40239
40252
|
});
|
|
40240
40253
|
}
|
|
40254
|
+
var COMBINED_ROW_DESC_MAX = 72;
|
|
40255
|
+
function renderCombinedWorkerFeed(rows, opts) {
|
|
40256
|
+
if (rows.length === 0)
|
|
40257
|
+
return null;
|
|
40258
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows));
|
|
40259
|
+
const rowLines = (r) => {
|
|
40260
|
+
const desc = escapeMarkdown(truncate(stripMarkdown(r.description).replace(/\s+/g, " ").trim() || "background task", COMBINED_ROW_DESC_MAX));
|
|
40261
|
+
const toolWord = r.toolCount === 1 ? "tool" : "tools";
|
|
40262
|
+
const modelLabel = formatModelLabel(r.model);
|
|
40263
|
+
const modelPart = modelLabel != null ? ` \u00b7 ${escapeMarkdown(modelLabel)}` : "";
|
|
40264
|
+
const header = `**${desc}** _\u00b7 ${formatFeedElapsed(r.elapsedMs)} \u00b7 ${r.toolCount} ${toolWord}${modelPart}_`;
|
|
40265
|
+
const stepClean = stripMarkdown(r.currentStep).replace(/\s+/g, " ").trim();
|
|
40266
|
+
const step = stepClean.length > 0 ? `\u2192 _${escapeMarkdown(truncate(stepClean, STATUS_LINE_MAX))}_` : `\u2192 _starting\u2026_`;
|
|
40267
|
+
return [header, step];
|
|
40268
|
+
};
|
|
40269
|
+
const compose = (visibleCount) => {
|
|
40270
|
+
const shown = rows.slice(0, visibleCount);
|
|
40271
|
+
const hidden = rows.length - shown.length;
|
|
40272
|
+
const out = [`\uD83D\uDEE0 **Workers** \u00b7 _${rows.length} running_`];
|
|
40273
|
+
for (const r of shown) {
|
|
40274
|
+
const [h, s] = rowLines(r);
|
|
40275
|
+
out.push(h, s);
|
|
40276
|
+
}
|
|
40277
|
+
if (hidden > 0)
|
|
40278
|
+
out.push(`_+${hidden} more working\u2026_`);
|
|
40279
|
+
return stackCardLines(out);
|
|
40280
|
+
};
|
|
40281
|
+
let visible = Math.min(rows.length, maxRows);
|
|
40282
|
+
let body = compose(visible);
|
|
40283
|
+
while (body.length > STATUS_CARD_CHAR_BUDGET && visible > 1) {
|
|
40284
|
+
visible -= 1;
|
|
40285
|
+
body = compose(visible);
|
|
40286
|
+
}
|
|
40287
|
+
return body;
|
|
40288
|
+
}
|
|
40241
40289
|
|
|
40242
40290
|
// retry-api-call.ts
|
|
40243
40291
|
var import_grammy = __toESM(require_mod2(), 1);
|
|
40292
|
+
var LOCAL_RESOURCE_EXHAUSTED = "LOCAL_RESOURCE_EXHAUSTED";
|
|
40244
40293
|
var FLOOD_WAIT_ACTIVE = "FLOOD_WAIT_ACTIVE";
|
|
40245
40294
|
function makeFloodWaitActiveError(retryAfterSec, untilTs, original) {
|
|
40246
40295
|
return Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
|
|
@@ -40371,13 +40420,16 @@ function createWorkerActivityFeed(opts) {
|
|
|
40371
40420
|
const minEditInterval = opts.minEditIntervalMs ?? 2500;
|
|
40372
40421
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000;
|
|
40373
40422
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000;
|
|
40423
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8));
|
|
40424
|
+
const reconcilePinFn = opts.reconcilePin ?? (() => {});
|
|
40374
40425
|
const setIntervalFn = opts.setInterval ?? ((cb, ms) => {
|
|
40375
40426
|
const t = setInterval(cb, ms);
|
|
40376
40427
|
t.unref?.();
|
|
40377
40428
|
return t;
|
|
40378
40429
|
});
|
|
40379
40430
|
const clearIntervalFn = opts.clearInterval ?? ((handle) => clearInterval(handle));
|
|
40380
|
-
const
|
|
40431
|
+
const groups = new Map;
|
|
40432
|
+
const agentIndex = new Map;
|
|
40381
40433
|
const finalized = new Set;
|
|
40382
40434
|
const FINALIZED_CAP = 256;
|
|
40383
40435
|
function markFinalized(agentId) {
|
|
@@ -40391,267 +40443,356 @@ function createWorkerActivityFeed(opts) {
|
|
|
40391
40443
|
}
|
|
40392
40444
|
}
|
|
40393
40445
|
let heartbeatTimer = null;
|
|
40394
|
-
function
|
|
40446
|
+
function feedKeyOf(chatId, threadId) {
|
|
40447
|
+
return `${chatId} ${threadId ?? ""}`;
|
|
40448
|
+
}
|
|
40449
|
+
function groupOfAgent(agentId) {
|
|
40450
|
+
const key = agentIndex.get(agentId);
|
|
40451
|
+
return key != null ? groups.get(key) : undefined;
|
|
40452
|
+
}
|
|
40453
|
+
function sendOptsFor(g) {
|
|
40395
40454
|
return {
|
|
40396
40455
|
disable_web_page_preview: true,
|
|
40397
40456
|
disable_notification: true,
|
|
40398
|
-
...
|
|
40457
|
+
...g.threadId != null ? { message_thread_id: g.threadId } : {}
|
|
40399
40458
|
};
|
|
40400
40459
|
}
|
|
40401
|
-
function noteRateLimited(
|
|
40460
|
+
function noteRateLimited(g, err, label) {
|
|
40402
40461
|
const retryAfter = extractRetryAfterSecs(err);
|
|
40403
40462
|
if (retryAfter == null)
|
|
40404
40463
|
return;
|
|
40405
|
-
|
|
40464
|
+
g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS;
|
|
40406
40465
|
log(`worker-feed: ${label} 429 \u2014 backing off ${retryAfter}s`);
|
|
40407
40466
|
}
|
|
40408
|
-
function parkIfFloodWindowOpen(
|
|
40467
|
+
function parkIfFloodWindowOpen(g) {
|
|
40409
40468
|
const remaining = floodWaitRemainingMs();
|
|
40410
40469
|
if (remaining <= 0)
|
|
40411
40470
|
return false;
|
|
40412
40471
|
const until = nowFn() + remaining + COOLDOWN_JITTER_MS;
|
|
40413
|
-
if (until >
|
|
40414
|
-
|
|
40472
|
+
if (until > g.cooldownUntil)
|
|
40473
|
+
g.cooldownUntil = until;
|
|
40415
40474
|
return true;
|
|
40416
40475
|
}
|
|
40417
|
-
function accumulateNarrative(
|
|
40476
|
+
function accumulateNarrative(row, view) {
|
|
40418
40477
|
const line = view.latestSummary.trim();
|
|
40419
40478
|
if (line.length === 0)
|
|
40420
40479
|
return;
|
|
40421
|
-
if (
|
|
40480
|
+
if (row.narrative.includes(line))
|
|
40422
40481
|
return;
|
|
40423
|
-
|
|
40424
|
-
|
|
40425
|
-
if (
|
|
40426
|
-
|
|
40427
|
-
}
|
|
40428
|
-
}
|
|
40429
|
-
|
|
40430
|
-
|
|
40431
|
-
|
|
40432
|
-
|
|
40433
|
-
|
|
40434
|
-
|
|
40435
|
-
|
|
40482
|
+
row.narrative.push(line);
|
|
40483
|
+
row.stepStartedAtMs = nowFn();
|
|
40484
|
+
if (row.narrative.length > STATUS_ROLLING_LINES) {
|
|
40485
|
+
row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES);
|
|
40486
|
+
}
|
|
40487
|
+
}
|
|
40488
|
+
function liveElapsed(row, now) {
|
|
40489
|
+
const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0;
|
|
40490
|
+
return Math.max(base, row.lastView?.elapsedMs ?? 0);
|
|
40491
|
+
}
|
|
40492
|
+
function runningRows(g) {
|
|
40493
|
+
return [...g.workers.values()].filter((w) => w.state === "running" && w.lastView != null).sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0));
|
|
40494
|
+
}
|
|
40495
|
+
function renderGroupBody(g, now, terminalRecap, heartbeat) {
|
|
40496
|
+
const running = runningRows(g);
|
|
40497
|
+
if (running.length === 0) {
|
|
40498
|
+
if (terminalRecap == null)
|
|
40499
|
+
return null;
|
|
40500
|
+
return renderWorkerActivity(terminalRecap);
|
|
40501
|
+
}
|
|
40502
|
+
const elapsedFor = (r) => heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now);
|
|
40503
|
+
if (running.length === 1) {
|
|
40504
|
+
const r = running[0];
|
|
40505
|
+
const view = {
|
|
40506
|
+
...r.lastView,
|
|
40507
|
+
elapsedMs: elapsedFor(r),
|
|
40508
|
+
narrativeLines: [...r.narrative]
|
|
40509
|
+
};
|
|
40510
|
+
let liveSuffix = "";
|
|
40511
|
+
if (heartbeat) {
|
|
40512
|
+
const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now);
|
|
40513
|
+
liveSuffix = formatStepSuffix(stepElapsed);
|
|
40514
|
+
}
|
|
40515
|
+
return renderWorkerActivity(view, liveSuffix);
|
|
40516
|
+
}
|
|
40517
|
+
const rows = running.map((r) => {
|
|
40518
|
+
const v = r.lastView;
|
|
40519
|
+
const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary;
|
|
40520
|
+
return {
|
|
40521
|
+
description: v.description,
|
|
40522
|
+
elapsedMs: elapsedFor(r),
|
|
40523
|
+
toolCount: v.toolCount,
|
|
40524
|
+
currentStep,
|
|
40525
|
+
model: v.model
|
|
40526
|
+
};
|
|
40527
|
+
});
|
|
40528
|
+
return renderCombinedWorkerFeed(rows, { maxRows });
|
|
40529
|
+
}
|
|
40530
|
+
function removeWorker(g, agentId) {
|
|
40531
|
+
g.workers.delete(agentId);
|
|
40532
|
+
agentIndex.delete(agentId);
|
|
40533
|
+
if (g.workers.size === 0)
|
|
40534
|
+
groups.delete(g.feedKey);
|
|
40535
|
+
}
|
|
40536
|
+
function syncPin(g) {
|
|
40537
|
+
const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null;
|
|
40538
|
+
reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId });
|
|
40539
|
+
}
|
|
40540
|
+
async function doRender(g, opts2 = {}) {
|
|
40541
|
+
const now = nowFn();
|
|
40542
|
+
const isTerminal = opts2.terminalRecap != null;
|
|
40543
|
+
const settleTerminal = () => {
|
|
40544
|
+
g.pendingFinalize = null;
|
|
40545
|
+
if (opts2.finishingAgentId != null)
|
|
40546
|
+
removeWorker(g, opts2.finishingAgentId);
|
|
40547
|
+
syncPin(g);
|
|
40548
|
+
};
|
|
40549
|
+
if (now < g.cooldownUntil) {
|
|
40550
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40551
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40552
|
+
return;
|
|
40553
|
+
}
|
|
40554
|
+
if (parkIfFloodWindowOpen(g)) {
|
|
40555
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40556
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40436
40557
|
return;
|
|
40437
|
-
|
|
40558
|
+
}
|
|
40559
|
+
const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false);
|
|
40560
|
+
if (body == null) {
|
|
40561
|
+
if (isTerminal)
|
|
40562
|
+
settleTerminal();
|
|
40438
40563
|
return;
|
|
40439
|
-
|
|
40440
|
-
if (
|
|
40441
|
-
|
|
40564
|
+
}
|
|
40565
|
+
if (g.messageId == null) {
|
|
40566
|
+
const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)));
|
|
40567
|
+
if (isTerminal) {
|
|
40568
|
+
settleTerminal();
|
|
40569
|
+
return;
|
|
40570
|
+
}
|
|
40571
|
+
if (maxElapsed < firstPaintMin)
|
|
40442
40572
|
return;
|
|
40443
40573
|
try {
|
|
40444
|
-
const sent = await opts.bot.sendMessage(
|
|
40574
|
+
const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g));
|
|
40445
40575
|
if (sent == null || typeof sent.message_id !== "number") {
|
|
40446
|
-
parkIfFloodWindowOpen(
|
|
40447
|
-
log(`worker-feed: first paint shed by send gate
|
|
40576
|
+
parkIfFloodWindowOpen(g);
|
|
40577
|
+
log(`worker-feed: first paint shed by send gate feed=${g.feedKey} \u2014 not delivered`);
|
|
40448
40578
|
return;
|
|
40449
40579
|
}
|
|
40450
|
-
|
|
40451
|
-
|
|
40452
|
-
|
|
40453
|
-
|
|
40580
|
+
g.messageId = sent.message_id;
|
|
40581
|
+
g.lastBody = body;
|
|
40582
|
+
g.lastEditAt = now;
|
|
40583
|
+
syncPin(g);
|
|
40584
|
+
log(`worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
|
|
40454
40585
|
} catch (err) {
|
|
40455
|
-
noteRateLimited(
|
|
40586
|
+
noteRateLimited(g, err, "send");
|
|
40456
40587
|
log(`worker-feed: send failed: ${err.message}`);
|
|
40457
40588
|
}
|
|
40458
40589
|
return;
|
|
40459
40590
|
}
|
|
40460
|
-
if (body ===
|
|
40461
|
-
|
|
40462
|
-
|
|
40463
|
-
return;
|
|
40464
|
-
try {
|
|
40465
|
-
const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
40466
|
-
if (isSendGateShed(res)) {
|
|
40467
|
-
parkIfFloodWindowOpen(h);
|
|
40468
|
-
return;
|
|
40469
|
-
}
|
|
40470
|
-
h.lastBody = body;
|
|
40471
|
-
h.lastEditAt = nowFn();
|
|
40472
|
-
log(`worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} bytes=${body.length}`);
|
|
40473
|
-
} catch (err) {
|
|
40474
|
-
const outcome = classifyEditError(err);
|
|
40475
|
-
if (outcome === "rate_limited") {
|
|
40476
|
-
noteRateLimited(h, err, "edit");
|
|
40477
|
-
return;
|
|
40478
|
-
}
|
|
40479
|
-
if (outcome === "not_modified") {
|
|
40480
|
-
h.lastBody = body;
|
|
40481
|
-
h.lastEditAt = nowFn();
|
|
40482
|
-
return;
|
|
40483
|
-
}
|
|
40484
|
-
if (outcome === "gone") {
|
|
40485
|
-
h.messageId = null;
|
|
40486
|
-
h.lastBody = null;
|
|
40487
|
-
return;
|
|
40488
|
-
}
|
|
40489
|
-
log(`worker-feed: edit transient error agent=${h.agentId}: ${err.message}`);
|
|
40490
|
-
}
|
|
40491
|
-
}
|
|
40492
|
-
async function doFinish(h, view) {
|
|
40493
|
-
h.finished = true;
|
|
40494
|
-
markFinalized(h.agentId);
|
|
40495
|
-
if (h.messageId == null) {
|
|
40496
|
-
h.pendingFinish = null;
|
|
40497
|
-
return;
|
|
40498
|
-
}
|
|
40499
|
-
if (parkIfFloodWindowOpen(h) || nowFn() < h.cooldownUntil) {
|
|
40500
|
-
h.pendingFinish = view;
|
|
40591
|
+
if (body === g.lastBody) {
|
|
40592
|
+
if (isTerminal)
|
|
40593
|
+
settleTerminal();
|
|
40501
40594
|
return;
|
|
40502
40595
|
}
|
|
40503
|
-
|
|
40504
|
-
if (body === h.lastBody) {
|
|
40505
|
-
h.pendingFinish = null;
|
|
40596
|
+
if (!opts2.force && now - g.lastEditAt < minEditInterval)
|
|
40506
40597
|
return;
|
|
40507
|
-
}
|
|
40508
40598
|
try {
|
|
40509
|
-
const res = await opts.bot.editMessageText(
|
|
40599
|
+
const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g));
|
|
40510
40600
|
if (isSendGateShed(res)) {
|
|
40511
|
-
parkIfFloodWindowOpen(
|
|
40512
|
-
|
|
40601
|
+
parkIfFloodWindowOpen(g);
|
|
40602
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40603
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40513
40604
|
return;
|
|
40514
40605
|
}
|
|
40515
|
-
|
|
40516
|
-
|
|
40517
|
-
|
|
40518
|
-
|
|
40606
|
+
g.lastBody = body;
|
|
40607
|
+
g.lastEditAt = now;
|
|
40608
|
+
if (isTerminal) {
|
|
40609
|
+
log(`worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? "-"} ` + `msgId=${g.messageId} agent=${opts2.finishingAgentId ?? "-"} ` + `state=${opts2.terminalRecap?.state ?? "done"} bytes=${body.length}`);
|
|
40610
|
+
} else {
|
|
40611
|
+
log(`worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
|
|
40612
|
+
}
|
|
40613
|
+
if (isTerminal)
|
|
40614
|
+
settleTerminal();
|
|
40519
40615
|
} catch (err) {
|
|
40520
40616
|
const outcome = classifyEditError(err);
|
|
40521
40617
|
if (outcome === "rate_limited") {
|
|
40522
|
-
noteRateLimited(
|
|
40523
|
-
|
|
40618
|
+
noteRateLimited(g, err, isTerminal ? "finish" : "edit");
|
|
40619
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40620
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40524
40621
|
return;
|
|
40525
40622
|
}
|
|
40526
40623
|
if (outcome === "not_modified") {
|
|
40527
|
-
|
|
40528
|
-
|
|
40529
|
-
|
|
40624
|
+
g.lastBody = body;
|
|
40625
|
+
g.lastEditAt = now;
|
|
40626
|
+
if (isTerminal)
|
|
40627
|
+
settleTerminal();
|
|
40530
40628
|
return;
|
|
40531
40629
|
}
|
|
40532
40630
|
if (outcome === "gone") {
|
|
40533
|
-
|
|
40631
|
+
g.messageId = null;
|
|
40632
|
+
g.lastBody = null;
|
|
40633
|
+
if (isTerminal)
|
|
40634
|
+
settleTerminal();
|
|
40635
|
+
else
|
|
40636
|
+
syncPin(g);
|
|
40534
40637
|
return;
|
|
40535
40638
|
}
|
|
40536
|
-
|
|
40537
|
-
|
|
40639
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40640
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40641
|
+
log(`worker-feed: edit transient error feed=${g.feedKey}: ${err.message}`);
|
|
40538
40642
|
}
|
|
40539
40643
|
}
|
|
40540
40644
|
function heartbeatTick() {
|
|
40541
40645
|
const now = nowFn();
|
|
40542
|
-
for (const
|
|
40543
|
-
if (
|
|
40544
|
-
|
|
40545
|
-
|
|
40546
|
-
|
|
40547
|
-
|
|
40548
|
-
log(`worker-feed: heartbeat finalize re-drive error ${h.agentId}: ${err.message}`);
|
|
40549
|
-
}).finally(() => {
|
|
40550
|
-
if (handles.get(h.agentId)?.pendingFinish == null) {
|
|
40551
|
-
handles.delete(h.agentId);
|
|
40552
|
-
}
|
|
40646
|
+
for (const g of [...groups.values()]) {
|
|
40647
|
+
if (g.pendingFinalize != null && now >= g.cooldownUntil) {
|
|
40648
|
+
const recap = g.pendingFinalize;
|
|
40649
|
+
const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId;
|
|
40650
|
+
g.chain = g.chain.then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId })).catch((err) => {
|
|
40651
|
+
log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${err.message}`);
|
|
40553
40652
|
});
|
|
40554
40653
|
continue;
|
|
40555
40654
|
}
|
|
40556
|
-
if (
|
|
40557
|
-
continue;
|
|
40558
|
-
if (h.lastView.state !== "running")
|
|
40655
|
+
if (now < g.cooldownUntil)
|
|
40559
40656
|
continue;
|
|
40560
|
-
|
|
40657
|
+
const running = runningRows(g);
|
|
40658
|
+
if (running.length === 0)
|
|
40561
40659
|
continue;
|
|
40562
|
-
|
|
40563
|
-
|
|
40564
|
-
if (
|
|
40660
|
+
if (g.messageId == null) {
|
|
40661
|
+
const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)));
|
|
40662
|
+
if (maxElapsed < firstPaintMin)
|
|
40565
40663
|
continue;
|
|
40566
|
-
|
|
40567
|
-
|
|
40568
|
-
log(`worker-feed: heartbeat first-paint chain error ${h.agentId}: ${err.message}`);
|
|
40664
|
+
g.chain = g.chain.then(() => doRender(g, {})).catch((err) => {
|
|
40665
|
+
log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${err.message}`);
|
|
40569
40666
|
});
|
|
40570
40667
|
continue;
|
|
40571
40668
|
}
|
|
40572
|
-
if (now -
|
|
40669
|
+
if (now - g.lastEditAt < minEditInterval)
|
|
40573
40670
|
continue;
|
|
40574
|
-
const stale = now -
|
|
40671
|
+
const stale = now - g.lastEditAt >= heartbeatTickMs;
|
|
40575
40672
|
if (!stale)
|
|
40576
40673
|
continue;
|
|
40577
|
-
|
|
40578
|
-
|
|
40579
|
-
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) };
|
|
40580
|
-
h.chain = h.chain.then(() => doUpdate(h, view, liveSuffix)).catch((err) => {
|
|
40581
|
-
log(`worker-feed: heartbeat chain error ${h.agentId}: ${err.message}`);
|
|
40674
|
+
g.chain = g.chain.then(() => doRender(g, { heartbeat: true })).catch((err) => {
|
|
40675
|
+
log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${err.message}`);
|
|
40582
40676
|
});
|
|
40583
40677
|
}
|
|
40584
40678
|
}
|
|
40585
40679
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs);
|
|
40586
40680
|
return {
|
|
40587
40681
|
has(agentId) {
|
|
40588
|
-
|
|
40682
|
+
const g = groupOfAgent(agentId);
|
|
40683
|
+
return g != null && g.messageId != null && g.workers.has(agentId);
|
|
40589
40684
|
},
|
|
40590
40685
|
messageIdOf(agentId) {
|
|
40591
|
-
return
|
|
40686
|
+
return groupOfAgent(agentId)?.messageId ?? null;
|
|
40687
|
+
},
|
|
40688
|
+
hasRunningInFeed(feedKey) {
|
|
40689
|
+
const g = groups.get(feedKey);
|
|
40690
|
+
return g != null && g.workers.size > 0;
|
|
40592
40691
|
},
|
|
40593
40692
|
get size() {
|
|
40594
|
-
|
|
40693
|
+
let n = 0;
|
|
40694
|
+
for (const g of groups.values())
|
|
40695
|
+
n += g.workers.size;
|
|
40696
|
+
return n;
|
|
40595
40697
|
},
|
|
40596
40698
|
update(agentId, chatId, view, threadId) {
|
|
40597
40699
|
if (chatId.length === 0)
|
|
40598
40700
|
return Promise.resolve();
|
|
40599
40701
|
if (finalized.has(agentId))
|
|
40600
40702
|
return Promise.resolve();
|
|
40601
|
-
const
|
|
40602
|
-
if (
|
|
40703
|
+
const existingRow = groupOfAgent(agentId)?.workers.get(agentId);
|
|
40704
|
+
if (existingRow?.finished === true)
|
|
40603
40705
|
return Promise.resolve();
|
|
40604
|
-
|
|
40605
|
-
|
|
40606
|
-
|
|
40607
|
-
|
|
40706
|
+
const feedKey = feedKeyOf(chatId, threadId);
|
|
40707
|
+
let g = groups.get(feedKey);
|
|
40708
|
+
if (g == null) {
|
|
40709
|
+
g = {
|
|
40710
|
+
feedKey,
|
|
40608
40711
|
chatId,
|
|
40609
40712
|
threadId,
|
|
40610
40713
|
messageId: null,
|
|
40611
40714
|
lastBody: null,
|
|
40612
40715
|
lastEditAt: 0,
|
|
40613
40716
|
cooldownUntil: 0,
|
|
40614
|
-
narrative: [],
|
|
40615
40717
|
chain: Promise.resolve(),
|
|
40718
|
+
workers: new Map,
|
|
40719
|
+
pendingFinalize: null
|
|
40720
|
+
};
|
|
40721
|
+
groups.set(feedKey, g);
|
|
40722
|
+
}
|
|
40723
|
+
let row = g.workers.get(agentId);
|
|
40724
|
+
if (row == null) {
|
|
40725
|
+
row = {
|
|
40726
|
+
agentId,
|
|
40727
|
+
narrative: [],
|
|
40616
40728
|
lastView: null,
|
|
40617
|
-
|
|
40618
|
-
stepStartedAtMs: null,
|
|
40729
|
+
state: "running",
|
|
40619
40730
|
finished: false,
|
|
40620
|
-
|
|
40731
|
+
dispatchAtMs: null,
|
|
40732
|
+
stepStartedAtMs: null
|
|
40621
40733
|
};
|
|
40622
|
-
|
|
40623
|
-
|
|
40624
|
-
|
|
40625
|
-
|
|
40734
|
+
g.workers.set(agentId, row);
|
|
40735
|
+
agentIndex.set(agentId, feedKey);
|
|
40736
|
+
}
|
|
40737
|
+
accumulateNarrative(row, view);
|
|
40738
|
+
row.state = "running";
|
|
40739
|
+
row.lastView = { ...view, narrativeLines: [...row.narrative] };
|
|
40740
|
+
if (row.dispatchAtMs == null)
|
|
40741
|
+
row.dispatchAtMs = nowFn() - view.elapsedMs;
|
|
40742
|
+
const group = g;
|
|
40743
|
+
group.chain = group.chain.then(() => doRender(group)).catch((err) => {
|
|
40626
40744
|
log(`worker-feed: update chain error ${agentId}: ${err.message}`);
|
|
40627
40745
|
});
|
|
40628
|
-
return
|
|
40746
|
+
return group.chain;
|
|
40629
40747
|
},
|
|
40630
40748
|
finish(agentId, view) {
|
|
40631
|
-
const
|
|
40632
|
-
|
|
40749
|
+
const g = groupOfAgent(agentId);
|
|
40750
|
+
const row = g?.workers.get(agentId);
|
|
40751
|
+
if (g == null || row == null) {
|
|
40752
|
+
markFinalized(agentId);
|
|
40633
40753
|
return Promise.resolve();
|
|
40634
|
-
|
|
40754
|
+
}
|
|
40755
|
+
row.finished = true;
|
|
40756
|
+
row.state = view.state === "failed" ? "failed" : "done";
|
|
40757
|
+
markFinalized(agentId);
|
|
40758
|
+
const group = g;
|
|
40759
|
+
group.chain = group.chain.then(() => {
|
|
40760
|
+
const others = runningRows(group).filter((w) => w.agentId !== agentId);
|
|
40761
|
+
if (others.length > 0) {
|
|
40762
|
+
removeWorker(group, agentId);
|
|
40763
|
+
syncPin(group);
|
|
40764
|
+
return doRender(group, { force: true });
|
|
40765
|
+
}
|
|
40766
|
+
const recap = { ...view, narrativeLines: [...row.narrative] };
|
|
40767
|
+
return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId });
|
|
40768
|
+
}).catch((err) => {
|
|
40635
40769
|
log(`worker-feed: finish chain error ${agentId}: ${err.message}`);
|
|
40636
|
-
}).finally(() => {
|
|
40637
|
-
if (handles.get(agentId)?.pendingFinish == null) {
|
|
40638
|
-
handles.delete(agentId);
|
|
40639
|
-
}
|
|
40640
40770
|
});
|
|
40641
|
-
return
|
|
40771
|
+
return group.chain;
|
|
40642
40772
|
},
|
|
40643
40773
|
drop(agentId) {
|
|
40644
40774
|
markFinalized(agentId);
|
|
40645
|
-
|
|
40775
|
+
const g = groupOfAgent(agentId);
|
|
40776
|
+
if (g == null)
|
|
40777
|
+
return;
|
|
40778
|
+
const hadMessage = g.messageId != null;
|
|
40779
|
+
removeWorker(g, agentId);
|
|
40780
|
+
if (hadMessage)
|
|
40781
|
+
syncPin(g);
|
|
40782
|
+
if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
|
|
40783
|
+
g.chain = g.chain.then(() => doRender(g, { force: true })).catch((err) => {
|
|
40784
|
+
log(`worker-feed: drop re-render error ${agentId}: ${err.message}`);
|
|
40785
|
+
});
|
|
40786
|
+
}
|
|
40646
40787
|
},
|
|
40647
40788
|
resurrect(agentId) {
|
|
40648
40789
|
const wasFinalized = finalized.delete(agentId);
|
|
40649
|
-
const
|
|
40650
|
-
if (
|
|
40651
|
-
|
|
40652
|
-
|
|
40790
|
+
const row = groupOfAgent(agentId)?.workers.get(agentId);
|
|
40791
|
+
if (row != null) {
|
|
40792
|
+
row.finished = false;
|
|
40793
|
+
row.state = "running";
|
|
40653
40794
|
}
|
|
40654
|
-
if (wasFinalized ||
|
|
40795
|
+
if (wasFinalized || row != null) {
|
|
40655
40796
|
log(`worker-feed: resurrect agent=${agentId} \u2014 cleared finalized gate; card will repaint on next running cue`);
|
|
40656
40797
|
}
|
|
40657
40798
|
},
|
|
@@ -44582,6 +44723,14 @@ function holdReasonFor(err) {
|
|
|
44582
44723
|
function isHeldUndeliverable2(pend) {
|
|
44583
44724
|
return pend.undeliverable != null;
|
|
44584
44725
|
}
|
|
44726
|
+
function applyDeliveredHoldReset(entry, now) {
|
|
44727
|
+
if (entry.undeliverable == null)
|
|
44728
|
+
return false;
|
|
44729
|
+
entry.undeliverable = null;
|
|
44730
|
+
entry.redeliveryFailures = 0;
|
|
44731
|
+
entry.startedAt = now;
|
|
44732
|
+
return true;
|
|
44733
|
+
}
|
|
44585
44734
|
var HELD_CARD_REDELIVERY_CAP = 3;
|
|
44586
44735
|
function selectHeldForRedelivery(entries, opts) {
|
|
44587
44736
|
const empty = { send: [], deferred: [] };
|
|
@@ -55213,7 +55362,7 @@ function isLocalResourceError(err) {
|
|
|
55213
55362
|
const msg = err instanceof Error ? err.message : String(err ?? "");
|
|
55214
55363
|
return /\b(ENOSPC|EDQUOT|EIO|ENOMEM)\b/.test(msg) || /no space left on device/i.test(msg) || /disk quota exceeded/i.test(msg);
|
|
55215
55364
|
}
|
|
55216
|
-
var
|
|
55365
|
+
var LOCAL_RESOURCE_EXHAUSTED2 = "LOCAL_RESOURCE_EXHAUSTED";
|
|
55217
55366
|
var GIVE_UP_MESSAGE = "retryApiCall: max retries exceeded";
|
|
55218
55367
|
var FLOOD_WAIT_ACTIVE2 = "FLOOD_WAIT_ACTIVE";
|
|
55219
55368
|
function makeFloodWaitActiveError2(retryAfterSec, untilTs, original) {
|
|
@@ -55269,13 +55418,13 @@ function createRetryApiCall2(config = {}) {
|
|
|
55269
55418
|
log?.(`telegram gateway: LOCAL resource exhaustion (${err.code ?? "disk/mem"}) \u2014 ` + `not retrying the send (would feed a flood ban); surfacing degraded state
|
|
55270
55419
|
`);
|
|
55271
55420
|
observer?.onGiveUp?.({ attempts: attempt + 1, error: err });
|
|
55272
|
-
throw Object.assign(new Error(
|
|
55421
|
+
throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED2), { original: err });
|
|
55273
55422
|
}
|
|
55274
55423
|
if (isGrammyErr && err.error_code === 429) {
|
|
55275
55424
|
const retryAfter = Number(err.parameters?.retry_after ?? 5);
|
|
55276
55425
|
const delayMs = retryAfter * 1000;
|
|
55277
55426
|
try {
|
|
55278
|
-
onFloodWait?.(retryAfter);
|
|
55427
|
+
onFloodWait?.(retryAfter, opts);
|
|
55279
55428
|
} catch {}
|
|
55280
55429
|
if (delayMs > maxFloodSleepMs) {
|
|
55281
55430
|
log?.(`telegram gateway: 429 flood ban of ${retryAfter}s exceeds the ` + `${Math.round(maxFloodSleepMs / 1000)}s in-process sleep ceiling \u2014 ` + `not sleeping it; surfacing degraded state
|
|
@@ -55444,16 +55593,25 @@ function stableStringify(value) {
|
|
|
55444
55593
|
});
|
|
55445
55594
|
}
|
|
55446
55595
|
var GROUP_TYPES2 = new Set(["group", "supergroup"]);
|
|
55596
|
+
var SEND_GATE_DEFAULTS = {
|
|
55597
|
+
globalPerSec: 25,
|
|
55598
|
+
globalBurst: 4,
|
|
55599
|
+
perChatPerSec: 1,
|
|
55600
|
+
perChatBurst: 3,
|
|
55601
|
+
perGroupPerMin: 18,
|
|
55602
|
+
perGroupBurst: 2,
|
|
55603
|
+
editFloorMs: 1500
|
|
55604
|
+
};
|
|
55447
55605
|
function createSendGate(config) {
|
|
55448
55606
|
const enabled2 = config.enabled;
|
|
55449
55607
|
const clock = config.clock ?? systemClock;
|
|
55450
|
-
const globalPerSec = config.globalPerSec ??
|
|
55451
|
-
const globalBurst = config.globalBurst ??
|
|
55452
|
-
const perChatPerSec = config.perChatPerSec ??
|
|
55453
|
-
const perChatBurst = config.perChatBurst ??
|
|
55454
|
-
const perGroupPerMin = config.perGroupPerMin ??
|
|
55455
|
-
const perGroupBurst = config.perGroupBurst ??
|
|
55456
|
-
const editFloorMs = config.editFloorMs ??
|
|
55608
|
+
const globalPerSec = config.globalPerSec ?? SEND_GATE_DEFAULTS.globalPerSec;
|
|
55609
|
+
const globalBurst = config.globalBurst ?? SEND_GATE_DEFAULTS.globalBurst;
|
|
55610
|
+
const perChatPerSec = config.perChatPerSec ?? SEND_GATE_DEFAULTS.perChatPerSec;
|
|
55611
|
+
const perChatBurst = config.perChatBurst ?? SEND_GATE_DEFAULTS.perChatBurst;
|
|
55612
|
+
const perGroupPerMin = config.perGroupPerMin ?? SEND_GATE_DEFAULTS.perGroupPerMin;
|
|
55613
|
+
const perGroupBurst = config.perGroupBurst ?? SEND_GATE_DEFAULTS.perGroupBurst;
|
|
55614
|
+
const editFloorMs = config.editFloorMs ?? SEND_GATE_DEFAULTS.editFloorMs;
|
|
55457
55615
|
const messageStateTtlMs = config.messageStateTtlMs ?? 60000;
|
|
55458
55616
|
const maxMessageStates = config.maxMessageStates ?? 5000;
|
|
55459
55617
|
const usefulTtlMs = config.usefulTtlMs ?? 120000;
|
|
@@ -55461,6 +55619,7 @@ function createSendGate(config) {
|
|
|
55461
55619
|
const criticalJitterMaxMs = config.criticalJitterMaxMs ?? 250;
|
|
55462
55620
|
const jitter = config.jitter ?? Math.random;
|
|
55463
55621
|
const onWindowOpen = config.onWindowOpen;
|
|
55622
|
+
const conservativeGlobalFloodScope = config.conservativeGlobalFloodScope ?? false;
|
|
55464
55623
|
const counters = {
|
|
55465
55624
|
sent: 0,
|
|
55466
55625
|
queued: 0,
|
|
@@ -55531,8 +55690,11 @@ function createSendGate(config) {
|
|
|
55531
55690
|
applyWindow(scopeKey, untilTs, true);
|
|
55532
55691
|
}
|
|
55533
55692
|
function openScopedWindowsForOpts(opts, untilTs) {
|
|
55534
|
-
|
|
55535
|
-
if (
|
|
55693
|
+
const hasChatScope = opts?.chat_id != null && opts.chat_id !== "";
|
|
55694
|
+
if (!hasChatScope || conservativeGlobalFloodScope) {
|
|
55695
|
+
applyWindow("global", untilTs, true);
|
|
55696
|
+
}
|
|
55697
|
+
if (hasChatScope) {
|
|
55536
55698
|
applyWindow(`chat:${opts.chat_id}`, untilTs, true);
|
|
55537
55699
|
if (opts.chatType && GROUP_TYPES2.has(opts.chatType)) {
|
|
55538
55700
|
applyWindow(`group:${opts.chat_id}`, untilTs, true);
|
|
@@ -55834,14 +55996,114 @@ function createSendGate(config) {
|
|
|
55834
55996
|
}
|
|
55835
55997
|
};
|
|
55836
55998
|
}
|
|
55837
|
-
|
|
55999
|
+
function openScopedFloodWindows(opts, untilTs) {
|
|
56000
|
+
if (!enabled2)
|
|
56001
|
+
return;
|
|
56002
|
+
openScopedWindowsForOpts(opts, untilTs);
|
|
56003
|
+
}
|
|
56004
|
+
return { gate, openFloodWindow, openScopedFloodWindows, stats };
|
|
55838
56005
|
}
|
|
55839
|
-
function
|
|
55840
|
-
const v = env.SWITCHROOM_TELEGRAM_SEND_GATE;
|
|
55841
|
-
if (v == null)
|
|
55842
|
-
return true;
|
|
56006
|
+
function isOffValue(v) {
|
|
55843
56007
|
const t = v.trim().toLowerCase();
|
|
55844
|
-
return
|
|
56008
|
+
return t === "0" || t === "false" || t === "off" || t === "no";
|
|
56009
|
+
}
|
|
56010
|
+
function parsePositiveNumber(raw) {
|
|
56011
|
+
if (raw == null || raw.trim() === "")
|
|
56012
|
+
return;
|
|
56013
|
+
const n = Number(raw);
|
|
56014
|
+
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
56015
|
+
}
|
|
56016
|
+
function parsePositiveInt(raw) {
|
|
56017
|
+
if (raw == null || raw.trim() === "")
|
|
56018
|
+
return;
|
|
56019
|
+
const n = Number(raw);
|
|
56020
|
+
return Number.isInteger(n) && n > 0 ? n : undefined;
|
|
56021
|
+
}
|
|
56022
|
+
function parseNonNegativeInt(raw) {
|
|
56023
|
+
if (raw == null || raw.trim() === "")
|
|
56024
|
+
return;
|
|
56025
|
+
const n = Number(raw);
|
|
56026
|
+
return Number.isInteger(n) && n >= 0 ? n : undefined;
|
|
56027
|
+
}
|
|
56028
|
+
function parseBoolFlag(raw) {
|
|
56029
|
+
if (raw == null || raw.trim() === "")
|
|
56030
|
+
return;
|
|
56031
|
+
const t = raw.trim().toLowerCase();
|
|
56032
|
+
if (t === "1" || t === "true" || t === "on" || t === "yes")
|
|
56033
|
+
return true;
|
|
56034
|
+
if (isOffValue(t))
|
|
56035
|
+
return false;
|
|
56036
|
+
return;
|
|
56037
|
+
}
|
|
56038
|
+
function sendGateConfigFromEnv(env = process.env) {
|
|
56039
|
+
const killSwitch = env.SWITCHROOM_TELEGRAM_SEND_GATE;
|
|
56040
|
+
let enabled2;
|
|
56041
|
+
if (killSwitch != null && killSwitch.trim() !== "") {
|
|
56042
|
+
enabled2 = !isOffValue(killSwitch);
|
|
56043
|
+
} else {
|
|
56044
|
+
const cfg = env.SWITCHROOM_TG_SEND_GATE_ENABLED;
|
|
56045
|
+
enabled2 = cfg != null && cfg.trim() !== "" ? !isOffValue(cfg) : true;
|
|
56046
|
+
}
|
|
56047
|
+
const out = { enabled: enabled2 };
|
|
56048
|
+
const globalPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_PER_SEC);
|
|
56049
|
+
if (globalPerSec !== undefined)
|
|
56050
|
+
out.globalPerSec = globalPerSec;
|
|
56051
|
+
const globalBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_BURST);
|
|
56052
|
+
if (globalBurst !== undefined)
|
|
56053
|
+
out.globalBurst = globalBurst;
|
|
56054
|
+
const perChatPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_PER_SEC);
|
|
56055
|
+
if (perChatPerSec !== undefined)
|
|
56056
|
+
out.perChatPerSec = perChatPerSec;
|
|
56057
|
+
const perChatBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_BURST);
|
|
56058
|
+
if (perChatBurst !== undefined)
|
|
56059
|
+
out.perChatBurst = perChatBurst;
|
|
56060
|
+
const perGroupPerMin = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_PER_MIN);
|
|
56061
|
+
if (perGroupPerMin !== undefined)
|
|
56062
|
+
out.perGroupPerMin = perGroupPerMin;
|
|
56063
|
+
const perGroupBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_BURST);
|
|
56064
|
+
if (perGroupBurst !== undefined)
|
|
56065
|
+
out.perGroupBurst = perGroupBurst;
|
|
56066
|
+
const editFloorMs = parseNonNegativeInt(env.SWITCHROOM_TG_SEND_GATE_EDIT_FLOOR_MS);
|
|
56067
|
+
if (editFloorMs !== undefined)
|
|
56068
|
+
out.editFloorMs = editFloorMs;
|
|
56069
|
+
const conservativeGlobal = parseBoolFlag(env.SWITCHROOM_TG_SEND_GATE_CONSERVATIVE_GLOBAL);
|
|
56070
|
+
if (conservativeGlobal !== undefined)
|
|
56071
|
+
out.conservativeGlobalFloodScope = conservativeGlobal;
|
|
56072
|
+
return out;
|
|
56073
|
+
}
|
|
56074
|
+
|
|
56075
|
+
// shared/local-time.ts
|
|
56076
|
+
function fmtLocalClock(ms, tz) {
|
|
56077
|
+
return new Intl.DateTimeFormat("en-US", {
|
|
56078
|
+
timeZone: tz,
|
|
56079
|
+
hour: "numeric",
|
|
56080
|
+
minute: "2-digit",
|
|
56081
|
+
hour12: true
|
|
56082
|
+
}).format(new Date(ms)).replace(/\s([AP])M$/, (_m, p) => `${p.toLowerCase()}m`);
|
|
56083
|
+
}
|
|
56084
|
+
function fmtLocalDate(ms, tz) {
|
|
56085
|
+
return new Intl.DateTimeFormat("en-GB", {
|
|
56086
|
+
timeZone: tz,
|
|
56087
|
+
day: "numeric",
|
|
56088
|
+
month: "short"
|
|
56089
|
+
}).format(new Date(ms));
|
|
56090
|
+
}
|
|
56091
|
+
function localDay(ms, tz) {
|
|
56092
|
+
return new Intl.DateTimeFormat("en-CA", {
|
|
56093
|
+
timeZone: tz,
|
|
56094
|
+
year: "numeric",
|
|
56095
|
+
month: "2-digit",
|
|
56096
|
+
day: "2-digit"
|
|
56097
|
+
}).format(new Date(ms));
|
|
56098
|
+
}
|
|
56099
|
+
function tzAbbrev(ms, tz) {
|
|
56100
|
+
const at = new Date(ms);
|
|
56101
|
+
for (const loc of ["en-US", "en-AU", "en-GB"]) {
|
|
56102
|
+
const v = new Intl.DateTimeFormat(loc, { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value;
|
|
56103
|
+
if (v && !/^(?:GMT|UTC)/i.test(v))
|
|
56104
|
+
return v;
|
|
56105
|
+
}
|
|
56106
|
+
return new Intl.DateTimeFormat("en-US", { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value ?? tz;
|
|
55845
56107
|
}
|
|
55846
56108
|
|
|
55847
56109
|
// send-gate-observability.ts
|
|
@@ -55878,8 +56140,18 @@ function createStatsLogger(config) {
|
|
|
55878
56140
|
function isAlertableScope(scopeKey) {
|
|
55879
56141
|
return scopeKey === "global" || scopeKey.startsWith("chat:") || scopeKey.startsWith("group:");
|
|
55880
56142
|
}
|
|
55881
|
-
function
|
|
55882
|
-
|
|
56143
|
+
function fmtLocalStamp(ms, tz, refMs) {
|
|
56144
|
+
const clock = fmtLocalClock(ms, tz);
|
|
56145
|
+
const sameDay = refMs != null && localDay(ms, tz) === localDay(refMs, tz);
|
|
56146
|
+
const body = sameDay ? clock : `${fmtLocalDate(ms, tz)} ${clock}`;
|
|
56147
|
+
return `${body} ${tzAbbrev(ms, tz)}`;
|
|
56148
|
+
}
|
|
56149
|
+
function fmtLocalRange(startMs, endMs, tz) {
|
|
56150
|
+
const sameDay = localDay(startMs, tz) === localDay(endMs, tz);
|
|
56151
|
+
if (sameDay) {
|
|
56152
|
+
return `${fmtLocalClock(startMs, tz)} to ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
|
|
56153
|
+
}
|
|
56154
|
+
return `${fmtLocalDate(startMs, tz)} ${fmtLocalClock(startMs, tz)} to ` + `${fmtLocalDate(endMs, tz)} ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
|
|
55883
56155
|
}
|
|
55884
56156
|
function fmtDur(ms) {
|
|
55885
56157
|
const s = Math.round(ms / 1000);
|
|
@@ -55891,6 +56163,7 @@ function fmtDur(ms) {
|
|
|
55891
56163
|
}
|
|
55892
56164
|
function createFloodWindowObserver(config) {
|
|
55893
56165
|
const alertThresholdMs = config.alertThresholdMs ?? 60000;
|
|
56166
|
+
const tz = config.tz ?? "UTC";
|
|
55894
56167
|
let lastSeen = new Map;
|
|
55895
56168
|
let firstTick = true;
|
|
55896
56169
|
const owedOnClose = new Set;
|
|
@@ -55911,14 +56184,14 @@ function createFloodWindowObserver(config) {
|
|
|
55911
56184
|
}
|
|
55912
56185
|
function openAlertText(w, now) {
|
|
55913
56186
|
const openFor = fmtDur(now - w.observedAt);
|
|
55914
|
-
return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${
|
|
56187
|
+
return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${fmtLocalStamp(w.untilTs, tz, now)}. ` + `Outbound to that scope is being suppressed.`;
|
|
55915
56188
|
}
|
|
55916
56189
|
function closeAlertText(recs) {
|
|
55917
56190
|
const observedAt = Math.min(...recs.map((r) => r.observedAt));
|
|
55918
56191
|
const untilTs = Math.max(...recs.map((r) => r.untilTs));
|
|
55919
56192
|
const scopes = recs.map((r) => `\`${r.scopeKey}\``).join(", ");
|
|
55920
56193
|
const plural = recs.length > 1 ? "s" : "";
|
|
55921
|
-
return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${
|
|
56194
|
+
return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${fmtLocalRange(observedAt, untilTs, tz)} ` + `(~${fmtDur(untilTs - observedAt)}). Some outbound messages during that ` + `window were suppressed.`;
|
|
55922
56195
|
}
|
|
55923
56196
|
async function tick() {
|
|
55924
56197
|
const s = config.stats();
|
|
@@ -63787,18 +64060,28 @@ function createFleetFallbackResumeGate(opts = {}) {
|
|
|
63787
64060
|
const maxAgeMs = opts.maxAgeMs ?? DEFAULT_RESUME_MAX_AGE_MS;
|
|
63788
64061
|
const singleFlightMs = opts.singleFlightMs ?? DEFAULT_RESUME_SINGLE_FLIGHT_MS;
|
|
63789
64062
|
let lastResumedAtMs = Number.NEGATIVE_INFINITY;
|
|
63790
|
-
function
|
|
64063
|
+
function peek(failedTurnStartedAtMs) {
|
|
63791
64064
|
const now = nowFn();
|
|
63792
64065
|
if (now - lastResumedAtMs < singleFlightMs)
|
|
63793
64066
|
return "skip-inflight";
|
|
63794
64067
|
if (failedTurnStartedAtMs != null && now - failedTurnStartedAtMs > maxAgeMs) {
|
|
63795
64068
|
return "skip-stale";
|
|
63796
64069
|
}
|
|
63797
|
-
lastResumedAtMs = now;
|
|
63798
64070
|
return "resume";
|
|
63799
64071
|
}
|
|
64072
|
+
function arm() {
|
|
64073
|
+
lastResumedAtMs = nowFn();
|
|
64074
|
+
}
|
|
64075
|
+
function decide(failedTurnStartedAtMs) {
|
|
64076
|
+
const verdict = peek(failedTurnStartedAtMs);
|
|
64077
|
+
if (verdict === "resume")
|
|
64078
|
+
arm();
|
|
64079
|
+
return verdict;
|
|
64080
|
+
}
|
|
63800
64081
|
return {
|
|
63801
64082
|
decide,
|
|
64083
|
+
peek,
|
|
64084
|
+
arm,
|
|
63802
64085
|
reset() {
|
|
63803
64086
|
lastResumedAtMs = Number.NEGATIVE_INFINITY;
|
|
63804
64087
|
},
|
|
@@ -64500,10 +64783,30 @@ function autoClassifyMidTurnInbound(i) {
|
|
|
64500
64783
|
return { decision: "queue", reason: "cross_topic", sameTopic: false };
|
|
64501
64784
|
return recent && i.msSinceLastAgentOutput <= i.topicSteerWindowMs ? { decision: "steer", reason: "same_topic_recent", sameTopic: true } : { decision: "queue", reason: "same_topic_stale", sameTopic: true };
|
|
64502
64785
|
}
|
|
64786
|
+
// raw-error-scrub.ts
|
|
64787
|
+
function stripRawErrorBytes(raw) {
|
|
64788
|
+
if (typeof raw !== "string" || raw.length === 0)
|
|
64789
|
+
return "";
|
|
64790
|
+
let s = raw;
|
|
64791
|
+
s = s.replace(/API Error:?\s*\d*\s*/gi, " ");
|
|
64792
|
+
s = s.replace(/\bb'[^']*'/g, " ");
|
|
64793
|
+
s = s.replace(/\bb"[^"]*"/g, " ");
|
|
64794
|
+
s = s.replace(/[\u00b7\-\s]*\{[\s\S]*?["']type["']\s*:\s*["']error["'][\s\S]*$/i, " ");
|
|
64795
|
+
s = s.replace(/^\s*\{[\s\S]*\}\s*$/g, " ");
|
|
64796
|
+
s = s.replace(/\s+/g, " ").replace(/[\u00b7:\-\s]+$/g, "").replace(/^[\u00b7:\-\s]+/g, "").trim();
|
|
64797
|
+
return s;
|
|
64798
|
+
}
|
|
64799
|
+
function extractRequestId(raw) {
|
|
64800
|
+
if (typeof raw !== "string" || raw.length === 0)
|
|
64801
|
+
return;
|
|
64802
|
+
const m = raw.match(/["']request[_-]?id["']\s*:\s*["']([A-Za-z0-9._-]+)["']/i) ?? raw.match(/\brequest[_-]?id[=:]\s*([A-Za-z0-9._-]+)/i);
|
|
64803
|
+
return m ? m[1] : undefined;
|
|
64804
|
+
}
|
|
64805
|
+
|
|
64503
64806
|
// operator-events.ts
|
|
64504
64807
|
function renderOperatorEvent(ev) {
|
|
64505
64808
|
const agent = escapeMarkdown(ev.agent);
|
|
64506
|
-
const detail = escapeMarkdown(ev.detail);
|
|
64809
|
+
const detail = escapeMarkdown(stripRawErrorBytes(ev.detail));
|
|
64507
64810
|
switch (ev.kind) {
|
|
64508
64811
|
case "credentials-expired":
|
|
64509
64812
|
return {
|
|
@@ -64720,6 +65023,13 @@ var transientUpstreamSignals = [
|
|
|
64720
65023
|
"would exceed your account\u2019s rate limit",
|
|
64721
65024
|
"would exceed your account's rate limit"
|
|
64722
65025
|
];
|
|
65026
|
+
function isTransientUpstreamSignal(text4) {
|
|
65027
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65028
|
+
return false;
|
|
65029
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65030
|
+
const lower = sample.toLowerCase();
|
|
65031
|
+
return transientUpstreamSignals.some((s) => lower.includes(s));
|
|
65032
|
+
}
|
|
64723
65033
|
var litellmProxyLocal429Signals = [
|
|
64724
65034
|
"deployment over user-defined ratelimit",
|
|
64725
65035
|
"model rate limit exceeded. tpm limit",
|
|
@@ -64740,6 +65050,56 @@ function isLitellmProxyLocal429(text4) {
|
|
|
64740
65050
|
return true;
|
|
64741
65051
|
return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
|
|
64742
65052
|
}
|
|
65053
|
+
function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
|
|
65054
|
+
const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
|
|
65055
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65056
|
+
return empty2;
|
|
65057
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65058
|
+
const lower = sample.toLowerCase();
|
|
65059
|
+
let limitType = null;
|
|
65060
|
+
let limit = null;
|
|
65061
|
+
const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
|
|
65062
|
+
if (eqLimit) {
|
|
65063
|
+
limitType = eqLimit[1];
|
|
65064
|
+
limit = Number(eqLimit[2]);
|
|
65065
|
+
}
|
|
65066
|
+
if (limitType == null) {
|
|
65067
|
+
const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
|
|
65068
|
+
if (v3Type)
|
|
65069
|
+
limitType = v3Type[1];
|
|
65070
|
+
}
|
|
65071
|
+
if (limit == null) {
|
|
65072
|
+
const v3Limit = lower.match(/current limit:\s*(\d+)/);
|
|
65073
|
+
if (v3Limit)
|
|
65074
|
+
limit = Number(v3Limit[1]);
|
|
65075
|
+
}
|
|
65076
|
+
let currentUsage = null;
|
|
65077
|
+
const usage = lower.match(/current usage=(\d+)/);
|
|
65078
|
+
if (usage)
|
|
65079
|
+
currentUsage = Number(usage[1]);
|
|
65080
|
+
let resetAtMs = null;
|
|
65081
|
+
const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
|
|
65082
|
+
if (resetsAt) {
|
|
65083
|
+
const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
|
|
65084
|
+
if (!Number.isNaN(d.getTime()))
|
|
65085
|
+
resetAtMs = d.getTime();
|
|
65086
|
+
}
|
|
65087
|
+
if (resetAtMs == null) {
|
|
65088
|
+
const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
|
|
65089
|
+
if (tryAgain) {
|
|
65090
|
+
const secs = Number(tryAgain[1]);
|
|
65091
|
+
if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
|
|
65092
|
+
resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
|
|
65093
|
+
}
|
|
65094
|
+
}
|
|
65095
|
+
}
|
|
65096
|
+
return {
|
|
65097
|
+
limitType,
|
|
65098
|
+
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
65099
|
+
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
65100
|
+
resetAtMs
|
|
65101
|
+
};
|
|
65102
|
+
}
|
|
64743
65103
|
function detectModelUnavailable(stderr) {
|
|
64744
65104
|
if (typeof stderr !== "string" || stderr.length === 0)
|
|
64745
65105
|
return null;
|
|
@@ -64933,54 +65293,358 @@ function parseRelativeDuration(s) {
|
|
|
64933
65293
|
}
|
|
64934
65294
|
return matched && total > 0 ? total : null;
|
|
64935
65295
|
}
|
|
64936
|
-
|
|
65296
|
+
|
|
65297
|
+
// throttle-tier.ts
|
|
65298
|
+
init_card_format();
|
|
65299
|
+
init_quota_check();
|
|
65300
|
+
var accountScopedThrottleSignals = [
|
|
65301
|
+
"would exceed your account's rate limit",
|
|
65302
|
+
"would exceed your account\u2019s rate limit",
|
|
65303
|
+
"not your account"
|
|
65304
|
+
];
|
|
65305
|
+
function isAccountScopedThrottle(text4) {
|
|
65306
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65307
|
+
return false;
|
|
65308
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65309
|
+
const lower = sample.toLowerCase();
|
|
65310
|
+
return accountScopedThrottleSignals.some((s) => lower.includes(s));
|
|
65311
|
+
}
|
|
65312
|
+
function classify429Detail(text4) {
|
|
65313
|
+
if (isAccountScopedThrottle(text4))
|
|
65314
|
+
return "account-scoped";
|
|
65315
|
+
if (isLitellmProxyLocal429(text4))
|
|
65316
|
+
return "litellm-local";
|
|
65317
|
+
return "generic-transient";
|
|
65318
|
+
}
|
|
65319
|
+
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
|
|
65320
|
+
var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
|
|
65321
|
+
function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS) {
|
|
65322
|
+
const last = prev.lastSentAtMsByAccount[account] ?? 0;
|
|
65323
|
+
if (now - last >= cooldownMs) {
|
|
65324
|
+
return {
|
|
65325
|
+
send: true,
|
|
65326
|
+
next: {
|
|
65327
|
+
lastSentAtMsByAccount: {
|
|
65328
|
+
...prev.lastSentAtMsByAccount,
|
|
65329
|
+
[account]: now
|
|
65330
|
+
}
|
|
65331
|
+
}
|
|
65332
|
+
};
|
|
65333
|
+
}
|
|
65334
|
+
return { send: false, next: prev };
|
|
65335
|
+
}
|
|
65336
|
+
function renderThrottleNotice(opts) {
|
|
64937
65337
|
const now = opts.now ?? new Date;
|
|
64938
|
-
const
|
|
64939
|
-
const
|
|
65338
|
+
const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
|
|
65339
|
+
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
64940
65340
|
const lines = [
|
|
64941
|
-
`\
|
|
64942
|
-
`
|
|
64943
|
-
|
|
65341
|
+
`\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
|
|
65342
|
+
`This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
|
|
65343
|
+
`_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
|
|
64944
65344
|
];
|
|
64945
|
-
if (opts.autoFallbackInFlight) {
|
|
64946
|
-
lines.push("_Auto-failover in progress \u2014 see the announcement below._");
|
|
64947
|
-
} else {
|
|
64948
|
-
lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
|
|
64949
|
-
}
|
|
64950
65345
|
return lines.join(`
|
|
64951
65346
|
`);
|
|
64952
65347
|
}
|
|
64953
|
-
function
|
|
64954
|
-
const
|
|
64955
|
-
|
|
64956
|
-
|
|
64957
|
-
|
|
64958
|
-
|
|
64959
|
-
|
|
64960
|
-
|
|
64961
|
-
|
|
64962
|
-
|
|
64963
|
-
|
|
65348
|
+
function renderThrottleEscalationNotice(opts) {
|
|
65349
|
+
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65350
|
+
const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
|
|
65351
|
+
const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
|
|
65352
|
+
return `${head}
|
|
65353
|
+
${tail}`;
|
|
65354
|
+
}
|
|
65355
|
+
// operator-events.ts
|
|
65356
|
+
function classifyClaudeError(raw) {
|
|
65357
|
+
try {
|
|
65358
|
+
return classifyInner(raw);
|
|
65359
|
+
} catch {
|
|
65360
|
+
return "unknown-4xx";
|
|
64964
65361
|
}
|
|
64965
65362
|
}
|
|
64966
|
-
function
|
|
64967
|
-
|
|
64968
|
-
|
|
64969
|
-
|
|
65363
|
+
function classifyInner(raw) {
|
|
65364
|
+
if (raw == null)
|
|
65365
|
+
return "unknown-4xx";
|
|
65366
|
+
const obj = typeof raw === "object" ? raw : {};
|
|
65367
|
+
const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
|
|
65368
|
+
const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
|
|
65369
|
+
const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
|
|
65370
|
+
const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
|
|
65371
|
+
const sdkCode = extractString(obj, "error_code") ?? "";
|
|
65372
|
+
if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
|
|
65373
|
+
const msg = message.toLowerCase();
|
|
65374
|
+
if (msg.includes("expired") || msg.includes("refresh")) {
|
|
65375
|
+
return "credentials-expired";
|
|
65376
|
+
}
|
|
65377
|
+
return "credentials-invalid";
|
|
64970
65378
|
}
|
|
64971
|
-
if (
|
|
64972
|
-
|
|
64973
|
-
return detected?.kind === "quota_exhausted" ? detected : null;
|
|
65379
|
+
if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
|
|
65380
|
+
return "credentials-invalid";
|
|
64974
65381
|
}
|
|
64975
|
-
if (
|
|
64976
|
-
return
|
|
65382
|
+
if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
|
|
65383
|
+
return "credit-exhausted";
|
|
64977
65384
|
}
|
|
64978
|
-
|
|
65385
|
+
if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
|
|
65386
|
+
return "rate-limited";
|
|
65387
|
+
}
|
|
65388
|
+
if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
|
|
65389
|
+
return "rate-limited";
|
|
65390
|
+
}
|
|
65391
|
+
if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
|
|
65392
|
+
return "agent-crashed";
|
|
65393
|
+
}
|
|
65394
|
+
if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
|
|
65395
|
+
return "agent-restarted-unexpectedly";
|
|
65396
|
+
}
|
|
65397
|
+
if (status != null) {
|
|
65398
|
+
if (status >= 400 && status < 500)
|
|
65399
|
+
return "unknown-4xx";
|
|
65400
|
+
if (status >= 500 && status < 600)
|
|
65401
|
+
return "unknown-5xx";
|
|
65402
|
+
}
|
|
65403
|
+
return "unknown-4xx";
|
|
65404
|
+
}
|
|
65405
|
+
function extractString(obj, key) {
|
|
65406
|
+
const v = obj[key];
|
|
65407
|
+
return typeof v === "string" && v.length > 0 ? v : null;
|
|
65408
|
+
}
|
|
65409
|
+
function extractNumber(obj, key) {
|
|
65410
|
+
const v = obj[key];
|
|
65411
|
+
return typeof v === "number" ? v : null;
|
|
65412
|
+
}
|
|
65413
|
+
function getNestedObj(obj, key) {
|
|
65414
|
+
const v = obj[key];
|
|
65415
|
+
return typeof v === "object" && v != null ? v : {};
|
|
64979
65416
|
}
|
|
65417
|
+
var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
|
|
65418
|
+
var cooldownMap2 = new Map;
|
|
64980
65419
|
|
|
64981
|
-
//
|
|
64982
|
-
|
|
64983
|
-
|
|
65420
|
+
// llm-error-present.ts
|
|
65421
|
+
function extractModel(raw) {
|
|
65422
|
+
const m = raw.match(/["']?model["']?\s*[=:]\s*["']?((?:claude|sr)[A-Za-z0-9._-]+)/i);
|
|
65423
|
+
return m ? m[1] : undefined;
|
|
65424
|
+
}
|
|
65425
|
+
var TRANSIENT_KINDS = new Set([
|
|
65426
|
+
"rate_limit",
|
|
65427
|
+
"overload_529",
|
|
65428
|
+
"transient"
|
|
65429
|
+
]);
|
|
65430
|
+
function isActionableKind(kind) {
|
|
65431
|
+
return kind === "auth" || kind === "quota_wall";
|
|
65432
|
+
}
|
|
65433
|
+
function parseLlmError(raw, retryState) {
|
|
65434
|
+
const text4 = typeof raw === "string" ? raw : "";
|
|
65435
|
+
const requestId = extractRequestId(text4);
|
|
65436
|
+
const model = extractModel(text4);
|
|
65437
|
+
const resetAt = parseResetTime(text4);
|
|
65438
|
+
const retryAfterMs = resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined;
|
|
65439
|
+
const { kind, source } = classifyKindAndSource(text4);
|
|
65440
|
+
let autoRetrying = false;
|
|
65441
|
+
let terminal = true;
|
|
65442
|
+
if (TRANSIENT_KINDS.has(kind)) {
|
|
65443
|
+
const { retryAttempt, maxRetries } = retryState ?? { retryAttempt: null, maxRetries: null };
|
|
65444
|
+
if (retryAttempt != null && maxRetries != null) {
|
|
65445
|
+
autoRetrying = retryAttempt < maxRetries;
|
|
65446
|
+
terminal = retryAttempt >= maxRetries;
|
|
65447
|
+
} else {
|
|
65448
|
+
autoRetrying = false;
|
|
65449
|
+
terminal = true;
|
|
65450
|
+
}
|
|
65451
|
+
}
|
|
65452
|
+
return {
|
|
65453
|
+
kind,
|
|
65454
|
+
coreText: buildCoreText(kind, source),
|
|
65455
|
+
...resetAt != null ? { resetAt } : {},
|
|
65456
|
+
...retryAfterMs != null ? { retryAfterMs } : {},
|
|
65457
|
+
...model != null ? { model } : {},
|
|
65458
|
+
...requestId != null ? { requestId } : {},
|
|
65459
|
+
source,
|
|
65460
|
+
autoRetrying,
|
|
65461
|
+
terminal
|
|
65462
|
+
};
|
|
65463
|
+
}
|
|
65464
|
+
function classifyKindAndSource(text4) {
|
|
65465
|
+
const lower = text4.toLowerCase();
|
|
65466
|
+
const claudeKind = classifyClaudeError({ message: text4, type: text4 });
|
|
65467
|
+
if (claudeKind === "credentials-expired" || claudeKind === "credentials-invalid") {
|
|
65468
|
+
return { kind: "auth", source: "anthropic" };
|
|
65469
|
+
}
|
|
65470
|
+
if (isLitellmProxyLocal429(text4)) {
|
|
65471
|
+
return { kind: "rate_limit", source: "litellm-local" };
|
|
65472
|
+
}
|
|
65473
|
+
const mu = detectModelUnavailable(text4);
|
|
65474
|
+
if (mu != null) {
|
|
65475
|
+
if (mu.kind === "quota_exhausted")
|
|
65476
|
+
return { kind: "quota_wall", source: "anthropic" };
|
|
65477
|
+
if (mu.kind === "network")
|
|
65478
|
+
return { kind: "transient", source: "network" };
|
|
65479
|
+
if (lower.includes("529") || lower.includes("overloaded")) {
|
|
65480
|
+
return { kind: "overload_529", source: "anthropic" };
|
|
65481
|
+
}
|
|
65482
|
+
const c = classify429Detail(text4);
|
|
65483
|
+
return {
|
|
65484
|
+
kind: "rate_limit",
|
|
65485
|
+
source: c === "litellm-local" ? "litellm-local" : "anthropic"
|
|
65486
|
+
};
|
|
65487
|
+
}
|
|
65488
|
+
if (claudeKind === "credit-exhausted") {
|
|
65489
|
+
return { kind: "quota_wall", source: "anthropic" };
|
|
65490
|
+
}
|
|
65491
|
+
if (claudeKind === "rate-limited") {
|
|
65492
|
+
return { kind: "rate_limit", source: "anthropic" };
|
|
65493
|
+
}
|
|
65494
|
+
if (claudeKind === "unknown-5xx") {
|
|
65495
|
+
return { kind: "overload_529", source: "anthropic" };
|
|
65496
|
+
}
|
|
65497
|
+
return { kind: "unknown", source: "anthropic" };
|
|
65498
|
+
}
|
|
65499
|
+
function buildCoreText(kind, source) {
|
|
65500
|
+
switch (kind) {
|
|
65501
|
+
case "rate_limit":
|
|
65502
|
+
return source === "litellm-local" ? "Hit the local proxy rate limit \u2014 retrying automatically." : "Rate limited by Anthropic \u2014 retrying automatically.";
|
|
65503
|
+
case "overload_529":
|
|
65504
|
+
return "Anthropic is overloaded (529) \u2014 retrying automatically.";
|
|
65505
|
+
case "quota_wall":
|
|
65506
|
+
return "Usage limit reached on this Claude subscription.";
|
|
65507
|
+
case "auth":
|
|
65508
|
+
return "Claude login needs re-authentication.";
|
|
65509
|
+
case "transient":
|
|
65510
|
+
return source === "network" ? "Couldn't reach Anthropic (network) \u2014 retrying automatically." : "A temporary upstream hiccup \u2014 retrying automatically.";
|
|
65511
|
+
case "unknown":
|
|
65512
|
+
return "The model returned an error.";
|
|
65513
|
+
}
|
|
65514
|
+
}
|
|
65515
|
+
function formatResetLocal(resetAt, tz, now = new Date) {
|
|
65516
|
+
if (resetAt == null)
|
|
65517
|
+
return "";
|
|
65518
|
+
const ms = resetAt.getTime();
|
|
65519
|
+
if (!Number.isFinite(ms))
|
|
65520
|
+
return "";
|
|
65521
|
+
const clock = fmtLocalClock(ms, tz);
|
|
65522
|
+
const abbrev = tzAbbrev(ms, tz);
|
|
65523
|
+
const rel = formatRelativeTail(ms - now.getTime());
|
|
65524
|
+
return rel ? `clears ~${clock} ${abbrev} (${rel})` : `clears ~${clock} ${abbrev}`;
|
|
65525
|
+
}
|
|
65526
|
+
function formatRelativeTail(deltaMs) {
|
|
65527
|
+
if (deltaMs <= 0)
|
|
65528
|
+
return "~now";
|
|
65529
|
+
const totalMin = Math.round(deltaMs / 60000);
|
|
65530
|
+
if (totalMin < 1)
|
|
65531
|
+
return "~in <1m";
|
|
65532
|
+
if (totalMin < 60)
|
|
65533
|
+
return `~in ${totalMin}m`;
|
|
65534
|
+
const hours = Math.floor(totalMin / 60);
|
|
65535
|
+
const mins = totalMin % 60;
|
|
65536
|
+
if (hours < 24)
|
|
65537
|
+
return mins > 0 ? `~in ${hours}h ${mins}m` : `~in ${hours}h`;
|
|
65538
|
+
const days = Math.floor(hours / 24);
|
|
65539
|
+
const remH = hours % 24;
|
|
65540
|
+
return remH > 0 ? `~in ${days}d ${remH}h` : `~in ${days}d`;
|
|
65541
|
+
}
|
|
65542
|
+
function renderLlmError(parsed, agent, tz, now = new Date) {
|
|
65543
|
+
const safeAgent = escapeAgent(agent);
|
|
65544
|
+
const emoji = kindEmoji(parsed.kind);
|
|
65545
|
+
const lines = [`${emoji} ${parsed.coreText} (**${safeAgent}**)`];
|
|
65546
|
+
const resetLine = formatResetLocal(parsed.resetAt, tz, now);
|
|
65547
|
+
if (resetLine)
|
|
65548
|
+
lines.push(`_${resetLine}_`);
|
|
65549
|
+
if (parsed.model)
|
|
65550
|
+
lines.push(`_model: ${escapeAgent(parsed.model)}_`);
|
|
65551
|
+
const text4 = lines.join(`
|
|
65552
|
+
`);
|
|
65553
|
+
switch (parsed.kind) {
|
|
65554
|
+
case "auth":
|
|
65555
|
+
return {
|
|
65556
|
+
text: text4,
|
|
65557
|
+
keyboard: {
|
|
65558
|
+
inline_keyboard: [
|
|
65559
|
+
[
|
|
65560
|
+
{ text: "\uD83D\uDD10 Reauth now", callback_data: `op:reauth:${encodeURIComponent(agent)}` },
|
|
65561
|
+
{ text: "\u274c Dismiss", callback_data: `op:dismiss:${encodeURIComponent(agent)}` }
|
|
65562
|
+
]
|
|
65563
|
+
]
|
|
65564
|
+
}
|
|
65565
|
+
};
|
|
65566
|
+
case "quota_wall":
|
|
65567
|
+
return {
|
|
65568
|
+
text: text4,
|
|
65569
|
+
keyboard: {
|
|
65570
|
+
inline_keyboard: [
|
|
65571
|
+
[{ text: "\u23f3 Wait", callback_data: `op:dismiss:${encodeURIComponent(agent)}` }]
|
|
65572
|
+
]
|
|
65573
|
+
}
|
|
65574
|
+
};
|
|
65575
|
+
default:
|
|
65576
|
+
return { text: text4 };
|
|
65577
|
+
}
|
|
65578
|
+
}
|
|
65579
|
+
function kindEmoji(kind) {
|
|
65580
|
+
switch (kind) {
|
|
65581
|
+
case "rate_limit":
|
|
65582
|
+
return "\uD83D\uDEA6";
|
|
65583
|
+
case "overload_529":
|
|
65584
|
+
return "\uD83D\uDD25";
|
|
65585
|
+
case "quota_wall":
|
|
65586
|
+
return "\u26a0\ufe0f";
|
|
65587
|
+
case "auth":
|
|
65588
|
+
return "\uD83D\uDD11";
|
|
65589
|
+
case "transient":
|
|
65590
|
+
return "\uD83C\uDF10";
|
|
65591
|
+
case "unknown":
|
|
65592
|
+
return "\u26a0\ufe0f";
|
|
65593
|
+
}
|
|
65594
|
+
}
|
|
65595
|
+
function escapeAgent(s) {
|
|
65596
|
+
return s.replace(/([_*`\[\]])/g, "\\$1");
|
|
65597
|
+
}
|
|
65598
|
+
var ERROR_COLLAPSE_WINDOW_MS = 60000;
|
|
65599
|
+
|
|
65600
|
+
class ErrorPresenceGate {
|
|
65601
|
+
claims = new Map;
|
|
65602
|
+
keyFor(parsed, agent, now) {
|
|
65603
|
+
if (parsed.requestId)
|
|
65604
|
+
return `rid:${parsed.requestId}`;
|
|
65605
|
+
const bucket = Math.floor(now / ERROR_COLLAPSE_WINDOW_MS);
|
|
65606
|
+
return `${parsed.kind}:${agent}:${bucket}`;
|
|
65607
|
+
}
|
|
65608
|
+
claim(key, now = Date.now()) {
|
|
65609
|
+
this.prune(now);
|
|
65610
|
+
const existing = this.claims.get(key);
|
|
65611
|
+
if (existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS) {
|
|
65612
|
+
return false;
|
|
65613
|
+
}
|
|
65614
|
+
this.claims.set(key, now);
|
|
65615
|
+
return true;
|
|
65616
|
+
}
|
|
65617
|
+
isClaimed(key, now = Date.now()) {
|
|
65618
|
+
const existing = this.claims.get(key);
|
|
65619
|
+
return existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS;
|
|
65620
|
+
}
|
|
65621
|
+
prune(now) {
|
|
65622
|
+
for (const [k, at] of this.claims) {
|
|
65623
|
+
if (now - at >= ERROR_COLLAPSE_WINDOW_MS)
|
|
65624
|
+
this.claims.delete(k);
|
|
65625
|
+
}
|
|
65626
|
+
}
|
|
65627
|
+
reset() {
|
|
65628
|
+
this.claims.clear();
|
|
65629
|
+
}
|
|
65630
|
+
}
|
|
65631
|
+
var errorPresenceGate = new ErrorPresenceGate;
|
|
65632
|
+
function decideErrorSurface(parsed, agent, opts = { claim: true }) {
|
|
65633
|
+
const now = opts.now ?? Date.now();
|
|
65634
|
+
const gate = opts.gate ?? errorPresenceGate;
|
|
65635
|
+
if (isActionableKind(parsed.kind)) {
|
|
65636
|
+
if (opts.claim)
|
|
65637
|
+
gate.claim(gate.keyFor(parsed, agent, now), now);
|
|
65638
|
+
return "render";
|
|
65639
|
+
}
|
|
65640
|
+
if (parsed.autoRetrying && !parsed.terminal)
|
|
65641
|
+
return "suppress";
|
|
65642
|
+
const key = gate.keyFor(parsed, agent, now);
|
|
65643
|
+
if (opts.claim) {
|
|
65644
|
+
return gate.claim(key, now) ? "render" : "suppress";
|
|
65645
|
+
}
|
|
65646
|
+
return gate.isClaimed(key, now) ? "suppress" : "render";
|
|
65647
|
+
}
|
|
64984
65648
|
|
|
64985
65649
|
// model-unavailable.ts
|
|
64986
65650
|
init_quota_check();
|
|
@@ -64995,13 +65659,6 @@ var transientUpstreamSignals2 = [
|
|
|
64995
65659
|
"would exceed your account\u2019s rate limit",
|
|
64996
65660
|
"would exceed your account's rate limit"
|
|
64997
65661
|
];
|
|
64998
|
-
function isTransientUpstreamSignal(text4) {
|
|
64999
|
-
if (typeof text4 !== "string" || text4.length === 0)
|
|
65000
|
-
return false;
|
|
65001
|
-
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65002
|
-
const lower = sample.toLowerCase();
|
|
65003
|
-
return transientUpstreamSignals2.some((s) => lower.includes(s));
|
|
65004
|
-
}
|
|
65005
65662
|
var litellmProxyLocal429Signals2 = [
|
|
65006
65663
|
"deployment over user-defined ratelimit",
|
|
65007
65664
|
"model rate limit exceeded. tpm limit",
|
|
@@ -65022,55 +65679,76 @@ function isLitellmProxyLocal4292(text4) {
|
|
|
65022
65679
|
return true;
|
|
65023
65680
|
return litellmV3LimiterSignalPair2.every((s) => lower.includes(s));
|
|
65024
65681
|
}
|
|
65025
|
-
function
|
|
65026
|
-
|
|
65027
|
-
|
|
65028
|
-
|
|
65029
|
-
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65682
|
+
function detectModelUnavailable2(stderr) {
|
|
65683
|
+
if (typeof stderr !== "string" || stderr.length === 0)
|
|
65684
|
+
return null;
|
|
65685
|
+
const sample = stderr.length > 16384 ? stderr.slice(0, 16384) : stderr;
|
|
65030
65686
|
const lower = sample.toLowerCase();
|
|
65031
|
-
|
|
65032
|
-
|
|
65033
|
-
|
|
65034
|
-
if (eqLimit) {
|
|
65035
|
-
limitType = eqLimit[1];
|
|
65036
|
-
limit = Number(eqLimit[2]);
|
|
65687
|
+
if (transientUpstreamSignals2.some((s) => lower.includes(s))) {
|
|
65688
|
+
const resetAt = parseResetTime2(sample);
|
|
65689
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
65037
65690
|
}
|
|
65038
|
-
if (
|
|
65039
|
-
const
|
|
65040
|
-
|
|
65041
|
-
limitType = v3Type[1];
|
|
65691
|
+
if (isLitellmProxyLocal4292(sample)) {
|
|
65692
|
+
const resetAt = parseResetTime2(sample);
|
|
65693
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
65042
65694
|
}
|
|
65043
|
-
|
|
65044
|
-
|
|
65045
|
-
|
|
65046
|
-
|
|
65695
|
+
const quotaSignals = [
|
|
65696
|
+
"out of extra usage",
|
|
65697
|
+
"extra usage",
|
|
65698
|
+
"credit_balance_too_low",
|
|
65699
|
+
"credit balance too low",
|
|
65700
|
+
"usage limit",
|
|
65701
|
+
"usage_limit",
|
|
65702
|
+
"quota exhausted",
|
|
65703
|
+
"quota_exhausted",
|
|
65704
|
+
"plan limit",
|
|
65705
|
+
"subscription limit",
|
|
65706
|
+
"hit your limit",
|
|
65707
|
+
"hit the limit",
|
|
65708
|
+
"session limit",
|
|
65709
|
+
"session cap"
|
|
65710
|
+
];
|
|
65711
|
+
if (quotaSignals.some((s) => lower.includes(s))) {
|
|
65712
|
+
const resetAt = parseResetTime2(sample);
|
|
65713
|
+
return resetAt !== undefined ? { kind: "quota_exhausted", resetAt, raw: stderr } : { kind: "quota_exhausted", raw: stderr };
|
|
65047
65714
|
}
|
|
65048
|
-
|
|
65049
|
-
|
|
65050
|
-
|
|
65051
|
-
|
|
65052
|
-
|
|
65053
|
-
|
|
65054
|
-
|
|
65055
|
-
|
|
65056
|
-
|
|
65057
|
-
|
|
65715
|
+
const overloadSignals = [
|
|
65716
|
+
"overloaded_error",
|
|
65717
|
+
"overloaded",
|
|
65718
|
+
"rate_limit_error",
|
|
65719
|
+
"rate limit",
|
|
65720
|
+
"rate-limited",
|
|
65721
|
+
"http 429",
|
|
65722
|
+
'"status":429',
|
|
65723
|
+
"status: 429",
|
|
65724
|
+
" 429 ",
|
|
65725
|
+
"503 service",
|
|
65726
|
+
"service unavailable",
|
|
65727
|
+
'"status":529',
|
|
65728
|
+
"http 529",
|
|
65729
|
+
" 529 "
|
|
65730
|
+
];
|
|
65731
|
+
if (overloadSignals.some((s) => lower.includes(s))) {
|
|
65732
|
+
const resetAt = parseResetTime2(sample);
|
|
65733
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
65058
65734
|
}
|
|
65059
|
-
|
|
65060
|
-
|
|
65061
|
-
|
|
65062
|
-
|
|
65063
|
-
|
|
65064
|
-
|
|
65065
|
-
|
|
65066
|
-
|
|
65735
|
+
const networkSignals = [
|
|
65736
|
+
"econnrefused",
|
|
65737
|
+
"econnreset",
|
|
65738
|
+
"etimedout",
|
|
65739
|
+
"enotfound",
|
|
65740
|
+
"eai_again",
|
|
65741
|
+
"fetch failed",
|
|
65742
|
+
"network error",
|
|
65743
|
+
"socket hang up",
|
|
65744
|
+
"request timed out",
|
|
65745
|
+
"connection refused",
|
|
65746
|
+
"getaddrinfo"
|
|
65747
|
+
];
|
|
65748
|
+
if (networkSignals.some((s) => lower.includes(s))) {
|
|
65749
|
+
return { kind: "network", raw: stderr };
|
|
65067
65750
|
}
|
|
65068
|
-
return
|
|
65069
|
-
limitType,
|
|
65070
|
-
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
65071
|
-
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
65072
|
-
resetAtMs
|
|
65073
|
-
};
|
|
65751
|
+
return null;
|
|
65074
65752
|
}
|
|
65075
65753
|
function parseResetTime2(text4, parseTimeNow = new Date) {
|
|
65076
65754
|
const lower = text4.toLowerCase();
|
|
@@ -65194,31 +65872,77 @@ function parseRelativeDuration2(s) {
|
|
|
65194
65872
|
}
|
|
65195
65873
|
return matched && total > 0 ? total : null;
|
|
65196
65874
|
}
|
|
65875
|
+
function formatModelUnavailableCard(detection, agent, opts = {}) {
|
|
65876
|
+
const now = opts.now ?? new Date;
|
|
65877
|
+
const slotPart = opts.slot ? ` (slot **${escapeMarkdown(opts.slot)}**)` : "";
|
|
65878
|
+
const reason = formatReason(detection, now);
|
|
65879
|
+
const lines = [
|
|
65880
|
+
`\u26a0\ufe0f **Model unavailable** on agent **${escapeMarkdown(agent)}**${slotPart}`,
|
|
65881
|
+
`Reason: ${reason}`,
|
|
65882
|
+
""
|
|
65883
|
+
];
|
|
65884
|
+
if (opts.autoFallbackInFlight) {
|
|
65885
|
+
lines.push("_Auto-failover in progress \u2014 see the announcement below._");
|
|
65886
|
+
} else {
|
|
65887
|
+
lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
|
|
65888
|
+
}
|
|
65889
|
+
return lines.join(`
|
|
65890
|
+
`);
|
|
65891
|
+
}
|
|
65892
|
+
function formatReason(d, now) {
|
|
65893
|
+
const reset2 = d.resetAt ? ` (${formatResetRelative(d.resetAt, now)})` : "";
|
|
65894
|
+
switch (d.kind) {
|
|
65895
|
+
case "quota_exhausted":
|
|
65896
|
+
return `quota exhausted${reset2}`;
|
|
65897
|
+
case "overload":
|
|
65898
|
+
return `model overloaded${reset2}`;
|
|
65899
|
+
case "rate_limited":
|
|
65900
|
+
return `account rate-limited${reset2}`;
|
|
65901
|
+
case "network":
|
|
65902
|
+
return "network unreachable";
|
|
65903
|
+
}
|
|
65904
|
+
}
|
|
65905
|
+
function resolveModelUnavailableFromOperatorEvent(ev) {
|
|
65906
|
+
const detail = typeof ev.detail === "string" ? ev.detail : "";
|
|
65907
|
+
if (ev.kind === "quota-exhausted") {
|
|
65908
|
+
return detectModelUnavailable2(detail) ?? { kind: "quota_exhausted", raw: detail };
|
|
65909
|
+
}
|
|
65910
|
+
if (ev.kind === "rate-limited") {
|
|
65911
|
+
const detected = detectModelUnavailable2(detail);
|
|
65912
|
+
return detected?.kind === "quota_exhausted" ? detected : null;
|
|
65913
|
+
}
|
|
65914
|
+
if (ev.kind === "unknown-5xx") {
|
|
65915
|
+
return detectModelUnavailable2(detail) ?? { kind: "overload", raw: detail };
|
|
65916
|
+
}
|
|
65917
|
+
return detectModelUnavailable2(detail);
|
|
65918
|
+
}
|
|
65197
65919
|
|
|
65198
65920
|
// throttle-tier.ts
|
|
65199
|
-
|
|
65921
|
+
init_card_format();
|
|
65922
|
+
init_quota_check();
|
|
65923
|
+
var accountScopedThrottleSignals2 = [
|
|
65200
65924
|
"would exceed your account's rate limit",
|
|
65201
65925
|
"would exceed your account\u2019s rate limit",
|
|
65202
65926
|
"not your account"
|
|
65203
65927
|
];
|
|
65204
|
-
function
|
|
65928
|
+
function isAccountScopedThrottle2(text4) {
|
|
65205
65929
|
if (typeof text4 !== "string" || text4.length === 0)
|
|
65206
65930
|
return false;
|
|
65207
65931
|
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65208
65932
|
const lower = sample.toLowerCase();
|
|
65209
|
-
return
|
|
65933
|
+
return accountScopedThrottleSignals2.some((s) => lower.includes(s));
|
|
65210
65934
|
}
|
|
65211
|
-
function
|
|
65212
|
-
if (
|
|
65935
|
+
function classify429Detail2(text4) {
|
|
65936
|
+
if (isAccountScopedThrottle2(text4))
|
|
65213
65937
|
return "account-scoped";
|
|
65214
|
-
if (
|
|
65938
|
+
if (isLitellmProxyLocal429(text4))
|
|
65215
65939
|
return "litellm-local";
|
|
65216
65940
|
return "generic-transient";
|
|
65217
65941
|
}
|
|
65218
65942
|
function build429ClassifiedMetric(opts) {
|
|
65219
65943
|
const detail = typeof opts.detail === "string" ? opts.detail : "";
|
|
65220
65944
|
const litellm = parseLitellmLimitDetail(detail, new Date(opts.now));
|
|
65221
|
-
const anthropicResetMs =
|
|
65945
|
+
const anthropicResetMs = parseResetTime(detail, new Date(opts.now))?.getTime() ?? null;
|
|
65222
65946
|
const resetAtMs = anthropicResetMs ?? litellm.resetAtMs;
|
|
65223
65947
|
return {
|
|
65224
65948
|
kind: "rate_limit_429_classified",
|
|
@@ -65232,20 +65956,20 @@ function build429ClassifiedMetric(opts) {
|
|
|
65232
65956
|
current_usage: litellm.currentUsage
|
|
65233
65957
|
};
|
|
65234
65958
|
}
|
|
65235
|
-
var
|
|
65959
|
+
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
|
|
65236
65960
|
var THROTTLE_DEFAULT_WAIT_MS = 60000;
|
|
65237
65961
|
function throttleRetryInPlaceMaxMs(env = process.env) {
|
|
65238
65962
|
const raw = env.SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS;
|
|
65239
65963
|
if (raw == null || raw === "")
|
|
65240
|
-
return
|
|
65964
|
+
return THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
|
|
65241
65965
|
const n = Number(raw);
|
|
65242
|
-
return Number.isFinite(n) && n > 0 ? n :
|
|
65966
|
+
return Number.isFinite(n) && n > 0 ? n : THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
|
|
65243
65967
|
}
|
|
65244
65968
|
function decideThrottleTier(opts) {
|
|
65245
65969
|
const { detail, now, thresholdMs } = opts;
|
|
65246
|
-
if (!
|
|
65970
|
+
if (!isAccountScopedThrottle2(detail))
|
|
65247
65971
|
return { action: "none" };
|
|
65248
|
-
const resetAt =
|
|
65972
|
+
const resetAt = parseResetTime(detail, new Date(now));
|
|
65249
65973
|
const resetAtMs = resetAt?.getTime();
|
|
65250
65974
|
if (resetAtMs == null || !Number.isFinite(resetAtMs) || resetAtMs <= now) {
|
|
65251
65975
|
return {
|
|
@@ -65259,47 +65983,7 @@ function decideThrottleTier(opts) {
|
|
|
65259
65983
|
}
|
|
65260
65984
|
return { action: "failover", resetAtMs };
|
|
65261
65985
|
}
|
|
65262
|
-
var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
|
|
65263
|
-
|
|
65264
|
-
// throttle-tier.ts
|
|
65265
|
-
init_card_format();
|
|
65266
|
-
init_quota_check();
|
|
65267
|
-
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
|
|
65268
65986
|
var THROTTLE_NOTICE_COOLDOWN_MS2 = 10 * 60000;
|
|
65269
|
-
function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS2) {
|
|
65270
|
-
const last = prev.lastSentAtMsByAccount[account] ?? 0;
|
|
65271
|
-
if (now - last >= cooldownMs) {
|
|
65272
|
-
return {
|
|
65273
|
-
send: true,
|
|
65274
|
-
next: {
|
|
65275
|
-
lastSentAtMsByAccount: {
|
|
65276
|
-
...prev.lastSentAtMsByAccount,
|
|
65277
|
-
[account]: now
|
|
65278
|
-
}
|
|
65279
|
-
}
|
|
65280
|
-
};
|
|
65281
|
-
}
|
|
65282
|
-
return { send: false, next: prev };
|
|
65283
|
-
}
|
|
65284
|
-
function renderThrottleNotice(opts) {
|
|
65285
|
-
const now = opts.now ?? new Date;
|
|
65286
|
-
const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
|
|
65287
|
-
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65288
|
-
const lines = [
|
|
65289
|
-
`\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
|
|
65290
|
-
`This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
|
|
65291
|
-
`_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
|
|
65292
|
-
];
|
|
65293
|
-
return lines.join(`
|
|
65294
|
-
`);
|
|
65295
|
-
}
|
|
65296
|
-
function renderThrottleEscalationNotice(opts) {
|
|
65297
|
-
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65298
|
-
const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
|
|
65299
|
-
const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
|
|
65300
|
-
return `${head}
|
|
65301
|
-
${tail}`;
|
|
65302
|
-
}
|
|
65303
65987
|
|
|
65304
65988
|
// gateway/throttle-tier-wiring.ts
|
|
65305
65989
|
var THROTTLE_RETRY_NUDGE_SLACK_MS = 5000;
|
|
@@ -65320,7 +66004,7 @@ function createThrottleTierRunner(deps) {
|
|
|
65320
66004
|
let granted = true;
|
|
65321
66005
|
if (client3 && account) {
|
|
65322
66006
|
try {
|
|
65323
|
-
granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`,
|
|
66007
|
+
granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`, THROTTLE_NOTICE_COOLDOWN_MS)).granted;
|
|
65324
66008
|
} catch {
|
|
65325
66009
|
granted = true;
|
|
65326
66010
|
}
|
|
@@ -69406,6 +70090,7 @@ var MODEL_ARG_RE2 = /^[A-Za-z0-9][A-Za-z0-9._/\[\]-]{0,99}$/;
|
|
|
69406
70090
|
function isValidModelArg2(arg) {
|
|
69407
70091
|
return MODEL_ARG_RE2.test(arg);
|
|
69408
70092
|
}
|
|
70093
|
+
var MODEL_CALLBACK_ALIAS2 = "mdl:alias:";
|
|
69409
70094
|
|
|
69410
70095
|
// gateway/session-model-file.ts
|
|
69411
70096
|
var SESSION_MODEL_FILE = ".session-model";
|
|
@@ -69472,6 +70157,172 @@ function clearSessionEffortFile(agentDir) {
|
|
|
69472
70157
|
rmSync4(join29(agentDir, SESSION_EFFORT_FILE), { force: true });
|
|
69473
70158
|
} catch {}
|
|
69474
70159
|
}
|
|
70160
|
+
var PREMIUM_RECOVERY_FILE = ".premium-recovery";
|
|
70161
|
+
function parsePremiumRecovery(text4) {
|
|
70162
|
+
try {
|
|
70163
|
+
const raw = JSON.parse(text4);
|
|
70164
|
+
if (typeof raw.premiumModel !== "string" || !isValidModelArg2(raw.premiumModel) || typeof raw.ts !== "number" || !Array.isArray(raw.chats) || raw.chats.length === 0 || !raw.chats.every((c) => typeof c === "string" && c.length > 0)) {
|
|
70165
|
+
return null;
|
|
70166
|
+
}
|
|
70167
|
+
return { premiumModel: raw.premiumModel, chats: raw.chats, ts: raw.ts };
|
|
70168
|
+
} catch {
|
|
70169
|
+
return null;
|
|
70170
|
+
}
|
|
70171
|
+
}
|
|
70172
|
+
function writePremiumRecoveryFile(agentDir, premiumModel, chats) {
|
|
70173
|
+
if (!isValidModelArg2(premiumModel)) {
|
|
70174
|
+
throw new Error(`refusing to persist non-canonical premium-recovery token: ${JSON.stringify(premiumModel)}`);
|
|
70175
|
+
}
|
|
70176
|
+
const clean = chats.filter((c) => typeof c === "string" && c.length > 0);
|
|
70177
|
+
if (clean.length === 0) {
|
|
70178
|
+
throw new Error("refusing to persist premium-recovery marker with no chats to notify");
|
|
70179
|
+
}
|
|
70180
|
+
atomicWrite(join29(agentDir, PREMIUM_RECOVERY_FILE), `${JSON.stringify({ premiumModel, chats: clean, ts: Date.now() })}
|
|
70181
|
+
`);
|
|
70182
|
+
}
|
|
70183
|
+
function readPremiumRecoveryFile(agentDir) {
|
|
70184
|
+
let raw;
|
|
70185
|
+
try {
|
|
70186
|
+
raw = readFileSync25(join29(agentDir, PREMIUM_RECOVERY_FILE), "utf8");
|
|
70187
|
+
} catch {
|
|
70188
|
+
return null;
|
|
70189
|
+
}
|
|
70190
|
+
const parsed = parsePremiumRecovery(raw);
|
|
70191
|
+
if (parsed == null) {
|
|
70192
|
+
clearPremiumRecoveryFile(agentDir);
|
|
70193
|
+
return null;
|
|
70194
|
+
}
|
|
70195
|
+
return parsed;
|
|
70196
|
+
}
|
|
70197
|
+
function clearPremiumRecoveryFile(agentDir) {
|
|
70198
|
+
try {
|
|
70199
|
+
rmSync4(join29(agentDir, PREMIUM_RECOVERY_FILE), { force: true });
|
|
70200
|
+
} catch {}
|
|
70201
|
+
}
|
|
70202
|
+
|
|
70203
|
+
// tier-downgrade.ts
|
|
70204
|
+
function decideTierDowngrade(input) {
|
|
70205
|
+
const configured = typeof input.configuredDefault === "string" ? input.configuredDefault.trim() : "";
|
|
70206
|
+
if (configured.length === 0) {
|
|
70207
|
+
return { action: "skip", reason: "unresolved" };
|
|
70208
|
+
}
|
|
70209
|
+
const override = input.sessionOverride;
|
|
70210
|
+
if (override == null || override.length === 0) {
|
|
70211
|
+
return { action: "skip", reason: "on-default" };
|
|
70212
|
+
}
|
|
70213
|
+
if (input.resolve(override) === input.resolve(configured)) {
|
|
70214
|
+
return { action: "skip", reason: "on-default" };
|
|
70215
|
+
}
|
|
70216
|
+
return { action: "downgrade", toModel: configured, fromModel: override };
|
|
70217
|
+
}
|
|
70218
|
+
function renderTierDowngradeNotice(fromModel, toModel, agent) {
|
|
70219
|
+
return `\u2935\ufe0f **Downgrading model to keep going** on agent **${agent}**
|
|
70220
|
+
` + `\`${fromModel}\` is overloaded across every account right now, so this turn is ` + `resuming on the default \`${toModel}\` to keep working. ` + `Re-issue \`/model ${fromModel}\` once it frees up \u2014 it won't switch back on its own.`;
|
|
70221
|
+
}
|
|
70222
|
+
function planTierDowngrade(decision, gateVerdict, agent) {
|
|
70223
|
+
if (decision.action !== "downgrade")
|
|
70224
|
+
return { kind: "skip" };
|
|
70225
|
+
if (gateVerdict === "skip-inflight")
|
|
70226
|
+
return { kind: "suppress" };
|
|
70227
|
+
if (gateVerdict === "skip-stale")
|
|
70228
|
+
return { kind: "skip" };
|
|
70229
|
+
return {
|
|
70230
|
+
kind: "downgrade",
|
|
70231
|
+
toModel: decision.toModel,
|
|
70232
|
+
fromModel: decision.fromModel,
|
|
70233
|
+
notice: renderTierDowngradeNotice(decision.fromModel, decision.toModel, agent)
|
|
70234
|
+
};
|
|
70235
|
+
}
|
|
70236
|
+
|
|
70237
|
+
// gateway/tier-downgrade-wiring.ts
|
|
70238
|
+
function runTierDowngrade(triggerAgent, deps) {
|
|
70239
|
+
const agentDir = deps.getAgentDir();
|
|
70240
|
+
if (!agentDir)
|
|
70241
|
+
return "skip";
|
|
70242
|
+
const configuredDefault = deps.getConfiguredDefault() ?? "";
|
|
70243
|
+
const decision = decideTierDowngrade({
|
|
70244
|
+
sessionOverride: deps.getSessionOverride(),
|
|
70245
|
+
configuredDefault,
|
|
70246
|
+
resolve: deps.resolve
|
|
70247
|
+
});
|
|
70248
|
+
const gateVerdict = deps.peekResumeGate();
|
|
70249
|
+
const plan = planTierDowngrade(decision, gateVerdict, triggerAgent);
|
|
70250
|
+
if (plan.kind === "skip") {
|
|
70251
|
+
if (decision.action === "downgrade") {
|
|
70252
|
+
deps.log(`[tier-downgrade] restart suppressed (${gateVerdict}) agent=${triggerAgent}`);
|
|
70253
|
+
}
|
|
70254
|
+
return "skip";
|
|
70255
|
+
}
|
|
70256
|
+
if (plan.kind === "suppress") {
|
|
70257
|
+
deps.log(`[tier-downgrade] give-up suppressed \u2014 a resume restart is already armed agent=${triggerAgent}`);
|
|
70258
|
+
return "restart-pending";
|
|
70259
|
+
}
|
|
70260
|
+
try {
|
|
70261
|
+
deps.writeCarrier(agentDir, plan.toModel, configuredDefault);
|
|
70262
|
+
} catch (err) {
|
|
70263
|
+
deps.log(`[tier-downgrade] failed to write session-model carrier \u2014 aborting downgrade: ${err?.message ?? err}`);
|
|
70264
|
+
return "skip";
|
|
70265
|
+
}
|
|
70266
|
+
deps.armResumeGate();
|
|
70267
|
+
deps.log(`[tier-downgrade] downgrading ${plan.fromModel} \u2192 ${plan.toModel} and resuming via self-restart agent=${triggerAgent}`);
|
|
70268
|
+
try {
|
|
70269
|
+
deps.writeRecoveryMarker(agentDir, plan.fromModel);
|
|
70270
|
+
} catch (err) {
|
|
70271
|
+
deps.log(`[tier-downgrade] premium-recovery marker write failed (non-fatal): ${err?.message ?? err}`);
|
|
70272
|
+
}
|
|
70273
|
+
deps.broadcastNotice(plan.notice);
|
|
70274
|
+
deps.selfRestart(deps.selfAgent(triggerAgent));
|
|
70275
|
+
return "downgraded";
|
|
70276
|
+
}
|
|
70277
|
+
|
|
70278
|
+
// premium-recovery.ts
|
|
70279
|
+
function renderPremiumRecoveryPing(premiumModel) {
|
|
70280
|
+
return {
|
|
70281
|
+
text: `\u2705 \`${premiumModel}\` is available again \u2014 tap to switch back to it for this session.`,
|
|
70282
|
+
buttonText: `Switch to ${premiumModel}`
|
|
70283
|
+
};
|
|
70284
|
+
}
|
|
70285
|
+
function premiumRecoveryClaimKey(agent, premiumModel) {
|
|
70286
|
+
return `premium-recovery:${agent}:${premiumModel}`;
|
|
70287
|
+
}
|
|
70288
|
+
|
|
70289
|
+
// gateway/premium-recovery-wiring.ts
|
|
70290
|
+
async function runPremiumRecoveryPing(deps) {
|
|
70291
|
+
const agentDir = deps.getAgentDir();
|
|
70292
|
+
if (!agentDir)
|
|
70293
|
+
return;
|
|
70294
|
+
const marker = deps.readMarker(agentDir);
|
|
70295
|
+
if (marker == null)
|
|
70296
|
+
return;
|
|
70297
|
+
if (!deps.decide())
|
|
70298
|
+
return;
|
|
70299
|
+
const agent = deps.getAgent();
|
|
70300
|
+
const granted = await deps.claimNotification(premiumRecoveryClaimKey(agent, marker.premiumModel));
|
|
70301
|
+
if (!granted) {
|
|
70302
|
+
deps.clearMarker(agentDir);
|
|
70303
|
+
return;
|
|
70304
|
+
}
|
|
70305
|
+
deps.clearMarker(agentDir);
|
|
70306
|
+
const ping = renderPremiumRecoveryPing(marker.premiumModel);
|
|
70307
|
+
const keyboard = {
|
|
70308
|
+
inline_keyboard: [
|
|
70309
|
+
[{ text: ping.buttonText, callback_data: `${MODEL_CALLBACK_ALIAS2}${marker.premiumModel}` }]
|
|
70310
|
+
]
|
|
70311
|
+
};
|
|
70312
|
+
const chats = marker.chats.length > 0 ? marker.chats : deps.fallbackChats();
|
|
70313
|
+
for (const chatId of chats) {
|
|
70314
|
+
deps.sendToChat(chatId, ping, keyboard);
|
|
70315
|
+
}
|
|
70316
|
+
deps.log(`[premium-recovery] ${marker.premiumModel} servable again \u2014 ping sent agent=${agent} chats=${chats.length}`);
|
|
70317
|
+
}
|
|
70318
|
+
|
|
70319
|
+
// premium-recovery.ts
|
|
70320
|
+
function decidePremiumRecovery(opts) {
|
|
70321
|
+
if (!opts.hasMarker)
|
|
70322
|
+
return { fire: false, reason: "no-marker" };
|
|
70323
|
+
const servable = opts.accounts.some((a) => !a.exhausted && !a.premiumWalled);
|
|
70324
|
+
return servable ? { fire: true, reason: "recovered" } : { fire: false, reason: "still-walled" };
|
|
70325
|
+
}
|
|
69475
70326
|
|
|
69476
70327
|
// ../src/agents/model-picker.ts
|
|
69477
70328
|
var HEADER_RE = /Select model/;
|
|
@@ -77113,71 +77964,6 @@ import {
|
|
|
77113
77964
|
} from "fs";
|
|
77114
77965
|
import { join as join39 } from "path";
|
|
77115
77966
|
|
|
77116
|
-
// operator-events.ts
|
|
77117
|
-
function classifyClaudeError(raw) {
|
|
77118
|
-
try {
|
|
77119
|
-
return classifyInner(raw);
|
|
77120
|
-
} catch {
|
|
77121
|
-
return "unknown-4xx";
|
|
77122
|
-
}
|
|
77123
|
-
}
|
|
77124
|
-
function classifyInner(raw) {
|
|
77125
|
-
if (raw == null)
|
|
77126
|
-
return "unknown-4xx";
|
|
77127
|
-
const obj = typeof raw === "object" ? raw : {};
|
|
77128
|
-
const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
|
|
77129
|
-
const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
|
|
77130
|
-
const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
|
|
77131
|
-
const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
|
|
77132
|
-
const sdkCode = extractString(obj, "error_code") ?? "";
|
|
77133
|
-
if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
|
|
77134
|
-
const msg = message.toLowerCase();
|
|
77135
|
-
if (msg.includes("expired") || msg.includes("refresh")) {
|
|
77136
|
-
return "credentials-expired";
|
|
77137
|
-
}
|
|
77138
|
-
return "credentials-invalid";
|
|
77139
|
-
}
|
|
77140
|
-
if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
|
|
77141
|
-
return "credentials-invalid";
|
|
77142
|
-
}
|
|
77143
|
-
if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
|
|
77144
|
-
return "credit-exhausted";
|
|
77145
|
-
}
|
|
77146
|
-
if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
|
|
77147
|
-
return "rate-limited";
|
|
77148
|
-
}
|
|
77149
|
-
if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
|
|
77150
|
-
return "rate-limited";
|
|
77151
|
-
}
|
|
77152
|
-
if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
|
|
77153
|
-
return "agent-crashed";
|
|
77154
|
-
}
|
|
77155
|
-
if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
|
|
77156
|
-
return "agent-restarted-unexpectedly";
|
|
77157
|
-
}
|
|
77158
|
-
if (status != null) {
|
|
77159
|
-
if (status >= 400 && status < 500)
|
|
77160
|
-
return "unknown-4xx";
|
|
77161
|
-
if (status >= 500 && status < 600)
|
|
77162
|
-
return "unknown-5xx";
|
|
77163
|
-
}
|
|
77164
|
-
return "unknown-4xx";
|
|
77165
|
-
}
|
|
77166
|
-
function extractString(obj, key) {
|
|
77167
|
-
const v = obj[key];
|
|
77168
|
-
return typeof v === "string" && v.length > 0 ? v : null;
|
|
77169
|
-
}
|
|
77170
|
-
function extractNumber(obj, key) {
|
|
77171
|
-
const v = obj[key];
|
|
77172
|
-
return typeof v === "number" ? v : null;
|
|
77173
|
-
}
|
|
77174
|
-
function getNestedObj(obj, key) {
|
|
77175
|
-
const v = obj[key];
|
|
77176
|
-
return typeof v === "object" && v != null ? v : {};
|
|
77177
|
-
}
|
|
77178
|
-
var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
|
|
77179
|
-
var cooldownMap2 = new Map;
|
|
77180
|
-
|
|
77181
77967
|
// session-tail.ts
|
|
77182
77968
|
function sanitizeCwdToProjectName(cwd) {
|
|
77183
77969
|
return cwd.replace(/[^a-zA-Z0-9]/g, "-");
|
|
@@ -77362,7 +78148,7 @@ function detectErrorInTranscriptLine(line) {
|
|
|
77362
78148
|
const errStr = typeof obj.error === "string" ? obj.error : "";
|
|
77363
78149
|
const text4 = extractAssistantText(obj);
|
|
77364
78150
|
const kind2 = status === 429 ? isTransientUpstreamSignal(`${text4}
|
|
77365
|
-
${errStr}`) ||
|
|
78151
|
+
${errStr}`) || isLitellmProxyLocal429(`${text4}
|
|
77366
78152
|
${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text4 });
|
|
77367
78153
|
return {
|
|
77368
78154
|
kind: kind2,
|
|
@@ -79671,6 +80457,29 @@ var RESOURCE_KEYS = ["path", "endpoint", "url", "resource", "route"];
|
|
|
79671
80457
|
var ARG_SUMMARY_MAX_KEYS = 4;
|
|
79672
80458
|
var ARG_VALUE_MAX = 40;
|
|
79673
80459
|
var ARG_SUMMARY_LINE_MAX = 180;
|
|
80460
|
+
var CONTEXT_NOISE_KEYS = new Set([
|
|
80461
|
+
"reason",
|
|
80462
|
+
"why",
|
|
80463
|
+
"chat_id",
|
|
80464
|
+
"message_id",
|
|
80465
|
+
"message_thread_id",
|
|
80466
|
+
"thread_id",
|
|
80467
|
+
"origin_turn_id",
|
|
80468
|
+
"reply_to",
|
|
80469
|
+
"quote",
|
|
80470
|
+
"quote_text",
|
|
80471
|
+
"format",
|
|
80472
|
+
"parse_mode",
|
|
80473
|
+
"disable_web_page_preview",
|
|
80474
|
+
"disable_notification",
|
|
80475
|
+
"protect_content",
|
|
80476
|
+
"single_use",
|
|
80477
|
+
"ack_text",
|
|
80478
|
+
"inline_keyboard",
|
|
80479
|
+
"file_id"
|
|
80480
|
+
]);
|
|
80481
|
+
var SENSITIVE_VALUE_KEY_RE = /token|secret|password|passwd|key|auth|dsn|conn|url|credential|cookie|session/i;
|
|
80482
|
+
var NON_HTTP_DSN_RE = /\b(?:postgres(?:ql)?|mysql|mariadb|redis|rediss|mongodb(?:\+srv)?|amqp|amqps):\/\/\S*@\S+/gi;
|
|
79674
80483
|
var MCP_TOOL_DESCRIPTIONS = {
|
|
79675
80484
|
"mcp__agent-config__config_get": "Read its own merged config",
|
|
79676
80485
|
"mcp__agent-config__cron_list": "List its own scheduled tasks",
|
|
@@ -79721,8 +80530,12 @@ function formatPermissionCardBody(opts) {
|
|
|
79721
80530
|
}
|
|
79722
80531
|
const callerReason = callerSuppliedReason(opts.inputPreview);
|
|
79723
80532
|
const rawWhy = (callerReason ?? "").replace(/\s+/g, " ").trim();
|
|
79724
|
-
|
|
79725
|
-
|
|
80533
|
+
if (rawWhy.length > 0) {
|
|
80534
|
+
const truncatedWhy = rawWhy.length > DESCRIPTION_LINE_MAX ? rawWhy.slice(0, DESCRIPTION_LINE_MAX - 1) + "\u2026" : rawWhy;
|
|
80535
|
+
lines.push(`why: _${escapeTgHtml(truncatedWhy)}_`);
|
|
80536
|
+
} else {
|
|
80537
|
+
lines.push(`context: _${escapeTgHtml(synthesizeContext(opts.toolName, opts.inputPreview))}_`);
|
|
80538
|
+
}
|
|
79726
80539
|
const argSummary = mcpArgSummary(opts.toolName, opts.inputPreview);
|
|
79727
80540
|
if (argSummary) {
|
|
79728
80541
|
lines.push(`\u21b3 _${escapeTgHtml(argSummary)}_`);
|
|
@@ -79996,6 +80809,50 @@ function callerSuppliedReason(inputPreview) {
|
|
|
79996
80809
|
}
|
|
79997
80810
|
return null;
|
|
79998
80811
|
}
|
|
80812
|
+
function synthesizeContext(toolName, inputPreview) {
|
|
80813
|
+
const input = parseInput2(inputPreview);
|
|
80814
|
+
const summary = input ? salientInputSummary(input) : null;
|
|
80815
|
+
const base = summary ?? naturalAction(toolName, inputPreview);
|
|
80816
|
+
const turnRef = input ? readString2(input, "origin_turn_id") : null;
|
|
80817
|
+
return turnRef ? `${base} \u00b7 turn ${shortTurnRef(turnRef)}` : base;
|
|
80818
|
+
}
|
|
80819
|
+
function shortTurnRef(turnId) {
|
|
80820
|
+
const t = turnId.trim();
|
|
80821
|
+
return t.length <= 10 ? t : `\u2026${t.slice(-8)}`;
|
|
80822
|
+
}
|
|
80823
|
+
function salientInputSummary(input) {
|
|
80824
|
+
const parts = [];
|
|
80825
|
+
for (const [key, value] of Object.entries(input)) {
|
|
80826
|
+
if (CONTEXT_NOISE_KEYS.has(key))
|
|
80827
|
+
continue;
|
|
80828
|
+
if (value == null)
|
|
80829
|
+
continue;
|
|
80830
|
+
if (parts.length >= ARG_SUMMARY_MAX_KEYS) {
|
|
80831
|
+
parts.push("\u2026");
|
|
80832
|
+
break;
|
|
80833
|
+
}
|
|
80834
|
+
if (typeof value === "object") {
|
|
80835
|
+
parts.push(key);
|
|
80836
|
+
continue;
|
|
80837
|
+
}
|
|
80838
|
+
const shown = truncate6(redactSalientValue(key, String(value)), ARG_VALUE_MAX);
|
|
80839
|
+
if (shown.length === 0)
|
|
80840
|
+
continue;
|
|
80841
|
+
parts.push(`${key}: ${shown}`);
|
|
80842
|
+
}
|
|
80843
|
+
if (parts.length === 0)
|
|
80844
|
+
return null;
|
|
80845
|
+
const joined = parts.join(", ");
|
|
80846
|
+
return joined.length > ARG_SUMMARY_LINE_MAX ? joined.slice(0, ARG_SUMMARY_LINE_MAX - 1) + "\u2026" : joined;
|
|
80847
|
+
}
|
|
80848
|
+
function redactSalientValue(key, value) {
|
|
80849
|
+
if (SENSITIVE_VALUE_KEY_RE.test(key))
|
|
80850
|
+
return REDACTED_MARKER;
|
|
80851
|
+
const dsnScrubbed = value.replace(NON_HTTP_DSN_RE, REDACTED_MARKER);
|
|
80852
|
+
const scrubbed = redact(`${key}=${dsnScrubbed}`);
|
|
80853
|
+
const prefix = `${key}=`;
|
|
80854
|
+
return scrubbed.startsWith(prefix) ? scrubbed.slice(prefix.length) : scrubbed;
|
|
80855
|
+
}
|
|
79999
80856
|
function extractReasonFromRaw(raw) {
|
|
80000
80857
|
const m = /"(?:reason|why)"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(raw);
|
|
80001
80858
|
if (!m)
|
|
@@ -81224,10 +82081,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
|
|
|
81224
82081
|
}
|
|
81225
82082
|
|
|
81226
82083
|
// ../src/build-info.ts
|
|
81227
|
-
var VERSION = "0.18.
|
|
81228
|
-
var COMMIT_SHA = "
|
|
81229
|
-
var COMMIT_DATE = "2026-07-
|
|
81230
|
-
var LATEST_PR =
|
|
82084
|
+
var VERSION = "0.18.18";
|
|
82085
|
+
var COMMIT_SHA = "a256b3b1";
|
|
82086
|
+
var COMMIT_DATE = "2026-07-13T03:10:38Z";
|
|
82087
|
+
var LATEST_PR = 3210;
|
|
81231
82088
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
81232
82089
|
|
|
81233
82090
|
// gateway/boot-version.ts
|
|
@@ -81280,6 +82137,8 @@ function classifyRejection(err, opts = {}) {
|
|
|
81280
82137
|
return "log_only";
|
|
81281
82138
|
if (err instanceof Error && err.message === FLOOD_WAIT_ACTIVE)
|
|
81282
82139
|
return "log_only";
|
|
82140
|
+
if (err instanceof Error && err.message === LOCAL_RESOURCE_EXHAUSTED)
|
|
82141
|
+
return "log_only";
|
|
81283
82142
|
if (!isGrammy)
|
|
81284
82143
|
return "shutdown";
|
|
81285
82144
|
const e = err;
|
|
@@ -81820,10 +82679,13 @@ async function emitLinearAgentActivity(args, deps = {}) {
|
|
|
81820
82679
|
`);
|
|
81821
82680
|
return { content: [{ type: "text", text: `Linear ${type} emitted on session ${sessionId}` }] };
|
|
81822
82681
|
}
|
|
82682
|
+
function captureDedupComment(dedupKey) {
|
|
82683
|
+
return `<!-- switchroom-capture: ${dedupKey} -->`;
|
|
82684
|
+
}
|
|
81823
82685
|
function captureDedupMarker(dedupKey) {
|
|
81824
82686
|
return `
|
|
81825
82687
|
|
|
81826
|
-
|
|
82688
|
+
${captureDedupComment(dedupKey)}`;
|
|
81827
82689
|
}
|
|
81828
82690
|
async function createLinearIssue(args, deps = {}) {
|
|
81829
82691
|
const log = deps.log ?? ((s) => process.stderr.write(s));
|
|
@@ -81872,9 +82734,11 @@ async function createLinearIssue(args, deps = {}) {
|
|
|
81872
82734
|
return { ok: true, data: json.data };
|
|
81873
82735
|
};
|
|
81874
82736
|
if (dedupKey) {
|
|
81875
|
-
const
|
|
82737
|
+
const marker = captureDedupComment(dedupKey);
|
|
82738
|
+
const search2 = await gql("query($term: String!) { searchIssues(term: $term, first: 25) { nodes { id url title description } } }", { term: dedupKey });
|
|
81876
82739
|
if (search2.ok) {
|
|
81877
|
-
const
|
|
82740
|
+
const nodes = search2.data?.searchIssues?.nodes ?? [];
|
|
82741
|
+
const hit = nodes.find((n) => typeof n.description === "string" && n.description.includes(marker));
|
|
81878
82742
|
if (hit?.url) {
|
|
81879
82743
|
log(`telegram gateway: linear_create_issue: dedup hit key=${dedupKey} agent=${agent}
|
|
81880
82744
|
`);
|
|
@@ -82681,8 +83545,15 @@ function buildBridgeDeadIdleNoticeInbound(args) {
|
|
|
82681
83545
|
}
|
|
82682
83546
|
};
|
|
82683
83547
|
}
|
|
82684
|
-
function isRealBridgeIdentity(agentName3) {
|
|
82685
|
-
|
|
83548
|
+
function isRealBridgeIdentity(agentName3, selfAgentName) {
|
|
83549
|
+
if (agentName3 == null || agentName3.length === 0)
|
|
83550
|
+
return false;
|
|
83551
|
+
if (isCronIdentity2(agentName3))
|
|
83552
|
+
return false;
|
|
83553
|
+
if (selfAgentName != null && selfAgentName.length > 0) {
|
|
83554
|
+
return agentName3 === selfAgentName;
|
|
83555
|
+
}
|
|
83556
|
+
return true;
|
|
82686
83557
|
}
|
|
82687
83558
|
function createBridgeDeadWatchdog(opts) {
|
|
82688
83559
|
const setTimer = opts.setTimer ?? ((fn, ms) => {
|
|
@@ -82696,6 +83567,7 @@ function createBridgeDeadWatchdog(opts) {
|
|
|
82696
83567
|
const readCrashTail = opts.readCrashTail ?? ((p, t) => readFreshCrashLogTail(p, { nowMs: t }));
|
|
82697
83568
|
const priorStreak = opts.priorStreak ?? 0;
|
|
82698
83569
|
const maxConsecutive = opts.maxConsecutive ?? MAX_CONSECUTIVE_ESCALATIONS;
|
|
83570
|
+
const selfAgentName = opts.selfAgentName;
|
|
82699
83571
|
let timer3 = null;
|
|
82700
83572
|
let bridgeRegistered = false;
|
|
82701
83573
|
let bridgeEverRegistered = false;
|
|
@@ -82773,14 +83645,14 @@ function createBridgeDeadWatchdog(opts) {
|
|
|
82773
83645
|
return {
|
|
82774
83646
|
arm: armInternal,
|
|
82775
83647
|
noteBridgeRegistered: (agentName3) => {
|
|
82776
|
-
if (!isRealBridgeIdentity(agentName3))
|
|
83648
|
+
if (!isRealBridgeIdentity(agentName3, selfAgentName))
|
|
82777
83649
|
return;
|
|
82778
83650
|
bridgeRegistered = true;
|
|
82779
83651
|
bridgeEverRegistered = true;
|
|
82780
83652
|
cancel();
|
|
82781
83653
|
},
|
|
82782
83654
|
noteBridgeDisconnected: (agentName3) => {
|
|
82783
|
-
if (!isRealBridgeIdentity(agentName3))
|
|
83655
|
+
if (!isRealBridgeIdentity(agentName3, selfAgentName))
|
|
82784
83656
|
return;
|
|
82785
83657
|
bridgeRegistered = false;
|
|
82786
83658
|
armInternal();
|
|
@@ -84887,8 +85759,9 @@ var PHOTO_EXTS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp"]);
|
|
|
84887
85759
|
var FLOOD_STATE_PATH = floodStatePath(STATE_DIR);
|
|
84888
85760
|
var FLOOD_WINDOWS_PATH = floodWindowsPath(STATE_DIR);
|
|
84889
85761
|
var recordFloodWindow = makeFloodWindowRecorder(FLOOD_WINDOWS_PATH);
|
|
85762
|
+
var sendGateConfig = sendGateConfigFromEnv();
|
|
84890
85763
|
var sendGate = createSendGate({
|
|
84891
|
-
|
|
85764
|
+
...sendGateConfig,
|
|
84892
85765
|
initialWindows: loadInitialFloodWindows(FLOOD_STATE_PATH, FLOOD_WINDOWS_PATH, Date.now()),
|
|
84893
85766
|
bootRamp: {},
|
|
84894
85767
|
onWindowOpen: (scopeKey, untilTs) => recordFloodWindow(scopeKey, untilTs)
|
|
@@ -84896,10 +85769,10 @@ var sendGate = createSendGate({
|
|
|
84896
85769
|
var probeFloodWaitRemainingMs = makeFloodWaitProbe2(FLOOD_STATE_PATH);
|
|
84897
85770
|
var rawRobustApiCall = createRetryApiCall2({
|
|
84898
85771
|
log: (line) => process.stderr.write(line),
|
|
84899
|
-
onFloodWait: (retryAfterSec) => {
|
|
85772
|
+
onFloodWait: (retryAfterSec, opts) => {
|
|
84900
85773
|
makeFloodWaitRecorder2(FLOOD_STATE_PATH)(retryAfterSec);
|
|
84901
85774
|
try {
|
|
84902
|
-
sendGate.
|
|
85775
|
+
sendGate.openScopedFloodWindows(opts, Date.now() + Math.max(0, retryAfterSec) * 1000);
|
|
84903
85776
|
} catch {}
|
|
84904
85777
|
},
|
|
84905
85778
|
floodWaitRemainingMs: probeFloodWaitRemainingMs
|
|
@@ -84935,6 +85808,7 @@ var sendGateStatsLogger = createStatsLogger({
|
|
|
84935
85808
|
var floodWindowObserver = createFloodWindowObserver({
|
|
84936
85809
|
clock: { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) },
|
|
84937
85810
|
log: (line) => process.stderr.write(line),
|
|
85811
|
+
tz: process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC",
|
|
84938
85812
|
stats: () => sendGate.stats(),
|
|
84939
85813
|
readWindows: (now) => readFloodWindows(FLOOD_WINDOWS_PATH, now),
|
|
84940
85814
|
markAlerted: (scopeKey, alertedAt) => markFloodWindowAlerted(FLOOD_WINDOWS_PATH, scopeKey, alertedAt, Date.now()),
|
|
@@ -84949,7 +85823,7 @@ var floodWindowObserver = createFloodWindowObserver({
|
|
|
84949
85823
|
await robustApiCall(() => bot.api.sendRichMessage(operator, richMessage2(text5), {}), { chat_id: String(operator), verb: "send-gate-flood-alert", priorityClass: "critical" });
|
|
84950
85824
|
}
|
|
84951
85825
|
});
|
|
84952
|
-
if (
|
|
85826
|
+
if (sendGateConfig.enabled) {
|
|
84953
85827
|
const observeTimer = setInterval(() => {
|
|
84954
85828
|
try {
|
|
84955
85829
|
sendGateStatsLogger.tick();
|
|
@@ -85419,7 +86293,8 @@ function isAutoFallbackCooldownActive(_agentName, now) {
|
|
|
85419
86293
|
async function editCardExpired(chatId, messageId, body) {
|
|
85420
86294
|
if (messageId == null)
|
|
85421
86295
|
return;
|
|
85422
|
-
|
|
86296
|
+
const text5 = richMessage2(body);
|
|
86297
|
+
await robustApiCall(() => lockedBot.api.editMessageText(chatId, messageId, text5, { reply_markup: { inline_keyboard: [] } }), { chat_id: chatId, verb: "card-expired.strip", priorityClass: "cosmetic", messageId, editPayload: body }).catch(() => {});
|
|
85423
86298
|
}
|
|
85424
86299
|
function recordMissedApproval(opts) {
|
|
85425
86300
|
if (!MISSED_APPROVAL_REOFFER_ENABLED)
|
|
@@ -85646,10 +86521,7 @@ function postPermissionCard(requestId, pend) {
|
|
|
85646
86521
|
if (live && sent && typeof sent.message_id === "number") {
|
|
85647
86522
|
const landedThreadId = sent.message_thread_id ?? undefined;
|
|
85648
86523
|
live.cards.push({ chatId, messageId: sent.message_id, threadId: landedThreadId });
|
|
85649
|
-
if (live.
|
|
85650
|
-
live.undeliverable = null;
|
|
85651
|
-
live.redeliveryFailures = 0;
|
|
85652
|
-
live.startedAt = Date.now();
|
|
86524
|
+
if (applyDeliveredHoldReset(live, Date.now())) {
|
|
85653
86525
|
reconcileBlockedApprovals();
|
|
85654
86526
|
process.stderr.write(`telegram gateway: permission-card RE-DELIVERED request=${requestId} tool=${live.tool_name} chat=${chatId} \u2014 the operator can answer now; ` + `TTL clock restarted (they had zero seconds while the channel was shut)
|
|
85655
86527
|
`);
|
|
@@ -85899,7 +86771,7 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85899
86771
|
let throttleEscalation = null;
|
|
85900
86772
|
let escalationFired = false;
|
|
85901
86773
|
let rateLimitedCooldownConsulted = false;
|
|
85902
|
-
const rateLimit429Classification = kind === "rate-limited" ?
|
|
86774
|
+
const rateLimit429Classification = kind === "rate-limited" ? classify429Detail2(event.detail) : null;
|
|
85903
86775
|
if (rateLimit429Classification != null && rateLimit429Classification !== "account-scoped") {
|
|
85904
86776
|
emitRuntimeMetric(build429ClassifiedMetric({
|
|
85905
86777
|
agent,
|
|
@@ -85991,6 +86863,18 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85991
86863
|
const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime());
|
|
85992
86864
|
fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt);
|
|
85993
86865
|
}
|
|
86866
|
+
} else if (kind === "rate-limited" || kind === "unknown-5xx") {
|
|
86867
|
+
const parsed = parseLlmError(event.detail);
|
|
86868
|
+
const now = Date.now();
|
|
86869
|
+
if (decideErrorSurface(parsed, agent, { claim: true, now }) === "suppress") {
|
|
86870
|
+
process.stderr.write(`telegram gateway: operator-event collapsed (error-presence-gate) agent=${agent} kind=${kind}
|
|
86871
|
+
`);
|
|
86872
|
+
return;
|
|
86873
|
+
}
|
|
86874
|
+
const tz = process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC";
|
|
86875
|
+
const r = renderLlmError(parsed, agent, tz, new Date(now));
|
|
86876
|
+
renderedText = r.text;
|
|
86877
|
+
renderedKeyboard = r.keyboard;
|
|
85994
86878
|
} else {
|
|
85995
86879
|
try {
|
|
85996
86880
|
const r = renderOperatorEvent(event);
|
|
@@ -86287,6 +87171,10 @@ async function runMidSessionCardReaper() {
|
|
|
86287
87171
|
const reaps = decideWorkerPinReaps({
|
|
86288
87172
|
pins: candidates,
|
|
86289
87173
|
statusOf: (agentId) => {
|
|
87174
|
+
if (agentId.startsWith("group:")) {
|
|
87175
|
+
const feedKey = agentId.slice("group:".length);
|
|
87176
|
+
return workerActivityFeed?.hasRunningInFeed(feedKey) ? "running" : "terminal";
|
|
87177
|
+
}
|
|
86290
87178
|
if (turnsDb == null)
|
|
86291
87179
|
return "unknown";
|
|
86292
87180
|
try {
|
|
@@ -86386,24 +87274,6 @@ async function reconcileStatusPinInner(pinKey, chatId, desired) {
|
|
|
86386
87274
|
statusPinPinnedAt.set(pinKey, Date.now());
|
|
86387
87275
|
}
|
|
86388
87276
|
}
|
|
86389
|
-
function reconcileWorkerPin(agentId, chatId, running) {
|
|
86390
|
-
if (!PIN_STATUS_WHILE_WORKING)
|
|
86391
|
-
return;
|
|
86392
|
-
const key = `wk:${agentId}`;
|
|
86393
|
-
if (!running) {
|
|
86394
|
-
const unpinChat = chatId ?? statusPinChatIds.get(key);
|
|
86395
|
-
if (unpinChat == null)
|
|
86396
|
-
return;
|
|
86397
|
-
reconcileStatusPin(key, unpinChat, { pinned: false });
|
|
86398
|
-
return;
|
|
86399
|
-
}
|
|
86400
|
-
if (chatId == null)
|
|
86401
|
-
return;
|
|
86402
|
-
const messageId = workerActivityFeed?.messageIdOf(agentId) ?? null;
|
|
86403
|
-
if (messageId == null)
|
|
86404
|
-
return;
|
|
86405
|
-
reconcileStatusPin(key, chatId, { pinned: true, messageId });
|
|
86406
|
-
}
|
|
86407
87277
|
async function unpinAllStatusPins() {
|
|
86408
87278
|
const keys = [...statusPinState.keys()];
|
|
86409
87279
|
for (const key of keys) {
|
|
@@ -87036,7 +87906,8 @@ var bridgeDeadWatchdog = createBridgeDeadWatchdog({
|
|
|
87036
87906
|
markerPath: join54(STATE_DIR, "bridge-dead-escalation.json"),
|
|
87037
87907
|
log: (line) => process.stderr.write(`${line}
|
|
87038
87908
|
`),
|
|
87039
|
-
priorStreak: bridgeDeadPriorStreak
|
|
87909
|
+
priorStreak: bridgeDeadPriorStreak,
|
|
87910
|
+
selfAgentName: process.env.SWITCHROOM_AGENT_NAME ?? ""
|
|
87040
87911
|
});
|
|
87041
87912
|
if (BRIDGE_DEAD_ESCALATION_ENABLED) {
|
|
87042
87913
|
bridgeDeadWatchdog.arm();
|
|
@@ -93190,11 +94061,13 @@ function recordTypedModelSwitch(reply, requestedModelArg, _deps) {
|
|
|
93190
94061
|
if (!reply.selectedModel)
|
|
93191
94062
|
return "";
|
|
93192
94063
|
sessionModelSource.setOverride(reply.selectedModel);
|
|
94064
|
+
clearPremiumRecoveryOnManualSwitch(reply.selectedModel);
|
|
93193
94065
|
return "";
|
|
93194
94066
|
}
|
|
93195
94067
|
function recordModelMenuSideEffects(outcome, modelDeps, cbChatId, cbThreadId, prevSessionModel) {
|
|
93196
94068
|
if (outcome.selectedModel) {
|
|
93197
94069
|
sessionModelSource.setOverride(outcome.selectedModel);
|
|
94070
|
+
clearPremiumRecoveryOnManualSwitch(outcome.selectedModel);
|
|
93198
94071
|
}
|
|
93199
94072
|
if (outcome.clearedDefault) {
|
|
93200
94073
|
const smDir = resolveAgentDirFromEnv();
|
|
@@ -94093,6 +94966,80 @@ function broadcastFleetFallbackFailure(triggerAgent, reason) {
|
|
|
94093
94966
|
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(html), {}), { chat_id, verb: "fleet-fallback:failure-notify" });
|
|
94094
94967
|
}
|
|
94095
94968
|
}
|
|
94969
|
+
function broadcastTierNotice(markdown) {
|
|
94970
|
+
const access = loadAccess();
|
|
94971
|
+
if (access.allowFrom.length === 0)
|
|
94972
|
+
return;
|
|
94973
|
+
for (const chat_id of access.allowFrom) {
|
|
94974
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(markdown), { disable_notification: true }), { chat_id: String(chat_id), verb: "tier-downgrade:notify" });
|
|
94975
|
+
}
|
|
94976
|
+
}
|
|
94977
|
+
function maybeTierDowngrade(triggerAgent) {
|
|
94978
|
+
return runTierDowngrade(triggerAgent, {
|
|
94979
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
94980
|
+
getConfiguredDefault: () => {
|
|
94981
|
+
const dir = resolveAgentDirFromEnv();
|
|
94982
|
+
if (!dir)
|
|
94983
|
+
return null;
|
|
94984
|
+
return resolveMainModel(readConfiguredDefaultModel(dir) ?? undefined);
|
|
94985
|
+
},
|
|
94986
|
+
getSessionOverride: () => sessionModelSource.getOverride(),
|
|
94987
|
+
resolve: (t) => resolveMainModel(t),
|
|
94988
|
+
peekResumeGate: () => fleetFallbackResumeGate.peek(newestActiveTurnStartedAtMs()),
|
|
94989
|
+
writeCarrier: (dir, toModel, cfg) => writeSessionModelFile(dir, toModel, cfg),
|
|
94990
|
+
armResumeGate: () => fleetFallbackResumeGate.arm(),
|
|
94991
|
+
writeRecoveryMarker: (dir, premiumModel) => {
|
|
94992
|
+
const chats = loadAccess().allowFrom.map((c) => String(c));
|
|
94993
|
+
if (chats.length > 0)
|
|
94994
|
+
writePremiumRecoveryFile(dir, premiumModel, chats);
|
|
94995
|
+
},
|
|
94996
|
+
broadcastNotice: (md) => broadcastTierNotice(md),
|
|
94997
|
+
selfRestart: (agent) => triggerSelfRestart(agent, "tier-downgrade-resume"),
|
|
94998
|
+
selfAgent: (t) => process.env.SWITCHROOM_AGENT_NAME ?? t,
|
|
94999
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
95000
|
+
`)
|
|
95001
|
+
});
|
|
95002
|
+
}
|
|
95003
|
+
async function maybePremiumRecoveryPing(brokerClient, accounts) {
|
|
95004
|
+
return runPremiumRecoveryPing({
|
|
95005
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
95006
|
+
readMarker: (dir) => readPremiumRecoveryFile(dir),
|
|
95007
|
+
clearMarker: (dir) => clearPremiumRecoveryFile(dir),
|
|
95008
|
+
getAgent: () => getMyAgentName(),
|
|
95009
|
+
decide: () => decidePremiumRecovery({
|
|
95010
|
+
hasMarker: true,
|
|
95011
|
+
accounts: accounts.map((a) => ({
|
|
95012
|
+
exhausted: a.exhausted,
|
|
95013
|
+
premiumWalled: a.premium_walled === true
|
|
95014
|
+
}))
|
|
95015
|
+
}).fire,
|
|
95016
|
+
claimNotification: (key) => claimQuotaNotification(brokerClient, key),
|
|
95017
|
+
fallbackChats: () => loadAccess().allowFrom.map((c) => String(c)),
|
|
95018
|
+
sendToChat: (chat_id, ping, keyboard) => {
|
|
95019
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(ping.text), {
|
|
95020
|
+
disable_notification: true,
|
|
95021
|
+
reply_markup: keyboard
|
|
95022
|
+
}), { chat_id: String(chat_id), verb: "premium-recovery:notify" });
|
|
95023
|
+
},
|
|
95024
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
95025
|
+
`)
|
|
95026
|
+
});
|
|
95027
|
+
}
|
|
95028
|
+
function clearPremiumRecoveryOnManualSwitch(appliedToken) {
|
|
95029
|
+
if (appliedToken == null || appliedToken.length === 0)
|
|
95030
|
+
return;
|
|
95031
|
+
const agentDir = resolveAgentDirFromEnv();
|
|
95032
|
+
if (!agentDir)
|
|
95033
|
+
return;
|
|
95034
|
+
const marker = readPremiumRecoveryFile(agentDir);
|
|
95035
|
+
if (marker == null)
|
|
95036
|
+
return;
|
|
95037
|
+
if (resolveMainModel(appliedToken) === resolveMainModel(marker.premiumModel)) {
|
|
95038
|
+
clearPremiumRecoveryFile(agentDir);
|
|
95039
|
+
process.stderr.write(`telegram gateway: [premium-recovery] marker cleared \u2014 user manually re-issued /model ${marker.premiumModel}
|
|
95040
|
+
`);
|
|
95041
|
+
}
|
|
95042
|
+
}
|
|
94096
95043
|
async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, trigger) {
|
|
94097
95044
|
try {
|
|
94098
95045
|
const client3 = await getAuthBrokerClient2(triggerAgent);
|
|
@@ -94126,6 +95073,10 @@ async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, tri
|
|
|
94126
95073
|
if (outcome.kind === "switched") {
|
|
94127
95074
|
fallbackAllBlockedNoticeState = { lastSentAtMs: 0 };
|
|
94128
95075
|
} else if (outcome.kind === "all-blocked") {
|
|
95076
|
+
const tier = maybeTierDowngrade(triggerAgent);
|
|
95077
|
+
if (tier === "downgraded" || tier === "restart-pending") {
|
|
95078
|
+
return false;
|
|
95079
|
+
}
|
|
94129
95080
|
const verdict = evaluateAllBlockedNotice(fallbackAllBlockedNoticeState, Date.now());
|
|
94130
95081
|
if (!verdict.send) {
|
|
94131
95082
|
process.stderr.write(`telegram gateway: [fleet-fallback] all-blocked card suppressed (cooldown) agent=${triggerAgent}
|
|
@@ -94235,6 +95186,10 @@ async function runQuotaWatch(opts = {}) {
|
|
|
94235
95186
|
if (!listStateData.accounts || listStateData.accounts.length === 0) {
|
|
94236
95187
|
return;
|
|
94237
95188
|
}
|
|
95189
|
+
await maybePremiumRecoveryPing(brokerClient, listStateData.accounts).catch((err) => {
|
|
95190
|
+
process.stderr.write(`telegram gateway: [premium-recovery] ping check failed (non-fatal): ${err?.message ?? err}
|
|
95191
|
+
`);
|
|
95192
|
+
});
|
|
94238
95193
|
const snapshots = buildSnapshotsFromCachedState2(listStateData);
|
|
94239
95194
|
let watchState = loadQuotaWatchState(stateDir);
|
|
94240
95195
|
const now = Date.now();
|
|
@@ -97336,6 +98291,10 @@ var didOneTimeSetup = false;
|
|
|
97336
98291
|
const watcherAgentDir = resolveAgentDirFromEnv();
|
|
97337
98292
|
if (watcherAgentDir != null) {
|
|
97338
98293
|
const workerFeedEnabled = isWorkerActivityFeedEnabled(process.env.SWITCHROOM_WORKER_ACTIVITY_FEED);
|
|
98294
|
+
const workerFeedMaxRows = (() => {
|
|
98295
|
+
const raw = Number(process.env.SWITCHROOM_TG_WORKER_FEED_MAX_ROWS);
|
|
98296
|
+
return Number.isInteger(raw) && raw > 0 ? raw : undefined;
|
|
98297
|
+
})();
|
|
97339
98298
|
const foregroundNestingEnabled = process.env.SWITCHROOM_FOREGROUND_SUBAGENT_NESTING !== "0";
|
|
97340
98299
|
const orphanStatusEnabled = isOrphanSubagentStatusEnabled(process.env.SWITCHROOM_ORPHAN_SUBAGENT_STATUS);
|
|
97341
98300
|
workerActivityFeed?.stop();
|
|
@@ -97354,6 +98313,20 @@ var didOneTimeSetup = false;
|
|
|
97354
98313
|
})
|
|
97355
98314
|
},
|
|
97356
98315
|
floodWaitRemainingMs: probeFloodWaitRemainingMs,
|
|
98316
|
+
maxRows: workerFeedMaxRows,
|
|
98317
|
+
reconcilePin: ({ feedKey, chatId, messageId }) => {
|
|
98318
|
+
if (!PIN_STATUS_WHILE_WORKING)
|
|
98319
|
+
return;
|
|
98320
|
+
const key = `wk:group:${feedKey}`;
|
|
98321
|
+
if (messageId != null) {
|
|
98322
|
+
reconcileStatusPin(key, chatId, { pinned: true, messageId });
|
|
98323
|
+
} else {
|
|
98324
|
+
const unpinChat = chatId || statusPinChatIds.get(key);
|
|
98325
|
+
if (unpinChat != null && unpinChat.length > 0) {
|
|
98326
|
+
reconcileStatusPin(key, unpinChat, { pinned: false });
|
|
98327
|
+
}
|
|
98328
|
+
}
|
|
98329
|
+
},
|
|
97357
98330
|
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
97358
98331
|
`)
|
|
97359
98332
|
});
|
|
@@ -97419,7 +98392,6 @@ var didOneTimeSetup = false;
|
|
|
97419
98392
|
state: outcome === "failed" ? "failed" : "done",
|
|
97420
98393
|
model: dispatch.feedModel ?? undefined
|
|
97421
98394
|
});
|
|
97422
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97423
98395
|
}
|
|
97424
98396
|
return;
|
|
97425
98397
|
}
|
|
@@ -97467,7 +98439,6 @@ var didOneTimeSetup = false;
|
|
|
97467
98439
|
state: outcome === "failed" ? "failed" : "done",
|
|
97468
98440
|
model: dispatch.feedModel ?? undefined
|
|
97469
98441
|
});
|
|
97470
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97471
98442
|
}
|
|
97472
98443
|
return;
|
|
97473
98444
|
}
|
|
@@ -97481,7 +98452,6 @@ var didOneTimeSetup = false;
|
|
|
97481
98452
|
state: outcome === "failed" ? "failed" : "done",
|
|
97482
98453
|
model: dispatch.feedModel ?? undefined
|
|
97483
98454
|
});
|
|
97484
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97485
98455
|
}
|
|
97486
98456
|
const handbackOrigin = resolveSubagentOriginChat(agentId);
|
|
97487
98457
|
const decision = decideSubagentHandback({
|
|
@@ -97555,7 +98525,7 @@ var didOneTimeSetup = false;
|
|
|
97555
98525
|
elapsedMs,
|
|
97556
98526
|
state: "running",
|
|
97557
98527
|
model: feedModel
|
|
97558
|
-
}, wk.threadId)
|
|
98528
|
+
}, wk.threadId);
|
|
97559
98529
|
return;
|
|
97560
98530
|
}
|
|
97561
98531
|
if (surface !== "nest")
|
|
@@ -97614,7 +98584,7 @@ var didOneTimeSetup = false;
|
|
|
97614
98584
|
elapsedMs,
|
|
97615
98585
|
state: "running",
|
|
97616
98586
|
model: feedModel
|
|
97617
|
-
}, wk.threadId)
|
|
98587
|
+
}, wk.threadId);
|
|
97618
98588
|
return;
|
|
97619
98589
|
}
|
|
97620
98590
|
const progressOrigin = resolveSubagentOriginChat(agentId);
|