switchroom 0.18.15 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +16 -0
- package/dist/auth-broker/index.js +445 -10
- package/dist/cli/notion-write-pretool.mjs +16 -0
- package/dist/cli/switchroom.js +654 -479
- package/dist/host-control/main.js +20 -1
- package/dist/vault/approvals/kernel-server.js +16 -0
- package/dist/vault/broker/server.js +16 -0
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1758 -661
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +64 -22
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +627 -291
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +141 -172
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/runtime-metrics.ts +16 -0
- package/telegram-plugin/send-gate-degraded.test.ts +161 -8
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +246 -23
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +5 -4
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +8 -3
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +212 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +543 -368
|
@@ -20509,6 +20509,19 @@ var init_schema = __esm(() => {
|
|
|
20509
20509
|
approval_timeout_minutes: exports_external.number().int().nonnegative().optional().describe("Operator approval-card lifetime (minutes) for the tool-use 'Allow " + "once' card and the vault grant decision wait. After this long with " + "no operator tap, the card auto-denies (a TIMEOUT, not a denial \u2014 the " + "agent is told not to retry). Default 60. hostd-gated verbs " + "(mcp__hostd__*) keep their own longer window; the hostd " + "config-propose card is not governed by this key."),
|
|
20510
20510
|
sub_agent_tick_interval_ms: exports_external.number().int().nonnegative().optional().describe("Heartbeat tick interval (ms) for sub-agent rendering. Forces a " + "re-render of the elapsed-time counter while sub-agents are running, " + "even during silent stretches between tool calls. Default 10000 (10 s). " + "Set to 0 to disable the elapsed-ticker path."),
|
|
20511
20511
|
edit_budget_threshold: exports_external.number().int().nonnegative().optional().describe("Telegram API edit budget per minute before the progress-card driver " + "falls back to a slower coalesce window. When a chat accumulates more " + "than this many card edits in the trailing 60 s, the driver switches " + "to a wider coalesce interval until the rate drops back. Default 18. " + "Increase if your gateway frequently bumps the Telegram edit-rate ceiling " + "with many parallel sub-agents; decrease for a more conservative buffer."),
|
|
20512
|
+
send_gate: exports_external.object({
|
|
20513
|
+
enabled: exports_external.boolean().optional().describe("Master switch for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts) \u2014 the token-bucket scheduler every " + "Bot API call transits so per-surface throttles can't add up past a " + "flood ceiling. ON by default. Precedence: the operator break-glass " + "env var SWITCHROOM_TELEGRAM_SEND_GATE (0/false/off/no) ALWAYS wins " + "when explicitly set; this key only decides when that env var is " + "unset. Omit to keep the gate on."),
|
|
20514
|
+
global_per_sec: exports_external.number().positive().optional().describe("Global bucket sustained rate (Bot API calls/sec across ALL chats). " + "Default 25 (headroom under Telegram's ~30/s). Must be > 0 \u2014 a zero " + "or negative rate would wedge all outbound sends. Omit to keep 25."),
|
|
20515
|
+
global_burst: exports_external.number().int().positive().describe("Global bucket burst capacity. Default 4; worst-case 1s window " + "admits global_burst + global_per_sec = 29 < 30, a real margin under " + "the ceiling. Must be an integer >= 1 (a 0 capacity never admits a " + "token and wedges sends). Omit to keep 4.").optional(),
|
|
20516
|
+
per_chat_per_sec: exports_external.number().positive().optional().describe("Per-chat sustained rate (calls/sec to a single chat). Default 1. " + "Must be > 0 (zero/negative wedges that chat). Omit to keep 1."),
|
|
20517
|
+
per_chat_burst: exports_external.number().int().positive().optional().describe("Per-chat burst capacity. Default 3. Must be an integer >= 1 " + "(0 wedges the chat's bucket). Omit to keep 3."),
|
|
20518
|
+
per_group_per_min: exports_external.number().positive().optional().describe("Per-group sustained rate (calls/min to a single group/supergroup). " + "Default 18 (headroom under Telegram's ~20/min group ceiling). Must " + "be > 0. Omit to keep 18."),
|
|
20519
|
+
per_group_burst: exports_external.number().int().positive().optional().describe("Per-group burst capacity. Default 2. Must be an integer >= 1 " + "(0 wedges the group's bucket). Omit to keep 2."),
|
|
20520
|
+
edit_floor_ms: exports_external.number().int().nonnegative().optional().describe("Minimum ms between successive edits of the SAME message_id " + "(last-write-wins coalescing enforces this floor). Default 1500 " + "(Telegram's ~1 edit/sec/message practical ceiling). 0 disables the " + "floor. Must be an integer >= 0. Omit to keep 1500.")
|
|
20521
|
+
}).optional().describe("Tunable rate limits for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts). Every key is optional and defaults to " + "the send gate's built-in value, so omitting the whole block reproduces " + "today's exact behaviour \u2014 this is pure operator tuning, no default is " + "changed. Cascades from defaults.channels.telegram.send_gate."),
|
|
20522
|
+
worker_feed: exports_external.object({
|
|
20523
|
+
max_rows: exports_external.number().int().positive().optional().describe("Max live-worker rows rendered in the COMBINED worker-activity feed " + "(2+ background workers in one chat/thread coalesce into ONE message; " + "telegram-plugin/worker-activity-feed.ts) before a compact " + "'+M more working\u2026' spill line \u2014 keeps the coalesced body compact and " + "legible (and under the rich-message wire ceiling). Default 8. A " + "single-worker chat renders the full \uD83D\uDEE0 Worker card and ignores this. " + "Must be an integer >= 1. Omit to keep 8.")
|
|
20524
|
+
}).optional().describe("Tuning for the coalesced worker-activity feed " + "(telegram-plugin/worker-activity-feed.ts). Cascades from " + "defaults.channels.telegram.worker_feed."),
|
|
20512
20525
|
stickers: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Sticker aliases for the `send_sticker` MCP tool (#576). Maps a " + "short alias name (e.g. 'happy', 'thinking') to a Telegram file_id. " + "Operator-curated \u2014 capture file_ids from inbound stickers the user " + "sends and add them here. The agent calls send_sticker(chat_id, " + "alias='happy') and the gateway resolves to the file_id at send " + "time. Aliases enable persona-flavored expressiveness without " + "exposing raw file_ids in the agent prompt. Personal-assistant / " + "health-coach personas benefit; coding agents typically don't " + "configure any."),
|
|
20513
20526
|
voice_in: exports_external.object({
|
|
20514
20527
|
enabled: exports_external.boolean().optional().describe("Master switch for voice-message transcription."),
|
|
@@ -20534,6 +20547,9 @@ var init_schema = __esm(() => {
|
|
|
20534
20547
|
window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
|
|
20535
20548
|
max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 \u2014 a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, \u2026). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
|
|
20536
20549
|
}).optional().describe("Inbound coalescing \u2014 how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
|
|
20550
|
+
litellm_notice: exports_external.object({
|
|
20551
|
+
window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 \u2014 see docs/auth.md \u00a7 " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
|
|
20552
|
+
}).optional().describe("Debounce tuning for the litellm-local throttle notice \u2014 the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
|
|
20537
20553
|
interrupt: exports_external.object({
|
|
20538
20554
|
safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
|
|
20539
20555
|
max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short \u2014 the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")
|
|
@@ -40235,6 +40251,96 @@ function renderActivityFeedWithNested(lines, childLines, final = false, liveSuff
|
|
|
40235
40251
|
stepCount
|
|
40236
40252
|
});
|
|
40237
40253
|
}
|
|
40254
|
+
var COMBINED_ROW_DESC_MAX = 72;
|
|
40255
|
+
function renderCombinedWorkerFeed(rows, opts) {
|
|
40256
|
+
if (rows.length === 0)
|
|
40257
|
+
return null;
|
|
40258
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows));
|
|
40259
|
+
const rowLines = (r) => {
|
|
40260
|
+
const desc = escapeMarkdown(truncate(stripMarkdown(r.description).replace(/\s+/g, " ").trim() || "background task", COMBINED_ROW_DESC_MAX));
|
|
40261
|
+
const toolWord = r.toolCount === 1 ? "tool" : "tools";
|
|
40262
|
+
const modelLabel = formatModelLabel(r.model);
|
|
40263
|
+
const modelPart = modelLabel != null ? ` \u00b7 ${escapeMarkdown(modelLabel)}` : "";
|
|
40264
|
+
const header = `**${desc}** _\u00b7 ${formatFeedElapsed(r.elapsedMs)} \u00b7 ${r.toolCount} ${toolWord}${modelPart}_`;
|
|
40265
|
+
const stepClean = stripMarkdown(r.currentStep).replace(/\s+/g, " ").trim();
|
|
40266
|
+
const step = stepClean.length > 0 ? `\u2192 _${escapeMarkdown(truncate(stepClean, STATUS_LINE_MAX))}_` : `\u2192 _starting\u2026_`;
|
|
40267
|
+
return [header, step];
|
|
40268
|
+
};
|
|
40269
|
+
const compose = (visibleCount) => {
|
|
40270
|
+
const shown = rows.slice(0, visibleCount);
|
|
40271
|
+
const hidden = rows.length - shown.length;
|
|
40272
|
+
const out = [`\uD83D\uDEE0 **Workers** \u00b7 _${rows.length} running_`];
|
|
40273
|
+
for (const r of shown) {
|
|
40274
|
+
const [h, s] = rowLines(r);
|
|
40275
|
+
out.push(h, s);
|
|
40276
|
+
}
|
|
40277
|
+
if (hidden > 0)
|
|
40278
|
+
out.push(`_+${hidden} more working\u2026_`);
|
|
40279
|
+
return stackCardLines(out);
|
|
40280
|
+
};
|
|
40281
|
+
let visible = Math.min(rows.length, maxRows);
|
|
40282
|
+
let body = compose(visible);
|
|
40283
|
+
while (body.length > STATUS_CARD_CHAR_BUDGET && visible > 1) {
|
|
40284
|
+
visible -= 1;
|
|
40285
|
+
body = compose(visible);
|
|
40286
|
+
}
|
|
40287
|
+
return body;
|
|
40288
|
+
}
|
|
40289
|
+
|
|
40290
|
+
// retry-api-call.ts
|
|
40291
|
+
var import_grammy = __toESM(require_mod2(), 1);
|
|
40292
|
+
var LOCAL_RESOURCE_EXHAUSTED = "LOCAL_RESOURCE_EXHAUSTED";
|
|
40293
|
+
var FLOOD_WAIT_ACTIVE = "FLOOD_WAIT_ACTIVE";
|
|
40294
|
+
function makeFloodWaitActiveError(retryAfterSec, untilTs, original) {
|
|
40295
|
+
return Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
|
|
40296
|
+
retryAfterSec,
|
|
40297
|
+
untilTs,
|
|
40298
|
+
error_code: 429,
|
|
40299
|
+
parameters: { retry_after: retryAfterSec },
|
|
40300
|
+
original
|
|
40301
|
+
});
|
|
40302
|
+
}
|
|
40303
|
+
function isFloodWaitActiveError(err) {
|
|
40304
|
+
return err instanceof Error && err.message === FLOOD_WAIT_ACTIVE;
|
|
40305
|
+
}
|
|
40306
|
+
async function retryWithThreadFallback(retry, send, opts) {
|
|
40307
|
+
try {
|
|
40308
|
+
return await retry(() => send(opts.threadId), {
|
|
40309
|
+
...opts.threadId != null ? { threadId: opts.threadId } : {},
|
|
40310
|
+
chat_id: opts.chat_id,
|
|
40311
|
+
...opts.verb != null ? { verb: opts.verb } : {}
|
|
40312
|
+
});
|
|
40313
|
+
} catch (err) {
|
|
40314
|
+
if (err instanceof Error && err.message === "THREAD_NOT_FOUND") {
|
|
40315
|
+
return await retry(() => send(undefined), {
|
|
40316
|
+
chat_id: opts.chat_id,
|
|
40317
|
+
...opts.verb != null ? { verb: opts.verb } : {}
|
|
40318
|
+
});
|
|
40319
|
+
}
|
|
40320
|
+
throw err;
|
|
40321
|
+
}
|
|
40322
|
+
}
|
|
40323
|
+
function isHtmlParseRejectError(err) {
|
|
40324
|
+
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
40325
|
+
return false;
|
|
40326
|
+
if (isMessageTooLongError(err))
|
|
40327
|
+
return false;
|
|
40328
|
+
const d = (err.description || "").toLowerCase();
|
|
40329
|
+
return d.includes("can't parse entities") || d.includes("can\u2019t parse entities") || d.includes("can't parse") || d.includes("can\u2019t parse") || d.includes("parse markdown") || d.includes("parse rich") || d.includes("unsupported start tag") || d.includes("unclosed start tag") || d.includes("can't find end of the entity") || d.includes("can\u2019t find end of the entity") || d.includes("expected end tag");
|
|
40330
|
+
}
|
|
40331
|
+
function isMessageTooLongError(err) {
|
|
40332
|
+
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
40333
|
+
return false;
|
|
40334
|
+
const d = (err.description || "").toLowerCase();
|
|
40335
|
+
return d.includes("rich_message_text_too_long") || d.includes("message_too_long") || d.includes("text_too_long") || d.includes("message is too long") || d.includes("text is too long");
|
|
40336
|
+
}
|
|
40337
|
+
|
|
40338
|
+
// send-gate.ts
|
|
40339
|
+
var SEND_GATE_SHED = Symbol.for("switchroom.send-gate.shed");
|
|
40340
|
+
function isSendGateShed(value) {
|
|
40341
|
+
return value === SEND_GATE_SHED;
|
|
40342
|
+
}
|
|
40343
|
+
var GROUP_TYPES = new Set(["group", "supergroup"]);
|
|
40238
40344
|
|
|
40239
40345
|
// worker-activity-feed.ts
|
|
40240
40346
|
function isWorkerActivityFeedEnabled(envVal) {
|
|
@@ -40310,16 +40416,20 @@ function classifyEditError(err) {
|
|
|
40310
40416
|
function createWorkerActivityFeed(opts) {
|
|
40311
40417
|
const log = opts.log ?? (() => {});
|
|
40312
40418
|
const nowFn = opts.now ?? Date.now;
|
|
40419
|
+
const floodWaitRemainingMs = opts.floodWaitRemainingMs ?? (() => 0);
|
|
40313
40420
|
const minEditInterval = opts.minEditIntervalMs ?? 2500;
|
|
40314
40421
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000;
|
|
40315
40422
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000;
|
|
40423
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8));
|
|
40424
|
+
const reconcilePinFn = opts.reconcilePin ?? (() => {});
|
|
40316
40425
|
const setIntervalFn = opts.setInterval ?? ((cb, ms) => {
|
|
40317
40426
|
const t = setInterval(cb, ms);
|
|
40318
40427
|
t.unref?.();
|
|
40319
40428
|
return t;
|
|
40320
40429
|
});
|
|
40321
40430
|
const clearIntervalFn = opts.clearInterval ?? ((handle) => clearInterval(handle));
|
|
40322
|
-
const
|
|
40431
|
+
const groups = new Map;
|
|
40432
|
+
const agentIndex = new Map;
|
|
40323
40433
|
const finalized = new Set;
|
|
40324
40434
|
const FINALIZED_CAP = 256;
|
|
40325
40435
|
function markFinalized(agentId) {
|
|
@@ -40333,242 +40443,356 @@ function createWorkerActivityFeed(opts) {
|
|
|
40333
40443
|
}
|
|
40334
40444
|
}
|
|
40335
40445
|
let heartbeatTimer = null;
|
|
40336
|
-
function
|
|
40446
|
+
function feedKeyOf(chatId, threadId) {
|
|
40447
|
+
return `${chatId} ${threadId ?? ""}`;
|
|
40448
|
+
}
|
|
40449
|
+
function groupOfAgent(agentId) {
|
|
40450
|
+
const key = agentIndex.get(agentId);
|
|
40451
|
+
return key != null ? groups.get(key) : undefined;
|
|
40452
|
+
}
|
|
40453
|
+
function sendOptsFor(g) {
|
|
40337
40454
|
return {
|
|
40338
40455
|
disable_web_page_preview: true,
|
|
40339
40456
|
disable_notification: true,
|
|
40340
|
-
...
|
|
40457
|
+
...g.threadId != null ? { message_thread_id: g.threadId } : {}
|
|
40341
40458
|
};
|
|
40342
40459
|
}
|
|
40343
|
-
function noteRateLimited(
|
|
40460
|
+
function noteRateLimited(g, err, label) {
|
|
40344
40461
|
const retryAfter = extractRetryAfterSecs(err);
|
|
40345
40462
|
if (retryAfter == null)
|
|
40346
40463
|
return;
|
|
40347
|
-
|
|
40464
|
+
g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS;
|
|
40348
40465
|
log(`worker-feed: ${label} 429 \u2014 backing off ${retryAfter}s`);
|
|
40349
40466
|
}
|
|
40350
|
-
function
|
|
40467
|
+
function parkIfFloodWindowOpen(g) {
|
|
40468
|
+
const remaining = floodWaitRemainingMs();
|
|
40469
|
+
if (remaining <= 0)
|
|
40470
|
+
return false;
|
|
40471
|
+
const until = nowFn() + remaining + COOLDOWN_JITTER_MS;
|
|
40472
|
+
if (until > g.cooldownUntil)
|
|
40473
|
+
g.cooldownUntil = until;
|
|
40474
|
+
return true;
|
|
40475
|
+
}
|
|
40476
|
+
function accumulateNarrative(row, view) {
|
|
40351
40477
|
const line = view.latestSummary.trim();
|
|
40352
40478
|
if (line.length === 0)
|
|
40353
40479
|
return;
|
|
40354
|
-
if (
|
|
40480
|
+
if (row.narrative.includes(line))
|
|
40355
40481
|
return;
|
|
40356
|
-
|
|
40357
|
-
|
|
40358
|
-
if (
|
|
40359
|
-
|
|
40360
|
-
}
|
|
40361
|
-
}
|
|
40362
|
-
|
|
40363
|
-
|
|
40364
|
-
|
|
40365
|
-
|
|
40366
|
-
|
|
40367
|
-
|
|
40368
|
-
|
|
40369
|
-
|
|
40370
|
-
const
|
|
40371
|
-
if (
|
|
40372
|
-
if (
|
|
40373
|
-
return;
|
|
40374
|
-
|
|
40375
|
-
|
|
40376
|
-
|
|
40377
|
-
|
|
40378
|
-
|
|
40379
|
-
|
|
40380
|
-
|
|
40381
|
-
|
|
40382
|
-
|
|
40482
|
+
row.narrative.push(line);
|
|
40483
|
+
row.stepStartedAtMs = nowFn();
|
|
40484
|
+
if (row.narrative.length > STATUS_ROLLING_LINES) {
|
|
40485
|
+
row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES);
|
|
40486
|
+
}
|
|
40487
|
+
}
|
|
40488
|
+
function liveElapsed(row, now) {
|
|
40489
|
+
const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0;
|
|
40490
|
+
return Math.max(base, row.lastView?.elapsedMs ?? 0);
|
|
40491
|
+
}
|
|
40492
|
+
function runningRows(g) {
|
|
40493
|
+
return [...g.workers.values()].filter((w) => w.state === "running" && w.lastView != null).sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0));
|
|
40494
|
+
}
|
|
40495
|
+
function renderGroupBody(g, now, terminalRecap, heartbeat) {
|
|
40496
|
+
const running = runningRows(g);
|
|
40497
|
+
if (running.length === 0) {
|
|
40498
|
+
if (terminalRecap == null)
|
|
40499
|
+
return null;
|
|
40500
|
+
return renderWorkerActivity(terminalRecap);
|
|
40501
|
+
}
|
|
40502
|
+
const elapsedFor = (r) => heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now);
|
|
40503
|
+
if (running.length === 1) {
|
|
40504
|
+
const r = running[0];
|
|
40505
|
+
const view = {
|
|
40506
|
+
...r.lastView,
|
|
40507
|
+
elapsedMs: elapsedFor(r),
|
|
40508
|
+
narrativeLines: [...r.narrative]
|
|
40509
|
+
};
|
|
40510
|
+
let liveSuffix = "";
|
|
40511
|
+
if (heartbeat) {
|
|
40512
|
+
const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now);
|
|
40513
|
+
liveSuffix = formatStepSuffix(stepElapsed);
|
|
40383
40514
|
}
|
|
40515
|
+
return renderWorkerActivity(view, liveSuffix);
|
|
40516
|
+
}
|
|
40517
|
+
const rows = running.map((r) => {
|
|
40518
|
+
const v = r.lastView;
|
|
40519
|
+
const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary;
|
|
40520
|
+
return {
|
|
40521
|
+
description: v.description,
|
|
40522
|
+
elapsedMs: elapsedFor(r),
|
|
40523
|
+
toolCount: v.toolCount,
|
|
40524
|
+
currentStep,
|
|
40525
|
+
model: v.model
|
|
40526
|
+
};
|
|
40527
|
+
});
|
|
40528
|
+
return renderCombinedWorkerFeed(rows, { maxRows });
|
|
40529
|
+
}
|
|
40530
|
+
function removeWorker(g, agentId) {
|
|
40531
|
+
g.workers.delete(agentId);
|
|
40532
|
+
agentIndex.delete(agentId);
|
|
40533
|
+
if (g.workers.size === 0)
|
|
40534
|
+
groups.delete(g.feedKey);
|
|
40535
|
+
}
|
|
40536
|
+
function syncPin(g) {
|
|
40537
|
+
const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null;
|
|
40538
|
+
reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId });
|
|
40539
|
+
}
|
|
40540
|
+
async function doRender(g, opts2 = {}) {
|
|
40541
|
+
const now = nowFn();
|
|
40542
|
+
const isTerminal = opts2.terminalRecap != null;
|
|
40543
|
+
const settleTerminal = () => {
|
|
40544
|
+
g.pendingFinalize = null;
|
|
40545
|
+
if (opts2.finishingAgentId != null)
|
|
40546
|
+
removeWorker(g, opts2.finishingAgentId);
|
|
40547
|
+
syncPin(g);
|
|
40548
|
+
};
|
|
40549
|
+
if (now < g.cooldownUntil) {
|
|
40550
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40551
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40384
40552
|
return;
|
|
40385
40553
|
}
|
|
40386
|
-
if (
|
|
40554
|
+
if (parkIfFloodWindowOpen(g)) {
|
|
40555
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40556
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40387
40557
|
return;
|
|
40388
|
-
|
|
40558
|
+
}
|
|
40559
|
+
const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false);
|
|
40560
|
+
if (body == null) {
|
|
40561
|
+
if (isTerminal)
|
|
40562
|
+
settleTerminal();
|
|
40389
40563
|
return;
|
|
40390
|
-
|
|
40391
|
-
|
|
40392
|
-
|
|
40393
|
-
|
|
40394
|
-
|
|
40395
|
-
} catch (err) {
|
|
40396
|
-
const outcome = classifyEditError(err);
|
|
40397
|
-
if (outcome === "rate_limited") {
|
|
40398
|
-
noteRateLimited(h, err, "edit");
|
|
40399
|
-
return;
|
|
40400
|
-
}
|
|
40401
|
-
if (outcome === "not_modified") {
|
|
40402
|
-
h.lastBody = body;
|
|
40403
|
-
h.lastEditAt = nowFn();
|
|
40564
|
+
}
|
|
40565
|
+
if (g.messageId == null) {
|
|
40566
|
+
const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)));
|
|
40567
|
+
if (isTerminal) {
|
|
40568
|
+
settleTerminal();
|
|
40404
40569
|
return;
|
|
40405
40570
|
}
|
|
40406
|
-
if (
|
|
40407
|
-
h.messageId = null;
|
|
40408
|
-
h.lastBody = null;
|
|
40571
|
+
if (maxElapsed < firstPaintMin)
|
|
40409
40572
|
return;
|
|
40573
|
+
try {
|
|
40574
|
+
const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g));
|
|
40575
|
+
if (sent == null || typeof sent.message_id !== "number") {
|
|
40576
|
+
parkIfFloodWindowOpen(g);
|
|
40577
|
+
log(`worker-feed: first paint shed by send gate feed=${g.feedKey} \u2014 not delivered`);
|
|
40578
|
+
return;
|
|
40579
|
+
}
|
|
40580
|
+
g.messageId = sent.message_id;
|
|
40581
|
+
g.lastBody = body;
|
|
40582
|
+
g.lastEditAt = now;
|
|
40583
|
+
syncPin(g);
|
|
40584
|
+
log(`worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
|
|
40585
|
+
} catch (err) {
|
|
40586
|
+
noteRateLimited(g, err, "send");
|
|
40587
|
+
log(`worker-feed: send failed: ${err.message}`);
|
|
40410
40588
|
}
|
|
40411
|
-
log(`worker-feed: edit transient error agent=${h.agentId}: ${err.message}`);
|
|
40412
|
-
}
|
|
40413
|
-
}
|
|
40414
|
-
async function doFinish(h, view) {
|
|
40415
|
-
h.finished = true;
|
|
40416
|
-
markFinalized(h.agentId);
|
|
40417
|
-
if (h.messageId == null) {
|
|
40418
|
-
h.pendingFinish = null;
|
|
40419
40589
|
return;
|
|
40420
40590
|
}
|
|
40421
|
-
if (
|
|
40422
|
-
|
|
40591
|
+
if (body === g.lastBody) {
|
|
40592
|
+
if (isTerminal)
|
|
40593
|
+
settleTerminal();
|
|
40423
40594
|
return;
|
|
40424
40595
|
}
|
|
40425
|
-
|
|
40426
|
-
if (body === h.lastBody) {
|
|
40427
|
-
h.pendingFinish = null;
|
|
40596
|
+
if (!opts2.force && now - g.lastEditAt < minEditInterval)
|
|
40428
40597
|
return;
|
|
40429
|
-
}
|
|
40430
40598
|
try {
|
|
40431
|
-
await opts.bot.editMessageText(
|
|
40432
|
-
|
|
40433
|
-
|
|
40434
|
-
|
|
40435
|
-
|
|
40599
|
+
const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g));
|
|
40600
|
+
if (isSendGateShed(res)) {
|
|
40601
|
+
parkIfFloodWindowOpen(g);
|
|
40602
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40603
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40604
|
+
return;
|
|
40605
|
+
}
|
|
40606
|
+
g.lastBody = body;
|
|
40607
|
+
g.lastEditAt = now;
|
|
40608
|
+
if (isTerminal) {
|
|
40609
|
+
log(`worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? "-"} ` + `msgId=${g.messageId} agent=${opts2.finishingAgentId ?? "-"} ` + `state=${opts2.terminalRecap?.state ?? "done"} bytes=${body.length}`);
|
|
40610
|
+
} else {
|
|
40611
|
+
log(`worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
|
|
40612
|
+
}
|
|
40613
|
+
if (isTerminal)
|
|
40614
|
+
settleTerminal();
|
|
40436
40615
|
} catch (err) {
|
|
40437
40616
|
const outcome = classifyEditError(err);
|
|
40438
40617
|
if (outcome === "rate_limited") {
|
|
40439
|
-
noteRateLimited(
|
|
40440
|
-
|
|
40618
|
+
noteRateLimited(g, err, isTerminal ? "finish" : "edit");
|
|
40619
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40620
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40441
40621
|
return;
|
|
40442
40622
|
}
|
|
40443
40623
|
if (outcome === "not_modified") {
|
|
40444
|
-
|
|
40445
|
-
|
|
40446
|
-
|
|
40624
|
+
g.lastBody = body;
|
|
40625
|
+
g.lastEditAt = now;
|
|
40626
|
+
if (isTerminal)
|
|
40627
|
+
settleTerminal();
|
|
40447
40628
|
return;
|
|
40448
40629
|
}
|
|
40449
40630
|
if (outcome === "gone") {
|
|
40450
|
-
|
|
40631
|
+
g.messageId = null;
|
|
40632
|
+
g.lastBody = null;
|
|
40633
|
+
if (isTerminal)
|
|
40634
|
+
settleTerminal();
|
|
40635
|
+
else
|
|
40636
|
+
syncPin(g);
|
|
40451
40637
|
return;
|
|
40452
40638
|
}
|
|
40453
|
-
|
|
40454
|
-
|
|
40639
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40640
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40641
|
+
log(`worker-feed: edit transient error feed=${g.feedKey}: ${err.message}`);
|
|
40455
40642
|
}
|
|
40456
40643
|
}
|
|
40457
40644
|
function heartbeatTick() {
|
|
40458
40645
|
const now = nowFn();
|
|
40459
|
-
for (const
|
|
40460
|
-
if (
|
|
40461
|
-
|
|
40462
|
-
|
|
40463
|
-
|
|
40464
|
-
|
|
40465
|
-
log(`worker-feed: heartbeat finalize re-drive error ${h.agentId}: ${err.message}`);
|
|
40466
|
-
}).finally(() => {
|
|
40467
|
-
if (handles.get(h.agentId)?.pendingFinish == null) {
|
|
40468
|
-
handles.delete(h.agentId);
|
|
40469
|
-
}
|
|
40646
|
+
for (const g of [...groups.values()]) {
|
|
40647
|
+
if (g.pendingFinalize != null && now >= g.cooldownUntil) {
|
|
40648
|
+
const recap = g.pendingFinalize;
|
|
40649
|
+
const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId;
|
|
40650
|
+
g.chain = g.chain.then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId })).catch((err) => {
|
|
40651
|
+
log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${err.message}`);
|
|
40470
40652
|
});
|
|
40471
40653
|
continue;
|
|
40472
40654
|
}
|
|
40473
|
-
if (
|
|
40655
|
+
if (now < g.cooldownUntil)
|
|
40474
40656
|
continue;
|
|
40475
|
-
|
|
40657
|
+
const running = runningRows(g);
|
|
40658
|
+
if (running.length === 0)
|
|
40476
40659
|
continue;
|
|
40477
|
-
if (
|
|
40478
|
-
|
|
40479
|
-
|
|
40480
|
-
if (h.messageId == null) {
|
|
40481
|
-
if (liveElapsed < firstPaintMin)
|
|
40660
|
+
if (g.messageId == null) {
|
|
40661
|
+
const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)));
|
|
40662
|
+
if (maxElapsed < firstPaintMin)
|
|
40482
40663
|
continue;
|
|
40483
|
-
|
|
40484
|
-
|
|
40485
|
-
log(`worker-feed: heartbeat first-paint chain error ${h.agentId}: ${err.message}`);
|
|
40664
|
+
g.chain = g.chain.then(() => doRender(g, {})).catch((err) => {
|
|
40665
|
+
log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${err.message}`);
|
|
40486
40666
|
});
|
|
40487
40667
|
continue;
|
|
40488
40668
|
}
|
|
40489
|
-
if (now -
|
|
40669
|
+
if (now - g.lastEditAt < minEditInterval)
|
|
40490
40670
|
continue;
|
|
40491
|
-
const stale = now -
|
|
40671
|
+
const stale = now - g.lastEditAt >= heartbeatTickMs;
|
|
40492
40672
|
if (!stale)
|
|
40493
40673
|
continue;
|
|
40494
|
-
|
|
40495
|
-
|
|
40496
|
-
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) };
|
|
40497
|
-
h.chain = h.chain.then(() => doUpdate(h, view, liveSuffix)).catch((err) => {
|
|
40498
|
-
log(`worker-feed: heartbeat chain error ${h.agentId}: ${err.message}`);
|
|
40674
|
+
g.chain = g.chain.then(() => doRender(g, { heartbeat: true })).catch((err) => {
|
|
40675
|
+
log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${err.message}`);
|
|
40499
40676
|
});
|
|
40500
40677
|
}
|
|
40501
40678
|
}
|
|
40502
40679
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs);
|
|
40503
40680
|
return {
|
|
40504
40681
|
has(agentId) {
|
|
40505
|
-
|
|
40682
|
+
const g = groupOfAgent(agentId);
|
|
40683
|
+
return g != null && g.messageId != null && g.workers.has(agentId);
|
|
40506
40684
|
},
|
|
40507
40685
|
messageIdOf(agentId) {
|
|
40508
|
-
return
|
|
40686
|
+
return groupOfAgent(agentId)?.messageId ?? null;
|
|
40687
|
+
},
|
|
40688
|
+
hasRunningInFeed(feedKey) {
|
|
40689
|
+
const g = groups.get(feedKey);
|
|
40690
|
+
return g != null && g.workers.size > 0;
|
|
40509
40691
|
},
|
|
40510
40692
|
get size() {
|
|
40511
|
-
|
|
40693
|
+
let n = 0;
|
|
40694
|
+
for (const g of groups.values())
|
|
40695
|
+
n += g.workers.size;
|
|
40696
|
+
return n;
|
|
40512
40697
|
},
|
|
40513
40698
|
update(agentId, chatId, view, threadId) {
|
|
40514
40699
|
if (chatId.length === 0)
|
|
40515
40700
|
return Promise.resolve();
|
|
40516
40701
|
if (finalized.has(agentId))
|
|
40517
40702
|
return Promise.resolve();
|
|
40518
|
-
const
|
|
40519
|
-
if (
|
|
40703
|
+
const existingRow = groupOfAgent(agentId)?.workers.get(agentId);
|
|
40704
|
+
if (existingRow?.finished === true)
|
|
40520
40705
|
return Promise.resolve();
|
|
40521
|
-
|
|
40522
|
-
|
|
40523
|
-
|
|
40524
|
-
|
|
40706
|
+
const feedKey = feedKeyOf(chatId, threadId);
|
|
40707
|
+
let g = groups.get(feedKey);
|
|
40708
|
+
if (g == null) {
|
|
40709
|
+
g = {
|
|
40710
|
+
feedKey,
|
|
40525
40711
|
chatId,
|
|
40526
40712
|
threadId,
|
|
40527
40713
|
messageId: null,
|
|
40528
40714
|
lastBody: null,
|
|
40529
40715
|
lastEditAt: 0,
|
|
40530
40716
|
cooldownUntil: 0,
|
|
40531
|
-
narrative: [],
|
|
40532
40717
|
chain: Promise.resolve(),
|
|
40718
|
+
workers: new Map,
|
|
40719
|
+
pendingFinalize: null
|
|
40720
|
+
};
|
|
40721
|
+
groups.set(feedKey, g);
|
|
40722
|
+
}
|
|
40723
|
+
let row = g.workers.get(agentId);
|
|
40724
|
+
if (row == null) {
|
|
40725
|
+
row = {
|
|
40726
|
+
agentId,
|
|
40727
|
+
narrative: [],
|
|
40533
40728
|
lastView: null,
|
|
40534
|
-
|
|
40535
|
-
stepStartedAtMs: null,
|
|
40729
|
+
state: "running",
|
|
40536
40730
|
finished: false,
|
|
40537
|
-
|
|
40731
|
+
dispatchAtMs: null,
|
|
40732
|
+
stepStartedAtMs: null
|
|
40538
40733
|
};
|
|
40539
|
-
|
|
40540
|
-
|
|
40541
|
-
|
|
40542
|
-
|
|
40734
|
+
g.workers.set(agentId, row);
|
|
40735
|
+
agentIndex.set(agentId, feedKey);
|
|
40736
|
+
}
|
|
40737
|
+
accumulateNarrative(row, view);
|
|
40738
|
+
row.state = "running";
|
|
40739
|
+
row.lastView = { ...view, narrativeLines: [...row.narrative] };
|
|
40740
|
+
if (row.dispatchAtMs == null)
|
|
40741
|
+
row.dispatchAtMs = nowFn() - view.elapsedMs;
|
|
40742
|
+
const group = g;
|
|
40743
|
+
group.chain = group.chain.then(() => doRender(group)).catch((err) => {
|
|
40543
40744
|
log(`worker-feed: update chain error ${agentId}: ${err.message}`);
|
|
40544
40745
|
});
|
|
40545
|
-
return
|
|
40746
|
+
return group.chain;
|
|
40546
40747
|
},
|
|
40547
40748
|
finish(agentId, view) {
|
|
40548
|
-
const
|
|
40549
|
-
|
|
40749
|
+
const g = groupOfAgent(agentId);
|
|
40750
|
+
const row = g?.workers.get(agentId);
|
|
40751
|
+
if (g == null || row == null) {
|
|
40752
|
+
markFinalized(agentId);
|
|
40550
40753
|
return Promise.resolve();
|
|
40551
|
-
|
|
40754
|
+
}
|
|
40755
|
+
row.finished = true;
|
|
40756
|
+
row.state = view.state === "failed" ? "failed" : "done";
|
|
40757
|
+
markFinalized(agentId);
|
|
40758
|
+
const group = g;
|
|
40759
|
+
group.chain = group.chain.then(() => {
|
|
40760
|
+
const others = runningRows(group).filter((w) => w.agentId !== agentId);
|
|
40761
|
+
if (others.length > 0) {
|
|
40762
|
+
removeWorker(group, agentId);
|
|
40763
|
+
syncPin(group);
|
|
40764
|
+
return doRender(group, { force: true });
|
|
40765
|
+
}
|
|
40766
|
+
const recap = { ...view, narrativeLines: [...row.narrative] };
|
|
40767
|
+
return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId });
|
|
40768
|
+
}).catch((err) => {
|
|
40552
40769
|
log(`worker-feed: finish chain error ${agentId}: ${err.message}`);
|
|
40553
|
-
}).finally(() => {
|
|
40554
|
-
if (handles.get(agentId)?.pendingFinish == null) {
|
|
40555
|
-
handles.delete(agentId);
|
|
40556
|
-
}
|
|
40557
40770
|
});
|
|
40558
|
-
return
|
|
40771
|
+
return group.chain;
|
|
40559
40772
|
},
|
|
40560
40773
|
drop(agentId) {
|
|
40561
40774
|
markFinalized(agentId);
|
|
40562
|
-
|
|
40775
|
+
const g = groupOfAgent(agentId);
|
|
40776
|
+
if (g == null)
|
|
40777
|
+
return;
|
|
40778
|
+
const hadMessage = g.messageId != null;
|
|
40779
|
+
removeWorker(g, agentId);
|
|
40780
|
+
if (hadMessage)
|
|
40781
|
+
syncPin(g);
|
|
40782
|
+
if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
|
|
40783
|
+
g.chain = g.chain.then(() => doRender(g, { force: true })).catch((err) => {
|
|
40784
|
+
log(`worker-feed: drop re-render error ${agentId}: ${err.message}`);
|
|
40785
|
+
});
|
|
40786
|
+
}
|
|
40563
40787
|
},
|
|
40564
40788
|
resurrect(agentId) {
|
|
40565
40789
|
const wasFinalized = finalized.delete(agentId);
|
|
40566
|
-
const
|
|
40567
|
-
if (
|
|
40568
|
-
|
|
40569
|
-
|
|
40790
|
+
const row = groupOfAgent(agentId)?.workers.get(agentId);
|
|
40791
|
+
if (row != null) {
|
|
40792
|
+
row.finished = false;
|
|
40793
|
+
row.state = "running";
|
|
40570
40794
|
}
|
|
40571
|
-
if (wasFinalized ||
|
|
40795
|
+
if (wasFinalized || row != null) {
|
|
40572
40796
|
log(`worker-feed: resurrect agent=${agentId} \u2014 cleared finalized gate; card will repaint on next running cue`);
|
|
40573
40797
|
}
|
|
40574
40798
|
},
|
|
@@ -41022,55 +41246,6 @@ var STALE_TAP_NOTICE = "This request already resolved (timed out) \u2014 ask aga
|
|
|
41022
41246
|
var import_grammy2 = __toESM(require_mod2(), 1);
|
|
41023
41247
|
var import_runner = __toESM(require_mod4(), 1);
|
|
41024
41248
|
import { AsyncLocalStorage } from "async_hooks";
|
|
41025
|
-
|
|
41026
|
-
// retry-api-call.ts
|
|
41027
|
-
var import_grammy = __toESM(require_mod2(), 1);
|
|
41028
|
-
var FLOOD_WAIT_ACTIVE = "FLOOD_WAIT_ACTIVE";
|
|
41029
|
-
function makeFloodWaitActiveError(retryAfterSec, untilTs, original) {
|
|
41030
|
-
return Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
|
|
41031
|
-
retryAfterSec,
|
|
41032
|
-
untilTs,
|
|
41033
|
-
error_code: 429,
|
|
41034
|
-
parameters: { retry_after: retryAfterSec },
|
|
41035
|
-
original
|
|
41036
|
-
});
|
|
41037
|
-
}
|
|
41038
|
-
function isFloodWaitActiveError(err) {
|
|
41039
|
-
return err instanceof Error && err.message === FLOOD_WAIT_ACTIVE;
|
|
41040
|
-
}
|
|
41041
|
-
async function retryWithThreadFallback(retry, send, opts) {
|
|
41042
|
-
try {
|
|
41043
|
-
return await retry(() => send(opts.threadId), {
|
|
41044
|
-
...opts.threadId != null ? { threadId: opts.threadId } : {},
|
|
41045
|
-
chat_id: opts.chat_id,
|
|
41046
|
-
...opts.verb != null ? { verb: opts.verb } : {}
|
|
41047
|
-
});
|
|
41048
|
-
} catch (err) {
|
|
41049
|
-
if (err instanceof Error && err.message === "THREAD_NOT_FOUND") {
|
|
41050
|
-
return await retry(() => send(undefined), {
|
|
41051
|
-
chat_id: opts.chat_id,
|
|
41052
|
-
...opts.verb != null ? { verb: opts.verb } : {}
|
|
41053
|
-
});
|
|
41054
|
-
}
|
|
41055
|
-
throw err;
|
|
41056
|
-
}
|
|
41057
|
-
}
|
|
41058
|
-
function isHtmlParseRejectError(err) {
|
|
41059
|
-
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
41060
|
-
return false;
|
|
41061
|
-
if (isMessageTooLongError(err))
|
|
41062
|
-
return false;
|
|
41063
|
-
const d = (err.description || "").toLowerCase();
|
|
41064
|
-
return d.includes("can't parse entities") || d.includes("can\u2019t parse entities") || d.includes("can't parse") || d.includes("can\u2019t parse") || d.includes("parse markdown") || d.includes("parse rich") || d.includes("unsupported start tag") || d.includes("unclosed start tag") || d.includes("can't find end of the entity") || d.includes("can\u2019t find end of the entity") || d.includes("expected end tag");
|
|
41065
|
-
}
|
|
41066
|
-
function isMessageTooLongError(err) {
|
|
41067
|
-
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
41068
|
-
return false;
|
|
41069
|
-
const d = (err.description || "").toLowerCase();
|
|
41070
|
-
return d.includes("rich_message_text_too_long") || d.includes("message_too_long") || d.includes("text_too_long") || d.includes("message is too long") || d.includes("text is too long");
|
|
41071
|
-
}
|
|
41072
|
-
|
|
41073
|
-
// shared/bot-runtime.ts
|
|
41074
41249
|
init_flood_circuit_breaker();
|
|
41075
41250
|
|
|
41076
41251
|
// shared/gw-trace-gate.ts
|
|
@@ -44548,6 +44723,14 @@ function holdReasonFor(err) {
|
|
|
44548
44723
|
function isHeldUndeliverable2(pend) {
|
|
44549
44724
|
return pend.undeliverable != null;
|
|
44550
44725
|
}
|
|
44726
|
+
function applyDeliveredHoldReset(entry, now) {
|
|
44727
|
+
if (entry.undeliverable == null)
|
|
44728
|
+
return false;
|
|
44729
|
+
entry.undeliverable = null;
|
|
44730
|
+
entry.redeliveryFailures = 0;
|
|
44731
|
+
entry.startedAt = now;
|
|
44732
|
+
return true;
|
|
44733
|
+
}
|
|
44551
44734
|
var HELD_CARD_REDELIVERY_CAP = 3;
|
|
44552
44735
|
function selectHeldForRedelivery(entries, opts) {
|
|
44553
44736
|
const empty = { send: [], deferred: [] };
|
|
@@ -45307,6 +45490,12 @@ function createTypingEmitter(deps) {
|
|
|
45307
45490
|
|
|
45308
45491
|
// draft-stream.ts
|
|
45309
45492
|
var TELEGRAM_MAX_CHARS = 32768;
|
|
45493
|
+
function makeDraftEditShedError(messageId) {
|
|
45494
|
+
return Object.assign(new Error(`draft edit shed by send gate (id=${messageId ?? "unknown"}); snapshot preserved for re-flush`), { draftEditShed: true });
|
|
45495
|
+
}
|
|
45496
|
+
function isDraftEditShedError(err) {
|
|
45497
|
+
return err instanceof Error && err.draftEditShed === true;
|
|
45498
|
+
}
|
|
45310
45499
|
var DEFAULT_DM_THROTTLE_MS = 400;
|
|
45311
45500
|
var DEFAULT_GROUP_THROTTLE_MS = 1000;
|
|
45312
45501
|
var MIN_THROTTLE_MS = 250;
|
|
@@ -45325,6 +45514,7 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45325
45514
|
let messageId = config.initialMessageId ?? null;
|
|
45326
45515
|
let pendingText = null;
|
|
45327
45516
|
let lastSentText = null;
|
|
45517
|
+
let shedText = null;
|
|
45328
45518
|
let lastSentAt = 0;
|
|
45329
45519
|
let inFlight = null;
|
|
45330
45520
|
const streamStartedAt = Date.now();
|
|
@@ -45368,11 +45558,16 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45368
45558
|
await sendViaMessage(textToSend);
|
|
45369
45559
|
lastSentText = textToSend;
|
|
45370
45560
|
lastSentAt = Date.now();
|
|
45561
|
+
shedText = null;
|
|
45371
45562
|
} catch (err) {
|
|
45372
45563
|
const msg = err.message ?? String(err);
|
|
45373
|
-
if (
|
|
45564
|
+
if (isDraftEditShedError(err)) {
|
|
45565
|
+
shedText = textToSend;
|
|
45566
|
+
log?.(`stream \u2192 shed by send gate (id: ${messageId}); snapshot preserved for re-flush`);
|
|
45567
|
+
} else if (/\bmessage is not modified\b/i.test(msg)) {
|
|
45374
45568
|
lastSentText = textToSend;
|
|
45375
45569
|
lastSentAt = Date.now();
|
|
45570
|
+
shedText = null;
|
|
45376
45571
|
log?.(`stream \u2192 not modified (id: ${messageId})`);
|
|
45377
45572
|
} else if (/\bmessage to edit not found\b/i.test(msg) || /\bMESSAGE_ID_INVALID\b/i.test(msg)) {
|
|
45378
45573
|
log?.(`stream \u2192 message not found (id: ${messageId}), re-sending`);
|
|
@@ -45465,10 +45660,12 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45465
45660
|
}
|
|
45466
45661
|
return waitPromise;
|
|
45467
45662
|
},
|
|
45468
|
-
async finalize() {
|
|
45663
|
+
async finalize(finalText) {
|
|
45469
45664
|
if (final)
|
|
45470
45665
|
return;
|
|
45471
45666
|
final = true;
|
|
45667
|
+
if (finalText != null && !stopped)
|
|
45668
|
+
pendingText = finalText;
|
|
45472
45669
|
if (scheduledTimer != null) {
|
|
45473
45670
|
clearTimeout(scheduledTimer);
|
|
45474
45671
|
scheduledTimer = null;
|
|
@@ -45476,6 +45673,10 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45476
45673
|
if (inFlight) {
|
|
45477
45674
|
await inFlight;
|
|
45478
45675
|
}
|
|
45676
|
+
if (pendingText == null && shedText != null && !stopped) {
|
|
45677
|
+
pendingText = shedText;
|
|
45678
|
+
}
|
|
45679
|
+
shedText = null;
|
|
45479
45680
|
if (pendingText != null && !stopped) {
|
|
45480
45681
|
await flush();
|
|
45481
45682
|
}
|
|
@@ -54912,28 +55113,41 @@ function createStreamController(cfg) {
|
|
|
54912
55113
|
return bot.api.editMessageText(chatId, id, piece.text, opts);
|
|
54913
55114
|
return bot.api.editMessageText(chatId, id, richMessage(piece.text), opts);
|
|
54914
55115
|
};
|
|
55116
|
+
let handleRef = null;
|
|
55117
|
+
const editGateOpts = (id, payload) => ({
|
|
55118
|
+
threadId,
|
|
55119
|
+
chat_id: chatId,
|
|
55120
|
+
messageId: id,
|
|
55121
|
+
editPayload: payload,
|
|
55122
|
+
priorityClass: handleRef?.isFinal() === true ? "critical" : "cosmetic"
|
|
55123
|
+
});
|
|
55124
|
+
const piecePayload = (piece) => piece.rich ? richMessage(piece.text) : piece.text;
|
|
54915
55125
|
const tailIds = [];
|
|
54916
55126
|
const tailLastText = [];
|
|
54917
55127
|
const upsertTail = async (ti, piece) => {
|
|
54918
55128
|
const existingId = tailIds[ti];
|
|
54919
55129
|
if (existingId != null) {
|
|
54920
55130
|
if (tailLastText[ti] === piece.text)
|
|
54921
|
-
return;
|
|
55131
|
+
return false;
|
|
54922
55132
|
try {
|
|
54923
|
-
await retry(() => editPiece(existingId, piece, baseOpts),
|
|
55133
|
+
const res = await retry(() => editPiece(existingId, piece, baseOpts), editGateOpts(existingId, piecePayload(piece)));
|
|
55134
|
+
if (isSendGateShed(res))
|
|
55135
|
+
return true;
|
|
54924
55136
|
tailLastText[ti] = piece.text;
|
|
54925
55137
|
onEdit?.(existingId, piece.text.length);
|
|
54926
55138
|
} catch (err) {
|
|
54927
55139
|
if (!literalText && piece.rich && isParseEntitiesError(err)) {
|
|
54928
55140
|
warn?.(`stream-controller: tail-piece #${ti + 1} edit parse-entities rejected \u2014 retrying same id=${existingId} as plain text (${err instanceof Error ? err.message : String(err)})`);
|
|
54929
|
-
await retry(() => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts),
|
|
55141
|
+
const res = await retry(() => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts), editGateOpts(existingId, piece.text));
|
|
55142
|
+
if (isSendGateShed(res))
|
|
55143
|
+
return true;
|
|
54930
55144
|
tailLastText[ti] = piece.text;
|
|
54931
55145
|
onEdit?.(existingId, piece.text.length);
|
|
54932
55146
|
} else {
|
|
54933
55147
|
warn?.(`stream-controller: tail-piece #${ti + 1} edit FAILED (id=${existingId}) \u2014 partial delivery, this piece may be stale (${err instanceof Error ? err.message : String(err)})`);
|
|
54934
55148
|
}
|
|
54935
55149
|
}
|
|
54936
|
-
return;
|
|
55150
|
+
return false;
|
|
54937
55151
|
}
|
|
54938
55152
|
try {
|
|
54939
55153
|
const sent = await retry(() => sendPiece(piece, sendOpts), { threadId, chat_id: chatId });
|
|
@@ -54951,8 +55165,9 @@ function createStreamController(cfg) {
|
|
|
54951
55165
|
warn?.(`stream-controller: tail-piece #${ti + 1} send FAILED \u2014 partial delivery, this and later pieces may be missing this flush (${err instanceof Error ? err.message : String(err)})`);
|
|
54952
55166
|
}
|
|
54953
55167
|
}
|
|
55168
|
+
return false;
|
|
54954
55169
|
};
|
|
54955
|
-
|
|
55170
|
+
const handle = createDraftStream(async (text4) => {
|
|
54956
55171
|
const pieces = renderPieces(text4);
|
|
54957
55172
|
const head = pieces[0];
|
|
54958
55173
|
let anchorId;
|
|
@@ -54977,22 +55192,34 @@ function createStreamController(cfg) {
|
|
|
54977
55192
|
}, async (id, text4) => {
|
|
54978
55193
|
const pieces = renderPieces(text4);
|
|
54979
55194
|
const head = pieces[0];
|
|
55195
|
+
let anchorShed = false;
|
|
54980
55196
|
try {
|
|
54981
|
-
await retry(() => editPiece(id, head, baseOpts),
|
|
54982
|
-
|
|
55197
|
+
const res = await retry(() => editPiece(id, head, baseOpts), editGateOpts(id, piecePayload(head)));
|
|
55198
|
+
if (isSendGateShed(res)) {
|
|
55199
|
+
anchorShed = true;
|
|
55200
|
+
} else {
|
|
55201
|
+
onEdit?.(id, head.text.length);
|
|
55202
|
+
}
|
|
54983
55203
|
} catch (err) {
|
|
54984
55204
|
if (!literalText && head.rich && isParseEntitiesError(err)) {
|
|
54985
55205
|
warn?.(`stream-controller: edit parse-entities rejected \u2014 retrying same id=${id} as plain text (${err instanceof Error ? err.message : String(err)})`);
|
|
54986
55206
|
const fallbackBody = pieces.length === 1 ? text4 : head.text;
|
|
54987
|
-
await retry(() => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts),
|
|
54988
|
-
|
|
55207
|
+
const res = await retry(() => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts), editGateOpts(id, fallbackBody));
|
|
55208
|
+
if (isSendGateShed(res))
|
|
55209
|
+
anchorShed = true;
|
|
55210
|
+
else
|
|
55211
|
+
onEdit?.(id, head.text.length);
|
|
54989
55212
|
} else {
|
|
54990
55213
|
throw err;
|
|
54991
55214
|
}
|
|
54992
55215
|
}
|
|
55216
|
+
let anyTailShed = false;
|
|
54993
55217
|
for (let pi = 1;pi < pieces.length; pi++) {
|
|
54994
|
-
await upsertTail(pi - 1, pieces[pi])
|
|
55218
|
+
if (await upsertTail(pi - 1, pieces[pi]))
|
|
55219
|
+
anyTailShed = true;
|
|
54995
55220
|
}
|
|
55221
|
+
if (anchorShed || anyTailShed)
|
|
55222
|
+
throw makeDraftEditShedError(id);
|
|
54996
55223
|
}, {
|
|
54997
55224
|
...throttleMs != null ? { throttleMs } : {},
|
|
54998
55225
|
...idleMs != null ? { idleMs } : {},
|
|
@@ -55002,6 +55229,8 @@ function createStreamController(cfg) {
|
|
|
55002
55229
|
...initialMessageId != null ? { initialMessageId } : {},
|
|
55003
55230
|
chatId
|
|
55004
55231
|
});
|
|
55232
|
+
handleRef = handle;
|
|
55233
|
+
return handle;
|
|
55005
55234
|
}
|
|
55006
55235
|
|
|
55007
55236
|
// pty-partial-handler.ts
|
|
@@ -55133,7 +55362,7 @@ function isLocalResourceError(err) {
|
|
|
55133
55362
|
const msg = err instanceof Error ? err.message : String(err ?? "");
|
|
55134
55363
|
return /\b(ENOSPC|EDQUOT|EIO|ENOMEM)\b/.test(msg) || /no space left on device/i.test(msg) || /disk quota exceeded/i.test(msg);
|
|
55135
55364
|
}
|
|
55136
|
-
var
|
|
55365
|
+
var LOCAL_RESOURCE_EXHAUSTED2 = "LOCAL_RESOURCE_EXHAUSTED";
|
|
55137
55366
|
var GIVE_UP_MESSAGE = "retryApiCall: max retries exceeded";
|
|
55138
55367
|
var FLOOD_WAIT_ACTIVE2 = "FLOOD_WAIT_ACTIVE";
|
|
55139
55368
|
function makeFloodWaitActiveError2(retryAfterSec, untilTs, original) {
|
|
@@ -55189,13 +55418,13 @@ function createRetryApiCall2(config = {}) {
|
|
|
55189
55418
|
log?.(`telegram gateway: LOCAL resource exhaustion (${err.code ?? "disk/mem"}) \u2014 ` + `not retrying the send (would feed a flood ban); surfacing degraded state
|
|
55190
55419
|
`);
|
|
55191
55420
|
observer?.onGiveUp?.({ attempts: attempt + 1, error: err });
|
|
55192
|
-
throw Object.assign(new Error(
|
|
55421
|
+
throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED2), { original: err });
|
|
55193
55422
|
}
|
|
55194
55423
|
if (isGrammyErr && err.error_code === 429) {
|
|
55195
55424
|
const retryAfter = Number(err.parameters?.retry_after ?? 5);
|
|
55196
55425
|
const delayMs = retryAfter * 1000;
|
|
55197
55426
|
try {
|
|
55198
|
-
onFloodWait?.(retryAfter);
|
|
55427
|
+
onFloodWait?.(retryAfter, opts);
|
|
55199
55428
|
} catch {}
|
|
55200
55429
|
if (delayMs > maxFloodSleepMs) {
|
|
55201
55430
|
log?.(`telegram gateway: 429 flood ban of ${retryAfter}s exceeds the ` + `${Math.round(maxFloodSleepMs / 1000)}s in-process sleep ceiling \u2014 ` + `not sleeping it; surfacing degraded state
|
|
@@ -55285,7 +55514,7 @@ var systemClock = {
|
|
|
55285
55514
|
sleep: (ms) => new Promise((r) => setTimeout(r, ms))
|
|
55286
55515
|
};
|
|
55287
55516
|
var UNTAGGED_SEND_CLASS = "critical";
|
|
55288
|
-
|
|
55517
|
+
var SEND_GATE_SHED2 = Symbol.for("switchroom.send-gate.shed");
|
|
55289
55518
|
class TokenBucket {
|
|
55290
55519
|
capacity;
|
|
55291
55520
|
refillPerMs;
|
|
@@ -55363,17 +55592,26 @@ function stableStringify(value) {
|
|
|
55363
55592
|
return v;
|
|
55364
55593
|
});
|
|
55365
55594
|
}
|
|
55366
|
-
var
|
|
55595
|
+
var GROUP_TYPES2 = new Set(["group", "supergroup"]);
|
|
55596
|
+
var SEND_GATE_DEFAULTS = {
|
|
55597
|
+
globalPerSec: 25,
|
|
55598
|
+
globalBurst: 4,
|
|
55599
|
+
perChatPerSec: 1,
|
|
55600
|
+
perChatBurst: 3,
|
|
55601
|
+
perGroupPerMin: 18,
|
|
55602
|
+
perGroupBurst: 2,
|
|
55603
|
+
editFloorMs: 1500
|
|
55604
|
+
};
|
|
55367
55605
|
function createSendGate(config) {
|
|
55368
55606
|
const enabled2 = config.enabled;
|
|
55369
55607
|
const clock = config.clock ?? systemClock;
|
|
55370
|
-
const globalPerSec = config.globalPerSec ??
|
|
55371
|
-
const globalBurst = config.globalBurst ??
|
|
55372
|
-
const perChatPerSec = config.perChatPerSec ??
|
|
55373
|
-
const perChatBurst = config.perChatBurst ??
|
|
55374
|
-
const perGroupPerMin = config.perGroupPerMin ??
|
|
55375
|
-
const perGroupBurst = config.perGroupBurst ??
|
|
55376
|
-
const editFloorMs = config.editFloorMs ??
|
|
55608
|
+
const globalPerSec = config.globalPerSec ?? SEND_GATE_DEFAULTS.globalPerSec;
|
|
55609
|
+
const globalBurst = config.globalBurst ?? SEND_GATE_DEFAULTS.globalBurst;
|
|
55610
|
+
const perChatPerSec = config.perChatPerSec ?? SEND_GATE_DEFAULTS.perChatPerSec;
|
|
55611
|
+
const perChatBurst = config.perChatBurst ?? SEND_GATE_DEFAULTS.perChatBurst;
|
|
55612
|
+
const perGroupPerMin = config.perGroupPerMin ?? SEND_GATE_DEFAULTS.perGroupPerMin;
|
|
55613
|
+
const perGroupBurst = config.perGroupBurst ?? SEND_GATE_DEFAULTS.perGroupBurst;
|
|
55614
|
+
const editFloorMs = config.editFloorMs ?? SEND_GATE_DEFAULTS.editFloorMs;
|
|
55377
55615
|
const messageStateTtlMs = config.messageStateTtlMs ?? 60000;
|
|
55378
55616
|
const maxMessageStates = config.maxMessageStates ?? 5000;
|
|
55379
55617
|
const usefulTtlMs = config.usefulTtlMs ?? 120000;
|
|
@@ -55381,6 +55619,7 @@ function createSendGate(config) {
|
|
|
55381
55619
|
const criticalJitterMaxMs = config.criticalJitterMaxMs ?? 250;
|
|
55382
55620
|
const jitter = config.jitter ?? Math.random;
|
|
55383
55621
|
const onWindowOpen = config.onWindowOpen;
|
|
55622
|
+
const conservativeGlobalFloodScope = config.conservativeGlobalFloodScope ?? false;
|
|
55384
55623
|
const counters = {
|
|
55385
55624
|
sent: 0,
|
|
55386
55625
|
queued: 0,
|
|
@@ -55439,7 +55678,7 @@ function createSendGate(config) {
|
|
|
55439
55678
|
const buckets = [globalBucket];
|
|
55440
55679
|
if (opts?.chat_id) {
|
|
55441
55680
|
buckets.push(chatBucket(opts.chat_id));
|
|
55442
|
-
if (opts.chatType &&
|
|
55681
|
+
if (opts.chatType && GROUP_TYPES2.has(opts.chatType)) {
|
|
55443
55682
|
buckets.push(groupBucket(opts.chat_id));
|
|
55444
55683
|
}
|
|
55445
55684
|
}
|
|
@@ -55451,10 +55690,13 @@ function createSendGate(config) {
|
|
|
55451
55690
|
applyWindow(scopeKey, untilTs, true);
|
|
55452
55691
|
}
|
|
55453
55692
|
function openScopedWindowsForOpts(opts, untilTs) {
|
|
55454
|
-
|
|
55455
|
-
if (
|
|
55693
|
+
const hasChatScope = opts?.chat_id != null && opts.chat_id !== "";
|
|
55694
|
+
if (!hasChatScope || conservativeGlobalFloodScope) {
|
|
55695
|
+
applyWindow("global", untilTs, true);
|
|
55696
|
+
}
|
|
55697
|
+
if (hasChatScope) {
|
|
55456
55698
|
applyWindow(`chat:${opts.chat_id}`, untilTs, true);
|
|
55457
|
-
if (opts.chatType &&
|
|
55699
|
+
if (opts.chatType && GROUP_TYPES2.has(opts.chatType)) {
|
|
55458
55700
|
applyWindow(`group:${opts.chat_id}`, untilTs, true);
|
|
55459
55701
|
}
|
|
55460
55702
|
}
|
|
@@ -55668,7 +55910,7 @@ function createSendGate(config) {
|
|
|
55668
55910
|
const msgWait = state.suppressedUntilMs > now ? state.suppressedUntilMs - now : 0;
|
|
55669
55911
|
if (wait > 0 || msgWait > 0) {
|
|
55670
55912
|
counters.shed++;
|
|
55671
|
-
return Promise.resolve(
|
|
55913
|
+
return Promise.resolve(SEND_GATE_SHED2);
|
|
55672
55914
|
}
|
|
55673
55915
|
}
|
|
55674
55916
|
if (state.pending) {
|
|
@@ -55713,7 +55955,7 @@ function createSendGate(config) {
|
|
|
55713
55955
|
const outcome = await admitPriority(bucketsFor(opts), priority);
|
|
55714
55956
|
if (outcome.result === "shed") {
|
|
55715
55957
|
counters.shed++;
|
|
55716
|
-
return;
|
|
55958
|
+
return SEND_GATE_SHED2;
|
|
55717
55959
|
}
|
|
55718
55960
|
if (outcome.result === "expired") {
|
|
55719
55961
|
counters.expired++;
|
|
@@ -55754,14 +55996,114 @@ function createSendGate(config) {
|
|
|
55754
55996
|
}
|
|
55755
55997
|
};
|
|
55756
55998
|
}
|
|
55757
|
-
|
|
55999
|
+
function openScopedFloodWindows(opts, untilTs) {
|
|
56000
|
+
if (!enabled2)
|
|
56001
|
+
return;
|
|
56002
|
+
openScopedWindowsForOpts(opts, untilTs);
|
|
56003
|
+
}
|
|
56004
|
+
return { gate, openFloodWindow, openScopedFloodWindows, stats };
|
|
55758
56005
|
}
|
|
55759
|
-
function
|
|
55760
|
-
const v = env.SWITCHROOM_TELEGRAM_SEND_GATE;
|
|
55761
|
-
if (v == null)
|
|
55762
|
-
return true;
|
|
56006
|
+
function isOffValue(v) {
|
|
55763
56007
|
const t = v.trim().toLowerCase();
|
|
55764
|
-
return
|
|
56008
|
+
return t === "0" || t === "false" || t === "off" || t === "no";
|
|
56009
|
+
}
|
|
56010
|
+
function parsePositiveNumber(raw) {
|
|
56011
|
+
if (raw == null || raw.trim() === "")
|
|
56012
|
+
return;
|
|
56013
|
+
const n = Number(raw);
|
|
56014
|
+
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
56015
|
+
}
|
|
56016
|
+
function parsePositiveInt(raw) {
|
|
56017
|
+
if (raw == null || raw.trim() === "")
|
|
56018
|
+
return;
|
|
56019
|
+
const n = Number(raw);
|
|
56020
|
+
return Number.isInteger(n) && n > 0 ? n : undefined;
|
|
56021
|
+
}
|
|
56022
|
+
function parseNonNegativeInt(raw) {
|
|
56023
|
+
if (raw == null || raw.trim() === "")
|
|
56024
|
+
return;
|
|
56025
|
+
const n = Number(raw);
|
|
56026
|
+
return Number.isInteger(n) && n >= 0 ? n : undefined;
|
|
56027
|
+
}
|
|
56028
|
+
function parseBoolFlag(raw) {
|
|
56029
|
+
if (raw == null || raw.trim() === "")
|
|
56030
|
+
return;
|
|
56031
|
+
const t = raw.trim().toLowerCase();
|
|
56032
|
+
if (t === "1" || t === "true" || t === "on" || t === "yes")
|
|
56033
|
+
return true;
|
|
56034
|
+
if (isOffValue(t))
|
|
56035
|
+
return false;
|
|
56036
|
+
return;
|
|
56037
|
+
}
|
|
56038
|
+
function sendGateConfigFromEnv(env = process.env) {
|
|
56039
|
+
const killSwitch = env.SWITCHROOM_TELEGRAM_SEND_GATE;
|
|
56040
|
+
let enabled2;
|
|
56041
|
+
if (killSwitch != null && killSwitch.trim() !== "") {
|
|
56042
|
+
enabled2 = !isOffValue(killSwitch);
|
|
56043
|
+
} else {
|
|
56044
|
+
const cfg = env.SWITCHROOM_TG_SEND_GATE_ENABLED;
|
|
56045
|
+
enabled2 = cfg != null && cfg.trim() !== "" ? !isOffValue(cfg) : true;
|
|
56046
|
+
}
|
|
56047
|
+
const out = { enabled: enabled2 };
|
|
56048
|
+
const globalPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_PER_SEC);
|
|
56049
|
+
if (globalPerSec !== undefined)
|
|
56050
|
+
out.globalPerSec = globalPerSec;
|
|
56051
|
+
const globalBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_BURST);
|
|
56052
|
+
if (globalBurst !== undefined)
|
|
56053
|
+
out.globalBurst = globalBurst;
|
|
56054
|
+
const perChatPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_PER_SEC);
|
|
56055
|
+
if (perChatPerSec !== undefined)
|
|
56056
|
+
out.perChatPerSec = perChatPerSec;
|
|
56057
|
+
const perChatBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_BURST);
|
|
56058
|
+
if (perChatBurst !== undefined)
|
|
56059
|
+
out.perChatBurst = perChatBurst;
|
|
56060
|
+
const perGroupPerMin = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_PER_MIN);
|
|
56061
|
+
if (perGroupPerMin !== undefined)
|
|
56062
|
+
out.perGroupPerMin = perGroupPerMin;
|
|
56063
|
+
const perGroupBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_BURST);
|
|
56064
|
+
if (perGroupBurst !== undefined)
|
|
56065
|
+
out.perGroupBurst = perGroupBurst;
|
|
56066
|
+
const editFloorMs = parseNonNegativeInt(env.SWITCHROOM_TG_SEND_GATE_EDIT_FLOOR_MS);
|
|
56067
|
+
if (editFloorMs !== undefined)
|
|
56068
|
+
out.editFloorMs = editFloorMs;
|
|
56069
|
+
const conservativeGlobal = parseBoolFlag(env.SWITCHROOM_TG_SEND_GATE_CONSERVATIVE_GLOBAL);
|
|
56070
|
+
if (conservativeGlobal !== undefined)
|
|
56071
|
+
out.conservativeGlobalFloodScope = conservativeGlobal;
|
|
56072
|
+
return out;
|
|
56073
|
+
}
|
|
56074
|
+
|
|
56075
|
+
// shared/local-time.ts
|
|
56076
|
+
function fmtLocalClock(ms, tz) {
|
|
56077
|
+
return new Intl.DateTimeFormat("en-US", {
|
|
56078
|
+
timeZone: tz,
|
|
56079
|
+
hour: "numeric",
|
|
56080
|
+
minute: "2-digit",
|
|
56081
|
+
hour12: true
|
|
56082
|
+
}).format(new Date(ms)).replace(/\s([AP])M$/, (_m, p) => `${p.toLowerCase()}m`);
|
|
56083
|
+
}
|
|
56084
|
+
function fmtLocalDate(ms, tz) {
|
|
56085
|
+
return new Intl.DateTimeFormat("en-GB", {
|
|
56086
|
+
timeZone: tz,
|
|
56087
|
+
day: "numeric",
|
|
56088
|
+
month: "short"
|
|
56089
|
+
}).format(new Date(ms));
|
|
56090
|
+
}
|
|
56091
|
+
function localDay(ms, tz) {
|
|
56092
|
+
return new Intl.DateTimeFormat("en-CA", {
|
|
56093
|
+
timeZone: tz,
|
|
56094
|
+
year: "numeric",
|
|
56095
|
+
month: "2-digit",
|
|
56096
|
+
day: "2-digit"
|
|
56097
|
+
}).format(new Date(ms));
|
|
56098
|
+
}
|
|
56099
|
+
function tzAbbrev(ms, tz) {
|
|
56100
|
+
const at = new Date(ms);
|
|
56101
|
+
for (const loc of ["en-US", "en-AU", "en-GB"]) {
|
|
56102
|
+
const v = new Intl.DateTimeFormat(loc, { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value;
|
|
56103
|
+
if (v && !/^(?:GMT|UTC)/i.test(v))
|
|
56104
|
+
return v;
|
|
56105
|
+
}
|
|
56106
|
+
return new Intl.DateTimeFormat("en-US", { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value ?? tz;
|
|
55765
56107
|
}
|
|
55766
56108
|
|
|
55767
56109
|
// send-gate-observability.ts
|
|
@@ -55798,8 +56140,18 @@ function createStatsLogger(config) {
|
|
|
55798
56140
|
function isAlertableScope(scopeKey) {
|
|
55799
56141
|
return scopeKey === "global" || scopeKey.startsWith("chat:") || scopeKey.startsWith("group:");
|
|
55800
56142
|
}
|
|
55801
|
-
function
|
|
55802
|
-
|
|
56143
|
+
function fmtLocalStamp(ms, tz, refMs) {
|
|
56144
|
+
const clock = fmtLocalClock(ms, tz);
|
|
56145
|
+
const sameDay = refMs != null && localDay(ms, tz) === localDay(refMs, tz);
|
|
56146
|
+
const body = sameDay ? clock : `${fmtLocalDate(ms, tz)} ${clock}`;
|
|
56147
|
+
return `${body} ${tzAbbrev(ms, tz)}`;
|
|
56148
|
+
}
|
|
56149
|
+
function fmtLocalRange(startMs, endMs, tz) {
|
|
56150
|
+
const sameDay = localDay(startMs, tz) === localDay(endMs, tz);
|
|
56151
|
+
if (sameDay) {
|
|
56152
|
+
return `${fmtLocalClock(startMs, tz)} to ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
|
|
56153
|
+
}
|
|
56154
|
+
return `${fmtLocalDate(startMs, tz)} ${fmtLocalClock(startMs, tz)} to ` + `${fmtLocalDate(endMs, tz)} ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
|
|
55803
56155
|
}
|
|
55804
56156
|
function fmtDur(ms) {
|
|
55805
56157
|
const s = Math.round(ms / 1000);
|
|
@@ -55811,6 +56163,7 @@ function fmtDur(ms) {
|
|
|
55811
56163
|
}
|
|
55812
56164
|
function createFloodWindowObserver(config) {
|
|
55813
56165
|
const alertThresholdMs = config.alertThresholdMs ?? 60000;
|
|
56166
|
+
const tz = config.tz ?? "UTC";
|
|
55814
56167
|
let lastSeen = new Map;
|
|
55815
56168
|
let firstTick = true;
|
|
55816
56169
|
const owedOnClose = new Set;
|
|
@@ -55831,14 +56184,14 @@ function createFloodWindowObserver(config) {
|
|
|
55831
56184
|
}
|
|
55832
56185
|
function openAlertText(w, now) {
|
|
55833
56186
|
const openFor = fmtDur(now - w.observedAt);
|
|
55834
|
-
return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${
|
|
56187
|
+
return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${fmtLocalStamp(w.untilTs, tz, now)}. ` + `Outbound to that scope is being suppressed.`;
|
|
55835
56188
|
}
|
|
55836
56189
|
function closeAlertText(recs) {
|
|
55837
56190
|
const observedAt = Math.min(...recs.map((r) => r.observedAt));
|
|
55838
56191
|
const untilTs = Math.max(...recs.map((r) => r.untilTs));
|
|
55839
56192
|
const scopes = recs.map((r) => `\`${r.scopeKey}\``).join(", ");
|
|
55840
56193
|
const plural = recs.length > 1 ? "s" : "";
|
|
55841
|
-
return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${
|
|
56194
|
+
return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${fmtLocalRange(observedAt, untilTs, tz)} ` + `(~${fmtDur(untilTs - observedAt)}). Some outbound messages during that ` + `window were suppressed.`;
|
|
55842
56195
|
}
|
|
55843
56196
|
async function tick() {
|
|
55844
56197
|
const s = config.stats();
|
|
@@ -63707,18 +64060,28 @@ function createFleetFallbackResumeGate(opts = {}) {
|
|
|
63707
64060
|
const maxAgeMs = opts.maxAgeMs ?? DEFAULT_RESUME_MAX_AGE_MS;
|
|
63708
64061
|
const singleFlightMs = opts.singleFlightMs ?? DEFAULT_RESUME_SINGLE_FLIGHT_MS;
|
|
63709
64062
|
let lastResumedAtMs = Number.NEGATIVE_INFINITY;
|
|
63710
|
-
function
|
|
64063
|
+
function peek(failedTurnStartedAtMs) {
|
|
63711
64064
|
const now = nowFn();
|
|
63712
64065
|
if (now - lastResumedAtMs < singleFlightMs)
|
|
63713
64066
|
return "skip-inflight";
|
|
63714
64067
|
if (failedTurnStartedAtMs != null && now - failedTurnStartedAtMs > maxAgeMs) {
|
|
63715
64068
|
return "skip-stale";
|
|
63716
64069
|
}
|
|
63717
|
-
lastResumedAtMs = now;
|
|
63718
64070
|
return "resume";
|
|
63719
64071
|
}
|
|
64072
|
+
function arm() {
|
|
64073
|
+
lastResumedAtMs = nowFn();
|
|
64074
|
+
}
|
|
64075
|
+
function decide(failedTurnStartedAtMs) {
|
|
64076
|
+
const verdict = peek(failedTurnStartedAtMs);
|
|
64077
|
+
if (verdict === "resume")
|
|
64078
|
+
arm();
|
|
64079
|
+
return verdict;
|
|
64080
|
+
}
|
|
63720
64081
|
return {
|
|
63721
64082
|
decide,
|
|
64083
|
+
peek,
|
|
64084
|
+
arm,
|
|
63722
64085
|
reset() {
|
|
63723
64086
|
lastResumedAtMs = Number.NEGATIVE_INFINITY;
|
|
63724
64087
|
},
|
|
@@ -64420,10 +64783,30 @@ function autoClassifyMidTurnInbound(i) {
|
|
|
64420
64783
|
return { decision: "queue", reason: "cross_topic", sameTopic: false };
|
|
64421
64784
|
return recent && i.msSinceLastAgentOutput <= i.topicSteerWindowMs ? { decision: "steer", reason: "same_topic_recent", sameTopic: true } : { decision: "queue", reason: "same_topic_stale", sameTopic: true };
|
|
64422
64785
|
}
|
|
64786
|
+
// raw-error-scrub.ts
|
|
64787
|
+
function stripRawErrorBytes(raw) {
|
|
64788
|
+
if (typeof raw !== "string" || raw.length === 0)
|
|
64789
|
+
return "";
|
|
64790
|
+
let s = raw;
|
|
64791
|
+
s = s.replace(/API Error:?\s*\d*\s*/gi, " ");
|
|
64792
|
+
s = s.replace(/\bb'[^']*'/g, " ");
|
|
64793
|
+
s = s.replace(/\bb"[^"]*"/g, " ");
|
|
64794
|
+
s = s.replace(/[\u00b7\-\s]*\{[\s\S]*?["']type["']\s*:\s*["']error["'][\s\S]*$/i, " ");
|
|
64795
|
+
s = s.replace(/^\s*\{[\s\S]*\}\s*$/g, " ");
|
|
64796
|
+
s = s.replace(/\s+/g, " ").replace(/[\u00b7:\-\s]+$/g, "").replace(/^[\u00b7:\-\s]+/g, "").trim();
|
|
64797
|
+
return s;
|
|
64798
|
+
}
|
|
64799
|
+
function extractRequestId(raw) {
|
|
64800
|
+
if (typeof raw !== "string" || raw.length === 0)
|
|
64801
|
+
return;
|
|
64802
|
+
const m = raw.match(/["']request[_-]?id["']\s*:\s*["']([A-Za-z0-9._-]+)["']/i) ?? raw.match(/\brequest[_-]?id[=:]\s*([A-Za-z0-9._-]+)/i);
|
|
64803
|
+
return m ? m[1] : undefined;
|
|
64804
|
+
}
|
|
64805
|
+
|
|
64423
64806
|
// operator-events.ts
|
|
64424
64807
|
function renderOperatorEvent(ev) {
|
|
64425
64808
|
const agent = escapeMarkdown(ev.agent);
|
|
64426
|
-
const detail = escapeMarkdown(ev.detail);
|
|
64809
|
+
const detail = escapeMarkdown(stripRawErrorBytes(ev.detail));
|
|
64427
64810
|
switch (ev.kind) {
|
|
64428
64811
|
case "credentials-expired":
|
|
64429
64812
|
return {
|
|
@@ -64640,6 +65023,13 @@ var transientUpstreamSignals = [
|
|
|
64640
65023
|
"would exceed your account\u2019s rate limit",
|
|
64641
65024
|
"would exceed your account's rate limit"
|
|
64642
65025
|
];
|
|
65026
|
+
function isTransientUpstreamSignal(text4) {
|
|
65027
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65028
|
+
return false;
|
|
65029
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65030
|
+
const lower = sample.toLowerCase();
|
|
65031
|
+
return transientUpstreamSignals.some((s) => lower.includes(s));
|
|
65032
|
+
}
|
|
64643
65033
|
var litellmProxyLocal429Signals = [
|
|
64644
65034
|
"deployment over user-defined ratelimit",
|
|
64645
65035
|
"model rate limit exceeded. tpm limit",
|
|
@@ -64660,6 +65050,56 @@ function isLitellmProxyLocal429(text4) {
|
|
|
64660
65050
|
return true;
|
|
64661
65051
|
return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
|
|
64662
65052
|
}
|
|
65053
|
+
function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
|
|
65054
|
+
const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
|
|
65055
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65056
|
+
return empty2;
|
|
65057
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65058
|
+
const lower = sample.toLowerCase();
|
|
65059
|
+
let limitType = null;
|
|
65060
|
+
let limit = null;
|
|
65061
|
+
const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
|
|
65062
|
+
if (eqLimit) {
|
|
65063
|
+
limitType = eqLimit[1];
|
|
65064
|
+
limit = Number(eqLimit[2]);
|
|
65065
|
+
}
|
|
65066
|
+
if (limitType == null) {
|
|
65067
|
+
const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
|
|
65068
|
+
if (v3Type)
|
|
65069
|
+
limitType = v3Type[1];
|
|
65070
|
+
}
|
|
65071
|
+
if (limit == null) {
|
|
65072
|
+
const v3Limit = lower.match(/current limit:\s*(\d+)/);
|
|
65073
|
+
if (v3Limit)
|
|
65074
|
+
limit = Number(v3Limit[1]);
|
|
65075
|
+
}
|
|
65076
|
+
let currentUsage = null;
|
|
65077
|
+
const usage = lower.match(/current usage=(\d+)/);
|
|
65078
|
+
if (usage)
|
|
65079
|
+
currentUsage = Number(usage[1]);
|
|
65080
|
+
let resetAtMs = null;
|
|
65081
|
+
const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
|
|
65082
|
+
if (resetsAt) {
|
|
65083
|
+
const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
|
|
65084
|
+
if (!Number.isNaN(d.getTime()))
|
|
65085
|
+
resetAtMs = d.getTime();
|
|
65086
|
+
}
|
|
65087
|
+
if (resetAtMs == null) {
|
|
65088
|
+
const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
|
|
65089
|
+
if (tryAgain) {
|
|
65090
|
+
const secs = Number(tryAgain[1]);
|
|
65091
|
+
if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
|
|
65092
|
+
resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
|
|
65093
|
+
}
|
|
65094
|
+
}
|
|
65095
|
+
}
|
|
65096
|
+
return {
|
|
65097
|
+
limitType,
|
|
65098
|
+
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
65099
|
+
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
65100
|
+
resetAtMs
|
|
65101
|
+
};
|
|
65102
|
+
}
|
|
64663
65103
|
function detectModelUnavailable(stderr) {
|
|
64664
65104
|
if (typeof stderr !== "string" || stderr.length === 0)
|
|
64665
65105
|
return null;
|
|
@@ -64853,54 +65293,358 @@ function parseRelativeDuration(s) {
|
|
|
64853
65293
|
}
|
|
64854
65294
|
return matched && total > 0 ? total : null;
|
|
64855
65295
|
}
|
|
64856
|
-
|
|
65296
|
+
|
|
65297
|
+
// throttle-tier.ts
|
|
65298
|
+
init_card_format();
|
|
65299
|
+
init_quota_check();
|
|
65300
|
+
var accountScopedThrottleSignals = [
|
|
65301
|
+
"would exceed your account's rate limit",
|
|
65302
|
+
"would exceed your account\u2019s rate limit",
|
|
65303
|
+
"not your account"
|
|
65304
|
+
];
|
|
65305
|
+
function isAccountScopedThrottle(text4) {
|
|
65306
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65307
|
+
return false;
|
|
65308
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65309
|
+
const lower = sample.toLowerCase();
|
|
65310
|
+
return accountScopedThrottleSignals.some((s) => lower.includes(s));
|
|
65311
|
+
}
|
|
65312
|
+
function classify429Detail(text4) {
|
|
65313
|
+
if (isAccountScopedThrottle(text4))
|
|
65314
|
+
return "account-scoped";
|
|
65315
|
+
if (isLitellmProxyLocal429(text4))
|
|
65316
|
+
return "litellm-local";
|
|
65317
|
+
return "generic-transient";
|
|
65318
|
+
}
|
|
65319
|
+
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
|
|
65320
|
+
var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
|
|
65321
|
+
function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS) {
|
|
65322
|
+
const last = prev.lastSentAtMsByAccount[account] ?? 0;
|
|
65323
|
+
if (now - last >= cooldownMs) {
|
|
65324
|
+
return {
|
|
65325
|
+
send: true,
|
|
65326
|
+
next: {
|
|
65327
|
+
lastSentAtMsByAccount: {
|
|
65328
|
+
...prev.lastSentAtMsByAccount,
|
|
65329
|
+
[account]: now
|
|
65330
|
+
}
|
|
65331
|
+
}
|
|
65332
|
+
};
|
|
65333
|
+
}
|
|
65334
|
+
return { send: false, next: prev };
|
|
65335
|
+
}
|
|
65336
|
+
function renderThrottleNotice(opts) {
|
|
64857
65337
|
const now = opts.now ?? new Date;
|
|
64858
|
-
const
|
|
64859
|
-
const
|
|
65338
|
+
const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
|
|
65339
|
+
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
64860
65340
|
const lines = [
|
|
64861
|
-
`\
|
|
64862
|
-
`
|
|
64863
|
-
|
|
65341
|
+
`\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
|
|
65342
|
+
`This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
|
|
65343
|
+
`_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
|
|
64864
65344
|
];
|
|
64865
|
-
if (opts.autoFallbackInFlight) {
|
|
64866
|
-
lines.push("_Auto-failover in progress \u2014 see the announcement below._");
|
|
64867
|
-
} else {
|
|
64868
|
-
lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
|
|
64869
|
-
}
|
|
64870
65345
|
return lines.join(`
|
|
64871
65346
|
`);
|
|
64872
65347
|
}
|
|
64873
|
-
function
|
|
64874
|
-
const
|
|
64875
|
-
|
|
64876
|
-
|
|
64877
|
-
|
|
64878
|
-
|
|
64879
|
-
|
|
64880
|
-
|
|
64881
|
-
|
|
64882
|
-
|
|
64883
|
-
|
|
65348
|
+
function renderThrottleEscalationNotice(opts) {
|
|
65349
|
+
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65350
|
+
const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
|
|
65351
|
+
const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
|
|
65352
|
+
return `${head}
|
|
65353
|
+
${tail}`;
|
|
65354
|
+
}
|
|
65355
|
+
// operator-events.ts
|
|
65356
|
+
function classifyClaudeError(raw) {
|
|
65357
|
+
try {
|
|
65358
|
+
return classifyInner(raw);
|
|
65359
|
+
} catch {
|
|
65360
|
+
return "unknown-4xx";
|
|
64884
65361
|
}
|
|
64885
65362
|
}
|
|
64886
|
-
function
|
|
64887
|
-
|
|
64888
|
-
|
|
64889
|
-
|
|
65363
|
+
function classifyInner(raw) {
|
|
65364
|
+
if (raw == null)
|
|
65365
|
+
return "unknown-4xx";
|
|
65366
|
+
const obj = typeof raw === "object" ? raw : {};
|
|
65367
|
+
const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
|
|
65368
|
+
const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
|
|
65369
|
+
const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
|
|
65370
|
+
const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
|
|
65371
|
+
const sdkCode = extractString(obj, "error_code") ?? "";
|
|
65372
|
+
if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
|
|
65373
|
+
const msg = message.toLowerCase();
|
|
65374
|
+
if (msg.includes("expired") || msg.includes("refresh")) {
|
|
65375
|
+
return "credentials-expired";
|
|
65376
|
+
}
|
|
65377
|
+
return "credentials-invalid";
|
|
64890
65378
|
}
|
|
64891
|
-
if (
|
|
64892
|
-
|
|
64893
|
-
return detected?.kind === "quota_exhausted" ? detected : null;
|
|
65379
|
+
if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
|
|
65380
|
+
return "credentials-invalid";
|
|
64894
65381
|
}
|
|
64895
|
-
if (
|
|
64896
|
-
return
|
|
65382
|
+
if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
|
|
65383
|
+
return "credit-exhausted";
|
|
65384
|
+
}
|
|
65385
|
+
if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
|
|
65386
|
+
return "rate-limited";
|
|
65387
|
+
}
|
|
65388
|
+
if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
|
|
65389
|
+
return "rate-limited";
|
|
65390
|
+
}
|
|
65391
|
+
if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
|
|
65392
|
+
return "agent-crashed";
|
|
64897
65393
|
}
|
|
64898
|
-
|
|
65394
|
+
if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
|
|
65395
|
+
return "agent-restarted-unexpectedly";
|
|
65396
|
+
}
|
|
65397
|
+
if (status != null) {
|
|
65398
|
+
if (status >= 400 && status < 500)
|
|
65399
|
+
return "unknown-4xx";
|
|
65400
|
+
if (status >= 500 && status < 600)
|
|
65401
|
+
return "unknown-5xx";
|
|
65402
|
+
}
|
|
65403
|
+
return "unknown-4xx";
|
|
65404
|
+
}
|
|
65405
|
+
function extractString(obj, key) {
|
|
65406
|
+
const v = obj[key];
|
|
65407
|
+
return typeof v === "string" && v.length > 0 ? v : null;
|
|
64899
65408
|
}
|
|
65409
|
+
function extractNumber(obj, key) {
|
|
65410
|
+
const v = obj[key];
|
|
65411
|
+
return typeof v === "number" ? v : null;
|
|
65412
|
+
}
|
|
65413
|
+
function getNestedObj(obj, key) {
|
|
65414
|
+
const v = obj[key];
|
|
65415
|
+
return typeof v === "object" && v != null ? v : {};
|
|
65416
|
+
}
|
|
65417
|
+
var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
|
|
65418
|
+
var cooldownMap2 = new Map;
|
|
64900
65419
|
|
|
64901
|
-
//
|
|
64902
|
-
|
|
64903
|
-
|
|
65420
|
+
// llm-error-present.ts
|
|
65421
|
+
function extractModel(raw) {
|
|
65422
|
+
const m = raw.match(/["']?model["']?\s*[=:]\s*["']?((?:claude|sr)[A-Za-z0-9._-]+)/i);
|
|
65423
|
+
return m ? m[1] : undefined;
|
|
65424
|
+
}
|
|
65425
|
+
var TRANSIENT_KINDS = new Set([
|
|
65426
|
+
"rate_limit",
|
|
65427
|
+
"overload_529",
|
|
65428
|
+
"transient"
|
|
65429
|
+
]);
|
|
65430
|
+
function isActionableKind(kind) {
|
|
65431
|
+
return kind === "auth" || kind === "quota_wall";
|
|
65432
|
+
}
|
|
65433
|
+
function parseLlmError(raw, retryState) {
|
|
65434
|
+
const text4 = typeof raw === "string" ? raw : "";
|
|
65435
|
+
const requestId = extractRequestId(text4);
|
|
65436
|
+
const model = extractModel(text4);
|
|
65437
|
+
const resetAt = parseResetTime(text4);
|
|
65438
|
+
const retryAfterMs = resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined;
|
|
65439
|
+
const { kind, source } = classifyKindAndSource(text4);
|
|
65440
|
+
let autoRetrying = false;
|
|
65441
|
+
let terminal = true;
|
|
65442
|
+
if (TRANSIENT_KINDS.has(kind)) {
|
|
65443
|
+
const { retryAttempt, maxRetries } = retryState ?? { retryAttempt: null, maxRetries: null };
|
|
65444
|
+
if (retryAttempt != null && maxRetries != null) {
|
|
65445
|
+
autoRetrying = retryAttempt < maxRetries;
|
|
65446
|
+
terminal = retryAttempt >= maxRetries;
|
|
65447
|
+
} else {
|
|
65448
|
+
autoRetrying = false;
|
|
65449
|
+
terminal = true;
|
|
65450
|
+
}
|
|
65451
|
+
}
|
|
65452
|
+
return {
|
|
65453
|
+
kind,
|
|
65454
|
+
coreText: buildCoreText(kind, source),
|
|
65455
|
+
...resetAt != null ? { resetAt } : {},
|
|
65456
|
+
...retryAfterMs != null ? { retryAfterMs } : {},
|
|
65457
|
+
...model != null ? { model } : {},
|
|
65458
|
+
...requestId != null ? { requestId } : {},
|
|
65459
|
+
source,
|
|
65460
|
+
autoRetrying,
|
|
65461
|
+
terminal
|
|
65462
|
+
};
|
|
65463
|
+
}
|
|
65464
|
+
function classifyKindAndSource(text4) {
|
|
65465
|
+
const lower = text4.toLowerCase();
|
|
65466
|
+
const claudeKind = classifyClaudeError({ message: text4, type: text4 });
|
|
65467
|
+
if (claudeKind === "credentials-expired" || claudeKind === "credentials-invalid") {
|
|
65468
|
+
return { kind: "auth", source: "anthropic" };
|
|
65469
|
+
}
|
|
65470
|
+
if (isLitellmProxyLocal429(text4)) {
|
|
65471
|
+
return { kind: "rate_limit", source: "litellm-local" };
|
|
65472
|
+
}
|
|
65473
|
+
const mu = detectModelUnavailable(text4);
|
|
65474
|
+
if (mu != null) {
|
|
65475
|
+
if (mu.kind === "quota_exhausted")
|
|
65476
|
+
return { kind: "quota_wall", source: "anthropic" };
|
|
65477
|
+
if (mu.kind === "network")
|
|
65478
|
+
return { kind: "transient", source: "network" };
|
|
65479
|
+
if (lower.includes("529") || lower.includes("overloaded")) {
|
|
65480
|
+
return { kind: "overload_529", source: "anthropic" };
|
|
65481
|
+
}
|
|
65482
|
+
const c = classify429Detail(text4);
|
|
65483
|
+
return {
|
|
65484
|
+
kind: "rate_limit",
|
|
65485
|
+
source: c === "litellm-local" ? "litellm-local" : "anthropic"
|
|
65486
|
+
};
|
|
65487
|
+
}
|
|
65488
|
+
if (claudeKind === "credit-exhausted") {
|
|
65489
|
+
return { kind: "quota_wall", source: "anthropic" };
|
|
65490
|
+
}
|
|
65491
|
+
if (claudeKind === "rate-limited") {
|
|
65492
|
+
return { kind: "rate_limit", source: "anthropic" };
|
|
65493
|
+
}
|
|
65494
|
+
if (claudeKind === "unknown-5xx") {
|
|
65495
|
+
return { kind: "overload_529", source: "anthropic" };
|
|
65496
|
+
}
|
|
65497
|
+
return { kind: "unknown", source: "anthropic" };
|
|
65498
|
+
}
|
|
65499
|
+
function buildCoreText(kind, source) {
|
|
65500
|
+
switch (kind) {
|
|
65501
|
+
case "rate_limit":
|
|
65502
|
+
return source === "litellm-local" ? "Hit the local proxy rate limit \u2014 retrying automatically." : "Rate limited by Anthropic \u2014 retrying automatically.";
|
|
65503
|
+
case "overload_529":
|
|
65504
|
+
return "Anthropic is overloaded (529) \u2014 retrying automatically.";
|
|
65505
|
+
case "quota_wall":
|
|
65506
|
+
return "Usage limit reached on this Claude subscription.";
|
|
65507
|
+
case "auth":
|
|
65508
|
+
return "Claude login needs re-authentication.";
|
|
65509
|
+
case "transient":
|
|
65510
|
+
return source === "network" ? "Couldn't reach Anthropic (network) \u2014 retrying automatically." : "A temporary upstream hiccup \u2014 retrying automatically.";
|
|
65511
|
+
case "unknown":
|
|
65512
|
+
return "The model returned an error.";
|
|
65513
|
+
}
|
|
65514
|
+
}
|
|
65515
|
+
function formatResetLocal(resetAt, tz, now = new Date) {
|
|
65516
|
+
if (resetAt == null)
|
|
65517
|
+
return "";
|
|
65518
|
+
const ms = resetAt.getTime();
|
|
65519
|
+
if (!Number.isFinite(ms))
|
|
65520
|
+
return "";
|
|
65521
|
+
const clock = fmtLocalClock(ms, tz);
|
|
65522
|
+
const abbrev = tzAbbrev(ms, tz);
|
|
65523
|
+
const rel = formatRelativeTail(ms - now.getTime());
|
|
65524
|
+
return rel ? `clears ~${clock} ${abbrev} (${rel})` : `clears ~${clock} ${abbrev}`;
|
|
65525
|
+
}
|
|
65526
|
+
function formatRelativeTail(deltaMs) {
|
|
65527
|
+
if (deltaMs <= 0)
|
|
65528
|
+
return "~now";
|
|
65529
|
+
const totalMin = Math.round(deltaMs / 60000);
|
|
65530
|
+
if (totalMin < 1)
|
|
65531
|
+
return "~in <1m";
|
|
65532
|
+
if (totalMin < 60)
|
|
65533
|
+
return `~in ${totalMin}m`;
|
|
65534
|
+
const hours = Math.floor(totalMin / 60);
|
|
65535
|
+
const mins = totalMin % 60;
|
|
65536
|
+
if (hours < 24)
|
|
65537
|
+
return mins > 0 ? `~in ${hours}h ${mins}m` : `~in ${hours}h`;
|
|
65538
|
+
const days = Math.floor(hours / 24);
|
|
65539
|
+
const remH = hours % 24;
|
|
65540
|
+
return remH > 0 ? `~in ${days}d ${remH}h` : `~in ${days}d`;
|
|
65541
|
+
}
|
|
65542
|
+
function renderLlmError(parsed, agent, tz, now = new Date) {
|
|
65543
|
+
const safeAgent = escapeAgent(agent);
|
|
65544
|
+
const emoji = kindEmoji(parsed.kind);
|
|
65545
|
+
const lines = [`${emoji} ${parsed.coreText} (**${safeAgent}**)`];
|
|
65546
|
+
const resetLine = formatResetLocal(parsed.resetAt, tz, now);
|
|
65547
|
+
if (resetLine)
|
|
65548
|
+
lines.push(`_${resetLine}_`);
|
|
65549
|
+
if (parsed.model)
|
|
65550
|
+
lines.push(`_model: ${escapeAgent(parsed.model)}_`);
|
|
65551
|
+
const text4 = lines.join(`
|
|
65552
|
+
`);
|
|
65553
|
+
switch (parsed.kind) {
|
|
65554
|
+
case "auth":
|
|
65555
|
+
return {
|
|
65556
|
+
text: text4,
|
|
65557
|
+
keyboard: {
|
|
65558
|
+
inline_keyboard: [
|
|
65559
|
+
[
|
|
65560
|
+
{ text: "\uD83D\uDD10 Reauth now", callback_data: `op:reauth:${encodeURIComponent(agent)}` },
|
|
65561
|
+
{ text: "\u274c Dismiss", callback_data: `op:dismiss:${encodeURIComponent(agent)}` }
|
|
65562
|
+
]
|
|
65563
|
+
]
|
|
65564
|
+
}
|
|
65565
|
+
};
|
|
65566
|
+
case "quota_wall":
|
|
65567
|
+
return {
|
|
65568
|
+
text: text4,
|
|
65569
|
+
keyboard: {
|
|
65570
|
+
inline_keyboard: [
|
|
65571
|
+
[{ text: "\u23f3 Wait", callback_data: `op:dismiss:${encodeURIComponent(agent)}` }]
|
|
65572
|
+
]
|
|
65573
|
+
}
|
|
65574
|
+
};
|
|
65575
|
+
default:
|
|
65576
|
+
return { text: text4 };
|
|
65577
|
+
}
|
|
65578
|
+
}
|
|
65579
|
+
function kindEmoji(kind) {
|
|
65580
|
+
switch (kind) {
|
|
65581
|
+
case "rate_limit":
|
|
65582
|
+
return "\uD83D\uDEA6";
|
|
65583
|
+
case "overload_529":
|
|
65584
|
+
return "\uD83D\uDD25";
|
|
65585
|
+
case "quota_wall":
|
|
65586
|
+
return "\u26a0\ufe0f";
|
|
65587
|
+
case "auth":
|
|
65588
|
+
return "\uD83D\uDD11";
|
|
65589
|
+
case "transient":
|
|
65590
|
+
return "\uD83C\uDF10";
|
|
65591
|
+
case "unknown":
|
|
65592
|
+
return "\u26a0\ufe0f";
|
|
65593
|
+
}
|
|
65594
|
+
}
|
|
65595
|
+
function escapeAgent(s) {
|
|
65596
|
+
return s.replace(/([_*`\[\]])/g, "\\$1");
|
|
65597
|
+
}
|
|
65598
|
+
var ERROR_COLLAPSE_WINDOW_MS = 60000;
|
|
65599
|
+
|
|
65600
|
+
class ErrorPresenceGate {
|
|
65601
|
+
claims = new Map;
|
|
65602
|
+
keyFor(parsed, agent, now) {
|
|
65603
|
+
if (parsed.requestId)
|
|
65604
|
+
return `rid:${parsed.requestId}`;
|
|
65605
|
+
const bucket = Math.floor(now / ERROR_COLLAPSE_WINDOW_MS);
|
|
65606
|
+
return `${parsed.kind}:${agent}:${bucket}`;
|
|
65607
|
+
}
|
|
65608
|
+
claim(key, now = Date.now()) {
|
|
65609
|
+
this.prune(now);
|
|
65610
|
+
const existing = this.claims.get(key);
|
|
65611
|
+
if (existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS) {
|
|
65612
|
+
return false;
|
|
65613
|
+
}
|
|
65614
|
+
this.claims.set(key, now);
|
|
65615
|
+
return true;
|
|
65616
|
+
}
|
|
65617
|
+
isClaimed(key, now = Date.now()) {
|
|
65618
|
+
const existing = this.claims.get(key);
|
|
65619
|
+
return existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS;
|
|
65620
|
+
}
|
|
65621
|
+
prune(now) {
|
|
65622
|
+
for (const [k, at] of this.claims) {
|
|
65623
|
+
if (now - at >= ERROR_COLLAPSE_WINDOW_MS)
|
|
65624
|
+
this.claims.delete(k);
|
|
65625
|
+
}
|
|
65626
|
+
}
|
|
65627
|
+
reset() {
|
|
65628
|
+
this.claims.clear();
|
|
65629
|
+
}
|
|
65630
|
+
}
|
|
65631
|
+
var errorPresenceGate = new ErrorPresenceGate;
|
|
65632
|
+
function decideErrorSurface(parsed, agent, opts = { claim: true }) {
|
|
65633
|
+
const now = opts.now ?? Date.now();
|
|
65634
|
+
const gate = opts.gate ?? errorPresenceGate;
|
|
65635
|
+
if (isActionableKind(parsed.kind)) {
|
|
65636
|
+
if (opts.claim)
|
|
65637
|
+
gate.claim(gate.keyFor(parsed, agent, now), now);
|
|
65638
|
+
return "render";
|
|
65639
|
+
}
|
|
65640
|
+
if (parsed.autoRetrying && !parsed.terminal)
|
|
65641
|
+
return "suppress";
|
|
65642
|
+
const key = gate.keyFor(parsed, agent, now);
|
|
65643
|
+
if (opts.claim) {
|
|
65644
|
+
return gate.claim(key, now) ? "render" : "suppress";
|
|
65645
|
+
}
|
|
65646
|
+
return gate.isClaimed(key, now) ? "suppress" : "render";
|
|
65647
|
+
}
|
|
64904
65648
|
|
|
64905
65649
|
// model-unavailable.ts
|
|
64906
65650
|
init_quota_check();
|
|
@@ -64915,13 +65659,6 @@ var transientUpstreamSignals2 = [
|
|
|
64915
65659
|
"would exceed your account\u2019s rate limit",
|
|
64916
65660
|
"would exceed your account's rate limit"
|
|
64917
65661
|
];
|
|
64918
|
-
function isTransientUpstreamSignal(text4) {
|
|
64919
|
-
if (typeof text4 !== "string" || text4.length === 0)
|
|
64920
|
-
return false;
|
|
64921
|
-
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
64922
|
-
const lower = sample.toLowerCase();
|
|
64923
|
-
return transientUpstreamSignals2.some((s) => lower.includes(s));
|
|
64924
|
-
}
|
|
64925
65662
|
var litellmProxyLocal429Signals2 = [
|
|
64926
65663
|
"deployment over user-defined ratelimit",
|
|
64927
65664
|
"model rate limit exceeded. tpm limit",
|
|
@@ -64942,55 +65679,76 @@ function isLitellmProxyLocal4292(text4) {
|
|
|
64942
65679
|
return true;
|
|
64943
65680
|
return litellmV3LimiterSignalPair2.every((s) => lower.includes(s));
|
|
64944
65681
|
}
|
|
64945
|
-
function
|
|
64946
|
-
|
|
64947
|
-
|
|
64948
|
-
|
|
64949
|
-
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65682
|
+
function detectModelUnavailable2(stderr) {
|
|
65683
|
+
if (typeof stderr !== "string" || stderr.length === 0)
|
|
65684
|
+
return null;
|
|
65685
|
+
const sample = stderr.length > 16384 ? stderr.slice(0, 16384) : stderr;
|
|
64950
65686
|
const lower = sample.toLowerCase();
|
|
64951
|
-
|
|
64952
|
-
|
|
64953
|
-
|
|
64954
|
-
if (eqLimit) {
|
|
64955
|
-
limitType = eqLimit[1];
|
|
64956
|
-
limit = Number(eqLimit[2]);
|
|
65687
|
+
if (transientUpstreamSignals2.some((s) => lower.includes(s))) {
|
|
65688
|
+
const resetAt = parseResetTime2(sample);
|
|
65689
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
64957
65690
|
}
|
|
64958
|
-
if (
|
|
64959
|
-
const
|
|
64960
|
-
|
|
64961
|
-
limitType = v3Type[1];
|
|
65691
|
+
if (isLitellmProxyLocal4292(sample)) {
|
|
65692
|
+
const resetAt = parseResetTime2(sample);
|
|
65693
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
64962
65694
|
}
|
|
64963
|
-
|
|
64964
|
-
|
|
64965
|
-
|
|
64966
|
-
|
|
65695
|
+
const quotaSignals = [
|
|
65696
|
+
"out of extra usage",
|
|
65697
|
+
"extra usage",
|
|
65698
|
+
"credit_balance_too_low",
|
|
65699
|
+
"credit balance too low",
|
|
65700
|
+
"usage limit",
|
|
65701
|
+
"usage_limit",
|
|
65702
|
+
"quota exhausted",
|
|
65703
|
+
"quota_exhausted",
|
|
65704
|
+
"plan limit",
|
|
65705
|
+
"subscription limit",
|
|
65706
|
+
"hit your limit",
|
|
65707
|
+
"hit the limit",
|
|
65708
|
+
"session limit",
|
|
65709
|
+
"session cap"
|
|
65710
|
+
];
|
|
65711
|
+
if (quotaSignals.some((s) => lower.includes(s))) {
|
|
65712
|
+
const resetAt = parseResetTime2(sample);
|
|
65713
|
+
return resetAt !== undefined ? { kind: "quota_exhausted", resetAt, raw: stderr } : { kind: "quota_exhausted", raw: stderr };
|
|
64967
65714
|
}
|
|
64968
|
-
|
|
64969
|
-
|
|
64970
|
-
|
|
64971
|
-
|
|
64972
|
-
|
|
64973
|
-
|
|
64974
|
-
|
|
64975
|
-
|
|
64976
|
-
|
|
64977
|
-
|
|
65715
|
+
const overloadSignals = [
|
|
65716
|
+
"overloaded_error",
|
|
65717
|
+
"overloaded",
|
|
65718
|
+
"rate_limit_error",
|
|
65719
|
+
"rate limit",
|
|
65720
|
+
"rate-limited",
|
|
65721
|
+
"http 429",
|
|
65722
|
+
'"status":429',
|
|
65723
|
+
"status: 429",
|
|
65724
|
+
" 429 ",
|
|
65725
|
+
"503 service",
|
|
65726
|
+
"service unavailable",
|
|
65727
|
+
'"status":529',
|
|
65728
|
+
"http 529",
|
|
65729
|
+
" 529 "
|
|
65730
|
+
];
|
|
65731
|
+
if (overloadSignals.some((s) => lower.includes(s))) {
|
|
65732
|
+
const resetAt = parseResetTime2(sample);
|
|
65733
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
64978
65734
|
}
|
|
64979
|
-
|
|
64980
|
-
|
|
64981
|
-
|
|
64982
|
-
|
|
64983
|
-
|
|
64984
|
-
|
|
64985
|
-
|
|
64986
|
-
|
|
65735
|
+
const networkSignals = [
|
|
65736
|
+
"econnrefused",
|
|
65737
|
+
"econnreset",
|
|
65738
|
+
"etimedout",
|
|
65739
|
+
"enotfound",
|
|
65740
|
+
"eai_again",
|
|
65741
|
+
"fetch failed",
|
|
65742
|
+
"network error",
|
|
65743
|
+
"socket hang up",
|
|
65744
|
+
"request timed out",
|
|
65745
|
+
"connection refused",
|
|
65746
|
+
"getaddrinfo"
|
|
65747
|
+
];
|
|
65748
|
+
if (networkSignals.some((s) => lower.includes(s))) {
|
|
65749
|
+
return { kind: "network", raw: stderr };
|
|
64987
65750
|
}
|
|
64988
|
-
return
|
|
64989
|
-
limitType,
|
|
64990
|
-
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
64991
|
-
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
64992
|
-
resetAtMs
|
|
64993
|
-
};
|
|
65751
|
+
return null;
|
|
64994
65752
|
}
|
|
64995
65753
|
function parseResetTime2(text4, parseTimeNow = new Date) {
|
|
64996
65754
|
const lower = text4.toLowerCase();
|
|
@@ -65114,31 +65872,77 @@ function parseRelativeDuration2(s) {
|
|
|
65114
65872
|
}
|
|
65115
65873
|
return matched && total > 0 ? total : null;
|
|
65116
65874
|
}
|
|
65875
|
+
function formatModelUnavailableCard(detection, agent, opts = {}) {
|
|
65876
|
+
const now = opts.now ?? new Date;
|
|
65877
|
+
const slotPart = opts.slot ? ` (slot **${escapeMarkdown(opts.slot)}**)` : "";
|
|
65878
|
+
const reason = formatReason(detection, now);
|
|
65879
|
+
const lines = [
|
|
65880
|
+
`\u26a0\ufe0f **Model unavailable** on agent **${escapeMarkdown(agent)}**${slotPart}`,
|
|
65881
|
+
`Reason: ${reason}`,
|
|
65882
|
+
""
|
|
65883
|
+
];
|
|
65884
|
+
if (opts.autoFallbackInFlight) {
|
|
65885
|
+
lines.push("_Auto-failover in progress \u2014 see the announcement below._");
|
|
65886
|
+
} else {
|
|
65887
|
+
lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
|
|
65888
|
+
}
|
|
65889
|
+
return lines.join(`
|
|
65890
|
+
`);
|
|
65891
|
+
}
|
|
65892
|
+
function formatReason(d, now) {
|
|
65893
|
+
const reset2 = d.resetAt ? ` (${formatResetRelative(d.resetAt, now)})` : "";
|
|
65894
|
+
switch (d.kind) {
|
|
65895
|
+
case "quota_exhausted":
|
|
65896
|
+
return `quota exhausted${reset2}`;
|
|
65897
|
+
case "overload":
|
|
65898
|
+
return `model overloaded${reset2}`;
|
|
65899
|
+
case "rate_limited":
|
|
65900
|
+
return `account rate-limited${reset2}`;
|
|
65901
|
+
case "network":
|
|
65902
|
+
return "network unreachable";
|
|
65903
|
+
}
|
|
65904
|
+
}
|
|
65905
|
+
function resolveModelUnavailableFromOperatorEvent(ev) {
|
|
65906
|
+
const detail = typeof ev.detail === "string" ? ev.detail : "";
|
|
65907
|
+
if (ev.kind === "quota-exhausted") {
|
|
65908
|
+
return detectModelUnavailable2(detail) ?? { kind: "quota_exhausted", raw: detail };
|
|
65909
|
+
}
|
|
65910
|
+
if (ev.kind === "rate-limited") {
|
|
65911
|
+
const detected = detectModelUnavailable2(detail);
|
|
65912
|
+
return detected?.kind === "quota_exhausted" ? detected : null;
|
|
65913
|
+
}
|
|
65914
|
+
if (ev.kind === "unknown-5xx") {
|
|
65915
|
+
return detectModelUnavailable2(detail) ?? { kind: "overload", raw: detail };
|
|
65916
|
+
}
|
|
65917
|
+
return detectModelUnavailable2(detail);
|
|
65918
|
+
}
|
|
65117
65919
|
|
|
65118
65920
|
// throttle-tier.ts
|
|
65119
|
-
|
|
65921
|
+
init_card_format();
|
|
65922
|
+
init_quota_check();
|
|
65923
|
+
var accountScopedThrottleSignals2 = [
|
|
65120
65924
|
"would exceed your account's rate limit",
|
|
65121
65925
|
"would exceed your account\u2019s rate limit",
|
|
65122
65926
|
"not your account"
|
|
65123
65927
|
];
|
|
65124
|
-
function
|
|
65928
|
+
function isAccountScopedThrottle2(text4) {
|
|
65125
65929
|
if (typeof text4 !== "string" || text4.length === 0)
|
|
65126
65930
|
return false;
|
|
65127
65931
|
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65128
65932
|
const lower = sample.toLowerCase();
|
|
65129
|
-
return
|
|
65933
|
+
return accountScopedThrottleSignals2.some((s) => lower.includes(s));
|
|
65130
65934
|
}
|
|
65131
|
-
function
|
|
65132
|
-
if (
|
|
65935
|
+
function classify429Detail2(text4) {
|
|
65936
|
+
if (isAccountScopedThrottle2(text4))
|
|
65133
65937
|
return "account-scoped";
|
|
65134
|
-
if (
|
|
65938
|
+
if (isLitellmProxyLocal429(text4))
|
|
65135
65939
|
return "litellm-local";
|
|
65136
65940
|
return "generic-transient";
|
|
65137
65941
|
}
|
|
65138
65942
|
function build429ClassifiedMetric(opts) {
|
|
65139
65943
|
const detail = typeof opts.detail === "string" ? opts.detail : "";
|
|
65140
65944
|
const litellm = parseLitellmLimitDetail(detail, new Date(opts.now));
|
|
65141
|
-
const anthropicResetMs =
|
|
65945
|
+
const anthropicResetMs = parseResetTime(detail, new Date(opts.now))?.getTime() ?? null;
|
|
65142
65946
|
const resetAtMs = anthropicResetMs ?? litellm.resetAtMs;
|
|
65143
65947
|
return {
|
|
65144
65948
|
kind: "rate_limit_429_classified",
|
|
@@ -65152,20 +65956,20 @@ function build429ClassifiedMetric(opts) {
|
|
|
65152
65956
|
current_usage: litellm.currentUsage
|
|
65153
65957
|
};
|
|
65154
65958
|
}
|
|
65155
|
-
var
|
|
65959
|
+
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
|
|
65156
65960
|
var THROTTLE_DEFAULT_WAIT_MS = 60000;
|
|
65157
65961
|
function throttleRetryInPlaceMaxMs(env = process.env) {
|
|
65158
65962
|
const raw = env.SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS;
|
|
65159
65963
|
if (raw == null || raw === "")
|
|
65160
|
-
return
|
|
65964
|
+
return THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
|
|
65161
65965
|
const n = Number(raw);
|
|
65162
|
-
return Number.isFinite(n) && n > 0 ? n :
|
|
65966
|
+
return Number.isFinite(n) && n > 0 ? n : THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
|
|
65163
65967
|
}
|
|
65164
65968
|
function decideThrottleTier(opts) {
|
|
65165
65969
|
const { detail, now, thresholdMs } = opts;
|
|
65166
|
-
if (!
|
|
65970
|
+
if (!isAccountScopedThrottle2(detail))
|
|
65167
65971
|
return { action: "none" };
|
|
65168
|
-
const resetAt =
|
|
65972
|
+
const resetAt = parseResetTime(detail, new Date(now));
|
|
65169
65973
|
const resetAtMs = resetAt?.getTime();
|
|
65170
65974
|
if (resetAtMs == null || !Number.isFinite(resetAtMs) || resetAtMs <= now) {
|
|
65171
65975
|
return {
|
|
@@ -65179,47 +65983,7 @@ function decideThrottleTier(opts) {
|
|
|
65179
65983
|
}
|
|
65180
65984
|
return { action: "failover", resetAtMs };
|
|
65181
65985
|
}
|
|
65182
|
-
var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
|
|
65183
|
-
|
|
65184
|
-
// throttle-tier.ts
|
|
65185
|
-
init_card_format();
|
|
65186
|
-
init_quota_check();
|
|
65187
|
-
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
|
|
65188
65986
|
var THROTTLE_NOTICE_COOLDOWN_MS2 = 10 * 60000;
|
|
65189
|
-
function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS2) {
|
|
65190
|
-
const last = prev.lastSentAtMsByAccount[account] ?? 0;
|
|
65191
|
-
if (now - last >= cooldownMs) {
|
|
65192
|
-
return {
|
|
65193
|
-
send: true,
|
|
65194
|
-
next: {
|
|
65195
|
-
lastSentAtMsByAccount: {
|
|
65196
|
-
...prev.lastSentAtMsByAccount,
|
|
65197
|
-
[account]: now
|
|
65198
|
-
}
|
|
65199
|
-
}
|
|
65200
|
-
};
|
|
65201
|
-
}
|
|
65202
|
-
return { send: false, next: prev };
|
|
65203
|
-
}
|
|
65204
|
-
function renderThrottleNotice(opts) {
|
|
65205
|
-
const now = opts.now ?? new Date;
|
|
65206
|
-
const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
|
|
65207
|
-
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65208
|
-
const lines = [
|
|
65209
|
-
`\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
|
|
65210
|
-
`This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
|
|
65211
|
-
`_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
|
|
65212
|
-
];
|
|
65213
|
-
return lines.join(`
|
|
65214
|
-
`);
|
|
65215
|
-
}
|
|
65216
|
-
function renderThrottleEscalationNotice(opts) {
|
|
65217
|
-
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65218
|
-
const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
|
|
65219
|
-
const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
|
|
65220
|
-
return `${head}
|
|
65221
|
-
${tail}`;
|
|
65222
|
-
}
|
|
65223
65987
|
|
|
65224
65988
|
// gateway/throttle-tier-wiring.ts
|
|
65225
65989
|
var THROTTLE_RETRY_NUDGE_SLACK_MS = 5000;
|
|
@@ -65240,7 +66004,7 @@ function createThrottleTierRunner(deps) {
|
|
|
65240
66004
|
let granted = true;
|
|
65241
66005
|
if (client3 && account) {
|
|
65242
66006
|
try {
|
|
65243
|
-
granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`,
|
|
66007
|
+
granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`, THROTTLE_NOTICE_COOLDOWN_MS)).granted;
|
|
65244
66008
|
} catch {
|
|
65245
66009
|
granted = true;
|
|
65246
66010
|
}
|
|
@@ -65325,6 +66089,132 @@ function createThrottleTierRunner(deps) {
|
|
|
65325
66089
|
};
|
|
65326
66090
|
}
|
|
65327
66091
|
|
|
66092
|
+
// litellm-local-notice.ts
|
|
66093
|
+
init_card_format();
|
|
66094
|
+
var LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT = 15 * 60000;
|
|
66095
|
+
function parseLitellmNoticeWindowMs(raw) {
|
|
66096
|
+
if (typeof raw !== "number")
|
|
66097
|
+
return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT;
|
|
66098
|
+
if (!Number.isFinite(raw) || raw <= 0)
|
|
66099
|
+
return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT;
|
|
66100
|
+
return raw;
|
|
66101
|
+
}
|
|
66102
|
+
|
|
66103
|
+
// litellm-local-notice.ts
|
|
66104
|
+
init_card_format();
|
|
66105
|
+
var LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT2 = 15 * 60000;
|
|
66106
|
+
function initialLitellmLocalNoticeState() {
|
|
66107
|
+
return { lastSentAtMsByAgent: {}, suppressedCountByAgent: {} };
|
|
66108
|
+
}
|
|
66109
|
+
function evaluateLitellmLocalNotice(prev, agent, now, windowMs = LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT2) {
|
|
66110
|
+
const last = prev.lastSentAtMsByAgent[agent];
|
|
66111
|
+
if (last == null || now - last >= windowMs) {
|
|
66112
|
+
return {
|
|
66113
|
+
send: true,
|
|
66114
|
+
suppressedSinceLastNotice: prev.suppressedCountByAgent[agent] ?? 0,
|
|
66115
|
+
next: {
|
|
66116
|
+
lastSentAtMsByAgent: { ...prev.lastSentAtMsByAgent, [agent]: now },
|
|
66117
|
+
suppressedCountByAgent: { ...prev.suppressedCountByAgent, [agent]: 0 }
|
|
66118
|
+
}
|
|
66119
|
+
};
|
|
66120
|
+
}
|
|
66121
|
+
return {
|
|
66122
|
+
send: false,
|
|
66123
|
+
suppressedSinceLastNotice: 0,
|
|
66124
|
+
next: {
|
|
66125
|
+
lastSentAtMsByAgent: prev.lastSentAtMsByAgent,
|
|
66126
|
+
suppressedCountByAgent: {
|
|
66127
|
+
...prev.suppressedCountByAgent,
|
|
66128
|
+
[agent]: (prev.suppressedCountByAgent[agent] ?? 0) + 1
|
|
66129
|
+
}
|
|
66130
|
+
}
|
|
66131
|
+
};
|
|
66132
|
+
}
|
|
66133
|
+
function renderLitellmLocalNotice(opts) {
|
|
66134
|
+
const agent = escapeMarkdown(opts.agent);
|
|
66135
|
+
const n = opts.suppressedSinceLastNotice;
|
|
66136
|
+
const lines = [
|
|
66137
|
+
`\uD83D\uDEA6 **Fleet token limiter engaged** \u2014 **${agent}** hit the local LiteLLM proxy cap (\`tpm_limit\`/\`rpm_limit\`).`,
|
|
66138
|
+
`This is switchroom's own fleet limiter smoothing a burst, not an Anthropic account limit \u2014 nothing is exhausted and no account was touched.`
|
|
66139
|
+
];
|
|
66140
|
+
if (n > 0) {
|
|
66141
|
+
lines.push(`Throttled ${n} more ${n === 1 ? "time" : "times"} since the last notice.`);
|
|
66142
|
+
}
|
|
66143
|
+
lines.push(`_The turn retries automatically \u2014 no action needed._`);
|
|
66144
|
+
return lines.join(`
|
|
66145
|
+
`);
|
|
66146
|
+
}
|
|
66147
|
+
function buildLitellmLocalNoticeMetric(opts) {
|
|
66148
|
+
return {
|
|
66149
|
+
kind: "litellm_local_429_notice",
|
|
66150
|
+
agent: opts.agent,
|
|
66151
|
+
suppressed_count: opts.suppressedCount,
|
|
66152
|
+
window_ms: opts.windowMs
|
|
66153
|
+
};
|
|
66154
|
+
}
|
|
66155
|
+
function isLitellmLocalNoticeEligible(classification) {
|
|
66156
|
+
return classification === "litellm-local";
|
|
66157
|
+
}
|
|
66158
|
+
|
|
66159
|
+
// gateway/litellm-local-notice-wiring.ts
|
|
66160
|
+
function decideRateLimitedSurface(opts) {
|
|
66161
|
+
if (isLitellmLocalNoticeEligible(opts.classification))
|
|
66162
|
+
return "litellm-local-notice";
|
|
66163
|
+
return opts.shouldEmitCard(opts.agent) ? "generic-card" : "cooldown-suppressed";
|
|
66164
|
+
}
|
|
66165
|
+
function createLitellmLocalNoticeRunner(deps) {
|
|
66166
|
+
const now = deps.now ?? (() => Date.now());
|
|
66167
|
+
let state3 = initialLitellmLocalNoticeState();
|
|
66168
|
+
function onRateLimited(classification, agent) {
|
|
66169
|
+
try {
|
|
66170
|
+
if (!isLitellmLocalNoticeEligible(classification))
|
|
66171
|
+
return "skipped";
|
|
66172
|
+
const windowMs = deps.windowMs();
|
|
66173
|
+
const verdict = evaluateLitellmLocalNotice(state3, agent, now(), windowMs);
|
|
66174
|
+
if (!verdict.send) {
|
|
66175
|
+
state3 = verdict.next;
|
|
66176
|
+
deps.log(`[litellm-local-notice] suppressed (cooldown) agent=${agent} ` + `suppressedSoFar=${state3.suppressedCountByAgent[agent] ?? 0}`);
|
|
66177
|
+
return "suppressed";
|
|
66178
|
+
}
|
|
66179
|
+
const chats = deps.listNoticeChats();
|
|
66180
|
+
const markdown = renderLitellmLocalNotice({
|
|
66181
|
+
agent,
|
|
66182
|
+
suppressedSinceLastNotice: verdict.suppressedSinceLastNotice
|
|
66183
|
+
});
|
|
66184
|
+
let issued = 0;
|
|
66185
|
+
for (const chatId of chats) {
|
|
66186
|
+
try {
|
|
66187
|
+
deps.sendNotice(chatId, markdown);
|
|
66188
|
+
issued++;
|
|
66189
|
+
} catch (err) {
|
|
66190
|
+
deps.log(`[litellm-local-notice] send failed chat=${chatId} agent=${agent}: ` + `${err?.message ?? err}`);
|
|
66191
|
+
}
|
|
66192
|
+
}
|
|
66193
|
+
if (issued === 0) {
|
|
66194
|
+
deps.log(`[litellm-local-notice] no sends issued (chats=${chats.length}) agent=${agent} \u2014 ` + `window not armed, will retry on the next event`);
|
|
66195
|
+
return "skipped";
|
|
66196
|
+
}
|
|
66197
|
+
state3 = verdict.next;
|
|
66198
|
+
deps.emitMetric(buildLitellmLocalNoticeMetric({
|
|
66199
|
+
agent,
|
|
66200
|
+
suppressedCount: verdict.suppressedSinceLastNotice,
|
|
66201
|
+
windowMs
|
|
66202
|
+
}));
|
|
66203
|
+
deps.log(`[litellm-local-notice] posted agent=${agent} chats=${issued} ` + `suppressedSinceLast=${verdict.suppressedSinceLastNotice} windowMs=${windowMs}`);
|
|
66204
|
+
return "sent";
|
|
66205
|
+
} catch (err) {
|
|
66206
|
+
try {
|
|
66207
|
+
deps.log(`[litellm-local-notice] error agent=${agent}: ${err?.message ?? err}`);
|
|
66208
|
+
} catch {}
|
|
66209
|
+
return "skipped";
|
|
66210
|
+
}
|
|
66211
|
+
}
|
|
66212
|
+
return {
|
|
66213
|
+
onRateLimited,
|
|
66214
|
+
inspect: () => ({ state: state3 })
|
|
66215
|
+
};
|
|
66216
|
+
}
|
|
66217
|
+
|
|
65328
66218
|
// auto-fallback-fleet.ts
|
|
65329
66219
|
init_card_format();
|
|
65330
66220
|
init_auth_snapshot_format();
|
|
@@ -68634,7 +69524,22 @@ function parseModelCommand(text4) {
|
|
|
68634
69524
|
}
|
|
68635
69525
|
return { kind: "set", model: arg };
|
|
68636
69526
|
}
|
|
68637
|
-
|
|
69527
|
+
function isModelCommandBusy(ctx) {
|
|
69528
|
+
return ctx.currentTurnActive || ctx.turnInFlight;
|
|
69529
|
+
}
|
|
69530
|
+
function planModelCommand(parsed, ctx) {
|
|
69531
|
+
if (parsed.kind === "show" && ctx.menuEnabled)
|
|
69532
|
+
return { kind: "menu" };
|
|
69533
|
+
if (parsed.kind === "set" && isModelCommandBusy(ctx)) {
|
|
69534
|
+
return { kind: "queue", target: expandSrAlias(parsed.model) };
|
|
69535
|
+
}
|
|
69536
|
+
return { kind: "apply", parsed };
|
|
69537
|
+
}
|
|
69538
|
+
function modelCommandReceiptLine(agent, parsed, busy) {
|
|
69539
|
+
const arg = parsed.kind === "set" ? parsed.model : parsed.kind === "show" ? "(show)" : "(help)";
|
|
69540
|
+
return `telegram gateway: gw /model received agent=${agent} kind=${parsed.kind} arg=${arg} busy=${busy}`;
|
|
69541
|
+
}
|
|
69542
|
+
var PERSIST_NOTE = "_Session-only \u2014 this override lasts until the agent\u2019s next restart, then reverts to the configured `model:`. `/model default` clears it now. To change the default permanently, set `model:` in switchroom.yaml._";
|
|
68638
69543
|
function helpText2(deps, reason) {
|
|
68639
69544
|
const srAliasExamples = Object.keys(SR_MODEL_ALIASES).map((a) => `\`${a}\``).join(" \u00b7 ");
|
|
68640
69545
|
const lines = [];
|
|
@@ -69185,15 +70090,11 @@ var MODEL_ARG_RE2 = /^[A-Za-z0-9][A-Za-z0-9._/\[\]-]{0,99}$/;
|
|
|
69185
70090
|
function isValidModelArg2(arg) {
|
|
69186
70091
|
return MODEL_ARG_RE2.test(arg);
|
|
69187
70092
|
}
|
|
70093
|
+
var MODEL_CALLBACK_ALIAS2 = "mdl:alias:";
|
|
69188
70094
|
|
|
69189
70095
|
// gateway/session-model-file.ts
|
|
69190
70096
|
var SESSION_MODEL_FILE = ".session-model";
|
|
69191
|
-
var RELAUNCH_MODEL_INTENT_FILE = ".relaunch-model-intent";
|
|
69192
70097
|
var CONFIGURED_DEFAULT_MODEL_FILE = ".configured-default-model";
|
|
69193
|
-
var SESSION_MODEL_BOOT_ATTEMPTS_FILE = ".session-model-boot-attempts";
|
|
69194
|
-
function intentForRestartReason(_reason) {
|
|
69195
|
-
return "keep";
|
|
69196
|
-
}
|
|
69197
70098
|
function atomicWrite(path2, content3) {
|
|
69198
70099
|
const tmp = `${path2}.tmp-${process.pid}-${Date.now()}`;
|
|
69199
70100
|
writeFileSync21(tmp, content3, "utf8");
|
|
@@ -69207,17 +70108,6 @@ function serializeSessionModel(rec) {
|
|
|
69207
70108
|
})}
|
|
69208
70109
|
`;
|
|
69209
70110
|
}
|
|
69210
|
-
function parseSessionModel(text4) {
|
|
69211
|
-
try {
|
|
69212
|
-
const raw = JSON.parse(text4);
|
|
69213
|
-
if (typeof raw.model !== "string" || typeof raw.configuredDefaultAtWrite !== "string" || typeof raw.ts !== "number" || !isValidModelArg2(raw.model)) {
|
|
69214
|
-
return null;
|
|
69215
|
-
}
|
|
69216
|
-
return { model: raw.model, configuredDefaultAtWrite: raw.configuredDefaultAtWrite, ts: raw.ts };
|
|
69217
|
-
} catch {
|
|
69218
|
-
return null;
|
|
69219
|
-
}
|
|
69220
|
-
}
|
|
69221
70111
|
function writeSessionModelFile(agentDir, model, configuredDefaultAtWrite) {
|
|
69222
70112
|
if (!isValidModelArg2(model)) {
|
|
69223
70113
|
throw new Error(`refusing to persist non-canonical session model token: ${JSON.stringify(model)}`);
|
|
@@ -69231,20 +70121,9 @@ function readSessionModelFileRaw(agentDir) {
|
|
|
69231
70121
|
return null;
|
|
69232
70122
|
}
|
|
69233
70123
|
}
|
|
69234
|
-
function readSessionModelFile(agentDir) {
|
|
69235
|
-
const raw = readSessionModelFileRaw(agentDir);
|
|
69236
|
-
return raw == null ? null : parseSessionModel(raw);
|
|
69237
|
-
}
|
|
69238
70124
|
function clearSessionModelFile(agentDir) {
|
|
69239
70125
|
try {
|
|
69240
70126
|
rmSync4(join29(agentDir, SESSION_MODEL_FILE), { force: true });
|
|
69241
|
-
rmSync4(join29(agentDir, ".session-model-kept-notified"), { force: true });
|
|
69242
|
-
rmSync4(join29(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true });
|
|
69243
|
-
} catch {}
|
|
69244
|
-
}
|
|
69245
|
-
function clearSessionModelBootAttempts(agentDir) {
|
|
69246
|
-
try {
|
|
69247
|
-
rmSync4(join29(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true });
|
|
69248
70127
|
} catch {}
|
|
69249
70128
|
}
|
|
69250
70129
|
function restoreSessionModelFileRaw(agentDir, raw) {
|
|
@@ -69256,40 +70135,6 @@ function restoreSessionModelFileRaw(agentDir, raw) {
|
|
|
69256
70135
|
atomicWrite(join29(agentDir, SESSION_MODEL_FILE), raw);
|
|
69257
70136
|
} catch {}
|
|
69258
70137
|
}
|
|
69259
|
-
function writeRelaunchModelIntent(agentDir, intent, reason) {
|
|
69260
|
-
try {
|
|
69261
|
-
atomicWrite(join29(agentDir, RELAUNCH_MODEL_INTENT_FILE), `${JSON.stringify({ intent, reason, ts: Date.now() })}
|
|
69262
|
-
`);
|
|
69263
|
-
} catch (err) {
|
|
69264
|
-
process.stderr.write(`telegram gateway: relaunch-model-intent write failed (boot will revert): ${err?.message ?? String(err)}
|
|
69265
|
-
`);
|
|
69266
|
-
}
|
|
69267
|
-
}
|
|
69268
|
-
var GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX = "gateway-shutdown:";
|
|
69269
|
-
function readRelaunchModelIntent(agentDir) {
|
|
69270
|
-
try {
|
|
69271
|
-
const raw = readFileSync25(join29(agentDir, RELAUNCH_MODEL_INTENT_FILE), "utf8");
|
|
69272
|
-
const parsed = JSON.parse(raw);
|
|
69273
|
-
if (parsed.intent !== "keep" && parsed.intent !== "revert" || typeof parsed.reason !== "string" || typeof parsed.ts !== "number") {
|
|
69274
|
-
return null;
|
|
69275
|
-
}
|
|
69276
|
-
return { intent: parsed.intent, reason: parsed.reason, ts: parsed.ts };
|
|
69277
|
-
} catch {
|
|
69278
|
-
return null;
|
|
69279
|
-
}
|
|
69280
|
-
}
|
|
69281
|
-
function clearStaleGatewayShutdownIntent(agentDir) {
|
|
69282
|
-
const rec = readRelaunchModelIntent(agentDir);
|
|
69283
|
-
if (rec == null || !rec.reason.startsWith(GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX))
|
|
69284
|
-
return false;
|
|
69285
|
-
clearRelaunchModelIntent(agentDir);
|
|
69286
|
-
return true;
|
|
69287
|
-
}
|
|
69288
|
-
function clearRelaunchModelIntent(agentDir) {
|
|
69289
|
-
try {
|
|
69290
|
-
rmSync4(join29(agentDir, RELAUNCH_MODEL_INTENT_FILE), { force: true });
|
|
69291
|
-
} catch {}
|
|
69292
|
-
}
|
|
69293
70138
|
function readConfiguredDefaultModel(agentDir) {
|
|
69294
70139
|
try {
|
|
69295
70140
|
const v = readFileSync25(join29(agentDir, CONFIGURED_DEFAULT_MODEL_FILE), "utf8").trim();
|
|
@@ -69300,37 +70145,185 @@ function readConfiguredDefaultModel(agentDir) {
|
|
|
69300
70145
|
}
|
|
69301
70146
|
var SESSION_EFFORT_FILE = ".session-effort";
|
|
69302
70147
|
var EFFORT_LEVEL_RE = /^(low|medium|high|xhigh|max)$/;
|
|
69303
|
-
function
|
|
70148
|
+
function writeSessionEffortFile(agentDir, level, configuredDefaultAtWrite) {
|
|
70149
|
+
if (!EFFORT_LEVEL_RE.test(level)) {
|
|
70150
|
+
throw new Error(`refusing to persist non-allowlisted effort level: ${JSON.stringify(level)}`);
|
|
70151
|
+
}
|
|
70152
|
+
atomicWrite(join29(agentDir, SESSION_EFFORT_FILE), `${JSON.stringify({ level, configuredDefaultAtWrite: configuredDefaultAtWrite ?? "", ts: Date.now() })}
|
|
70153
|
+
`);
|
|
70154
|
+
}
|
|
70155
|
+
function clearSessionEffortFile(agentDir) {
|
|
70156
|
+
try {
|
|
70157
|
+
rmSync4(join29(agentDir, SESSION_EFFORT_FILE), { force: true });
|
|
70158
|
+
} catch {}
|
|
70159
|
+
}
|
|
70160
|
+
var PREMIUM_RECOVERY_FILE = ".premium-recovery";
|
|
70161
|
+
function parsePremiumRecovery(text4) {
|
|
69304
70162
|
try {
|
|
69305
70163
|
const raw = JSON.parse(text4);
|
|
69306
|
-
if (typeof raw.
|
|
70164
|
+
if (typeof raw.premiumModel !== "string" || !isValidModelArg2(raw.premiumModel) || typeof raw.ts !== "number" || !Array.isArray(raw.chats) || raw.chats.length === 0 || !raw.chats.every((c) => typeof c === "string" && c.length > 0)) {
|
|
69307
70165
|
return null;
|
|
69308
70166
|
}
|
|
69309
|
-
return {
|
|
70167
|
+
return { premiumModel: raw.premiumModel, chats: raw.chats, ts: raw.ts };
|
|
69310
70168
|
} catch {
|
|
69311
70169
|
return null;
|
|
69312
70170
|
}
|
|
69313
70171
|
}
|
|
69314
|
-
function
|
|
69315
|
-
if (!
|
|
69316
|
-
throw new Error(`refusing to persist non-
|
|
70172
|
+
function writePremiumRecoveryFile(agentDir, premiumModel, chats) {
|
|
70173
|
+
if (!isValidModelArg2(premiumModel)) {
|
|
70174
|
+
throw new Error(`refusing to persist non-canonical premium-recovery token: ${JSON.stringify(premiumModel)}`);
|
|
69317
70175
|
}
|
|
69318
|
-
|
|
70176
|
+
const clean = chats.filter((c) => typeof c === "string" && c.length > 0);
|
|
70177
|
+
if (clean.length === 0) {
|
|
70178
|
+
throw new Error("refusing to persist premium-recovery marker with no chats to notify");
|
|
70179
|
+
}
|
|
70180
|
+
atomicWrite(join29(agentDir, PREMIUM_RECOVERY_FILE), `${JSON.stringify({ premiumModel, chats: clean, ts: Date.now() })}
|
|
69319
70181
|
`);
|
|
69320
70182
|
}
|
|
69321
|
-
function
|
|
70183
|
+
function readPremiumRecoveryFile(agentDir) {
|
|
70184
|
+
let raw;
|
|
69322
70185
|
try {
|
|
69323
|
-
|
|
70186
|
+
raw = readFileSync25(join29(agentDir, PREMIUM_RECOVERY_FILE), "utf8");
|
|
69324
70187
|
} catch {
|
|
69325
70188
|
return null;
|
|
69326
70189
|
}
|
|
70190
|
+
const parsed = parsePremiumRecovery(raw);
|
|
70191
|
+
if (parsed == null) {
|
|
70192
|
+
clearPremiumRecoveryFile(agentDir);
|
|
70193
|
+
return null;
|
|
70194
|
+
}
|
|
70195
|
+
return parsed;
|
|
69327
70196
|
}
|
|
69328
|
-
function
|
|
70197
|
+
function clearPremiumRecoveryFile(agentDir) {
|
|
69329
70198
|
try {
|
|
69330
|
-
rmSync4(join29(agentDir,
|
|
70199
|
+
rmSync4(join29(agentDir, PREMIUM_RECOVERY_FILE), { force: true });
|
|
69331
70200
|
} catch {}
|
|
69332
70201
|
}
|
|
69333
70202
|
|
|
70203
|
+
// tier-downgrade.ts
|
|
70204
|
+
function decideTierDowngrade(input) {
|
|
70205
|
+
const configured = typeof input.configuredDefault === "string" ? input.configuredDefault.trim() : "";
|
|
70206
|
+
if (configured.length === 0) {
|
|
70207
|
+
return { action: "skip", reason: "unresolved" };
|
|
70208
|
+
}
|
|
70209
|
+
const override = input.sessionOverride;
|
|
70210
|
+
if (override == null || override.length === 0) {
|
|
70211
|
+
return { action: "skip", reason: "on-default" };
|
|
70212
|
+
}
|
|
70213
|
+
if (input.resolve(override) === input.resolve(configured)) {
|
|
70214
|
+
return { action: "skip", reason: "on-default" };
|
|
70215
|
+
}
|
|
70216
|
+
return { action: "downgrade", toModel: configured, fromModel: override };
|
|
70217
|
+
}
|
|
70218
|
+
function renderTierDowngradeNotice(fromModel, toModel, agent) {
|
|
70219
|
+
return `\u2935\ufe0f **Downgrading model to keep going** on agent **${agent}**
|
|
70220
|
+
` + `\`${fromModel}\` is overloaded across every account right now, so this turn is ` + `resuming on the default \`${toModel}\` to keep working. ` + `Re-issue \`/model ${fromModel}\` once it frees up \u2014 it won't switch back on its own.`;
|
|
70221
|
+
}
|
|
70222
|
+
function planTierDowngrade(decision, gateVerdict, agent) {
|
|
70223
|
+
if (decision.action !== "downgrade")
|
|
70224
|
+
return { kind: "skip" };
|
|
70225
|
+
if (gateVerdict === "skip-inflight")
|
|
70226
|
+
return { kind: "suppress" };
|
|
70227
|
+
if (gateVerdict === "skip-stale")
|
|
70228
|
+
return { kind: "skip" };
|
|
70229
|
+
return {
|
|
70230
|
+
kind: "downgrade",
|
|
70231
|
+
toModel: decision.toModel,
|
|
70232
|
+
fromModel: decision.fromModel,
|
|
70233
|
+
notice: renderTierDowngradeNotice(decision.fromModel, decision.toModel, agent)
|
|
70234
|
+
};
|
|
70235
|
+
}
|
|
70236
|
+
|
|
70237
|
+
// gateway/tier-downgrade-wiring.ts
|
|
70238
|
+
function runTierDowngrade(triggerAgent, deps) {
|
|
70239
|
+
const agentDir = deps.getAgentDir();
|
|
70240
|
+
if (!agentDir)
|
|
70241
|
+
return "skip";
|
|
70242
|
+
const configuredDefault = deps.getConfiguredDefault() ?? "";
|
|
70243
|
+
const decision = decideTierDowngrade({
|
|
70244
|
+
sessionOverride: deps.getSessionOverride(),
|
|
70245
|
+
configuredDefault,
|
|
70246
|
+
resolve: deps.resolve
|
|
70247
|
+
});
|
|
70248
|
+
const gateVerdict = deps.peekResumeGate();
|
|
70249
|
+
const plan = planTierDowngrade(decision, gateVerdict, triggerAgent);
|
|
70250
|
+
if (plan.kind === "skip") {
|
|
70251
|
+
if (decision.action === "downgrade") {
|
|
70252
|
+
deps.log(`[tier-downgrade] restart suppressed (${gateVerdict}) agent=${triggerAgent}`);
|
|
70253
|
+
}
|
|
70254
|
+
return "skip";
|
|
70255
|
+
}
|
|
70256
|
+
if (plan.kind === "suppress") {
|
|
70257
|
+
deps.log(`[tier-downgrade] give-up suppressed \u2014 a resume restart is already armed agent=${triggerAgent}`);
|
|
70258
|
+
return "restart-pending";
|
|
70259
|
+
}
|
|
70260
|
+
try {
|
|
70261
|
+
deps.writeCarrier(agentDir, plan.toModel, configuredDefault);
|
|
70262
|
+
} catch (err) {
|
|
70263
|
+
deps.log(`[tier-downgrade] failed to write session-model carrier \u2014 aborting downgrade: ${err?.message ?? err}`);
|
|
70264
|
+
return "skip";
|
|
70265
|
+
}
|
|
70266
|
+
deps.armResumeGate();
|
|
70267
|
+
deps.log(`[tier-downgrade] downgrading ${plan.fromModel} \u2192 ${plan.toModel} and resuming via self-restart agent=${triggerAgent}`);
|
|
70268
|
+
try {
|
|
70269
|
+
deps.writeRecoveryMarker(agentDir, plan.fromModel);
|
|
70270
|
+
} catch (err) {
|
|
70271
|
+
deps.log(`[tier-downgrade] premium-recovery marker write failed (non-fatal): ${err?.message ?? err}`);
|
|
70272
|
+
}
|
|
70273
|
+
deps.broadcastNotice(plan.notice);
|
|
70274
|
+
deps.selfRestart(deps.selfAgent(triggerAgent));
|
|
70275
|
+
return "downgraded";
|
|
70276
|
+
}
|
|
70277
|
+
|
|
70278
|
+
// premium-recovery.ts
|
|
70279
|
+
function renderPremiumRecoveryPing(premiumModel) {
|
|
70280
|
+
return {
|
|
70281
|
+
text: `\u2705 \`${premiumModel}\` is available again \u2014 tap to switch back to it for this session.`,
|
|
70282
|
+
buttonText: `Switch to ${premiumModel}`
|
|
70283
|
+
};
|
|
70284
|
+
}
|
|
70285
|
+
function premiumRecoveryClaimKey(agent, premiumModel) {
|
|
70286
|
+
return `premium-recovery:${agent}:${premiumModel}`;
|
|
70287
|
+
}
|
|
70288
|
+
|
|
70289
|
+
// gateway/premium-recovery-wiring.ts
|
|
70290
|
+
async function runPremiumRecoveryPing(deps) {
|
|
70291
|
+
const agentDir = deps.getAgentDir();
|
|
70292
|
+
if (!agentDir)
|
|
70293
|
+
return;
|
|
70294
|
+
const marker = deps.readMarker(agentDir);
|
|
70295
|
+
if (marker == null)
|
|
70296
|
+
return;
|
|
70297
|
+
if (!deps.decide())
|
|
70298
|
+
return;
|
|
70299
|
+
const agent = deps.getAgent();
|
|
70300
|
+
const granted = await deps.claimNotification(premiumRecoveryClaimKey(agent, marker.premiumModel));
|
|
70301
|
+
if (!granted) {
|
|
70302
|
+
deps.clearMarker(agentDir);
|
|
70303
|
+
return;
|
|
70304
|
+
}
|
|
70305
|
+
deps.clearMarker(agentDir);
|
|
70306
|
+
const ping = renderPremiumRecoveryPing(marker.premiumModel);
|
|
70307
|
+
const keyboard = {
|
|
70308
|
+
inline_keyboard: [
|
|
70309
|
+
[{ text: ping.buttonText, callback_data: `${MODEL_CALLBACK_ALIAS2}${marker.premiumModel}` }]
|
|
70310
|
+
]
|
|
70311
|
+
};
|
|
70312
|
+
const chats = marker.chats.length > 0 ? marker.chats : deps.fallbackChats();
|
|
70313
|
+
for (const chatId of chats) {
|
|
70314
|
+
deps.sendToChat(chatId, ping, keyboard);
|
|
70315
|
+
}
|
|
70316
|
+
deps.log(`[premium-recovery] ${marker.premiumModel} servable again \u2014 ping sent agent=${agent} chats=${chats.length}`);
|
|
70317
|
+
}
|
|
70318
|
+
|
|
70319
|
+
// premium-recovery.ts
|
|
70320
|
+
function decidePremiumRecovery(opts) {
|
|
70321
|
+
if (!opts.hasMarker)
|
|
70322
|
+
return { fire: false, reason: "no-marker" };
|
|
70323
|
+
const servable = opts.accounts.some((a) => !a.exhausted && !a.premiumWalled);
|
|
70324
|
+
return servable ? { fire: true, reason: "recovered" } : { fire: false, reason: "still-walled" };
|
|
70325
|
+
}
|
|
70326
|
+
|
|
69334
70327
|
// ../src/agents/model-picker.ts
|
|
69335
70328
|
var HEADER_RE = /Select model/;
|
|
69336
70329
|
var OPTION_RE = /^\s*(\u276f)?\s*(\d+)\.\s+(.*)$/;
|
|
@@ -69699,7 +70692,7 @@ function parseEffortCommand(text4) {
|
|
|
69699
70692
|
}
|
|
69700
70693
|
return { kind: "set", level: arg.toLowerCase() };
|
|
69701
70694
|
}
|
|
69702
|
-
var PERSIST_NOTE2 = "
|
|
70695
|
+
var PERSIST_NOTE2 = "_Session-only \u2014 this override lasts until the agent\u2019s next restart, then reverts to the configured `thinking_effort:`. `/effort default` clears it now. To change the default permanently, set `thinking_effort:` in switchroom.yaml._";
|
|
69703
70696
|
var LEVELS_INLINE = EFFORT_LEVELS.map((l) => `\`${l}\``).join(" \u00b7 ");
|
|
69704
70697
|
function helpText3(deps, reason) {
|
|
69705
70698
|
const lines = [];
|
|
@@ -76971,71 +77964,6 @@ import {
|
|
|
76971
77964
|
} from "fs";
|
|
76972
77965
|
import { join as join39 } from "path";
|
|
76973
77966
|
|
|
76974
|
-
// operator-events.ts
|
|
76975
|
-
function classifyClaudeError(raw) {
|
|
76976
|
-
try {
|
|
76977
|
-
return classifyInner(raw);
|
|
76978
|
-
} catch {
|
|
76979
|
-
return "unknown-4xx";
|
|
76980
|
-
}
|
|
76981
|
-
}
|
|
76982
|
-
function classifyInner(raw) {
|
|
76983
|
-
if (raw == null)
|
|
76984
|
-
return "unknown-4xx";
|
|
76985
|
-
const obj = typeof raw === "object" ? raw : {};
|
|
76986
|
-
const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
|
|
76987
|
-
const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
|
|
76988
|
-
const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
|
|
76989
|
-
const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
|
|
76990
|
-
const sdkCode = extractString(obj, "error_code") ?? "";
|
|
76991
|
-
if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
|
|
76992
|
-
const msg = message.toLowerCase();
|
|
76993
|
-
if (msg.includes("expired") || msg.includes("refresh")) {
|
|
76994
|
-
return "credentials-expired";
|
|
76995
|
-
}
|
|
76996
|
-
return "credentials-invalid";
|
|
76997
|
-
}
|
|
76998
|
-
if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
|
|
76999
|
-
return "credentials-invalid";
|
|
77000
|
-
}
|
|
77001
|
-
if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
|
|
77002
|
-
return "credit-exhausted";
|
|
77003
|
-
}
|
|
77004
|
-
if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
|
|
77005
|
-
return "rate-limited";
|
|
77006
|
-
}
|
|
77007
|
-
if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
|
|
77008
|
-
return "rate-limited";
|
|
77009
|
-
}
|
|
77010
|
-
if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
|
|
77011
|
-
return "agent-crashed";
|
|
77012
|
-
}
|
|
77013
|
-
if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
|
|
77014
|
-
return "agent-restarted-unexpectedly";
|
|
77015
|
-
}
|
|
77016
|
-
if (status != null) {
|
|
77017
|
-
if (status >= 400 && status < 500)
|
|
77018
|
-
return "unknown-4xx";
|
|
77019
|
-
if (status >= 500 && status < 600)
|
|
77020
|
-
return "unknown-5xx";
|
|
77021
|
-
}
|
|
77022
|
-
return "unknown-4xx";
|
|
77023
|
-
}
|
|
77024
|
-
function extractString(obj, key) {
|
|
77025
|
-
const v = obj[key];
|
|
77026
|
-
return typeof v === "string" && v.length > 0 ? v : null;
|
|
77027
|
-
}
|
|
77028
|
-
function extractNumber(obj, key) {
|
|
77029
|
-
const v = obj[key];
|
|
77030
|
-
return typeof v === "number" ? v : null;
|
|
77031
|
-
}
|
|
77032
|
-
function getNestedObj(obj, key) {
|
|
77033
|
-
const v = obj[key];
|
|
77034
|
-
return typeof v === "object" && v != null ? v : {};
|
|
77035
|
-
}
|
|
77036
|
-
var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
|
|
77037
|
-
var cooldownMap2 = new Map;
|
|
77038
|
-
|
|
77039
77967
|
// session-tail.ts
|
|
77040
77968
|
function sanitizeCwdToProjectName(cwd) {
|
|
77041
77969
|
return cwd.replace(/[^a-zA-Z0-9]/g, "-");
|
|
@@ -77220,7 +78148,7 @@ function detectErrorInTranscriptLine(line) {
|
|
|
77220
78148
|
const errStr = typeof obj.error === "string" ? obj.error : "";
|
|
77221
78149
|
const text4 = extractAssistantText(obj);
|
|
77222
78150
|
const kind2 = status === 429 ? isTransientUpstreamSignal(`${text4}
|
|
77223
|
-
${errStr}`) ||
|
|
78151
|
+
${errStr}`) || isLitellmProxyLocal429(`${text4}
|
|
77224
78152
|
${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text4 });
|
|
77225
78153
|
return {
|
|
77226
78154
|
kind: kind2,
|
|
@@ -79529,6 +80457,29 @@ var RESOURCE_KEYS = ["path", "endpoint", "url", "resource", "route"];
|
|
|
79529
80457
|
var ARG_SUMMARY_MAX_KEYS = 4;
|
|
79530
80458
|
var ARG_VALUE_MAX = 40;
|
|
79531
80459
|
var ARG_SUMMARY_LINE_MAX = 180;
|
|
80460
|
+
var CONTEXT_NOISE_KEYS = new Set([
|
|
80461
|
+
"reason",
|
|
80462
|
+
"why",
|
|
80463
|
+
"chat_id",
|
|
80464
|
+
"message_id",
|
|
80465
|
+
"message_thread_id",
|
|
80466
|
+
"thread_id",
|
|
80467
|
+
"origin_turn_id",
|
|
80468
|
+
"reply_to",
|
|
80469
|
+
"quote",
|
|
80470
|
+
"quote_text",
|
|
80471
|
+
"format",
|
|
80472
|
+
"parse_mode",
|
|
80473
|
+
"disable_web_page_preview",
|
|
80474
|
+
"disable_notification",
|
|
80475
|
+
"protect_content",
|
|
80476
|
+
"single_use",
|
|
80477
|
+
"ack_text",
|
|
80478
|
+
"inline_keyboard",
|
|
80479
|
+
"file_id"
|
|
80480
|
+
]);
|
|
80481
|
+
var SENSITIVE_VALUE_KEY_RE = /token|secret|password|passwd|key|auth|dsn|conn|url|credential|cookie|session/i;
|
|
80482
|
+
var NON_HTTP_DSN_RE = /\b(?:postgres(?:ql)?|mysql|mariadb|redis|rediss|mongodb(?:\+srv)?|amqp|amqps):\/\/\S*@\S+/gi;
|
|
79532
80483
|
var MCP_TOOL_DESCRIPTIONS = {
|
|
79533
80484
|
"mcp__agent-config__config_get": "Read its own merged config",
|
|
79534
80485
|
"mcp__agent-config__cron_list": "List its own scheduled tasks",
|
|
@@ -79579,8 +80530,12 @@ function formatPermissionCardBody(opts) {
|
|
|
79579
80530
|
}
|
|
79580
80531
|
const callerReason = callerSuppliedReason(opts.inputPreview);
|
|
79581
80532
|
const rawWhy = (callerReason ?? "").replace(/\s+/g, " ").trim();
|
|
79582
|
-
|
|
79583
|
-
|
|
80533
|
+
if (rawWhy.length > 0) {
|
|
80534
|
+
const truncatedWhy = rawWhy.length > DESCRIPTION_LINE_MAX ? rawWhy.slice(0, DESCRIPTION_LINE_MAX - 1) + "\u2026" : rawWhy;
|
|
80535
|
+
lines.push(`why: _${escapeTgHtml(truncatedWhy)}_`);
|
|
80536
|
+
} else {
|
|
80537
|
+
lines.push(`context: _${escapeTgHtml(synthesizeContext(opts.toolName, opts.inputPreview))}_`);
|
|
80538
|
+
}
|
|
79584
80539
|
const argSummary = mcpArgSummary(opts.toolName, opts.inputPreview);
|
|
79585
80540
|
if (argSummary) {
|
|
79586
80541
|
lines.push(`\u21b3 _${escapeTgHtml(argSummary)}_`);
|
|
@@ -79854,6 +80809,50 @@ function callerSuppliedReason(inputPreview) {
|
|
|
79854
80809
|
}
|
|
79855
80810
|
return null;
|
|
79856
80811
|
}
|
|
80812
|
+
function synthesizeContext(toolName, inputPreview) {
|
|
80813
|
+
const input = parseInput2(inputPreview);
|
|
80814
|
+
const summary = input ? salientInputSummary(input) : null;
|
|
80815
|
+
const base = summary ?? naturalAction(toolName, inputPreview);
|
|
80816
|
+
const turnRef = input ? readString2(input, "origin_turn_id") : null;
|
|
80817
|
+
return turnRef ? `${base} \u00b7 turn ${shortTurnRef(turnRef)}` : base;
|
|
80818
|
+
}
|
|
80819
|
+
function shortTurnRef(turnId) {
|
|
80820
|
+
const t = turnId.trim();
|
|
80821
|
+
return t.length <= 10 ? t : `\u2026${t.slice(-8)}`;
|
|
80822
|
+
}
|
|
80823
|
+
function salientInputSummary(input) {
|
|
80824
|
+
const parts = [];
|
|
80825
|
+
for (const [key, value] of Object.entries(input)) {
|
|
80826
|
+
if (CONTEXT_NOISE_KEYS.has(key))
|
|
80827
|
+
continue;
|
|
80828
|
+
if (value == null)
|
|
80829
|
+
continue;
|
|
80830
|
+
if (parts.length >= ARG_SUMMARY_MAX_KEYS) {
|
|
80831
|
+
parts.push("\u2026");
|
|
80832
|
+
break;
|
|
80833
|
+
}
|
|
80834
|
+
if (typeof value === "object") {
|
|
80835
|
+
parts.push(key);
|
|
80836
|
+
continue;
|
|
80837
|
+
}
|
|
80838
|
+
const shown = truncate6(redactSalientValue(key, String(value)), ARG_VALUE_MAX);
|
|
80839
|
+
if (shown.length === 0)
|
|
80840
|
+
continue;
|
|
80841
|
+
parts.push(`${key}: ${shown}`);
|
|
80842
|
+
}
|
|
80843
|
+
if (parts.length === 0)
|
|
80844
|
+
return null;
|
|
80845
|
+
const joined = parts.join(", ");
|
|
80846
|
+
return joined.length > ARG_SUMMARY_LINE_MAX ? joined.slice(0, ARG_SUMMARY_LINE_MAX - 1) + "\u2026" : joined;
|
|
80847
|
+
}
|
|
80848
|
+
function redactSalientValue(key, value) {
|
|
80849
|
+
if (SENSITIVE_VALUE_KEY_RE.test(key))
|
|
80850
|
+
return REDACTED_MARKER;
|
|
80851
|
+
const dsnScrubbed = value.replace(NON_HTTP_DSN_RE, REDACTED_MARKER);
|
|
80852
|
+
const scrubbed = redact(`${key}=${dsnScrubbed}`);
|
|
80853
|
+
const prefix = `${key}=`;
|
|
80854
|
+
return scrubbed.startsWith(prefix) ? scrubbed.slice(prefix.length) : scrubbed;
|
|
80855
|
+
}
|
|
79857
80856
|
function extractReasonFromRaw(raw) {
|
|
79858
80857
|
const m = /"(?:reason|why)"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(raw);
|
|
79859
80858
|
if (!m)
|
|
@@ -80744,7 +81743,8 @@ function buildFleetRollMessage(roll, now) {
|
|
|
80744
81743
|
const pctPart = typeof roll.pct === "number" ? ` at ${fmtPct(roll.pct)}` : "";
|
|
80745
81744
|
const resetPart = typeof roll.exhausted_until === "number" && roll.exhausted_until > now ? ` (resets ${formatRelative(new Date(roll.exhausted_until), new Date(now))})` : "";
|
|
80746
81745
|
const softAvoid = roll.reason === "soft-avoid";
|
|
80747
|
-
const
|
|
81746
|
+
const modelTierWall = roll.reason === "model-tier-wall";
|
|
81747
|
+
const causeLine = softAvoid ? `Proactive switch \u2014 \`${codeSpanSafe(roll.from)}\` is approaching its limits (${winLabel}${pctPart})${resetPart}, so the fleet moved early instead of hitting the wall.` : modelTierWall ? `Flagship (premium) tier weekly limit reached on \`${codeSpanSafe(roll.from)}\`${resetPart} \u2014 opus/haiku on that account are unaffected.` : `${winLabel}${pctPart} on \`${codeSpanSafe(roll.from)}\`${resetPart}.`;
|
|
80748
81748
|
return [
|
|
80749
81749
|
`\uD83D\uDD01 **Switched fleet to \`${codeSpanSafe(roll.to)}\`**`,
|
|
80750
81750
|
``,
|
|
@@ -81081,10 +82081,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
|
|
|
81081
82081
|
}
|
|
81082
82082
|
|
|
81083
82083
|
// ../src/build-info.ts
|
|
81084
|
-
var VERSION = "0.18.
|
|
81085
|
-
var COMMIT_SHA = "
|
|
81086
|
-
var COMMIT_DATE = "2026-07-
|
|
81087
|
-
var LATEST_PR =
|
|
82084
|
+
var VERSION = "0.18.18";
|
|
82085
|
+
var COMMIT_SHA = "a256b3b1";
|
|
82086
|
+
var COMMIT_DATE = "2026-07-13T03:10:38Z";
|
|
82087
|
+
var LATEST_PR = 3210;
|
|
81088
82088
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
81089
82089
|
|
|
81090
82090
|
// gateway/boot-version.ts
|
|
@@ -81137,6 +82137,8 @@ function classifyRejection(err, opts = {}) {
|
|
|
81137
82137
|
return "log_only";
|
|
81138
82138
|
if (err instanceof Error && err.message === FLOOD_WAIT_ACTIVE)
|
|
81139
82139
|
return "log_only";
|
|
82140
|
+
if (err instanceof Error && err.message === LOCAL_RESOURCE_EXHAUSTED)
|
|
82141
|
+
return "log_only";
|
|
81140
82142
|
if (!isGrammy)
|
|
81141
82143
|
return "shutdown";
|
|
81142
82144
|
const e = err;
|
|
@@ -81677,10 +82679,13 @@ async function emitLinearAgentActivity(args, deps = {}) {
|
|
|
81677
82679
|
`);
|
|
81678
82680
|
return { content: [{ type: "text", text: `Linear ${type} emitted on session ${sessionId}` }] };
|
|
81679
82681
|
}
|
|
82682
|
+
function captureDedupComment(dedupKey) {
|
|
82683
|
+
return `<!-- switchroom-capture: ${dedupKey} -->`;
|
|
82684
|
+
}
|
|
81680
82685
|
function captureDedupMarker(dedupKey) {
|
|
81681
82686
|
return `
|
|
81682
82687
|
|
|
81683
|
-
|
|
82688
|
+
${captureDedupComment(dedupKey)}`;
|
|
81684
82689
|
}
|
|
81685
82690
|
async function createLinearIssue(args, deps = {}) {
|
|
81686
82691
|
const log = deps.log ?? ((s) => process.stderr.write(s));
|
|
@@ -81729,9 +82734,11 @@ async function createLinearIssue(args, deps = {}) {
|
|
|
81729
82734
|
return { ok: true, data: json.data };
|
|
81730
82735
|
};
|
|
81731
82736
|
if (dedupKey) {
|
|
81732
|
-
const
|
|
82737
|
+
const marker = captureDedupComment(dedupKey);
|
|
82738
|
+
const search2 = await gql("query($term: String!) { searchIssues(term: $term, first: 25) { nodes { id url title description } } }", { term: dedupKey });
|
|
81733
82739
|
if (search2.ok) {
|
|
81734
|
-
const
|
|
82740
|
+
const nodes = search2.data?.searchIssues?.nodes ?? [];
|
|
82741
|
+
const hit = nodes.find((n) => typeof n.description === "string" && n.description.includes(marker));
|
|
81735
82742
|
if (hit?.url) {
|
|
81736
82743
|
log(`telegram gateway: linear_create_issue: dedup hit key=${dedupKey} agent=${agent}
|
|
81737
82744
|
`);
|
|
@@ -82538,8 +83545,15 @@ function buildBridgeDeadIdleNoticeInbound(args) {
|
|
|
82538
83545
|
}
|
|
82539
83546
|
};
|
|
82540
83547
|
}
|
|
82541
|
-
function isRealBridgeIdentity(agentName3) {
|
|
82542
|
-
|
|
83548
|
+
function isRealBridgeIdentity(agentName3, selfAgentName) {
|
|
83549
|
+
if (agentName3 == null || agentName3.length === 0)
|
|
83550
|
+
return false;
|
|
83551
|
+
if (isCronIdentity2(agentName3))
|
|
83552
|
+
return false;
|
|
83553
|
+
if (selfAgentName != null && selfAgentName.length > 0) {
|
|
83554
|
+
return agentName3 === selfAgentName;
|
|
83555
|
+
}
|
|
83556
|
+
return true;
|
|
82543
83557
|
}
|
|
82544
83558
|
function createBridgeDeadWatchdog(opts) {
|
|
82545
83559
|
const setTimer = opts.setTimer ?? ((fn, ms) => {
|
|
@@ -82553,6 +83567,7 @@ function createBridgeDeadWatchdog(opts) {
|
|
|
82553
83567
|
const readCrashTail = opts.readCrashTail ?? ((p, t) => readFreshCrashLogTail(p, { nowMs: t }));
|
|
82554
83568
|
const priorStreak = opts.priorStreak ?? 0;
|
|
82555
83569
|
const maxConsecutive = opts.maxConsecutive ?? MAX_CONSECUTIVE_ESCALATIONS;
|
|
83570
|
+
const selfAgentName = opts.selfAgentName;
|
|
82556
83571
|
let timer3 = null;
|
|
82557
83572
|
let bridgeRegistered = false;
|
|
82558
83573
|
let bridgeEverRegistered = false;
|
|
@@ -82630,14 +83645,14 @@ function createBridgeDeadWatchdog(opts) {
|
|
|
82630
83645
|
return {
|
|
82631
83646
|
arm: armInternal,
|
|
82632
83647
|
noteBridgeRegistered: (agentName3) => {
|
|
82633
|
-
if (!isRealBridgeIdentity(agentName3))
|
|
83648
|
+
if (!isRealBridgeIdentity(agentName3, selfAgentName))
|
|
82634
83649
|
return;
|
|
82635
83650
|
bridgeRegistered = true;
|
|
82636
83651
|
bridgeEverRegistered = true;
|
|
82637
83652
|
cancel();
|
|
82638
83653
|
},
|
|
82639
83654
|
noteBridgeDisconnected: (agentName3) => {
|
|
82640
|
-
if (!isRealBridgeIdentity(agentName3))
|
|
83655
|
+
if (!isRealBridgeIdentity(agentName3, selfAgentName))
|
|
82641
83656
|
return;
|
|
82642
83657
|
bridgeRegistered = false;
|
|
82643
83658
|
armInternal();
|
|
@@ -82969,11 +83984,6 @@ function triggerSelfRestart(targetAgent, reason, delayMs = 300) {
|
|
|
82969
83984
|
`);
|
|
82970
83985
|
return false;
|
|
82971
83986
|
}
|
|
82972
|
-
{
|
|
82973
|
-
const smDir = resolveAgentDirFromEnv();
|
|
82974
|
-
if (smDir)
|
|
82975
|
-
writeRelaunchModelIntent(smDir, intentForRestartReason(reason), reason);
|
|
82976
|
-
}
|
|
82977
83987
|
process.stderr.write(`telegram gateway: restart-via-SIGTERM-PID1 agent=${targetAgent} reason=${reason} (docker)
|
|
82978
83988
|
`);
|
|
82979
83989
|
setTimeout(() => {
|
|
@@ -82986,11 +83996,6 @@ function triggerSelfRestart(targetAgent, reason, delayMs = 300) {
|
|
|
82986
83996
|
}, delayMs).unref();
|
|
82987
83997
|
return true;
|
|
82988
83998
|
}
|
|
82989
|
-
if (targetAgent === selfAgent) {
|
|
82990
|
-
const smDir = resolveAgentDirFromEnv();
|
|
82991
|
-
if (smDir)
|
|
82992
|
-
writeRelaunchModelIntent(smDir, intentForRestartReason(reason), reason);
|
|
82993
|
-
}
|
|
82994
83999
|
process.stderr.write(`telegram gateway: restart-via-systemctl agent=${targetAgent} reason=${reason}
|
|
82995
84000
|
`);
|
|
82996
84001
|
try {
|
|
@@ -83005,13 +84010,6 @@ function triggerSelfRestart(targetAgent, reason, delayMs = 300) {
|
|
|
83005
84010
|
return false;
|
|
83006
84011
|
}
|
|
83007
84012
|
}
|
|
83008
|
-
{
|
|
83009
|
-
const bootSmDir = resolveAgentDirFromEnv();
|
|
83010
|
-
if (bootSmDir != null && clearStaleGatewayShutdownIntent(bootSmDir)) {
|
|
83011
|
-
process.stderr.write(`telegram gateway: cleared stale gateway-shutdown relaunch-model intent (previous bounce was gateway-only \u2014 container never restarted)
|
|
83012
|
-
`);
|
|
83013
|
-
}
|
|
83014
|
-
}
|
|
83015
84013
|
var cachedClaudeCliVersion = undefined;
|
|
83016
84014
|
function getClaudeCliVersion() {
|
|
83017
84015
|
if (cachedClaudeCliVersion !== undefined)
|
|
@@ -83215,6 +84213,7 @@ function readAccessFile() {
|
|
|
83215
84213
|
parseMode: parsed.parseMode,
|
|
83216
84214
|
disableLinkPreview: parsed.disableLinkPreview,
|
|
83217
84215
|
coalescingGapMs: parsed.coalescingGapMs,
|
|
84216
|
+
litellmNoticeWindowMs: parsed.litellmNoticeWindowMs,
|
|
83218
84217
|
coalesceMaxAttachments: parsed.coalesceMaxAttachments,
|
|
83219
84218
|
interruptSafeBoundary: parsed.interruptSafeBoundary,
|
|
83220
84219
|
interruptMaxWaitMs: parsed.interruptMaxWaitMs,
|
|
@@ -84760,8 +85759,9 @@ var PHOTO_EXTS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp"]);
|
|
|
84760
85759
|
var FLOOD_STATE_PATH = floodStatePath(STATE_DIR);
|
|
84761
85760
|
var FLOOD_WINDOWS_PATH = floodWindowsPath(STATE_DIR);
|
|
84762
85761
|
var recordFloodWindow = makeFloodWindowRecorder(FLOOD_WINDOWS_PATH);
|
|
85762
|
+
var sendGateConfig = sendGateConfigFromEnv();
|
|
84763
85763
|
var sendGate = createSendGate({
|
|
84764
|
-
|
|
85764
|
+
...sendGateConfig,
|
|
84765
85765
|
initialWindows: loadInitialFloodWindows(FLOOD_STATE_PATH, FLOOD_WINDOWS_PATH, Date.now()),
|
|
84766
85766
|
bootRamp: {},
|
|
84767
85767
|
onWindowOpen: (scopeKey, untilTs) => recordFloodWindow(scopeKey, untilTs)
|
|
@@ -84769,10 +85769,10 @@ var sendGate = createSendGate({
|
|
|
84769
85769
|
var probeFloodWaitRemainingMs = makeFloodWaitProbe2(FLOOD_STATE_PATH);
|
|
84770
85770
|
var rawRobustApiCall = createRetryApiCall2({
|
|
84771
85771
|
log: (line) => process.stderr.write(line),
|
|
84772
|
-
onFloodWait: (retryAfterSec) => {
|
|
85772
|
+
onFloodWait: (retryAfterSec, opts) => {
|
|
84773
85773
|
makeFloodWaitRecorder2(FLOOD_STATE_PATH)(retryAfterSec);
|
|
84774
85774
|
try {
|
|
84775
|
-
sendGate.
|
|
85775
|
+
sendGate.openScopedFloodWindows(opts, Date.now() + Math.max(0, retryAfterSec) * 1000);
|
|
84776
85776
|
} catch {}
|
|
84777
85777
|
},
|
|
84778
85778
|
floodWaitRemainingMs: probeFloodWaitRemainingMs
|
|
@@ -84808,6 +85808,7 @@ var sendGateStatsLogger = createStatsLogger({
|
|
|
84808
85808
|
var floodWindowObserver = createFloodWindowObserver({
|
|
84809
85809
|
clock: { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) },
|
|
84810
85810
|
log: (line) => process.stderr.write(line),
|
|
85811
|
+
tz: process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC",
|
|
84811
85812
|
stats: () => sendGate.stats(),
|
|
84812
85813
|
readWindows: (now) => readFloodWindows(FLOOD_WINDOWS_PATH, now),
|
|
84813
85814
|
markAlerted: (scopeKey, alertedAt) => markFloodWindowAlerted(FLOOD_WINDOWS_PATH, scopeKey, alertedAt, Date.now()),
|
|
@@ -84822,7 +85823,7 @@ var floodWindowObserver = createFloodWindowObserver({
|
|
|
84822
85823
|
await robustApiCall(() => bot.api.sendRichMessage(operator, richMessage2(text5), {}), { chat_id: String(operator), verb: "send-gate-flood-alert", priorityClass: "critical" });
|
|
84823
85824
|
}
|
|
84824
85825
|
});
|
|
84825
|
-
if (
|
|
85826
|
+
if (sendGateConfig.enabled) {
|
|
84826
85827
|
const observeTimer = setInterval(() => {
|
|
84827
85828
|
try {
|
|
84828
85829
|
sendGateStatsLogger.tick();
|
|
@@ -85292,7 +86293,8 @@ function isAutoFallbackCooldownActive(_agentName, now) {
|
|
|
85292
86293
|
async function editCardExpired(chatId, messageId, body) {
|
|
85293
86294
|
if (messageId == null)
|
|
85294
86295
|
return;
|
|
85295
|
-
|
|
86296
|
+
const text5 = richMessage2(body);
|
|
86297
|
+
await robustApiCall(() => lockedBot.api.editMessageText(chatId, messageId, text5, { reply_markup: { inline_keyboard: [] } }), { chat_id: chatId, verb: "card-expired.strip", priorityClass: "cosmetic", messageId, editPayload: body }).catch(() => {});
|
|
85296
86298
|
}
|
|
85297
86299
|
function recordMissedApproval(opts) {
|
|
85298
86300
|
if (!MISSED_APPROVAL_REOFFER_ENABLED)
|
|
@@ -85519,10 +86521,7 @@ function postPermissionCard(requestId, pend) {
|
|
|
85519
86521
|
if (live && sent && typeof sent.message_id === "number") {
|
|
85520
86522
|
const landedThreadId = sent.message_thread_id ?? undefined;
|
|
85521
86523
|
live.cards.push({ chatId, messageId: sent.message_id, threadId: landedThreadId });
|
|
85522
|
-
if (live.
|
|
85523
|
-
live.undeliverable = null;
|
|
85524
|
-
live.redeliveryFailures = 0;
|
|
85525
|
-
live.startedAt = Date.now();
|
|
86524
|
+
if (applyDeliveredHoldReset(live, Date.now())) {
|
|
85526
86525
|
reconcileBlockedApprovals();
|
|
85527
86526
|
process.stderr.write(`telegram gateway: permission-card RE-DELIVERED request=${requestId} tool=${live.tool_name} chat=${chatId} \u2014 the operator can answer now; ` + `TTL clock restarted (they had zero seconds while the channel was shut)
|
|
85528
86527
|
`);
|
|
@@ -85771,7 +86770,8 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85771
86770
|
const { agent, kind } = event;
|
|
85772
86771
|
let throttleEscalation = null;
|
|
85773
86772
|
let escalationFired = false;
|
|
85774
|
-
|
|
86773
|
+
let rateLimitedCooldownConsulted = false;
|
|
86774
|
+
const rateLimit429Classification = kind === "rate-limited" ? classify429Detail2(event.detail) : null;
|
|
85775
86775
|
if (rateLimit429Classification != null && rateLimit429Classification !== "account-scoped") {
|
|
85776
86776
|
emitRuntimeMetric(build429ClassifiedMetric({
|
|
85777
86777
|
agent,
|
|
@@ -85780,10 +86780,28 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85780
86780
|
action: "calm",
|
|
85781
86781
|
now: Date.now()
|
|
85782
86782
|
}));
|
|
85783
|
-
|
|
86783
|
+
const surface = decideRateLimitedSurface({
|
|
86784
|
+
classification: rateLimit429Classification,
|
|
86785
|
+
agent,
|
|
86786
|
+
shouldEmitCard: (a) => shouldEmitOperatorEvent(a, "rate-limited")
|
|
86787
|
+
});
|
|
86788
|
+
if (surface === "litellm-local-notice") {
|
|
85784
86789
|
process.stderr.write(`telegram gateway: 429 classified litellm-proxy-local agent=${agent} \u2014 ` + `calm path, no account attribution, no failover
|
|
85785
86790
|
`);
|
|
86791
|
+
const outcome = litellmLocalNoticeRunner.onRateLimited("litellm-local", agent);
|
|
86792
|
+
if (outcome === "sent") {
|
|
86793
|
+
try {
|
|
86794
|
+
recordOperatorEvent(event);
|
|
86795
|
+
} catch {}
|
|
86796
|
+
}
|
|
86797
|
+
return;
|
|
85786
86798
|
}
|
|
86799
|
+
if (surface === "cooldown-suppressed") {
|
|
86800
|
+
process.stderr.write(`telegram gateway: operator-event suppressed (cooldown) agent=${agent} kind=${kind}
|
|
86801
|
+
`);
|
|
86802
|
+
return;
|
|
86803
|
+
}
|
|
86804
|
+
rateLimitedCooldownConsulted = true;
|
|
85787
86805
|
}
|
|
85788
86806
|
if (rateLimit429Classification === "account-scoped") {
|
|
85789
86807
|
const throttleDecision = decideThrottleTier({
|
|
@@ -85816,7 +86834,7 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85816
86834
|
}
|
|
85817
86835
|
}
|
|
85818
86836
|
}
|
|
85819
|
-
if (!shouldEmitOperatorEvent(agent, kind)) {
|
|
86837
|
+
if (!rateLimitedCooldownConsulted && !shouldEmitOperatorEvent(agent, kind)) {
|
|
85820
86838
|
process.stderr.write(`telegram gateway: operator-event suppressed (cooldown) agent=${agent} kind=${kind}
|
|
85821
86839
|
`);
|
|
85822
86840
|
return;
|
|
@@ -85845,6 +86863,18 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85845
86863
|
const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime());
|
|
85846
86864
|
fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt);
|
|
85847
86865
|
}
|
|
86866
|
+
} else if (kind === "rate-limited" || kind === "unknown-5xx") {
|
|
86867
|
+
const parsed = parseLlmError(event.detail);
|
|
86868
|
+
const now = Date.now();
|
|
86869
|
+
if (decideErrorSurface(parsed, agent, { claim: true, now }) === "suppress") {
|
|
86870
|
+
process.stderr.write(`telegram gateway: operator-event collapsed (error-presence-gate) agent=${agent} kind=${kind}
|
|
86871
|
+
`);
|
|
86872
|
+
return;
|
|
86873
|
+
}
|
|
86874
|
+
const tz = process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC";
|
|
86875
|
+
const r = renderLlmError(parsed, agent, tz, new Date(now));
|
|
86876
|
+
renderedText = r.text;
|
|
86877
|
+
renderedKeyboard = r.keyboard;
|
|
85848
86878
|
} else {
|
|
85849
86879
|
try {
|
|
85850
86880
|
const r = renderOperatorEvent(event);
|
|
@@ -86141,6 +87171,10 @@ async function runMidSessionCardReaper() {
|
|
|
86141
87171
|
const reaps = decideWorkerPinReaps({
|
|
86142
87172
|
pins: candidates,
|
|
86143
87173
|
statusOf: (agentId) => {
|
|
87174
|
+
if (agentId.startsWith("group:")) {
|
|
87175
|
+
const feedKey = agentId.slice("group:".length);
|
|
87176
|
+
return workerActivityFeed?.hasRunningInFeed(feedKey) ? "running" : "terminal";
|
|
87177
|
+
}
|
|
86144
87178
|
if (turnsDb == null)
|
|
86145
87179
|
return "unknown";
|
|
86146
87180
|
try {
|
|
@@ -86240,24 +87274,6 @@ async function reconcileStatusPinInner(pinKey, chatId, desired) {
|
|
|
86240
87274
|
statusPinPinnedAt.set(pinKey, Date.now());
|
|
86241
87275
|
}
|
|
86242
87276
|
}
|
|
86243
|
-
function reconcileWorkerPin(agentId, chatId, running) {
|
|
86244
|
-
if (!PIN_STATUS_WHILE_WORKING)
|
|
86245
|
-
return;
|
|
86246
|
-
const key = `wk:${agentId}`;
|
|
86247
|
-
if (!running) {
|
|
86248
|
-
const unpinChat = chatId ?? statusPinChatIds.get(key);
|
|
86249
|
-
if (unpinChat == null)
|
|
86250
|
-
return;
|
|
86251
|
-
reconcileStatusPin(key, unpinChat, { pinned: false });
|
|
86252
|
-
return;
|
|
86253
|
-
}
|
|
86254
|
-
if (chatId == null)
|
|
86255
|
-
return;
|
|
86256
|
-
const messageId = workerActivityFeed?.messageIdOf(agentId) ?? null;
|
|
86257
|
-
if (messageId == null)
|
|
86258
|
-
return;
|
|
86259
|
-
reconcileStatusPin(key, chatId, { pinned: true, messageId });
|
|
86260
|
-
}
|
|
86261
87277
|
async function unpinAllStatusPins() {
|
|
86262
87278
|
const keys = [...statusPinState.keys()];
|
|
86263
87279
|
for (const key of keys) {
|
|
@@ -86890,7 +87906,8 @@ var bridgeDeadWatchdog = createBridgeDeadWatchdog({
|
|
|
86890
87906
|
markerPath: join54(STATE_DIR, "bridge-dead-escalation.json"),
|
|
86891
87907
|
log: (line) => process.stderr.write(`${line}
|
|
86892
87908
|
`),
|
|
86893
|
-
priorStreak: bridgeDeadPriorStreak
|
|
87909
|
+
priorStreak: bridgeDeadPriorStreak,
|
|
87910
|
+
selfAgentName: process.env.SWITCHROOM_AGENT_NAME ?? ""
|
|
86894
87911
|
});
|
|
86895
87912
|
if (BRIDGE_DEAD_ESCALATION_ENABLED) {
|
|
86896
87913
|
bridgeDeadWatchdog.arm();
|
|
@@ -86914,11 +87931,6 @@ var ipcServer = createIpcServer({
|
|
|
86914
87931
|
}
|
|
86915
87932
|
const bridgeUpEffects = client3.agentName != null ? shadowEmit({ kind: "bridgeUp", at: Date.now() }) : [];
|
|
86916
87933
|
bridgeDeadWatchdog.noteBridgeRegistered(client3.agentName);
|
|
86917
|
-
if (client3.agentName != null) {
|
|
86918
|
-
const smBootDir = resolveAgentDirFromEnv();
|
|
86919
|
-
if (smBootDir != null)
|
|
86920
|
-
clearSessionModelBootAttempts(smBootDir);
|
|
86921
|
-
}
|
|
86922
87934
|
client3.send({ type: "status", status: "agent_connected" });
|
|
86923
87935
|
if (client3.agentName != null) {
|
|
86924
87936
|
if (isDispatchEnabled()) {
|
|
@@ -92987,11 +93999,6 @@ function buildModelDeps(restartCtx) {
|
|
|
92987
93999
|
});
|
|
92988
94000
|
}
|
|
92989
94001
|
stampUserRestartReason(reason);
|
|
92990
|
-
{
|
|
92991
|
-
const smDir = resolveAgentDirFromEnv();
|
|
92992
|
-
if (smDir)
|
|
92993
|
-
writeRelaunchModelIntent(smDir, "keep", reason);
|
|
92994
|
-
}
|
|
92995
94002
|
await sweepBeforeSelfRestart();
|
|
92996
94003
|
const hostdResp = await tryHostdDispatch2(name, {
|
|
92997
94004
|
v: 1,
|
|
@@ -93006,11 +94013,6 @@ function buildModelDeps(restartCtx) {
|
|
|
93006
94013
|
}
|
|
93007
94014
|
if (hostdResp.result !== "started" && hostdResp.result !== "completed") {
|
|
93008
94015
|
clearRestartMarker();
|
|
93009
|
-
{
|
|
93010
|
-
const smDir = resolveAgentDirFromEnv();
|
|
93011
|
-
if (smDir)
|
|
93012
|
-
clearRelaunchModelIntent(smDir);
|
|
93013
|
-
}
|
|
93014
94016
|
throw new Error(`hostd restart failed (result=${hostdResp.result}): ${hostdResp.error ?? "(no details)"}`);
|
|
93015
94017
|
}
|
|
93016
94018
|
},
|
|
@@ -93028,7 +94030,6 @@ function buildModelDeps(restartCtx) {
|
|
|
93028
94030
|
if (err?.code !== "restart_in_flight") {
|
|
93029
94031
|
restoreSessionModelFileRaw(agentDir, prevFileRaw);
|
|
93030
94032
|
sessionModelSource.setOverride(prevOverride);
|
|
93031
|
-
clearRelaunchModelIntent(agentDir);
|
|
93032
94033
|
}
|
|
93033
94034
|
throw err;
|
|
93034
94035
|
}
|
|
@@ -93047,12 +94048,12 @@ function modelMenuReplyMarkup(reply) {
|
|
|
93047
94048
|
}
|
|
93048
94049
|
return kb;
|
|
93049
94050
|
}
|
|
93050
|
-
function recordTypedModelSwitch(reply, requestedModelArg,
|
|
94051
|
+
function recordTypedModelSwitch(reply, requestedModelArg, _deps) {
|
|
93051
94052
|
const requested = requestedModelArg != null ? expandSrAlias(requestedModelArg) : null;
|
|
93052
94053
|
if (requested?.toLowerCase() === "default") {
|
|
93053
|
-
const
|
|
93054
|
-
if (
|
|
93055
|
-
clearSessionModelFile(
|
|
94054
|
+
const smDir = resolveAgentDirFromEnv();
|
|
94055
|
+
if (smDir)
|
|
94056
|
+
clearSessionModelFile(smDir);
|
|
93056
94057
|
if (reply.selectedModel)
|
|
93057
94058
|
sessionModelSource.setOverride(null);
|
|
93058
94059
|
return "";
|
|
@@ -93060,33 +94061,13 @@ function recordTypedModelSwitch(reply, requestedModelArg, deps) {
|
|
|
93060
94061
|
if (!reply.selectedModel)
|
|
93061
94062
|
return "";
|
|
93062
94063
|
sessionModelSource.setOverride(reply.selectedModel);
|
|
93063
|
-
|
|
93064
|
-
if (smDir && requested && isValidModelArg(requested) && !isSrModel(requested)) {
|
|
93065
|
-
try {
|
|
93066
|
-
writeSessionModelFile(smDir, requested, readConfiguredDefaultModel(smDir) ?? resolveMainModel(deps.getConfiguredModel() ?? undefined));
|
|
93067
|
-
} catch (err) {
|
|
93068
|
-
process.stderr.write(`telegram gateway: session-model persist failed (typed /model): ${err?.message ?? String(err)}
|
|
93069
|
-
`);
|
|
93070
|
-
return `
|
|
93071
|
-
\u26A0\uFE0F Couldn\u2019t persist the sticky override \u2014 the switch is live now but won\u2019t survive a relaunch.`;
|
|
93072
|
-
}
|
|
93073
|
-
}
|
|
94064
|
+
clearPremiumRecoveryOnManualSwitch(reply.selectedModel);
|
|
93074
94065
|
return "";
|
|
93075
94066
|
}
|
|
93076
94067
|
function recordModelMenuSideEffects(outcome, modelDeps, cbChatId, cbThreadId, prevSessionModel) {
|
|
93077
94068
|
if (outcome.selectedModel) {
|
|
93078
94069
|
sessionModelSource.setOverride(outcome.selectedModel);
|
|
93079
|
-
|
|
93080
|
-
if (smDir && outcome.selectedModelToken) {
|
|
93081
|
-
try {
|
|
93082
|
-
writeSessionModelFile(smDir, outcome.selectedModelToken, readConfiguredDefaultModel(smDir) ?? resolveMainModel(modelDeps.getConfiguredModel() ?? undefined));
|
|
93083
|
-
} catch (err) {
|
|
93084
|
-
outcome.reply.text += `
|
|
93085
|
-
\u26A0\uFE0F Couldn\u2019t persist the sticky override \u2014 the switch is live now but won\u2019t survive a relaunch.`;
|
|
93086
|
-
process.stderr.write(`telegram gateway: session-model persist failed (menu): ${err?.message ?? String(err)}
|
|
93087
|
-
`);
|
|
93088
|
-
}
|
|
93089
|
-
}
|
|
94070
|
+
clearPremiumRecoveryOnManualSwitch(outcome.selectedModel);
|
|
93090
94071
|
}
|
|
93091
94072
|
if (outcome.clearedDefault) {
|
|
93092
94073
|
const smDir = resolveAgentDirFromEnv();
|
|
@@ -93183,7 +94164,6 @@ function persistQueuedCommandForRestart(action) {
|
|
|
93183
94164
|
}
|
|
93184
94165
|
const configured = readConfiguredDefaultModel(agentDir) ?? resolveMainModel(undefined);
|
|
93185
94166
|
writeSessionModelFile(agentDir, expandSrAlias(action.arg), configured);
|
|
93186
|
-
writeRelaunchModelIntent(agentDir, "keep", "queued /model carried across restart");
|
|
93187
94167
|
break;
|
|
93188
94168
|
}
|
|
93189
94169
|
case "clear-model":
|
|
@@ -93240,14 +94220,38 @@ bot.command("model", async (ctx) => {
|
|
|
93240
94220
|
const parsed = parseModelCommand(text5) ?? { kind: "show" };
|
|
93241
94221
|
const chatId = String(ctx.chat.id);
|
|
93242
94222
|
const threadId = resolveThreadId(chatId, ctx.message?.message_thread_id);
|
|
94223
|
+
const busyNow = currentTurn !== null || turnInFlightForGate();
|
|
94224
|
+
process.stderr.write(modelCommandReceiptLine(getMyAgentName(), parsed, busyNow) + `
|
|
94225
|
+
`);
|
|
94226
|
+
if (HISTORY_ENABLED && ctx.message?.message_id != null) {
|
|
94227
|
+
try {
|
|
94228
|
+
recordInbound({
|
|
94229
|
+
chat_id: chatId,
|
|
94230
|
+
thread_id: threadId ?? null,
|
|
94231
|
+
message_id: ctx.message.message_id,
|
|
94232
|
+
user: ctx.from?.username ?? (ctx.from?.id != null ? String(ctx.from.id) : null),
|
|
94233
|
+
user_id: ctx.from?.id != null ? String(ctx.from.id) : null,
|
|
94234
|
+
ts: ctx.message.date ?? Math.floor(Date.now() / 1000),
|
|
94235
|
+
text: text5
|
|
94236
|
+
});
|
|
94237
|
+
} catch (err) {
|
|
94238
|
+
process.stderr.write(`telegram gateway: /model recordInbound failed: ${err?.message ?? String(err)}
|
|
94239
|
+
`);
|
|
94240
|
+
}
|
|
94241
|
+
}
|
|
93243
94242
|
const deps = buildModelDeps({ chatId, threadId });
|
|
93244
|
-
|
|
94243
|
+
const disposition = planModelCommand(parsed, {
|
|
94244
|
+
currentTurnActive: currentTurn !== null,
|
|
94245
|
+
turnInFlight: turnInFlightForGate(),
|
|
94246
|
+
menuEnabled: process.env.SWITCHROOM_MODEL_MENU !== "0"
|
|
94247
|
+
});
|
|
94248
|
+
if (disposition.kind === "menu") {
|
|
93245
94249
|
const menu = await buildModelMenu(deps);
|
|
93246
94250
|
await switchroomReply(ctx, menu.text, { html: true, reply_markup: modelMenuReplyMarkup(menu) });
|
|
93247
94251
|
return;
|
|
93248
94252
|
}
|
|
93249
|
-
if (
|
|
93250
|
-
const target =
|
|
94253
|
+
if (disposition.kind === "queue") {
|
|
94254
|
+
const target = disposition.target;
|
|
93251
94255
|
const sent = await ctx.replyWithRichMessage(richMessage2(hardenCardBreaks2(ackText("model", target, escapeHtmlForTg2))), threadId != null ? { message_thread_id: threadId } : {});
|
|
93252
94256
|
enqueueSessionCommand({
|
|
93253
94257
|
kind: "model",
|
|
@@ -93266,34 +94270,24 @@ bot.command("model", async (ctx) => {
|
|
|
93266
94270
|
const persistWarning = recordTypedModelSwitch(reply, parsed.kind === "set" ? parsed.model : null, deps);
|
|
93267
94271
|
await switchroomReply(ctx, reply.text + persistWarning, { html: reply.html });
|
|
93268
94272
|
});
|
|
94273
|
+
var sessionEffortOverride = null;
|
|
93269
94274
|
function buildEffortDeps() {
|
|
93270
94275
|
return {
|
|
93271
94276
|
applyEffort: async (agent, level) => {
|
|
93272
94277
|
const result = await applyEffort(agent, level);
|
|
93273
|
-
if (result.ok)
|
|
93274
|
-
|
|
93275
|
-
if (agentDir) {
|
|
93276
|
-
try {
|
|
93277
|
-
writeSessionEffortFile(agentDir, level, getConfiguredEffortForPersist());
|
|
93278
|
-
} catch (err) {
|
|
93279
|
-
process.stderr.write(`telegram gateway: session-effort persist failed level=${level}: ${err?.message ?? String(err)}
|
|
93280
|
-
`);
|
|
93281
|
-
}
|
|
93282
|
-
}
|
|
93283
|
-
}
|
|
94278
|
+
if (result.ok)
|
|
94279
|
+
sessionEffortOverride = level;
|
|
93284
94280
|
return result;
|
|
93285
94281
|
},
|
|
93286
94282
|
getAgentName: getMyAgentName,
|
|
93287
94283
|
getConfiguredEffort: () => getConfiguredEffortForPersist(),
|
|
93288
94284
|
clearSessionEffort: () => {
|
|
94285
|
+
sessionEffortOverride = null;
|
|
93289
94286
|
const agentDir = resolveAgentDirFromEnv();
|
|
93290
94287
|
if (agentDir)
|
|
93291
94288
|
clearSessionEffortFile(agentDir);
|
|
93292
94289
|
},
|
|
93293
|
-
getSessionEffort: () =>
|
|
93294
|
-
const agentDir = resolveAgentDirFromEnv();
|
|
93295
|
-
return agentDir ? readSessionEffortFile(agentDir)?.level ?? null : null;
|
|
93296
|
-
},
|
|
94290
|
+
getSessionEffort: () => sessionEffortOverride,
|
|
93297
94291
|
escapeHtml: escapeHtmlForTg2
|
|
93298
94292
|
};
|
|
93299
94293
|
}
|
|
@@ -93415,11 +94409,6 @@ bot.command("restart", async (ctx) => {
|
|
|
93415
94409
|
} catch {}
|
|
93416
94410
|
writeRestartMarker({ chat_id: chatId, thread_id: threadId ?? null, ack_message_id: ackId, ts: Date.now() });
|
|
93417
94411
|
stampUserRestartReason("user: /restart from chat");
|
|
93418
|
-
{
|
|
93419
|
-
const smDir = resolveAgentDirFromEnv();
|
|
93420
|
-
if (smDir)
|
|
93421
|
-
writeRelaunchModelIntent(smDir, "keep", "user: /restart from chat");
|
|
93422
|
-
}
|
|
93423
94412
|
await sweepBeforeSelfRestart();
|
|
93424
94413
|
const hostdResp = await tryHostdDispatch2(getMyAgentName(), {
|
|
93425
94414
|
v: 1,
|
|
@@ -93516,9 +94505,6 @@ async function handleNewCommand(ctx) {
|
|
|
93516
94505
|
}
|
|
93517
94506
|
}
|
|
93518
94507
|
stampUserRestartReason(`user: /${kind} from chat`);
|
|
93519
|
-
if (agentDir != null) {
|
|
93520
|
-
writeRelaunchModelIntent(agentDir, "keep", `user: /${kind} from chat`);
|
|
93521
|
-
}
|
|
93522
94508
|
await sweepBeforeSelfRestart();
|
|
93523
94509
|
const hostdResp = await tryHostdDispatch2(getMyAgentName(), {
|
|
93524
94510
|
v: 1,
|
|
@@ -93940,6 +94926,26 @@ var throttleTierRunner = createThrottleTierRunner({
|
|
|
93940
94926
|
log: (m) => process.stderr.write(`telegram gateway: ${m}
|
|
93941
94927
|
`)
|
|
93942
94928
|
});
|
|
94929
|
+
var litellmLocalNoticeRunner = createLitellmLocalNoticeRunner({
|
|
94930
|
+
listNoticeChats: () => loadAccess().allowFrom,
|
|
94931
|
+
sendNotice: (chat_id, markdown) => {
|
|
94932
|
+
const noticeTopic = resolveAgentOutboundTopic({ kind: "compact-watchdog" });
|
|
94933
|
+
const noticeSupergroup = resolveAgentSupergroupChatId();
|
|
94934
|
+
const noticeThread = topicForRecipient({
|
|
94935
|
+
recipientChatId: chat_id,
|
|
94936
|
+
resolvedTopic: noticeTopic,
|
|
94937
|
+
supergroupChatId: noticeSupergroup
|
|
94938
|
+
});
|
|
94939
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(markdown), {
|
|
94940
|
+
disable_notification: true,
|
|
94941
|
+
...noticeThread != null ? { message_thread_id: noticeThread } : {}
|
|
94942
|
+
}), { chat_id: String(chat_id), verb: "litellm-local-notice:notify" });
|
|
94943
|
+
},
|
|
94944
|
+
windowMs: () => parseLitellmNoticeWindowMs(loadAccess().litellmNoticeWindowMs),
|
|
94945
|
+
emitMetric: (event) => emitRuntimeMetric(event),
|
|
94946
|
+
log: (m) => process.stderr.write(`telegram gateway: ${m}
|
|
94947
|
+
`)
|
|
94948
|
+
});
|
|
93943
94949
|
var fallbackFailureNoticeState = { lastSentAtMs: 0 };
|
|
93944
94950
|
var fallbackAllBlockedNoticeState = { lastSentAtMs: 0 };
|
|
93945
94951
|
function broadcastFleetFallbackFailure(triggerAgent, reason) {
|
|
@@ -93960,6 +94966,80 @@ function broadcastFleetFallbackFailure(triggerAgent, reason) {
|
|
|
93960
94966
|
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(html), {}), { chat_id, verb: "fleet-fallback:failure-notify" });
|
|
93961
94967
|
}
|
|
93962
94968
|
}
|
|
94969
|
+
function broadcastTierNotice(markdown) {
|
|
94970
|
+
const access = loadAccess();
|
|
94971
|
+
if (access.allowFrom.length === 0)
|
|
94972
|
+
return;
|
|
94973
|
+
for (const chat_id of access.allowFrom) {
|
|
94974
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(markdown), { disable_notification: true }), { chat_id: String(chat_id), verb: "tier-downgrade:notify" });
|
|
94975
|
+
}
|
|
94976
|
+
}
|
|
94977
|
+
function maybeTierDowngrade(triggerAgent) {
|
|
94978
|
+
return runTierDowngrade(triggerAgent, {
|
|
94979
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
94980
|
+
getConfiguredDefault: () => {
|
|
94981
|
+
const dir = resolveAgentDirFromEnv();
|
|
94982
|
+
if (!dir)
|
|
94983
|
+
return null;
|
|
94984
|
+
return resolveMainModel(readConfiguredDefaultModel(dir) ?? undefined);
|
|
94985
|
+
},
|
|
94986
|
+
getSessionOverride: () => sessionModelSource.getOverride(),
|
|
94987
|
+
resolve: (t) => resolveMainModel(t),
|
|
94988
|
+
peekResumeGate: () => fleetFallbackResumeGate.peek(newestActiveTurnStartedAtMs()),
|
|
94989
|
+
writeCarrier: (dir, toModel, cfg) => writeSessionModelFile(dir, toModel, cfg),
|
|
94990
|
+
armResumeGate: () => fleetFallbackResumeGate.arm(),
|
|
94991
|
+
writeRecoveryMarker: (dir, premiumModel) => {
|
|
94992
|
+
const chats = loadAccess().allowFrom.map((c) => String(c));
|
|
94993
|
+
if (chats.length > 0)
|
|
94994
|
+
writePremiumRecoveryFile(dir, premiumModel, chats);
|
|
94995
|
+
},
|
|
94996
|
+
broadcastNotice: (md) => broadcastTierNotice(md),
|
|
94997
|
+
selfRestart: (agent) => triggerSelfRestart(agent, "tier-downgrade-resume"),
|
|
94998
|
+
selfAgent: (t) => process.env.SWITCHROOM_AGENT_NAME ?? t,
|
|
94999
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
95000
|
+
`)
|
|
95001
|
+
});
|
|
95002
|
+
}
|
|
95003
|
+
async function maybePremiumRecoveryPing(brokerClient, accounts) {
|
|
95004
|
+
return runPremiumRecoveryPing({
|
|
95005
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
95006
|
+
readMarker: (dir) => readPremiumRecoveryFile(dir),
|
|
95007
|
+
clearMarker: (dir) => clearPremiumRecoveryFile(dir),
|
|
95008
|
+
getAgent: () => getMyAgentName(),
|
|
95009
|
+
decide: () => decidePremiumRecovery({
|
|
95010
|
+
hasMarker: true,
|
|
95011
|
+
accounts: accounts.map((a) => ({
|
|
95012
|
+
exhausted: a.exhausted,
|
|
95013
|
+
premiumWalled: a.premium_walled === true
|
|
95014
|
+
}))
|
|
95015
|
+
}).fire,
|
|
95016
|
+
claimNotification: (key) => claimQuotaNotification(brokerClient, key),
|
|
95017
|
+
fallbackChats: () => loadAccess().allowFrom.map((c) => String(c)),
|
|
95018
|
+
sendToChat: (chat_id, ping, keyboard) => {
|
|
95019
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(ping.text), {
|
|
95020
|
+
disable_notification: true,
|
|
95021
|
+
reply_markup: keyboard
|
|
95022
|
+
}), { chat_id: String(chat_id), verb: "premium-recovery:notify" });
|
|
95023
|
+
},
|
|
95024
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
95025
|
+
`)
|
|
95026
|
+
});
|
|
95027
|
+
}
|
|
95028
|
+
function clearPremiumRecoveryOnManualSwitch(appliedToken) {
|
|
95029
|
+
if (appliedToken == null || appliedToken.length === 0)
|
|
95030
|
+
return;
|
|
95031
|
+
const agentDir = resolveAgentDirFromEnv();
|
|
95032
|
+
if (!agentDir)
|
|
95033
|
+
return;
|
|
95034
|
+
const marker = readPremiumRecoveryFile(agentDir);
|
|
95035
|
+
if (marker == null)
|
|
95036
|
+
return;
|
|
95037
|
+
if (resolveMainModel(appliedToken) === resolveMainModel(marker.premiumModel)) {
|
|
95038
|
+
clearPremiumRecoveryFile(agentDir);
|
|
95039
|
+
process.stderr.write(`telegram gateway: [premium-recovery] marker cleared \u2014 user manually re-issued /model ${marker.premiumModel}
|
|
95040
|
+
`);
|
|
95041
|
+
}
|
|
95042
|
+
}
|
|
93963
95043
|
async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, trigger) {
|
|
93964
95044
|
try {
|
|
93965
95045
|
const client3 = await getAuthBrokerClient2(triggerAgent);
|
|
@@ -93993,6 +95073,10 @@ async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, tri
|
|
|
93993
95073
|
if (outcome.kind === "switched") {
|
|
93994
95074
|
fallbackAllBlockedNoticeState = { lastSentAtMs: 0 };
|
|
93995
95075
|
} else if (outcome.kind === "all-blocked") {
|
|
95076
|
+
const tier = maybeTierDowngrade(triggerAgent);
|
|
95077
|
+
if (tier === "downgraded" || tier === "restart-pending") {
|
|
95078
|
+
return false;
|
|
95079
|
+
}
|
|
93996
95080
|
const verdict = evaluateAllBlockedNotice(fallbackAllBlockedNoticeState, Date.now());
|
|
93997
95081
|
if (!verdict.send) {
|
|
93998
95082
|
process.stderr.write(`telegram gateway: [fleet-fallback] all-blocked card suppressed (cooldown) agent=${triggerAgent}
|
|
@@ -94102,6 +95186,10 @@ async function runQuotaWatch(opts = {}) {
|
|
|
94102
95186
|
if (!listStateData.accounts || listStateData.accounts.length === 0) {
|
|
94103
95187
|
return;
|
|
94104
95188
|
}
|
|
95189
|
+
await maybePremiumRecoveryPing(brokerClient, listStateData.accounts).catch((err) => {
|
|
95190
|
+
process.stderr.write(`telegram gateway: [premium-recovery] ping check failed (non-fatal): ${err?.message ?? err}
|
|
95191
|
+
`);
|
|
95192
|
+
});
|
|
94105
95193
|
const snapshots = buildSnapshotsFromCachedState2(listStateData);
|
|
94106
95194
|
let watchState = loadQuotaWatchState(stateDir);
|
|
94107
95195
|
const now = Date.now();
|
|
@@ -96639,21 +97727,6 @@ async function shutdown(signal) {
|
|
|
96639
97727
|
`);
|
|
96640
97728
|
} catch (err) {
|
|
96641
97729
|
process.stderr.write(`telegram gateway: shutdown.clean_marker_write_failed err=${err.message}
|
|
96642
|
-
`);
|
|
96643
|
-
}
|
|
96644
|
-
try {
|
|
96645
|
-
const smDir = resolveAgentDirFromEnv();
|
|
96646
|
-
if (smDir != null) {
|
|
96647
|
-
const hasOverride = readSessionModelFile(smDir) != null;
|
|
96648
|
-
const intentAlreadyStamped = existsSync50(join54(smDir, RELAUNCH_MODEL_INTENT_FILE));
|
|
96649
|
-
if (hasOverride && !intentAlreadyStamped) {
|
|
96650
|
-
writeRelaunchModelIntent(smDir, "keep", `${GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX} graceful ${signal} shutdown (deploy/rolling restart) \u2014 preserving user-chosen session model`);
|
|
96651
|
-
process.stderr.write(`telegram gateway: shutdown.session_model_keep_stamped signal=${signal}
|
|
96652
|
-
`);
|
|
96653
|
-
}
|
|
96654
|
-
}
|
|
96655
|
-
} catch (err) {
|
|
96656
|
-
process.stderr.write(`telegram gateway: shutdown.session_model_keep_stamp_failed err=${err.message}
|
|
96657
97730
|
`);
|
|
96658
97731
|
}
|
|
96659
97732
|
} else {
|
|
@@ -97109,6 +98182,14 @@ var didOneTimeSetup = false;
|
|
|
97109
98182
|
sessionModelSource.setOverride(launched.length > 0 && launched !== configured ? launched : null);
|
|
97110
98183
|
} catch {}
|
|
97111
98184
|
}
|
|
98185
|
+
const activeEffortPath = join54(smAgentDir, ".active-session-effort");
|
|
98186
|
+
if (existsSync50(activeEffortPath)) {
|
|
98187
|
+
try {
|
|
98188
|
+
const launchedEffort = readFileSync54(activeEffortPath, "utf8").trim();
|
|
98189
|
+
const configuredEffort = getConfiguredEffortForPersist();
|
|
98190
|
+
sessionEffortOverride = launchedEffort.length > 0 && launchedEffort !== configuredEffort ? launchedEffort : null;
|
|
98191
|
+
} catch {}
|
|
98192
|
+
}
|
|
97112
98193
|
const alertPath = join54(smAgentDir, ".session-model-alert");
|
|
97113
98194
|
if (existsSync50(alertPath)) {
|
|
97114
98195
|
let alertText = null;
|
|
@@ -97210,6 +98291,10 @@ var didOneTimeSetup = false;
|
|
|
97210
98291
|
const watcherAgentDir = resolveAgentDirFromEnv();
|
|
97211
98292
|
if (watcherAgentDir != null) {
|
|
97212
98293
|
const workerFeedEnabled = isWorkerActivityFeedEnabled(process.env.SWITCHROOM_WORKER_ACTIVITY_FEED);
|
|
98294
|
+
const workerFeedMaxRows = (() => {
|
|
98295
|
+
const raw = Number(process.env.SWITCHROOM_TG_WORKER_FEED_MAX_ROWS);
|
|
98296
|
+
return Number.isInteger(raw) && raw > 0 ? raw : undefined;
|
|
98297
|
+
})();
|
|
97213
98298
|
const foregroundNestingEnabled = process.env.SWITCHROOM_FOREGROUND_SUBAGENT_NESTING !== "0";
|
|
97214
98299
|
const orphanStatusEnabled = isOrphanSubagentStatusEnabled(process.env.SWITCHROOM_ORPHAN_SUBAGENT_STATUS);
|
|
97215
98300
|
workerActivityFeed?.stop();
|
|
@@ -97227,6 +98312,21 @@ var didOneTimeSetup = false;
|
|
|
97227
98312
|
editPayload: text5
|
|
97228
98313
|
})
|
|
97229
98314
|
},
|
|
98315
|
+
floodWaitRemainingMs: probeFloodWaitRemainingMs,
|
|
98316
|
+
maxRows: workerFeedMaxRows,
|
|
98317
|
+
reconcilePin: ({ feedKey, chatId, messageId }) => {
|
|
98318
|
+
if (!PIN_STATUS_WHILE_WORKING)
|
|
98319
|
+
return;
|
|
98320
|
+
const key = `wk:group:${feedKey}`;
|
|
98321
|
+
if (messageId != null) {
|
|
98322
|
+
reconcileStatusPin(key, chatId, { pinned: true, messageId });
|
|
98323
|
+
} else {
|
|
98324
|
+
const unpinChat = chatId || statusPinChatIds.get(key);
|
|
98325
|
+
if (unpinChat != null && unpinChat.length > 0) {
|
|
98326
|
+
reconcileStatusPin(key, unpinChat, { pinned: false });
|
|
98327
|
+
}
|
|
98328
|
+
}
|
|
98329
|
+
},
|
|
97230
98330
|
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
97231
98331
|
`)
|
|
97232
98332
|
});
|
|
@@ -97292,7 +98392,6 @@ var didOneTimeSetup = false;
|
|
|
97292
98392
|
state: outcome === "failed" ? "failed" : "done",
|
|
97293
98393
|
model: dispatch.feedModel ?? undefined
|
|
97294
98394
|
});
|
|
97295
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97296
98395
|
}
|
|
97297
98396
|
return;
|
|
97298
98397
|
}
|
|
@@ -97340,7 +98439,6 @@ var didOneTimeSetup = false;
|
|
|
97340
98439
|
state: outcome === "failed" ? "failed" : "done",
|
|
97341
98440
|
model: dispatch.feedModel ?? undefined
|
|
97342
98441
|
});
|
|
97343
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97344
98442
|
}
|
|
97345
98443
|
return;
|
|
97346
98444
|
}
|
|
@@ -97354,7 +98452,6 @@ var didOneTimeSetup = false;
|
|
|
97354
98452
|
state: outcome === "failed" ? "failed" : "done",
|
|
97355
98453
|
model: dispatch.feedModel ?? undefined
|
|
97356
98454
|
});
|
|
97357
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97358
98455
|
}
|
|
97359
98456
|
const handbackOrigin = resolveSubagentOriginChat(agentId);
|
|
97360
98457
|
const decision = decideSubagentHandback({
|
|
@@ -97428,7 +98525,7 @@ var didOneTimeSetup = false;
|
|
|
97428
98525
|
elapsedMs,
|
|
97429
98526
|
state: "running",
|
|
97430
98527
|
model: feedModel
|
|
97431
|
-
}, wk.threadId)
|
|
98528
|
+
}, wk.threadId);
|
|
97432
98529
|
return;
|
|
97433
98530
|
}
|
|
97434
98531
|
if (surface !== "nest")
|
|
@@ -97487,7 +98584,7 @@ var didOneTimeSetup = false;
|
|
|
97487
98584
|
elapsedMs,
|
|
97488
98585
|
state: "running",
|
|
97489
98586
|
model: feedModel
|
|
97490
|
-
}, wk.threadId)
|
|
98587
|
+
}, wk.threadId);
|
|
97491
98588
|
return;
|
|
97492
98589
|
}
|
|
97493
98590
|
const progressOrigin = resolveSubagentOriginChat(agentId);
|