switchroom 0.18.14 → 0.18.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +3 -0
- package/dist/auth-broker/index.js +473 -49
- package/dist/cli/notion-write-pretool.mjs +3 -0
- package/dist/cli/switchroom.js +1200 -1067
- package/dist/host-control/main.js +56 -51
- package/dist/vault/approvals/kernel-server.js +19 -12
- package/dist/vault/broker/server.js +675 -668
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/dist/bridge/bridge.js +21 -0
- package/telegram-plugin/dist/gateway/gateway.js +531 -259
- package/telegram-plugin/dist/server.js +22 -1
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +3 -4
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +310 -219
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/session-model-file.ts +38 -172
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/model-unavailable.ts +214 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/runtime-metrics.ts +47 -0
- package/telegram-plugin/send-gate-degraded.test.ts +9 -7
- package/telegram-plugin/send-gate.ts +34 -4
- package/telegram-plugin/session-tail.ts +14 -2
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +2 -2
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/model-unavailable.test.ts +187 -0
- package/telegram-plugin/tests/operator-events-session-tail.test.ts +55 -0
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +2 -2
- package/telegram-plugin/tests/runtime-metrics.test.ts +24 -0
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/throttle-tier.test.ts +176 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +207 -0
- package/telegram-plugin/throttle-tier.ts +98 -1
- package/telegram-plugin/worker-activity-feed.ts +83 -8
|
@@ -20534,6 +20534,9 @@ var init_schema = __esm(() => {
|
|
|
20534
20534
|
window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
|
|
20535
20535
|
max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 \u2014 a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, \u2026). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
|
|
20536
20536
|
}).optional().describe("Inbound coalescing \u2014 how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
|
|
20537
|
+
litellm_notice: exports_external.object({
|
|
20538
|
+
window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 \u2014 see docs/auth.md \u00a7 " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
|
|
20539
|
+
}).optional().describe("Debounce tuning for the litellm-local throttle notice \u2014 the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
|
|
20537
20540
|
interrupt: exports_external.object({
|
|
20538
20541
|
safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
|
|
20539
20542
|
max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short \u2014 the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")
|
|
@@ -40236,6 +40239,60 @@ function renderActivityFeedWithNested(lines, childLines, final = false, liveSuff
|
|
|
40236
40239
|
});
|
|
40237
40240
|
}
|
|
40238
40241
|
|
|
40242
|
+
// retry-api-call.ts
|
|
40243
|
+
var import_grammy = __toESM(require_mod2(), 1);
|
|
40244
|
+
var FLOOD_WAIT_ACTIVE = "FLOOD_WAIT_ACTIVE";
|
|
40245
|
+
function makeFloodWaitActiveError(retryAfterSec, untilTs, original) {
|
|
40246
|
+
return Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
|
|
40247
|
+
retryAfterSec,
|
|
40248
|
+
untilTs,
|
|
40249
|
+
error_code: 429,
|
|
40250
|
+
parameters: { retry_after: retryAfterSec },
|
|
40251
|
+
original
|
|
40252
|
+
});
|
|
40253
|
+
}
|
|
40254
|
+
function isFloodWaitActiveError(err) {
|
|
40255
|
+
return err instanceof Error && err.message === FLOOD_WAIT_ACTIVE;
|
|
40256
|
+
}
|
|
40257
|
+
async function retryWithThreadFallback(retry, send, opts) {
|
|
40258
|
+
try {
|
|
40259
|
+
return await retry(() => send(opts.threadId), {
|
|
40260
|
+
...opts.threadId != null ? { threadId: opts.threadId } : {},
|
|
40261
|
+
chat_id: opts.chat_id,
|
|
40262
|
+
...opts.verb != null ? { verb: opts.verb } : {}
|
|
40263
|
+
});
|
|
40264
|
+
} catch (err) {
|
|
40265
|
+
if (err instanceof Error && err.message === "THREAD_NOT_FOUND") {
|
|
40266
|
+
return await retry(() => send(undefined), {
|
|
40267
|
+
chat_id: opts.chat_id,
|
|
40268
|
+
...opts.verb != null ? { verb: opts.verb } : {}
|
|
40269
|
+
});
|
|
40270
|
+
}
|
|
40271
|
+
throw err;
|
|
40272
|
+
}
|
|
40273
|
+
}
|
|
40274
|
+
function isHtmlParseRejectError(err) {
|
|
40275
|
+
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
40276
|
+
return false;
|
|
40277
|
+
if (isMessageTooLongError(err))
|
|
40278
|
+
return false;
|
|
40279
|
+
const d = (err.description || "").toLowerCase();
|
|
40280
|
+
return d.includes("can't parse entities") || d.includes("can\u2019t parse entities") || d.includes("can't parse") || d.includes("can\u2019t parse") || d.includes("parse markdown") || d.includes("parse rich") || d.includes("unsupported start tag") || d.includes("unclosed start tag") || d.includes("can't find end of the entity") || d.includes("can\u2019t find end of the entity") || d.includes("expected end tag");
|
|
40281
|
+
}
|
|
40282
|
+
function isMessageTooLongError(err) {
|
|
40283
|
+
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
40284
|
+
return false;
|
|
40285
|
+
const d = (err.description || "").toLowerCase();
|
|
40286
|
+
return d.includes("rich_message_text_too_long") || d.includes("message_too_long") || d.includes("text_too_long") || d.includes("message is too long") || d.includes("text is too long");
|
|
40287
|
+
}
|
|
40288
|
+
|
|
40289
|
+
// send-gate.ts
|
|
40290
|
+
var SEND_GATE_SHED = Symbol.for("switchroom.send-gate.shed");
|
|
40291
|
+
function isSendGateShed(value) {
|
|
40292
|
+
return value === SEND_GATE_SHED;
|
|
40293
|
+
}
|
|
40294
|
+
var GROUP_TYPES = new Set(["group", "supergroup"]);
|
|
40295
|
+
|
|
40239
40296
|
// worker-activity-feed.ts
|
|
40240
40297
|
function isWorkerActivityFeedEnabled(envVal) {
|
|
40241
40298
|
return envVal !== "0";
|
|
@@ -40310,6 +40367,7 @@ function classifyEditError(err) {
|
|
|
40310
40367
|
function createWorkerActivityFeed(opts) {
|
|
40311
40368
|
const log = opts.log ?? (() => {});
|
|
40312
40369
|
const nowFn = opts.now ?? Date.now;
|
|
40370
|
+
const floodWaitRemainingMs = opts.floodWaitRemainingMs ?? (() => 0);
|
|
40313
40371
|
const minEditInterval = opts.minEditIntervalMs ?? 2500;
|
|
40314
40372
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000;
|
|
40315
40373
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000;
|
|
@@ -40347,6 +40405,15 @@ function createWorkerActivityFeed(opts) {
|
|
|
40347
40405
|
h.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS;
|
|
40348
40406
|
log(`worker-feed: ${label} 429 \u2014 backing off ${retryAfter}s`);
|
|
40349
40407
|
}
|
|
40408
|
+
function parkIfFloodWindowOpen(h) {
|
|
40409
|
+
const remaining = floodWaitRemainingMs();
|
|
40410
|
+
if (remaining <= 0)
|
|
40411
|
+
return false;
|
|
40412
|
+
const until = nowFn() + remaining + COOLDOWN_JITTER_MS;
|
|
40413
|
+
if (until > h.cooldownUntil)
|
|
40414
|
+
h.cooldownUntil = until;
|
|
40415
|
+
return true;
|
|
40416
|
+
}
|
|
40350
40417
|
function accumulateNarrative(h, view) {
|
|
40351
40418
|
const line = view.latestSummary.trim();
|
|
40352
40419
|
if (line.length === 0)
|
|
@@ -40367,12 +40434,19 @@ function createWorkerActivityFeed(opts) {
|
|
|
40367
40434
|
h.dispatchAtMs = nowFn() - view.elapsedMs;
|
|
40368
40435
|
if (nowFn() < h.cooldownUntil)
|
|
40369
40436
|
return;
|
|
40437
|
+
if (parkIfFloodWindowOpen(h))
|
|
40438
|
+
return;
|
|
40370
40439
|
const body = renderWorkerActivity(merged, liveSuffix);
|
|
40371
40440
|
if (h.messageId == null) {
|
|
40372
40441
|
if (view.elapsedMs < firstPaintMin)
|
|
40373
40442
|
return;
|
|
40374
40443
|
try {
|
|
40375
40444
|
const sent = await opts.bot.sendMessage(h.chatId, body, sendOptsFor(h));
|
|
40445
|
+
if (sent == null || typeof sent.message_id !== "number") {
|
|
40446
|
+
parkIfFloodWindowOpen(h);
|
|
40447
|
+
log(`worker-feed: first paint shed by send gate agent=${h.agentId} \u2014 not delivered`);
|
|
40448
|
+
return;
|
|
40449
|
+
}
|
|
40376
40450
|
h.messageId = sent.message_id;
|
|
40377
40451
|
h.lastBody = body;
|
|
40378
40452
|
h.lastEditAt = nowFn();
|
|
@@ -40388,7 +40462,11 @@ function createWorkerActivityFeed(opts) {
|
|
|
40388
40462
|
if (nowFn() - h.lastEditAt < minEditInterval)
|
|
40389
40463
|
return;
|
|
40390
40464
|
try {
|
|
40391
|
-
await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
40465
|
+
const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
40466
|
+
if (isSendGateShed(res)) {
|
|
40467
|
+
parkIfFloodWindowOpen(h);
|
|
40468
|
+
return;
|
|
40469
|
+
}
|
|
40392
40470
|
h.lastBody = body;
|
|
40393
40471
|
h.lastEditAt = nowFn();
|
|
40394
40472
|
log(`worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} bytes=${body.length}`);
|
|
@@ -40418,7 +40496,7 @@ function createWorkerActivityFeed(opts) {
|
|
|
40418
40496
|
h.pendingFinish = null;
|
|
40419
40497
|
return;
|
|
40420
40498
|
}
|
|
40421
|
-
if (nowFn() < h.cooldownUntil) {
|
|
40499
|
+
if (parkIfFloodWindowOpen(h) || nowFn() < h.cooldownUntil) {
|
|
40422
40500
|
h.pendingFinish = view;
|
|
40423
40501
|
return;
|
|
40424
40502
|
}
|
|
@@ -40428,7 +40506,12 @@ function createWorkerActivityFeed(opts) {
|
|
|
40428
40506
|
return;
|
|
40429
40507
|
}
|
|
40430
40508
|
try {
|
|
40431
|
-
await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
40509
|
+
const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
40510
|
+
if (isSendGateShed(res)) {
|
|
40511
|
+
parkIfFloodWindowOpen(h);
|
|
40512
|
+
h.pendingFinish = view;
|
|
40513
|
+
return;
|
|
40514
|
+
}
|
|
40432
40515
|
h.lastBody = body;
|
|
40433
40516
|
h.lastEditAt = nowFn();
|
|
40434
40517
|
h.pendingFinish = null;
|
|
@@ -41022,55 +41105,6 @@ var STALE_TAP_NOTICE = "This request already resolved (timed out) \u2014 ask aga
|
|
|
41022
41105
|
var import_grammy2 = __toESM(require_mod2(), 1);
|
|
41023
41106
|
var import_runner = __toESM(require_mod4(), 1);
|
|
41024
41107
|
import { AsyncLocalStorage } from "async_hooks";
|
|
41025
|
-
|
|
41026
|
-
// retry-api-call.ts
|
|
41027
|
-
var import_grammy = __toESM(require_mod2(), 1);
|
|
41028
|
-
var FLOOD_WAIT_ACTIVE = "FLOOD_WAIT_ACTIVE";
|
|
41029
|
-
function makeFloodWaitActiveError(retryAfterSec, untilTs, original) {
|
|
41030
|
-
return Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
|
|
41031
|
-
retryAfterSec,
|
|
41032
|
-
untilTs,
|
|
41033
|
-
error_code: 429,
|
|
41034
|
-
parameters: { retry_after: retryAfterSec },
|
|
41035
|
-
original
|
|
41036
|
-
});
|
|
41037
|
-
}
|
|
41038
|
-
function isFloodWaitActiveError(err) {
|
|
41039
|
-
return err instanceof Error && err.message === FLOOD_WAIT_ACTIVE;
|
|
41040
|
-
}
|
|
41041
|
-
async function retryWithThreadFallback(retry, send, opts) {
|
|
41042
|
-
try {
|
|
41043
|
-
return await retry(() => send(opts.threadId), {
|
|
41044
|
-
...opts.threadId != null ? { threadId: opts.threadId } : {},
|
|
41045
|
-
chat_id: opts.chat_id,
|
|
41046
|
-
...opts.verb != null ? { verb: opts.verb } : {}
|
|
41047
|
-
});
|
|
41048
|
-
} catch (err) {
|
|
41049
|
-
if (err instanceof Error && err.message === "THREAD_NOT_FOUND") {
|
|
41050
|
-
return await retry(() => send(undefined), {
|
|
41051
|
-
chat_id: opts.chat_id,
|
|
41052
|
-
...opts.verb != null ? { verb: opts.verb } : {}
|
|
41053
|
-
});
|
|
41054
|
-
}
|
|
41055
|
-
throw err;
|
|
41056
|
-
}
|
|
41057
|
-
}
|
|
41058
|
-
function isHtmlParseRejectError(err) {
|
|
41059
|
-
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
41060
|
-
return false;
|
|
41061
|
-
if (isMessageTooLongError(err))
|
|
41062
|
-
return false;
|
|
41063
|
-
const d = (err.description || "").toLowerCase();
|
|
41064
|
-
return d.includes("can't parse entities") || d.includes("can\u2019t parse entities") || d.includes("can't parse") || d.includes("can\u2019t parse") || d.includes("parse markdown") || d.includes("parse rich") || d.includes("unsupported start tag") || d.includes("unclosed start tag") || d.includes("can't find end of the entity") || d.includes("can\u2019t find end of the entity") || d.includes("expected end tag");
|
|
41065
|
-
}
|
|
41066
|
-
function isMessageTooLongError(err) {
|
|
41067
|
-
if (!(err instanceof import_grammy.GrammyError) || err.error_code !== 400)
|
|
41068
|
-
return false;
|
|
41069
|
-
const d = (err.description || "").toLowerCase();
|
|
41070
|
-
return d.includes("rich_message_text_too_long") || d.includes("message_too_long") || d.includes("text_too_long") || d.includes("message is too long") || d.includes("text is too long");
|
|
41071
|
-
}
|
|
41072
|
-
|
|
41073
|
-
// shared/bot-runtime.ts
|
|
41074
41108
|
init_flood_circuit_breaker();
|
|
41075
41109
|
|
|
41076
41110
|
// shared/gw-trace-gate.ts
|
|
@@ -45307,6 +45341,12 @@ function createTypingEmitter(deps) {
|
|
|
45307
45341
|
|
|
45308
45342
|
// draft-stream.ts
|
|
45309
45343
|
var TELEGRAM_MAX_CHARS = 32768;
|
|
45344
|
+
function makeDraftEditShedError(messageId) {
|
|
45345
|
+
return Object.assign(new Error(`draft edit shed by send gate (id=${messageId ?? "unknown"}); snapshot preserved for re-flush`), { draftEditShed: true });
|
|
45346
|
+
}
|
|
45347
|
+
function isDraftEditShedError(err) {
|
|
45348
|
+
return err instanceof Error && err.draftEditShed === true;
|
|
45349
|
+
}
|
|
45310
45350
|
var DEFAULT_DM_THROTTLE_MS = 400;
|
|
45311
45351
|
var DEFAULT_GROUP_THROTTLE_MS = 1000;
|
|
45312
45352
|
var MIN_THROTTLE_MS = 250;
|
|
@@ -45325,6 +45365,7 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45325
45365
|
let messageId = config.initialMessageId ?? null;
|
|
45326
45366
|
let pendingText = null;
|
|
45327
45367
|
let lastSentText = null;
|
|
45368
|
+
let shedText = null;
|
|
45328
45369
|
let lastSentAt = 0;
|
|
45329
45370
|
let inFlight = null;
|
|
45330
45371
|
const streamStartedAt = Date.now();
|
|
@@ -45368,11 +45409,16 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45368
45409
|
await sendViaMessage(textToSend);
|
|
45369
45410
|
lastSentText = textToSend;
|
|
45370
45411
|
lastSentAt = Date.now();
|
|
45412
|
+
shedText = null;
|
|
45371
45413
|
} catch (err) {
|
|
45372
45414
|
const msg = err.message ?? String(err);
|
|
45373
|
-
if (
|
|
45415
|
+
if (isDraftEditShedError(err)) {
|
|
45416
|
+
shedText = textToSend;
|
|
45417
|
+
log?.(`stream \u2192 shed by send gate (id: ${messageId}); snapshot preserved for re-flush`);
|
|
45418
|
+
} else if (/\bmessage is not modified\b/i.test(msg)) {
|
|
45374
45419
|
lastSentText = textToSend;
|
|
45375
45420
|
lastSentAt = Date.now();
|
|
45421
|
+
shedText = null;
|
|
45376
45422
|
log?.(`stream \u2192 not modified (id: ${messageId})`);
|
|
45377
45423
|
} else if (/\bmessage to edit not found\b/i.test(msg) || /\bMESSAGE_ID_INVALID\b/i.test(msg)) {
|
|
45378
45424
|
log?.(`stream \u2192 message not found (id: ${messageId}), re-sending`);
|
|
@@ -45465,10 +45511,12 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45465
45511
|
}
|
|
45466
45512
|
return waitPromise;
|
|
45467
45513
|
},
|
|
45468
|
-
async finalize() {
|
|
45514
|
+
async finalize(finalText) {
|
|
45469
45515
|
if (final)
|
|
45470
45516
|
return;
|
|
45471
45517
|
final = true;
|
|
45518
|
+
if (finalText != null && !stopped)
|
|
45519
|
+
pendingText = finalText;
|
|
45472
45520
|
if (scheduledTimer != null) {
|
|
45473
45521
|
clearTimeout(scheduledTimer);
|
|
45474
45522
|
scheduledTimer = null;
|
|
@@ -45476,6 +45524,10 @@ function createDraftStream(send, edit, config = {}) {
|
|
|
45476
45524
|
if (inFlight) {
|
|
45477
45525
|
await inFlight;
|
|
45478
45526
|
}
|
|
45527
|
+
if (pendingText == null && shedText != null && !stopped) {
|
|
45528
|
+
pendingText = shedText;
|
|
45529
|
+
}
|
|
45530
|
+
shedText = null;
|
|
45479
45531
|
if (pendingText != null && !stopped) {
|
|
45480
45532
|
await flush();
|
|
45481
45533
|
}
|
|
@@ -54912,28 +54964,41 @@ function createStreamController(cfg) {
|
|
|
54912
54964
|
return bot.api.editMessageText(chatId, id, piece.text, opts);
|
|
54913
54965
|
return bot.api.editMessageText(chatId, id, richMessage(piece.text), opts);
|
|
54914
54966
|
};
|
|
54967
|
+
let handleRef = null;
|
|
54968
|
+
const editGateOpts = (id, payload) => ({
|
|
54969
|
+
threadId,
|
|
54970
|
+
chat_id: chatId,
|
|
54971
|
+
messageId: id,
|
|
54972
|
+
editPayload: payload,
|
|
54973
|
+
priorityClass: handleRef?.isFinal() === true ? "critical" : "cosmetic"
|
|
54974
|
+
});
|
|
54975
|
+
const piecePayload = (piece) => piece.rich ? richMessage(piece.text) : piece.text;
|
|
54915
54976
|
const tailIds = [];
|
|
54916
54977
|
const tailLastText = [];
|
|
54917
54978
|
const upsertTail = async (ti, piece) => {
|
|
54918
54979
|
const existingId = tailIds[ti];
|
|
54919
54980
|
if (existingId != null) {
|
|
54920
54981
|
if (tailLastText[ti] === piece.text)
|
|
54921
|
-
return;
|
|
54982
|
+
return false;
|
|
54922
54983
|
try {
|
|
54923
|
-
await retry(() => editPiece(existingId, piece, baseOpts),
|
|
54984
|
+
const res = await retry(() => editPiece(existingId, piece, baseOpts), editGateOpts(existingId, piecePayload(piece)));
|
|
54985
|
+
if (isSendGateShed(res))
|
|
54986
|
+
return true;
|
|
54924
54987
|
tailLastText[ti] = piece.text;
|
|
54925
54988
|
onEdit?.(existingId, piece.text.length);
|
|
54926
54989
|
} catch (err) {
|
|
54927
54990
|
if (!literalText && piece.rich && isParseEntitiesError(err)) {
|
|
54928
54991
|
warn?.(`stream-controller: tail-piece #${ti + 1} edit parse-entities rejected \u2014 retrying same id=${existingId} as plain text (${err instanceof Error ? err.message : String(err)})`);
|
|
54929
|
-
await retry(() => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts),
|
|
54992
|
+
const res = await retry(() => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts), editGateOpts(existingId, piece.text));
|
|
54993
|
+
if (isSendGateShed(res))
|
|
54994
|
+
return true;
|
|
54930
54995
|
tailLastText[ti] = piece.text;
|
|
54931
54996
|
onEdit?.(existingId, piece.text.length);
|
|
54932
54997
|
} else {
|
|
54933
54998
|
warn?.(`stream-controller: tail-piece #${ti + 1} edit FAILED (id=${existingId}) \u2014 partial delivery, this piece may be stale (${err instanceof Error ? err.message : String(err)})`);
|
|
54934
54999
|
}
|
|
54935
55000
|
}
|
|
54936
|
-
return;
|
|
55001
|
+
return false;
|
|
54937
55002
|
}
|
|
54938
55003
|
try {
|
|
54939
55004
|
const sent = await retry(() => sendPiece(piece, sendOpts), { threadId, chat_id: chatId });
|
|
@@ -54951,8 +55016,9 @@ function createStreamController(cfg) {
|
|
|
54951
55016
|
warn?.(`stream-controller: tail-piece #${ti + 1} send FAILED \u2014 partial delivery, this and later pieces may be missing this flush (${err instanceof Error ? err.message : String(err)})`);
|
|
54952
55017
|
}
|
|
54953
55018
|
}
|
|
55019
|
+
return false;
|
|
54954
55020
|
};
|
|
54955
|
-
|
|
55021
|
+
const handle = createDraftStream(async (text4) => {
|
|
54956
55022
|
const pieces = renderPieces(text4);
|
|
54957
55023
|
const head = pieces[0];
|
|
54958
55024
|
let anchorId;
|
|
@@ -54977,22 +55043,34 @@ function createStreamController(cfg) {
|
|
|
54977
55043
|
}, async (id, text4) => {
|
|
54978
55044
|
const pieces = renderPieces(text4);
|
|
54979
55045
|
const head = pieces[0];
|
|
55046
|
+
let anchorShed = false;
|
|
54980
55047
|
try {
|
|
54981
|
-
await retry(() => editPiece(id, head, baseOpts),
|
|
54982
|
-
|
|
55048
|
+
const res = await retry(() => editPiece(id, head, baseOpts), editGateOpts(id, piecePayload(head)));
|
|
55049
|
+
if (isSendGateShed(res)) {
|
|
55050
|
+
anchorShed = true;
|
|
55051
|
+
} else {
|
|
55052
|
+
onEdit?.(id, head.text.length);
|
|
55053
|
+
}
|
|
54983
55054
|
} catch (err) {
|
|
54984
55055
|
if (!literalText && head.rich && isParseEntitiesError(err)) {
|
|
54985
55056
|
warn?.(`stream-controller: edit parse-entities rejected \u2014 retrying same id=${id} as plain text (${err instanceof Error ? err.message : String(err)})`);
|
|
54986
55057
|
const fallbackBody = pieces.length === 1 ? text4 : head.text;
|
|
54987
|
-
await retry(() => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts),
|
|
54988
|
-
|
|
55058
|
+
const res = await retry(() => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts), editGateOpts(id, fallbackBody));
|
|
55059
|
+
if (isSendGateShed(res))
|
|
55060
|
+
anchorShed = true;
|
|
55061
|
+
else
|
|
55062
|
+
onEdit?.(id, head.text.length);
|
|
54989
55063
|
} else {
|
|
54990
55064
|
throw err;
|
|
54991
55065
|
}
|
|
54992
55066
|
}
|
|
55067
|
+
let anyTailShed = false;
|
|
54993
55068
|
for (let pi = 1;pi < pieces.length; pi++) {
|
|
54994
|
-
await upsertTail(pi - 1, pieces[pi])
|
|
55069
|
+
if (await upsertTail(pi - 1, pieces[pi]))
|
|
55070
|
+
anyTailShed = true;
|
|
54995
55071
|
}
|
|
55072
|
+
if (anchorShed || anyTailShed)
|
|
55073
|
+
throw makeDraftEditShedError(id);
|
|
54996
55074
|
}, {
|
|
54997
55075
|
...throttleMs != null ? { throttleMs } : {},
|
|
54998
55076
|
...idleMs != null ? { idleMs } : {},
|
|
@@ -55002,6 +55080,8 @@ function createStreamController(cfg) {
|
|
|
55002
55080
|
...initialMessageId != null ? { initialMessageId } : {},
|
|
55003
55081
|
chatId
|
|
55004
55082
|
});
|
|
55083
|
+
handleRef = handle;
|
|
55084
|
+
return handle;
|
|
55005
55085
|
}
|
|
55006
55086
|
|
|
55007
55087
|
// pty-partial-handler.ts
|
|
@@ -55285,7 +55365,7 @@ var systemClock = {
|
|
|
55285
55365
|
sleep: (ms) => new Promise((r) => setTimeout(r, ms))
|
|
55286
55366
|
};
|
|
55287
55367
|
var UNTAGGED_SEND_CLASS = "critical";
|
|
55288
|
-
|
|
55368
|
+
var SEND_GATE_SHED2 = Symbol.for("switchroom.send-gate.shed");
|
|
55289
55369
|
class TokenBucket {
|
|
55290
55370
|
capacity;
|
|
55291
55371
|
refillPerMs;
|
|
@@ -55363,7 +55443,7 @@ function stableStringify(value) {
|
|
|
55363
55443
|
return v;
|
|
55364
55444
|
});
|
|
55365
55445
|
}
|
|
55366
|
-
var
|
|
55446
|
+
var GROUP_TYPES2 = new Set(["group", "supergroup"]);
|
|
55367
55447
|
function createSendGate(config) {
|
|
55368
55448
|
const enabled2 = config.enabled;
|
|
55369
55449
|
const clock = config.clock ?? systemClock;
|
|
@@ -55439,7 +55519,7 @@ function createSendGate(config) {
|
|
|
55439
55519
|
const buckets = [globalBucket];
|
|
55440
55520
|
if (opts?.chat_id) {
|
|
55441
55521
|
buckets.push(chatBucket(opts.chat_id));
|
|
55442
|
-
if (opts.chatType &&
|
|
55522
|
+
if (opts.chatType && GROUP_TYPES2.has(opts.chatType)) {
|
|
55443
55523
|
buckets.push(groupBucket(opts.chat_id));
|
|
55444
55524
|
}
|
|
55445
55525
|
}
|
|
@@ -55454,7 +55534,7 @@ function createSendGate(config) {
|
|
|
55454
55534
|
applyWindow("global", untilTs, true);
|
|
55455
55535
|
if (opts?.chat_id) {
|
|
55456
55536
|
applyWindow(`chat:${opts.chat_id}`, untilTs, true);
|
|
55457
|
-
if (opts.chatType &&
|
|
55537
|
+
if (opts.chatType && GROUP_TYPES2.has(opts.chatType)) {
|
|
55458
55538
|
applyWindow(`group:${opts.chat_id}`, untilTs, true);
|
|
55459
55539
|
}
|
|
55460
55540
|
}
|
|
@@ -55668,7 +55748,7 @@ function createSendGate(config) {
|
|
|
55668
55748
|
const msgWait = state.suppressedUntilMs > now ? state.suppressedUntilMs - now : 0;
|
|
55669
55749
|
if (wait > 0 || msgWait > 0) {
|
|
55670
55750
|
counters.shed++;
|
|
55671
|
-
return Promise.resolve(
|
|
55751
|
+
return Promise.resolve(SEND_GATE_SHED2);
|
|
55672
55752
|
}
|
|
55673
55753
|
}
|
|
55674
55754
|
if (state.pending) {
|
|
@@ -55713,7 +55793,7 @@ function createSendGate(config) {
|
|
|
55713
55793
|
const outcome = await admitPriority(bucketsFor(opts), priority);
|
|
55714
55794
|
if (outcome.result === "shed") {
|
|
55715
55795
|
counters.shed++;
|
|
55716
|
-
return;
|
|
55796
|
+
return SEND_GATE_SHED2;
|
|
55717
55797
|
}
|
|
55718
55798
|
if (outcome.result === "expired") {
|
|
55719
55799
|
counters.expired++;
|
|
@@ -64640,6 +64720,26 @@ var transientUpstreamSignals = [
|
|
|
64640
64720
|
"would exceed your account\u2019s rate limit",
|
|
64641
64721
|
"would exceed your account's rate limit"
|
|
64642
64722
|
];
|
|
64723
|
+
var litellmProxyLocal429Signals = [
|
|
64724
|
+
"deployment over user-defined ratelimit",
|
|
64725
|
+
"model rate limit exceeded. tpm limit",
|
|
64726
|
+
"model rate limit exceeded. rpm limit",
|
|
64727
|
+
"deployment over defined rpm limit",
|
|
64728
|
+
"no deployments available for selected model",
|
|
64729
|
+
"litellm rate limit handler",
|
|
64730
|
+
"crossed tpm / rpm",
|
|
64731
|
+
"max parallel request limit reached"
|
|
64732
|
+
];
|
|
64733
|
+
var litellmV3LimiterSignalPair = ["rate limit exceeded for ", "limit type:"];
|
|
64734
|
+
function isLitellmProxyLocal429(text4) {
|
|
64735
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
64736
|
+
return false;
|
|
64737
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
64738
|
+
const lower = sample.toLowerCase();
|
|
64739
|
+
if (litellmProxyLocal429Signals.some((s) => lower.includes(s)))
|
|
64740
|
+
return true;
|
|
64741
|
+
return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
|
|
64742
|
+
}
|
|
64643
64743
|
function detectModelUnavailable(stderr) {
|
|
64644
64744
|
if (typeof stderr !== "string" || stderr.length === 0)
|
|
64645
64745
|
return null;
|
|
@@ -64649,6 +64749,10 @@ function detectModelUnavailable(stderr) {
|
|
|
64649
64749
|
const resetAt = parseResetTime(sample);
|
|
64650
64750
|
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
64651
64751
|
}
|
|
64752
|
+
if (isLitellmProxyLocal429(sample)) {
|
|
64753
|
+
const resetAt = parseResetTime(sample);
|
|
64754
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
64755
|
+
}
|
|
64652
64756
|
const quotaSignals = [
|
|
64653
64757
|
"out of extra usage",
|
|
64654
64758
|
"extra usage",
|
|
@@ -64898,6 +65002,76 @@ function isTransientUpstreamSignal(text4) {
|
|
|
64898
65002
|
const lower = sample.toLowerCase();
|
|
64899
65003
|
return transientUpstreamSignals2.some((s) => lower.includes(s));
|
|
64900
65004
|
}
|
|
65005
|
+
var litellmProxyLocal429Signals2 = [
|
|
65006
|
+
"deployment over user-defined ratelimit",
|
|
65007
|
+
"model rate limit exceeded. tpm limit",
|
|
65008
|
+
"model rate limit exceeded. rpm limit",
|
|
65009
|
+
"deployment over defined rpm limit",
|
|
65010
|
+
"no deployments available for selected model",
|
|
65011
|
+
"litellm rate limit handler",
|
|
65012
|
+
"crossed tpm / rpm",
|
|
65013
|
+
"max parallel request limit reached"
|
|
65014
|
+
];
|
|
65015
|
+
var litellmV3LimiterSignalPair2 = ["rate limit exceeded for ", "limit type:"];
|
|
65016
|
+
function isLitellmProxyLocal4292(text4) {
|
|
65017
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65018
|
+
return false;
|
|
65019
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65020
|
+
const lower = sample.toLowerCase();
|
|
65021
|
+
if (litellmProxyLocal429Signals2.some((s) => lower.includes(s)))
|
|
65022
|
+
return true;
|
|
65023
|
+
return litellmV3LimiterSignalPair2.every((s) => lower.includes(s));
|
|
65024
|
+
}
|
|
65025
|
+
function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
|
|
65026
|
+
const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
|
|
65027
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65028
|
+
return empty2;
|
|
65029
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65030
|
+
const lower = sample.toLowerCase();
|
|
65031
|
+
let limitType = null;
|
|
65032
|
+
let limit = null;
|
|
65033
|
+
const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
|
|
65034
|
+
if (eqLimit) {
|
|
65035
|
+
limitType = eqLimit[1];
|
|
65036
|
+
limit = Number(eqLimit[2]);
|
|
65037
|
+
}
|
|
65038
|
+
if (limitType == null) {
|
|
65039
|
+
const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
|
|
65040
|
+
if (v3Type)
|
|
65041
|
+
limitType = v3Type[1];
|
|
65042
|
+
}
|
|
65043
|
+
if (limit == null) {
|
|
65044
|
+
const v3Limit = lower.match(/current limit:\s*(\d+)/);
|
|
65045
|
+
if (v3Limit)
|
|
65046
|
+
limit = Number(v3Limit[1]);
|
|
65047
|
+
}
|
|
65048
|
+
let currentUsage = null;
|
|
65049
|
+
const usage = lower.match(/current usage=(\d+)/);
|
|
65050
|
+
if (usage)
|
|
65051
|
+
currentUsage = Number(usage[1]);
|
|
65052
|
+
let resetAtMs = null;
|
|
65053
|
+
const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
|
|
65054
|
+
if (resetsAt) {
|
|
65055
|
+
const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
|
|
65056
|
+
if (!Number.isNaN(d.getTime()))
|
|
65057
|
+
resetAtMs = d.getTime();
|
|
65058
|
+
}
|
|
65059
|
+
if (resetAtMs == null) {
|
|
65060
|
+
const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
|
|
65061
|
+
if (tryAgain) {
|
|
65062
|
+
const secs = Number(tryAgain[1]);
|
|
65063
|
+
if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
|
|
65064
|
+
resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
|
|
65065
|
+
}
|
|
65066
|
+
}
|
|
65067
|
+
}
|
|
65068
|
+
return {
|
|
65069
|
+
limitType,
|
|
65070
|
+
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
65071
|
+
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
65072
|
+
resetAtMs
|
|
65073
|
+
};
|
|
65074
|
+
}
|
|
64901
65075
|
function parseResetTime2(text4, parseTimeNow = new Date) {
|
|
64902
65076
|
const lower = text4.toLowerCase();
|
|
64903
65077
|
const retryAfter = lower.match(/retry[\s-]*after[:\s]+(\d+)\s*(seconds?|s\b|minutes?|m\b|hours?|h\b)?/);
|
|
@@ -65034,6 +65208,30 @@ function isAccountScopedThrottle(text4) {
|
|
|
65034
65208
|
const lower = sample.toLowerCase();
|
|
65035
65209
|
return accountScopedThrottleSignals.some((s) => lower.includes(s));
|
|
65036
65210
|
}
|
|
65211
|
+
function classify429Detail(text4) {
|
|
65212
|
+
if (isAccountScopedThrottle(text4))
|
|
65213
|
+
return "account-scoped";
|
|
65214
|
+
if (isLitellmProxyLocal4292(text4))
|
|
65215
|
+
return "litellm-local";
|
|
65216
|
+
return "generic-transient";
|
|
65217
|
+
}
|
|
65218
|
+
function build429ClassifiedMetric(opts) {
|
|
65219
|
+
const detail = typeof opts.detail === "string" ? opts.detail : "";
|
|
65220
|
+
const litellm = parseLitellmLimitDetail(detail, new Date(opts.now));
|
|
65221
|
+
const anthropicResetMs = parseResetTime2(detail, new Date(opts.now))?.getTime() ?? null;
|
|
65222
|
+
const resetAtMs = anthropicResetMs ?? litellm.resetAtMs;
|
|
65223
|
+
return {
|
|
65224
|
+
kind: "rate_limit_429_classified",
|
|
65225
|
+
agent: opts.agent,
|
|
65226
|
+
classification: opts.classification,
|
|
65227
|
+
action: opts.action,
|
|
65228
|
+
reset_at_ms: resetAtMs,
|
|
65229
|
+
reset_in_ms: resetAtMs != null ? Math.max(0, resetAtMs - opts.now) : null,
|
|
65230
|
+
limit_type: litellm.limitType,
|
|
65231
|
+
limit: litellm.limit,
|
|
65232
|
+
current_usage: litellm.currentUsage
|
|
65233
|
+
};
|
|
65234
|
+
}
|
|
65037
65235
|
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
|
|
65038
65236
|
var THROTTLE_DEFAULT_WAIT_MS = 60000;
|
|
65039
65237
|
function throttleRetryInPlaceMaxMs(env = process.env) {
|
|
@@ -65207,6 +65405,132 @@ function createThrottleTierRunner(deps) {
|
|
|
65207
65405
|
};
|
|
65208
65406
|
}
|
|
65209
65407
|
|
|
65408
|
+
// litellm-local-notice.ts
|
|
65409
|
+
init_card_format();
|
|
65410
|
+
var LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT = 15 * 60000;
|
|
65411
|
+
function parseLitellmNoticeWindowMs(raw) {
|
|
65412
|
+
if (typeof raw !== "number")
|
|
65413
|
+
return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT;
|
|
65414
|
+
if (!Number.isFinite(raw) || raw <= 0)
|
|
65415
|
+
return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT;
|
|
65416
|
+
return raw;
|
|
65417
|
+
}
|
|
65418
|
+
|
|
65419
|
+
// litellm-local-notice.ts
|
|
65420
|
+
init_card_format();
|
|
65421
|
+
var LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT2 = 15 * 60000;
|
|
65422
|
+
function initialLitellmLocalNoticeState() {
|
|
65423
|
+
return { lastSentAtMsByAgent: {}, suppressedCountByAgent: {} };
|
|
65424
|
+
}
|
|
65425
|
+
function evaluateLitellmLocalNotice(prev, agent, now, windowMs = LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT2) {
|
|
65426
|
+
const last = prev.lastSentAtMsByAgent[agent];
|
|
65427
|
+
if (last == null || now - last >= windowMs) {
|
|
65428
|
+
return {
|
|
65429
|
+
send: true,
|
|
65430
|
+
suppressedSinceLastNotice: prev.suppressedCountByAgent[agent] ?? 0,
|
|
65431
|
+
next: {
|
|
65432
|
+
lastSentAtMsByAgent: { ...prev.lastSentAtMsByAgent, [agent]: now },
|
|
65433
|
+
suppressedCountByAgent: { ...prev.suppressedCountByAgent, [agent]: 0 }
|
|
65434
|
+
}
|
|
65435
|
+
};
|
|
65436
|
+
}
|
|
65437
|
+
return {
|
|
65438
|
+
send: false,
|
|
65439
|
+
suppressedSinceLastNotice: 0,
|
|
65440
|
+
next: {
|
|
65441
|
+
lastSentAtMsByAgent: prev.lastSentAtMsByAgent,
|
|
65442
|
+
suppressedCountByAgent: {
|
|
65443
|
+
...prev.suppressedCountByAgent,
|
|
65444
|
+
[agent]: (prev.suppressedCountByAgent[agent] ?? 0) + 1
|
|
65445
|
+
}
|
|
65446
|
+
}
|
|
65447
|
+
};
|
|
65448
|
+
}
|
|
65449
|
+
function renderLitellmLocalNotice(opts) {
|
|
65450
|
+
const agent = escapeMarkdown(opts.agent);
|
|
65451
|
+
const n = opts.suppressedSinceLastNotice;
|
|
65452
|
+
const lines = [
|
|
65453
|
+
`\uD83D\uDEA6 **Fleet token limiter engaged** \u2014 **${agent}** hit the local LiteLLM proxy cap (\`tpm_limit\`/\`rpm_limit\`).`,
|
|
65454
|
+
`This is switchroom's own fleet limiter smoothing a burst, not an Anthropic account limit \u2014 nothing is exhausted and no account was touched.`
|
|
65455
|
+
];
|
|
65456
|
+
if (n > 0) {
|
|
65457
|
+
lines.push(`Throttled ${n} more ${n === 1 ? "time" : "times"} since the last notice.`);
|
|
65458
|
+
}
|
|
65459
|
+
lines.push(`_The turn retries automatically \u2014 no action needed._`);
|
|
65460
|
+
return lines.join(`
|
|
65461
|
+
`);
|
|
65462
|
+
}
|
|
65463
|
+
function buildLitellmLocalNoticeMetric(opts) {
|
|
65464
|
+
return {
|
|
65465
|
+
kind: "litellm_local_429_notice",
|
|
65466
|
+
agent: opts.agent,
|
|
65467
|
+
suppressed_count: opts.suppressedCount,
|
|
65468
|
+
window_ms: opts.windowMs
|
|
65469
|
+
};
|
|
65470
|
+
}
|
|
65471
|
+
function isLitellmLocalNoticeEligible(classification) {
|
|
65472
|
+
return classification === "litellm-local";
|
|
65473
|
+
}
|
|
65474
|
+
|
|
65475
|
+
// gateway/litellm-local-notice-wiring.ts
|
|
65476
|
+
function decideRateLimitedSurface(opts) {
|
|
65477
|
+
if (isLitellmLocalNoticeEligible(opts.classification))
|
|
65478
|
+
return "litellm-local-notice";
|
|
65479
|
+
return opts.shouldEmitCard(opts.agent) ? "generic-card" : "cooldown-suppressed";
|
|
65480
|
+
}
|
|
65481
|
+
function createLitellmLocalNoticeRunner(deps) {
|
|
65482
|
+
const now = deps.now ?? (() => Date.now());
|
|
65483
|
+
let state3 = initialLitellmLocalNoticeState();
|
|
65484
|
+
function onRateLimited(classification, agent) {
|
|
65485
|
+
try {
|
|
65486
|
+
if (!isLitellmLocalNoticeEligible(classification))
|
|
65487
|
+
return "skipped";
|
|
65488
|
+
const windowMs = deps.windowMs();
|
|
65489
|
+
const verdict = evaluateLitellmLocalNotice(state3, agent, now(), windowMs);
|
|
65490
|
+
if (!verdict.send) {
|
|
65491
|
+
state3 = verdict.next;
|
|
65492
|
+
deps.log(`[litellm-local-notice] suppressed (cooldown) agent=${agent} ` + `suppressedSoFar=${state3.suppressedCountByAgent[agent] ?? 0}`);
|
|
65493
|
+
return "suppressed";
|
|
65494
|
+
}
|
|
65495
|
+
const chats = deps.listNoticeChats();
|
|
65496
|
+
const markdown = renderLitellmLocalNotice({
|
|
65497
|
+
agent,
|
|
65498
|
+
suppressedSinceLastNotice: verdict.suppressedSinceLastNotice
|
|
65499
|
+
});
|
|
65500
|
+
let issued = 0;
|
|
65501
|
+
for (const chatId of chats) {
|
|
65502
|
+
try {
|
|
65503
|
+
deps.sendNotice(chatId, markdown);
|
|
65504
|
+
issued++;
|
|
65505
|
+
} catch (err) {
|
|
65506
|
+
deps.log(`[litellm-local-notice] send failed chat=${chatId} agent=${agent}: ` + `${err?.message ?? err}`);
|
|
65507
|
+
}
|
|
65508
|
+
}
|
|
65509
|
+
if (issued === 0) {
|
|
65510
|
+
deps.log(`[litellm-local-notice] no sends issued (chats=${chats.length}) agent=${agent} \u2014 ` + `window not armed, will retry on the next event`);
|
|
65511
|
+
return "skipped";
|
|
65512
|
+
}
|
|
65513
|
+
state3 = verdict.next;
|
|
65514
|
+
deps.emitMetric(buildLitellmLocalNoticeMetric({
|
|
65515
|
+
agent,
|
|
65516
|
+
suppressedCount: verdict.suppressedSinceLastNotice,
|
|
65517
|
+
windowMs
|
|
65518
|
+
}));
|
|
65519
|
+
deps.log(`[litellm-local-notice] posted agent=${agent} chats=${issued} ` + `suppressedSinceLast=${verdict.suppressedSinceLastNotice} windowMs=${windowMs}`);
|
|
65520
|
+
return "sent";
|
|
65521
|
+
} catch (err) {
|
|
65522
|
+
try {
|
|
65523
|
+
deps.log(`[litellm-local-notice] error agent=${agent}: ${err?.message ?? err}`);
|
|
65524
|
+
} catch {}
|
|
65525
|
+
return "skipped";
|
|
65526
|
+
}
|
|
65527
|
+
}
|
|
65528
|
+
return {
|
|
65529
|
+
onRateLimited,
|
|
65530
|
+
inspect: () => ({ state: state3 })
|
|
65531
|
+
};
|
|
65532
|
+
}
|
|
65533
|
+
|
|
65210
65534
|
// auto-fallback-fleet.ts
|
|
65211
65535
|
init_card_format();
|
|
65212
65536
|
init_auth_snapshot_format();
|
|
@@ -68516,7 +68840,22 @@ function parseModelCommand(text4) {
|
|
|
68516
68840
|
}
|
|
68517
68841
|
return { kind: "set", model: arg };
|
|
68518
68842
|
}
|
|
68519
|
-
|
|
68843
|
+
function isModelCommandBusy(ctx) {
|
|
68844
|
+
return ctx.currentTurnActive || ctx.turnInFlight;
|
|
68845
|
+
}
|
|
68846
|
+
function planModelCommand(parsed, ctx) {
|
|
68847
|
+
if (parsed.kind === "show" && ctx.menuEnabled)
|
|
68848
|
+
return { kind: "menu" };
|
|
68849
|
+
if (parsed.kind === "set" && isModelCommandBusy(ctx)) {
|
|
68850
|
+
return { kind: "queue", target: expandSrAlias(parsed.model) };
|
|
68851
|
+
}
|
|
68852
|
+
return { kind: "apply", parsed };
|
|
68853
|
+
}
|
|
68854
|
+
function modelCommandReceiptLine(agent, parsed, busy) {
|
|
68855
|
+
const arg = parsed.kind === "set" ? parsed.model : parsed.kind === "show" ? "(show)" : "(help)";
|
|
68856
|
+
return `telegram gateway: gw /model received agent=${agent} kind=${parsed.kind} arg=${arg} busy=${busy}`;
|
|
68857
|
+
}
|
|
68858
|
+
var PERSIST_NOTE = "_Session-only \u2014 this override lasts until the agent\u2019s next restart, then reverts to the configured `model:`. `/model default` clears it now. To change the default permanently, set `model:` in switchroom.yaml._";
|
|
68520
68859
|
function helpText2(deps, reason) {
|
|
68521
68860
|
const srAliasExamples = Object.keys(SR_MODEL_ALIASES).map((a) => `\`${a}\``).join(" \u00b7 ");
|
|
68522
68861
|
const lines = [];
|
|
@@ -69070,12 +69409,7 @@ function isValidModelArg2(arg) {
|
|
|
69070
69409
|
|
|
69071
69410
|
// gateway/session-model-file.ts
|
|
69072
69411
|
var SESSION_MODEL_FILE = ".session-model";
|
|
69073
|
-
var RELAUNCH_MODEL_INTENT_FILE = ".relaunch-model-intent";
|
|
69074
69412
|
var CONFIGURED_DEFAULT_MODEL_FILE = ".configured-default-model";
|
|
69075
|
-
var SESSION_MODEL_BOOT_ATTEMPTS_FILE = ".session-model-boot-attempts";
|
|
69076
|
-
function intentForRestartReason(_reason) {
|
|
69077
|
-
return "keep";
|
|
69078
|
-
}
|
|
69079
69413
|
function atomicWrite(path2, content3) {
|
|
69080
69414
|
const tmp = `${path2}.tmp-${process.pid}-${Date.now()}`;
|
|
69081
69415
|
writeFileSync21(tmp, content3, "utf8");
|
|
@@ -69089,17 +69423,6 @@ function serializeSessionModel(rec) {
|
|
|
69089
69423
|
})}
|
|
69090
69424
|
`;
|
|
69091
69425
|
}
|
|
69092
|
-
function parseSessionModel(text4) {
|
|
69093
|
-
try {
|
|
69094
|
-
const raw = JSON.parse(text4);
|
|
69095
|
-
if (typeof raw.model !== "string" || typeof raw.configuredDefaultAtWrite !== "string" || typeof raw.ts !== "number" || !isValidModelArg2(raw.model)) {
|
|
69096
|
-
return null;
|
|
69097
|
-
}
|
|
69098
|
-
return { model: raw.model, configuredDefaultAtWrite: raw.configuredDefaultAtWrite, ts: raw.ts };
|
|
69099
|
-
} catch {
|
|
69100
|
-
return null;
|
|
69101
|
-
}
|
|
69102
|
-
}
|
|
69103
69426
|
function writeSessionModelFile(agentDir, model, configuredDefaultAtWrite) {
|
|
69104
69427
|
if (!isValidModelArg2(model)) {
|
|
69105
69428
|
throw new Error(`refusing to persist non-canonical session model token: ${JSON.stringify(model)}`);
|
|
@@ -69113,20 +69436,9 @@ function readSessionModelFileRaw(agentDir) {
|
|
|
69113
69436
|
return null;
|
|
69114
69437
|
}
|
|
69115
69438
|
}
|
|
69116
|
-
function readSessionModelFile(agentDir) {
|
|
69117
|
-
const raw = readSessionModelFileRaw(agentDir);
|
|
69118
|
-
return raw == null ? null : parseSessionModel(raw);
|
|
69119
|
-
}
|
|
69120
69439
|
function clearSessionModelFile(agentDir) {
|
|
69121
69440
|
try {
|
|
69122
69441
|
rmSync4(join29(agentDir, SESSION_MODEL_FILE), { force: true });
|
|
69123
|
-
rmSync4(join29(agentDir, ".session-model-kept-notified"), { force: true });
|
|
69124
|
-
rmSync4(join29(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true });
|
|
69125
|
-
} catch {}
|
|
69126
|
-
}
|
|
69127
|
-
function clearSessionModelBootAttempts(agentDir) {
|
|
69128
|
-
try {
|
|
69129
|
-
rmSync4(join29(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true });
|
|
69130
69442
|
} catch {}
|
|
69131
69443
|
}
|
|
69132
69444
|
function restoreSessionModelFileRaw(agentDir, raw) {
|
|
@@ -69138,40 +69450,6 @@ function restoreSessionModelFileRaw(agentDir, raw) {
|
|
|
69138
69450
|
atomicWrite(join29(agentDir, SESSION_MODEL_FILE), raw);
|
|
69139
69451
|
} catch {}
|
|
69140
69452
|
}
|
|
69141
|
-
function writeRelaunchModelIntent(agentDir, intent, reason) {
|
|
69142
|
-
try {
|
|
69143
|
-
atomicWrite(join29(agentDir, RELAUNCH_MODEL_INTENT_FILE), `${JSON.stringify({ intent, reason, ts: Date.now() })}
|
|
69144
|
-
`);
|
|
69145
|
-
} catch (err) {
|
|
69146
|
-
process.stderr.write(`telegram gateway: relaunch-model-intent write failed (boot will revert): ${err?.message ?? String(err)}
|
|
69147
|
-
`);
|
|
69148
|
-
}
|
|
69149
|
-
}
|
|
69150
|
-
var GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX = "gateway-shutdown:";
|
|
69151
|
-
function readRelaunchModelIntent(agentDir) {
|
|
69152
|
-
try {
|
|
69153
|
-
const raw = readFileSync25(join29(agentDir, RELAUNCH_MODEL_INTENT_FILE), "utf8");
|
|
69154
|
-
const parsed = JSON.parse(raw);
|
|
69155
|
-
if (parsed.intent !== "keep" && parsed.intent !== "revert" || typeof parsed.reason !== "string" || typeof parsed.ts !== "number") {
|
|
69156
|
-
return null;
|
|
69157
|
-
}
|
|
69158
|
-
return { intent: parsed.intent, reason: parsed.reason, ts: parsed.ts };
|
|
69159
|
-
} catch {
|
|
69160
|
-
return null;
|
|
69161
|
-
}
|
|
69162
|
-
}
|
|
69163
|
-
function clearStaleGatewayShutdownIntent(agentDir) {
|
|
69164
|
-
const rec = readRelaunchModelIntent(agentDir);
|
|
69165
|
-
if (rec == null || !rec.reason.startsWith(GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX))
|
|
69166
|
-
return false;
|
|
69167
|
-
clearRelaunchModelIntent(agentDir);
|
|
69168
|
-
return true;
|
|
69169
|
-
}
|
|
69170
|
-
function clearRelaunchModelIntent(agentDir) {
|
|
69171
|
-
try {
|
|
69172
|
-
rmSync4(join29(agentDir, RELAUNCH_MODEL_INTENT_FILE), { force: true });
|
|
69173
|
-
} catch {}
|
|
69174
|
-
}
|
|
69175
69453
|
function readConfiguredDefaultModel(agentDir) {
|
|
69176
69454
|
try {
|
|
69177
69455
|
const v = readFileSync25(join29(agentDir, CONFIGURED_DEFAULT_MODEL_FILE), "utf8").trim();
|
|
@@ -69182,17 +69460,6 @@ function readConfiguredDefaultModel(agentDir) {
|
|
|
69182
69460
|
}
|
|
69183
69461
|
var SESSION_EFFORT_FILE = ".session-effort";
|
|
69184
69462
|
var EFFORT_LEVEL_RE = /^(low|medium|high|xhigh|max)$/;
|
|
69185
|
-
function parseSessionEffort(text4) {
|
|
69186
|
-
try {
|
|
69187
|
-
const raw = JSON.parse(text4);
|
|
69188
|
-
if (typeof raw.level !== "string" || typeof raw.configuredDefaultAtWrite !== "string" || typeof raw.ts !== "number" || !EFFORT_LEVEL_RE.test(raw.level)) {
|
|
69189
|
-
return null;
|
|
69190
|
-
}
|
|
69191
|
-
return { level: raw.level, configuredDefaultAtWrite: raw.configuredDefaultAtWrite, ts: raw.ts };
|
|
69192
|
-
} catch {
|
|
69193
|
-
return null;
|
|
69194
|
-
}
|
|
69195
|
-
}
|
|
69196
69463
|
function writeSessionEffortFile(agentDir, level, configuredDefaultAtWrite) {
|
|
69197
69464
|
if (!EFFORT_LEVEL_RE.test(level)) {
|
|
69198
69465
|
throw new Error(`refusing to persist non-allowlisted effort level: ${JSON.stringify(level)}`);
|
|
@@ -69200,13 +69467,6 @@ function writeSessionEffortFile(agentDir, level, configuredDefaultAtWrite) {
|
|
|
69200
69467
|
atomicWrite(join29(agentDir, SESSION_EFFORT_FILE), `${JSON.stringify({ level, configuredDefaultAtWrite: configuredDefaultAtWrite ?? "", ts: Date.now() })}
|
|
69201
69468
|
`);
|
|
69202
69469
|
}
|
|
69203
|
-
function readSessionEffortFile(agentDir) {
|
|
69204
|
-
try {
|
|
69205
|
-
return parseSessionEffort(readFileSync25(join29(agentDir, SESSION_EFFORT_FILE), "utf8"));
|
|
69206
|
-
} catch {
|
|
69207
|
-
return null;
|
|
69208
|
-
}
|
|
69209
|
-
}
|
|
69210
69470
|
function clearSessionEffortFile(agentDir) {
|
|
69211
69471
|
try {
|
|
69212
69472
|
rmSync4(join29(agentDir, SESSION_EFFORT_FILE), { force: true });
|
|
@@ -69426,6 +69686,11 @@ function extractConfirmation(pane) {
|
|
|
69426
69686
|
// ../src/agents/scaffold.ts
|
|
69427
69687
|
import { join as join32, resolve as resolve6 } from "node:path";
|
|
69428
69688
|
init_atomic();
|
|
69689
|
+
|
|
69690
|
+
// ../src/agents/agent-uid.ts
|
|
69691
|
+
init_peercred();
|
|
69692
|
+
|
|
69693
|
+
// ../src/agents/scaffold.ts
|
|
69429
69694
|
init_schema();
|
|
69430
69695
|
|
|
69431
69696
|
// ../src/config/users.ts
|
|
@@ -69576,7 +69841,7 @@ function parseEffortCommand(text4) {
|
|
|
69576
69841
|
}
|
|
69577
69842
|
return { kind: "set", level: arg.toLowerCase() };
|
|
69578
69843
|
}
|
|
69579
|
-
var PERSIST_NOTE2 = "
|
|
69844
|
+
var PERSIST_NOTE2 = "_Session-only \u2014 this override lasts until the agent\u2019s next restart, then reverts to the configured `thinking_effort:`. `/effort default` clears it now. To change the default permanently, set `thinking_effort:` in switchroom.yaml._";
|
|
69580
69845
|
var LEVELS_INLINE = EFFORT_LEVELS.map((l) => `\`${l}\``).join(" \u00b7 ");
|
|
69581
69846
|
function helpText3(deps, reason) {
|
|
69582
69847
|
const lines = [];
|
|
@@ -77097,6 +77362,7 @@ function detectErrorInTranscriptLine(line) {
|
|
|
77097
77362
|
const errStr = typeof obj.error === "string" ? obj.error : "";
|
|
77098
77363
|
const text4 = extractAssistantText(obj);
|
|
77099
77364
|
const kind2 = status === 429 ? isTransientUpstreamSignal(`${text4}
|
|
77365
|
+
${errStr}`) || isLitellmProxyLocal4292(`${text4}
|
|
77100
77366
|
${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text4 });
|
|
77101
77367
|
return {
|
|
77102
77368
|
kind: kind2,
|
|
@@ -80620,7 +80886,8 @@ function buildFleetRollMessage(roll, now) {
|
|
|
80620
80886
|
const pctPart = typeof roll.pct === "number" ? ` at ${fmtPct(roll.pct)}` : "";
|
|
80621
80887
|
const resetPart = typeof roll.exhausted_until === "number" && roll.exhausted_until > now ? ` (resets ${formatRelative(new Date(roll.exhausted_until), new Date(now))})` : "";
|
|
80622
80888
|
const softAvoid = roll.reason === "soft-avoid";
|
|
80623
|
-
const
|
|
80889
|
+
const modelTierWall = roll.reason === "model-tier-wall";
|
|
80890
|
+
const causeLine = softAvoid ? `Proactive switch \u2014 \`${codeSpanSafe(roll.from)}\` is approaching its limits (${winLabel}${pctPart})${resetPart}, so the fleet moved early instead of hitting the wall.` : modelTierWall ? `Flagship (premium) tier weekly limit reached on \`${codeSpanSafe(roll.from)}\`${resetPart} \u2014 opus/haiku on that account are unaffected.` : `${winLabel}${pctPart} on \`${codeSpanSafe(roll.from)}\`${resetPart}.`;
|
|
80624
80891
|
return [
|
|
80625
80892
|
`\uD83D\uDD01 **Switched fleet to \`${codeSpanSafe(roll.to)}\`**`,
|
|
80626
80893
|
``,
|
|
@@ -80957,10 +81224,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
|
|
|
80957
81224
|
}
|
|
80958
81225
|
|
|
80959
81226
|
// ../src/build-info.ts
|
|
80960
|
-
var VERSION = "0.18.
|
|
80961
|
-
var COMMIT_SHA = "
|
|
80962
|
-
var COMMIT_DATE = "2026-07-
|
|
80963
|
-
var LATEST_PR =
|
|
81227
|
+
var VERSION = "0.18.17";
|
|
81228
|
+
var COMMIT_SHA = "372a4968";
|
|
81229
|
+
var COMMIT_DATE = "2026-07-12T09:18:05Z";
|
|
81230
|
+
var LATEST_PR = 3190;
|
|
80964
81231
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
80965
81232
|
|
|
80966
81233
|
// gateway/boot-version.ts
|
|
@@ -82845,11 +83112,6 @@ function triggerSelfRestart(targetAgent, reason, delayMs = 300) {
|
|
|
82845
83112
|
`);
|
|
82846
83113
|
return false;
|
|
82847
83114
|
}
|
|
82848
|
-
{
|
|
82849
|
-
const smDir = resolveAgentDirFromEnv();
|
|
82850
|
-
if (smDir)
|
|
82851
|
-
writeRelaunchModelIntent(smDir, intentForRestartReason(reason), reason);
|
|
82852
|
-
}
|
|
82853
83115
|
process.stderr.write(`telegram gateway: restart-via-SIGTERM-PID1 agent=${targetAgent} reason=${reason} (docker)
|
|
82854
83116
|
`);
|
|
82855
83117
|
setTimeout(() => {
|
|
@@ -82862,11 +83124,6 @@ function triggerSelfRestart(targetAgent, reason, delayMs = 300) {
|
|
|
82862
83124
|
}, delayMs).unref();
|
|
82863
83125
|
return true;
|
|
82864
83126
|
}
|
|
82865
|
-
if (targetAgent === selfAgent) {
|
|
82866
|
-
const smDir = resolveAgentDirFromEnv();
|
|
82867
|
-
if (smDir)
|
|
82868
|
-
writeRelaunchModelIntent(smDir, intentForRestartReason(reason), reason);
|
|
82869
|
-
}
|
|
82870
83127
|
process.stderr.write(`telegram gateway: restart-via-systemctl agent=${targetAgent} reason=${reason}
|
|
82871
83128
|
`);
|
|
82872
83129
|
try {
|
|
@@ -82881,13 +83138,6 @@ function triggerSelfRestart(targetAgent, reason, delayMs = 300) {
|
|
|
82881
83138
|
return false;
|
|
82882
83139
|
}
|
|
82883
83140
|
}
|
|
82884
|
-
{
|
|
82885
|
-
const bootSmDir = resolveAgentDirFromEnv();
|
|
82886
|
-
if (bootSmDir != null && clearStaleGatewayShutdownIntent(bootSmDir)) {
|
|
82887
|
-
process.stderr.write(`telegram gateway: cleared stale gateway-shutdown relaunch-model intent (previous bounce was gateway-only \u2014 container never restarted)
|
|
82888
|
-
`);
|
|
82889
|
-
}
|
|
82890
|
-
}
|
|
82891
83141
|
var cachedClaudeCliVersion = undefined;
|
|
82892
83142
|
function getClaudeCliVersion() {
|
|
82893
83143
|
if (cachedClaudeCliVersion !== undefined)
|
|
@@ -83091,6 +83341,7 @@ function readAccessFile() {
|
|
|
83091
83341
|
parseMode: parsed.parseMode,
|
|
83092
83342
|
disableLinkPreview: parsed.disableLinkPreview,
|
|
83093
83343
|
coalescingGapMs: parsed.coalescingGapMs,
|
|
83344
|
+
litellmNoticeWindowMs: parsed.litellmNoticeWindowMs,
|
|
83094
83345
|
coalesceMaxAttachments: parsed.coalesceMaxAttachments,
|
|
83095
83346
|
interruptSafeBoundary: parsed.interruptSafeBoundary,
|
|
83096
83347
|
interruptMaxWaitMs: parsed.interruptMaxWaitMs,
|
|
@@ -85647,12 +85898,52 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85647
85898
|
const { agent, kind } = event;
|
|
85648
85899
|
let throttleEscalation = null;
|
|
85649
85900
|
let escalationFired = false;
|
|
85650
|
-
|
|
85901
|
+
let rateLimitedCooldownConsulted = false;
|
|
85902
|
+
const rateLimit429Classification = kind === "rate-limited" ? classify429Detail(event.detail) : null;
|
|
85903
|
+
if (rateLimit429Classification != null && rateLimit429Classification !== "account-scoped") {
|
|
85904
|
+
emitRuntimeMetric(build429ClassifiedMetric({
|
|
85905
|
+
agent,
|
|
85906
|
+
detail: event.detail,
|
|
85907
|
+
classification: rateLimit429Classification,
|
|
85908
|
+
action: "calm",
|
|
85909
|
+
now: Date.now()
|
|
85910
|
+
}));
|
|
85911
|
+
const surface = decideRateLimitedSurface({
|
|
85912
|
+
classification: rateLimit429Classification,
|
|
85913
|
+
agent,
|
|
85914
|
+
shouldEmitCard: (a) => shouldEmitOperatorEvent(a, "rate-limited")
|
|
85915
|
+
});
|
|
85916
|
+
if (surface === "litellm-local-notice") {
|
|
85917
|
+
process.stderr.write(`telegram gateway: 429 classified litellm-proxy-local agent=${agent} \u2014 ` + `calm path, no account attribution, no failover
|
|
85918
|
+
`);
|
|
85919
|
+
const outcome = litellmLocalNoticeRunner.onRateLimited("litellm-local", agent);
|
|
85920
|
+
if (outcome === "sent") {
|
|
85921
|
+
try {
|
|
85922
|
+
recordOperatorEvent(event);
|
|
85923
|
+
} catch {}
|
|
85924
|
+
}
|
|
85925
|
+
return;
|
|
85926
|
+
}
|
|
85927
|
+
if (surface === "cooldown-suppressed") {
|
|
85928
|
+
process.stderr.write(`telegram gateway: operator-event suppressed (cooldown) agent=${agent} kind=${kind}
|
|
85929
|
+
`);
|
|
85930
|
+
return;
|
|
85931
|
+
}
|
|
85932
|
+
rateLimitedCooldownConsulted = true;
|
|
85933
|
+
}
|
|
85934
|
+
if (rateLimit429Classification === "account-scoped") {
|
|
85651
85935
|
const throttleDecision = decideThrottleTier({
|
|
85652
85936
|
detail: event.detail,
|
|
85653
85937
|
now: Date.now(),
|
|
85654
85938
|
thresholdMs: throttleRetryInPlaceMaxMs()
|
|
85655
85939
|
});
|
|
85940
|
+
emitRuntimeMetric(build429ClassifiedMetric({
|
|
85941
|
+
agent,
|
|
85942
|
+
detail: event.detail,
|
|
85943
|
+
classification: "account-scoped",
|
|
85944
|
+
action: throttleDecision.action === "failover" ? "failover" : "throttle",
|
|
85945
|
+
now: Date.now()
|
|
85946
|
+
}));
|
|
85656
85947
|
if (throttleDecision.action === "throttle") {
|
|
85657
85948
|
process.stderr.write(`telegram gateway: throttle-tier staying-put agent=${agent} until=${new Date(throttleDecision.throttledUntilMs).toISOString()} parsedReset=${throttleDecision.resetParsed}
|
|
85658
85949
|
`);
|
|
@@ -85671,7 +85962,7 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85671
85962
|
}
|
|
85672
85963
|
}
|
|
85673
85964
|
}
|
|
85674
|
-
if (!shouldEmitOperatorEvent(agent, kind)) {
|
|
85965
|
+
if (!rateLimitedCooldownConsulted && !shouldEmitOperatorEvent(agent, kind)) {
|
|
85675
85966
|
process.stderr.write(`telegram gateway: operator-event suppressed (cooldown) agent=${agent} kind=${kind}
|
|
85676
85967
|
`);
|
|
85677
85968
|
return;
|
|
@@ -86769,11 +87060,6 @@ var ipcServer = createIpcServer({
|
|
|
86769
87060
|
}
|
|
86770
87061
|
const bridgeUpEffects = client3.agentName != null ? shadowEmit({ kind: "bridgeUp", at: Date.now() }) : [];
|
|
86771
87062
|
bridgeDeadWatchdog.noteBridgeRegistered(client3.agentName);
|
|
86772
|
-
if (client3.agentName != null) {
|
|
86773
|
-
const smBootDir = resolveAgentDirFromEnv();
|
|
86774
|
-
if (smBootDir != null)
|
|
86775
|
-
clearSessionModelBootAttempts(smBootDir);
|
|
86776
|
-
}
|
|
86777
87063
|
client3.send({ type: "status", status: "agent_connected" });
|
|
86778
87064
|
if (client3.agentName != null) {
|
|
86779
87065
|
if (isDispatchEnabled()) {
|
|
@@ -92842,11 +93128,6 @@ function buildModelDeps(restartCtx) {
|
|
|
92842
93128
|
});
|
|
92843
93129
|
}
|
|
92844
93130
|
stampUserRestartReason(reason);
|
|
92845
|
-
{
|
|
92846
|
-
const smDir = resolveAgentDirFromEnv();
|
|
92847
|
-
if (smDir)
|
|
92848
|
-
writeRelaunchModelIntent(smDir, "keep", reason);
|
|
92849
|
-
}
|
|
92850
93131
|
await sweepBeforeSelfRestart();
|
|
92851
93132
|
const hostdResp = await tryHostdDispatch2(name, {
|
|
92852
93133
|
v: 1,
|
|
@@ -92861,11 +93142,6 @@ function buildModelDeps(restartCtx) {
|
|
|
92861
93142
|
}
|
|
92862
93143
|
if (hostdResp.result !== "started" && hostdResp.result !== "completed") {
|
|
92863
93144
|
clearRestartMarker();
|
|
92864
|
-
{
|
|
92865
|
-
const smDir = resolveAgentDirFromEnv();
|
|
92866
|
-
if (smDir)
|
|
92867
|
-
clearRelaunchModelIntent(smDir);
|
|
92868
|
-
}
|
|
92869
93145
|
throw new Error(`hostd restart failed (result=${hostdResp.result}): ${hostdResp.error ?? "(no details)"}`);
|
|
92870
93146
|
}
|
|
92871
93147
|
},
|
|
@@ -92883,7 +93159,6 @@ function buildModelDeps(restartCtx) {
|
|
|
92883
93159
|
if (err?.code !== "restart_in_flight") {
|
|
92884
93160
|
restoreSessionModelFileRaw(agentDir, prevFileRaw);
|
|
92885
93161
|
sessionModelSource.setOverride(prevOverride);
|
|
92886
|
-
clearRelaunchModelIntent(agentDir);
|
|
92887
93162
|
}
|
|
92888
93163
|
throw err;
|
|
92889
93164
|
}
|
|
@@ -92902,12 +93177,12 @@ function modelMenuReplyMarkup(reply) {
|
|
|
92902
93177
|
}
|
|
92903
93178
|
return kb;
|
|
92904
93179
|
}
|
|
92905
|
-
function recordTypedModelSwitch(reply, requestedModelArg,
|
|
93180
|
+
function recordTypedModelSwitch(reply, requestedModelArg, _deps) {
|
|
92906
93181
|
const requested = requestedModelArg != null ? expandSrAlias(requestedModelArg) : null;
|
|
92907
93182
|
if (requested?.toLowerCase() === "default") {
|
|
92908
|
-
const
|
|
92909
|
-
if (
|
|
92910
|
-
clearSessionModelFile(
|
|
93183
|
+
const smDir = resolveAgentDirFromEnv();
|
|
93184
|
+
if (smDir)
|
|
93185
|
+
clearSessionModelFile(smDir);
|
|
92911
93186
|
if (reply.selectedModel)
|
|
92912
93187
|
sessionModelSource.setOverride(null);
|
|
92913
93188
|
return "";
|
|
@@ -92915,33 +93190,11 @@ function recordTypedModelSwitch(reply, requestedModelArg, deps) {
|
|
|
92915
93190
|
if (!reply.selectedModel)
|
|
92916
93191
|
return "";
|
|
92917
93192
|
sessionModelSource.setOverride(reply.selectedModel);
|
|
92918
|
-
const smDir = resolveAgentDirFromEnv();
|
|
92919
|
-
if (smDir && requested && isValidModelArg(requested) && !isSrModel(requested)) {
|
|
92920
|
-
try {
|
|
92921
|
-
writeSessionModelFile(smDir, requested, readConfiguredDefaultModel(smDir) ?? resolveMainModel(deps.getConfiguredModel() ?? undefined));
|
|
92922
|
-
} catch (err) {
|
|
92923
|
-
process.stderr.write(`telegram gateway: session-model persist failed (typed /model): ${err?.message ?? String(err)}
|
|
92924
|
-
`);
|
|
92925
|
-
return `
|
|
92926
|
-
\u26A0\uFE0F Couldn\u2019t persist the sticky override \u2014 the switch is live now but won\u2019t survive a relaunch.`;
|
|
92927
|
-
}
|
|
92928
|
-
}
|
|
92929
93193
|
return "";
|
|
92930
93194
|
}
|
|
92931
93195
|
function recordModelMenuSideEffects(outcome, modelDeps, cbChatId, cbThreadId, prevSessionModel) {
|
|
92932
93196
|
if (outcome.selectedModel) {
|
|
92933
93197
|
sessionModelSource.setOverride(outcome.selectedModel);
|
|
92934
|
-
const smDir = resolveAgentDirFromEnv();
|
|
92935
|
-
if (smDir && outcome.selectedModelToken) {
|
|
92936
|
-
try {
|
|
92937
|
-
writeSessionModelFile(smDir, outcome.selectedModelToken, readConfiguredDefaultModel(smDir) ?? resolveMainModel(modelDeps.getConfiguredModel() ?? undefined));
|
|
92938
|
-
} catch (err) {
|
|
92939
|
-
outcome.reply.text += `
|
|
92940
|
-
\u26A0\uFE0F Couldn\u2019t persist the sticky override \u2014 the switch is live now but won\u2019t survive a relaunch.`;
|
|
92941
|
-
process.stderr.write(`telegram gateway: session-model persist failed (menu): ${err?.message ?? String(err)}
|
|
92942
|
-
`);
|
|
92943
|
-
}
|
|
92944
|
-
}
|
|
92945
93198
|
}
|
|
92946
93199
|
if (outcome.clearedDefault) {
|
|
92947
93200
|
const smDir = resolveAgentDirFromEnv();
|
|
@@ -93038,7 +93291,6 @@ function persistQueuedCommandForRestart(action) {
|
|
|
93038
93291
|
}
|
|
93039
93292
|
const configured = readConfiguredDefaultModel(agentDir) ?? resolveMainModel(undefined);
|
|
93040
93293
|
writeSessionModelFile(agentDir, expandSrAlias(action.arg), configured);
|
|
93041
|
-
writeRelaunchModelIntent(agentDir, "keep", "queued /model carried across restart");
|
|
93042
93294
|
break;
|
|
93043
93295
|
}
|
|
93044
93296
|
case "clear-model":
|
|
@@ -93095,14 +93347,38 @@ bot.command("model", async (ctx) => {
|
|
|
93095
93347
|
const parsed = parseModelCommand(text5) ?? { kind: "show" };
|
|
93096
93348
|
const chatId = String(ctx.chat.id);
|
|
93097
93349
|
const threadId = resolveThreadId(chatId, ctx.message?.message_thread_id);
|
|
93350
|
+
const busyNow = currentTurn !== null || turnInFlightForGate();
|
|
93351
|
+
process.stderr.write(modelCommandReceiptLine(getMyAgentName(), parsed, busyNow) + `
|
|
93352
|
+
`);
|
|
93353
|
+
if (HISTORY_ENABLED && ctx.message?.message_id != null) {
|
|
93354
|
+
try {
|
|
93355
|
+
recordInbound({
|
|
93356
|
+
chat_id: chatId,
|
|
93357
|
+
thread_id: threadId ?? null,
|
|
93358
|
+
message_id: ctx.message.message_id,
|
|
93359
|
+
user: ctx.from?.username ?? (ctx.from?.id != null ? String(ctx.from.id) : null),
|
|
93360
|
+
user_id: ctx.from?.id != null ? String(ctx.from.id) : null,
|
|
93361
|
+
ts: ctx.message.date ?? Math.floor(Date.now() / 1000),
|
|
93362
|
+
text: text5
|
|
93363
|
+
});
|
|
93364
|
+
} catch (err) {
|
|
93365
|
+
process.stderr.write(`telegram gateway: /model recordInbound failed: ${err?.message ?? String(err)}
|
|
93366
|
+
`);
|
|
93367
|
+
}
|
|
93368
|
+
}
|
|
93098
93369
|
const deps = buildModelDeps({ chatId, threadId });
|
|
93099
|
-
|
|
93370
|
+
const disposition = planModelCommand(parsed, {
|
|
93371
|
+
currentTurnActive: currentTurn !== null,
|
|
93372
|
+
turnInFlight: turnInFlightForGate(),
|
|
93373
|
+
menuEnabled: process.env.SWITCHROOM_MODEL_MENU !== "0"
|
|
93374
|
+
});
|
|
93375
|
+
if (disposition.kind === "menu") {
|
|
93100
93376
|
const menu = await buildModelMenu(deps);
|
|
93101
93377
|
await switchroomReply(ctx, menu.text, { html: true, reply_markup: modelMenuReplyMarkup(menu) });
|
|
93102
93378
|
return;
|
|
93103
93379
|
}
|
|
93104
|
-
if (
|
|
93105
|
-
const target =
|
|
93380
|
+
if (disposition.kind === "queue") {
|
|
93381
|
+
const target = disposition.target;
|
|
93106
93382
|
const sent = await ctx.replyWithRichMessage(richMessage2(hardenCardBreaks2(ackText("model", target, escapeHtmlForTg2))), threadId != null ? { message_thread_id: threadId } : {});
|
|
93107
93383
|
enqueueSessionCommand({
|
|
93108
93384
|
kind: "model",
|
|
@@ -93121,34 +93397,24 @@ bot.command("model", async (ctx) => {
|
|
|
93121
93397
|
const persistWarning = recordTypedModelSwitch(reply, parsed.kind === "set" ? parsed.model : null, deps);
|
|
93122
93398
|
await switchroomReply(ctx, reply.text + persistWarning, { html: reply.html });
|
|
93123
93399
|
});
|
|
93400
|
+
var sessionEffortOverride = null;
|
|
93124
93401
|
function buildEffortDeps() {
|
|
93125
93402
|
return {
|
|
93126
93403
|
applyEffort: async (agent, level) => {
|
|
93127
93404
|
const result = await applyEffort(agent, level);
|
|
93128
|
-
if (result.ok)
|
|
93129
|
-
|
|
93130
|
-
if (agentDir) {
|
|
93131
|
-
try {
|
|
93132
|
-
writeSessionEffortFile(agentDir, level, getConfiguredEffortForPersist());
|
|
93133
|
-
} catch (err) {
|
|
93134
|
-
process.stderr.write(`telegram gateway: session-effort persist failed level=${level}: ${err?.message ?? String(err)}
|
|
93135
|
-
`);
|
|
93136
|
-
}
|
|
93137
|
-
}
|
|
93138
|
-
}
|
|
93405
|
+
if (result.ok)
|
|
93406
|
+
sessionEffortOverride = level;
|
|
93139
93407
|
return result;
|
|
93140
93408
|
},
|
|
93141
93409
|
getAgentName: getMyAgentName,
|
|
93142
93410
|
getConfiguredEffort: () => getConfiguredEffortForPersist(),
|
|
93143
93411
|
clearSessionEffort: () => {
|
|
93412
|
+
sessionEffortOverride = null;
|
|
93144
93413
|
const agentDir = resolveAgentDirFromEnv();
|
|
93145
93414
|
if (agentDir)
|
|
93146
93415
|
clearSessionEffortFile(agentDir);
|
|
93147
93416
|
},
|
|
93148
|
-
getSessionEffort: () =>
|
|
93149
|
-
const agentDir = resolveAgentDirFromEnv();
|
|
93150
|
-
return agentDir ? readSessionEffortFile(agentDir)?.level ?? null : null;
|
|
93151
|
-
},
|
|
93417
|
+
getSessionEffort: () => sessionEffortOverride,
|
|
93152
93418
|
escapeHtml: escapeHtmlForTg2
|
|
93153
93419
|
};
|
|
93154
93420
|
}
|
|
@@ -93270,11 +93536,6 @@ bot.command("restart", async (ctx) => {
|
|
|
93270
93536
|
} catch {}
|
|
93271
93537
|
writeRestartMarker({ chat_id: chatId, thread_id: threadId ?? null, ack_message_id: ackId, ts: Date.now() });
|
|
93272
93538
|
stampUserRestartReason("user: /restart from chat");
|
|
93273
|
-
{
|
|
93274
|
-
const smDir = resolveAgentDirFromEnv();
|
|
93275
|
-
if (smDir)
|
|
93276
|
-
writeRelaunchModelIntent(smDir, "keep", "user: /restart from chat");
|
|
93277
|
-
}
|
|
93278
93539
|
await sweepBeforeSelfRestart();
|
|
93279
93540
|
const hostdResp = await tryHostdDispatch2(getMyAgentName(), {
|
|
93280
93541
|
v: 1,
|
|
@@ -93371,9 +93632,6 @@ async function handleNewCommand(ctx) {
|
|
|
93371
93632
|
}
|
|
93372
93633
|
}
|
|
93373
93634
|
stampUserRestartReason(`user: /${kind} from chat`);
|
|
93374
|
-
if (agentDir != null) {
|
|
93375
|
-
writeRelaunchModelIntent(agentDir, "keep", `user: /${kind} from chat`);
|
|
93376
|
-
}
|
|
93377
93635
|
await sweepBeforeSelfRestart();
|
|
93378
93636
|
const hostdResp = await tryHostdDispatch2(getMyAgentName(), {
|
|
93379
93637
|
v: 1,
|
|
@@ -93795,6 +94053,26 @@ var throttleTierRunner = createThrottleTierRunner({
|
|
|
93795
94053
|
log: (m) => process.stderr.write(`telegram gateway: ${m}
|
|
93796
94054
|
`)
|
|
93797
94055
|
});
|
|
94056
|
+
var litellmLocalNoticeRunner = createLitellmLocalNoticeRunner({
|
|
94057
|
+
listNoticeChats: () => loadAccess().allowFrom,
|
|
94058
|
+
sendNotice: (chat_id, markdown) => {
|
|
94059
|
+
const noticeTopic = resolveAgentOutboundTopic({ kind: "compact-watchdog" });
|
|
94060
|
+
const noticeSupergroup = resolveAgentSupergroupChatId();
|
|
94061
|
+
const noticeThread = topicForRecipient({
|
|
94062
|
+
recipientChatId: chat_id,
|
|
94063
|
+
resolvedTopic: noticeTopic,
|
|
94064
|
+
supergroupChatId: noticeSupergroup
|
|
94065
|
+
});
|
|
94066
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(markdown), {
|
|
94067
|
+
disable_notification: true,
|
|
94068
|
+
...noticeThread != null ? { message_thread_id: noticeThread } : {}
|
|
94069
|
+
}), { chat_id: String(chat_id), verb: "litellm-local-notice:notify" });
|
|
94070
|
+
},
|
|
94071
|
+
windowMs: () => parseLitellmNoticeWindowMs(loadAccess().litellmNoticeWindowMs),
|
|
94072
|
+
emitMetric: (event) => emitRuntimeMetric(event),
|
|
94073
|
+
log: (m) => process.stderr.write(`telegram gateway: ${m}
|
|
94074
|
+
`)
|
|
94075
|
+
});
|
|
93798
94076
|
var fallbackFailureNoticeState = { lastSentAtMs: 0 };
|
|
93799
94077
|
var fallbackAllBlockedNoticeState = { lastSentAtMs: 0 };
|
|
93800
94078
|
function broadcastFleetFallbackFailure(triggerAgent, reason) {
|
|
@@ -96494,21 +96772,6 @@ async function shutdown(signal) {
|
|
|
96494
96772
|
`);
|
|
96495
96773
|
} catch (err) {
|
|
96496
96774
|
process.stderr.write(`telegram gateway: shutdown.clean_marker_write_failed err=${err.message}
|
|
96497
|
-
`);
|
|
96498
|
-
}
|
|
96499
|
-
try {
|
|
96500
|
-
const smDir = resolveAgentDirFromEnv();
|
|
96501
|
-
if (smDir != null) {
|
|
96502
|
-
const hasOverride = readSessionModelFile(smDir) != null;
|
|
96503
|
-
const intentAlreadyStamped = existsSync50(join54(smDir, RELAUNCH_MODEL_INTENT_FILE));
|
|
96504
|
-
if (hasOverride && !intentAlreadyStamped) {
|
|
96505
|
-
writeRelaunchModelIntent(smDir, "keep", `${GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX} graceful ${signal} shutdown (deploy/rolling restart) \u2014 preserving user-chosen session model`);
|
|
96506
|
-
process.stderr.write(`telegram gateway: shutdown.session_model_keep_stamped signal=${signal}
|
|
96507
|
-
`);
|
|
96508
|
-
}
|
|
96509
|
-
}
|
|
96510
|
-
} catch (err) {
|
|
96511
|
-
process.stderr.write(`telegram gateway: shutdown.session_model_keep_stamp_failed err=${err.message}
|
|
96512
96775
|
`);
|
|
96513
96776
|
}
|
|
96514
96777
|
} else {
|
|
@@ -96964,6 +97227,14 @@ var didOneTimeSetup = false;
|
|
|
96964
97227
|
sessionModelSource.setOverride(launched.length > 0 && launched !== configured ? launched : null);
|
|
96965
97228
|
} catch {}
|
|
96966
97229
|
}
|
|
97230
|
+
const activeEffortPath = join54(smAgentDir, ".active-session-effort");
|
|
97231
|
+
if (existsSync50(activeEffortPath)) {
|
|
97232
|
+
try {
|
|
97233
|
+
const launchedEffort = readFileSync54(activeEffortPath, "utf8").trim();
|
|
97234
|
+
const configuredEffort = getConfiguredEffortForPersist();
|
|
97235
|
+
sessionEffortOverride = launchedEffort.length > 0 && launchedEffort !== configuredEffort ? launchedEffort : null;
|
|
97236
|
+
} catch {}
|
|
97237
|
+
}
|
|
96967
97238
|
const alertPath = join54(smAgentDir, ".session-model-alert");
|
|
96968
97239
|
if (existsSync50(alertPath)) {
|
|
96969
97240
|
let alertText = null;
|
|
@@ -97082,6 +97353,7 @@ var didOneTimeSetup = false;
|
|
|
97082
97353
|
editPayload: text5
|
|
97083
97354
|
})
|
|
97084
97355
|
},
|
|
97356
|
+
floodWaitRemainingMs: probeFloodWaitRemainingMs,
|
|
97085
97357
|
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
97086
97358
|
`)
|
|
97087
97359
|
});
|