switchroom 0.18.17 → 0.18.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1544 -619
- package/telegram-plugin/dist/server.js +32 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/format.ts +137 -213
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +399 -85
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/outbound-send-path.ts +9 -7
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +474 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-reply-handler.ts +5 -14
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/format-consistency.test.ts +68 -53
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +5 -6
- package/telegram-plugin/tests/formatting-torture-set.ts +1 -1
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +26 -0
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +481 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +4 -3
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +42 -100
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +9 -12
- package/telegram-plugin/tests/telegram-format.test.ts +86 -31
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +17 -21
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/turn-flush-safety.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -6824,99 +6824,28 @@ function hardenCardBreaks(text) {
|
|
|
6824
6824
|
}
|
|
6825
6825
|
return restore(pieces.join(""));
|
|
6826
6826
|
}
|
|
6827
|
-
function addParagraphSpacers(text) {
|
|
6828
|
-
if (!text.includes(`
|
|
6829
|
-
|
|
6830
|
-
`))
|
|
6831
|
-
return text;
|
|
6832
|
-
const nonce = Math.random().toString(36).slice(2);
|
|
6833
|
-
const { masked, restore, placeholder } = maskCodeRegions(text, nonce);
|
|
6834
|
-
if (!masked.includes(`
|
|
6835
|
-
|
|
6836
|
-
`))
|
|
6837
|
-
return restore(masked);
|
|
6838
|
-
const spacerLine = PARAGRAPH_SPACER;
|
|
6839
|
-
const isBlankLine = (line) => /^[ \t\r\f\v]*$/.test(line);
|
|
6840
|
-
const asciiTrim = (line) => line.replace(/^[ \t\r\f\v]+|[ \t\r\f\v]+$/g, "");
|
|
6841
|
-
const blockKind = (line) => {
|
|
6842
|
-
if (asciiTrim(line) === spacerLine)
|
|
6843
|
-
return "spacer";
|
|
6844
|
-
if (isFenceOpenLine(line, placeholder))
|
|
6845
|
-
return "fence";
|
|
6846
|
-
if (isListItemLine(line))
|
|
6847
|
-
return "list";
|
|
6848
|
-
if (isTableRowLine(line) || isTableDelimiterLine(line))
|
|
6849
|
-
return "table";
|
|
6850
|
-
if (isBlockquoteLine(line))
|
|
6851
|
-
return "quote";
|
|
6852
|
-
if (isHeadingLine(line))
|
|
6853
|
-
return "heading";
|
|
6854
|
-
if (/^(-{3,}|\*{3,}|_{3,})\s*$/.test(line.trimStart()))
|
|
6855
|
-
return "divider";
|
|
6856
|
-
return "prose";
|
|
6857
|
-
};
|
|
6858
|
-
const SAME_KIND_TIGHT = new Set([
|
|
6859
|
-
"list",
|
|
6860
|
-
"table",
|
|
6861
|
-
"quote",
|
|
6862
|
-
"fence",
|
|
6863
|
-
"divider"
|
|
6864
|
-
]);
|
|
6865
|
-
const shouldSpaceGap = (above, below) => {
|
|
6866
|
-
const a = blockKind(above);
|
|
6867
|
-
const b = blockKind(below);
|
|
6868
|
-
if (a === "spacer" || b === "spacer")
|
|
6869
|
-
return false;
|
|
6870
|
-
if (a === b && SAME_KIND_TIGHT.has(a))
|
|
6871
|
-
return false;
|
|
6872
|
-
return true;
|
|
6873
|
-
};
|
|
6874
|
-
const lines = masked.split(`
|
|
6875
|
-
`);
|
|
6876
|
-
const out = [];
|
|
6877
|
-
for (let i = 0;i < lines.length; i++) {
|
|
6878
|
-
const line = lines[i];
|
|
6879
|
-
const isBlank = isBlankLine(line);
|
|
6880
|
-
if (isBlank) {
|
|
6881
|
-
const prevEmitted = out.length > 0 ? out[out.length - 1] : null;
|
|
6882
|
-
const prevIsBlank = prevEmitted != null && isBlankLine(prevEmitted);
|
|
6883
|
-
if (!prevIsBlank) {
|
|
6884
|
-
const above = lastNonBlank(out, isBlankLine);
|
|
6885
|
-
const below = nextNonBlank(lines, i + 1, isBlankLine);
|
|
6886
|
-
const alreadySpaced = above != null && asciiTrim(above) === spacerLine || below != null && asciiTrim(below) === spacerLine;
|
|
6887
|
-
if (!alreadySpaced && above != null && below != null && shouldSpaceGap(above, below)) {
|
|
6888
|
-
out.push("");
|
|
6889
|
-
out.push(spacerLine);
|
|
6890
|
-
out.push("");
|
|
6891
|
-
continue;
|
|
6892
|
-
}
|
|
6893
|
-
}
|
|
6894
|
-
}
|
|
6895
|
-
out.push(line);
|
|
6896
|
-
}
|
|
6897
|
-
return restore(out.join(`
|
|
6898
|
-
`));
|
|
6899
|
-
}
|
|
6900
|
-
function lastNonBlank(arr, isBlank) {
|
|
6901
|
-
for (let i = arr.length - 1;i >= 0; i--) {
|
|
6902
|
-
if (!isBlank(arr[i]))
|
|
6903
|
-
return arr[i];
|
|
6904
|
-
}
|
|
6905
|
-
return null;
|
|
6906
|
-
}
|
|
6907
|
-
function nextNonBlank(lines, from, isBlank) {
|
|
6908
|
-
for (let i = from;i < lines.length; i++) {
|
|
6909
|
-
if (!isBlank(lines[i]))
|
|
6910
|
-
return lines[i];
|
|
6911
|
-
}
|
|
6912
|
-
return null;
|
|
6913
|
-
}
|
|
6914
6827
|
function normalizePunctuation(text) {
|
|
6915
6828
|
if (!/[\u2014\u2013\u2022\u00b7]/.test(text))
|
|
6916
6829
|
return text;
|
|
6917
6830
|
const nonce = Math.random().toString(36).slice(2);
|
|
6918
6831
|
const { masked, restore } = maskCodeRegions(text, nonce);
|
|
6919
|
-
|
|
6832
|
+
const linkMasks = [];
|
|
6833
|
+
const LINK_MASK_PH = `\x00RML${nonce}_`;
|
|
6834
|
+
const maskedLinks = masked.replace(/(\]\()([^)\n]*)(\))/g, (_m, open, href, close) => {
|
|
6835
|
+
const idx = linkMasks.length;
|
|
6836
|
+
linkMasks.push(href);
|
|
6837
|
+
return `${open}${LINK_MASK_PH}${idx}\x00${close}`;
|
|
6838
|
+
});
|
|
6839
|
+
const maskedAutolinks = maskedLinks.replace(/(<)([a-zA-Z][a-zA-Z0-9+.-]*:[^>\s]*)(>)/g, (_m, lt, uri, gt) => {
|
|
6840
|
+
const idx = linkMasks.length;
|
|
6841
|
+
linkMasks.push(uri);
|
|
6842
|
+
return `${lt}${LINK_MASK_PH}${idx}\x00${gt}`;
|
|
6843
|
+
});
|
|
6844
|
+
const escNonce = nonce.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
6845
|
+
const linkRestoreRe = new RegExp(`\x00RML${escNonce}_(\\d+)\x00`, "g");
|
|
6846
|
+
const restoreLinks = (s) => s.replace(linkRestoreRe, (_m, idx) => linkMasks[Number(idx)] ?? _m);
|
|
6847
|
+
let out = maskedAutolinks.replace(/(\S)[ \t][\u2014\u2013][ \t](?=(\S))/g, (_m, a, b) => /\d/.test(a) && /\d/.test(b) ? `${a}-` : `${a}, `).replace(/(\w)\u2014(?=(\w))/g, (_m, a, b) => /\d/.test(a) && /\d/.test(b) ? `${a}-` : `${a}, `).replace(/(\w)\u2013(?=\w)/g, "$1-");
|
|
6848
|
+
out = restoreLinks(out);
|
|
6920
6849
|
out = out.split(`
|
|
6921
6850
|
`).map((line) => line.replace(/^([ \t]*)[\u2022\u00b7][ \t]+/, "$1- ")).join(`
|
|
6922
6851
|
`);
|
|
@@ -7109,6 +7038,7 @@ function splitMarkdownChunks(text, maxLen = RICH_MESSAGE_MAX_CHARS) {
|
|
|
7109
7038
|
}
|
|
7110
7039
|
cut = backOffOpenFence(rest, cut);
|
|
7111
7040
|
cut = backOffTableRow(rest, cut);
|
|
7041
|
+
cut = backOffOpenInline(rest, cut);
|
|
7112
7042
|
if (cut <= 0) {
|
|
7113
7043
|
const sliced = hardSliceToCap(rest, maxLen);
|
|
7114
7044
|
chunks.push(stripBoundarySpacers(sliced[0], "trailing"));
|
|
@@ -7121,11 +7051,10 @@ function splitMarkdownChunks(text, maxLen = RICH_MESSAGE_MAX_CHARS) {
|
|
|
7121
7051
|
return chunks.map((c) => stripBoundarySpacers(c, "trailing"));
|
|
7122
7052
|
}
|
|
7123
7053
|
function stripBoundarySpacers(chunk, side) {
|
|
7124
|
-
const sp = PARAGRAPH_SPACER;
|
|
7125
7054
|
if (side === "leading") {
|
|
7126
|
-
return chunk.replace(
|
|
7055
|
+
return chunk.replace(/^(?:[ \t]*\n)+/, "");
|
|
7127
7056
|
}
|
|
7128
|
-
return chunk.replace(
|
|
7057
|
+
return chunk.replace(/(?:\n[ \t]*)+$/, "");
|
|
7129
7058
|
}
|
|
7130
7059
|
function backOffOpenFence(text, cut) {
|
|
7131
7060
|
if (cut <= 0 || cut >= text.length)
|
|
@@ -7154,7 +7083,38 @@ function backOffTableRow(text, cut) {
|
|
|
7154
7083
|
}
|
|
7155
7084
|
return cut;
|
|
7156
7085
|
}
|
|
7157
|
-
|
|
7086
|
+
function backOffOpenInline(text, cut) {
|
|
7087
|
+
if (cut <= 0 || cut >= text.length)
|
|
7088
|
+
return cut;
|
|
7089
|
+
let earliest = cut;
|
|
7090
|
+
for (const re of INLINE_SPAN_PATTERNS) {
|
|
7091
|
+
re.lastIndex = 0;
|
|
7092
|
+
let m;
|
|
7093
|
+
while ((m = re.exec(text)) !== null) {
|
|
7094
|
+
const start = m.index;
|
|
7095
|
+
const end = start + m[0].length;
|
|
7096
|
+
if (start < cut && cut < end && start < earliest)
|
|
7097
|
+
earliest = start;
|
|
7098
|
+
if (start >= cut)
|
|
7099
|
+
break;
|
|
7100
|
+
if (re.lastIndex === start)
|
|
7101
|
+
re.lastIndex = start + 1;
|
|
7102
|
+
}
|
|
7103
|
+
}
|
|
7104
|
+
return earliest;
|
|
7105
|
+
}
|
|
7106
|
+
var RICH_MESSAGE_MAX_CHARS = 32768, INLINE_SPAN_PATTERNS;
|
|
7107
|
+
var init_format = __esm(() => {
|
|
7108
|
+
INLINE_SPAN_PATTERNS = [
|
|
7109
|
+
/`[^`\n]+`/g,
|
|
7110
|
+
/\*\*\*[^*\n]+\*\*\*/g,
|
|
7111
|
+
/___[^_\n]+___/g,
|
|
7112
|
+
/\*\*[^*\n]+\*\*/g,
|
|
7113
|
+
/__[^_\n]+__/g,
|
|
7114
|
+
/(?<![\w*])_[^_\n]+_(?![\w*])/g,
|
|
7115
|
+
/\[[^\]\n]*\]\([^)\n]*\)/g
|
|
7116
|
+
];
|
|
7117
|
+
});
|
|
7158
7118
|
|
|
7159
7119
|
// text-voice-scrub.ts
|
|
7160
7120
|
function enabled() {
|
|
@@ -7339,6 +7299,7 @@ function cleanWorkerResultParagraph(s) {
|
|
|
7339
7299
|
return kept.join(" ").replace(/\s+/g, " ").trim();
|
|
7340
7300
|
}
|
|
7341
7301
|
var init_card_format = __esm(() => {
|
|
7302
|
+
init_format();
|
|
7342
7303
|
init_text_voice_scrub();
|
|
7343
7304
|
});
|
|
7344
7305
|
|
|
@@ -7480,6 +7441,7 @@ function ttlMsFromToken(token) {
|
|
|
7480
7441
|
}
|
|
7481
7442
|
var import_grammy3;
|
|
7482
7443
|
var init_approval_card = __esm(() => {
|
|
7444
|
+
init_format();
|
|
7483
7445
|
import_grammy3 = __toESM(require_mod2(), 1);
|
|
7484
7446
|
});
|
|
7485
7447
|
|
|
@@ -20509,6 +20471,19 @@ var init_schema = __esm(() => {
|
|
|
20509
20471
|
approval_timeout_minutes: exports_external.number().int().nonnegative().optional().describe("Operator approval-card lifetime (minutes) for the tool-use 'Allow " + "once' card and the vault grant decision wait. After this long with " + "no operator tap, the card auto-denies (a TIMEOUT, not a denial \u2014 the " + "agent is told not to retry). Default 60. hostd-gated verbs " + "(mcp__hostd__*) keep their own longer window; the hostd " + "config-propose card is not governed by this key."),
|
|
20510
20472
|
sub_agent_tick_interval_ms: exports_external.number().int().nonnegative().optional().describe("Heartbeat tick interval (ms) for sub-agent rendering. Forces a " + "re-render of the elapsed-time counter while sub-agents are running, " + "even during silent stretches between tool calls. Default 10000 (10 s). " + "Set to 0 to disable the elapsed-ticker path."),
|
|
20511
20473
|
edit_budget_threshold: exports_external.number().int().nonnegative().optional().describe("Telegram API edit budget per minute before the progress-card driver " + "falls back to a slower coalesce window. When a chat accumulates more " + "than this many card edits in the trailing 60 s, the driver switches " + "to a wider coalesce interval until the rate drops back. Default 18. " + "Increase if your gateway frequently bumps the Telegram edit-rate ceiling " + "with many parallel sub-agents; decrease for a more conservative buffer."),
|
|
20474
|
+
send_gate: exports_external.object({
|
|
20475
|
+
enabled: exports_external.boolean().optional().describe("Master switch for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts) \u2014 the token-bucket scheduler every " + "Bot API call transits so per-surface throttles can't add up past a " + "flood ceiling. ON by default. Precedence: the operator break-glass " + "env var SWITCHROOM_TELEGRAM_SEND_GATE (0/false/off/no) ALWAYS wins " + "when explicitly set; this key only decides when that env var is " + "unset. Omit to keep the gate on."),
|
|
20476
|
+
global_per_sec: exports_external.number().positive().optional().describe("Global bucket sustained rate (Bot API calls/sec across ALL chats). " + "Default 25 (headroom under Telegram's ~30/s). Must be > 0 \u2014 a zero " + "or negative rate would wedge all outbound sends. Omit to keep 25."),
|
|
20477
|
+
global_burst: exports_external.number().int().positive().describe("Global bucket burst capacity. Default 4; worst-case 1s window " + "admits global_burst + global_per_sec = 29 < 30, a real margin under " + "the ceiling. Must be an integer >= 1 (a 0 capacity never admits a " + "token and wedges sends). Omit to keep 4.").optional(),
|
|
20478
|
+
per_chat_per_sec: exports_external.number().positive().optional().describe("Per-chat sustained rate (calls/sec to a single chat). Default 1. " + "Must be > 0 (zero/negative wedges that chat). Omit to keep 1."),
|
|
20479
|
+
per_chat_burst: exports_external.number().int().positive().optional().describe("Per-chat burst capacity. Default 3. Must be an integer >= 1 " + "(0 wedges the chat's bucket). Omit to keep 3."),
|
|
20480
|
+
per_group_per_min: exports_external.number().positive().optional().describe("Per-group sustained rate (calls/min to a single group/supergroup). " + "Default 18 (headroom under Telegram's ~20/min group ceiling). Must " + "be > 0. Omit to keep 18."),
|
|
20481
|
+
per_group_burst: exports_external.number().int().positive().optional().describe("Per-group burst capacity. Default 2. Must be an integer >= 1 " + "(0 wedges the group's bucket). Omit to keep 2."),
|
|
20482
|
+
edit_floor_ms: exports_external.number().int().nonnegative().optional().describe("Minimum ms between successive edits of the SAME message_id " + "(last-write-wins coalescing enforces this floor). Default 1500 " + "(Telegram's ~1 edit/sec/message practical ceiling). 0 disables the " + "floor. Must be an integer >= 0. Omit to keep 1500.")
|
|
20483
|
+
}).optional().describe("Tunable rate limits for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts). Every key is optional and defaults to " + "the send gate's built-in value, so omitting the whole block reproduces " + "today's exact behaviour \u2014 this is pure operator tuning, no default is " + "changed. Cascades from defaults.channels.telegram.send_gate."),
|
|
20484
|
+
worker_feed: exports_external.object({
|
|
20485
|
+
max_rows: exports_external.number().int().positive().optional().describe("Max live-worker rows rendered in the COMBINED worker-activity feed " + "(2+ background workers in one chat/thread coalesce into ONE message; " + "telegram-plugin/worker-activity-feed.ts) before a compact " + "'+M more working\u2026' spill line \u2014 keeps the coalesced body compact and " + "legible (and under the rich-message wire ceiling). Default 8. A " + "single-worker chat renders the full \uD83D\uDEE0 Worker card and ignores this. " + "Must be an integer >= 1. Omit to keep 8.")
|
|
20486
|
+
}).optional().describe("Tuning for the coalesced worker-activity feed " + "(telegram-plugin/worker-activity-feed.ts). Cascades from " + "defaults.channels.telegram.worker_feed."),
|
|
20512
20487
|
stickers: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Sticker aliases for the `send_sticker` MCP tool (#576). Maps a " + "short alias name (e.g. 'happy', 'thinking') to a Telegram file_id. " + "Operator-curated \u2014 capture file_ids from inbound stickers the user " + "sends and add them here. The agent calls send_sticker(chat_id, " + "alias='happy') and the gateway resolves to the file_id at send " + "time. Aliases enable persona-flavored expressiveness without " + "exposing raw file_ids in the agent prompt. Personal-assistant / " + "health-coach personas benefit; coding agents typically don't " + "configure any."),
|
|
20513
20488
|
voice_in: exports_external.object({
|
|
20514
20489
|
enabled: exports_external.boolean().optional().describe("Master switch for voice-message transcription."),
|
|
@@ -36872,6 +36847,7 @@ function parseConfigApprovalCallback(data) {
|
|
|
36872
36847
|
var pending, TELEGRAM_SENDMESSAGE_LIMIT2 = 32768, RENDERED_BODY_CAP2 = 32000, REASON_MAX_CHARS = 500, REASON_ELLIPSIS = "\u2026", DIFF_SENTINEL = `
|
|
36873
36848
|
[\u2026 diff continues, see attached file]`;
|
|
36874
36849
|
var init_config_approval_handler = __esm(() => {
|
|
36850
|
+
init_format();
|
|
36875
36851
|
pending = new Map;
|
|
36876
36852
|
});
|
|
36877
36853
|
|
|
@@ -36996,6 +36972,7 @@ ${detail}`)));
|
|
|
36996
36972
|
});
|
|
36997
36973
|
}
|
|
36998
36974
|
var init_approvals_commands = __esm(() => {
|
|
36975
|
+
init_format();
|
|
36999
36976
|
init_rich_send();
|
|
37000
36977
|
init_client3();
|
|
37001
36978
|
});
|
|
@@ -39765,6 +39742,7 @@ class DeferredDoneReactions {
|
|
|
39765
39742
|
init_card_format();
|
|
39766
39743
|
|
|
39767
39744
|
// status-no-truncate.ts
|
|
39745
|
+
init_format();
|
|
39768
39746
|
var STATUS_ROLLING_LINES = 5;
|
|
39769
39747
|
var STATUS_LINE_MAX = 200;
|
|
39770
39748
|
var STATUS_CARD_CHAR_BUDGET = RICH_MESSAGE_MAX_CHARS;
|
|
@@ -40238,9 +40216,45 @@ function renderActivityFeedWithNested(lines, childLines, final = false, liveSuff
|
|
|
40238
40216
|
stepCount
|
|
40239
40217
|
});
|
|
40240
40218
|
}
|
|
40219
|
+
var COMBINED_ROW_DESC_MAX = 72;
|
|
40220
|
+
function renderCombinedWorkerFeed(rows, opts) {
|
|
40221
|
+
if (rows.length === 0)
|
|
40222
|
+
return null;
|
|
40223
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows));
|
|
40224
|
+
const rowLines = (r) => {
|
|
40225
|
+
const desc = escapeMarkdown(truncate(stripMarkdown(r.description).replace(/\s+/g, " ").trim() || "background task", COMBINED_ROW_DESC_MAX));
|
|
40226
|
+
const toolWord = r.toolCount === 1 ? "tool" : "tools";
|
|
40227
|
+
const modelLabel = formatModelLabel(r.model);
|
|
40228
|
+
const modelPart = modelLabel != null ? ` \u00b7 ${escapeMarkdown(modelLabel)}` : "";
|
|
40229
|
+
const header = `**${desc}** _\u00b7 ${formatFeedElapsed(r.elapsedMs)} \u00b7 ${r.toolCount} ${toolWord}${modelPart}_`;
|
|
40230
|
+
const stepClean = stripMarkdown(r.currentStep).replace(/\s+/g, " ").trim();
|
|
40231
|
+
const step = stepClean.length > 0 ? `\u2192 _${escapeMarkdown(truncate(stepClean, STATUS_LINE_MAX))}_` : `\u2192 _starting\u2026_`;
|
|
40232
|
+
return [header, step];
|
|
40233
|
+
};
|
|
40234
|
+
const compose = (visibleCount) => {
|
|
40235
|
+
const shown = rows.slice(0, visibleCount);
|
|
40236
|
+
const hidden = rows.length - shown.length;
|
|
40237
|
+
const out = [`\uD83D\uDEE0 **Workers** \u00b7 _${rows.length} running_`];
|
|
40238
|
+
for (const r of shown) {
|
|
40239
|
+
const [h, s] = rowLines(r);
|
|
40240
|
+
out.push(h, s);
|
|
40241
|
+
}
|
|
40242
|
+
if (hidden > 0)
|
|
40243
|
+
out.push(`_+${hidden} more working\u2026_`);
|
|
40244
|
+
return stackCardLines(out);
|
|
40245
|
+
};
|
|
40246
|
+
let visible = Math.min(rows.length, maxRows);
|
|
40247
|
+
let body = compose(visible);
|
|
40248
|
+
while (body.length > STATUS_CARD_CHAR_BUDGET && visible > 1) {
|
|
40249
|
+
visible -= 1;
|
|
40250
|
+
body = compose(visible);
|
|
40251
|
+
}
|
|
40252
|
+
return body;
|
|
40253
|
+
}
|
|
40241
40254
|
|
|
40242
40255
|
// retry-api-call.ts
|
|
40243
40256
|
var import_grammy = __toESM(require_mod2(), 1);
|
|
40257
|
+
var LOCAL_RESOURCE_EXHAUSTED = "LOCAL_RESOURCE_EXHAUSTED";
|
|
40244
40258
|
var FLOOD_WAIT_ACTIVE = "FLOOD_WAIT_ACTIVE";
|
|
40245
40259
|
function makeFloodWaitActiveError(retryAfterSec, untilTs, original) {
|
|
40246
40260
|
return Object.assign(new Error(FLOOD_WAIT_ACTIVE), {
|
|
@@ -40371,13 +40385,16 @@ function createWorkerActivityFeed(opts) {
|
|
|
40371
40385
|
const minEditInterval = opts.minEditIntervalMs ?? 2500;
|
|
40372
40386
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000;
|
|
40373
40387
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000;
|
|
40388
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8));
|
|
40389
|
+
const reconcilePinFn = opts.reconcilePin ?? (() => {});
|
|
40374
40390
|
const setIntervalFn = opts.setInterval ?? ((cb, ms) => {
|
|
40375
40391
|
const t = setInterval(cb, ms);
|
|
40376
40392
|
t.unref?.();
|
|
40377
40393
|
return t;
|
|
40378
40394
|
});
|
|
40379
40395
|
const clearIntervalFn = opts.clearInterval ?? ((handle) => clearInterval(handle));
|
|
40380
|
-
const
|
|
40396
|
+
const groups = new Map;
|
|
40397
|
+
const agentIndex = new Map;
|
|
40381
40398
|
const finalized = new Set;
|
|
40382
40399
|
const FINALIZED_CAP = 256;
|
|
40383
40400
|
function markFinalized(agentId) {
|
|
@@ -40391,267 +40408,356 @@ function createWorkerActivityFeed(opts) {
|
|
|
40391
40408
|
}
|
|
40392
40409
|
}
|
|
40393
40410
|
let heartbeatTimer = null;
|
|
40394
|
-
function
|
|
40411
|
+
function feedKeyOf(chatId, threadId) {
|
|
40412
|
+
return `${chatId} ${threadId ?? ""}`;
|
|
40413
|
+
}
|
|
40414
|
+
function groupOfAgent(agentId) {
|
|
40415
|
+
const key = agentIndex.get(agentId);
|
|
40416
|
+
return key != null ? groups.get(key) : undefined;
|
|
40417
|
+
}
|
|
40418
|
+
function sendOptsFor(g) {
|
|
40395
40419
|
return {
|
|
40396
40420
|
disable_web_page_preview: true,
|
|
40397
40421
|
disable_notification: true,
|
|
40398
|
-
...
|
|
40422
|
+
...g.threadId != null ? { message_thread_id: g.threadId } : {}
|
|
40399
40423
|
};
|
|
40400
40424
|
}
|
|
40401
|
-
function noteRateLimited(
|
|
40425
|
+
function noteRateLimited(g, err, label) {
|
|
40402
40426
|
const retryAfter = extractRetryAfterSecs(err);
|
|
40403
40427
|
if (retryAfter == null)
|
|
40404
40428
|
return;
|
|
40405
|
-
|
|
40429
|
+
g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS;
|
|
40406
40430
|
log(`worker-feed: ${label} 429 \u2014 backing off ${retryAfter}s`);
|
|
40407
40431
|
}
|
|
40408
|
-
function parkIfFloodWindowOpen(
|
|
40432
|
+
function parkIfFloodWindowOpen(g) {
|
|
40409
40433
|
const remaining = floodWaitRemainingMs();
|
|
40410
40434
|
if (remaining <= 0)
|
|
40411
40435
|
return false;
|
|
40412
40436
|
const until = nowFn() + remaining + COOLDOWN_JITTER_MS;
|
|
40413
|
-
if (until >
|
|
40414
|
-
|
|
40437
|
+
if (until > g.cooldownUntil)
|
|
40438
|
+
g.cooldownUntil = until;
|
|
40415
40439
|
return true;
|
|
40416
40440
|
}
|
|
40417
|
-
function accumulateNarrative(
|
|
40441
|
+
function accumulateNarrative(row, view) {
|
|
40418
40442
|
const line = view.latestSummary.trim();
|
|
40419
40443
|
if (line.length === 0)
|
|
40420
40444
|
return;
|
|
40421
|
-
if (
|
|
40445
|
+
if (row.narrative.includes(line))
|
|
40422
40446
|
return;
|
|
40423
|
-
|
|
40424
|
-
|
|
40425
|
-
if (
|
|
40426
|
-
|
|
40427
|
-
}
|
|
40428
|
-
}
|
|
40429
|
-
|
|
40430
|
-
|
|
40431
|
-
|
|
40432
|
-
|
|
40433
|
-
|
|
40434
|
-
|
|
40435
|
-
|
|
40447
|
+
row.narrative.push(line);
|
|
40448
|
+
row.stepStartedAtMs = nowFn();
|
|
40449
|
+
if (row.narrative.length > STATUS_ROLLING_LINES) {
|
|
40450
|
+
row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES);
|
|
40451
|
+
}
|
|
40452
|
+
}
|
|
40453
|
+
function liveElapsed(row, now) {
|
|
40454
|
+
const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0;
|
|
40455
|
+
return Math.max(base, row.lastView?.elapsedMs ?? 0);
|
|
40456
|
+
}
|
|
40457
|
+
function runningRows(g) {
|
|
40458
|
+
return [...g.workers.values()].filter((w) => w.state === "running" && w.lastView != null).sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0));
|
|
40459
|
+
}
|
|
40460
|
+
function renderGroupBody(g, now, terminalRecap, heartbeat) {
|
|
40461
|
+
const running = runningRows(g);
|
|
40462
|
+
if (running.length === 0) {
|
|
40463
|
+
if (terminalRecap == null)
|
|
40464
|
+
return null;
|
|
40465
|
+
return renderWorkerActivity(terminalRecap);
|
|
40466
|
+
}
|
|
40467
|
+
const elapsedFor = (r) => heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now);
|
|
40468
|
+
if (running.length === 1) {
|
|
40469
|
+
const r = running[0];
|
|
40470
|
+
const view = {
|
|
40471
|
+
...r.lastView,
|
|
40472
|
+
elapsedMs: elapsedFor(r),
|
|
40473
|
+
narrativeLines: [...r.narrative]
|
|
40474
|
+
};
|
|
40475
|
+
let liveSuffix = "";
|
|
40476
|
+
if (heartbeat) {
|
|
40477
|
+
const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now);
|
|
40478
|
+
liveSuffix = formatStepSuffix(stepElapsed);
|
|
40479
|
+
}
|
|
40480
|
+
return renderWorkerActivity(view, liveSuffix);
|
|
40481
|
+
}
|
|
40482
|
+
const rows = running.map((r) => {
|
|
40483
|
+
const v = r.lastView;
|
|
40484
|
+
const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary;
|
|
40485
|
+
return {
|
|
40486
|
+
description: v.description,
|
|
40487
|
+
elapsedMs: elapsedFor(r),
|
|
40488
|
+
toolCount: v.toolCount,
|
|
40489
|
+
currentStep,
|
|
40490
|
+
model: v.model
|
|
40491
|
+
};
|
|
40492
|
+
});
|
|
40493
|
+
return renderCombinedWorkerFeed(rows, { maxRows });
|
|
40494
|
+
}
|
|
40495
|
+
function removeWorker(g, agentId) {
|
|
40496
|
+
g.workers.delete(agentId);
|
|
40497
|
+
agentIndex.delete(agentId);
|
|
40498
|
+
if (g.workers.size === 0)
|
|
40499
|
+
groups.delete(g.feedKey);
|
|
40500
|
+
}
|
|
40501
|
+
function syncPin(g) {
|
|
40502
|
+
const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null;
|
|
40503
|
+
reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId });
|
|
40504
|
+
}
|
|
40505
|
+
async function doRender(g, opts2 = {}) {
|
|
40506
|
+
const now = nowFn();
|
|
40507
|
+
const isTerminal = opts2.terminalRecap != null;
|
|
40508
|
+
const settleTerminal = () => {
|
|
40509
|
+
g.pendingFinalize = null;
|
|
40510
|
+
if (opts2.finishingAgentId != null)
|
|
40511
|
+
removeWorker(g, opts2.finishingAgentId);
|
|
40512
|
+
syncPin(g);
|
|
40513
|
+
};
|
|
40514
|
+
if (now < g.cooldownUntil) {
|
|
40515
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40516
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40517
|
+
return;
|
|
40518
|
+
}
|
|
40519
|
+
if (parkIfFloodWindowOpen(g)) {
|
|
40520
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40521
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40436
40522
|
return;
|
|
40437
|
-
|
|
40523
|
+
}
|
|
40524
|
+
const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false);
|
|
40525
|
+
if (body == null) {
|
|
40526
|
+
if (isTerminal)
|
|
40527
|
+
settleTerminal();
|
|
40438
40528
|
return;
|
|
40439
|
-
|
|
40440
|
-
if (
|
|
40441
|
-
|
|
40529
|
+
}
|
|
40530
|
+
if (g.messageId == null) {
|
|
40531
|
+
const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)));
|
|
40532
|
+
if (isTerminal) {
|
|
40533
|
+
settleTerminal();
|
|
40534
|
+
return;
|
|
40535
|
+
}
|
|
40536
|
+
if (maxElapsed < firstPaintMin)
|
|
40442
40537
|
return;
|
|
40443
40538
|
try {
|
|
40444
|
-
const sent = await opts.bot.sendMessage(
|
|
40539
|
+
const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g));
|
|
40445
40540
|
if (sent == null || typeof sent.message_id !== "number") {
|
|
40446
|
-
parkIfFloodWindowOpen(
|
|
40447
|
-
log(`worker-feed: first paint shed by send gate
|
|
40541
|
+
parkIfFloodWindowOpen(g);
|
|
40542
|
+
log(`worker-feed: first paint shed by send gate feed=${g.feedKey} \u2014 not delivered`);
|
|
40448
40543
|
return;
|
|
40449
40544
|
}
|
|
40450
|
-
|
|
40451
|
-
|
|
40452
|
-
|
|
40453
|
-
|
|
40545
|
+
g.messageId = sent.message_id;
|
|
40546
|
+
g.lastBody = body;
|
|
40547
|
+
g.lastEditAt = now;
|
|
40548
|
+
syncPin(g);
|
|
40549
|
+
log(`worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
|
|
40454
40550
|
} catch (err) {
|
|
40455
|
-
noteRateLimited(
|
|
40551
|
+
noteRateLimited(g, err, "send");
|
|
40456
40552
|
log(`worker-feed: send failed: ${err.message}`);
|
|
40457
40553
|
}
|
|
40458
40554
|
return;
|
|
40459
40555
|
}
|
|
40460
|
-
if (body ===
|
|
40461
|
-
|
|
40462
|
-
|
|
40463
|
-
return;
|
|
40464
|
-
try {
|
|
40465
|
-
const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
40466
|
-
if (isSendGateShed(res)) {
|
|
40467
|
-
parkIfFloodWindowOpen(h);
|
|
40468
|
-
return;
|
|
40469
|
-
}
|
|
40470
|
-
h.lastBody = body;
|
|
40471
|
-
h.lastEditAt = nowFn();
|
|
40472
|
-
log(`worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} bytes=${body.length}`);
|
|
40473
|
-
} catch (err) {
|
|
40474
|
-
const outcome = classifyEditError(err);
|
|
40475
|
-
if (outcome === "rate_limited") {
|
|
40476
|
-
noteRateLimited(h, err, "edit");
|
|
40477
|
-
return;
|
|
40478
|
-
}
|
|
40479
|
-
if (outcome === "not_modified") {
|
|
40480
|
-
h.lastBody = body;
|
|
40481
|
-
h.lastEditAt = nowFn();
|
|
40482
|
-
return;
|
|
40483
|
-
}
|
|
40484
|
-
if (outcome === "gone") {
|
|
40485
|
-
h.messageId = null;
|
|
40486
|
-
h.lastBody = null;
|
|
40487
|
-
return;
|
|
40488
|
-
}
|
|
40489
|
-
log(`worker-feed: edit transient error agent=${h.agentId}: ${err.message}`);
|
|
40490
|
-
}
|
|
40491
|
-
}
|
|
40492
|
-
async function doFinish(h, view) {
|
|
40493
|
-
h.finished = true;
|
|
40494
|
-
markFinalized(h.agentId);
|
|
40495
|
-
if (h.messageId == null) {
|
|
40496
|
-
h.pendingFinish = null;
|
|
40497
|
-
return;
|
|
40498
|
-
}
|
|
40499
|
-
if (parkIfFloodWindowOpen(h) || nowFn() < h.cooldownUntil) {
|
|
40500
|
-
h.pendingFinish = view;
|
|
40556
|
+
if (body === g.lastBody) {
|
|
40557
|
+
if (isTerminal)
|
|
40558
|
+
settleTerminal();
|
|
40501
40559
|
return;
|
|
40502
40560
|
}
|
|
40503
|
-
|
|
40504
|
-
if (body === h.lastBody) {
|
|
40505
|
-
h.pendingFinish = null;
|
|
40561
|
+
if (!opts2.force && now - g.lastEditAt < minEditInterval)
|
|
40506
40562
|
return;
|
|
40507
|
-
}
|
|
40508
40563
|
try {
|
|
40509
|
-
const res = await opts.bot.editMessageText(
|
|
40564
|
+
const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g));
|
|
40510
40565
|
if (isSendGateShed(res)) {
|
|
40511
|
-
parkIfFloodWindowOpen(
|
|
40512
|
-
|
|
40566
|
+
parkIfFloodWindowOpen(g);
|
|
40567
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40568
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40513
40569
|
return;
|
|
40514
40570
|
}
|
|
40515
|
-
|
|
40516
|
-
|
|
40517
|
-
|
|
40518
|
-
|
|
40571
|
+
g.lastBody = body;
|
|
40572
|
+
g.lastEditAt = now;
|
|
40573
|
+
if (isTerminal) {
|
|
40574
|
+
log(`worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? "-"} ` + `msgId=${g.messageId} agent=${opts2.finishingAgentId ?? "-"} ` + `state=${opts2.terminalRecap?.state ?? "done"} bytes=${body.length}`);
|
|
40575
|
+
} else {
|
|
40576
|
+
log(`worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` + `thread=${g.threadId ?? "-"} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`);
|
|
40577
|
+
}
|
|
40578
|
+
if (isTerminal)
|
|
40579
|
+
settleTerminal();
|
|
40519
40580
|
} catch (err) {
|
|
40520
40581
|
const outcome = classifyEditError(err);
|
|
40521
40582
|
if (outcome === "rate_limited") {
|
|
40522
|
-
noteRateLimited(
|
|
40523
|
-
|
|
40583
|
+
noteRateLimited(g, err, isTerminal ? "finish" : "edit");
|
|
40584
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40585
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40524
40586
|
return;
|
|
40525
40587
|
}
|
|
40526
40588
|
if (outcome === "not_modified") {
|
|
40527
|
-
|
|
40528
|
-
|
|
40529
|
-
|
|
40589
|
+
g.lastBody = body;
|
|
40590
|
+
g.lastEditAt = now;
|
|
40591
|
+
if (isTerminal)
|
|
40592
|
+
settleTerminal();
|
|
40530
40593
|
return;
|
|
40531
40594
|
}
|
|
40532
40595
|
if (outcome === "gone") {
|
|
40533
|
-
|
|
40596
|
+
g.messageId = null;
|
|
40597
|
+
g.lastBody = null;
|
|
40598
|
+
if (isTerminal)
|
|
40599
|
+
settleTerminal();
|
|
40600
|
+
else
|
|
40601
|
+
syncPin(g);
|
|
40534
40602
|
return;
|
|
40535
40603
|
}
|
|
40536
|
-
|
|
40537
|
-
|
|
40604
|
+
if (isTerminal && opts2.terminalRecap != null)
|
|
40605
|
+
g.pendingFinalize = opts2.terminalRecap;
|
|
40606
|
+
log(`worker-feed: edit transient error feed=${g.feedKey}: ${err.message}`);
|
|
40538
40607
|
}
|
|
40539
40608
|
}
|
|
40540
40609
|
function heartbeatTick() {
|
|
40541
40610
|
const now = nowFn();
|
|
40542
|
-
for (const
|
|
40543
|
-
if (
|
|
40544
|
-
|
|
40545
|
-
|
|
40546
|
-
|
|
40547
|
-
|
|
40548
|
-
log(`worker-feed: heartbeat finalize re-drive error ${h.agentId}: ${err.message}`);
|
|
40549
|
-
}).finally(() => {
|
|
40550
|
-
if (handles.get(h.agentId)?.pendingFinish == null) {
|
|
40551
|
-
handles.delete(h.agentId);
|
|
40552
|
-
}
|
|
40611
|
+
for (const g of [...groups.values()]) {
|
|
40612
|
+
if (g.pendingFinalize != null && now >= g.cooldownUntil) {
|
|
40613
|
+
const recap = g.pendingFinalize;
|
|
40614
|
+
const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId;
|
|
40615
|
+
g.chain = g.chain.then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId })).catch((err) => {
|
|
40616
|
+
log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${err.message}`);
|
|
40553
40617
|
});
|
|
40554
40618
|
continue;
|
|
40555
40619
|
}
|
|
40556
|
-
if (
|
|
40557
|
-
continue;
|
|
40558
|
-
if (h.lastView.state !== "running")
|
|
40620
|
+
if (now < g.cooldownUntil)
|
|
40559
40621
|
continue;
|
|
40560
|
-
|
|
40622
|
+
const running = runningRows(g);
|
|
40623
|
+
if (running.length === 0)
|
|
40561
40624
|
continue;
|
|
40562
|
-
|
|
40563
|
-
|
|
40564
|
-
if (
|
|
40625
|
+
if (g.messageId == null) {
|
|
40626
|
+
const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)));
|
|
40627
|
+
if (maxElapsed < firstPaintMin)
|
|
40565
40628
|
continue;
|
|
40566
|
-
|
|
40567
|
-
|
|
40568
|
-
log(`worker-feed: heartbeat first-paint chain error ${h.agentId}: ${err.message}`);
|
|
40629
|
+
g.chain = g.chain.then(() => doRender(g, {})).catch((err) => {
|
|
40630
|
+
log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${err.message}`);
|
|
40569
40631
|
});
|
|
40570
40632
|
continue;
|
|
40571
40633
|
}
|
|
40572
|
-
if (now -
|
|
40634
|
+
if (now - g.lastEditAt < minEditInterval)
|
|
40573
40635
|
continue;
|
|
40574
|
-
const stale = now -
|
|
40636
|
+
const stale = now - g.lastEditAt >= heartbeatTickMs;
|
|
40575
40637
|
if (!stale)
|
|
40576
40638
|
continue;
|
|
40577
|
-
|
|
40578
|
-
|
|
40579
|
-
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) };
|
|
40580
|
-
h.chain = h.chain.then(() => doUpdate(h, view, liveSuffix)).catch((err) => {
|
|
40581
|
-
log(`worker-feed: heartbeat chain error ${h.agentId}: ${err.message}`);
|
|
40639
|
+
g.chain = g.chain.then(() => doRender(g, { heartbeat: true })).catch((err) => {
|
|
40640
|
+
log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${err.message}`);
|
|
40582
40641
|
});
|
|
40583
40642
|
}
|
|
40584
40643
|
}
|
|
40585
40644
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs);
|
|
40586
40645
|
return {
|
|
40587
40646
|
has(agentId) {
|
|
40588
|
-
|
|
40647
|
+
const g = groupOfAgent(agentId);
|
|
40648
|
+
return g != null && g.messageId != null && g.workers.has(agentId);
|
|
40589
40649
|
},
|
|
40590
40650
|
messageIdOf(agentId) {
|
|
40591
|
-
return
|
|
40651
|
+
return groupOfAgent(agentId)?.messageId ?? null;
|
|
40652
|
+
},
|
|
40653
|
+
hasRunningInFeed(feedKey) {
|
|
40654
|
+
const g = groups.get(feedKey);
|
|
40655
|
+
return g != null && g.workers.size > 0;
|
|
40592
40656
|
},
|
|
40593
40657
|
get size() {
|
|
40594
|
-
|
|
40658
|
+
let n = 0;
|
|
40659
|
+
for (const g of groups.values())
|
|
40660
|
+
n += g.workers.size;
|
|
40661
|
+
return n;
|
|
40595
40662
|
},
|
|
40596
40663
|
update(agentId, chatId, view, threadId) {
|
|
40597
40664
|
if (chatId.length === 0)
|
|
40598
40665
|
return Promise.resolve();
|
|
40599
40666
|
if (finalized.has(agentId))
|
|
40600
40667
|
return Promise.resolve();
|
|
40601
|
-
const
|
|
40602
|
-
if (
|
|
40668
|
+
const existingRow = groupOfAgent(agentId)?.workers.get(agentId);
|
|
40669
|
+
if (existingRow?.finished === true)
|
|
40603
40670
|
return Promise.resolve();
|
|
40604
|
-
|
|
40605
|
-
|
|
40606
|
-
|
|
40607
|
-
|
|
40671
|
+
const feedKey = feedKeyOf(chatId, threadId);
|
|
40672
|
+
let g = groups.get(feedKey);
|
|
40673
|
+
if (g == null) {
|
|
40674
|
+
g = {
|
|
40675
|
+
feedKey,
|
|
40608
40676
|
chatId,
|
|
40609
40677
|
threadId,
|
|
40610
40678
|
messageId: null,
|
|
40611
40679
|
lastBody: null,
|
|
40612
40680
|
lastEditAt: 0,
|
|
40613
40681
|
cooldownUntil: 0,
|
|
40614
|
-
narrative: [],
|
|
40615
40682
|
chain: Promise.resolve(),
|
|
40683
|
+
workers: new Map,
|
|
40684
|
+
pendingFinalize: null
|
|
40685
|
+
};
|
|
40686
|
+
groups.set(feedKey, g);
|
|
40687
|
+
}
|
|
40688
|
+
let row = g.workers.get(agentId);
|
|
40689
|
+
if (row == null) {
|
|
40690
|
+
row = {
|
|
40691
|
+
agentId,
|
|
40692
|
+
narrative: [],
|
|
40616
40693
|
lastView: null,
|
|
40617
|
-
|
|
40618
|
-
stepStartedAtMs: null,
|
|
40694
|
+
state: "running",
|
|
40619
40695
|
finished: false,
|
|
40620
|
-
|
|
40696
|
+
dispatchAtMs: null,
|
|
40697
|
+
stepStartedAtMs: null
|
|
40621
40698
|
};
|
|
40622
|
-
|
|
40623
|
-
|
|
40624
|
-
|
|
40625
|
-
|
|
40699
|
+
g.workers.set(agentId, row);
|
|
40700
|
+
agentIndex.set(agentId, feedKey);
|
|
40701
|
+
}
|
|
40702
|
+
accumulateNarrative(row, view);
|
|
40703
|
+
row.state = "running";
|
|
40704
|
+
row.lastView = { ...view, narrativeLines: [...row.narrative] };
|
|
40705
|
+
if (row.dispatchAtMs == null)
|
|
40706
|
+
row.dispatchAtMs = nowFn() - view.elapsedMs;
|
|
40707
|
+
const group = g;
|
|
40708
|
+
group.chain = group.chain.then(() => doRender(group)).catch((err) => {
|
|
40626
40709
|
log(`worker-feed: update chain error ${agentId}: ${err.message}`);
|
|
40627
40710
|
});
|
|
40628
|
-
return
|
|
40711
|
+
return group.chain;
|
|
40629
40712
|
},
|
|
40630
40713
|
finish(agentId, view) {
|
|
40631
|
-
const
|
|
40632
|
-
|
|
40714
|
+
const g = groupOfAgent(agentId);
|
|
40715
|
+
const row = g?.workers.get(agentId);
|
|
40716
|
+
if (g == null || row == null) {
|
|
40717
|
+
markFinalized(agentId);
|
|
40633
40718
|
return Promise.resolve();
|
|
40634
|
-
|
|
40719
|
+
}
|
|
40720
|
+
row.finished = true;
|
|
40721
|
+
row.state = view.state === "failed" ? "failed" : "done";
|
|
40722
|
+
markFinalized(agentId);
|
|
40723
|
+
const group = g;
|
|
40724
|
+
group.chain = group.chain.then(() => {
|
|
40725
|
+
const others = runningRows(group).filter((w) => w.agentId !== agentId);
|
|
40726
|
+
if (others.length > 0) {
|
|
40727
|
+
removeWorker(group, agentId);
|
|
40728
|
+
syncPin(group);
|
|
40729
|
+
return doRender(group, { force: true });
|
|
40730
|
+
}
|
|
40731
|
+
const recap = { ...view, narrativeLines: [...row.narrative] };
|
|
40732
|
+
return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId });
|
|
40733
|
+
}).catch((err) => {
|
|
40635
40734
|
log(`worker-feed: finish chain error ${agentId}: ${err.message}`);
|
|
40636
|
-
}).finally(() => {
|
|
40637
|
-
if (handles.get(agentId)?.pendingFinish == null) {
|
|
40638
|
-
handles.delete(agentId);
|
|
40639
|
-
}
|
|
40640
40735
|
});
|
|
40641
|
-
return
|
|
40736
|
+
return group.chain;
|
|
40642
40737
|
},
|
|
40643
40738
|
drop(agentId) {
|
|
40644
40739
|
markFinalized(agentId);
|
|
40645
|
-
|
|
40740
|
+
const g = groupOfAgent(agentId);
|
|
40741
|
+
if (g == null)
|
|
40742
|
+
return;
|
|
40743
|
+
const hadMessage = g.messageId != null;
|
|
40744
|
+
removeWorker(g, agentId);
|
|
40745
|
+
if (hadMessage)
|
|
40746
|
+
syncPin(g);
|
|
40747
|
+
if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
|
|
40748
|
+
g.chain = g.chain.then(() => doRender(g, { force: true })).catch((err) => {
|
|
40749
|
+
log(`worker-feed: drop re-render error ${agentId}: ${err.message}`);
|
|
40750
|
+
});
|
|
40751
|
+
}
|
|
40646
40752
|
},
|
|
40647
40753
|
resurrect(agentId) {
|
|
40648
40754
|
const wasFinalized = finalized.delete(agentId);
|
|
40649
|
-
const
|
|
40650
|
-
if (
|
|
40651
|
-
|
|
40652
|
-
|
|
40755
|
+
const row = groupOfAgent(agentId)?.workers.get(agentId);
|
|
40756
|
+
if (row != null) {
|
|
40757
|
+
row.finished = false;
|
|
40758
|
+
row.state = "running";
|
|
40653
40759
|
}
|
|
40654
|
-
if (wasFinalized ||
|
|
40760
|
+
if (wasFinalized || row != null) {
|
|
40655
40761
|
log(`worker-feed: resurrect agent=${agentId} \u2014 cleared finalized gate; card will repaint on next running cue`);
|
|
40656
40762
|
}
|
|
40657
40763
|
},
|
|
@@ -41106,6 +41212,7 @@ var import_grammy2 = __toESM(require_mod2(), 1);
|
|
|
41106
41212
|
var import_runner = __toESM(require_mod4(), 1);
|
|
41107
41213
|
import { AsyncLocalStorage } from "async_hooks";
|
|
41108
41214
|
init_flood_circuit_breaker();
|
|
41215
|
+
init_format();
|
|
41109
41216
|
|
|
41110
41217
|
// shared/gw-trace-gate.ts
|
|
41111
41218
|
function computeGwTraceVerbose(flag) {
|
|
@@ -41140,6 +41247,7 @@ function escapeHtmlForTg(text) {
|
|
|
41140
41247
|
|
|
41141
41248
|
// gateway/vault-request-access-card.ts
|
|
41142
41249
|
init_approval_card();
|
|
41250
|
+
init_format();
|
|
41143
41251
|
function renderVaultRequestAccessCard(req) {
|
|
41144
41252
|
const lines = [];
|
|
41145
41253
|
const scopeLabel = req.scope === "write" ? "write" : "read";
|
|
@@ -44582,6 +44690,14 @@ function holdReasonFor(err) {
|
|
|
44582
44690
|
function isHeldUndeliverable2(pend) {
|
|
44583
44691
|
return pend.undeliverable != null;
|
|
44584
44692
|
}
|
|
44693
|
+
function applyDeliveredHoldReset(entry, now) {
|
|
44694
|
+
if (entry.undeliverable == null)
|
|
44695
|
+
return false;
|
|
44696
|
+
entry.undeliverable = null;
|
|
44697
|
+
entry.redeliveryFailures = 0;
|
|
44698
|
+
entry.startedAt = now;
|
|
44699
|
+
return true;
|
|
44700
|
+
}
|
|
44585
44701
|
var HELD_CARD_REDELIVERY_CAP = 3;
|
|
44586
44702
|
function selectHeldForRedelivery(entries, opts) {
|
|
44587
44703
|
const empty = { send: [], deferred: [] };
|
|
@@ -54668,6 +54784,7 @@ function parse2(markdown) {
|
|
|
54668
54784
|
}
|
|
54669
54785
|
|
|
54670
54786
|
// render/render.ts
|
|
54787
|
+
init_format();
|
|
54671
54788
|
function renderInline(node2, ctx = {}) {
|
|
54672
54789
|
switch (node2.type) {
|
|
54673
54790
|
case "plain":
|
|
@@ -54873,6 +54990,7 @@ function renderSafe(doc, source, maxLen = RICH_MESSAGE_MAX_CHARS) {
|
|
|
54873
54990
|
}
|
|
54874
54991
|
|
|
54875
54992
|
// render/rich-render.ts
|
|
54993
|
+
init_format();
|
|
54876
54994
|
var PLAIN_TEXT_MAX_CHARS = 4096;
|
|
54877
54995
|
function parseRichRenderEnabled(raw) {
|
|
54878
54996
|
if (raw == null)
|
|
@@ -55158,6 +55276,10 @@ function handlePtyPartialPure(text4, state, deps) {
|
|
|
55158
55276
|
stream.update(text4).catch(() => {});
|
|
55159
55277
|
return created ? "update-new" : "update-existing";
|
|
55160
55278
|
}
|
|
55279
|
+
|
|
55280
|
+
// stream-reply-handler.ts
|
|
55281
|
+
init_format();
|
|
55282
|
+
|
|
55161
55283
|
// chat-lock.ts
|
|
55162
55284
|
function createChatLock() {
|
|
55163
55285
|
const chains = new Map;
|
|
@@ -55213,7 +55335,7 @@ function isLocalResourceError(err) {
|
|
|
55213
55335
|
const msg = err instanceof Error ? err.message : String(err ?? "");
|
|
55214
55336
|
return /\b(ENOSPC|EDQUOT|EIO|ENOMEM)\b/.test(msg) || /no space left on device/i.test(msg) || /disk quota exceeded/i.test(msg);
|
|
55215
55337
|
}
|
|
55216
|
-
var
|
|
55338
|
+
var LOCAL_RESOURCE_EXHAUSTED2 = "LOCAL_RESOURCE_EXHAUSTED";
|
|
55217
55339
|
var GIVE_UP_MESSAGE = "retryApiCall: max retries exceeded";
|
|
55218
55340
|
var FLOOD_WAIT_ACTIVE2 = "FLOOD_WAIT_ACTIVE";
|
|
55219
55341
|
function makeFloodWaitActiveError2(retryAfterSec, untilTs, original) {
|
|
@@ -55269,13 +55391,13 @@ function createRetryApiCall2(config = {}) {
|
|
|
55269
55391
|
log?.(`telegram gateway: LOCAL resource exhaustion (${err.code ?? "disk/mem"}) \u2014 ` + `not retrying the send (would feed a flood ban); surfacing degraded state
|
|
55270
55392
|
`);
|
|
55271
55393
|
observer?.onGiveUp?.({ attempts: attempt + 1, error: err });
|
|
55272
|
-
throw Object.assign(new Error(
|
|
55394
|
+
throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED2), { original: err });
|
|
55273
55395
|
}
|
|
55274
55396
|
if (isGrammyErr && err.error_code === 429) {
|
|
55275
55397
|
const retryAfter = Number(err.parameters?.retry_after ?? 5);
|
|
55276
55398
|
const delayMs = retryAfter * 1000;
|
|
55277
55399
|
try {
|
|
55278
|
-
onFloodWait?.(retryAfter);
|
|
55400
|
+
onFloodWait?.(retryAfter, opts);
|
|
55279
55401
|
} catch {}
|
|
55280
55402
|
if (delayMs > maxFloodSleepMs) {
|
|
55281
55403
|
log?.(`telegram gateway: 429 flood ban of ${retryAfter}s exceeds the ` + `${Math.round(maxFloodSleepMs / 1000)}s in-process sleep ceiling \u2014 ` + `not sleeping it; surfacing degraded state
|
|
@@ -55444,16 +55566,25 @@ function stableStringify(value) {
|
|
|
55444
55566
|
});
|
|
55445
55567
|
}
|
|
55446
55568
|
var GROUP_TYPES2 = new Set(["group", "supergroup"]);
|
|
55569
|
+
var SEND_GATE_DEFAULTS = {
|
|
55570
|
+
globalPerSec: 25,
|
|
55571
|
+
globalBurst: 4,
|
|
55572
|
+
perChatPerSec: 1,
|
|
55573
|
+
perChatBurst: 3,
|
|
55574
|
+
perGroupPerMin: 18,
|
|
55575
|
+
perGroupBurst: 2,
|
|
55576
|
+
editFloorMs: 1500
|
|
55577
|
+
};
|
|
55447
55578
|
function createSendGate(config) {
|
|
55448
55579
|
const enabled2 = config.enabled;
|
|
55449
55580
|
const clock = config.clock ?? systemClock;
|
|
55450
|
-
const globalPerSec = config.globalPerSec ??
|
|
55451
|
-
const globalBurst = config.globalBurst ??
|
|
55452
|
-
const perChatPerSec = config.perChatPerSec ??
|
|
55453
|
-
const perChatBurst = config.perChatBurst ??
|
|
55454
|
-
const perGroupPerMin = config.perGroupPerMin ??
|
|
55455
|
-
const perGroupBurst = config.perGroupBurst ??
|
|
55456
|
-
const editFloorMs = config.editFloorMs ??
|
|
55581
|
+
const globalPerSec = config.globalPerSec ?? SEND_GATE_DEFAULTS.globalPerSec;
|
|
55582
|
+
const globalBurst = config.globalBurst ?? SEND_GATE_DEFAULTS.globalBurst;
|
|
55583
|
+
const perChatPerSec = config.perChatPerSec ?? SEND_GATE_DEFAULTS.perChatPerSec;
|
|
55584
|
+
const perChatBurst = config.perChatBurst ?? SEND_GATE_DEFAULTS.perChatBurst;
|
|
55585
|
+
const perGroupPerMin = config.perGroupPerMin ?? SEND_GATE_DEFAULTS.perGroupPerMin;
|
|
55586
|
+
const perGroupBurst = config.perGroupBurst ?? SEND_GATE_DEFAULTS.perGroupBurst;
|
|
55587
|
+
const editFloorMs = config.editFloorMs ?? SEND_GATE_DEFAULTS.editFloorMs;
|
|
55457
55588
|
const messageStateTtlMs = config.messageStateTtlMs ?? 60000;
|
|
55458
55589
|
const maxMessageStates = config.maxMessageStates ?? 5000;
|
|
55459
55590
|
const usefulTtlMs = config.usefulTtlMs ?? 120000;
|
|
@@ -55461,6 +55592,7 @@ function createSendGate(config) {
|
|
|
55461
55592
|
const criticalJitterMaxMs = config.criticalJitterMaxMs ?? 250;
|
|
55462
55593
|
const jitter = config.jitter ?? Math.random;
|
|
55463
55594
|
const onWindowOpen = config.onWindowOpen;
|
|
55595
|
+
const conservativeGlobalFloodScope = config.conservativeGlobalFloodScope ?? false;
|
|
55464
55596
|
const counters = {
|
|
55465
55597
|
sent: 0,
|
|
55466
55598
|
queued: 0,
|
|
@@ -55531,8 +55663,11 @@ function createSendGate(config) {
|
|
|
55531
55663
|
applyWindow(scopeKey, untilTs, true);
|
|
55532
55664
|
}
|
|
55533
55665
|
function openScopedWindowsForOpts(opts, untilTs) {
|
|
55534
|
-
|
|
55535
|
-
if (
|
|
55666
|
+
const hasChatScope = opts?.chat_id != null && opts.chat_id !== "";
|
|
55667
|
+
if (!hasChatScope || conservativeGlobalFloodScope) {
|
|
55668
|
+
applyWindow("global", untilTs, true);
|
|
55669
|
+
}
|
|
55670
|
+
if (hasChatScope) {
|
|
55536
55671
|
applyWindow(`chat:${opts.chat_id}`, untilTs, true);
|
|
55537
55672
|
if (opts.chatType && GROUP_TYPES2.has(opts.chatType)) {
|
|
55538
55673
|
applyWindow(`group:${opts.chat_id}`, untilTs, true);
|
|
@@ -55834,14 +55969,114 @@ function createSendGate(config) {
|
|
|
55834
55969
|
}
|
|
55835
55970
|
};
|
|
55836
55971
|
}
|
|
55837
|
-
|
|
55972
|
+
function openScopedFloodWindows(opts, untilTs) {
|
|
55973
|
+
if (!enabled2)
|
|
55974
|
+
return;
|
|
55975
|
+
openScopedWindowsForOpts(opts, untilTs);
|
|
55976
|
+
}
|
|
55977
|
+
return { gate, openFloodWindow, openScopedFloodWindows, stats };
|
|
55838
55978
|
}
|
|
55839
|
-
function
|
|
55840
|
-
const v = env.SWITCHROOM_TELEGRAM_SEND_GATE;
|
|
55841
|
-
if (v == null)
|
|
55842
|
-
return true;
|
|
55979
|
+
function isOffValue(v) {
|
|
55843
55980
|
const t = v.trim().toLowerCase();
|
|
55844
|
-
return
|
|
55981
|
+
return t === "0" || t === "false" || t === "off" || t === "no";
|
|
55982
|
+
}
|
|
55983
|
+
function parsePositiveNumber(raw) {
|
|
55984
|
+
if (raw == null || raw.trim() === "")
|
|
55985
|
+
return;
|
|
55986
|
+
const n = Number(raw);
|
|
55987
|
+
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
55988
|
+
}
|
|
55989
|
+
function parsePositiveInt(raw) {
|
|
55990
|
+
if (raw == null || raw.trim() === "")
|
|
55991
|
+
return;
|
|
55992
|
+
const n = Number(raw);
|
|
55993
|
+
return Number.isInteger(n) && n > 0 ? n : undefined;
|
|
55994
|
+
}
|
|
55995
|
+
function parseNonNegativeInt(raw) {
|
|
55996
|
+
if (raw == null || raw.trim() === "")
|
|
55997
|
+
return;
|
|
55998
|
+
const n = Number(raw);
|
|
55999
|
+
return Number.isInteger(n) && n >= 0 ? n : undefined;
|
|
56000
|
+
}
|
|
56001
|
+
function parseBoolFlag(raw) {
|
|
56002
|
+
if (raw == null || raw.trim() === "")
|
|
56003
|
+
return;
|
|
56004
|
+
const t = raw.trim().toLowerCase();
|
|
56005
|
+
if (t === "1" || t === "true" || t === "on" || t === "yes")
|
|
56006
|
+
return true;
|
|
56007
|
+
if (isOffValue(t))
|
|
56008
|
+
return false;
|
|
56009
|
+
return;
|
|
56010
|
+
}
|
|
56011
|
+
function sendGateConfigFromEnv(env = process.env) {
|
|
56012
|
+
const killSwitch = env.SWITCHROOM_TELEGRAM_SEND_GATE;
|
|
56013
|
+
let enabled2;
|
|
56014
|
+
if (killSwitch != null && killSwitch.trim() !== "") {
|
|
56015
|
+
enabled2 = !isOffValue(killSwitch);
|
|
56016
|
+
} else {
|
|
56017
|
+
const cfg = env.SWITCHROOM_TG_SEND_GATE_ENABLED;
|
|
56018
|
+
enabled2 = cfg != null && cfg.trim() !== "" ? !isOffValue(cfg) : true;
|
|
56019
|
+
}
|
|
56020
|
+
const out = { enabled: enabled2 };
|
|
56021
|
+
const globalPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_PER_SEC);
|
|
56022
|
+
if (globalPerSec !== undefined)
|
|
56023
|
+
out.globalPerSec = globalPerSec;
|
|
56024
|
+
const globalBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_GLOBAL_BURST);
|
|
56025
|
+
if (globalBurst !== undefined)
|
|
56026
|
+
out.globalBurst = globalBurst;
|
|
56027
|
+
const perChatPerSec = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_PER_SEC);
|
|
56028
|
+
if (perChatPerSec !== undefined)
|
|
56029
|
+
out.perChatPerSec = perChatPerSec;
|
|
56030
|
+
const perChatBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_CHAT_BURST);
|
|
56031
|
+
if (perChatBurst !== undefined)
|
|
56032
|
+
out.perChatBurst = perChatBurst;
|
|
56033
|
+
const perGroupPerMin = parsePositiveNumber(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_PER_MIN);
|
|
56034
|
+
if (perGroupPerMin !== undefined)
|
|
56035
|
+
out.perGroupPerMin = perGroupPerMin;
|
|
56036
|
+
const perGroupBurst = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_GROUP_BURST);
|
|
56037
|
+
if (perGroupBurst !== undefined)
|
|
56038
|
+
out.perGroupBurst = perGroupBurst;
|
|
56039
|
+
const editFloorMs = parseNonNegativeInt(env.SWITCHROOM_TG_SEND_GATE_EDIT_FLOOR_MS);
|
|
56040
|
+
if (editFloorMs !== undefined)
|
|
56041
|
+
out.editFloorMs = editFloorMs;
|
|
56042
|
+
const conservativeGlobal = parseBoolFlag(env.SWITCHROOM_TG_SEND_GATE_CONSERVATIVE_GLOBAL);
|
|
56043
|
+
if (conservativeGlobal !== undefined)
|
|
56044
|
+
out.conservativeGlobalFloodScope = conservativeGlobal;
|
|
56045
|
+
return out;
|
|
56046
|
+
}
|
|
56047
|
+
|
|
56048
|
+
// shared/local-time.ts
|
|
56049
|
+
function fmtLocalClock(ms, tz) {
|
|
56050
|
+
return new Intl.DateTimeFormat("en-US", {
|
|
56051
|
+
timeZone: tz,
|
|
56052
|
+
hour: "numeric",
|
|
56053
|
+
minute: "2-digit",
|
|
56054
|
+
hour12: true
|
|
56055
|
+
}).format(new Date(ms)).replace(/\s([AP])M$/, (_m, p) => `${p.toLowerCase()}m`);
|
|
56056
|
+
}
|
|
56057
|
+
function fmtLocalDate(ms, tz) {
|
|
56058
|
+
return new Intl.DateTimeFormat("en-GB", {
|
|
56059
|
+
timeZone: tz,
|
|
56060
|
+
day: "numeric",
|
|
56061
|
+
month: "short"
|
|
56062
|
+
}).format(new Date(ms));
|
|
56063
|
+
}
|
|
56064
|
+
function localDay(ms, tz) {
|
|
56065
|
+
return new Intl.DateTimeFormat("en-CA", {
|
|
56066
|
+
timeZone: tz,
|
|
56067
|
+
year: "numeric",
|
|
56068
|
+
month: "2-digit",
|
|
56069
|
+
day: "2-digit"
|
|
56070
|
+
}).format(new Date(ms));
|
|
56071
|
+
}
|
|
56072
|
+
function tzAbbrev(ms, tz) {
|
|
56073
|
+
const at = new Date(ms);
|
|
56074
|
+
for (const loc of ["en-US", "en-AU", "en-GB"]) {
|
|
56075
|
+
const v = new Intl.DateTimeFormat(loc, { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value;
|
|
56076
|
+
if (v && !/^(?:GMT|UTC)/i.test(v))
|
|
56077
|
+
return v;
|
|
56078
|
+
}
|
|
56079
|
+
return new Intl.DateTimeFormat("en-US", { timeZone: tz, timeZoneName: "short" }).formatToParts(at).find((p) => p.type === "timeZoneName")?.value ?? tz;
|
|
55845
56080
|
}
|
|
55846
56081
|
|
|
55847
56082
|
// send-gate-observability.ts
|
|
@@ -55878,8 +56113,18 @@ function createStatsLogger(config) {
|
|
|
55878
56113
|
function isAlertableScope(scopeKey) {
|
|
55879
56114
|
return scopeKey === "global" || scopeKey.startsWith("chat:") || scopeKey.startsWith("group:");
|
|
55880
56115
|
}
|
|
55881
|
-
function
|
|
55882
|
-
|
|
56116
|
+
function fmtLocalStamp(ms, tz, refMs) {
|
|
56117
|
+
const clock = fmtLocalClock(ms, tz);
|
|
56118
|
+
const sameDay = refMs != null && localDay(ms, tz) === localDay(refMs, tz);
|
|
56119
|
+
const body = sameDay ? clock : `${fmtLocalDate(ms, tz)} ${clock}`;
|
|
56120
|
+
return `${body} ${tzAbbrev(ms, tz)}`;
|
|
56121
|
+
}
|
|
56122
|
+
function fmtLocalRange(startMs, endMs, tz) {
|
|
56123
|
+
const sameDay = localDay(startMs, tz) === localDay(endMs, tz);
|
|
56124
|
+
if (sameDay) {
|
|
56125
|
+
return `${fmtLocalClock(startMs, tz)} to ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
|
|
56126
|
+
}
|
|
56127
|
+
return `${fmtLocalDate(startMs, tz)} ${fmtLocalClock(startMs, tz)} to ` + `${fmtLocalDate(endMs, tz)} ${fmtLocalClock(endMs, tz)} ${tzAbbrev(endMs, tz)}`;
|
|
55883
56128
|
}
|
|
55884
56129
|
function fmtDur(ms) {
|
|
55885
56130
|
const s = Math.round(ms / 1000);
|
|
@@ -55891,6 +56136,7 @@ function fmtDur(ms) {
|
|
|
55891
56136
|
}
|
|
55892
56137
|
function createFloodWindowObserver(config) {
|
|
55893
56138
|
const alertThresholdMs = config.alertThresholdMs ?? 60000;
|
|
56139
|
+
const tz = config.tz ?? "UTC";
|
|
55894
56140
|
let lastSeen = new Map;
|
|
55895
56141
|
let firstTick = true;
|
|
55896
56142
|
const owedOnClose = new Set;
|
|
@@ -55911,14 +56157,14 @@ function createFloodWindowObserver(config) {
|
|
|
55911
56157
|
}
|
|
55912
56158
|
function openAlertText(w, now) {
|
|
55913
56159
|
const openFor = fmtDur(now - w.observedAt);
|
|
55914
|
-
return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${
|
|
56160
|
+
return `\u26a0\ufe0f Telegram flood ban active (scope \`${w.scopeKey}\`). ` + `Open for ${openFor}, expected to clear at ${fmtLocalStamp(w.untilTs, tz, now)}. ` + `Outbound to that scope is being suppressed.`;
|
|
55915
56161
|
}
|
|
55916
56162
|
function closeAlertText(recs) {
|
|
55917
56163
|
const observedAt = Math.min(...recs.map((r) => r.observedAt));
|
|
55918
56164
|
const untilTs = Math.max(...recs.map((r) => r.untilTs));
|
|
55919
56165
|
const scopes = recs.map((r) => `\`${r.scopeKey}\``).join(", ");
|
|
55920
56166
|
const plural = recs.length > 1 ? "s" : "";
|
|
55921
|
-
return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${
|
|
56167
|
+
return `\u26a0\ufe0f Telegram flood ban cleared (scope${plural} ${scopes}). ` + `The bot was banned from ${fmtLocalRange(observedAt, untilTs, tz)} ` + `(~${fmtDur(untilTs - observedAt)}). Some outbound messages during that ` + `window were suppressed.`;
|
|
55922
56168
|
}
|
|
55923
56169
|
async function tick() {
|
|
55924
56170
|
const s = config.stats();
|
|
@@ -56128,6 +56374,7 @@ var import_runner2 = __toESM(require_mod4(), 1);
|
|
|
56128
56374
|
import { createHash as createHash2 } from "crypto";
|
|
56129
56375
|
import { AsyncLocalStorage as AsyncLocalStorage2 } from "async_hooks";
|
|
56130
56376
|
init_flood_circuit_breaker();
|
|
56377
|
+
init_format();
|
|
56131
56378
|
var tgPostTagStore2 = new AsyncLocalStorage2;
|
|
56132
56379
|
function _getTgPostTags() {
|
|
56133
56380
|
return tgPostTagStore2.getStore();
|
|
@@ -61224,6 +61471,7 @@ function decideOverPing(input) {
|
|
|
61224
61471
|
}
|
|
61225
61472
|
|
|
61226
61473
|
// silent-reply-anchor.ts
|
|
61474
|
+
init_format();
|
|
61227
61475
|
var TELEGRAM_MSG_CAP = RICH_MESSAGE_MAX_CHARS;
|
|
61228
61476
|
function enabled3() {
|
|
61229
61477
|
const v = process.env.SWITCHROOM_DISABLE_SILENT_REPLY_AUTOEDIT;
|
|
@@ -62361,6 +62609,7 @@ function safeResolvePersonName(directory, opts, rawFallback) {
|
|
|
62361
62609
|
}
|
|
62362
62610
|
|
|
62363
62611
|
// gateway/auth-command.ts
|
|
62612
|
+
init_format();
|
|
62364
62613
|
init_auth_snapshot_format();
|
|
62365
62614
|
init_demo_mask();
|
|
62366
62615
|
var AUTH_RM_CONFIRM_TTL_MS = 60000;
|
|
@@ -63787,18 +64036,28 @@ function createFleetFallbackResumeGate(opts = {}) {
|
|
|
63787
64036
|
const maxAgeMs = opts.maxAgeMs ?? DEFAULT_RESUME_MAX_AGE_MS;
|
|
63788
64037
|
const singleFlightMs = opts.singleFlightMs ?? DEFAULT_RESUME_SINGLE_FLIGHT_MS;
|
|
63789
64038
|
let lastResumedAtMs = Number.NEGATIVE_INFINITY;
|
|
63790
|
-
function
|
|
64039
|
+
function peek(failedTurnStartedAtMs) {
|
|
63791
64040
|
const now = nowFn();
|
|
63792
64041
|
if (now - lastResumedAtMs < singleFlightMs)
|
|
63793
64042
|
return "skip-inflight";
|
|
63794
64043
|
if (failedTurnStartedAtMs != null && now - failedTurnStartedAtMs > maxAgeMs) {
|
|
63795
64044
|
return "skip-stale";
|
|
63796
64045
|
}
|
|
63797
|
-
lastResumedAtMs = now;
|
|
63798
64046
|
return "resume";
|
|
63799
64047
|
}
|
|
64048
|
+
function arm() {
|
|
64049
|
+
lastResumedAtMs = nowFn();
|
|
64050
|
+
}
|
|
64051
|
+
function decide(failedTurnStartedAtMs) {
|
|
64052
|
+
const verdict = peek(failedTurnStartedAtMs);
|
|
64053
|
+
if (verdict === "resume")
|
|
64054
|
+
arm();
|
|
64055
|
+
return verdict;
|
|
64056
|
+
}
|
|
63800
64057
|
return {
|
|
63801
64058
|
decide,
|
|
64059
|
+
peek,
|
|
64060
|
+
arm,
|
|
63802
64061
|
reset() {
|
|
63803
64062
|
lastResumedAtMs = Number.NEGATIVE_INFINITY;
|
|
63804
64063
|
},
|
|
@@ -64500,10 +64759,34 @@ function autoClassifyMidTurnInbound(i) {
|
|
|
64500
64759
|
return { decision: "queue", reason: "cross_topic", sameTopic: false };
|
|
64501
64760
|
return recent && i.msSinceLastAgentOutput <= i.topicSteerWindowMs ? { decision: "steer", reason: "same_topic_recent", sameTopic: true } : { decision: "queue", reason: "same_topic_stale", sameTopic: true };
|
|
64502
64761
|
}
|
|
64762
|
+
|
|
64763
|
+
// operator-events.ts
|
|
64764
|
+
init_format();
|
|
64765
|
+
|
|
64766
|
+
// raw-error-scrub.ts
|
|
64767
|
+
function stripRawErrorBytes(raw) {
|
|
64768
|
+
if (typeof raw !== "string" || raw.length === 0)
|
|
64769
|
+
return "";
|
|
64770
|
+
let s = raw;
|
|
64771
|
+
s = s.replace(/API Error:?\s*\d*\s*/gi, " ");
|
|
64772
|
+
s = s.replace(/\bb'[^']*'/g, " ");
|
|
64773
|
+
s = s.replace(/\bb"[^"]*"/g, " ");
|
|
64774
|
+
s = s.replace(/[\u00b7\-\s]*\{[\s\S]*?["']type["']\s*:\s*["']error["'][\s\S]*$/i, " ");
|
|
64775
|
+
s = s.replace(/^\s*\{[\s\S]*\}\s*$/g, " ");
|
|
64776
|
+
s = s.replace(/\s+/g, " ").replace(/[\u00b7:\-\s]+$/g, "").replace(/^[\u00b7:\-\s]+/g, "").trim();
|
|
64777
|
+
return s;
|
|
64778
|
+
}
|
|
64779
|
+
function extractRequestId(raw) {
|
|
64780
|
+
if (typeof raw !== "string" || raw.length === 0)
|
|
64781
|
+
return;
|
|
64782
|
+
const m = raw.match(/["']request[_-]?id["']\s*:\s*["']([A-Za-z0-9._-]+)["']/i) ?? raw.match(/\brequest[_-]?id[=:]\s*([A-Za-z0-9._-]+)/i);
|
|
64783
|
+
return m ? m[1] : undefined;
|
|
64784
|
+
}
|
|
64785
|
+
|
|
64503
64786
|
// operator-events.ts
|
|
64504
64787
|
function renderOperatorEvent(ev) {
|
|
64505
64788
|
const agent = escapeMarkdown(ev.agent);
|
|
64506
|
-
const detail = escapeMarkdown(ev.detail);
|
|
64789
|
+
const detail = escapeMarkdown(stripRawErrorBytes(ev.detail));
|
|
64507
64790
|
switch (ev.kind) {
|
|
64508
64791
|
case "credentials-expired":
|
|
64509
64792
|
return {
|
|
@@ -64720,6 +65003,13 @@ var transientUpstreamSignals = [
|
|
|
64720
65003
|
"would exceed your account\u2019s rate limit",
|
|
64721
65004
|
"would exceed your account's rate limit"
|
|
64722
65005
|
];
|
|
65006
|
+
function isTransientUpstreamSignal(text4) {
|
|
65007
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65008
|
+
return false;
|
|
65009
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65010
|
+
const lower = sample.toLowerCase();
|
|
65011
|
+
return transientUpstreamSignals.some((s) => lower.includes(s));
|
|
65012
|
+
}
|
|
64723
65013
|
var litellmProxyLocal429Signals = [
|
|
64724
65014
|
"deployment over user-defined ratelimit",
|
|
64725
65015
|
"model rate limit exceeded. tpm limit",
|
|
@@ -64740,6 +65030,56 @@ function isLitellmProxyLocal429(text4) {
|
|
|
64740
65030
|
return true;
|
|
64741
65031
|
return litellmV3LimiterSignalPair.every((s) => lower.includes(s));
|
|
64742
65032
|
}
|
|
65033
|
+
function parseLitellmLimitDetail(text4, parseTimeNow = new Date) {
|
|
65034
|
+
const empty2 = { limitType: null, limit: null, currentUsage: null, resetAtMs: null };
|
|
65035
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65036
|
+
return empty2;
|
|
65037
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65038
|
+
const lower = sample.toLowerCase();
|
|
65039
|
+
let limitType = null;
|
|
65040
|
+
let limit = null;
|
|
65041
|
+
const eqLimit = lower.match(/\b([tr]pm)[ _]limit[=:]\s*(\d+)/);
|
|
65042
|
+
if (eqLimit) {
|
|
65043
|
+
limitType = eqLimit[1];
|
|
65044
|
+
limit = Number(eqLimit[2]);
|
|
65045
|
+
}
|
|
65046
|
+
if (limitType == null) {
|
|
65047
|
+
const v3Type = lower.match(/limit type:\s*(tokens|requests|max_parallel_requests)/);
|
|
65048
|
+
if (v3Type)
|
|
65049
|
+
limitType = v3Type[1];
|
|
65050
|
+
}
|
|
65051
|
+
if (limit == null) {
|
|
65052
|
+
const v3Limit = lower.match(/current limit:\s*(\d+)/);
|
|
65053
|
+
if (v3Limit)
|
|
65054
|
+
limit = Number(v3Limit[1]);
|
|
65055
|
+
}
|
|
65056
|
+
let currentUsage = null;
|
|
65057
|
+
const usage = lower.match(/current usage=(\d+)/);
|
|
65058
|
+
if (usage)
|
|
65059
|
+
currentUsage = Number(usage[1]);
|
|
65060
|
+
let resetAtMs = null;
|
|
65061
|
+
const resetsAt = sample.match(/limit resets at:\s*(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}) UTC/i);
|
|
65062
|
+
if (resetsAt) {
|
|
65063
|
+
const d = new Date(`${resetsAt[1]}T${resetsAt[2]}Z`);
|
|
65064
|
+
if (!Number.isNaN(d.getTime()))
|
|
65065
|
+
resetAtMs = d.getTime();
|
|
65066
|
+
}
|
|
65067
|
+
if (resetAtMs == null) {
|
|
65068
|
+
const tryAgain = lower.match(/try again in\s+(\d+(?:\.\d+)?)\s*seconds/);
|
|
65069
|
+
if (tryAgain) {
|
|
65070
|
+
const secs = Number(tryAgain[1]);
|
|
65071
|
+
if (Number.isFinite(secs) && secs > 0 && secs < 7 * 24 * 3600) {
|
|
65072
|
+
resetAtMs = parseTimeNow.getTime() + Math.round(secs * 1000);
|
|
65073
|
+
}
|
|
65074
|
+
}
|
|
65075
|
+
}
|
|
65076
|
+
return {
|
|
65077
|
+
limitType,
|
|
65078
|
+
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
65079
|
+
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
65080
|
+
resetAtMs
|
|
65081
|
+
};
|
|
65082
|
+
}
|
|
64743
65083
|
function detectModelUnavailable(stderr) {
|
|
64744
65084
|
if (typeof stderr !== "string" || stderr.length === 0)
|
|
64745
65085
|
return null;
|
|
@@ -64933,54 +65273,366 @@ function parseRelativeDuration(s) {
|
|
|
64933
65273
|
}
|
|
64934
65274
|
return matched && total > 0 ? total : null;
|
|
64935
65275
|
}
|
|
64936
|
-
|
|
65276
|
+
|
|
65277
|
+
// throttle-tier.ts
|
|
65278
|
+
init_card_format();
|
|
65279
|
+
init_quota_check();
|
|
65280
|
+
var accountScopedThrottleSignals = [
|
|
65281
|
+
"would exceed your account's rate limit",
|
|
65282
|
+
"would exceed your account\u2019s rate limit",
|
|
65283
|
+
"not your account"
|
|
65284
|
+
];
|
|
65285
|
+
function isAccountScopedThrottle(text4) {
|
|
65286
|
+
if (typeof text4 !== "string" || text4.length === 0)
|
|
65287
|
+
return false;
|
|
65288
|
+
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65289
|
+
const lower = sample.toLowerCase();
|
|
65290
|
+
return accountScopedThrottleSignals.some((s) => lower.includes(s));
|
|
65291
|
+
}
|
|
65292
|
+
function classify429Detail(text4) {
|
|
65293
|
+
if (isAccountScopedThrottle(text4))
|
|
65294
|
+
return "account-scoped";
|
|
65295
|
+
if (isLitellmProxyLocal429(text4))
|
|
65296
|
+
return "litellm-local";
|
|
65297
|
+
return "generic-transient";
|
|
65298
|
+
}
|
|
65299
|
+
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT = 5 * 60000;
|
|
65300
|
+
var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
|
|
65301
|
+
function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS) {
|
|
65302
|
+
const last = prev.lastSentAtMsByAccount[account] ?? 0;
|
|
65303
|
+
if (now - last >= cooldownMs) {
|
|
65304
|
+
return {
|
|
65305
|
+
send: true,
|
|
65306
|
+
next: {
|
|
65307
|
+
lastSentAtMsByAccount: {
|
|
65308
|
+
...prev.lastSentAtMsByAccount,
|
|
65309
|
+
[account]: now
|
|
65310
|
+
}
|
|
65311
|
+
}
|
|
65312
|
+
};
|
|
65313
|
+
}
|
|
65314
|
+
return { send: false, next: prev };
|
|
65315
|
+
}
|
|
65316
|
+
function renderThrottleNotice(opts) {
|
|
64937
65317
|
const now = opts.now ?? new Date;
|
|
64938
|
-
const
|
|
64939
|
-
const
|
|
65318
|
+
const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
|
|
65319
|
+
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
64940
65320
|
const lines = [
|
|
64941
|
-
`\
|
|
64942
|
-
`
|
|
64943
|
-
|
|
65321
|
+
`\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
|
|
65322
|
+
`This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
|
|
65323
|
+
`_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
|
|
64944
65324
|
];
|
|
64945
|
-
if (opts.autoFallbackInFlight) {
|
|
64946
|
-
lines.push("_Auto-failover in progress \u2014 see the announcement below._");
|
|
64947
|
-
} else {
|
|
64948
|
-
lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
|
|
64949
|
-
}
|
|
64950
65325
|
return lines.join(`
|
|
64951
65326
|
`);
|
|
64952
65327
|
}
|
|
64953
|
-
function
|
|
64954
|
-
const
|
|
64955
|
-
|
|
64956
|
-
|
|
64957
|
-
|
|
64958
|
-
|
|
64959
|
-
|
|
64960
|
-
|
|
64961
|
-
|
|
64962
|
-
|
|
64963
|
-
|
|
65328
|
+
function renderThrottleEscalationNotice(opts) {
|
|
65329
|
+
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65330
|
+
const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
|
|
65331
|
+
const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
|
|
65332
|
+
return `${head}
|
|
65333
|
+
${tail}`;
|
|
65334
|
+
}
|
|
65335
|
+
|
|
65336
|
+
// operator-events.ts
|
|
65337
|
+
init_format();
|
|
65338
|
+
function classifyClaudeError(raw) {
|
|
65339
|
+
try {
|
|
65340
|
+
return classifyInner(raw);
|
|
65341
|
+
} catch {
|
|
65342
|
+
return "unknown-4xx";
|
|
64964
65343
|
}
|
|
64965
65344
|
}
|
|
64966
|
-
function
|
|
64967
|
-
|
|
64968
|
-
|
|
64969
|
-
|
|
65345
|
+
function classifyInner(raw) {
|
|
65346
|
+
if (raw == null)
|
|
65347
|
+
return "unknown-4xx";
|
|
65348
|
+
const obj = typeof raw === "object" ? raw : {};
|
|
65349
|
+
const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
|
|
65350
|
+
const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
|
|
65351
|
+
const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
|
|
65352
|
+
const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
|
|
65353
|
+
const sdkCode = extractString(obj, "error_code") ?? "";
|
|
65354
|
+
if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
|
|
65355
|
+
const msg = message.toLowerCase();
|
|
65356
|
+
if (msg.includes("expired") || msg.includes("refresh")) {
|
|
65357
|
+
return "credentials-expired";
|
|
65358
|
+
}
|
|
65359
|
+
return "credentials-invalid";
|
|
64970
65360
|
}
|
|
64971
|
-
if (
|
|
64972
|
-
|
|
64973
|
-
return detected?.kind === "quota_exhausted" ? detected : null;
|
|
65361
|
+
if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
|
|
65362
|
+
return "credentials-invalid";
|
|
64974
65363
|
}
|
|
64975
|
-
if (
|
|
64976
|
-
return
|
|
65364
|
+
if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
|
|
65365
|
+
return "credit-exhausted";
|
|
65366
|
+
}
|
|
65367
|
+
if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
|
|
65368
|
+
return "rate-limited";
|
|
64977
65369
|
}
|
|
64978
|
-
|
|
65370
|
+
if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
|
|
65371
|
+
return "rate-limited";
|
|
65372
|
+
}
|
|
65373
|
+
if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
|
|
65374
|
+
return "agent-crashed";
|
|
65375
|
+
}
|
|
65376
|
+
if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
|
|
65377
|
+
return "agent-restarted-unexpectedly";
|
|
65378
|
+
}
|
|
65379
|
+
if (status != null) {
|
|
65380
|
+
if (status >= 400 && status < 500)
|
|
65381
|
+
return "unknown-4xx";
|
|
65382
|
+
if (status >= 500 && status < 600)
|
|
65383
|
+
return "unknown-5xx";
|
|
65384
|
+
}
|
|
65385
|
+
return "unknown-4xx";
|
|
65386
|
+
}
|
|
65387
|
+
function extractString(obj, key) {
|
|
65388
|
+
const v = obj[key];
|
|
65389
|
+
return typeof v === "string" && v.length > 0 ? v : null;
|
|
64979
65390
|
}
|
|
65391
|
+
function extractNumber(obj, key) {
|
|
65392
|
+
const v = obj[key];
|
|
65393
|
+
return typeof v === "number" ? v : null;
|
|
65394
|
+
}
|
|
65395
|
+
function getNestedObj(obj, key) {
|
|
65396
|
+
const v = obj[key];
|
|
65397
|
+
return typeof v === "object" && v != null ? v : {};
|
|
65398
|
+
}
|
|
65399
|
+
var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
|
|
65400
|
+
var cooldownMap2 = new Map;
|
|
64980
65401
|
|
|
64981
|
-
//
|
|
64982
|
-
|
|
64983
|
-
|
|
65402
|
+
// llm-error-present.ts
|
|
65403
|
+
function extractModel(raw) {
|
|
65404
|
+
const m = raw.match(/["']?model["']?\s*[=:]\s*["']?((?:claude|sr)[A-Za-z0-9._-]+)/i);
|
|
65405
|
+
return m ? m[1] : undefined;
|
|
65406
|
+
}
|
|
65407
|
+
var TRANSIENT_KINDS = new Set([
|
|
65408
|
+
"rate_limit",
|
|
65409
|
+
"overload_529",
|
|
65410
|
+
"transient"
|
|
65411
|
+
]);
|
|
65412
|
+
function isActionableKind(kind) {
|
|
65413
|
+
return kind === "auth" || kind === "quota_wall";
|
|
65414
|
+
}
|
|
65415
|
+
function parseLlmError(raw, retryState) {
|
|
65416
|
+
const text4 = typeof raw === "string" ? raw : "";
|
|
65417
|
+
const requestId = extractRequestId(text4);
|
|
65418
|
+
const model = extractModel(text4);
|
|
65419
|
+
const resetAt = parseResetTime(text4);
|
|
65420
|
+
const retryAfterMs = resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined;
|
|
65421
|
+
const { kind, source } = classifyKindAndSource(text4);
|
|
65422
|
+
let autoRetrying = false;
|
|
65423
|
+
let terminal = true;
|
|
65424
|
+
if (TRANSIENT_KINDS.has(kind)) {
|
|
65425
|
+
const { retryAttempt, maxRetries } = retryState ?? { retryAttempt: null, maxRetries: null };
|
|
65426
|
+
if (retryAttempt != null && maxRetries != null) {
|
|
65427
|
+
autoRetrying = retryAttempt < maxRetries;
|
|
65428
|
+
terminal = retryAttempt >= maxRetries;
|
|
65429
|
+
} else {
|
|
65430
|
+
autoRetrying = false;
|
|
65431
|
+
terminal = true;
|
|
65432
|
+
}
|
|
65433
|
+
}
|
|
65434
|
+
return {
|
|
65435
|
+
kind,
|
|
65436
|
+
coreText: buildCoreText(kind, source),
|
|
65437
|
+
...resetAt != null ? { resetAt } : {},
|
|
65438
|
+
...retryAfterMs != null ? { retryAfterMs } : {},
|
|
65439
|
+
...model != null ? { model } : {},
|
|
65440
|
+
...requestId != null ? { requestId } : {},
|
|
65441
|
+
source,
|
|
65442
|
+
autoRetrying,
|
|
65443
|
+
terminal
|
|
65444
|
+
};
|
|
65445
|
+
}
|
|
65446
|
+
function classifyKindAndSource(text4) {
|
|
65447
|
+
const lower = text4.toLowerCase();
|
|
65448
|
+
const claudeKind = classifyClaudeError({ message: text4, type: text4 });
|
|
65449
|
+
if (claudeKind === "credentials-expired" || claudeKind === "credentials-invalid") {
|
|
65450
|
+
return { kind: "auth", source: "anthropic" };
|
|
65451
|
+
}
|
|
65452
|
+
if (isLitellmProxyLocal429(text4)) {
|
|
65453
|
+
return { kind: "rate_limit", source: "litellm-local" };
|
|
65454
|
+
}
|
|
65455
|
+
const mu = detectModelUnavailable(text4);
|
|
65456
|
+
if (mu != null) {
|
|
65457
|
+
if (mu.kind === "quota_exhausted")
|
|
65458
|
+
return { kind: "quota_wall", source: "anthropic" };
|
|
65459
|
+
if (mu.kind === "network")
|
|
65460
|
+
return { kind: "transient", source: "network" };
|
|
65461
|
+
if (lower.includes("529") || lower.includes("overloaded")) {
|
|
65462
|
+
return { kind: "overload_529", source: "anthropic" };
|
|
65463
|
+
}
|
|
65464
|
+
const c = classify429Detail(text4);
|
|
65465
|
+
return {
|
|
65466
|
+
kind: "rate_limit",
|
|
65467
|
+
source: c === "litellm-local" ? "litellm-local" : "anthropic"
|
|
65468
|
+
};
|
|
65469
|
+
}
|
|
65470
|
+
if (claudeKind === "credit-exhausted") {
|
|
65471
|
+
return { kind: "quota_wall", source: "anthropic" };
|
|
65472
|
+
}
|
|
65473
|
+
if (claudeKind === "rate-limited") {
|
|
65474
|
+
return { kind: "rate_limit", source: "anthropic" };
|
|
65475
|
+
}
|
|
65476
|
+
if (claudeKind === "unknown-5xx") {
|
|
65477
|
+
return { kind: "overload_529", source: "anthropic" };
|
|
65478
|
+
}
|
|
65479
|
+
return { kind: "unknown", source: "anthropic" };
|
|
65480
|
+
}
|
|
65481
|
+
function buildCoreText(kind, source) {
|
|
65482
|
+
switch (kind) {
|
|
65483
|
+
case "rate_limit":
|
|
65484
|
+
return source === "litellm-local" ? "Hit the local proxy rate limit \u2014 retrying automatically." : "Rate limited by Anthropic \u2014 retrying automatically.";
|
|
65485
|
+
case "overload_529":
|
|
65486
|
+
return "Anthropic is overloaded (529) \u2014 retrying automatically.";
|
|
65487
|
+
case "quota_wall":
|
|
65488
|
+
return "Usage limit reached on this Claude subscription.";
|
|
65489
|
+
case "auth":
|
|
65490
|
+
return "Claude login needs re-authentication.";
|
|
65491
|
+
case "transient":
|
|
65492
|
+
return source === "network" ? "Couldn't reach Anthropic (network) \u2014 retrying automatically." : "A temporary upstream hiccup \u2014 retrying automatically.";
|
|
65493
|
+
case "unknown":
|
|
65494
|
+
return "The model returned an error.";
|
|
65495
|
+
}
|
|
65496
|
+
}
|
|
65497
|
+
function formatResetLocal(resetAt, tz, now = new Date) {
|
|
65498
|
+
if (resetAt == null)
|
|
65499
|
+
return "";
|
|
65500
|
+
const ms = resetAt.getTime();
|
|
65501
|
+
if (!Number.isFinite(ms))
|
|
65502
|
+
return "";
|
|
65503
|
+
const clock = fmtLocalClock(ms, tz);
|
|
65504
|
+
const abbrev = tzAbbrev(ms, tz);
|
|
65505
|
+
const rel = formatRelativeTail(ms - now.getTime());
|
|
65506
|
+
return rel ? `clears ~${clock} ${abbrev} (${rel})` : `clears ~${clock} ${abbrev}`;
|
|
65507
|
+
}
|
|
65508
|
+
function formatRelativeTail(deltaMs) {
|
|
65509
|
+
if (deltaMs <= 0)
|
|
65510
|
+
return "~now";
|
|
65511
|
+
const totalMin = Math.round(deltaMs / 60000);
|
|
65512
|
+
if (totalMin < 1)
|
|
65513
|
+
return "~in <1m";
|
|
65514
|
+
if (totalMin < 60)
|
|
65515
|
+
return `~in ${totalMin}m`;
|
|
65516
|
+
const hours = Math.floor(totalMin / 60);
|
|
65517
|
+
const mins = totalMin % 60;
|
|
65518
|
+
if (hours < 24)
|
|
65519
|
+
return mins > 0 ? `~in ${hours}h ${mins}m` : `~in ${hours}h`;
|
|
65520
|
+
const days = Math.floor(hours / 24);
|
|
65521
|
+
const remH = hours % 24;
|
|
65522
|
+
return remH > 0 ? `~in ${days}d ${remH}h` : `~in ${days}d`;
|
|
65523
|
+
}
|
|
65524
|
+
function formatResetClock(resetAt, tz) {
|
|
65525
|
+
if (resetAt == null)
|
|
65526
|
+
return "";
|
|
65527
|
+
const ms = resetAt.getTime();
|
|
65528
|
+
if (!Number.isFinite(ms))
|
|
65529
|
+
return "";
|
|
65530
|
+
return `${fmtLocalClock(ms, tz)} ${tzAbbrev(ms, tz)}`;
|
|
65531
|
+
}
|
|
65532
|
+
function buildRecommendation(parsed, tz) {
|
|
65533
|
+
switch (parsed.kind) {
|
|
65534
|
+
case "auth":
|
|
65535
|
+
return "\u2192 Re-authenticate this account to continue.";
|
|
65536
|
+
case "quota_wall": {
|
|
65537
|
+
const reset2 = formatResetClock(parsed.resetAt, tz);
|
|
65538
|
+
return reset2 ? `\u2192 Switch to another account, or wait for the quota to reset at ${reset2}.` : "\u2192 Switch to another account, or wait for the quota to reset.";
|
|
65539
|
+
}
|
|
65540
|
+
default:
|
|
65541
|
+
return;
|
|
65542
|
+
}
|
|
65543
|
+
}
|
|
65544
|
+
function renderLlmError(parsed, agent, tz, now = new Date) {
|
|
65545
|
+
const safeAgent = escapeAgent(agent);
|
|
65546
|
+
const emoji = kindEmoji(parsed.kind);
|
|
65547
|
+
const lines = [`${emoji} ${parsed.coreText} (**${safeAgent}**)`];
|
|
65548
|
+
const resetLine = formatResetLocal(parsed.resetAt, tz, now);
|
|
65549
|
+
if (resetLine)
|
|
65550
|
+
lines.push(`_${resetLine}_`);
|
|
65551
|
+
if (parsed.model)
|
|
65552
|
+
lines.push(`_model: ${escapeAgent(parsed.model)}_`);
|
|
65553
|
+
const recommendation2 = buildRecommendation(parsed, tz);
|
|
65554
|
+
if (recommendation2)
|
|
65555
|
+
lines.push(recommendation2);
|
|
65556
|
+
return { text: lines.join(`
|
|
65557
|
+
`) };
|
|
65558
|
+
}
|
|
65559
|
+
function renderLlmErrorSafe(parsed, agent, tz, now = new Date) {
|
|
65560
|
+
try {
|
|
65561
|
+
return renderLlmError(parsed, agent, tz, now);
|
|
65562
|
+
} catch {
|
|
65563
|
+
const safeAgent = escapeAgent(agent);
|
|
65564
|
+
return { text: `${kindEmoji(parsed.kind)} ${parsed.coreText} (**${safeAgent}**)` };
|
|
65565
|
+
}
|
|
65566
|
+
}
|
|
65567
|
+
function kindEmoji(kind) {
|
|
65568
|
+
switch (kind) {
|
|
65569
|
+
case "rate_limit":
|
|
65570
|
+
return "\uD83D\uDEA6";
|
|
65571
|
+
case "overload_529":
|
|
65572
|
+
return "\uD83D\uDD25";
|
|
65573
|
+
case "quota_wall":
|
|
65574
|
+
return "\u26a0\ufe0f";
|
|
65575
|
+
case "auth":
|
|
65576
|
+
return "\uD83D\uDD11";
|
|
65577
|
+
case "transient":
|
|
65578
|
+
return "\uD83C\uDF10";
|
|
65579
|
+
case "unknown":
|
|
65580
|
+
return "\u26a0\ufe0f";
|
|
65581
|
+
}
|
|
65582
|
+
}
|
|
65583
|
+
function escapeAgent(s) {
|
|
65584
|
+
return s.replace(/([_*`\[\]])/g, "\\$1");
|
|
65585
|
+
}
|
|
65586
|
+
var ERROR_COLLAPSE_WINDOW_MS = 60000;
|
|
65587
|
+
|
|
65588
|
+
class ErrorPresenceGate {
|
|
65589
|
+
claims = new Map;
|
|
65590
|
+
keyFor(parsed, agent, now) {
|
|
65591
|
+
if (parsed.requestId)
|
|
65592
|
+
return `rid:${parsed.requestId}`;
|
|
65593
|
+
const bucket = Math.floor(now / ERROR_COLLAPSE_WINDOW_MS);
|
|
65594
|
+
return `${parsed.kind}:${agent}:${bucket}`;
|
|
65595
|
+
}
|
|
65596
|
+
claim(key, now = Date.now()) {
|
|
65597
|
+
this.prune(now);
|
|
65598
|
+
const existing = this.claims.get(key);
|
|
65599
|
+
if (existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS) {
|
|
65600
|
+
return false;
|
|
65601
|
+
}
|
|
65602
|
+
this.claims.set(key, now);
|
|
65603
|
+
return true;
|
|
65604
|
+
}
|
|
65605
|
+
isClaimed(key, now = Date.now()) {
|
|
65606
|
+
const existing = this.claims.get(key);
|
|
65607
|
+
return existing != null && now - existing < ERROR_COLLAPSE_WINDOW_MS;
|
|
65608
|
+
}
|
|
65609
|
+
prune(now) {
|
|
65610
|
+
for (const [k, at] of this.claims) {
|
|
65611
|
+
if (now - at >= ERROR_COLLAPSE_WINDOW_MS)
|
|
65612
|
+
this.claims.delete(k);
|
|
65613
|
+
}
|
|
65614
|
+
}
|
|
65615
|
+
reset() {
|
|
65616
|
+
this.claims.clear();
|
|
65617
|
+
}
|
|
65618
|
+
}
|
|
65619
|
+
var errorPresenceGate = new ErrorPresenceGate;
|
|
65620
|
+
function decideErrorSurface(parsed, agent, opts = { claim: true }) {
|
|
65621
|
+
const now = opts.now ?? Date.now();
|
|
65622
|
+
const gate = opts.gate ?? errorPresenceGate;
|
|
65623
|
+
if (isActionableKind(parsed.kind)) {
|
|
65624
|
+
if (opts.claim)
|
|
65625
|
+
gate.claim(gate.keyFor(parsed, agent, now), now);
|
|
65626
|
+
return "render";
|
|
65627
|
+
}
|
|
65628
|
+
if (parsed.autoRetrying && !parsed.terminal)
|
|
65629
|
+
return "suppress";
|
|
65630
|
+
const key = gate.keyFor(parsed, agent, now);
|
|
65631
|
+
if (opts.claim) {
|
|
65632
|
+
return gate.claim(key, now) ? "render" : "suppress";
|
|
65633
|
+
}
|
|
65634
|
+
return gate.isClaimed(key, now) ? "suppress" : "render";
|
|
65635
|
+
}
|
|
64984
65636
|
|
|
64985
65637
|
// model-unavailable.ts
|
|
64986
65638
|
init_quota_check();
|
|
@@ -64995,13 +65647,6 @@ var transientUpstreamSignals2 = [
|
|
|
64995
65647
|
"would exceed your account\u2019s rate limit",
|
|
64996
65648
|
"would exceed your account's rate limit"
|
|
64997
65649
|
];
|
|
64998
|
-
function isTransientUpstreamSignal(text4) {
|
|
64999
|
-
if (typeof text4 !== "string" || text4.length === 0)
|
|
65000
|
-
return false;
|
|
65001
|
-
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65002
|
-
const lower = sample.toLowerCase();
|
|
65003
|
-
return transientUpstreamSignals2.some((s) => lower.includes(s));
|
|
65004
|
-
}
|
|
65005
65650
|
var litellmProxyLocal429Signals2 = [
|
|
65006
65651
|
"deployment over user-defined ratelimit",
|
|
65007
65652
|
"model rate limit exceeded. tpm limit",
|
|
@@ -65022,55 +65667,76 @@ function isLitellmProxyLocal4292(text4) {
|
|
|
65022
65667
|
return true;
|
|
65023
65668
|
return litellmV3LimiterSignalPair2.every((s) => lower.includes(s));
|
|
65024
65669
|
}
|
|
65025
|
-
function
|
|
65026
|
-
|
|
65027
|
-
|
|
65028
|
-
|
|
65029
|
-
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65670
|
+
function detectModelUnavailable2(stderr) {
|
|
65671
|
+
if (typeof stderr !== "string" || stderr.length === 0)
|
|
65672
|
+
return null;
|
|
65673
|
+
const sample = stderr.length > 16384 ? stderr.slice(0, 16384) : stderr;
|
|
65030
65674
|
const lower = sample.toLowerCase();
|
|
65031
|
-
|
|
65032
|
-
|
|
65033
|
-
|
|
65034
|
-
if (eqLimit) {
|
|
65035
|
-
limitType = eqLimit[1];
|
|
65036
|
-
limit = Number(eqLimit[2]);
|
|
65675
|
+
if (transientUpstreamSignals2.some((s) => lower.includes(s))) {
|
|
65676
|
+
const resetAt = parseResetTime2(sample);
|
|
65677
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
65037
65678
|
}
|
|
65038
|
-
if (
|
|
65039
|
-
const
|
|
65040
|
-
|
|
65041
|
-
limitType = v3Type[1];
|
|
65679
|
+
if (isLitellmProxyLocal4292(sample)) {
|
|
65680
|
+
const resetAt = parseResetTime2(sample);
|
|
65681
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
65042
65682
|
}
|
|
65043
|
-
|
|
65044
|
-
|
|
65045
|
-
|
|
65046
|
-
|
|
65683
|
+
const quotaSignals = [
|
|
65684
|
+
"out of extra usage",
|
|
65685
|
+
"extra usage",
|
|
65686
|
+
"credit_balance_too_low",
|
|
65687
|
+
"credit balance too low",
|
|
65688
|
+
"usage limit",
|
|
65689
|
+
"usage_limit",
|
|
65690
|
+
"quota exhausted",
|
|
65691
|
+
"quota_exhausted",
|
|
65692
|
+
"plan limit",
|
|
65693
|
+
"subscription limit",
|
|
65694
|
+
"hit your limit",
|
|
65695
|
+
"hit the limit",
|
|
65696
|
+
"session limit",
|
|
65697
|
+
"session cap"
|
|
65698
|
+
];
|
|
65699
|
+
if (quotaSignals.some((s) => lower.includes(s))) {
|
|
65700
|
+
const resetAt = parseResetTime2(sample);
|
|
65701
|
+
return resetAt !== undefined ? { kind: "quota_exhausted", resetAt, raw: stderr } : { kind: "quota_exhausted", raw: stderr };
|
|
65047
65702
|
}
|
|
65048
|
-
|
|
65049
|
-
|
|
65050
|
-
|
|
65051
|
-
|
|
65052
|
-
|
|
65053
|
-
|
|
65054
|
-
|
|
65055
|
-
|
|
65056
|
-
|
|
65057
|
-
|
|
65703
|
+
const overloadSignals = [
|
|
65704
|
+
"overloaded_error",
|
|
65705
|
+
"overloaded",
|
|
65706
|
+
"rate_limit_error",
|
|
65707
|
+
"rate limit",
|
|
65708
|
+
"rate-limited",
|
|
65709
|
+
"http 429",
|
|
65710
|
+
'"status":429',
|
|
65711
|
+
"status: 429",
|
|
65712
|
+
" 429 ",
|
|
65713
|
+
"503 service",
|
|
65714
|
+
"service unavailable",
|
|
65715
|
+
'"status":529',
|
|
65716
|
+
"http 529",
|
|
65717
|
+
" 529 "
|
|
65718
|
+
];
|
|
65719
|
+
if (overloadSignals.some((s) => lower.includes(s))) {
|
|
65720
|
+
const resetAt = parseResetTime2(sample);
|
|
65721
|
+
return resetAt !== undefined ? { kind: "overload", resetAt, raw: stderr } : { kind: "overload", raw: stderr };
|
|
65058
65722
|
}
|
|
65059
|
-
|
|
65060
|
-
|
|
65061
|
-
|
|
65062
|
-
|
|
65063
|
-
|
|
65064
|
-
|
|
65065
|
-
|
|
65066
|
-
|
|
65723
|
+
const networkSignals = [
|
|
65724
|
+
"econnrefused",
|
|
65725
|
+
"econnreset",
|
|
65726
|
+
"etimedout",
|
|
65727
|
+
"enotfound",
|
|
65728
|
+
"eai_again",
|
|
65729
|
+
"fetch failed",
|
|
65730
|
+
"network error",
|
|
65731
|
+
"socket hang up",
|
|
65732
|
+
"request timed out",
|
|
65733
|
+
"connection refused",
|
|
65734
|
+
"getaddrinfo"
|
|
65735
|
+
];
|
|
65736
|
+
if (networkSignals.some((s) => lower.includes(s))) {
|
|
65737
|
+
return { kind: "network", raw: stderr };
|
|
65067
65738
|
}
|
|
65068
|
-
return
|
|
65069
|
-
limitType,
|
|
65070
|
-
limit: limit != null && Number.isFinite(limit) ? limit : null,
|
|
65071
|
-
currentUsage: currentUsage != null && Number.isFinite(currentUsage) ? currentUsage : null,
|
|
65072
|
-
resetAtMs
|
|
65073
|
-
};
|
|
65739
|
+
return null;
|
|
65074
65740
|
}
|
|
65075
65741
|
function parseResetTime2(text4, parseTimeNow = new Date) {
|
|
65076
65742
|
const lower = text4.toLowerCase();
|
|
@@ -65194,31 +65860,77 @@ function parseRelativeDuration2(s) {
|
|
|
65194
65860
|
}
|
|
65195
65861
|
return matched && total > 0 ? total : null;
|
|
65196
65862
|
}
|
|
65863
|
+
function formatModelUnavailableCard(detection, agent, opts = {}) {
|
|
65864
|
+
const now = opts.now ?? new Date;
|
|
65865
|
+
const slotPart = opts.slot ? ` (slot **${escapeMarkdown(opts.slot)}**)` : "";
|
|
65866
|
+
const reason = formatReason(detection, now);
|
|
65867
|
+
const lines = [
|
|
65868
|
+
`\u26a0\ufe0f **Model unavailable** on agent **${escapeMarkdown(agent)}**${slotPart}`,
|
|
65869
|
+
`Reason: ${reason}`,
|
|
65870
|
+
""
|
|
65871
|
+
];
|
|
65872
|
+
if (opts.autoFallbackInFlight) {
|
|
65873
|
+
lines.push("_Auto-failover in progress \u2014 see the announcement below._");
|
|
65874
|
+
} else {
|
|
65875
|
+
lines.push("**What to try**", "\u2022 `/auth use <label>` \u2014 switch the fleet to a healthy account", "\u2022 `/auth add` \u2014 attach another subscription", "\u2022 `/usage` \u2014 show quota breakdown");
|
|
65876
|
+
}
|
|
65877
|
+
return lines.join(`
|
|
65878
|
+
`);
|
|
65879
|
+
}
|
|
65880
|
+
function formatReason(d, now) {
|
|
65881
|
+
const reset2 = d.resetAt ? ` (${formatResetRelative(d.resetAt, now)})` : "";
|
|
65882
|
+
switch (d.kind) {
|
|
65883
|
+
case "quota_exhausted":
|
|
65884
|
+
return `quota exhausted${reset2}`;
|
|
65885
|
+
case "overload":
|
|
65886
|
+
return `model overloaded${reset2}`;
|
|
65887
|
+
case "rate_limited":
|
|
65888
|
+
return `account rate-limited${reset2}`;
|
|
65889
|
+
case "network":
|
|
65890
|
+
return "network unreachable";
|
|
65891
|
+
}
|
|
65892
|
+
}
|
|
65893
|
+
function resolveModelUnavailableFromOperatorEvent(ev) {
|
|
65894
|
+
const detail = typeof ev.detail === "string" ? ev.detail : "";
|
|
65895
|
+
if (ev.kind === "quota-exhausted") {
|
|
65896
|
+
return detectModelUnavailable2(detail) ?? { kind: "quota_exhausted", raw: detail };
|
|
65897
|
+
}
|
|
65898
|
+
if (ev.kind === "rate-limited") {
|
|
65899
|
+
const detected = detectModelUnavailable2(detail);
|
|
65900
|
+
return detected?.kind === "quota_exhausted" ? detected : null;
|
|
65901
|
+
}
|
|
65902
|
+
if (ev.kind === "unknown-5xx") {
|
|
65903
|
+
return detectModelUnavailable2(detail) ?? { kind: "overload", raw: detail };
|
|
65904
|
+
}
|
|
65905
|
+
return detectModelUnavailable2(detail);
|
|
65906
|
+
}
|
|
65197
65907
|
|
|
65198
65908
|
// throttle-tier.ts
|
|
65199
|
-
|
|
65909
|
+
init_card_format();
|
|
65910
|
+
init_quota_check();
|
|
65911
|
+
var accountScopedThrottleSignals2 = [
|
|
65200
65912
|
"would exceed your account's rate limit",
|
|
65201
65913
|
"would exceed your account\u2019s rate limit",
|
|
65202
65914
|
"not your account"
|
|
65203
65915
|
];
|
|
65204
|
-
function
|
|
65916
|
+
function isAccountScopedThrottle2(text4) {
|
|
65205
65917
|
if (typeof text4 !== "string" || text4.length === 0)
|
|
65206
65918
|
return false;
|
|
65207
65919
|
const sample = text4.length > 16384 ? text4.slice(0, 16384) : text4;
|
|
65208
65920
|
const lower = sample.toLowerCase();
|
|
65209
|
-
return
|
|
65921
|
+
return accountScopedThrottleSignals2.some((s) => lower.includes(s));
|
|
65210
65922
|
}
|
|
65211
|
-
function
|
|
65212
|
-
if (
|
|
65923
|
+
function classify429Detail2(text4) {
|
|
65924
|
+
if (isAccountScopedThrottle2(text4))
|
|
65213
65925
|
return "account-scoped";
|
|
65214
|
-
if (
|
|
65926
|
+
if (isLitellmProxyLocal429(text4))
|
|
65215
65927
|
return "litellm-local";
|
|
65216
65928
|
return "generic-transient";
|
|
65217
65929
|
}
|
|
65218
65930
|
function build429ClassifiedMetric(opts) {
|
|
65219
65931
|
const detail = typeof opts.detail === "string" ? opts.detail : "";
|
|
65220
65932
|
const litellm = parseLitellmLimitDetail(detail, new Date(opts.now));
|
|
65221
|
-
const anthropicResetMs =
|
|
65933
|
+
const anthropicResetMs = parseResetTime(detail, new Date(opts.now))?.getTime() ?? null;
|
|
65222
65934
|
const resetAtMs = anthropicResetMs ?? litellm.resetAtMs;
|
|
65223
65935
|
return {
|
|
65224
65936
|
kind: "rate_limit_429_classified",
|
|
@@ -65232,20 +65944,20 @@ function build429ClassifiedMetric(opts) {
|
|
|
65232
65944
|
current_usage: litellm.currentUsage
|
|
65233
65945
|
};
|
|
65234
65946
|
}
|
|
65235
|
-
var
|
|
65947
|
+
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
|
|
65236
65948
|
var THROTTLE_DEFAULT_WAIT_MS = 60000;
|
|
65237
65949
|
function throttleRetryInPlaceMaxMs(env = process.env) {
|
|
65238
65950
|
const raw = env.SWITCHROOM_THROTTLE_RETRY_IN_PLACE_MAX_MS;
|
|
65239
65951
|
if (raw == null || raw === "")
|
|
65240
|
-
return
|
|
65952
|
+
return THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
|
|
65241
65953
|
const n = Number(raw);
|
|
65242
|
-
return Number.isFinite(n) && n > 0 ? n :
|
|
65954
|
+
return Number.isFinite(n) && n > 0 ? n : THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2;
|
|
65243
65955
|
}
|
|
65244
65956
|
function decideThrottleTier(opts) {
|
|
65245
65957
|
const { detail, now, thresholdMs } = opts;
|
|
65246
|
-
if (!
|
|
65958
|
+
if (!isAccountScopedThrottle2(detail))
|
|
65247
65959
|
return { action: "none" };
|
|
65248
|
-
const resetAt =
|
|
65960
|
+
const resetAt = parseResetTime(detail, new Date(now));
|
|
65249
65961
|
const resetAtMs = resetAt?.getTime();
|
|
65250
65962
|
if (resetAtMs == null || !Number.isFinite(resetAtMs) || resetAtMs <= now) {
|
|
65251
65963
|
return {
|
|
@@ -65259,47 +65971,7 @@ function decideThrottleTier(opts) {
|
|
|
65259
65971
|
}
|
|
65260
65972
|
return { action: "failover", resetAtMs };
|
|
65261
65973
|
}
|
|
65262
|
-
var THROTTLE_NOTICE_COOLDOWN_MS = 10 * 60000;
|
|
65263
|
-
|
|
65264
|
-
// throttle-tier.ts
|
|
65265
|
-
init_card_format();
|
|
65266
|
-
init_quota_check();
|
|
65267
|
-
var THROTTLE_RETRY_IN_PLACE_MAX_MS_DEFAULT2 = 5 * 60000;
|
|
65268
65974
|
var THROTTLE_NOTICE_COOLDOWN_MS2 = 10 * 60000;
|
|
65269
|
-
function evaluateThrottleNotice(prev, account, now, cooldownMs = THROTTLE_NOTICE_COOLDOWN_MS2) {
|
|
65270
|
-
const last = prev.lastSentAtMsByAccount[account] ?? 0;
|
|
65271
|
-
if (now - last >= cooldownMs) {
|
|
65272
|
-
return {
|
|
65273
|
-
send: true,
|
|
65274
|
-
next: {
|
|
65275
|
-
lastSentAtMsByAccount: {
|
|
65276
|
-
...prev.lastSentAtMsByAccount,
|
|
65277
|
-
[account]: now
|
|
65278
|
-
}
|
|
65279
|
-
}
|
|
65280
|
-
};
|
|
65281
|
-
}
|
|
65282
|
-
return { send: false, next: prev };
|
|
65283
|
-
}
|
|
65284
|
-
function renderThrottleNotice(opts) {
|
|
65285
|
-
const now = opts.now ?? new Date;
|
|
65286
|
-
const resetStr = formatResetRelative(new Date(opts.throttledUntilMs), now);
|
|
65287
|
-
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65288
|
-
const lines = [
|
|
65289
|
-
`\uD83D\uDEA6 **Rate-limited, staying put** \u2014 ${acct} hit a transient rate limit on **${escapeMarkdown(opts.agent)}**.`,
|
|
65290
|
-
`This is a short throttle, not a quota wall \u2014 ${opts.resetParsed ? resetStr : `no reset given, retrying in ~60s`}.`,
|
|
65291
|
-
`_Staying on ${acct}; no failover needed. The turn retries automatically after the reset._`
|
|
65292
|
-
];
|
|
65293
|
-
return lines.join(`
|
|
65294
|
-
`);
|
|
65295
|
-
}
|
|
65296
|
-
function renderThrottleEscalationNotice(opts) {
|
|
65297
|
-
const acct = opts.account ? `\`${escapeMarkdown(opts.account)}\`` : "the active account";
|
|
65298
|
-
const head = `\u26d4\ufe0f **Rate limit was actually a wall** \u2014 repeated 429s on ${acct} ` + `(trigger: **${escapeMarkdown(opts.agent)}**) were corroborated by a live quota probe.`;
|
|
65299
|
-
const tail = opts.rolledTo ? `Marked exhausted and rolled to \`${escapeMarkdown(opts.rolledTo)}\`.` : `Marked exhausted \u2014 no fallback account had quota (all blocked). ` + `Use \`/auth add <label>\` to attach another subscription.`;
|
|
65300
|
-
return `${head}
|
|
65301
|
-
${tail}`;
|
|
65302
|
-
}
|
|
65303
65975
|
|
|
65304
65976
|
// gateway/throttle-tier-wiring.ts
|
|
65305
65977
|
var THROTTLE_RETRY_NUDGE_SLACK_MS = 5000;
|
|
@@ -65320,7 +65992,7 @@ function createThrottleTierRunner(deps) {
|
|
|
65320
65992
|
let granted = true;
|
|
65321
65993
|
if (client3 && account) {
|
|
65322
65994
|
try {
|
|
65323
|
-
granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`,
|
|
65995
|
+
granted = (await client3.claimNotification(`${keyPrefix}:${account}:${chatId}`, THROTTLE_NOTICE_COOLDOWN_MS)).granted;
|
|
65324
65996
|
} catch {
|
|
65325
65997
|
granted = true;
|
|
65326
65998
|
}
|
|
@@ -65851,100 +66523,28 @@ function hardenCardBreaks2(text4) {
|
|
|
65851
66523
|
}
|
|
65852
66524
|
return restore2(pieces.join(""));
|
|
65853
66525
|
}
|
|
65854
|
-
var PARAGRAPH_SPACER2 = "\u00a0";
|
|
65855
|
-
function addParagraphSpacers2(text4) {
|
|
65856
|
-
if (!text4.includes(`
|
|
65857
|
-
|
|
65858
|
-
`))
|
|
65859
|
-
return text4;
|
|
65860
|
-
const nonce = Math.random().toString(36).slice(2);
|
|
65861
|
-
const { masked, restore: restore2, placeholder } = maskCodeRegions2(text4, nonce);
|
|
65862
|
-
if (!masked.includes(`
|
|
65863
|
-
|
|
65864
|
-
`))
|
|
65865
|
-
return restore2(masked);
|
|
65866
|
-
const spacerLine = PARAGRAPH_SPACER2;
|
|
65867
|
-
const isBlankLine = (line) => /^[ \t\r\f\v]*$/.test(line);
|
|
65868
|
-
const asciiTrim = (line) => line.replace(/^[ \t\r\f\v]+|[ \t\r\f\v]+$/g, "");
|
|
65869
|
-
const blockKind = (line) => {
|
|
65870
|
-
if (asciiTrim(line) === spacerLine)
|
|
65871
|
-
return "spacer";
|
|
65872
|
-
if (isFenceOpenLine2(line, placeholder))
|
|
65873
|
-
return "fence";
|
|
65874
|
-
if (isListItemLine2(line))
|
|
65875
|
-
return "list";
|
|
65876
|
-
if (isTableRowLine2(line) || isTableDelimiterLine2(line))
|
|
65877
|
-
return "table";
|
|
65878
|
-
if (isBlockquoteLine2(line))
|
|
65879
|
-
return "quote";
|
|
65880
|
-
if (isHeadingLine2(line))
|
|
65881
|
-
return "heading";
|
|
65882
|
-
if (/^(-{3,}|\*{3,}|_{3,})\s*$/.test(line.trimStart()))
|
|
65883
|
-
return "divider";
|
|
65884
|
-
return "prose";
|
|
65885
|
-
};
|
|
65886
|
-
const SAME_KIND_TIGHT = new Set([
|
|
65887
|
-
"list",
|
|
65888
|
-
"table",
|
|
65889
|
-
"quote",
|
|
65890
|
-
"fence",
|
|
65891
|
-
"divider"
|
|
65892
|
-
]);
|
|
65893
|
-
const shouldSpaceGap = (above, below) => {
|
|
65894
|
-
const a = blockKind(above);
|
|
65895
|
-
const b = blockKind(below);
|
|
65896
|
-
if (a === "spacer" || b === "spacer")
|
|
65897
|
-
return false;
|
|
65898
|
-
if (a === b && SAME_KIND_TIGHT.has(a))
|
|
65899
|
-
return false;
|
|
65900
|
-
return true;
|
|
65901
|
-
};
|
|
65902
|
-
const lines = masked.split(`
|
|
65903
|
-
`);
|
|
65904
|
-
const out = [];
|
|
65905
|
-
for (let i = 0;i < lines.length; i++) {
|
|
65906
|
-
const line = lines[i];
|
|
65907
|
-
const isBlank = isBlankLine(line);
|
|
65908
|
-
if (isBlank) {
|
|
65909
|
-
const prevEmitted = out.length > 0 ? out[out.length - 1] : null;
|
|
65910
|
-
const prevIsBlank = prevEmitted != null && isBlankLine(prevEmitted);
|
|
65911
|
-
if (!prevIsBlank) {
|
|
65912
|
-
const above = lastNonBlank2(out, isBlankLine);
|
|
65913
|
-
const below = nextNonBlank2(lines, i + 1, isBlankLine);
|
|
65914
|
-
const alreadySpaced = above != null && asciiTrim(above) === spacerLine || below != null && asciiTrim(below) === spacerLine;
|
|
65915
|
-
if (!alreadySpaced && above != null && below != null && shouldSpaceGap(above, below)) {
|
|
65916
|
-
out.push("");
|
|
65917
|
-
out.push(spacerLine);
|
|
65918
|
-
out.push("");
|
|
65919
|
-
continue;
|
|
65920
|
-
}
|
|
65921
|
-
}
|
|
65922
|
-
}
|
|
65923
|
-
out.push(line);
|
|
65924
|
-
}
|
|
65925
|
-
return restore2(out.join(`
|
|
65926
|
-
`));
|
|
65927
|
-
}
|
|
65928
|
-
function lastNonBlank2(arr, isBlank) {
|
|
65929
|
-
for (let i = arr.length - 1;i >= 0; i--) {
|
|
65930
|
-
if (!isBlank(arr[i]))
|
|
65931
|
-
return arr[i];
|
|
65932
|
-
}
|
|
65933
|
-
return null;
|
|
65934
|
-
}
|
|
65935
|
-
function nextNonBlank2(lines, from, isBlank) {
|
|
65936
|
-
for (let i = from;i < lines.length; i++) {
|
|
65937
|
-
if (!isBlank(lines[i]))
|
|
65938
|
-
return lines[i];
|
|
65939
|
-
}
|
|
65940
|
-
return null;
|
|
65941
|
-
}
|
|
65942
66526
|
function normalizePunctuation2(text4) {
|
|
65943
66527
|
if (!/[\u2014\u2013\u2022\u00b7]/.test(text4))
|
|
65944
66528
|
return text4;
|
|
65945
66529
|
const nonce = Math.random().toString(36).slice(2);
|
|
65946
66530
|
const { masked, restore: restore2 } = maskCodeRegions2(text4, nonce);
|
|
65947
|
-
|
|
66531
|
+
const linkMasks = [];
|
|
66532
|
+
const LINK_MASK_PH = `\x00RML${nonce}_`;
|
|
66533
|
+
const maskedLinks = masked.replace(/(\]\()([^)\n]*)(\))/g, (_m, open, href, close) => {
|
|
66534
|
+
const idx = linkMasks.length;
|
|
66535
|
+
linkMasks.push(href);
|
|
66536
|
+
return `${open}${LINK_MASK_PH}${idx}\x00${close}`;
|
|
66537
|
+
});
|
|
66538
|
+
const maskedAutolinks = maskedLinks.replace(/(<)([a-zA-Z][a-zA-Z0-9+.-]*:[^>\s]*)(>)/g, (_m, lt, uri, gt) => {
|
|
66539
|
+
const idx = linkMasks.length;
|
|
66540
|
+
linkMasks.push(uri);
|
|
66541
|
+
return `${lt}${LINK_MASK_PH}${idx}\x00${gt}`;
|
|
66542
|
+
});
|
|
66543
|
+
const escNonce = nonce.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
66544
|
+
const linkRestoreRe = new RegExp(`\x00RML${escNonce}_(\\d+)\x00`, "g");
|
|
66545
|
+
const restoreLinks = (s) => s.replace(linkRestoreRe, (_m, idx) => linkMasks[Number(idx)] ?? _m);
|
|
66546
|
+
let out = maskedAutolinks.replace(/(\S)[ \t][\u2014\u2013][ \t](?=(\S))/g, (_m, a, b) => /\d/.test(a) && /\d/.test(b) ? `${a}-` : `${a}, `).replace(/(\w)\u2014(?=(\w))/g, (_m, a, b) => /\d/.test(a) && /\d/.test(b) ? `${a}-` : `${a}, `).replace(/(\w)\u2013(?=\w)/g, "$1-");
|
|
66547
|
+
out = restoreLinks(out);
|
|
65948
66548
|
out = out.split(`
|
|
65949
66549
|
`).map((line) => line.replace(/^([ \t]*)[\u2022\u00b7][ \t]+/, "$1- ")).join(`
|
|
65950
66550
|
`);
|
|
@@ -66137,6 +66737,7 @@ function splitMarkdownChunks2(text4, maxLen = RICH_MESSAGE_MAX_CHARS2) {
|
|
|
66137
66737
|
}
|
|
66138
66738
|
cut = backOffOpenFence2(rest, cut);
|
|
66139
66739
|
cut = backOffTableRow2(rest, cut);
|
|
66740
|
+
cut = backOffOpenInline2(rest, cut);
|
|
66140
66741
|
if (cut <= 0) {
|
|
66141
66742
|
const sliced = hardSliceToCap2(rest, maxLen);
|
|
66142
66743
|
chunks.push(stripBoundarySpacers2(sliced[0], "trailing"));
|
|
@@ -66149,11 +66750,10 @@ function splitMarkdownChunks2(text4, maxLen = RICH_MESSAGE_MAX_CHARS2) {
|
|
|
66149
66750
|
return chunks.map((c) => stripBoundarySpacers2(c, "trailing"));
|
|
66150
66751
|
}
|
|
66151
66752
|
function stripBoundarySpacers2(chunk2, side) {
|
|
66152
|
-
const sp = PARAGRAPH_SPACER2;
|
|
66153
66753
|
if (side === "leading") {
|
|
66154
|
-
return chunk2.replace(
|
|
66754
|
+
return chunk2.replace(/^(?:[ \t]*\n)+/, "");
|
|
66155
66755
|
}
|
|
66156
|
-
return chunk2.replace(
|
|
66756
|
+
return chunk2.replace(/(?:\n[ \t]*)+$/, "");
|
|
66157
66757
|
}
|
|
66158
66758
|
function backOffOpenFence2(text4, cut) {
|
|
66159
66759
|
if (cut <= 0 || cut >= text4.length)
|
|
@@ -66182,6 +66782,35 @@ function backOffTableRow2(text4, cut) {
|
|
|
66182
66782
|
}
|
|
66183
66783
|
return cut;
|
|
66184
66784
|
}
|
|
66785
|
+
var INLINE_SPAN_PATTERNS2 = [
|
|
66786
|
+
/`[^`\n]+`/g,
|
|
66787
|
+
/\*\*\*[^*\n]+\*\*\*/g,
|
|
66788
|
+
/___[^_\n]+___/g,
|
|
66789
|
+
/\*\*[^*\n]+\*\*/g,
|
|
66790
|
+
/__[^_\n]+__/g,
|
|
66791
|
+
/(?<![\w*])_[^_\n]+_(?![\w*])/g,
|
|
66792
|
+
/\[[^\]\n]*\]\([^)\n]*\)/g
|
|
66793
|
+
];
|
|
66794
|
+
function backOffOpenInline2(text4, cut) {
|
|
66795
|
+
if (cut <= 0 || cut >= text4.length)
|
|
66796
|
+
return cut;
|
|
66797
|
+
let earliest = cut;
|
|
66798
|
+
for (const re of INLINE_SPAN_PATTERNS2) {
|
|
66799
|
+
re.lastIndex = 0;
|
|
66800
|
+
let m;
|
|
66801
|
+
while ((m = re.exec(text4)) !== null) {
|
|
66802
|
+
const start = m.index;
|
|
66803
|
+
const end = start + m[0].length;
|
|
66804
|
+
if (start < cut && cut < end && start < earliest)
|
|
66805
|
+
earliest = start;
|
|
66806
|
+
if (start >= cut)
|
|
66807
|
+
break;
|
|
66808
|
+
if (re.lastIndex === start)
|
|
66809
|
+
re.lastIndex = start + 1;
|
|
66810
|
+
}
|
|
66811
|
+
}
|
|
66812
|
+
return earliest;
|
|
66813
|
+
}
|
|
66185
66814
|
|
|
66186
66815
|
// rich-send.ts
|
|
66187
66816
|
var import_grammy8 = __toESM(require_mod2(), 1);
|
|
@@ -66300,6 +66929,7 @@ function scrubVoice2(text4) {
|
|
|
66300
66929
|
}
|
|
66301
66930
|
|
|
66302
66931
|
// gateway/outbound-send-path.ts
|
|
66932
|
+
init_format();
|
|
66303
66933
|
init_text_voice_scrub();
|
|
66304
66934
|
function normalizeOutboundBody(rawText, site, redact2) {
|
|
66305
66935
|
let text4 = normalizeParagraphBreaks(repairEscapedWhitespace(rawText));
|
|
@@ -66313,8 +66943,8 @@ function normalizeOutboundBody(rawText, site, redact2) {
|
|
|
66313
66943
|
}
|
|
66314
66944
|
return { text: text4, voiceReplaced };
|
|
66315
66945
|
}
|
|
66316
|
-
function computeEffectiveText(text4,
|
|
66317
|
-
return
|
|
66946
|
+
function computeEffectiveText(text4, _literalText) {
|
|
66947
|
+
return text4;
|
|
66318
66948
|
}
|
|
66319
66949
|
function computeReplyChunks(args) {
|
|
66320
66950
|
const { effectiveText, literalText, limit, chunkMode } = args;
|
|
@@ -68655,7 +69285,9 @@ function makeTmuxRunner2(tmuxBin) {
|
|
|
68655
69285
|
}
|
|
68656
69286
|
};
|
|
68657
69287
|
}
|
|
69288
|
+
|
|
68658
69289
|
// stream-reply-handler.ts
|
|
69290
|
+
init_format();
|
|
68659
69291
|
function buildAccentHeader(accent) {
|
|
68660
69292
|
switch (accent) {
|
|
68661
69293
|
case "in-progress":
|
|
@@ -69406,6 +70038,7 @@ var MODEL_ARG_RE2 = /^[A-Za-z0-9][A-Za-z0-9._/\[\]-]{0,99}$/;
|
|
|
69406
70038
|
function isValidModelArg2(arg) {
|
|
69407
70039
|
return MODEL_ARG_RE2.test(arg);
|
|
69408
70040
|
}
|
|
70041
|
+
var MODEL_CALLBACK_ALIAS2 = "mdl:alias:";
|
|
69409
70042
|
|
|
69410
70043
|
// gateway/session-model-file.ts
|
|
69411
70044
|
var SESSION_MODEL_FILE = ".session-model";
|
|
@@ -69472,6 +70105,172 @@ function clearSessionEffortFile(agentDir) {
|
|
|
69472
70105
|
rmSync4(join29(agentDir, SESSION_EFFORT_FILE), { force: true });
|
|
69473
70106
|
} catch {}
|
|
69474
70107
|
}
|
|
70108
|
+
var PREMIUM_RECOVERY_FILE = ".premium-recovery";
|
|
70109
|
+
function parsePremiumRecovery(text4) {
|
|
70110
|
+
try {
|
|
70111
|
+
const raw = JSON.parse(text4);
|
|
70112
|
+
if (typeof raw.premiumModel !== "string" || !isValidModelArg2(raw.premiumModel) || typeof raw.ts !== "number" || !Array.isArray(raw.chats) || raw.chats.length === 0 || !raw.chats.every((c) => typeof c === "string" && c.length > 0)) {
|
|
70113
|
+
return null;
|
|
70114
|
+
}
|
|
70115
|
+
return { premiumModel: raw.premiumModel, chats: raw.chats, ts: raw.ts };
|
|
70116
|
+
} catch {
|
|
70117
|
+
return null;
|
|
70118
|
+
}
|
|
70119
|
+
}
|
|
70120
|
+
function writePremiumRecoveryFile(agentDir, premiumModel, chats) {
|
|
70121
|
+
if (!isValidModelArg2(premiumModel)) {
|
|
70122
|
+
throw new Error(`refusing to persist non-canonical premium-recovery token: ${JSON.stringify(premiumModel)}`);
|
|
70123
|
+
}
|
|
70124
|
+
const clean = chats.filter((c) => typeof c === "string" && c.length > 0);
|
|
70125
|
+
if (clean.length === 0) {
|
|
70126
|
+
throw new Error("refusing to persist premium-recovery marker with no chats to notify");
|
|
70127
|
+
}
|
|
70128
|
+
atomicWrite(join29(agentDir, PREMIUM_RECOVERY_FILE), `${JSON.stringify({ premiumModel, chats: clean, ts: Date.now() })}
|
|
70129
|
+
`);
|
|
70130
|
+
}
|
|
70131
|
+
function readPremiumRecoveryFile(agentDir) {
|
|
70132
|
+
let raw;
|
|
70133
|
+
try {
|
|
70134
|
+
raw = readFileSync25(join29(agentDir, PREMIUM_RECOVERY_FILE), "utf8");
|
|
70135
|
+
} catch {
|
|
70136
|
+
return null;
|
|
70137
|
+
}
|
|
70138
|
+
const parsed = parsePremiumRecovery(raw);
|
|
70139
|
+
if (parsed == null) {
|
|
70140
|
+
clearPremiumRecoveryFile(agentDir);
|
|
70141
|
+
return null;
|
|
70142
|
+
}
|
|
70143
|
+
return parsed;
|
|
70144
|
+
}
|
|
70145
|
+
function clearPremiumRecoveryFile(agentDir) {
|
|
70146
|
+
try {
|
|
70147
|
+
rmSync4(join29(agentDir, PREMIUM_RECOVERY_FILE), { force: true });
|
|
70148
|
+
} catch {}
|
|
70149
|
+
}
|
|
70150
|
+
|
|
70151
|
+
// tier-downgrade.ts
|
|
70152
|
+
function decideTierDowngrade(input) {
|
|
70153
|
+
const configured = typeof input.configuredDefault === "string" ? input.configuredDefault.trim() : "";
|
|
70154
|
+
if (configured.length === 0) {
|
|
70155
|
+
return { action: "skip", reason: "unresolved" };
|
|
70156
|
+
}
|
|
70157
|
+
const override = input.sessionOverride;
|
|
70158
|
+
if (override == null || override.length === 0) {
|
|
70159
|
+
return { action: "skip", reason: "on-default" };
|
|
70160
|
+
}
|
|
70161
|
+
if (input.resolve(override) === input.resolve(configured)) {
|
|
70162
|
+
return { action: "skip", reason: "on-default" };
|
|
70163
|
+
}
|
|
70164
|
+
return { action: "downgrade", toModel: configured, fromModel: override };
|
|
70165
|
+
}
|
|
70166
|
+
function renderTierDowngradeNotice(fromModel, toModel, agent) {
|
|
70167
|
+
return `\u2935\ufe0f **Downgrading model to keep going** on agent **${agent}**
|
|
70168
|
+
` + `\`${fromModel}\` is overloaded across every account right now, so this turn is ` + `resuming on the default \`${toModel}\` to keep working. ` + `Re-issue \`/model ${fromModel}\` once it frees up \u2014 it won't switch back on its own.`;
|
|
70169
|
+
}
|
|
70170
|
+
function planTierDowngrade(decision, gateVerdict, agent) {
|
|
70171
|
+
if (decision.action !== "downgrade")
|
|
70172
|
+
return { kind: "skip" };
|
|
70173
|
+
if (gateVerdict === "skip-inflight")
|
|
70174
|
+
return { kind: "suppress" };
|
|
70175
|
+
if (gateVerdict === "skip-stale")
|
|
70176
|
+
return { kind: "skip" };
|
|
70177
|
+
return {
|
|
70178
|
+
kind: "downgrade",
|
|
70179
|
+
toModel: decision.toModel,
|
|
70180
|
+
fromModel: decision.fromModel,
|
|
70181
|
+
notice: renderTierDowngradeNotice(decision.fromModel, decision.toModel, agent)
|
|
70182
|
+
};
|
|
70183
|
+
}
|
|
70184
|
+
|
|
70185
|
+
// gateway/tier-downgrade-wiring.ts
|
|
70186
|
+
function runTierDowngrade(triggerAgent, deps) {
|
|
70187
|
+
const agentDir = deps.getAgentDir();
|
|
70188
|
+
if (!agentDir)
|
|
70189
|
+
return "skip";
|
|
70190
|
+
const configuredDefault = deps.getConfiguredDefault() ?? "";
|
|
70191
|
+
const decision = decideTierDowngrade({
|
|
70192
|
+
sessionOverride: deps.getSessionOverride(),
|
|
70193
|
+
configuredDefault,
|
|
70194
|
+
resolve: deps.resolve
|
|
70195
|
+
});
|
|
70196
|
+
const gateVerdict = deps.peekResumeGate();
|
|
70197
|
+
const plan = planTierDowngrade(decision, gateVerdict, triggerAgent);
|
|
70198
|
+
if (plan.kind === "skip") {
|
|
70199
|
+
if (decision.action === "downgrade") {
|
|
70200
|
+
deps.log(`[tier-downgrade] restart suppressed (${gateVerdict}) agent=${triggerAgent}`);
|
|
70201
|
+
}
|
|
70202
|
+
return "skip";
|
|
70203
|
+
}
|
|
70204
|
+
if (plan.kind === "suppress") {
|
|
70205
|
+
deps.log(`[tier-downgrade] give-up suppressed \u2014 a resume restart is already armed agent=${triggerAgent}`);
|
|
70206
|
+
return "restart-pending";
|
|
70207
|
+
}
|
|
70208
|
+
try {
|
|
70209
|
+
deps.writeCarrier(agentDir, plan.toModel, configuredDefault);
|
|
70210
|
+
} catch (err) {
|
|
70211
|
+
deps.log(`[tier-downgrade] failed to write session-model carrier \u2014 aborting downgrade: ${err?.message ?? err}`);
|
|
70212
|
+
return "skip";
|
|
70213
|
+
}
|
|
70214
|
+
deps.armResumeGate();
|
|
70215
|
+
deps.log(`[tier-downgrade] downgrading ${plan.fromModel} \u2192 ${plan.toModel} and resuming via self-restart agent=${triggerAgent}`);
|
|
70216
|
+
try {
|
|
70217
|
+
deps.writeRecoveryMarker(agentDir, plan.fromModel);
|
|
70218
|
+
} catch (err) {
|
|
70219
|
+
deps.log(`[tier-downgrade] premium-recovery marker write failed (non-fatal): ${err?.message ?? err}`);
|
|
70220
|
+
}
|
|
70221
|
+
deps.broadcastNotice(plan.notice);
|
|
70222
|
+
deps.selfRestart(deps.selfAgent(triggerAgent));
|
|
70223
|
+
return "downgraded";
|
|
70224
|
+
}
|
|
70225
|
+
|
|
70226
|
+
// premium-recovery.ts
|
|
70227
|
+
function renderPremiumRecoveryPing(premiumModel) {
|
|
70228
|
+
return {
|
|
70229
|
+
text: `\u2705 \`${premiumModel}\` is available again \u2014 tap to switch back to it for this session.`,
|
|
70230
|
+
buttonText: `Switch to ${premiumModel}`
|
|
70231
|
+
};
|
|
70232
|
+
}
|
|
70233
|
+
function premiumRecoveryClaimKey(agent, premiumModel) {
|
|
70234
|
+
return `premium-recovery:${agent}:${premiumModel}`;
|
|
70235
|
+
}
|
|
70236
|
+
|
|
70237
|
+
// gateway/premium-recovery-wiring.ts
|
|
70238
|
+
async function runPremiumRecoveryPing(deps) {
|
|
70239
|
+
const agentDir = deps.getAgentDir();
|
|
70240
|
+
if (!agentDir)
|
|
70241
|
+
return;
|
|
70242
|
+
const marker = deps.readMarker(agentDir);
|
|
70243
|
+
if (marker == null)
|
|
70244
|
+
return;
|
|
70245
|
+
if (!deps.decide())
|
|
70246
|
+
return;
|
|
70247
|
+
const agent = deps.getAgent();
|
|
70248
|
+
const granted = await deps.claimNotification(premiumRecoveryClaimKey(agent, marker.premiumModel));
|
|
70249
|
+
if (!granted) {
|
|
70250
|
+
deps.clearMarker(agentDir);
|
|
70251
|
+
return;
|
|
70252
|
+
}
|
|
70253
|
+
deps.clearMarker(agentDir);
|
|
70254
|
+
const ping = renderPremiumRecoveryPing(marker.premiumModel);
|
|
70255
|
+
const keyboard = {
|
|
70256
|
+
inline_keyboard: [
|
|
70257
|
+
[{ text: ping.buttonText, callback_data: `${MODEL_CALLBACK_ALIAS2}${marker.premiumModel}` }]
|
|
70258
|
+
]
|
|
70259
|
+
};
|
|
70260
|
+
const chats = marker.chats.length > 0 ? marker.chats : deps.fallbackChats();
|
|
70261
|
+
for (const chatId of chats) {
|
|
70262
|
+
deps.sendToChat(chatId, ping, keyboard);
|
|
70263
|
+
}
|
|
70264
|
+
deps.log(`[premium-recovery] ${marker.premiumModel} servable again \u2014 ping sent agent=${agent} chats=${chats.length}`);
|
|
70265
|
+
}
|
|
70266
|
+
|
|
70267
|
+
// premium-recovery.ts
|
|
70268
|
+
function decidePremiumRecovery(opts) {
|
|
70269
|
+
if (!opts.hasMarker)
|
|
70270
|
+
return { fire: false, reason: "no-marker" };
|
|
70271
|
+
const servable = opts.accounts.some((a) => !a.exhausted && !a.premiumWalled);
|
|
70272
|
+
return servable ? { fire: true, reason: "recovered" } : { fire: false, reason: "still-walled" };
|
|
70273
|
+
}
|
|
69475
70274
|
|
|
69476
70275
|
// ../src/agents/model-picker.ts
|
|
69477
70276
|
var HEADER_RE = /Select model/;
|
|
@@ -72318,6 +73117,7 @@ function recordWebhookEvent(rec, deps = {}) {
|
|
|
72318
73117
|
}
|
|
72319
73118
|
|
|
72320
73119
|
// gateway/ipc-server.ts
|
|
73120
|
+
init_format();
|
|
72321
73121
|
import { renameSync as renameSync11, unlinkSync as unlinkSync14, chmodSync as chmodSync9 } from "fs";
|
|
72322
73122
|
var MAX_BUFFER_SIZE = 1024 * 1024;
|
|
72323
73123
|
var VALID_OPERATOR_KINDS = new Set([
|
|
@@ -73023,6 +73823,7 @@ function validateInput(input) {
|
|
|
73023
73823
|
}
|
|
73024
73824
|
|
|
73025
73825
|
// gateway/drive-write-approval.ts
|
|
73826
|
+
init_format();
|
|
73026
73827
|
var DEFAULT_TTL_MS = 5 * 60 * 1000;
|
|
73027
73828
|
var MAX_TTL_MS = 30 * 60 * 1000;
|
|
73028
73829
|
var MIN_TTL_MS = 30 * 1000;
|
|
@@ -73153,6 +73954,7 @@ function clampTtl(requested, fallback, min, max) {
|
|
|
73153
73954
|
}
|
|
73154
73955
|
|
|
73155
73956
|
// gateway/ms365-write-approval.ts
|
|
73957
|
+
init_format();
|
|
73156
73958
|
function validateMs365Preview(input) {
|
|
73157
73959
|
if (!input || typeof input !== "object")
|
|
73158
73960
|
return null;
|
|
@@ -73331,6 +74133,7 @@ async function handleRequestMs365Approval(client3, msg, deps) {
|
|
|
73331
74133
|
}
|
|
73332
74134
|
|
|
73333
74135
|
// gateway/diff-preview-card.ts
|
|
74136
|
+
init_format();
|
|
73334
74137
|
var import_grammy10 = __toESM(require_mod2(), 1);
|
|
73335
74138
|
var REQUEST_ID_RE = /^[0-9a-f]{32}$/;
|
|
73336
74139
|
var PENDING_FILE_ID_SENTINEL = "pending-create";
|
|
@@ -75489,6 +76292,7 @@ function maybeFireWarmup(ctx) {
|
|
|
75489
76292
|
|
|
75490
76293
|
// gateway/mental-model-propose-card.ts
|
|
75491
76294
|
init_approval_card();
|
|
76295
|
+
init_format();
|
|
75492
76296
|
function renderMentalModelProposeCard(req) {
|
|
75493
76297
|
const lines = [];
|
|
75494
76298
|
lines.push(`\uD83E\uDDE0 **${escapeHtmlForTg(req.agent)}** proposes a mental model`);
|
|
@@ -77113,71 +77917,6 @@ import {
|
|
|
77113
77917
|
} from "fs";
|
|
77114
77918
|
import { join as join39 } from "path";
|
|
77115
77919
|
|
|
77116
|
-
// operator-events.ts
|
|
77117
|
-
function classifyClaudeError(raw) {
|
|
77118
|
-
try {
|
|
77119
|
-
return classifyInner(raw);
|
|
77120
|
-
} catch {
|
|
77121
|
-
return "unknown-4xx";
|
|
77122
|
-
}
|
|
77123
|
-
}
|
|
77124
|
-
function classifyInner(raw) {
|
|
77125
|
-
if (raw == null)
|
|
77126
|
-
return "unknown-4xx";
|
|
77127
|
-
const obj = typeof raw === "object" ? raw : {};
|
|
77128
|
-
const errorType = extractString(obj, "error_type") ?? extractString(obj, "type") ?? extractString(getNestedObj(obj, "error"), "type") ?? "";
|
|
77129
|
-
const errorCode = extractString(obj, "code") ?? extractString(getNestedObj(obj, "error"), "code") ?? "";
|
|
77130
|
-
const message = extractString(obj, "message") ?? extractString(getNestedObj(obj, "error"), "message") ?? (typeof raw === "string" ? raw : "") ?? "";
|
|
77131
|
-
const status = extractNumber(obj, "status") ?? extractNumber(obj, "statusCode") ?? extractNumber(obj, "status_code") ?? null;
|
|
77132
|
-
const sdkCode = extractString(obj, "error_code") ?? "";
|
|
77133
|
-
if (errorType === "authentication_error" || errorCode === "authentication_error" || sdkCode === "authentication_error" || message.toLowerCase().includes("authentication_error")) {
|
|
77134
|
-
const msg = message.toLowerCase();
|
|
77135
|
-
if (msg.includes("expired") || msg.includes("refresh")) {
|
|
77136
|
-
return "credentials-expired";
|
|
77137
|
-
}
|
|
77138
|
-
return "credentials-invalid";
|
|
77139
|
-
}
|
|
77140
|
-
if (errorType === "invalid_api_key" || errorCode === "invalid_api_key" || sdkCode === "invalid_api_key" || message.toLowerCase().includes("invalid_api_key") || message.toLowerCase().includes("invalid api key")) {
|
|
77141
|
-
return "credentials-invalid";
|
|
77142
|
-
}
|
|
77143
|
-
if (errorType === "credit_balance_too_low" || errorCode === "credit_balance_too_low" || sdkCode === "credit_balance_too_low" || message.toLowerCase().includes("credit_balance_too_low") || message.toLowerCase().includes("credit balance")) {
|
|
77144
|
-
return "credit-exhausted";
|
|
77145
|
-
}
|
|
77146
|
-
if (errorType === "rate_limit_error" || errorCode === "rate_limit_error" || sdkCode === "rate_limit_error" || message.toLowerCase().includes("rate_limit_error") || message.toLowerCase().includes("rate limit")) {
|
|
77147
|
-
return "rate-limited";
|
|
77148
|
-
}
|
|
77149
|
-
if (errorType === "overloaded_error" || errorCode === "overloaded_error" || sdkCode === "overloaded_error" || message.toLowerCase().includes("overloaded_error") || message.toLowerCase().includes("overloaded")) {
|
|
77150
|
-
return "rate-limited";
|
|
77151
|
-
}
|
|
77152
|
-
if (errorType === "agent-crashed" || errorCode === "agent-crashed") {
|
|
77153
|
-
return "agent-crashed";
|
|
77154
|
-
}
|
|
77155
|
-
if (errorType === "agent-restarted-unexpectedly" || errorCode === "agent-restarted-unexpectedly") {
|
|
77156
|
-
return "agent-restarted-unexpectedly";
|
|
77157
|
-
}
|
|
77158
|
-
if (status != null) {
|
|
77159
|
-
if (status >= 400 && status < 500)
|
|
77160
|
-
return "unknown-4xx";
|
|
77161
|
-
if (status >= 500 && status < 600)
|
|
77162
|
-
return "unknown-5xx";
|
|
77163
|
-
}
|
|
77164
|
-
return "unknown-4xx";
|
|
77165
|
-
}
|
|
77166
|
-
function extractString(obj, key) {
|
|
77167
|
-
const v = obj[key];
|
|
77168
|
-
return typeof v === "string" && v.length > 0 ? v : null;
|
|
77169
|
-
}
|
|
77170
|
-
function extractNumber(obj, key) {
|
|
77171
|
-
const v = obj[key];
|
|
77172
|
-
return typeof v === "number" ? v : null;
|
|
77173
|
-
}
|
|
77174
|
-
function getNestedObj(obj, key) {
|
|
77175
|
-
const v = obj[key];
|
|
77176
|
-
return typeof v === "object" && v != null ? v : {};
|
|
77177
|
-
}
|
|
77178
|
-
var DEFAULT_OPERATOR_EVENT_COOLDOWN_MS2 = 5 * 60000;
|
|
77179
|
-
var cooldownMap2 = new Map;
|
|
77180
|
-
|
|
77181
77920
|
// session-tail.ts
|
|
77182
77921
|
function sanitizeCwdToProjectName(cwd) {
|
|
77183
77922
|
return cwd.replace(/[^a-zA-Z0-9]/g, "-");
|
|
@@ -77362,7 +78101,7 @@ function detectErrorInTranscriptLine(line) {
|
|
|
77362
78101
|
const errStr = typeof obj.error === "string" ? obj.error : "";
|
|
77363
78102
|
const text4 = extractAssistantText(obj);
|
|
77364
78103
|
const kind2 = status === 429 ? isTransientUpstreamSignal(`${text4}
|
|
77365
|
-
${errStr}`) ||
|
|
78104
|
+
${errStr}`) || isLitellmProxyLocal429(`${text4}
|
|
77366
78105
|
${errStr}`) ? "rate-limited" : "quota-exhausted" : classifyClaudeError({ type: errStr, status, message: text4 });
|
|
77367
78106
|
return {
|
|
77368
78107
|
kind: kind2,
|
|
@@ -79671,6 +80410,29 @@ var RESOURCE_KEYS = ["path", "endpoint", "url", "resource", "route"];
|
|
|
79671
80410
|
var ARG_SUMMARY_MAX_KEYS = 4;
|
|
79672
80411
|
var ARG_VALUE_MAX = 40;
|
|
79673
80412
|
var ARG_SUMMARY_LINE_MAX = 180;
|
|
80413
|
+
var CONTEXT_NOISE_KEYS = new Set([
|
|
80414
|
+
"reason",
|
|
80415
|
+
"why",
|
|
80416
|
+
"chat_id",
|
|
80417
|
+
"message_id",
|
|
80418
|
+
"message_thread_id",
|
|
80419
|
+
"thread_id",
|
|
80420
|
+
"origin_turn_id",
|
|
80421
|
+
"reply_to",
|
|
80422
|
+
"quote",
|
|
80423
|
+
"quote_text",
|
|
80424
|
+
"format",
|
|
80425
|
+
"parse_mode",
|
|
80426
|
+
"disable_web_page_preview",
|
|
80427
|
+
"disable_notification",
|
|
80428
|
+
"protect_content",
|
|
80429
|
+
"single_use",
|
|
80430
|
+
"ack_text",
|
|
80431
|
+
"inline_keyboard",
|
|
80432
|
+
"file_id"
|
|
80433
|
+
]);
|
|
80434
|
+
var SENSITIVE_VALUE_KEY_RE = /token|secret|password|passwd|key|auth|dsn|conn|url|credential|cookie|session/i;
|
|
80435
|
+
var NON_HTTP_DSN_RE = /\b(?:postgres(?:ql)?|mysql|mariadb|redis|rediss|mongodb(?:\+srv)?|amqp|amqps):\/\/\S*@\S+/gi;
|
|
79674
80436
|
var MCP_TOOL_DESCRIPTIONS = {
|
|
79675
80437
|
"mcp__agent-config__config_get": "Read its own merged config",
|
|
79676
80438
|
"mcp__agent-config__cron_list": "List its own scheduled tasks",
|
|
@@ -79721,8 +80483,12 @@ function formatPermissionCardBody(opts) {
|
|
|
79721
80483
|
}
|
|
79722
80484
|
const callerReason = callerSuppliedReason(opts.inputPreview);
|
|
79723
80485
|
const rawWhy = (callerReason ?? "").replace(/\s+/g, " ").trim();
|
|
79724
|
-
|
|
79725
|
-
|
|
80486
|
+
if (rawWhy.length > 0) {
|
|
80487
|
+
const truncatedWhy = rawWhy.length > DESCRIPTION_LINE_MAX ? rawWhy.slice(0, DESCRIPTION_LINE_MAX - 1) + "\u2026" : rawWhy;
|
|
80488
|
+
lines.push(`why: _${escapeTgHtml(truncatedWhy)}_`);
|
|
80489
|
+
} else {
|
|
80490
|
+
lines.push(`context: _${escapeTgHtml(synthesizeContext(opts.toolName, opts.inputPreview))}_`);
|
|
80491
|
+
}
|
|
79726
80492
|
const argSummary = mcpArgSummary(opts.toolName, opts.inputPreview);
|
|
79727
80493
|
if (argSummary) {
|
|
79728
80494
|
lines.push(`\u21b3 _${escapeTgHtml(argSummary)}_`);
|
|
@@ -79996,6 +80762,50 @@ function callerSuppliedReason(inputPreview) {
|
|
|
79996
80762
|
}
|
|
79997
80763
|
return null;
|
|
79998
80764
|
}
|
|
80765
|
+
function synthesizeContext(toolName, inputPreview) {
|
|
80766
|
+
const input = parseInput2(inputPreview);
|
|
80767
|
+
const summary = input ? salientInputSummary(input) : null;
|
|
80768
|
+
const base = summary ?? naturalAction(toolName, inputPreview);
|
|
80769
|
+
const turnRef = input ? readString2(input, "origin_turn_id") : null;
|
|
80770
|
+
return turnRef ? `${base} \u00b7 turn ${shortTurnRef(turnRef)}` : base;
|
|
80771
|
+
}
|
|
80772
|
+
function shortTurnRef(turnId) {
|
|
80773
|
+
const t = turnId.trim();
|
|
80774
|
+
return t.length <= 10 ? t : `\u2026${t.slice(-8)}`;
|
|
80775
|
+
}
|
|
80776
|
+
function salientInputSummary(input) {
|
|
80777
|
+
const parts = [];
|
|
80778
|
+
for (const [key, value] of Object.entries(input)) {
|
|
80779
|
+
if (CONTEXT_NOISE_KEYS.has(key))
|
|
80780
|
+
continue;
|
|
80781
|
+
if (value == null)
|
|
80782
|
+
continue;
|
|
80783
|
+
if (parts.length >= ARG_SUMMARY_MAX_KEYS) {
|
|
80784
|
+
parts.push("\u2026");
|
|
80785
|
+
break;
|
|
80786
|
+
}
|
|
80787
|
+
if (typeof value === "object") {
|
|
80788
|
+
parts.push(key);
|
|
80789
|
+
continue;
|
|
80790
|
+
}
|
|
80791
|
+
const shown = truncate6(redactSalientValue(key, String(value)), ARG_VALUE_MAX);
|
|
80792
|
+
if (shown.length === 0)
|
|
80793
|
+
continue;
|
|
80794
|
+
parts.push(`${key}: ${shown}`);
|
|
80795
|
+
}
|
|
80796
|
+
if (parts.length === 0)
|
|
80797
|
+
return null;
|
|
80798
|
+
const joined = parts.join(", ");
|
|
80799
|
+
return joined.length > ARG_SUMMARY_LINE_MAX ? joined.slice(0, ARG_SUMMARY_LINE_MAX - 1) + "\u2026" : joined;
|
|
80800
|
+
}
|
|
80801
|
+
function redactSalientValue(key, value) {
|
|
80802
|
+
if (SENSITIVE_VALUE_KEY_RE.test(key))
|
|
80803
|
+
return REDACTED_MARKER;
|
|
80804
|
+
const dsnScrubbed = value.replace(NON_HTTP_DSN_RE, REDACTED_MARKER);
|
|
80805
|
+
const scrubbed = redact(`${key}=${dsnScrubbed}`);
|
|
80806
|
+
const prefix = `${key}=`;
|
|
80807
|
+
return scrubbed.startsWith(prefix) ? scrubbed.slice(prefix.length) : scrubbed;
|
|
80808
|
+
}
|
|
79999
80809
|
function extractReasonFromRaw(raw) {
|
|
80000
80810
|
const m = /"(?:reason|why)"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(raw);
|
|
80001
80811
|
if (!m)
|
|
@@ -81224,10 +82034,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
|
|
|
81224
82034
|
}
|
|
81225
82035
|
|
|
81226
82036
|
// ../src/build-info.ts
|
|
81227
|
-
var VERSION = "0.18.
|
|
81228
|
-
var COMMIT_SHA = "
|
|
81229
|
-
var COMMIT_DATE = "2026-07-
|
|
81230
|
-
var LATEST_PR =
|
|
82037
|
+
var VERSION = "0.18.19";
|
|
82038
|
+
var COMMIT_SHA = "34c72776";
|
|
82039
|
+
var COMMIT_DATE = "2026-07-13T14:06:25+10:00";
|
|
82040
|
+
var LATEST_PR = 3212;
|
|
81231
82041
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
81232
82042
|
|
|
81233
82043
|
// gateway/boot-version.ts
|
|
@@ -81280,6 +82090,8 @@ function classifyRejection(err, opts = {}) {
|
|
|
81280
82090
|
return "log_only";
|
|
81281
82091
|
if (err instanceof Error && err.message === FLOOD_WAIT_ACTIVE)
|
|
81282
82092
|
return "log_only";
|
|
82093
|
+
if (err instanceof Error && err.message === LOCAL_RESOURCE_EXHAUSTED)
|
|
82094
|
+
return "log_only";
|
|
81283
82095
|
if (!isGrammy)
|
|
81284
82096
|
return "shutdown";
|
|
81285
82097
|
const e = err;
|
|
@@ -81482,6 +82294,7 @@ async function listGrantsViaBroker2(agent, opts) {
|
|
|
81482
82294
|
}
|
|
81483
82295
|
|
|
81484
82296
|
// gateway/linear-activity.ts
|
|
82297
|
+
init_format();
|
|
81485
82298
|
init_client();
|
|
81486
82299
|
|
|
81487
82300
|
// ../src/linear/oauth-refresh.ts
|
|
@@ -81820,10 +82633,13 @@ async function emitLinearAgentActivity(args, deps = {}) {
|
|
|
81820
82633
|
`);
|
|
81821
82634
|
return { content: [{ type: "text", text: `Linear ${type} emitted on session ${sessionId}` }] };
|
|
81822
82635
|
}
|
|
82636
|
+
function captureDedupComment(dedupKey) {
|
|
82637
|
+
return `<!-- switchroom-capture: ${dedupKey} -->`;
|
|
82638
|
+
}
|
|
81823
82639
|
function captureDedupMarker(dedupKey) {
|
|
81824
82640
|
return `
|
|
81825
82641
|
|
|
81826
|
-
|
|
82642
|
+
${captureDedupComment(dedupKey)}`;
|
|
81827
82643
|
}
|
|
81828
82644
|
async function createLinearIssue(args, deps = {}) {
|
|
81829
82645
|
const log = deps.log ?? ((s) => process.stderr.write(s));
|
|
@@ -81872,9 +82688,11 @@ async function createLinearIssue(args, deps = {}) {
|
|
|
81872
82688
|
return { ok: true, data: json.data };
|
|
81873
82689
|
};
|
|
81874
82690
|
if (dedupKey) {
|
|
81875
|
-
const
|
|
82691
|
+
const marker = captureDedupComment(dedupKey);
|
|
82692
|
+
const search2 = await gql("query($term: String!) { searchIssues(term: $term, first: 25) { nodes { id url title description } } }", { term: dedupKey });
|
|
81876
82693
|
if (search2.ok) {
|
|
81877
|
-
const
|
|
82694
|
+
const nodes = search2.data?.searchIssues?.nodes ?? [];
|
|
82695
|
+
const hit = nodes.find((n) => typeof n.description === "string" && n.description.includes(marker));
|
|
81878
82696
|
if (hit?.url) {
|
|
81879
82697
|
log(`telegram gateway: linear_create_issue: dedup hit key=${dedupKey} agent=${agent}
|
|
81880
82698
|
`);
|
|
@@ -82681,8 +83499,15 @@ function buildBridgeDeadIdleNoticeInbound(args) {
|
|
|
82681
83499
|
}
|
|
82682
83500
|
};
|
|
82683
83501
|
}
|
|
82684
|
-
function isRealBridgeIdentity(agentName3) {
|
|
82685
|
-
|
|
83502
|
+
function isRealBridgeIdentity(agentName3, selfAgentName) {
|
|
83503
|
+
if (agentName3 == null || agentName3.length === 0)
|
|
83504
|
+
return false;
|
|
83505
|
+
if (isCronIdentity2(agentName3))
|
|
83506
|
+
return false;
|
|
83507
|
+
if (selfAgentName != null && selfAgentName.length > 0) {
|
|
83508
|
+
return agentName3 === selfAgentName;
|
|
83509
|
+
}
|
|
83510
|
+
return true;
|
|
82686
83511
|
}
|
|
82687
83512
|
function createBridgeDeadWatchdog(opts) {
|
|
82688
83513
|
const setTimer = opts.setTimer ?? ((fn, ms) => {
|
|
@@ -82696,6 +83521,7 @@ function createBridgeDeadWatchdog(opts) {
|
|
|
82696
83521
|
const readCrashTail = opts.readCrashTail ?? ((p, t) => readFreshCrashLogTail(p, { nowMs: t }));
|
|
82697
83522
|
const priorStreak = opts.priorStreak ?? 0;
|
|
82698
83523
|
const maxConsecutive = opts.maxConsecutive ?? MAX_CONSECUTIVE_ESCALATIONS;
|
|
83524
|
+
const selfAgentName = opts.selfAgentName;
|
|
82699
83525
|
let timer3 = null;
|
|
82700
83526
|
let bridgeRegistered = false;
|
|
82701
83527
|
let bridgeEverRegistered = false;
|
|
@@ -82773,14 +83599,14 @@ function createBridgeDeadWatchdog(opts) {
|
|
|
82773
83599
|
return {
|
|
82774
83600
|
arm: armInternal,
|
|
82775
83601
|
noteBridgeRegistered: (agentName3) => {
|
|
82776
|
-
if (!isRealBridgeIdentity(agentName3))
|
|
83602
|
+
if (!isRealBridgeIdentity(agentName3, selfAgentName))
|
|
82777
83603
|
return;
|
|
82778
83604
|
bridgeRegistered = true;
|
|
82779
83605
|
bridgeEverRegistered = true;
|
|
82780
83606
|
cancel();
|
|
82781
83607
|
},
|
|
82782
83608
|
noteBridgeDisconnected: (agentName3) => {
|
|
82783
|
-
if (!isRealBridgeIdentity(agentName3))
|
|
83609
|
+
if (!isRealBridgeIdentity(agentName3, selfAgentName))
|
|
82784
83610
|
return;
|
|
82785
83611
|
bridgeRegistered = false;
|
|
82786
83612
|
armInternal();
|
|
@@ -84887,8 +85713,9 @@ var PHOTO_EXTS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp"]);
|
|
|
84887
85713
|
var FLOOD_STATE_PATH = floodStatePath(STATE_DIR);
|
|
84888
85714
|
var FLOOD_WINDOWS_PATH = floodWindowsPath(STATE_DIR);
|
|
84889
85715
|
var recordFloodWindow = makeFloodWindowRecorder(FLOOD_WINDOWS_PATH);
|
|
85716
|
+
var sendGateConfig = sendGateConfigFromEnv();
|
|
84890
85717
|
var sendGate = createSendGate({
|
|
84891
|
-
|
|
85718
|
+
...sendGateConfig,
|
|
84892
85719
|
initialWindows: loadInitialFloodWindows(FLOOD_STATE_PATH, FLOOD_WINDOWS_PATH, Date.now()),
|
|
84893
85720
|
bootRamp: {},
|
|
84894
85721
|
onWindowOpen: (scopeKey, untilTs) => recordFloodWindow(scopeKey, untilTs)
|
|
@@ -84896,10 +85723,10 @@ var sendGate = createSendGate({
|
|
|
84896
85723
|
var probeFloodWaitRemainingMs = makeFloodWaitProbe2(FLOOD_STATE_PATH);
|
|
84897
85724
|
var rawRobustApiCall = createRetryApiCall2({
|
|
84898
85725
|
log: (line) => process.stderr.write(line),
|
|
84899
|
-
onFloodWait: (retryAfterSec) => {
|
|
85726
|
+
onFloodWait: (retryAfterSec, opts) => {
|
|
84900
85727
|
makeFloodWaitRecorder2(FLOOD_STATE_PATH)(retryAfterSec);
|
|
84901
85728
|
try {
|
|
84902
|
-
sendGate.
|
|
85729
|
+
sendGate.openScopedFloodWindows(opts, Date.now() + Math.max(0, retryAfterSec) * 1000);
|
|
84903
85730
|
} catch {}
|
|
84904
85731
|
},
|
|
84905
85732
|
floodWaitRemainingMs: probeFloodWaitRemainingMs
|
|
@@ -84935,6 +85762,7 @@ var sendGateStatsLogger = createStatsLogger({
|
|
|
84935
85762
|
var floodWindowObserver = createFloodWindowObserver({
|
|
84936
85763
|
clock: { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) },
|
|
84937
85764
|
log: (line) => process.stderr.write(line),
|
|
85765
|
+
tz: process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC",
|
|
84938
85766
|
stats: () => sendGate.stats(),
|
|
84939
85767
|
readWindows: (now) => readFloodWindows(FLOOD_WINDOWS_PATH, now),
|
|
84940
85768
|
markAlerted: (scopeKey, alertedAt) => markFloodWindowAlerted(FLOOD_WINDOWS_PATH, scopeKey, alertedAt, Date.now()),
|
|
@@ -84949,7 +85777,7 @@ var floodWindowObserver = createFloodWindowObserver({
|
|
|
84949
85777
|
await robustApiCall(() => bot.api.sendRichMessage(operator, richMessage2(text5), {}), { chat_id: String(operator), verb: "send-gate-flood-alert", priorityClass: "critical" });
|
|
84950
85778
|
}
|
|
84951
85779
|
});
|
|
84952
|
-
if (
|
|
85780
|
+
if (sendGateConfig.enabled) {
|
|
84953
85781
|
const observeTimer = setInterval(() => {
|
|
84954
85782
|
try {
|
|
84955
85783
|
sendGateStatsLogger.tick();
|
|
@@ -85419,7 +86247,8 @@ function isAutoFallbackCooldownActive(_agentName, now) {
|
|
|
85419
86247
|
async function editCardExpired(chatId, messageId, body) {
|
|
85420
86248
|
if (messageId == null)
|
|
85421
86249
|
return;
|
|
85422
|
-
|
|
86250
|
+
const text5 = richMessage2(body);
|
|
86251
|
+
await robustApiCall(() => lockedBot.api.editMessageText(chatId, messageId, text5, { reply_markup: { inline_keyboard: [] } }), { chat_id: chatId, verb: "card-expired.strip", priorityClass: "cosmetic", messageId, editPayload: body }).catch(() => {});
|
|
85423
86252
|
}
|
|
85424
86253
|
function recordMissedApproval(opts) {
|
|
85425
86254
|
if (!MISSED_APPROVAL_REOFFER_ENABLED)
|
|
@@ -85646,10 +86475,7 @@ function postPermissionCard(requestId, pend) {
|
|
|
85646
86475
|
if (live && sent && typeof sent.message_id === "number") {
|
|
85647
86476
|
const landedThreadId = sent.message_thread_id ?? undefined;
|
|
85648
86477
|
live.cards.push({ chatId, messageId: sent.message_id, threadId: landedThreadId });
|
|
85649
|
-
if (live.
|
|
85650
|
-
live.undeliverable = null;
|
|
85651
|
-
live.redeliveryFailures = 0;
|
|
85652
|
-
live.startedAt = Date.now();
|
|
86478
|
+
if (applyDeliveredHoldReset(live, Date.now())) {
|
|
85653
86479
|
reconcileBlockedApprovals();
|
|
85654
86480
|
process.stderr.write(`telegram gateway: permission-card RE-DELIVERED request=${requestId} tool=${live.tool_name} chat=${chatId} \u2014 the operator can answer now; ` + `TTL clock restarted (they had zero seconds while the channel was shut)
|
|
85655
86481
|
`);
|
|
@@ -85896,10 +86722,11 @@ var inboundCoalescer = createInboundCoalescer({
|
|
|
85896
86722
|
});
|
|
85897
86723
|
function emitGatewayOperatorEvent(event) {
|
|
85898
86724
|
const { agent, kind } = event;
|
|
86725
|
+
event = { ...event, detail: redactOutboundText(event.detail, "operator_event") };
|
|
85899
86726
|
let throttleEscalation = null;
|
|
85900
86727
|
let escalationFired = false;
|
|
85901
86728
|
let rateLimitedCooldownConsulted = false;
|
|
85902
|
-
const rateLimit429Classification = kind === "rate-limited" ?
|
|
86729
|
+
const rateLimit429Classification = kind === "rate-limited" ? classify429Detail2(event.detail) : null;
|
|
85903
86730
|
if (rateLimit429Classification != null && rateLimit429Classification !== "account-scoped") {
|
|
85904
86731
|
emitRuntimeMetric(build429ClassifiedMetric({
|
|
85905
86732
|
agent,
|
|
@@ -85991,6 +86818,18 @@ function emitGatewayOperatorEvent(event) {
|
|
|
85991
86818
|
const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime());
|
|
85992
86819
|
fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt);
|
|
85993
86820
|
}
|
|
86821
|
+
} else if (kind === "rate-limited" || kind === "unknown-5xx") {
|
|
86822
|
+
const parsed = parseLlmError(event.detail);
|
|
86823
|
+
const now = Date.now();
|
|
86824
|
+
if (decideErrorSurface(parsed, agent, { claim: true, now }) === "suppress") {
|
|
86825
|
+
process.stderr.write(`telegram gateway: operator-event collapsed (error-presence-gate) agent=${agent} kind=${kind}
|
|
86826
|
+
`);
|
|
86827
|
+
return;
|
|
86828
|
+
}
|
|
86829
|
+
const tz = process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? "UTC";
|
|
86830
|
+
const r = renderLlmErrorSafe(parsed, agent, tz, new Date(now));
|
|
86831
|
+
renderedText = r.text;
|
|
86832
|
+
renderedKeyboard = undefined;
|
|
85994
86833
|
} else {
|
|
85995
86834
|
try {
|
|
85996
86835
|
const r = renderOperatorEvent(event);
|
|
@@ -86287,6 +87126,10 @@ async function runMidSessionCardReaper() {
|
|
|
86287
87126
|
const reaps = decideWorkerPinReaps({
|
|
86288
87127
|
pins: candidates,
|
|
86289
87128
|
statusOf: (agentId) => {
|
|
87129
|
+
if (agentId.startsWith("group:")) {
|
|
87130
|
+
const feedKey = agentId.slice("group:".length);
|
|
87131
|
+
return workerActivityFeed?.hasRunningInFeed(feedKey) ? "running" : "terminal";
|
|
87132
|
+
}
|
|
86290
87133
|
if (turnsDb == null)
|
|
86291
87134
|
return "unknown";
|
|
86292
87135
|
try {
|
|
@@ -86386,24 +87229,6 @@ async function reconcileStatusPinInner(pinKey, chatId, desired) {
|
|
|
86386
87229
|
statusPinPinnedAt.set(pinKey, Date.now());
|
|
86387
87230
|
}
|
|
86388
87231
|
}
|
|
86389
|
-
function reconcileWorkerPin(agentId, chatId, running) {
|
|
86390
|
-
if (!PIN_STATUS_WHILE_WORKING)
|
|
86391
|
-
return;
|
|
86392
|
-
const key = `wk:${agentId}`;
|
|
86393
|
-
if (!running) {
|
|
86394
|
-
const unpinChat = chatId ?? statusPinChatIds.get(key);
|
|
86395
|
-
if (unpinChat == null)
|
|
86396
|
-
return;
|
|
86397
|
-
reconcileStatusPin(key, unpinChat, { pinned: false });
|
|
86398
|
-
return;
|
|
86399
|
-
}
|
|
86400
|
-
if (chatId == null)
|
|
86401
|
-
return;
|
|
86402
|
-
const messageId = workerActivityFeed?.messageIdOf(agentId) ?? null;
|
|
86403
|
-
if (messageId == null)
|
|
86404
|
-
return;
|
|
86405
|
-
reconcileStatusPin(key, chatId, { pinned: true, messageId });
|
|
86406
|
-
}
|
|
86407
87232
|
async function unpinAllStatusPins() {
|
|
86408
87233
|
const keys = [...statusPinState.keys()];
|
|
86409
87234
|
for (const key of keys) {
|
|
@@ -87036,7 +87861,8 @@ var bridgeDeadWatchdog = createBridgeDeadWatchdog({
|
|
|
87036
87861
|
markerPath: join54(STATE_DIR, "bridge-dead-escalation.json"),
|
|
87037
87862
|
log: (line) => process.stderr.write(`${line}
|
|
87038
87863
|
`),
|
|
87039
|
-
priorStreak: bridgeDeadPriorStreak
|
|
87864
|
+
priorStreak: bridgeDeadPriorStreak,
|
|
87865
|
+
selfAgentName: process.env.SWITCHROOM_AGENT_NAME ?? ""
|
|
87040
87866
|
});
|
|
87041
87867
|
if (BRIDGE_DEAD_ESCALATION_ENABLED) {
|
|
87042
87868
|
bridgeDeadWatchdog.arm();
|
|
@@ -89662,7 +90488,7 @@ async function executeEditMessage(args) {
|
|
|
89662
90488
|
editRawText = normalizeParagraphBreaks2(editRawText);
|
|
89663
90489
|
editRawText = redactOutboundText(editRawText, "edit_message");
|
|
89664
90490
|
if (!editLiteralText)
|
|
89665
|
-
editRawText =
|
|
90491
|
+
editRawText = stripExcessBold2(normalizePunctuation2(editRawText));
|
|
89666
90492
|
{
|
|
89667
90493
|
const scrub = scrubVoice2(editRawText);
|
|
89668
90494
|
if (scrub.replaced > 0) {
|
|
@@ -90837,7 +91663,7 @@ function handleSessionEvent(ev) {
|
|
|
90837
91663
|
link_preview_options: { is_disabled: true }
|
|
90838
91664
|
};
|
|
90839
91665
|
const limit = RICH_MESSAGE_MAX_CHARS2;
|
|
90840
|
-
const renderedText =
|
|
91666
|
+
const renderedText = capturedText;
|
|
90841
91667
|
const htmlChunks = splitMarkdownChunks2(renderedText, limit);
|
|
90842
91668
|
const sentIds = [];
|
|
90843
91669
|
try {
|
|
@@ -93190,11 +94016,13 @@ function recordTypedModelSwitch(reply, requestedModelArg, _deps) {
|
|
|
93190
94016
|
if (!reply.selectedModel)
|
|
93191
94017
|
return "";
|
|
93192
94018
|
sessionModelSource.setOverride(reply.selectedModel);
|
|
94019
|
+
clearPremiumRecoveryOnManualSwitch(reply.selectedModel);
|
|
93193
94020
|
return "";
|
|
93194
94021
|
}
|
|
93195
94022
|
function recordModelMenuSideEffects(outcome, modelDeps, cbChatId, cbThreadId, prevSessionModel) {
|
|
93196
94023
|
if (outcome.selectedModel) {
|
|
93197
94024
|
sessionModelSource.setOverride(outcome.selectedModel);
|
|
94025
|
+
clearPremiumRecoveryOnManualSwitch(outcome.selectedModel);
|
|
93198
94026
|
}
|
|
93199
94027
|
if (outcome.clearedDefault) {
|
|
93200
94028
|
const smDir = resolveAgentDirFromEnv();
|
|
@@ -94093,6 +94921,80 @@ function broadcastFleetFallbackFailure(triggerAgent, reason) {
|
|
|
94093
94921
|
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(html), {}), { chat_id, verb: "fleet-fallback:failure-notify" });
|
|
94094
94922
|
}
|
|
94095
94923
|
}
|
|
94924
|
+
function broadcastTierNotice(markdown) {
|
|
94925
|
+
const access = loadAccess();
|
|
94926
|
+
if (access.allowFrom.length === 0)
|
|
94927
|
+
return;
|
|
94928
|
+
for (const chat_id of access.allowFrom) {
|
|
94929
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(markdown), { disable_notification: true }), { chat_id: String(chat_id), verb: "tier-downgrade:notify" });
|
|
94930
|
+
}
|
|
94931
|
+
}
|
|
94932
|
+
function maybeTierDowngrade(triggerAgent) {
|
|
94933
|
+
return runTierDowngrade(triggerAgent, {
|
|
94934
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
94935
|
+
getConfiguredDefault: () => {
|
|
94936
|
+
const dir = resolveAgentDirFromEnv();
|
|
94937
|
+
if (!dir)
|
|
94938
|
+
return null;
|
|
94939
|
+
return resolveMainModel(readConfiguredDefaultModel(dir) ?? undefined);
|
|
94940
|
+
},
|
|
94941
|
+
getSessionOverride: () => sessionModelSource.getOverride(),
|
|
94942
|
+
resolve: (t) => resolveMainModel(t),
|
|
94943
|
+
peekResumeGate: () => fleetFallbackResumeGate.peek(newestActiveTurnStartedAtMs()),
|
|
94944
|
+
writeCarrier: (dir, toModel, cfg) => writeSessionModelFile(dir, toModel, cfg),
|
|
94945
|
+
armResumeGate: () => fleetFallbackResumeGate.arm(),
|
|
94946
|
+
writeRecoveryMarker: (dir, premiumModel) => {
|
|
94947
|
+
const chats = loadAccess().allowFrom.map((c) => String(c));
|
|
94948
|
+
if (chats.length > 0)
|
|
94949
|
+
writePremiumRecoveryFile(dir, premiumModel, chats);
|
|
94950
|
+
},
|
|
94951
|
+
broadcastNotice: (md) => broadcastTierNotice(md),
|
|
94952
|
+
selfRestart: (agent) => triggerSelfRestart(agent, "tier-downgrade-resume"),
|
|
94953
|
+
selfAgent: (t) => process.env.SWITCHROOM_AGENT_NAME ?? t,
|
|
94954
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
94955
|
+
`)
|
|
94956
|
+
});
|
|
94957
|
+
}
|
|
94958
|
+
async function maybePremiumRecoveryPing(brokerClient, accounts) {
|
|
94959
|
+
return runPremiumRecoveryPing({
|
|
94960
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
94961
|
+
readMarker: (dir) => readPremiumRecoveryFile(dir),
|
|
94962
|
+
clearMarker: (dir) => clearPremiumRecoveryFile(dir),
|
|
94963
|
+
getAgent: () => getMyAgentName(),
|
|
94964
|
+
decide: () => decidePremiumRecovery({
|
|
94965
|
+
hasMarker: true,
|
|
94966
|
+
accounts: accounts.map((a) => ({
|
|
94967
|
+
exhausted: a.exhausted,
|
|
94968
|
+
premiumWalled: a.premium_walled === true
|
|
94969
|
+
}))
|
|
94970
|
+
}).fire,
|
|
94971
|
+
claimNotification: (key) => claimQuotaNotification(brokerClient, key),
|
|
94972
|
+
fallbackChats: () => loadAccess().allowFrom.map((c) => String(c)),
|
|
94973
|
+
sendToChat: (chat_id, ping, keyboard) => {
|
|
94974
|
+
swallowingApiCall(() => bot.api.sendRichMessage(chat_id, richMessage2(ping.text), {
|
|
94975
|
+
disable_notification: true,
|
|
94976
|
+
reply_markup: keyboard
|
|
94977
|
+
}), { chat_id: String(chat_id), verb: "premium-recovery:notify" });
|
|
94978
|
+
},
|
|
94979
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
94980
|
+
`)
|
|
94981
|
+
});
|
|
94982
|
+
}
|
|
94983
|
+
function clearPremiumRecoveryOnManualSwitch(appliedToken) {
|
|
94984
|
+
if (appliedToken == null || appliedToken.length === 0)
|
|
94985
|
+
return;
|
|
94986
|
+
const agentDir = resolveAgentDirFromEnv();
|
|
94987
|
+
if (!agentDir)
|
|
94988
|
+
return;
|
|
94989
|
+
const marker = readPremiumRecoveryFile(agentDir);
|
|
94990
|
+
if (marker == null)
|
|
94991
|
+
return;
|
|
94992
|
+
if (resolveMainModel(appliedToken) === resolveMainModel(marker.premiumModel)) {
|
|
94993
|
+
clearPremiumRecoveryFile(agentDir);
|
|
94994
|
+
process.stderr.write(`telegram gateway: [premium-recovery] marker cleared \u2014 user manually re-issued /model ${marker.premiumModel}
|
|
94995
|
+
`);
|
|
94996
|
+
}
|
|
94997
|
+
}
|
|
94096
94998
|
async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, trigger) {
|
|
94097
94999
|
try {
|
|
94098
95000
|
const client3 = await getAuthBrokerClient2(triggerAgent);
|
|
@@ -94126,6 +95028,10 @@ async function doFireFleetAutoFallback(triggerAgent, untilMs, parsedResetAt, tri
|
|
|
94126
95028
|
if (outcome.kind === "switched") {
|
|
94127
95029
|
fallbackAllBlockedNoticeState = { lastSentAtMs: 0 };
|
|
94128
95030
|
} else if (outcome.kind === "all-blocked") {
|
|
95031
|
+
const tier = maybeTierDowngrade(triggerAgent);
|
|
95032
|
+
if (tier === "downgraded" || tier === "restart-pending") {
|
|
95033
|
+
return false;
|
|
95034
|
+
}
|
|
94129
95035
|
const verdict = evaluateAllBlockedNotice(fallbackAllBlockedNoticeState, Date.now());
|
|
94130
95036
|
if (!verdict.send) {
|
|
94131
95037
|
process.stderr.write(`telegram gateway: [fleet-fallback] all-blocked card suppressed (cooldown) agent=${triggerAgent}
|
|
@@ -94235,6 +95141,10 @@ async function runQuotaWatch(opts = {}) {
|
|
|
94235
95141
|
if (!listStateData.accounts || listStateData.accounts.length === 0) {
|
|
94236
95142
|
return;
|
|
94237
95143
|
}
|
|
95144
|
+
await maybePremiumRecoveryPing(brokerClient, listStateData.accounts).catch((err) => {
|
|
95145
|
+
process.stderr.write(`telegram gateway: [premium-recovery] ping check failed (non-fatal): ${err?.message ?? err}
|
|
95146
|
+
`);
|
|
95147
|
+
});
|
|
94238
95148
|
const snapshots = buildSnapshotsFromCachedState2(listStateData);
|
|
94239
95149
|
let watchState = loadQuotaWatchState(stateDir);
|
|
94240
95150
|
const now = Date.now();
|
|
@@ -97336,6 +98246,10 @@ var didOneTimeSetup = false;
|
|
|
97336
98246
|
const watcherAgentDir = resolveAgentDirFromEnv();
|
|
97337
98247
|
if (watcherAgentDir != null) {
|
|
97338
98248
|
const workerFeedEnabled = isWorkerActivityFeedEnabled(process.env.SWITCHROOM_WORKER_ACTIVITY_FEED);
|
|
98249
|
+
const workerFeedMaxRows = (() => {
|
|
98250
|
+
const raw = Number(process.env.SWITCHROOM_TG_WORKER_FEED_MAX_ROWS);
|
|
98251
|
+
return Number.isInteger(raw) && raw > 0 ? raw : undefined;
|
|
98252
|
+
})();
|
|
97339
98253
|
const foregroundNestingEnabled = process.env.SWITCHROOM_FOREGROUND_SUBAGENT_NESTING !== "0";
|
|
97340
98254
|
const orphanStatusEnabled = isOrphanSubagentStatusEnabled(process.env.SWITCHROOM_ORPHAN_SUBAGENT_STATUS);
|
|
97341
98255
|
workerActivityFeed?.stop();
|
|
@@ -97354,6 +98268,20 @@ var didOneTimeSetup = false;
|
|
|
97354
98268
|
})
|
|
97355
98269
|
},
|
|
97356
98270
|
floodWaitRemainingMs: probeFloodWaitRemainingMs,
|
|
98271
|
+
maxRows: workerFeedMaxRows,
|
|
98272
|
+
reconcilePin: ({ feedKey, chatId, messageId }) => {
|
|
98273
|
+
if (!PIN_STATUS_WHILE_WORKING)
|
|
98274
|
+
return;
|
|
98275
|
+
const key = `wk:group:${feedKey}`;
|
|
98276
|
+
if (messageId != null) {
|
|
98277
|
+
reconcileStatusPin(key, chatId, { pinned: true, messageId });
|
|
98278
|
+
} else {
|
|
98279
|
+
const unpinChat = chatId || statusPinChatIds.get(key);
|
|
98280
|
+
if (unpinChat != null && unpinChat.length > 0) {
|
|
98281
|
+
reconcileStatusPin(key, unpinChat, { pinned: false });
|
|
98282
|
+
}
|
|
98283
|
+
}
|
|
98284
|
+
},
|
|
97357
98285
|
log: (msg) => process.stderr.write(`telegram gateway: ${msg}
|
|
97358
98286
|
`)
|
|
97359
98287
|
});
|
|
@@ -97419,7 +98347,6 @@ var didOneTimeSetup = false;
|
|
|
97419
98347
|
state: outcome === "failed" ? "failed" : "done",
|
|
97420
98348
|
model: dispatch.feedModel ?? undefined
|
|
97421
98349
|
});
|
|
97422
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97423
98350
|
}
|
|
97424
98351
|
return;
|
|
97425
98352
|
}
|
|
@@ -97467,7 +98394,6 @@ var didOneTimeSetup = false;
|
|
|
97467
98394
|
state: outcome === "failed" ? "failed" : "done",
|
|
97468
98395
|
model: dispatch.feedModel ?? undefined
|
|
97469
98396
|
});
|
|
97470
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97471
98397
|
}
|
|
97472
98398
|
return;
|
|
97473
98399
|
}
|
|
@@ -97481,7 +98407,6 @@ var didOneTimeSetup = false;
|
|
|
97481
98407
|
state: outcome === "failed" ? "failed" : "done",
|
|
97482
98408
|
model: dispatch.feedModel ?? undefined
|
|
97483
98409
|
});
|
|
97484
|
-
reconcileWorkerPin(agentId, null, false);
|
|
97485
98410
|
}
|
|
97486
98411
|
const handbackOrigin = resolveSubagentOriginChat(agentId);
|
|
97487
98412
|
const decision = decideSubagentHandback({
|
|
@@ -97555,7 +98480,7 @@ var didOneTimeSetup = false;
|
|
|
97555
98480
|
elapsedMs,
|
|
97556
98481
|
state: "running",
|
|
97557
98482
|
model: feedModel
|
|
97558
|
-
}, wk.threadId)
|
|
98483
|
+
}, wk.threadId);
|
|
97559
98484
|
return;
|
|
97560
98485
|
}
|
|
97561
98486
|
if (surface !== "nest")
|
|
@@ -97614,7 +98539,7 @@ var didOneTimeSetup = false;
|
|
|
97614
98539
|
elapsedMs,
|
|
97615
98540
|
state: "running",
|
|
97616
98541
|
model: feedModel
|
|
97617
|
-
}, wk.threadId)
|
|
98542
|
+
}, wk.threadId);
|
|
97618
98543
|
return;
|
|
97619
98544
|
}
|
|
97620
98545
|
const progressOrigin = resolveSubagentOriginChat(agentId);
|