switchroom 0.18.17 → 0.18.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1544 -619
- package/telegram-plugin/dist/server.js +32 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/format.ts +137 -213
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +399 -85
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/outbound-send-path.ts +9 -7
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +474 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-reply-handler.ts +5 -14
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/format-consistency.test.ts +68 -53
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +5 -6
- package/telegram-plugin/tests/formatting-torture-set.ts +1 -1
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +26 -0
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +481 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +4 -3
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +42 -100
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +9 -12
- package/telegram-plugin/tests/telegram-format.test.ts +86 -31
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +17 -21
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/turn-flush-safety.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
import { basename } from "node:path";
|
|
21
21
|
import { escapeMarkdown } from "./card-format.js";
|
|
22
22
|
import { prettyMcpServer, type ScopeOption } from "./permission-rule.js";
|
|
23
|
-
import { redact } from "./secret-detect/redact.js";
|
|
23
|
+
import { redact, REDACTED_MARKER } from "./secret-detect/redact.js";
|
|
24
24
|
|
|
25
25
|
const COMMAND_TITLE_MAX = 48;
|
|
26
26
|
const DESCRIPTION_LINE_MAX = 240;
|
|
@@ -35,6 +35,55 @@ const ARG_SUMMARY_MAX_KEYS = 4; // how many payload keys to surface on the card
|
|
|
35
35
|
const ARG_VALUE_MAX = 40; // per-value truncation in the arg-summary line
|
|
36
36
|
const ARG_SUMMARY_LINE_MAX = 180; // total cap for the arg-summary line
|
|
37
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Input keys that are routing / formatting / id noise, never operator-
|
|
40
|
+
* meaningful context. Stripped when synthesizing the fallback `context:`
|
|
41
|
+
* line (#3167) so it surfaces the substance (the reply text, the command,
|
|
42
|
+
* the query) and not `chat_id` / `format` / `disable_notification` chrome.
|
|
43
|
+
* `reason`/`why` are here too because their presence takes the `why:` path —
|
|
44
|
+
* they never reach the synthesizer.
|
|
45
|
+
*/
|
|
46
|
+
const CONTEXT_NOISE_KEYS = new Set([
|
|
47
|
+
"reason",
|
|
48
|
+
"why",
|
|
49
|
+
"chat_id",
|
|
50
|
+
"message_id",
|
|
51
|
+
"message_thread_id",
|
|
52
|
+
"thread_id",
|
|
53
|
+
"origin_turn_id",
|
|
54
|
+
"reply_to",
|
|
55
|
+
"quote",
|
|
56
|
+
"quote_text",
|
|
57
|
+
"format",
|
|
58
|
+
"parse_mode",
|
|
59
|
+
"disable_web_page_preview",
|
|
60
|
+
"disable_notification",
|
|
61
|
+
"protect_content",
|
|
62
|
+
"single_use",
|
|
63
|
+
"ack_text",
|
|
64
|
+
"inline_keyboard",
|
|
65
|
+
"file_id",
|
|
66
|
+
]);
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Input keys whose VALUE is credential-shaped by name — hard-masked whole,
|
|
70
|
+
* never partially revealed (#3167 review). The token-shape detectors in
|
|
71
|
+
* `redact()` need a random-looking value to fire, so a low-entropy password
|
|
72
|
+
* or a short secret under one of these keys would otherwise slip through the
|
|
73
|
+
* synthesized `context:` line. Matching the key is the reliable signal.
|
|
74
|
+
*/
|
|
75
|
+
const SENSITIVE_VALUE_KEY_RE =
|
|
76
|
+
/token|secret|password|passwd|key|auth|dsn|conn|url|credential|cookie|session/i;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Credential-bearing DSN schemes that `redactUrls()` (inside `redact()`) does
|
|
80
|
+
* NOT cover — it only handles http(s)/ws(s)/ftp. A `postgres://user:pass@host`
|
|
81
|
+
* DATABASE_URL is a real vault-key shape here, so mask the whole DSN when its
|
|
82
|
+
* authority carries a `user:pass@` credential (#3167 review).
|
|
83
|
+
*/
|
|
84
|
+
const NON_HTTP_DSN_RE =
|
|
85
|
+
/\b(?:postgres(?:ql)?|mysql|mariadb|redis|rediss|mongodb(?:\+srv)?|amqp|amqps):\/\/\S*@\S+/gi;
|
|
86
|
+
|
|
38
87
|
/**
|
|
39
88
|
* Human verb-phrases for switchroom-managed MCP tools. The raw
|
|
40
89
|
* `mcp__<server>__<tool>` name is operator-hostile. Phrases are written
|
|
@@ -141,15 +190,23 @@ export function formatPermissionCardBody(opts: {
|
|
|
141
190
|
// static schema description (#2469).
|
|
142
191
|
const callerReason = callerSuppliedReason(opts.inputPreview);
|
|
143
192
|
const rawWhy = (callerReason ?? "").replace(/\s+/g, " ").trim();
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
193
|
+
if (rawWhy.length > 0) {
|
|
194
|
+
const truncatedWhy =
|
|
195
|
+
rawWhy.length > DESCRIPTION_LINE_MAX
|
|
196
|
+
? rawWhy.slice(0, DESCRIPTION_LINE_MAX - 1) + "…"
|
|
197
|
+
: rawWhy;
|
|
198
|
+
lines.push(`why: _${escapeTgHtml(truncatedWhy)}_`);
|
|
199
|
+
} else {
|
|
200
|
+
// No caller reason. Many tools (reply, react, edit_message, …) carry no
|
|
201
|
+
// `reason`/`why` argument at all, so a bare "why: not provided" gives the
|
|
202
|
+
// operator nothing to decide on — the post-rollout permission storm filled
|
|
203
|
+
// the chat with contentless cards (#3167). Fall back to an honest,
|
|
204
|
+
// redaction-safe `context:` line synthesized from the tool + its salient
|
|
205
|
+
// input, so the card always carries something meaningful. The distinct
|
|
206
|
+
// label keeps the agent's omission of a rationale visible (it is NOT a
|
|
207
|
+
// fabricated "why"), while still surfacing what the tool is about to do.
|
|
208
|
+
lines.push(`context: _${escapeTgHtml(synthesizeContext(opts.toolName, opts.inputPreview))}_`);
|
|
209
|
+
}
|
|
153
210
|
|
|
154
211
|
// Third line (REST-wrapper MCP writes only): a redaction-safe summary of
|
|
155
212
|
// the payload so the operator can see WHAT is being sent, not just the
|
|
@@ -597,6 +654,111 @@ function callerSuppliedReason(inputPreview: string | undefined): string | null {
|
|
|
597
654
|
return null;
|
|
598
655
|
}
|
|
599
656
|
|
|
657
|
+
/**
|
|
658
|
+
* Honest, redaction-safe fallback for the card's rationale line when the
|
|
659
|
+
* caller supplied no `reason`/`why` (#3167). Synthesizes context from a
|
|
660
|
+
* summary of the tool's salient input fields — the reply text, the command,
|
|
661
|
+
* the search query — so the operator always has something to decide on
|
|
662
|
+
* instead of a contentless "not provided". Every value passes through
|
|
663
|
+
* `redact()`; ids / routing / formatting keys are stripped. Falls back to
|
|
664
|
+
* the natural action phrase when the input exposes nothing salient.
|
|
665
|
+
* Deterministic (no model in the loop): same input → same line. Exported
|
|
666
|
+
* for unit testing.
|
|
667
|
+
*/
|
|
668
|
+
export function synthesizeContext(
|
|
669
|
+
toolName: string,
|
|
670
|
+
inputPreview: string | undefined,
|
|
671
|
+
): string {
|
|
672
|
+
const input = parseInput(inputPreview);
|
|
673
|
+
const summary = input ? salientInputSummary(input) : null;
|
|
674
|
+
// Prefer the salient input summary (the reply text, the query, the diff
|
|
675
|
+
// target) — the substance the title's verb-phrase does NOT already carry.
|
|
676
|
+
// When the input exposes nothing meaningful, fall back to the natural
|
|
677
|
+
// action so the line still names the tool rather than reading empty.
|
|
678
|
+
const base = summary ?? naturalAction(toolName, inputPreview);
|
|
679
|
+
// Acceptance criteria (#3167) call for "tool + summarized input +
|
|
680
|
+
// originating turn". When the call carries a forum-topic origin turn,
|
|
681
|
+
// append a compact reference so the operator can tie the card to the turn
|
|
682
|
+
// that spawned it. Kept short — it's a routing id, not prose.
|
|
683
|
+
const turnRef = input ? readString(input, "origin_turn_id") : null;
|
|
684
|
+
return turnRef ? `${base} · turn ${shortTurnRef(turnRef)}` : base;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/** Compact display form of an origin-turn id — the tail is the distinguishing
|
|
688
|
+
* part; a full opaque id would swamp the line. */
|
|
689
|
+
function shortTurnRef(turnId: string): string {
|
|
690
|
+
const t = turnId.trim();
|
|
691
|
+
return t.length <= 10 ? t : `…${t.slice(-8)}`;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
/**
|
|
695
|
+
* A compact, redaction-safe summary of the operator-meaningful scalar fields
|
|
696
|
+
* of a tool input — used to build the synthesized `context:` line (#3167).
|
|
697
|
+
* Skips {@link CONTEXT_NOISE_KEYS} (ids/routing/formatting), surfaces up to
|
|
698
|
+
* {@link ARG_SUMMARY_MAX_KEYS} scalar `key: value` pairs (each redacted +
|
|
699
|
+
* truncated), and renders nested objects/arrays as the bare key name (no
|
|
700
|
+
* value dump — avoids leaking PII/secrets and oversized blobs). Returns null
|
|
701
|
+
* when nothing meaningful remains, so the caller falls back to the bare
|
|
702
|
+
* action phrase.
|
|
703
|
+
*
|
|
704
|
+
* Free-text fields (e.g. a reply `text`) surface as content — a conscious
|
|
705
|
+
* choice (#3167 review, LOW-1): the operator NEEDS to see *what* is being
|
|
706
|
+
* sent to judge the card. Exposure is bounded to {@link ARG_VALUE_MAX} chars
|
|
707
|
+
* and every value is redaction-passed via {@link redactSalientValue}, so a
|
|
708
|
+
* card can never leak more than the outbound reply already does in history
|
|
709
|
+
* (both use the same `redact()` chokepoint). We do not mask free text to a
|
|
710
|
+
* placeholder, which would gut the card's usefulness.
|
|
711
|
+
*/
|
|
712
|
+
function salientInputSummary(input: Record<string, unknown>): string | null {
|
|
713
|
+
const parts: string[] = [];
|
|
714
|
+
for (const [key, value] of Object.entries(input)) {
|
|
715
|
+
if (CONTEXT_NOISE_KEYS.has(key)) continue;
|
|
716
|
+
if (value == null) continue;
|
|
717
|
+
if (parts.length >= ARG_SUMMARY_MAX_KEYS) {
|
|
718
|
+
parts.push("…");
|
|
719
|
+
break;
|
|
720
|
+
}
|
|
721
|
+
if (typeof value === "object") {
|
|
722
|
+
parts.push(key); // nested object/array → key name only, never dumped
|
|
723
|
+
continue;
|
|
724
|
+
}
|
|
725
|
+
const shown = truncate(redactSalientValue(key, String(value)), ARG_VALUE_MAX);
|
|
726
|
+
if (shown.length === 0) continue;
|
|
727
|
+
parts.push(`${key}: ${shown}`);
|
|
728
|
+
}
|
|
729
|
+
if (parts.length === 0) return null;
|
|
730
|
+
const joined = parts.join(", ");
|
|
731
|
+
return joined.length > ARG_SUMMARY_LINE_MAX
|
|
732
|
+
? joined.slice(0, ARG_SUMMARY_LINE_MAX - 1) + "…"
|
|
733
|
+
: joined;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/**
|
|
737
|
+
* Redact a single salient value for display on the synthesized `context:`
|
|
738
|
+
* line, given its input KEY (#3167 review). Three layers, defense-in-depth:
|
|
739
|
+
*
|
|
740
|
+
* 1. Hard-mask whole when the key is credential-shaped
|
|
741
|
+
* ({@link SENSITIVE_VALUE_KEY_RE}) — the token-shape detectors need a
|
|
742
|
+
* random-looking value, so a low-entropy password / short secret under a
|
|
743
|
+
* `token`/`password`/`api_key` key would otherwise leak. The key name is
|
|
744
|
+
* the reliable signal.
|
|
745
|
+
* 2. Mask non-http DSN credentials ({@link NON_HTTP_DSN_RE}) that
|
|
746
|
+
* `redactUrls()` misses (postgres://user:pass@host, mysql://, redis://…).
|
|
747
|
+
* 3. Run `redact()` WITH the key restored as `key=value` context, so the
|
|
748
|
+
* contextual detectors (`kv_entropy` / `env_key_value`) that require the
|
|
749
|
+
* `key[:=]value` shape in one string can fire — then strip the synthetic
|
|
750
|
+
* `key=` prefix. Redacting the BARE value (the pre-review bug) blinded
|
|
751
|
+
* those detectors, since `redact()` deliberately excludes the
|
|
752
|
+
* low-precision `generic_high_entropy` fallback.
|
|
753
|
+
*/
|
|
754
|
+
function redactSalientValue(key: string, value: string): string {
|
|
755
|
+
if (SENSITIVE_VALUE_KEY_RE.test(key)) return REDACTED_MARKER;
|
|
756
|
+
const dsnScrubbed = value.replace(NON_HTTP_DSN_RE, REDACTED_MARKER);
|
|
757
|
+
const scrubbed = redact(`${key}=${dsnScrubbed}`);
|
|
758
|
+
const prefix = `${key}=`;
|
|
759
|
+
return scrubbed.startsWith(prefix) ? scrubbed.slice(prefix.length) : scrubbed;
|
|
760
|
+
}
|
|
761
|
+
|
|
600
762
|
/**
|
|
601
763
|
* Regex-based fallback to extract a `reason` or `why` value from a raw
|
|
602
764
|
* (possibly truncated / invalid-JSON) inputPreview string. Mirrors
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* premium-recovery.ts — "your premium model is servable again" ping (pure layer).
|
|
3
|
+
*
|
|
4
|
+
* Companion to tier-downgrade.ts. When a premium `/model` selection (e.g.
|
|
5
|
+
* `/model fable`) is walled fleet-wide, the tier-downgrade tier resumes the
|
|
6
|
+
* turn on the configured default and tells the user to re-issue `/model fable`
|
|
7
|
+
* once it frees up (there is NO automatic revert — the override is session-
|
|
8
|
+
* scoped and dies on the downgrade restart). This module closes that loop with
|
|
9
|
+
* a HEADS-UP: the moment the premium tier can be served again, ping the chat
|
|
10
|
+
* ONCE with a one-tap "switch back" button.
|
|
11
|
+
*
|
|
12
|
+
* DETERMINISTIC recovery signal (P0 deterministic-controls — never model-
|
|
13
|
+
* decided). The signal is the auth-broker's own per-account eligibility, read
|
|
14
|
+
* off `list-state` (the SAME state the gateway's `runQuotaWatch` tick already
|
|
15
|
+
* polls every 15 min — no new poller, no broker change):
|
|
16
|
+
* - `account.exhausted` — live-authoritative `isAccountExhausted` verdict
|
|
17
|
+
* (5h/7d wall or an unexpired `exhausted_until` mark).
|
|
18
|
+
* - `account.premium_walled` — live-authoritative `isAccountPremiumWalled`
|
|
19
|
+
* verdict; `isModelTierWalled` is a pure timestamp compare
|
|
20
|
+
* (`premium_walled_until > now`) unless a fresher canary read the flagship
|
|
21
|
+
* tier `allowed`. (src/auth/broker/model-tier-quota.ts / account-eligibility.ts.)
|
|
22
|
+
* The tier-downgrade fires from the account-swap `all-blocked` verdict — i.e.
|
|
23
|
+
* EVERY account was exhausted or premium-walled. Recovery is the exact
|
|
24
|
+
* complement: at least ONE account is neither. No clock of our own, no model
|
|
25
|
+
* judgement — the broker already decided; we read its decision.
|
|
26
|
+
*
|
|
27
|
+
* AT-MOST-ONCE. The gateway holds a consume-once `.premium-recovery` marker
|
|
28
|
+
* (session-model-file.ts) recording the dropped premium token + the chats to
|
|
29
|
+
* notify. `decidePremiumRecovery` is a pure predicate; the gateway clears the
|
|
30
|
+
* marker BEFORE sending (never-storm) and additionally takes a fleet-wide
|
|
31
|
+
* `claim-notification` so a bounce or a second gateway tick can't double-fire.
|
|
32
|
+
* The marker is also cleared the instant the user manually re-issues
|
|
33
|
+
* `/model <premium>` (no stale ping).
|
|
34
|
+
*
|
|
35
|
+
* PURE — no I/O, no clock. The gateway does the list-state read, the marker
|
|
36
|
+
* FS, the claim, and the send off these verdicts.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/** The minimal per-account eligibility view the recovery predicate needs —
|
|
40
|
+
* both fields are the broker's live-authoritative verdicts from `list-state`. */
|
|
41
|
+
export interface AccountRecoveryView {
|
|
42
|
+
/** `isAccountExhausted` — a 5h/7d wall or an unexpired `exhausted_until` mark. */
|
|
43
|
+
exhausted: boolean
|
|
44
|
+
/** `isAccountPremiumWalled` — the flagship (`7d_oi`) tier wall verdict. */
|
|
45
|
+
premiumWalled: boolean
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface PremiumRecoveryDecision {
|
|
49
|
+
/** Fire exactly one ping now (a marker is pending AND the premium tier is
|
|
50
|
+
* servable again). The gateway clears the marker + claims before sending. */
|
|
51
|
+
fire: boolean
|
|
52
|
+
/** Why not, for the log (never surfaced to the user). */
|
|
53
|
+
reason: 'no-marker' | 'still-walled' | 'recovered'
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Decide whether the premium tier has recovered for a pending downgrade.
|
|
58
|
+
* PURE. Fires iff a marker is pending AND at least one account can serve the
|
|
59
|
+
* premium tier again (neither exhausted nor premium-walled) — the exact
|
|
60
|
+
* complement of the `all-blocked` condition that fired the downgrade.
|
|
61
|
+
*/
|
|
62
|
+
export function decidePremiumRecovery(opts: {
|
|
63
|
+
hasMarker: boolean
|
|
64
|
+
accounts: ReadonlyArray<AccountRecoveryView>
|
|
65
|
+
}): PremiumRecoveryDecision {
|
|
66
|
+
if (!opts.hasMarker) return { fire: false, reason: 'no-marker' }
|
|
67
|
+
const servable = opts.accounts.some((a) => !a.exhausted && !a.premiumWalled)
|
|
68
|
+
return servable
|
|
69
|
+
? { fire: true, reason: 'recovered' }
|
|
70
|
+
: { fire: false, reason: 'still-walled' }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface PremiumRecoveryPing {
|
|
74
|
+
/** Telegram-formatted (rich-markdown) body. */
|
|
75
|
+
text: string
|
|
76
|
+
/** Inline-keyboard button label. */
|
|
77
|
+
buttonText: string
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The user-facing recovery ping. PURE + deterministic so the wording + button
|
|
82
|
+
* are pinned by a test. Honest: the switch it offers is SESSION-SCOPED (the
|
|
83
|
+
* same `/model` semantics as everywhere else — it reverts on the next restart),
|
|
84
|
+
* so it never promises a durable pin.
|
|
85
|
+
*/
|
|
86
|
+
export function renderPremiumRecoveryPing(premiumModel: string): PremiumRecoveryPing {
|
|
87
|
+
return {
|
|
88
|
+
text:
|
|
89
|
+
`✅ \`${premiumModel}\` is available again — tap to switch back to it for this session.`,
|
|
90
|
+
buttonText: `Switch to ${premiumModel}`,
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The fleet-wide `claim-notification` key for a premium-recovery ping. Keyed on
|
|
96
|
+
* agent + the specific premium token so two different dropped models never
|
|
97
|
+
* dedup each other, and a bounce mid-window can't re-send the same one.
|
|
98
|
+
*/
|
|
99
|
+
export function premiumRecoveryClaimKey(agent: string, premiumModel: string): string {
|
|
100
|
+
return `premium-recovery:${agent}:${premiumModel}`
|
|
101
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* raw-error-scrub.ts — the ONE scrubber that removes raw-API-error bytes from a
|
|
3
|
+
* string before it reaches a user, shared by every surface that might print an
|
|
4
|
+
* error `detail`.
|
|
5
|
+
*
|
|
6
|
+
* Zero dependencies (a leaf module) so both the pure `operator-events.ts`
|
|
7
|
+
* renderer AND `llm-error-present.ts` can import it without a cycle. Anchored on
|
|
8
|
+
* the same markers as `looksLikeRawApiError` (pty-partial-handler.ts): the CLI's
|
|
9
|
+
* Python `· b'{…}'` byte-blob, a trailing `{"type":"error"…}` JSON object, and an
|
|
10
|
+
* `API Error:` prefix.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Strip the raw-API-error bytes a source can smuggle into an otherwise-human
|
|
15
|
+
* string. Total — never throws; a clean human string passes through unchanged
|
|
16
|
+
* (modulo whitespace tidy-up).
|
|
17
|
+
*/
|
|
18
|
+
export function stripRawErrorBytes(raw: string): string {
|
|
19
|
+
if (typeof raw !== 'string' || raw.length === 0) return ''
|
|
20
|
+
let s = raw
|
|
21
|
+
// 1. `API Error:` / `API Error 429:` prefix anywhere.
|
|
22
|
+
s = s.replace(/API Error:?\s*\d*\s*/gi, ' ')
|
|
23
|
+
// 2. Python byte-blob render: ` b'{…}'` or ` b"{…}"` (the CLI's raw-body echo).
|
|
24
|
+
s = s.replace(/\bb'[^']*'/g, ' ')
|
|
25
|
+
s = s.replace(/\bb"[^"]*"/g, ' ')
|
|
26
|
+
// 3. A JSON error object `{"type":"error"…}` (or `'type': 'error'`) and
|
|
27
|
+
// everything after it — these blobs are always trailing on the real lines,
|
|
28
|
+
// and brace-balanced stripping is not worth the fragility.
|
|
29
|
+
s = s.replace(/[·\-\s]*\{[\s\S]*?["']type["']\s*:\s*["']error["'][\s\S]*$/i, ' ')
|
|
30
|
+
// 4. A bare leading/trailing JSON object with no human text around it.
|
|
31
|
+
s = s.replace(/^\s*\{[\s\S]*\}\s*$/g, ' ')
|
|
32
|
+
// 5. Tidy: collapse whitespace, drop dangling separators.
|
|
33
|
+
s = s.replace(/\s+/g, ' ').replace(/[·:\-\s]+$/g, '').replace(/^[·:\-\s]+/g, '').trim()
|
|
34
|
+
return s
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Pull an Anthropic `request_id` out of a raw error string, or undefined. The
|
|
39
|
+
* id lives in the trailing byte-blob (`…,"request_id":"req_abc"}'`) and can sit
|
|
40
|
+
* far into a long body — see `truncateDetailPreservingRequestId`. Total.
|
|
41
|
+
*/
|
|
42
|
+
export function extractRequestId(raw: string): string | undefined {
|
|
43
|
+
if (typeof raw !== 'string' || raw.length === 0) return undefined
|
|
44
|
+
const m =
|
|
45
|
+
raw.match(/["']request[_-]?id["']\s*:\s*["']([A-Za-z0-9._-]+)["']/i) ??
|
|
46
|
+
raw.match(/\brequest[_-]?id[=:]\s*([A-Za-z0-9._-]+)/i)
|
|
47
|
+
return m ? m[1] : undefined
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Truncate an error `detail` to `max` chars WITHOUT losing the `request_id`.
|
|
52
|
+
*
|
|
53
|
+
* Why this exists: the bridge forwards operator-event detail truncated to 1000
|
|
54
|
+
* chars (OPERATOR_EVENT_DETAIL_MAX), but the Anthropic `request_id` lives in the
|
|
55
|
+
* trailing byte-blob, which can sit PAST char 1000. A naive `slice(0, 1000)`
|
|
56
|
+
* drops it, so the cross-surface dedup gate silently degrades from the reliable
|
|
57
|
+
* `rid:` EXACT key to the coarse `${kind}:${agent}:${bucket}` key — collapsing
|
|
58
|
+
* two genuinely-distinct same-kind errors within 60s into one card.
|
|
59
|
+
*
|
|
60
|
+
* Fix: extract the id from the FULL body first; if the plain head would drop it,
|
|
61
|
+
* append `request_id=<id>` in the freed tail budget (`extractRequestId` matches
|
|
62
|
+
* that form), keeping the result ≤ `max`. Total — never throws.
|
|
63
|
+
*/
|
|
64
|
+
export function truncateDetailPreservingRequestId(detail: string, max: number): string {
|
|
65
|
+
if (typeof detail !== 'string') return ''
|
|
66
|
+
if (detail.length <= max) return detail
|
|
67
|
+
const rid = extractRequestId(detail)
|
|
68
|
+
const head = detail.slice(0, max)
|
|
69
|
+
if (rid == null || head.includes(rid)) return head
|
|
70
|
+
const suffix = ` request_id=${rid}`
|
|
71
|
+
const headBudget = Math.max(0, max - suffix.length)
|
|
72
|
+
return `${detail.slice(0, headBudget)}${suffix}`
|
|
73
|
+
}
|
|
@@ -121,8 +121,14 @@ export interface RetryApiCallConfig {
|
|
|
121
121
|
* a container restart during an active ban can suppress non-essential
|
|
122
122
|
* sends (boot cards) instead of feeding the same per-bot-token flood
|
|
123
123
|
* counter and prolonging the ban. Best-effort; a throw here is swallowed.
|
|
124
|
+
*
|
|
125
|
+
* The call's `opts` are passed through (#3111) so the hook can open a
|
|
126
|
+
* SCOPE-PRECISE send-gate window: this fires even for a SHORT slept-and-retried
|
|
127
|
+
* 429 that never throws `FLOOD_WAIT_ACTIVE`, so without the opts the gateway
|
|
128
|
+
* could only open a blanket `global` window for those. Existing callers that
|
|
129
|
+
* ignore the second argument are unaffected.
|
|
124
130
|
*/
|
|
125
|
-
onFloodWait?: (retryAfterSec: number) => void
|
|
131
|
+
onFloodWait?: (retryAfterSec: number, opts?: RetryCallOpts) => void
|
|
126
132
|
}
|
|
127
133
|
|
|
128
134
|
/**
|
|
@@ -353,7 +359,7 @@ export function createRetryApiCall(
|
|
|
353
359
|
// Persist the flood window so a restart during the ban can suppress
|
|
354
360
|
// non-essential sends instead of extending it (#2923 circuit breaker).
|
|
355
361
|
try {
|
|
356
|
-
onFloodWait?.(retryAfter)
|
|
362
|
+
onFloodWait?.(retryAfter, opts)
|
|
357
363
|
} catch {
|
|
358
364
|
/* best-effort — never let the breaker hook break the retry path */
|
|
359
365
|
}
|
|
@@ -566,11 +566,162 @@ describe('send-gate PR2: opening a window from a 429, and persistence hook', ()
|
|
|
566
566
|
).rejects.toBe(floodErr)
|
|
567
567
|
|
|
568
568
|
const scopes = opened.map((o) => o.scopeKey).sort()
|
|
569
|
+
// #3111: a chat-bound 429 opens ONLY the chat/group/msg-edit scopes — NOT a
|
|
570
|
+
// coincident `global` window (which would suppress unrelated chats).
|
|
569
571
|
// H1: msg-edit scope carries the chat_id (msg-edit:7:3), not the bare id.
|
|
570
|
-
expect(scopes).toEqual(['chat:7', '
|
|
572
|
+
expect(scopes).toEqual(['chat:7', 'group:7', 'msg-edit:7:3'])
|
|
571
573
|
// After the window opens, a later cosmetic on the same chat sheds.
|
|
572
574
|
const shed = await gate.gate(async () => 'x', { chat_id: '7', priorityClass: 'cosmetic' })
|
|
573
575
|
expect(shed).toBe(SEND_GATE_SHED)
|
|
574
576
|
expect(gate.stats().global.shed).toBe(1)
|
|
575
577
|
})
|
|
576
578
|
})
|
|
579
|
+
|
|
580
|
+
describe('send-gate #3111: scope-precise flood windows', () => {
|
|
581
|
+
// A structured FLOOD_WAIT_ACTIVE the wrapped `fn` throws to simulate a 429
|
|
582
|
+
// whose remaining window (`untilTs`) exceeds the fail-fast ceiling.
|
|
583
|
+
function floodErrUntil(untilTs: number) {
|
|
584
|
+
const sec = Math.ceil(untilTs / 1000)
|
|
585
|
+
return Object.assign(new Error('FLOOD_WAIT_ACTIVE'), {
|
|
586
|
+
retryAfterSec: sec,
|
|
587
|
+
untilTs,
|
|
588
|
+
error_code: 429 as const,
|
|
589
|
+
parameters: { retry_after: sec },
|
|
590
|
+
original: null,
|
|
591
|
+
})
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
it('a chat:A 429 suppresses only chat:A — an unrelated chat:B critical still sends', async () => {
|
|
595
|
+
const clock = new FakeClock()
|
|
596
|
+
const opened: string[] = []
|
|
597
|
+
const gate = createSendGate({
|
|
598
|
+
enabled: true,
|
|
599
|
+
clock,
|
|
600
|
+
criticalFailFastMs: 60_000,
|
|
601
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
602
|
+
})
|
|
603
|
+
|
|
604
|
+
// A LONG (10-min) 429 on chat:A opens the flood window via the gate's catch.
|
|
605
|
+
const floodErr = floodErrUntil(600_000)
|
|
606
|
+
await expect(
|
|
607
|
+
gate.gate(
|
|
608
|
+
async () => {
|
|
609
|
+
throw floodErr
|
|
610
|
+
},
|
|
611
|
+
{ chat_id: 'A', priorityClass: 'critical' },
|
|
612
|
+
),
|
|
613
|
+
).rejects.toBe(floodErr)
|
|
614
|
+
|
|
615
|
+
// Only chat:A was suppressed — pre-#3111 this ALSO opened `global`.
|
|
616
|
+
expect(opened.sort()).toEqual(['chat:A'])
|
|
617
|
+
// The 429 came from `fn`, not a fail-fast admission.
|
|
618
|
+
expect(gate.stats().global.failedFast).toBe(0)
|
|
619
|
+
|
|
620
|
+
// An unrelated chat:B critical is NOT fail-fasted — it sends. Pre-#3111 the
|
|
621
|
+
// coincident global window would have fail-fasted this critical too.
|
|
622
|
+
const bResult = await gate.gate(async () => 'B-sent', {
|
|
623
|
+
chat_id: 'B',
|
|
624
|
+
priorityClass: 'critical',
|
|
625
|
+
})
|
|
626
|
+
expect(bResult).toBe('B-sent')
|
|
627
|
+
expect(gate.stats().global.failedFast).toBe(0)
|
|
628
|
+
expect(gate.stats().global.sent).toBe(1)
|
|
629
|
+
|
|
630
|
+
// A chat:A critical IS fail-fasted — its window exceeds the ceiling.
|
|
631
|
+
let caught: unknown
|
|
632
|
+
await gate
|
|
633
|
+
.gate(async () => 'A-sent', { chat_id: 'A', priorityClass: 'critical' })
|
|
634
|
+
.catch((e) => {
|
|
635
|
+
caught = e
|
|
636
|
+
})
|
|
637
|
+
expect(isFloodWaitActiveError(caught)).toBe(true)
|
|
638
|
+
expect(gate.stats().global.failedFast).toBe(1)
|
|
639
|
+
// chat:B's critical really was the only additional send.
|
|
640
|
+
expect(gate.stats().global.sent).toBe(1)
|
|
641
|
+
})
|
|
642
|
+
|
|
643
|
+
it('a global 429 (no chat scope) still opens the global window and suppresses everywhere', async () => {
|
|
644
|
+
const clock = new FakeClock()
|
|
645
|
+
const opened: string[] = []
|
|
646
|
+
const gate = createSendGate({
|
|
647
|
+
enabled: true,
|
|
648
|
+
clock,
|
|
649
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
650
|
+
})
|
|
651
|
+
|
|
652
|
+
// A 429 on a call with NO chat_id → the evidence IS global.
|
|
653
|
+
const floodErr = floodErrUntil(HOUR)
|
|
654
|
+
await expect(
|
|
655
|
+
gate.gate(
|
|
656
|
+
async () => {
|
|
657
|
+
throw floodErr
|
|
658
|
+
},
|
|
659
|
+
{ priorityClass: 'critical' },
|
|
660
|
+
),
|
|
661
|
+
).rejects.toBe(floodErr)
|
|
662
|
+
|
|
663
|
+
expect(opened).toEqual(['global'])
|
|
664
|
+
// An unrelated chat's cosmetic sheds — global covers every scope.
|
|
665
|
+
const shed = await gate.gate(async () => 'x', { chat_id: 'Z', priorityClass: 'cosmetic' })
|
|
666
|
+
expect(shed).toBe(SEND_GATE_SHED)
|
|
667
|
+
expect(gate.stats().global.shed).toBe(1)
|
|
668
|
+
})
|
|
669
|
+
|
|
670
|
+
it('conservativeGlobalFloodScope restores the pre-#3111 always-open-global posture', async () => {
|
|
671
|
+
const clock = new FakeClock()
|
|
672
|
+
const opened: string[] = []
|
|
673
|
+
const gate = createSendGate({
|
|
674
|
+
enabled: true,
|
|
675
|
+
clock,
|
|
676
|
+
conservativeGlobalFloodScope: true,
|
|
677
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
678
|
+
})
|
|
679
|
+
|
|
680
|
+
const floodErr = floodErrUntil(HOUR)
|
|
681
|
+
await expect(
|
|
682
|
+
gate.gate(
|
|
683
|
+
async () => {
|
|
684
|
+
throw floodErr
|
|
685
|
+
},
|
|
686
|
+
{ chat_id: 'A', priorityClass: 'critical' },
|
|
687
|
+
),
|
|
688
|
+
).rejects.toBe(floodErr)
|
|
689
|
+
|
|
690
|
+
// Conservative posture: a chat-bound 429 ALSO opens the global window.
|
|
691
|
+
expect(opened.sort()).toEqual(['chat:A', 'global'])
|
|
692
|
+
})
|
|
693
|
+
|
|
694
|
+
it('openScopedFloodWindows (the gateway onFloodWait seam) opens scope-precise windows', async () => {
|
|
695
|
+
const clock = new FakeClock()
|
|
696
|
+
const opened: string[] = []
|
|
697
|
+
const gate = createSendGate({
|
|
698
|
+
enabled: true,
|
|
699
|
+
clock,
|
|
700
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
701
|
+
})
|
|
702
|
+
|
|
703
|
+
// Simulates the gateway's onFloodWait hook for a SHORT slept-and-retried 429
|
|
704
|
+
// on a supergroup chat — the gate's own FLOOD_WAIT_ACTIVE catch never fires
|
|
705
|
+
// for those, so this seam must carry the scope precision.
|
|
706
|
+
gate.openScopedFloodWindows({ chat_id: '9', chatType: 'supergroup' }, HOUR)
|
|
707
|
+
expect(opened.sort()).toEqual(['chat:9', 'group:9'])
|
|
708
|
+
|
|
709
|
+
// A no-chat-scope 429 opens global.
|
|
710
|
+
opened.length = 0
|
|
711
|
+
gate.openScopedFloodWindows(undefined, HOUR)
|
|
712
|
+
expect(opened).toEqual(['global'])
|
|
713
|
+
})
|
|
714
|
+
|
|
715
|
+
it('openScopedFloodWindows is a pure no-op when the gate is disabled', async () => {
|
|
716
|
+
const clock = new FakeClock()
|
|
717
|
+
const opened: string[] = []
|
|
718
|
+
const gate = createSendGate({
|
|
719
|
+
enabled: false,
|
|
720
|
+
clock,
|
|
721
|
+
onWindowOpen: (scopeKey) => opened.push(scopeKey),
|
|
722
|
+
})
|
|
723
|
+
gate.openScopedFloodWindows({ chat_id: 'A' }, HOUR)
|
|
724
|
+
gate.openScopedFloodWindows(undefined, HOUR)
|
|
725
|
+
expect(opened).toEqual([])
|
|
726
|
+
})
|
|
727
|
+
})
|