switchroom 0.19.25 → 0.19.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +61 -2
- package/dist/auth-broker/index.js +125 -8
- package/dist/cli/notion-write-pretool.mjs +61 -2
- package/dist/cli/switchroom.js +2347 -1104
- package/dist/host-control/main.js +126 -9
- package/dist/vault/approvals/kernel-server.js +124 -8
- package/dist/vault/broker/server.js +124 -8
- package/package.json +6 -2
- package/profiles/_base/cron-session.sh.hbs +14 -0
- package/profiles/_base/start.sh.hbs +145 -4
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +93 -1
- package/telegram-plugin/dist/gateway/gateway.js +2213 -1204
- package/telegram-plugin/dist/server.js +96 -1
- package/telegram-plugin/edit-flood-fuse.ts +637 -56
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +103 -112
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/periodic-sweep-guard.ts +86 -0
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +180 -0
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +72 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +11 -0
- package/telegram-plugin/status-pin-driver.ts +33 -17
- package/telegram-plugin/status-pin.ts +51 -5
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +291 -0
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +316 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/periodic-sweep-guard.test.ts +151 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +24 -18
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +244 -0
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-shutdown-wiring.test.ts +94 -0
- package/telegram-plugin/tests/status-pin-store.test.ts +179 -64
- package/telegram-plugin/tests/status-pin.test.ts +184 -7
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +29 -19
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tier-downgrade.ts +3 -2
- package/telegram-plugin/tool-activity-summary.ts +239 -322
- package/telegram-plugin/uat/assertions.ts +33 -3
- package/telegram-plugin/uat/feed-matcher.test.ts +36 -0
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-channel.test.ts +9 -2
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-dm.test.ts +9 -2
- package/telegram-plugin/worker-activity-feed.ts +109 -30
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_request_timeout.py +241 -0
- package/vendor/hindsight-memory/settings.json +1 -1
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { createPeriodicSweepGuard } from "../gateway/periodic-sweep-guard.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Single-flight guard for the interval-driven mid-session card reaper.
|
|
6
|
+
*
|
|
7
|
+
* Pre-fix the interval dispatched `void runMidSessionCardReaper()` directly, so
|
|
8
|
+
* a pass that parked behind a flood-wait (this gateway took a 62-minute flood
|
|
9
|
+
* ban on 2026-07-25) let EVERY subsequent tick start another concurrent pass —
|
|
10
|
+
* unbounded fan-out racing the same store rows and firing duplicate unpins into
|
|
11
|
+
* the rate limit that caused the stall. These assert the observable outcome:
|
|
12
|
+
* the number of times `run` actually started.
|
|
13
|
+
*/
|
|
14
|
+
describe("createPeriodicSweepGuard", () => {
|
|
15
|
+
/** A pass we can hold open, counting starts and completions. */
|
|
16
|
+
function heldRun() {
|
|
17
|
+
let release!: () => void;
|
|
18
|
+
let gate = new Promise<void>((r) => {
|
|
19
|
+
release = r;
|
|
20
|
+
});
|
|
21
|
+
let starts = 0;
|
|
22
|
+
let finishes = 0;
|
|
23
|
+
const run = async () => {
|
|
24
|
+
starts += 1;
|
|
25
|
+
await gate;
|
|
26
|
+
finishes += 1;
|
|
27
|
+
};
|
|
28
|
+
return {
|
|
29
|
+
run,
|
|
30
|
+
starts: () => starts,
|
|
31
|
+
finishes: () => finishes,
|
|
32
|
+
release: () => {
|
|
33
|
+
release();
|
|
34
|
+
// Re-arm so a later pass can be held again.
|
|
35
|
+
gate = new Promise<void>((r) => {
|
|
36
|
+
release = r;
|
|
37
|
+
});
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
it("runs the pass when idle", async () => {
|
|
43
|
+
let ran = 0;
|
|
44
|
+
const g = createPeriodicSweepGuard({
|
|
45
|
+
run: async () => {
|
|
46
|
+
ran += 1;
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
await g.tick();
|
|
50
|
+
expect(ran).toBe(1);
|
|
51
|
+
expect(g.isRunning()).toBe(false);
|
|
52
|
+
expect(g.skipped()).toBe(0);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("SKIPS every tick that arrives while a pass is still in flight (the fan-out fix)", async () => {
|
|
56
|
+
const h = heldRun();
|
|
57
|
+
const skips: number[] = [];
|
|
58
|
+
const g = createPeriodicSweepGuard({ run: h.run, onSkip: () => skips.push(1) });
|
|
59
|
+
|
|
60
|
+
const inFlight = g.tick();
|
|
61
|
+
await Promise.resolve();
|
|
62
|
+
expect(g.isRunning()).toBe(true);
|
|
63
|
+
expect(h.starts()).toBe(1);
|
|
64
|
+
|
|
65
|
+
// Ten more interval ticks land during the stall. Pre-fix these were ten
|
|
66
|
+
// more concurrent passes; now they are dropped.
|
|
67
|
+
await Promise.all([
|
|
68
|
+
g.tick(),
|
|
69
|
+
g.tick(),
|
|
70
|
+
g.tick(),
|
|
71
|
+
g.tick(),
|
|
72
|
+
g.tick(),
|
|
73
|
+
g.tick(),
|
|
74
|
+
g.tick(),
|
|
75
|
+
g.tick(),
|
|
76
|
+
g.tick(),
|
|
77
|
+
g.tick(),
|
|
78
|
+
]);
|
|
79
|
+
expect(h.starts()).toBe(1); // still exactly ONE pass ever started
|
|
80
|
+
expect(g.skipped()).toBe(10);
|
|
81
|
+
expect(skips).toHaveLength(10);
|
|
82
|
+
|
|
83
|
+
h.release();
|
|
84
|
+
await inFlight;
|
|
85
|
+
expect(h.finishes()).toBe(1);
|
|
86
|
+
expect(g.isRunning()).toBe(false);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it("re-arms after the in-flight pass settles, so the sweep is not disabled forever", async () => {
|
|
90
|
+
const h = heldRun();
|
|
91
|
+
const g = createPeriodicSweepGuard({ run: h.run });
|
|
92
|
+
const first = g.tick();
|
|
93
|
+
await Promise.resolve();
|
|
94
|
+
await g.tick(); // skipped
|
|
95
|
+
h.release();
|
|
96
|
+
await first;
|
|
97
|
+
|
|
98
|
+
const second = g.tick();
|
|
99
|
+
await Promise.resolve();
|
|
100
|
+
expect(h.starts()).toBe(2); // the NEXT tick genuinely ran
|
|
101
|
+
h.release();
|
|
102
|
+
await second;
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it("absorbs a throwing pass: tick never rejects, the guard re-arms, onError sees it", async () => {
|
|
106
|
+
const seen: unknown[] = [];
|
|
107
|
+
let calls = 0;
|
|
108
|
+
const g = createPeriodicSweepGuard({
|
|
109
|
+
run: async () => {
|
|
110
|
+
calls += 1;
|
|
111
|
+
throw new Error(`boom ${calls}`);
|
|
112
|
+
},
|
|
113
|
+
onError: (e) => seen.push(e),
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
// A rejection escaping here reaches the gateway's `unhandledRejection`
|
|
117
|
+
// handler, which CRASHES the process — a cosmetic sweep must never do that.
|
|
118
|
+
await expect(g.tick()).resolves.toBeUndefined();
|
|
119
|
+
expect(g.isRunning()).toBe(false);
|
|
120
|
+
await expect(g.tick()).resolves.toBeUndefined();
|
|
121
|
+
expect(calls).toBe(2); // a throw does not wedge the running flag
|
|
122
|
+
expect(seen).toHaveLength(2);
|
|
123
|
+
expect((seen[0] as Error).message).toBe("boom 1");
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("a throwing observer cannot break the guard", async () => {
|
|
127
|
+
const h = heldRun();
|
|
128
|
+
const g = createPeriodicSweepGuard({
|
|
129
|
+
run: h.run,
|
|
130
|
+
onSkip: () => {
|
|
131
|
+
throw new Error("broken logger");
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
const first = g.tick();
|
|
135
|
+
await Promise.resolve();
|
|
136
|
+
await expect(g.tick()).resolves.toBeUndefined();
|
|
137
|
+
expect(g.skipped()).toBe(1);
|
|
138
|
+
h.release();
|
|
139
|
+
await first;
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("onError is optional — a throwing pass with no observer is still absorbed", async () => {
|
|
143
|
+
const g = createPeriodicSweepGuard({
|
|
144
|
+
run: async () => {
|
|
145
|
+
throw new Error("boom");
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
await expect(g.tick()).resolves.toBeUndefined();
|
|
149
|
+
expect(g.isRunning()).toBe(false);
|
|
150
|
+
});
|
|
151
|
+
});
|
|
@@ -134,9 +134,9 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
|
|
|
134
134
|
|
|
135
135
|
it('kills the exact artifacts from the report', () => {
|
|
136
136
|
const collapsed = collapsePreview(body)
|
|
137
|
-
// 1. the count/ordinal collision — glance line into row 1's ordinal.
|
|
138
|
-
//
|
|
139
|
-
//
|
|
137
|
+
// 1. the count/ordinal collision — glance line into row 1's ordinal. Since
|
|
138
|
+
// #3842 the row header is FLUSH (the #3820 card-level indent is gone),
|
|
139
|
+
// so the separator is the ONLY thing holding this seam apart.
|
|
140
140
|
// (The reported spelling was `3 running1.`; with the packed glance line
|
|
141
141
|
// the same seam now reads `… 512.3k tok` -> `1. Fix issue`, so assert on
|
|
142
142
|
// the CURRENT last token of line 1 — an assertion on the old spelling
|
|
@@ -144,14 +144,14 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
|
|
|
144
144
|
expect(collapsed).not.toContain('running1.')
|
|
145
145
|
expect(collapsed).not.toContain('tok1.')
|
|
146
146
|
expect(collapsed).toContain(`tok${NB}1. Fix issue`)
|
|
147
|
-
// 2. the mid-word ✓ (model tag running into the step trail).
|
|
148
|
-
//
|
|
149
|
-
//
|
|
150
|
-
//
|
|
147
|
+
// 2. the mid-word ✓ (model tag running into the step trail). A step line
|
|
148
|
+
// still leads with WORKER_STEP_INDENT (three U+2800), so this seam is
|
|
149
|
+
// separator + step indent — asserted against the constant rather than a
|
|
150
|
+
// hardcoded run.
|
|
151
151
|
expect(collapsed).not.toContain('opus 5✓')
|
|
152
152
|
expect(collapsed).toContain(`opus 5${NB}${WORKER_STEP_INDENT}✓`)
|
|
153
153
|
// 3. the step trail running into the next step, and into the next row's
|
|
154
|
-
// header (that last seam is separator
|
|
154
|
+
// header (post-#3842 that last seam is the separator alone).
|
|
155
155
|
expect(collapsed).not.toContain('gateway.ts→')
|
|
156
156
|
expect(collapsed).toContain(`gateway.ts${NB}${WORKER_STEP_INDENT}→`)
|
|
157
157
|
expect(collapsed).not.toContain('search2.')
|
|
@@ -170,20 +170,24 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
|
|
|
170
170
|
})
|
|
171
171
|
|
|
172
172
|
it('CONTROL: the same lines joined without collapseSafe still mash (pre-fix shape)', () => {
|
|
173
|
-
// Discriminator: re-join
|
|
174
|
-
//
|
|
173
|
+
// Discriminator: re-join rendered card lines with the separator removed and
|
|
174
|
+
// show the collapsed preview mashes again. Without this, the assertions
|
|
175
175
|
// above could all be passing for reasons unrelated to the fix.
|
|
176
176
|
//
|
|
177
|
-
// The
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
//
|
|
181
|
-
|
|
182
|
-
|
|
177
|
+
// The control runs on the 🤖 AGENT card. Post-#3842 the single-worker card
|
|
178
|
+
// is flush too, so either would isolate the separator's contribution; the
|
|
179
|
+
// agent card is kept because it has no step indent on ANY line, so no
|
|
180
|
+
// future indent change can quietly make this control vacuous.
|
|
181
|
+
const agent = renderActivityFeed([
|
|
182
|
+
'Reading gateway.ts',
|
|
183
|
+
'Searching memory',
|
|
184
|
+
'Running tests',
|
|
185
|
+
])!
|
|
186
|
+
const lines = rawCardLines(agent).map((l) => l.replace(new RegExp(NB + '$'), ''))
|
|
183
187
|
const preFix = stackCardLines(lines)
|
|
184
188
|
const collapsed = collapsePreview(preFix)
|
|
185
|
-
expect(collapsed).toContain('
|
|
186
|
-
expect(collapsed).toContain('
|
|
189
|
+
expect(collapsed).toContain('gateway.ts✓ Searching')
|
|
190
|
+
expect(collapsed).toContain('memory→ Running')
|
|
187
191
|
// …and the property assertion itself would have failed on it.
|
|
188
192
|
expect(() => expectNoMashedSeams(preFix)).toThrow()
|
|
189
193
|
})
|
|
@@ -217,6 +221,8 @@ describe('single-worker / agent status card survives the collapse too (#3666)',
|
|
|
217
221
|
expectNoMashedSeams(body)
|
|
218
222
|
const collapsed = collapsePreview(body)
|
|
219
223
|
expect(collapsed).not.toContain('toolsstarting')
|
|
224
|
+
// #3842: the single-worker card is flush, so the separator alone holds the
|
|
225
|
+
// hand-rolled `starting…` seam apart — nothing else masks a regression.
|
|
220
226
|
expect(collapsed).toContain(`0 tools${NB}starting`)
|
|
221
227
|
})
|
|
222
228
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* extracted from gateway.ts (switchroom#2996 P6 cluster F). Asserts the
|
|
4
4
|
* chat-scoped ownership guard (only OUR pins get their service message
|
|
5
5
|
* deleted), the reconcile-store race retry, and best-effort logging on a
|
|
6
|
-
* delete failure — against injected
|
|
6
|
+
* delete failure — against an injected claim registry + a mock deleteServiceMessage.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { describe, it, expect, vi } from 'vitest'
|
|
@@ -11,13 +11,18 @@ import {
|
|
|
11
11
|
handlePinnedMessage,
|
|
12
12
|
type PinnedMessageHandlerDeps,
|
|
13
13
|
} from '../gateway/pinned-message-handler.js'
|
|
14
|
+
import type { StatusPinClaim } from '../gateway/status-pin-retarget.js'
|
|
15
|
+
|
|
16
|
+
/** One claim, the single record the gateway now keeps per pin key (#3809). */
|
|
17
|
+
function claim(messageId: number, chatId: string): StatusPinClaim {
|
|
18
|
+
return { messageId, chatId, pinnedAt: 1000 }
|
|
19
|
+
}
|
|
14
20
|
|
|
15
21
|
function makeDeps(over: Partial<PinnedMessageHandlerDeps> = {}) {
|
|
16
22
|
const deleteServiceMessage = vi.fn(async () => {})
|
|
17
23
|
const log = vi.fn()
|
|
18
24
|
const deps: PinnedMessageHandlerDeps = {
|
|
19
|
-
|
|
20
|
-
statusPinChatIds: new Map(),
|
|
25
|
+
statusPinClaims: new Map(),
|
|
21
26
|
deleteServiceMessage,
|
|
22
27
|
log,
|
|
23
28
|
...over,
|
|
@@ -38,8 +43,7 @@ function ctxWith(chatId: number, pinnedMessageId: number | undefined, serviceMsg
|
|
|
38
43
|
describe('handlePinnedMessage — ownership guard', () => {
|
|
39
44
|
it('deletes the service message when the pin is ours in this chat', async () => {
|
|
40
45
|
const { deps, deleteServiceMessage } = makeDeps({
|
|
41
|
-
|
|
42
|
-
statusPinChatIds: new Map([['fg:a', '42']]),
|
|
46
|
+
statusPinClaims: new Map([['fg:a', claim(555, '42')]]),
|
|
43
47
|
})
|
|
44
48
|
await handlePinnedMessage(ctxWith(42, 555, 999), deps)
|
|
45
49
|
expect(deleteServiceMessage).toHaveBeenCalledWith('42', 999)
|
|
@@ -47,8 +51,8 @@ describe('handlePinnedMessage — ownership guard', () => {
|
|
|
47
51
|
|
|
48
52
|
it('does NOT delete when the pinned id matches but the tracked chat differs', async () => {
|
|
49
53
|
const { deps, deleteServiceMessage } = makeDeps({
|
|
50
|
-
|
|
51
|
-
|
|
54
|
+
// tracked in a DIFFERENT chat
|
|
55
|
+
statusPinClaims: new Map([['fg:a', claim(555, '99')]]),
|
|
52
56
|
})
|
|
53
57
|
await handlePinnedMessage(ctxWith(42, 555, 999), deps)
|
|
54
58
|
expect(deleteServiceMessage).not.toHaveBeenCalled()
|
|
@@ -75,16 +79,13 @@ describe('handlePinnedMessage — reconcile-store race', () => {
|
|
|
75
79
|
// #3354 bun-test failure). The handler waits 250ms before its single
|
|
76
80
|
// re-check; populate the store inside that window and await the real
|
|
77
81
|
// delay.
|
|
78
|
-
const
|
|
79
|
-
const chatIds = new Map<string, string>()
|
|
82
|
+
const claims = new Map<string, StatusPinClaim>()
|
|
80
83
|
const { deps, deleteServiceMessage } = makeDeps({
|
|
81
|
-
|
|
82
|
-
statusPinChatIds: chatIds,
|
|
84
|
+
statusPinClaims: claims,
|
|
83
85
|
})
|
|
84
86
|
const p = handlePinnedMessage(ctxWith(42, 555, 999), deps)
|
|
85
87
|
// Store catches up during the 250ms race window.
|
|
86
|
-
|
|
87
|
-
chatIds.set('fg:a', '42')
|
|
88
|
+
claims.set('fg:a', claim(555, '42'))
|
|
88
89
|
await p
|
|
89
90
|
expect(deleteServiceMessage).toHaveBeenCalledWith('42', 999)
|
|
90
91
|
})
|
|
@@ -93,8 +94,7 @@ describe('handlePinnedMessage — reconcile-store race', () => {
|
|
|
93
94
|
describe('handlePinnedMessage — best-effort delete failure', () => {
|
|
94
95
|
it('logs a concise reason when the delete throws', async () => {
|
|
95
96
|
const { deps, log } = makeDeps({
|
|
96
|
-
|
|
97
|
-
statusPinChatIds: new Map([['fg:a', '42']]),
|
|
97
|
+
statusPinClaims: new Map([['fg:a', claim(555, '42')]]),
|
|
98
98
|
deleteServiceMessage: vi.fn(async () => {
|
|
99
99
|
throw new Error('not enough rights')
|
|
100
100
|
}),
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* provider-credit-402.test.ts — outcome tests for the OpenRouter/OpenAI/
|
|
3
|
+
* Perplexity credit-error leak.
|
|
4
|
+
*
|
|
5
|
+
* THE BUG (audited 2026-07-28). OpenRouter answers an exhausted balance with
|
|
6
|
+
* HTTP 402 `payment_required` / "insufficient credits". None of switchroom's
|
|
7
|
+
* quota/credit wordings matched it — every one of them was an ANTHROPIC
|
|
8
|
+
* wording — so `classifyClaudeError` fell through to `unknown-4xx`.
|
|
9
|
+
* `unknown-4xx` is NOT in `OPERATOR_ACTIONABLE_KINDS`, so
|
|
10
|
+
* `decideOperatorEventAudience` broadcast it to EVERY allowlist chat: an end
|
|
11
|
+
* user got a card carrying the raw vendor error in a code span and a "🔐
|
|
12
|
+
* Reauth" button they cannot act on. That violates Ken's standing rule that an
|
|
13
|
+
* operator-actionable error must never reach an end user.
|
|
14
|
+
*
|
|
15
|
+
* These assert the OBSERVABLE RESULT, composing the SAME production functions
|
|
16
|
+
* in the SAME order `emitGatewayOperatorEvent` (gateway.ts ~:8134-8290) calls
|
|
17
|
+
* them — classify → resolveModelUnavailableFromOperatorEvent → render →
|
|
18
|
+
* decideOperatorEventAudience → renderUserFacingFailureNotice — so a
|
|
19
|
+
* regression anywhere along that chain fails here:
|
|
20
|
+
*
|
|
21
|
+
* 1. a real OpenRouter 402 body produces an OPERATOR card, and
|
|
22
|
+
* 2. the text a non-operator user sees is the brief plain-language notice,
|
|
23
|
+
* carrying NO raw error text, NO vendor name, NO status code.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { describe, it, expect } from 'vitest'
|
|
27
|
+
import {
|
|
28
|
+
classifyClaudeError,
|
|
29
|
+
renderOperatorEvent,
|
|
30
|
+
decideOperatorEventAudience,
|
|
31
|
+
isOperatorActionableKind,
|
|
32
|
+
renderUserFacingFailureNotice,
|
|
33
|
+
type OperatorEvent,
|
|
34
|
+
} from '../operator-events.js'
|
|
35
|
+
import { resolveModelUnavailableFromOperatorEvent } from '../model-unavailable.js'
|
|
36
|
+
import { parseLlmError, isActionableKind } from '../llm-error-present.js'
|
|
37
|
+
import { attributeProvider, detectProviderCreditExhaustion } from '../provider-credit.js'
|
|
38
|
+
|
|
39
|
+
// ── Verbatim vendor shapes (docs verified 2026-07-28) ───────────────────────
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* OpenRouter through LiteLLM. Wording per
|
|
43
|
+
* https://openrouter.ai/docs/api-reference/errors — 402, typed code
|
|
44
|
+
* `payment_required`, "insufficient credits. Add more credits and retry".
|
|
45
|
+
*/
|
|
46
|
+
const OPENROUTER_402 =
|
|
47
|
+
'litellm.APIError: OpenrouterException - {"error":{"code":402,"message":"Your account or API key has insufficient credits. Add more credits and retry the request.","metadata":{"provider_name":"openrouter"}}}'
|
|
48
|
+
|
|
49
|
+
/** The per-key variant: the ACCOUNT has money, the key's `limit_remaining` is spent. */
|
|
50
|
+
const OPENROUTER_402_PER_KEY =
|
|
51
|
+
'openrouter.ai returned 402: This request requires more credits, or fewer max_tokens.'
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* OpenAI's exhausted balance. NOTE the status is 429, not 402
|
|
55
|
+
* (https://platform.openai.com/docs/guides/error-codes) — the case that makes
|
|
56
|
+
* "just match 402" insufficient.
|
|
57
|
+
*/
|
|
58
|
+
const OPENAI_INSUFFICIENT_QUOTA =
|
|
59
|
+
'litellm.RateLimitError: OpenAIException - {"error":{"message":"You exceeded your current quota, please check your plan and billing details.","type":"insufficient_quota","code":"insufficient_quota"}} (api.openai.com)'
|
|
60
|
+
|
|
61
|
+
/** A genuine ANTHROPIC credit wall — must keep its own kind and its own card. */
|
|
62
|
+
const ANTHROPIC_CREDIT =
|
|
63
|
+
'{"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low to access the Claude API."}}'
|
|
64
|
+
|
|
65
|
+
/** An ordinary transient 429 — must NOT be dragged into the credit class. */
|
|
66
|
+
const PLAIN_RATE_LIMIT =
|
|
67
|
+
'{"type":"error","error":{"type":"rate_limit_error","message":"Number of requests has exceeded your rate limit"}}'
|
|
68
|
+
|
|
69
|
+
const ALLOW = ['5000000001' /* operator */, '5000000002' /* end user */, '5000000003']
|
|
70
|
+
|
|
71
|
+
function makeEvent(detail: string): OperatorEvent {
|
|
72
|
+
const kind = classifyClaudeError({ message: detail, type: detail })
|
|
73
|
+
return {
|
|
74
|
+
kind,
|
|
75
|
+
agent: 'klanker',
|
|
76
|
+
detail,
|
|
77
|
+
suggestedActions: [],
|
|
78
|
+
firstSeenAt: new Date('2026-07-28T10:00:00Z'),
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The gateway's observable outcome for one error string: what the OPERATOR
|
|
84
|
+
* sees and what a NON-OPERATOR user sees. Composed from the same production
|
|
85
|
+
* functions, in the gateway's order (gateway.ts ~:8134-8290).
|
|
86
|
+
*/
|
|
87
|
+
function route(detail: string): {
|
|
88
|
+
kind: string
|
|
89
|
+
firedFleetFailover: boolean
|
|
90
|
+
operatorText: string | null
|
|
91
|
+
operatorChats: string[]
|
|
92
|
+
userText: string | null
|
|
93
|
+
userChats: string[]
|
|
94
|
+
} {
|
|
95
|
+
const ev = makeEvent(detail)
|
|
96
|
+
// The gateway renders the "⚠️ Model unavailable" card AND fires
|
|
97
|
+
// fireFleetAutoFallback iff this resolves to a quota_exhausted detection.
|
|
98
|
+
const mu = resolveModelUnavailableFromOperatorEvent(ev)
|
|
99
|
+
const firedFleetFailover = mu?.kind === 'quota_exhausted'
|
|
100
|
+
const { operatorChats, userNoticeChats } = decideOperatorEventAudience(
|
|
101
|
+
ev.kind,
|
|
102
|
+
ALLOW,
|
|
103
|
+
ALLOW[0],
|
|
104
|
+
)
|
|
105
|
+
return {
|
|
106
|
+
kind: ev.kind,
|
|
107
|
+
firedFleetFailover,
|
|
108
|
+
operatorText: operatorChats.length > 0 ? renderOperatorEvent(ev).text : null,
|
|
109
|
+
operatorChats,
|
|
110
|
+
userText: userNoticeChats.length > 0 ? renderUserFacingFailureNotice() : null,
|
|
111
|
+
userChats: userNoticeChats,
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
describe('OpenRouter 402 — the end-user leak is closed', () => {
|
|
116
|
+
it('produces an OPERATOR card and gives the user only the brief notice', () => {
|
|
117
|
+
const r = route(OPENROUTER_402)
|
|
118
|
+
|
|
119
|
+
// 1. Classified as the operator-only provider-credit kind.
|
|
120
|
+
expect(r.kind).toBe('provider-credit-exhausted')
|
|
121
|
+
expect(isOperatorActionableKind('provider-credit-exhausted')).toBe(true)
|
|
122
|
+
|
|
123
|
+
// 2. The full card goes to the OPERATOR ONLY.
|
|
124
|
+
expect(r.operatorChats).toEqual(['5000000001'])
|
|
125
|
+
expect(r.userChats).toEqual(['5000000002', '5000000003'])
|
|
126
|
+
|
|
127
|
+
// 3. The operator card is actionable and provider-correct.
|
|
128
|
+
expect(r.operatorText).toContain('OpenRouter')
|
|
129
|
+
expect(r.operatorText).toContain('openrouter/api-key') // vault key NAME
|
|
130
|
+
expect(r.operatorText).toContain('https://openrouter.ai/credits')
|
|
131
|
+
|
|
132
|
+
// 4. THE LEAK ASSERTION: nothing an end user receives carries the raw
|
|
133
|
+
// vendor error, the vendor name, or the status code.
|
|
134
|
+
expect(r.userText).toBe(renderUserFacingFailureNotice())
|
|
135
|
+
for (const fragment of [
|
|
136
|
+
'insufficient credits',
|
|
137
|
+
'OpenRouter',
|
|
138
|
+
'openrouter',
|
|
139
|
+
'402',
|
|
140
|
+
'payment_required',
|
|
141
|
+
'litellm',
|
|
142
|
+
'api key',
|
|
143
|
+
]) {
|
|
144
|
+
expect(r.userText!.toLowerCase()).not.toContain(fragment.toLowerCase())
|
|
145
|
+
}
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
it('never advertises an Anthropic remedy — /auth cannot buy OpenRouter credit', () => {
|
|
149
|
+
const { operatorText } = route(OPENROUTER_402)
|
|
150
|
+
// The `credit-exhausted` (Anthropic) card's recommendation sentence must
|
|
151
|
+
// not appear — the card may only NAME `/auth use` to rule it OUT.
|
|
152
|
+
expect(operatorText).not.toContain('Use `/auth use <label>`')
|
|
153
|
+
expect(operatorText).not.toContain('/auth add')
|
|
154
|
+
expect(operatorText).toContain('`/auth use` will NOT fix this')
|
|
155
|
+
// and no re-auth button: the key is valid, it is out of money
|
|
156
|
+
const rendered = renderOperatorEvent(makeEvent(OPENROUTER_402))
|
|
157
|
+
const buttons = rendered.keyboard.inline_keyboard.flat().map(b => b.text)
|
|
158
|
+
expect(buttons).toEqual(['❌ Dismiss'])
|
|
159
|
+
expect(JSON.stringify(rendered.keyboard)).not.toContain('reauth')
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
it('does NOT fire an Anthropic fleet failover (nothing about Anthropic is exhausted)', () => {
|
|
163
|
+
expect(route(OPENROUTER_402).firedFleetFailover).toBe(false)
|
|
164
|
+
expect(route(OPENROUTER_402_PER_KEY).firedFleetFailover).toBe(false)
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
it('covers the per-key credit-limit variant, not just a zero account balance', () => {
|
|
168
|
+
const r = route(OPENROUTER_402_PER_KEY)
|
|
169
|
+
expect(r.kind).toBe('provider-credit-exhausted')
|
|
170
|
+
expect(r.userChats.length).toBe(2)
|
|
171
|
+
expect(r.userText).toBe(renderUserFacingFailureNotice())
|
|
172
|
+
})
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
describe('OpenAI + Perplexity have the same gap and are covered', () => {
|
|
176
|
+
it('OpenAI insufficient_quota (HTTP 429!) is a credit wall, not a transient throttle', () => {
|
|
177
|
+
const r = route(OPENAI_INSUFFICIENT_QUOTA)
|
|
178
|
+
expect(r.kind).toBe('provider-credit-exhausted')
|
|
179
|
+
expect(r.operatorText).toContain('OpenAI')
|
|
180
|
+
expect(r.operatorText).toContain('openai/api-key')
|
|
181
|
+
// The bug this guards: 429 wording routing to "Rate limited … will retry
|
|
182
|
+
// automatically", which is false — nothing retries a spent balance.
|
|
183
|
+
expect(r.operatorText).not.toContain('Will retry automatically')
|
|
184
|
+
expect(r.userChats.length).toBe(2)
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
it('attributes a Perplexity balance error to the Perplexity console + key name', () => {
|
|
188
|
+
const entry = attributeProvider('api.perplexity.ai responded 401: insufficient credits')
|
|
189
|
+
expect(entry?.id).toBe('perplexity')
|
|
190
|
+
expect(entry?.vaultKey).toBe('perplexity/api-key')
|
|
191
|
+
expect(detectProviderCreditExhaustion('api.perplexity.ai: insufficient credits')).not.toBeNull()
|
|
192
|
+
})
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
describe('no collateral damage to the Anthropic paths', () => {
|
|
196
|
+
it('an Anthropic credit-balance wall keeps its own kind and its slot-switch remedy', () => {
|
|
197
|
+
const r = route(ANTHROPIC_CREDIT)
|
|
198
|
+
expect(r.kind).toBe('credit-exhausted')
|
|
199
|
+
expect(r.operatorText).toContain('/auth use')
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
it('an ordinary rate limit is untouched — still broadcast, still transient', () => {
|
|
203
|
+
const r = route(PLAIN_RATE_LIMIT)
|
|
204
|
+
expect(r.kind).toBe('rate-limited')
|
|
205
|
+
expect(r.operatorChats).toEqual(ALLOW)
|
|
206
|
+
expect(r.userChats).toEqual([])
|
|
207
|
+
})
|
|
208
|
+
|
|
209
|
+
it('a bare "402" substring in unrelated text is NOT a credit wall', () => {
|
|
210
|
+
// model ids, token counts and request ids routinely contain "402".
|
|
211
|
+
expect(detectProviderCreditExhaustion('model=gpt-402-turbo request_id=req_402x')).toBeNull()
|
|
212
|
+
expect(classifyClaudeError({ message: 'timeout after 402 ms', type: '' })).not.toBe(
|
|
213
|
+
'provider-credit-exhausted',
|
|
214
|
+
)
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
it('a structured HTTP 402 status is honoured even with no recognisable wording', () => {
|
|
218
|
+
expect(classifyClaudeError({ message: 'upstream refused', status: 402 })).toBe(
|
|
219
|
+
'provider-credit-exhausted',
|
|
220
|
+
)
|
|
221
|
+
})
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
describe('the user-facing LLM error surface', () => {
|
|
225
|
+
it('renders a diagnosis-free one-liner, never the raw 402 body', () => {
|
|
226
|
+
const parsed = parseLlmError(OPENROUTER_402)
|
|
227
|
+
expect(parsed.kind).toBe('provider_credit')
|
|
228
|
+
expect(parsed.providerId).toBe('openrouter')
|
|
229
|
+
expect(parsed.coreText).toBe(
|
|
230
|
+
'An upstream model provider is out of credit — the operator has been notified.',
|
|
231
|
+
)
|
|
232
|
+
expect(parsed.coreText).not.toContain('402')
|
|
233
|
+
expect(parsed.coreText).not.toContain('insufficient credits')
|
|
234
|
+
// Never claim a retry will fix it — a spent balance does not self-heal.
|
|
235
|
+
expect(parsed.coreText).not.toContain('retrying automatically')
|
|
236
|
+
expect(parsed.terminal).toBe(true)
|
|
237
|
+
expect(parsed.autoRetrying).toBe(false)
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
it('is always-actionable, so a dedup window can never silence it', () => {
|
|
241
|
+
expect(isActionableKind('provider_credit')).toBe(true)
|
|
242
|
+
})
|
|
243
|
+
})
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* claimed-but-never-painted pin, or (worse, Defect B's shape) a dropped claim
|
|
8
8
|
* plus a deleted durable row for a message that is still pinned.
|
|
9
9
|
*
|
|
10
|
-
* The assertions below are OUTCOMES read off `
|
|
10
|
+
* The assertions below are OUTCOMES read off `executePinLeg`'s returned state
|
|
11
11
|
* (does the gateway still hold a record of the pinned message?), not "was some
|
|
12
12
|
* function called".
|
|
13
13
|
*/
|
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
type PinCapableBot,
|
|
21
21
|
} from '../gateway/status-pin-api.js'
|
|
22
22
|
import { SEND_GATE_SHED } from '../send-gate.js'
|
|
23
|
-
import {
|
|
23
|
+
import { executePinLeg } from '../status-pin-driver.js'
|
|
24
24
|
|
|
25
25
|
/** A bot whose API always succeeds — so any failure below comes from the gate
|
|
26
26
|
* seam, never from Telegram. */
|
|
@@ -38,14 +38,14 @@ const shedRobust = async () => SEND_GATE_SHED as unknown
|
|
|
38
38
|
describe('assertLanded — a shed send must not look like a landed one (#3664)', () => {
|
|
39
39
|
it('a shed unpin does NOT drop the claim — the message is still pinned', async () => {
|
|
40
40
|
const api = createStatusPinApi(liveBot, shedRobust)
|
|
41
|
-
const next = await
|
|
41
|
+
const next = await executePinLeg({
|
|
42
42
|
api,
|
|
43
43
|
chatId: '-100123',
|
|
44
44
|
prevState: { messageId: 715 },
|
|
45
|
-
|
|
45
|
+
action: { kind: 'unpin', messageId: 715 },
|
|
46
46
|
onError: () => {},
|
|
47
47
|
})
|
|
48
|
-
// Without assertLanded the shed resolves success-shaped,
|
|
48
|
+
// Without assertLanded the shed resolves success-shaped, executePinLeg
|
|
49
49
|
// returns null, and reconcileAndPersistStatusPin then DELETES the durable
|
|
50
50
|
// row for a message that is provably still pinned — Defect B, reopened.
|
|
51
51
|
expect(next).toEqual({ messageId: 715 })
|
|
@@ -53,11 +53,11 @@ describe('assertLanded — a shed send must not look like a landed one (#3664)',
|
|
|
53
53
|
|
|
54
54
|
it('a shed pin does not claim the message', async () => {
|
|
55
55
|
const api = createStatusPinApi(liveBot, shedRobust)
|
|
56
|
-
const next = await
|
|
56
|
+
const next = await executePinLeg({
|
|
57
57
|
api,
|
|
58
58
|
chatId: '-100123',
|
|
59
59
|
prevState: null,
|
|
60
|
-
|
|
60
|
+
action: { kind: 'pin', messageId: 715 },
|
|
61
61
|
onError: () => {},
|
|
62
62
|
})
|
|
63
63
|
// Claiming it would leave the gateway believing a pin exists that never
|
|
@@ -84,11 +84,11 @@ describe('assertLanded — a shed send must not look like a landed one (#3664)',
|
|
|
84
84
|
const api = createStatusPinApi(liveBot, async () => undefined)
|
|
85
85
|
await expect(api.unpinChatMessage('-100123', 715)).resolves.toBeUndefined()
|
|
86
86
|
|
|
87
|
-
const next = await
|
|
87
|
+
const next = await executePinLeg({
|
|
88
88
|
api,
|
|
89
89
|
chatId: '-100123',
|
|
90
90
|
prevState: { messageId: 715 },
|
|
91
|
-
|
|
91
|
+
action: { kind: 'unpin', messageId: 715 },
|
|
92
92
|
onError: () => {},
|
|
93
93
|
})
|
|
94
94
|
expect(next).toBeNull()
|
|
@@ -127,11 +127,11 @@ describe('assertBotReady — named backstop for a pre-ready caller (#3664)', ()
|
|
|
127
127
|
|
|
128
128
|
it('a pre-ready unpin failure RETAINS the claim (it never reached Telegram)', async () => {
|
|
129
129
|
const api = createStatusPinApi(() => undefined, passthrough)
|
|
130
|
-
const next = await
|
|
130
|
+
const next = await executePinLeg({
|
|
131
131
|
api,
|
|
132
132
|
chatId: '-100123',
|
|
133
133
|
prevState: { messageId: 715 },
|
|
134
|
-
|
|
134
|
+
action: { kind: 'unpin', messageId: 715 },
|
|
135
135
|
onError: () => {},
|
|
136
136
|
})
|
|
137
137
|
expect(next).toEqual({ messageId: 715 })
|