switchroom 0.19.26 → 0.19.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +55 -0
- package/dist/auth-broker/index.js +116 -6
- package/dist/cli/notion-write-pretool.mjs +55 -0
- package/dist/cli/switchroom.js +2055 -1193
- package/dist/host-control/main.js +117 -7
- package/dist/vault/approvals/kernel-server.js +114 -6
- package/dist/vault/broker/server.js +114 -6
- package/package.json +6 -2
- package/profiles/_base/cron-session.sh.hbs +8 -0
- package/profiles/_base/start.sh.hbs +105 -0
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +93 -1
- package/telegram-plugin/dist/gateway/gateway.js +2009 -1166
- package/telegram-plugin/dist/server.js +96 -1
- package/telegram-plugin/edit-flood-fuse.ts +637 -56
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +58 -68
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +72 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +10 -48
- package/telegram-plugin/status-pin-driver.ts +33 -45
- package/telegram-plugin/status-pin.ts +18 -1
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +316 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
- package/telegram-plugin/tests/status-pin.test.ts +56 -5
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tool-activity-summary.ts +239 -365
- package/telegram-plugin/uat/assertions.ts +22 -11
- package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
- package/telegram-plugin/worker-activity-feed.ts +105 -47
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/settings.json +1 -1
|
@@ -33,7 +33,6 @@ import {
|
|
|
33
33
|
STATUS_CARD_CHAR_BUDGET,
|
|
34
34
|
STATUS_LINE_MAX,
|
|
35
35
|
WORKER_STEP_INDENT,
|
|
36
|
-
SUBORDINATE_LINE_INDENT,
|
|
37
36
|
} from '../status-no-truncate.js'
|
|
38
37
|
|
|
39
38
|
/**
|
|
@@ -135,31 +134,28 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
|
|
|
135
134
|
|
|
136
135
|
it('kills the exact artifacts from the report', () => {
|
|
137
136
|
const collapsed = collapsePreview(body)
|
|
138
|
-
// 1. the count/ordinal collision — glance line into row 1's ordinal.
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
// agent card), so this seam is separator + card indent.
|
|
137
|
+
// 1. the count/ordinal collision — glance line into row 1's ordinal. Since
|
|
138
|
+
// #3842 the row header is FLUSH (the #3820 card-level indent is gone),
|
|
139
|
+
// so the separator is the ONLY thing holding this seam apart.
|
|
142
140
|
// (The reported spelling was `3 running1.`; with the packed glance line
|
|
143
141
|
// the same seam now reads `… 512.3k tok` -> `1. Fix issue`, so assert on
|
|
144
142
|
// the CURRENT last token of line 1 — an assertion on the old spelling
|
|
145
143
|
// alone would be vacuously green.)
|
|
146
144
|
expect(collapsed).not.toContain('running1.')
|
|
147
145
|
expect(collapsed).not.toContain('tok1.')
|
|
148
|
-
expect(collapsed).toContain(`tok${NB}
|
|
149
|
-
// 2. the mid-word ✓ (model tag running into the step trail).
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
146
|
+
expect(collapsed).toContain(`tok${NB}1. Fix issue`)
|
|
147
|
+
// 2. the mid-word ✓ (model tag running into the step trail). A step line
|
|
148
|
+
// still leads with WORKER_STEP_INDENT (three U+2800), so this seam is
|
|
149
|
+
// separator + step indent — asserted against the constant rather than a
|
|
150
|
+
// hardcoded run.
|
|
153
151
|
expect(collapsed).not.toContain('opus 5✓')
|
|
154
|
-
expect(collapsed).toContain(`opus 5${NB}${
|
|
152
|
+
expect(collapsed).toContain(`opus 5${NB}${WORKER_STEP_INDENT}✓`)
|
|
155
153
|
// 3. the step trail running into the next step, and into the next row's
|
|
156
|
-
// header (post-#
|
|
154
|
+
// header (post-#3842 that last seam is the separator alone).
|
|
157
155
|
expect(collapsed).not.toContain('gateway.ts→')
|
|
158
|
-
expect(collapsed).toContain(
|
|
159
|
-
`gateway.ts${NB}${SUBORDINATE_LINE_INDENT}${WORKER_STEP_INDENT}→`,
|
|
160
|
-
)
|
|
156
|
+
expect(collapsed).toContain(`gateway.ts${NB}${WORKER_STEP_INDENT}→`)
|
|
161
157
|
expect(collapsed).not.toContain('search2.')
|
|
162
|
-
expect(collapsed).toContain(`search${NB}
|
|
158
|
+
expect(collapsed).toContain(`search${NB}2.`)
|
|
163
159
|
})
|
|
164
160
|
|
|
165
161
|
it('leads with a self-contained glance that ends in a unit word, not a bare number', () => {
|
|
@@ -178,13 +174,10 @@ describe('combined worker card survives the pinned-bar collapse (#3666)', () =>
|
|
|
178
174
|
// show the collapsed preview mashes again. Without this, the assertions
|
|
179
175
|
// above could all be passing for reasons unrelated to the fix.
|
|
180
176
|
//
|
|
181
|
-
// The control runs on the 🤖 AGENT card
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
// separator's contribution. The agent card is the surface where the
|
|
186
|
-
// separator is still the ONLY thing holding the seams apart, which is
|
|
187
|
-
// exactly what this control must measure.
|
|
177
|
+
// The control runs on the 🤖 AGENT card. Post-#3842 the single-worker card
|
|
178
|
+
// is flush too, so either would isolate the separator's contribution; the
|
|
179
|
+
// agent card is kept because it has no step indent on ANY line, so no
|
|
180
|
+
// future indent change can quietly make this control vacuous.
|
|
188
181
|
const agent = renderActivityFeed([
|
|
189
182
|
'Reading gateway.ts',
|
|
190
183
|
'Searching memory',
|
|
@@ -228,7 +221,9 @@ describe('single-worker / agent status card survives the collapse too (#3666)',
|
|
|
228
221
|
expectNoMashedSeams(body)
|
|
229
222
|
const collapsed = collapsePreview(body)
|
|
230
223
|
expect(collapsed).not.toContain('toolsstarting')
|
|
231
|
-
|
|
224
|
+
// #3842: the single-worker card is flush, so the separator alone holds the
|
|
225
|
+
// hand-rolled `starting…` seam apart — nothing else masks a regression.
|
|
226
|
+
expect(collapsed).toContain(`0 tools${NB}starting`)
|
|
232
227
|
})
|
|
233
228
|
|
|
234
229
|
it('the nested child block stays separated even though its indent is ASCII (#3668)', () => {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* extracted from gateway.ts (switchroom#2996 P6 cluster F). Asserts the
|
|
4
4
|
* chat-scoped ownership guard (only OUR pins get their service message
|
|
5
5
|
* deleted), the reconcile-store race retry, and best-effort logging on a
|
|
6
|
-
* delete failure — against injected
|
|
6
|
+
* delete failure — against an injected claim registry + a mock deleteServiceMessage.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { describe, it, expect, vi } from 'vitest'
|
|
@@ -11,13 +11,18 @@ import {
|
|
|
11
11
|
handlePinnedMessage,
|
|
12
12
|
type PinnedMessageHandlerDeps,
|
|
13
13
|
} from '../gateway/pinned-message-handler.js'
|
|
14
|
+
import type { StatusPinClaim } from '../gateway/status-pin-retarget.js'
|
|
15
|
+
|
|
16
|
+
/** One claim, the single record the gateway now keeps per pin key (#3809). */
|
|
17
|
+
function claim(messageId: number, chatId: string): StatusPinClaim {
|
|
18
|
+
return { messageId, chatId, pinnedAt: 1000 }
|
|
19
|
+
}
|
|
14
20
|
|
|
15
21
|
function makeDeps(over: Partial<PinnedMessageHandlerDeps> = {}) {
|
|
16
22
|
const deleteServiceMessage = vi.fn(async () => {})
|
|
17
23
|
const log = vi.fn()
|
|
18
24
|
const deps: PinnedMessageHandlerDeps = {
|
|
19
|
-
|
|
20
|
-
statusPinChatIds: new Map(),
|
|
25
|
+
statusPinClaims: new Map(),
|
|
21
26
|
deleteServiceMessage,
|
|
22
27
|
log,
|
|
23
28
|
...over,
|
|
@@ -38,8 +43,7 @@ function ctxWith(chatId: number, pinnedMessageId: number | undefined, serviceMsg
|
|
|
38
43
|
describe('handlePinnedMessage — ownership guard', () => {
|
|
39
44
|
it('deletes the service message when the pin is ours in this chat', async () => {
|
|
40
45
|
const { deps, deleteServiceMessage } = makeDeps({
|
|
41
|
-
|
|
42
|
-
statusPinChatIds: new Map([['fg:a', '42']]),
|
|
46
|
+
statusPinClaims: new Map([['fg:a', claim(555, '42')]]),
|
|
43
47
|
})
|
|
44
48
|
await handlePinnedMessage(ctxWith(42, 555, 999), deps)
|
|
45
49
|
expect(deleteServiceMessage).toHaveBeenCalledWith('42', 999)
|
|
@@ -47,8 +51,8 @@ describe('handlePinnedMessage — ownership guard', () => {
|
|
|
47
51
|
|
|
48
52
|
it('does NOT delete when the pinned id matches but the tracked chat differs', async () => {
|
|
49
53
|
const { deps, deleteServiceMessage } = makeDeps({
|
|
50
|
-
|
|
51
|
-
|
|
54
|
+
// tracked in a DIFFERENT chat
|
|
55
|
+
statusPinClaims: new Map([['fg:a', claim(555, '99')]]),
|
|
52
56
|
})
|
|
53
57
|
await handlePinnedMessage(ctxWith(42, 555, 999), deps)
|
|
54
58
|
expect(deleteServiceMessage).not.toHaveBeenCalled()
|
|
@@ -75,16 +79,13 @@ describe('handlePinnedMessage — reconcile-store race', () => {
|
|
|
75
79
|
// #3354 bun-test failure). The handler waits 250ms before its single
|
|
76
80
|
// re-check; populate the store inside that window and await the real
|
|
77
81
|
// delay.
|
|
78
|
-
const
|
|
79
|
-
const chatIds = new Map<string, string>()
|
|
82
|
+
const claims = new Map<string, StatusPinClaim>()
|
|
80
83
|
const { deps, deleteServiceMessage } = makeDeps({
|
|
81
|
-
|
|
82
|
-
statusPinChatIds: chatIds,
|
|
84
|
+
statusPinClaims: claims,
|
|
83
85
|
})
|
|
84
86
|
const p = handlePinnedMessage(ctxWith(42, 555, 999), deps)
|
|
85
87
|
// Store catches up during the 250ms race window.
|
|
86
|
-
|
|
87
|
-
chatIds.set('fg:a', '42')
|
|
88
|
+
claims.set('fg:a', claim(555, '42'))
|
|
88
89
|
await p
|
|
89
90
|
expect(deleteServiceMessage).toHaveBeenCalledWith('42', 999)
|
|
90
91
|
})
|
|
@@ -93,8 +94,7 @@ describe('handlePinnedMessage — reconcile-store race', () => {
|
|
|
93
94
|
describe('handlePinnedMessage — best-effort delete failure', () => {
|
|
94
95
|
it('logs a concise reason when the delete throws', async () => {
|
|
95
96
|
const { deps, log } = makeDeps({
|
|
96
|
-
|
|
97
|
-
statusPinChatIds: new Map([['fg:a', '42']]),
|
|
97
|
+
statusPinClaims: new Map([['fg:a', claim(555, '42')]]),
|
|
98
98
|
deleteServiceMessage: vi.fn(async () => {
|
|
99
99
|
throw new Error('not enough rights')
|
|
100
100
|
}),
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* provider-credit-402.test.ts — outcome tests for the OpenRouter/OpenAI/
|
|
3
|
+
* Perplexity credit-error leak.
|
|
4
|
+
*
|
|
5
|
+
* THE BUG (audited 2026-07-28). OpenRouter answers an exhausted balance with
|
|
6
|
+
* HTTP 402 `payment_required` / "insufficient credits". None of switchroom's
|
|
7
|
+
* quota/credit wordings matched it — every one of them was an ANTHROPIC
|
|
8
|
+
* wording — so `classifyClaudeError` fell through to `unknown-4xx`.
|
|
9
|
+
* `unknown-4xx` is NOT in `OPERATOR_ACTIONABLE_KINDS`, so
|
|
10
|
+
* `decideOperatorEventAudience` broadcast it to EVERY allowlist chat: an end
|
|
11
|
+
* user got a card carrying the raw vendor error in a code span and a "🔐
|
|
12
|
+
* Reauth" button they cannot act on. That violates Ken's standing rule that an
|
|
13
|
+
* operator-actionable error must never reach an end user.
|
|
14
|
+
*
|
|
15
|
+
* These assert the OBSERVABLE RESULT, composing the SAME production functions
|
|
16
|
+
* in the SAME order `emitGatewayOperatorEvent` (gateway.ts ~:8134-8290) calls
|
|
17
|
+
* them — classify → resolveModelUnavailableFromOperatorEvent → render →
|
|
18
|
+
* decideOperatorEventAudience → renderUserFacingFailureNotice — so a
|
|
19
|
+
* regression anywhere along that chain fails here:
|
|
20
|
+
*
|
|
21
|
+
* 1. a real OpenRouter 402 body produces an OPERATOR card, and
|
|
22
|
+
* 2. the text a non-operator user sees is the brief plain-language notice,
|
|
23
|
+
* carrying NO raw error text, NO vendor name, NO status code.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { describe, it, expect } from 'vitest'
|
|
27
|
+
import {
|
|
28
|
+
classifyClaudeError,
|
|
29
|
+
renderOperatorEvent,
|
|
30
|
+
decideOperatorEventAudience,
|
|
31
|
+
isOperatorActionableKind,
|
|
32
|
+
renderUserFacingFailureNotice,
|
|
33
|
+
type OperatorEvent,
|
|
34
|
+
} from '../operator-events.js'
|
|
35
|
+
import { resolveModelUnavailableFromOperatorEvent } from '../model-unavailable.js'
|
|
36
|
+
import { parseLlmError, isActionableKind } from '../llm-error-present.js'
|
|
37
|
+
import { attributeProvider, detectProviderCreditExhaustion } from '../provider-credit.js'
|
|
38
|
+
|
|
39
|
+
// ── Verbatim vendor shapes (docs verified 2026-07-28) ───────────────────────
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* OpenRouter through LiteLLM. Wording per
|
|
43
|
+
* https://openrouter.ai/docs/api-reference/errors — 402, typed code
|
|
44
|
+
* `payment_required`, "insufficient credits. Add more credits and retry".
|
|
45
|
+
*/
|
|
46
|
+
const OPENROUTER_402 =
|
|
47
|
+
'litellm.APIError: OpenrouterException - {"error":{"code":402,"message":"Your account or API key has insufficient credits. Add more credits and retry the request.","metadata":{"provider_name":"openrouter"}}}'
|
|
48
|
+
|
|
49
|
+
/** The per-key variant: the ACCOUNT has money, the key's `limit_remaining` is spent. */
|
|
50
|
+
const OPENROUTER_402_PER_KEY =
|
|
51
|
+
'openrouter.ai returned 402: This request requires more credits, or fewer max_tokens.'
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* OpenAI's exhausted balance. NOTE the status is 429, not 402
|
|
55
|
+
* (https://platform.openai.com/docs/guides/error-codes) — the case that makes
|
|
56
|
+
* "just match 402" insufficient.
|
|
57
|
+
*/
|
|
58
|
+
const OPENAI_INSUFFICIENT_QUOTA =
|
|
59
|
+
'litellm.RateLimitError: OpenAIException - {"error":{"message":"You exceeded your current quota, please check your plan and billing details.","type":"insufficient_quota","code":"insufficient_quota"}} (api.openai.com)'
|
|
60
|
+
|
|
61
|
+
/** A genuine ANTHROPIC credit wall — must keep its own kind and its own card. */
|
|
62
|
+
const ANTHROPIC_CREDIT =
|
|
63
|
+
'{"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low to access the Claude API."}}'
|
|
64
|
+
|
|
65
|
+
/** An ordinary transient 429 — must NOT be dragged into the credit class. */
|
|
66
|
+
const PLAIN_RATE_LIMIT =
|
|
67
|
+
'{"type":"error","error":{"type":"rate_limit_error","message":"Number of requests has exceeded your rate limit"}}'
|
|
68
|
+
|
|
69
|
+
const ALLOW = ['5000000001' /* operator */, '5000000002' /* end user */, '5000000003']
|
|
70
|
+
|
|
71
|
+
function makeEvent(detail: string): OperatorEvent {
|
|
72
|
+
const kind = classifyClaudeError({ message: detail, type: detail })
|
|
73
|
+
return {
|
|
74
|
+
kind,
|
|
75
|
+
agent: 'klanker',
|
|
76
|
+
detail,
|
|
77
|
+
suggestedActions: [],
|
|
78
|
+
firstSeenAt: new Date('2026-07-28T10:00:00Z'),
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The gateway's observable outcome for one error string: what the OPERATOR
|
|
84
|
+
* sees and what a NON-OPERATOR user sees. Composed from the same production
|
|
85
|
+
* functions, in the gateway's order (gateway.ts ~:8134-8290).
|
|
86
|
+
*/
|
|
87
|
+
function route(detail: string): {
|
|
88
|
+
kind: string
|
|
89
|
+
firedFleetFailover: boolean
|
|
90
|
+
operatorText: string | null
|
|
91
|
+
operatorChats: string[]
|
|
92
|
+
userText: string | null
|
|
93
|
+
userChats: string[]
|
|
94
|
+
} {
|
|
95
|
+
const ev = makeEvent(detail)
|
|
96
|
+
// The gateway renders the "⚠️ Model unavailable" card AND fires
|
|
97
|
+
// fireFleetAutoFallback iff this resolves to a quota_exhausted detection.
|
|
98
|
+
const mu = resolveModelUnavailableFromOperatorEvent(ev)
|
|
99
|
+
const firedFleetFailover = mu?.kind === 'quota_exhausted'
|
|
100
|
+
const { operatorChats, userNoticeChats } = decideOperatorEventAudience(
|
|
101
|
+
ev.kind,
|
|
102
|
+
ALLOW,
|
|
103
|
+
ALLOW[0],
|
|
104
|
+
)
|
|
105
|
+
return {
|
|
106
|
+
kind: ev.kind,
|
|
107
|
+
firedFleetFailover,
|
|
108
|
+
operatorText: operatorChats.length > 0 ? renderOperatorEvent(ev).text : null,
|
|
109
|
+
operatorChats,
|
|
110
|
+
userText: userNoticeChats.length > 0 ? renderUserFacingFailureNotice() : null,
|
|
111
|
+
userChats: userNoticeChats,
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
describe('OpenRouter 402 — the end-user leak is closed', () => {
|
|
116
|
+
it('produces an OPERATOR card and gives the user only the brief notice', () => {
|
|
117
|
+
const r = route(OPENROUTER_402)
|
|
118
|
+
|
|
119
|
+
// 1. Classified as the operator-only provider-credit kind.
|
|
120
|
+
expect(r.kind).toBe('provider-credit-exhausted')
|
|
121
|
+
expect(isOperatorActionableKind('provider-credit-exhausted')).toBe(true)
|
|
122
|
+
|
|
123
|
+
// 2. The full card goes to the OPERATOR ONLY.
|
|
124
|
+
expect(r.operatorChats).toEqual(['5000000001'])
|
|
125
|
+
expect(r.userChats).toEqual(['5000000002', '5000000003'])
|
|
126
|
+
|
|
127
|
+
// 3. The operator card is actionable and provider-correct.
|
|
128
|
+
expect(r.operatorText).toContain('OpenRouter')
|
|
129
|
+
expect(r.operatorText).toContain('openrouter/api-key') // vault key NAME
|
|
130
|
+
expect(r.operatorText).toContain('https://openrouter.ai/credits')
|
|
131
|
+
|
|
132
|
+
// 4. THE LEAK ASSERTION: nothing an end user receives carries the raw
|
|
133
|
+
// vendor error, the vendor name, or the status code.
|
|
134
|
+
expect(r.userText).toBe(renderUserFacingFailureNotice())
|
|
135
|
+
for (const fragment of [
|
|
136
|
+
'insufficient credits',
|
|
137
|
+
'OpenRouter',
|
|
138
|
+
'openrouter',
|
|
139
|
+
'402',
|
|
140
|
+
'payment_required',
|
|
141
|
+
'litellm',
|
|
142
|
+
'api key',
|
|
143
|
+
]) {
|
|
144
|
+
expect(r.userText!.toLowerCase()).not.toContain(fragment.toLowerCase())
|
|
145
|
+
}
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
it('never advertises an Anthropic remedy — /auth cannot buy OpenRouter credit', () => {
|
|
149
|
+
const { operatorText } = route(OPENROUTER_402)
|
|
150
|
+
// The `credit-exhausted` (Anthropic) card's recommendation sentence must
|
|
151
|
+
// not appear — the card may only NAME `/auth use` to rule it OUT.
|
|
152
|
+
expect(operatorText).not.toContain('Use `/auth use <label>`')
|
|
153
|
+
expect(operatorText).not.toContain('/auth add')
|
|
154
|
+
expect(operatorText).toContain('`/auth use` will NOT fix this')
|
|
155
|
+
// and no re-auth button: the key is valid, it is out of money
|
|
156
|
+
const rendered = renderOperatorEvent(makeEvent(OPENROUTER_402))
|
|
157
|
+
const buttons = rendered.keyboard.inline_keyboard.flat().map(b => b.text)
|
|
158
|
+
expect(buttons).toEqual(['❌ Dismiss'])
|
|
159
|
+
expect(JSON.stringify(rendered.keyboard)).not.toContain('reauth')
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
it('does NOT fire an Anthropic fleet failover (nothing about Anthropic is exhausted)', () => {
|
|
163
|
+
expect(route(OPENROUTER_402).firedFleetFailover).toBe(false)
|
|
164
|
+
expect(route(OPENROUTER_402_PER_KEY).firedFleetFailover).toBe(false)
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
it('covers the per-key credit-limit variant, not just a zero account balance', () => {
|
|
168
|
+
const r = route(OPENROUTER_402_PER_KEY)
|
|
169
|
+
expect(r.kind).toBe('provider-credit-exhausted')
|
|
170
|
+
expect(r.userChats.length).toBe(2)
|
|
171
|
+
expect(r.userText).toBe(renderUserFacingFailureNotice())
|
|
172
|
+
})
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
describe('OpenAI + Perplexity have the same gap and are covered', () => {
|
|
176
|
+
it('OpenAI insufficient_quota (HTTP 429!) is a credit wall, not a transient throttle', () => {
|
|
177
|
+
const r = route(OPENAI_INSUFFICIENT_QUOTA)
|
|
178
|
+
expect(r.kind).toBe('provider-credit-exhausted')
|
|
179
|
+
expect(r.operatorText).toContain('OpenAI')
|
|
180
|
+
expect(r.operatorText).toContain('openai/api-key')
|
|
181
|
+
// The bug this guards: 429 wording routing to "Rate limited … will retry
|
|
182
|
+
// automatically", which is false — nothing retries a spent balance.
|
|
183
|
+
expect(r.operatorText).not.toContain('Will retry automatically')
|
|
184
|
+
expect(r.userChats.length).toBe(2)
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
it('attributes a Perplexity balance error to the Perplexity console + key name', () => {
|
|
188
|
+
const entry = attributeProvider('api.perplexity.ai responded 401: insufficient credits')
|
|
189
|
+
expect(entry?.id).toBe('perplexity')
|
|
190
|
+
expect(entry?.vaultKey).toBe('perplexity/api-key')
|
|
191
|
+
expect(detectProviderCreditExhaustion('api.perplexity.ai: insufficient credits')).not.toBeNull()
|
|
192
|
+
})
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
describe('no collateral damage to the Anthropic paths', () => {
|
|
196
|
+
it('an Anthropic credit-balance wall keeps its own kind and its slot-switch remedy', () => {
|
|
197
|
+
const r = route(ANTHROPIC_CREDIT)
|
|
198
|
+
expect(r.kind).toBe('credit-exhausted')
|
|
199
|
+
expect(r.operatorText).toContain('/auth use')
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
it('an ordinary rate limit is untouched — still broadcast, still transient', () => {
|
|
203
|
+
const r = route(PLAIN_RATE_LIMIT)
|
|
204
|
+
expect(r.kind).toBe('rate-limited')
|
|
205
|
+
expect(r.operatorChats).toEqual(ALLOW)
|
|
206
|
+
expect(r.userChats).toEqual([])
|
|
207
|
+
})
|
|
208
|
+
|
|
209
|
+
it('a bare "402" substring in unrelated text is NOT a credit wall', () => {
|
|
210
|
+
// model ids, token counts and request ids routinely contain "402".
|
|
211
|
+
expect(detectProviderCreditExhaustion('model=gpt-402-turbo request_id=req_402x')).toBeNull()
|
|
212
|
+
expect(classifyClaudeError({ message: 'timeout after 402 ms', type: '' })).not.toBe(
|
|
213
|
+
'provider-credit-exhausted',
|
|
214
|
+
)
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
it('a structured HTTP 402 status is honoured even with no recognisable wording', () => {
|
|
218
|
+
expect(classifyClaudeError({ message: 'upstream refused', status: 402 })).toBe(
|
|
219
|
+
'provider-credit-exhausted',
|
|
220
|
+
)
|
|
221
|
+
})
|
|
222
|
+
})
|
|
223
|
+
|
|
224
|
+
describe('the user-facing LLM error surface', () => {
|
|
225
|
+
it('renders a diagnosis-free one-liner, never the raw 402 body', () => {
|
|
226
|
+
const parsed = parseLlmError(OPENROUTER_402)
|
|
227
|
+
expect(parsed.kind).toBe('provider_credit')
|
|
228
|
+
expect(parsed.providerId).toBe('openrouter')
|
|
229
|
+
expect(parsed.coreText).toBe(
|
|
230
|
+
'An upstream model provider is out of credit — the operator has been notified.',
|
|
231
|
+
)
|
|
232
|
+
expect(parsed.coreText).not.toContain('402')
|
|
233
|
+
expect(parsed.coreText).not.toContain('insufficient credits')
|
|
234
|
+
// Never claim a retry will fix it — a spent balance does not self-heal.
|
|
235
|
+
expect(parsed.coreText).not.toContain('retrying automatically')
|
|
236
|
+
expect(parsed.terminal).toBe(true)
|
|
237
|
+
expect(parsed.autoRetrying).toBe(false)
|
|
238
|
+
})
|
|
239
|
+
|
|
240
|
+
it('is always-actionable, so a dedup window can never silence it', () => {
|
|
241
|
+
expect(isActionableKind('provider_credit')).toBe(true)
|
|
242
|
+
})
|
|
243
|
+
})
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* claimed-but-never-painted pin, or (worse, Defect B's shape) a dropped claim
|
|
8
8
|
* plus a deleted durable row for a message that is still pinned.
|
|
9
9
|
*
|
|
10
|
-
* The assertions below are OUTCOMES read off `
|
|
10
|
+
* The assertions below are OUTCOMES read off `executePinLeg`'s returned state
|
|
11
11
|
* (does the gateway still hold a record of the pinned message?), not "was some
|
|
12
12
|
* function called".
|
|
13
13
|
*/
|
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
type PinCapableBot,
|
|
21
21
|
} from '../gateway/status-pin-api.js'
|
|
22
22
|
import { SEND_GATE_SHED } from '../send-gate.js'
|
|
23
|
-
import {
|
|
23
|
+
import { executePinLeg } from '../status-pin-driver.js'
|
|
24
24
|
|
|
25
25
|
/** A bot whose API always succeeds — so any failure below comes from the gate
|
|
26
26
|
* seam, never from Telegram. */
|
|
@@ -38,14 +38,14 @@ const shedRobust = async () => SEND_GATE_SHED as unknown
|
|
|
38
38
|
describe('assertLanded — a shed send must not look like a landed one (#3664)', () => {
|
|
39
39
|
it('a shed unpin does NOT drop the claim — the message is still pinned', async () => {
|
|
40
40
|
const api = createStatusPinApi(liveBot, shedRobust)
|
|
41
|
-
const next = await
|
|
41
|
+
const next = await executePinLeg({
|
|
42
42
|
api,
|
|
43
43
|
chatId: '-100123',
|
|
44
44
|
prevState: { messageId: 715 },
|
|
45
|
-
|
|
45
|
+
action: { kind: 'unpin', messageId: 715 },
|
|
46
46
|
onError: () => {},
|
|
47
47
|
})
|
|
48
|
-
// Without assertLanded the shed resolves success-shaped,
|
|
48
|
+
// Without assertLanded the shed resolves success-shaped, executePinLeg
|
|
49
49
|
// returns null, and reconcileAndPersistStatusPin then DELETES the durable
|
|
50
50
|
// row for a message that is provably still pinned — Defect B, reopened.
|
|
51
51
|
expect(next).toEqual({ messageId: 715 })
|
|
@@ -53,11 +53,11 @@ describe('assertLanded — a shed send must not look like a landed one (#3664)',
|
|
|
53
53
|
|
|
54
54
|
it('a shed pin does not claim the message', async () => {
|
|
55
55
|
const api = createStatusPinApi(liveBot, shedRobust)
|
|
56
|
-
const next = await
|
|
56
|
+
const next = await executePinLeg({
|
|
57
57
|
api,
|
|
58
58
|
chatId: '-100123',
|
|
59
59
|
prevState: null,
|
|
60
|
-
|
|
60
|
+
action: { kind: 'pin', messageId: 715 },
|
|
61
61
|
onError: () => {},
|
|
62
62
|
})
|
|
63
63
|
// Claiming it would leave the gateway believing a pin exists that never
|
|
@@ -84,11 +84,11 @@ describe('assertLanded — a shed send must not look like a landed one (#3664)',
|
|
|
84
84
|
const api = createStatusPinApi(liveBot, async () => undefined)
|
|
85
85
|
await expect(api.unpinChatMessage('-100123', 715)).resolves.toBeUndefined()
|
|
86
86
|
|
|
87
|
-
const next = await
|
|
87
|
+
const next = await executePinLeg({
|
|
88
88
|
api,
|
|
89
89
|
chatId: '-100123',
|
|
90
90
|
prevState: { messageId: 715 },
|
|
91
|
-
|
|
91
|
+
action: { kind: 'unpin', messageId: 715 },
|
|
92
92
|
onError: () => {},
|
|
93
93
|
})
|
|
94
94
|
expect(next).toBeNull()
|
|
@@ -127,11 +127,11 @@ describe('assertBotReady — named backstop for a pre-ready caller (#3664)', ()
|
|
|
127
127
|
|
|
128
128
|
it('a pre-ready unpin failure RETAINS the claim (it never reached Telegram)', async () => {
|
|
129
129
|
const api = createStatusPinApi(() => undefined, passthrough)
|
|
130
|
-
const next = await
|
|
130
|
+
const next = await executePinLeg({
|
|
131
131
|
api,
|
|
132
132
|
chatId: '-100123',
|
|
133
133
|
prevState: { messageId: 715 },
|
|
134
|
-
|
|
134
|
+
action: { kind: 'unpin', messageId: 715 },
|
|
135
135
|
onError: () => {},
|
|
136
136
|
})
|
|
137
137
|
expect(next).toEqual({ messageId: 715 })
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* We cannot import gateway.ts directly (its module top-level runs the whole boot
|
|
12
12
|
* IIFE incl. the startup mutex + bot.start). Instead we model the gateway's
|
|
13
13
|
* status-pin subsystem faithfully with the SAME functions gateway.ts calls
|
|
14
|
-
* (
|
|
14
|
+
* (runStatusPinReconcile, executePinLeg, runStatusPinBootCleanup) over an
|
|
15
15
|
* in-memory fs + a fake Telegram whose pin set we can inspect — so a regression
|
|
16
16
|
* in the ordering / cleanup contract reds here.
|
|
17
17
|
*
|
|
@@ -25,18 +25,28 @@ import { readFileSync } from "node:fs";
|
|
|
25
25
|
import { fileURLToPath } from "node:url";
|
|
26
26
|
import {
|
|
27
27
|
loadStatusPins,
|
|
28
|
+
type PersistedStatusPin,
|
|
28
29
|
persistStatusPins,
|
|
29
30
|
reconcileAndPersistStatusPin,
|
|
30
31
|
runStatusPinBootCleanup,
|
|
31
32
|
type StatusPinStoreFsSeam,
|
|
32
33
|
} from "../gateway/status-pin-store.js";
|
|
33
|
-
import {
|
|
34
|
-
import type {
|
|
35
|
-
import {
|
|
34
|
+
import { executePinLeg } from "../status-pin-driver.js";
|
|
35
|
+
import type { DesiredPin, PinState } from "../status-pin.js";
|
|
36
|
+
import {
|
|
37
|
+
runStatusPinReconcile,
|
|
38
|
+
type StatusPinClaim,
|
|
39
|
+
} from "../gateway/status-pin-retarget.js";
|
|
36
40
|
import { makeFloodWaitActiveError } from "../retry-api-call.js";
|
|
37
41
|
|
|
38
42
|
const PATH = "/state/agent/telegram/status-pins.json";
|
|
39
43
|
|
|
44
|
+
/** Drop the #3810 claim-age stamp so a row's identity fields can be compared
|
|
45
|
+
* literally. The age itself is asserted in status-pin-store.test.ts. */
|
|
46
|
+
function idOnly(rows: PersistedStatusPin[]): Omit<PersistedStatusPin, "pinnedAt">[] {
|
|
47
|
+
return rows.map(({ pinnedAt: _age, ...rest }) => rest);
|
|
48
|
+
}
|
|
49
|
+
|
|
40
50
|
function memFs(seed: Record<string, string> = {}) {
|
|
41
51
|
const files = new Map<string, string>(Object.entries(seed));
|
|
42
52
|
const fs: StatusPinStoreFsSeam = {
|
|
@@ -78,33 +88,22 @@ function fakeTelegram() {
|
|
|
78
88
|
* with injectable fs + api so a single test can span a "crash" and a fresh boot.
|
|
79
89
|
*/
|
|
80
90
|
function makeGateway(fs: StatusPinStoreFsSeam, tg: ReturnType<typeof fakeTelegram>) {
|
|
81
|
-
|
|
82
|
-
const
|
|
91
|
+
// ONE claim registry — the same single record the gateway keeps (#3809).
|
|
92
|
+
const statusPinClaims = new Map<string, StatusPinClaim>();
|
|
83
93
|
|
|
84
94
|
async function reconcileStatusPin(pinKey: string, chatId: string, desired: DesiredPin) {
|
|
85
|
-
const
|
|
86
|
-
const
|
|
87
|
-
|
|
88
|
-
action.kind === "pin"
|
|
89
|
-
? ({ kind: "pin", messageId: action.messageId } as const)
|
|
90
|
-
: ({ kind: "clear" } as const);
|
|
91
|
-
const next = await reconcileAndPersistStatusPin({
|
|
92
|
-
path: PATH,
|
|
93
|
-
fs,
|
|
95
|
+
const claim = statusPinClaims.get(pinKey);
|
|
96
|
+
const prev: PinState | null = claim == null ? null : { messageId: claim.messageId };
|
|
97
|
+
await runStatusPinReconcile({
|
|
94
98
|
pinKey,
|
|
95
99
|
chatId,
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
+
prev,
|
|
101
|
+
desired,
|
|
102
|
+
persist: { path: PATH, fs },
|
|
103
|
+
runPin: (action, from) =>
|
|
104
|
+
executePinLeg({ api: tg.api, chatId, prevState: from, action }),
|
|
105
|
+
claims: statusPinClaims,
|
|
100
106
|
});
|
|
101
|
-
if (next == null) {
|
|
102
|
-
statusPinState.delete(pinKey);
|
|
103
|
-
statusPinChatIds.delete(pinKey);
|
|
104
|
-
} else {
|
|
105
|
-
statusPinState.set(pinKey, next);
|
|
106
|
-
statusPinChatIds.set(pinKey, chatId);
|
|
107
|
-
}
|
|
108
107
|
}
|
|
109
108
|
|
|
110
109
|
async function bootCleanup() {
|
|
@@ -116,7 +115,7 @@ function makeGateway(fs: StatusPinStoreFsSeam, tg: ReturnType<typeof fakeTelegra
|
|
|
116
115
|
});
|
|
117
116
|
}
|
|
118
117
|
|
|
119
|
-
return {
|
|
118
|
+
return { statusPinClaims, reconcileStatusPin, bootCleanup };
|
|
120
119
|
}
|
|
121
120
|
|
|
122
121
|
describe("status-pin boot recovery (gateway wiring)", () => {
|
|
@@ -130,7 +129,7 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
130
129
|
|
|
131
130
|
// Pin is live in Telegram and recorded on disk (confirmed).
|
|
132
131
|
expect(tg.pinned.has("-100123:715")).toBe(true);
|
|
133
|
-
expect(loadStatusPins(PATH, fs)).toEqual([
|
|
132
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([
|
|
134
133
|
{ pinKey: "fg:c:3", chatId: "-100123", messageId: 715 },
|
|
135
134
|
]);
|
|
136
135
|
|
|
@@ -144,7 +143,7 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
144
143
|
|
|
145
144
|
expect(res).toEqual({ cleared: 1, retained: 0, kept: 0, total: 1 });
|
|
146
145
|
expect(tg.pinned.has("-100123:715")).toBe(false); // orphan unpinned
|
|
147
|
-
expect(loadStatusPins(PATH, fs)).toEqual([]); // store emptied
|
|
146
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([]); // store emptied
|
|
148
147
|
});
|
|
149
148
|
|
|
150
149
|
it("(B) pending pin (crash inside the persist-after-pin window) → recovered next boot", async () => {
|
|
@@ -171,7 +170,7 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
171
170
|
|
|
172
171
|
// Orphan is live in Telegram; a PENDING record is on disk.
|
|
173
172
|
expect(tg.pinned.has("-100123:715")).toBe(true);
|
|
174
|
-
expect(loadStatusPins(PATH, fs)).toEqual([
|
|
173
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([
|
|
175
174
|
{ pinKey: "fg:c:3", chatId: "-100123", messageId: 715, pending: true },
|
|
176
175
|
]);
|
|
177
176
|
|
|
@@ -180,7 +179,7 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
180
179
|
const res = await gw2.bootCleanup();
|
|
181
180
|
expect(res).toEqual({ cleared: 1, retained: 0, kept: 0, total: 1 });
|
|
182
181
|
expect(tg.pinned.has("-100123:715")).toBe(false);
|
|
183
|
-
expect(loadStatusPins(PATH, fs)).toEqual([]);
|
|
182
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([]);
|
|
184
183
|
});
|
|
185
184
|
|
|
186
185
|
it("(C) #3664: a never-confirmed unpin (FLOOD_WAIT_ACTIVE) KEEPS the claim + the durable row, and the next boot clears the pin", async () => {
|
|
@@ -216,9 +215,9 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
216
215
|
// The pin is STILL UP …
|
|
217
216
|
expect(pinned.has("-100123:715")).toBe(true);
|
|
218
217
|
// … and BOTH records survive: the in-memory claim …
|
|
219
|
-
expect(gw1.
|
|
218
|
+
expect(gw1.statusPinClaims.get("fg:c:3")?.messageId).toBe(715);
|
|
220
219
|
// … and the durable row (what boot cleanup reads).
|
|
221
|
-
expect(loadStatusPins(PATH, fs)).toEqual([
|
|
220
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([
|
|
222
221
|
{ pinKey: "fg:c:3", chatId: "-100123", messageId: 715 },
|
|
223
222
|
]);
|
|
224
223
|
|
|
@@ -226,8 +225,8 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
226
225
|
floodOpen = false;
|
|
227
226
|
await gw1.reconcileStatusPin("fg:c:3", "-100123", { pinned: false });
|
|
228
227
|
expect(pinned.has("-100123:715")).toBe(false);
|
|
229
|
-
expect(gw1.
|
|
230
|
-
expect(loadStatusPins(PATH, fs)).toEqual([]);
|
|
228
|
+
expect(gw1.statusPinClaims.has("fg:c:3")).toBe(false);
|
|
229
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([]);
|
|
231
230
|
});
|
|
232
231
|
|
|
233
232
|
it("(D) #3664: if the process dies while the flood window is open, the retained row lets the NEXT boot clear the orphan", async () => {
|
|
@@ -257,7 +256,7 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
257
256
|
const res = await gw2.bootCleanup();
|
|
258
257
|
expect(res).toEqual({ cleared: 1, retained: 0, kept: 0, total: 1 });
|
|
259
258
|
expect(pinned.has("-100123:715")).toBe(false);
|
|
260
|
-
expect(loadStatusPins(PATH, fs)).toEqual([]);
|
|
259
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([]);
|
|
261
260
|
});
|
|
262
261
|
|
|
263
262
|
it("clean shutdown (sweep DID run) leaves nothing for boot cleanup to do", async () => {
|
|
@@ -271,7 +270,7 @@ describe("status-pin boot recovery (gateway wiring)", () => {
|
|
|
271
270
|
// Clean sweep: unpin the key.
|
|
272
271
|
await gw1.reconcileStatusPin("fg:c:3", "-100123", { pinned: false });
|
|
273
272
|
expect(tg.pinned.size).toBe(0);
|
|
274
|
-
expect(loadStatusPins(PATH, fs)).toEqual([]);
|
|
273
|
+
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([]);
|
|
275
274
|
|
|
276
275
|
const gw2 = makeGateway(fs, tg);
|
|
277
276
|
expect(await gw2.bootCleanup()).toEqual({ cleared: 0, retained: 0, kept: 0, total: 0 });
|