switchroom 0.19.25 → 0.19.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +61 -2
- package/dist/auth-broker/index.js +125 -8
- package/dist/cli/notion-write-pretool.mjs +61 -2
- package/dist/cli/switchroom.js +2347 -1104
- package/dist/host-control/main.js +126 -9
- package/dist/vault/approvals/kernel-server.js +124 -8
- package/dist/vault/broker/server.js +124 -8
- package/package.json +6 -2
- package/profiles/_base/cron-session.sh.hbs +14 -0
- package/profiles/_base/start.sh.hbs +145 -4
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +93 -1
- package/telegram-plugin/dist/gateway/gateway.js +2213 -1204
- package/telegram-plugin/dist/server.js +96 -1
- package/telegram-plugin/edit-flood-fuse.ts +637 -56
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +103 -112
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/periodic-sweep-guard.ts +86 -0
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +180 -0
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +72 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +11 -0
- package/telegram-plugin/status-pin-driver.ts +33 -17
- package/telegram-plugin/status-pin.ts +51 -5
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +291 -0
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +316 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/periodic-sweep-guard.test.ts +151 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +24 -18
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +244 -0
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-shutdown-wiring.test.ts +94 -0
- package/telegram-plugin/tests/status-pin-store.test.ts +179 -64
- package/telegram-plugin/tests/status-pin.test.ts +184 -7
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +29 -19
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tier-downgrade.ts +3 -2
- package/telegram-plugin/tool-activity-summary.ts +239 -322
- package/telegram-plugin/uat/assertions.ts +33 -3
- package/telegram-plugin/uat/feed-matcher.test.ts +36 -0
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-channel.test.ts +9 -2
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-dm.test.ts +9 -2
- package/telegram-plugin/worker-activity-feed.ts +109 -30
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_request_timeout.py +241 -0
- package/vendor/hindsight-memory/settings.json +1 -1
|
@@ -70,6 +70,27 @@ export interface PersistedStatusPin {
|
|
|
70
70
|
* unpin fails (flood-wait exhausted / transient 5xx); the row is retained
|
|
71
71
|
* for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
|
|
72
72
|
attempts?: number
|
|
73
|
+
/**
|
|
74
|
+
* Wall-clock ms the pin was FIRST claimed for this key+message (#3810).
|
|
75
|
+
*
|
|
76
|
+
* Why the store needs it: the mid-session `wk:` reaper folds in rows that
|
|
77
|
+
* exist on disk but have NO in-memory claim (`storeOnlyWorkerPinCandidates`).
|
|
78
|
+
* The store used to persist no timestamp, so those candidates were stamped
|
|
79
|
+
* `pinnedAt = now` on every pass — deliberately, to avoid a spurious unpin of
|
|
80
|
+
* a pin whose real age was unknown, but with the side effect that the TTL
|
|
81
|
+
* gate could NEVER fire for a store orphan. Combined with a `wk:<agentId>`
|
|
82
|
+
* whose turnsDb row is gone (verdict `'unknown'`, never `'terminal'`), such a
|
|
83
|
+
* row was never mid-session reaped AT ALL — it waited for the next boot, the
|
|
84
|
+
* exact "stale pin glued to the top of the chat" failure the reaper exists to
|
|
85
|
+
* prevent.
|
|
86
|
+
*
|
|
87
|
+
* Recording the real claim time removes the guess: a store orphan now ages
|
|
88
|
+
* honestly and the ordinary TTL applies. Optional so a v1/v2 snapshot still
|
|
89
|
+
* loads; a row without it keeps the old conservative `now` stamp (terminal-
|
|
90
|
+
* only reaping), which self-clears within one restart since every write from
|
|
91
|
+
* this version onward carries the field.
|
|
92
|
+
*/
|
|
93
|
+
pinnedAt?: number
|
|
73
94
|
}
|
|
74
95
|
|
|
75
96
|
/** How many boots may retry a failing boot-cleanup unpin before the row is
|
|
@@ -80,13 +101,18 @@ export interface PersistedStatusPin {
|
|
|
80
101
|
export const BOOT_UNPIN_MAX_ATTEMPTS = 5
|
|
81
102
|
|
|
82
103
|
/** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
|
|
83
|
-
* pin (pending undefined). v2 adds the optional `pending` flag.
|
|
84
|
-
* fail-open — an
|
|
104
|
+
* pin (pending undefined). v2 adds the optional `pending` flag. v3 adds the
|
|
105
|
+
* optional `pinnedAt` claim timestamp (#3810). All load fail-open — an
|
|
106
|
+
* unknown/newer version yields []. Every field added since v1 is optional, so
|
|
107
|
+
* the versions are mutually readable and a downgrade degrades rather than
|
|
108
|
+
* breaks. */
|
|
85
109
|
interface SnapshotEnvelope {
|
|
86
|
-
v: 1 | 2
|
|
110
|
+
v: 1 | 2 | 3
|
|
87
111
|
pins: PersistedStatusPin[]
|
|
88
112
|
}
|
|
89
113
|
|
|
114
|
+
const SNAPSHOT_VERSIONS = new Set([1, 2, 3])
|
|
115
|
+
|
|
90
116
|
function isPinRow(x: unknown): x is PersistedStatusPin {
|
|
91
117
|
if (x == null || typeof x !== 'object') return false
|
|
92
118
|
const o = x as Record<string, unknown>
|
|
@@ -98,7 +124,8 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
|
|
|
98
124
|
typeof o.messageId === 'number' &&
|
|
99
125
|
(o.pending === undefined || typeof o.pending === 'boolean') &&
|
|
100
126
|
(o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
|
|
101
|
-
(o.attempts === undefined || typeof o.attempts === 'number')
|
|
127
|
+
(o.attempts === undefined || typeof o.attempts === 'number') &&
|
|
128
|
+
(o.pinnedAt === undefined || typeof o.pinnedAt === 'number')
|
|
102
129
|
)
|
|
103
130
|
}
|
|
104
131
|
|
|
@@ -127,7 +154,7 @@ export function loadStatusPins(
|
|
|
127
154
|
}
|
|
128
155
|
if (parsed == null || typeof parsed !== 'object') return []
|
|
129
156
|
const env = parsed as Record<string, unknown>
|
|
130
|
-
if (
|
|
157
|
+
if (typeof env.v !== 'number' || !SNAPSHOT_VERSIONS.has(env.v) || !Array.isArray(env.pins)) return []
|
|
131
158
|
return env.pins.filter(isPinRow)
|
|
132
159
|
}
|
|
133
160
|
|
|
@@ -143,7 +170,7 @@ export function persistStatusPins(
|
|
|
143
170
|
snapshot: readonly PersistedStatusPin[],
|
|
144
171
|
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
145
172
|
): void {
|
|
146
|
-
const env: SnapshotEnvelope = { v:
|
|
173
|
+
const env: SnapshotEnvelope = { v: 3, pins: [...snapshot] }
|
|
147
174
|
const tmp = path + '.tmp'
|
|
148
175
|
try {
|
|
149
176
|
fs.writeFileSync(tmp, JSON.stringify(env))
|
|
@@ -427,16 +454,34 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
427
454
|
/** Execute the real pin/unpin; returns the confirmed message id (pin) or
|
|
428
455
|
* null (cleared). Must never throw — API errors are swallowed inside. */
|
|
429
456
|
applyPin: () => Promise<{ messageId: number } | null>
|
|
457
|
+
/** Clock for the `pinnedAt` stamp (#3810). Defaults to wall clock. */
|
|
458
|
+
now?: number
|
|
430
459
|
log?: (line: string) => void
|
|
431
460
|
}): Promise<{ messageId: number } | null> {
|
|
432
461
|
const { path, fs, pinKey, chatId, op } = args
|
|
433
462
|
const log = args.log ?? ((l: string) => process.stderr.write(l))
|
|
463
|
+
const now = args.now ?? Date.now()
|
|
464
|
+
|
|
465
|
+
/**
|
|
466
|
+
* The claim age to persist for `messageId`. Carried forward from the row
|
|
467
|
+
* already on disk when it names the SAME message (a steady-state re-write
|
|
468
|
+
* must not reset the age and hand a stale pin a fresh TTL lease); stamped
|
|
469
|
+
* `now` for a genuinely new pin. Mirrors the in-memory `pinnedAt` rule in
|
|
470
|
+
* `status-pin-retarget.ts` so disk and memory can't disagree about age.
|
|
471
|
+
*/
|
|
472
|
+
const claimAge = (messageId: number): number => {
|
|
473
|
+
const existing = loadStatusPins(path, fs).find((p) => p.pinKey === pinKey)
|
|
474
|
+
return existing?.messageId === messageId && existing.pinnedAt != null
|
|
475
|
+
? existing.pinnedAt
|
|
476
|
+
: now
|
|
477
|
+
}
|
|
434
478
|
|
|
435
479
|
// Hold the per-path lock across the WHOLE op so no other writer (a banner
|
|
436
480
|
// persist, or another key's reconcile) can rebuild/overwrite the file during
|
|
437
481
|
// the applyPin await window and drop this key's pending/confirmed row.
|
|
438
482
|
return withStoreLock(path, async () => {
|
|
439
483
|
if (op.kind === 'pin') {
|
|
484
|
+
const pinnedAt = claimAge(op.messageId)
|
|
440
485
|
// Persist INTENT first, marked pending — BEFORE the pin API call. If we
|
|
441
486
|
// crash after the pin lands but before the confirm rewrite, this pending
|
|
442
487
|
// record is what boot cleanup uses to unpin the orphan.
|
|
@@ -444,7 +489,7 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
444
489
|
path,
|
|
445
490
|
fs,
|
|
446
491
|
pinKey,
|
|
447
|
-
{ pinKey, chatId, messageId: op.messageId, pending: true },
|
|
492
|
+
{ pinKey, chatId, messageId: op.messageId, pending: true, pinnedAt },
|
|
448
493
|
log,
|
|
449
494
|
)
|
|
450
495
|
const next = await args.applyPin()
|
|
@@ -459,7 +504,7 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
459
504
|
path,
|
|
460
505
|
fs,
|
|
461
506
|
pinKey,
|
|
462
|
-
{ pinKey, chatId, messageId: next.messageId },
|
|
507
|
+
{ pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
|
|
463
508
|
log,
|
|
464
509
|
)
|
|
465
510
|
return next
|
|
@@ -488,11 +533,15 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
488
533
|
if (next == null) {
|
|
489
534
|
applyStatusPinRow(path, fs, pinKey, null, log)
|
|
490
535
|
} else {
|
|
536
|
+
// Steady-state noop-clear (the worker feed calls syncPin on every edit):
|
|
537
|
+
// the pin is still up, so the row is preserved AND so is its original
|
|
538
|
+
// claim age — re-stamping it here would hand a genuinely old pin a fresh
|
|
539
|
+
// TTL lease on every feed edit and make it immortal (#3810).
|
|
491
540
|
applyStatusPinRow(
|
|
492
541
|
path,
|
|
493
542
|
fs,
|
|
494
543
|
pinKey,
|
|
495
|
-
{ pinKey, chatId, messageId: next.messageId },
|
|
544
|
+
{ pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
|
|
496
545
|
log,
|
|
497
546
|
)
|
|
498
547
|
}
|
|
@@ -99,6 +99,47 @@ export interface StoreOrphanRow {
|
|
|
99
99
|
pending?: boolean
|
|
100
100
|
/** Time-scoped `tool:` pin. Never a worker orphan; excluded. */
|
|
101
101
|
expiresAt?: number
|
|
102
|
+
/** Wall-clock ms the claim was first taken (#3810). Absent on a pre-v3 row. */
|
|
103
|
+
pinnedAt?: number
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The live worker-activity feed's liveness probe, as the reaper needs it.
|
|
108
|
+
* Nullable ON PURPOSE — see `groupPinStatus`.
|
|
109
|
+
*/
|
|
110
|
+
export interface WorkerFeedLivenessProbe {
|
|
111
|
+
hasRunningInFeed(feedKey: string): boolean
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Verdict for a GROUP-level pin key (`wk:group:<feedKey>`, the shape the
|
|
116
|
+
* coalesced feed produces since #3207). Vouched off the live feed rather than
|
|
117
|
+
* the sub-agent registry: `group:<feedKey>` is not a jsonl agent id.
|
|
118
|
+
*
|
|
119
|
+
* #3811 — the gateway used to write this as
|
|
120
|
+
* `feed?.hasRunningInFeed(key) ? 'running' : 'terminal'`, which collapses two
|
|
121
|
+
* very different facts into one verdict:
|
|
122
|
+
*
|
|
123
|
+
* - "a live feed exists and affirmatively says this group is done" → terminal
|
|
124
|
+
* - "there is NO feed object to ask" → nothing known
|
|
125
|
+
*
|
|
126
|
+
* A `'terminal'` verdict reaps at ANY age — it bypasses the TTL exemption
|
|
127
|
+
* entirely — so the second case could unpin a GENUINELY LIVE group pin out from
|
|
128
|
+
* under running workers during a window where `workerActivityFeed` is null
|
|
129
|
+
* (teardown/rebuild on a bridge flap, and every tick before the first bridge
|
|
130
|
+
* connect). That is the inverse of the stale-pin bug and much harder to notice:
|
|
131
|
+
* the user loses the pin for work that is still in flight.
|
|
132
|
+
*
|
|
133
|
+
* "No feed to ask" is `'unknown'`: the TTL still applies, so a truly stale pin
|
|
134
|
+
* is still bounded, but nothing is reaped on the strength of a question we
|
|
135
|
+
* never got to ask.
|
|
136
|
+
*/
|
|
137
|
+
export function groupPinStatus(
|
|
138
|
+
feed: WorkerFeedLivenessProbe | null | undefined,
|
|
139
|
+
feedKey: string,
|
|
140
|
+
): WorkerRegistryStatus {
|
|
141
|
+
if (feed == null) return 'unknown'
|
|
142
|
+
return feed.hasRunningInFeed(feedKey) ? 'running' : 'terminal'
|
|
102
143
|
}
|
|
103
144
|
|
|
104
145
|
/**
|
|
@@ -110,13 +151,21 @@ export interface StoreOrphanRow {
|
|
|
110
151
|
* it too, group-safely (per-message unpin of a bot-tracked row — never an
|
|
111
152
|
* unpin-all, never a human pin).
|
|
112
153
|
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
154
|
+
* AGE (#3810). A v3+ row carries the real `pinnedAt` the claim was taken at, so
|
|
155
|
+
* a store orphan ages honestly and the ordinary TTL gate applies to it exactly
|
|
156
|
+
* as it does to an in-memory claim. A pre-v3 row has no timestamp and keeps the
|
|
157
|
+
* old conservative stamp (`pinnedAt = now`, i.e. TERMINAL-verdict reaping only)
|
|
158
|
+
* rather than guessing an age — but that case now self-clears within one
|
|
159
|
+
* restart, because every write from v3 onward records the field.
|
|
160
|
+
*
|
|
161
|
+
* Why the honest age matters: `storeOnlyWorkerPinCandidates` was the ONLY net
|
|
162
|
+
* for a `wk:` row with no in-memory claim, and with the TTL permanently
|
|
163
|
+
* disabled a `wk:<agentId>` row whose turnsDb row had been pruned (verdict
|
|
164
|
+
* `'unknown'`, never `'terminal'`) was never mid-session reaped at all — its
|
|
165
|
+
* stale `🛠 Worker` pin sat at the top of the chat until the next gateway boot.
|
|
166
|
+
*
|
|
117
167
|
* `pending` rows (pin API in-flight) and time-scoped `tool:` rows are excluded,
|
|
118
|
-
* as are rows already tracked in memory (the in-memory reaper owns those
|
|
119
|
-
* a real `pinnedAt` that can drive the TTL).
|
|
168
|
+
* as are rows already tracked in memory (the in-memory reaper owns those).
|
|
120
169
|
*/
|
|
121
170
|
export function storeOnlyWorkerPinCandidates(args: {
|
|
122
171
|
rows: Iterable<StoreOrphanRow>
|
|
@@ -133,7 +182,7 @@ export function storeOnlyWorkerPinCandidates(args: {
|
|
|
133
182
|
out.push({
|
|
134
183
|
pinKey: r.pinKey,
|
|
135
184
|
chatId: r.chatId,
|
|
136
|
-
pinnedAt: args.now,
|
|
185
|
+
pinnedAt: r.pinnedAt ?? args.now,
|
|
137
186
|
messageId: r.messageId,
|
|
138
187
|
})
|
|
139
188
|
}
|
|
@@ -34,6 +34,12 @@ import {
|
|
|
34
34
|
parseResetTime,
|
|
35
35
|
} from './model-unavailable.js'
|
|
36
36
|
import { classify429Detail } from './throttle-tier.js'
|
|
37
|
+
import {
|
|
38
|
+
PROVIDER_CREDIT_REGISTRY,
|
|
39
|
+
attributeProvider,
|
|
40
|
+
describeProviderCreditRemedy,
|
|
41
|
+
type ProviderCreditEntry,
|
|
42
|
+
} from './provider-credit.js'
|
|
37
43
|
import { classifyClaudeError } from './operator-events.js'
|
|
38
44
|
import { stripRawErrorBytes, extractRequestId } from './raw-error-scrub.js'
|
|
39
45
|
import { fmtLocalClock, tzAbbrev } from './shared/local-time.js'
|
|
@@ -48,10 +54,24 @@ export type LlmErrorKind =
|
|
|
48
54
|
| 'quota_wall'
|
|
49
55
|
| 'auth'
|
|
50
56
|
| 'infra_misconfig'
|
|
57
|
+
/**
|
|
58
|
+
* A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) is out of
|
|
59
|
+
* credit. OPERATOR-actionable, never user-actionable — its `coreText` is a
|
|
60
|
+
* deliberately diagnosis-free one-liner and its recommendation names the
|
|
61
|
+
* vendor console, never `/auth`. Kept distinct from `quota_wall` (the
|
|
62
|
+
* Anthropic subscription window, which resets on a clock and is addressed by
|
|
63
|
+
* switching account slots); a vendor balance resets only when someone pays.
|
|
64
|
+
*/
|
|
65
|
+
| 'provider_credit'
|
|
51
66
|
| 'transient'
|
|
52
67
|
| 'unknown'
|
|
53
68
|
|
|
54
|
-
export type LlmErrorSource =
|
|
69
|
+
export type LlmErrorSource =
|
|
70
|
+
| 'anthropic'
|
|
71
|
+
| 'litellm-local'
|
|
72
|
+
/** A paid third-party model provider reached THROUGH the proxy (OpenRouter, OpenAI). */
|
|
73
|
+
| 'external-provider'
|
|
74
|
+
| 'network'
|
|
55
75
|
|
|
56
76
|
export interface ParsedLlmError {
|
|
57
77
|
kind: LlmErrorKind
|
|
@@ -65,6 +85,14 @@ export interface ParsedLlmError {
|
|
|
65
85
|
model?: string
|
|
66
86
|
/** Anthropic `request_id`, when present — the strongest dedup key. */
|
|
67
87
|
requestId?: string
|
|
88
|
+
/**
|
|
89
|
+
* Registry id of the third-party provider the error was attributed to
|
|
90
|
+
* (`openrouter` / `openai` / `perplexity`), when `kind === 'provider_credit'`
|
|
91
|
+
* and the raw text named one. An ID only — never raw error text, never a
|
|
92
|
+
* credential — so the recommendation line can point at the right console
|
|
93
|
+
* without this struct ever carrying the raw string it was parsed from.
|
|
94
|
+
*/
|
|
95
|
+
providerId?: string
|
|
68
96
|
source: LlmErrorSource
|
|
69
97
|
/** True when the harness is still retrying this error internally (mid-retry). */
|
|
70
98
|
autoRetrying: boolean
|
|
@@ -96,7 +124,10 @@ const TRANSIENT_KINDS: ReadonlySet<LlmErrorKind> = new Set<LlmErrorKind>([
|
|
|
96
124
|
|
|
97
125
|
/** The always-actionable kinds — NEVER silenced, even inside a collapse window. */
|
|
98
126
|
export function isActionableKind(kind: LlmErrorKind): boolean {
|
|
99
|
-
|
|
127
|
+
// `provider_credit` joins the never-silenced set: an unpaid vendor balance
|
|
128
|
+
// does not clear on its own, so collapsing it inside a dedup window would
|
|
129
|
+
// hide the one signal that needs a human.
|
|
130
|
+
return kind === 'auth' || kind === 'quota_wall' || kind === 'provider_credit'
|
|
100
131
|
}
|
|
101
132
|
|
|
102
133
|
/**
|
|
@@ -118,6 +149,8 @@ export function parseLlmError(
|
|
|
118
149
|
resetAt != null ? Math.max(0, resetAt.getTime() - Date.now()) : undefined
|
|
119
150
|
|
|
120
151
|
const { kind, source } = classifyKindAndSource(text)
|
|
152
|
+
const providerId =
|
|
153
|
+
kind === 'provider_credit' ? (attributeProvider(text)?.id ?? undefined) : undefined
|
|
121
154
|
|
|
122
155
|
// Retry / terminal semantics. Only the transient family can be mid-retry;
|
|
123
156
|
// auth / quota_wall / model_unavailable are terminal by construction.
|
|
@@ -143,6 +176,7 @@ export function parseLlmError(
|
|
|
143
176
|
...(retryAfterMs != null ? { retryAfterMs } : {}),
|
|
144
177
|
...(model != null ? { model } : {}),
|
|
145
178
|
...(requestId != null ? { requestId } : {}),
|
|
179
|
+
...(providerId != null ? { providerId } : {}),
|
|
146
180
|
source,
|
|
147
181
|
autoRetrying,
|
|
148
182
|
terminal,
|
|
@@ -165,6 +199,15 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
|
|
|
165
199
|
|
|
166
200
|
// 1. Auth — always terminal, always actionable.
|
|
167
201
|
const claudeKind = classifyClaudeError({ message: text, type: text })
|
|
202
|
+
// 1b. THIRD-PARTY provider credit wall. Placed immediately after the auth
|
|
203
|
+
// resolution and BEFORE isLitellmProxyLocal429 / detectModelUnavailable:
|
|
204
|
+
// OpenAI reports an exhausted balance as HTTP 429 `insufficient_quota`,
|
|
205
|
+
// and the 429 matchers below would otherwise swallow it as a transient
|
|
206
|
+
// "retrying automatically" — the single most misleading thing to tell
|
|
207
|
+
// someone whose provider has no money left.
|
|
208
|
+
if (claudeKind === 'provider-credit-exhausted') {
|
|
209
|
+
return { kind: 'provider_credit', source: 'external-provider' }
|
|
210
|
+
}
|
|
168
211
|
if (claudeKind === 'proxy-misconfig') {
|
|
169
212
|
return { kind: 'infra_misconfig', source: 'litellm-local' }
|
|
170
213
|
}
|
|
@@ -216,6 +259,12 @@ function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmE
|
|
|
216
259
|
return { kind: 'unknown', source: 'anthropic' }
|
|
217
260
|
}
|
|
218
261
|
|
|
262
|
+
/** Resolve a registry entry from a persisted `providerId`, or null. */
|
|
263
|
+
function providerById(id: string | undefined): ProviderCreditEntry | null {
|
|
264
|
+
if (id == null) return null
|
|
265
|
+
return PROVIDER_CREDIT_REGISTRY.find(p => p.id === id) ?? null
|
|
266
|
+
}
|
|
267
|
+
|
|
219
268
|
function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
|
|
220
269
|
switch (kind) {
|
|
221
270
|
case 'rate_limit':
|
|
@@ -230,6 +279,10 @@ function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
|
|
|
230
279
|
return 'Claude login needs re-authentication.'
|
|
231
280
|
case 'infra_misconfig':
|
|
232
281
|
return 'Local model-gateway auth misconfig (proxy fallback dropped the OAuth header).'
|
|
282
|
+
case 'provider_credit':
|
|
283
|
+
// Diagnosis-free by design — this string is the ceiling on what a
|
|
284
|
+
// non-operator may ever be told about a vendor billing wall.
|
|
285
|
+
return 'An upstream model provider is out of credit — the operator has been notified.'
|
|
233
286
|
case 'transient':
|
|
234
287
|
return source === 'network'
|
|
235
288
|
? "Couldn't reach Anthropic (network) — retrying automatically."
|
|
@@ -302,6 +355,10 @@ function buildRecommendation(parsed: ParsedLlmError, tz: string): string | undef
|
|
|
302
355
|
// Operator-facing — NO re-auth wording (the login is fine). Points at the
|
|
303
356
|
// real remedy: the local LiteLLM proxy fallback config.
|
|
304
357
|
return '→ Fix the LiteLLM proxy fallback config (deployment missing OAuth passthrough).'
|
|
358
|
+
case 'provider_credit':
|
|
359
|
+
// Operator-facing. NO `/auth` wording: the credential is valid, the
|
|
360
|
+
// balance is empty, and rotating an Anthropic slot changes nothing.
|
|
361
|
+
return `→ ${describeProviderCreditRemedy(providerById(parsed.providerId))}`
|
|
305
362
|
case 'quota_wall': {
|
|
306
363
|
const reset = formatResetClock(parsed.resetAt, tz)
|
|
307
364
|
return reset
|
|
@@ -377,6 +434,8 @@ function kindEmoji(kind: LlmErrorKind): string {
|
|
|
377
434
|
return '🔑'
|
|
378
435
|
case 'infra_misconfig':
|
|
379
436
|
return '🛠️'
|
|
437
|
+
case 'provider_credit':
|
|
438
|
+
return '💳'
|
|
380
439
|
case 'transient':
|
|
381
440
|
return '🌐'
|
|
382
441
|
case 'unknown':
|
|
@@ -790,6 +790,14 @@ export function resolveModelUnavailableFromOperatorEvent(
|
|
|
790
790
|
ev: OperatorEventLike,
|
|
791
791
|
): ModelUnavailableDetection | null {
|
|
792
792
|
const detail = typeof ev.detail === 'string' ? ev.detail : ''
|
|
793
|
+
// A THIRD-PARTY provider credit wall is never "model unavailable" in the
|
|
794
|
+
// Anthropic sense. Returning a detection here would route the event into the
|
|
795
|
+
// `modelUnavailable` branch of emitGatewayOperatorEvent, which renders the
|
|
796
|
+
// "⚠️ Model unavailable" card AND — for a `quota_exhausted` detection —
|
|
797
|
+
// fires `fireFleetAutoFallback`, benching a perfectly healthy Anthropic
|
|
798
|
+
// account slot because OpenRouter ran out of money. Return null so the event
|
|
799
|
+
// takes the plain `renderOperatorEvent` path with its provider-specific card.
|
|
800
|
+
if (ev.kind === 'provider-credit-exhausted') return null
|
|
793
801
|
if (ev.kind === 'quota-exhausted') {
|
|
794
802
|
return detectModelUnavailable(detail) ?? { kind: 'quota_exhausted', raw: detail }
|
|
795
803
|
}
|
|
@@ -15,6 +15,11 @@
|
|
|
15
15
|
import { escapeMarkdown } from './format.js'
|
|
16
16
|
import { stripRawErrorBytes } from './raw-error-scrub.js'
|
|
17
17
|
import { isLitellmProxyAuthMisconfig } from './model-unavailable.js'
|
|
18
|
+
import {
|
|
19
|
+
attributeProvider,
|
|
20
|
+
describeProviderCreditRemedy,
|
|
21
|
+
detectProviderCreditExhaustion,
|
|
22
|
+
} from './provider-credit.js'
|
|
18
23
|
|
|
19
24
|
// ─── Taxonomy ────────────────────────────────────────────────────────────────
|
|
20
25
|
|
|
@@ -23,6 +28,16 @@ export type OperatorEventKind =
|
|
|
23
28
|
| 'credentials-invalid'
|
|
24
29
|
| 'proxy-misconfig'
|
|
25
30
|
| 'credit-exhausted'
|
|
31
|
+
/**
|
|
32
|
+
* A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) reports no
|
|
33
|
+
* credit remaining — HTTP 402 `payment_required`, "insufficient credits",
|
|
34
|
+
* OpenAI's `insufficient_quota`, etc. Distinct from `credit-exhausted`,
|
|
35
|
+
* which is the ANTHROPIC balance and whose remedy is `/auth use <slot>`:
|
|
36
|
+
* rotating an Anthropic account slot buys zero OpenRouter tokens, so the two
|
|
37
|
+
* must not share a card. Operator-actionable (top up in the vendor console),
|
|
38
|
+
* never user-actionable — see {@link OPERATOR_ACTIONABLE_KINDS}.
|
|
39
|
+
*/
|
|
40
|
+
| 'provider-credit-exhausted'
|
|
26
41
|
| 'quota-exhausted'
|
|
27
42
|
| 'rate-limited'
|
|
28
43
|
| 'agent-crashed'
|
|
@@ -134,16 +149,38 @@ function classifyInner(raw: unknown): OperatorEventKind {
|
|
|
134
149
|
return 'credentials-invalid'
|
|
135
150
|
}
|
|
136
151
|
|
|
152
|
+
// The full scan surface for the provider-credit decision — same
|
|
153
|
+
// type/code/message union isLitellmProxyAuthMisconfig uses, because LiteLLM
|
|
154
|
+
// stamps the upstream vendor's wording in whichever field it has room for.
|
|
155
|
+
const creditScanText = `${errorType}\n${errorCode}\n${sdkCode}\n${message}`
|
|
156
|
+
|
|
157
|
+
// ANTHROPIC credit wall. Guarded with `attributeProvider(...) == null` so an
|
|
158
|
+
// OpenRouter/OpenAI/Perplexity error that happens to say "credit balance"
|
|
159
|
+
// does NOT get the Anthropic card, whose remedy (`/auth use <slot>`) is
|
|
160
|
+
// useless against a third-party balance. Unattributed credit-balance wording
|
|
161
|
+
// keeps its historical Anthropic reading — that is where it has always come
|
|
162
|
+
// from, and the alternative (guessing a vendor) sends the operator to the
|
|
163
|
+
// wrong console.
|
|
137
164
|
if (
|
|
138
|
-
errorType === 'credit_balance_too_low' ||
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
165
|
+
(errorType === 'credit_balance_too_low' ||
|
|
166
|
+
errorCode === 'credit_balance_too_low' ||
|
|
167
|
+
sdkCode === 'credit_balance_too_low' ||
|
|
168
|
+
message.toLowerCase().includes('credit_balance_too_low') ||
|
|
169
|
+
message.toLowerCase().includes('credit balance')) &&
|
|
170
|
+
attributeProvider(creditScanText) == null
|
|
143
171
|
) {
|
|
144
172
|
return 'credit-exhausted'
|
|
145
173
|
}
|
|
146
174
|
|
|
175
|
+
// THIRD-PARTY provider credit wall — the OpenRouter 402 leak (#external-credit).
|
|
176
|
+
// Runs BEFORE the rate-limit branch on purpose: OpenAI signals an exhausted
|
|
177
|
+
// balance as HTTP *429* `insufficient_quota`, so a rate-limit-first ordering
|
|
178
|
+
// would classify a billing wall as a transient throttle and tell the operator
|
|
179
|
+
// to wait for a reset that will never come.
|
|
180
|
+
if (detectProviderCreditExhaustion(creditScanText, status) != null) {
|
|
181
|
+
return 'provider-credit-exhausted'
|
|
182
|
+
}
|
|
183
|
+
|
|
147
184
|
if (
|
|
148
185
|
errorType === 'rate_limit_error' ||
|
|
149
186
|
errorCode === 'rate_limit_error' ||
|
|
@@ -316,6 +353,31 @@ export function renderOperatorEvent(ev: OperatorEvent): RenderResult {
|
|
|
316
353
|
},
|
|
317
354
|
}
|
|
318
355
|
|
|
356
|
+
// A third-party provider's balance, NOT Anthropic's. Deliberately carries
|
|
357
|
+
// NO `/auth use` / `/auth add` language (those rotate Anthropic account
|
|
358
|
+
// slots and buy zero OpenRouter tokens) and NO "🔐 Reauth" button (the key
|
|
359
|
+
// is valid; it is out of money). The remedy names the provider, the VAULT
|
|
360
|
+
// KEY NAME — never a value — and the console to act in. Dismiss-only.
|
|
361
|
+
case 'provider-credit-exhausted': {
|
|
362
|
+
const provider = attributeProvider(ev.detail)
|
|
363
|
+
const who = provider != null ? escapeMarkdown(provider.label) : 'An upstream model provider'
|
|
364
|
+
return {
|
|
365
|
+
text: [
|
|
366
|
+
`💳 **${provider != null ? `${who} credit exhausted` : 'Upstream provider credit exhausted'}** — hit by **${agent}**.`,
|
|
367
|
+
detail ? `_${detail}_` : '',
|
|
368
|
+
describeProviderCreditRemedy(provider),
|
|
369
|
+
`Anthropic account slots are unaffected — \`/auth use\` will NOT fix this.`,
|
|
370
|
+
]
|
|
371
|
+
.filter(Boolean)
|
|
372
|
+
.join('\n'),
|
|
373
|
+
keyboard: {
|
|
374
|
+
inline_keyboard: [
|
|
375
|
+
[{ text: '❌ Dismiss', callback_data: `op:dismiss:${encodeURIComponent(ev.agent)}` }],
|
|
376
|
+
],
|
|
377
|
+
},
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
319
381
|
case 'quota-exhausted':
|
|
320
382
|
// Canonical quota-exhausted text (migrated from auto-fallback.ts).
|
|
321
383
|
// auto-fallback.ts's buildSwitchedMessage / buildAllExhaustedMessage
|
|
@@ -544,6 +606,11 @@ export const OPERATOR_ACTIONABLE_KINDS: ReadonlySet<OperatorEventKind> = new Set
|
|
|
544
606
|
'credentials-expired',
|
|
545
607
|
'credentials-invalid',
|
|
546
608
|
'credit-exhausted',
|
|
609
|
+
// The OpenRouter/OpenAI/Perplexity 402 class. Only the operator can top up a
|
|
610
|
+
// vendor balance; an end user shown "insufficient credits" can do nothing but
|
|
611
|
+
// lose confidence. This membership is what routes it to the operator card and
|
|
612
|
+
// hands the user the brief plain-language notice instead.
|
|
613
|
+
'provider-credit-exhausted',
|
|
547
614
|
'proxy-misconfig',
|
|
548
615
|
])
|
|
549
616
|
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Outbound priority class, propagated to the grammY transformer stack.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this module exists ────────────────────────────────────────────────
|
|
5
|
+
* The send gate (`send-gate.ts`) knows every outbound call's `priorityClass`
|
|
6
|
+
* — `cosmetic` (a repaint of a progress card), `useful`, or `critical` (the
|
|
7
|
+
* operator's actual answer). The edit-flood fuse (`edit-flood-fuse.ts`) is a
|
|
8
|
+
* grammY API transformer, and a transformer sees only `(method, payload)`.
|
|
9
|
+
* It therefore could NOT tell a liveness repaint apart from a reply, and had
|
|
10
|
+
* to treat every edit identically.
|
|
11
|
+
*
|
|
12
|
+
* That blindness is the core defect behind the 2026-07-27 incident: agent
|
|
13
|
+
* `overlord` sustained ~17 activity-card edits/min for hours, earned a
|
|
14
|
+
* 15908-second (4.4h) per-chat flood ban, and the ban took out ALL replies.
|
|
15
|
+
* The fuse's per-chat edit ceiling at the time was 30/60s, so it never bound
|
|
16
|
+
* — a purely cosmetic surface was allowed to consume the whole chat's real
|
|
17
|
+
* Telegram budget, and the answer path paid for it.
|
|
18
|
+
*
|
|
19
|
+
* A class-aware fuse can do the two things a class-blind one cannot:
|
|
20
|
+
* 1. hold COSMETIC traffic to a rate far below Telegram's per-chat ceiling,
|
|
21
|
+
* without throttling approval cards or answers; and
|
|
22
|
+
* 2. RESERVE part of the per-chat budget so a reply is never starved by
|
|
23
|
+
* repaints.
|
|
24
|
+
*
|
|
25
|
+
* ── Why AsyncLocalStorage ─────────────────────────────────────────────────
|
|
26
|
+
* The class must travel from the send gate (well above grammY) down to the
|
|
27
|
+
* transformer, through an arbitrary async chain, WITHOUT smuggling a
|
|
28
|
+
* non-Bot-API field into the outbound payload. `AsyncLocalStorage` is the
|
|
29
|
+
* Node-native mechanism for exactly that, and the plugin already uses the
|
|
30
|
+
* same pattern for `tg-post` log tags (`shared/bot-runtime.ts`).
|
|
31
|
+
*
|
|
32
|
+
* This module deliberately has NO imports beyond `async_hooks`: both
|
|
33
|
+
* `send-gate.ts` (the writer) and `edit-flood-fuse.ts` (the reader) import
|
|
34
|
+
* it, so anything heavier would create an import cycle.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import { AsyncLocalStorage } from 'async_hooks'
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Mirrors `send-gate.ts`'s `PriorityClass`. Declared here rather than
|
|
41
|
+
* imported so this module stays dependency-free (see the docblock); the
|
|
42
|
+
* send gate's `PriorityClass` is assignable to it and a compile-time
|
|
43
|
+
* assertion in `send-gate.ts` keeps the two in lockstep.
|
|
44
|
+
*/
|
|
45
|
+
export type OutboundClass = 'critical' | 'useful' | 'cosmetic'
|
|
46
|
+
|
|
47
|
+
const store = new AsyncLocalStorage<OutboundClass>()
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Run `fn` with `cls` visible to every Telegram API call it makes, including
|
|
51
|
+
* across awaits. The send gate wraps each admitted call in this.
|
|
52
|
+
*/
|
|
53
|
+
export function withOutboundClass<T>(cls: OutboundClass, fn: () => T): T {
|
|
54
|
+
return store.run(cls, fn)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** The class of the call currently in flight, or undefined when untagged. */
|
|
58
|
+
export function currentOutboundClass(): OutboundClass | undefined {
|
|
59
|
+
return store.getStore()
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The class the fuse assigns to a call that arrived with no tag.
|
|
64
|
+
*
|
|
65
|
+
* Asymmetric ON PURPOSE:
|
|
66
|
+
*
|
|
67
|
+
* - an untagged EDIT is treated as `cosmetic`. An edit that never went
|
|
68
|
+
* through the send gate is, by construction, exactly the shape of the
|
|
69
|
+
* incident (the activity card edited one message id with no key and no
|
|
70
|
+
* class). Defaulting it to cosmetic means a NEW unkeyed edit call site
|
|
71
|
+
* is rate-limited by default instead of being invisible until it earns
|
|
72
|
+
* a ban. Dropping a repaint is free — the next render carries full
|
|
73
|
+
* state.
|
|
74
|
+
* - an untagged SEND is treated as `critical`, matching the send gate's
|
|
75
|
+
* own `UNTAGGED_SEND_CLASS`. Sends create user-visible output and are
|
|
76
|
+
* never dropped by the fuse; the conservative default here is the one
|
|
77
|
+
* that cannot lose an answer.
|
|
78
|
+
*/
|
|
79
|
+
export function defaultOutboundClass(isEdit: boolean): OutboundClass {
|
|
80
|
+
return isEdit ? 'cosmetic' : 'critical'
|
|
81
|
+
}
|