switchroom 0.18.15 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +16 -0
- package/dist/auth-broker/index.js +445 -10
- package/dist/cli/notion-write-pretool.mjs +16 -0
- package/dist/cli/switchroom.js +654 -479
- package/dist/host-control/main.js +20 -1
- package/dist/vault/approvals/kernel-server.js +16 -0
- package/dist/vault/broker/server.js +16 -0
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1758 -661
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +64 -22
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +627 -291
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +141 -172
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/runtime-metrics.ts +16 -0
- package/telegram-plugin/send-gate-degraded.test.ts +161 -8
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +246 -23
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +5 -4
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +8 -3
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +212 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +543 -368
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration test for the tier-downgrade GLUE (`runTierDowngrade`,
|
|
3
|
+
* tier-downgrade-wiring.ts) — the gateway orchestration the pure
|
|
4
|
+
* `decideTierDowngrade` / `renderTierDowngradeNotice` tests do NOT cover.
|
|
5
|
+
*
|
|
6
|
+
* This is the test the @overlord review asked for (the MEDIUM "reverts to
|
|
7
|
+
* fable" misinform slipped through precisely because the glue that emits the
|
|
8
|
+
* BROADCAST notice text had no test — only the pure renderer did). It drives
|
|
9
|
+
* the real runner with injected seams and PINS the broadcast notice:
|
|
10
|
+
* - asserts the resume-on-DEFAULT wording + the manual re-issue instruction;
|
|
11
|
+
* - FAILS on any revert-to-premium phrasing (the exact regression).
|
|
12
|
+
* It also covers the resume-gate suppression path and the carrier-before-arm
|
|
13
|
+
* ordering invariant.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, it, expect } from 'vitest'
|
|
17
|
+
import {
|
|
18
|
+
runTierDowngrade,
|
|
19
|
+
type TierDowngradeRunnerDeps,
|
|
20
|
+
type TierDowngradeOutcome,
|
|
21
|
+
} from '../gateway/tier-downgrade-wiring.js'
|
|
22
|
+
|
|
23
|
+
interface Recorder {
|
|
24
|
+
carrier: Array<{ dir: string; toModel: string; cfg: string }>
|
|
25
|
+
arms: number
|
|
26
|
+
markers: Array<{ dir: string; premiumModel: string }>
|
|
27
|
+
notices: string[]
|
|
28
|
+
restarts: string[]
|
|
29
|
+
logs: string[]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function makeDeps(
|
|
33
|
+
over: Partial<TierDowngradeRunnerDeps> & {
|
|
34
|
+
sessionOverride?: string | null
|
|
35
|
+
configuredDefault?: string | null
|
|
36
|
+
gateVerdict?: 'resume' | 'skip-inflight' | 'skip-stale'
|
|
37
|
+
carrierThrows?: boolean
|
|
38
|
+
markerThrows?: boolean
|
|
39
|
+
agentDir?: string | null
|
|
40
|
+
} = {},
|
|
41
|
+
): { deps: TierDowngradeRunnerDeps; rec: Recorder } {
|
|
42
|
+
const rec: Recorder = { carrier: [], arms: 0, markers: [], notices: [], restarts: [], logs: [] }
|
|
43
|
+
// A near-identity resolver like the real resolveMainModel: the unset/`default`
|
|
44
|
+
// sentinel maps to the switchroom default, everything else is itself.
|
|
45
|
+
const resolve = (t: string): string => (t === '' || t === 'default' ? 'claude-opus-4-8' : t)
|
|
46
|
+
const deps: TierDowngradeRunnerDeps = {
|
|
47
|
+
getAgentDir: () => (over.agentDir === undefined ? '/tmp/agent' : over.agentDir),
|
|
48
|
+
getConfiguredDefault: () => (over.configuredDefault === undefined ? 'opus' : over.configuredDefault),
|
|
49
|
+
getSessionOverride: () => (over.sessionOverride === undefined ? 'fable' : over.sessionOverride),
|
|
50
|
+
resolve,
|
|
51
|
+
peekResumeGate: () => over.gateVerdict ?? 'resume',
|
|
52
|
+
writeCarrier: (dir, toModel, cfg) => {
|
|
53
|
+
if (over.carrierThrows) throw new Error('disk full')
|
|
54
|
+
rec.carrier.push({ dir, toModel, cfg })
|
|
55
|
+
},
|
|
56
|
+
armResumeGate: () => {
|
|
57
|
+
rec.arms += 1
|
|
58
|
+
},
|
|
59
|
+
writeRecoveryMarker: (dir, premiumModel) => {
|
|
60
|
+
if (over.markerThrows) throw new Error('marker write failed')
|
|
61
|
+
rec.markers.push({ dir, premiumModel })
|
|
62
|
+
},
|
|
63
|
+
broadcastNotice: (md) => rec.notices.push(md),
|
|
64
|
+
selfRestart: (agent) => rec.restarts.push(agent),
|
|
65
|
+
selfAgent: (t) => `self-${t}`,
|
|
66
|
+
log: (msg) => rec.logs.push(msg),
|
|
67
|
+
...over,
|
|
68
|
+
}
|
|
69
|
+
return { deps, rec }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
describe('runTierDowngrade — broadcast notice honesty (the pinned regression)', () => {
|
|
73
|
+
it('downgrade path broadcasts EXACTLY ONE notice with resume-on-default wording', () => {
|
|
74
|
+
const { deps, rec } = makeDeps({ sessionOverride: 'fable', configuredDefault: 'opus' })
|
|
75
|
+
const outcome: TierDowngradeOutcome = runTierDowngrade('klanker', deps)
|
|
76
|
+
expect(outcome).toBe('downgraded')
|
|
77
|
+
expect(rec.notices).toHaveLength(1)
|
|
78
|
+
const notice = rec.notices[0]
|
|
79
|
+
// Resume-on-DEFAULT + the manual re-issue instruction.
|
|
80
|
+
expect(notice).toContain('`opus`')
|
|
81
|
+
expect(notice).toContain('/model fable')
|
|
82
|
+
expect(notice.toLowerCase()).toContain("won't switch back on its own")
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
it('the notice NEVER promises a revert to the premium model (the MEDIUM regression)', () => {
|
|
86
|
+
const { deps, rec } = makeDeps({ sessionOverride: 'fable', configuredDefault: 'opus' })
|
|
87
|
+
runTierDowngrade('klanker', deps)
|
|
88
|
+
const notice = rec.notices[0].toLowerCase()
|
|
89
|
+
// The exact false phrasing from the pre-5d9bbf44 revision, and its kin.
|
|
90
|
+
expect(notice).not.toContain('reverts to `fable`')
|
|
91
|
+
expect(notice).not.toContain('revert to `fable`')
|
|
92
|
+
expect(notice).not.toMatch(/reverts? to fable/)
|
|
93
|
+
expect(notice).not.toMatch(/switch(es)? back.*on (the )?next restart/)
|
|
94
|
+
expect(notice).not.toMatch(/automatically.*(fable|premium)/)
|
|
95
|
+
})
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
describe('runTierDowngrade — side-effect ordering + outcomes', () => {
|
|
99
|
+
it('downgrade: carrier written BEFORE the arm, marker + notice + restart all fire', () => {
|
|
100
|
+
const { deps, rec } = makeDeps({ sessionOverride: 'fable', configuredDefault: 'opus' })
|
|
101
|
+
const outcome = runTierDowngrade('klanker', deps)
|
|
102
|
+
expect(outcome).toBe('downgraded')
|
|
103
|
+
expect(rec.carrier).toEqual([{ dir: '/tmp/agent', toModel: 'opus', cfg: 'opus' }])
|
|
104
|
+
expect(rec.arms).toBe(1)
|
|
105
|
+
expect(rec.markers).toEqual([{ dir: '/tmp/agent', premiumModel: 'fable' }])
|
|
106
|
+
expect(rec.restarts).toEqual(['self-klanker'])
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
it('carrier write THROWS → abort: NO arm, NO marker, NO notice, NO restart (skip)', () => {
|
|
110
|
+
const { deps, rec } = makeDeps({ carrierThrows: true })
|
|
111
|
+
const outcome = runTierDowngrade('klanker', deps)
|
|
112
|
+
expect(outcome).toBe('skip')
|
|
113
|
+
expect(rec.arms).toBe(0)
|
|
114
|
+
expect(rec.markers).toHaveLength(0)
|
|
115
|
+
expect(rec.notices).toHaveLength(0)
|
|
116
|
+
expect(rec.restarts).toHaveLength(0)
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
it('marker write THROWS → downgrade still completes (marker is best-effort)', () => {
|
|
120
|
+
const { deps, rec } = makeDeps({ markerThrows: true })
|
|
121
|
+
const outcome = runTierDowngrade('klanker', deps)
|
|
122
|
+
expect(outcome).toBe('downgraded')
|
|
123
|
+
expect(rec.arms).toBe(1)
|
|
124
|
+
expect(rec.notices).toHaveLength(1)
|
|
125
|
+
expect(rec.restarts).toEqual(['self-klanker'])
|
|
126
|
+
expect(rec.markers).toHaveLength(0)
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it('resume-gate skip-inflight → restart-pending, NO notice, NO restart, NO carrier', () => {
|
|
130
|
+
const { deps, rec } = makeDeps({ gateVerdict: 'skip-inflight' })
|
|
131
|
+
const outcome = runTierDowngrade('klanker', deps)
|
|
132
|
+
expect(outcome).toBe('restart-pending')
|
|
133
|
+
expect(rec.carrier).toHaveLength(0)
|
|
134
|
+
expect(rec.arms).toBe(0)
|
|
135
|
+
expect(rec.notices).toHaveLength(0)
|
|
136
|
+
expect(rec.restarts).toHaveLength(0)
|
|
137
|
+
})
|
|
138
|
+
|
|
139
|
+
it('resume-gate skip-stale → skip (give up), NO notice/restart', () => {
|
|
140
|
+
const { deps, rec } = makeDeps({ gateVerdict: 'skip-stale' })
|
|
141
|
+
const outcome = runTierDowngrade('klanker', deps)
|
|
142
|
+
expect(outcome).toBe('skip')
|
|
143
|
+
expect(rec.notices).toHaveLength(0)
|
|
144
|
+
expect(rec.restarts).toHaveLength(0)
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
it('session on the configured default (override null) → skip, no downgrade', () => {
|
|
148
|
+
const { deps, rec } = makeDeps({ sessionOverride: null })
|
|
149
|
+
const outcome = runTierDowngrade('klanker', deps)
|
|
150
|
+
expect(outcome).toBe('skip')
|
|
151
|
+
expect(rec.notices).toHaveLength(0)
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
it('unresolved configured default → skip (never downgrade blind)', () => {
|
|
155
|
+
const { deps, rec } = makeDeps({ configuredDefault: '' })
|
|
156
|
+
const outcome = runTierDowngrade('klanker', deps)
|
|
157
|
+
expect(outcome).toBe('skip')
|
|
158
|
+
expect(rec.carrier).toHaveLength(0)
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
it('no agent dir → skip immediately', () => {
|
|
162
|
+
const { deps } = makeDeps({ agentDir: null })
|
|
163
|
+
expect(runTierDowngrade('klanker', deps)).toBe('skip')
|
|
164
|
+
})
|
|
165
|
+
})
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for the MODEL-TIER downgrade failover (second recovery tier).
|
|
3
|
+
*
|
|
4
|
+
* The pure `decideTierDowngrade` owns the decision the gateway consults inside
|
|
5
|
+
* the `all-blocked` branch of doFireFleetAutoFallback — i.e. AFTER account-swap
|
|
6
|
+
* has been tried and found no account still serving the walled premium model.
|
|
7
|
+
* `planTierDowngrade` then routes that decision + the resume-gate verdict into
|
|
8
|
+
* what the gateway should DO (downgrade / suppress the give-up / fall through),
|
|
9
|
+
* and `renderTierDowngradeNotice` owns the exact user-facing wording. All three
|
|
10
|
+
* are pure so the decision, the concurrent-turn suppression (LOW-9a), and the
|
|
11
|
+
* honesty of the broadcast (no "revert to the premium model" promise) are pinned
|
|
12
|
+
* without a process restart.
|
|
13
|
+
*
|
|
14
|
+
* Contract (post loop-guard reconciliation, option b — the ceremonial
|
|
15
|
+
* `.tier-downgrade-attempts` counter was REMOVED):
|
|
16
|
+
* - overload + the session is NOT on a premium model (override null, or the
|
|
17
|
+
* override resolves to the configured default) → NO downgrade ('skip'); the
|
|
18
|
+
* account-swap path already handled it or there is no lower tier to fall to.
|
|
19
|
+
* This is also the natural loop bound: after the downgrade boot the override
|
|
20
|
+
* is gone, so a re-entry lands on 'on-default' and never re-downgrades.
|
|
21
|
+
* - overload + a premium model walled across ALL accounts → 'downgrade' to the
|
|
22
|
+
* configured default.
|
|
23
|
+
* - a resume restart already armed in this process (gate 'skip-inflight') →
|
|
24
|
+
* 'suppress': no give-up card, no re-downgrade (the armed restart resumes).
|
|
25
|
+
* - the downgrade broadcast states resume-on-default + manual re-issue and
|
|
26
|
+
* carries NO promise of an automatic revert to the premium model.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { describe, it, expect } from 'vitest'
|
|
30
|
+
import {
|
|
31
|
+
decideTierDowngrade,
|
|
32
|
+
planTierDowngrade,
|
|
33
|
+
renderTierDowngradeNotice,
|
|
34
|
+
} from '../tier-downgrade.js'
|
|
35
|
+
|
|
36
|
+
// The gateway passes resolveMainModel; here a near-identity resolver (maps the
|
|
37
|
+
// unset/`default` sentinel to the switchroom default, exactly like the real one)
|
|
38
|
+
// is enough to exercise the canonicalization guard.
|
|
39
|
+
const resolve = (t: string): string =>
|
|
40
|
+
t === '' || t === 'default' ? 'claude-sonnet-5' : t
|
|
41
|
+
|
|
42
|
+
describe('decideTierDowngrade — precedence + downgrade target', () => {
|
|
43
|
+
it('NO downgrade when the session is on the configured default (override null)', () => {
|
|
44
|
+
// Overload + account-swap all-blocked, but nothing premium is active — the
|
|
45
|
+
// account-swap path owns this and there is no lower tier. => skip.
|
|
46
|
+
const d = decideTierDowngrade({ sessionOverride: null, configuredDefault: 'opus', resolve })
|
|
47
|
+
expect(d).toEqual({ action: 'skip', reason: 'on-default' })
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
it('NO downgrade when the override resolves to the configured default (alias vs id)', () => {
|
|
51
|
+
// A premium-looking override that is actually the default in another spelling
|
|
52
|
+
// must never read as a premium tier (that would loop). Same resolver both
|
|
53
|
+
// sides → on-default.
|
|
54
|
+
const d = decideTierDowngrade({
|
|
55
|
+
sessionOverride: 'default',
|
|
56
|
+
configuredDefault: 'claude-sonnet-5',
|
|
57
|
+
resolve,
|
|
58
|
+
})
|
|
59
|
+
expect(d).toEqual({ action: 'skip', reason: 'on-default' })
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
it('NO downgrade (and no blind fall) when the configured default is unreadable', () => {
|
|
63
|
+
const d = decideTierDowngrade({ sessionOverride: 'fable', configuredDefault: null, resolve })
|
|
64
|
+
expect(d).toEqual({ action: 'skip', reason: 'unresolved' })
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
it('DOWNGRADES a walled premium model to the configured default', () => {
|
|
68
|
+
const d = decideTierDowngrade({ sessionOverride: 'fable', configuredDefault: 'opus', resolve })
|
|
69
|
+
expect(d).toEqual({ action: 'downgrade', toModel: 'opus', fromModel: 'fable' })
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('downgrade target is GENERAL — any premium model → the agent default (not hardcoded fable→opus)', () => {
|
|
73
|
+
const d = decideTierDowngrade({
|
|
74
|
+
sessionOverride: 'sr-x-ai/grok-4',
|
|
75
|
+
configuredDefault: 'claude-sonnet-5',
|
|
76
|
+
resolve,
|
|
77
|
+
})
|
|
78
|
+
expect(d).toEqual({ action: 'downgrade', toModel: 'claude-sonnet-5', fromModel: 'sr-x-ai/grok-4' })
|
|
79
|
+
})
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
describe('renderTierDowngradeNotice — HONEST wording (no revert-to-premium promise)', () => {
|
|
83
|
+
const notice = renderTierDowngradeNotice('fable', 'opus', 'klanker')
|
|
84
|
+
|
|
85
|
+
it('states the turn resumes on the DEFAULT to keep working', () => {
|
|
86
|
+
expect(notice).toContain('opus')
|
|
87
|
+
expect(notice.toLowerCase()).toContain('resuming on the default')
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
it('tells the user to RE-ISSUE the premium /model themselves', () => {
|
|
91
|
+
expect(notice).toContain('/model fable')
|
|
92
|
+
expect(notice.toLowerCase()).toContain("won't switch back on its own")
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
it("carries NO promise of an automatic revert to the premium/fable model", () => {
|
|
96
|
+
// The HIGH finding: the prior wording lied ("It reverts to `fable` on the
|
|
97
|
+
// next restart"). Assert none of those false promises can creep back.
|
|
98
|
+
const lower = notice.toLowerCase()
|
|
99
|
+
expect(lower).not.toMatch(/reverts? to `?fable/)
|
|
100
|
+
expect(lower).not.toContain('reverts to `fable`')
|
|
101
|
+
expect(lower).not.toContain('on the next restart')
|
|
102
|
+
expect(lower).not.toMatch(/revert(s|ing)? to the premium/)
|
|
103
|
+
expect(lower).not.toMatch(/back to `?fable`? (on|at|after)/)
|
|
104
|
+
})
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
describe('planTierDowngrade — routing (downgrade / suppress / skip)', () => {
|
|
108
|
+
const skipDecision = { action: 'skip', reason: 'on-default' } as const
|
|
109
|
+
const downgradeDecision = { action: 'downgrade', toModel: 'opus', fromModel: 'fable' } as const
|
|
110
|
+
|
|
111
|
+
it("a 'skip' decision routes to 'skip' regardless of the gate verdict", () => {
|
|
112
|
+
expect(planTierDowngrade(skipDecision, 'resume', 'ag').kind).toBe('skip')
|
|
113
|
+
expect(planTierDowngrade(skipDecision, 'skip-inflight', 'ag').kind).toBe('skip')
|
|
114
|
+
expect(planTierDowngrade(skipDecision, 'skip-stale', 'ag').kind).toBe('skip')
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
it("downgrade + gate 'resume' → 'downgrade' with the honest notice", () => {
|
|
118
|
+
const plan = planTierDowngrade(downgradeDecision, 'resume', 'klanker')
|
|
119
|
+
expect(plan.kind).toBe('downgrade')
|
|
120
|
+
if (plan.kind === 'downgrade') {
|
|
121
|
+
expect(plan.toModel).toBe('opus')
|
|
122
|
+
expect(plan.fromModel).toBe('fable')
|
|
123
|
+
// The plan carries EXACTLY the honest renderer output — no revert promise.
|
|
124
|
+
expect(plan.notice).toBe(renderTierDowngradeNotice('fable', 'opus', 'klanker'))
|
|
125
|
+
expect(plan.notice.toLowerCase()).not.toContain('on the next restart')
|
|
126
|
+
}
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it("LOW-9a: downgrade + gate 'skip-inflight' → 'suppress' (a resume restart is already armed, no give-up card)", () => {
|
|
130
|
+
// Two turns hit all-blocked in the same pre-restart process. Turn 1 armed a
|
|
131
|
+
// downgrade (or account-swap) restart. Turn 2 must NOT broadcast a "could
|
|
132
|
+
// not be recovered" give-up — the armed restart resumes the interrupted turn.
|
|
133
|
+
const plan = planTierDowngrade(downgradeDecision, 'skip-inflight', 'klanker')
|
|
134
|
+
expect(plan.kind).toBe('suppress')
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
it("downgrade + gate 'skip-stale' → 'skip' (turn too old to resume; fall through to the all-blocked card)", () => {
|
|
138
|
+
const plan = planTierDowngrade(downgradeDecision, 'skip-stale', 'klanker')
|
|
139
|
+
expect(plan.kind).toBe('skip')
|
|
140
|
+
})
|
|
141
|
+
})
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { describe, it, expect } from 'bun:test'
|
|
12
12
|
import { GrammyError } from 'grammy'
|
|
13
13
|
import { classifyRejection } from '../gateway/unhandled-rejection-policy.js'
|
|
14
|
-
import { FLOOD_WAIT_ACTIVE } from '../retry-api-call.js'
|
|
14
|
+
import { FLOOD_WAIT_ACTIVE, LOCAL_RESOURCE_EXHAUSTED } from '../retry-api-call.js'
|
|
15
15
|
|
|
16
16
|
// ── Real GrammyError fixtures ──────────────────────────────────────────────
|
|
17
17
|
|
|
@@ -241,3 +241,29 @@ describe('classifyRejection — FLOOD_WAIT_ACTIVE marker (#3084)', () => {
|
|
|
241
241
|
expect(classifyRejection(new Error('FLOOD_WAIT_ACTIVE-ish but not it'))).toBe('shutdown')
|
|
242
242
|
})
|
|
243
243
|
})
|
|
244
|
+
|
|
245
|
+
describe('classifyRejection — LOCAL_RESOURCE_EXHAUSTED marker (#3099)', () => {
|
|
246
|
+
// retry-api-call throws this plain Error marker (retry-api-call.ts:344) when a
|
|
247
|
+
// send fails on LOCAL disk/memory exhaustion (ENOSPC/EDQUOT/EIO/ENOMEM) rather
|
|
248
|
+
// than retrying it (#2923). Like its sibling FLOOD_WAIT_ACTIVE it is a plain
|
|
249
|
+
// Error, not a GrammyError, so without an explicit entry it fell into the
|
|
250
|
+
// `!isGrammy → shutdown` branch — crashing the gateway when the box is ALREADY
|
|
251
|
+
// out of disk, which drives a fresh round of boot-time sends/staging writes at
|
|
252
|
+
// an exhausted resource (the amplification the #2923 marker exists to avoid).
|
|
253
|
+
it('returns "log_only" for a leaked LOCAL_RESOURCE_EXHAUSTED marker', () => {
|
|
254
|
+
// Mirror the real throw shape from retry-api-call.ts:344 —
|
|
255
|
+
// `Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED), { original: err })`.
|
|
256
|
+
const err = Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED), {
|
|
257
|
+
original: Object.assign(new Error('ENOSPC: no space left on device'), {
|
|
258
|
+
code: 'ENOSPC',
|
|
259
|
+
}),
|
|
260
|
+
})
|
|
261
|
+
expect(classifyRejection(err)).toBe('log_only')
|
|
262
|
+
})
|
|
263
|
+
|
|
264
|
+
it('still returns "shutdown" for an unrelated plain Error', () => {
|
|
265
|
+
expect(
|
|
266
|
+
classifyRejection(new Error('LOCAL_RESOURCE_EXHAUSTED-ish but not it')),
|
|
267
|
+
).toBe('shutdown')
|
|
268
|
+
})
|
|
269
|
+
})
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
type BotApiForWorkerFeed,
|
|
8
8
|
} from '../worker-activity-feed.js'
|
|
9
9
|
import { STATUS_ROLLING_LINES, STATUS_LINE_MAX } from '../status-no-truncate.js'
|
|
10
|
+
import { SEND_GATE_SHED } from '../send-gate.js'
|
|
10
11
|
|
|
11
12
|
describe('isWorkerActivityFeedEnabled (default ON)', () => {
|
|
12
13
|
it('defaults to true when the env var is unset', () => {
|
|
@@ -532,17 +533,20 @@ describe('createWorkerActivityFeed — log sink', () => {
|
|
|
532
533
|
const edit = logs.find((l) => l.startsWith('worker-feed: edit'))
|
|
533
534
|
const finish = logs.find((l) => l.startsWith('worker-feed: finish'))
|
|
534
535
|
|
|
536
|
+
// The feed message is per-(chat,thread) now (workers coalesce), so the
|
|
537
|
+
// paint/edit lines are feed-scoped; the terminal `finish` line still names
|
|
538
|
+
// the finishing worker + its state.
|
|
535
539
|
expect(paint).toBeDefined()
|
|
536
|
-
expect(paint).toContain('agent=w-research')
|
|
537
540
|
expect(paint).toContain('chat=chat-9')
|
|
538
541
|
expect(paint).toContain('thread=7')
|
|
539
542
|
expect(paint).toMatch(/msgId=\d+/)
|
|
540
543
|
expect(paint).toMatch(/bytes=\d+/)
|
|
541
544
|
|
|
542
545
|
expect(edit).toBeDefined()
|
|
543
|
-
expect(edit).toContain('
|
|
546
|
+
expect(edit).toContain('chat=chat-9')
|
|
544
547
|
|
|
545
548
|
expect(finish).toBeDefined()
|
|
549
|
+
expect(finish).toContain('agent=w-research')
|
|
546
550
|
expect(finish).toContain('state=done')
|
|
547
551
|
})
|
|
548
552
|
|
|
@@ -1336,3 +1340,209 @@ describe('narrative dedup — non-adjacent repeats collapse (A,B,A)', () => {
|
|
|
1336
1340
|
expect(last.text).toContain('step-repeat')
|
|
1337
1341
|
})
|
|
1338
1342
|
})
|
|
1343
|
+
|
|
1344
|
+
// ─── send-gate shed contract (#3174) ────────────────────────────────────────
|
|
1345
|
+
//
|
|
1346
|
+
// The feed's send/edit adapters transit the deterministic send gate
|
|
1347
|
+
// (telegram-plugin/send-gate.ts, #3084), wired at the robustApiCall layer. The
|
|
1348
|
+
// gate SHEDS a call — resolves `undefined` WITHOUT hitting the API — when a
|
|
1349
|
+
// flood window is open (or a `useful` send exceeds its queue TTL, or a cosmetic
|
|
1350
|
+
// edit finds no free token). Before this fix the first-paint path dereferenced
|
|
1351
|
+
// `sent.message_id` on that `undefined`, throwing every heartbeat tick (~6s) for
|
|
1352
|
+
// the whole ban — 578 `undefined is not an object (evaluating 'sent.message_id')`
|
|
1353
|
+
// crashes in one 6h flood ban on 2026-07-12. These tests pin the shed contract:
|
|
1354
|
+
// (a) an open flood window parks the handle and makes ZERO api calls; (b) an
|
|
1355
|
+
// undefined send is treated as NOT-delivered (no crash, no phantom message id);
|
|
1356
|
+
// (c)/(d) an undefined edit is not recorded as on-screen (shed honesty, mirroring
|
|
1357
|
+
// PR #3173) so the update re-sends once the gate clears.
|
|
1358
|
+
|
|
1359
|
+
interface GateBot extends BotApiForWorkerFeed {
|
|
1360
|
+
/** Times sendMessage was INVOKED (attempts), delivered or shed. */
|
|
1361
|
+
sendCalls: number
|
|
1362
|
+
/** Times editMessageText was INVOKED (attempts), delivered or shed. */
|
|
1363
|
+
editCalls: number
|
|
1364
|
+
/** DELIVERED sends (gate admitted). */
|
|
1365
|
+
sent: Array<{ text: string }>
|
|
1366
|
+
/** DELIVERED edits (gate admitted). */
|
|
1367
|
+
edits: Array<{ messageId: number; text: string }>
|
|
1368
|
+
/**
|
|
1369
|
+
* One-shot: shed the next send even with the window closed. Faithful to the
|
|
1370
|
+
* gate: a worker-feed SEND is `useful` (never `cosmetic`), so a gate that
|
|
1371
|
+
* can't admit it resolves `undefined` (queue-TTL `expired`) — NEVER the
|
|
1372
|
+
* cosmetic-only SEND_GATE_SHED sentinel. The feed's send-shed detection is
|
|
1373
|
+
* structural (`typeof sent.message_id !== 'number'`), catching either.
|
|
1374
|
+
*/
|
|
1375
|
+
shedNextSend: boolean
|
|
1376
|
+
/**
|
|
1377
|
+
* One-shot: shed the next EDIT even with the window closed. Faithful to the
|
|
1378
|
+
* gate: worker-feed edits are `cosmetic`, so a gate shed resolves the
|
|
1379
|
+
* distinguishable SEND_GATE_SHED sentinel (#3110 F1) — NOT `undefined` (which
|
|
1380
|
+
* the gate reserves for a benign no-op drop whose payload IS on screen).
|
|
1381
|
+
*/
|
|
1382
|
+
shedNextEdit: boolean
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
/**
|
|
1386
|
+
* A bot adapter that models the send gate: it SHEDS (without recording a
|
|
1387
|
+
* delivery) whenever the injected flood probe reports an open window, or a
|
|
1388
|
+
* one-shot `shedNext*` flag is set (the race where the window opens between the
|
|
1389
|
+
* feed's probe read and the send). Faithful to the post-#3110 contract: a shed
|
|
1390
|
+
* EDIT resolves the SEND_GATE_SHED sentinel (cosmetic shed), a shed SEND
|
|
1391
|
+
* resolves `undefined` (`useful` queue-TTL expiry). A DELIVERED edit resolves a
|
|
1392
|
+
* non-shed value (`{}` — grammy returns `true`/`Message`).
|
|
1393
|
+
*/
|
|
1394
|
+
function makeGateBot(floodRemaining: () => number): GateBot {
|
|
1395
|
+
let nextId = 1000
|
|
1396
|
+
const gb: GateBot = {
|
|
1397
|
+
sendCalls: 0,
|
|
1398
|
+
editCalls: 0,
|
|
1399
|
+
sent: [],
|
|
1400
|
+
edits: [],
|
|
1401
|
+
shedNextSend: false,
|
|
1402
|
+
shedNextEdit: false,
|
|
1403
|
+
sendMessage: async (_chatId, text) => {
|
|
1404
|
+
gb.sendCalls++
|
|
1405
|
+
if (floodRemaining() > 0 || gb.shedNextSend) {
|
|
1406
|
+
gb.shedNextSend = false
|
|
1407
|
+
return undefined as unknown as { message_id: number }
|
|
1408
|
+
}
|
|
1409
|
+
gb.sent.push({ text })
|
|
1410
|
+
return { message_id: nextId++ }
|
|
1411
|
+
},
|
|
1412
|
+
editMessageText: async (_chatId, messageId, text) => {
|
|
1413
|
+
gb.editCalls++
|
|
1414
|
+
if (floodRemaining() > 0 || gb.shedNextEdit) {
|
|
1415
|
+
gb.shedNextEdit = false
|
|
1416
|
+
return SEND_GATE_SHED as unknown as undefined
|
|
1417
|
+
}
|
|
1418
|
+
gb.edits.push({ messageId, text })
|
|
1419
|
+
return {}
|
|
1420
|
+
},
|
|
1421
|
+
}
|
|
1422
|
+
return gb
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
describe('worker-feed send-gate shed contract', () => {
|
|
1426
|
+
it('makes ZERO api calls while a flood window is open, then paints full state after it closes', async () => {
|
|
1427
|
+
let clock = 0
|
|
1428
|
+
const untilTs = 6 * 60 * 60 * 1000 // a 6h ban, as observed 2026-07-12
|
|
1429
|
+
const probe = () => Math.max(0, untilTs - clock)
|
|
1430
|
+
const bot = makeGateBot(probe)
|
|
1431
|
+
const feed = createWorkerActivityFeed({
|
|
1432
|
+
bot,
|
|
1433
|
+
now: () => clock,
|
|
1434
|
+
firstPaintMinMs: 0,
|
|
1435
|
+
floodWaitRemainingMs: probe,
|
|
1436
|
+
})
|
|
1437
|
+
|
|
1438
|
+
// Drive the heartbeat cadence hammering the feed for the whole ban.
|
|
1439
|
+
for (let i = 0; i < 30; i++) {
|
|
1440
|
+
clock += 6000
|
|
1441
|
+
await feed.update('w1', 'chat', view({ elapsedMs: clock }))
|
|
1442
|
+
}
|
|
1443
|
+
// Parked on the window — never even called the API (the whole point).
|
|
1444
|
+
expect(bot.sendCalls).toBe(0)
|
|
1445
|
+
expect(bot.sent).toHaveLength(0)
|
|
1446
|
+
expect(feed.has('w1')).toBe(false)
|
|
1447
|
+
|
|
1448
|
+
// Window closes: the next tick paints full state exactly once.
|
|
1449
|
+
clock = untilTs + 1000
|
|
1450
|
+
await feed.update('w1', 'chat', view({ elapsedMs: clock }))
|
|
1451
|
+
expect(bot.sendCalls).toBe(1)
|
|
1452
|
+
expect(bot.sent).toHaveLength(1)
|
|
1453
|
+
expect(bot.sent[0].text).toContain('🛠 **Worker**')
|
|
1454
|
+
expect(feed.has('w1')).toBe(true)
|
|
1455
|
+
})
|
|
1456
|
+
|
|
1457
|
+
it('treats an undefined send result as not-delivered — no crash, no phantom message id', async () => {
|
|
1458
|
+
let clock = 10_000
|
|
1459
|
+
const logs: string[] = []
|
|
1460
|
+
const bot = makeGateBot(() => 0) // probe reads clear …
|
|
1461
|
+
bot.shedNextSend = true // … but the gate sheds this send (race)
|
|
1462
|
+
const feed = createWorkerActivityFeed({
|
|
1463
|
+
bot,
|
|
1464
|
+
now: () => clock,
|
|
1465
|
+
firstPaintMinMs: 0,
|
|
1466
|
+
floodWaitRemainingMs: () => 0,
|
|
1467
|
+
log: (m) => logs.push(m),
|
|
1468
|
+
})
|
|
1469
|
+
|
|
1470
|
+
await feed.update('w1', 'chat', view())
|
|
1471
|
+
expect(bot.sendCalls).toBe(1) // it attempted the send
|
|
1472
|
+
expect(bot.sent).toHaveLength(0) // but nothing was delivered
|
|
1473
|
+
expect(feed.messageIdOf('w1')).toBeNull() // and no phantom message id recorded
|
|
1474
|
+
expect(feed.has('w1')).toBe(false)
|
|
1475
|
+
// Regression pin: the old code dereferenced `undefined.message_id` and logged
|
|
1476
|
+
// it as a hard "send failed"; the shed is now a recognized, clean skip.
|
|
1477
|
+
expect(logs.some((l) => l.startsWith('worker-feed: send failed'))).toBe(false)
|
|
1478
|
+
expect(logs.some((l) => l.includes('shed by send gate'))).toBe(true)
|
|
1479
|
+
|
|
1480
|
+
// The next paint (gate no longer shedding) lands cleanly.
|
|
1481
|
+
clock = 20_000
|
|
1482
|
+
await feed.update('w1', 'chat', view())
|
|
1483
|
+
expect(bot.sent).toHaveLength(1)
|
|
1484
|
+
expect(feed.has('w1')).toBe(true)
|
|
1485
|
+
})
|
|
1486
|
+
|
|
1487
|
+
it('does not record a shed cosmetic edit as on-screen — the update re-sends after the gate clears', async () => {
|
|
1488
|
+
let clock = 10_000
|
|
1489
|
+
const bot = makeGateBot(() => 0)
|
|
1490
|
+
const feed = createWorkerActivityFeed({
|
|
1491
|
+
bot,
|
|
1492
|
+
now: () => clock,
|
|
1493
|
+
firstPaintMinMs: 0,
|
|
1494
|
+
minEditIntervalMs: 0,
|
|
1495
|
+
floodWaitRemainingMs: () => 0,
|
|
1496
|
+
})
|
|
1497
|
+
|
|
1498
|
+
await feed.update('w1', 'chat', view({ toolCount: 1 }))
|
|
1499
|
+
expect(bot.sent).toHaveLength(1)
|
|
1500
|
+
|
|
1501
|
+
// Gate sheds the edit (undefined) — it is NOT on screen.
|
|
1502
|
+
clock = 20_000
|
|
1503
|
+
bot.shedNextEdit = true
|
|
1504
|
+
await feed.update('w1', 'chat', view({ toolCount: 2 }))
|
|
1505
|
+
expect(bot.editCalls).toBe(1)
|
|
1506
|
+
expect(bot.edits).toHaveLength(0)
|
|
1507
|
+
|
|
1508
|
+
// Shed honesty: the SAME update re-sends once the gate clears — the shed
|
|
1509
|
+
// payload was never recorded as `lastBody`, so it is not dropped as a dup.
|
|
1510
|
+
clock = 30_000
|
|
1511
|
+
await feed.update('w1', 'chat', view({ toolCount: 2 }))
|
|
1512
|
+
expect(bot.edits).toHaveLength(1)
|
|
1513
|
+
expect(bot.edits[0].text).toContain('2 tools')
|
|
1514
|
+
})
|
|
1515
|
+
|
|
1516
|
+
it('does not falsely finalize on a shed terminal edit — re-drives the finalize once the gate clears', async () => {
|
|
1517
|
+
let clock = 10_000
|
|
1518
|
+
const bot = makeGateBot(() => 0)
|
|
1519
|
+
const feed = createWorkerActivityFeed({
|
|
1520
|
+
bot,
|
|
1521
|
+
now: () => clock,
|
|
1522
|
+
firstPaintMinMs: 0,
|
|
1523
|
+
minEditIntervalMs: 0,
|
|
1524
|
+
floodWaitRemainingMs: () => 0,
|
|
1525
|
+
})
|
|
1526
|
+
|
|
1527
|
+
await feed.update('w1', 'chat', view({ toolCount: 1 }))
|
|
1528
|
+
expect(bot.sent).toHaveLength(1)
|
|
1529
|
+
|
|
1530
|
+
// Gate sheds the terminal edit (undefined).
|
|
1531
|
+
clock = 20_000
|
|
1532
|
+
bot.shedNextEdit = true
|
|
1533
|
+
await feed.finish('w1', view({ state: 'done', toolCount: 5 }))
|
|
1534
|
+
expect(bot.editCalls).toBe(1)
|
|
1535
|
+
expect(bot.edits).toHaveLength(0)
|
|
1536
|
+
// NOT finalized: the handle survives (pendingFinish staged) rather than
|
|
1537
|
+
// being torn down with the card frozen on its last running render.
|
|
1538
|
+
expect(feed.has('w1')).toBe(true)
|
|
1539
|
+
|
|
1540
|
+
// Re-drive with the gate clear: the terminal recap lands and the handle
|
|
1541
|
+
// finalizes.
|
|
1542
|
+
clock = 30_000
|
|
1543
|
+
await feed.finish('w1', view({ state: 'done', toolCount: 5 }))
|
|
1544
|
+
expect(bot.edits).toHaveLength(1)
|
|
1545
|
+
expect(bot.edits[0].text).toContain('_done · 5 tools')
|
|
1546
|
+
expect(feed.has('w1')).toBe(false)
|
|
1547
|
+
})
|
|
1548
|
+
})
|