switchroom 0.20.8 → 0.20.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/handoff-briefing.sh +57 -5
- package/bin/working-state-reload-hook.sh +262 -0
- package/dist/agent-scheduler/index.js +16 -13
- package/dist/auth-broker/index.js +70 -30
- package/dist/cli/autoaccept-poll.js +5 -3
- package/dist/cli/drive-write-pretool.mjs +5 -3
- package/dist/cli/ms-365-write-pretool.mjs +5 -3
- package/dist/cli/notion-write-pretool.mjs +6 -6
- package/dist/cli/switchroom.js +42 -13
- package/dist/host-control/main.js +7 -7
- package/dist/vault/approvals/kernel-server.js +6 -6
- package/dist/vault/broker/server.js +6 -6
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +49 -0
- package/profiles/default/CLAUDE.md.hbs +12 -13
- package/telegram-plugin/ask-user.ts +6 -7
- package/telegram-plugin/dist/gateway/gateway.js +192 -66
- package/telegram-plugin/gateway/auth-broker-client.ts +1 -1
- package/telegram-plugin/gateway/auth-command.ts +4 -2
- package/telegram-plugin/gateway/checklist-fallback.ts +8 -1
- package/telegram-plugin/gateway/gateway.ts +8 -4
- package/telegram-plugin/gateway/outbound-send-path.ts +9 -1
- package/telegram-plugin/gateway/subagent-handback-inbound-builder.ts +21 -1
- package/telegram-plugin/gateway/subagent-handback-marker.ts +94 -0
- package/telegram-plugin/gateway/throttle-tier-wiring.ts +93 -18
- package/telegram-plugin/render/emphasis-guard.ts +92 -12
- package/telegram-plugin/render/line-start-guard.ts +27 -2
- package/telegram-plugin/sticker-aliases.ts +12 -14
- package/telegram-plugin/tests/ask-user.test.ts +15 -0
- package/telegram-plugin/tests/checklist-fallback.test.ts +21 -0
- package/telegram-plugin/tests/handback-tasknotif-dedup.test.ts +248 -0
- package/telegram-plugin/tests/render/emphasis-guard.test.ts +105 -6
- package/telegram-plugin/tests/render/heading-guard-blockquote-glued-hash.test.ts +123 -36
- package/telegram-plugin/tests/reply-quote-wire.test.ts +47 -0
- package/telegram-plugin/tests/sticker-aliases.test.ts +43 -0
- package/telegram-plugin/tests/throttle-tier-probe-only.test.ts +216 -0
- package/telegram-plugin/tests/throttle-tier-route-429-wiring.test.ts +92 -0
- package/telegram-plugin/tests/throttle-tier-route-429.test.ts +71 -0
- package/telegram-plugin/throttle-tier.ts +59 -0
- package/vendor/hindsight-memory/CHANGELOG.md +31 -0
- package/vendor/hindsight-memory/hooks/hooks.json +2 -1
- package/vendor/hindsight-memory/scripts/session_start.py +35 -8
- package/vendor/hindsight-memory/scripts/tests/test_session_start_durability.py +107 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the generic-transient PROBE-ONLY 429 path (#failover-429-corroborate).
|
|
3
|
+
*
|
|
4
|
+
* This pins the OUTCOMES the gateway promises for a bare `rate_limit_error`
|
|
5
|
+
* (generic-transient) 429 after Ken picked Option A (probe-only):
|
|
6
|
+
*
|
|
7
|
+
* - generic-transient + HEALTHY probe → the probe's ONLY effect is the
|
|
8
|
+
* silent broker quota refresh. Account-inert at the runner: NO throttle
|
|
9
|
+
* notice, NO self-restart nudge, NO `throttled_until` soft-defer, NO second
|
|
10
|
+
* card. The calm rate-limited card the gateway already emitted stays the
|
|
11
|
+
* ONLY user-visible output.
|
|
12
|
+
* - generic-transient + WALL (probe corroborates) → the escalation path runs
|
|
13
|
+
* exactly like the account-scoped `fire`: the corroborated-wall announcement
|
|
14
|
+
* posts and the dead turn is resumed. No redundant calm card is emitted from
|
|
15
|
+
* this branch — the announcement IS the output.
|
|
16
|
+
* - litellm-local → the runner does NOT fire at all (account-inert by
|
|
17
|
+
* mechanism, not discipline): the request never reached Anthropic, so
|
|
18
|
+
* account state must not be touched. `account-scoped` likewise takes its own
|
|
19
|
+
* `fire` path, not `fireProbeOnly`. Pinned via the pure classification gate
|
|
20
|
+
* the gateway consults before calling `fireProbeOnly`.
|
|
21
|
+
*
|
|
22
|
+
* The runner side is exercised with fully injected deps (no gateway import),
|
|
23
|
+
* mirroring throttle-tier-wiring.test.ts.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { describe, it, expect } from 'vitest'
|
|
27
|
+
import {
|
|
28
|
+
createThrottleTierRunner,
|
|
29
|
+
type ThrottleBrokerClient,
|
|
30
|
+
type ThrottleTierRunnerDeps,
|
|
31
|
+
} from '../gateway/throttle-tier-wiring.js'
|
|
32
|
+
import { classification429WarrantsCorroboration } from '../throttle-tier.js'
|
|
33
|
+
|
|
34
|
+
const NOW = Date.UTC(2026, 6, 12, 8, 0, 0)
|
|
35
|
+
|
|
36
|
+
interface Harness {
|
|
37
|
+
deps: ThrottleTierRunnerDeps
|
|
38
|
+
calls: {
|
|
39
|
+
markThrottled: Array<{ until: number; probeOnly?: boolean }>
|
|
40
|
+
claims: string[]
|
|
41
|
+
notices: Array<{ chatId: string | number; markdown: string }>
|
|
42
|
+
deferrals: string[]
|
|
43
|
+
restarts: string[]
|
|
44
|
+
logs: string[]
|
|
45
|
+
timers: Array<{ ms: number; fn: () => void; cancelled: boolean }>
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function makeHarness(opts: {
|
|
50
|
+
markThrottledResult?:
|
|
51
|
+
| { account: string; throttled_until: number; escalated: boolean; rolledTo?: string | null }
|
|
52
|
+
| 'unreachable'
|
|
53
|
+
| 'throw'
|
|
54
|
+
turnInFlight?: () => boolean
|
|
55
|
+
newestTurnStartedAt?: () => number | null
|
|
56
|
+
} = {}): Harness {
|
|
57
|
+
const nowMs = NOW
|
|
58
|
+
const calls: Harness['calls'] = {
|
|
59
|
+
markThrottled: [],
|
|
60
|
+
claims: [],
|
|
61
|
+
notices: [],
|
|
62
|
+
deferrals: [],
|
|
63
|
+
restarts: [],
|
|
64
|
+
logs: [],
|
|
65
|
+
timers: [],
|
|
66
|
+
}
|
|
67
|
+
const client: ThrottleBrokerClient = {
|
|
68
|
+
async markThrottled(until: number, probeOnly?: boolean) {
|
|
69
|
+
calls.markThrottled.push({ until, probeOnly })
|
|
70
|
+
if (opts.markThrottledResult === 'throw') throw new Error('boom')
|
|
71
|
+
const r = opts.markThrottledResult
|
|
72
|
+
if (r && r !== 'unreachable') return r
|
|
73
|
+
return { account: 'alice', throttled_until: until, escalated: false, rolledTo: null }
|
|
74
|
+
},
|
|
75
|
+
async claimNotification(key: string) {
|
|
76
|
+
calls.claims.push(key)
|
|
77
|
+
return { granted: true }
|
|
78
|
+
},
|
|
79
|
+
}
|
|
80
|
+
const deps: ThrottleTierRunnerDeps = {
|
|
81
|
+
agentName: 'carrie',
|
|
82
|
+
getBrokerClient: async () =>
|
|
83
|
+
opts.markThrottledResult === 'unreachable' ? null : client,
|
|
84
|
+
listNoticeChats: () => ['111', '222'],
|
|
85
|
+
sendNotice: (chatId, markdown) => calls.notices.push({ chatId, markdown }),
|
|
86
|
+
resumeDecide: () => 'resume',
|
|
87
|
+
newestActiveTurnStartedAtMs: opts.newestTurnStartedAt ?? (() => null),
|
|
88
|
+
turnInFlight: opts.turnInFlight ?? (() => false),
|
|
89
|
+
deferRestartToTurnComplete: (_agent, reason) => calls.deferrals.push(reason),
|
|
90
|
+
restartNow: (_agent, reason) => calls.restarts.push(reason),
|
|
91
|
+
log: (m) => calls.logs.push(m),
|
|
92
|
+
now: () => nowMs,
|
|
93
|
+
schedule: (fn, ms) => {
|
|
94
|
+
const t = { ms, fn, cancelled: false }
|
|
95
|
+
calls.timers.push(t)
|
|
96
|
+
return { cancel: () => { t.cancelled = true } }
|
|
97
|
+
},
|
|
98
|
+
jitterMs: () => 0,
|
|
99
|
+
}
|
|
100
|
+
return { deps, calls }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
describe('generic-transient probe-only — classification gate (litellm-local never fires)', () => {
|
|
104
|
+
it('fires the corroboration probe ONLY for generic-transient', () => {
|
|
105
|
+
expect(classification429WarrantsCorroboration('generic-transient')).toBe(true)
|
|
106
|
+
// litellm-local: request never reached Anthropic — account-inert by mechanism.
|
|
107
|
+
expect(classification429WarrantsCorroboration('litellm-local')).toBe(false)
|
|
108
|
+
// account-scoped: runs its own throttle tier / failover (fire, not fireProbeOnly).
|
|
109
|
+
expect(classification429WarrantsCorroboration('account-scoped')).toBe(false)
|
|
110
|
+
// null: not a rate-limited event.
|
|
111
|
+
expect(classification429WarrantsCorroboration(null)).toBe(false)
|
|
112
|
+
})
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
describe('generic-transient probe-only — HEALTHY probe is account-inert', () => {
|
|
116
|
+
it('probes the broker but emits NOTHING else (calm card stays the only output)', async () => {
|
|
117
|
+
const h = makeHarness() // default result: escalated:false (healthy)
|
|
118
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
119
|
+
await runner.fireProbeOnly('carrie')
|
|
120
|
+
|
|
121
|
+
// The probe reached the broker in probe-only mode…
|
|
122
|
+
expect(h.calls.markThrottled).toHaveLength(1)
|
|
123
|
+
expect(h.calls.markThrottled[0].probeOnly).toBe(true)
|
|
124
|
+
|
|
125
|
+
// …but produced NO user-visible or account-mutating side effects:
|
|
126
|
+
expect(h.calls.notices).toHaveLength(0) // no throttle notice, no second card
|
|
127
|
+
expect(h.calls.claims).toHaveLength(0) // no fleet-dedup broadcast at all
|
|
128
|
+
expect(h.calls.restarts).toHaveLength(0) // no self-restart nudge
|
|
129
|
+
expect(h.calls.deferrals).toHaveLength(0)
|
|
130
|
+
expect(h.calls.timers).toHaveLength(0) // no throttled_until soft-defer / retry nudge
|
|
131
|
+
expect(runner.inspect().nudgePending).toBe(false)
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
it('a broker-unreachable probe is inert and never throws', async () => {
|
|
135
|
+
const h = makeHarness({ markThrottledResult: 'unreachable' })
|
|
136
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
137
|
+
await expect(runner.fireProbeOnly('carrie')).resolves.toBeUndefined()
|
|
138
|
+
expect(h.calls.markThrottled).toHaveLength(0)
|
|
139
|
+
expect(h.calls.notices).toHaveLength(0)
|
|
140
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
141
|
+
expect(h.calls.timers).toHaveLength(0)
|
|
142
|
+
})
|
|
143
|
+
|
|
144
|
+
it('a markThrottled throw is swallowed and stays inert (no card, no restart)', async () => {
|
|
145
|
+
const h = makeHarness({ markThrottledResult: 'throw' })
|
|
146
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
147
|
+
await expect(runner.fireProbeOnly('carrie')).resolves.toBeUndefined()
|
|
148
|
+
expect(h.calls.notices).toHaveLength(0)
|
|
149
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
150
|
+
expect(h.calls.timers).toHaveLength(0)
|
|
151
|
+
})
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
describe('generic-transient probe-only — WALL corroborated → escalation', () => {
|
|
155
|
+
it('posts the corroborated-wall announcement and resumes immediately (no calm card, no nudge timer)', async () => {
|
|
156
|
+
const h = makeHarness({
|
|
157
|
+
markThrottledResult: {
|
|
158
|
+
account: 'alice',
|
|
159
|
+
throttled_until: NOW + 60_000,
|
|
160
|
+
escalated: true,
|
|
161
|
+
rolledTo: 'bob',
|
|
162
|
+
},
|
|
163
|
+
})
|
|
164
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
165
|
+
await runner.fireProbeOnly('carrie')
|
|
166
|
+
|
|
167
|
+
// Probe ran in probe-only mode, then escalation took over.
|
|
168
|
+
expect(h.calls.markThrottled[0].probeOnly).toBe(true)
|
|
169
|
+
|
|
170
|
+
// The escalation announcement (fleet-deduped), NOT the staying-put notice.
|
|
171
|
+
expect(h.calls.claims).toEqual([
|
|
172
|
+
`throttle-escalation:alice:111`,
|
|
173
|
+
`throttle-escalation:alice:222`,
|
|
174
|
+
])
|
|
175
|
+
expect(h.calls.notices).toHaveLength(2)
|
|
176
|
+
expect(h.calls.notices[0].markdown).toContain('actually a wall')
|
|
177
|
+
expect(h.calls.notices[0].markdown).toContain('bob')
|
|
178
|
+
|
|
179
|
+
// Immediate resume via the escalation lever; NO delayed retry nudge armed.
|
|
180
|
+
expect(h.calls.restarts).toEqual(['throttle-escalation-resume'])
|
|
181
|
+
expect(h.calls.timers).toHaveLength(0)
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
it('escalated with rolledTo=null (all blocked) announces but does NOT restart', async () => {
|
|
185
|
+
const h = makeHarness({
|
|
186
|
+
markThrottledResult: {
|
|
187
|
+
account: 'alice',
|
|
188
|
+
throttled_until: NOW + 60_000,
|
|
189
|
+
escalated: true,
|
|
190
|
+
rolledTo: null,
|
|
191
|
+
},
|
|
192
|
+
})
|
|
193
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
194
|
+
await runner.fireProbeOnly('carrie')
|
|
195
|
+
expect(h.calls.notices[0].markdown).toContain('all blocked')
|
|
196
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
197
|
+
expect(h.calls.timers).toHaveLength(0)
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
it('escalated resume respects the live-turn guards (defers under the dead turn gate)', async () => {
|
|
201
|
+
const h = makeHarness({
|
|
202
|
+
markThrottledResult: {
|
|
203
|
+
account: 'alice',
|
|
204
|
+
throttled_until: NOW + 60_000,
|
|
205
|
+
escalated: true,
|
|
206
|
+
rolledTo: 'bob',
|
|
207
|
+
},
|
|
208
|
+
turnInFlight: () => true,
|
|
209
|
+
newestTurnStartedAt: () => NOW - 60_000, // the dead turn still holds the gate
|
|
210
|
+
})
|
|
211
|
+
const runner = createThrottleTierRunner(h.deps)
|
|
212
|
+
await runner.fireProbeOnly('carrie')
|
|
213
|
+
expect(h.calls.restarts).toHaveLength(0)
|
|
214
|
+
expect(h.calls.deferrals).toEqual(['throttle-escalation-resume'])
|
|
215
|
+
})
|
|
216
|
+
})
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* L1 SEAM-BOUNDARY GUARD for the 429 classify→route wiring (follow-up to
|
|
3
|
+
* switchroom#4381 / #4379).
|
|
4
|
+
*
|
|
5
|
+
* `throttle-tier-route-429.test.ts` proves the ROUTING LOGIC of the pure seam
|
|
6
|
+
* `routeRateLimit429` in isolation (generic-transient → fireProbeOnly;
|
|
7
|
+
* litellm-local / account-scoped / null → no probe). But nothing asserted that
|
|
8
|
+
* the GATEWAY still calls that seam at its 429-handling path. That is the exact
|
|
9
|
+
* gap the extraction opened: a future edit deleting the gateway callsite
|
|
10
|
+
* (gateway.ts, `emitGatewayOperatorEvent` rate-limited calm branch) would
|
|
11
|
+
* silently drop the corroboration probe for every `generic-transient` 429 —
|
|
12
|
+
* a real 5h/7d wall hiding behind transient wording would again die with no
|
|
13
|
+
* failover — while the seam's own unit suite stayed fully green.
|
|
14
|
+
*
|
|
15
|
+
* gateway.ts is a side-effecting IIFE that cannot be imported in a unit test
|
|
16
|
+
* (same constraint documented in turn-flush-suppression-wiring.test.ts /
|
|
17
|
+
* activity-card-wiring.test.ts): `emitGatewayOperatorEvent` is a module-scoped,
|
|
18
|
+
* un-exported function reachable only after the gateway boots. So an
|
|
19
|
+
* outcome-level "drive emitGatewayOperatorEvent and observe fireProbeOnly" test
|
|
20
|
+
* is infeasible without exporting internal gateway state — scope creep the
|
|
21
|
+
* repo deliberately avoids. Following the established wiring-guard pattern,
|
|
22
|
+
* this is a STRUCTURAL assertion that pins the load-bearing call site. It
|
|
23
|
+
* COMPLEMENTS the seam suite: the routing outcomes are proven there; this
|
|
24
|
+
* guards that the gateway actually routes through them.
|
|
25
|
+
*
|
|
26
|
+
* The outcome that flips on the regression: with the callsite present, a
|
|
27
|
+
* `generic-transient` 429 reaches `routeRateLimit429(..., throttleTierRunner,
|
|
28
|
+
* agent)` and fires the probe on the REAL runner; delete or neuter the
|
|
29
|
+
* callsite and these assertions go red. Verified fails-red by deleting
|
|
30
|
+
* gateway.ts:7831 locally (see PR body).
|
|
31
|
+
*/
|
|
32
|
+
import { describe, it, expect } from 'vitest'
|
|
33
|
+
import { readFileSync } from 'node:fs'
|
|
34
|
+
import { resolve } from 'node:path'
|
|
35
|
+
|
|
36
|
+
const gatewaySrc = readFileSync(resolve(__dirname, '..', 'gateway', 'gateway.ts'), 'utf-8')
|
|
37
|
+
|
|
38
|
+
function between(src: string, startMarker: string, endMarker: string): string {
|
|
39
|
+
const after = src.split(startMarker)[1] ?? ''
|
|
40
|
+
return after.split(endMarker)[0] ?? ''
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Strip comment lines so prose (a docstring mentioning the call, or a comment
|
|
44
|
+
* describing the OLD shape) can neither satisfy nor trip an assertion about
|
|
45
|
+
* the actual CODE. */
|
|
46
|
+
function codeOnly(src: string): string {
|
|
47
|
+
return src
|
|
48
|
+
.split('\n')
|
|
49
|
+
.filter((l) => !l.trim().startsWith('//'))
|
|
50
|
+
.filter((l) => !l.trim().startsWith('*') && !l.trim().startsWith('/*'))
|
|
51
|
+
.join('\n')
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
describe('429 route-seam wiring — the gateway calls routeRateLimit429 (switchroom#4381 L1)', () => {
|
|
55
|
+
// The calm-429 branch: from the classification binding through the surface
|
|
56
|
+
// decision that immediately follows the seam call.
|
|
57
|
+
const branch = between(
|
|
58
|
+
gatewaySrc,
|
|
59
|
+
'const rateLimit429Classification =',
|
|
60
|
+
"if (surface === 'litellm-local-notice')",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
it('the calm-429 branch routes the classification through routeRateLimit429', () => {
|
|
64
|
+
expect(branch.length).toBeGreaterThan(100)
|
|
65
|
+
const code = codeOnly(branch)
|
|
66
|
+
// The load-bearing call site. Deleting it drops the corroboration probe for
|
|
67
|
+
// every generic-transient 429 with the seam's unit suite still green — the
|
|
68
|
+
// exact regression this guard exists to catch.
|
|
69
|
+
expect(code).toMatch(/routeRateLimit429\(/)
|
|
70
|
+
// It must be fed the CLASSIFICATION computed for this event, not a literal
|
|
71
|
+
// or a stale variable — otherwise the wrong family would (not) be probed.
|
|
72
|
+
expect(code).toMatch(
|
|
73
|
+
/routeRateLimit429\(\s*rateLimit429Classification\s*,/,
|
|
74
|
+
)
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
it('the seam is handed the REAL throttleTierRunner (a live probe, not a no-op)', () => {
|
|
78
|
+
const code = codeOnly(branch)
|
|
79
|
+
// Passing anything other than the gateway's real runner (e.g. a stub) would
|
|
80
|
+
// make the probe fire into the void — the branch outcome would look correct
|
|
81
|
+
// in a naive grep but corroborate nothing.
|
|
82
|
+
expect(code).toMatch(
|
|
83
|
+
/routeRateLimit429\(\s*rateLimit429Classification\s*,\s*throttleTierRunner\s*,\s*agent\s*\)/,
|
|
84
|
+
)
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('routeRateLimit429 is imported into the gateway (the seam is actually linked)', () => {
|
|
88
|
+
// A dangling call to an unimported symbol would fail tsc, but pinning the
|
|
89
|
+
// import makes the dependency edge explicit and its removal a red test too.
|
|
90
|
+
expect(codeOnly(gatewaySrc)).toMatch(/\brouteRateLimit429\b/)
|
|
91
|
+
})
|
|
92
|
+
})
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the classify→route WIRING of the gateway's 429 corroboration path
|
|
3
|
+
* (#failover-429-corroborate, follow-up switchroom#4379).
|
|
4
|
+
*
|
|
5
|
+
* The pure gate `classification429WarrantsCorroboration` is already unit-tested
|
|
6
|
+
* (throttle-tier-probe-only.test.ts), and the runner's `fireProbeOnly` outcomes
|
|
7
|
+
* are pinned there too. What was NOT covered is the WIRING that connects them:
|
|
8
|
+
* the gateway callsite (gateway.ts, `handleOperatorEvent` rate-limited branch)
|
|
9
|
+
* that decides whether to invoke `throttleTierRunner.fireProbeOnly(agent)` for a
|
|
10
|
+
* given classification.
|
|
11
|
+
*
|
|
12
|
+
* gateway.ts is hard to unit-test in isolation and is under a hard line ratchet,
|
|
13
|
+
* so rather than stand up a gateway harness the decision was extracted into a
|
|
14
|
+
* pure, injectable seam — `routeRateLimit429(classification, runner, agent)` in
|
|
15
|
+
* throttle-tier.ts — which the gateway callsite now calls verbatim. These tests
|
|
16
|
+
* exercise that seam with a fake runner that records which entrypoint fired,
|
|
17
|
+
* pinning the three routing outcomes the gateway promises:
|
|
18
|
+
*
|
|
19
|
+
* - generic-transient → routes to fireProbeOnly (the probe fires)
|
|
20
|
+
* - litellm-local → does NOT route to fireProbeOnly (account-inert by
|
|
21
|
+
* mechanism: the request never reached Anthropic)
|
|
22
|
+
* - account-scoped → does NOT route to fireProbeOnly (that classification
|
|
23
|
+
* takes its own `fire` path, not the probe)
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { describe, it, expect } from 'vitest'
|
|
27
|
+
import { routeRateLimit429, type RateLimit429ProbeRunner } from '../throttle-tier.js'
|
|
28
|
+
|
|
29
|
+
interface FakeRunner extends RateLimit429ProbeRunner {
|
|
30
|
+
probeCalls: string[]
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function makeFakeRunner(): FakeRunner {
|
|
34
|
+
const probeCalls: string[] = []
|
|
35
|
+
return {
|
|
36
|
+
probeCalls,
|
|
37
|
+
async fireProbeOnly(triggerAgent: string) {
|
|
38
|
+
probeCalls.push(triggerAgent)
|
|
39
|
+
},
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
describe('routeRateLimit429 — gateway classify→route wiring (switchroom#4379)', () => {
|
|
44
|
+
it('generic-transient → routes to fireProbeOnly', () => {
|
|
45
|
+
const runner = makeFakeRunner()
|
|
46
|
+
const fired = routeRateLimit429('generic-transient', runner, 'carrie')
|
|
47
|
+
expect(fired).toBe(true)
|
|
48
|
+
expect(runner.probeCalls).toEqual(['carrie'])
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
it('litellm-local → does NOT route to fireProbeOnly (stays account-inert)', () => {
|
|
52
|
+
const runner = makeFakeRunner()
|
|
53
|
+
const fired = routeRateLimit429('litellm-local', runner, 'carrie')
|
|
54
|
+
expect(fired).toBe(false)
|
|
55
|
+
expect(runner.probeCalls).toHaveLength(0)
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
it('account-scoped → does NOT route to fireProbeOnly (uses fire, not the probe)', () => {
|
|
59
|
+
const runner = makeFakeRunner()
|
|
60
|
+
const fired = routeRateLimit429('account-scoped', runner, 'carrie')
|
|
61
|
+
expect(fired).toBe(false)
|
|
62
|
+
expect(runner.probeCalls).toHaveLength(0)
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
it('null (not a rate-limited event) → does NOT route to fireProbeOnly', () => {
|
|
66
|
+
const runner = makeFakeRunner()
|
|
67
|
+
const fired = routeRateLimit429(null, runner, 'carrie')
|
|
68
|
+
expect(fired).toBe(false)
|
|
69
|
+
expect(runner.probeCalls).toHaveLength(0)
|
|
70
|
+
})
|
|
71
|
+
})
|
|
@@ -120,6 +120,65 @@ export function classify429Detail(text: string): RateLimit429Classification {
|
|
|
120
120
|
return 'generic-transient'
|
|
121
121
|
}
|
|
122
122
|
|
|
123
|
+
/**
|
|
124
|
+
* #failover-429-corroborate — does this terminal 429 classification warrant a
|
|
125
|
+
* fire-and-forget broker corroboration probe (throttle-tier runner's
|
|
126
|
+
* PROBE-ONLY entrypoint) IN ADDITION to the calm rate-limited card?
|
|
127
|
+
*
|
|
128
|
+
* TRUE for `generic-transient` ONLY. That family (a bare `rate_limit_error`
|
|
129
|
+
* body / novel wording that matches neither the account-scoped negation
|
|
130
|
+
* strings nor litellm-local) used to drop on the calm-card floor with NO broker
|
|
131
|
+
* contact — so a genuine 5h/7d wall hiding behind transient wording was never
|
|
132
|
+
* probed, and the turn died with no failover. Routing it through the runner's
|
|
133
|
+
* `fireProbeOnly` lets the broker take ONE live quota probe (rate-bounded) and
|
|
134
|
+
* convert a real wall into failover. A HEALTHY probe is fully account-inert:
|
|
135
|
+
* probe-only records NO `throttled_until`, sends NO throttle notice, arms NO
|
|
136
|
+
* self-restart nudge, and adds NO second card — so the calm rate-limited card
|
|
137
|
+
* stays the ONLY user-visible output, exactly as before this path existed.
|
|
138
|
+
*
|
|
139
|
+
* FALSE (never corroborate) for:
|
|
140
|
+
* - `litellm-local` — INVARIANT: the request never reached Anthropic (the
|
|
141
|
+
* proxy's own tpm/rpm limiter tripped), so account state must not be
|
|
142
|
+
* touched. This mechanism, not discipline, keeps that path account-inert.
|
|
143
|
+
* - `account-scoped` — already runs its own throttle tier / failover path.
|
|
144
|
+
* - `null` — not a rate-limited event.
|
|
145
|
+
*/
|
|
146
|
+
export function classification429WarrantsCorroboration(
|
|
147
|
+
classification: RateLimit429Classification | null,
|
|
148
|
+
): boolean {
|
|
149
|
+
return classification === 'generic-transient'
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Minimal runner shape the classify→route seam needs: just the PROBE-ONLY
|
|
154
|
+
* entrypoint. The full gateway `ThrottleTierRunner` satisfies it structurally,
|
|
155
|
+
* and a test can supply a fake that records the call.
|
|
156
|
+
*/
|
|
157
|
+
export interface RateLimit429ProbeRunner {
|
|
158
|
+
fireProbeOnly(triggerAgent: string): Promise<void>
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* #failover-429-corroborate — the classify→route WIRING extracted from the
|
|
163
|
+
* gateway callsite so the routing decision (not just the gate) is unit-testable
|
|
164
|
+
* without a gateway harness. Fires the runner's PROBE-ONLY entrypoint IFF the
|
|
165
|
+
* classification warrants corroboration (`generic-transient` only, per
|
|
166
|
+
* `classification429WarrantsCorroboration`); `litellm-local`, `account-scoped`,
|
|
167
|
+
* and `null` never fire it. Fire-and-forget: the runner promise is intentionally
|
|
168
|
+
* not awaited (the gateway must not block the operator-event path on a probe).
|
|
169
|
+
* Returns `true` when a probe was fired, `false` otherwise — observable in tests
|
|
170
|
+
* and unused at the gateway callsite.
|
|
171
|
+
*/
|
|
172
|
+
export function routeRateLimit429(
|
|
173
|
+
classification: RateLimit429Classification | null,
|
|
174
|
+
runner: RateLimit429ProbeRunner,
|
|
175
|
+
agent: string,
|
|
176
|
+
): boolean {
|
|
177
|
+
if (!classification429WarrantsCorroboration(classification)) return false
|
|
178
|
+
void runner.fireProbeOnly(agent)
|
|
179
|
+
return true
|
|
180
|
+
}
|
|
181
|
+
|
|
123
182
|
/**
|
|
124
183
|
* Build the `rate_limit_429_classified` runtime metric for one terminal
|
|
125
184
|
* rate-limited operator event — the instrumentation that lets an operator
|
|
@@ -54,6 +54,37 @@
|
|
|
54
54
|
|
|
55
55
|
### Changed (switchroom divergence)
|
|
56
56
|
|
|
57
|
+
- **SessionStart hook runs async so its durability work stops being killed
|
|
58
|
+
mid-drain** (`hooks/hooks.json`, `scripts/session_start.py`). The hook does
|
|
59
|
+
the recovery a prior session's abrupt death skipped — drain the
|
|
60
|
+
SessionEnd-queued retains (#1071) and reconcile un-committed transcript turns
|
|
61
|
+
(#3244). Those two carry independent 4s wall-clock budgets
|
|
62
|
+
(`HINDSIGHT_DRAIN_BUDGET_S` / `HINDSIGHT_RECONCILE_BUDGET_S`), each sized in
|
|
63
|
+
isolation against the old synchronous 5s SessionStart timeout, plus a ~2s
|
|
64
|
+
Mode-1 external-server health probe. Summed, they routinely overran 5s, so
|
|
65
|
+
Claude Code SIGKILLed the hook part-way through — truncating exactly the
|
|
66
|
+
durability work it exists to do and letting the pending-retains backlog grow
|
|
67
|
+
(fleet transcripts carry 1000+ `hook_cancelled` attachments for
|
|
68
|
+
`session_start.py`, all `timedOut: true` with `durationMs > 5000`; a
|
|
69
|
+
successful firing injects no `additionalContext` and so leaves no record,
|
|
70
|
+
which made the failures look like ~100% of firings when the true rate is
|
|
71
|
+
unmeasurable from attachments). Setting `"async": true` on the hook (the same
|
|
72
|
+
non-blocking pattern the Stop-event `retain.py` already uses) detaches it from
|
|
73
|
+
the SessionStart critical path: it injects no context, so nothing depends on
|
|
74
|
+
it finishing first and async's dropped context costs nothing. The `timeout`
|
|
75
|
+
ceiling rises 5s → 30s purely as a background-lifetime bound above the ~10s
|
|
76
|
+
summed sub-budgets — it never re-introduces a startup block, because drain and
|
|
77
|
+
reconcile self-cap at their budgets regardless. The drain stays in the hook
|
|
78
|
+
(not deferred wholly to the `hindsight-drain` sidecar, which `start.sh` only
|
|
79
|
+
starts conditionally — when it is absent this hook is the sole backlog path)
|
|
80
|
+
and `reconcile_tail`, which has no sidecar equivalent, stays here and now runs
|
|
81
|
+
to completion; its over-budget remainder still resumes on the next boot.
|
|
82
|
+
Concurrent drain with the sidecar stays safe via `drain_pending`'s exclusive
|
|
83
|
+
`fcntl.flock`. Pinned by `scripts/tests/test_session_start_durability.py`
|
|
84
|
+
(drain + reconcile are still invoked, in order) and
|
|
85
|
+
`tests/hindsight-session-start-async.test.ts` (the async flag + budget-clearing
|
|
86
|
+
ceiling survive the scaffold's hooks-override round-trip).
|
|
87
|
+
|
|
57
88
|
- **Recall/retain hygiene guard-rail batch** (`scripts/recall.py`,
|
|
58
89
|
`scripts/subagent_retain.py`, `scripts/lib/content.py`, `scripts/tests/**`).
|
|
59
90
|
Closes guard-rail debt in the fork's hook scripts before extending them:
|
|
@@ -1,12 +1,39 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""SessionStart hook: health check +
|
|
3
|
-
|
|
4
|
-
Fires once when a Claude Code session begins
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
2
|
+
"""SessionStart hook: health check + durability drain/reconcile.
|
|
3
|
+
|
|
4
|
+
Fires once when a Claude Code session begins (startup / compact / clear).
|
|
5
|
+
This is the Claude Code equivalent of Openclaw's service.start() — verify
|
|
6
|
+
the server is reachable early, then do the durability work that a prior
|
|
7
|
+
session's abrupt death may have skipped: drain SessionEnd-queued retains
|
|
8
|
+
(#1071) and reconcile un-committed transcript turns (#3244).
|
|
9
|
+
|
|
10
|
+
Runs ASYNC (``"async": true`` in hooks/hooks.json), for a reason specific
|
|
11
|
+
to THIS hook: it injects NO additionalContext — every return path is a
|
|
12
|
+
pure side effect — so nothing in the session depends on it finishing
|
|
13
|
+
before the first prompt, and losing async's dropped context costs us
|
|
14
|
+
nothing here (unlike recall.py, which must stay synchronous to inject).
|
|
15
|
+
|
|
16
|
+
Why async matters: drain and reconcile each carry an independent
|
|
17
|
+
wall-clock budget (``HINDSIGHT_DRAIN_BUDGET_S`` / ``HINDSIGHT_RECONCILE_BUDGET_S``,
|
|
18
|
+
4s each) plus a Mode-1 health probe (~2s). Those budgets were each sized
|
|
19
|
+
against the old synchronous 5s SessionStart timeout in ISOLATION, but they
|
|
20
|
+
STACK on one hook: drain(4s) + reconcile(4s, added later in #3244) +
|
|
21
|
+
probe(2s) routinely overran 5s, so the hook was SIGKILLed mid-drain on a
|
|
22
|
+
large share of firings (see the fleet-wide ``hook_cancelled`` transcript
|
|
23
|
+
attachments) — truncating exactly the durability work it exists to do,
|
|
24
|
+
and growing the pending-retains backlog it was meant to clear. Async
|
|
25
|
+
detaches it from the SessionStart critical path so the bounded, resumable
|
|
26
|
+
drain/reconcile can run to completion instead of being cut off. Both are
|
|
27
|
+
crash-safe under an eventual reap anyway: drain holds an ``fcntl.flock``
|
|
28
|
+
the kernel releases on death and re-queues unsent entries, and reconcile
|
|
29
|
+
is idempotent and resumes any over-budget remainder on the next boot.
|
|
30
|
+
|
|
31
|
+
Keeping the drain here (rather than deferring wholly to the hindsight-drain
|
|
32
|
+
sidecar) is deliberate: that sidecar is only CONDITIONALLY started — its
|
|
33
|
+
own "NOT STARTED" branch in start.sh notes that when it is absent the
|
|
34
|
+
memory queue is drained ONLY at session boot — so this hook stays the
|
|
35
|
+
backlog path for agents without the sidecar. reconcile_tail has no sidecar
|
|
36
|
+
equivalent at all and lives only here.
|
|
10
37
|
"""
|
|
11
38
|
|
|
12
39
|
import json
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""The SessionStart hook must still DO its durability work.
|
|
2
|
+
|
|
3
|
+
The hook was made ``"async": true`` (hooks/hooks.json) so a stacked
|
|
4
|
+
drain + reconcile + health-probe budget can no longer overrun the
|
|
5
|
+
SessionStart timeout and get the process SIGKILLed mid-drain. Async only
|
|
6
|
+
helps if the two durability calls are still MADE on the healthy path —
|
|
7
|
+
if a refactor drops one, the queue silently stops draining and abrupt-kill
|
|
8
|
+
turns stop being recovered, which is the exact outage the hook exists to
|
|
9
|
+
prevent and which no attachment record would surface (a successful
|
|
10
|
+
SessionStart hook that injects no context leaves no transcript trace).
|
|
11
|
+
|
|
12
|
+
So this pins the OUTCOME, not the wiring: on a reachable server,
|
|
13
|
+
``session_start.main()`` invokes ``drain_pending.drain`` and then
|
|
14
|
+
``reconcile_tail.reconcile``, in that order (reconcile runs AFTER the
|
|
15
|
+
drain by design — it recovers what SessionEnd never managed to enqueue).
|
|
16
|
+
|
|
17
|
+
Lives under ``scripts/tests/`` because that is the only python test
|
|
18
|
+
directory CI discovers (``ci-tests-python.yml`` runs ``unittest discover``
|
|
19
|
+
from ``vendor/hindsight-memory/scripts``).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import io
|
|
23
|
+
import os
|
|
24
|
+
import sys
|
|
25
|
+
import unittest
|
|
26
|
+
import unittest.mock
|
|
27
|
+
|
|
28
|
+
SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
29
|
+
if SCRIPTS_DIR not in sys.path:
|
|
30
|
+
sys.path.insert(0, SCRIPTS_DIR)
|
|
31
|
+
|
|
32
|
+
import drain_pending # noqa: E402
|
|
33
|
+
import reconcile_tail # noqa: E402
|
|
34
|
+
import session_start # noqa: E402
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class _ReachableClient:
|
|
38
|
+
"""Stand-in for HindsightClient — reachable, no network."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, *_a, **_kw):
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
def health_check(self, timeout=5, retries=3):
|
|
44
|
+
return True
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class DurabilityWorkStillRunsTest(unittest.TestCase):
|
|
48
|
+
CONFIG = {"autoRetain": True, "autoRecall": True}
|
|
49
|
+
|
|
50
|
+
def _run_main(self, config=None):
|
|
51
|
+
"""Run ``session_start.main()`` against a reachable server with the
|
|
52
|
+
two durability calls stubbed, returning the ordered call log."""
|
|
53
|
+
calls = []
|
|
54
|
+
|
|
55
|
+
def fake_drain(cfg):
|
|
56
|
+
calls.append("drain")
|
|
57
|
+
|
|
58
|
+
def fake_reconcile(cfg, hook_input=None):
|
|
59
|
+
calls.append("reconcile")
|
|
60
|
+
|
|
61
|
+
cfg = dict(self.CONFIG if config is None else config)
|
|
62
|
+
with unittest.mock.patch.object(drain_pending, "drain", fake_drain), \
|
|
63
|
+
unittest.mock.patch.object(reconcile_tail, "reconcile", fake_reconcile), \
|
|
64
|
+
unittest.mock.patch.object(session_start, "load_config", lambda: cfg), \
|
|
65
|
+
unittest.mock.patch.object(
|
|
66
|
+
session_start,
|
|
67
|
+
"get_api_url",
|
|
68
|
+
lambda c, debug_fn=None, allow_daemon_start=True: (
|
|
69
|
+
"http://127.0.0.1:9/none"
|
|
70
|
+
),
|
|
71
|
+
), \
|
|
72
|
+
unittest.mock.patch.object(
|
|
73
|
+
session_start, "HindsightClient", _ReachableClient
|
|
74
|
+
), \
|
|
75
|
+
unittest.mock.patch.object(sys, "stdin", io.StringIO("{}")):
|
|
76
|
+
session_start.main()
|
|
77
|
+
return calls
|
|
78
|
+
|
|
79
|
+
def test_drain_and_reconcile_both_run_on_a_reachable_server(self):
|
|
80
|
+
calls = self._run_main()
|
|
81
|
+
self.assertIn("drain", calls, "queued retains must still be drained")
|
|
82
|
+
self.assertIn(
|
|
83
|
+
"reconcile",
|
|
84
|
+
calls,
|
|
85
|
+
"un-committed abrupt-kill turns must still be reconciled",
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def test_reconcile_runs_after_the_drain(self):
|
|
89
|
+
calls = self._run_main()
|
|
90
|
+
self.assertEqual(
|
|
91
|
+
calls,
|
|
92
|
+
["drain", "reconcile"],
|
|
93
|
+
"reconcile recovers what SessionEnd never enqueued, so it must "
|
|
94
|
+
"run AFTER the drain replays what it did",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def test_disabled_memory_skips_both(self):
|
|
98
|
+
"""The control case: with both autoRecall and autoRetain off, the
|
|
99
|
+
hook returns before touching the durability path. Without this the
|
|
100
|
+
assertions above could be satisfied by calls that fire
|
|
101
|
+
unconditionally."""
|
|
102
|
+
calls = self._run_main(config={"autoRetain": False, "autoRecall": False})
|
|
103
|
+
self.assertEqual(calls, [])
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
if __name__ == "__main__":
|
|
107
|
+
unittest.main()
|