switchroom 0.20.8 → 0.20.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/bin/handoff-briefing.sh +57 -5
  2. package/bin/working-state-reload-hook.sh +262 -0
  3. package/dist/agent-scheduler/index.js +16 -13
  4. package/dist/auth-broker/index.js +70 -30
  5. package/dist/cli/autoaccept-poll.js +5 -3
  6. package/dist/cli/drive-write-pretool.mjs +5 -3
  7. package/dist/cli/ms-365-write-pretool.mjs +5 -3
  8. package/dist/cli/notion-write-pretool.mjs +6 -6
  9. package/dist/cli/switchroom.js +42 -13
  10. package/dist/host-control/main.js +7 -7
  11. package/dist/vault/approvals/kernel-server.js +6 -6
  12. package/dist/vault/broker/server.js +6 -6
  13. package/package.json +1 -1
  14. package/profiles/_base/start.sh.hbs +49 -0
  15. package/profiles/default/CLAUDE.md.hbs +12 -13
  16. package/telegram-plugin/ask-user.ts +6 -7
  17. package/telegram-plugin/dist/gateway/gateway.js +192 -66
  18. package/telegram-plugin/gateway/auth-broker-client.ts +1 -1
  19. package/telegram-plugin/gateway/auth-command.ts +4 -2
  20. package/telegram-plugin/gateway/checklist-fallback.ts +8 -1
  21. package/telegram-plugin/gateway/gateway.ts +8 -4
  22. package/telegram-plugin/gateway/outbound-send-path.ts +9 -1
  23. package/telegram-plugin/gateway/subagent-handback-inbound-builder.ts +21 -1
  24. package/telegram-plugin/gateway/subagent-handback-marker.ts +94 -0
  25. package/telegram-plugin/gateway/throttle-tier-wiring.ts +93 -18
  26. package/telegram-plugin/render/emphasis-guard.ts +92 -12
  27. package/telegram-plugin/render/line-start-guard.ts +27 -2
  28. package/telegram-plugin/sticker-aliases.ts +12 -14
  29. package/telegram-plugin/tests/ask-user.test.ts +15 -0
  30. package/telegram-plugin/tests/checklist-fallback.test.ts +21 -0
  31. package/telegram-plugin/tests/handback-tasknotif-dedup.test.ts +248 -0
  32. package/telegram-plugin/tests/render/emphasis-guard.test.ts +105 -6
  33. package/telegram-plugin/tests/render/heading-guard-blockquote-glued-hash.test.ts +123 -36
  34. package/telegram-plugin/tests/reply-quote-wire.test.ts +47 -0
  35. package/telegram-plugin/tests/sticker-aliases.test.ts +43 -0
  36. package/telegram-plugin/tests/throttle-tier-probe-only.test.ts +216 -0
  37. package/telegram-plugin/tests/throttle-tier-route-429-wiring.test.ts +92 -0
  38. package/telegram-plugin/tests/throttle-tier-route-429.test.ts +71 -0
  39. package/telegram-plugin/throttle-tier.ts +59 -0
  40. package/vendor/hindsight-memory/CHANGELOG.md +31 -0
  41. package/vendor/hindsight-memory/hooks/hooks.json +2 -1
  42. package/vendor/hindsight-memory/scripts/session_start.py +35 -8
  43. package/vendor/hindsight-memory/scripts/tests/test_session_start_durability.py +107 -0
@@ -0,0 +1,216 @@
1
+ /**
2
+ * Tests for the generic-transient PROBE-ONLY 429 path (#failover-429-corroborate).
3
+ *
4
+ * This pins the OUTCOMES the gateway promises for a bare `rate_limit_error`
5
+ * (generic-transient) 429 after Ken picked Option A (probe-only):
6
+ *
7
+ * - generic-transient + HEALTHY probe → the probe's ONLY effect is the
8
+ * silent broker quota refresh. Account-inert at the runner: NO throttle
9
+ * notice, NO self-restart nudge, NO `throttled_until` soft-defer, NO second
10
+ * card. The calm rate-limited card the gateway already emitted stays the
11
+ * ONLY user-visible output.
12
+ * - generic-transient + WALL (probe corroborates) → the escalation path runs
13
+ * exactly like the account-scoped `fire`: the corroborated-wall announcement
14
+ * posts and the dead turn is resumed. No redundant calm card is emitted from
15
+ * this branch — the announcement IS the output.
16
+ * - litellm-local → the runner does NOT fire at all (account-inert by
17
+ * mechanism, not discipline): the request never reached Anthropic, so
18
+ * account state must not be touched. `account-scoped` likewise takes its own
19
+ * `fire` path, not `fireProbeOnly`. Pinned via the pure classification gate
20
+ * the gateway consults before calling `fireProbeOnly`.
21
+ *
22
+ * The runner side is exercised with fully injected deps (no gateway import),
23
+ * mirroring throttle-tier-wiring.test.ts.
24
+ */
25
+
26
+ import { describe, it, expect } from 'vitest'
27
+ import {
28
+ createThrottleTierRunner,
29
+ type ThrottleBrokerClient,
30
+ type ThrottleTierRunnerDeps,
31
+ } from '../gateway/throttle-tier-wiring.js'
32
+ import { classification429WarrantsCorroboration } from '../throttle-tier.js'
33
+
34
+ const NOW = Date.UTC(2026, 6, 12, 8, 0, 0)
35
+
36
+ interface Harness {
37
+ deps: ThrottleTierRunnerDeps
38
+ calls: {
39
+ markThrottled: Array<{ until: number; probeOnly?: boolean }>
40
+ claims: string[]
41
+ notices: Array<{ chatId: string | number; markdown: string }>
42
+ deferrals: string[]
43
+ restarts: string[]
44
+ logs: string[]
45
+ timers: Array<{ ms: number; fn: () => void; cancelled: boolean }>
46
+ }
47
+ }
48
+
49
+ function makeHarness(opts: {
50
+ markThrottledResult?:
51
+ | { account: string; throttled_until: number; escalated: boolean; rolledTo?: string | null }
52
+ | 'unreachable'
53
+ | 'throw'
54
+ turnInFlight?: () => boolean
55
+ newestTurnStartedAt?: () => number | null
56
+ } = {}): Harness {
57
+ const nowMs = NOW
58
+ const calls: Harness['calls'] = {
59
+ markThrottled: [],
60
+ claims: [],
61
+ notices: [],
62
+ deferrals: [],
63
+ restarts: [],
64
+ logs: [],
65
+ timers: [],
66
+ }
67
+ const client: ThrottleBrokerClient = {
68
+ async markThrottled(until: number, probeOnly?: boolean) {
69
+ calls.markThrottled.push({ until, probeOnly })
70
+ if (opts.markThrottledResult === 'throw') throw new Error('boom')
71
+ const r = opts.markThrottledResult
72
+ if (r && r !== 'unreachable') return r
73
+ return { account: 'alice', throttled_until: until, escalated: false, rolledTo: null }
74
+ },
75
+ async claimNotification(key: string) {
76
+ calls.claims.push(key)
77
+ return { granted: true }
78
+ },
79
+ }
80
+ const deps: ThrottleTierRunnerDeps = {
81
+ agentName: 'carrie',
82
+ getBrokerClient: async () =>
83
+ opts.markThrottledResult === 'unreachable' ? null : client,
84
+ listNoticeChats: () => ['111', '222'],
85
+ sendNotice: (chatId, markdown) => calls.notices.push({ chatId, markdown }),
86
+ resumeDecide: () => 'resume',
87
+ newestActiveTurnStartedAtMs: opts.newestTurnStartedAt ?? (() => null),
88
+ turnInFlight: opts.turnInFlight ?? (() => false),
89
+ deferRestartToTurnComplete: (_agent, reason) => calls.deferrals.push(reason),
90
+ restartNow: (_agent, reason) => calls.restarts.push(reason),
91
+ log: (m) => calls.logs.push(m),
92
+ now: () => nowMs,
93
+ schedule: (fn, ms) => {
94
+ const t = { ms, fn, cancelled: false }
95
+ calls.timers.push(t)
96
+ return { cancel: () => { t.cancelled = true } }
97
+ },
98
+ jitterMs: () => 0,
99
+ }
100
+ return { deps, calls }
101
+ }
102
+
103
+ describe('generic-transient probe-only — classification gate (litellm-local never fires)', () => {
104
+ it('fires the corroboration probe ONLY for generic-transient', () => {
105
+ expect(classification429WarrantsCorroboration('generic-transient')).toBe(true)
106
+ // litellm-local: request never reached Anthropic — account-inert by mechanism.
107
+ expect(classification429WarrantsCorroboration('litellm-local')).toBe(false)
108
+ // account-scoped: runs its own throttle tier / failover (fire, not fireProbeOnly).
109
+ expect(classification429WarrantsCorroboration('account-scoped')).toBe(false)
110
+ // null: not a rate-limited event.
111
+ expect(classification429WarrantsCorroboration(null)).toBe(false)
112
+ })
113
+ })
114
+
115
+ describe('generic-transient probe-only — HEALTHY probe is account-inert', () => {
116
+ it('probes the broker but emits NOTHING else (calm card stays the only output)', async () => {
117
+ const h = makeHarness() // default result: escalated:false (healthy)
118
+ const runner = createThrottleTierRunner(h.deps)
119
+ await runner.fireProbeOnly('carrie')
120
+
121
+ // The probe reached the broker in probe-only mode…
122
+ expect(h.calls.markThrottled).toHaveLength(1)
123
+ expect(h.calls.markThrottled[0].probeOnly).toBe(true)
124
+
125
+ // …but produced NO user-visible or account-mutating side effects:
126
+ expect(h.calls.notices).toHaveLength(0) // no throttle notice, no second card
127
+ expect(h.calls.claims).toHaveLength(0) // no fleet-dedup broadcast at all
128
+ expect(h.calls.restarts).toHaveLength(0) // no self-restart nudge
129
+ expect(h.calls.deferrals).toHaveLength(0)
130
+ expect(h.calls.timers).toHaveLength(0) // no throttled_until soft-defer / retry nudge
131
+ expect(runner.inspect().nudgePending).toBe(false)
132
+ })
133
+
134
+ it('a broker-unreachable probe is inert and never throws', async () => {
135
+ const h = makeHarness({ markThrottledResult: 'unreachable' })
136
+ const runner = createThrottleTierRunner(h.deps)
137
+ await expect(runner.fireProbeOnly('carrie')).resolves.toBeUndefined()
138
+ expect(h.calls.markThrottled).toHaveLength(0)
139
+ expect(h.calls.notices).toHaveLength(0)
140
+ expect(h.calls.restarts).toHaveLength(0)
141
+ expect(h.calls.timers).toHaveLength(0)
142
+ })
143
+
144
+ it('a markThrottled throw is swallowed and stays inert (no card, no restart)', async () => {
145
+ const h = makeHarness({ markThrottledResult: 'throw' })
146
+ const runner = createThrottleTierRunner(h.deps)
147
+ await expect(runner.fireProbeOnly('carrie')).resolves.toBeUndefined()
148
+ expect(h.calls.notices).toHaveLength(0)
149
+ expect(h.calls.restarts).toHaveLength(0)
150
+ expect(h.calls.timers).toHaveLength(0)
151
+ })
152
+ })
153
+
154
+ describe('generic-transient probe-only — WALL corroborated → escalation', () => {
155
+ it('posts the corroborated-wall announcement and resumes immediately (no calm card, no nudge timer)', async () => {
156
+ const h = makeHarness({
157
+ markThrottledResult: {
158
+ account: 'alice',
159
+ throttled_until: NOW + 60_000,
160
+ escalated: true,
161
+ rolledTo: 'bob',
162
+ },
163
+ })
164
+ const runner = createThrottleTierRunner(h.deps)
165
+ await runner.fireProbeOnly('carrie')
166
+
167
+ // Probe ran in probe-only mode, then escalation took over.
168
+ expect(h.calls.markThrottled[0].probeOnly).toBe(true)
169
+
170
+ // The escalation announcement (fleet-deduped), NOT the staying-put notice.
171
+ expect(h.calls.claims).toEqual([
172
+ `throttle-escalation:alice:111`,
173
+ `throttle-escalation:alice:222`,
174
+ ])
175
+ expect(h.calls.notices).toHaveLength(2)
176
+ expect(h.calls.notices[0].markdown).toContain('actually a wall')
177
+ expect(h.calls.notices[0].markdown).toContain('bob')
178
+
179
+ // Immediate resume via the escalation lever; NO delayed retry nudge armed.
180
+ expect(h.calls.restarts).toEqual(['throttle-escalation-resume'])
181
+ expect(h.calls.timers).toHaveLength(0)
182
+ })
183
+
184
+ it('escalated with rolledTo=null (all blocked) announces but does NOT restart', async () => {
185
+ const h = makeHarness({
186
+ markThrottledResult: {
187
+ account: 'alice',
188
+ throttled_until: NOW + 60_000,
189
+ escalated: true,
190
+ rolledTo: null,
191
+ },
192
+ })
193
+ const runner = createThrottleTierRunner(h.deps)
194
+ await runner.fireProbeOnly('carrie')
195
+ expect(h.calls.notices[0].markdown).toContain('all blocked')
196
+ expect(h.calls.restarts).toHaveLength(0)
197
+ expect(h.calls.timers).toHaveLength(0)
198
+ })
199
+
200
+ it('escalated resume respects the live-turn guards (defers under the dead turn gate)', async () => {
201
+ const h = makeHarness({
202
+ markThrottledResult: {
203
+ account: 'alice',
204
+ throttled_until: NOW + 60_000,
205
+ escalated: true,
206
+ rolledTo: 'bob',
207
+ },
208
+ turnInFlight: () => true,
209
+ newestTurnStartedAt: () => NOW - 60_000, // the dead turn still holds the gate
210
+ })
211
+ const runner = createThrottleTierRunner(h.deps)
212
+ await runner.fireProbeOnly('carrie')
213
+ expect(h.calls.restarts).toHaveLength(0)
214
+ expect(h.calls.deferrals).toEqual(['throttle-escalation-resume'])
215
+ })
216
+ })
@@ -0,0 +1,92 @@
1
+ /**
2
+ * L1 SEAM-BOUNDARY GUARD for the 429 classify→route wiring (follow-up to
3
+ * switchroom#4381 / #4379).
4
+ *
5
+ * `throttle-tier-route-429.test.ts` proves the ROUTING LOGIC of the pure seam
6
+ * `routeRateLimit429` in isolation (generic-transient → fireProbeOnly;
7
+ * litellm-local / account-scoped / null → no probe). But nothing asserted that
8
+ * the GATEWAY still calls that seam at its 429-handling path. That is the exact
9
+ * gap the extraction opened: a future edit deleting the gateway callsite
10
+ * (gateway.ts, `emitGatewayOperatorEvent` rate-limited calm branch) would
11
+ * silently drop the corroboration probe for every `generic-transient` 429 —
12
+ * a real 5h/7d wall hiding behind transient wording would again die with no
13
+ * failover — while the seam's own unit suite stayed fully green.
14
+ *
15
+ * gateway.ts is a side-effecting IIFE that cannot be imported in a unit test
16
+ * (same constraint documented in turn-flush-suppression-wiring.test.ts /
17
+ * activity-card-wiring.test.ts): `emitGatewayOperatorEvent` is a module-scoped,
18
+ * un-exported function reachable only after the gateway boots. So an
19
+ * outcome-level "drive emitGatewayOperatorEvent and observe fireProbeOnly" test
20
+ * is infeasible without exporting internal gateway state — scope creep the
21
+ * repo deliberately avoids. Following the established wiring-guard pattern,
22
+ * this is a STRUCTURAL assertion that pins the load-bearing call site. It
23
+ * COMPLEMENTS the seam suite: the routing outcomes are proven there; this
24
+ * guards that the gateway actually routes through them.
25
+ *
26
+ * The outcome that flips on the regression: with the callsite present, a
27
+ * `generic-transient` 429 reaches `routeRateLimit429(..., throttleTierRunner,
28
+ * agent)` and fires the probe on the REAL runner; delete or neuter the
29
+ * callsite and these assertions go red. Verified fails-red by deleting
30
+ * gateway.ts:7831 locally (see PR body).
31
+ */
32
+ import { describe, it, expect } from 'vitest'
33
+ import { readFileSync } from 'node:fs'
34
+ import { resolve } from 'node:path'
35
+
36
+ const gatewaySrc = readFileSync(resolve(__dirname, '..', 'gateway', 'gateway.ts'), 'utf-8')
37
+
38
+ function between(src: string, startMarker: string, endMarker: string): string {
39
+ const after = src.split(startMarker)[1] ?? ''
40
+ return after.split(endMarker)[0] ?? ''
41
+ }
42
+
43
+ /** Strip comment lines so prose (a docstring mentioning the call, or a comment
44
+ * describing the OLD shape) can neither satisfy nor trip an assertion about
45
+ * the actual CODE. */
46
+ function codeOnly(src: string): string {
47
+ return src
48
+ .split('\n')
49
+ .filter((l) => !l.trim().startsWith('//'))
50
+ .filter((l) => !l.trim().startsWith('*') && !l.trim().startsWith('/*'))
51
+ .join('\n')
52
+ }
53
+
54
+ describe('429 route-seam wiring — the gateway calls routeRateLimit429 (switchroom#4381 L1)', () => {
55
+ // The calm-429 branch: from the classification binding through the surface
56
+ // decision that immediately follows the seam call.
57
+ const branch = between(
58
+ gatewaySrc,
59
+ 'const rateLimit429Classification =',
60
+ "if (surface === 'litellm-local-notice')",
61
+ )
62
+
63
+ it('the calm-429 branch routes the classification through routeRateLimit429', () => {
64
+ expect(branch.length).toBeGreaterThan(100)
65
+ const code = codeOnly(branch)
66
+ // The load-bearing call site. Deleting it drops the corroboration probe for
67
+ // every generic-transient 429 with the seam's unit suite still green — the
68
+ // exact regression this guard exists to catch.
69
+ expect(code).toMatch(/routeRateLimit429\(/)
70
+ // It must be fed the CLASSIFICATION computed for this event, not a literal
71
+ // or a stale variable — otherwise the wrong family would (not) be probed.
72
+ expect(code).toMatch(
73
+ /routeRateLimit429\(\s*rateLimit429Classification\s*,/,
74
+ )
75
+ })
76
+
77
+ it('the seam is handed the REAL throttleTierRunner (a live probe, not a no-op)', () => {
78
+ const code = codeOnly(branch)
79
+ // Passing anything other than the gateway's real runner (e.g. a stub) would
80
+ // make the probe fire into the void — the branch outcome would look correct
81
+ // in a naive grep but corroborate nothing.
82
+ expect(code).toMatch(
83
+ /routeRateLimit429\(\s*rateLimit429Classification\s*,\s*throttleTierRunner\s*,\s*agent\s*\)/,
84
+ )
85
+ })
86
+
87
+ it('routeRateLimit429 is imported into the gateway (the seam is actually linked)', () => {
88
+ // A dangling call to an unimported symbol would fail tsc, but pinning the
89
+ // import makes the dependency edge explicit and its removal a red test too.
90
+ expect(codeOnly(gatewaySrc)).toMatch(/\brouteRateLimit429\b/)
91
+ })
92
+ })
@@ -0,0 +1,71 @@
1
+ /**
2
+ * Tests for the classify→route WIRING of the gateway's 429 corroboration path
3
+ * (#failover-429-corroborate, follow-up switchroom#4379).
4
+ *
5
+ * The pure gate `classification429WarrantsCorroboration` is already unit-tested
6
+ * (throttle-tier-probe-only.test.ts), and the runner's `fireProbeOnly` outcomes
7
+ * are pinned there too. What was NOT covered is the WIRING that connects them:
8
+ * the gateway callsite (gateway.ts, `handleOperatorEvent` rate-limited branch)
9
+ * that decides whether to invoke `throttleTierRunner.fireProbeOnly(agent)` for a
10
+ * given classification.
11
+ *
12
+ * gateway.ts is hard to unit-test in isolation and is under a hard line ratchet,
13
+ * so rather than stand up a gateway harness the decision was extracted into a
14
+ * pure, injectable seam — `routeRateLimit429(classification, runner, agent)` in
15
+ * throttle-tier.ts — which the gateway callsite now calls verbatim. These tests
16
+ * exercise that seam with a fake runner that records which entrypoint fired,
17
+ * pinning the three routing outcomes the gateway promises:
18
+ *
19
+ * - generic-transient → routes to fireProbeOnly (the probe fires)
20
+ * - litellm-local → does NOT route to fireProbeOnly (account-inert by
21
+ * mechanism: the request never reached Anthropic)
22
+ * - account-scoped → does NOT route to fireProbeOnly (that classification
23
+ * takes its own `fire` path, not the probe)
24
+ */
25
+
26
+ import { describe, it, expect } from 'vitest'
27
+ import { routeRateLimit429, type RateLimit429ProbeRunner } from '../throttle-tier.js'
28
+
29
+ interface FakeRunner extends RateLimit429ProbeRunner {
30
+ probeCalls: string[]
31
+ }
32
+
33
+ function makeFakeRunner(): FakeRunner {
34
+ const probeCalls: string[] = []
35
+ return {
36
+ probeCalls,
37
+ async fireProbeOnly(triggerAgent: string) {
38
+ probeCalls.push(triggerAgent)
39
+ },
40
+ }
41
+ }
42
+
43
+ describe('routeRateLimit429 — gateway classify→route wiring (switchroom#4379)', () => {
44
+ it('generic-transient → routes to fireProbeOnly', () => {
45
+ const runner = makeFakeRunner()
46
+ const fired = routeRateLimit429('generic-transient', runner, 'carrie')
47
+ expect(fired).toBe(true)
48
+ expect(runner.probeCalls).toEqual(['carrie'])
49
+ })
50
+
51
+ it('litellm-local → does NOT route to fireProbeOnly (stays account-inert)', () => {
52
+ const runner = makeFakeRunner()
53
+ const fired = routeRateLimit429('litellm-local', runner, 'carrie')
54
+ expect(fired).toBe(false)
55
+ expect(runner.probeCalls).toHaveLength(0)
56
+ })
57
+
58
+ it('account-scoped → does NOT route to fireProbeOnly (uses fire, not the probe)', () => {
59
+ const runner = makeFakeRunner()
60
+ const fired = routeRateLimit429('account-scoped', runner, 'carrie')
61
+ expect(fired).toBe(false)
62
+ expect(runner.probeCalls).toHaveLength(0)
63
+ })
64
+
65
+ it('null (not a rate-limited event) → does NOT route to fireProbeOnly', () => {
66
+ const runner = makeFakeRunner()
67
+ const fired = routeRateLimit429(null, runner, 'carrie')
68
+ expect(fired).toBe(false)
69
+ expect(runner.probeCalls).toHaveLength(0)
70
+ })
71
+ })
@@ -120,6 +120,65 @@ export function classify429Detail(text: string): RateLimit429Classification {
120
120
  return 'generic-transient'
121
121
  }
122
122
 
123
+ /**
124
+ * #failover-429-corroborate — does this terminal 429 classification warrant a
125
+ * fire-and-forget broker corroboration probe (throttle-tier runner's
126
+ * PROBE-ONLY entrypoint) IN ADDITION to the calm rate-limited card?
127
+ *
128
+ * TRUE for `generic-transient` ONLY. That family (a bare `rate_limit_error`
129
+ * body / novel wording that matches neither the account-scoped negation
130
+ * strings nor litellm-local) used to drop on the calm-card floor with NO broker
131
+ * contact — so a genuine 5h/7d wall hiding behind transient wording was never
132
+ * probed, and the turn died with no failover. Routing it through the runner's
133
+ * `fireProbeOnly` lets the broker take ONE live quota probe (rate-bounded) and
134
+ * convert a real wall into failover. A HEALTHY probe is fully account-inert:
135
+ * probe-only records NO `throttled_until`, sends NO throttle notice, arms NO
136
+ * self-restart nudge, and adds NO second card — so the calm rate-limited card
137
+ * stays the ONLY user-visible output, exactly as before this path existed.
138
+ *
139
+ * FALSE (never corroborate) for:
140
+ * - `litellm-local` — INVARIANT: the request never reached Anthropic (the
141
+ * proxy's own tpm/rpm limiter tripped), so account state must not be
142
+ * touched. This mechanism, not discipline, keeps that path account-inert.
143
+ * - `account-scoped` — already runs its own throttle tier / failover path.
144
+ * - `null` — not a rate-limited event.
145
+ */
146
+ export function classification429WarrantsCorroboration(
147
+ classification: RateLimit429Classification | null,
148
+ ): boolean {
149
+ return classification === 'generic-transient'
150
+ }
151
+
152
+ /**
153
+ * Minimal runner shape the classify→route seam needs: just the PROBE-ONLY
154
+ * entrypoint. The full gateway `ThrottleTierRunner` satisfies it structurally,
155
+ * and a test can supply a fake that records the call.
156
+ */
157
+ export interface RateLimit429ProbeRunner {
158
+ fireProbeOnly(triggerAgent: string): Promise<void>
159
+ }
160
+
161
+ /**
162
+ * #failover-429-corroborate — the classify→route WIRING extracted from the
163
+ * gateway callsite so the routing decision (not just the gate) is unit-testable
164
+ * without a gateway harness. Fires the runner's PROBE-ONLY entrypoint IFF the
165
+ * classification warrants corroboration (`generic-transient` only, per
166
+ * `classification429WarrantsCorroboration`); `litellm-local`, `account-scoped`,
167
+ * and `null` never fire it. Fire-and-forget: the runner promise is intentionally
168
+ * not awaited (the gateway must not block the operator-event path on a probe).
169
+ * Returns `true` when a probe was fired, `false` otherwise — observable in tests
170
+ * and unused at the gateway callsite.
171
+ */
172
+ export function routeRateLimit429(
173
+ classification: RateLimit429Classification | null,
174
+ runner: RateLimit429ProbeRunner,
175
+ agent: string,
176
+ ): boolean {
177
+ if (!classification429WarrantsCorroboration(classification)) return false
178
+ void runner.fireProbeOnly(agent)
179
+ return true
180
+ }
181
+
123
182
  /**
124
183
  * Build the `rate_limit_429_classified` runtime metric for one terminal
125
184
  * rate-limited operator event — the instrumentation that lets an operator
@@ -54,6 +54,37 @@
54
54
 
55
55
  ### Changed (switchroom divergence)
56
56
 
57
+ - **SessionStart hook runs async so its durability work stops being killed
58
+ mid-drain** (`hooks/hooks.json`, `scripts/session_start.py`). The hook does
59
+ the recovery a prior session's abrupt death skipped — drain the
60
+ SessionEnd-queued retains (#1071) and reconcile un-committed transcript turns
61
+ (#3244). Those two carry independent 4s wall-clock budgets
62
+ (`HINDSIGHT_DRAIN_BUDGET_S` / `HINDSIGHT_RECONCILE_BUDGET_S`), each sized in
63
+ isolation against the old synchronous 5s SessionStart timeout, plus a ~2s
64
+ Mode-1 external-server health probe. Summed, they routinely overran 5s, so
65
+ Claude Code SIGKILLed the hook part-way through — truncating exactly the
66
+ durability work it exists to do and letting the pending-retains backlog grow
67
+ (fleet transcripts carry 1000+ `hook_cancelled` attachments for
68
+ `session_start.py`, all `timedOut: true` with `durationMs > 5000`; a
69
+ successful firing injects no `additionalContext` and so leaves no record,
70
+ which made the failures look like ~100% of firings when the true rate is
71
+ unmeasurable from attachments). Setting `"async": true` on the hook (the same
72
+ non-blocking pattern the Stop-event `retain.py` already uses) detaches it from
73
+ the SessionStart critical path: it injects no context, so nothing depends on
74
+ it finishing first and async's dropped context costs nothing. The `timeout`
75
+ ceiling rises 5s → 30s purely as a background-lifetime bound above the ~10s
76
+ summed sub-budgets — it never re-introduces a startup block, because drain and
77
+ reconcile self-cap at their budgets regardless. The drain stays in the hook
78
+ (not deferred wholly to the `hindsight-drain` sidecar, which `start.sh` only
79
+ starts conditionally — when it is absent this hook is the sole backlog path)
80
+ and `reconcile_tail`, which has no sidecar equivalent, stays here and now runs
81
+ to completion; its over-budget remainder still resumes on the next boot.
82
+ Concurrent drain with the sidecar stays safe via `drain_pending`'s exclusive
83
+ `fcntl.flock`. Pinned by `scripts/tests/test_session_start_durability.py`
84
+ (drain + reconcile are still invoked, in order) and
85
+ `tests/hindsight-session-start-async.test.ts` (the async flag + budget-clearing
86
+ ceiling survive the scaffold's hooks-override round-trip).
87
+
57
88
  - **Recall/retain hygiene guard-rail batch** (`scripts/recall.py`,
58
89
  `scripts/subagent_retain.py`, `scripts/lib/content.py`, `scripts/tests/**`).
59
90
  Closes guard-rail debt in the fork's hook scripts before extending them:
@@ -6,7 +6,8 @@
6
6
  {
7
7
  "type": "command",
8
8
  "command": "python3 \"${CLAUDE_PLUGIN_ROOT}/scripts/session_start.py\"",
9
- "timeout": 5
9
+ "async": true,
10
+ "timeout": 30
10
11
  }
11
12
  ]
12
13
  }
@@ -1,12 +1,39 @@
1
1
  #!/usr/bin/env python3
2
- """SessionStart hook: health check + session logging.
3
-
4
- Fires once when a Claude Code session begins. Uses additionalContext
5
- (supported on SessionStart) to inject an initial system note if
6
- Hindsight is available.
7
-
8
- This is the Claude Code equivalent of Openclaw's service.start()
9
- verify the server is reachable early, before the first prompt.
2
+ """SessionStart hook: health check + durability drain/reconcile.
3
+
4
+ Fires once when a Claude Code session begins (startup / compact / clear).
5
+ This is the Claude Code equivalent of Openclaw's service.start() — verify
6
+ the server is reachable early, then do the durability work that a prior
7
+ session's abrupt death may have skipped: drain SessionEnd-queued retains
8
+ (#1071) and reconcile un-committed transcript turns (#3244).
9
+
10
+ Runs ASYNC (``"async": true`` in hooks/hooks.json), for a reason specific
11
+ to THIS hook: it injects NO additionalContext — every return path is a
12
+ pure side effect — so nothing in the session depends on it finishing
13
+ before the first prompt, and losing async's dropped context costs us
14
+ nothing here (unlike recall.py, which must stay synchronous to inject).
15
+
16
+ Why async matters: drain and reconcile each carry an independent
17
+ wall-clock budget (``HINDSIGHT_DRAIN_BUDGET_S`` / ``HINDSIGHT_RECONCILE_BUDGET_S``,
18
+ 4s each) plus a Mode-1 health probe (~2s). Those budgets were each sized
19
+ against the old synchronous 5s SessionStart timeout in ISOLATION, but they
20
+ STACK on one hook: drain(4s) + reconcile(4s, added later in #3244) +
21
+ probe(2s) routinely overran 5s, so the hook was SIGKILLed mid-drain on a
22
+ large share of firings (see the fleet-wide ``hook_cancelled`` transcript
23
+ attachments) — truncating exactly the durability work it exists to do,
24
+ and growing the pending-retains backlog it was meant to clear. Async
25
+ detaches it from the SessionStart critical path so the bounded, resumable
26
+ drain/reconcile can run to completion instead of being cut off. Both are
27
+ crash-safe under an eventual reap anyway: drain holds an ``fcntl.flock``
28
+ the kernel releases on death and re-queues unsent entries, and reconcile
29
+ is idempotent and resumes any over-budget remainder on the next boot.
30
+
31
+ Keeping the drain here (rather than deferring wholly to the hindsight-drain
32
+ sidecar) is deliberate: that sidecar is only CONDITIONALLY started — its
33
+ own "NOT STARTED" branch in start.sh notes that when it is absent the
34
+ memory queue is drained ONLY at session boot — so this hook stays the
35
+ backlog path for agents without the sidecar. reconcile_tail has no sidecar
36
+ equivalent at all and lives only here.
10
37
  """
11
38
 
12
39
  import json
@@ -0,0 +1,107 @@
1
+ """The SessionStart hook must still DO its durability work.
2
+
3
+ The hook was made ``"async": true`` (hooks/hooks.json) so a stacked
4
+ drain + reconcile + health-probe budget can no longer overrun the
5
+ SessionStart timeout and get the process SIGKILLed mid-drain. Async only
6
+ helps if the two durability calls are still MADE on the healthy path —
7
+ if a refactor drops one, the queue silently stops draining and abrupt-kill
8
+ turns stop being recovered, which is the exact outage the hook exists to
9
+ prevent and which no attachment record would surface (a successful
10
+ SessionStart hook that injects no context leaves no transcript trace).
11
+
12
+ So this pins the OUTCOME, not the wiring: on a reachable server,
13
+ ``session_start.main()`` invokes ``drain_pending.drain`` and then
14
+ ``reconcile_tail.reconcile``, in that order (reconcile runs AFTER the
15
+ drain by design — it recovers what SessionEnd never managed to enqueue).
16
+
17
+ Lives under ``scripts/tests/`` because that is the only python test
18
+ directory CI discovers (``ci-tests-python.yml`` runs ``unittest discover``
19
+ from ``vendor/hindsight-memory/scripts``).
20
+ """
21
+
22
+ import io
23
+ import os
24
+ import sys
25
+ import unittest
26
+ import unittest.mock
27
+
28
+ SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
29
+ if SCRIPTS_DIR not in sys.path:
30
+ sys.path.insert(0, SCRIPTS_DIR)
31
+
32
+ import drain_pending # noqa: E402
33
+ import reconcile_tail # noqa: E402
34
+ import session_start # noqa: E402
35
+
36
+
37
+ class _ReachableClient:
38
+ """Stand-in for HindsightClient — reachable, no network."""
39
+
40
+ def __init__(self, *_a, **_kw):
41
+ pass
42
+
43
+ def health_check(self, timeout=5, retries=3):
44
+ return True
45
+
46
+
47
+ class DurabilityWorkStillRunsTest(unittest.TestCase):
48
+ CONFIG = {"autoRetain": True, "autoRecall": True}
49
+
50
+ def _run_main(self, config=None):
51
+ """Run ``session_start.main()`` against a reachable server with the
52
+ two durability calls stubbed, returning the ordered call log."""
53
+ calls = []
54
+
55
+ def fake_drain(cfg):
56
+ calls.append("drain")
57
+
58
+ def fake_reconcile(cfg, hook_input=None):
59
+ calls.append("reconcile")
60
+
61
+ cfg = dict(self.CONFIG if config is None else config)
62
+ with unittest.mock.patch.object(drain_pending, "drain", fake_drain), \
63
+ unittest.mock.patch.object(reconcile_tail, "reconcile", fake_reconcile), \
64
+ unittest.mock.patch.object(session_start, "load_config", lambda: cfg), \
65
+ unittest.mock.patch.object(
66
+ session_start,
67
+ "get_api_url",
68
+ lambda c, debug_fn=None, allow_daemon_start=True: (
69
+ "http://127.0.0.1:9/none"
70
+ ),
71
+ ), \
72
+ unittest.mock.patch.object(
73
+ session_start, "HindsightClient", _ReachableClient
74
+ ), \
75
+ unittest.mock.patch.object(sys, "stdin", io.StringIO("{}")):
76
+ session_start.main()
77
+ return calls
78
+
79
+ def test_drain_and_reconcile_both_run_on_a_reachable_server(self):
80
+ calls = self._run_main()
81
+ self.assertIn("drain", calls, "queued retains must still be drained")
82
+ self.assertIn(
83
+ "reconcile",
84
+ calls,
85
+ "un-committed abrupt-kill turns must still be reconciled",
86
+ )
87
+
88
+ def test_reconcile_runs_after_the_drain(self):
89
+ calls = self._run_main()
90
+ self.assertEqual(
91
+ calls,
92
+ ["drain", "reconcile"],
93
+ "reconcile recovers what SessionEnd never enqueued, so it must "
94
+ "run AFTER the drain replays what it did",
95
+ )
96
+
97
+ def test_disabled_memory_skips_both(self):
98
+ """The control case: with both autoRecall and autoRetain off, the
99
+ hook returns before touching the durability path. Without this the
100
+ assertions above could be satisfied by calls that fire
101
+ unconditionally."""
102
+ calls = self._run_main(config={"autoRetain": False, "autoRecall": False})
103
+ self.assertEqual(calls, [])
104
+
105
+
106
+ if __name__ == "__main__":
107
+ unittest.main()