switchroom 0.12.18 → 0.12.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +85 -81
- package/dist/auth-broker/index.js +85 -81
- package/dist/cli/drive-write-pretool.mjs +10 -10
- package/dist/cli/skill-validate-pretool.mjs +72 -72
- package/dist/cli/switchroom.js +1284 -998
- package/dist/host-control/main.js +104 -100
- package/dist/vault/approvals/kernel-server.js +87 -83
- package/dist/vault/broker/server.js +94 -90
- package/package.json +2 -2
- package/telegram-plugin/dist/bridge/bridge.js +112 -112
- package/telegram-plugin/dist/gateway/gateway.js +448 -209
- package/telegram-plugin/dist/server.js +160 -160
- package/telegram-plugin/gateway/gateway.ts +180 -13
- package/telegram-plugin/gateway/inbound-delivery-gate.ts +85 -0
- package/telegram-plugin/gateway/inbound-spool.ts +272 -0
- package/telegram-plugin/gateway/pending-inbound-buffer.ts +42 -3
- package/telegram-plugin/gateway/turn-state-purge.ts +71 -0
- package/telegram-plugin/tests/inbound-delivery-gate.test.ts +53 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +229 -0
- package/telegram-plugin/tests/pending-inbound-buffer.test.ts +66 -0
- package/telegram-plugin/tests/turn-state-purge.test.ts +109 -0
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
import type { InboundMessage } from './ipc-protocol.js'
|
|
33
|
+
import type { InboundSpool } from './inbound-spool.js'
|
|
33
34
|
|
|
34
35
|
/** Default cap per agent. Tuned for `should fit a reasonable backlog of
|
|
35
36
|
* approval cards stacked while bridge is offline` but no more. */
|
|
@@ -52,6 +53,19 @@ export interface PendingInboundBuffer {
|
|
|
52
53
|
export interface PendingInboundBufferOptions {
|
|
53
54
|
capPerAgent?: number
|
|
54
55
|
log?: (line: string) => void
|
|
56
|
+
/**
|
|
57
|
+
* Durable spool. When set, every `push` is also recorded on the
|
|
58
|
+
* persistent per-agent volume so a gateway/container restart cannot
|
|
59
|
+
* silently lose the message (the finn/carrie incident class). The
|
|
60
|
+
* in-memory queue stays the hot path + cap; the spool is the
|
|
61
|
+
* crash-survivable record, acked only on confirmed delivery (by
|
|
62
|
+
* `redeliverBufferedInbound`/`idleDrainTick`), boot-replayed by the
|
|
63
|
+
* gateway, and escalated-then-dropped if undeliverable past its
|
|
64
|
+
* bound. The in-memory cap eviction does NOT touch the spool — an
|
|
65
|
+
* evicted-from-memory entry survives in the spool (strictly safer
|
|
66
|
+
* than the old silent in-memory drop).
|
|
67
|
+
*/
|
|
68
|
+
spool?: InboundSpool
|
|
55
69
|
}
|
|
56
70
|
|
|
57
71
|
/**
|
|
@@ -72,6 +86,7 @@ export function redeliverBufferedInbound(
|
|
|
72
86
|
buffer: PendingInboundBuffer,
|
|
73
87
|
agent: string,
|
|
74
88
|
send: (msg: InboundMessage) => boolean,
|
|
89
|
+
spool?: InboundSpool,
|
|
75
90
|
): { drained: number; redelivered: number; rebuffered: number } {
|
|
76
91
|
const pending = buffer.drain(agent)
|
|
77
92
|
let redelivered = 0
|
|
@@ -85,6 +100,11 @@ export function redeliverBufferedInbound(
|
|
|
85
100
|
}
|
|
86
101
|
if (delivered) {
|
|
87
102
|
redelivered++
|
|
103
|
+
// Confirmed delivery to a live registered bridge → the durable
|
|
104
|
+
// promise is kept; tombstone the spool entry so it is NOT
|
|
105
|
+
// boot-replayed again. A miss leaves it spooled (re-pushed below
|
|
106
|
+
// AND still live in the spool) for the next drain / escalation.
|
|
107
|
+
spool?.ack(msg)
|
|
88
108
|
} else {
|
|
89
109
|
buffer.push(agent, msg)
|
|
90
110
|
rebuffered++
|
|
@@ -107,8 +127,19 @@ export function redeliverBufferedInbound(
|
|
|
107
127
|
* which would re-buffer+log-spin every tick; onClientRegistered
|
|
108
128
|
* will drain on the eventual reconnect instead)
|
|
109
129
|
* - otherwise → `redeliverBufferedInbound` (lossless: re-buffers any
|
|
110
|
-
* per-message miss).
|
|
111
|
-
*
|
|
130
|
+
* per-message miss).
|
|
131
|
+
*
|
|
132
|
+
* NOTE (#1556): a message delivered mid-turn is NOT safely queued by
|
|
133
|
+
* the bridge — the prior "queued normally, same as a live arrival"
|
|
134
|
+
* claim here was the false assumption behind the lawgpt composer
|
|
135
|
+
* wedge. claude types a mid-turn channel notification into its TUI
|
|
136
|
+
* composer and the auto-submit races turn-completion, stranding it.
|
|
137
|
+
* The `idleDrainTick` caller therefore also gates on
|
|
138
|
+
* `activeTurnStartedAt.size === 0`, so this function is never invoked
|
|
139
|
+
* mid-turn. The Telegram `handleInbound` delivery path is turn-gated
|
|
140
|
+
* (gateway.ts); the `inject_inbound` cron/synthetic path is a separate
|
|
141
|
+
* delivery contract and deliberately not gated — see
|
|
142
|
+
* `inbound-delivery-gate.ts`.
|
|
112
143
|
*
|
|
113
144
|
* Returns the redeliver counts only when it actually ran, else null
|
|
114
145
|
* (so the caller logs only on a real flush).
|
|
@@ -118,11 +149,12 @@ export function idleDrainTick(
|
|
|
118
149
|
agent: string,
|
|
119
150
|
isBridgeAlive: () => boolean,
|
|
120
151
|
send: (msg: InboundMessage) => boolean,
|
|
152
|
+
spool?: InboundSpool,
|
|
121
153
|
): { drained: number; redelivered: number; rebuffered: number } | null {
|
|
122
154
|
if (!agent) return null
|
|
123
155
|
if (buffer.depth(agent) === 0) return null
|
|
124
156
|
if (!isBridgeAlive()) return null
|
|
125
|
-
return redeliverBufferedInbound(buffer, agent, send)
|
|
157
|
+
return redeliverBufferedInbound(buffer, agent, send, spool)
|
|
126
158
|
}
|
|
127
159
|
|
|
128
160
|
export function createPendingInboundBuffer(
|
|
@@ -130,6 +162,7 @@ export function createPendingInboundBuffer(
|
|
|
130
162
|
): PendingInboundBuffer {
|
|
131
163
|
const cap = opts.capPerAgent ?? DEFAULT_PENDING_INBOUND_CAP
|
|
132
164
|
const log = opts.log ?? ((line: string) => process.stderr.write(line))
|
|
165
|
+
const spool = opts.spool
|
|
133
166
|
const queues = new Map<string, InboundMessage[]>()
|
|
134
167
|
|
|
135
168
|
return {
|
|
@@ -149,6 +182,12 @@ export function createPendingInboundBuffer(
|
|
|
149
182
|
)
|
|
150
183
|
}
|
|
151
184
|
q.push(msg)
|
|
185
|
+
// Durable record FIRST-class to the in-memory queue: spool BEFORE
|
|
186
|
+
// returning, regardless of the cap eviction above — an entry the
|
|
187
|
+
// in-memory cap drops still survives in the spool (boot-replayed /
|
|
188
|
+
// escalated), which is the whole point. spool.put dedups by
|
|
189
|
+
// spoolId so a boot-replay re-push is a no-op here.
|
|
190
|
+
spool?.put(agent, msg)
|
|
152
191
|
log(
|
|
153
192
|
`pending-inbound-buffer: agent=${agent} buffered source=${msg.meta?.source ?? '-'} ` +
|
|
154
193
|
`depth_after=${q.length} evicted=${evicted}\n`,
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* turn-state-purge.ts — defense-in-depth cleanup for the silence-poke
|
|
3
|
+
* framework-fallback (gateway.ts).
|
|
4
|
+
*
|
|
5
|
+
* The fallback's `purgeReactionTracking(fbKey)` clears the canonical
|
|
6
|
+
* `statusKey(chatId, threadId)` for the exact chat+thread that armed
|
|
7
|
+
* the silence-poke. But `activeTurnStartedAt` (and its siblings) can
|
|
8
|
+
* legitimately hold MORE than one entry for the same chat — e.g.:
|
|
9
|
+
*
|
|
10
|
+
* - a normal turn-end handler null'd `currentTurn` but skipped
|
|
11
|
+
* `purgeReactionTracking` on a code-path that didn't reach it
|
|
12
|
+
* (gateway.ts:5887/5921/6193 — purgeReactionTracking calls in
|
|
13
|
+
* those handlers are conditional/not on every branch), so
|
|
14
|
+
* `activeTurnStartedAt[oldKey]` is dangling when the fallback
|
|
15
|
+
* fires for a NEW key
|
|
16
|
+
* - a multi-thread chat (topic-based group, or a `null` vs
|
|
17
|
+
* `undefined` thread variant) has entries under different keys of
|
|
18
|
+
* the SAME chat
|
|
19
|
+
*
|
|
20
|
+
* In either case, the fallback purges `fbKey` but a sibling key for
|
|
21
|
+
* the same chat remains → `activeTurnStartedAt.size > 0` persists →
|
|
22
|
+
* `#1556`'s turn-gate holds every new inbound forever → user sees
|
|
23
|
+
* "not responding" while claude is actually idle (gymbro + klanker,
|
|
24
|
+
* 2026-05-20).
|
|
25
|
+
*
|
|
26
|
+
* This helper sweeps any sibling keys for the firing chat after the
|
|
27
|
+
* canonical `purgeReactionTracking(fbKey)`, via the same purger.
|
|
28
|
+
* Multi-chat safe: it only touches keys whose chatId matches — other
|
|
29
|
+
* chats' active turns are untouched, preserving the deliberate
|
|
30
|
+
* multi-chat safety #1546 kept.
|
|
31
|
+
*
|
|
32
|
+
* Key shape: `statusKey(chatId, threadId)` emits
|
|
33
|
+
* `${chatId}:${threadId ?? '_'}`. Chat ids are numeric strings (no
|
|
34
|
+
* `:` inside), so `indexOf(':')` is a safe prefix delimiter. A
|
|
35
|
+
* "prefix-without-separator" false positive (e.g. chatId `123` vs key
|
|
36
|
+
* `1234:_`) is structurally impossible because the helper splits on
|
|
37
|
+
* `:` and equates the prefix, NOT a string-startsWith.
|
|
38
|
+
*
|
|
39
|
+
* Pure / dependency-free so the multi-chat-safety and key-shape logic
|
|
40
|
+
* are unit-testable without standing up a gateway — mirrors the
|
|
41
|
+
* `#1544 / #1546 / #1549 / #1558` pure-seam idiom.
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
export interface PurgeStaleTurnsResult {
|
|
45
|
+
/** Keys for `chatId` that the helper purged via the callback. */
|
|
46
|
+
purged: string[]
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function purgeStaleTurnsForChat(
|
|
50
|
+
chatId: string,
|
|
51
|
+
keys: Iterable<string>,
|
|
52
|
+
purger: (key: string) => void,
|
|
53
|
+
): PurgeStaleTurnsResult {
|
|
54
|
+
if (!chatId) return { purged: [] }
|
|
55
|
+
const purged: string[] = []
|
|
56
|
+
// Snapshot first — `purger` typically deletes from the same Map the
|
|
57
|
+
// caller is iterating; mutating during iteration is correct on JS
|
|
58
|
+
// Maps but a snapshot makes the contract explicit and lets the
|
|
59
|
+
// helper accept any iterable.
|
|
60
|
+
const snapshot: string[] = []
|
|
61
|
+
for (const k of keys) snapshot.push(k)
|
|
62
|
+
for (const key of snapshot) {
|
|
63
|
+
const sep = key.indexOf(':')
|
|
64
|
+
if (sep < 0) continue // malformed / non-statusKey shape — skip
|
|
65
|
+
const keyChat = key.slice(0, sep)
|
|
66
|
+
if (keyChat !== chatId) continue
|
|
67
|
+
purger(key)
|
|
68
|
+
purged.push(key)
|
|
69
|
+
}
|
|
70
|
+
return { purged }
|
|
71
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest'
|
|
2
|
+
|
|
3
|
+
import { decideInboundDelivery } from '../gateway/inbound-delivery-gate.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Regression coverage for #1556 — the lawgpt composer wedge.
|
|
7
|
+
*
|
|
8
|
+
* Before this gate, the gateway sent every inbound to the bridge
|
|
9
|
+
* immediately, buffering only when the bridge was offline. A
|
|
10
|
+
* non-steering message that arrived mid-turn was typed into claude's
|
|
11
|
+
* TUI composer and stranded when the auto-submit raced
|
|
12
|
+
* turn-completion. The deterministic invariant the gate enforces:
|
|
13
|
+
*
|
|
14
|
+
* a non-steering inbound is delivered ONLY when no turn is in flight.
|
|
15
|
+
*
|
|
16
|
+
* Steering (/steer, /s) is the sole exemption — reaching claude
|
|
17
|
+
* mid-turn is the entire point of that feature.
|
|
18
|
+
*/
|
|
19
|
+
describe('decideInboundDelivery', () => {
|
|
20
|
+
it('delivers immediately when claude is idle (no turn in flight)', () => {
|
|
21
|
+
expect(
|
|
22
|
+
decideInboundDelivery({ turnInFlight: false, isSteering: false }),
|
|
23
|
+
).toBe('deliver')
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
it('BUFFERS a non-steering message that arrives mid-turn (the wedge fix)', () => {
|
|
27
|
+
expect(
|
|
28
|
+
decideInboundDelivery({ turnInFlight: true, isSteering: false }),
|
|
29
|
+
).toBe('buffer-until-idle')
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
it('delivers a steering message mid-turn (steering is intentionally exempt)', () => {
|
|
33
|
+
expect(
|
|
34
|
+
decideInboundDelivery({ turnInFlight: true, isSteering: true }),
|
|
35
|
+
).toBe('deliver')
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
it('delivers a steering message when idle (steer with no active turn)', () => {
|
|
39
|
+
expect(
|
|
40
|
+
decideInboundDelivery({ turnInFlight: false, isSteering: true }),
|
|
41
|
+
).toBe('deliver')
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
it('is total: the ONLY deferral path is mid-turn AND not steering', () => {
|
|
45
|
+
for (const turnInFlight of [true, false]) {
|
|
46
|
+
for (const isSteering of [true, false]) {
|
|
47
|
+
const decision = decideInboundDelivery({ turnInFlight, isSteering })
|
|
48
|
+
const expectBuffer = turnInFlight && !isSteering
|
|
49
|
+
expect(decision).toBe(expectBuffer ? 'buffer-until-idle' : 'deliver')
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
})
|
|
53
|
+
})
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* inbound-spool — durable, crash-tolerant inbound spool.
|
|
3
|
+
*
|
|
4
|
+
* Pins the determinism guarantee: a buffered inbound survives a
|
|
5
|
+
* gateway/container restart (it's on the persistent volume), is
|
|
6
|
+
* replayed un-acked, acked only on confirmed delivery, deduped by a
|
|
7
|
+
* stable id, and escalated-then-dropped if undeliverable past its
|
|
8
|
+
* bound — so the "your message is queued" promise is ALWAYS resolved
|
|
9
|
+
* (delivered or visibly retracted), never silently lost (the
|
|
10
|
+
* finn/carrie lost-on-restart incident class, 2026-05-19).
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, it, expect } from 'vitest'
|
|
14
|
+
import {
|
|
15
|
+
createInboundSpool,
|
|
16
|
+
spoolId,
|
|
17
|
+
type InboundSpoolFsSeam,
|
|
18
|
+
} from '../gateway/inbound-spool.js'
|
|
19
|
+
import type { InboundMessage } from '../gateway/ipc-protocol.js'
|
|
20
|
+
|
|
21
|
+
function msg(over: Partial<InboundMessage> = {}): InboundMessage {
|
|
22
|
+
return {
|
|
23
|
+
type: 'inbound',
|
|
24
|
+
chatId: 'c1',
|
|
25
|
+
messageId: 1001,
|
|
26
|
+
user: 'ken',
|
|
27
|
+
userId: 42,
|
|
28
|
+
ts: 1000,
|
|
29
|
+
text: 'hello',
|
|
30
|
+
meta: {},
|
|
31
|
+
...over,
|
|
32
|
+
} as InboundMessage
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** In-memory fake fs keyed by path. Models append, full rewrite, and
|
|
36
|
+
* atomic rename (so the tmp→rename compaction path is exercised). */
|
|
37
|
+
function fakeFs(): InboundSpoolFsSeam & { dump(p?: string): string } {
|
|
38
|
+
const files = new Map<string, string>()
|
|
39
|
+
return {
|
|
40
|
+
appendFileSync: (p, d) => files.set(p, (files.get(p) ?? '') + d),
|
|
41
|
+
readFileSync: (p) => files.get(p) ?? '',
|
|
42
|
+
writeFileSync: (p, d) => files.set(p, d),
|
|
43
|
+
renameSync: (from, to) => {
|
|
44
|
+
files.set(to, files.get(from) ?? '')
|
|
45
|
+
files.delete(from)
|
|
46
|
+
},
|
|
47
|
+
existsSync: (p) => files.has(p),
|
|
48
|
+
statSizeSync: (p) => Buffer.byteLength(files.get(p) ?? ''),
|
|
49
|
+
dump: (p = PATH) => files.get(p) ?? '',
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const PATH = '/state/agent/telegram/inbound-spool.jsonl'
|
|
54
|
+
|
|
55
|
+
describe('spoolId — stable dedup key', () => {
|
|
56
|
+
it('real Telegram message → m:chat:msgId', () => {
|
|
57
|
+
expect(spoolId(msg({ chatId: 'c9', messageId: 55 }))).toBe('m:c9:55')
|
|
58
|
+
})
|
|
59
|
+
it('synthetic (messageId 0) → s:chat:source:ts (distinct events do not collapse)', () => {
|
|
60
|
+
const a = spoolId(msg({ messageId: 0, meta: { source: 'cron' }, ts: 100 }))
|
|
61
|
+
const b = spoolId(msg({ messageId: 0, meta: { source: 'cron' }, ts: 200 }))
|
|
62
|
+
expect(a).toBe('s:c1:cron:100')
|
|
63
|
+
expect(a).not.toBe(b) // different ts = different logical event
|
|
64
|
+
})
|
|
65
|
+
it('same logical synthetic retried (same ts) dedups to the same id', () => {
|
|
66
|
+
const a = spoolId(msg({ messageId: 0, meta: { source: 'cron' }, ts: 100 }))
|
|
67
|
+
const b = spoolId(msg({ messageId: 0, meta: { source: 'cron' }, ts: 100 }))
|
|
68
|
+
expect(a).toBe(b)
|
|
69
|
+
})
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
describe('inbound-spool — put / ack / dedup', () => {
|
|
73
|
+
it('put records a live entry; dedups a re-put of the same id', () => {
|
|
74
|
+
const fs = fakeFs()
|
|
75
|
+
const s = createInboundSpool({ path: PATH, fs })
|
|
76
|
+
expect(s.put('carrie', msg({ messageId: 7 }))).toBe(true)
|
|
77
|
+
expect(s.put('carrie', msg({ messageId: 7 }))).toBe(false) // dedup
|
|
78
|
+
expect(s.liveCount()).toBe(1)
|
|
79
|
+
expect(s.liveEntries()).toHaveLength(1)
|
|
80
|
+
expect(s.liveEntries()[0].agent).toBe('carrie')
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
it('ack tombstones the entry; ack is idempotent / unknown-id safe', () => {
|
|
84
|
+
const fs = fakeFs()
|
|
85
|
+
const s = createInboundSpool({ path: PATH, fs })
|
|
86
|
+
const m = msg({ messageId: 7 })
|
|
87
|
+
s.put('carrie', m)
|
|
88
|
+
s.ack(m)
|
|
89
|
+
expect(s.liveCount()).toBe(0)
|
|
90
|
+
s.ack(m) // idempotent
|
|
91
|
+
s.ack(msg({ messageId: 999 })) // unknown id, no throw
|
|
92
|
+
expect(s.liveCount()).toBe(0)
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
it('liveEntries is oldest-first (replay order)', () => {
|
|
96
|
+
const fs = fakeFs()
|
|
97
|
+
const s = createInboundSpool({ path: PATH, fs })
|
|
98
|
+
s.put('a', msg({ messageId: 1 }))
|
|
99
|
+
s.put('a', msg({ messageId: 2 }))
|
|
100
|
+
s.put('a', msg({ messageId: 3 }))
|
|
101
|
+
expect(s.liveEntries().map((e) => e.msg.messageId)).toEqual([1, 2, 3])
|
|
102
|
+
})
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
describe('inbound-spool — crash-survivable replay (the core guarantee)', () => {
|
|
106
|
+
it('a fresh spool over an existing file rebuilds live state (survives restart)', () => {
|
|
107
|
+
const fs = fakeFs()
|
|
108
|
+
const s1 = createInboundSpool({ path: PATH, fs })
|
|
109
|
+
s1.put('carrie', msg({ messageId: 7, text: 'the craft message' }))
|
|
110
|
+
s1.put('carrie', msg({ messageId: 8 }))
|
|
111
|
+
s1.ack(msg({ messageId: 8 })) // 8 delivered before the "crash"
|
|
112
|
+
// Simulate gateway/container restart: brand-new spool, SAME file.
|
|
113
|
+
const s2 = createInboundSpool({ path: PATH, fs })
|
|
114
|
+
expect(s2.liveCount()).toBe(1)
|
|
115
|
+
const live = s2.liveEntries()
|
|
116
|
+
expect(live[0].msg.messageId).toBe(7)
|
|
117
|
+
expect(live[0].msg.text).toBe('the craft message') // full payload survived
|
|
118
|
+
expect(live[0].agent).toBe('carrie')
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
it('tolerates a torn final line (crash mid-append) — skips it, keeps the rest', () => {
|
|
122
|
+
const fs = fakeFs()
|
|
123
|
+
const s1 = createInboundSpool({ path: PATH, fs })
|
|
124
|
+
s1.put('carrie', msg({ messageId: 7 }))
|
|
125
|
+
// Append a half-written record (no newline, invalid JSON tail).
|
|
126
|
+
fs.appendFileSync(PATH, '{"t":"put","id":"m:c1:8","agen')
|
|
127
|
+
const s2 = createInboundSpool({ path: PATH, fs })
|
|
128
|
+
expect(s2.liveCount()).toBe(1) // the torn line is ignored, 7 survives
|
|
129
|
+
expect(s2.liveEntries()[0].msg.messageId).toBe(7)
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
it('ignores any line that does not pass the shape check', () => {
|
|
133
|
+
const fs = fakeFs()
|
|
134
|
+
fs.appendFileSync(PATH, 'not json\n')
|
|
135
|
+
fs.appendFileSync(PATH, '{"t":"put"}\n') // missing id/msg/agent
|
|
136
|
+
fs.appendFileSync(PATH, '{"t":"weird","id":"x"}\n')
|
|
137
|
+
fs.appendFileSync(
|
|
138
|
+
PATH,
|
|
139
|
+
JSON.stringify({ t: 'put', id: 'm:c1:7', agent: 'a', firstAt: 1, msg: msg({ messageId: 7 }) }) + '\n',
|
|
140
|
+
)
|
|
141
|
+
const s = createInboundSpool({ path: PATH, fs })
|
|
142
|
+
expect(s.liveCount()).toBe(1)
|
|
143
|
+
expect(s.liveEntries()[0].msg.messageId).toBe(7)
|
|
144
|
+
})
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
describe('inbound-spool — bounded escalation (promise always resolved)', () => {
|
|
148
|
+
it('escalates+drops only entries older than the bound; younger untouched', () => {
|
|
149
|
+
const fs = fakeFs()
|
|
150
|
+
let t = 1_000_000
|
|
151
|
+
const s = createInboundSpool({
|
|
152
|
+
path: PATH,
|
|
153
|
+
fs,
|
|
154
|
+
now: () => t,
|
|
155
|
+
escalateAfterMs: 10_000,
|
|
156
|
+
})
|
|
157
|
+
s.put('carrie', msg({ messageId: 1 })) // firstAt = 1_000_000
|
|
158
|
+
t = 1_005_000
|
|
159
|
+
s.put('carrie', msg({ messageId: 2 })) // firstAt = 1_005_000
|
|
160
|
+
t = 1_012_000 // msg1 is 12s old (>10s bound), msg2 is 7s old
|
|
161
|
+
const escalated: number[] = []
|
|
162
|
+
const n = s.sweepEscalations((e) => escalated.push(e.msg.messageId as number))
|
|
163
|
+
expect(n).toBe(1)
|
|
164
|
+
expect(escalated).toEqual([1])
|
|
165
|
+
expect(s.liveCount()).toBe(1) // msg2 still live
|
|
166
|
+
expect(s.liveEntries()[0].msg.messageId).toBe(2)
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
it('an escalated id stays dropped across a restart (tombstoned, not replayed)', () => {
|
|
170
|
+
const fs = fakeFs()
|
|
171
|
+
let t = 0
|
|
172
|
+
const s1 = createInboundSpool({ path: PATH, fs, now: () => t, escalateAfterMs: 100 })
|
|
173
|
+
s1.put('a', msg({ messageId: 1 }))
|
|
174
|
+
t = 1000
|
|
175
|
+
expect(s1.sweepEscalations(() => {})).toBe(1)
|
|
176
|
+
const s2 = createInboundSpool({ path: PATH, fs })
|
|
177
|
+
expect(s2.liveCount()).toBe(0) // not resurrected on replay
|
|
178
|
+
})
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
describe('inbound-spool — robustness', () => {
|
|
182
|
+
it('a failing appendFileSync does not throw and keeps in-memory live state', () => {
|
|
183
|
+
const fs = fakeFs()
|
|
184
|
+
fs.appendFileSync = () => {
|
|
185
|
+
throw new Error('ENOSPC')
|
|
186
|
+
}
|
|
187
|
+
const logs: string[] = []
|
|
188
|
+
const s = createInboundSpool({ path: PATH, fs, log: (l) => logs.push(l) })
|
|
189
|
+
expect(() => s.put('a', msg({ messageId: 1 }))).not.toThrow()
|
|
190
|
+
expect(s.liveCount()).toBe(1) // live delivery still works (degraded durability)
|
|
191
|
+
expect(logs.join('')).toContain('durability degraded')
|
|
192
|
+
})
|
|
193
|
+
|
|
194
|
+
it('compacts once past the size bound, dropping acked ids', () => {
|
|
195
|
+
const fs = fakeFs()
|
|
196
|
+
const s = createInboundSpool({ path: PATH, fs, compactAtBytes: 200 })
|
|
197
|
+
for (let i = 1; i <= 20; i++) {
|
|
198
|
+
s.put('a', msg({ messageId: i, text: 'x'.repeat(50) }))
|
|
199
|
+
s.ack(msg({ messageId: i }))
|
|
200
|
+
}
|
|
201
|
+
s.put('a', msg({ messageId: 999 }))
|
|
202
|
+
// After compaction the file holds only the one live id, not the
|
|
203
|
+
// 20 acked put+ack pairs.
|
|
204
|
+
const s2 = createInboundSpool({ path: PATH, fs })
|
|
205
|
+
expect(s2.liveCount()).toBe(1)
|
|
206
|
+
expect(s2.liveEntries()[0].msg.messageId).toBe(999)
|
|
207
|
+
expect(fs.dump().split('\n').filter(Boolean).length).toBeLessThan(5)
|
|
208
|
+
})
|
|
209
|
+
|
|
210
|
+
it('atomic compaction: a rename crash leaves the ORIGINAL log intact (no loss)', () => {
|
|
211
|
+
const fs = fakeFs()
|
|
212
|
+
const s = createInboundSpool({ path: PATH, fs, compactAtBytes: 200 })
|
|
213
|
+
for (let i = 1; i <= 10; i++) {
|
|
214
|
+
s.put('a', msg({ messageId: i, text: 'y'.repeat(50) }))
|
|
215
|
+
}
|
|
216
|
+
// Simulate a crash AFTER the tmp write but BEFORE/at the rename.
|
|
217
|
+
fs.renameSync = () => {
|
|
218
|
+
throw new Error('crash mid-compact')
|
|
219
|
+
}
|
|
220
|
+
s.put('a', msg({ messageId: 11, text: 'y'.repeat(50) })) // triggers maybeCompact → rename throws
|
|
221
|
+
// Original append-only log must still hold every live entry — the
|
|
222
|
+
// failed compaction is a no-op, never a truncation.
|
|
223
|
+
const s2 = createInboundSpool({ path: PATH, fs })
|
|
224
|
+
expect(s2.liveCount()).toBe(11)
|
|
225
|
+
expect(s2.liveEntries().map((e) => e.msg.messageId)).toEqual([
|
|
226
|
+
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
|
|
227
|
+
])
|
|
228
|
+
})
|
|
229
|
+
})
|
|
@@ -247,3 +247,69 @@ describe('idleDrainTick — the 3rd drain trigger (finn orphan gap, 2026-05-19)'
|
|
|
247
247
|
expect(probed).toBe(false) // cheap path: Map.get only, no bridge probe, no log
|
|
248
248
|
})
|
|
249
249
|
})
|
|
250
|
+
|
|
251
|
+
describe('durable-spool integration (finn/carrie lost-on-restart fix)', () => {
|
|
252
|
+
function spySpool() {
|
|
253
|
+
const puts: string[] = []
|
|
254
|
+
const acks: string[] = []
|
|
255
|
+
return {
|
|
256
|
+
puts,
|
|
257
|
+
acks,
|
|
258
|
+
spool: {
|
|
259
|
+
put: (_a: string, m: InboundMessage) => {
|
|
260
|
+
puts.push(String(m.messageId))
|
|
261
|
+
return true
|
|
262
|
+
},
|
|
263
|
+
ack: (m: InboundMessage) => {
|
|
264
|
+
acks.push(String(m.messageId))
|
|
265
|
+
},
|
|
266
|
+
liveEntries: () => [],
|
|
267
|
+
sweepEscalations: () => 0,
|
|
268
|
+
liveCount: () => 0,
|
|
269
|
+
},
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
it('every push is durably spooled (chokepoint for all ~10 push sites)', () => {
|
|
274
|
+
const sp = spySpool()
|
|
275
|
+
const buf = createPendingInboundBuffer({ log: () => {}, spool: sp.spool as never })
|
|
276
|
+
buf.push('carrie', inbound('user', 7))
|
|
277
|
+
buf.push('carrie', inbound('user', 8))
|
|
278
|
+
expect(sp.puts).toEqual(['7', '8'])
|
|
279
|
+
})
|
|
280
|
+
|
|
281
|
+
it('a CONFIRMED delivery acks the spool; a miss does NOT (stays durable)', () => {
|
|
282
|
+
const sp = spySpool()
|
|
283
|
+
const buf = createPendingInboundBuffer({ log: () => {}, spool: sp.spool as never })
|
|
284
|
+
buf.push('carrie', inbound('user', 7))
|
|
285
|
+
// delivery succeeds → spool acked
|
|
286
|
+
redeliverBufferedInbound(buf, 'carrie', () => true, sp.spool as never)
|
|
287
|
+
expect(sp.acks).toEqual(['7'])
|
|
288
|
+
|
|
289
|
+
// delivery misses → NOT acked, re-buffered + still spooled
|
|
290
|
+
const sp2 = spySpool()
|
|
291
|
+
const buf2 = createPendingInboundBuffer({ log: () => {}, spool: sp2.spool as never })
|
|
292
|
+
buf2.push('carrie', inbound('user', 9))
|
|
293
|
+
redeliverBufferedInbound(buf2, 'carrie', () => false, sp2.spool as never)
|
|
294
|
+
expect(sp2.acks).toEqual([]) // never acked on a miss → survives for retry/escalation
|
|
295
|
+
expect(buf2.depth('carrie')).toBe(1) // re-buffered in memory too
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
it('idleDrainTick threads the spool through (ack only on delivered)', () => {
|
|
299
|
+
const sp = spySpool()
|
|
300
|
+
const buf = createPendingInboundBuffer({ log: () => {}, spool: sp.spool as never })
|
|
301
|
+
buf.push('carrie', inbound('user', 7))
|
|
302
|
+
idleDrainTick(buf, 'carrie', () => true, () => true, sp.spool as never)
|
|
303
|
+
expect(sp.acks).toEqual(['7'])
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
it('works with no spool (back-compat: undefined spool is a no-op)', () => {
|
|
307
|
+
const buf = createPendingInboundBuffer({ log: () => {} })
|
|
308
|
+
buf.push('carrie', inbound('user', 7))
|
|
309
|
+
expect(redeliverBufferedInbound(buf, 'carrie', () => true)).toEqual({
|
|
310
|
+
drained: 1,
|
|
311
|
+
redelivered: 1,
|
|
312
|
+
rebuffered: 0,
|
|
313
|
+
})
|
|
314
|
+
})
|
|
315
|
+
})
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* turn-state-purge — the silence-poke fallback's defense-in-depth
|
|
3
|
+
* cleanup. Pins the multi-chat safety + key-shape correctness that
|
|
4
|
+
* the gymbro/klanker held-mid-turn fix (2026-05-20) depends on.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { describe, it, expect } from 'vitest'
|
|
8
|
+
import { purgeStaleTurnsForChat } from '../gateway/turn-state-purge.js'
|
|
9
|
+
|
|
10
|
+
function statusKey(chatId: string, threadId?: number): string {
|
|
11
|
+
return `${chatId}:${threadId ?? '_'}`
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
describe('purgeStaleTurnsForChat', () => {
|
|
15
|
+
it('returns [] and calls nothing when keys is empty', () => {
|
|
16
|
+
let calls = 0
|
|
17
|
+
const r = purgeStaleTurnsForChat('123', [], () => calls++)
|
|
18
|
+
expect(r.purged).toEqual([])
|
|
19
|
+
expect(calls).toBe(0)
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
it('returns [] and calls nothing when chatId is empty (guard)', () => {
|
|
23
|
+
let calls = 0
|
|
24
|
+
const r = purgeStaleTurnsForChat('', [statusKey('123')], () => calls++)
|
|
25
|
+
expect(r.purged).toEqual([])
|
|
26
|
+
expect(calls).toBe(0)
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
it('purges the canonical DM key for the firing chat (typical case)', () => {
|
|
30
|
+
const purged: string[] = []
|
|
31
|
+
const r = purgeStaleTurnsForChat(
|
|
32
|
+
'12345',
|
|
33
|
+
[statusKey('12345')],
|
|
34
|
+
(k) => purged.push(k),
|
|
35
|
+
)
|
|
36
|
+
expect(r.purged).toEqual(['12345:_'])
|
|
37
|
+
expect(purged).toEqual(['12345:_'])
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
it('purges every sibling key for the same chat (multi-thread / dangling)', () => {
|
|
41
|
+
// This is the gymbro/klanker symptom: a stale activeTurnStartedAt
|
|
42
|
+
// entry under a different thread of the same chat survives the
|
|
43
|
+
// canonical purgeReactionTracking(fbKey).
|
|
44
|
+
const purged: string[] = []
|
|
45
|
+
const r = purgeStaleTurnsForChat(
|
|
46
|
+
'12345',
|
|
47
|
+
[
|
|
48
|
+
statusKey('12345'), // DM, no thread
|
|
49
|
+
statusKey('12345', 42), // thread 42
|
|
50
|
+
statusKey('12345', 99), // thread 99
|
|
51
|
+
],
|
|
52
|
+
(k) => purged.push(k),
|
|
53
|
+
)
|
|
54
|
+
expect(r.purged.sort()).toEqual(['12345:42', '12345:99', '12345:_'])
|
|
55
|
+
expect(purged).toHaveLength(3)
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
it('NEVER touches keys for OTHER chats (multi-chat safety — the #1546 guard)', () => {
|
|
59
|
+
const purged: string[] = []
|
|
60
|
+
const r = purgeStaleTurnsForChat(
|
|
61
|
+
'12345',
|
|
62
|
+
[
|
|
63
|
+
statusKey('12345'), // target chat — purge
|
|
64
|
+
statusKey('-1001234567890'), // different chat — must NOT touch
|
|
65
|
+
statusKey('-1001234567890', 7), // different chat thread — must NOT touch
|
|
66
|
+
],
|
|
67
|
+
(k) => purged.push(k),
|
|
68
|
+
)
|
|
69
|
+
expect(r.purged).toEqual(['12345:_'])
|
|
70
|
+
expect(purged).toEqual(['12345:_']) // ONLY the target chat
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
it('does not false-match on a chatId-prefix superstring (e.g. 123 vs 1234)', () => {
|
|
74
|
+
// If we used a naive startsWith on the WHOLE key, chatId "123"
|
|
75
|
+
// would falsely match key "1234:_". The helper splits on `:` and
|
|
76
|
+
// equates the chatId slice, which prevents that.
|
|
77
|
+
const purged: string[] = []
|
|
78
|
+
const r = purgeStaleTurnsForChat(
|
|
79
|
+
'123',
|
|
80
|
+
[statusKey('123'), statusKey('1234'), statusKey('12')],
|
|
81
|
+
(k) => purged.push(k),
|
|
82
|
+
)
|
|
83
|
+
expect(r.purged).toEqual(['123:_']) // ONLY the exact match
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
it('skips malformed keys (no `:`)', () => {
|
|
87
|
+
let calls = 0
|
|
88
|
+
const r = purgeStaleTurnsForChat(
|
|
89
|
+
'123',
|
|
90
|
+
['malformed', '', 'no-colon-here', statusKey('123')],
|
|
91
|
+
() => calls++,
|
|
92
|
+
)
|
|
93
|
+
expect(r.purged).toEqual(['123:_'])
|
|
94
|
+
expect(calls).toBe(1)
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
it('accepts any iterable (snapshots before iteration — purger may delete)', () => {
|
|
98
|
+
// The real call site iterates `activeTurnStartedAt.keys()` and
|
|
99
|
+
// the purger deletes from that same Map. Snapshotting in the
|
|
100
|
+
// helper prevents iterator skew.
|
|
101
|
+
const map = new Map<string, number>()
|
|
102
|
+
map.set('123:_', 1)
|
|
103
|
+
map.set('123:7', 2)
|
|
104
|
+
map.set('999:_', 3)
|
|
105
|
+
const r = purgeStaleTurnsForChat('123', map.keys(), (k) => map.delete(k))
|
|
106
|
+
expect(r.purged.sort()).toEqual(['123:7', '123:_'])
|
|
107
|
+
expect([...map.keys()]).toEqual(['999:_']) // multi-chat safety preserved
|
|
108
|
+
})
|
|
109
|
+
})
|