switchroom 0.19.30 → 0.19.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +1121 -584
- package/dist/host-control/main.js +151 -73
- package/package.json +1 -1
- package/profiles/_base/cron-session.sh.hbs +5 -1
- package/profiles/_base/start.sh.hbs +15 -1
- package/telegram-plugin/dist/bridge/bridge.js +3 -0
- package/telegram-plugin/dist/gateway/gateway.js +1660 -677
- package/telegram-plugin/dist/server.js +3 -0
- package/telegram-plugin/edit-flood-fuse.ts +70 -20
- package/telegram-plugin/gateway/boot-beacon.ts +364 -0
- package/telegram-plugin/gateway/boot-sweep-gate.ts +20 -15
- package/telegram-plugin/gateway/gateway.ts +87 -88
- package/telegram-plugin/gateway/inbound-spool.ts +39 -0
- package/telegram-plugin/gateway/narrative-lane.ts +12 -0
- package/telegram-plugin/gateway/obligation-store.ts +28 -0
- package/telegram-plugin/gateway/stale-pin-sweep-store.ts +221 -0
- package/telegram-plugin/gateway/stale-pin-sweep-wiring.ts +211 -0
- package/telegram-plugin/gateway/stale-pin-sweep.test.ts +804 -0
- package/telegram-plugin/gateway/stale-pin-sweep.ts +1146 -0
- package/telegram-plugin/gateway/status-pin-retarget.ts +15 -2
- package/telegram-plugin/gateway/status-pin-store.ts +33 -11
- package/telegram-plugin/gateway/stream-render.ts +578 -321
- package/telegram-plugin/registry/turns-schema.ts +21 -1
- package/telegram-plugin/retry-api-call.ts +46 -21
- package/telegram-plugin/session-tail.ts +13 -0
- package/telegram-plugin/shared/bot-runtime.ts +61 -17
- package/telegram-plugin/shared/gw-trace-gate.ts +18 -2
- package/telegram-plugin/tests/activity-card-wiring.test.ts +7 -7
- package/telegram-plugin/tests/activity-drain-fuse-drop-not-failure.test.ts +324 -0
- package/telegram-plugin/tests/agent-card-result-footer.test.ts +193 -0
- package/telegram-plugin/tests/boot-beacon.test.ts +462 -0
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +6 -6
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +42 -31
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool-progress.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +134 -5
- package/telegram-plugin/tests/narrative-lane-golden.test.ts +28 -0
- package/telegram-plugin/tests/obligation-determinism.test.ts +2 -0
- package/telegram-plugin/tests/obligation-store.test.ts +67 -1
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +26 -5
- package/telegram-plugin/tests/stream-render-golden.test.ts +20 -5
- package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +161 -0
- package/telegram-plugin/tests/turn-mint-defers-until-dequeue.test.ts +196 -0
- package/telegram-plugin/tests/turn-mint-harness.ts +155 -0
- package/telegram-plugin/tests/turn-supersede-finalizes-prior-card.test.ts +124 -0
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +30 -0
- package/telegram-plugin/tool-activity-summary.ts +104 -38
- package/telegram-plugin/worker-activity-feed.ts +33 -16
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +0 -251
- package/telegram-plugin/gateway/dm-pin-sweep.ts +0 -178
|
@@ -19,17 +19,17 @@ import { resolve } from 'node:path'
|
|
|
19
19
|
const gatewaySrc = readFileSync(resolve(__dirname, '..', 'gateway', 'gateway.ts'), 'utf-8')
|
|
20
20
|
|
|
21
21
|
describe('boot pin sweep wiring (#3664)', () => {
|
|
22
|
-
it('never dispatches
|
|
22
|
+
it('never dispatches runBootPinCleanupAndStalePinSweep() directly — only through the gate', () => {
|
|
23
23
|
// The original bug shape, verbatim: a fire-and-forget call at module-eval
|
|
24
24
|
// time. Any direct invocation (voided or awaited) bypasses the bot-ready
|
|
25
25
|
// half of the precondition.
|
|
26
26
|
const directCalls =
|
|
27
|
-
gatewaySrc.match(/(?<!function\s)(?<![.\w])
|
|
27
|
+
gatewaySrc.match(/(?<!function\s)(?<![.\w])runBootPinCleanupAndStalePinSweep\s*\(/g) ?? []
|
|
28
28
|
expect(directCalls).toEqual([])
|
|
29
29
|
})
|
|
30
30
|
|
|
31
31
|
it('passes the sweep to the gate as a reference, and arms it at both mutex sites', () => {
|
|
32
|
-
expect(gatewaySrc).toContain('createBootSweepGate({ run:
|
|
32
|
+
expect(gatewaySrc).toContain('createBootSweepGate({ run: runBootPinCleanupAndStalePinSweep')
|
|
33
33
|
// Both startup-lock outcomes (mutex acquired, and the non-atomic
|
|
34
34
|
// writePidFile fallback) arm — neither may dispatch on its own.
|
|
35
35
|
const armSites = gatewaySrc.match(/bootPinSweepGate\.arm\(\)/g) ?? []
|
|
@@ -63,15 +63,15 @@ describe('boot pin sweep wiring (#3664)', () => {
|
|
|
63
63
|
expect(inlineAwaits).toEqual([])
|
|
64
64
|
}
|
|
65
65
|
// The DM-eligibility flip is what a throw used to strand, so it must live
|
|
66
|
-
// INSIDE the
|
|
66
|
+
// INSIDE the enableSweep step callback — not as a statement sequenced
|
|
67
67
|
// after the reapers, where one rejection skips it for the whole session.
|
|
68
68
|
const lines = gatewaySrc.split('\n')
|
|
69
69
|
const flipIdxs = lines
|
|
70
70
|
.map((l, i) => ({ l, i }))
|
|
71
|
-
.filter(({ l }) => /^\s*
|
|
71
|
+
.filter(({ l }) => /^\s*stalePinSweepEligible = true\s*$/.test(l))
|
|
72
72
|
.map(({ i }) => i)
|
|
73
73
|
expect(flipIdxs.length).toBe(1)
|
|
74
|
-
expect(lines[flipIdxs[0] - 1]).toContain('
|
|
74
|
+
expect(lines[flipIdxs[0] - 1]).toContain('enableSweep: () => {')
|
|
75
75
|
})
|
|
76
76
|
|
|
77
77
|
it('routes the pin API through the asserting seam, never a raw lockedBot.api pin', () => {
|
|
@@ -133,23 +133,23 @@ describe('createBootSweepGate (#3664 Defect A)', () => {
|
|
|
133
133
|
* Per-step isolation (#3664 salvage S2).
|
|
134
134
|
*
|
|
135
135
|
* The steps used to be a bare sequential `await` chain in gateway.ts. A throw
|
|
136
|
-
* from either card reaper skipped `
|
|
137
|
-
* the
|
|
138
|
-
* disabled
|
|
136
|
+
* from either card reaper skipped `stalePinSweepEligible = true`, which
|
|
137
|
+
* authorises the stale-pin drain for the WHOLE SESSION — so one throwing reaper
|
|
138
|
+
* silently disabled stale-pin cleanup, including every later lazy first-inbound
|
|
139
139
|
* sweep. That became newly reachable the moment #3664 made the sweep actually
|
|
140
140
|
* run for the first time.
|
|
141
141
|
*
|
|
142
|
-
* The outcome asserted is "was the
|
|
143
|
-
* swept", not "was a wrapper called".
|
|
142
|
+
* The outcome asserted is "was the drain enabled and were the recorded
|
|
143
|
+
* (chat, thread) targets swept", not "was a wrapper called".
|
|
144
144
|
*/
|
|
145
145
|
describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
146
146
|
function deps(over: Partial<BootPinSweepSteps> = {}) {
|
|
147
147
|
const order: string[] = []
|
|
148
148
|
const logs: string[] = []
|
|
149
149
|
const base: BootPinSweepSteps = {
|
|
150
|
-
|
|
150
|
+
scanSweepTargets: () => {
|
|
151
151
|
order.push('scan')
|
|
152
|
-
return ['
|
|
152
|
+
return [{ chatId: '900000001' }]
|
|
153
153
|
},
|
|
154
154
|
statusPinCleanup: async () => {
|
|
155
155
|
order.push('status-pins')
|
|
@@ -160,9 +160,9 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
160
160
|
queuedCardReaper: async () => {
|
|
161
161
|
order.push('queued-cards')
|
|
162
162
|
},
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
order.push(`
|
|
163
|
+
enableSweep: () => order.push('enable-sweep'),
|
|
164
|
+
sweepTarget: async (t) => {
|
|
165
|
+
order.push(`sweep:${t.chatId}:${t.threadId ?? '-'}`)
|
|
166
166
|
},
|
|
167
167
|
log: (l) => logs.push(l),
|
|
168
168
|
}
|
|
@@ -177,8 +177,8 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
177
177
|
'status-pins',
|
|
178
178
|
'activity-cards',
|
|
179
179
|
'queued-cards',
|
|
180
|
-
'enable-
|
|
181
|
-
'
|
|
180
|
+
'enable-sweep',
|
|
181
|
+
'sweep:900000001:-',
|
|
182
182
|
])
|
|
183
183
|
})
|
|
184
184
|
|
|
@@ -190,7 +190,13 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
190
190
|
})
|
|
191
191
|
await runBootPinSweepSteps(d.deps)
|
|
192
192
|
// The DM path is still enabled and the recorded DM chat is still swept.
|
|
193
|
-
expect(d.order).toEqual([
|
|
193
|
+
expect(d.order).toEqual([
|
|
194
|
+
'scan',
|
|
195
|
+
'status-pins',
|
|
196
|
+
'queued-cards',
|
|
197
|
+
'enable-sweep',
|
|
198
|
+
'sweep:900000001:-',
|
|
199
|
+
])
|
|
194
200
|
expect(d.logs.join('')).toContain("step 'activity-card-reaper' failed: boom")
|
|
195
201
|
})
|
|
196
202
|
|
|
@@ -204,32 +210,37 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
204
210
|
queuedCardReaper: boom,
|
|
205
211
|
})
|
|
206
212
|
await runBootPinSweepSteps(d.deps)
|
|
207
|
-
expect(d.order).toEqual(['scan', 'enable-
|
|
213
|
+
expect(d.order).toEqual(['scan', 'enable-sweep', 'sweep:900000001:-'])
|
|
208
214
|
})
|
|
209
215
|
|
|
210
216
|
it('a failing store scan still lets the reapers and the DM enable run', async () => {
|
|
211
217
|
const d = deps({
|
|
212
|
-
|
|
218
|
+
scanSweepTargets: () => {
|
|
213
219
|
throw new Error('ENOENT')
|
|
214
220
|
},
|
|
215
221
|
})
|
|
216
222
|
await runBootPinSweepSteps(d.deps)
|
|
217
|
-
// No
|
|
218
|
-
expect(d.order).toEqual(['status-pins', 'activity-cards', 'queued-cards', 'enable-
|
|
219
|
-
expect(d.logs.join('')).toContain("step '
|
|
223
|
+
// No targets to sweep, but nothing behind the scan is stranded.
|
|
224
|
+
expect(d.order).toEqual(['status-pins', 'activity-cards', 'queued-cards', 'enable-sweep'])
|
|
225
|
+
expect(d.logs.join('')).toContain("step 'sweep-target-scan' failed: ENOENT")
|
|
220
226
|
})
|
|
221
227
|
|
|
222
|
-
it('one failing
|
|
228
|
+
it('one failing target does not block the next one, and topics stay distinct', async () => {
|
|
223
229
|
const swept: string[] = []
|
|
224
230
|
const d = deps({
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
231
|
+
scanSweepTargets: () => [
|
|
232
|
+
{ chatId: '-1009000000001', threadId: 5 },
|
|
233
|
+
{ chatId: '-1009000000001', threadId: 9 },
|
|
234
|
+
],
|
|
235
|
+
sweepTarget: async (t) => {
|
|
236
|
+
if (t.threadId === 5) throw new Error('kicked')
|
|
237
|
+
swept.push(`${t.chatId}:${t.threadId}`)
|
|
229
238
|
},
|
|
230
239
|
})
|
|
231
240
|
await runBootPinSweepSteps(d.deps)
|
|
232
|
-
expect(swept).toEqual(['
|
|
241
|
+
expect(swept).toEqual(['-1009000000001:9'])
|
|
242
|
+
// Each target gets its own named step, keyed by (chat, thread).
|
|
243
|
+
expect(d.logs.join('')).toContain("step 'stale-pin-sweep:-1009000000001:5' failed: kicked")
|
|
233
244
|
})
|
|
234
245
|
|
|
235
246
|
it('logs a non-Error throw honestly instead of "undefined"', async () => {
|
|
@@ -250,7 +261,7 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
250
261
|
const order: string[] = []
|
|
251
262
|
await expect(
|
|
252
263
|
runBootPinSweepSteps({
|
|
253
|
-
|
|
264
|
+
scanSweepTargets: () => [],
|
|
254
265
|
statusPinCleanup: async () => {
|
|
255
266
|
throw new Error('boom')
|
|
256
267
|
},
|
|
@@ -260,14 +271,14 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
260
271
|
queuedCardReaper: async () => {
|
|
261
272
|
order.push('queued-cards')
|
|
262
273
|
},
|
|
263
|
-
|
|
264
|
-
|
|
274
|
+
enableSweep: () => order.push('enable-sweep'),
|
|
275
|
+
sweepTarget: async () => {},
|
|
265
276
|
log: () => {
|
|
266
277
|
throw new Error('EPIPE')
|
|
267
278
|
},
|
|
268
279
|
}),
|
|
269
280
|
).resolves.toBeUndefined()
|
|
270
|
-
expect(order).toEqual(['activity-cards', 'queued-cards', 'enable-
|
|
281
|
+
expect(order).toEqual(['activity-cards', 'queued-cards', 'enable-sweep'])
|
|
271
282
|
})
|
|
272
283
|
|
|
273
284
|
it('never rejects — the caller fire-and-forgets it', async () => {
|
|
@@ -276,16 +287,16 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
276
287
|
}
|
|
277
288
|
await expect(
|
|
278
289
|
runBootPinSweepSteps({
|
|
279
|
-
|
|
290
|
+
scanSweepTargets: () => {
|
|
280
291
|
throw new Error('boom')
|
|
281
292
|
},
|
|
282
293
|
statusPinCleanup: boom,
|
|
283
294
|
activityCardReaper: boom,
|
|
284
295
|
queuedCardReaper: boom,
|
|
285
|
-
|
|
296
|
+
enableSweep: () => {
|
|
286
297
|
throw new Error('boom')
|
|
287
298
|
},
|
|
288
|
-
|
|
299
|
+
sweepTarget: boom,
|
|
289
300
|
log: () => {},
|
|
290
301
|
}),
|
|
291
302
|
).resolves.toBeUndefined()
|
|
@@ -245,6 +245,8 @@ describe('dispatchEffects — inbound-routing effects (PR3c wiring)', () => {
|
|
|
245
245
|
},
|
|
246
246
|
existsSync: (p: string) => files.has(p),
|
|
247
247
|
statSizeSync: (p: string) => Buffer.byteLength(files.get(p) ?? ''),
|
|
248
|
+
fsyncFileSync: () => {},
|
|
249
|
+
fsyncDirSync: () => {},
|
|
248
250
|
}
|
|
249
251
|
const spool = createInboundSpool({ path: spoolPath, fs, log: () => {} })
|
|
250
252
|
const buffer = createPendingInboundBuffer({ spool, log: () => {} })
|
|
@@ -33,20 +33,33 @@ function msg(over: Partial<InboundMessage> = {}): InboundMessage {
|
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
/** In-memory fake fs keyed by path. Models append, full rewrite, and
|
|
36
|
-
* atomic rename (so the tmp→rename compaction path is exercised).
|
|
37
|
-
|
|
36
|
+
* atomic rename (so the tmp→rename compaction path is exercised).
|
|
37
|
+
* `ops` records the mutating/durability calls in order, so tests can pin
|
|
38
|
+
* the fsync ORDERING the crash-durability guarantee depends on. */
|
|
39
|
+
function fakeFs(): InboundSpoolFsSeam & { dump(p?: string): string; ops: string[] } {
|
|
38
40
|
const files = new Map<string, string>()
|
|
41
|
+
const ops: string[] = []
|
|
39
42
|
return {
|
|
40
|
-
appendFileSync: (p, d) =>
|
|
43
|
+
appendFileSync: (p, d) => {
|
|
44
|
+
ops.push(`append:${p}`)
|
|
45
|
+
files.set(p, (files.get(p) ?? '') + d)
|
|
46
|
+
},
|
|
41
47
|
readFileSync: (p) => files.get(p) ?? '',
|
|
42
|
-
writeFileSync: (p, d) =>
|
|
48
|
+
writeFileSync: (p, d) => {
|
|
49
|
+
ops.push(`write:${p}`)
|
|
50
|
+
files.set(p, d)
|
|
51
|
+
},
|
|
43
52
|
renameSync: (from, to) => {
|
|
53
|
+
ops.push(`rename:${from}->${to}`)
|
|
44
54
|
files.set(to, files.get(from) ?? '')
|
|
45
55
|
files.delete(from)
|
|
46
56
|
},
|
|
47
57
|
existsSync: (p) => files.has(p),
|
|
48
58
|
statSizeSync: (p) => Buffer.byteLength(files.get(p) ?? ''),
|
|
59
|
+
fsyncFileSync: (p) => ops.push(`fsyncFile:${p}`),
|
|
60
|
+
fsyncDirSync: (p) => ops.push(`fsyncDir:${p}`),
|
|
49
61
|
dump: (p = PATH) => files.get(p) ?? '',
|
|
62
|
+
ops,
|
|
50
63
|
}
|
|
51
64
|
}
|
|
52
65
|
|
|
@@ -478,6 +491,92 @@ describe('inbound-spool — robustness', () => {
|
|
|
478
491
|
})
|
|
479
492
|
})
|
|
480
493
|
|
|
494
|
+
describe('inbound-spool — power-cut durability (fsync)', () => {
|
|
495
|
+
it('fsyncs the spool file on every appended record', () => {
|
|
496
|
+
const fs = fakeFs()
|
|
497
|
+
const s = createInboundSpool({ path: PATH, fs, log: () => {} })
|
|
498
|
+
s.put('a', msg({ messageId: 1 }))
|
|
499
|
+
s.ack(msg({ messageId: 1 }))
|
|
500
|
+
// Both the put record and the ack tombstone are durability-critical:
|
|
501
|
+
// an un-synced ack replays an already-answered message after a power
|
|
502
|
+
// cut, an un-synced put loses it entirely.
|
|
503
|
+
expect(fs.ops).toEqual([
|
|
504
|
+
`append:${PATH}`,
|
|
505
|
+
`fsyncFile:${PATH}`,
|
|
506
|
+
`append:${PATH}`,
|
|
507
|
+
`fsyncFile:${PATH}`,
|
|
508
|
+
])
|
|
509
|
+
})
|
|
510
|
+
|
|
511
|
+
it('compaction fsyncs the tmp file BEFORE the rename and the directory AFTER', () => {
|
|
512
|
+
const fs = fakeFs()
|
|
513
|
+
const s = createInboundSpool({ path: PATH, fs, compactAtBytes: 200, log: () => {} })
|
|
514
|
+
for (let i = 1; i <= 10; i++) s.put('a', msg({ messageId: i, text: 'y'.repeat(50) }))
|
|
515
|
+
// The bound triggers several compactions across the 10 puts; each one
|
|
516
|
+
// must follow the same four-step sequence, so pin the last.
|
|
517
|
+
expect(fs.ops.slice(-4)).toEqual([
|
|
518
|
+
`write:${PATH}.compact.tmp`,
|
|
519
|
+
`fsyncFile:${PATH}.compact.tmp`,
|
|
520
|
+
`rename:${PATH}.compact.tmp->${PATH}`,
|
|
521
|
+
'fsyncDir:/state/agent/telegram',
|
|
522
|
+
])
|
|
523
|
+
})
|
|
524
|
+
|
|
525
|
+
it('a failed compaction directory fsync is not reported as a failed compaction', () => {
|
|
526
|
+
// The rename already landed, so the compacted log IS on disk; only the
|
|
527
|
+
// power-cut upgrade is missing. Calling it "compact FAILED" would send an
|
|
528
|
+
// operator hunting a rewrite that actually worked.
|
|
529
|
+
const fs = fakeFs()
|
|
530
|
+
const logs: string[] = []
|
|
531
|
+
fs.fsyncDirSync = () => {
|
|
532
|
+
throw new Error('EIO: i/o error, fsync dir')
|
|
533
|
+
}
|
|
534
|
+
const s = createInboundSpool({ path: PATH, fs, compactAtBytes: 200, log: (l) => logs.push(l) })
|
|
535
|
+
for (let i = 1; i <= 10; i++) s.put('a', msg({ messageId: i, text: 'y'.repeat(50) }))
|
|
536
|
+
expect(logs.join('')).toContain('compact directory fsync FAILED')
|
|
537
|
+
expect(logs.join('')).toContain('compacted path=')
|
|
538
|
+
// And the compacted log is intact and readable — nothing was rolled back.
|
|
539
|
+
const s2 = createInboundSpool({ path: PATH, fs, log: () => {} })
|
|
540
|
+
expect(s2.liveCount()).toBe(10)
|
|
541
|
+
})
|
|
542
|
+
|
|
543
|
+
it('an un-acked entry survives a POWER CUT, not just a process kill', () => {
|
|
544
|
+
// Models the distinction the fsync exists for: `appendFileSync` returning
|
|
545
|
+
// puts the bytes in the page cache (enough to survive SIGKILL — the
|
|
546
|
+
// process dies, the kernel still holds them), but only fsync puts them on
|
|
547
|
+
// stable storage. `powerCut()` discards everything not fsync'd, which is
|
|
548
|
+
// what a host power loss actually does.
|
|
549
|
+
const cache = new Map<string, string>()
|
|
550
|
+
const platter = new Map<string, string>()
|
|
551
|
+
const fs: InboundSpoolFsSeam = {
|
|
552
|
+
appendFileSync: (p, d) => cache.set(p, (cache.get(p) ?? '') + d),
|
|
553
|
+
readFileSync: (p) => cache.get(p) ?? '',
|
|
554
|
+
writeFileSync: (p, d) => cache.set(p, d),
|
|
555
|
+
renameSync: (from, to) => {
|
|
556
|
+
cache.set(to, cache.get(from) ?? '')
|
|
557
|
+
cache.delete(from)
|
|
558
|
+
},
|
|
559
|
+
existsSync: (p) => cache.has(p),
|
|
560
|
+
statSizeSync: (p) => Buffer.byteLength(cache.get(p) ?? ''),
|
|
561
|
+
fsyncFileSync: (p) => {
|
|
562
|
+
if (cache.has(p)) platter.set(p, cache.get(p)!)
|
|
563
|
+
},
|
|
564
|
+
fsyncDirSync: () => {},
|
|
565
|
+
}
|
|
566
|
+
const powerCut = (): void => {
|
|
567
|
+
cache.clear()
|
|
568
|
+
for (const [p, d] of platter) cache.set(p, d)
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const s = createInboundSpool({ path: PATH, fs, log: () => {} })
|
|
572
|
+
s.put('a', msg({ messageId: 7, text: 'answer me' }))
|
|
573
|
+
powerCut()
|
|
574
|
+
|
|
575
|
+
const rebooted = createInboundSpool({ path: PATH, fs, log: () => {} })
|
|
576
|
+
expect(rebooted.liveEntries().map((e) => e.msg.messageId)).toEqual([7])
|
|
577
|
+
})
|
|
578
|
+
})
|
|
579
|
+
|
|
481
580
|
describe('inbound-spool — #2789 C: multi-message drop notice reports the real count', () => {
|
|
482
581
|
it('passes the true per-chat dropped count to the coalesced notice', () => {
|
|
483
582
|
const fs = fakeFs()
|
|
@@ -549,10 +648,14 @@ describe('inbound-spool — #2789 B: spool write failure surfaces a health signa
|
|
|
549
648
|
// An fs whose appendFileSync can be toggled to fail, modelling a full /
|
|
550
649
|
// unwritable persistent volume. The append swallows the error (delivery
|
|
551
650
|
// must not break) but the degradation must be SURFACED, not silent.
|
|
552
|
-
function toggleableFs(): InboundSpoolFsSeam & { fail: boolean } {
|
|
651
|
+
function toggleableFs(): InboundSpoolFsSeam & { fail: boolean; failFsync: boolean } {
|
|
553
652
|
const files = new Map<string, string>()
|
|
554
653
|
const seam = {
|
|
555
654
|
fail: false,
|
|
655
|
+
// A failing fsync is its own degradation mode: the bytes reached the
|
|
656
|
+
// page cache but NOT stable storage, so the durable promise is gone
|
|
657
|
+
// even though appendFileSync returned cleanly.
|
|
658
|
+
failFsync: false,
|
|
556
659
|
appendFileSync(p: string, d: string) {
|
|
557
660
|
if (seam.fail) throw new Error('ENOSPC: no space left on device')
|
|
558
661
|
files.set(p, (files.get(p) ?? '') + d)
|
|
@@ -565,6 +668,10 @@ describe('inbound-spool — #2789 B: spool write failure surfaces a health signa
|
|
|
565
668
|
},
|
|
566
669
|
existsSync: (p: string) => files.has(p),
|
|
567
670
|
statSizeSync: (p: string) => Buffer.byteLength(files.get(p) ?? ''),
|
|
671
|
+
fsyncFileSync: (_p: string) => {
|
|
672
|
+
if (seam.failFsync) throw new Error('EIO: i/o error, fsync')
|
|
673
|
+
},
|
|
674
|
+
fsyncDirSync: (_p: string) => {},
|
|
568
675
|
}
|
|
569
676
|
return seam
|
|
570
677
|
}
|
|
@@ -588,6 +695,28 @@ describe('inbound-spool — #2789 B: spool write failure surfaces a health signa
|
|
|
588
695
|
expect(events).toEqual([{ degraded: true, consecutiveFailures: 1 }])
|
|
589
696
|
})
|
|
590
697
|
|
|
698
|
+
it('a failing fsync degrades too — a cached-but-unsynced append is NOT durable', () => {
|
|
699
|
+
// The subtle one: appendFileSync succeeds, so the old code would have
|
|
700
|
+
// reported a healthy spool while the record was one power cut from gone.
|
|
701
|
+
const fs = toggleableFs()
|
|
702
|
+
const events: { degraded: boolean }[] = []
|
|
703
|
+
const logs: string[] = []
|
|
704
|
+
const s = createInboundSpool({
|
|
705
|
+
path: PATH,
|
|
706
|
+
fs,
|
|
707
|
+
log: (l) => logs.push(l),
|
|
708
|
+
onDegraded: (info) => events.push({ degraded: info.degraded }),
|
|
709
|
+
})
|
|
710
|
+
fs.failFsync = true
|
|
711
|
+
expect(() => s.put('a', msg({ messageId: 1, ts: 1 }))).not.toThrow()
|
|
712
|
+
expect(s.isDegraded()).toBe(true)
|
|
713
|
+
expect(events).toEqual([{ degraded: true }])
|
|
714
|
+
expect(logs.join('')).toContain('durability degraded')
|
|
715
|
+
// Live delivery is unaffected — degradation is a durability claim, not a
|
|
716
|
+
// delivery failure.
|
|
717
|
+
expect(s.liveCount()).toBe(1)
|
|
718
|
+
})
|
|
719
|
+
|
|
591
720
|
it('does not re-fire the degraded signal on every failing append (latched)', () => {
|
|
592
721
|
const fs = toggleableFs()
|
|
593
722
|
const events: boolean[] = []
|
|
@@ -211,6 +211,34 @@ describe('narrative-lane golden — SHOW path opens then edits ONE feed card', (
|
|
|
211
211
|
expect(final!).toMatch(/3 steps/)
|
|
212
212
|
expect(live!).not.toMatch(/3 steps/)
|
|
213
213
|
})
|
|
214
|
+
|
|
215
|
+
it('composeTurnActivity: the FINAL render ends with the ✅ + delivered-answer footer', () => {
|
|
216
|
+
// Parity with the 🛠 worker card, which has always closed on `✅ <summary>`.
|
|
217
|
+
// The agent card's analogue of the worker's `latestSummary` is the answer
|
|
218
|
+
// the turn actually delivered (`turn.lastReplyText`); the lane threads it
|
|
219
|
+
// into the header as `resultText` and the SHARED `deriveCardResult` renders
|
|
220
|
+
// it. Fails on the pre-fix lane, which passed no result text at all.
|
|
221
|
+
const { lane } = makeLane()
|
|
222
|
+
const turn = makeLaneTurn(lane, { labeledToolCount: 2 })
|
|
223
|
+
turn.mirrorLines.push('Read the file', 'Ran the tests')
|
|
224
|
+
turn.lastReplyText = 'Standing by for the background merge waiter to complete.'
|
|
225
|
+
const live = lane.composeTurnActivity(turn)
|
|
226
|
+
const final = lane.composeTurnActivity(turn, true)
|
|
227
|
+
expect(final!).toContain('─────')
|
|
228
|
+
expect(final!.endsWith('✅ _Standing by for the background merge waiter to complete._')).toBe(true)
|
|
229
|
+
// The LIVE render must not leak the footer — the turn is not finished.
|
|
230
|
+
expect(live!).not.toContain('✅')
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
it('composeTurnActivity: a turn that delivered no answer renders NO footer (no fabrication)', () => {
|
|
234
|
+
const { lane } = makeLane()
|
|
235
|
+
const turn = makeLaneTurn(lane, { labeledToolCount: 2 })
|
|
236
|
+
turn.mirrorLines.push('Read the file', 'Ran the tests')
|
|
237
|
+
turn.lastReplyText = ''
|
|
238
|
+
const final = lane.composeTurnActivity(turn, true)
|
|
239
|
+
expect(final!).not.toContain('─────')
|
|
240
|
+
expect(final!).not.toContain('✅')
|
|
241
|
+
})
|
|
214
242
|
})
|
|
215
243
|
|
|
216
244
|
describe('narrative-lane golden — the narrative-dedup gate at turn end', () => {
|
|
@@ -27,6 +27,12 @@ function memFs(seed: Record<string, string> = {}) {
|
|
|
27
27
|
files.delete(a);
|
|
28
28
|
},
|
|
29
29
|
existsSync: (p) => files.has(p),
|
|
30
|
+
fsyncFileSync: (p) => {
|
|
31
|
+
calls.push(`fsyncFile:${p}`);
|
|
32
|
+
},
|
|
33
|
+
fsyncDirSync: (p) => {
|
|
34
|
+
calls.push(`fsyncDir:${p}`);
|
|
35
|
+
},
|
|
30
36
|
};
|
|
31
37
|
return { fs, files, calls };
|
|
32
38
|
}
|
|
@@ -63,7 +69,12 @@ describe("obligation-store", () => {
|
|
|
63
69
|
it("persists atomically: writes a sibling .tmp then renames over the path", () => {
|
|
64
70
|
const { fs, calls, files } = memFs();
|
|
65
71
|
persistObligations(PATH, fs, [ob("c:3#1")]);
|
|
66
|
-
expect(calls).toEqual([
|
|
72
|
+
expect(calls).toEqual([
|
|
73
|
+
`write:${PATH}.tmp`,
|
|
74
|
+
`fsyncFile:${PATH}.tmp`,
|
|
75
|
+
`rename:${PATH}.tmp->${PATH}`,
|
|
76
|
+
"fsyncDir:/state/agent/telegram",
|
|
77
|
+
]);
|
|
67
78
|
// The tmp is gone (renamed); only the real path remains.
|
|
68
79
|
expect(files.has(PATH)).toBe(true);
|
|
69
80
|
expect(files.has(`${PATH}.tmp`)).toBe(false);
|
|
@@ -170,8 +181,63 @@ describe("obligation-store", () => {
|
|
|
170
181
|
},
|
|
171
182
|
renameSync: () => {},
|
|
172
183
|
existsSync: () => false,
|
|
184
|
+
fsyncFileSync: () => {},
|
|
185
|
+
fsyncDirSync: () => {},
|
|
173
186
|
};
|
|
174
187
|
expect(() => persistObligations(PATH, fs, [ob("c:3#1")], (l) => logs.push(l))).not.toThrow();
|
|
175
188
|
expect(logs.join("")).toContain("persist FAILED");
|
|
176
189
|
});
|
|
177
190
|
});
|
|
191
|
+
|
|
192
|
+
describe("obligation-store — power-cut durability (fsync ordering)", () => {
|
|
193
|
+
it("fsyncs the tmp file BEFORE the rename and the directory AFTER it", () => {
|
|
194
|
+
const { fs, calls } = memFs();
|
|
195
|
+
persistObligations(PATH, fs, [ob("c:3#1")]);
|
|
196
|
+
// The exact sequence IS the guarantee: syncing the directory before the
|
|
197
|
+
// rename, or renaming before the tmp file's data is on stable storage,
|
|
198
|
+
// publishes a name that can point at bytes that were never written.
|
|
199
|
+
expect(calls).toEqual([
|
|
200
|
+
`write:${PATH}.tmp`,
|
|
201
|
+
`fsyncFile:${PATH}.tmp`,
|
|
202
|
+
`rename:${PATH}.tmp->${PATH}`,
|
|
203
|
+
"fsyncDir:/state/agent/telegram",
|
|
204
|
+
]);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("a failed tmp fsync aborts BEFORE the rename — the previous snapshot is left intact", () => {
|
|
208
|
+
// Publishing an unsynced tmp over a good snapshot would be strictly worse
|
|
209
|
+
// than not writing at all, so the rename must not happen.
|
|
210
|
+
const prior = JSON.stringify({ v: 1, obligations: [ob("c:3#1")] });
|
|
211
|
+
const { fs, files } = memFs({ [PATH]: prior });
|
|
212
|
+
const logs: string[] = [];
|
|
213
|
+
const failing: ObligationStoreFsSeam = {
|
|
214
|
+
...fs,
|
|
215
|
+
fsyncFileSync: () => {
|
|
216
|
+
throw new Error("EIO: i/o error, fsync");
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
expect(() => persistObligations(PATH, failing, [ob("c:3#2")], (l) => logs.push(l))).not.toThrow();
|
|
220
|
+
expect(logs.join("")).toContain("persist FAILED");
|
|
221
|
+
expect(files.get(PATH)).toBe(prior);
|
|
222
|
+
expect(loadObligations(PATH, fs)[0]!.originTurnId).toBe("c:3#1");
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it("a failed DIRECTORY fsync keeps the written snapshot and is not reported as a failed persist", () => {
|
|
226
|
+
// The rename already landed, so the snapshot is correct for a process
|
|
227
|
+
// crash; only the power-cut upgrade is missing. Mislabelling it
|
|
228
|
+
// "persist FAILED" would send an operator hunting a write that worked.
|
|
229
|
+
const { fs, files } = memFs();
|
|
230
|
+
const logs: string[] = [];
|
|
231
|
+
const failing: ObligationStoreFsSeam = {
|
|
232
|
+
...fs,
|
|
233
|
+
fsyncDirSync: () => {
|
|
234
|
+
throw new Error("EIO: i/o error, fsync dir");
|
|
235
|
+
},
|
|
236
|
+
};
|
|
237
|
+
expect(() => persistObligations(PATH, failing, [ob("c:3#1")], (l) => logs.push(l))).not.toThrow();
|
|
238
|
+
expect(logs.join("")).toContain("directory fsync FAILED");
|
|
239
|
+
expect(logs.join("")).not.toContain("persist FAILED");
|
|
240
|
+
expect(files.has(PATH)).toBe(true);
|
|
241
|
+
expect(loadObligations(PATH, fs)).toHaveLength(1);
|
|
242
|
+
});
|
|
243
|
+
});
|
|
@@ -311,7 +311,7 @@ describe("status-pin boot cleanup is mutex-gated (structural)", () => {
|
|
|
311
311
|
|
|
312
312
|
it("every boot pin-cleanup call site follows the won-lock check", () => {
|
|
313
313
|
// #3026: boot cleanup is now invoked via the mutex-gated orchestrator
|
|
314
|
-
//
|
|
314
|
+
// runBootPinCleanupAndStalePinSweep(), which awaits statusPinBootCleanup()
|
|
315
315
|
// (+ the other reapers) and then runs the DM stale-pin sweep. The
|
|
316
316
|
// invariant is unchanged: cleanup must only run after the lock is won.
|
|
317
317
|
const lines = gatewaySrc.split("\n");
|
|
@@ -344,7 +344,7 @@ describe("status-pin boot cleanup is mutex-gated (structural)", () => {
|
|
|
344
344
|
);
|
|
345
345
|
expect(declIdx).toBeGreaterThan(-1);
|
|
346
346
|
const orchestratorIdx = lines.findIndex((l) =>
|
|
347
|
-
/function
|
|
347
|
+
/function runBootPinCleanupAndStalePinSweep/.test(l),
|
|
348
348
|
);
|
|
349
349
|
expect(orchestratorIdx).toBeGreaterThan(-1);
|
|
350
350
|
const invocationIdxs = lines
|
|
@@ -365,7 +365,7 @@ describe("status-pin boot cleanup is mutex-gated (structural)", () => {
|
|
|
365
365
|
|
|
366
366
|
it("the blocked (losing) boot exits before reaching cleanup", () => {
|
|
367
367
|
// In the mutex block, `outcome.status === 'blocked'` must lead to
|
|
368
|
-
// process.exit(1) BEFORE the winner's
|
|
368
|
+
// process.exit(1) BEFORE the winner's runBootPinCleanupAndStalePinSweep() call
|
|
369
369
|
// in that block — so a loser never touches the shared store.
|
|
370
370
|
const blockedIdx = gatewaySrc.indexOf("outcome.status === 'blocked'");
|
|
371
371
|
expect(blockedIdx).toBeGreaterThan(-1);
|
|
@@ -117,8 +117,8 @@ describe("status-pin-store", () => {
|
|
|
117
117
|
});
|
|
118
118
|
|
|
119
119
|
it("fail-open: wrong envelope version / shape loads as []", () => {
|
|
120
|
-
//
|
|
121
|
-
const { fs: f1 } = memFs({ [PATH]: JSON.stringify({ v:
|
|
120
|
+
// v5 is an unknown FUTURE version (v1–v4 are the supported set).
|
|
121
|
+
const { fs: f1 } = memFs({ [PATH]: JSON.stringify({ v: 5, pins: [pin()] }) });
|
|
122
122
|
expect(loadStatusPins(PATH, f1)).toEqual([]);
|
|
123
123
|
const { fs: f2 } = memFs({ [PATH]: JSON.stringify({ v: 1, pins: "nope" }) });
|
|
124
124
|
expect(loadStatusPins(PATH, f2)).toEqual([]);
|
|
@@ -548,13 +548,34 @@ describe("status-pin-store — envelope version compat", () => {
|
|
|
548
548
|
expect(loaded[0].messageId).toBe(715);
|
|
549
549
|
});
|
|
550
550
|
|
|
551
|
-
it("round-trips a v2 pending flag and writes
|
|
551
|
+
it("round-trips a v2 pending flag and writes v4 envelopes", () => {
|
|
552
552
|
const { fs, files } = memFs();
|
|
553
553
|
persistStatusPins(PATH, fs, [pin({ pending: true })]);
|
|
554
|
-
expect(JSON.parse(files.get(PATH)!).v).toBe(
|
|
554
|
+
expect(JSON.parse(files.get(PATH)!).v).toBe(4);
|
|
555
555
|
expect(loadStatusPins(PATH, fs)[0].pending).toBe(true);
|
|
556
556
|
});
|
|
557
557
|
|
|
558
|
+
it("v4 persists the forum topic a pin lives in, so a later sweep can aim at it", () => {
|
|
559
|
+
// Telegram's pin stack is chat-wide and `unpinChatMessage` takes no
|
|
560
|
+
// message_thread_id, so `unpinAllForumTopicMessages(chat, thread)` is the
|
|
561
|
+
// only topic-scoped drain a bot has — and it needs this field. Without it a
|
|
562
|
+
// boot after a crash knows a forum chat has orphans but not where.
|
|
563
|
+
const { fs, files } = memFs();
|
|
564
|
+
persistStatusPins(PATH, fs, [pin({ threadId: 77 })]);
|
|
565
|
+
expect(JSON.parse(files.get(PATH)!).v).toBe(4);
|
|
566
|
+
expect(loadStatusPins(PATH, fs)[0].threadId).toBe(77);
|
|
567
|
+
});
|
|
568
|
+
|
|
569
|
+
it("drops a row with a non-numeric threadId", () => {
|
|
570
|
+
const { fs } = memFs({
|
|
571
|
+
[PATH]: JSON.stringify({
|
|
572
|
+
v: 4,
|
|
573
|
+
pins: [pin({ messageId: 715 }), { pinKey: "k", chatId: "c", messageId: 1, threadId: "5" }],
|
|
574
|
+
}),
|
|
575
|
+
});
|
|
576
|
+
expect(loadStatusPins(PATH, fs)).toHaveLength(1);
|
|
577
|
+
});
|
|
578
|
+
|
|
558
579
|
it("#3810: a v3 row round-trips its pinnedAt claim age; a v1/v2 row loads without one", () => {
|
|
559
580
|
const { fs } = memFs({
|
|
560
581
|
[PATH]: JSON.stringify({
|
|
@@ -584,7 +605,7 @@ describe("status-pin-store — envelope version compat", () => {
|
|
|
584
605
|
});
|
|
585
606
|
|
|
586
607
|
it("rejects an unknown future envelope version (fail-open [])", () => {
|
|
587
|
-
const { fs } = memFs({ [PATH]: JSON.stringify({ v:
|
|
608
|
+
const { fs } = memFs({ [PATH]: JSON.stringify({ v: 5, pins: [pin()] }) });
|
|
588
609
|
expect(idOnly(loadStatusPins(PATH, fs))).toEqual([]);
|
|
589
610
|
});
|
|
590
611
|
|