switchroom 0.19.29 → 0.19.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +2044 -900
- package/dist/host-control/main.js +162 -80
- package/package.json +1 -1
- package/profiles/_base/cron-session.sh.hbs +5 -1
- package/profiles/_base/start.sh.hbs +15 -1
- package/telegram-plugin/dist/gateway/gateway.js +1435 -551
- package/telegram-plugin/edit-flood-fuse.ts +70 -20
- package/telegram-plugin/gateway/boot-beacon.ts +364 -0
- package/telegram-plugin/gateway/boot-sweep-gate.ts +20 -15
- package/telegram-plugin/gateway/gateway.ts +87 -88
- package/telegram-plugin/gateway/inbound-spool.ts +39 -0
- package/telegram-plugin/gateway/narrative-lane.ts +12 -0
- package/telegram-plugin/gateway/obligation-store.ts +28 -0
- package/telegram-plugin/gateway/stale-pin-sweep-store.ts +221 -0
- package/telegram-plugin/gateway/stale-pin-sweep-wiring.ts +211 -0
- package/telegram-plugin/gateway/stale-pin-sweep.test.ts +804 -0
- package/telegram-plugin/gateway/stale-pin-sweep.ts +1146 -0
- package/telegram-plugin/gateway/status-pin-retarget.ts +15 -2
- package/telegram-plugin/gateway/status-pin-store.ts +33 -11
- package/telegram-plugin/registry/turns-schema.ts +21 -1
- package/telegram-plugin/retry-api-call.ts +46 -21
- package/telegram-plugin/shared/bot-runtime.ts +61 -17
- package/telegram-plugin/shared/gw-trace-gate.ts +18 -2
- package/telegram-plugin/tests/activity-card-wiring.test.ts +7 -7
- package/telegram-plugin/tests/activity-drain-fuse-drop-not-failure.test.ts +324 -0
- package/telegram-plugin/tests/agent-card-result-footer.test.ts +193 -0
- package/telegram-plugin/tests/boot-beacon.test.ts +462 -0
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +6 -6
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +42 -31
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool-progress.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +134 -5
- package/telegram-plugin/tests/narrative-lane-golden.test.ts +28 -0
- package/telegram-plugin/tests/obligation-determinism.test.ts +2 -0
- package/telegram-plugin/tests/obligation-store.test.ts +67 -1
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +26 -5
- package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +161 -0
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +30 -0
- package/telegram-plugin/tool-activity-summary.ts +104 -38
- package/telegram-plugin/worker-activity-feed.ts +33 -16
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +0 -251
- package/telegram-plugin/gateway/dm-pin-sweep.ts +0 -178
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gateway boot beacon — the write-only forensic record a later boot-cause
|
|
3
|
+
* classifier will read.
|
|
4
|
+
*
|
|
5
|
+
* Two things are pinned here, and they are the two that can silently break:
|
|
6
|
+
*
|
|
7
|
+
* 1. **Graceful degradation.** Every cgroup / `/proc` read is optional. A
|
|
8
|
+
* cgroup v1 host, a kernel with no `random/boot_id`, a truncated or
|
|
9
|
+
* garbage file — each must OMIT that one field and still produce a
|
|
10
|
+
* usable beacon. This module runs on a 5s tick inside every gateway in
|
|
11
|
+
* the fleet; a throw here would crash-loop every agent on rollout, so the
|
|
12
|
+
* malformed-input cases assert "no throw + field absent", not just a
|
|
13
|
+
* happy-path parse.
|
|
14
|
+
*
|
|
15
|
+
* 2. **The absent/unlimited distinction.** `memory.max` of the literal
|
|
16
|
+
* `max` (unlimited cgroup) must survive as `'max'`, NOT collapse to the
|
|
17
|
+
* same "field missing" shape as an unreadable file. The classifier needs
|
|
18
|
+
* to tell "a limit-triggered OOM was impossible" from "we don't know".
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { describe, it, expect } from 'vitest'
|
|
22
|
+
import { mkdtempSync, readFileSync, writeFileSync, rmSync, existsSync, statSync } from 'node:fs'
|
|
23
|
+
import { join } from 'node:path'
|
|
24
|
+
import { tmpdir } from 'node:os'
|
|
25
|
+
import {
|
|
26
|
+
attachBootBeacon,
|
|
27
|
+
BOOT_BEACON_FILE,
|
|
28
|
+
BOOT_BEACON_INTERVAL_MS,
|
|
29
|
+
buildBootBeacon,
|
|
30
|
+
parseHostBootId,
|
|
31
|
+
parseMemoryBytes,
|
|
32
|
+
parseMemoryMax,
|
|
33
|
+
parseOomKillCount,
|
|
34
|
+
sampleBeaconHost,
|
|
35
|
+
serializeBootBeacon,
|
|
36
|
+
tickBootBeacon,
|
|
37
|
+
writeBootBeaconFile,
|
|
38
|
+
type BootBeacon,
|
|
39
|
+
} from '../gateway/boot-beacon.js'
|
|
40
|
+
|
|
41
|
+
const REAL_MEMORY_EVENTS = 'low 0\nhigh 0\nmax 0\noom 0\noom_kill 7\noom_group_kill 0\n'
|
|
42
|
+
|
|
43
|
+
describe('boot-beacon parsers', () => {
|
|
44
|
+
describe('parseHostBootId', () => {
|
|
45
|
+
it('extracts the boot_id uuid from the /proc file contents', () => {
|
|
46
|
+
expect(parseHostBootId('8917e33b-05c6-4509-8b4b-7e39d98c4c7f\n'))
|
|
47
|
+
.toBe('8917e33b-05c6-4509-8b4b-7e39d98c4c7f')
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
it('returns undefined for a missing file (null read), empty, or blank contents', () => {
|
|
51
|
+
expect(parseHostBootId(null)).toBeUndefined()
|
|
52
|
+
expect(parseHostBootId(undefined)).toBeUndefined()
|
|
53
|
+
expect(parseHostBootId('')).toBeUndefined()
|
|
54
|
+
expect(parseHostBootId(' \n ')).toBeUndefined()
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
it('rejects multi-token junk rather than poisoning the host-reboot comparison', () => {
|
|
58
|
+
// A garbage boot_id is WORSE than an absent one: the classifier compares
|
|
59
|
+
// this value across boots to decide "host rebooted" vs "our process was
|
|
60
|
+
// killed". Junk that happens to differ between reads would fabricate a
|
|
61
|
+
// host reboot, so anything unexpected must degrade to absent.
|
|
62
|
+
expect(parseHostBootId('<html>not a boot id</html>')).toBeUndefined()
|
|
63
|
+
expect(parseHostBootId('abc def')).toBeUndefined()
|
|
64
|
+
expect(parseHostBootId('x'.repeat(500))).toBeUndefined()
|
|
65
|
+
})
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
describe('parseOomKillCount', () => {
|
|
69
|
+
it('reads oom_kill out of a real cgroup v2 memory.events table', () => {
|
|
70
|
+
expect(parseOomKillCount(REAL_MEMORY_EVENTS)).toBe(7)
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
it('reads oom_kill, not oom — reclaim-resolved OOM events are not kills', () => {
|
|
74
|
+
expect(parseOomKillCount('oom 42\noom_kill 0\n')).toBe(0)
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
it('tolerates oom_group_kill sharing the oom_kill prefix', () => {
|
|
78
|
+
expect(parseOomKillCount('oom_group_kill 9\noom_kill 3\n')).toBe(3)
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
it('returns undefined when the key is absent (older kernel) or the file is unreadable', () => {
|
|
82
|
+
expect(parseOomKillCount('low 0\nhigh 0\n')).toBeUndefined()
|
|
83
|
+
expect(parseOomKillCount(null)).toBeUndefined()
|
|
84
|
+
expect(parseOomKillCount('')).toBeUndefined()
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('returns undefined for a malformed value instead of NaN or a throw', () => {
|
|
88
|
+
expect(parseOomKillCount('oom_kill notanumber\n')).toBeUndefined()
|
|
89
|
+
expect(parseOomKillCount('oom_kill -1\n')).toBeUndefined()
|
|
90
|
+
expect(parseOomKillCount('oom_kill\n')).toBeUndefined()
|
|
91
|
+
expect(parseOomKillCount('oom_kill 1 2\n')).toBeUndefined()
|
|
92
|
+
// Binary garbage (a cgroup v1 host serving something else at this path).
|
|
93
|
+
expect(() => parseOomKillCount('\x00\x01\x02')).not.toThrow()
|
|
94
|
+
expect(parseOomKillCount('\x00\x01\x02')).toBeUndefined()
|
|
95
|
+
})
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
describe('parseMemoryBytes', () => {
|
|
99
|
+
it('parses memory.current', () => {
|
|
100
|
+
expect(parseMemoryBytes('2846650368\n')).toBe(2846650368)
|
|
101
|
+
expect(parseMemoryBytes('0')).toBe(0)
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
it('degrades to undefined for unreadable / non-numeric / non-integer contents', () => {
|
|
105
|
+
expect(parseMemoryBytes(null)).toBeUndefined()
|
|
106
|
+
expect(parseMemoryBytes('')).toBeUndefined()
|
|
107
|
+
expect(parseMemoryBytes('max')).toBeUndefined()
|
|
108
|
+
expect(parseMemoryBytes('12.5')).toBeUndefined()
|
|
109
|
+
expect(parseMemoryBytes('-5')).toBeUndefined()
|
|
110
|
+
// Beyond Number.MAX_SAFE_INTEGER — a lossy parse is worse than absent.
|
|
111
|
+
expect(parseMemoryBytes('99999999999999999999')).toBeUndefined()
|
|
112
|
+
})
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
describe('parseMemoryMax', () => {
|
|
116
|
+
it('parses a byte limit', () => {
|
|
117
|
+
expect(parseMemoryMax('17179869184\n')).toBe(17179869184)
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
it('preserves the literal "max" — unlimited is NOT the same as unknown', () => {
|
|
121
|
+
expect(parseMemoryMax('max\n')).toBe('max')
|
|
122
|
+
// An unreadable file omits the field; an unlimited cgroup reports 'max'.
|
|
123
|
+
expect(parseMemoryMax(null)).toBeUndefined()
|
|
124
|
+
expect(parseMemoryMax('garbage')).toBeUndefined()
|
|
125
|
+
})
|
|
126
|
+
})
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
describe('buildBootBeacon', () => {
|
|
130
|
+
it('carries every field through when the full host sample is available', () => {
|
|
131
|
+
const beacon = buildBootBeacon({
|
|
132
|
+
bootId: 'boot-1',
|
|
133
|
+
pid: 4242,
|
|
134
|
+
wallMs: 1_700_000_000_000,
|
|
135
|
+
monotonicMs: 12_345,
|
|
136
|
+
sample: {
|
|
137
|
+
hostBootId: 'host-1',
|
|
138
|
+
oomKillCount: 7,
|
|
139
|
+
memCurrent: 100,
|
|
140
|
+
memMax: 200,
|
|
141
|
+
},
|
|
142
|
+
})
|
|
143
|
+
expect(beacon).toEqual({
|
|
144
|
+
bootId: 'boot-1',
|
|
145
|
+
pid: 4242,
|
|
146
|
+
wallMs: 1_700_000_000_000,
|
|
147
|
+
monotonicMs: 12_345,
|
|
148
|
+
hostBootId: 'host-1',
|
|
149
|
+
oomKillCount: 7,
|
|
150
|
+
memCurrent: 100,
|
|
151
|
+
memMax: 200,
|
|
152
|
+
})
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
it('OMITS unavailable fields rather than emitting undefined/null keys', () => {
|
|
156
|
+
// A non-cgroup-v2 host degrades to the process-local core. "Absent" must
|
|
157
|
+
// be one unambiguous shape for the future reader, so the keys must not
|
|
158
|
+
// exist at all.
|
|
159
|
+
const beacon = buildBootBeacon({
|
|
160
|
+
bootId: 'boot-1',
|
|
161
|
+
pid: 1,
|
|
162
|
+
wallMs: 5,
|
|
163
|
+
monotonicMs: 6,
|
|
164
|
+
sample: {},
|
|
165
|
+
})
|
|
166
|
+
expect(beacon).toEqual({ bootId: 'boot-1', pid: 1, wallMs: 5, monotonicMs: 6 })
|
|
167
|
+
expect(Object.keys(beacon).sort()).toEqual(['bootId', 'monotonicMs', 'pid', 'wallMs'])
|
|
168
|
+
expect(JSON.parse(serializeBootBeacon(beacon))).not.toHaveProperty('hostBootId')
|
|
169
|
+
expect(JSON.parse(serializeBootBeacon(beacon))).not.toHaveProperty('oomKillCount')
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
it('keeps a zero oomKillCount — 0 is a real reading the classifier diffs against', () => {
|
|
173
|
+
const beacon = buildBootBeacon({
|
|
174
|
+
bootId: 'b', pid: 1, wallMs: 1, monotonicMs: 1,
|
|
175
|
+
sample: { oomKillCount: 0, memCurrent: 0 },
|
|
176
|
+
})
|
|
177
|
+
expect(beacon.oomKillCount).toBe(0)
|
|
178
|
+
expect(beacon.memCurrent).toBe(0)
|
|
179
|
+
expect(serializeBootBeacon(beacon)).toContain('"oomKillCount":0')
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
it('serialises to one JSON line with a trailing newline', () => {
|
|
183
|
+
const text = serializeBootBeacon(buildBootBeacon({
|
|
184
|
+
bootId: 'b', pid: 1, wallMs: 2, monotonicMs: 3,
|
|
185
|
+
}))
|
|
186
|
+
expect(text.endsWith('\n')).toBe(true)
|
|
187
|
+
expect(text.trimEnd().includes('\n')).toBe(false)
|
|
188
|
+
expect(JSON.parse(text)).toEqual({ bootId: 'b', pid: 1, wallMs: 2, monotonicMs: 3 })
|
|
189
|
+
})
|
|
190
|
+
})
|
|
191
|
+
|
|
192
|
+
describe('sampleBeaconHost', () => {
|
|
193
|
+
it('reads a cgroup v2 layout end to end from injected paths', () => {
|
|
194
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-cg-'))
|
|
195
|
+
try {
|
|
196
|
+
writeFileSync(join(dir, 'boot_id'), 'host-boot-abc\n')
|
|
197
|
+
writeFileSync(join(dir, 'memory.events'), REAL_MEMORY_EVENTS)
|
|
198
|
+
writeFileSync(join(dir, 'memory.current'), '2846650368\n')
|
|
199
|
+
writeFileSync(join(dir, 'memory.max'), '17179869184\n')
|
|
200
|
+
expect(sampleBeaconHost({
|
|
201
|
+
hostBootId: join(dir, 'boot_id'),
|
|
202
|
+
memoryEvents: join(dir, 'memory.events'),
|
|
203
|
+
memoryCurrent: join(dir, 'memory.current'),
|
|
204
|
+
memoryMax: join(dir, 'memory.max'),
|
|
205
|
+
})).toEqual({
|
|
206
|
+
hostBootId: 'host-boot-abc',
|
|
207
|
+
oomKillCount: 7,
|
|
208
|
+
memCurrent: 2846650368,
|
|
209
|
+
memMax: 17179869184,
|
|
210
|
+
})
|
|
211
|
+
} finally {
|
|
212
|
+
rmSync(dir, { recursive: true, force: true })
|
|
213
|
+
}
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
it('degrades to an empty sample when NOTHING is readable (cgroup v1 host)', () => {
|
|
217
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-cg1-'))
|
|
218
|
+
try {
|
|
219
|
+
const missing = {
|
|
220
|
+
hostBootId: join(dir, 'nope-boot_id'),
|
|
221
|
+
memoryEvents: join(dir, 'nope-memory.events'),
|
|
222
|
+
memoryCurrent: join(dir, 'nope-memory.current'),
|
|
223
|
+
memoryMax: join(dir, 'nope-memory.max'),
|
|
224
|
+
}
|
|
225
|
+
let sample: ReturnType<typeof sampleBeaconHost> | undefined
|
|
226
|
+
expect(() => { sample = sampleBeaconHost(missing) }).not.toThrow()
|
|
227
|
+
expect(sample).toEqual({})
|
|
228
|
+
} finally {
|
|
229
|
+
rmSync(dir, { recursive: true, force: true })
|
|
230
|
+
}
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
it('degrades PER FIELD — one unreadable file does not take the others down', () => {
|
|
234
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-partial-'))
|
|
235
|
+
try {
|
|
236
|
+
writeFileSync(join(dir, 'boot_id'), 'host-boot-abc\n')
|
|
237
|
+
// memory.events missing entirely; memory.current garbage; memory.max unlimited.
|
|
238
|
+
writeFileSync(join(dir, 'memory.current'), 'not-a-number\n')
|
|
239
|
+
writeFileSync(join(dir, 'memory.max'), 'max\n')
|
|
240
|
+
const sample = sampleBeaconHost({
|
|
241
|
+
hostBootId: join(dir, 'boot_id'),
|
|
242
|
+
memoryEvents: join(dir, 'memory.events'),
|
|
243
|
+
memoryCurrent: join(dir, 'memory.current'),
|
|
244
|
+
memoryMax: join(dir, 'memory.max'),
|
|
245
|
+
})
|
|
246
|
+
expect(sample).toEqual({ hostBootId: 'host-boot-abc', memMax: 'max' })
|
|
247
|
+
} finally {
|
|
248
|
+
rmSync(dir, { recursive: true, force: true })
|
|
249
|
+
}
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
it('does not throw when a cgroup path resolves to a DIRECTORY (EISDIR)', () => {
|
|
253
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-eisdir-'))
|
|
254
|
+
try {
|
|
255
|
+
expect(() => sampleBeaconHost({
|
|
256
|
+
hostBootId: dir,
|
|
257
|
+
memoryEvents: dir,
|
|
258
|
+
memoryCurrent: dir,
|
|
259
|
+
memoryMax: dir,
|
|
260
|
+
})).not.toThrow()
|
|
261
|
+
expect(sampleBeaconHost({ hostBootId: dir, memoryEvents: dir, memoryCurrent: dir, memoryMax: dir }))
|
|
262
|
+
.toEqual({})
|
|
263
|
+
} finally {
|
|
264
|
+
rmSync(dir, { recursive: true, force: true })
|
|
265
|
+
}
|
|
266
|
+
})
|
|
267
|
+
})
|
|
268
|
+
|
|
269
|
+
describe('writeBootBeaconFile', () => {
|
|
270
|
+
const beacon: BootBeacon = {
|
|
271
|
+
bootId: 'boot-1', pid: 99, wallMs: 1234, monotonicMs: 56, hostBootId: 'host-1',
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
it('writes the beacon under the state dir, creating the dir 0700 if needed', () => {
|
|
275
|
+
const root = mkdtempSync(join(tmpdir(), 'beacon-w-'))
|
|
276
|
+
try {
|
|
277
|
+
const stateDir = join(root, 'nested', 'state')
|
|
278
|
+
expect(writeBootBeaconFile(stateDir, beacon)).toBe(true)
|
|
279
|
+
const path = join(stateDir, BOOT_BEACON_FILE)
|
|
280
|
+
expect(JSON.parse(readFileSync(path, 'utf-8'))).toEqual(beacon)
|
|
281
|
+
// STATE_DIR also holds access.json / .env. A 5s tick that recreated it
|
|
282
|
+
// at a default mode would loosen permissions on all of that.
|
|
283
|
+
expect(statSync(stateDir).mode & 0o777).toBe(0o700)
|
|
284
|
+
expect(statSync(path).mode & 0o777).toBe(0o600)
|
|
285
|
+
} finally {
|
|
286
|
+
rmSync(root, { recursive: true, force: true })
|
|
287
|
+
}
|
|
288
|
+
})
|
|
289
|
+
|
|
290
|
+
it('leaves no tmp file behind after a successful write (rename, not copy)', () => {
|
|
291
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-w-'))
|
|
292
|
+
try {
|
|
293
|
+
writeBootBeaconFile(dir, beacon)
|
|
294
|
+
expect(existsSync(join(dir, `${BOOT_BEACON_FILE}.tmp-${process.pid}`))).toBe(false)
|
|
295
|
+
} finally {
|
|
296
|
+
rmSync(dir, { recursive: true, force: true })
|
|
297
|
+
}
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
it('overwrites the previous beacon in place on the next tick', () => {
|
|
301
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-w-'))
|
|
302
|
+
try {
|
|
303
|
+
writeBootBeaconFile(dir, beacon)
|
|
304
|
+
writeBootBeaconFile(dir, { ...beacon, wallMs: 9999, oomKillCount: 3 })
|
|
305
|
+
const read = JSON.parse(readFileSync(join(dir, BOOT_BEACON_FILE), 'utf-8'))
|
|
306
|
+
expect(read.wallMs).toBe(9999)
|
|
307
|
+
expect(read.oomKillCount).toBe(3)
|
|
308
|
+
} finally {
|
|
309
|
+
rmSync(dir, { recursive: true, force: true })
|
|
310
|
+
}
|
|
311
|
+
})
|
|
312
|
+
|
|
313
|
+
it('returns false instead of throwing when the state dir cannot be created', () => {
|
|
314
|
+
const root = mkdtempSync(join(tmpdir(), 'beacon-bad-'))
|
|
315
|
+
try {
|
|
316
|
+
// A regular FILE where the state dir should be — mkdirSync throws ENOTDIR.
|
|
317
|
+
const notADir = join(root, 'occupied')
|
|
318
|
+
writeFileSync(notADir, 'i am a file')
|
|
319
|
+
let result: boolean | undefined
|
|
320
|
+
expect(() => { result = writeBootBeaconFile(join(notADir, 'state'), beacon) }).not.toThrow()
|
|
321
|
+
expect(result).toBe(false)
|
|
322
|
+
} finally {
|
|
323
|
+
rmSync(root, { recursive: true, force: true })
|
|
324
|
+
}
|
|
325
|
+
})
|
|
326
|
+
})
|
|
327
|
+
|
|
328
|
+
describe('tickBootBeacon', () => {
|
|
329
|
+
it('writes a well-formed beacon carrying this process pid and the given boot id', () => {
|
|
330
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-tick-'))
|
|
331
|
+
try {
|
|
332
|
+
expect(tickBootBeacon(dir, 'boot-under-test')).toBe(true)
|
|
333
|
+
const read = JSON.parse(readFileSync(join(dir, BOOT_BEACON_FILE), 'utf-8')) as BootBeacon
|
|
334
|
+
expect(read.bootId).toBe('boot-under-test')
|
|
335
|
+
expect(read.pid).toBe(process.pid)
|
|
336
|
+
expect(read.wallMs).toBeGreaterThan(1_700_000_000_000)
|
|
337
|
+
expect(read.monotonicMs).toBeGreaterThanOrEqual(0)
|
|
338
|
+
// Host/cgroup fields are environment-dependent: assert TYPE when present,
|
|
339
|
+
// and that their absence is a clean omission, never null/NaN.
|
|
340
|
+
if ('hostBootId' in read) expect(typeof read.hostBootId).toBe('string')
|
|
341
|
+
if ('oomKillCount' in read) expect(Number.isInteger(read.oomKillCount)).toBe(true)
|
|
342
|
+
if ('memCurrent' in read) expect(Number.isInteger(read.memCurrent)).toBe(true)
|
|
343
|
+
if ('memMax' in read) expect(['number', 'string']).toContain(typeof read.memMax)
|
|
344
|
+
for (const value of Object.values(read)) expect(value).not.toBeNull()
|
|
345
|
+
} finally {
|
|
346
|
+
rmSync(dir, { recursive: true, force: true })
|
|
347
|
+
}
|
|
348
|
+
})
|
|
349
|
+
|
|
350
|
+
it('advances wallMs across ticks so the next boot can bound the time of death', async () => {
|
|
351
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-tick-'))
|
|
352
|
+
try {
|
|
353
|
+
tickBootBeacon(dir, 'b')
|
|
354
|
+
const first = JSON.parse(readFileSync(join(dir, BOOT_BEACON_FILE), 'utf-8')) as BootBeacon
|
|
355
|
+
await new Promise(resolve => setTimeout(resolve, 25))
|
|
356
|
+
tickBootBeacon(dir, 'b')
|
|
357
|
+
const second = JSON.parse(readFileSync(join(dir, BOOT_BEACON_FILE), 'utf-8')) as BootBeacon
|
|
358
|
+
expect(second.wallMs).toBeGreaterThanOrEqual(first.wallMs)
|
|
359
|
+
expect(second.monotonicMs).toBeGreaterThan(first.monotonicMs)
|
|
360
|
+
} finally {
|
|
361
|
+
rmSync(dir, { recursive: true, force: true })
|
|
362
|
+
}
|
|
363
|
+
})
|
|
364
|
+
|
|
365
|
+
it('never throws on an unwritable state dir — a tick failure must not reach the gateway interval', () => {
|
|
366
|
+
const root = mkdtempSync(join(tmpdir(), 'beacon-tick-bad-'))
|
|
367
|
+
try {
|
|
368
|
+
const notADir = join(root, 'occupied')
|
|
369
|
+
writeFileSync(notADir, 'i am a file')
|
|
370
|
+
let result: boolean | undefined
|
|
371
|
+
expect(() => { result = tickBootBeacon(join(notADir, 'state'), 'b') }).not.toThrow()
|
|
372
|
+
expect(result).toBe(false)
|
|
373
|
+
} finally {
|
|
374
|
+
rmSync(root, { recursive: true, force: true })
|
|
375
|
+
}
|
|
376
|
+
})
|
|
377
|
+
|
|
378
|
+
})
|
|
379
|
+
|
|
380
|
+
describe('attachBootBeacon', () => {
|
|
381
|
+
it('writes a beacon IMMEDIATELY, before the first interval elapses', () => {
|
|
382
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-attach-'))
|
|
383
|
+
try {
|
|
384
|
+
attachBootBeacon(dir, () => {})
|
|
385
|
+
// Not "after the returned tick runs" — at attach time. A gateway that
|
|
386
|
+
// dies in its first 5s must still have recorded this host's boot_id.
|
|
387
|
+
expect(existsSync(join(dir, BOOT_BEACON_FILE))).toBe(true)
|
|
388
|
+
} finally {
|
|
389
|
+
rmSync(dir, { recursive: true, force: true })
|
|
390
|
+
}
|
|
391
|
+
})
|
|
392
|
+
|
|
393
|
+
it('refreshes the beacon and runs the host tick, beacon first', () => {
|
|
394
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-attach-'))
|
|
395
|
+
try {
|
|
396
|
+
const order: string[] = []
|
|
397
|
+
const tick = attachBootBeacon(dir, () => {
|
|
398
|
+
order.push(`next:${JSON.parse(readFileSync(join(dir, BOOT_BEACON_FILE), 'utf-8')).bootId}`)
|
|
399
|
+
})
|
|
400
|
+
// Stamp a sentinel over the attach-time beacon so "the tick refreshed it"
|
|
401
|
+
// is observable without depending on the clock advancing between writes.
|
|
402
|
+
writeFileSync(join(dir, BOOT_BEACON_FILE), '{"bootId":"stale-sentinel"}\n')
|
|
403
|
+
tick()
|
|
404
|
+
const after = JSON.parse(readFileSync(join(dir, BOOT_BEACON_FILE), 'utf-8')) as BootBeacon
|
|
405
|
+
expect(after.bootId).not.toBe('stale-sentinel')
|
|
406
|
+
expect(after.pid).toBe(process.pid)
|
|
407
|
+
// The host tick observed the ALREADY-refreshed beacon — proving the
|
|
408
|
+
// beacon write precedes it, so a throw from `next` cannot starve it.
|
|
409
|
+
expect(order).toEqual([`next:${after.bootId}`])
|
|
410
|
+
} finally {
|
|
411
|
+
rmSync(dir, { recursive: true, force: true })
|
|
412
|
+
}
|
|
413
|
+
})
|
|
414
|
+
|
|
415
|
+
it('does not swallow the host tick\'s errors — existing behaviour is unchanged', () => {
|
|
416
|
+
const dir = mkdtempSync(join(tmpdir(), 'beacon-attach-'))
|
|
417
|
+
try {
|
|
418
|
+
const tick = attachBootBeacon(dir, () => { throw new Error('host tick blew up') })
|
|
419
|
+
expect(() => tick()).toThrow('host tick blew up')
|
|
420
|
+
// …and the beacon still landed, because it is written first.
|
|
421
|
+
expect(existsSync(join(dir, BOOT_BEACON_FILE))).toBe(true)
|
|
422
|
+
} finally {
|
|
423
|
+
rmSync(dir, { recursive: true, force: true })
|
|
424
|
+
}
|
|
425
|
+
})
|
|
426
|
+
|
|
427
|
+
it('still runs the host tick when the beacon write fails', () => {
|
|
428
|
+
// The beacon is forensic garnish; it must never gate the tick it rides on.
|
|
429
|
+
const root = mkdtempSync(join(tmpdir(), 'beacon-attach-bad-'))
|
|
430
|
+
try {
|
|
431
|
+
const notADir = join(root, 'occupied')
|
|
432
|
+
writeFileSync(notADir, 'i am a file')
|
|
433
|
+
let ran = false
|
|
434
|
+
const tick = attachBootBeacon(join(notADir, 'state'), () => { ran = true })
|
|
435
|
+
expect(() => tick()).not.toThrow()
|
|
436
|
+
expect(ran).toBe(true)
|
|
437
|
+
} finally {
|
|
438
|
+
rmSync(root, { recursive: true, force: true })
|
|
439
|
+
}
|
|
440
|
+
})
|
|
441
|
+
})
|
|
442
|
+
|
|
443
|
+
describe('gateway wiring', () => {
|
|
444
|
+
// Source-scrape rather than booting the 24k-line gateway module (which has
|
|
445
|
+
// load-bearing module-eval ordering — see the #3392 TDZ scar). The outcome
|
|
446
|
+
// that matters is that the beacon is actually DRIVEN: an unwired writer is
|
|
447
|
+
// an empty forensic record, and nothing else in the tree would go red.
|
|
448
|
+
const gatewaySrc = readFileSync(
|
|
449
|
+
new URL('../gateway/gateway.ts', import.meta.url), 'utf-8',
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
it('rides the existing shared 5s approval tick rather than adding a timer', () => {
|
|
453
|
+
expect(gatewaySrc).toContain("import { attachBootBeacon } from './boot-beacon.js'")
|
|
454
|
+
expect(gatewaySrc).toContain(
|
|
455
|
+
'setInterval(attachBootBeacon(STATE_DIR, checkApprovals), 5000)',
|
|
456
|
+
)
|
|
457
|
+
// Exactly one wiring site, and it is the pre-existing interval — a second
|
|
458
|
+
// `attachBootBeacon` call would mean a second beacon writer.
|
|
459
|
+
expect(gatewaySrc.match(/attachBootBeacon\(STATE_DIR/g)?.length).toBe(1)
|
|
460
|
+
expect(BOOT_BEACON_INTERVAL_MS).toBe(5_000)
|
|
461
|
+
})
|
|
462
|
+
})
|
|
@@ -19,17 +19,17 @@ import { resolve } from 'node:path'
|
|
|
19
19
|
const gatewaySrc = readFileSync(resolve(__dirname, '..', 'gateway', 'gateway.ts'), 'utf-8')
|
|
20
20
|
|
|
21
21
|
describe('boot pin sweep wiring (#3664)', () => {
|
|
22
|
-
it('never dispatches
|
|
22
|
+
it('never dispatches runBootPinCleanupAndStalePinSweep() directly — only through the gate', () => {
|
|
23
23
|
// The original bug shape, verbatim: a fire-and-forget call at module-eval
|
|
24
24
|
// time. Any direct invocation (voided or awaited) bypasses the bot-ready
|
|
25
25
|
// half of the precondition.
|
|
26
26
|
const directCalls =
|
|
27
|
-
gatewaySrc.match(/(?<!function\s)(?<![.\w])
|
|
27
|
+
gatewaySrc.match(/(?<!function\s)(?<![.\w])runBootPinCleanupAndStalePinSweep\s*\(/g) ?? []
|
|
28
28
|
expect(directCalls).toEqual([])
|
|
29
29
|
})
|
|
30
30
|
|
|
31
31
|
it('passes the sweep to the gate as a reference, and arms it at both mutex sites', () => {
|
|
32
|
-
expect(gatewaySrc).toContain('createBootSweepGate({ run:
|
|
32
|
+
expect(gatewaySrc).toContain('createBootSweepGate({ run: runBootPinCleanupAndStalePinSweep')
|
|
33
33
|
// Both startup-lock outcomes (mutex acquired, and the non-atomic
|
|
34
34
|
// writePidFile fallback) arm — neither may dispatch on its own.
|
|
35
35
|
const armSites = gatewaySrc.match(/bootPinSweepGate\.arm\(\)/g) ?? []
|
|
@@ -63,15 +63,15 @@ describe('boot pin sweep wiring (#3664)', () => {
|
|
|
63
63
|
expect(inlineAwaits).toEqual([])
|
|
64
64
|
}
|
|
65
65
|
// The DM-eligibility flip is what a throw used to strand, so it must live
|
|
66
|
-
// INSIDE the
|
|
66
|
+
// INSIDE the enableSweep step callback — not as a statement sequenced
|
|
67
67
|
// after the reapers, where one rejection skips it for the whole session.
|
|
68
68
|
const lines = gatewaySrc.split('\n')
|
|
69
69
|
const flipIdxs = lines
|
|
70
70
|
.map((l, i) => ({ l, i }))
|
|
71
|
-
.filter(({ l }) => /^\s*
|
|
71
|
+
.filter(({ l }) => /^\s*stalePinSweepEligible = true\s*$/.test(l))
|
|
72
72
|
.map(({ i }) => i)
|
|
73
73
|
expect(flipIdxs.length).toBe(1)
|
|
74
|
-
expect(lines[flipIdxs[0] - 1]).toContain('
|
|
74
|
+
expect(lines[flipIdxs[0] - 1]).toContain('enableSweep: () => {')
|
|
75
75
|
})
|
|
76
76
|
|
|
77
77
|
it('routes the pin API through the asserting seam, never a raw lockedBot.api pin', () => {
|
|
@@ -133,23 +133,23 @@ describe('createBootSweepGate (#3664 Defect A)', () => {
|
|
|
133
133
|
* Per-step isolation (#3664 salvage S2).
|
|
134
134
|
*
|
|
135
135
|
* The steps used to be a bare sequential `await` chain in gateway.ts. A throw
|
|
136
|
-
* from either card reaper skipped `
|
|
137
|
-
* the
|
|
138
|
-
* disabled
|
|
136
|
+
* from either card reaper skipped `stalePinSweepEligible = true`, which
|
|
137
|
+
* authorises the stale-pin drain for the WHOLE SESSION — so one throwing reaper
|
|
138
|
+
* silently disabled stale-pin cleanup, including every later lazy first-inbound
|
|
139
139
|
* sweep. That became newly reachable the moment #3664 made the sweep actually
|
|
140
140
|
* run for the first time.
|
|
141
141
|
*
|
|
142
|
-
* The outcome asserted is "was the
|
|
143
|
-
* swept", not "was a wrapper called".
|
|
142
|
+
* The outcome asserted is "was the drain enabled and were the recorded
|
|
143
|
+
* (chat, thread) targets swept", not "was a wrapper called".
|
|
144
144
|
*/
|
|
145
145
|
describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
146
146
|
function deps(over: Partial<BootPinSweepSteps> = {}) {
|
|
147
147
|
const order: string[] = []
|
|
148
148
|
const logs: string[] = []
|
|
149
149
|
const base: BootPinSweepSteps = {
|
|
150
|
-
|
|
150
|
+
scanSweepTargets: () => {
|
|
151
151
|
order.push('scan')
|
|
152
|
-
return ['
|
|
152
|
+
return [{ chatId: '900000001' }]
|
|
153
153
|
},
|
|
154
154
|
statusPinCleanup: async () => {
|
|
155
155
|
order.push('status-pins')
|
|
@@ -160,9 +160,9 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
160
160
|
queuedCardReaper: async () => {
|
|
161
161
|
order.push('queued-cards')
|
|
162
162
|
},
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
order.push(`
|
|
163
|
+
enableSweep: () => order.push('enable-sweep'),
|
|
164
|
+
sweepTarget: async (t) => {
|
|
165
|
+
order.push(`sweep:${t.chatId}:${t.threadId ?? '-'}`)
|
|
166
166
|
},
|
|
167
167
|
log: (l) => logs.push(l),
|
|
168
168
|
}
|
|
@@ -177,8 +177,8 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
177
177
|
'status-pins',
|
|
178
178
|
'activity-cards',
|
|
179
179
|
'queued-cards',
|
|
180
|
-
'enable-
|
|
181
|
-
'
|
|
180
|
+
'enable-sweep',
|
|
181
|
+
'sweep:900000001:-',
|
|
182
182
|
])
|
|
183
183
|
})
|
|
184
184
|
|
|
@@ -190,7 +190,13 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
190
190
|
})
|
|
191
191
|
await runBootPinSweepSteps(d.deps)
|
|
192
192
|
// The DM path is still enabled and the recorded DM chat is still swept.
|
|
193
|
-
expect(d.order).toEqual([
|
|
193
|
+
expect(d.order).toEqual([
|
|
194
|
+
'scan',
|
|
195
|
+
'status-pins',
|
|
196
|
+
'queued-cards',
|
|
197
|
+
'enable-sweep',
|
|
198
|
+
'sweep:900000001:-',
|
|
199
|
+
])
|
|
194
200
|
expect(d.logs.join('')).toContain("step 'activity-card-reaper' failed: boom")
|
|
195
201
|
})
|
|
196
202
|
|
|
@@ -204,32 +210,37 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
204
210
|
queuedCardReaper: boom,
|
|
205
211
|
})
|
|
206
212
|
await runBootPinSweepSteps(d.deps)
|
|
207
|
-
expect(d.order).toEqual(['scan', 'enable-
|
|
213
|
+
expect(d.order).toEqual(['scan', 'enable-sweep', 'sweep:900000001:-'])
|
|
208
214
|
})
|
|
209
215
|
|
|
210
216
|
it('a failing store scan still lets the reapers and the DM enable run', async () => {
|
|
211
217
|
const d = deps({
|
|
212
|
-
|
|
218
|
+
scanSweepTargets: () => {
|
|
213
219
|
throw new Error('ENOENT')
|
|
214
220
|
},
|
|
215
221
|
})
|
|
216
222
|
await runBootPinSweepSteps(d.deps)
|
|
217
|
-
// No
|
|
218
|
-
expect(d.order).toEqual(['status-pins', 'activity-cards', 'queued-cards', 'enable-
|
|
219
|
-
expect(d.logs.join('')).toContain("step '
|
|
223
|
+
// No targets to sweep, but nothing behind the scan is stranded.
|
|
224
|
+
expect(d.order).toEqual(['status-pins', 'activity-cards', 'queued-cards', 'enable-sweep'])
|
|
225
|
+
expect(d.logs.join('')).toContain("step 'sweep-target-scan' failed: ENOENT")
|
|
220
226
|
})
|
|
221
227
|
|
|
222
|
-
it('one failing
|
|
228
|
+
it('one failing target does not block the next one, and topics stay distinct', async () => {
|
|
223
229
|
const swept: string[] = []
|
|
224
230
|
const d = deps({
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
231
|
+
scanSweepTargets: () => [
|
|
232
|
+
{ chatId: '-1009000000001', threadId: 5 },
|
|
233
|
+
{ chatId: '-1009000000001', threadId: 9 },
|
|
234
|
+
],
|
|
235
|
+
sweepTarget: async (t) => {
|
|
236
|
+
if (t.threadId === 5) throw new Error('kicked')
|
|
237
|
+
swept.push(`${t.chatId}:${t.threadId}`)
|
|
229
238
|
},
|
|
230
239
|
})
|
|
231
240
|
await runBootPinSweepSteps(d.deps)
|
|
232
|
-
expect(swept).toEqual(['
|
|
241
|
+
expect(swept).toEqual(['-1009000000001:9'])
|
|
242
|
+
// Each target gets its own named step, keyed by (chat, thread).
|
|
243
|
+
expect(d.logs.join('')).toContain("step 'stale-pin-sweep:-1009000000001:5' failed: kicked")
|
|
233
244
|
})
|
|
234
245
|
|
|
235
246
|
it('logs a non-Error throw honestly instead of "undefined"', async () => {
|
|
@@ -250,7 +261,7 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
250
261
|
const order: string[] = []
|
|
251
262
|
await expect(
|
|
252
263
|
runBootPinSweepSteps({
|
|
253
|
-
|
|
264
|
+
scanSweepTargets: () => [],
|
|
254
265
|
statusPinCleanup: async () => {
|
|
255
266
|
throw new Error('boom')
|
|
256
267
|
},
|
|
@@ -260,14 +271,14 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
260
271
|
queuedCardReaper: async () => {
|
|
261
272
|
order.push('queued-cards')
|
|
262
273
|
},
|
|
263
|
-
|
|
264
|
-
|
|
274
|
+
enableSweep: () => order.push('enable-sweep'),
|
|
275
|
+
sweepTarget: async () => {},
|
|
265
276
|
log: () => {
|
|
266
277
|
throw new Error('EPIPE')
|
|
267
278
|
},
|
|
268
279
|
}),
|
|
269
280
|
).resolves.toBeUndefined()
|
|
270
|
-
expect(order).toEqual(['activity-cards', 'queued-cards', 'enable-
|
|
281
|
+
expect(order).toEqual(['activity-cards', 'queued-cards', 'enable-sweep'])
|
|
271
282
|
})
|
|
272
283
|
|
|
273
284
|
it('never rejects — the caller fire-and-forgets it', async () => {
|
|
@@ -276,16 +287,16 @@ describe('runBootPinSweepSteps (#3664 salvage S2)', () => {
|
|
|
276
287
|
}
|
|
277
288
|
await expect(
|
|
278
289
|
runBootPinSweepSteps({
|
|
279
|
-
|
|
290
|
+
scanSweepTargets: () => {
|
|
280
291
|
throw new Error('boom')
|
|
281
292
|
},
|
|
282
293
|
statusPinCleanup: boom,
|
|
283
294
|
activityCardReaper: boom,
|
|
284
295
|
queuedCardReaper: boom,
|
|
285
|
-
|
|
296
|
+
enableSweep: () => {
|
|
286
297
|
throw new Error('boom')
|
|
287
298
|
},
|
|
288
|
-
|
|
299
|
+
sweepTarget: boom,
|
|
289
300
|
log: () => {},
|
|
290
301
|
}),
|
|
291
302
|
).resolves.toBeUndefined()
|
|
@@ -245,6 +245,8 @@ describe('dispatchEffects — inbound-routing effects (PR3c wiring)', () => {
|
|
|
245
245
|
},
|
|
246
246
|
existsSync: (p: string) => files.has(p),
|
|
247
247
|
statSizeSync: (p: string) => Buffer.byteLength(files.get(p) ?? ''),
|
|
248
|
+
fsyncFileSync: () => {},
|
|
249
|
+
fsyncDirSync: () => {},
|
|
248
250
|
}
|
|
249
251
|
const spool = createInboundSpool({ path: spoolPath, fs, log: () => {} })
|
|
250
252
|
const buffer = createPendingInboundBuffer({ spool, log: () => {} })
|