switchroom 0.19.30 → 0.19.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +1121 -584
- package/dist/host-control/main.js +151 -73
- package/package.json +1 -1
- package/profiles/_base/cron-session.sh.hbs +5 -1
- package/profiles/_base/start.sh.hbs +15 -1
- package/telegram-plugin/dist/gateway/gateway.js +1435 -551
- package/telegram-plugin/edit-flood-fuse.ts +70 -20
- package/telegram-plugin/gateway/boot-beacon.ts +364 -0
- package/telegram-plugin/gateway/boot-sweep-gate.ts +20 -15
- package/telegram-plugin/gateway/gateway.ts +87 -88
- package/telegram-plugin/gateway/inbound-spool.ts +39 -0
- package/telegram-plugin/gateway/narrative-lane.ts +12 -0
- package/telegram-plugin/gateway/obligation-store.ts +28 -0
- package/telegram-plugin/gateway/stale-pin-sweep-store.ts +221 -0
- package/telegram-plugin/gateway/stale-pin-sweep-wiring.ts +211 -0
- package/telegram-plugin/gateway/stale-pin-sweep.test.ts +804 -0
- package/telegram-plugin/gateway/stale-pin-sweep.ts +1146 -0
- package/telegram-plugin/gateway/status-pin-retarget.ts +15 -2
- package/telegram-plugin/gateway/status-pin-store.ts +33 -11
- package/telegram-plugin/registry/turns-schema.ts +21 -1
- package/telegram-plugin/retry-api-call.ts +46 -21
- package/telegram-plugin/shared/bot-runtime.ts +61 -17
- package/telegram-plugin/shared/gw-trace-gate.ts +18 -2
- package/telegram-plugin/tests/activity-card-wiring.test.ts +7 -7
- package/telegram-plugin/tests/activity-drain-fuse-drop-not-failure.test.ts +324 -0
- package/telegram-plugin/tests/agent-card-result-footer.test.ts +193 -0
- package/telegram-plugin/tests/boot-beacon.test.ts +462 -0
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +6 -6
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +42 -31
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool-progress.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +134 -5
- package/telegram-plugin/tests/narrative-lane-golden.test.ts +28 -0
- package/telegram-plugin/tests/obligation-determinism.test.ts +2 -0
- package/telegram-plugin/tests/obligation-store.test.ts +67 -1
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +26 -5
- package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +161 -0
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +30 -0
- package/telegram-plugin/tool-activity-summary.ts +104 -38
- package/telegram-plugin/worker-activity-feed.ts +33 -16
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +0 -251
- package/telegram-plugin/gateway/dm-pin-sweep.ts +0 -178
|
@@ -69,12 +69,16 @@
|
|
|
69
69
|
* lost". Two rules follow, and both are load-bearing:
|
|
70
70
|
* - per-MESSAGE over-budget edits SUPERSEDE (newest wins, older frame is
|
|
71
71
|
* discarded) — safe, because the frame that killed it will paint;
|
|
72
|
-
* -
|
|
73
|
-
* that same message is still in flight to repaint it. When
|
|
74
|
-
* frame is the only one for its card (a turn-final `finalize`,
|
|
75
|
-
* RELEASED late instead of dropped. Dropping it would freeze the
|
|
76
|
-
* mid-run AND return `true` to the send gate, which would then record a
|
|
77
|
-
* never-painted payload as on-screen and no-op-skip every retry.
|
|
72
|
+
* - an over-budget edit may only be DROPPED AT THE DEFER DEADLINE while a
|
|
73
|
+
* NEWER frame for that same message is still in flight to repaint it. When
|
|
74
|
+
* the waiting frame is the only one for its card (a turn-final `finalize`,
|
|
75
|
+
* say) it is RELEASED late instead of dropped. Dropping it would freeze the
|
|
76
|
+
* card mid-run AND return `true` to the send gate, which would then record a
|
|
77
|
+
* never-painted payload as on-screen and no-op-skip every retry. This rule
|
|
78
|
+
* (`dropGuard`) binds on ALL THREE tiers — per-message included. It was
|
|
79
|
+
* originally wired into the two per-chat tiers only, which left the
|
|
80
|
+
* per-message tier dropping a lone terminal frame unconditionally at
|
|
81
|
+
* `maxDeferMs` and freezing the card by exactly the route R1 forbids.
|
|
78
82
|
*
|
|
79
83
|
* ── 2026-07-27: the fuse existed and the ban happened anyway ──────────────
|
|
80
84
|
* The ceilings above were sized against the in-repo pacers, not against what
|
|
@@ -599,8 +603,35 @@ export function editFloodFuseConfigFromEnv(
|
|
|
599
603
|
return cfg
|
|
600
604
|
}
|
|
601
605
|
|
|
602
|
-
/**
|
|
603
|
-
|
|
606
|
+
/**
|
|
607
|
+
* What a DROPPED call resolves to.
|
|
608
|
+
*
|
|
609
|
+
* `apply` is a grammY **transformer**, and a transformer's contract is the raw
|
|
610
|
+
* `ApiResponse` ENVELOPE (`{ok: true, result}` / `{ok: false, error_code,
|
|
611
|
+
* description}`) — not the unwrapped result. grammY's `Api.callApi` does:
|
|
612
|
+
*
|
|
613
|
+
* ```js
|
|
614
|
+
* const data = await this.call(method, payload, signal)
|
|
615
|
+
* if (data.ok) return data.result
|
|
616
|
+
* else throw toGrammyError(data, method, payload)
|
|
617
|
+
* ```
|
|
618
|
+
* (grammy `out/core/client.js`)
|
|
619
|
+
*
|
|
620
|
+
* This constant used to be the bare `true` — the *result* an edit resolves to,
|
|
621
|
+
* with the envelope omitted. `(true).ok` is `undefined`, so every deliberate
|
|
622
|
+
* fuse drop took the `else` branch and surfaced as
|
|
623
|
+
* `GrammyError: Call to 'editMessageText' failed! (undefined: undefined)`
|
|
624
|
+
* (`err.error_code` / `err.description` are both absent on a boolean). Callers
|
|
625
|
+
* then counted the fuse's INTENDED rate-limiting as a transport failure — which
|
|
626
|
+
* is how a healthy turn reached `activityDrainFailures=8` and was flagged
|
|
627
|
+
* DEGRADED for behaving exactly as designed (narrative-lane.ts drain catch).
|
|
628
|
+
*
|
|
629
|
+
* Wrapping it in the envelope restores the contract the rest of this file
|
|
630
|
+
* already documents: a dropped call resolves as the benign no-op described in
|
|
631
|
+
* the header ("A superseded or over-deferred edit is DROPPED (resolved as a
|
|
632
|
+
* benign no-op)") and returns `true` to the send gate, as the R1 note assumes.
|
|
633
|
+
*/
|
|
634
|
+
const DROPPED_RESULT = { ok: true as const, result: true }
|
|
604
635
|
|
|
605
636
|
interface Window {
|
|
606
637
|
ts: number[]
|
|
@@ -961,11 +992,15 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
961
992
|
maxFor: (now: number) => number,
|
|
962
993
|
method: string, mode: WaitMode, cls: OutboundClass,
|
|
963
994
|
/**
|
|
964
|
-
* Consulted ONLY at the defer deadline for `mode
|
|
965
|
-
* converts the drop into a late
|
|
966
|
-
* will ever paint its message, so
|
|
967
|
-
* frame", it is freezing a card. Absent
|
|
968
|
-
* pre-review behaviour).
|
|
995
|
+
* Consulted ONLY at the defer deadline, for every non-`release` mode
|
|
996
|
+
* (`drop` AND `supersede`). Returning false converts the drop into a late
|
|
997
|
+
* release: this call is the last thing that will ever paint its message, so
|
|
998
|
+
* losing it is not "shedding a stale frame", it is freezing a card. Absent
|
|
999
|
+
* ⇒ drop unconditionally (the pre-review behaviour).
|
|
1000
|
+
*
|
|
1001
|
+
* Note this is the DEADLINE guard only. In `supersede` mode a genuinely
|
|
1002
|
+
* newer frame still kills a waiter outright via `w.waiter.kill()` — that is
|
|
1003
|
+
* last-write-wins and is unaffected by this guard.
|
|
969
1004
|
*/
|
|
970
1005
|
dropGuard: (() => boolean) | undefined,
|
|
971
1006
|
/**
|
|
@@ -1118,21 +1153,36 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
1118
1153
|
// Counted BEFORE the first await so a frame that arrives while an older
|
|
1119
1154
|
// one is waiting is visible to that older one's `dropGuard`.
|
|
1120
1155
|
mw.inflight++
|
|
1156
|
+
// R1: only drop while something newer for THIS message is still in
|
|
1157
|
+
// flight to repaint it. `> 1` = this frame plus at least one newer.
|
|
1158
|
+
const dropGuard = (): boolean => mw.inflight > 1
|
|
1159
|
+
// Cosmetic frames share ONE overshoot budget per chat across the
|
|
1160
|
+
// per-message and per-chat tiers, so a frame cannot late-release twice on
|
|
1161
|
+
// its way out.
|
|
1162
|
+
const lateKey = cls === 'cosmetic' ? `lr:${chat}` : undefined
|
|
1121
1163
|
try {
|
|
1164
|
+
// R1 applies to the per-MESSAGE tier too. `supersede` mode still lets a
|
|
1165
|
+
// genuinely-newer frame kill this one mid-wait (last-write-wins, the
|
|
1166
|
+
// `killed` path), but at the DEFER DEADLINE this tier used to pass
|
|
1167
|
+
// `dropGuard: undefined`, which `awaitRoom` reads as "drop
|
|
1168
|
+
// unconditionally". A LONE frame — the terminal `finalize` of a card,
|
|
1169
|
+
// whose whole job is to replace "→ in-progress" with the done state and
|
|
1170
|
+
// which by definition has nothing newer coming — was therefore dropped
|
|
1171
|
+
// here after `maxDeferMs`, freezing the card mid-step. That is the exact
|
|
1172
|
+
// failure R1 was written to prevent; it was only ever wired into the two
|
|
1173
|
+
// per-chat tiers. Passing the same guard makes a lone frame LATE-RELEASE
|
|
1174
|
+
// instead, bounded for cosmetic traffic by the shared `lateKey`
|
|
1175
|
+
// overshoot budget (non-cosmetic edits — finalize, approval cards — are
|
|
1176
|
+
// low-cadence by nature and already late-release unbounded on both
|
|
1177
|
+
// per-chat tiers, so this adds no new class of overshoot).
|
|
1122
1178
|
const msgSlot = await awaitRoom(
|
|
1123
1179
|
msgKey, perMessageWindowMs,
|
|
1124
1180
|
cls === 'cosmetic'
|
|
1125
1181
|
? (t) => ceiling(cosmeticPerMessageMax, t)
|
|
1126
1182
|
: (t) => classCeiling(perMessageMax, cls, t),
|
|
1127
|
-
method, 'supersede', cls,
|
|
1183
|
+
method, 'supersede', cls, dropGuard, deadline, lateKey,
|
|
1128
1184
|
)
|
|
1129
1185
|
if (msgSlot === null) return DROPPED_RESULT as unknown as R
|
|
1130
|
-
// R1: only drop while something newer for THIS message is still in
|
|
1131
|
-
// flight to repaint it. `> 1` = this frame plus at least one newer.
|
|
1132
|
-
const dropGuard = (): boolean => mw.inflight > 1
|
|
1133
|
-
// Cosmetic frames share ONE overshoot budget per chat across both
|
|
1134
|
-
// per-chat tiers, so a frame cannot late-release twice on its way out.
|
|
1135
|
-
const lateKey = cls === 'cosmetic' ? `lr:${chat}` : undefined
|
|
1136
1186
|
const reserved: Array<[string, number]> = [[msgKey, msgSlot]]
|
|
1137
1187
|
const giveBack = (): void => { for (const [k, at] of reserved) unreserve(k, at) }
|
|
1138
1188
|
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gateway boot beacon — write-only forensic evidence for the NEXT boot.
|
|
3
|
+
*
|
|
4
|
+
* The problem this exists to feed
|
|
5
|
+
* ------------------------------
|
|
6
|
+
* When an agent's container dies, the next boot cannot tell WHY. Every open
|
|
7
|
+
* turn that isn't classified as a hang is blanket-stamped `ended_via='restart'`
|
|
8
|
+
* (`registry/turns-schema.ts` `sweepOpenTurns`), so an OOM kill, a bare
|
|
9
|
+
* SIGKILL, an operator restart and a host power loss are indistinguishable —
|
|
10
|
+
* and the user gets told "the hang-watchdog killed it" for what was actually a
|
|
11
|
+
* power cut.
|
|
12
|
+
*
|
|
13
|
+
* NOTHING READS THIS BEACON YET. This module is deliberately write-only: it
|
|
14
|
+
* lays down the evidence a later classifier will read, so that the classifier
|
|
15
|
+
* lands with real historical data behind it instead of shipping blind. Keeping
|
|
16
|
+
* the write and the read in separate changes also means this one carries zero
|
|
17
|
+
* behaviour risk — there is no decision anywhere in the tree that depends on it.
|
|
18
|
+
*
|
|
19
|
+
* What each field will later discriminate
|
|
20
|
+
* ---------------------------------------
|
|
21
|
+
* bootId Fresh UUID per gateway PROCESS. A beacon whose bootId differs
|
|
22
|
+
* from ours proves the file was written by a previous process
|
|
23
|
+
* (i.e. it is genuinely evidence about a death, not our own
|
|
24
|
+
* in-flight state).
|
|
25
|
+
* hostBootId `/proc/sys/kernel/random/boot_id` — regenerated by the kernel
|
|
26
|
+
* on every host boot. THE load-bearing field: at next boot, an
|
|
27
|
+
* UNCHANGED host boot_id plus a missing clean-shutdown marker
|
|
28
|
+
* means OUR process died (SIGKILL/OOM/crash) on a host that kept
|
|
29
|
+
* running; a CHANGED boot_id means the HOST rebooted (power loss,
|
|
30
|
+
* kernel panic, hypervisor stop). It is the only clean
|
|
31
|
+
* discriminator between those two.
|
|
32
|
+
* pid Distinguishes "same process still alive" from a respawn when
|
|
33
|
+
* bootId alone is ambiguous (e.g. a beacon restored from a
|
|
34
|
+
* snapshot).
|
|
35
|
+
* wallMs Last-known-alive wall clock. Bounds the time of death to
|
|
36
|
+
* within one beacon interval.
|
|
37
|
+
* monotonicMs Monotonic ms since process start. Compared against the wallMs
|
|
38
|
+
* delta, a divergence reveals a wall-clock jump (NTP step, host
|
|
39
|
+
* suspend/resume) rather than a real elapsed gap.
|
|
40
|
+
* oomKillCount The cgroup v2 `oom_kill` counter, SAMPLED LIVE. See below.
|
|
41
|
+
* memCurrent `/sys/fs/cgroup/memory.current` at the last sample — how close
|
|
42
|
+
* to the limit we were moments before death.
|
|
43
|
+
* memMax `/sys/fs/cgroup/memory.max`; the literal string `'max'` when
|
|
44
|
+
* the cgroup is unlimited (a real, distinct state from "could
|
|
45
|
+
* not read", which omits the field entirely).
|
|
46
|
+
*
|
|
47
|
+
* Why oomKillCount MUST be sampled live rather than read at boot
|
|
48
|
+
* -------------------------------------------------------------
|
|
49
|
+
* The cgroup `oom_kill` counter resets when the cgroup is recreated, which is
|
|
50
|
+
* exactly what happens when a container restarts. Read cold at boot it is
|
|
51
|
+
* therefore always 0 and carries no information. Sampled by the LIVE gateway
|
|
52
|
+
* into this beacon and compared against the PRIOR beacon, an increment is
|
|
53
|
+
* direct proof of an OOM kill — including the case that is otherwise invisible,
|
|
54
|
+
* where a child process (the `claude` process) is OOM-killed while PID 1
|
|
55
|
+
* survives and the container never restarts at all.
|
|
56
|
+
*
|
|
57
|
+
* Degradation contract (hard requirement)
|
|
58
|
+
* ---------------------------------------
|
|
59
|
+
* Every `/proc` and cgroup read is individually try/catch'd and each field is
|
|
60
|
+
* independently optional. A cgroup v1 host, a kernel without
|
|
61
|
+
* `random/boot_id`, a read-only or namespaced `/sys`, a permission error, a
|
|
62
|
+
* truncated file — all degrade to OMITTING that one field. This module must
|
|
63
|
+
* NEVER throw into the gateway tick: a crash here would take down every agent
|
|
64
|
+
* in the fleet on rollout.
|
|
65
|
+
*/
|
|
66
|
+
|
|
67
|
+
import { randomUUID } from 'node:crypto'
|
|
68
|
+
import {
|
|
69
|
+
closeSync,
|
|
70
|
+
fsyncSync,
|
|
71
|
+
mkdirSync,
|
|
72
|
+
openSync,
|
|
73
|
+
readFileSync,
|
|
74
|
+
renameSync,
|
|
75
|
+
unlinkSync,
|
|
76
|
+
writeFileSync,
|
|
77
|
+
} from 'node:fs'
|
|
78
|
+
import { join } from 'node:path'
|
|
79
|
+
|
|
80
|
+
/** Filename under `TELEGRAM_STATE_DIR`. */
|
|
81
|
+
export const BOOT_BEACON_FILE = 'gateway-beacon.json'
|
|
82
|
+
|
|
83
|
+
/** How often the live gateway refreshes the beacon. 5s is the accepted
|
|
84
|
+
* loss window: the beacon bounds the time of death to within one interval,
|
|
85
|
+
* and a sub-5s cadence buys resolution nobody consumes. */
|
|
86
|
+
export const BOOT_BEACON_INTERVAL_MS = 5_000
|
|
87
|
+
|
|
88
|
+
export const HOST_BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id'
|
|
89
|
+
export const CGROUP_MEMORY_EVENTS_PATH = '/sys/fs/cgroup/memory.events'
|
|
90
|
+
export const CGROUP_MEMORY_CURRENT_PATH = '/sys/fs/cgroup/memory.current'
|
|
91
|
+
export const CGROUP_MEMORY_MAX_PATH = '/sys/fs/cgroup/memory.max'
|
|
92
|
+
|
|
93
|
+
/** Defensive cap on how much of a `/proc`/cgroup value we keep. These files are
|
|
94
|
+
* tens of bytes; anything larger is a host we don't understand and we'd rather
|
|
95
|
+
* drop the field than serialise junk into the beacon. */
|
|
96
|
+
const MAX_FIELD_CHARS = 256
|
|
97
|
+
|
|
98
|
+
export interface BootBeacon {
|
|
99
|
+
/** Fresh UUID per gateway process. Always present. */
|
|
100
|
+
bootId: string
|
|
101
|
+
/** `/proc/sys/kernel/random/boot_id`. Omitted when unreadable. */
|
|
102
|
+
hostBootId?: string
|
|
103
|
+
/** Gateway process pid. Always present. */
|
|
104
|
+
pid: number
|
|
105
|
+
/** `Date.now()` at sample time. Always present. */
|
|
106
|
+
wallMs: number
|
|
107
|
+
/** Monotonic ms since process start. Always present. */
|
|
108
|
+
monotonicMs: number
|
|
109
|
+
/** cgroup v2 `oom_kill` counter. Omitted when unreadable. */
|
|
110
|
+
oomKillCount?: number
|
|
111
|
+
/** `/sys/fs/cgroup/memory.current` in bytes. Omitted when unreadable. */
|
|
112
|
+
memCurrent?: number
|
|
113
|
+
/** `/sys/fs/cgroup/memory.max` in bytes, or the literal `'max'` when the
|
|
114
|
+
* cgroup is unlimited. Omitted when unreadable. */
|
|
115
|
+
memMax?: number | 'max'
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** The host/cgroup half of a beacon — every field independently optional. */
|
|
119
|
+
export interface BeaconHostSample {
|
|
120
|
+
hostBootId?: string
|
|
121
|
+
oomKillCount?: number
|
|
122
|
+
memCurrent?: number
|
|
123
|
+
memMax?: number | 'max'
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ─── Pure parsers ─────────────────────────────────────────────────────────
|
|
127
|
+
// Each takes the RAW file contents and returns `undefined` for anything it
|
|
128
|
+
// can't confidently interpret. They never throw, so a malformed file degrades
|
|
129
|
+
// to an omitted field rather than a crash.
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Parse `/proc/sys/kernel/random/boot_id`: a single UUID line. Anything that
|
|
133
|
+
* isn't a plausible non-empty single-line token is rejected, so a cgroup v1 /
|
|
134
|
+
* non-Linux host that somehow returns HTML or an empty file omits the field
|
|
135
|
+
* instead of poisoning the classifier's "did the host reboot?" comparison —
|
|
136
|
+
* the one question this field exists to answer.
|
|
137
|
+
*/
|
|
138
|
+
export function parseHostBootId(raw: string | null | undefined): string | undefined {
|
|
139
|
+
if (typeof raw !== 'string') return undefined
|
|
140
|
+
const trimmed = raw.trim()
|
|
141
|
+
if (trimmed.length === 0 || trimmed.length > MAX_FIELD_CHARS) return undefined
|
|
142
|
+
if (/\s/.test(trimmed)) return undefined
|
|
143
|
+
return trimmed
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Parse the `oom_kill` counter out of cgroup v2 `memory.events`, which is a
|
|
148
|
+
* `key value` table:
|
|
149
|
+
*
|
|
150
|
+
* low 0
|
|
151
|
+
* high 0
|
|
152
|
+
* max 0
|
|
153
|
+
* oom 0
|
|
154
|
+
* oom_kill 3
|
|
155
|
+
* oom_group_kill 0
|
|
156
|
+
*
|
|
157
|
+
* Returns `undefined` when the key is absent (older kernels predate
|
|
158
|
+
* `oom_kill`) or the value isn't a non-negative integer. Deliberately reads
|
|
159
|
+
* `oom_kill`, NOT `oom`: `oom` counts times the cgroup hit its limit and
|
|
160
|
+
* entered OOM handling, which reclaim can resolve without killing anything;
|
|
161
|
+
* `oom_kill` counts processes actually killed, which is the event we need to
|
|
162
|
+
* attribute a death to.
|
|
163
|
+
*/
|
|
164
|
+
export function parseOomKillCount(raw: string | null | undefined): number | undefined {
|
|
165
|
+
if (typeof raw !== 'string') return undefined
|
|
166
|
+
for (const line of raw.split('\n')) {
|
|
167
|
+
const parts = line.trim().split(/\s+/)
|
|
168
|
+
if (parts.length !== 2 || parts[0] !== 'oom_kill') continue
|
|
169
|
+
return parseNonNegativeInt(parts[1])
|
|
170
|
+
}
|
|
171
|
+
return undefined
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Parse a cgroup v2 byte-count file (`memory.current`). Returns `undefined`
|
|
176
|
+
* for empty, non-numeric, negative or non-integer contents.
|
|
177
|
+
*/
|
|
178
|
+
export function parseMemoryBytes(raw: string | null | undefined): number | undefined {
|
|
179
|
+
if (typeof raw !== 'string') return undefined
|
|
180
|
+
return parseNonNegativeInt(raw.trim())
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Parse `memory.max`, which is either a byte count or the literal `max`
|
|
185
|
+
* meaning "no limit". `'max'` is returned as itself rather than folded to
|
|
186
|
+
* `undefined` so the later classifier can tell "unlimited cgroup, a
|
|
187
|
+
* limit-triggered OOM is impossible" apart from "we couldn't read the file".
|
|
188
|
+
*/
|
|
189
|
+
export function parseMemoryMax(raw: string | null | undefined): number | 'max' | undefined {
|
|
190
|
+
if (typeof raw !== 'string') return undefined
|
|
191
|
+
const trimmed = raw.trim()
|
|
192
|
+
if (trimmed === 'max') return 'max'
|
|
193
|
+
return parseNonNegativeInt(trimmed)
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function parseNonNegativeInt(text: string | undefined): number | undefined {
|
|
197
|
+
if (typeof text !== 'string') return undefined
|
|
198
|
+
const trimmed = text.trim()
|
|
199
|
+
if (trimmed.length === 0 || trimmed.length > MAX_FIELD_CHARS) return undefined
|
|
200
|
+
if (!/^\d+$/.test(trimmed)) return undefined
|
|
201
|
+
const value = Number(trimmed)
|
|
202
|
+
return Number.isSafeInteger(value) ? value : undefined
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// ─── Pure assembly + serialisation ────────────────────────────────────────
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Assemble a beacon from an already-taken host sample. Pure: no clock, no fs,
|
|
209
|
+
* no pid lookup — every input is an argument, so the exact bytes that hit disk
|
|
210
|
+
* are unit-testable. Optional sample fields are omitted from the object rather
|
|
211
|
+
* than emitted as `undefined`/`null`, keeping "absent" a single unambiguous
|
|
212
|
+
* shape for the future reader.
|
|
213
|
+
*/
|
|
214
|
+
export function buildBootBeacon(input: {
|
|
215
|
+
bootId: string
|
|
216
|
+
pid: number
|
|
217
|
+
wallMs: number
|
|
218
|
+
monotonicMs: number
|
|
219
|
+
sample?: BeaconHostSample
|
|
220
|
+
}): BootBeacon {
|
|
221
|
+
const beacon: BootBeacon = {
|
|
222
|
+
bootId: input.bootId,
|
|
223
|
+
pid: input.pid,
|
|
224
|
+
wallMs: input.wallMs,
|
|
225
|
+
monotonicMs: input.monotonicMs,
|
|
226
|
+
}
|
|
227
|
+
const sample = input.sample ?? {}
|
|
228
|
+
if (sample.hostBootId !== undefined) beacon.hostBootId = sample.hostBootId
|
|
229
|
+
if (sample.oomKillCount !== undefined) beacon.oomKillCount = sample.oomKillCount
|
|
230
|
+
if (sample.memCurrent !== undefined) beacon.memCurrent = sample.memCurrent
|
|
231
|
+
if (sample.memMax !== undefined) beacon.memMax = sample.memMax
|
|
232
|
+
return beacon
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Serialise to the exact bytes written to disk (single line + trailing \n). */
|
|
236
|
+
export function serializeBootBeacon(beacon: BootBeacon): string {
|
|
237
|
+
return `${JSON.stringify(beacon)}\n`
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ─── Impure edges ─────────────────────────────────────────────────────────
|
|
241
|
+
|
|
242
|
+
/** Read a small text file, returning `null` on ANY failure (missing, EACCES,
|
|
243
|
+
* EIO, a directory, cgroup v1 layout, …). */
|
|
244
|
+
function readTextOrNull(path: string): string | null {
|
|
245
|
+
try {
|
|
246
|
+
return readFileSync(path, 'utf-8')
|
|
247
|
+
} catch {
|
|
248
|
+
return null
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Sample the host + cgroup v2 state. Every read is independently try/catch'd
|
|
254
|
+
* (via `readTextOrNull`) and every parse independently returns `undefined`, so
|
|
255
|
+
* a cgroup v1 host degrades to a beacon carrying only bootId/pid/wall/monotonic
|
|
256
|
+
* rather than throwing.
|
|
257
|
+
*/
|
|
258
|
+
export function sampleBeaconHost(paths: {
|
|
259
|
+
hostBootId?: string
|
|
260
|
+
memoryEvents?: string
|
|
261
|
+
memoryCurrent?: string
|
|
262
|
+
memoryMax?: string
|
|
263
|
+
} = {}): BeaconHostSample {
|
|
264
|
+
const sample: BeaconHostSample = {}
|
|
265
|
+
const hostBootId = parseHostBootId(readTextOrNull(paths.hostBootId ?? HOST_BOOT_ID_PATH))
|
|
266
|
+
if (hostBootId !== undefined) sample.hostBootId = hostBootId
|
|
267
|
+
const oomKillCount = parseOomKillCount(readTextOrNull(paths.memoryEvents ?? CGROUP_MEMORY_EVENTS_PATH))
|
|
268
|
+
if (oomKillCount !== undefined) sample.oomKillCount = oomKillCount
|
|
269
|
+
const memCurrent = parseMemoryBytes(readTextOrNull(paths.memoryCurrent ?? CGROUP_MEMORY_CURRENT_PATH))
|
|
270
|
+
if (memCurrent !== undefined) sample.memCurrent = memCurrent
|
|
271
|
+
const memMax = parseMemoryMax(readTextOrNull(paths.memoryMax ?? CGROUP_MEMORY_MAX_PATH))
|
|
272
|
+
if (memMax !== undefined) sample.memMax = memMax
|
|
273
|
+
return sample
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Durably write the beacon: tmp file → write → `fsync` → `rename`. The fsync
|
|
278
|
+
* is the point of the whole exercise — this file's only consumer is the boot
|
|
279
|
+
* AFTER a hard kill or a host power loss, so contents still sitting in the page
|
|
280
|
+
* cache at power-off would be exactly the evidence we lose in the case we most
|
|
281
|
+
* need it. `rename` over the same directory is atomic, so a reader can never
|
|
282
|
+
* observe a torn beacon.
|
|
283
|
+
*
|
|
284
|
+
* (We do not additionally fsync the containing directory. That would harden the
|
|
285
|
+
* rename's own durability, but the beacon is refreshed every 5s and losing the
|
|
286
|
+
* final rename costs at most one interval of resolution — the same loss window
|
|
287
|
+
* we already accept.)
|
|
288
|
+
*
|
|
289
|
+
* Returns true when the beacon landed. Never throws.
|
|
290
|
+
*/
|
|
291
|
+
export function writeBootBeaconFile(stateDir: string, beacon: BootBeacon): boolean {
|
|
292
|
+
const path = join(stateDir, BOOT_BEACON_FILE)
|
|
293
|
+
const tmp = `${path}.tmp-${process.pid}`
|
|
294
|
+
let fd: number | undefined
|
|
295
|
+
try {
|
|
296
|
+
// mode 0o700 matters: STATE_DIR also holds access.json / .env, and the
|
|
297
|
+
// gateway creates it 0o700 at boot. If it were ever missing here, a
|
|
298
|
+
// default-mode recreate by this 5s tick would silently loosen the
|
|
299
|
+
// permissions on that whole directory.
|
|
300
|
+
mkdirSync(stateDir, { recursive: true, mode: 0o700 })
|
|
301
|
+
fd = openSync(tmp, 'w', 0o600)
|
|
302
|
+
// writeFileSync (not writeSync) on the fd: it loops internally, so a short
|
|
303
|
+
// write can't leave a truncated beacon that we then fsync and rename into
|
|
304
|
+
// place as if it were whole.
|
|
305
|
+
writeFileSync(fd, serializeBootBeacon(beacon))
|
|
306
|
+
fsyncSync(fd)
|
|
307
|
+
closeSync(fd)
|
|
308
|
+
fd = undefined
|
|
309
|
+
renameSync(tmp, path)
|
|
310
|
+
return true
|
|
311
|
+
} catch {
|
|
312
|
+
if (fd !== undefined) {
|
|
313
|
+
try { closeSync(fd) } catch { /* best effort */ }
|
|
314
|
+
}
|
|
315
|
+
try { unlinkSync(tmp) } catch { /* best effort — tmp may not exist */ }
|
|
316
|
+
return false
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** This gateway process's boot id. Stable for the process lifetime; a change
|
|
321
|
+
* across two beacons proves a respawn happened between them. */
|
|
322
|
+
export const GATEWAY_BOOT_ID = randomUUID()
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Take one sample and write it. Safe to call from a shared gateway tick: it
|
|
326
|
+
* swallows every failure and returns false rather than letting anything escape
|
|
327
|
+
* into a `setInterval` callback (which would reach `uncaughtException`).
|
|
328
|
+
*/
|
|
329
|
+
export function tickBootBeacon(stateDir: string, bootId: string = GATEWAY_BOOT_ID): boolean {
|
|
330
|
+
try {
|
|
331
|
+
return writeBootBeaconFile(stateDir, buildBootBeacon({
|
|
332
|
+
bootId,
|
|
333
|
+
pid: process.pid,
|
|
334
|
+
wallMs: Date.now(),
|
|
335
|
+
monotonicMs: Math.round(performance.now()),
|
|
336
|
+
sample: sampleBeaconHost(),
|
|
337
|
+
}))
|
|
338
|
+
} catch {
|
|
339
|
+
return false
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Attach the beacon to an EXISTING gateway tick rather than starting a second
|
|
345
|
+
* timer. Writes one beacon immediately (so a gateway that dies inside the first
|
|
346
|
+
* interval still leaves a record of which host boot it was living on), and
|
|
347
|
+
* returns a tick function that writes the beacon and then runs `next`.
|
|
348
|
+
*
|
|
349
|
+
* The beacon is written BEFORE `next` so a throw from the host tick's own work
|
|
350
|
+
* can never starve the forensic write — and this wrapper deliberately does not
|
|
351
|
+
* catch on `next`'s behalf: swallowing the host tick's errors would change
|
|
352
|
+
* existing behaviour, which is exactly what this write-only change must not do.
|
|
353
|
+
*
|
|
354
|
+
* Composed here rather than inline in `gateway.ts` so the wiring is one call
|
|
355
|
+
* site in that file (#2996 anti-inflation ratchet) and so this shape is
|
|
356
|
+
* unit-testable without booting the gateway module.
|
|
357
|
+
*/
|
|
358
|
+
export function attachBootBeacon(stateDir: string, next: () => void): () => void {
|
|
359
|
+
tickBootBeacon(stateDir)
|
|
360
|
+
return () => {
|
|
361
|
+
tickBootBeacon(stateDir)
|
|
362
|
+
next()
|
|
363
|
+
}
|
|
364
|
+
}
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import type { SweepTarget } from './stale-pin-sweep.js'
|
|
2
|
+
import { sweepTargetKey } from './stale-pin-sweep-store.js'
|
|
3
|
+
|
|
1
4
|
/**
|
|
2
5
|
* boot-sweep-gate.ts — control flow for the boot pin sweep (#3664): WHEN it may
|
|
3
6
|
* start (`createBootSweepGate`, the two-condition arming gate) and HOW its steps
|
|
@@ -5,7 +8,7 @@
|
|
|
5
8
|
*
|
|
6
9
|
* Why this exists
|
|
7
10
|
* ---------------
|
|
8
|
-
* The boot orphan sweep (`
|
|
11
|
+
* The boot orphan sweep (`runBootPinCleanupAndStalePinSweep` → `statusPinBootCleanup`
|
|
9
12
|
* → `unpinChatMessage`) has TWO independent preconditions, and they are
|
|
10
13
|
* satisfied at two different points of the gateway's boot:
|
|
11
14
|
*
|
|
@@ -91,16 +94,16 @@ export function createBootSweepGate(args: {
|
|
|
91
94
|
/** Dependencies of {@link runBootPinSweepSteps}. All side effects are injected
|
|
92
95
|
* so the sequencing contract is provable without a gateway. */
|
|
93
96
|
export interface BootPinSweepSteps {
|
|
94
|
-
/** Pure fs scan for
|
|
95
|
-
* BEFORE the reapers, which empty those same stores. */
|
|
96
|
-
|
|
97
|
+
/** Pure fs scan for the `(chat, thread)` targets with a prior-session pin
|
|
98
|
+
* record. MUST run BEFORE the reapers, which empty those same stores. */
|
|
99
|
+
scanSweepTargets: () => SweepTarget[]
|
|
97
100
|
statusPinCleanup: () => Promise<unknown>
|
|
98
101
|
activityCardReaper: () => Promise<unknown>
|
|
99
102
|
queuedCardReaper: () => Promise<unknown>
|
|
100
|
-
/** Flips the flag authorising the
|
|
103
|
+
/** Flips the flag authorising the stale-pin drain — for this sweep AND for
|
|
101
104
|
* later lazy first-inbound sweeps. */
|
|
102
|
-
|
|
103
|
-
|
|
105
|
+
enableSweep: () => void
|
|
106
|
+
sweepTarget: (target: SweepTarget) => Promise<unknown>
|
|
104
107
|
log?: (line: string) => void
|
|
105
108
|
}
|
|
106
109
|
|
|
@@ -133,9 +136,9 @@ async function step(name: string, fn: () => Promise<unknown>, log: (line: string
|
|
|
133
136
|
* bare sequential `await` chain. `runStatusPinBootCleanup` absorbs its own
|
|
134
137
|
* per-row throws, but the two card reapers issue real Bot API calls and can
|
|
135
138
|
* reject outright — and a rejection there did not merely skip the later
|
|
136
|
-
* reapers, it skipped `
|
|
137
|
-
*
|
|
138
|
-
* first-inbound sweep), so one throwing reaper silently disabled
|
|
139
|
+
* reapers, it skipped `enableSweep()`. That flag authorises the stale-pin
|
|
140
|
+
* drain for the WHOLE SESSION (the boot sweep AND every later lazy
|
|
141
|
+
* first-inbound sweep), so one throwing reaper silently disabled stale-pin
|
|
139
142
|
* cleanup entirely.
|
|
140
143
|
*
|
|
141
144
|
* That was unreachable before #3664 only because the sweep never actually ran
|
|
@@ -149,9 +152,9 @@ async function step(name: string, fn: () => Promise<unknown>, log: (line: string
|
|
|
149
152
|
export async function runBootPinSweepSteps(deps: BootPinSweepSteps): Promise<void> {
|
|
150
153
|
const log = deps.log ?? ((l: string) => process.stderr.write(l))
|
|
151
154
|
|
|
152
|
-
let
|
|
153
|
-
await step('
|
|
154
|
-
|
|
155
|
+
let targets: SweepTarget[] = []
|
|
156
|
+
await step('sweep-target-scan', async () => {
|
|
157
|
+
targets = deps.scanSweepTargets()
|
|
155
158
|
}, log)
|
|
156
159
|
|
|
157
160
|
await step('status-pin-cleanup', deps.statusPinCleanup, log)
|
|
@@ -159,6 +162,8 @@ export async function runBootPinSweepSteps(deps: BootPinSweepSteps): Promise<voi
|
|
|
159
162
|
await step('queued-card-reaper', deps.queuedCardReaper, log)
|
|
160
163
|
|
|
161
164
|
// Unconditional: reached even when every step above threw. See the docblock.
|
|
162
|
-
await step('enable-
|
|
163
|
-
for (const
|
|
165
|
+
await step('enable-sweep', async () => deps.enableSweep(), log)
|
|
166
|
+
for (const t of targets) {
|
|
167
|
+
await step(`stale-pin-sweep:${sweepTargetKey(t.chatId, t.threadId)}`, () => deps.sweepTarget(t), log)
|
|
168
|
+
}
|
|
164
169
|
}
|