switchroom 0.19.17 → 0.19.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/run-hook.sh +148 -0
- package/bin/workspace-dynamic-hook.sh +147 -38
- package/dist/agent-scheduler/index.js +11 -3
- package/dist/auth-broker/index.js +29 -4
- package/dist/cli/notion-write-pretool.mjs +11 -3
- package/dist/cli/switchroom.js +8306 -7619
- package/dist/host-control/main.js +626 -36
- package/dist/vault/approvals/kernel-server.js +30 -5
- package/dist/vault/broker/server.js +71 -18
- package/package.json +3 -2
- package/profiles/_base/start.sh.hbs +8 -4
- package/profiles/coding/CLAUDE.md.hbs +1 -1
- package/profiles/default/CLAUDE.md.hbs +3 -3
- package/profiles/executive-assistant/CLAUDE.md.hbs +1 -1
- package/profiles/health-coach/CLAUDE.md.hbs +1 -1
- package/skills/mental-model-curator/SKILL.md +8 -6
- package/telegram-plugin/bridge/bridge.ts +11 -19
- package/telegram-plugin/bridge/mcp-instructions.ts +87 -0
- package/telegram-plugin/dist/bridge/bridge.js +15 -20
- package/telegram-plugin/dist/gateway/gateway.js +763 -373
- package/telegram-plugin/dist/server.js +19 -20
- package/telegram-plugin/gateway/boot-card.ts +5 -1
- package/telegram-plugin/gateway/boot-probes.ts +113 -0
- package/telegram-plugin/gateway/config-approval-handler.test.ts +54 -0
- package/telegram-plugin/gateway/config-approval-handler.ts +16 -1
- package/telegram-plugin/gateway/disconnect-flush.ts +17 -0
- package/telegram-plugin/gateway/gateway.ts +43 -1
- package/telegram-plugin/gateway/handback-preturn-signal.ts +61 -7
- package/telegram-plugin/gateway/ipc-protocol.ts +5 -0
- package/telegram-plugin/gateway/ipc-server.ts +13 -0
- package/telegram-plugin/gateway/liveness-wiring.ts +125 -5
- package/telegram-plugin/gateway/obligation-ledger.ts +84 -4
- package/telegram-plugin/gateway/resume-inbound-builder.ts +13 -4
- package/telegram-plugin/gateway/stream-render.ts +24 -5
- package/telegram-plugin/hooks/secret-guard-pretool.mjs +249 -76
- package/telegram-plugin/registry/turns-schema.test.ts +8 -3
- package/telegram-plugin/registry/turns-schema.ts +40 -12
- package/telegram-plugin/runtime-metrics.ts +14 -0
- package/telegram-plugin/silence-poke.ts +138 -0
- package/telegram-plugin/tests/boot-probe-drift.test.ts +152 -0
- package/telegram-plugin/tests/gateway-disconnect-flush.test.ts +32 -0
- package/telegram-plugin/tests/handback-preturn-signal.test.ts +62 -0
- package/telegram-plugin/tests/helpers/liveness-wiring-fixture.ts +178 -0
- package/telegram-plugin/tests/ipc-server-validate-config-approval.test.ts +95 -0
- package/telegram-plugin/tests/mcp-instructions-budget.test.ts +184 -0
- package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +22 -2
- package/telegram-plugin/tests/obligation-determinism.test.ts +114 -3
- package/telegram-plugin/tests/obligation-ledger.test.ts +310 -0
- package/telegram-plugin/tests/registry-turns.test.ts +13 -0
- package/telegram-plugin/tests/resume-inbound-builder.test.ts +15 -0
- package/telegram-plugin/tests/secret-guard-pretool.test.ts +347 -16
- package/telegram-plugin/tests/silence-poke-orphan-reap.test.ts +392 -0
- package/telegram-plugin/tests/silence-poke-teardown-notice.test.ts +301 -0
- package/telegram-plugin/tests/stream-render-golden.test.ts +103 -1
- package/telegram-plugin/tests/tts-normalize.test.ts +43 -0
- package/telegram-plugin/tests/voice-normalize-text.test.ts +212 -3
- package/telegram-plugin/tts-normalize.ts +6 -4
- package/telegram-plugin/voice-normalize-text.ts +168 -11
- package/vendor/hindsight-memory/CHANGELOG.md +73 -0
- package/vendor/hindsight-memory/scripts/lib/config.py +8 -3
- package/vendor/hindsight-memory/scripts/lib/directives.py +62 -4
- package/vendor/hindsight-memory/scripts/recall.py +257 -12
- package/vendor/hindsight-memory/scripts/retain.py +12 -6
- package/vendor/hindsight-memory/scripts/tests/test_directives.py +80 -9
- package/vendor/hindsight-memory/scripts/tests/test_recall_integration.py +362 -18
- package/vendor/hindsight-memory/settings.json +1 -1
|
@@ -10,15 +10,75 @@
|
|
|
10
10
|
* Output: exit 0 + empty stdout → allow.
|
|
11
11
|
* exit 0 + JSON on stdout with `decision: "block"` + `reason` → block.
|
|
12
12
|
*
|
|
13
|
-
* Performance note (
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
* keys × every tool call, that compounded to seconds of overhead per turn.
|
|
13
|
+
* Performance note (#472 finding #7, then #3543):
|
|
14
|
+
* Generation 1 forked `switchroom vault list` + `switchroom vault get` per
|
|
15
|
+
* key — each fork paid ~785ms of CLI cold-start. Seconds per turn.
|
|
17
16
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
17
|
+
* Generation 2 spoke the broker's NDJSON unix-socket protocol directly
|
|
18
|
+
* (see src/vault/broker/protocol.ts) but issued `list` + N `get` requests
|
|
19
|
+
* SEQUENTIALLY over one connection. Its header claimed "sub-10ms total
|
|
20
|
+
* even with several keys". That claim was wrong by ~15×: measured
|
|
21
|
+
* in-container on 2026-07-25 the hook cost ~145ms per tool call, of which
|
|
22
|
+
* ~132ms was 1 + N sequential round trips at ~19ms each (#3543). A false
|
|
23
|
+
* perf claim in a header comment is how that stayed invisible; the numbers
|
|
24
|
+
* below are measured, not assumed.
|
|
25
|
+
*
|
|
26
|
+
* Generation 3 (this one) keeps the same two logical phases but fans the
|
|
27
|
+
* `get` phase out CONCURRENTLY, one request per connection, so the CLIENT
|
|
28
|
+
* issues ~2 round trips' worth of requests instead of 1 + N serialized
|
|
29
|
+
* ones.
|
|
30
|
+
*
|
|
31
|
+
* ⚠ READ THIS BEFORE QUOTING A NUMBER. Client concurrency is NOT the
|
|
32
|
+
* whole cost, and an earlier draft of this header claimed "~2 round trips
|
|
33
|
+
* regardless of key count" — which was false against the real broker, for
|
|
34
|
+
* the same reason generation 2's "sub-10ms" claim was false. Whenever the
|
|
35
|
+
* request carries a token (which is ALWAYS, in an agent container:
|
|
36
|
+
* readVaultToken() below always finds `.vault-token`), the broker runs
|
|
37
|
+
* `validateGrant` per request — `list` at src/vault/broker/server.ts:1105
|
|
38
|
+
* and every `get` at :1284 — and that awaits a `bcryptjs.compare`
|
|
39
|
+
* (src/vault/grants.ts, BCRYPT_COST=10). bcryptjs is pure JS and BLOCKS
|
|
40
|
+
* the broker's single event loop: measured 2026-07-25, one compare is
|
|
41
|
+
* ~57ms on the fleet host and 8 "concurrent" compares took 459ms — fully
|
|
42
|
+
* serialized. The broker is shared by every agent, so N keys cost the
|
|
43
|
+
* broker (N+1) × compare of serialized CPU that no amount of client
|
|
44
|
+
* fan-out can shorten.
|
|
45
|
+
*
|
|
46
|
+
* That is why #3574 also added an exact positive-result memo for the
|
|
47
|
+
* bcrypt comparison in src/vault/grants.ts (revocation/expiry/key_allow
|
|
48
|
+
* are still read live from SQLite on every call — only the immutable
|
|
49
|
+
* "does this secret hash to this digest" fact is memoized). With it, the
|
|
50
|
+
* per-request broker CPU collapses to a Map lookup after the first
|
|
51
|
+
* request and the fan-out delivers what it promises. Without it, this
|
|
52
|
+
* hook's real cost is dominated by broker CPU, not by round trips.
|
|
53
|
+
*
|
|
54
|
+
* The failure mode when broker CPU DOES saturate is a security
|
|
55
|
+
* degradation, not a hang: on deadline expiry we keep whatever arrived
|
|
56
|
+
* and scan a PARTIAL key set, so a secret can be guarded on one tool call
|
|
57
|
+
* and unguarded on the next. `warnOnUnreachableGets` below exists so that
|
|
58
|
+
* can never happen silently.
|
|
59
|
+
*
|
|
60
|
+
* Why one request per connection rather than pipelining N gets down the
|
|
61
|
+
* single connection: the broker dispatches each line to an async handler
|
|
62
|
+
* WITHOUT awaiting it (src/vault/broker/server.ts:924), and the `get`
|
|
63
|
+
* handler awaits `validateGrant` before replying (server.ts:1284). So
|
|
64
|
+
* pipelined responses are not guaranteed to come back in request order,
|
|
65
|
+
* and the wire format carries no request id to correlate them by
|
|
66
|
+
* (protocol.ts:452 OkEntryResponseSchema is `{ ok, entry }` — no key, no
|
|
67
|
+
* id). One request per connection is exactly the shape the protocol
|
|
68
|
+
* documents ("one request per connection turn, one response") and needs
|
|
69
|
+
* no broker change, so it ships without a broker restart.
|
|
70
|
+
*
|
|
71
|
+
* Why not a TTL cache of vault values: values rotate at runtime through
|
|
72
|
+
* the broker's `put` op, and this hook cannot see the vault file to key an
|
|
73
|
+
* invalidation off its mtime. A cache would let a freshly-rotated secret
|
|
74
|
+
* go unguarded for the cache lifetime — i.e. it would make the guard fail
|
|
75
|
+
* open in a case where it currently blocks. Rejected on those grounds.
|
|
76
|
+
*
|
|
77
|
+
* Measured client-side only (fake broker with an ASYNC 19ms service delay,
|
|
78
|
+
* 6 keys, median of 5): sequential 167.7ms → concurrent ~59ms. Treat this
|
|
79
|
+
* as an upper bound on what the fan-out alone buys — a fake broker that
|
|
80
|
+
* sleeps does not model the real broker's synchronous bcrypt CPU. See the
|
|
81
|
+
* CPU-bound fake-broker test in the sibling test file.
|
|
22
82
|
*
|
|
23
83
|
* When the broker is unreachable (not running, socket missing, denied)
|
|
24
84
|
* the hook fails open — same behavior as before. Vault security is owned
|
|
@@ -37,6 +97,34 @@ const BROKER_SOCKET =
|
|
|
37
97
|
?? join(homedir(), '.switchroom', 'vault-broker.sock')
|
|
38
98
|
const BROKER_TIMEOUT_MS = 1500
|
|
39
99
|
const MIN_VALUE_LENGTH_TO_GUARD = 8
|
|
100
|
+
/**
|
|
101
|
+
* Max concurrent `get` connections to the broker (#3543). The fan-out is
|
|
102
|
+
* what makes this hook O(1) round trips instead of O(N), but an unbounded
|
|
103
|
+
* fan-out on a large vault would open one unix socket per key at once. 32
|
|
104
|
+
* keeps a typical vault (single digits to low tens of keys) at a single
|
|
105
|
+
* round trip while capping the worst case at ceil(N/32) rounds.
|
|
106
|
+
*
|
|
107
|
+
* ⚠ COUPLING — this hook now needs up to MAX_CONCURRENT_GETS + 1 broker
|
|
108
|
+
* connections where generation 2 needed exactly ONE. That makes the guard
|
|
109
|
+
* sensitive to a limit the broker does not currently impose. Verified as of
|
|
110
|
+
* 2026-07-25: src/vault/broker/server.ts sets no `maxConnections`, no rate
|
|
111
|
+
* limit and no per-connection mutex, `validateGrant` is a read-only SQLite
|
|
112
|
+
* lookup, and the socket is bind-mounted into agent containers with no
|
|
113
|
+
* relay in between — so nothing caps us today.
|
|
114
|
+
*
|
|
115
|
+
* But if a future operator adds DoS hardening (server.maxConnections, a
|
|
116
|
+
* connection rate limiter, an inetd-style relay), the surplus `get`
|
|
117
|
+
* connections would be refused, `brokerRequest` would resolve null for
|
|
118
|
+
* those keys, and a tool call carrying one of their secrets would turn
|
|
119
|
+
* from BLOCK into a silent allow. Reproduced against a 1-connection and a
|
|
120
|
+
* 3-connection fake broker: old = BLOCK, new = allow.
|
|
121
|
+
*
|
|
122
|
+
* The deterministic guard for that is `warnOnUnreachableGets` below — a
|
|
123
|
+
* connection cap can no longer degrade the guard silently; it always
|
|
124
|
+
* leaves a line on stderr (captured by bin/run-hook.sh). If you ever cap
|
|
125
|
+
* broker connections, lower MAX_CONCURRENT_GETS below that cap.
|
|
126
|
+
*/
|
|
127
|
+
const MAX_CONCURRENT_GETS = 32
|
|
40
128
|
|
|
41
129
|
// ─── Stdin ────────────────────────────────────────────────────────────────
|
|
42
130
|
|
|
@@ -72,92 +160,177 @@ function readVaultToken() {
|
|
|
72
160
|
// ─── Inline NDJSON broker client ──────────────────────────────────────────
|
|
73
161
|
|
|
74
162
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
163
|
+
* Run exactly ONE request/response turn on a fresh connection, then close.
|
|
164
|
+
*
|
|
165
|
+
* This is the unit the broker protocol documents (protocol.ts:8-11 — "one
|
|
166
|
+
* request per connection turn, one response"). Keeping it to one request per
|
|
167
|
+
* socket is what makes it safe to run many of these concurrently: no
|
|
168
|
+
* response-ordering assumption is needed, because each socket only ever
|
|
169
|
+
* carries one response.
|
|
170
|
+
*
|
|
171
|
+
* Never rejects. Resolves `null` on ANY failure — connect refused, socket
|
|
172
|
+
* error, deadline passed, short read, malformed JSON. Callers treat `null`
|
|
173
|
+
* as "this key contributed nothing", which is the same fail-open posture the
|
|
174
|
+
* sequential implementation had.
|
|
175
|
+
*
|
|
176
|
+
* @param {object} req request object to serialize
|
|
177
|
+
* @param {number} deadline absolute `Date.now()` ms after which we give up
|
|
178
|
+
* @returns {Promise<object|null>} parsed response object, or null
|
|
78
179
|
*/
|
|
79
|
-
function
|
|
180
|
+
function brokerRequest(req, deadline) {
|
|
80
181
|
return new Promise((resolve) => {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
let
|
|
84
|
-
let done = false
|
|
85
|
-
let pending = null // { resolve(line) }
|
|
182
|
+
let settled = false
|
|
183
|
+
let sock
|
|
184
|
+
let timer
|
|
86
185
|
const finish = (result) => {
|
|
87
|
-
if (
|
|
88
|
-
|
|
89
|
-
|
|
186
|
+
if (settled) return
|
|
187
|
+
settled = true
|
|
188
|
+
clearTimeout(timer)
|
|
189
|
+
try { sock?.destroy() } catch { /* best-effort */ }
|
|
90
190
|
resolve(result)
|
|
91
191
|
}
|
|
92
192
|
|
|
93
|
-
const
|
|
193
|
+
const remaining = deadline - Date.now()
|
|
194
|
+
if (remaining <= 0) return finish(null)
|
|
195
|
+
timer = setTimeout(() => finish(null), remaining)
|
|
94
196
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
}
|
|
197
|
+
try {
|
|
198
|
+
sock = connect(BROKER_SOCKET)
|
|
199
|
+
} catch {
|
|
200
|
+
return finish(null)
|
|
201
|
+
}
|
|
100
202
|
|
|
203
|
+
let buf = ''
|
|
204
|
+
sock.on('error', () => finish(null))
|
|
205
|
+
// A close before we saw a full line is a failure, not an empty success.
|
|
206
|
+
sock.on('close', () => finish(null))
|
|
101
207
|
sock.on('data', (chunk) => {
|
|
102
208
|
buf += chunk.toString('utf8')
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
const p = pending
|
|
110
|
-
pending = null
|
|
111
|
-
p.resolve(line)
|
|
112
|
-
}
|
|
113
|
-
}
|
|
209
|
+
const idx = buf.indexOf('\n')
|
|
210
|
+
if (idx < 0) return
|
|
211
|
+
const line = buf.slice(0, idx)
|
|
212
|
+
let parsed
|
|
213
|
+
try { parsed = JSON.parse(line) } catch { return finish(null) }
|
|
214
|
+
finish(parsed)
|
|
114
215
|
})
|
|
115
|
-
|
|
116
|
-
function send(req) {
|
|
117
|
-
return new Promise((respond) => {
|
|
118
|
-
pending = { resolve: respond }
|
|
119
|
-
sock.write(JSON.stringify(req) + '\n')
|
|
120
|
-
})
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
sock.on('connect', async () => {
|
|
216
|
+
sock.on('connect', () => {
|
|
124
217
|
try {
|
|
125
|
-
|
|
126
|
-
const listReq = token ? { v: 1, op: 'list', token } : { v: 1, op: 'list' }
|
|
127
|
-
const listLine = await send(listReq)
|
|
128
|
-
let listRsp
|
|
129
|
-
try { listRsp = JSON.parse(listLine) } catch { return finish([]) }
|
|
130
|
-
if (!listRsp || listRsp.ok !== true || !Array.isArray(listRsp.keys)) {
|
|
131
|
-
// LOCKED / DENIED / etc. — fall through, no values.
|
|
132
|
-
return finish([])
|
|
133
|
-
}
|
|
134
|
-
// 2. fetch each value
|
|
135
|
-
const values = []
|
|
136
|
-
for (const k of listRsp.keys) {
|
|
137
|
-
const getReq = token
|
|
138
|
-
? { v: 1, op: 'get', key: k, token }
|
|
139
|
-
: { v: 1, op: 'get', key: k }
|
|
140
|
-
const getLine = await send(getReq)
|
|
141
|
-
let getRsp
|
|
142
|
-
try { getRsp = JSON.parse(getLine) } catch { continue }
|
|
143
|
-
if (!getRsp || getRsp.ok !== true || !getRsp.entry) continue
|
|
144
|
-
// Only string-kind entries are scannable haystack candidates;
|
|
145
|
-
// binary/files entries can't be substring-matched against a
|
|
146
|
-
// tool-input string in any meaningful way.
|
|
147
|
-
if (getRsp.entry.kind === 'string'
|
|
148
|
-
&& typeof getRsp.entry.value === 'string'
|
|
149
|
-
&& getRsp.entry.value.length >= MIN_VALUE_LENGTH_TO_GUARD) {
|
|
150
|
-
values.push({ key: k, value: getRsp.entry.value })
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
finish(values)
|
|
218
|
+
sock.write(JSON.stringify(req) + '\n')
|
|
154
219
|
} catch {
|
|
155
|
-
finish(
|
|
220
|
+
finish(null)
|
|
156
221
|
}
|
|
157
222
|
})
|
|
158
223
|
})
|
|
159
224
|
}
|
|
160
225
|
|
|
226
|
+
/**
|
|
227
|
+
* Map `worker` over `items` with at most `limit` in flight at once.
|
|
228
|
+
* Preserves input order in the result array.
|
|
229
|
+
*/
|
|
230
|
+
async function mapWithConcurrency(items, limit, worker) {
|
|
231
|
+
const results = new Array(items.length)
|
|
232
|
+
let cursor = 0
|
|
233
|
+
const runners = new Array(Math.min(limit, items.length)).fill(0).map(async () => {
|
|
234
|
+
for (;;) {
|
|
235
|
+
const i = cursor++
|
|
236
|
+
if (i >= items.length) return
|
|
237
|
+
results[i] = await worker(items[i], i)
|
|
238
|
+
}
|
|
239
|
+
})
|
|
240
|
+
await Promise.all(runners)
|
|
241
|
+
return results
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Load every guardable vault value: one `list` turn, then a CONCURRENT fan-out
|
|
246
|
+
* of `get` turns (#3543 — this replaces a sequential 1 + N round-trip loop).
|
|
247
|
+
*
|
|
248
|
+
* Fail-open contract, unchanged from the sequential version except where
|
|
249
|
+
* noted:
|
|
250
|
+
* - `list` fails / is denied / broker locked → `[]` (hook allows).
|
|
251
|
+
* - an individual `get` fails or is denied → that key is skipped, the
|
|
252
|
+
* remaining keys are still guarded.
|
|
253
|
+
* - overall deadline expires → whatever HAS been fetched
|
|
254
|
+
* is returned and still scanned. The sequential version discarded
|
|
255
|
+
* everything on timeout, so this is strictly MORE coverage, never less.
|
|
256
|
+
*/
|
|
257
|
+
async function loadVaultValuesViaBroker() {
|
|
258
|
+
const deadline = Date.now() + BROKER_TIMEOUT_MS
|
|
259
|
+
const token = readVaultToken()
|
|
260
|
+
|
|
261
|
+
// 1. list keys — one round trip.
|
|
262
|
+
const listRsp = await brokerRequest(
|
|
263
|
+
token ? { v: 1, op: 'list', token } : { v: 1, op: 'list' },
|
|
264
|
+
deadline,
|
|
265
|
+
)
|
|
266
|
+
if (!listRsp || listRsp.ok !== true || !Array.isArray(listRsp.keys)) {
|
|
267
|
+
// LOCKED / DENIED / unreachable / malformed — no values.
|
|
268
|
+
return []
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// 2. fetch every value concurrently — one round trip regardless of N
|
|
272
|
+
// (up to MAX_CONCURRENT_GETS in flight).
|
|
273
|
+
// `unreachable` counts keys whose `get` produced NO response at all
|
|
274
|
+
// (connect refused, socket error, deadline) as opposed to a broker
|
|
275
|
+
// reply of DENIED/UNKNOWN_KEY. That distinction is the connection-cap
|
|
276
|
+
// signal — see the MAX_CONCURRENT_GETS note above.
|
|
277
|
+
let unreachable = 0
|
|
278
|
+
const fetched = await mapWithConcurrency(
|
|
279
|
+
listRsp.keys,
|
|
280
|
+
MAX_CONCURRENT_GETS,
|
|
281
|
+
async (k) => {
|
|
282
|
+
const getRsp = await brokerRequest(
|
|
283
|
+
token ? { v: 1, op: 'get', key: k, token } : { v: 1, op: 'get', key: k },
|
|
284
|
+
deadline,
|
|
285
|
+
)
|
|
286
|
+
if (getRsp === null) unreachable++
|
|
287
|
+
if (!getRsp || getRsp.ok !== true || !getRsp.entry) return null
|
|
288
|
+
// Only string-kind entries are scannable haystack candidates;
|
|
289
|
+
// binary/files entries can't be substring-matched against a
|
|
290
|
+
// tool-input string in any meaningful way.
|
|
291
|
+
if (getRsp.entry.kind === 'string'
|
|
292
|
+
&& typeof getRsp.entry.value === 'string'
|
|
293
|
+
&& getRsp.entry.value.length >= MIN_VALUE_LENGTH_TO_GUARD) {
|
|
294
|
+
return { key: k, value: getRsp.entry.value }
|
|
295
|
+
}
|
|
296
|
+
return null
|
|
297
|
+
},
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
warnOnUnreachableGets(unreachable, listRsp.keys.length)
|
|
301
|
+
|
|
302
|
+
return fetched.filter((v) => v !== null)
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Deterministic degradation signal (#3574 review).
|
|
307
|
+
*
|
|
308
|
+
* The fan-out silently drops any key whose `get` never came back, and a
|
|
309
|
+
* dropped key is an UNGUARDED secret — exactly the failure a broker
|
|
310
|
+
* connection cap would produce. Prose in a comment cannot catch that at
|
|
311
|
+
* runtime, so emit one line on stderr instead. bin/run-hook.sh captures
|
|
312
|
+
* hook stderr, so `grep secret-guard ...` answers "is the guard degraded?"
|
|
313
|
+
* without anyone having to reason about connection budgets.
|
|
314
|
+
*
|
|
315
|
+
* One aggregate line, not one per key: bounded output regardless of vault
|
|
316
|
+
* size. stderr only — stdout is the hook's decision channel and must stay
|
|
317
|
+
* empty on allow.
|
|
318
|
+
*/
|
|
319
|
+
function warnOnUnreachableGets(unreachable, total) {
|
|
320
|
+
if (unreachable <= 0) return
|
|
321
|
+
try {
|
|
322
|
+
process.stderr.write(
|
|
323
|
+
`secret-guard-pretool: DEGRADED — ${unreachable}/${total} vault `
|
|
324
|
+
+ `key(s) unresolved within the ${BROKER_TIMEOUT_MS}ms deadline `
|
|
325
|
+
+ `(no broker response: deadline expired, connection refused, or `
|
|
326
|
+
+ `socket error). Those secrets are NOT guarded for this tool call. `
|
|
327
|
+
+ `Likely causes: broker CPU saturation (per-request bcrypt in `
|
|
328
|
+
+ `validateGrant), or a broker connection cap below `
|
|
329
|
+
+ `MAX_CONCURRENT_GETS (currently ${MAX_CONCURRENT_GETS}).\n`,
|
|
330
|
+
)
|
|
331
|
+
} catch { /* stderr closed — never let logging change the decision */ }
|
|
332
|
+
}
|
|
333
|
+
|
|
161
334
|
// ─── Scan ─────────────────────────────────────────────────────────────────
|
|
162
335
|
|
|
163
336
|
function scanToolInput(toolInput, vaultValues) {
|
|
@@ -169,7 +169,7 @@ describe('reapStaleOpenTurns (#2918 mid-session sweep)', () => {
|
|
|
169
169
|
db.prepare('UPDATE turns SET started_at = ? WHERE turn_key = ?').run(startedAt, turnKey)
|
|
170
170
|
}
|
|
171
171
|
|
|
172
|
-
it(
|
|
172
|
+
it("stamps an ownerless open row aged past the TTL as 'reaped_stale', not 'restart' (#3555)", () => {
|
|
173
173
|
const db = openTurnsDbInMemory()
|
|
174
174
|
const now = 1_000_000_000_000
|
|
175
175
|
recordTurnStart(db, { turnKey: 'dm:dead', chatId: '111' })
|
|
@@ -183,7 +183,12 @@ describe('reapStaleOpenTurns (#2918 mid-session sweep)', () => {
|
|
|
183
183
|
expect(res.reapedTurnKeys).toEqual(['dm:dead'])
|
|
184
184
|
const turn = getTurnByKey(db, 'dm:dead')
|
|
185
185
|
expect(turn?.ended_at).toBe(now)
|
|
186
|
-
|
|
186
|
+
// #3555: this sweep is NOT a restart. Stamping 'restart' contaminated
|
|
187
|
+
// every restart statistic derived from the column (1055 rows on one host,
|
|
188
|
+
// 91.7% clustered at the 15-min reaper TTL, against exactly ONE actual
|
|
189
|
+
// gateway exit in the same window).
|
|
190
|
+
expect(turn?.ended_via).toBe('reaped_stale')
|
|
191
|
+
expect(turn?.ended_via).not.toBe('restart')
|
|
187
192
|
db.close()
|
|
188
193
|
})
|
|
189
194
|
|
|
@@ -233,7 +238,7 @@ describe('reapStaleOpenTurns (#2918 mid-session sweep)', () => {
|
|
|
233
238
|
now,
|
|
234
239
|
})
|
|
235
240
|
expect(res.reapedTurnKeys).toEqual(['dm:dead'])
|
|
236
|
-
expect(getTurnByKey(db, 'dm:dead')?.ended_via).toBe('
|
|
241
|
+
expect(getTurnByKey(db, 'dm:dead')?.ended_via).toBe('reaped_stale')
|
|
237
242
|
expect(getTurnByKey(db, 'dm:live')?.ended_at).toBeNull()
|
|
238
243
|
db.close()
|
|
239
244
|
})
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* thread_id TEXT -- nullable: forum topics only
|
|
18
18
|
* started_at INTEGER NOT NULL -- unix ms
|
|
19
19
|
* ended_at INTEGER -- nullable until turn ends
|
|
20
|
-
* ended_via TEXT -- 'stop'
|
|
20
|
+
* ended_via TEXT -- 'stop'|'sigterm'|'restart'|'reaped_stale'|'timeout'|'unknown'
|
|
21
21
|
* last_assistant_msg_id TEXT -- last outbound message_id in this turn
|
|
22
22
|
* last_assistant_done INTEGER -- 0|1; 1 = stream_reply done=true sent
|
|
23
23
|
* last_user_msg_id TEXT -- inbound message_id that started the turn
|
|
@@ -84,7 +84,26 @@ function loadDatabaseClass(): SqliteDatabaseConstructor {
|
|
|
84
84
|
// Types
|
|
85
85
|
// ---------------------------------------------------------------------------
|
|
86
86
|
|
|
87
|
-
|
|
87
|
+
/**
|
|
88
|
+
* How a turn stopped being open.
|
|
89
|
+
*
|
|
90
|
+
* `'reaped_stale'` (#3555) is distinct from `'restart'` on purpose. The
|
|
91
|
+
* mid-session sweep (`reapStaleOpenTurns`) used to stamp `'restart'`, which
|
|
92
|
+
* made every restart-derived statistic unusable: 1055 rows on one host, 967
|
|
93
|
+
* of them (91.7%) clustered at 15-19 min — the signature of the 15-min
|
|
94
|
+
* `MID_SESSION_CARD_REAPER_TTL_MS` on a 5-min sweep, NOT of gateway
|
|
95
|
+
* restarts (the same window's log holds exactly ONE `[supervise] gateway
|
|
96
|
+
* exited`). Behaviourally the two are identical — both are clean interrupts,
|
|
97
|
+
* both are in `INTERRUPTED_VIA`, both map to a `'resume'` — but they are now
|
|
98
|
+
* separable in the data.
|
|
99
|
+
*/
|
|
100
|
+
export type TurnEndedVia =
|
|
101
|
+
| 'stop'
|
|
102
|
+
| 'sigterm'
|
|
103
|
+
| 'restart'
|
|
104
|
+
| 'reaped_stale'
|
|
105
|
+
| 'timeout'
|
|
106
|
+
| 'unknown'
|
|
88
107
|
|
|
89
108
|
export interface Turn {
|
|
90
109
|
turn_key: string
|
|
@@ -554,7 +573,7 @@ export interface ReapStaleOpenTurnsOpts {
|
|
|
554
573
|
}
|
|
555
574
|
|
|
556
575
|
export interface ReapStaleOpenTurnsResult {
|
|
557
|
-
/** Rows stamped `ended_via='
|
|
576
|
+
/** Rows stamped `ended_via='reaped_stale'` by this sweep (#3555). */
|
|
558
577
|
reaped: number
|
|
559
578
|
/** The turn_keys that were stamped, for logging / card finalization. */
|
|
560
579
|
reapedTurnKeys: string[]
|
|
@@ -568,15 +587,19 @@ export interface ReapStaleOpenTurnsResult {
|
|
|
568
587
|
* leaves its row `ended_at IS NULL`, and its activity card keeps spinning
|
|
569
588
|
* until the NEXT gateway boot (often many hours later). This sweep runs on a
|
|
570
589
|
* periodic timer inside the live gateway and stamps those ownerless open rows
|
|
571
|
-
*
|
|
572
|
-
* reaper uses for a non-hung orphan) so the stale card can be finalized
|
|
573
|
-
* without waiting for a restart.
|
|
590
|
+
* so the stale card can be finalized without waiting for a restart.
|
|
574
591
|
*
|
|
575
592
|
* CORRECTNESS: only rows that are BOTH (a) not owned by any live turn
|
|
576
593
|
* (`turn_key ∉ activeTurnKeys`) AND (b) older than `ttlMs` are swept. A
|
|
577
594
|
* healthy in-flight turn — however long it runs — is always in
|
|
578
|
-
* `activeTurnKeys` and is never touched.
|
|
579
|
-
*
|
|
595
|
+
* `activeTurnKeys` and is never touched.
|
|
596
|
+
*
|
|
597
|
+
* #3555: this used to stamp `'restart'`, borrowing the boot reaper's
|
|
598
|
+
* classification. That was a lie in the data — nothing here restarted; the
|
|
599
|
+
* sweep merely gave up on a row after `ttlMs`. It stamps `'reaped_stale'`
|
|
600
|
+
* now. `'reaped_stale'` is a member of `INTERRUPTED_VIA` and maps to the
|
|
601
|
+
* same `'resume'` in `selectResumeBuilder`, so the resume/report policy is
|
|
602
|
+
* byte-for-byte unchanged; only the forensic label differs.
|
|
580
603
|
*/
|
|
581
604
|
export function reapStaleOpenTurns(
|
|
582
605
|
db: SqliteDatabase,
|
|
@@ -594,7 +617,7 @@ export function reapStaleOpenTurns(
|
|
|
594
617
|
const stamp = db.prepare(`
|
|
595
618
|
UPDATE turns
|
|
596
619
|
SET ended_at = ?,
|
|
597
|
-
ended_via = '
|
|
620
|
+
ended_via = 'reaped_stale',
|
|
598
621
|
updated_at = ?
|
|
599
622
|
WHERE turn_key = ? AND ended_at IS NULL
|
|
600
623
|
`)
|
|
@@ -688,6 +711,7 @@ export function listTurnsForAgent(
|
|
|
688
711
|
/** ended_via values that mean "this turn did not finish on its own". */
|
|
689
712
|
const INTERRUPTED_VIA: ReadonlySet<TurnEndedVia> = new Set<TurnEndedVia>([
|
|
690
713
|
'restart',
|
|
714
|
+
'reaped_stale', // #3555 — same clean-interrupt semantics, honest label
|
|
691
715
|
'sigterm',
|
|
692
716
|
'timeout',
|
|
693
717
|
'unknown',
|
|
@@ -782,9 +806,13 @@ export function markAnswerRedelivered(
|
|
|
782
806
|
|
|
783
807
|
/**
|
|
784
808
|
* Return the single most-recently-started turn IFF it was interrupted
|
|
785
|
-
* (`ended_at IS NULL`, or `ended_via` in {restart,
|
|
786
|
-
* unknown}). Returns null when the latest
|
|
787
|
-
* or there are no turns at all.
|
|
809
|
+
* (`ended_at IS NULL`, or `ended_via` in {restart, reaped_stale, sigterm,
|
|
810
|
+
* timeout, unknown} — i.e. `INTERRUPTED_VIA`). Returns null when the latest
|
|
811
|
+
* turn ended cleanly (`'stop'`) or there are no turns at all.
|
|
812
|
+
*
|
|
813
|
+
* `'reaped_stale'` (#3555) is in that set for the same reason `'restart'` is:
|
|
814
|
+
* a row the mid-session sweep gave up on was still a turn the user was owed
|
|
815
|
+
* an answer for. Omitting it would silently drop the resume.
|
|
788
816
|
*
|
|
789
817
|
* This is the resume gate. Keying on the *latest* turn (not "latest
|
|
790
818
|
* interrupted turn anywhere in history") is deliberate: once the agent
|
|
@@ -77,6 +77,20 @@ export type RuntimeMetricEvent =
|
|
|
77
77
|
fallback_kind: 'working' | 'thinking'
|
|
78
78
|
silence_ms: number
|
|
79
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* #3552 — per-turn silence-poke state dropped by the orphan reaper: the turn
|
|
82
|
+
* behind `key` is provably over (no `activeTurnStartedAt` entry AND no
|
|
83
|
+
* current turn) yet its state was still armed, so it is disarmed on the poll
|
|
84
|
+
* tick rather than 300s later at fire time. A healthy fleet trends this
|
|
85
|
+
* toward zero; a persistent stream of it names a turn-end path that is
|
|
86
|
+
* failing to call `silencePoke.endTurn`. `silence_ms` is how long the state
|
|
87
|
+
* had already been orphaned when the reaper caught it.
|
|
88
|
+
*/
|
|
89
|
+
| {
|
|
90
|
+
kind: 'silence_poke_orphan_reaped'
|
|
91
|
+
key: string
|
|
92
|
+
silence_ms: number
|
|
93
|
+
}
|
|
80
94
|
/**
|
|
81
95
|
* #2527 — mid-turn liveness floor decision. `decision: 'fire'` when the
|
|
82
96
|
* quiet "still on it" beat was sent; otherwise the machine-readable skip
|