switchroom 0.14.21 → 0.14.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +0 -1
- package/dist/auth-broker/index.js +0 -1
- package/dist/cli/notion-write-pretool.mjs +0 -1
- package/dist/cli/switchroom.js +14 -6
- package/dist/host-control/main.js +0 -1
- package/dist/vault/approvals/kernel-server.js +0 -1
- package/dist/vault/broker/server.js +0 -1
- package/package.json +3 -3
- package/profiles/_base/start.sh.hbs +11 -24
- package/profiles/_shared/telegram-style.md.hbs +2 -2
- package/profiles/default/CLAUDE.md.hbs +4 -1
- package/skills/switchroom-runtime/SKILL.md +6 -16
- package/telegram-plugin/agent-dir.ts +15 -0
- package/telegram-plugin/dist/gateway/gateway.js +788 -513
- package/telegram-plugin/gateway/gateway.ts +216 -61
- package/telegram-plugin/gateway/inbound-spool.ts +15 -0
- package/telegram-plugin/gateway/resume-inbound-builder.ts +180 -0
- package/telegram-plugin/registry/turns-schema.ts +138 -33
- package/telegram-plugin/stream-reply-handler.ts +1 -11
- package/telegram-plugin/subagent-watcher.ts +79 -5
- package/telegram-plugin/tests/agent-dir.test.ts +25 -0
- package/telegram-plugin/tests/e2e.test.ts +2 -77
- package/telegram-plugin/tests/inbound-spool.test.ts +45 -0
- package/telegram-plugin/tests/multi-turn-continuity.test.ts +0 -1
- package/telegram-plugin/tests/outbound-ordering.test.ts +0 -1
- package/telegram-plugin/tests/parse-mode-rotation.test.ts +0 -1
- package/telegram-plugin/tests/races.test.ts +0 -26
- package/telegram-plugin/tests/registry-turns.test.ts +106 -29
- package/telegram-plugin/tests/resume-inbound-builder.test.ts +182 -0
- package/telegram-plugin/tests/status-accent.test.ts +0 -1
- package/telegram-plugin/tests/stream-reply-error-paths.test.ts +0 -1
- package/telegram-plugin/tests/stream-reply-handler.test.ts +0 -24
- package/telegram-plugin/tests/streaming-e2e.test.ts +0 -1
- package/telegram-plugin/tests/streaming-orchestration.test.ts +0 -1
- package/telegram-plugin/tests/subagent-registry-bugs.test.ts +7 -3
- package/telegram-plugin/tests/subagent-watcher-handback-gaps.test.ts +293 -0
- package/telegram-plugin/tests/subagent-watcher.test.ts +23 -15
- package/telegram-plugin/tests/tool-activity-summary.test.ts +44 -0
- package/telegram-plugin/tests/turns-writer.test.ts +16 -6
- package/telegram-plugin/tool-activity-summary.ts +55 -0
- package/telegram-plugin/uat/driver.ts +3 -1
- package/telegram-plugin/handoff-continuity.ts +0 -206
- package/telegram-plugin/tests/handoff-continuity.test.ts +0 -262
|
@@ -28,11 +28,12 @@
|
|
|
28
28
|
* updated_at INTEGER NOT NULL
|
|
29
29
|
*
|
|
30
30
|
* Boot-time usage:
|
|
31
|
-
* On every gateway boot, call `
|
|
32
|
-
* after opening the DB. Any turn with `ended_at IS NULL` was
|
|
33
|
-
* mid-flight (SIGKILL, OOM, power loss) — it never
|
|
34
|
-
* a clean-shutdown marker.
|
|
35
|
-
*
|
|
31
|
+
* On every gateway boot, call `markOrphanedWithTimeoutClassification(db, …)`
|
|
32
|
+
* immediately after opening the DB. Any turn with `ended_at IS NULL` was
|
|
33
|
+
* killed mid-flight (SIGKILL, OOM, power loss, operator restart) — it never
|
|
34
|
+
* got a chance to write a clean-shutdown marker. The classifier stamps the
|
|
35
|
+
* in-flight turn `'timeout'` when its hang-marker is stale and `'restart'`
|
|
36
|
+
* otherwise; the gateway then resumes or reports accordingly.
|
|
36
37
|
*/
|
|
37
38
|
|
|
38
39
|
import { chmodSync, mkdirSync } from 'fs'
|
|
@@ -98,6 +99,15 @@ export interface Turn {
|
|
|
98
99
|
user_prompt_preview: string | null
|
|
99
100
|
assistant_reply_preview: string | null
|
|
100
101
|
tool_call_count: number | null
|
|
102
|
+
/**
|
|
103
|
+
* Forensic snapshot persisted by the boot-time classifier when a turn is
|
|
104
|
+
* stamped `ended_via='timeout'` (the hang-watchdog window elapsed with no
|
|
105
|
+
* tool progress). Carries the idle duration so a *later* boot can rebuild
|
|
106
|
+
* the watchdog-report inbound after the on-disk turn-active marker — the
|
|
107
|
+
* only live source of the idle age — has already been swept. Null for
|
|
108
|
+
* cleanly-restarted (`'restart'`) orphans.
|
|
109
|
+
*/
|
|
110
|
+
interrupt_reason: string | null
|
|
101
111
|
created_at: number
|
|
102
112
|
updated_at: number
|
|
103
113
|
}
|
|
@@ -137,6 +147,7 @@ const SCHEMA_SQL = `
|
|
|
137
147
|
user_prompt_preview TEXT,
|
|
138
148
|
assistant_reply_preview TEXT,
|
|
139
149
|
tool_call_count INTEGER,
|
|
150
|
+
interrupt_reason TEXT,
|
|
140
151
|
created_at INTEGER NOT NULL,
|
|
141
152
|
updated_at INTEGER NOT NULL
|
|
142
153
|
);
|
|
@@ -151,13 +162,21 @@ const PHASE1_MIGRATIONS = [
|
|
|
151
162
|
`ALTER TABLE turns ADD COLUMN tool_call_count INTEGER`,
|
|
152
163
|
]
|
|
153
164
|
|
|
165
|
+
// Column added for honest-restart-resume. Persists the idle snapshot the
|
|
166
|
+
// boot classifier captures when stamping a turn 'timeout' (see
|
|
167
|
+
// `markOrphanedWithTimeoutClassification`).
|
|
168
|
+
const PHASE2_MIGRATIONS = [
|
|
169
|
+
`ALTER TABLE turns ADD COLUMN interrupt_reason TEXT`,
|
|
170
|
+
]
|
|
171
|
+
|
|
154
172
|
function applySchema(db: SqliteDatabase): void {
|
|
155
173
|
db.exec('PRAGMA journal_mode = WAL')
|
|
156
174
|
db.exec('PRAGMA synchronous = NORMAL')
|
|
157
175
|
db.exec(SCHEMA_SQL)
|
|
158
|
-
// Run migrations
|
|
159
|
-
//
|
|
160
|
-
|
|
176
|
+
// Run migrations. SQLite doesn't support "ADD COLUMN IF NOT EXISTS", so
|
|
177
|
+
// we swallow the "duplicate column" error to stay idempotent on
|
|
178
|
+
// pre-existing registry.db files.
|
|
179
|
+
for (const sql of [...PHASE1_MIGRATIONS, ...PHASE2_MIGRATIONS]) {
|
|
161
180
|
try {
|
|
162
181
|
db.exec(sql)
|
|
163
182
|
} catch (err) {
|
|
@@ -225,6 +244,7 @@ interface RawTurnRow {
|
|
|
225
244
|
user_prompt_preview: string | null
|
|
226
245
|
assistant_reply_preview: string | null
|
|
227
246
|
tool_call_count: number | null
|
|
247
|
+
interrupt_reason: string | null
|
|
228
248
|
created_at: number
|
|
229
249
|
updated_at: number
|
|
230
250
|
}
|
|
@@ -244,6 +264,7 @@ function mapRow(row: RawTurnRow): Turn {
|
|
|
244
264
|
user_prompt_preview: row.user_prompt_preview,
|
|
245
265
|
assistant_reply_preview: row.assistant_reply_preview,
|
|
246
266
|
tool_call_count: row.tool_call_count,
|
|
267
|
+
interrupt_reason: row.interrupt_reason,
|
|
247
268
|
created_at: row.created_at,
|
|
248
269
|
updated_at: row.updated_at,
|
|
249
270
|
}
|
|
@@ -283,7 +304,7 @@ export function recordTurnStart(db: SqliteDatabase, args: RecordTurnStartArgs):
|
|
|
283
304
|
* tool-call count.
|
|
284
305
|
*
|
|
285
306
|
* No-ops gracefully if `turnKey` is not found (turn may have already been
|
|
286
|
-
* swept by `
|
|
307
|
+
* swept by `markOrphanedWithTimeoutClassification` on a prior boot).
|
|
287
308
|
*/
|
|
288
309
|
export function recordTurnEnd(db: SqliteDatabase, args: RecordTurnEndArgs): void {
|
|
289
310
|
const now = Date.now()
|
|
@@ -327,27 +348,96 @@ export function findOrphanedTurns(db: SqliteDatabase, chatId: string): Turn[] {
|
|
|
327
348
|
return rows.map(mapRow)
|
|
328
349
|
}
|
|
329
350
|
|
|
351
|
+
export interface OrphanClassifyOpts {
|
|
352
|
+
/**
|
|
353
|
+
* `turnKey` from the on-disk `turn-active.json` marker — the single
|
|
354
|
+
* in-flight turn the hang-watchdog tracks. Null when no marker is
|
|
355
|
+
* present at boot (the previous process exited cleanly between turns).
|
|
356
|
+
*/
|
|
357
|
+
markerTurnKey?: string | null
|
|
358
|
+
/**
|
|
359
|
+
* Age in ms of the `turn-active.json` marker's mtime at boot, or null
|
|
360
|
+
* when no marker is present. The marker's mtime is bumped on every
|
|
361
|
+
* tool_use, so this is "ms since the last observable progress" of the
|
|
362
|
+
* in-flight turn.
|
|
363
|
+
*/
|
|
364
|
+
markerAgeMs?: number | null
|
|
365
|
+
/**
|
|
366
|
+
* Hang-watchdog threshold in ms (`TURN_HANG_SECS * 1000`, default
|
|
367
|
+
* 300_000). A marker older than this means the in-flight turn made no
|
|
368
|
+
* tool progress for at least the watchdog window — i.e. it was (or,
|
|
369
|
+
* under Docker where the watchdog is disabled, *would have been*)
|
|
370
|
+
* killed as a hang rather than cleanly restarted. That distinction is
|
|
371
|
+
* the whole point: a hung turn is reported, a live one is resumed.
|
|
372
|
+
*/
|
|
373
|
+
hangThresholdMs: number
|
|
374
|
+
/**
|
|
375
|
+
* Opaque snapshot persisted to `interrupt_reason` for the
|
|
376
|
+
* timeout-classified turn so a later boot can rebuild the watchdog
|
|
377
|
+
* report after the marker has been swept.
|
|
378
|
+
*/
|
|
379
|
+
reasonSnapshot?: string | null
|
|
380
|
+
/** Injectable clock for tests. */
|
|
381
|
+
now?: number
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
export interface OrphanClassifyResult {
|
|
385
|
+
/** Total rows stamped (timeout + restart). */
|
|
386
|
+
reaped: number
|
|
387
|
+
/** turn_key stamped 'timeout', or null if none qualified as a hang. */
|
|
388
|
+
timeoutTurnKey: string | null
|
|
389
|
+
}
|
|
390
|
+
|
|
330
391
|
/**
|
|
331
|
-
* Boot-time reaper. Sweeps ALL turns
|
|
332
|
-
*
|
|
333
|
-
*
|
|
392
|
+
* Boot-time reaper + classifier. Sweeps ALL turns with `ended_at IS NULL`
|
|
393
|
+
* (killed mid-flight: SIGKILL / OOM / hard reboot / operator restart) and
|
|
394
|
+
* stamps an `ended_via`:
|
|
334
395
|
*
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
*
|
|
396
|
+
* - the in-flight turn (matched by `markerTurnKey`) is stamped
|
|
397
|
+
* `'timeout'` IFF its marker is older than `hangThresholdMs` — it
|
|
398
|
+
* stalled with no tool progress for the full watchdog window, so it's
|
|
399
|
+
* reported-not-resumed; its `interrupt_reason` carries `reasonSnapshot`.
|
|
400
|
+
* - every other open turn (and the in-flight one when it was making
|
|
401
|
+
* progress) is stamped `'restart'` — a clean interrupt, eligible for
|
|
402
|
+
* blanket resume.
|
|
338
403
|
*
|
|
339
|
-
*
|
|
404
|
+
* Call this once immediately after `openTurnsDb`, BEFORE any new turns are
|
|
405
|
+
* recorded for the current boot, and BEFORE the turn-active marker is
|
|
406
|
+
* swept (the classifier needs the marker's mtime).
|
|
340
407
|
*/
|
|
341
|
-
export function
|
|
342
|
-
|
|
343
|
-
|
|
408
|
+
export function markOrphanedWithTimeoutClassification(
|
|
409
|
+
db: SqliteDatabase,
|
|
410
|
+
opts: OrphanClassifyOpts,
|
|
411
|
+
): OrphanClassifyResult {
|
|
412
|
+
const now = opts.now ?? Date.now()
|
|
413
|
+
const isHang =
|
|
414
|
+
opts.markerAgeMs != null &&
|
|
415
|
+
opts.markerAgeMs >= opts.hangThresholdMs &&
|
|
416
|
+
opts.markerTurnKey != null &&
|
|
417
|
+
opts.markerTurnKey.length > 0
|
|
418
|
+
|
|
419
|
+
let timeoutTurnKey: string | null = null
|
|
420
|
+
if (isHang) {
|
|
421
|
+
const r = db.prepare(`
|
|
422
|
+
UPDATE turns
|
|
423
|
+
SET ended_at = ?,
|
|
424
|
+
ended_via = 'timeout',
|
|
425
|
+
interrupt_reason = ?,
|
|
426
|
+
updated_at = ?
|
|
427
|
+
WHERE turn_key = ? AND ended_at IS NULL
|
|
428
|
+
`).run(now, opts.reasonSnapshot ?? null, now, opts.markerTurnKey) as { changes: number }
|
|
429
|
+
if (r.changes > 0) timeoutTurnKey = opts.markerTurnKey ?? null
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
const rest = db.prepare(`
|
|
344
433
|
UPDATE turns
|
|
345
434
|
SET ended_at = ?,
|
|
346
435
|
ended_via = 'restart',
|
|
347
436
|
updated_at = ?
|
|
348
437
|
WHERE ended_at IS NULL
|
|
349
438
|
`).run(now, now) as { changes: number }
|
|
350
|
-
|
|
439
|
+
|
|
440
|
+
return { reaped: (timeoutTurnKey ? 1 : 0) + rest.changes, timeoutTurnKey }
|
|
351
441
|
}
|
|
352
442
|
|
|
353
443
|
/**
|
|
@@ -392,26 +482,41 @@ export function listTurnsForAgent(
|
|
|
392
482
|
return rows.map(mapRow)
|
|
393
483
|
}
|
|
394
484
|
|
|
485
|
+
/** ended_via values that mean "this turn did not finish on its own". */
|
|
486
|
+
const INTERRUPTED_VIA: ReadonlySet<TurnEndedVia> = new Set<TurnEndedVia>([
|
|
487
|
+
'restart',
|
|
488
|
+
'sigterm',
|
|
489
|
+
'timeout',
|
|
490
|
+
'unknown',
|
|
491
|
+
])
|
|
492
|
+
|
|
395
493
|
/**
|
|
396
|
-
*
|
|
397
|
-
* (`
|
|
398
|
-
*
|
|
399
|
-
*
|
|
494
|
+
* Return the single most-recently-started turn IFF it was interrupted
|
|
495
|
+
* (`ended_at IS NULL`, or `ended_via` in {restart, sigterm, timeout,
|
|
496
|
+
* unknown}). Returns null when the latest turn ended cleanly (`'stop'`)
|
|
497
|
+
* or there are no turns at all.
|
|
400
498
|
*
|
|
401
|
-
*
|
|
499
|
+
* This is the resume gate. Keying on the *latest* turn (not "latest
|
|
500
|
+
* interrupted turn anywhere in history") is deliberate: once the agent
|
|
501
|
+
* resumes and that follow-up turn ends `'stop'`, the latest turn is clean
|
|
502
|
+
* and this returns null — so a completed resume is never re-fired on the
|
|
503
|
+
* next restart. The older `findMostRecentInterruptedTurn` had the inverse
|
|
504
|
+
* bug: a clean latest turn didn't shadow a stale interrupted one, so it
|
|
505
|
+
* would resurface already-handled work indefinitely.
|
|
402
506
|
*
|
|
403
|
-
*
|
|
404
|
-
*
|
|
405
|
-
*
|
|
406
|
-
* the user remembers, and that's `started_at`.
|
|
507
|
+
* Ordering uses `started_at DESC` (not `updated_at`) so the boot reaper,
|
|
508
|
+
* which mass-stamps orphans with identical timestamps, can't reorder the
|
|
509
|
+
* temporal "last turn" the user actually remembers.
|
|
407
510
|
*/
|
|
408
|
-
export function
|
|
511
|
+
export function findLatestTurnIfInterrupted(db: SqliteDatabase): Turn | null {
|
|
409
512
|
const row = db.prepare(`
|
|
410
513
|
SELECT * FROM turns
|
|
411
|
-
WHERE ended_at IS NULL
|
|
412
|
-
OR ended_via IN ('restart', 'sigterm', 'timeout')
|
|
413
514
|
ORDER BY started_at DESC
|
|
414
515
|
LIMIT 1
|
|
415
516
|
`).get() as RawTurnRow | undefined
|
|
416
|
-
|
|
517
|
+
if (!row) return null
|
|
518
|
+
const turn = mapRow(row)
|
|
519
|
+
if (turn.ended_at == null) return turn
|
|
520
|
+
if (turn.ended_via != null && INTERRUPTED_VIA.has(turn.ended_via)) return turn
|
|
521
|
+
return null
|
|
417
522
|
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Contract:
|
|
10
10
|
* - First call for a chat+thread: creates a stream via
|
|
11
|
-
* createStreamController
|
|
11
|
+
* createStreamController.
|
|
12
12
|
* - Subsequent calls: reuse the existing stream, push the new text.
|
|
13
13
|
* - `done=true`: finalize, delete the map entry, fire status-reaction
|
|
14
14
|
* completion, and (if history enabled) record the final message.
|
|
@@ -171,8 +171,6 @@ export interface StreamReplyDeps {
|
|
|
171
171
|
escapeMarkdownV2: (text: string) => string
|
|
172
172
|
/** Whitespace repair applied to the raw caller text. */
|
|
173
173
|
repairEscapedWhitespace: (text: string) => string
|
|
174
|
-
/** Resolves the handoff prefix for a first-chunk stream. Empty string if none. */
|
|
175
|
-
takeHandoffPrefix: (format: 'html' | 'markdownv2' | 'text') => string
|
|
176
174
|
/** Validates the chat id against the access list. Throws on deny. */
|
|
177
175
|
assertAllowedChat: (chatId: string) => void
|
|
178
176
|
/** Resolves the effective thread id (explicit, last-inbound, or undefined). */
|
|
@@ -445,14 +443,6 @@ export async function handleStreamReply(
|
|
|
445
443
|
streamExisted,
|
|
446
444
|
})
|
|
447
445
|
|
|
448
|
-
// First chunk of a session: consume any pending handoff prefix.
|
|
449
|
-
if (!stream) {
|
|
450
|
-
const prefix = deps.takeHandoffPrefix(
|
|
451
|
-
format === 'html' ? 'html' : format === 'markdownv2' ? 'markdownv2' : 'text',
|
|
452
|
-
)
|
|
453
|
-
if (prefix.length > 0) effectiveText = prefix + effectiveText
|
|
454
|
-
}
|
|
455
|
-
|
|
456
446
|
if (!stream) {
|
|
457
447
|
// Resolve the effective quote-reply target. Explicit `reply_to` wins;
|
|
458
448
|
// otherwise (unless the caller opted out with `quote:false`) fall back
|
|
@@ -40,7 +40,7 @@ import {
|
|
|
40
40
|
} from 'fs'
|
|
41
41
|
import { basename, join } from 'path'
|
|
42
42
|
import { homedir } from 'os'
|
|
43
|
-
import { projectSubagentLine, sanitizeCwdToProjectName } from './session-tail.js'
|
|
43
|
+
import { projectSubagentLine, sanitizeCwdToProjectName, detectErrorInTranscriptLine } from './session-tail.js'
|
|
44
44
|
import { sanitiseToolArg } from './fleet-state.js'
|
|
45
45
|
import { escapeHtml, truncate } from './card-format.js'
|
|
46
46
|
import { bumpSubagentActivity, recordSubagentStall, recordSubagentResume, recordSubagentEnd, reapStuckRunningRows } from './registry/subagents-schema.js'
|
|
@@ -142,6 +142,21 @@ export interface WorkerEntry {
|
|
|
142
142
|
* dead, the file is just left over from a prior session.
|
|
143
143
|
*/
|
|
144
144
|
historical: boolean
|
|
145
|
+
/**
|
|
146
|
+
* True once a TERMINAL error line — a model API failure / quota
|
|
147
|
+
* exhaustion / crash, NOT an in-flight retry or a routine tool-level
|
|
148
|
+
* `is_error` result — has been observed in this worker's own
|
|
149
|
+
* transcript. Drives the `failed` terminal outcome so the handback
|
|
150
|
+
* tells the user the delegated work did NOT complete, instead of
|
|
151
|
+
* dressing a dead worker up as `completed`. Classified by
|
|
152
|
+
* `detectErrorInTranscriptLine` (the same gate the operator-event
|
|
153
|
+
* path uses), so transient mid-retry errors are excluded.
|
|
154
|
+
*/
|
|
155
|
+
errored?: boolean
|
|
156
|
+
/** Human-readable detail from the terminal error line, surfaced in the
|
|
157
|
+
* failed handback's "what it reported before failing" slot when the
|
|
158
|
+
* worker left no narrative result of its own. */
|
|
159
|
+
errorDetail?: string
|
|
145
160
|
}
|
|
146
161
|
|
|
147
162
|
export interface SubagentWatcherConfig {
|
|
@@ -611,6 +626,20 @@ export function readSubTail(
|
|
|
611
626
|
const startState = { hasEmittedStart: tail.hasEmittedStart }
|
|
612
627
|
for (const line of lines) {
|
|
613
628
|
if (!line) continue
|
|
629
|
+
// Gap 2 (failure honesty): a terminal error line in the worker's
|
|
630
|
+
// OWN transcript — a model API failure, quota exhaustion, or crash —
|
|
631
|
+
// means the worker FAILED, not finished. Reuse the operator-event
|
|
632
|
+
// classifier: `terminal:true` excludes in-flight retries (a 529 mid-
|
|
633
|
+
// backoff is `terminal:false`), and tool-level `is_error` results
|
|
634
|
+
// never reach here (they parse as `sub_agent_tool_result`, which is
|
|
635
|
+
// routine mid-run noise, not a worker death). The flag persists on
|
|
636
|
+
// the entry; the terminal transition (real turn_end OR stall
|
|
637
|
+
// synthesis) reads it to emit `failed` instead of `completed`.
|
|
638
|
+
const errInfo = detectErrorInTranscriptLine(line)
|
|
639
|
+
if (errInfo?.terminal) {
|
|
640
|
+
entry.errored = true
|
|
641
|
+
if (errInfo.detail) entry.errorDetail = errInfo.detail.slice(0, SUBAGENT_RESULT_TEXT_MAX)
|
|
642
|
+
}
|
|
614
643
|
const events = projectSubagentLine(line, entry.agentId, startState)
|
|
615
644
|
for (const ev of events) {
|
|
616
645
|
const idleSecBeforeBump = Math.round((now - entry.lastActivityAt) / 1000)
|
|
@@ -716,7 +745,10 @@ export function readSubTail(
|
|
|
716
745
|
recordSubagentEnd(db, {
|
|
717
746
|
id: rowRef.id,
|
|
718
747
|
endedAt: now,
|
|
719
|
-
|
|
748
|
+
// Gap 2: keep the audit row honest — a worker that hit a
|
|
749
|
+
// terminal transcript error is `failed`, matching the
|
|
750
|
+
// handback outcome computed in maybySendStateTransition.
|
|
751
|
+
status: entry.errored ? 'failed' : 'completed',
|
|
720
752
|
})
|
|
721
753
|
}
|
|
722
754
|
} catch (dbErr) {
|
|
@@ -917,6 +949,34 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
917
949
|
log?.(`subagent-watcher: description updated for ${agentId}: ${desc}`)
|
|
918
950
|
}, fs, log, db, parentStateDir, config.onUnstall, undefined, config.onProgress)
|
|
919
951
|
|
|
952
|
+
// Gap 1 (restart survival): a file still RUNNING at boot is a LIVE
|
|
953
|
+
// worker that predates this watcher — typically one dispatched in a
|
|
954
|
+
// prior gateway life and still in-flight across a restart / fleet
|
|
955
|
+
// rollout, NOT a stale already-finished file. `historical` must
|
|
956
|
+
// suppress replay only for done-at-boot files; an in-flight-at-boot
|
|
957
|
+
// worker the user is still waiting on must get full live treatment:
|
|
958
|
+
// progress nudges, the stall-synthesis safety net (checkStalls skips
|
|
959
|
+
// historical entries), and a real `completed`/`failed` handback rather
|
|
960
|
+
// than a dropped `orphan`. Promote it to a live entry here. (A file
|
|
961
|
+
// already `done` at boot stays historical and is short-circuited just
|
|
962
|
+
// below — it finished before this session.)
|
|
963
|
+
if (isHistorical && entry.state === 'running') {
|
|
964
|
+
entry.historical = false
|
|
965
|
+
log?.(`subagent-watcher: ${agentId} was in-flight at boot — promoting to live (predates watcher; user still awaiting handback)`)
|
|
966
|
+
// The prior gateway life's registration normally linked
|
|
967
|
+
// jsonl_agent_id already, but re-run the backfill idempotently in
|
|
968
|
+
// case that life crashed before the link persisted — the handback's
|
|
969
|
+
// isBackground lookup is keyed on jsonl_agent_id, and an unlinked row
|
|
970
|
+
// would mis-resolve the worker as foreground and drop the handback.
|
|
971
|
+
if (db != null) {
|
|
972
|
+
try {
|
|
973
|
+
backfillJsonlAgentId(db, filePath, agentId, log)
|
|
974
|
+
} catch (err) {
|
|
975
|
+
log?.(`subagent-watcher: backfill error for ${agentId}: ${(err as Error).message}`)
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
|
|
920
980
|
// If the JSONL already contained a turn_end at registration time
|
|
921
981
|
// (file written-then-watched), fire the state-transition + completion
|
|
922
982
|
// notification now. Otherwise the FSWatcher callback handles it on
|
|
@@ -980,11 +1040,22 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
980
1040
|
config.onFinish({
|
|
981
1041
|
agentId,
|
|
982
1042
|
state: entry.state,
|
|
983
|
-
|
|
1043
|
+
// Gap 2: a terminal error observed in the transcript wins over
|
|
1044
|
+
// the completed/orphan classification — a worker that crashed
|
|
1045
|
+
// is `failed`, even if it later wrote a turn_end or aged into
|
|
1046
|
+
// stall synthesis. `orphan` remains for genuinely stale
|
|
1047
|
+
// done-at-boot rows (which never reach this path; see
|
|
1048
|
+
// registerAgent's short-circuit + Gap 1 promotion).
|
|
1049
|
+
outcome: entry.errored ? 'failed' : entry.historical ? 'orphan' : 'completed',
|
|
984
1050
|
toolCount: entry.toolCount,
|
|
985
1051
|
durationMs: nowFn() - entry.dispatchedAt,
|
|
986
1052
|
description: entry.description,
|
|
987
|
-
|
|
1053
|
+
// For a failure, fall back to the error detail when the worker
|
|
1054
|
+
// left no narrative of its own — so the handback's "what it
|
|
1055
|
+
// reported before failing" slot is never empty on a crash.
|
|
1056
|
+
resultText: entry.errored
|
|
1057
|
+
? entry.lastResultText || entry.errorDetail || ''
|
|
1058
|
+
: entry.lastResultText,
|
|
988
1059
|
})
|
|
989
1060
|
} catch (cbErr) {
|
|
990
1061
|
log?.(`subagent-watcher: onFinish callback error ${agentId}: ${(cbErr as Error).message}`)
|
|
@@ -1151,7 +1222,10 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1151
1222
|
recordSubagentEnd(db, {
|
|
1152
1223
|
id: rowRef.id,
|
|
1153
1224
|
endedAt: n,
|
|
1154
|
-
|
|
1225
|
+
// Gap 2: a worker that hit a terminal transcript error before
|
|
1226
|
+
// going silent is `failed`, not `completed` — keep the audit
|
|
1227
|
+
// row consistent with the handback outcome.
|
|
1228
|
+
status: entry.errored ? 'failed' : 'completed',
|
|
1155
1229
|
})
|
|
1156
1230
|
}
|
|
1157
1231
|
} catch (dbErr) {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { describe, it, expect, afterEach } from "vitest";
|
|
2
|
+
import { resolveAgentDirFromEnv } from "../agent-dir.js";
|
|
3
|
+
|
|
4
|
+
describe("resolveAgentDirFromEnv", () => {
|
|
5
|
+
const prior = process.env.TELEGRAM_STATE_DIR;
|
|
6
|
+
afterEach(() => {
|
|
7
|
+
if (prior === undefined) delete process.env.TELEGRAM_STATE_DIR;
|
|
8
|
+
else process.env.TELEGRAM_STATE_DIR = prior;
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it("returns dirname of TELEGRAM_STATE_DIR", () => {
|
|
12
|
+
process.env.TELEGRAM_STATE_DIR = "/foo/bar/agent/telegram";
|
|
13
|
+
expect(resolveAgentDirFromEnv()).toBe("/foo/bar/agent");
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it("returns null when env unset", () => {
|
|
17
|
+
delete process.env.TELEGRAM_STATE_DIR;
|
|
18
|
+
expect(resolveAgentDirFromEnv()).toBeNull();
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it("returns null when env is empty string", () => {
|
|
22
|
+
process.env.TELEGRAM_STATE_DIR = " ";
|
|
23
|
+
expect(resolveAgentDirFromEnv()).toBeNull();
|
|
24
|
+
});
|
|
25
|
+
});
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* of this test file and brittle w.r.t. upstream churn.
|
|
11
11
|
*
|
|
12
12
|
* Instead, following the existing project convention
|
|
13
|
-
* (see steering.test.ts
|
|
13
|
+
* (see steering.test.ts), we exercise each
|
|
14
14
|
* specified scenario through the same pure helper modules that server.ts
|
|
15
15
|
* calls. Where a scenario lives inside server.ts's in-memory state
|
|
16
16
|
* (activeTurnStartedAt, activeStatusReactions, suppressPtyPreview), we
|
|
@@ -18,23 +18,13 @@
|
|
|
18
18
|
* server.ts uses. The helpers and the state shape are the contract —
|
|
19
19
|
* if they don't regress, the integrated behaviour doesn't regress.
|
|
20
20
|
*/
|
|
21
|
-
import { describe, it, expect, beforeEach, afterEach
|
|
22
|
-
import { mkdtempSync, rmSync, writeFileSync, existsSync } from 'node:fs'
|
|
23
|
-
import { tmpdir } from 'node:os'
|
|
24
|
-
import { join } from 'node:path'
|
|
21
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
25
22
|
|
|
26
23
|
import {
|
|
27
24
|
parseQueuePrefix,
|
|
28
25
|
formatPriorAssistantPreview,
|
|
29
26
|
buildChannelMetaAttributes,
|
|
30
27
|
} from '../steering.js'
|
|
31
|
-
import {
|
|
32
|
-
consumeHandoffTopic,
|
|
33
|
-
readHandoffTopic,
|
|
34
|
-
formatHandoffLine,
|
|
35
|
-
shouldShowHandoffLine,
|
|
36
|
-
HANDOFF_TOPIC_FILENAME,
|
|
37
|
-
} from '../handoff-continuity.js'
|
|
38
28
|
import {
|
|
39
29
|
isContextExhaustionText,
|
|
40
30
|
shouldArmOrphanedReplyTimeout,
|
|
@@ -64,7 +54,6 @@ interface PluginState {
|
|
|
64
54
|
currentSessionChatId: string | null
|
|
65
55
|
currentSessionThreadId: number | undefined
|
|
66
56
|
currentTurnStartedAt: number
|
|
67
|
-
handoffTopicUsed: boolean
|
|
68
57
|
}
|
|
69
58
|
|
|
70
59
|
function freshState(): PluginState {
|
|
@@ -75,7 +64,6 @@ function freshState(): PluginState {
|
|
|
75
64
|
currentSessionChatId: null,
|
|
76
65
|
currentSessionThreadId: undefined,
|
|
77
66
|
currentTurnStartedAt: 0,
|
|
78
|
-
handoffTopicUsed: false,
|
|
79
67
|
}
|
|
80
68
|
}
|
|
81
69
|
|
|
@@ -271,69 +259,6 @@ describe('E2E: turn lifecycle cleanup', () => {
|
|
|
271
259
|
})
|
|
272
260
|
})
|
|
273
261
|
|
|
274
|
-
// ---------------------------------------------------------------------------
|
|
275
|
-
// Handoff continuity
|
|
276
|
-
// ---------------------------------------------------------------------------
|
|
277
|
-
|
|
278
|
-
describe('E2E: handoff continuity', () => {
|
|
279
|
-
let tmp: string
|
|
280
|
-
const priorEnv = { ...process.env }
|
|
281
|
-
|
|
282
|
-
beforeEach(() => {
|
|
283
|
-
tmp = mkdtempSync(join(tmpdir(), 'handoff-e2e-'))
|
|
284
|
-
})
|
|
285
|
-
|
|
286
|
-
afterEach(() => {
|
|
287
|
-
rmSync(tmp, { recursive: true, force: true })
|
|
288
|
-
process.env = { ...priorEnv }
|
|
289
|
-
})
|
|
290
|
-
|
|
291
|
-
it('bootstrap with sidecar + show-line=true → first reply prepends the line', () => {
|
|
292
|
-
writeFileSync(join(tmp, HANDOFF_TOPIC_FILENAME), 'shipped the feature\n', 'utf8')
|
|
293
|
-
process.env.SWITCHROOM_HANDOFF_SHOW_LINE = 'true'
|
|
294
|
-
expect(shouldShowHandoffLine()).toBe(true)
|
|
295
|
-
const topic = consumeHandoffTopic(tmp)
|
|
296
|
-
expect(topic).toBe('shipped the feature')
|
|
297
|
-
const line = formatHandoffLine(topic!, 'html')
|
|
298
|
-
expect(line).toContain('shipped the feature')
|
|
299
|
-
expect(line).toMatch(/^<i>/)
|
|
300
|
-
})
|
|
301
|
-
|
|
302
|
-
it('bootstrap with sidecar + show-line=false → no prefix', () => {
|
|
303
|
-
writeFileSync(join(tmp, HANDOFF_TOPIC_FILENAME), 'x\n', 'utf8')
|
|
304
|
-
process.env.SWITCHROOM_HANDOFF_SHOW_LINE = 'false'
|
|
305
|
-
expect(shouldShowHandoffLine()).toBe(false)
|
|
306
|
-
})
|
|
307
|
-
|
|
308
|
-
it('bootstrap with no sidecar → no prefix', () => {
|
|
309
|
-
expect(readHandoffTopic(tmp)).toBeNull()
|
|
310
|
-
expect(consumeHandoffTopic(tmp)).toBeNull()
|
|
311
|
-
})
|
|
312
|
-
|
|
313
|
-
it('consuming topic is one-shot — second call returns null + sidecar deleted', () => {
|
|
314
|
-
writeFileSync(join(tmp, HANDOFF_TOPIC_FILENAME), 'topic\n', 'utf8')
|
|
315
|
-
expect(consumeHandoffTopic(tmp)).toBe('topic')
|
|
316
|
-
expect(existsSync(join(tmp, HANDOFF_TOPIC_FILENAME))).toBe(false)
|
|
317
|
-
expect(consumeHandoffTopic(tmp)).toBeNull()
|
|
318
|
-
})
|
|
319
|
-
|
|
320
|
-
it('stream_reply: once topic consumed, subsequent stream chunks do not re-prefix', () => {
|
|
321
|
-
// Model: the plugin tracks handoffTopicUsed after first reply/stream_reply
|
|
322
|
-
// use. The second and later stream edits on the same stream read the flag
|
|
323
|
-
// and skip prepending.
|
|
324
|
-
writeFileSync(join(tmp, HANDOFF_TOPIC_FILENAME), 't\n', 'utf8')
|
|
325
|
-
const s = freshState()
|
|
326
|
-
expect(s.handoffTopicUsed).toBe(false)
|
|
327
|
-
// first chunk
|
|
328
|
-
const topic = consumeHandoffTopic(tmp)
|
|
329
|
-
expect(topic).toBe('t')
|
|
330
|
-
s.handoffTopicUsed = true
|
|
331
|
-
// simulate next chunk arriving — should not consume
|
|
332
|
-
expect(consumeHandoffTopic(tmp)).toBeNull()
|
|
333
|
-
expect(s.handoffTopicUsed).toBe(true)
|
|
334
|
-
})
|
|
335
|
-
})
|
|
336
|
-
|
|
337
262
|
// ---------------------------------------------------------------------------
|
|
338
263
|
// Context exhaustion
|
|
339
264
|
// ---------------------------------------------------------------------------
|
|
@@ -106,6 +106,51 @@ describe('spoolId — stable dedup key', () => {
|
|
|
106
106
|
// messageId > 0 → legacy m:<chat>:<msgId> still wins.
|
|
107
107
|
expect(a).toBe('m:c1:555')
|
|
108
108
|
})
|
|
109
|
+
// honest-restart-resume: a boot-resume inbound is minted with a fresh
|
|
110
|
+
// ts/messageId every boot, so without a turn-keyed id an operator who
|
|
111
|
+
// restarts twice before the agent drains the first resume would stack
|
|
112
|
+
// N resumes of the same turn. Keying on resume_turn_key collapses them.
|
|
113
|
+
it('resume_interrupted → s:resume:<turn_key>, stable across boots (fresh ts/messageId)', () => {
|
|
114
|
+
const a = spoolId(
|
|
115
|
+
msg({
|
|
116
|
+
messageId: 1700_000_000_000,
|
|
117
|
+
ts: 1700_000_000_000,
|
|
118
|
+
meta: { source: 'resume_interrupted', resume_turn_key: '12345:11' },
|
|
119
|
+
}),
|
|
120
|
+
)
|
|
121
|
+
const b = spoolId(
|
|
122
|
+
msg({
|
|
123
|
+
messageId: 1700_000_999_999,
|
|
124
|
+
ts: 1700_000_999_999,
|
|
125
|
+
meta: { source: 'resume_interrupted', resume_turn_key: '12345:11' },
|
|
126
|
+
}),
|
|
127
|
+
)
|
|
128
|
+
expect(a).toBe('s:resume:12345:11')
|
|
129
|
+
expect(b).toBe(a)
|
|
130
|
+
})
|
|
131
|
+
it('resume_watchdog_timeout shares the s:resume namespace (one turn is one or the other)', () => {
|
|
132
|
+
const interrupted = spoolId(
|
|
133
|
+
msg({ messageId: 0, meta: { source: 'resume_interrupted', resume_turn_key: 'k:1' } }),
|
|
134
|
+
)
|
|
135
|
+
const timeout = spoolId(
|
|
136
|
+
msg({ messageId: 0, meta: { source: 'resume_watchdog_timeout', resume_turn_key: 'k:1' } }),
|
|
137
|
+
)
|
|
138
|
+
expect(timeout).toBe('s:resume:k:1')
|
|
139
|
+
expect(timeout).toBe(interrupted)
|
|
140
|
+
})
|
|
141
|
+
it('resume inbounds for distinct turns stay distinct', () => {
|
|
142
|
+
const a = spoolId(
|
|
143
|
+
msg({ messageId: 0, meta: { source: 'resume_interrupted', resume_turn_key: 'k:1' } }),
|
|
144
|
+
)
|
|
145
|
+
const b = spoolId(
|
|
146
|
+
msg({ messageId: 0, meta: { source: 'resume_interrupted', resume_turn_key: 'k:2' } }),
|
|
147
|
+
)
|
|
148
|
+
expect(a).not.toBe(b)
|
|
149
|
+
})
|
|
150
|
+
it('resume source without a turn_key falls back to legacy id (no crash)', () => {
|
|
151
|
+
const a = spoolId(msg({ messageId: 777, meta: { source: 'resume_interrupted' }, ts: 100 }))
|
|
152
|
+
expect(a).toBe('m:c1:777')
|
|
153
|
+
})
|
|
109
154
|
})
|
|
110
155
|
|
|
111
156
|
describe('inbound-spool — subagent_handback dedup across restart re-build (#1719)', () => {
|
|
@@ -31,7 +31,6 @@ function makeDeps(bot: FakeBot, overrides?: Partial<StreamReplyDeps>): StreamRep
|
|
|
31
31
|
markdownToHtml: (t) => realMarkdownToHtml(t),
|
|
32
32
|
escapeMarkdownV2: (t) => t,
|
|
33
33
|
repairEscapedWhitespace: (t) => t,
|
|
34
|
-
takeHandoffPrefix: () => '',
|
|
35
34
|
assertAllowedChat: () => {},
|
|
36
35
|
resolveThreadId: (_, explicit) => (explicit != null ? Number(explicit) : undefined),
|
|
37
36
|
disableLinkPreview: true,
|
|
@@ -437,7 +437,6 @@ describe('wrapBot + handleStreamReply + reply ordering', () => {
|
|
|
437
437
|
markdownToHtml: (t) => t,
|
|
438
438
|
escapeMarkdownV2: (t) => t,
|
|
439
439
|
repairEscapedWhitespace: (t) => t,
|
|
440
|
-
takeHandoffPrefix: () => '',
|
|
441
440
|
assertAllowedChat: () => {},
|
|
442
441
|
resolveThreadId: () => undefined,
|
|
443
442
|
disableLinkPreview: true,
|
|
@@ -31,7 +31,6 @@ function makeDeps(bot: FakeBot, overrides?: Partial<StreamReplyDeps>): StreamRep
|
|
|
31
31
|
markdownToHtml: (t) => realMarkdownToHtml(t),
|
|
32
32
|
escapeMarkdownV2: (t) => `ESC(${t})`,
|
|
33
33
|
repairEscapedWhitespace: (t) => t,
|
|
34
|
-
takeHandoffPrefix: () => '',
|
|
35
34
|
assertAllowedChat: () => {},
|
|
36
35
|
resolveThreadId: (_, explicit) => (explicit != null ? Number(explicit) : undefined),
|
|
37
36
|
disableLinkPreview: true,
|