switchroom 0.18.24 → 0.18.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +59 -11
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -0
- package/telegram-plugin/dist/gateway/gateway.js +1489 -829
- package/telegram-plugin/dist/server.js +26 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +7 -0
- package/telegram-plugin/gateway/gateway.ts +314 -3
- package/telegram-plugin/gateway/model-command.ts +188 -56
- package/telegram-plugin/gateway/redelivery-decision.ts +139 -0
- package/telegram-plugin/gateway/vault-grant-inbound-builders.ts +42 -1
- package/telegram-plugin/history.ts +118 -0
- package/telegram-plugin/registry/turns-schema.ts +89 -1
- package/telegram-plugin/session-tail.ts +185 -0
- package/telegram-plugin/subagent-watcher.ts +45 -0
- package/telegram-plugin/tests/crash-redelivery-resume-exclusion.test.ts +133 -0
- package/telegram-plugin/tests/crash-redelivery-wiring.test.ts +72 -0
- package/telegram-plugin/tests/history.test.ts +91 -0
- package/telegram-plugin/tests/model-command.test.ts +189 -12
- package/telegram-plugin/tests/redelivery-decision.test.ts +84 -0
- package/telegram-plugin/tests/registry-turns.test.ts +51 -0
- package/telegram-plugin/tests/session-model-source.test.ts +11 -0
- package/telegram-plugin/tests/session-tail.test.ts +145 -0
- package/telegram-plugin/tests/subagent-watcher.test.ts +50 -0
- package/telegram-plugin/tests/tool-activity-summary.test.ts +109 -0
- package/telegram-plugin/tests/trailing-answer-projector.test.ts +124 -0
- package/telegram-plugin/tests/vault-grant-inbound-builders.test.ts +125 -0
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +306 -0
- package/telegram-plugin/tool-activity-summary.ts +54 -3
- package/telegram-plugin/worker-activity-feed.ts +104 -0
- package/vendor/hindsight-memory/scripts/backfill_transcripts.py +762 -0
- package/vendor/hindsight-memory/scripts/drain_pending.py +13 -1
- package/vendor/hindsight-memory/scripts/lib/client.py +14 -4
- package/vendor/hindsight-memory/scripts/lib/config.py +8 -0
- package/vendor/hindsight-memory/scripts/lib/pacing.py +102 -0
- package/vendor/hindsight-memory/scripts/lib/watermark.py +213 -0
- package/vendor/hindsight-memory/scripts/reconcile_tail.py +344 -0
- package/vendor/hindsight-memory/scripts/retain.py +299 -143
- package/vendor/hindsight-memory/scripts/session_start.py +14 -0
- package/vendor/hindsight-memory/scripts/tests/test_backfill.py +362 -0
- package/vendor/hindsight-memory/scripts/tests/test_reconcile_durability.py +350 -0
- package/vendor/hindsight-memory/tests/test_hooks.py +8 -2
|
@@ -120,6 +120,26 @@ export interface Turn {
|
|
|
120
120
|
* Null for turns that were never resumed.
|
|
121
121
|
*/
|
|
122
122
|
resumed_at: number | null
|
|
123
|
+
/**
|
|
124
|
+
* The claude session id (the `<sessionId>.jsonl` transcript stem) that
|
|
125
|
+
* produced this turn's assistant output, stamped DURING the turn as soon as
|
|
126
|
+
* the first session event is observed (see `stampTurnSessionId`). Crash-
|
|
127
|
+
* survival redelivery uses this to resolve the EXACT transcript file for an
|
|
128
|
+
* interrupted turn, instead of `findActiveSessionFile`'s most-recent-mtime
|
|
129
|
+
* heuristic — which can shadow the new boot session's own transcript. Null
|
|
130
|
+
* until the turn produces its first session event (or for pre-migration rows).
|
|
131
|
+
*/
|
|
132
|
+
session_id: string | null
|
|
133
|
+
/**
|
|
134
|
+
* Ms epoch at which the interrupted turn's captured-but-undelivered final
|
|
135
|
+
* answer was re-sent to the user at boot (crash-survival redelivery). This is
|
|
136
|
+
* the at-most-once ledger for redelivery — first-write-wins via
|
|
137
|
+
* `WHERE answer_redelivered_at IS NULL` (see `markAnswerRedelivered`). Kept on
|
|
138
|
+
* a SEPARATE marker from `resumed_at` because the two concerns have different
|
|
139
|
+
* correctness contracts (resume = at-most-once side-effect replay; redelivery
|
|
140
|
+
* = at-most-once answer send) and a turn can be both. Null until redelivered.
|
|
141
|
+
*/
|
|
142
|
+
answer_redelivered_at: number | null
|
|
123
143
|
created_at: number
|
|
124
144
|
updated_at: number
|
|
125
145
|
}
|
|
@@ -190,6 +210,15 @@ const PHASE3_MIGRATIONS = [
|
|
|
190
210
|
`ALTER TABLE turns ADD COLUMN resumed_at INTEGER`,
|
|
191
211
|
]
|
|
192
212
|
|
|
213
|
+
// Columns added for crash-survival redelivery. `session_id` pins the exact
|
|
214
|
+
// transcript file for an interrupted turn (so redelivery never resolves the
|
|
215
|
+
// wrong session via a most-recent-file heuristic); `answer_redelivered_at` is
|
|
216
|
+
// the at-most-once redelivery ledger (stamped synchronously with the re-send).
|
|
217
|
+
const PHASE4_MIGRATIONS = [
|
|
218
|
+
`ALTER TABLE turns ADD COLUMN session_id TEXT`,
|
|
219
|
+
`ALTER TABLE turns ADD COLUMN answer_redelivered_at INTEGER`,
|
|
220
|
+
]
|
|
221
|
+
|
|
193
222
|
function applySchema(db: SqliteDatabase): void {
|
|
194
223
|
db.exec('PRAGMA journal_mode = WAL')
|
|
195
224
|
db.exec('PRAGMA synchronous = NORMAL')
|
|
@@ -206,7 +235,7 @@ function applySchema(db: SqliteDatabase): void {
|
|
|
206
235
|
// Run migrations. SQLite doesn't support "ADD COLUMN IF NOT EXISTS", so
|
|
207
236
|
// we swallow the "duplicate column" error to stay idempotent on
|
|
208
237
|
// pre-existing registry.db files.
|
|
209
|
-
for (const sql of [...PHASE1_MIGRATIONS, ...PHASE2_MIGRATIONS, ...PHASE3_MIGRATIONS]) {
|
|
238
|
+
for (const sql of [...PHASE1_MIGRATIONS, ...PHASE2_MIGRATIONS, ...PHASE3_MIGRATIONS, ...PHASE4_MIGRATIONS]) {
|
|
210
239
|
try {
|
|
211
240
|
db.exec(sql)
|
|
212
241
|
} catch (err) {
|
|
@@ -283,6 +312,8 @@ interface RawTurnRow {
|
|
|
283
312
|
tool_call_count: number | null
|
|
284
313
|
interrupt_reason: string | null
|
|
285
314
|
resumed_at: number | null
|
|
315
|
+
session_id: string | null
|
|
316
|
+
answer_redelivered_at: number | null
|
|
286
317
|
created_at: number
|
|
287
318
|
updated_at: number
|
|
288
319
|
}
|
|
@@ -304,6 +335,8 @@ function mapRow(row: RawTurnRow): Turn {
|
|
|
304
335
|
tool_call_count: row.tool_call_count,
|
|
305
336
|
interrupt_reason: row.interrupt_reason,
|
|
306
337
|
resumed_at: row.resumed_at,
|
|
338
|
+
session_id: row.session_id ?? null,
|
|
339
|
+
answer_redelivered_at: row.answer_redelivered_at ?? null,
|
|
307
340
|
created_at: row.created_at,
|
|
308
341
|
updated_at: row.updated_at,
|
|
309
342
|
}
|
|
@@ -692,6 +725,61 @@ export function markTurnResumed(
|
|
|
692
725
|
`).run(now, now, turnKey)
|
|
693
726
|
}
|
|
694
727
|
|
|
728
|
+
/**
|
|
729
|
+
* Stamp the claude `session_id` (the `<sessionId>.jsonl` transcript stem) on a
|
|
730
|
+
* turn the FIRST time it is observed, DURING the turn. First-write-wins via
|
|
731
|
+
* `WHERE session_id IS NULL` so the hot session-event path can call this on
|
|
732
|
+
* every event cheaply and idempotently. This must run while the turn is live
|
|
733
|
+
* (before any crash) so crash-survival redelivery can resolve the exact
|
|
734
|
+
* transcript file for an interrupted turn — never the most-recent-mtime file,
|
|
735
|
+
* which a fresh boot session would shadow. No-ops if `turnKey` is not found.
|
|
736
|
+
*/
|
|
737
|
+
export function stampTurnSessionId(
|
|
738
|
+
db: SqliteDatabase,
|
|
739
|
+
turnKey: string,
|
|
740
|
+
sessionId: string,
|
|
741
|
+
now: number = Date.now(),
|
|
742
|
+
): void {
|
|
743
|
+
if (!sessionId) return
|
|
744
|
+
db.prepare(`
|
|
745
|
+
UPDATE turns
|
|
746
|
+
SET session_id = ?,
|
|
747
|
+
updated_at = ?
|
|
748
|
+
WHERE turn_key = ? AND session_id IS NULL
|
|
749
|
+
`).run(sessionId, now, turnKey)
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* Stamp `answer_redelivered_at` on an interrupted turn at the moment its
|
|
754
|
+
* captured-but-undelivered final answer has been re-sent at boot (crash-
|
|
755
|
+
* survival redelivery). This is the at-most-once ledger for redelivery: once
|
|
756
|
+
* stamped, the redelivery decision skips the turn on any later restart.
|
|
757
|
+
*
|
|
758
|
+
* Ordering: the caller stamps SYNCHRONOUSLY with the send (immediately after
|
|
759
|
+
* the send resolves), the same discipline as `markTurnResumed`. A residual
|
|
760
|
+
* race remains — the window between the Telegram send completing and this row
|
|
761
|
+
* (plus the send's own `role='assistant'` history row) becoming durable. A
|
|
762
|
+
* crash landing in that window could re-send on the next boot; the durable
|
|
763
|
+
* text-identity delivery oracle (matching the projected answer text against
|
|
764
|
+
* delivered `messages` rows) is what CATCHES that duplicate, so redelivery is
|
|
765
|
+
* at-most-once modulo detection, never a silent double-send of a fresh answer.
|
|
766
|
+
*
|
|
767
|
+
* Idempotent and first-write-wins (`WHERE answer_redelivered_at IS NULL`).
|
|
768
|
+
* No-ops if `turnKey` is not found.
|
|
769
|
+
*/
|
|
770
|
+
export function markAnswerRedelivered(
|
|
771
|
+
db: SqliteDatabase,
|
|
772
|
+
turnKey: string,
|
|
773
|
+
now: number = Date.now(),
|
|
774
|
+
): void {
|
|
775
|
+
db.prepare(`
|
|
776
|
+
UPDATE turns
|
|
777
|
+
SET answer_redelivered_at = ?,
|
|
778
|
+
updated_at = ?
|
|
779
|
+
WHERE turn_key = ? AND answer_redelivered_at IS NULL
|
|
780
|
+
`).run(now, now, turnKey)
|
|
781
|
+
}
|
|
782
|
+
|
|
695
783
|
/**
|
|
696
784
|
* Return the single most-recently-started turn IFF it was interrupted
|
|
697
785
|
* (`ended_at IS NULL`, or `ended_via` in {restart, sigterm, timeout,
|
|
@@ -118,6 +118,17 @@ export type SessionEvent =
|
|
|
118
118
|
// and reserved for a future staging-skip optimization; do not assume the
|
|
119
119
|
// gate keys on it.
|
|
120
120
|
| { kind: 'text'; text: string; blockIndex: number; lastInMessage: boolean }
|
|
121
|
+
// Per-assistant-message token usage for the MAIN agent, extracted from
|
|
122
|
+
// `message.usage` on each `type:"assistant"` transcript line. `totalTokens`
|
|
123
|
+
// is the NEW-work delta for THIS message (input + output + cache_creation,
|
|
124
|
+
// via sumUsageTokens; cache_read is deliberately excluded — replayed cached
|
|
125
|
+
// context, not new work). `messageId` is `message.id` —
|
|
126
|
+
// REQUIRED for dedup: Claude Code persists one logical assistant message as
|
|
127
|
+
// MULTIPLE JSONL lines sharing one `message.id`, each stamped with the SAME
|
|
128
|
+
// `usage` block, so the accumulator must count a given `messageId` only once
|
|
129
|
+
// (naive summing across lines over-counts). Null messageId → un-dedupable,
|
|
130
|
+
// counted as-is. Mirrors `sub_agent_usage` but for the parent's OWN tokens.
|
|
131
|
+
| { kind: 'usage'; messageId: string | null; totalTokens: number }
|
|
121
132
|
| { kind: 'tool_result'; toolUseId: string; toolName: string | null; isError?: boolean; errorText?: string }
|
|
122
133
|
// `reason` is set ONLY by an internal gateway-synthesized turn_end (never by
|
|
123
134
|
// the JSONL projection). `answer-ready-quiescence` (PR A) marks the positive
|
|
@@ -134,6 +145,17 @@ export type SessionEvent =
|
|
|
134
145
|
// `model` kind (sentinel-filtered, emitted first) but agent-scoped so the
|
|
135
146
|
// watcher can track it per WorkerEntry and thread it onto the worker card.
|
|
136
147
|
| { kind: 'sub_agent_model'; agentId: string; model: string }
|
|
148
|
+
// Per-assistant-message token usage for a SUB-AGENT, extracted from
|
|
149
|
+
// `message.usage` on each `type:"assistant"` transcript line. `totalTokens`
|
|
150
|
+
// is the NEW-work delta for THIS message (input + output + cache_creation;
|
|
151
|
+
// cache_read is deliberately excluded — replayed cached context, not new
|
|
152
|
+
// work). `messageId` is `message.id` — REQUIRED for dedup: Claude
|
|
153
|
+
// Code ≥2.1.x persists one logical assistant message as MULTIPLE JSONL lines
|
|
154
|
+
// sharing one `message.id`, each stamped with the SAME `usage` block, so the
|
|
155
|
+
// watcher must count a given `messageId` only once (naive summing across
|
|
156
|
+
// lines 2-3x over-counts; verified against live worker jsonl — 131 usage
|
|
157
|
+
// lines / 59 unique ids). Null messageId → un-dedupable, counted as-is.
|
|
158
|
+
| { kind: 'sub_agent_usage'; agentId: string; messageId: string | null; totalTokens: number }
|
|
137
159
|
| { kind: 'sub_agent_tool_use'; agentId: string; toolUseId: string | null; toolName: string; input?: Record<string, unknown>; precomputedLabel?: string }
|
|
138
160
|
// Same shared contract as the main-agent `text` kind — see its doc above
|
|
139
161
|
// (including the `lastInMessage` projection-artifact note). The wire-kind
|
|
@@ -283,6 +305,34 @@ export function projectAssistantTextBlocks(
|
|
|
283
305
|
* projectSubagentLine). A thinking-only or empty line returns false; the real
|
|
284
306
|
* terminal rides the following content line, which also carries `end_turn`.
|
|
285
307
|
*/
|
|
308
|
+
/**
|
|
309
|
+
* Sum the NEW token work carried by a single assistant message's `usage`
|
|
310
|
+
* object: `input_tokens + output_tokens + cache_creation_input_tokens`. Every
|
|
311
|
+
* field is guarded with `?? 0` — Claude Code omits fields that are zero/absent
|
|
312
|
+
* on some messages. A non-object (or missing) usage returns 0 so the caller
|
|
313
|
+
* can skip a no-usage line.
|
|
314
|
+
*
|
|
315
|
+
* `cache_read_input_tokens` is DELIBERATELY EXCLUDED. On a prompt-cached turn
|
|
316
|
+
* it is replayed context (billed at ~10% and doing no new work), and it
|
|
317
|
+
* dominates the raw total — including it made the displayed number 2-5x bigger
|
|
318
|
+
* than the actual work done this turn and misread as a cost/effort figure. The
|
|
319
|
+
* three fields kept here represent new tokens processed this turn: fresh input,
|
|
320
|
+
* generated output, and newly-written cache. The nested `iterations` /
|
|
321
|
+
* `cache_creation` breakdowns are subsets already reflected in the top-level
|
|
322
|
+
* fields — never add them, that double-counts. Verified against live worker
|
|
323
|
+
* jsonl.
|
|
324
|
+
*/
|
|
325
|
+
export function sumUsageTokens(usage: unknown): number {
|
|
326
|
+
if (usage == null || typeof usage !== 'object') return 0
|
|
327
|
+
const u = usage as Record<string, unknown>
|
|
328
|
+
const n = (v: unknown): number => (typeof v === 'number' && Number.isFinite(v) ? v : 0)
|
|
329
|
+
return (
|
|
330
|
+
n(u.input_tokens) +
|
|
331
|
+
n(u.output_tokens) +
|
|
332
|
+
n(u.cache_creation_input_tokens)
|
|
333
|
+
)
|
|
334
|
+
}
|
|
335
|
+
|
|
286
336
|
export function assistantLineCarriesAnswerSurface(
|
|
287
337
|
content: Array<Record<string, unknown>> | undefined,
|
|
288
338
|
): boolean {
|
|
@@ -303,6 +353,108 @@ export function assistantLineCarriesAnswerSurface(
|
|
|
303
353
|
return false
|
|
304
354
|
}
|
|
305
355
|
|
|
356
|
+
export interface TrailingAnswer {
|
|
357
|
+
/** Concatenated trailing assistant text of the last turn (after the last
|
|
358
|
+
* tool_use / turn boundary). Empty when there is none to redeliver. */
|
|
359
|
+
text: string
|
|
360
|
+
/** True iff the last content-bearing event of the transcript was text — i.e.
|
|
361
|
+
* the turn ended on an answer, not a dangling tool_use (mid-stream). Bounds
|
|
362
|
+
* the preamble-vs-final ambiguity for crash-survival redelivery. */
|
|
363
|
+
trailingIsText: boolean
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Re-project the TRAILING assistant answer of the last turn from a claude
|
|
368
|
+
* session transcript's full text. Pure — reuses the same `projectTranscriptLine`
|
|
369
|
+
* kernel the live tail uses, so it inherits the `isApiErrorMessage` suppression
|
|
370
|
+
* (a usage-limit error line is NEVER resurfaced as an answer) and the empty-block
|
|
371
|
+
* drop. Used by crash-survival redelivery to recover the finished-but-never-sent
|
|
372
|
+
* answer from disk after a pre-flush crash.
|
|
373
|
+
*
|
|
374
|
+
* Semantics: walk the event stream in order; the "answer buffer" accumulates
|
|
375
|
+
* `text` events and is RESET by any `tool_use` (the answer-so-far was a preamble
|
|
376
|
+
* to a tool call) or by a turn boundary. What remains at end-of-file is the
|
|
377
|
+
* trailing answer of the last turn. `trailingIsText` is true only when the final
|
|
378
|
+
* content-bearing event was that text (not a tool_use), so a turn killed mid-tool
|
|
379
|
+
* never redelivers a stale preamble as an answer.
|
|
380
|
+
*
|
|
381
|
+
* TURN BOUNDARY (diff-review defect #2). A boundary is BOTH an `enqueue`
|
|
382
|
+
* queue-operation AND a real `type:"user"` message line. The kernel
|
|
383
|
+
* (`projectTranscriptLine`) emits nothing for a plain user text line — it only
|
|
384
|
+
* projects `tool_result` blocks out of `type:"user"` — so relying on `enqueue`
|
|
385
|
+
* alone would let two turns separated by a plain user line (no intervening
|
|
386
|
+
* tool_use) CONCATENATE. `isRealUserTurnBoundary` detects that separator
|
|
387
|
+
* directly so only the LAST turn's trailing text is projected.
|
|
388
|
+
*/
|
|
389
|
+
export function projectTrailingAnswerFromTranscript(transcriptText: string): TrailingAnswer {
|
|
390
|
+
const buf: string[] = []
|
|
391
|
+
let lastMeaningful: 'text' | 'tool_use' | null = null
|
|
392
|
+
for (const rawLine of transcriptText.split('\n')) {
|
|
393
|
+
const line = rawLine.trim()
|
|
394
|
+
if (!line) continue
|
|
395
|
+
if (isRealUserTurnBoundary(line)) {
|
|
396
|
+
// A real user message opens a new turn — discard any prior turn's tail.
|
|
397
|
+
buf.length = 0
|
|
398
|
+
lastMeaningful = null
|
|
399
|
+
continue
|
|
400
|
+
}
|
|
401
|
+
for (const ev of projectTranscriptLine(line)) {
|
|
402
|
+
if (ev.kind === 'enqueue') {
|
|
403
|
+
// New inbound turn — any prior turn's trailing text is not this turn's.
|
|
404
|
+
buf.length = 0
|
|
405
|
+
lastMeaningful = null
|
|
406
|
+
} else if (ev.kind === 'tool_use') {
|
|
407
|
+
buf.length = 0
|
|
408
|
+
lastMeaningful = 'tool_use'
|
|
409
|
+
} else if (ev.kind === 'text') {
|
|
410
|
+
const t = (ev as { text?: string }).text ?? ''
|
|
411
|
+
if (t.trim().length > 0) {
|
|
412
|
+
buf.push(t)
|
|
413
|
+
lastMeaningful = 'text'
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
// thinking / model / dequeue / tool_result etc. do not affect the answer.
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
const text = buf.join('').trim()
|
|
420
|
+
return { text, trailingIsText: lastMeaningful === 'text' && text.length > 0 }
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Detect a real inbound-user turn separator in a claude session JSONL.
|
|
425
|
+
*
|
|
426
|
+
* A `type:"user"` line is EITHER a genuine user message (its `message.content`
|
|
427
|
+
* is a string, or an array carrying a `{type:"text"}` block) OR a tool_result
|
|
428
|
+
* carrier (`message.content` is an array of `{type:"tool_result"}` blocks only).
|
|
429
|
+
* Only the former opens a new turn. The main projection kernel emits NO event
|
|
430
|
+
* for a genuine user text line (it projects tool_result blocks only), so the
|
|
431
|
+
* trailing-answer projector needs this to break turns that are separated by a
|
|
432
|
+
* plain user line rather than an interleaved `enqueue` queue-operation. Pure.
|
|
433
|
+
*/
|
|
434
|
+
export function isRealUserTurnBoundary(line: string): boolean {
|
|
435
|
+
let obj: Record<string, unknown>
|
|
436
|
+
try {
|
|
437
|
+
obj = JSON.parse(line)
|
|
438
|
+
} catch {
|
|
439
|
+
return false
|
|
440
|
+
}
|
|
441
|
+
if (obj.type !== 'user') return false
|
|
442
|
+
const message = obj.message as Record<string, unknown> | undefined
|
|
443
|
+
const content = message?.content
|
|
444
|
+
if (typeof content === 'string') return content.trim().length > 0
|
|
445
|
+
if (Array.isArray(content)) {
|
|
446
|
+
// A genuine user message carries a text block; a tool_result carrier does
|
|
447
|
+
// not. Presence of any text block ⇒ real user turn.
|
|
448
|
+
for (const c of content) {
|
|
449
|
+
if (typeof c === 'object' && c != null && (c as Record<string, unknown>).type === 'text') {
|
|
450
|
+
const t = String((c as Record<string, unknown>).text ?? '')
|
|
451
|
+
if (t.trim().length > 0) return true
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return false
|
|
456
|
+
}
|
|
457
|
+
|
|
306
458
|
/**
|
|
307
459
|
* Project a single transcript line into a SessionEvent (or null if it's
|
|
308
460
|
* uninteresting noise). Caller is responsible for the JSON parse — if a
|
|
@@ -364,6 +516,23 @@ export function projectTranscriptLine(line: string): SessionEvent[] {
|
|
|
364
516
|
if (typeof mainModel === 'string' && !isModelSentinel(mainModel)) {
|
|
365
517
|
events.push({ kind: 'model', model: mainModel })
|
|
366
518
|
}
|
|
519
|
+
// Per-message token usage (MAIN tier): surface the summed delta so the
|
|
520
|
+
// gateway can accumulate the parent's OWN running total for the turn card's
|
|
521
|
+
// metrics line. `messageId` (message.id) rides along so the accumulator
|
|
522
|
+
// dedups the multi-line split-message shape (one logical message → many
|
|
523
|
+
// JSONL lines, one shared `usage`). Emitted only when a usage object with a
|
|
524
|
+
// non-zero total exists — a no-usage line contributes nothing and is
|
|
525
|
+
// skipped. Mirrors the sub-agent emit below; this counts the parent alone
|
|
526
|
+
// (sub-agents report their own tokens on their worker-feed rows).
|
|
527
|
+
const mainUsageTotal = sumUsageTokens(message?.usage)
|
|
528
|
+
if (mainUsageTotal > 0) {
|
|
529
|
+
const mainMsgId = message?.id
|
|
530
|
+
events.push({
|
|
531
|
+
kind: 'usage',
|
|
532
|
+
messageId: typeof mainMsgId === 'string' ? mainMsgId : null,
|
|
533
|
+
totalTokens: mainUsageTotal,
|
|
534
|
+
})
|
|
535
|
+
}
|
|
367
536
|
// Text→narrative projection comes from the ONE shared kernel
|
|
368
537
|
// (projectAssistantTextBlocks): it owns the empty-drop + blockIndex +
|
|
369
538
|
// lastInMessage contract. We emit its events at their source positions
|
|
@@ -518,6 +687,22 @@ export function projectSubagentLine(
|
|
|
518
687
|
if (typeof subModel === 'string' && !isModelSentinel(subModel)) {
|
|
519
688
|
events.push({ kind: 'sub_agent_model', agentId, model: subModel })
|
|
520
689
|
}
|
|
690
|
+
// Per-message token usage: surface the summed delta so the watcher can
|
|
691
|
+
// accumulate a running total for the worker-activity card's metrics line.
|
|
692
|
+
// `messageId` (message.id) rides along so the watcher dedups the multi-line
|
|
693
|
+
// split-message shape (one logical message → many JSONL lines, one shared
|
|
694
|
+
// `usage`). Emitted only when a usage object with a non-zero total exists —
|
|
695
|
+
// a no-usage line contributes nothing and is skipped here.
|
|
696
|
+
const subUsageTotal = sumUsageTokens(message?.usage)
|
|
697
|
+
if (subUsageTotal > 0) {
|
|
698
|
+
const subMsgId = message?.id
|
|
699
|
+
events.push({
|
|
700
|
+
kind: 'sub_agent_usage',
|
|
701
|
+
agentId,
|
|
702
|
+
messageId: typeof subMsgId === 'string' ? subMsgId : null,
|
|
703
|
+
totalTokens: subUsageTotal,
|
|
704
|
+
})
|
|
705
|
+
}
|
|
521
706
|
// Text→narrative projection comes from the SAME shared kernel as the
|
|
522
707
|
// main agent (projectAssistantTextBlocks): one source for the empty-drop
|
|
523
708
|
// + blockIndex + lastInMessage contract. The `make` adapter only changes
|
|
@@ -125,6 +125,19 @@ export interface WorkerEntry {
|
|
|
125
125
|
lastActivityAt: number
|
|
126
126
|
/** Number of tool calls seen so far. */
|
|
127
127
|
toolCount: number
|
|
128
|
+
/**
|
|
129
|
+
* Running TOTAL tokens across every assistant message the worker has emitted
|
|
130
|
+
* (input + output + cache_creation, summed via sumUsageTokens; cache_read is
|
|
131
|
+
* excluded — replayed cached context, not new work).
|
|
132
|
+
* Accumulated from `sub_agent_usage` events, deduped by `seenUsageMessageIds`
|
|
133
|
+
* so the multi-line split-message shape (one logical message persisted as
|
|
134
|
+
* several JSONL lines sharing one `message.id` + identical `usage`) counts
|
|
135
|
+
* once. Rendered on the worker card's metrics line. 0 for a worker that never
|
|
136
|
+
* emitted a usage block (e.g. a non-Claude/litellm transcript).
|
|
137
|
+
*/
|
|
138
|
+
totalTokens: number
|
|
139
|
+
/** message.id set already folded into `totalTokens` (usage dedup). */
|
|
140
|
+
seenUsageMessageIds: Set<string>
|
|
128
141
|
/** True once a stall notification has been sent (suppresses repeat). */
|
|
129
142
|
stallNotified: boolean
|
|
130
143
|
/**
|
|
@@ -557,6 +570,9 @@ export interface SubagentWatcherConfig {
|
|
|
557
570
|
state: WorkerState
|
|
558
571
|
outcome: 'completed' | 'failed' | 'orphan'
|
|
559
572
|
toolCount: number
|
|
573
|
+
/** Final running TOTAL tokens across the worker's whole life
|
|
574
|
+
* (`WorkerEntry.totalTokens`). For the terminal worker-feed card. */
|
|
575
|
+
totalTokens: number
|
|
560
576
|
durationMs: number
|
|
561
577
|
/** Dispatch-time task description, for the handback envelope. */
|
|
562
578
|
description: string
|
|
@@ -615,6 +631,10 @@ export interface SubagentWatcherConfig {
|
|
|
615
631
|
lastTool: { name: string; sanitisedArg: string } | null
|
|
616
632
|
/** Tool-use count observed so far. */
|
|
617
633
|
toolCount: number
|
|
634
|
+
/** Running TOTAL tokens across the worker's assistant messages so far
|
|
635
|
+
* (`WorkerEntry.totalTokens`). Threaded onto the worker/nested card's
|
|
636
|
+
* metrics line. 0 for a worker that has emitted no usage yet / ever. */
|
|
637
|
+
totalTokens: number
|
|
618
638
|
/** Friendly display line for THIS tick. Set on `sub_agent_tool_use`
|
|
619
639
|
* events to a `describeToolUse` label ("Reading X", "Running a
|
|
620
640
|
* command") so a foreground sub-agent that runs tools without
|
|
@@ -1089,6 +1109,8 @@ export function readSubTail(
|
|
|
1089
1109
|
lastTool: { name: string; sanitisedArg: string } | null
|
|
1090
1110
|
/** Tool-use count observed so far. */
|
|
1091
1111
|
toolCount: number
|
|
1112
|
+
/** Running total tokens so far (see SubagentWatcherConfig.onProgress). */
|
|
1113
|
+
totalTokens: number
|
|
1092
1114
|
/** Friendly display line for THIS tick (set on tool ticks; see the
|
|
1093
1115
|
* SubagentWatcherConfig.onProgress doc). */
|
|
1094
1116
|
progressLine?: string
|
|
@@ -1179,6 +1201,7 @@ export function readSubTail(
|
|
|
1179
1201
|
},
|
|
1180
1202
|
lastTool: entry.lastTool,
|
|
1181
1203
|
toolCount: entry.toolCount,
|
|
1204
|
+
totalTokens: entry.totalTokens,
|
|
1182
1205
|
model: entry.currentModel,
|
|
1183
1206
|
skeleton: true,
|
|
1184
1207
|
})
|
|
@@ -1320,6 +1343,7 @@ export function readSubTail(
|
|
|
1320
1343
|
},
|
|
1321
1344
|
lastTool: entry.lastTool,
|
|
1322
1345
|
toolCount: entry.toolCount,
|
|
1346
|
+
totalTokens: entry.totalTokens,
|
|
1323
1347
|
model: entry.currentModel,
|
|
1324
1348
|
})
|
|
1325
1349
|
return true
|
|
@@ -1502,6 +1526,22 @@ export function readSubTail(
|
|
|
1502
1526
|
}
|
|
1503
1527
|
continue
|
|
1504
1528
|
}
|
|
1529
|
+
if (ev.kind === 'sub_agent_usage') {
|
|
1530
|
+
// Accumulate the worker's running total tokens, deduped by
|
|
1531
|
+
// message.id: the ≥2.1.x split-message shape stamps the SAME `usage`
|
|
1532
|
+
// block on every JSONL line of one logical assistant message, so
|
|
1533
|
+
// counting each line would 2-3x over-count. A null messageId is
|
|
1534
|
+
// un-dedupable (older/edge shapes) — count it as-is (its usage is
|
|
1535
|
+
// real). No card render here; the total rides the next onProgress
|
|
1536
|
+
// tick's payload like the model does.
|
|
1537
|
+
if (ev.messageId == null) {
|
|
1538
|
+
entry.totalTokens += ev.totalTokens
|
|
1539
|
+
} else if (!entry.seenUsageMessageIds.has(ev.messageId)) {
|
|
1540
|
+
entry.seenUsageMessageIds.add(ev.messageId)
|
|
1541
|
+
entry.totalTokens += ev.totalTokens
|
|
1542
|
+
}
|
|
1543
|
+
continue
|
|
1544
|
+
}
|
|
1505
1545
|
if (ev.kind === 'sub_agent_tool_use') {
|
|
1506
1546
|
// Narrative-dedup gate step 2: a sub_agent_text block was pending;
|
|
1507
1547
|
// this tool is the lookahead that decides it (SHOW unless it drafts
|
|
@@ -1567,6 +1607,7 @@ export function readSubTail(
|
|
|
1567
1607
|
},
|
|
1568
1608
|
lastTool: entry.lastTool,
|
|
1569
1609
|
toolCount: entry.toolCount,
|
|
1610
|
+
totalTokens: entry.totalTokens,
|
|
1570
1611
|
progressLine: toolLine,
|
|
1571
1612
|
model: entry.currentModel,
|
|
1572
1613
|
})
|
|
@@ -1892,6 +1933,8 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1892
1933
|
dispatchedAt: n,
|
|
1893
1934
|
lastActivityAt: n,
|
|
1894
1935
|
toolCount: 0,
|
|
1936
|
+
totalTokens: 0,
|
|
1937
|
+
seenUsageMessageIds: new Set<string>(),
|
|
1895
1938
|
stallNotified: false,
|
|
1896
1939
|
stalledAt: null,
|
|
1897
1940
|
completionNotified: false,
|
|
@@ -2157,6 +2200,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
2157
2200
|
// registerAgent's short-circuit + Gap 1 promotion).
|
|
2158
2201
|
outcome: entry.errored ? 'failed' : entry.historical ? 'orphan' : 'completed',
|
|
2159
2202
|
toolCount: entry.toolCount,
|
|
2203
|
+
totalTokens: entry.totalTokens,
|
|
2160
2204
|
durationMs: nowFn() - entry.dispatchedAt,
|
|
2161
2205
|
description: entry.description,
|
|
2162
2206
|
// For a failure, fall back to the error detail when the worker
|
|
@@ -2184,6 +2228,7 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
2184
2228
|
state: entry.state,
|
|
2185
2229
|
outcome: 'failed',
|
|
2186
2230
|
toolCount: entry.toolCount,
|
|
2231
|
+
totalTokens: entry.totalTokens,
|
|
2187
2232
|
durationMs: nowFn() - entry.dispatchedAt,
|
|
2188
2233
|
description: entry.description,
|
|
2189
2234
|
resultText: entry.lastResultText,
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest'
|
|
2
|
+
import { decideRedeliverCapture } from '../gateway/redelivery-decision.js'
|
|
3
|
+
import {
|
|
4
|
+
decideBootResumeKind,
|
|
5
|
+
RESUME_SYNTHETIC_PROMPT_PREFIX,
|
|
6
|
+
} from '../gateway/resume-inbound-builder.js'
|
|
7
|
+
import type { Turn, TurnEndedVia } from '../registry/turns-schema.js'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Pins the MUTUAL EXCLUSION between crash-survival redelivery and the resume
|
|
11
|
+
* synthetic — the double-send guard. The gateway boot block composes exactly
|
|
12
|
+
* these two pure predicates: `decideBootResumeKind` classifies the interrupted
|
|
13
|
+
* turn, then `decideRedeliverCapture({ willBeResumed: kind === 'resume', ... })`
|
|
14
|
+
* decides whether to ALSO stage a redelivery.
|
|
15
|
+
*
|
|
16
|
+
* The load-bearing outcome: an interrupted turn that WILL be resumed (the model
|
|
17
|
+
* re-runs and emits a fresh answer) must NOT also redeliver its recovered draft
|
|
18
|
+
* — otherwise the same answer reaches the user twice. Conversely, a turn that
|
|
19
|
+
* will NOT be resumed (watchdog report, boot_resume:never suppression,
|
|
20
|
+
* resume-of-a-resume loop-guard) MUST redeliver, because nothing else re-answers.
|
|
21
|
+
*
|
|
22
|
+
* This mirrors the gateway wiring exactly, so it asserts the real composed
|
|
23
|
+
* outcome, not an isolated code path.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
const RESUME_MAX_AGE_MS = 10_800_000 // 3h, gateway default
|
|
27
|
+
|
|
28
|
+
function makeTurn(over: Partial<Turn> = {}): Turn {
|
|
29
|
+
return {
|
|
30
|
+
turn_key: '900001:_#7',
|
|
31
|
+
chat_id: '900001',
|
|
32
|
+
thread_id: null,
|
|
33
|
+
started_at: Date.now() - 60_000, // 1 min ago — well within maxAge
|
|
34
|
+
ended_at: null,
|
|
35
|
+
ended_via: null,
|
|
36
|
+
last_assistant_msg_id: null,
|
|
37
|
+
last_assistant_done: null,
|
|
38
|
+
last_user_msg_id: null,
|
|
39
|
+
user_prompt_preview: 'deploy the staging stack',
|
|
40
|
+
assistant_reply_preview: null,
|
|
41
|
+
tool_call_count: 2,
|
|
42
|
+
interrupt_reason: null,
|
|
43
|
+
resumed_at: null,
|
|
44
|
+
session_id: 'sess-abcd', // durably pinned → redelivery is eligible on the floor
|
|
45
|
+
...over,
|
|
46
|
+
} as Turn
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Compose the two predicates exactly as the gateway boot block does. */
|
|
50
|
+
function composeGate(turn: Turn, suppressed: boolean) {
|
|
51
|
+
const bootResumeKind = decideBootResumeKind({
|
|
52
|
+
pending: turn,
|
|
53
|
+
suppressed,
|
|
54
|
+
ageMs: Math.max(0, Date.now() - turn.started_at),
|
|
55
|
+
maxAgeMs: RESUME_MAX_AGE_MS,
|
|
56
|
+
})
|
|
57
|
+
const capture = decideRedeliverCapture({
|
|
58
|
+
willBeResumed: bootResumeKind === 'resume',
|
|
59
|
+
hasSessionId: Boolean(turn.session_id),
|
|
60
|
+
})
|
|
61
|
+
return { bootResumeKind, capture }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
describe('crash-redelivery ↔ resume mutual exclusion (no double-send)', () => {
|
|
65
|
+
it('(a) an interrupted turn that WILL be resumed does NOT also redeliver', () => {
|
|
66
|
+
// ended_via 'restart' → decideBootResumeKind returns 'resume': the model
|
|
67
|
+
// re-runs and emits a fresh answer that supersedes any recovered draft.
|
|
68
|
+
const turn = makeTurn({ ended_via: 'restart' as TurnEndedVia })
|
|
69
|
+
const { bootResumeKind, capture } = composeGate(turn, /* suppressed */ false)
|
|
70
|
+
expect(bootResumeKind).toBe('resume')
|
|
71
|
+
expect(capture.capture).toBe(false)
|
|
72
|
+
expect(capture.skipReason).toBe('will-be-resumed')
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
it('(a2) a still-open (ended_via=null) killed-mid-flight turn is resumed, not redelivered', () => {
|
|
76
|
+
const turn = makeTurn({ ended_via: null })
|
|
77
|
+
const { bootResumeKind, capture } = composeGate(turn, false)
|
|
78
|
+
expect(bootResumeKind).toBe('resume')
|
|
79
|
+
expect(capture.capture).toBe(false)
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
it('(b) a watchdog-timeout turn (report, no auto re-answer) DOES redeliver', () => {
|
|
83
|
+
// ended_via 'timeout' → 'report': the synthetic only ASKS the user whether
|
|
84
|
+
// to retry — it does not auto-re-answer. Redelivery is the correct recovery.
|
|
85
|
+
const turn = makeTurn({ ended_via: 'timeout' as TurnEndedVia })
|
|
86
|
+
const { bootResumeKind, capture } = composeGate(turn, false)
|
|
87
|
+
expect(bootResumeKind).toBe('report')
|
|
88
|
+
expect(capture.capture).toBe(true)
|
|
89
|
+
expect(capture.skipReason).toBeUndefined()
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
it('(b2) a boot_resume:never-suppressed turn (defer-suppressed) DOES redeliver', () => {
|
|
93
|
+
// suppressed=true → 'defer-suppressed': no synthetic re-run, so redelivery
|
|
94
|
+
// is the ONLY recovery send.
|
|
95
|
+
const turn = makeTurn({ ended_via: 'restart' as TurnEndedVia })
|
|
96
|
+
const { bootResumeKind, capture } = composeGate(turn, /* suppressed */ true)
|
|
97
|
+
expect(bootResumeKind).toBe('defer-suppressed')
|
|
98
|
+
expect(capture.capture).toBe(true)
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
it('(b3) a resume-of-a-resume (loop-guard → defer-loop) turn DOES redeliver', () => {
|
|
102
|
+
// A turn whose prompt is itself a resume synthetic → 'defer-loop': the chain
|
|
103
|
+
// is capped, no re-run happens, so redelivery must still recover the answer.
|
|
104
|
+
const turn = makeTurn({
|
|
105
|
+
ended_via: 'restart' as TurnEndedVia,
|
|
106
|
+
user_prompt_preview: `${RESUME_SYNTHETIC_PROMPT_PREFIX} Continue the interrupted deploy.`,
|
|
107
|
+
})
|
|
108
|
+
const { bootResumeKind, capture } = composeGate(turn, false)
|
|
109
|
+
expect(bootResumeKind).toBe('defer-loop')
|
|
110
|
+
expect(capture.capture).toBe(true)
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
it('(b4) a STALE resume downgraded to report DOES redeliver (not suppressed)', () => {
|
|
114
|
+
// A 'restart' turn older than maxAge downgrades resume→report in
|
|
115
|
+
// selectResumeBuilder. Because the final kind is 'report' (no re-run),
|
|
116
|
+
// redelivery must fire — the gate keys on the FINAL kind, not the raw
|
|
117
|
+
// ended_via.
|
|
118
|
+
const turn = makeTurn({
|
|
119
|
+
ended_via: 'restart' as TurnEndedVia,
|
|
120
|
+
started_at: Date.now() - (RESUME_MAX_AGE_MS + 60_000),
|
|
121
|
+
})
|
|
122
|
+
const { bootResumeKind, capture } = composeGate(turn, false)
|
|
123
|
+
expect(bootResumeKind).toBe('report')
|
|
124
|
+
expect(capture.capture).toBe(true)
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
it('eligibility floor still applies: no session_id → no redelivery even when not resumed', () => {
|
|
128
|
+
const turn = makeTurn({ ended_via: 'timeout' as TurnEndedVia, session_id: null })
|
|
129
|
+
const { capture } = composeGate(turn, false)
|
|
130
|
+
expect(capture.capture).toBe(false)
|
|
131
|
+
expect(capture.skipReason).toBe('no-session-id')
|
|
132
|
+
})
|
|
133
|
+
})
|