switchroom 0.18.19 → 0.18.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/ms-365-write-pretool.mjs +92 -20
- package/dist/cli/switchroom.js +59 -6
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_shared/delegation-golden-rule.md.hbs +9 -0
- package/profiles/_shared/dev-protocol.md.hbs +2 -0
- package/profiles/_shared/execution-discipline.md.hbs +2 -2
- package/profiles/coding/CLAUDE.md.hbs +1 -1
- package/telegram-plugin/answer-ready-flush.ts +187 -0
- package/telegram-plugin/dist/gateway/gateway.js +1114 -184
- package/telegram-plugin/format.ts +179 -20
- package/telegram-plugin/gateway/cron-session.ts +32 -0
- package/telegram-plugin/gateway/gateway.ts +794 -106
- package/telegram-plugin/gateway/idle-clear.ts +170 -0
- package/telegram-plugin/gateway/inject-handler.ts +11 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +9 -9
- package/telegram-plugin/gateway/subagent-progress-inbound-builder.ts +17 -0
- package/telegram-plugin/gateway/turn-record-status.ts +134 -0
- package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +23 -0
- package/telegram-plugin/hooks/silent-end-scan.mjs +98 -8
- package/telegram-plugin/narrative-flush.ts +181 -0
- package/telegram-plugin/pending-work-progress.ts +65 -1
- package/telegram-plugin/registry/subagents-schema.ts +6 -0
- package/telegram-plugin/session-tail.ts +6 -1
- package/telegram-plugin/silent-end.ts +182 -0
- package/telegram-plugin/stream-reply-handler.ts +14 -5
- package/telegram-plugin/subagent-watcher.ts +330 -82
- package/telegram-plugin/tests/answer-ready-flush.test.ts +343 -0
- package/telegram-plugin/tests/cron-inject-idle-clock.test.ts +54 -0
- package/telegram-plugin/tests/emission-authority-facade.test.ts +13 -10
- package/telegram-plugin/tests/format-consistency.test.ts +54 -34
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +6 -5
- package/telegram-plugin/tests/formatting-torture-set.ts +1 -1
- package/telegram-plugin/tests/idle-clear.test.ts +315 -37
- package/telegram-plugin/tests/narrative-flush.test.ts +213 -0
- package/telegram-plugin/tests/narrative-splice-before-finalize.test.ts +167 -0
- package/telegram-plugin/tests/nested-worker-visibility-harness.test.ts +20 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +5 -4
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +100 -42
- package/telegram-plugin/tests/paragraph-spacer-golden.test.ts +150 -0
- package/telegram-plugin/tests/per-topic-current-turn.test.ts +4 -1
- package/telegram-plugin/tests/silent-end-interrupt-stop-scan.test.ts +194 -0
- package/telegram-plugin/tests/silent-end.test.ts +296 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +12 -9
- package/telegram-plugin/tests/subagent-progress-inbound-builder.test.ts +30 -0
- package/telegram-plugin/tests/subagent-watcher-first-paint-independence.test.ts +171 -0
- package/telegram-plugin/tests/subagent-watcher-narrative-early-paint.test.ts +220 -0
- package/telegram-plugin/tests/subagent-watcher.test.ts +13 -12
- package/telegram-plugin/tests/telegram-format.test.ts +36 -23
- package/telegram-plugin/tests/turn-flush-safety.test.ts +21 -17
- package/telegram-plugin/tests/turn-record-status.test.ts +119 -0
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +218 -1
- package/telegram-plugin/tests/worker-feed-terminal-cleanup.test.ts +254 -0
- package/telegram-plugin/tests/worker-feed-terminal-state-truthful.test.ts +165 -0
- package/telegram-plugin/tool-activity-summary.ts +78 -16
- package/telegram-plugin/turn-flush-safety.ts +4 -4
- package/telegram-plugin/worker-activity-feed.ts +181 -30
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Residual B regression: a worker cleaned up by the TTL / authoritative
|
|
3
|
+
* `terminate` sweep — i.e. reaped / vanished / crashed WITHOUT ever delivering
|
|
4
|
+
* a clean `onFinish` result — must render a TRUTHFUL terminal state, NOT "done".
|
|
5
|
+
*
|
|
6
|
+
* `terminateWorker` synthesises the recap when NO clean finish arrived (a clean
|
|
7
|
+
* `onFinish` removes the row first, making `terminate` a no-op; an errored
|
|
8
|
+
* worker goes through `onFinish(outcome:'failed')`). So a row still present at
|
|
9
|
+
* terminate time genuinely ended without a result → it must render as the
|
|
10
|
+
* `incomplete` terminal ("incomplete · …"), never "done".
|
|
11
|
+
*
|
|
12
|
+
* OUTCOME assertions on the finalized card body (RED if `terminate` reverts to
|
|
13
|
+
* `state:'done'`), applied to the generic worker row so the guarantee holds at
|
|
14
|
+
* every nesting level (the feed keys rows by agentId, depth-agnostic):
|
|
15
|
+
* - a solo reaped worker's terminal card reads `incomplete`, never `done`;
|
|
16
|
+
* - a genuinely-`finish`ed worker still reads `done` with its result.
|
|
17
|
+
*/
|
|
18
|
+
import { describe, it, expect } from 'vitest'
|
|
19
|
+
import {
|
|
20
|
+
createWorkerActivityFeed,
|
|
21
|
+
renderWorkerActivity,
|
|
22
|
+
type BotApiForWorkerFeed,
|
|
23
|
+
type WorkerActivityView,
|
|
24
|
+
} from '../worker-activity-feed.js'
|
|
25
|
+
|
|
26
|
+
function makeFeed(now: () => number) {
|
|
27
|
+
const sends: { text: string }[] = []
|
|
28
|
+
const edits: { messageId: number; text: string }[] = []
|
|
29
|
+
let seq = 900
|
|
30
|
+
const bot: BotApiForWorkerFeed = {
|
|
31
|
+
sendMessage: async (_c, text) => { sends.push({ text }); return { message_id: seq++ } },
|
|
32
|
+
editMessageText: async (_c, messageId, text) => { edits.push({ messageId, text }); return true },
|
|
33
|
+
}
|
|
34
|
+
const feed = createWorkerActivityFeed({
|
|
35
|
+
bot,
|
|
36
|
+
now,
|
|
37
|
+
minEditIntervalMs: 0,
|
|
38
|
+
firstPaintMinMs: 0,
|
|
39
|
+
setInterval: () => 0,
|
|
40
|
+
clearInterval: () => {},
|
|
41
|
+
})
|
|
42
|
+
return { feed, sends, edits }
|
|
43
|
+
}
|
|
44
|
+
const runningView = (desc: string, step: string, elapsedMs: number): WorkerActivityView => ({
|
|
45
|
+
description: desc, lastTool: null, toolCount: 3, latestSummary: step, elapsedMs, state: 'running',
|
|
46
|
+
})
|
|
47
|
+
async function drain(): Promise<void> {
|
|
48
|
+
for (let i = 0; i < 12; i++) await new Promise((r) => setImmediate(r))
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
describe('worker terminal state is truthful for reaped workers (Residual B)', () => {
|
|
52
|
+
it('a reaped/vanished worker (terminate, no clean finish) renders `incomplete`, never `done`', async () => {
|
|
53
|
+
let clock = 1000
|
|
54
|
+
const { feed, edits } = makeFeed(() => clock)
|
|
55
|
+
clock = 1000
|
|
56
|
+
await feed.update('w', 'chat', runningView('background job', 'doing work', 1000))
|
|
57
|
+
await drain()
|
|
58
|
+
clock = 5000
|
|
59
|
+
// Authoritative sweep / TTL backstop: no onFinish ever arrived.
|
|
60
|
+
await feed.terminate('w')
|
|
61
|
+
await drain()
|
|
62
|
+
|
|
63
|
+
const last = edits[edits.length - 1].text
|
|
64
|
+
expect(last, 'reaped worker must NOT read as done').not.toMatch(/\bdone\b/)
|
|
65
|
+
expect(last, 'reaped worker reads the truthful incomplete terminal').toContain('incomplete')
|
|
66
|
+
expect(feed.size).toBe(0)
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
it('a genuinely finished worker still renders `done` with its result', async () => {
|
|
70
|
+
let clock = 1000
|
|
71
|
+
const { feed, edits } = makeFeed(() => clock)
|
|
72
|
+
await feed.update('w', 'chat', runningView('background job', 'doing work', 1000))
|
|
73
|
+
await drain()
|
|
74
|
+
clock = 2000
|
|
75
|
+
await feed.finish('w', {
|
|
76
|
+
description: 'background job',
|
|
77
|
+
lastTool: null,
|
|
78
|
+
toolCount: 5,
|
|
79
|
+
latestSummary: 'the delivered result paragraph',
|
|
80
|
+
elapsedMs: 2000,
|
|
81
|
+
state: 'done',
|
|
82
|
+
})
|
|
83
|
+
await drain()
|
|
84
|
+
const last = edits[edits.length - 1].text
|
|
85
|
+
expect(last).toContain('done')
|
|
86
|
+
expect(last).toContain('the delivered result paragraph')
|
|
87
|
+
expect(last).not.toContain('incomplete')
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
// #3233 terminal-window race: the skeleton first-paint cue fires on EVERY
|
|
91
|
+
// no-growth poll, so one can land on a row that `finish()` just finalized
|
|
92
|
+
// (watcher poll cadence ≈ seconds). The `finalized` latch must absorb it —
|
|
93
|
+
// a late skeleton `update()` must NOT resurrect a fresh `running` card on an
|
|
94
|
+
// already-done worker. This exercises that path with a skeleton-shaped cue
|
|
95
|
+
// (empty step line, running state) arriving after finish.
|
|
96
|
+
it('a late skeleton update() after finish() is absorbed by the finalized latch (no resurrection)', async () => {
|
|
97
|
+
let clock = 1000
|
|
98
|
+
const { feed, edits } = makeFeed(() => clock)
|
|
99
|
+
await feed.update('w', 'chat', runningView('background job', 'doing work', 1000))
|
|
100
|
+
await drain()
|
|
101
|
+
clock = 2000
|
|
102
|
+
await feed.finish('w', {
|
|
103
|
+
description: 'background job',
|
|
104
|
+
lastTool: null,
|
|
105
|
+
toolCount: 5,
|
|
106
|
+
latestSummary: 'the delivered result paragraph',
|
|
107
|
+
elapsedMs: 2000,
|
|
108
|
+
state: 'done',
|
|
109
|
+
})
|
|
110
|
+
await drain()
|
|
111
|
+
const editsAfterFinish = edits.length
|
|
112
|
+
// Late skeleton cue (empty latestSummary, running state) — the shape the
|
|
113
|
+
// watcher emits on a no-growth poll — lands AFTER finalization.
|
|
114
|
+
clock = 2500
|
|
115
|
+
await feed.update('w', 'chat', {
|
|
116
|
+
description: 'background job',
|
|
117
|
+
lastTool: null,
|
|
118
|
+
toolCount: 5,
|
|
119
|
+
latestSummary: '',
|
|
120
|
+
elapsedMs: 2500,
|
|
121
|
+
state: 'running',
|
|
122
|
+
})
|
|
123
|
+
await drain()
|
|
124
|
+
// No further edit, no resurrection: the terminal card stays `done`.
|
|
125
|
+
expect(edits.length).toBe(editsAfterFinish)
|
|
126
|
+
expect(edits[edits.length - 1].text).toContain('done')
|
|
127
|
+
expect(feed.has('w')).toBe(false)
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
it('renderWorkerActivity renders the `incomplete` state as a finished card without a fabricated result', () => {
|
|
131
|
+
const card = renderWorkerActivity({
|
|
132
|
+
description: 'background job',
|
|
133
|
+
lastTool: null,
|
|
134
|
+
toolCount: 3,
|
|
135
|
+
latestSummary: '', // reaped: no result text
|
|
136
|
+
elapsedMs: 4000,
|
|
137
|
+
state: 'incomplete',
|
|
138
|
+
})
|
|
139
|
+
expect(card).toContain('incomplete')
|
|
140
|
+
expect(card).not.toMatch(/\bdone\b/)
|
|
141
|
+
// No fabricated result paragraph (latestSummary empty) → no ✅ result block.
|
|
142
|
+
expect(card).not.toContain('✅')
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
it('renderWorkerActivity NEVER renders a result/⚠️ block for `incomplete`, even with a non-empty latestSummary (deterministic guard, not caller-discipline)', () => {
|
|
146
|
+
// Latent-trap regression guard: the live call site (terminateWorker) always
|
|
147
|
+
// passes latestSummary:'' for a reaped worker, but a future/direct caller
|
|
148
|
+
// could pass stray text. The truthful-no-result invariant must be enforced
|
|
149
|
+
// in the renderer — an `incomplete` worker produced no result, so it must
|
|
150
|
+
// never fabricate a ⚠️-prefixed result paragraph regardless of the summary.
|
|
151
|
+
const card = renderWorkerActivity({
|
|
152
|
+
description: 'background job',
|
|
153
|
+
lastTool: null,
|
|
154
|
+
toolCount: 3,
|
|
155
|
+
latestSummary: 'this looks like a real result but the worker never finished',
|
|
156
|
+
elapsedMs: 4000,
|
|
157
|
+
state: 'incomplete',
|
|
158
|
+
})
|
|
159
|
+
expect(card).toContain('incomplete')
|
|
160
|
+
// The renderer maps a non-`done` finished result to the ⚠️ emoji; the guard
|
|
161
|
+
// must suppress that block entirely for `incomplete`.
|
|
162
|
+
expect(card).not.toContain('⚠️')
|
|
163
|
+
expect(card).not.toContain('this looks like a real result')
|
|
164
|
+
})
|
|
165
|
+
})
|
|
@@ -170,8 +170,10 @@ export function clipNarrative(s: string): string {
|
|
|
170
170
|
* `description` — italicised task description (optional)
|
|
171
171
|
* `elapsedMs` — wall-clock elapsed, rendered via `formatFeedElapsed`
|
|
172
172
|
* `toolCount` — labeled tool calls this turn
|
|
173
|
-
* `state` — 'running' | 'done' | 'failed' (controls the
|
|
174
|
-
*
|
|
173
|
+
* `state` — 'running' | 'done' | 'failed' | 'incomplete' (controls the
|
|
174
|
+
* status line wording; 'failed'/'incomplete' render
|
|
175
|
+
* `failed · …` / `incomplete · …` so a failed or reaped worker
|
|
176
|
+
* never reads as done)
|
|
175
177
|
*
|
|
176
178
|
* Returns a two-element array of ready Telegram HTML lines (no trailing newline).
|
|
177
179
|
*/
|
|
@@ -181,7 +183,7 @@ export function renderActivityHeader(
|
|
|
181
183
|
description: string,
|
|
182
184
|
elapsedMs: number,
|
|
183
185
|
toolCount: number,
|
|
184
|
-
state: 'running' | 'done' | 'failed',
|
|
186
|
+
state: 'running' | 'done' | 'failed' | 'incomplete',
|
|
185
187
|
model?: string,
|
|
186
188
|
): [string, string] {
|
|
187
189
|
const toolWord = toolCount === 1 ? 'tool' : 'tools'
|
|
@@ -283,7 +285,7 @@ export interface StatusCardHeader {
|
|
|
283
285
|
description?: string
|
|
284
286
|
elapsedMs: number
|
|
285
287
|
toolCount: number
|
|
286
|
-
state: 'running' | 'done' | 'failed'
|
|
288
|
+
state: 'running' | 'done' | 'failed' | 'incomplete'
|
|
287
289
|
/** Live model id (raw, e.g. `claude-opus-4-8`) — rendered as a short friendly
|
|
288
290
|
* tag on the metrics line via `formatModelLabel`. Omitted when unknown. */
|
|
289
291
|
model?: string
|
|
@@ -582,6 +584,16 @@ export interface CombinedWorkerRow {
|
|
|
582
584
|
toolCount: number
|
|
583
585
|
/** The worker's latest step line (raw prose or friendly tool label). */
|
|
584
586
|
currentStep: string
|
|
587
|
+
/**
|
|
588
|
+
* The worker's accumulated narrative history (oldest→newest, raw/unescaped),
|
|
589
|
+
* already deduped + rolling-window capped upstream (STATUS_ROLLING_LINES).
|
|
590
|
+
* When present + non-empty, the combined feed paints the worker's last-K
|
|
591
|
+
* lines as a `✓`/`→` step trail (prior steps struck, newest in-progress) —
|
|
592
|
+
* the same idiom as the single-worker card — with K set by the adaptive
|
|
593
|
+
* per-worker line budget. Absent/empty → falls back to the single
|
|
594
|
+
* `currentStep` line (back-compat for direct callers).
|
|
595
|
+
*/
|
|
596
|
+
historyLines?: string[]
|
|
585
597
|
/** Live model id (raw, e.g. `claude-opus-4-8`); omitted when unknown. */
|
|
586
598
|
model?: string
|
|
587
599
|
}
|
|
@@ -593,17 +605,53 @@ export interface CombinedWorkerFeedOpts {
|
|
|
593
605
|
maxRows: number
|
|
594
606
|
}
|
|
595
607
|
|
|
608
|
+
/**
|
|
609
|
+
* Total per-worker BODY line budget for the combined feed — the sum, across all
|
|
610
|
+
* visible workers, of (one header line + that worker's history lines). The top
|
|
611
|
+
* `🛠 Workers · N running` line and the `+M more working…` spill are OUTSIDE
|
|
612
|
+
* this budget (fixed chrome). 13 is chosen so the card stays a compact glance,
|
|
613
|
+
* not a wall: at 2 workers it yields the full 5-line history each (2·1 header +
|
|
614
|
+
* 2·5 history = 12 ≤ 13), and it degrades to a single history line each by ~6
|
|
615
|
+
* workers — matching the pre-adaptive one-line-per-worker floor while never
|
|
616
|
+
* letting a 2–3 worker fan-out lose its narrative trail.
|
|
617
|
+
*/
|
|
618
|
+
const MAX_COMBINED_BODY_LINES = 13
|
|
619
|
+
/** Each visible worker costs one header line before any history. */
|
|
620
|
+
const PER_WORKER_HEADER_COST = 1
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* Deterministic per-worker history depth for `w` visible workers:
|
|
624
|
+
* clamp( floor( (BUDGET − headerCost·w) / w ), 1, STATUS_ROLLING_LINES )
|
|
625
|
+
* So 2 workers → 5 lines each, 3 → 3, 4 → 2, ≥6 → 1 (today's single-line floor
|
|
626
|
+
* is the graceful-degradation floor, never below it). Pure function of the
|
|
627
|
+
* visible worker count — no model input, consistent with deterministic controls.
|
|
628
|
+
*/
|
|
629
|
+
export function combinedHistoryDepth(w: number): number {
|
|
630
|
+
if (w <= 0) return 1
|
|
631
|
+
const raw = Math.floor((MAX_COMBINED_BODY_LINES - PER_WORKER_HEADER_COST * w) / w)
|
|
632
|
+
return Math.max(1, Math.min(STATUS_ROLLING_LINES, raw))
|
|
633
|
+
}
|
|
634
|
+
|
|
596
635
|
/**
|
|
597
636
|
* Render N≥1 live workers into ONE combined feed body (ready Telegram
|
|
598
637
|
* markdown; callers send verbatim — do NOT re-escape). Layout:
|
|
599
638
|
*
|
|
600
639
|
* 🛠 **Workers** · _N running_
|
|
601
640
|
* **{desc1}** _· {elapsed} · {n} tools_
|
|
602
|
-
*
|
|
641
|
+
* ~~_✓ {earlier step}_~~
|
|
642
|
+
* **→ {newest step}**
|
|
603
643
|
* **{desc2}** _· {elapsed} · {n} tools_
|
|
604
|
-
*
|
|
644
|
+
* **→ {newest step}**
|
|
605
645
|
* _+M more working…_
|
|
606
646
|
*
|
|
647
|
+
* ADAPTIVE DENSITY: each visible worker renders its last-K narrative lines as a
|
|
648
|
+
* `✓`/`→` trail (prior steps struck, newest bold in-progress) — the single-
|
|
649
|
+
* worker card's idiom — where K = `combinedHistoryDepth(visibleCount)` splits a
|
|
650
|
+
* fixed body-line budget across the running workers. So 2 workers each show
|
|
651
|
+
* their full recent history and a 6-way fan-out degrades to one line each, the
|
|
652
|
+
* card staying bounded regardless of fan-out. When a worker has no history yet
|
|
653
|
+
* it falls back to a single `→ starting…`/currentStep line.
|
|
654
|
+
*
|
|
607
655
|
* Pure. Rows are rendered in the order supplied (the manager passes them
|
|
608
656
|
* dispatch-order, oldest first). `maxRows` caps the visible rows; the hidden
|
|
609
657
|
* remainder collapses to a single `+M more working…` line. A total-budget
|
|
@@ -618,29 +666,43 @@ export function renderCombinedWorkerFeed(
|
|
|
618
666
|
if (rows.length === 0) return null
|
|
619
667
|
const maxRows = Math.max(1, Math.floor(opts.maxRows))
|
|
620
668
|
|
|
621
|
-
const
|
|
669
|
+
const rowHeader = (r: CombinedWorkerRow): string => {
|
|
622
670
|
const desc = escapeMarkdown(
|
|
623
671
|
truncate(stripMarkdown(r.description).replace(/\s+/g, ' ').trim() || 'background task', COMBINED_ROW_DESC_MAX),
|
|
624
672
|
)
|
|
625
673
|
const toolWord = r.toolCount === 1 ? 'tool' : 'tools'
|
|
626
674
|
const modelLabel = formatModelLabel(r.model)
|
|
627
675
|
const modelPart = modelLabel != null ? ` · ${escapeMarkdown(modelLabel)}` : ''
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
676
|
+
return `**${desc}** _· ${formatFeedElapsed(r.elapsedMs)} · ${r.toolCount} ${toolWord}${modelPart}_`
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// Raw (unescaped) history for a worker, oldest→newest, empty lines stripped.
|
|
680
|
+
// Falls back to the single currentStep when no history was supplied.
|
|
681
|
+
const rowHistory = (r: CombinedWorkerRow): string[] => {
|
|
682
|
+
const src = r.historyLines != null && r.historyLines.length > 0 ? r.historyLines : [r.currentStep]
|
|
683
|
+
return src.filter((s) => s != null && stripMarkdown(s).replace(/\s+/g, ' ').trim().length > 0)
|
|
635
684
|
}
|
|
636
685
|
|
|
637
686
|
const compose = (visibleCount: number): string => {
|
|
638
687
|
const shown = rows.slice(0, visibleCount)
|
|
639
688
|
const hidden = rows.length - shown.length
|
|
689
|
+
// Adaptive depth: split the fixed body-line budget across the VISIBLE
|
|
690
|
+
// workers so the card stays bounded regardless of fan-out.
|
|
691
|
+
const depth = combinedHistoryDepth(shown.length)
|
|
640
692
|
const out: string[] = [`🛠 **Workers** · _${rows.length} running_`]
|
|
641
693
|
for (const r of shown) {
|
|
642
|
-
|
|
643
|
-
|
|
694
|
+
out.push(rowHeader(r))
|
|
695
|
+
const hist = rowHistory(r)
|
|
696
|
+
if (hist.length === 0) {
|
|
697
|
+
out.push('→ _starting…_')
|
|
698
|
+
continue
|
|
699
|
+
}
|
|
700
|
+
// Paint the last-K history lines with the SAME `✓`/`→` idiom as the
|
|
701
|
+
// single-worker card: escape each raw line through the shared per-line
|
|
702
|
+
// pipeline (escapeStepLine), then renderStepFeed strikes the prior steps
|
|
703
|
+
// and bolds the newest in-progress step.
|
|
704
|
+
const esc = hist.slice(-depth).map(escapeStepLine)
|
|
705
|
+
renderStepFeed(out, esc, false)
|
|
644
706
|
}
|
|
645
707
|
if (hidden > 0) out.push(`_+${hidden} more working…_`)
|
|
646
708
|
return stackCardLines(out)
|
|
@@ -197,10 +197,10 @@ export function decideTurnFlush(input: FlushDecisionInput): FlushDecision {
|
|
|
197
197
|
// lone `\n` collapses adjacent blocks into one run — on the Bot API 10.1
|
|
198
198
|
// rich-markdown path (#2669) a single newline is a soft break, so the blocks
|
|
199
199
|
// render as an undifferentiated wall-of-text. `\n\n` is the GFM paragraph
|
|
200
|
-
// separator; the
|
|
201
|
-
//
|
|
202
|
-
//
|
|
203
|
-
//
|
|
200
|
+
// separator; the gateway send path then wedges visible spacers into those
|
|
201
|
+
// gaps via addParagraphSpacers (mirroring the reply path, #2692) so the
|
|
202
|
+
// paragraphs render with real separation (the rich GFM renderer otherwise
|
|
203
|
+
// shows a bare `\n\n` gap TIGHT).
|
|
204
204
|
//
|
|
205
205
|
// The silent-marker guards below are unaffected by this change:
|
|
206
206
|
// isSilentFlushMarker length-guards the whole joined string; the composite /
|
|
@@ -69,7 +69,20 @@ export function isWorkerActivityFeedEnabled(envVal: string | undefined): boolean
|
|
|
69
69
|
return envVal !== '0'
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
/**
|
|
73
|
+
* Terminal states a worker card can render.
|
|
74
|
+
* - 'running' — live.
|
|
75
|
+
* - 'done' — clean finish WITH a result (✅).
|
|
76
|
+
* - 'failed' — clean finish reporting a failure / crash observed in the
|
|
77
|
+
* transcript (⚠️).
|
|
78
|
+
* - 'incomplete' — force-reaped / vanished / crashed WITHOUT any clean finish
|
|
79
|
+
* (the TTL sweep or the watcher's authoritative
|
|
80
|
+
* `onTerminalCleanup` synthesised the terminal because
|
|
81
|
+
* neither `onFinish` nor a turn_end ever delivered a result).
|
|
82
|
+
* Renders `incomplete · …` so a reaped worker NEVER reads as
|
|
83
|
+
* "done" — the truthful "ended without result" terminal.
|
|
84
|
+
*/
|
|
85
|
+
export type WorkerActivityState = 'running' | 'done' | 'failed' | 'incomplete'
|
|
73
86
|
|
|
74
87
|
/** The render-relevant snapshot of a worker at one instant. */
|
|
75
88
|
export interface WorkerActivityView {
|
|
@@ -141,7 +154,7 @@ const DESC_MAX = 80
|
|
|
141
154
|
*/
|
|
142
155
|
export function renderWorkerActivity(v: WorkerActivityView, liveSuffix = ''): string {
|
|
143
156
|
const desc = truncate(stripMarkdown(v.description).trim() || 'background task', DESC_MAX)
|
|
144
|
-
const finished = v.state === 'done' || v.state === 'failed'
|
|
157
|
+
const finished = v.state === 'done' || v.state === 'failed' || v.state === 'incomplete'
|
|
145
158
|
|
|
146
159
|
// Raw narrative steps (unstripped/unescaped) — the unified renderer runs the
|
|
147
160
|
// full per-line pipeline (stripMarkdown → collapse ws → clip → escape).
|
|
@@ -169,8 +182,15 @@ export function renderWorkerActivity(v: WorkerActivityView, liveSuffix = ''): st
|
|
|
169
182
|
|
|
170
183
|
// Terminal: latestSummary carries the worker's final result text (gateway
|
|
171
184
|
// onFinish), distinct from the running narrative steps. Pass it as `result`.
|
|
185
|
+
//
|
|
186
|
+
// Truthful-no-result invariant (deterministic control, not caller-discipline):
|
|
187
|
+
// an `incomplete` worker produced NO result, so it must NEVER render a result
|
|
188
|
+
// block regardless of whatever `latestSummary` happens to carry. The current
|
|
189
|
+
// call site (terminateWorker) always sets latestSummary:'' for incomplete, but
|
|
190
|
+
// enforce the invariant HERE so any future/direct caller can't fabricate a
|
|
191
|
+
// `⚠️`-prefixed result paragraph out of stray summary text.
|
|
172
192
|
let result: { emoji: string; text: string } | undefined
|
|
173
|
-
if (finished) {
|
|
193
|
+
if (finished && v.state !== 'incomplete') {
|
|
174
194
|
const text = cleanWorkerResultParagraph(v.latestSummary)
|
|
175
195
|
if (text.length > 0) result = { emoji: v.state === 'done' ? '✅' : '⚠️', text }
|
|
176
196
|
}
|
|
@@ -257,6 +277,27 @@ export interface WorkerActivityFeedOpts {
|
|
|
257
277
|
* `channels.telegram.worker_feed.max_rows` via the config cascade.
|
|
258
278
|
*/
|
|
259
279
|
maxRows?: number
|
|
280
|
+
/**
|
|
281
|
+
* Backstop TTL (ms): a worker row that has received no `update()` cue for
|
|
282
|
+
* longer than this AND is not already finished is force-terminated by the
|
|
283
|
+
* heartbeat sweep — its row is removed, and when it was the last live worker
|
|
284
|
+
* the shared card collapses to its terminal summary and unpins. This is the
|
|
285
|
+
* durable guard against an immortal card if BOTH terminal signals are missed
|
|
286
|
+
* (the gateway's `onFinish` AND the watcher's `onTerminalCleanup` sweep).
|
|
287
|
+
*
|
|
288
|
+
* The gateway DERIVES this in code from the watcher's effective in-flight
|
|
289
|
+
* terminal cap (`resolveInflightTerminalCapMs()` — the same env/default the
|
|
290
|
+
* watcher resolves) plus a margin, so the invariant "never reap a row the
|
|
291
|
+
* watcher still considers live" holds even if an operator raises the cap via
|
|
292
|
+
* `SWITCHROOM_SUBAGENT_INFLIGHT_TERMINAL_CAP_MS`. A worker mid-very-long tool
|
|
293
|
+
* can go silent up to the cap before the watcher declares it terminal, so any
|
|
294
|
+
* row still present past cap + margin is a definitive leak — the watcher would
|
|
295
|
+
* already have swept a live worker.
|
|
296
|
+
*
|
|
297
|
+
* Fallback default (this module, for direct / non-gateway callers) 50 min.
|
|
298
|
+
* Tests inject a small value.
|
|
299
|
+
*/
|
|
300
|
+
staleWorkerTtlMs?: number
|
|
260
301
|
/**
|
|
261
302
|
* Group-level status-pin reconcile hook (#3207 review). Because workers now
|
|
262
303
|
* COALESCE into one shared message, the pin MUST follow the GROUP lifecycle,
|
|
@@ -309,6 +350,15 @@ interface WorkerRow {
|
|
|
309
350
|
* stable sort order within the combined feed.
|
|
310
351
|
*/
|
|
311
352
|
dispatchAtMs: number | null
|
|
353
|
+
/**
|
|
354
|
+
* Wall-clock ms of the most recent `update()` cue for this worker (its last
|
|
355
|
+
* observed liveness). The heartbeat's backstop TTL sweep force-terminates a
|
|
356
|
+
* row whose `lastUpdateAt` is older than `staleWorkerTtlMs` — the durable
|
|
357
|
+
* guard that no card can go immortal even if every terminal signal (onFinish
|
|
358
|
+
* AND the onTerminalCleanup sweep) is somehow missed. Stamped on row creation
|
|
359
|
+
* and on every `update()`.
|
|
360
|
+
*/
|
|
361
|
+
lastUpdateAt: number
|
|
312
362
|
/**
|
|
313
363
|
* Wall-clock ms the CURRENT step started — stamped whenever a NEW narrative
|
|
314
364
|
* line lands (the `→` line changes). The heartbeat's step suffix shows the
|
|
@@ -432,6 +482,17 @@ export interface WorkerActivityFeed {
|
|
|
432
482
|
* in the group — force the terminal recap edit. No-op if the worker was
|
|
433
483
|
* never tracked. */
|
|
434
484
|
finish(agentId: string, view: WorkerActivityView): Promise<void>
|
|
485
|
+
/**
|
|
486
|
+
* Force a worker terminal from an AUTHORITATIVE watcher sweep
|
|
487
|
+
* (`onTerminalCleanup`) or the backstop TTL, WITHOUT an external result view:
|
|
488
|
+
* the terminal recap is synthesised from the worker's own last-known row
|
|
489
|
+
* state (state → `done`, no fabricated result text — the real result reaches
|
|
490
|
+
* the user via the separate handback, if one ran). When it was the last live
|
|
491
|
+
* worker the shared card collapses to its terminal summary and unpins; with
|
|
492
|
+
* siblings, its row is dropped and the combined body re-renders. Idempotent —
|
|
493
|
+
* a no-op if the worker was already removed (e.g. `onFinish` fired first).
|
|
494
|
+
*/
|
|
495
|
+
terminate(agentId: string): Promise<void>
|
|
435
496
|
/** Forget a worker's state without a recap edit (e.g. error path); re-renders
|
|
436
497
|
* the group so the dropped worker disappears from the combined body. */
|
|
437
498
|
drop(agentId: string): void
|
|
@@ -461,6 +522,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
461
522
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000
|
|
462
523
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
|
|
463
524
|
const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
|
|
525
|
+
const staleWorkerTtlMs = Math.max(1, Math.floor(opts.staleWorkerTtlMs ?? 50 * 60_000))
|
|
464
526
|
const reconcilePinFn = opts.reconcilePin ?? (() => {})
|
|
465
527
|
const setIntervalFn =
|
|
466
528
|
opts.setInterval ??
|
|
@@ -616,6 +678,11 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
616
678
|
elapsedMs: elapsedFor(r),
|
|
617
679
|
toolCount: v.toolCount,
|
|
618
680
|
currentStep,
|
|
681
|
+
// Full per-worker rolling history (oldest→newest) so the combined feed
|
|
682
|
+
// can paint an adaptive-depth ✓/→ trail, not just the latest line. The
|
|
683
|
+
// renderer clamps depth to the shared body-line budget; when a worker
|
|
684
|
+
// has no narrative yet this is empty and it falls back to currentStep.
|
|
685
|
+
historyLines: r.narrative.length > 0 ? [...r.narrative] : undefined,
|
|
619
686
|
model: v.model,
|
|
620
687
|
}
|
|
621
688
|
})
|
|
@@ -785,10 +852,110 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
785
852
|
}
|
|
786
853
|
}
|
|
787
854
|
|
|
855
|
+
/**
|
|
856
|
+
* Shared terminal-finalize path for a worker whose row is still tracked.
|
|
857
|
+
* Latches the row terminal synchronously (so a late `running` cue on the
|
|
858
|
+
* chain can't resurrect it), then on the chain either drops the row + re-
|
|
859
|
+
* renders the surviving siblings, or — when it was the last live worker —
|
|
860
|
+
* finalizes the shared message to its terminal recap. Used by `finish` (with
|
|
861
|
+
* the gateway's onFinish view) AND by `terminate` (authoritative watcher
|
|
862
|
+
* sweep / TTL backstop, view synthesised from the row's own last state).
|
|
863
|
+
*/
|
|
864
|
+
function finalizeWorker(group: FeedGroup, agentId: string, row: WorkerRow, view: WorkerActivityView): Promise<void> {
|
|
865
|
+
row.finished = true
|
|
866
|
+
// Preserve the truthful terminal state through to the row (done / failed /
|
|
867
|
+
// incomplete) — never collapse a reaped 'incomplete' into 'done'. view.state
|
|
868
|
+
// is always terminal on this path (finalize is only reached for a finishing
|
|
869
|
+
// worker), so threading it verbatim is correct.
|
|
870
|
+
row.state = view.state
|
|
871
|
+
markFinalized(agentId)
|
|
872
|
+
group.chain = group.chain
|
|
873
|
+
.then(() => {
|
|
874
|
+
const others = runningRows(group).filter((w) => w.agentId !== agentId)
|
|
875
|
+
if (others.length > 0) {
|
|
876
|
+
// Siblings still live → drop this row from the combined body and re-
|
|
877
|
+
// render the running set. The group pin STAYS (siblings still need
|
|
878
|
+
// the shared message).
|
|
879
|
+
removeWorker(group, agentId)
|
|
880
|
+
syncPin(group)
|
|
881
|
+
return doRender(group, { force: true })
|
|
882
|
+
}
|
|
883
|
+
// Last live worker → finalize the shared message to its terminal recap.
|
|
884
|
+
const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
|
|
885
|
+
return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
|
|
886
|
+
})
|
|
887
|
+
.catch((err) => {
|
|
888
|
+
log(`worker-feed: finalize chain error ${agentId}: ${(err as Error).message}`)
|
|
889
|
+
})
|
|
890
|
+
return group.chain
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
/**
|
|
894
|
+
* Force a worker terminal without an external result view (authoritative
|
|
895
|
+
* `onTerminalCleanup` sweep or the TTL backstop). Synthesises the recap from
|
|
896
|
+
* the row's own last-known state — state `incomplete` (NOT `done`), NO
|
|
897
|
+
* fabricated result text (the real result, if any, reaches the user via the
|
|
898
|
+
* separate handback). Reaching here means NO clean finish arrived: a clean
|
|
899
|
+
* `onFinish` removes the row FIRST (this call is then a no-op), and an
|
|
900
|
+
* errored worker goes through `onFinish(outcome:'failed')` → `finish` — so a
|
|
901
|
+
* row still present at terminate time genuinely ended WITHOUT a result. It
|
|
902
|
+
* must therefore render `incomplete · …`, never "done" (a crash/vanish read
|
|
903
|
+
* as "done" was the residual this fixes). Idempotent: a no-op once the worker
|
|
904
|
+
* was already removed (onFinish first). Depth-generic: operates on the
|
|
905
|
+
* generic worker row by agentId, identical at every nesting level.
|
|
906
|
+
*/
|
|
907
|
+
function terminateWorker(agentId: string): Promise<void> {
|
|
908
|
+
const g = groupOfAgent(agentId)
|
|
909
|
+
const row = g?.workers.get(agentId)
|
|
910
|
+
if (g == null || row == null) {
|
|
911
|
+
// Already gone (onFinish removed it) — latch finalized so a late cue can't
|
|
912
|
+
// resurrect, and no-op.
|
|
913
|
+
markFinalized(agentId)
|
|
914
|
+
return Promise.resolve()
|
|
915
|
+
}
|
|
916
|
+
if (row.finished) {
|
|
917
|
+
// Terminal already latched; the chain will settle removal. No-op.
|
|
918
|
+
return g.chain
|
|
919
|
+
}
|
|
920
|
+
const lv = row.lastView
|
|
921
|
+
const view: WorkerActivityView = {
|
|
922
|
+
description: lv?.description ?? 'background task',
|
|
923
|
+
lastTool: null,
|
|
924
|
+
toolCount: lv?.toolCount ?? 0,
|
|
925
|
+
// No fabricated result paragraph — an authoritative sweep can't know what
|
|
926
|
+
// the worker returned; the terminal card shows the header struck-through
|
|
927
|
+
// as `incomplete`, and the handback (if it ran) carries the actual result.
|
|
928
|
+
latestSummary: '',
|
|
929
|
+
elapsedMs: liveElapsed(row, nowFn()),
|
|
930
|
+
state: 'incomplete',
|
|
931
|
+
model: lv?.model,
|
|
932
|
+
}
|
|
933
|
+
return finalizeWorker(g, agentId, row, view)
|
|
934
|
+
}
|
|
935
|
+
|
|
788
936
|
// Arm the heartbeat once at construction. The real timer is `.unref()`'d so
|
|
789
937
|
// it never keeps the process alive; tests inject setInterval/clearInterval.
|
|
790
938
|
function heartbeatTick(): void {
|
|
791
939
|
const now = nowFn()
|
|
940
|
+
// Backstop TTL sweep: force-terminate any worker row that has gone silent
|
|
941
|
+
// past `staleWorkerTtlMs` (no `update()` cue AND not already finished). This
|
|
942
|
+
// is the durable guard against an immortal card if BOTH terminal signals
|
|
943
|
+
// are missed — the gateway's `onFinish` AND the watcher's authoritative
|
|
944
|
+
// `onTerminalCleanup` sweep. Collect the stale agent ids first (terminate
|
|
945
|
+
// mutates the group's worker map), then terminate each through its chain so
|
|
946
|
+
// the render/unpin happens under the normal cooldown/flood guards.
|
|
947
|
+
const staleAgentIds: string[] = []
|
|
948
|
+
for (const g of groups.values()) {
|
|
949
|
+
for (const row of g.workers.values()) {
|
|
950
|
+
if (!row.finished && now - row.lastUpdateAt >= staleWorkerTtlMs) {
|
|
951
|
+
staleAgentIds.push(row.agentId)
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
for (const agentId of staleAgentIds) {
|
|
956
|
+
log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (groupOfAgent(agentId)?.workers.get(agentId)?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
|
|
957
|
+
void terminateWorker(agentId)
|
|
958
|
+
}
|
|
792
959
|
for (const g of [...groups.values()]) {
|
|
793
960
|
// Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
|
|
794
961
|
// window was staged on `pendingFinalize`. Re-drive it once the cooldown
|
|
@@ -888,12 +1055,16 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
888
1055
|
lastView: null,
|
|
889
1056
|
state: 'running',
|
|
890
1057
|
finished: false,
|
|
1058
|
+
lastUpdateAt: nowFn(),
|
|
891
1059
|
dispatchAtMs: null,
|
|
892
1060
|
stepStartedAtMs: null,
|
|
893
1061
|
}
|
|
894
1062
|
g.workers.set(agentId, row)
|
|
895
1063
|
agentIndex.set(agentId, feedKey)
|
|
896
1064
|
}
|
|
1065
|
+
// Stamp liveness for the backstop TTL sweep (this is the worker's most
|
|
1066
|
+
// recent observed activity cue).
|
|
1067
|
+
row.lastUpdateAt = nowFn()
|
|
897
1068
|
// Accumulate before the gate so a throttled tick still grows the
|
|
898
1069
|
// narrative — it surfaces on the next edit that does fire.
|
|
899
1070
|
accumulateNarrative(row, view)
|
|
@@ -916,33 +1087,13 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
916
1087
|
markFinalized(agentId)
|
|
917
1088
|
return Promise.resolve()
|
|
918
1089
|
}
|
|
919
|
-
// Latch
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
.then(() => {
|
|
927
|
-
const others = runningRows(group).filter((w) => w.agentId !== agentId)
|
|
928
|
-
if (others.length > 0) {
|
|
929
|
-
// Siblings still live → drop this row from the combined body and
|
|
930
|
-
// re-render the running set. The result reaches the user via the
|
|
931
|
-
// separate handback, never folded into this cosmetic edit. The
|
|
932
|
-
// group pin STAYS (siblings still need the shared message) — a
|
|
933
|
-
// per-worker unpin here was the #3207 review blocker.
|
|
934
|
-
removeWorker(group, agentId)
|
|
935
|
-
syncPin(group)
|
|
936
|
-
return doRender(group, { force: true })
|
|
937
|
-
}
|
|
938
|
-
// Last live worker → finalize the shared message to its terminal recap.
|
|
939
|
-
const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
|
|
940
|
-
return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
|
|
941
|
-
})
|
|
942
|
-
.catch((err) => {
|
|
943
|
-
log(`worker-feed: finish chain error ${agentId}: ${(err as Error).message}`)
|
|
944
|
-
})
|
|
945
|
-
return group.chain
|
|
1090
|
+
// Latch + finalize via the shared path (drop-with-siblings / terminal
|
|
1091
|
+
// recap on the last worker). Synchronous latch inside finalizeWorker
|
|
1092
|
+
// stops a late `running` cue on the chain from resurrecting the row.
|
|
1093
|
+
return finalizeWorker(g, agentId, row, view)
|
|
1094
|
+
},
|
|
1095
|
+
terminate(agentId) {
|
|
1096
|
+
return terminateWorker(agentId)
|
|
946
1097
|
},
|
|
947
1098
|
drop(agentId) {
|
|
948
1099
|
// A dropped worker is also done — mark finalized so a late tick can't
|