switchroom 0.16.46 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +83 -81
- package/dist/auth-broker/index.js +104 -88
- package/dist/cli/autoaccept-poll.js +8 -8
- package/dist/cli/drive-write-pretool.mjs +10 -15
- package/dist/cli/notion-write-pretool.mjs +85 -83
- package/dist/cli/skill-validate-pretool.mjs +91 -91
- package/dist/cli/switchroom.js +1720 -1392
- package/dist/cli/ui/index.html +84 -12
- package/dist/host-control/main.js +209 -173
- package/dist/vault/approvals/kernel-server.js +86 -83
- package/dist/vault/broker/server.js +284 -139
- package/package.json +3 -3
- package/profiles/_base/cron-session.sh.hbs +1 -1
- package/profiles/_base/start.sh.hbs +54 -3
- package/skills/switchroom-architecture/telegram.md +8 -15
- package/skills/switchroom-cli/SKILL.md +4 -5
- package/skills/telegram-test-harness/SKILL.md +1 -1
- package/telegram-plugin/README.md +18 -29
- package/telegram-plugin/bridge/bridge.ts +1 -41
- package/telegram-plugin/bridge/tool-filter.ts +3 -4
- package/telegram-plugin/dist/bridge/bridge.js +120 -155
- package/telegram-plugin/dist/gateway/gateway.js +1127 -1029
- package/telegram-plugin/dist/server.js +168 -203
- package/telegram-plugin/gateway/busy-key-reaper.ts +113 -0
- package/telegram-plugin/gateway/disconnect-flush.ts +11 -0
- package/telegram-plugin/gateway/escalation-bridge-gate.ts +46 -0
- package/telegram-plugin/gateway/gate-parity-probe.ts +102 -0
- package/telegram-plugin/gateway/gateway.ts +566 -631
- package/telegram-plugin/gateway/inbound-delivery-confirm.ts +89 -7
- package/telegram-plugin/gateway/inbound-spool.ts +108 -10
- package/telegram-plugin/gateway/model-command.ts +51 -3
- package/telegram-plugin/gateway/pending-inbound-buffer.ts +26 -0
- package/telegram-plugin/gateway/represent-guard.ts +28 -11
- package/telegram-plugin/gateway/status-pin-store.ts +124 -45
- package/telegram-plugin/gateway/worker-feed-dispatch.ts +19 -0
- package/telegram-plugin/history.ts +5 -0
- package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +1 -2
- package/telegram-plugin/hooks/subagent-tracker-pretool.mjs +9 -1
- package/telegram-plugin/registry/subagents-schema.ts +126 -1
- package/telegram-plugin/registry/turns-schema.ts +65 -1
- package/telegram-plugin/session-tail.ts +26 -4
- package/telegram-plugin/slot-banner-driver.ts +42 -2
- package/telegram-plugin/status-query-telemetry.ts +100 -0
- package/telegram-plugin/stream-reply-handler.ts +15 -16
- package/telegram-plugin/subagent-watcher.ts +182 -30
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +4 -10
- package/telegram-plugin/tests/busy-key-reaper.test.ts +191 -0
- package/telegram-plugin/tests/emission-authority-facade.test.ts +11 -17
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +5 -26
- package/telegram-plugin/tests/escalation-bridge-gate.test.ts +38 -0
- package/telegram-plugin/tests/gate-parity-probe.test.ts +171 -0
- package/telegram-plugin/tests/gateway-disconnect-flush.test.ts +13 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +14 -11
- package/telegram-plugin/tests/inbound-delivery-confirm.test.ts +146 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +143 -0
- package/telegram-plugin/tests/model-command.test.ts +54 -1
- package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +5 -11
- package/telegram-plugin/tests/nested-worker-visibility-harness.test.ts +329 -0
- package/telegram-plugin/tests/pending-inbound-buffer.test.ts +53 -0
- package/telegram-plugin/tests/progress-update-redact.test.ts +99 -0
- package/telegram-plugin/tests/registry-turns.test.ts +67 -0
- package/telegram-plugin/tests/represent-guard.test.ts +42 -6
- package/telegram-plugin/tests/resume-inbound-builder.test.ts +1 -0
- package/telegram-plugin/tests/session-tail.test.ts +10 -1
- package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +246 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +0 -14
- package/telegram-plugin/tests/status-pin-store.test.ts +220 -5
- package/telegram-plugin/tests/status-query-telemetry.test.ts +115 -0
- package/telegram-plugin/tests/subagent-nested-dispatch.test.ts +209 -0
- package/telegram-plugin/tests/subagent-tracker-hooks.test.ts +37 -0
- package/telegram-plugin/tests/subagent-watcher-boot-promotion-replay.test.ts +167 -0
- package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +46 -3
- package/telegram-plugin/tests/subagent-watcher-stall-notification.test.ts +70 -0
- package/telegram-plugin/tests/tool-activity-summary.test.ts +16 -0
- package/telegram-plugin/tests/tool-filter.test.ts +1 -3
- package/telegram-plugin/tests/tool-label-pretool.test.ts +1 -4
- package/telegram-plugin/tests/turn-flush-safety.test.ts +222 -1
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +46 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +202 -9
- package/telegram-plugin/tests/worker-feed-dispatch.test.ts +25 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +295 -0
- package/telegram-plugin/tool-activity-summary.ts +19 -0
- package/telegram-plugin/turn-flush-safety.ts +16 -1
- package/telegram-plugin/uat/scenarios/jtbd-answer-pings.test.ts +8 -9
- package/telegram-plugin/uat/scenarios/jtbd-foreground-feed-visibility-dm.test.ts +1 -1
- package/telegram-plugin/uat/scenarios/jtbd-narration-intent-dm.test.ts +1 -1
- package/telegram-plugin/worker-activity-feed.ts +75 -15
- package/vendor/hindsight-memory/CHANGELOG.md +24 -0
- package/vendor/hindsight-memory/README.md +5 -0
- package/vendor/hindsight-memory/scripts/lib/client.py +31 -1
- package/vendor/hindsight-memory/scripts/lib/config.py +41 -2
- package/vendor/hindsight-memory/scripts/lib/content.py +4 -1
- package/vendor/hindsight-memory/scripts/lib/daemon.py +11 -2
- package/vendor/hindsight-memory/scripts/recall.py +74 -1
- package/vendor/hindsight-memory/scripts/retain.py +8 -1
- package/vendor/hindsight-memory/scripts/tests/test_config_client_casts.py +111 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_integration.py +85 -1
- package/vendor/hindsight-memory/scripts/tests/test_recall_tag_filters.py +107 -0
- package/vendor/hindsight-memory/settings.json +4 -0
- package/vendor/hindsight-memory/tests/test_client.py +130 -0
- package/vendor/hindsight-memory/tests/test_config.py +47 -0
- package/vendor/hindsight-memory/tests/test_content.py +18 -0
- package/vendor/hindsight-memory/tests/test_hooks.py +62 -0
- package/telegram-plugin/gateway/error-envelope-card.ts +0 -64
- package/telegram-plugin/gateway/resolve-calling-subagent.ts +0 -78
- package/telegram-plugin/silent-reply.ts +0 -58
- package/telegram-plugin/tests/error-envelope-unlock-card.test.ts +0 -79
- package/telegram-plugin/tests/resolve-calling-subagent.test.ts +0 -269
- package/telegram-plugin/tests/silent-reply-guard.test.ts +0 -122
|
@@ -72,6 +72,23 @@ export interface RefreshBannerArgs {
|
|
|
72
72
|
/** Optional API-failure observer. Phase identifies which Bot API
|
|
73
73
|
* call failed so the caller can log meaningfully. Default: silent. */
|
|
74
74
|
onError?: (phase: 'pin' | 'edit' | 'unpin', err: unknown) => void;
|
|
75
|
+
/** Optional durable-persistence hooks so an orphaned banner pin is
|
|
76
|
+
* recoverable across a gateway crash. The gateway wires these to the
|
|
77
|
+
* shared status-pin store (a distinct `banner:` pinKey), mirroring the
|
|
78
|
+
* status-pin store's persist-BEFORE-pin ordering. All hooks are
|
|
79
|
+
* best-effort — the driver guards each call so a throwing hook can never
|
|
80
|
+
* break banner pinning (persistence is cosmetic-recovery, never load-bearing
|
|
81
|
+
* for the live pin). Omit entirely to disable persistence (unit tests). */
|
|
82
|
+
persist?: {
|
|
83
|
+
/** Record a PENDING pin: the banner message was sent but the pin API call
|
|
84
|
+
* has not yet confirmed. Called AFTER sendMessage, BEFORE pinChatMessage,
|
|
85
|
+
* so a crash in that window leaves a recoverable record on disk. */
|
|
86
|
+
pending?: (chatId: string, messageId: number) => void;
|
|
87
|
+
/** Rewrite the record as confirmed once the pin lands. */
|
|
88
|
+
confirm?: (chatId: string, messageId: number) => void;
|
|
89
|
+
/** Drop the persisted record (banner unpinned, cleared, or pin failed). */
|
|
90
|
+
clear?: () => void;
|
|
91
|
+
};
|
|
75
92
|
}
|
|
76
93
|
|
|
77
94
|
/**
|
|
@@ -95,6 +112,17 @@ export async function refreshBanner(
|
|
|
95
112
|
args.defaultSlot,
|
|
96
113
|
);
|
|
97
114
|
|
|
115
|
+
// Persistence hooks are cosmetic-recovery; a throwing hook must never break
|
|
116
|
+
// live banner pinning. Guard every call.
|
|
117
|
+
const safePersist = (fn: (() => void) | undefined) => {
|
|
118
|
+
if (!fn) return;
|
|
119
|
+
try {
|
|
120
|
+
fn();
|
|
121
|
+
} catch {
|
|
122
|
+
/* best-effort — persistence failures degrade to in-memory-only */
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
|
|
98
126
|
if (action.kind === 'noop') return args.prevState;
|
|
99
127
|
|
|
100
128
|
if (action.kind === 'unpin') {
|
|
@@ -105,7 +133,10 @@ export async function refreshBanner(
|
|
|
105
133
|
}
|
|
106
134
|
// Even if unpin failed, drop our claim — the message may have been
|
|
107
135
|
// unpinned out-of-band (operator did it manually) and re-pinning
|
|
108
|
-
// would be more confusing than surfacing it again later.
|
|
136
|
+
// would be more confusing than surfacing it again later. Drop the
|
|
137
|
+
// persisted record too (unpin-then-clear mirrors the status-pin store:
|
|
138
|
+
// a crash between unpin and clear just re-unpins next boot, idempotent).
|
|
139
|
+
safePersist(args.persist?.clear);
|
|
109
140
|
return null;
|
|
110
141
|
}
|
|
111
142
|
|
|
@@ -122,15 +153,24 @@ export async function refreshBanner(
|
|
|
122
153
|
args.onError?.('pin', err);
|
|
123
154
|
return args.prevState;
|
|
124
155
|
}
|
|
156
|
+
// Persist INTENT (pending) BEFORE the pin API call: a crash between the pin
|
|
157
|
+
// landing and its confirm rewrite leaves a pending record that boot cleanup
|
|
158
|
+
// unpins next boot — closing the persist-after-pin leak.
|
|
159
|
+
safePersist(() => args.persist?.pending?.(String(args.ownerChatId), sent.message_id));
|
|
125
160
|
try {
|
|
126
161
|
await args.bot.api.pinChatMessage(args.ownerChatId, sent.message_id, {
|
|
127
162
|
disable_notification: true,
|
|
128
163
|
});
|
|
129
164
|
} catch (err) {
|
|
130
165
|
args.onError?.('pin', err);
|
|
131
|
-
// sendMessage succeeded but pin failed — don't claim the message
|
|
166
|
+
// sendMessage succeeded but pin failed — don't claim the message, and
|
|
167
|
+
// drop the pending record so we don't leave a phantom claim for a pin
|
|
168
|
+
// that never landed.
|
|
169
|
+
safePersist(args.persist?.clear);
|
|
132
170
|
return args.prevState;
|
|
133
171
|
}
|
|
172
|
+
// Pin confirmed — rewrite the record without the pending flag.
|
|
173
|
+
safePersist(() => args.persist?.confirm?.(String(args.ownerChatId), sent.message_id));
|
|
134
174
|
return { messageId: sent.message_id, slot: action.slot };
|
|
135
175
|
}
|
|
136
176
|
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Truthful `status-query` telemetry — issue #109 (truthful-telemetry PR).
|
|
3
|
+
*
|
|
4
|
+
* When the user has to ask "status?" mid-turn, the gateway logs a snapshot of
|
|
5
|
+
* what is ACTUALLY in flight so the frequency of surface failures can be
|
|
6
|
+
* counted. This module is the pure decision at the heart of that log line.
|
|
7
|
+
*
|
|
8
|
+
* History / why this is pure: the snapshot used to be read from the pinned
|
|
9
|
+
* progress card via `progressDriver.peek(...)`. That card was retired
|
|
10
|
+
* (#1122/#1126) and `progressDriver` is now permanently `null`, so the old
|
|
11
|
+
* peek ALWAYS reported idle/zero regardless of live background work — a lying
|
|
12
|
+
* diagnostic that actively misled debugging. The gateway now reads the LIVE
|
|
13
|
+
* surfaces (running-background-worker DB count, the `🛠 Worker` activity-feed
|
|
14
|
+
* size, the cross-turn pending-async-dispatch flag, and whether a turn is
|
|
15
|
+
* active) and passes them here. A live background worker ⇒ non-idle.
|
|
16
|
+
*
|
|
17
|
+
* A `ux-failure` is emitted ONLY when `idle` is true — every surface is
|
|
18
|
+
* genuinely zero AND no turn is active, i.e. the user asked "status?" and
|
|
19
|
+
* there really was nothing in flight for the surfaces to have shown.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export interface StatusQuerySurfaces {
|
|
23
|
+
/** A turn is currently active for this chat key (`activeTurnStartedAt`). */
|
|
24
|
+
turnActive: boolean
|
|
25
|
+
/**
|
|
26
|
+
* Age of the active turn in whole seconds, or `-1` when no turn is active
|
|
27
|
+
* (numeric sentinel so structured-log parsers avoid string branches).
|
|
28
|
+
*/
|
|
29
|
+
turnAgeS: number
|
|
30
|
+
/** Background sub-agent workers currently `running` (registry DB count). */
|
|
31
|
+
runningWorkers: number
|
|
32
|
+
/** Workers tracked by the edit-in-place `🛠 Worker` activity feed. */
|
|
33
|
+
workerFeedSize: number
|
|
34
|
+
/** Cross-turn pending-async dispatch for this chat key: 1 if pending, else 0. */
|
|
35
|
+
pendingAsync: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type StatusQueryStage = 'idle' | 'turn-active' | 'background-work'
|
|
39
|
+
|
|
40
|
+
export interface StatusQueryTelemetry {
|
|
41
|
+
stage: StatusQueryStage
|
|
42
|
+
/**
|
|
43
|
+
* True only when NOTHING is live on any surface. Gates the `ux-failure`
|
|
44
|
+
* emission: a live turn or a live background worker means the surfaces DID
|
|
45
|
+
* have something to show, so the "status?" is not a clean surface failure.
|
|
46
|
+
*/
|
|
47
|
+
idle: boolean
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Derive the truthful stage + idle verdict from the live surfaces. Pure and
|
|
52
|
+
* total so it can be unit-tested without standing up the gateway.
|
|
53
|
+
*/
|
|
54
|
+
export function deriveStatusQueryTelemetry(s: StatusQuerySurfaces): StatusQueryTelemetry {
|
|
55
|
+
const idle =
|
|
56
|
+
!s.turnActive &&
|
|
57
|
+
s.runningWorkers <= 0 &&
|
|
58
|
+
s.workerFeedSize <= 0 &&
|
|
59
|
+
s.pendingAsync <= 0
|
|
60
|
+
const stage: StatusQueryStage = idle
|
|
61
|
+
? 'idle'
|
|
62
|
+
: s.turnActive
|
|
63
|
+
? 'turn-active'
|
|
64
|
+
: 'background-work'
|
|
65
|
+
return { stage, idle }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Render the primary structured log line (grep anchor: "status-query").
|
|
70
|
+
* Kept here so the field layout is single-sourced with the derive logic and
|
|
71
|
+
* covered by the same test.
|
|
72
|
+
*/
|
|
73
|
+
export function formatStatusQueryLine(
|
|
74
|
+
agentName: string,
|
|
75
|
+
chatId: string,
|
|
76
|
+
thread: string,
|
|
77
|
+
s: StatusQuerySurfaces,
|
|
78
|
+
t: StatusQueryTelemetry,
|
|
79
|
+
): string {
|
|
80
|
+
return (
|
|
81
|
+
`telegram gateway: status-query agent=${agentName} chat_id=${chatId} thread=${thread} ` +
|
|
82
|
+
`stage=${t.stage} turn_age_s=${s.turnAgeS} running_workers=${s.runningWorkers} ` +
|
|
83
|
+
`worker_feed=${s.workerFeedSize} pending_async=${s.pendingAsync}\n`
|
|
84
|
+
)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Render the `ux-failure` line (grep anchor: "ux-failure: status-query"). Only
|
|
89
|
+
* call this when `t.idle` is true.
|
|
90
|
+
*/
|
|
91
|
+
export function formatStatusQueryUxFailureLine(
|
|
92
|
+
agentName: string,
|
|
93
|
+
chatId: string,
|
|
94
|
+
thread: string,
|
|
95
|
+
): string {
|
|
96
|
+
return (
|
|
97
|
+
`telegram gateway: ux-failure: status-query agent=${agentName} chat_id=${chatId} thread=${thread} ` +
|
|
98
|
+
`stage=idle turn_age_s=-1 running_workers=0 worker_feed=0 pending_async=0\n`
|
|
99
|
+
)
|
|
100
|
+
}
|
|
@@ -216,25 +216,24 @@ export interface StreamReplyDeps {
|
|
|
216
216
|
sameAsLast: boolean
|
|
217
217
|
}) => void
|
|
218
218
|
/**
|
|
219
|
-
* Optional:
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
* re-
|
|
219
|
+
* Optional: turn-complete hook. Historically wired by the gateway to
|
|
220
|
+
* `progressDriver.forceCompleteTurn(...)` so a `stream_reply(done=true)` on
|
|
221
|
+
* the default (unnamed) lane acted as an authoritative turn-complete signal
|
|
222
|
+
* equal to session-tail `turn_end`. The pinned progress card was retired
|
|
223
|
+
* (#1122/#1126) and `progressDriver` is permanently null, so the gateway now
|
|
224
|
+
* passes a no-op wrapper for this dep — the hook is inert until a future
|
|
225
|
+
* consumer re-attaches it. Skipped when args.lane is 'progress'. Safe to
|
|
226
|
+
* leave unset.
|
|
226
227
|
*/
|
|
227
228
|
forceCompleteTurn?: (chatId: string, threadId: number | undefined) => void
|
|
228
229
|
/**
|
|
229
|
-
* Optional:
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
*
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
* stream produced a non-null messageId. Safe to leave unset for callers
|
|
237
|
-
* that don't use the driver.
|
|
230
|
+
* Optional: outbound-delivery counter hook. Historically wired to
|
|
231
|
+
* `progressDriver.recordOutboundDelivered(...)`, called BEFORE
|
|
232
|
+
* `forceCompleteTurn` so the driver's per-turn outbound counter was non-zero
|
|
233
|
+
* when the terminal render fired (issue #310). With the progress card retired
|
|
234
|
+
* (#1122/#1126, null driver) the gateway passes a no-op wrapper — inert until
|
|
235
|
+
* re-attached. Only called on the default (unnamed) lane when `done=true` and
|
|
236
|
+
* the stream produced a non-null messageId. Safe to leave unset.
|
|
238
237
|
*/
|
|
239
238
|
recordOutboundDelivered?: (chatId: string, threadId: number | undefined) => void
|
|
240
239
|
/** Whether to persist outbound history. */
|
|
@@ -48,7 +48,7 @@ import { sanitiseToolArg } from './fleet-state.js'
|
|
|
48
48
|
import { clipNarrative, describeToolUse } from './tool-activity-summary.js'
|
|
49
49
|
import { REPLY_TOOLS, isDraftOfReply } from './narrative-dedup.js'
|
|
50
50
|
import { truncate } from './card-format.js'
|
|
51
|
-
import { bumpSubagentActivity, recordSubagentStall, recordSubagentResume, recordSubagentEnd, reapStuckRunningRows, countRunningBackgroundSubagents } from './registry/subagents-schema.js'
|
|
51
|
+
import { bumpSubagentActivity, recordSubagentStall, recordSubagentResume, recordSubagentEnd, reapStuckRunningRows, countRunningBackgroundSubagents, recordNestedSubagentDispatch } from './registry/subagents-schema.js'
|
|
52
52
|
import { touchTurnActiveMarker } from './gateway/turn-active-marker.js'
|
|
53
53
|
|
|
54
54
|
// ─── Types ───────────────────────────────────────────────────────────────────
|
|
@@ -183,6 +183,16 @@ export interface WorkerEntry {
|
|
|
183
183
|
* `turn.lastReplyText`.
|
|
184
184
|
*/
|
|
185
185
|
lastReplyText?: string
|
|
186
|
+
/**
|
|
187
|
+
* Wall-clock ms of the most recent RETRY of `backfillJsonlAgentId` from the
|
|
188
|
+
* liveness path (readSubTail's "row not in DB yet" branch). The one-shot
|
|
189
|
+
* backfill at registration loses the race for a NESTED worker: its row is
|
|
190
|
+
* written by recordNestedSubagentDispatch only when the watcher reads the
|
|
191
|
+
* PARENT worker's dispatch line, which can land after the child's own
|
|
192
|
+
* registration. Retrying (throttled) closes the "liveness skip … Phase 2
|
|
193
|
+
* Pre hook pending" window that froze nested cards on "starting…".
|
|
194
|
+
*/
|
|
195
|
+
lastBackfillAttemptAt?: number
|
|
186
196
|
}
|
|
187
197
|
|
|
188
198
|
export interface SubagentWatcherConfig {
|
|
@@ -293,37 +303,47 @@ export interface SubagentWatcherConfig {
|
|
|
293
303
|
/**
|
|
294
304
|
* Option C: callback fired when a stall is detected for a running sub-agent.
|
|
295
305
|
* Called with the sub-agent's agentId, idle ms, and description string.
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
*
|
|
306
|
+
* The `stallNotified` flag prevents duplicate calls for the same sub-agent
|
|
307
|
+
* across subsequent poll ticks.
|
|
308
|
+
*
|
|
309
|
+
* NOTE: this used to be wired in gateway.ts to `progressDriver.onSubAgentStall`
|
|
310
|
+
* so the pinned progress card could render a ⚠️ stall badge. That card was
|
|
311
|
+
* retired (#1122/#1126) and the gateway wiring was removed (the dead no-op
|
|
312
|
+
* falsely implied a stall renders a visual badge). The callback is retained
|
|
313
|
+
* as an unwired hook — PR 2 will re-wire it to repaint the live `🛠 Worker`
|
|
314
|
+
* activity feed on stall. Currently no gateway consumer is attached.
|
|
300
315
|
*/
|
|
301
316
|
onStall?: (agentId: string, idleMs: number, description: string) => void
|
|
302
317
|
/**
|
|
303
318
|
* Symmetric to `onStall`: fires when a previously-stalled sub-agent's
|
|
304
319
|
* JSONL grows again (text emission, tool use, turn_end — anything that
|
|
305
|
-
* moves last_activity_at).
|
|
306
|
-
* in gateway.ts so the pinned card clears the ⚠ Stalled badge as soon
|
|
307
|
-
* as activity resumes, instead of waiting on the next render tick.
|
|
320
|
+
* moves last_activity_at).
|
|
308
321
|
*
|
|
309
322
|
* Each stall→resume cycle fires exactly once: the watcher resets
|
|
310
323
|
* `entry.stallNotified` on resume, so a sub-agent that stalls again
|
|
311
324
|
* later in the same lifetime is detected (and reported) again.
|
|
325
|
+
*
|
|
326
|
+
* NOTE: formerly wired to `progressDriver.onSubAgentUnstall` (retired card,
|
|
327
|
+
* #1122/#1126) — the gateway wiring was removed with `onStall`. Retained as
|
|
328
|
+
* an unwired hook for PR 2's live-feed repaint.
|
|
312
329
|
*/
|
|
313
330
|
onUnstall?: (agentId: string, description: string) => void
|
|
314
331
|
/**
|
|
315
332
|
* RFC §Bug 6: fires when the watcher synthesises a terminal transition
|
|
316
333
|
* for a stalled sub-agent (no explicit `sub_agent_turn_end` line in
|
|
317
334
|
* the JSONL after `silentStallTerminalMs` past the stall notification).
|
|
318
|
-
* Wired in gateway.ts to push a synthetic
|
|
319
|
-
* `{kind:'sub_agent_turn_end', agentId}` event into the progress
|
|
320
|
-
* driver so the pinned card can release its deferred-completion gate
|
|
321
|
-
* for the background dispatch.
|
|
322
335
|
*
|
|
323
336
|
* Idempotent: each sub-agent triggers this at most once per lifetime
|
|
324
|
-
* (guarded by `entry.stallTerminalSynthesised`).
|
|
325
|
-
*
|
|
326
|
-
*
|
|
337
|
+
* (guarded by `entry.stallTerminalSynthesised`).
|
|
338
|
+
*
|
|
339
|
+
* NOTE: formerly wired in gateway.ts to push a synthetic
|
|
340
|
+
* `{kind:'sub_agent_turn_end', agentId}` into the progress driver so the
|
|
341
|
+
* pinned card could release its deferred-completion gate. The card is retired
|
|
342
|
+
* (#1122/#1126) and that dead wiring was removed. The completion path does
|
|
343
|
+
* NOT depend on this callback: the silent-stall synthesis loop in
|
|
344
|
+
* `checkStalls()` writes the terminal registry-DB row itself
|
|
345
|
+
* (`recordSubagentEnd`) and fires `maybySendStateTransition` → `onFinish`
|
|
346
|
+
* (the handback) regardless. Retained as an unwired hook.
|
|
327
347
|
*/
|
|
328
348
|
onStallTerminal?: (agentId: string, description: string) => void
|
|
329
349
|
/**
|
|
@@ -439,6 +459,28 @@ const DEFAULT_SILENT_SYNTHESIS_STALL_THRESHOLD_MS = 300_000
|
|
|
439
459
|
*/
|
|
440
460
|
const DEFAULT_SILENT_STALL_TERMINAL_MS = 300_000
|
|
441
461
|
|
|
462
|
+
/**
|
|
463
|
+
* Tools that legitimately run for minutes with ZERO intervening JSONL
|
|
464
|
+
* writes: the worker emits ONE `sub_agent_tool_use` line when the command
|
|
465
|
+
* starts, then nothing until it returns. `Bash` (a build, `npm test`, a long
|
|
466
|
+
* curl, a git clone) is the canonical case. A worker whose last observed tool
|
|
467
|
+
* is one of these is NOT stalled just because the JSONL went quiet — the
|
|
468
|
+
* 60s active-loop threshold misfires on exactly this population, producing the
|
|
469
|
+
* false "stall detected (idle 60s)" the operator hit while a background worker
|
|
470
|
+
* was mid-`Bash`. Pass-1 stall detection widens the idle threshold for these
|
|
471
|
+
* to the same silent-synthesis window used for not-yet-started workers, so a
|
|
472
|
+
* long command is given room before it counts as stalled. The pass-2 terminal
|
|
473
|
+
* synthesis is unchanged — it still releases the deferred-completion gate the
|
|
474
|
+
* fixed window after a stall IS eventually flagged.
|
|
475
|
+
*/
|
|
476
|
+
const LONG_RUNNING_TOOLS: ReadonlySet<string> = new Set(['Bash'])
|
|
477
|
+
|
|
478
|
+
/** True when the tool named legitimately runs quiet for minutes (see
|
|
479
|
+
* LONG_RUNNING_TOOLS). Null/undefined tool name → false. */
|
|
480
|
+
function isLongRunningTool(name: string | null | undefined): boolean {
|
|
481
|
+
return name != null && LONG_RUNNING_TOOLS.has(name)
|
|
482
|
+
}
|
|
483
|
+
|
|
442
484
|
/**
|
|
443
485
|
* Freshness window for the boot-scan "in-flight at boot → promote to
|
|
444
486
|
* live" path. A worker file still in `running` state at boot is only
|
|
@@ -505,6 +547,13 @@ const DEFAULT_REAPER_INTERVAL_MS = 15 * 60_000 // 15 minutes
|
|
|
505
547
|
*/
|
|
506
548
|
const TERMINAL_CLEANUP_GRACE_MS = 30_000
|
|
507
549
|
|
|
550
|
+
/**
|
|
551
|
+
* Throttle for the liveness-path retry of `backfillJsonlAgentId` (see
|
|
552
|
+
* WorkerEntry.lastBackfillAttemptAt). Cheap (one meta.json read + two
|
|
553
|
+
* indexed lookups) but no reason to run it on every 1s poll tick.
|
|
554
|
+
*/
|
|
555
|
+
const BACKFILL_RETRY_INTERVAL_MS = 3000
|
|
556
|
+
|
|
508
557
|
// ─── JSONL tail per sub-agent ─────────────────────────────────────────────
|
|
509
558
|
|
|
510
559
|
interface SubTail {
|
|
@@ -642,21 +691,52 @@ export function backfillJsonlAgentId(
|
|
|
642
691
|
// throws out of the watcher poll loop.
|
|
643
692
|
try {
|
|
644
693
|
const linkedRow = db
|
|
645
|
-
.prepare('SELECT started_at, parent_turn_key FROM subagents WHERE id = ?')
|
|
646
|
-
.get(candidateId) as { started_at: number; parent_turn_key: string | null } | null
|
|
694
|
+
.prepare('SELECT started_at, parent_turn_key, parent_agent_id FROM subagents WHERE id = ?')
|
|
695
|
+
.get(candidateId) as { started_at: number; parent_turn_key: string | null; parent_agent_id?: string | null } | null
|
|
647
696
|
if (linkedRow != null && linkedRow.parent_turn_key == null) {
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
if (
|
|
697
|
+
// Nested-parent inheritance FIRST: a depth-2+ worker's dispatching
|
|
698
|
+
// context is another sub-agent, not a gateway turn — its origin is its
|
|
699
|
+
// ancestor chain's turn key (stamped on the depth-1 row while the main
|
|
700
|
+
// turn was still active). The time-window match below can never be
|
|
701
|
+
// right for it (the turn had typically ended before the nested dispatch
|
|
702
|
+
// happened, and overlapping windows mis-attribute).
|
|
703
|
+
let resolvedKey: string | null = null
|
|
704
|
+
if (linkedRow.parent_agent_id != null && linkedRow.parent_agent_id.length > 0) {
|
|
705
|
+
const seen = new Set<string>([agentId])
|
|
706
|
+
let cursor: string | null = linkedRow.parent_agent_id
|
|
707
|
+
for (let hop = 0; hop < 5 && cursor != null && !seen.has(cursor); hop++) {
|
|
708
|
+
seen.add(cursor)
|
|
709
|
+
const parentRow = db
|
|
710
|
+
.prepare('SELECT parent_turn_key, parent_agent_id FROM subagents WHERE jsonl_agent_id = ? LIMIT 1')
|
|
711
|
+
.get(cursor) as { parent_turn_key: string | null; parent_agent_id?: string | null } | null
|
|
712
|
+
if (parentRow == null) break
|
|
713
|
+
if (parentRow.parent_turn_key != null && parentRow.parent_turn_key.length > 0) {
|
|
714
|
+
resolvedKey = parentRow.parent_turn_key
|
|
715
|
+
break
|
|
716
|
+
}
|
|
717
|
+
cursor = parentRow.parent_agent_id ?? null
|
|
718
|
+
}
|
|
719
|
+
if (resolvedKey != null) {
|
|
720
|
+
log?.(`subagent-watcher: backfill parent_turn_key ${candidateId} → ${resolvedKey} (inherited via nested-parent chain)`)
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
if (resolvedKey == null) {
|
|
724
|
+
const turn = db
|
|
725
|
+
.prepare(
|
|
726
|
+
`SELECT turn_key FROM turns
|
|
727
|
+
WHERE started_at <= ? AND (ended_at IS NULL OR ended_at >= ?)
|
|
728
|
+
ORDER BY started_at DESC LIMIT 1`,
|
|
729
|
+
)
|
|
730
|
+
.get(linkedRow.started_at, linkedRow.started_at) as { turn_key: string } | null
|
|
731
|
+
if (turn?.turn_key != null) {
|
|
732
|
+
resolvedKey = turn.turn_key
|
|
733
|
+
log?.(`subagent-watcher: backfill parent_turn_key ${candidateId} → ${resolvedKey}`)
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
if (resolvedKey != null) {
|
|
656
737
|
db
|
|
657
738
|
.prepare('UPDATE subagents SET parent_turn_key = ? WHERE id = ?')
|
|
658
|
-
.run(
|
|
659
|
-
log?.(`subagent-watcher: backfill parent_turn_key ${candidateId} → ${turn.turn_key}`)
|
|
739
|
+
.run(resolvedKey, candidateId)
|
|
660
740
|
}
|
|
661
741
|
}
|
|
662
742
|
} catch (err) {
|
|
@@ -751,6 +831,22 @@ export function readSubTail(
|
|
|
751
831
|
.get(entry.agentId) as { id: string; background: number } | null
|
|
752
832
|
if (existing == null) {
|
|
753
833
|
log?.(`subagent-watcher: liveness skip ${entry.agentId} — row not in DB yet (Phase 2 Pre hook pending)`)
|
|
834
|
+
// Retry the jsonl link (throttled). Registration's one-shot backfill
|
|
835
|
+
// loses the race for a NESTED worker whose row is only written when
|
|
836
|
+
// the watcher reads the PARENT's dispatch line — without this retry
|
|
837
|
+
// the row never links, the worker is misclassified foreground, and
|
|
838
|
+
// its card freezes on "starting…" forever (the depth-2+ freeze).
|
|
839
|
+
if (
|
|
840
|
+
entry.lastBackfillAttemptAt == null ||
|
|
841
|
+
now - entry.lastBackfillAttemptAt >= BACKFILL_RETRY_INTERVAL_MS
|
|
842
|
+
) {
|
|
843
|
+
entry.lastBackfillAttemptAt = now
|
|
844
|
+
try {
|
|
845
|
+
backfillJsonlAgentId(db, entry.filePath, entry.agentId, log)
|
|
846
|
+
} catch (bfErr) {
|
|
847
|
+
log?.(`subagent-watcher: backfill retry error ${entry.agentId}: ${(bfErr as Error).message}`)
|
|
848
|
+
}
|
|
849
|
+
}
|
|
754
850
|
} else {
|
|
755
851
|
bumpSubagentActivity(db, { id: existing.id, ts: now })
|
|
756
852
|
isForeground = existing.background === 0
|
|
@@ -947,6 +1043,36 @@ export function readSubTail(
|
|
|
947
1043
|
}
|
|
948
1044
|
}
|
|
949
1045
|
}
|
|
1046
|
+
} else if (ev.kind === 'sub_agent_nested_spawn') {
|
|
1047
|
+
// Nested (depth-2+) dispatch keying: this worker just dispatched a
|
|
1048
|
+
// sub-agent of its own. The PreToolUse hook can't attribute it (the
|
|
1049
|
+
// main turn's turn-active.json marker is long gone for a background
|
|
1050
|
+
// worker, and under concurrent nested dispatch the hook's write can
|
|
1051
|
+
// be lost entirely) — but WE are reading the authoritative dispatch
|
|
1052
|
+
// line right now. Record/repair the child's registry row: ensure it
|
|
1053
|
+
// exists (keyed on the dispatch tool_use_id, which the child's
|
|
1054
|
+
// meta.json `toolUseId` links against), stamp `parent_agent_id` =
|
|
1055
|
+
// this worker's jsonl stem, and inherit `parent_turn_key`
|
|
1056
|
+
// transitively so origin-chat routing works at any depth. Rendering
|
|
1057
|
+
// is unchanged (design §5.5 "no recursion in rendering") — this is
|
|
1058
|
+
// registry keying only. Fire-and-forget: a DB hiccup must never
|
|
1059
|
+
// break the tail loop.
|
|
1060
|
+
if (db != null && ev.toolUseId != null && ev.toolUseId.length > 0) {
|
|
1061
|
+
try {
|
|
1062
|
+
const nestedInput = (ev.input ?? {}) as Record<string, unknown>
|
|
1063
|
+
recordNestedSubagentDispatch(db, {
|
|
1064
|
+
toolUseId: ev.toolUseId,
|
|
1065
|
+
parentJsonlAgentId: entry.agentId,
|
|
1066
|
+
agentType: typeof nestedInput.subagent_type === 'string' ? nestedInput.subagent_type : null,
|
|
1067
|
+
description: typeof nestedInput.description === 'string' ? nestedInput.description : null,
|
|
1068
|
+
background: nestedInput.run_in_background === true,
|
|
1069
|
+
now,
|
|
1070
|
+
})
|
|
1071
|
+
log?.(`subagent-watcher: nested dispatch recorded parent=${entry.agentId} toolUseId=${ev.toolUseId}`)
|
|
1072
|
+
} catch (dbErr) {
|
|
1073
|
+
log?.(`subagent-watcher: nested dispatch record error ${entry.agentId}: ${(dbErr as Error).message}`)
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
950
1076
|
} else if (ev.kind === 'sub_agent_text') {
|
|
951
1077
|
// Do NOT overwrite description with narrative text — description is
|
|
952
1078
|
// set at dispatch time (from the parent Agent/Task tool_use input)
|
|
@@ -1260,6 +1386,23 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1260
1386
|
log?.(`subagent-watcher: backfill error for ${agentId}: ${(err as Error).message}`)
|
|
1261
1387
|
}
|
|
1262
1388
|
}
|
|
1389
|
+
// Historical/onProgress gate-ordering fix: the initial read above ran
|
|
1390
|
+
// while `entry.historical` was still TRUE, so every onProgress cue it
|
|
1391
|
+
// would have fired was suppressed by the `!entry.historical` guard —
|
|
1392
|
+
// and the tail cursor is now at EOF, so for a quiet worker no later
|
|
1393
|
+
// tick ever re-fires them. The card painted as a bare stub and froze
|
|
1394
|
+
// on "starting…" forever. Re-read from the start now that the entry
|
|
1395
|
+
// is live: the replayed events rebuild toolCount/lastTool/narrative
|
|
1396
|
+
// AND fire onProgress so the worker's card reaches real activity.
|
|
1397
|
+
entry.toolCount = 0
|
|
1398
|
+
entry.lastTool = null
|
|
1399
|
+
entry.pendingNarrative = null
|
|
1400
|
+
tail.cursor = 0
|
|
1401
|
+
tail.pendingPartial = ''
|
|
1402
|
+
tail.hasEmittedStart = false
|
|
1403
|
+
readSubTail(entry, tail, n, (desc) => {
|
|
1404
|
+
log?.(`subagent-watcher: description updated for ${agentId}: ${desc}`)
|
|
1405
|
+
}, fs, log, db, parentStateDir, config.onUnstall, undefined, config.onProgress)
|
|
1263
1406
|
}
|
|
1264
1407
|
}
|
|
1265
1408
|
|
|
@@ -1436,9 +1579,18 @@ export function startSubagentWatcher(config: SubagentWatcherConfig): SubagentWat
|
|
|
1436
1579
|
// tool_use. Once tools have started, switch to the tighter loop
|
|
1437
1580
|
// threshold — frequent JSONL writes mean 60s of silence is a
|
|
1438
1581
|
// strong signal the sub-agent is genuinely stuck.
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1582
|
+
// A worker that hasn't fired any tools yet is mid-silent-synthesis; a
|
|
1583
|
+
// worker whose LAST tool is a known long-runner (e.g. `Bash`) is mid-
|
|
1584
|
+
// command — both legitimately go quiet in the JSONL for minutes, so both
|
|
1585
|
+
// use the wider silent-synthesis window instead of the tight 60s active-
|
|
1586
|
+
// loop threshold. Without the long-runner arm a background worker running
|
|
1587
|
+
// a long `Bash`/`npm test` was falsely flagged "stall detected (idle
|
|
1588
|
+
// 60s)" while genuinely alive. Pass-2 terminal synthesis (below) is
|
|
1589
|
+
// unaffected: it still fires the fixed window after a stall IS flagged.
|
|
1590
|
+
const threshold =
|
|
1591
|
+
entry.toolCount === 0 || isLongRunningTool(entry.lastTool?.name)
|
|
1592
|
+
? silentSynthesisStallThresholdMs
|
|
1593
|
+
: stallThresholdMs
|
|
1442
1594
|
if (idleMs >= threshold) {
|
|
1443
1595
|
entry.stallNotified = true
|
|
1444
1596
|
entry.stalledAt = n
|
|
@@ -126,22 +126,16 @@ describe('buffer-gate release decoupled from final-answer classification', () =>
|
|
|
126
126
|
expect(releaseIdx).toBeGreaterThan(gateBlockClose)
|
|
127
127
|
})
|
|
128
128
|
|
|
129
|
-
it('
|
|
130
|
-
const post =
|
|
131
|
-
gatewaySrc.split('async function executeStreamReply')[1]
|
|
132
|
-
?.split('\nasync function ')[0] ?? ''
|
|
133
|
-
expect(post).toMatch(/releaseTurnBufferGate\(statusKey\(/)
|
|
134
|
-
})
|
|
135
|
-
|
|
136
|
-
it('the helper is invoked from executeReply / executeStreamReply only — not from new mid-turn paths', () => {
|
|
129
|
+
it('the helper is invoked from executeReply only — not from new mid-turn paths', () => {
|
|
137
130
|
// Sanity: nothing else should call releaseTurnBufferGate. The
|
|
138
131
|
// helper is narrow on purpose. If future code adds new
|
|
139
132
|
// callsites that aren't reply-finalize, the steer-vs-queue
|
|
140
133
|
// semantics could drift.
|
|
141
134
|
const callMatches = gatewaySrc.match(/releaseTurnBufferGate\(/g) ?? []
|
|
142
|
-
// Definition +
|
|
135
|
+
// Definition + 1 callsite (executeReply) = 2. (The retired
|
|
136
|
+
// stream_reply tool's callsite was removed with executeStreamReply.)
|
|
143
137
|
// If this count grows the test catches it; reviewer must justify
|
|
144
138
|
// any new callsite.
|
|
145
|
-
expect(callMatches.length).toBe(
|
|
139
|
+
expect(callMatches.length).toBe(2)
|
|
146
140
|
})
|
|
147
141
|
})
|