switchroom 0.19.4 → 0.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-broker/index.js +7 -3
- package/dist/cli/autoaccept-poll.js +8 -2
- package/dist/cli/switchroom.js +20 -5
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +67 -4
- package/telegram-plugin/dist/gateway/gateway.js +524 -293
- package/telegram-plugin/gateway/command-format.ts +253 -0
- package/telegram-plugin/gateway/gateway-heartbeat.ts +72 -0
- package/telegram-plugin/gateway/gateway.ts +97 -255
- package/telegram-plugin/gateway/hang-restart-decision.ts +189 -0
- package/telegram-plugin/gateway/liveness-wiring.ts +35 -1
- package/telegram-plugin/gateway/session-model-file.ts +13 -0
- package/telegram-plugin/gateway/stream-render.ts +18 -1
- package/telegram-plugin/gateway/turn-active-marker.ts +29 -17
- package/telegram-plugin/gateway/worker-feed-dispatch.ts +139 -0
- package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +87 -36
- package/telegram-plugin/hooks/silent-end-scan.mjs +263 -3
- package/telegram-plugin/render/line-start-guard.ts +76 -4
- package/telegram-plugin/rich-send.ts +8 -1
- package/telegram-plugin/tests/command-format.test.ts +212 -0
- package/telegram-plugin/tests/gateway-heartbeat.test.ts +70 -0
- package/telegram-plugin/tests/hang-restart-decision.test.ts +146 -0
- package/telegram-plugin/tests/hang-restart-marker-integration.test.ts +98 -0
- package/telegram-plugin/tests/render/heading-guard-blockquote-glued-hash.test.ts +86 -0
- package/telegram-plugin/tests/render/heading-guard.test.ts +114 -0
- package/telegram-plugin/tests/render/rich-corpus-seam-regression.test.ts +76 -0
- package/telegram-plugin/tests/silent-end-interrupt-stop-integration.test.ts +63 -0
- package/telegram-plugin/tests/silent-end-interrupt-stop-scan.test.ts +60 -16
- package/telegram-plugin/tests/silent-end-single-writer-election.test.ts +193 -0
- package/telegram-plugin/tests/silent-end.test.ts +60 -5
- package/telegram-plugin/tests/worker-feed-origin-race-defer.test.ts +321 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hang-restart-decision.ts — the progress-based hang-restart discriminator
|
|
3
|
+
* (Stage B safety net).
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS EXISTS
|
|
6
|
+
*
|
|
7
|
+
* The silence-poke framework fallback (`liveness-wiring.ts onFrameworkFallback`)
|
|
8
|
+
* is the last-resort unwedge, but it only tears down GATEWAY state — it never
|
|
9
|
+
* kills the wedged `claude` child. So a turn that hangs mid-tool-call recovers
|
|
10
|
+
* its conversation slot but leaves the child stuck; the "carrie" incident
|
|
11
|
+
* needed a MANUAL restart. Stage B closes that: when the fallback fires with a
|
|
12
|
+
* tool still mid-call AND no real progress, escalate to an actual restart via
|
|
13
|
+
* the SIGTERM-PID1 path, then let the boot classifier route recovery through
|
|
14
|
+
* the ask-first `resume_watchdog_timeout` inbound (never assertive auto-resume).
|
|
15
|
+
*
|
|
16
|
+
* THE DISCRIMINATOR (the crux — do NOT key on "tool open N seconds")
|
|
17
|
+
*
|
|
18
|
+
* A healthy 20-minute research turn must NOT be killed. The honest progress
|
|
19
|
+
* signal is the turn-active marker's mtime: it is touched on every `tool_use`
|
|
20
|
+
* AND on foreground sub-agent JSONL growth (`subagent-watcher.ts`), so a turn
|
|
21
|
+
* that keeps advancing it is working; a stale mtime means work stopped.
|
|
22
|
+
*
|
|
23
|
+
* markerAgeMs (from readTurnActiveMarkerAgeMs) small → progress → NOT a hang
|
|
24
|
+
* markerAgeMs large, or null (no progress signal at all) → stale → hang
|
|
25
|
+
*
|
|
26
|
+
* THE MARKER-ADVANCEMENT BLIND SPOT + THE TOOL-CLASS GUARD (review Finding A)
|
|
27
|
+
*
|
|
28
|
+
* The marker advances only on `tool_use` and sub-agent JSONL growth. A healthy
|
|
29
|
+
* long SINGLE FOREGROUND tool with no sub-agent and no interim tool_use never
|
|
30
|
+
* advances it — a 15-min foreground `Bash` build/test, a big `WebFetch`/crawl,
|
|
31
|
+
* a multi-minute research call. At the 15-min silence ceiling that turn is
|
|
32
|
+
* mid-tool with a stale marker and would be WRONGLY restarted by marker
|
|
33
|
+
* staleness alone. So we additionally protect known-long-running tool classes:
|
|
34
|
+
* if any in-flight tool is a `background`/long-fetch/research class, we do NOT
|
|
35
|
+
* restart, regardless of marker age.
|
|
36
|
+
*
|
|
37
|
+
* RESIDUAL RISK (documented, not silently accepted): a genuine hang INSIDE a
|
|
38
|
+
* protected long-running tool (a truly-wedged `Bash`, a `WebFetch` that never
|
|
39
|
+
* returns) is indistinguishable from healthy long work by any signal the
|
|
40
|
+
* gateway has — the marker cannot advance mid-single-tool either way. Those
|
|
41
|
+
* are NOT auto-restarted here; recovery for that class relies on the tool's
|
|
42
|
+
* own timeout, an operator `/restart`, or the ordinary state-teardown. We
|
|
43
|
+
* choose to never false-kill a healthy long tool over catching the rare
|
|
44
|
+
* hung-inside-a-long-tool case, because the false-kill is the common,
|
|
45
|
+
* user-visible harm and the discriminator exists precisely to avoid it.
|
|
46
|
+
*
|
|
47
|
+
* Recovery routes through the ask-first resume path so a turn that would hang
|
|
48
|
+
* again the same way is reported to the user, not silently re-run — that is
|
|
49
|
+
* also the storm guard (see § STORM GUARD below).
|
|
50
|
+
*
|
|
51
|
+
* Pure + deterministic: the clock, the marker age, and the in-flight tool names
|
|
52
|
+
* are all injected. The gateway owns the SIGTERM side effect.
|
|
53
|
+
*/
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Tool classes ported from Stage A's tool-call-deadline taxonomy. `background`
|
|
57
|
+
* covers sub-agent dispatch + long Bash; `human` covers approval waits;
|
|
58
|
+
* `standard` is everything else. Stage B additionally treats a broader set of
|
|
59
|
+
* single-foreground long tools as hang-restart-protected (see
|
|
60
|
+
* `isHangRestartProtectedTool`).
|
|
61
|
+
*/
|
|
62
|
+
export type ToolDeadlineClass = 'standard' | 'human' | 'background'
|
|
63
|
+
|
|
64
|
+
/** Human-in-the-loop waits. */
|
|
65
|
+
const HUMAN_WAIT_TOOLS = new Set<string>(['ask_user'])
|
|
66
|
+
/** Sub-agent dispatch — progress is sub-agent JSONL growth. */
|
|
67
|
+
const BACKGROUND_TOOLS = new Set<string>(['Task', 'Agent'])
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Classify a tool into the Stage A taxonomy. `Bash` with `run_in_background` is
|
|
71
|
+
* `background`; `Task`/`Agent` are `background`; `ask_user` is `human`.
|
|
72
|
+
*/
|
|
73
|
+
export function classifyToolClass(
|
|
74
|
+
toolName: string,
|
|
75
|
+
opts: { awaitingApproval?: boolean; backgroundBash?: boolean } = {},
|
|
76
|
+
): ToolDeadlineClass {
|
|
77
|
+
if (opts.awaitingApproval === true) return 'human'
|
|
78
|
+
if (HUMAN_WAIT_TOOLS.has(toolName)) return 'human'
|
|
79
|
+
if (BACKGROUND_TOOLS.has(toolName)) return 'background'
|
|
80
|
+
if (toolName === 'Bash' && opts.backgroundBash === true) return 'background'
|
|
81
|
+
return 'standard'
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Exact-name tools that legitimately run long as a SINGLE foreground call
|
|
86
|
+
* without touching the marker, so their staleness is not a hang signal.
|
|
87
|
+
*/
|
|
88
|
+
const HANG_PROTECTED_EXACT = new Set<string>([
|
|
89
|
+
'Task', 'Agent', // sub-agent dispatch (background class)
|
|
90
|
+
'Bash', // foreground builds/tests routinely run many minutes
|
|
91
|
+
'WebFetch', 'WebSearch',
|
|
92
|
+
])
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Name substrings marking long-running fetch/crawl/research MCP tools (matched
|
|
96
|
+
* case-insensitively) — perplexity_research, deep research, site crawls, etc.
|
|
97
|
+
*/
|
|
98
|
+
const HANG_PROTECTED_SUBSTRINGS = ['research', 'perplexity', 'crawl', 'deep_research', 'webkite']
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* True when a tool is a known-long-running / opaque-progress class that must
|
|
102
|
+
* NOT be hang-restarted on marker staleness (review Finding A). Covers the
|
|
103
|
+
* `background` class plus foreground Bash, web fetch/search, and research/crawl
|
|
104
|
+
* MCP tools. `ask_user` (`human`) is also protected — an operator wait is not a
|
|
105
|
+
* hang. See the module header § RESIDUAL RISK for the tradeoff.
|
|
106
|
+
*/
|
|
107
|
+
export function isHangRestartProtectedTool(toolName: string): boolean {
|
|
108
|
+
if (HANG_PROTECTED_EXACT.has(toolName)) return true
|
|
109
|
+
if (classifyToolClass(toolName) !== 'standard') return true // background / human
|
|
110
|
+
const lower = toolName.toLowerCase()
|
|
111
|
+
return HANG_PROTECTED_SUBSTRINGS.some((s) => lower.includes(s))
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export interface HangRestartInput {
|
|
115
|
+
/** Names of the tools in flight when the fallback fired (silence-poke
|
|
116
|
+
* snapshot). Empty ⇒ not a mid-tool fallback (ordinary teardown owns it). */
|
|
117
|
+
inFlightToolNames: string[]
|
|
118
|
+
/** Turn-active marker mtime age in ms (`readTurnActiveMarkerAgeMs`), or null
|
|
119
|
+
* when the marker is absent/unstattable (no progress signal). */
|
|
120
|
+
markerAgeMs: number | null
|
|
121
|
+
/** Staleness ceiling: marker age at/above this counts as "no progress". Keyed
|
|
122
|
+
* to the same TURN_HANG_SECS the boot classifier uses so the live decision
|
|
123
|
+
* and the boot reclassification agree on what "stalled" means. */
|
|
124
|
+
stalenessThresholdMs: number
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export interface HangRestartDecision {
|
|
128
|
+
restart: boolean
|
|
129
|
+
reason: string
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Decide whether a mid-tool framework-fallback should escalate to a real
|
|
134
|
+
* restart. Pure.
|
|
135
|
+
*
|
|
136
|
+
* - not mid-tool → no restart (ordinary teardown path owns it)
|
|
137
|
+
* - a protected long tool in flight → no restart (healthy long tool / sub-
|
|
138
|
+
* agent / research — the crux false-positive we
|
|
139
|
+
* must never kill; see § RESIDUAL RISK)
|
|
140
|
+
* - marker still advancing → no restart (real progress)
|
|
141
|
+
* - marker stale or absent → RESTART (genuine hang: mid-tool + no progress
|
|
142
|
+
* + no protected long tool that would explain it)
|
|
143
|
+
*/
|
|
144
|
+
export function decideHangRestart(input: HangRestartInput): HangRestartDecision {
|
|
145
|
+
if (input.inFlightToolNames.length === 0) return { restart: false, reason: 'not-mid-tool' }
|
|
146
|
+
// Finding A: never restart while a known-long-running tool is in flight — a
|
|
147
|
+
// stale marker is EXPECTED for a single long foreground tool.
|
|
148
|
+
const protectedName = input.inFlightToolNames.find(isHangRestartProtectedTool)
|
|
149
|
+
if (protectedName != null) {
|
|
150
|
+
return { restart: false, reason: `protected-long-tool:${protectedName}` }
|
|
151
|
+
}
|
|
152
|
+
// A working turn keeps touching the marker (tool_use + sub-agent JSONL
|
|
153
|
+
// growth), so a SMALL non-null age means real progress.
|
|
154
|
+
if (input.markerAgeMs != null && input.markerAgeMs < input.stalenessThresholdMs) {
|
|
155
|
+
return { restart: false, reason: 'marker-advancing' }
|
|
156
|
+
}
|
|
157
|
+
return { restart: true, reason: 'mid-tool-marker-stale' }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Default staleness ceiling (ms) — mirrors the watchdog's TURN_HANG_SECS=300. */
|
|
161
|
+
export const DEFAULT_HANG_STALENESS_MS = 300_000
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Resolve the staleness ceiling from the environment, keyed to the SAME
|
|
165
|
+
* `TURN_HANG_SECS` the boot classifier reads, so a live kill and the boot
|
|
166
|
+
* reclassification agree on "stalled". Blank/garbage/non-positive → default.
|
|
167
|
+
*/
|
|
168
|
+
export function hangStalenessMs(
|
|
169
|
+
env: Record<string, string | undefined> = process.env,
|
|
170
|
+
): number {
|
|
171
|
+
const raw = env.TURN_HANG_SECS
|
|
172
|
+
const n = Number(raw)
|
|
173
|
+
if (!Number.isFinite(n) || n <= 0) return DEFAULT_HANG_STALENESS_MS
|
|
174
|
+
return Math.floor(n * 1000)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* § STORM GUARD (review Finding B)
|
|
179
|
+
*
|
|
180
|
+
* There is deliberately NO restart cooldown here. A cooldown could only engage
|
|
181
|
+
* if two hang-restarts fired close together, but two mid-tool framework
|
|
182
|
+
* fallbacks are >= the 15-min silence-defer ceiling apart (they only fire after
|
|
183
|
+
* that ceiling), so a sub-15-min cooldown can never engage on the organic path
|
|
184
|
+
* — a dead guard giving false confidence. The REAL storm guard is the ask-first
|
|
185
|
+
* `resume_watchdog_timeout` recovery: after a hang-restart the agent does NOT
|
|
186
|
+
* auto-resume the wedged work — it reports to the user and asks — so the same
|
|
187
|
+
* hang cannot immediately recur without fresh human action. That is sufficient
|
|
188
|
+
* and self-documenting; a persisted cooldown file is not needed.
|
|
189
|
+
*/
|
|
@@ -24,7 +24,8 @@ import { emitRuntimeMetric } from '../runtime-metrics.js'
|
|
|
24
24
|
import { logStreamingEvent } from '../streaming-metrics.js'
|
|
25
25
|
import { clearSilentEndState } from '../silent-end.js'
|
|
26
26
|
import { purgeStaleTurnsForChat } from './turn-state-purge.js'
|
|
27
|
-
import { removeTurnActiveMarker } from './turn-active-marker.js'
|
|
27
|
+
import { removeTurnActiveMarker, readTurnActiveMarkerAgeMs } from './turn-active-marker.js'
|
|
28
|
+
import { decideHangRestart } from './hang-restart-decision.js'
|
|
28
29
|
import { recordTurnEnd } from '../registry/turns-schema.js'
|
|
29
30
|
import { hostdGetStatusOnce } from './hostd-dispatch.js'
|
|
30
31
|
import { formatUpdateStatusLine } from './update-status-line.js'
|
|
@@ -60,6 +61,7 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
|
|
|
60
61
|
trackRedeliveredInbound,
|
|
61
62
|
closeActivityLane,
|
|
62
63
|
closeProgressLane,
|
|
64
|
+
hangRestart,
|
|
63
65
|
} = deps
|
|
64
66
|
|
|
65
67
|
return {
|
|
@@ -160,6 +162,38 @@ export function buildSilencePokeOptions(deps: LivenessWiringDeps): Parameters<ty
|
|
|
160
162
|
return
|
|
161
163
|
}
|
|
162
164
|
|
|
165
|
+
// Stage B: escalate a mid-tool hang to a REAL restart. The fallback fires
|
|
166
|
+
// here with a tool still in flight only after silence crossed the 15-min
|
|
167
|
+
// defer ceiling (isLegitimatelyWorking held it that long). The honest
|
|
168
|
+
// hang/health discriminator is the turn-active marker's mtime age: a healthy
|
|
169
|
+
// turn keeps advancing it (tool_use + sub-agent JSONL growth), a true hang
|
|
170
|
+
// lets it go stale. A known-long-running tool in flight (Task/Agent/Bash/
|
|
171
|
+
// WebFetch/research) is protected regardless of marker age — a single long
|
|
172
|
+
// foreground tool legitimately can't advance the marker (Finding A). Only a
|
|
173
|
+
// stale (or absent) marker with no protected long tool triggers the
|
|
174
|
+
// SIGTERM-PID1 restart; the boot classifier then routes recovery through the
|
|
175
|
+
// ask-first resume_watchdog_timeout inbound. We return BEFORE the state-
|
|
176
|
+
// teardown so the turn-active marker survives for the boot reclassification.
|
|
177
|
+
if (hangRestart != null && ctx.inFlightTools.length > 0) {
|
|
178
|
+
const markerAgeMs = readTurnActiveMarkerAgeMs(STATE_DIR)
|
|
179
|
+
const inFlightToolNames = ctx.inFlightTools.map((t) => t.name)
|
|
180
|
+
const decision = decideHangRestart({
|
|
181
|
+
inFlightToolNames,
|
|
182
|
+
markerAgeMs,
|
|
183
|
+
stalenessThresholdMs: hangRestart.stalenessThresholdMs,
|
|
184
|
+
})
|
|
185
|
+
process.stderr.write(
|
|
186
|
+
`telegram gateway: [hang-watchdog] fallback mid-tool chat=${ctx.chatId} ` +
|
|
187
|
+
`thread=${ctx.threadId ?? '-'} silence_ms=${ctx.silenceMs} ` +
|
|
188
|
+
`tools=${inFlightToolNames.join(',')} marker_age_ms=${markerAgeMs ?? 'null'} ` +
|
|
189
|
+
`restart=${decision.restart} reason=${decision.reason}\n`,
|
|
190
|
+
)
|
|
191
|
+
if (decision.restart) {
|
|
192
|
+
hangRestart.request(decision.reason, markerAgeMs ?? ctx.silenceMs)
|
|
193
|
+
return
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
163
197
|
// Deterministic in-flight update status (klanker incident). If this
|
|
164
198
|
// gateway dispatched an update_apply that's still running, the
|
|
165
199
|
// recurring framework fallback carries hostd's REAL phase + elapsed
|
|
@@ -104,6 +104,19 @@ export function writeSessionModelFile(
|
|
|
104
104
|
if (!isValidModelArg(model)) {
|
|
105
105
|
throw new Error(`refusing to persist non-canonical session model token: ${JSON.stringify(model)}`)
|
|
106
106
|
}
|
|
107
|
+
// A FRESH carrier gets a FRESH retry budget: clear any bounded-retry counter
|
|
108
|
+
// left over from a prior carrier's boots. Without this, an orphan counter
|
|
109
|
+
// survives the whole session on a healthy snapshot-apply boot (the apply
|
|
110
|
+
// branch re-writes the live counter AFTER the gateway's healthy-boot consume
|
|
111
|
+
// already deleted it), gets snapshotted alongside the NEW carrier at the next
|
|
112
|
+
// boot, and burns the new override's retry budget from a stale count — three
|
|
113
|
+
// such sessions in a row and a perfectly healthy /model is refused with a
|
|
114
|
+
// false "did not reach a healthy boot after 3 attempts" revert.
|
|
115
|
+
try {
|
|
116
|
+
rmSync(join(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true })
|
|
117
|
+
} catch {
|
|
118
|
+
/* best-effort — worst case is a stale count, bounded and non-fatal */
|
|
119
|
+
}
|
|
107
120
|
atomicWrite(
|
|
108
121
|
join(agentDir, SESSION_MODEL_FILE),
|
|
109
122
|
serializeSessionModel({ model, configuredDefaultAtWrite, ts: Date.now() }),
|
|
@@ -2018,6 +2018,23 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
|
|
|
2018
2018
|
// represent. The delivery closes the obligation + records dedup, so
|
|
2019
2019
|
// the represent and a late reply-tool retry are both suppressed —
|
|
2020
2020
|
// captured-prose delivery and represent are mutually exclusive.
|
|
2021
|
+
// Capture-divergence corner (duplicate-message fix): when this is a
|
|
2022
|
+
// ZERO-reply silent-end AND the gateway's OWN capture came up empty
|
|
2023
|
+
// (so `decideTurnFlush` had nothing to flush — its 'empty-text'
|
|
2024
|
+
// skip), the captured-prose bridge is the ONLY delivery machine
|
|
2025
|
+
// left. The Stop hook scanned the transcript and DID find the
|
|
2026
|
+
// model's short trailing answer, persisting it as `pendingText`. In
|
|
2027
|
+
// that corner the 200-char substance floor would wrongly drop a
|
|
2028
|
+
// legitimately-short answer that no other machine can deliver, so
|
|
2029
|
+
// lower the floor to 1. This is scoped tightly: only when the model
|
|
2030
|
+
// never called reply (`replyCalled === false`) AND the gateway
|
|
2031
|
+
// captured nothing — the interim-ack case (replyCalled) keeps the
|
|
2032
|
+
// full 200 floor (a short closer after a real reply is not a dropped
|
|
2033
|
+
// answer, and the hook already refuses to persist <200 there).
|
|
2034
|
+
const gatewayCapturedEmpty =
|
|
2035
|
+
turn.capturedText.join('\n\n').trim().length === 0
|
|
2036
|
+
const proseMinChars =
|
|
2037
|
+
!turn.replyCalled && gatewayCapturedEmpty ? 1 : CAPTURED_PROSE_MIN_CHARS
|
|
2021
2038
|
const proseDecision = CAPTURED_PROSE_DELIVERY_ENABLED
|
|
2022
2039
|
? decideCapturedProseDelivery({
|
|
2023
2040
|
turnKey: tKey,
|
|
@@ -2025,7 +2042,7 @@ export function handleSessionEvent(deps: StreamRenderDeps, ev: SessionEvent): vo
|
|
|
2025
2042
|
// belong to THIS turn, not a stale carryover from a prior turn
|
|
2026
2043
|
// on the same chat/thread (tKey is not per-turn unique).
|
|
2027
2044
|
turnId: turn.turnId,
|
|
2028
|
-
minChars:
|
|
2045
|
+
minChars: proseMinChars,
|
|
2029
2046
|
})
|
|
2030
2047
|
: { deliver: false as const, reason: 'no-state' as const }
|
|
2031
2048
|
if (proseDecision.deliver && proseDecision.text != null) {
|
|
@@ -1,26 +1,38 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Turn-active liveness marker (#412).
|
|
3
3
|
*
|
|
4
|
-
* Writes `<STATE_DIR>/turn-active.json` on turn_start, touches its mtime
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* Writes `<STATE_DIR>/turn-active.json` on turn_start, touches its mtime on
|
|
5
|
+
* every tool_use AND (via the subagent-watcher, #501) on foreground sub-agent
|
|
6
|
+
* JSONL growth, removes it on turn_complete. The mtime is therefore "ms since
|
|
7
|
+
* last observable progress": it advances while a turn — even a long one running
|
|
8
|
+
* one big tool or a sub-agent — is genuinely working, and goes stale only when
|
|
9
|
+
* work stops (a wedge).
|
|
9
10
|
*
|
|
10
|
-
* Why this exists: PR #410 raised the journal-silence detector to 4000s
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
11
|
+
* Why this exists: PR #410 raised the journal-silence detector to 4000s to kill
|
|
12
|
+
* false positives on chat-cadence agents that legitimately idle for hours
|
|
13
|
+
* between turns. That left a gap — Stop-hook deadlocks (the original failure
|
|
14
|
+
* mode #116 tracked) are no longer caught under default thresholds. The
|
|
15
|
+
* distinguisher is "in-turn-and-silent" vs "between-turns-and-silent": the
|
|
16
|
+
* former is a wedge, the latter is healthy idle. This marker exists exactly
|
|
17
|
+
* during in-turn windows, so its staleness uniquely indicates the wedge.
|
|
15
18
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* the wedge.
|
|
19
|
+
* WHO CONSUMES THE STALENESS (contract corrected — the historical
|
|
20
|
+
* `bin/bridge-watchdog.sh` bash watchdog this header once named was never
|
|
21
|
+
* committed to the repo; do not reintroduce a reference to it):
|
|
20
22
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
23
|
+
* - The gateway BOOT classifier (`markOrphanedWithTimeoutClassification` in
|
|
24
|
+
* gateway.ts): on restart, a marker whose mtime is older than TURN_HANG_SECS
|
|
25
|
+
* (default 300s) reclassifies the orphaned turn as `timeout`, routing the
|
|
26
|
+
* next session to the ask-first `resume_watchdog_timeout` inbound.
|
|
27
|
+
* - The LIVE Stage B hang-restart (`hang-restart-decision.ts`, consulted from
|
|
28
|
+
* the silence-poke framework fallback): a mid-tool fallback with a stale
|
|
29
|
+
* marker escalates to a real SIGTERM-PID1 restart. `readTurnActiveMarkerAgeMs`
|
|
30
|
+
* below is the shared read.
|
|
31
|
+
* - The obligation / phantom-turn sweeps (`readTurnActiveMarkerAgeMs`,
|
|
32
|
+
* `effectiveTurnAgeMs`): a small age means work is still in flight, so a
|
|
33
|
+
* "did I miss this?" re-send is suppressed.
|
|
34
|
+
*
|
|
35
|
+
* Pure file I/O; the mtime is the honest cross-process progress signal.
|
|
24
36
|
*/
|
|
25
37
|
|
|
26
38
|
import {
|
|
@@ -40,6 +40,145 @@ export function handleWorkerResume(
|
|
|
40
40
|
log(`telegram gateway: worker ${agentId} card RE-SURFACED — resumed via SendMessage after a genuine terminal (issue #3373)`)
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
/**
|
|
44
|
+
* Worker-feed origin-race defer decision (issue: DM-misrouted worker card).
|
|
45
|
+
*
|
|
46
|
+
* The gateway picks a worker card's destination on the FIRST progress tick of
|
|
47
|
+
* a new sub-agent. That tick can beat the async `jsonl_agent_id` backfill that
|
|
48
|
+
* links the sub-agent's registry row to its origin turn (retried ~every 3s).
|
|
49
|
+
* Until the link lands, origin resolution returns null and the gateway's
|
|
50
|
+
* `resolveWorkerFeedChat` hard-falls back to the owner DM — CREATING the card
|
|
51
|
+
* there. The origin (supergroup + forum topic) resolves ~3s later, but the DM
|
|
52
|
+
* card already exists and stays the visible one.
|
|
53
|
+
*
|
|
54
|
+
* Decision: DEFER card creation while the agent is not yet linked to its
|
|
55
|
+
* origin AND no card exists yet for it. The subagent-watcher re-fires within
|
|
56
|
+
* seconds; once the backfill completes the origin resolves and the card is
|
|
57
|
+
* created in the correct chat+topic. Only CARD CREATION is deferred — if a
|
|
58
|
+
* card already exists, updates always proceed (`defer:false`). A bounded
|
|
59
|
+
* counter caps the wait: after `maxDeferrals` unlinked ticks (pathological
|
|
60
|
+
* backfill failure) it stops deferring so active work always gets a card.
|
|
61
|
+
*
|
|
62
|
+
* Pure so the seam is unit-testable — see worker-feed-dispatch.test.ts. The
|
|
63
|
+
* gateway must never inline this decision again.
|
|
64
|
+
*/
|
|
65
|
+
export function decideWorkerFeedOriginDefer(input: {
|
|
66
|
+
/** True once `resolveSubagentOriginChat` returns a chat (row linked). */
|
|
67
|
+
originResolved: boolean
|
|
68
|
+
/** True if the feed already has a posted message for this worker. */
|
|
69
|
+
cardExists: boolean
|
|
70
|
+
/** Consecutive prior deferrals for this agent (0 on the first tick). */
|
|
71
|
+
priorDeferrals: number
|
|
72
|
+
/** Max consecutive deferrals before painting anyway. */
|
|
73
|
+
maxDeferrals: number
|
|
74
|
+
}): { defer: boolean; deferrals: number } {
|
|
75
|
+
const { originResolved, cardExists, priorDeferrals, maxDeferrals } = input
|
|
76
|
+
// A card already exists, or the origin has resolved: never defer.
|
|
77
|
+
if (originResolved || cardExists) return { defer: false, deferrals: 0 }
|
|
78
|
+
const deferrals = priorDeferrals + 1
|
|
79
|
+
// Bounded: stop deferring once we've waited long enough for the backfill.
|
|
80
|
+
if (deferrals >= maxDeferrals) return { defer: false, deferrals }
|
|
81
|
+
return { defer: true, deferrals }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The FULL worker-feed destination decision for a single onProgress tick,
|
|
86
|
+
* extracted verbatim from the gateway's inline `onProgress` block (issue
|
|
87
|
+
* #3460). It folds two concerns that the gateway used to run inline:
|
|
88
|
+
*
|
|
89
|
+
* 1. the origin-race defer choice (`decideWorkerFeedOriginDefer`), and
|
|
90
|
+
* 2. the chat/thread RESOLUTION the gateway's `resolveWorkerFeedChat`
|
|
91
|
+
* performed (origin chat → fleet chat / stamp-turn fallback → owner DM),
|
|
92
|
+
* including the exhausted-defer stamp-turn forum-thread carry (#3458).
|
|
93
|
+
*
|
|
94
|
+
* Returning a plain decision object lets the gateway keep only a thin
|
|
95
|
+
* delegation (defer-map bookkeeping + the two audit logs + the feed.update)
|
|
96
|
+
* and gives this whole path REAL regression coverage — the test drives THIS
|
|
97
|
+
* function, the same code the gateway runs, instead of a hand-rolled replica.
|
|
98
|
+
*
|
|
99
|
+
* Pure and side-effect-free: the two audit-log side effects the gateway used
|
|
100
|
+
* to emit inline (`exhausted`, `ownerDmFallback`) are returned as flags so the
|
|
101
|
+
* caller performs them against its module-level state. Behavior — routing — is
|
|
102
|
+
* identical to the prior inline path; only the seam moved.
|
|
103
|
+
*
|
|
104
|
+
* Precedence for a PAINT (mirrors `resolveWorkerFeedChat`):
|
|
105
|
+
* origin chat (when resolved, non-empty) → fleet chat, else stamp-turn chat
|
|
106
|
+
* when no fleet chat is configured (carrying the stamp-turn forum topic) →
|
|
107
|
+
* owner DM (durable floor). Never returns an empty chat for a paint unless
|
|
108
|
+
* every source is empty.
|
|
109
|
+
*/
|
|
110
|
+
export type WorkerFeedDestination =
|
|
111
|
+
| { action: 'defer'; deferrals: number }
|
|
112
|
+
| {
|
|
113
|
+
action: 'paint'
|
|
114
|
+
chatId: string
|
|
115
|
+
threadId: number | undefined
|
|
116
|
+
/** New consecutive-deferral count to persist (0 once painting resumes). */
|
|
117
|
+
deferrals: number
|
|
118
|
+
/** True when painting only because the bounded defer cap was hit and the
|
|
119
|
+
* origin never linked — the gateway logs the "never linked" audit line. */
|
|
120
|
+
exhausted: boolean
|
|
121
|
+
/** True when the paint fell all the way to the owner DM (origin unresolved
|
|
122
|
+
* AND no fleet/stamp chat) — the gateway logs the once-per-agent misroute. */
|
|
123
|
+
ownerDmFallback: boolean
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function decideWorkerFeedDestination(input: {
|
|
127
|
+
/** Resolved origin chat/topic (`resolveSubagentOriginChat`), or null if the
|
|
128
|
+
* `jsonl_agent_id` backfill hasn't linked the row to its origin turn yet. */
|
|
129
|
+
origin: { chatId: string; threadId?: number } | null
|
|
130
|
+
/** True if the feed already has a posted message for this worker. */
|
|
131
|
+
cardExists: boolean
|
|
132
|
+
/** Consecutive prior deferrals for this agent (0 on the first tick). */
|
|
133
|
+
priorDeferrals: number
|
|
134
|
+
/** Max consecutive deferrals before painting anyway. */
|
|
135
|
+
maxDeferrals: number
|
|
136
|
+
/** The gateway's outer `fleetChatId` (may be empty). */
|
|
137
|
+
fleetChatId: string
|
|
138
|
+
/** Live turn's chat (`stampTurn.sessionChatId`) — the stamp-turn fallback
|
|
139
|
+
* used only when no fleet chat is configured. */
|
|
140
|
+
stampChatId?: string
|
|
141
|
+
/** Live turn's forum topic (`stampTurn.sessionThreadId`), carried on the
|
|
142
|
+
* stamp-turn fallback so an exhausted-defer paint lands in the origin
|
|
143
|
+
* topic, not General (#3458). */
|
|
144
|
+
stampThreadId?: number
|
|
145
|
+
/** Owner DM chat id (`loadAccess().allowFrom[0]`) — the durable floor. */
|
|
146
|
+
ownerDm: string
|
|
147
|
+
}): WorkerFeedDestination {
|
|
148
|
+
const { origin, cardExists, priorDeferrals, maxDeferrals, fleetChatId, stampChatId, stampThreadId, ownerDm } = input
|
|
149
|
+
const originResolved = origin != null
|
|
150
|
+
const { defer, deferrals } = decideWorkerFeedOriginDefer({
|
|
151
|
+
originResolved,
|
|
152
|
+
cardExists,
|
|
153
|
+
priorDeferrals,
|
|
154
|
+
maxDeferrals,
|
|
155
|
+
})
|
|
156
|
+
if (defer) return { action: 'defer', deferrals }
|
|
157
|
+
// Painting: exhausted-defer iff we're painting despite no origin and no card.
|
|
158
|
+
const exhausted = !originResolved && !cardExists
|
|
159
|
+
// Prefer the live turn's chat/topic over the owner DM when we must fall back
|
|
160
|
+
// (a misroute at least lands near the work) — only when NO fleet chat is set.
|
|
161
|
+
const usingStampFallback = fleetChatId.length === 0
|
|
162
|
+
const workerFleetChatId = usingStampFallback ? stampChatId ?? fleetChatId : fleetChatId
|
|
163
|
+
const fallbackThreadId = usingStampFallback ? stampThreadId : undefined
|
|
164
|
+
// resolveWorkerFeedChat precedence:
|
|
165
|
+
if (origin != null && origin.chatId.length > 0) {
|
|
166
|
+
return { action: 'paint', chatId: origin.chatId, threadId: origin.threadId, deferrals, exhausted, ownerDmFallback: false }
|
|
167
|
+
}
|
|
168
|
+
if (workerFleetChatId.length > 0) {
|
|
169
|
+
return { action: 'paint', chatId: workerFleetChatId, threadId: fallbackThreadId, deferrals, exhausted, ownerDmFallback: false }
|
|
170
|
+
}
|
|
171
|
+
const ownerDmFallback = origin == null && workerFleetChatId.length === 0 && ownerDm.length > 0
|
|
172
|
+
return {
|
|
173
|
+
action: 'paint',
|
|
174
|
+
chatId: ownerDm,
|
|
175
|
+
threadId: origin?.threadId ?? fallbackThreadId,
|
|
176
|
+
deferrals,
|
|
177
|
+
exhausted,
|
|
178
|
+
ownerDmFallback,
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
43
182
|
export interface WorkerFeedDispatch {
|
|
44
183
|
/** True when the sub-agent was dispatched with `run_in_background: true`. */
|
|
45
184
|
isBackground: boolean
|
|
@@ -58,7 +58,13 @@ import { readFileSync, writeFileSync, existsSync } from 'node:fs'
|
|
|
58
58
|
import { join } from 'node:path'
|
|
59
59
|
import { homedir } from 'node:os'
|
|
60
60
|
|
|
61
|
-
import {
|
|
61
|
+
import {
|
|
62
|
+
scanTurnForFinalReply,
|
|
63
|
+
decideStopHookDisposition,
|
|
64
|
+
isTurnFlushSafetyEnabledEnv,
|
|
65
|
+
isCapturedProseDeliveryEnabledEnv,
|
|
66
|
+
isGatewayHeartbeatFresh,
|
|
67
|
+
} from './silent-end-scan.mjs'
|
|
62
68
|
|
|
63
69
|
// MUST stay in sync with SILENT_END_MAX_RETRIES in telegram-plugin/silent-end.ts
|
|
64
70
|
// (this hook is a standalone .mjs and can't import the TS module).
|
|
@@ -77,6 +83,54 @@ function getStateDir() {
|
|
|
77
83
|
return process.env.TELEGRAM_STATE_DIR ?? join(homedir(), '.claude', 'channels', 'telegram')
|
|
78
84
|
}
|
|
79
85
|
|
|
86
|
+
/**
|
|
87
|
+
* Build the state-file payload the gateway reads back: carries `turnKey` /
|
|
88
|
+
* `chatId` / `threadId` / per-turn `turnId` nonce and the Option-A
|
|
89
|
+
* `pendingText` bridge, with the given `retryCount`. Stale carryover values
|
|
90
|
+
* from the prior on-disk state (`base`) are explicitly dropped when THIS turn
|
|
91
|
+
* has no derivable nonce / no deliverable prose.
|
|
92
|
+
*
|
|
93
|
+
* @param {object} base Prior on-disk state (spread as a starting point).
|
|
94
|
+
* @param {ReturnType<import('./silent-end-scan.mjs').scanTurnForFinalReply>} decision
|
|
95
|
+
* @param {number} retryCount
|
|
96
|
+
*/
|
|
97
|
+
function buildNextState(base, decision, retryCount) {
|
|
98
|
+
const next = { ...base, retryCount, timestamp: Date.now() }
|
|
99
|
+
if (decision.turnKey) {
|
|
100
|
+
next.turnKey = decision.turnKey
|
|
101
|
+
next.chatId = decision.chatId
|
|
102
|
+
if (decision.threadId != null) next.threadId = decision.threadId
|
|
103
|
+
if (decision.turnId) next.turnId = decision.turnId
|
|
104
|
+
else delete next.turnId
|
|
105
|
+
} else {
|
|
106
|
+
delete next.turnId
|
|
107
|
+
}
|
|
108
|
+
if (typeof decision.pendingText === 'string' && decision.pendingText.length > 0) {
|
|
109
|
+
next.pendingText = decision.pendingText
|
|
110
|
+
} else {
|
|
111
|
+
delete next.pendingText
|
|
112
|
+
}
|
|
113
|
+
return next
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Persist the elected state file (single-writer election allow path).
|
|
118
|
+
* retryCount is left UNCHANGED (this is a hand-off to the gateway's delivery
|
|
119
|
+
* machine, not a re-prompt). Fail-open on write error — an allow never loops.
|
|
120
|
+
*
|
|
121
|
+
* @param {string} statePath
|
|
122
|
+
* @param {object} base
|
|
123
|
+
* @param {ReturnType<import('./silent-end-scan.mjs').scanTurnForFinalReply>} decision
|
|
124
|
+
*/
|
|
125
|
+
function writeElectedState(statePath, base, decision) {
|
|
126
|
+
const retryCount = typeof base.retryCount === 'number' ? base.retryCount : 0
|
|
127
|
+
try {
|
|
128
|
+
writeFileSync(statePath, JSON.stringify(buildNextState(base, decision, retryCount)), 'utf8')
|
|
129
|
+
} catch (err) {
|
|
130
|
+
process.stderr.write(`[silent-end-interrupt] failed to write elected state file: ${err.message}\n`)
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
80
134
|
function main() {
|
|
81
135
|
const raw = readStdin().trim()
|
|
82
136
|
if (!raw) process.exit(0)
|
|
@@ -136,6 +190,34 @@ function main() {
|
|
|
136
190
|
|
|
137
191
|
const retryCount = typeof state.retryCount === 'number' ? state.retryCount : 0
|
|
138
192
|
|
|
193
|
+
// ── Single-writer election (duplicate-message fix) ────────────────
|
|
194
|
+
// On a would-BLOCK scan, ALLOW the stop (while still writing the state
|
|
195
|
+
// file so the gateway's delivery machines have their input) IFF a
|
|
196
|
+
// gateway delivery machine is PROVABLY going to deliver the trailing
|
|
197
|
+
// prose. Otherwise BLOCK exactly as today. See
|
|
198
|
+
// `decideStopHookDisposition` in silent-end-scan.mjs for the four
|
|
199
|
+
// never-drop gates. This eliminates the double-send: the gateway flush
|
|
200
|
+
// / captured-prose bridge is the single writer; the hook no longer
|
|
201
|
+
// re-prompts a reworded reply that defeats the exact-match dedup.
|
|
202
|
+
const disposition = decideStopHookDisposition({
|
|
203
|
+
scan: decision,
|
|
204
|
+
retryCount,
|
|
205
|
+
turnFlushSafetyEnabled: isTurnFlushSafetyEnabledEnv(process.env),
|
|
206
|
+
capturedProseDeliveryEnabled: isCapturedProseDeliveryEnabledEnv(process.env),
|
|
207
|
+
gatewayLive: isGatewayHeartbeatFresh(stateDir),
|
|
208
|
+
})
|
|
209
|
+
if (disposition.action === 'allow-elected') {
|
|
210
|
+
// Persist the state file (turnKey / turnId / pendingText) so the
|
|
211
|
+
// gateway's turn-end path delivers the answer — retryCount stays at 0
|
|
212
|
+
// (this is NOT a re-prompt, it's a hand-off to the single writer).
|
|
213
|
+
writeElectedState(statePath, state, decision)
|
|
214
|
+
process.stderr.write(
|
|
215
|
+
`[silent-end-interrupt] single-writer election ALLOWED stop ` +
|
|
216
|
+
`(scan=${decision.reason} elect=${disposition.reason}) — gateway will deliver\n`,
|
|
217
|
+
)
|
|
218
|
+
process.exit(0)
|
|
219
|
+
}
|
|
220
|
+
|
|
139
221
|
if (retryCount >= MAX_RETRIES) {
|
|
140
222
|
// Budget spent. Let the session end so the gateway's
|
|
141
223
|
// `silent-end.ts:recordUndeliveredTurnEnd` path delivers the
|
|
@@ -161,41 +243,10 @@ function main() {
|
|
|
161
243
|
// doubles the effective re-prompt budget vs. the design. With turnKey
|
|
162
244
|
// present (same chatKey shape the gateway uses), the match succeeds
|
|
163
245
|
// and the budget is honored.
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
}
|
|
169
|
-
if (decision.turnKey) {
|
|
170
|
-
nextState.turnKey = decision.turnKey
|
|
171
|
-
nextState.chatId = decision.chatId
|
|
172
|
-
if (decision.threadId != null) {
|
|
173
|
-
nextState.threadId = decision.threadId
|
|
174
|
-
}
|
|
175
|
-
// Per-turn nonce (Finding 3, #3228). The gateway requires this to match
|
|
176
|
-
// the live turn's `turnId` before delivering `pendingText`, so a stale
|
|
177
|
-
// record left over from a prior turn on the same chat/thread can never
|
|
178
|
-
// deliver a previous turn's answer on a later one. Explicitly drop a
|
|
179
|
-
// carried-over `turnId` from the spread `...state` when THIS turn has no
|
|
180
|
-
// derivable nonce, so an old value never lingers.
|
|
181
|
-
if (decision.turnId) nextState.turnId = decision.turnId
|
|
182
|
-
else delete nextState.turnId
|
|
183
|
-
} else {
|
|
184
|
-
delete nextState.turnId
|
|
185
|
-
}
|
|
186
|
-
// Option A transcript-prose bridge: when the scan isolated a substantive
|
|
187
|
-
// final answer the model wrote as plain text but never sent through the
|
|
188
|
-
// reply tool, persist it so the gateway's turn-end path can deliver it
|
|
189
|
-
// directly on the first silent-end (instead of relying on this hook's
|
|
190
|
-
// re-prompt / the obligation represent to eventually recover it). The
|
|
191
|
-
// gateway reads this field back out of the same state file. Explicitly
|
|
192
|
-
// clear a stale carryover value from a prior turn's spread `...state` when
|
|
193
|
-
// THIS turn has no deliverable prose, so an old answer is never re-sent.
|
|
194
|
-
if (typeof decision.pendingText === 'string' && decision.pendingText.length > 0) {
|
|
195
|
-
nextState.pendingText = decision.pendingText
|
|
196
|
-
} else {
|
|
197
|
-
delete nextState.pendingText
|
|
198
|
-
}
|
|
246
|
+
//
|
|
247
|
+
// Per-turn nonce (Finding 3, #3228) and the Option-A `pendingText` bridge
|
|
248
|
+
// are plumbed by `buildNextState`.
|
|
249
|
+
const nextState = buildNextState(state, decision, retryCount + 1)
|
|
199
250
|
try {
|
|
200
251
|
writeFileSync(statePath, JSON.stringify(nextState), 'utf8')
|
|
201
252
|
} catch (err) {
|