typeclaw 0.47.0 → 0.47.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/git-nudge.ts +4 -12
- package/src/agent/retry-same-ref.ts +89 -9
- package/src/agent/session-origin.ts +8 -6
- package/src/agent/subagents.ts +88 -15
- package/src/agent/tools/channel-reply.ts +13 -13
- package/src/agent/tools/channel-send.ts +3 -3
- package/src/agent/turn-runner.ts +48 -3
- package/src/bundled-plugins/agent-browser/index.ts +8 -5
- package/src/bundled-plugins/agent-browser/shim-install.ts +30 -9
- package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +4 -11
- package/src/bundled-plugins/memory/dreaming.ts +26 -66
- package/src/bundled-plugins/operator/operator.ts +9 -2
- package/src/bundled-plugins/researcher/researcher.ts +4 -6
- package/src/channels/continuation-willingness.ts +2 -2
- package/src/channels/github-false-receipt.ts +4 -4
- package/src/channels/github-rereview-guard.ts +8 -8
- package/src/channels/router.ts +178 -58
- package/src/git/run.ts +60 -0
- package/src/init/dockerfile.ts +36 -18
- package/src/init/index.ts +1 -1
- package/src/skills/typeclaw-channel-github/SKILL.md +4 -4
- package/src/skills/typeclaw-monorepo/SKILL.md +1 -1
- package/src/skills/typeclaw-troubleshooting/SKILL.md +1 -1
package/package.json
CHANGED
package/src/agent/git-nudge.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { hooklessGitArgs } from '@/git/hookless'
|
|
2
1
|
import { resolveAgentGit } from '@/git/resolve-agent-git'
|
|
2
|
+
import { runGit } from '@/git/run'
|
|
3
3
|
|
|
4
4
|
const MAX_LISTED_PATHS = 10
|
|
5
5
|
|
|
@@ -65,22 +65,14 @@ function filterAgentOwned(paths: readonly string[]): string[] {
|
|
|
65
65
|
return paths.filter((p) => !RUNTIME_OWNED_PREFIXES.some((prefix) => p.startsWith(prefix)))
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
// Mirrors the spawn pattern in `src/container/start.ts` `commitSystemFile`.
|
|
69
68
|
const defaultDeps: GitNudgeDeps = {
|
|
70
69
|
async readStatus(agentDir, gitArgs) {
|
|
71
70
|
const bun = getBun()
|
|
72
71
|
if (!bun) return null
|
|
73
72
|
try {
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
stdout: 'pipe',
|
|
78
|
-
stderr: 'pipe',
|
|
79
|
-
})
|
|
80
|
-
const exit = await proc.exited
|
|
81
|
-
if (exit !== 0) return null
|
|
82
|
-
const text = await new Response(proc.stdout).text()
|
|
83
|
-
return parsePorcelain(text)
|
|
73
|
+
const { exitCode, stdout } = await runGit(bun, agentDir, [...gitArgs, 'status', '--porcelain=v1'])
|
|
74
|
+
if (exitCode !== 0) return null
|
|
75
|
+
return parsePorcelain(stdout)
|
|
84
76
|
} catch {
|
|
85
77
|
return null
|
|
86
78
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AgentSession } from './index'
|
|
2
|
-
import { isRetryableSameRef, subscribeProviderErrors } from './provider-error'
|
|
2
|
+
import { isRetryableSameRef, isThrottleOrOverload, subscribeProviderErrors } from './provider-error'
|
|
3
3
|
|
|
4
4
|
// Same-ref retry policy. Conservative on purpose: the pi-ai provider layer ALSO
|
|
5
5
|
// retries transport/5xx blips underneath us (WebSocket→SSE fallback + its own
|
|
@@ -12,6 +12,45 @@ export const RETRIES_PER_REF = 1
|
|
|
12
12
|
const BASE_DELAY_MS = 1_000
|
|
13
13
|
const MAX_DELAY_MS = 5_000
|
|
14
14
|
|
|
15
|
+
// How much added latency a call site can absorb before the caller would rather
|
|
16
|
+
// see a failure. Deliberately NOT the model-profile name: the retry layer cares
|
|
17
|
+
// about latency tolerance, not which tier the operator configured. 'responsive'
|
|
18
|
+
// is every interactive path (channel chat, TUI, slash commands, cron) and keeps
|
|
19
|
+
// the historical fast-fail behavior; 'patient' is a background path where a
|
|
20
|
+
// human is already waiting minutes for a considered answer. Today the only
|
|
21
|
+
// producer is a subagent whose REQUESTED profile is `deep` (reviewer,
|
|
22
|
+
// researcher) — see resolveRetryPolicy in subagents.ts.
|
|
23
|
+
export type RetryPolicy = 'responsive' | 'patient'
|
|
24
|
+
|
|
25
|
+
// Capacity backoff, used ONLY for throttle/overload under the 'patient' policy.
|
|
26
|
+
// Kept separate from retryBackoffMs above on purpose: that curve (1–5s) is sized
|
|
27
|
+
// for a one-off socket blip, and widening it globally would add dead air to the
|
|
28
|
+
// TUI, slash commands, multimodal look-at, cron, and ordinary network errors.
|
|
29
|
+
// An overload is a capacity outage measured in minutes, not milliseconds — the
|
|
30
|
+
// incident that motivated this burned every retry in 4 seconds against a ~60s
|
|
31
|
+
// outage — so it gets its own, much longer curve.
|
|
32
|
+
//
|
|
33
|
+
// Nominal delays are 10s, 20s, 30s, 30s, 30s, 30s (±20% jitter), so six retries
|
|
34
|
+
// span roughly 120–180s of waiting and place the last attempt late enough to
|
|
35
|
+
// outlive a two-minute blip.
|
|
36
|
+
export const PATIENT_OVERLOAD_RETRIES = 6
|
|
37
|
+
const PATIENT_OVERLOAD_BASE_MS = 10_000
|
|
38
|
+
const PATIENT_OVERLOAD_CAP_MS = 30_000
|
|
39
|
+
// Ceiling on when the LAST retry may START, not a hard wall on total elapsed
|
|
40
|
+
// time: an attempt already in flight still runs to the observer's own 300s
|
|
41
|
+
// overall deadline. The pathological upper bound is therefore ~8m, comfortably
|
|
42
|
+
// inside the 30m reviewer/researcher spawn timeout.
|
|
43
|
+
export const PATIENT_OVERLOAD_WINDOW_MS = 180_000
|
|
44
|
+
|
|
45
|
+
// Proportional (±20%) rather than full jitter: full jitter can return ~0ms,
|
|
46
|
+
// which against a capacity outage means retrying instantly into the same wall.
|
|
47
|
+
// The floor keeps every wait meaningful while still decorrelating concurrent
|
|
48
|
+
// subagents recovering from one upstream blip.
|
|
49
|
+
export function patientOverloadBackoffMs(attempt: number, random: () => number = Math.random): number {
|
|
50
|
+
const nominal = Math.min(PATIENT_OVERLOAD_CAP_MS, PATIENT_OVERLOAD_BASE_MS * 2 ** attempt)
|
|
51
|
+
return Math.floor(nominal * (0.8 + random() * 0.4))
|
|
52
|
+
}
|
|
53
|
+
|
|
15
54
|
// Full-jitter exponential backoff: random in [0, min(cap, base·2^attempt)].
|
|
16
55
|
// Jitter decorrelates concurrent turns (multiple channels/subagents recovering
|
|
17
56
|
// from the same upstream blip) so they don't retry in lockstep and re-collide.
|
|
@@ -58,7 +97,7 @@ type ContinuableAgent = {
|
|
|
58
97
|
// no `agent.continue`) fails CLOSED — the caller falls back / surfaces instead.
|
|
59
98
|
export async function retryTurnOnPersistentSession(
|
|
60
99
|
session: AgentSession,
|
|
61
|
-
opts: { attempt: number; signal?: AbortSignal; random?: () => number } = { attempt: 0 },
|
|
100
|
+
opts: { attempt: number; signal?: AbortSignal; random?: () => number; delayMs?: number } = { attempt: 0 },
|
|
62
101
|
): Promise<boolean> {
|
|
63
102
|
const agent = (session as { agent?: ContinuableAgent }).agent
|
|
64
103
|
if (!agent || typeof agent.continue !== 'function') return false
|
|
@@ -67,7 +106,7 @@ export async function retryTurnOnPersistentSession(
|
|
|
67
106
|
const leafRole = (messages[messages.length - 1] as { role?: unknown }).role
|
|
68
107
|
if (leafRole !== 'assistant' && leafRole !== 'user') return false
|
|
69
108
|
|
|
70
|
-
await sleep(retryBackoffMs(opts.attempt, opts.random), opts.signal)
|
|
109
|
+
await sleep(opts.delayMs ?? retryBackoffMs(opts.attempt, opts.random), opts.signal)
|
|
71
110
|
if (opts.signal?.aborted) return false
|
|
72
111
|
|
|
73
112
|
// Drop only a trailing assistant error leaf; a trailing user message is already
|
|
@@ -153,11 +192,20 @@ function hasSameEntries(messages: unknown, expected: unknown[]): boolean {
|
|
|
153
192
|
// WITHOUT the race (typeclaw owns the soft-error signal). On a non-retryable
|
|
154
193
|
// failure it does exactly what a bare prompt() did: the throw propagates, or the
|
|
155
194
|
// soft error stays on the leaf for the caller to read.
|
|
195
|
+
export type SameRefPromptResult = { success: boolean; lastError?: Error }
|
|
196
|
+
|
|
156
197
|
export async function promptWithSameRefRetryOnly(
|
|
157
198
|
session: AgentSession,
|
|
158
199
|
text: string,
|
|
159
200
|
promptOpts?: Parameters<AgentSession['prompt']>[1],
|
|
160
|
-
|
|
201
|
+
opts: {
|
|
202
|
+
retryPolicy?: RetryPolicy
|
|
203
|
+
random?: () => number
|
|
204
|
+
now?: () => number
|
|
205
|
+
patientBackoffMs?: (attempt: number) => number
|
|
206
|
+
} = {},
|
|
207
|
+
): Promise<SameRefPromptResult> {
|
|
208
|
+
const now = opts.now ?? (() => performance.now())
|
|
161
209
|
let softError: Error | undefined
|
|
162
210
|
// Feature-detect subscribe: some lightweight call sites / test fakes pass a
|
|
163
211
|
// session without an event stream. Without it we simply can't observe soft
|
|
@@ -173,29 +221,61 @@ export async function promptWithSameRefRetryOnly(
|
|
|
173
221
|
// transcript shape → retryTurnOnPersistentSession returns false) still
|
|
174
222
|
// surfaces the original failure instead of resolving as a phantom success.
|
|
175
223
|
let priorHardError: Error | undefined
|
|
224
|
+
let lastError: Error | undefined
|
|
225
|
+
let overloadRetries = 0
|
|
226
|
+
let patientWindowStartedAt: number | undefined
|
|
227
|
+
let nextDelayMs: number | undefined
|
|
176
228
|
for (let attempt = 0; ; attempt++) {
|
|
177
229
|
softError = undefined
|
|
178
230
|
let hardError: Error | undefined
|
|
179
231
|
try {
|
|
180
232
|
if (attempt === 0) {
|
|
181
233
|
await session.prompt(text, promptOpts)
|
|
182
|
-
} else if (
|
|
234
|
+
} else if (
|
|
235
|
+
!(await retryTurnOnPersistentSession(session, {
|
|
236
|
+
attempt: attempt - 1,
|
|
237
|
+
...(nextDelayMs !== undefined ? { delayMs: nextDelayMs } : {}),
|
|
238
|
+
}))
|
|
239
|
+
) {
|
|
183
240
|
// Continue-recipe not applicable: replay never happened, so the prior
|
|
184
241
|
// failure stands — re-throw a hard error, or return to leave a soft
|
|
185
242
|
// error on the leaf (bare-prompt() semantics).
|
|
186
243
|
if (priorHardError !== undefined) throw priorHardError
|
|
187
|
-
return
|
|
244
|
+
return { success: false, ...(lastError !== undefined ? { lastError } : {}) }
|
|
188
245
|
}
|
|
189
246
|
} catch (err) {
|
|
190
247
|
hardError = err instanceof Error ? err : new Error(String(err))
|
|
191
248
|
}
|
|
192
249
|
const error = hardError ?? softError
|
|
193
|
-
if (error === undefined) return
|
|
194
|
-
|
|
250
|
+
if (error === undefined) return { success: true }
|
|
251
|
+
lastError = error
|
|
252
|
+
// A patient call site rides out a capacity outage on the same ref: there is
|
|
253
|
+
// no cross-ref failover on this path, so declining to retry an overload
|
|
254
|
+
// (isRetryableSameRef excludes it, expecting failover to handle it) would
|
|
255
|
+
// leave it with no recovery at all.
|
|
256
|
+
// The delay is budgeted BEFORE the retry is accepted: an elapsed-only check
|
|
257
|
+
// would let a retry taken just under the deadline sleep past it.
|
|
258
|
+
const nowMs = now()
|
|
259
|
+
const patientDelayMs =
|
|
260
|
+
opts.retryPolicy === 'patient' && isThrottleOrOverload(error.message)
|
|
261
|
+
? (opts.patientBackoffMs ?? ((n: number) => patientOverloadBackoffMs(n, opts.random)))(overloadRetries)
|
|
262
|
+
: undefined
|
|
263
|
+
const patientElapsedMs = patientWindowStartedAt === undefined ? 0 : nowMs - patientWindowStartedAt
|
|
264
|
+
const patientOverload =
|
|
265
|
+
patientDelayMs !== undefined &&
|
|
266
|
+
overloadRetries < PATIENT_OVERLOAD_RETRIES &&
|
|
267
|
+
patientElapsedMs + patientDelayMs <= PATIENT_OVERLOAD_WINDOW_MS
|
|
268
|
+
if (patientOverload) {
|
|
269
|
+
patientWindowStartedAt ??= nowMs
|
|
270
|
+
nextDelayMs = patientDelayMs
|
|
271
|
+
overloadRetries++
|
|
272
|
+
} else if (attempt < RETRIES_PER_REF && isRetryableSameRef(error.message)) {
|
|
273
|
+
nextDelayMs = undefined
|
|
274
|
+
} else {
|
|
195
275
|
// Out of budget or not retryable: preserve bare-prompt() semantics — a
|
|
196
276
|
// hard error throws; a soft error stays on the leaf for the caller.
|
|
197
277
|
if (hardError !== undefined) throw hardError
|
|
198
|
-
return
|
|
278
|
+
return { success: false, lastError: error }
|
|
199
279
|
}
|
|
200
280
|
priorHardError = hardError
|
|
201
281
|
}
|
|
@@ -486,15 +486,17 @@ function renderChannelOrigin(
|
|
|
486
486
|
...(origin.adapter === 'github'
|
|
487
487
|
? [
|
|
488
488
|
'tool call, keep working and post the answer with a single final',
|
|
489
|
-
'`channel_reply
|
|
489
|
+
'`channel_reply({ text, more_work_this_turn: false })`.',
|
|
490
|
+
'Do not post an "On it" ack comment first — the runtime',
|
|
490
491
|
'already added an :eyes: reaction; use `channel_react` for explicit ack.',
|
|
491
492
|
]
|
|
492
493
|
: [
|
|
493
494
|
'tool call, send a one-line ack first via `channel_reply({ text: "On it.",',
|
|
494
|
-
'
|
|
495
|
-
'`channel_reply
|
|
496
|
-
'
|
|
497
|
-
'the ack
|
|
495
|
+
'more_work_this_turn: true })`, keep working, then send the answer with a',
|
|
496
|
+
'final `channel_reply({ text, more_work_this_turn: false })`.',
|
|
497
|
+
'The ack is not your reply; the answer is. Once the answer lands, end your turn.',
|
|
498
|
+
'`more_work_this_turn: true` is mandatory on the ack or the turn ends there',
|
|
499
|
+
'and drops the fetch/subagent/actual answer.',
|
|
498
500
|
]),
|
|
499
501
|
'',
|
|
500
502
|
'**Backgrounded work does not end the obligation.** If you spawn a',
|
|
@@ -508,7 +510,7 @@ function renderChannelOrigin(
|
|
|
508
510
|
'',
|
|
509
511
|
'Do not send a second reply just to rephrase, restate, or "confirm" what you already said.',
|
|
510
512
|
'',
|
|
511
|
-
'To reply here, call `channel_reply({ text })`. Addressing (including the',
|
|
513
|
+
'To reply here, call `channel_reply({ text, more_work_this_turn })`. Addressing (including the',
|
|
512
514
|
`thread${origin.thread !== null ? '' : ' — none here, this is a channel-root session'}) is filled in; you don't need`,
|
|
513
515
|
'to copy these fields:',
|
|
514
516
|
'',
|
package/src/agent/subagents.ts
CHANGED
|
@@ -10,8 +10,8 @@ import { applyTurnThinkingLevel } from './attention-escalation'
|
|
|
10
10
|
import { type AgentSession, createSession, type PluginSessionWiring } from './index'
|
|
11
11
|
import { resolveFallbackChain } from './model-fallback'
|
|
12
12
|
import { applyModelRuntimeOverrides } from './model-overrides'
|
|
13
|
-
import { isFailoverWorthy, subscribeProviderErrors } from './provider-error'
|
|
14
|
-
import { promptWithSameRefRetryOnly } from './retry-same-ref'
|
|
13
|
+
import { detectHardProviderError, isFailoverWorthy, subscribeProviderErrors } from './provider-error'
|
|
14
|
+
import { promptWithSameRefRetryOnly, type RetryPolicy } from './retry-same-ref'
|
|
15
15
|
import type { SubagentBashPolicy } from './reviewer-bash-policy'
|
|
16
16
|
import type { SessionOrigin } from './session-origin'
|
|
17
17
|
import {
|
|
@@ -300,8 +300,35 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
|
|
|
300
300
|
normalizeSubagentSession(await createSessionForSubagent(subagent, sessionOptions))
|
|
301
301
|
let aborted = false
|
|
302
302
|
let activeModelRef = resolveFallbackChain(getConfig().models, resolveSubagentProfile(subagent, sessionOptions))[0]!
|
|
303
|
-
|
|
303
|
+
// Latency tolerance is decided ONCE here, from the REQUESTED profile — not the
|
|
304
|
+
// one resolveProfile() landed on. A `deep` subagent whose operator never
|
|
305
|
+
// configured `models.deep` still runs on a background path nobody is watching
|
|
306
|
+
// live, so it stays patient; a spawn explicitly overridden onto `fast` is an
|
|
307
|
+
// explicit request for responsiveness and loses it.
|
|
308
|
+
const retryPolicy: RetryPolicy =
|
|
309
|
+
resolveSubagentProfile(subagent, sessionOptions) === 'deep' ? 'patient' : 'responsive'
|
|
310
|
+
// MUST be updated after every turn — initial, drain, and nudge — so a later
|
|
311
|
+
// success clears an earlier failure and the guard below reads the true state.
|
|
312
|
+
let latestTurnFailure: Error | undefined
|
|
304
313
|
const requiredBlockTag = REQUIRED_FINAL_BLOCK[name]
|
|
314
|
+
// A provider failure reaches us as EITHER a soft leaf error or a hard throw
|
|
315
|
+
// (the turn runner rethrows a hard outcome once the chain is spent, and
|
|
316
|
+
// promptWithSameRefRetryOnly rethrows an exhausted hard error). Recording only
|
|
317
|
+
// the soft shape would let a thrown outage escape as a subagent crash and skip
|
|
318
|
+
// the required-block fallback entirely.
|
|
319
|
+
//
|
|
320
|
+
// Absorbing the throw is only safe where that fallback exists to turn it into
|
|
321
|
+
// an honest synthetic result. A subagent with no required block has no such
|
|
322
|
+
// surface, so swallowing there would report a clean ok:true for a turn that
|
|
323
|
+
// never ran. Those keep failing, as does anything unrecognized — loop-guard
|
|
324
|
+
// aborts, spawn timeouts, real bugs.
|
|
325
|
+
const recordHardProviderFailure = (err: unknown): void => {
|
|
326
|
+
if (requiredBlockTag === undefined) throw err
|
|
327
|
+
const detected = detectHardProviderError(err)
|
|
328
|
+
if (detected === null) throw err
|
|
329
|
+
latestTurnFailure = new Error(detected.message)
|
|
330
|
+
}
|
|
331
|
+
let drainWatch: SubagentDrainWatch | undefined
|
|
305
332
|
const capture = attachFinalMessageCapture(session, requiredBlockTag, options.onFinalMessageCaptured ?? (() => {}))
|
|
306
333
|
if (options.onSessionCreated !== undefined) {
|
|
307
334
|
options.onSessionCreated({
|
|
@@ -346,6 +373,7 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
|
|
|
346
373
|
refs: resolveFallbackChain(getConfig().models, resolveSubagentProfile(subagent, sessionOptions)),
|
|
347
374
|
currentModelRef: activeModelRef,
|
|
348
375
|
profile: resolveSubagentProfile(subagent, sessionOptions),
|
|
376
|
+
retryPolicy,
|
|
349
377
|
session,
|
|
350
378
|
text: turnText,
|
|
351
379
|
skipProviderErrorSubscription: true,
|
|
@@ -367,7 +395,10 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
|
|
|
367
395
|
},
|
|
368
396
|
})
|
|
369
397
|
if (result.success) activeModelRef = result.refUsed
|
|
398
|
+
latestTurnFailure = result.success ? undefined : result.lastError
|
|
370
399
|
throwIfLoopGuardAborted(session)
|
|
400
|
+
} catch (err) {
|
|
401
|
+
recordHardProviderFailure(err)
|
|
371
402
|
} finally {
|
|
372
403
|
if (hooks && turnEvent !== undefined) {
|
|
373
404
|
await hooks.runSessionTurnEnd(turnEvent)
|
|
@@ -377,7 +408,17 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
|
|
|
377
408
|
await runSubagentDrain(drainWatch, {
|
|
378
409
|
drain: backgroundDrain,
|
|
379
410
|
prompt: async (text) => {
|
|
380
|
-
|
|
411
|
+
try {
|
|
412
|
+
const drained = await promptWithSameRefRetryOnly(
|
|
413
|
+
session,
|
|
414
|
+
`${renderTurnTimeAnchor()}\n\n${text}`,
|
|
415
|
+
undefined,
|
|
416
|
+
{ retryPolicy },
|
|
417
|
+
)
|
|
418
|
+
latestTurnFailure = drained.success ? undefined : drained.lastError
|
|
419
|
+
} catch (err) {
|
|
420
|
+
recordHardProviderFailure(err)
|
|
421
|
+
}
|
|
381
422
|
throwIfLoopGuardAborted(session)
|
|
382
423
|
},
|
|
383
424
|
cancelled: () => aborted,
|
|
@@ -392,23 +433,39 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
|
|
|
392
433
|
// final contract-repair pass, not another research phase; it deliberately
|
|
393
434
|
// does NOT re-run the drain.
|
|
394
435
|
if (requiredBlockTag !== undefined) {
|
|
436
|
+
// `latestTurnFailure` set means the last turn died on an upstream provider
|
|
437
|
+
// error rather than finishing without the block. Nudging is then pointless
|
|
438
|
+
// — the nudge is itself a model call, so it hits the same failing provider
|
|
439
|
+
// and returns in ms, burning the whole budget in seconds against an outage
|
|
440
|
+
// that outlives it. Skip straight to a fallback that names the real cause.
|
|
395
441
|
for (
|
|
396
442
|
let attempt = 1;
|
|
397
|
-
!aborted &&
|
|
443
|
+
!aborted &&
|
|
444
|
+
!capture.hasRequiredBlock() &&
|
|
445
|
+
latestTurnFailure === undefined &&
|
|
446
|
+
attempt <= MAX_REQUIRED_BLOCK_RETRIES;
|
|
398
447
|
attempt++
|
|
399
448
|
) {
|
|
400
449
|
console.warn(
|
|
401
450
|
`[subagent] ${name} required_block_retry attempt=${attempt}/${MAX_REQUIRED_BLOCK_RETRIES} tag=${requiredBlockTag}`,
|
|
402
451
|
)
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
452
|
+
try {
|
|
453
|
+
const nudge = await promptWithSameRefRetryOnly(
|
|
454
|
+
session,
|
|
455
|
+
`${renderTurnTimeAnchor()}\n\n${renderRequiredBlockRetryNudge(requiredBlockTag)}`,
|
|
456
|
+
undefined,
|
|
457
|
+
{ retryPolicy },
|
|
458
|
+
)
|
|
459
|
+
if (!nudge.success) latestTurnFailure = nudge.lastError
|
|
460
|
+
} catch (err) {
|
|
461
|
+
recordHardProviderFailure(err)
|
|
462
|
+
}
|
|
407
463
|
throwIfLoopGuardAborted(session)
|
|
408
464
|
}
|
|
409
465
|
if (!aborted && !capture.hasRequiredBlock()) {
|
|
410
|
-
|
|
411
|
-
|
|
466
|
+
const reason: MissingBlockReason = latestTurnFailure !== undefined ? 'provider-failure' : 'missing-block'
|
|
467
|
+
console.warn(`[subagent] ${name} required_block_fallback tag=${requiredBlockTag} reason=${reason}`)
|
|
468
|
+
capture.setSyntheticFinalMessage(renderMissingRequiredBlockFallback(name, requiredBlockTag, reason))
|
|
412
469
|
}
|
|
413
470
|
}
|
|
414
471
|
if (hooks && sessionId !== undefined) {
|
|
@@ -686,14 +743,26 @@ Output exactly one <${tag}> block and nothing else.`
|
|
|
686
743
|
// src/skills/typeclaw-channel-github/SKILL.md.
|
|
687
744
|
export const INCOMPLETE_REVIEW_VERDICT = 'incomplete-review-not-a-verdict'
|
|
688
745
|
|
|
746
|
+
// Why the block is missing. 'missing-block' is the model's own doing — it ended
|
|
747
|
+
// its turn without emitting the block. 'provider-failure' is an upstream outage:
|
|
748
|
+
// the turn never ran to completion at all. Both produce the SAME sentinel verdict
|
|
749
|
+
// (the downstream contract is identical — no analysis landed), but the prose must
|
|
750
|
+
// name the real cause, or an operator reading a capacity outage is told the
|
|
751
|
+
// subagent misbehaved and re-runs it straight into the same wall.
|
|
752
|
+
type MissingBlockReason = 'missing-block' | 'provider-failure'
|
|
753
|
+
|
|
689
754
|
// The terminal graceful fallback when the nudges are exhausted. It fabricates NO
|
|
690
755
|
// findings — only a structured "could not complete" notice — so the parent gets
|
|
691
756
|
// a usable result instead of stale `<analysis>` or a hard error.
|
|
692
|
-
function renderMissingRequiredBlockFallback(name: string, tag: FinalBlockTag): string {
|
|
757
|
+
function renderMissingRequiredBlockFallback(name: string, tag: FinalBlockTag, reason: MissingBlockReason): string {
|
|
758
|
+
const providerFailed = reason === 'provider-failure'
|
|
759
|
+
const cause = providerFailed
|
|
760
|
+
? `the upstream model provider kept failing (capacity/outage), so the turn never completed`
|
|
761
|
+
: `it ended without emitting the required <${tag}> block`
|
|
693
762
|
if (tag === 'report') {
|
|
694
763
|
return `<report>
|
|
695
764
|
<summary>
|
|
696
|
-
The ${name} subagent could not complete a research report in this run:
|
|
765
|
+
The ${name} subagent could not complete a research report in this run: ${cause}${providerFailed ? '' : ' (a known cause is the report tool not completing)'}. Do not treat any earlier analysis text as findings — rerun the researcher or gather the sources directly if the answer is still needed.
|
|
697
766
|
</summary>
|
|
698
767
|
<report_file>
|
|
699
768
|
none
|
|
@@ -707,16 +776,20 @@ The original research request remains unresolved; rerun the ${name} subagent or
|
|
|
707
776
|
</report>`
|
|
708
777
|
}
|
|
709
778
|
if (tag === 'review') {
|
|
779
|
+
const headline = providerFailed ? 'UPSTREAM PROVIDER FAILURE — NOT A VERDICT' : 'INCOMPLETE REVIEW — NOT A VERDICT'
|
|
780
|
+
const remedy = providerFailed
|
|
781
|
+
? 'This is an infrastructure failure, not a judgement on the code. Re-request the review once the provider recovers; do not report it as a review that found nothing.'
|
|
782
|
+
: 'Re-request the review; if it still cannot complete, post an honest "review could not be completed" note and leave existing review state unchanged.'
|
|
710
783
|
return `<review>
|
|
711
784
|
<summary>
|
|
712
|
-
|
|
785
|
+
${headline}. The ${name} subagent could not complete a review in this run: ${cause}. No findings were produced and no verdict was reached — do not treat any earlier text as a verdict, and do NOT map this to APPROVE/REQUEST_CHANGES/COMMENT (including on a re-review). ${remedy}
|
|
713
786
|
</summary>
|
|
714
787
|
<findings>
|
|
715
788
|
</findings>
|
|
716
789
|
<verdict>${INCOMPLETE_REVIEW_VERDICT}</verdict>
|
|
717
790
|
</review>`
|
|
718
791
|
}
|
|
719
|
-
return `<${tag}>\nThe ${name} subagent
|
|
792
|
+
return `<${tag}>\nThe ${name} subagent could not produce the required <${tag}> block (${cause}) and could not recover; rerun it or inspect the transcript.\n</${tag}>`
|
|
720
793
|
}
|
|
721
794
|
|
|
722
795
|
type SubagentCapture = {
|
|
@@ -83,17 +83,17 @@ export function createChannelReplyTool({
|
|
|
83
83
|
},
|
|
84
84
|
),
|
|
85
85
|
),
|
|
86
|
-
|
|
86
|
+
more_work_this_turn: Type.Boolean({
|
|
87
87
|
description:
|
|
88
|
-
'REQUIRED on every channel_reply — you must explicitly choose, there is no default. Set `true` when this reply is a mid-turn status update (e.g. "working on it…") and you still have
|
|
88
|
+
'REQUIRED on every channel_reply — you must explicitly choose, there is no default. Set `true` when this reply is a mid-turn status update (e.g. "working on it…") and you still have MORE WORK to do THIS turn — fetching data, running a tool, spawning a subagent, then replying again; `true` keeps the turn alive so that follow-up actually runs. ' +
|
|
89
89
|
'Set `false` when this reply is your final message for the turn (the common case). ' +
|
|
90
|
-
'This choice is mandatory precisely because a missing value used to default to ending the turn silently: a successful reply ends the turn unless `
|
|
90
|
+
'This choice is mandatory precisely because a missing value used to default to ending the turn silently: a successful reply ends the turn unless `more_work_this_turn` is `true`, so a `false` on an ack you meant to keep working from drops the work you promised. ' +
|
|
91
91
|
'Do not set `true` just to seem responsive; only when genuine multi-step work follows in the same turn.',
|
|
92
92
|
}),
|
|
93
93
|
resolve_review_thread: Type.Optional(
|
|
94
94
|
Type.Boolean({
|
|
95
95
|
description:
|
|
96
|
-
'GitHub PR review threads ONLY. On a TERMINAL (`
|
|
96
|
+
'GitHub PR review threads ONLY. On a TERMINAL (`more_work_this_turn:false`) github PR review-thread reply that carries `text`, this is REQUIRED: you must set it to `true` or `false` and omitting it is rejected — you will be told to re-call with an explicit choice. (It stays a normal optional field everywhere else: ignored on Slack/Discord/Telegram/KakaoTalk and any non-github session, on github replies outside a review thread, on attachments-only replies, and on mid-turn `more_work_this_turn:true` status updates — leave it unset there.) ' +
|
|
97
97
|
'Set `true` when your `text` acknowledges the concern is fixed/verified/addressed (e.g. "verified at <sha>", "thanks, that resolves it"): the runtime resolves the thread BEFORE posting, so the close-out actually happens in this same turn — a successful reply ends the turn, so a resolve deferred to "later" never runs. ' +
|
|
98
98
|
"Set `false` when the thread should stay open (partial fix, disagreement, mid-discussion). It is safe to set `true` by default: the runtime resolves ONLY if the thread's root comment is yours — it refuses (and blocks the reply) on a human reviewer's thread, so you never close someone else's open question. You need not pre-check authorship; let the runtime enforce ownership.",
|
|
99
99
|
}),
|
|
@@ -103,7 +103,7 @@ export function createChannelReplyTool({
|
|
|
103
103
|
async execute(_toolCallId, params) {
|
|
104
104
|
let text = params.text
|
|
105
105
|
const attachments = params.attachments
|
|
106
|
-
const keepTurnAlive = params.
|
|
106
|
+
const keepTurnAlive = params.more_work_this_turn === true
|
|
107
107
|
if ((text === undefined || text === '') && (attachments === undefined || attachments.length === 0)) {
|
|
108
108
|
logger.warn(formatChannelToolFailure('channel_reply', 'missing text and attachments'))
|
|
109
109
|
return {
|
|
@@ -164,7 +164,7 @@ export function createChannelReplyTool({
|
|
|
164
164
|
const missingResolveChoice = missingReviewThreadResolveChoiceError({
|
|
165
165
|
origin,
|
|
166
166
|
text,
|
|
167
|
-
|
|
167
|
+
moreWorkThisTurn: keepTurnAlive,
|
|
168
168
|
resolveReviewThread: params.resolve_review_thread,
|
|
169
169
|
})
|
|
170
170
|
if (missingResolveChoice) {
|
|
@@ -185,7 +185,7 @@ export function createChannelReplyTool({
|
|
|
185
185
|
chat: origin.chat,
|
|
186
186
|
thread: origin.thread,
|
|
187
187
|
text,
|
|
188
|
-
|
|
188
|
+
moreWorkThisTurn: keepTurnAlive,
|
|
189
189
|
resolveReviewThread: params.resolve_review_thread === true,
|
|
190
190
|
})
|
|
191
191
|
if (falseReceipt.kind === 'block') {
|
|
@@ -208,7 +208,7 @@ export function createChannelReplyTool({
|
|
|
208
208
|
thread: origin.thread,
|
|
209
209
|
text,
|
|
210
210
|
wantsResolve: params.resolve_review_thread === true,
|
|
211
|
-
|
|
211
|
+
moreWorkThisTurn: keepTurnAlive,
|
|
212
212
|
getReviewState: (req) => router.getReviewState(req),
|
|
213
213
|
})
|
|
214
214
|
if (rereview.block) {
|
|
@@ -259,19 +259,19 @@ export function createChannelReplyTool({
|
|
|
259
259
|
),
|
|
260
260
|
)
|
|
261
261
|
}
|
|
262
|
-
// `
|
|
262
|
+
// `more_work_this_turn` is read by the router's terminal hook (installChannelReplyTerminalHook),
|
|
263
263
|
// not by this tool — it suppresses the post-reply abort so a multi-step turn
|
|
264
264
|
// keeps going. Success-only: a denied reply never ran, so there is no turn to keep.
|
|
265
265
|
const details: {
|
|
266
266
|
ok: boolean
|
|
267
267
|
error?: string
|
|
268
|
-
|
|
268
|
+
more_work_this_turn?: boolean
|
|
269
269
|
messageId?: string
|
|
270
270
|
messageIds?: readonly string[]
|
|
271
271
|
} = result.ok
|
|
272
272
|
? {
|
|
273
273
|
ok: true,
|
|
274
|
-
...(keepTurnAlive ? {
|
|
274
|
+
...(keepTurnAlive ? { more_work_this_turn: true } : {}),
|
|
275
275
|
...(result.messageId !== undefined ? { messageId: result.messageId } : {}),
|
|
276
276
|
...(result.messageIds !== undefined ? { messageIds: result.messageIds } : {}),
|
|
277
277
|
}
|
|
@@ -329,13 +329,13 @@ export function createChannelReplyTool({
|
|
|
329
329
|
function missingReviewThreadResolveChoiceError(input: {
|
|
330
330
|
origin: ChannelReplyOrigin
|
|
331
331
|
text: string | undefined
|
|
332
|
-
|
|
332
|
+
moreWorkThisTurn: boolean
|
|
333
333
|
resolveReviewThread: boolean | undefined
|
|
334
334
|
}): string {
|
|
335
335
|
const isGithubPrReviewThread =
|
|
336
336
|
input.origin.adapter === 'github' && /^pr:\d+$/.test(input.origin.chat) && input.origin.thread !== null
|
|
337
337
|
const hasText = input.text !== undefined && input.text.trim() !== ''
|
|
338
|
-
if (!isGithubPrReviewThread || !hasText || input.
|
|
338
|
+
if (!isGithubPrReviewThread || !hasText || input.moreWorkThisTurn || input.resolveReviewThread !== undefined) {
|
|
339
339
|
return ''
|
|
340
340
|
}
|
|
341
341
|
return (
|
|
@@ -185,7 +185,7 @@ export function createChannelSendTool({
|
|
|
185
185
|
// via channel_send, and the model kept acknowledging "addressed" without
|
|
186
186
|
// the flag, stranding the thread — channel_reply forced the choice but
|
|
187
187
|
// channel_send did not, so the most-used resolve path was the least
|
|
188
|
-
// guarded. A send is always terminal, so there is no
|
|
188
|
+
// guarded. A send is always terminal, so there is no more-work exemption.
|
|
189
189
|
const missingResolveChoice = missingReviewThreadResolveChoiceError({
|
|
190
190
|
adapter,
|
|
191
191
|
chat: params.chat,
|
|
@@ -209,7 +209,7 @@ export function createChannelSendTool({
|
|
|
209
209
|
chat: params.chat,
|
|
210
210
|
thread: params.thread ?? null,
|
|
211
211
|
text: bodyText,
|
|
212
|
-
|
|
212
|
+
moreWorkThisTurn: false,
|
|
213
213
|
resolveReviewThread: wantsResolve,
|
|
214
214
|
})
|
|
215
215
|
if (falseReceipt.kind === 'block') {
|
|
@@ -231,7 +231,7 @@ export function createChannelSendTool({
|
|
|
231
231
|
thread: params.thread ?? null,
|
|
232
232
|
text: bodyText,
|
|
233
233
|
wantsResolve,
|
|
234
|
-
|
|
234
|
+
moreWorkThisTurn: false,
|
|
235
235
|
getReviewState: (req) => router.getReviewState(req),
|
|
236
236
|
})
|
|
237
237
|
if (rereview.block) {
|
package/src/agent/turn-runner.ts
CHANGED
|
@@ -5,9 +5,18 @@ import {
|
|
|
5
5
|
detectProviderError,
|
|
6
6
|
isObserverTtfbTimeout,
|
|
7
7
|
isRetryableSameRef,
|
|
8
|
+
isThrottleOrOverload,
|
|
8
9
|
subscribeProviderErrors,
|
|
9
10
|
} from './provider-error'
|
|
10
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
PATIENT_OVERLOAD_RETRIES,
|
|
13
|
+
PATIENT_OVERLOAD_WINDOW_MS,
|
|
14
|
+
patientOverloadBackoffMs,
|
|
15
|
+
RETRIES_PER_REF,
|
|
16
|
+
retryTurnAfterCompletedToolResult,
|
|
17
|
+
retryTurnOnPersistentSession,
|
|
18
|
+
type RetryPolicy,
|
|
19
|
+
} from './retry-same-ref'
|
|
11
20
|
import { modelThrottleCircuit, type ThrottleCircuit } from './throttle-circuit'
|
|
12
21
|
|
|
13
22
|
export type PersistentTurnAttempt = {
|
|
@@ -42,11 +51,13 @@ export async function promptPersistentTurnWithFallback(opts: {
|
|
|
42
51
|
shouldFailover: (err: Error) => boolean
|
|
43
52
|
setModelForRef: (ref: ModelRef) => Promise<void>
|
|
44
53
|
profile?: string
|
|
54
|
+
retryPolicy?: RetryPolicy
|
|
45
55
|
circuit?: ThrottleCircuit
|
|
46
56
|
skipProviderErrorSubscription?: boolean
|
|
47
57
|
detectSoftErrorFromLeaf?: boolean
|
|
48
58
|
authorizeRetryAfterCompletedToolResult?: () => boolean
|
|
49
59
|
retryRandom?: () => number
|
|
60
|
+
patientBackoffMs?: (attempt: number) => number
|
|
50
61
|
onRetryBackoffStart?: () => void
|
|
51
62
|
beforeAttempt?: (ref: ModelRef) => void
|
|
52
63
|
onAttemptFailed?: (attempt: PersistentTurnAttempt) => void
|
|
@@ -107,6 +118,9 @@ export async function promptPersistentTurnWithFallback(opts: {
|
|
|
107
118
|
// later iterations resume via agent.continue() (no user-message re-append).
|
|
108
119
|
let outcome: AttemptOutcome | undefined
|
|
109
120
|
let retryAfterCompletedToolResult = false
|
|
121
|
+
let overloadRetries = 0
|
|
122
|
+
let patientWindowStartedAt: number | undefined
|
|
123
|
+
let nextDelayMs: number | undefined
|
|
110
124
|
for (let retry = 0; ; retry++) {
|
|
111
125
|
softError = undefined
|
|
112
126
|
const attemptStartedAt = now()
|
|
@@ -119,10 +133,14 @@ export async function promptPersistentTurnWithFallback(opts: {
|
|
|
119
133
|
? await retryTurnAfterCompletedToolResult(opts.session, {
|
|
120
134
|
attempt: retry - 1,
|
|
121
135
|
authorize: () => opts.authorizeRetryAfterCompletedToolResult?.() === true,
|
|
136
|
+
...(nextDelayMs !== undefined ? { delayMs: nextDelayMs } : {}),
|
|
122
137
|
...(opts.retryRandom !== undefined ? { random: opts.retryRandom } : {}),
|
|
123
138
|
...(opts.onRetryBackoffStart !== undefined ? { onBackoffStart: opts.onRetryBackoffStart } : {}),
|
|
124
139
|
})
|
|
125
|
-
: await retryTurnOnPersistentSession(opts.session, {
|
|
140
|
+
: await retryTurnOnPersistentSession(opts.session, {
|
|
141
|
+
attempt: retry - 1,
|
|
142
|
+
...(nextDelayMs !== undefined ? { delayMs: nextDelayMs } : {}),
|
|
143
|
+
})
|
|
126
144
|
// The safe continue-recipe couldn't apply: keep the PREVIOUS failure
|
|
127
145
|
// outcome (never cleared) and let it drive the advance/return below.
|
|
128
146
|
if (!retried) break
|
|
@@ -145,7 +163,27 @@ export async function promptPersistentTurnWithFallback(opts: {
|
|
|
145
163
|
// Retry within budget only when the failure is same-ref retryable and
|
|
146
164
|
// either no visible/tool activity occurred, or the caller explicitly
|
|
147
165
|
// authorizes the strict post-tool-result resume recipe.
|
|
148
|
-
|
|
166
|
+
// Overload means "this ref is out of capacity NOW", so the standing policy
|
|
167
|
+
// is to fail OVER rather than burn same-ref retries — which is why
|
|
168
|
+
// isRetryableSameRef excludes it. But on the LAST usable ref there is
|
|
169
|
+
// nothing to fail over to, and a single-ref chain is all last, so that
|
|
170
|
+
// policy leaves a capacity outage with no recovery whatsoever. A patient
|
|
171
|
+
// call site rides it out here instead; a responsive one still fails fast.
|
|
172
|
+
// Budget the delay BEFORE accepting the retry. Testing only the elapsed
|
|
173
|
+
// window lets a retry accepted just under the deadline sleep past it, so
|
|
174
|
+
// the next provider attempt would START after the window we advertise.
|
|
175
|
+
const nowMs = now()
|
|
176
|
+
const patientDelayMs =
|
|
177
|
+
opts.retryPolicy === 'patient' && isLast && isThrottleOrOverload(outcome.error.message)
|
|
178
|
+
? (opts.patientBackoffMs ?? ((n: number) => patientOverloadBackoffMs(n, opts.retryRandom)))(overloadRetries)
|
|
179
|
+
: undefined
|
|
180
|
+
const patientElapsedMs = patientWindowStartedAt === undefined ? 0 : nowMs - patientWindowStartedAt
|
|
181
|
+
const patientOverload =
|
|
182
|
+
patientDelayMs !== undefined &&
|
|
183
|
+
overloadRetries < PATIENT_OVERLOAD_RETRIES &&
|
|
184
|
+
patientElapsedMs + patientDelayMs <= PATIENT_OVERLOAD_WINDOW_MS
|
|
185
|
+
const retryableWithinBudget =
|
|
186
|
+
patientOverload || (retry < RETRIES_PER_REF && isRetryableSameRef(outcome.error.message))
|
|
149
187
|
const mayRetryWithoutActivity = !activity.producedAssistantOutput && !activity.startedToolExecution
|
|
150
188
|
retryAfterCompletedToolResult =
|
|
151
189
|
retryableWithinBudget &&
|
|
@@ -153,6 +191,13 @@ export async function promptPersistentTurnWithFallback(opts: {
|
|
|
153
191
|
opts.authorizeRetryAfterCompletedToolResult?.() === true
|
|
154
192
|
const mayRetry = retryableWithinBudget && (mayRetryWithoutActivity || retryAfterCompletedToolResult)
|
|
155
193
|
if (!mayRetry) break
|
|
194
|
+
if (patientOverload) {
|
|
195
|
+
patientWindowStartedAt ??= nowMs
|
|
196
|
+
nextDelayMs = patientDelayMs
|
|
197
|
+
overloadRetries++
|
|
198
|
+
} else {
|
|
199
|
+
nextDelayMs = undefined
|
|
200
|
+
}
|
|
156
201
|
}
|
|
157
202
|
|
|
158
203
|
if (outcome === undefined) {
|