typeclaw 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -6
- package/src/agent/index.ts +11 -0
- package/src/agent/model-fallback.ts +59 -40
- package/src/agent/multimodal/look-at.ts +2 -1
- package/src/agent/provider-error.ts +71 -18
- package/src/agent/retry-same-ref.ts +139 -0
- package/src/agent/subagents.ts +6 -2
- package/src/agent/tools/channel-edit.ts +144 -0
- package/src/agent/tools/channel-fetch-attachment.ts +6 -5
- package/src/agent/tools/curl-impersonate.ts +40 -1
- package/src/agent/tools/webfetch/fetch.ts +86 -3
- package/src/agent/tools/webfetch/tool.ts +26 -3
- package/src/agent/tools/webfetch/types.ts +8 -0
- package/src/agent/turn-runner.ts +65 -36
- package/src/bundled-plugins/agent-browser/skills/agent-browser/SKILL.md +28 -11
- package/src/bundled-plugins/reviewer/skills/code-review.ts +22 -2
- package/src/channels/adapters/discord-bot-edit.ts +49 -0
- package/src/channels/adapters/discord-bot.ts +5 -0
- package/src/channels/adapters/discord-edit.ts +51 -0
- package/src/channels/adapters/discord.ts +4 -0
- package/src/channels/adapters/slack-bot-edit.ts +58 -0
- package/src/channels/adapters/slack-bot.ts +18 -6
- package/src/channels/adapters/slack-edit.ts +47 -0
- package/src/channels/adapters/slack.ts +5 -0
- package/src/channels/adapters/teams-edit.ts +61 -0
- package/src/channels/adapters/teams.ts +4 -0
- package/src/channels/adapters/telegram-bot-edit.ts +51 -0
- package/src/channels/adapters/telegram-bot.ts +5 -0
- package/src/channels/adapters/webex-bot.ts +5 -0
- package/src/channels/adapters/webex-edit.ts +47 -0
- package/src/channels/adapters/webex.ts +5 -0
- package/src/channels/continuation-willingness.ts +227 -35
- package/src/channels/router.ts +180 -14
- package/src/channels/schema.ts +24 -0
- package/src/channels/types.ts +36 -0
- package/src/server/command-runner.ts +2 -1
- package/src/server/index.ts +2 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "typeclaw",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.45.0",
|
|
4
4
|
"homepage": "https://github.com/typeclaw/typeclaw#readme",
|
|
5
5
|
"bugs": {
|
|
6
6
|
"url": "https://github.com/typeclaw/typeclaw/issues"
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"access": "public"
|
|
32
32
|
},
|
|
33
33
|
"scripts": {
|
|
34
|
-
"typecheck": "
|
|
34
|
+
"typecheck": "tsc --noEmit",
|
|
35
35
|
"lint": "oxlint",
|
|
36
36
|
"format": "oxfmt --write .",
|
|
37
37
|
"format:check": "oxfmt --check .",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"@mariozechner/pi-tui": "^0.73.1",
|
|
51
51
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
52
52
|
"@mozilla/readability": "^0.6.0",
|
|
53
|
-
"agent-messenger": "^2.
|
|
53
|
+
"agent-messenger": "^2.30.0",
|
|
54
54
|
"cheerio": "^1.2.0",
|
|
55
55
|
"citty": "^0.2.2",
|
|
56
56
|
"cron-parser": "^5.5.0",
|
|
@@ -69,12 +69,13 @@
|
|
|
69
69
|
"@types/qrcode": "^1.5.6",
|
|
70
70
|
"@types/sinonjs__fake-timers": "^15.0.1",
|
|
71
71
|
"@types/turndown": "^5.0.6",
|
|
72
|
-
"@typescript/
|
|
72
|
+
"@typescript/typescript6": "^6.0.2",
|
|
73
73
|
"oxfmt": "^0.45.0",
|
|
74
|
-
"oxlint": "^1.60.0"
|
|
74
|
+
"oxlint": "^1.60.0",
|
|
75
|
+
"typescript": "^7.0.2"
|
|
75
76
|
},
|
|
76
77
|
"peerDependencies": {
|
|
77
|
-
"typescript": "^5"
|
|
78
|
+
"typescript": "^5 || ^6 || ^7"
|
|
78
79
|
},
|
|
79
80
|
"engines": {
|
|
80
81
|
"bun": ">=1.1.0"
|
package/src/agent/index.ts
CHANGED
|
@@ -72,6 +72,7 @@ import {
|
|
|
72
72
|
import { attachToolNotFoundNudge } from './tool-not-found-nudge'
|
|
73
73
|
import { createBudgetState, type ToolResultBudget, wrapToolDefinitionWithBudget } from './tool-result-budget'
|
|
74
74
|
import { createChannelDisengageTool } from './tools/channel-disengage'
|
|
75
|
+
import { createChannelEditTool } from './tools/channel-edit'
|
|
75
76
|
import { createChannelFetchAttachmentTool, resolveInboxBaseDir } from './tools/channel-fetch-attachment'
|
|
76
77
|
import { createChannelHistoryTool } from './tools/channel-history'
|
|
77
78
|
import { createChannelReactTool } from './tools/channel-react'
|
|
@@ -480,6 +481,14 @@ export async function createSessionWithDispose(options: CreateSessionOptions = {
|
|
|
480
481
|
customTools,
|
|
481
482
|
...(thinkingLevel ? { thinkingLevel } : {}),
|
|
482
483
|
})
|
|
484
|
+
// typeclaw owns retry/fallback in its turn drivers, so the SDK's own
|
|
485
|
+
// same-model auto-retry must be OFF. Leaving it on races typeclaw's
|
|
486
|
+
// soft-error capture: the SDK can silently recover a `stopReason:'error'` turn
|
|
487
|
+
// while typeclaw has already latched the failure, reporting a recovered turn as
|
|
488
|
+
// failed (and burning an unnecessary failover). Disabling it makes a soft error
|
|
489
|
+
// a deterministic, typeclaw-owned signal. Compaction/context-overflow recovery
|
|
490
|
+
// is independent (gated on compaction settings), so this does not disable those.
|
|
491
|
+
;(session as { setAutoRetryEnabled?: (enabled: boolean) => void }).setAutoRetryEnabled?.(false)
|
|
483
492
|
const getAbortReason = () => abortHolder.reason
|
|
484
493
|
const sessionWithAbortReason = Object.assign(session, { getAbortReason })
|
|
485
494
|
|
|
@@ -704,6 +713,7 @@ export function buildChannelTools(
|
|
|
704
713
|
...(sessionId !== undefined ? { sessionId } : {}),
|
|
705
714
|
}),
|
|
706
715
|
)
|
|
716
|
+
tools.push(createChannelEditTool({ router: channelRouter }))
|
|
707
717
|
// Read the live turn origin, falling back to the static snapshot when no
|
|
708
718
|
// getter is wired (composition tests). `reactionRef` is per-turn, so the
|
|
709
719
|
// getter is what makes reactions work outside tests.
|
|
@@ -735,6 +745,7 @@ export function buildChannelTools(
|
|
|
735
745
|
} else {
|
|
736
746
|
tools.push(createChannelSendTool({ router: channelRouter }))
|
|
737
747
|
tools.push(createChannelReadTool({ router: channelRouter }))
|
|
748
|
+
tools.push(createChannelEditTool({ router: channelRouter }))
|
|
738
749
|
}
|
|
739
750
|
return tools
|
|
740
751
|
}
|
|
@@ -3,7 +3,8 @@ import type { Models } from '@/config/config'
|
|
|
3
3
|
import type { KnownModelRef, ModelRef } from '@/config/providers'
|
|
4
4
|
|
|
5
5
|
import type { AgentSession } from './index'
|
|
6
|
-
import { subscribeProviderErrors } from './provider-error'
|
|
6
|
+
import { isRetryableSameRef, subscribeProviderErrors } from './provider-error'
|
|
7
|
+
import { RETRIES_PER_REF, sleepBackoff } from './retry-same-ref'
|
|
7
8
|
import { renderTurnTimeAnchor } from './system-prompt'
|
|
8
9
|
|
|
9
10
|
// Result of a single fallback-aware prompt run.
|
|
@@ -87,53 +88,71 @@ export async function promptWithFallback<TRef extends FallbackModelRef>(opts: {
|
|
|
87
88
|
for (let i = 0; i < opts.refs.length; i++) {
|
|
88
89
|
const ref = opts.refs[i]!
|
|
89
90
|
const isLast = i === opts.refs.length - 1
|
|
91
|
+
// Try this ref, replaying the SAME ref on a transient failure before we give
|
|
92
|
+
// up on it. Cron sessions are cheap and fresh-per-attempt, so a same-ref
|
|
93
|
+
// retry just recreates the session — no state surgery needed.
|
|
94
|
+
const outcome = await attemptRefWithRetry(ref, opts)
|
|
95
|
+
if (outcome.kind === 'success') {
|
|
96
|
+
attempts.push({ ref, outcome: 'success' })
|
|
97
|
+
return { success: true, refUsed: ref, attempts, session: outcome.session, dispose: outcome.dispose }
|
|
98
|
+
}
|
|
99
|
+
attempts.push({ ref, outcome: outcome.kind, errorMessage: outcome.error.message })
|
|
100
|
+
lastError = outcome.error
|
|
101
|
+
const stop = isLast || !failoverGate(outcome.error)
|
|
102
|
+
if (!stop) opts.onAttemptFailed?.({ ref, outcome: outcome.kind, errorMessage: outcome.error.message })
|
|
103
|
+
// The failed ref's session is spent either way — dispose it. On stop we still
|
|
104
|
+
// hand the caller the (now-disposed) session for its final surface, with a
|
|
105
|
+
// no-op dispose so a double-dispose can't happen.
|
|
106
|
+
await outcome.dispose()
|
|
107
|
+
if (stop) {
|
|
108
|
+
return { success: false, refUsed: ref, attempts, session: outcome.session, dispose: async () => {}, lastError }
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
throw new Error('promptWithFallback: unreachable — loop terminated without returning')
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
type RefAttemptOutcome =
|
|
115
|
+
| { kind: 'success'; session: AgentSession; dispose: () => Promise<void> }
|
|
116
|
+
| { kind: 'hard' | 'soft'; error: Error; session: AgentSession; dispose: () => Promise<void> }
|
|
117
|
+
|
|
118
|
+
// One ref, with same-ref retry. Each attempt gets a fresh session (cron's
|
|
119
|
+
// pattern); a retryable failure disposes it and recreates the same ref up to
|
|
120
|
+
// RETRIES_PER_REF times before returning failure to the chain. The returned
|
|
121
|
+
// session/dispose is always from the LAST attempt so the caller can surface or
|
|
122
|
+
// keep it exactly as before.
|
|
123
|
+
async function attemptRefWithRetry<TRef extends FallbackModelRef>(
|
|
124
|
+
ref: TRef,
|
|
125
|
+
opts: {
|
|
126
|
+
text: string
|
|
127
|
+
createSessionForRef: (ref: TRef) => Promise<{ session: AgentSession; dispose: () => Promise<void> }>
|
|
128
|
+
},
|
|
129
|
+
): Promise<RefAttemptOutcome> {
|
|
130
|
+
for (let attempt = 0; ; attempt++) {
|
|
90
131
|
const { session, dispose } = await opts.createSessionForRef(ref)
|
|
91
|
-
// Capture the first soft error per attempt. The
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
// `softError` is populated if a soft error occurred.
|
|
132
|
+
// Capture the first soft error per attempt. The listener fires synchronously
|
|
133
|
+
// off `message_end`, which lands BEFORE `session.prompt()` resolves, so by
|
|
134
|
+
// the time `await` returns `softError` is populated if one occurred.
|
|
95
135
|
let softError: Error | undefined
|
|
96
136
|
const unsub = subscribeProviderErrors(session, (err) => {
|
|
97
137
|
if (!softError) softError = new Error(err.message)
|
|
98
138
|
})
|
|
139
|
+
let outcome: RefAttemptOutcome
|
|
99
140
|
try {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
attempts.push(attempt)
|
|
106
|
-
lastError = error
|
|
107
|
-
const stop = isLast || !failoverGate(error)
|
|
108
|
-
if (!stop) opts.onAttemptFailed?.(attempt)
|
|
109
|
-
unsub()
|
|
110
|
-
await dispose()
|
|
111
|
-
if (stop) {
|
|
112
|
-
return { success: false, refUsed: ref, attempts, session, dispose: async () => {}, lastError }
|
|
113
|
-
}
|
|
114
|
-
continue
|
|
115
|
-
}
|
|
116
|
-
if (softError !== undefined) {
|
|
117
|
-
const attempt: FallbackAttempt<TRef> = { ref, outcome: 'soft', errorMessage: softError.message }
|
|
118
|
-
attempts.push(attempt)
|
|
119
|
-
lastError = softError
|
|
120
|
-
const stop = isLast || !failoverGate(softError)
|
|
121
|
-
if (!stop) opts.onAttemptFailed?.(attempt)
|
|
122
|
-
unsub()
|
|
123
|
-
await dispose()
|
|
124
|
-
if (stop) {
|
|
125
|
-
return { success: false, refUsed: ref, attempts, session, dispose: async () => {}, lastError }
|
|
126
|
-
}
|
|
127
|
-
continue
|
|
128
|
-
}
|
|
129
|
-
attempts.push({ ref, outcome: 'success' })
|
|
130
|
-
unsub()
|
|
131
|
-
return { success: true, refUsed: ref, attempts, session, dispose }
|
|
141
|
+
await session.prompt(`${renderTurnTimeAnchor()}\n\n${opts.text}`)
|
|
142
|
+
outcome =
|
|
143
|
+
softError === undefined
|
|
144
|
+
? { kind: 'success', session, dispose }
|
|
145
|
+
: { kind: 'soft', error: softError, session, dispose }
|
|
132
146
|
} catch (err) {
|
|
147
|
+
outcome = { kind: 'hard', error: err instanceof Error ? err : new Error(String(err)), session, dispose }
|
|
148
|
+
} finally {
|
|
133
149
|
unsub()
|
|
134
|
-
await dispose()
|
|
135
|
-
throw err
|
|
136
150
|
}
|
|
151
|
+
if (outcome.kind === 'success') return outcome
|
|
152
|
+
const canRetry = attempt < RETRIES_PER_REF && isRetryableSameRef(outcome.error.message)
|
|
153
|
+
if (!canRetry) return outcome
|
|
154
|
+
// Retryable and budget remains: drop this session and replay the same ref.
|
|
155
|
+
await dispose()
|
|
156
|
+
await sleepBackoff(attempt)
|
|
137
157
|
}
|
|
138
|
-
throw new Error('promptWithFallback: unreachable — loop terminated without returning')
|
|
139
158
|
}
|
|
@@ -12,6 +12,7 @@ import { providerForModelRef } from '@/config/providers'
|
|
|
12
12
|
import { LLM_FETCH_OBSERVER_TIMEOUTS, type LlmFetchObservedRequestInit } from '@/run/llm-fetch-observer'
|
|
13
13
|
import { SecretsBackend } from '@/secrets'
|
|
14
14
|
|
|
15
|
+
import { promptWithSameRefRetryOnly } from '../retry-same-ref'
|
|
15
16
|
import { buildMultimodalLookerSystemPrompt, resolveImage, type ImageInput } from './looker'
|
|
16
17
|
|
|
17
18
|
type ImageParam = { url: string } | { path: string } | { data: string; mimeType: string }
|
|
@@ -188,7 +189,7 @@ async function runLookAtImages(imageContents: ImageContent[], prompt: string | u
|
|
|
188
189
|
})
|
|
189
190
|
|
|
190
191
|
try {
|
|
191
|
-
await session
|
|
192
|
+
await promptWithSameRefRetryOnly(session, userText, { images: imageContents })
|
|
192
193
|
const text = extractLastAssistantText(session.messages)
|
|
193
194
|
if (text === null) {
|
|
194
195
|
return errorResult('multimodal-looker returned no text response', {
|
|
@@ -33,19 +33,36 @@ const GENERIC_SAFE_NOTICE = 'The upstream LLM provider failed. Operators can che
|
|
|
33
33
|
// and surface it. The literal is the system's own token — English by design.
|
|
34
34
|
const OBSERVER_TIMEOUT = /\(typeclaw observer timeout\)/i
|
|
35
35
|
|
|
36
|
+
// Transport-layer failure: the request died at the connection/session before a
|
|
37
|
+
// usable response — an expired live session, a WebSocket upgrade that never
|
|
38
|
+
// reached `101 Switching Protocols`, or a dropped transport. Unlike an
|
|
39
|
+
// account-wide auth/billing fault, this is transient AND ref-specific: a
|
|
40
|
+
// different ref opens its own session, so failing OVER can succeed. Observed from
|
|
41
|
+
// Codex/ChatGPT (`provider_transport_failure`, expired ChatGPT session, a
|
|
42
|
+
// `wss://…/codex/responses` non-101 upgrade). Provider protocol tokens — English
|
|
43
|
+
// by design (the system-token exception to the multi-language rule).
|
|
44
|
+
const TRANSPORT_FAILURE =
|
|
45
|
+
/provider[_ -]?transport[_ -]?failure|(?:websocket|ws) connection.*failed|expected 101|session expired/i
|
|
46
|
+
|
|
47
|
+
// Auth failure: the provider rejected our credentials (bad/expired/missing API
|
|
48
|
+
// key, 401, failed authentication). Account-wide — a different ref shares the
|
|
49
|
+
// same credential problem — so this is BOTH a redacted safe-message class (a
|
|
50
|
+
// 401 body can carry a Bearer token) AND a NON_FAILOVER_FAULT reason. Shared as
|
|
51
|
+
// one constant so the two uses can never drift: previously the safe-message copy
|
|
52
|
+
// matched `api key ... expired` but the failover copy didn't, letting
|
|
53
|
+
// `session expired: api key expired` wrongly fail over past the auth guard.
|
|
54
|
+
const AUTH_FAULT =
|
|
55
|
+
/\b(401|unauthori[sz]ed|invalid[_ -]?api[_ -]?key|api key.*(?:invalid|expired|missing)|authentication failed|invalid bearer)\b/i
|
|
56
|
+
|
|
36
57
|
// Each entry pairs a narrow matcher against the raw provider text with the
|
|
37
58
|
// canonical, leak-free sentence shown in channels. Matchers are intentionally
|
|
38
59
|
// specific: a miss falls through to GENERIC_SAFE_NOTICE rather than echoing raw
|
|
39
60
|
// text, so adding a new class is opt-in and never widens what we expose.
|
|
40
61
|
const SAFE_CLASSES: ReadonlyArray<{ match: RegExp; safe: string }> = [
|
|
41
62
|
{
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
|
|
45
|
-
// names the operator action (check the API key) without echoing the raw
|
|
46
|
-
// error, whose body can carry a Bearer token.
|
|
47
|
-
match:
|
|
48
|
-
/\b(401|unauthori[sz]ed|invalid[_ -]?api[_ -]?key|api key.*(?:invalid|expired|missing)|authentication failed|invalid bearer)\b/i,
|
|
63
|
+
// Matched first because a 401 body can also mention "account", which would
|
|
64
|
+
// otherwise fall into the billing class below.
|
|
65
|
+
match: AUTH_FAULT,
|
|
49
66
|
safe: 'The upstream LLM provider rejected the request as unauthorized. Operators should check the provider API key configuration and `typeclaw logs`.',
|
|
50
67
|
},
|
|
51
68
|
{
|
|
@@ -71,6 +88,10 @@ const SAFE_CLASSES: ReadonlyArray<{ match: RegExp; safe: string }> = [
|
|
|
71
88
|
match: OBSERVER_TIMEOUT,
|
|
72
89
|
safe: 'The upstream LLM provider stopped responding and the request timed out. Try again shortly.',
|
|
73
90
|
},
|
|
91
|
+
{
|
|
92
|
+
match: TRANSPORT_FAILURE,
|
|
93
|
+
safe: 'The connection to the upstream LLM provider dropped (session/transport failure). Try again shortly.',
|
|
94
|
+
},
|
|
74
95
|
]
|
|
75
96
|
|
|
76
97
|
function toSafeMessage(raw: string): string {
|
|
@@ -92,9 +113,12 @@ const THROTTLE_OR_OVERLOAD =
|
|
|
92
113
|
// help (same account) and would mask a config error the operator must fix. This
|
|
93
114
|
// is checked BEFORE the throttle match because providers often pair a `429`
|
|
94
115
|
// status with a quota/billing/auth reason (e.g. `429 insufficient quota`) — the
|
|
95
|
-
// status code alone must not force a pointless failover.
|
|
96
|
-
|
|
97
|
-
|
|
116
|
+
// status code alone must not force a pointless failover. The auth arm reuses the
|
|
117
|
+
// shared `AUTH_FAULT` source so it stays in lockstep with the safe-message class.
|
|
118
|
+
const NON_FAILOVER_FAULT = new RegExp(
|
|
119
|
+
`\\bcyber_policy\\b|insufficient.*(?:quota|credit|fund|balance)|\\bquota\\b|billing|payment|account is not active|${AUTH_FAULT.source}`,
|
|
120
|
+
'i',
|
|
121
|
+
)
|
|
98
122
|
|
|
99
123
|
export function isThrottleOrOverload(raw: string): boolean {
|
|
100
124
|
if (NON_FAILOVER_FAULT.test(raw)) return false
|
|
@@ -102,14 +126,40 @@ export function isThrottleOrOverload(raw: string): boolean {
|
|
|
102
126
|
}
|
|
103
127
|
|
|
104
128
|
// Failover predicate for turn-drivers' `shouldFailover`: rotate to the next model
|
|
105
|
-
// ref on a transient capacity signal (throttle/overload)
|
|
106
|
-
// timeout
|
|
107
|
-
//
|
|
108
|
-
// fail OVER before the caller surfaces a failure notice.
|
|
109
|
-
// (billing/quota/auth)
|
|
110
|
-
//
|
|
129
|
+
// ref on a transient capacity signal (throttle/overload), an observer stall
|
|
130
|
+
// timeout, or a transport/session failure. A stall or a dropped session is as
|
|
131
|
+
// transient as an overload and another ref (its own session/transport) may well
|
|
132
|
+
// succeed, so it must fail OVER before the caller surfaces a failure notice.
|
|
133
|
+
// Account-wide faults (billing/quota/auth) are excluded up front — a different
|
|
134
|
+
// ref shares the same account problem, so switching can't help.
|
|
111
135
|
export function isFailoverWorthy(raw: string): boolean {
|
|
112
|
-
|
|
136
|
+
if (NON_FAILOVER_FAULT.test(raw)) return false
|
|
137
|
+
return isThrottleOrOverload(raw) || OBSERVER_TIMEOUT.test(raw) || TRANSPORT_FAILURE.test(raw)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Transient network / upstream 5xx signals that a same-model replay may clear.
|
|
141
|
+
// Distinct from THROTTLE_OR_OVERLOAD (429/503/overload) on purpose: an overload
|
|
142
|
+
// means the model is out of capacity NOW, so the right move is to fail OVER to a
|
|
143
|
+
// different ref, not hammer the same one — whereas a socket hang-up or a 500 is a
|
|
144
|
+
// one-off blip a replay against the SAME ref often fixes. `\b` anchors the 5xx
|
|
145
|
+
// codes so they don't match digit runs in prose. Provider protocol tokens —
|
|
146
|
+
// English by design (the system-token exception to the multi-language rule).
|
|
147
|
+
const NETWORK_OR_5XX =
|
|
148
|
+
/econnreset|econnrefused|etimedout|enetunreach|socket hang up|network.?error|connection.?(?:error|refused|reset|lost)|fetch failed|other side closed|reset before headers|http2 request did not get a response|\b(?:500|502|504)\b/i
|
|
149
|
+
|
|
150
|
+
// Retry predicate for typeclaw-owned SAME-REF retry: replay the SAME model ref
|
|
151
|
+
// before advancing the chain. Deliberately NARROWER than `isFailoverWorthy` —
|
|
152
|
+
// only transient failures a same-model replay can plausibly clear: transport/
|
|
153
|
+
// session drops, observer stalls, and network/5xx blips. It intentionally EXCLUDES
|
|
154
|
+
// throttle/overload (429/503) — those mean "this ref is out of capacity now", so
|
|
155
|
+
// they should fail OVER to another ref rather than burn same-ref retries — and
|
|
156
|
+
// account-wide faults (auth/billing/quota/cyber_policy), which must surface. It
|
|
157
|
+
// does NOT match context-overflow: that stays on the SDK compaction path and must
|
|
158
|
+
// never be treated as a retryable provider failure.
|
|
159
|
+
export function isRetryableSameRef(raw: string): boolean {
|
|
160
|
+
if (NON_FAILOVER_FAULT.test(raw)) return false
|
|
161
|
+
if (isThrottleOrOverload(raw)) return false
|
|
162
|
+
return TRANSPORT_FAILURE.test(raw) || OBSERVER_TIMEOUT.test(raw) || NETWORK_OR_5XX.test(raw)
|
|
113
163
|
}
|
|
114
164
|
|
|
115
165
|
export function detectProviderError(message: unknown): DetectedProviderError | null {
|
|
@@ -134,7 +184,10 @@ export function detectProviderError(message: unknown): DetectedProviderError | n
|
|
|
134
184
|
export function detectHardProviderError(err: unknown): DetectedProviderError | null {
|
|
135
185
|
const message = err instanceof Error ? err.message : String(err)
|
|
136
186
|
const isProviderFailure =
|
|
137
|
-
isThrottleOrOverload(message) ||
|
|
187
|
+
isThrottleOrOverload(message) ||
|
|
188
|
+
NON_FAILOVER_FAULT.test(message) ||
|
|
189
|
+
OBSERVER_TIMEOUT.test(message) ||
|
|
190
|
+
TRANSPORT_FAILURE.test(message)
|
|
138
191
|
if (!isProviderFailure) return null
|
|
139
192
|
return { message, safeMessage: toSafeMessage(message) }
|
|
140
193
|
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import type { AgentSession } from './index'
|
|
2
|
+
import { isRetryableSameRef, subscribeProviderErrors } from './provider-error'
|
|
3
|
+
|
|
4
|
+
// Same-ref retry policy. Conservative on purpose: the pi-ai provider layer ALSO
|
|
5
|
+
// retries transport/5xx blips underneath us (WebSocket→SSE fallback + its own
|
|
6
|
+
// SSE retry loop), and each typeclaw attempt is bounded by the observer timeouts
|
|
7
|
+
// (TTFB 15s / idle 120s / overall 300s). Stacking a large per-ref retry count on
|
|
8
|
+
// top of that turns one outage into minutes of dead air, so we replay the same
|
|
9
|
+
// ref just ONCE by default and rely on cross-ref fallback for anything the single
|
|
10
|
+
// replay can't clear.
|
|
11
|
+
export const RETRIES_PER_REF = 1
|
|
12
|
+
const BASE_DELAY_MS = 1_000
|
|
13
|
+
const MAX_DELAY_MS = 5_000
|
|
14
|
+
|
|
15
|
+
// Full-jitter exponential backoff: random in [0, min(cap, base·2^attempt)].
|
|
16
|
+
// Jitter decorrelates concurrent turns (multiple channels/subagents recovering
|
|
17
|
+
// from the same upstream blip) so they don't retry in lockstep and re-collide.
|
|
18
|
+
export function retryBackoffMs(attempt: number, random: () => number = Math.random): number {
|
|
19
|
+
const ceiling = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** attempt)
|
|
20
|
+
return Math.floor(random() * ceiling)
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
24
|
+
if (ms <= 0) return Promise.resolve()
|
|
25
|
+
return new Promise((resolve) => {
|
|
26
|
+
const timer = setTimeout(resolve, ms)
|
|
27
|
+
signal?.addEventListener('abort', () => {
|
|
28
|
+
clearTimeout(timer)
|
|
29
|
+
resolve()
|
|
30
|
+
})
|
|
31
|
+
})
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// For callers that recreate the session themselves (cron) and only need the delay.
|
|
35
|
+
export function sleepBackoff(attempt: number, signal?: AbortSignal, random?: () => number): Promise<void> {
|
|
36
|
+
return sleep(retryBackoffMs(attempt, random), signal)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
type ContinuableAgent = {
|
|
40
|
+
state?: { messages?: unknown }
|
|
41
|
+
continue?: () => Promise<void>
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Replay the CURRENT turn on a persistent session WITHOUT re-appending the user
|
|
45
|
+
// message. Mirrors the SDK's own auto-retry shape, resuming via `agent.continue()`
|
|
46
|
+
// (which replays from the trailing user/tool-result message). Re-calling
|
|
47
|
+
// `session.prompt(text)` instead would duplicate the user message and corrupt
|
|
48
|
+
// history, so we never do that.
|
|
49
|
+
//
|
|
50
|
+
// Two resolved shapes are retryable, matching `continue()`'s contract that the
|
|
51
|
+
// last message be user/tool-result:
|
|
52
|
+
// - trailing ASSISTANT error leaf → pop it, then continue (soft error, or a
|
|
53
|
+
// hard throw AFTER a partial assistant message was written)
|
|
54
|
+
// - trailing USER message → the provider died BEFORE writing any assistant
|
|
55
|
+
// message (the reported incident: transport/session failure before stream
|
|
56
|
+
// start). Nothing to pop — continue() replays the user turn as-is.
|
|
57
|
+
// Any other trailing shape (tool-result mid-execution, custom, empty transcript,
|
|
58
|
+
// no `agent.continue`) fails CLOSED — the caller falls back / surfaces instead.
|
|
59
|
+
export async function retryTurnOnPersistentSession(
|
|
60
|
+
session: AgentSession,
|
|
61
|
+
opts: { attempt: number; signal?: AbortSignal; random?: () => number } = { attempt: 0 },
|
|
62
|
+
): Promise<boolean> {
|
|
63
|
+
const agent = (session as { agent?: ContinuableAgent }).agent
|
|
64
|
+
if (!agent || typeof agent.continue !== 'function') return false
|
|
65
|
+
const messages = agent.state?.messages
|
|
66
|
+
if (!Array.isArray(messages) || messages.length === 0) return false
|
|
67
|
+
const leafRole = (messages[messages.length - 1] as { role?: unknown }).role
|
|
68
|
+
if (leafRole !== 'assistant' && leafRole !== 'user') return false
|
|
69
|
+
|
|
70
|
+
await sleep(retryBackoffMs(opts.attempt, opts.random), opts.signal)
|
|
71
|
+
if (opts.signal?.aborted) return false
|
|
72
|
+
|
|
73
|
+
// Drop only a trailing assistant error leaf; a trailing user message is already
|
|
74
|
+
// the shape continue() wants, so leave it untouched (and never re-appended).
|
|
75
|
+
if (leafRole === 'assistant') {
|
|
76
|
+
;(agent.state as { messages: unknown }).messages = messages.slice(0, -1)
|
|
77
|
+
}
|
|
78
|
+
await agent.continue()
|
|
79
|
+
return true
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Same-ref retry for DIRECT `session.prompt()` call sites that bypass the model
|
|
83
|
+
// fallback helpers (non-stream TUI, slash commands, subagent drain/required-block
|
|
84
|
+
// nudges, multimodal look-at). These lost the SDK's built-in retry when it was
|
|
85
|
+
// disabled globally to kill the soft-error race; this restores equivalent
|
|
86
|
+
// same-model resilience WITHOUT cross-ref fallback (there's no chain here) and
|
|
87
|
+
// WITHOUT the race (typeclaw owns the soft-error signal). On a non-retryable
|
|
88
|
+
// failure it does exactly what a bare prompt() did: the throw propagates, or the
|
|
89
|
+
// soft error stays on the leaf for the caller to read.
|
|
90
|
+
export async function promptWithSameRefRetryOnly(
|
|
91
|
+
session: AgentSession,
|
|
92
|
+
text: string,
|
|
93
|
+
promptOpts?: Parameters<AgentSession['prompt']>[1],
|
|
94
|
+
): Promise<void> {
|
|
95
|
+
let softError: Error | undefined
|
|
96
|
+
// Feature-detect subscribe: some lightweight call sites / test fakes pass a
|
|
97
|
+
// session without an event stream. Without it we simply can't observe soft
|
|
98
|
+
// errors — the wrapper then only retries hard throws, still safe.
|
|
99
|
+
const canSubscribe = typeof (session as { subscribe?: unknown }).subscribe === 'function'
|
|
100
|
+
const unsub = canSubscribe
|
|
101
|
+
? subscribeProviderErrors(session, (err) => {
|
|
102
|
+
if (!softError) softError = new Error(err.message)
|
|
103
|
+
})
|
|
104
|
+
: () => {}
|
|
105
|
+
try {
|
|
106
|
+
// Carry the last attempt's failure so a retry that CAN'T run (unsafe
|
|
107
|
+
// transcript shape → retryTurnOnPersistentSession returns false) still
|
|
108
|
+
// surfaces the original failure instead of resolving as a phantom success.
|
|
109
|
+
let priorHardError: Error | undefined
|
|
110
|
+
for (let attempt = 0; ; attempt++) {
|
|
111
|
+
softError = undefined
|
|
112
|
+
let hardError: Error | undefined
|
|
113
|
+
try {
|
|
114
|
+
if (attempt === 0) {
|
|
115
|
+
await session.prompt(text, promptOpts)
|
|
116
|
+
} else if (!(await retryTurnOnPersistentSession(session, { attempt: attempt - 1 }))) {
|
|
117
|
+
// Continue-recipe not applicable: replay never happened, so the prior
|
|
118
|
+
// failure stands — re-throw a hard error, or return to leave a soft
|
|
119
|
+
// error on the leaf (bare-prompt() semantics).
|
|
120
|
+
if (priorHardError !== undefined) throw priorHardError
|
|
121
|
+
return
|
|
122
|
+
}
|
|
123
|
+
} catch (err) {
|
|
124
|
+
hardError = err instanceof Error ? err : new Error(String(err))
|
|
125
|
+
}
|
|
126
|
+
const error = hardError ?? softError
|
|
127
|
+
if (error === undefined) return
|
|
128
|
+
if (attempt >= RETRIES_PER_REF || !isRetryableSameRef(error.message)) {
|
|
129
|
+
// Out of budget or not retryable: preserve bare-prompt() semantics — a
|
|
130
|
+
// hard error throws; a soft error stays on the leaf for the caller.
|
|
131
|
+
if (hardError !== undefined) throw hardError
|
|
132
|
+
return
|
|
133
|
+
}
|
|
134
|
+
priorHardError = hardError
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
unsub()
|
|
138
|
+
}
|
|
139
|
+
}
|
package/src/agent/subagents.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { type AgentSession, createSession, type PluginSessionWiring } from './in
|
|
|
11
11
|
import { resolveFallbackChain } from './model-fallback'
|
|
12
12
|
import { applyModelRuntimeOverrides } from './model-overrides'
|
|
13
13
|
import { isFailoverWorthy, subscribeProviderErrors } from './provider-error'
|
|
14
|
+
import { promptWithSameRefRetryOnly } from './retry-same-ref'
|
|
14
15
|
import type { SubagentBashPolicy } from './reviewer-bash-policy'
|
|
15
16
|
import type { SessionOrigin } from './session-origin'
|
|
16
17
|
import {
|
|
@@ -375,7 +376,7 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
|
|
|
375
376
|
await runSubagentDrain(drainWatch, {
|
|
376
377
|
drain: backgroundDrain,
|
|
377
378
|
prompt: async (text) => {
|
|
378
|
-
await session
|
|
379
|
+
await promptWithSameRefRetryOnly(session, `${renderTurnTimeAnchor()}\n\n${text}`)
|
|
379
380
|
},
|
|
380
381
|
cancelled: () => aborted,
|
|
381
382
|
})
|
|
@@ -395,7 +396,10 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
|
|
|
395
396
|
console.warn(
|
|
396
397
|
`[subagent] ${name} required_block_retry attempt=${attempt}/${MAX_REQUIRED_BLOCK_RETRIES} tag=${requiredBlockTag}`,
|
|
397
398
|
)
|
|
398
|
-
await
|
|
399
|
+
await promptWithSameRefRetryOnly(
|
|
400
|
+
session,
|
|
401
|
+
`${renderTurnTimeAnchor()}\n\n${renderRequiredBlockRetryNudge(requiredBlockTag)}`,
|
|
402
|
+
)
|
|
399
403
|
}
|
|
400
404
|
if (!aborted && !capture.hasRequiredBlock()) {
|
|
401
405
|
console.warn(`[subagent] ${name} required_block_fallback tag=${requiredBlockTag}`)
|