typeclaw 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -6
- package/src/agent/index.ts +11 -0
- package/src/agent/model-fallback.ts +59 -40
- package/src/agent/multimodal/look-at.ts +2 -1
- package/src/agent/provider-error.ts +71 -18
- package/src/agent/retry-same-ref.ts +139 -0
- package/src/agent/subagents.ts +6 -2
- package/src/agent/tools/channel-edit.ts +144 -0
- package/src/agent/tools/channel-fetch-attachment.ts +6 -5
- package/src/agent/tools/curl-impersonate.ts +40 -1
- package/src/agent/tools/webfetch/fetch.ts +86 -3
- package/src/agent/tools/webfetch/tool.ts +26 -3
- package/src/agent/tools/webfetch/types.ts +8 -0
- package/src/agent/turn-runner.ts +65 -36
- package/src/bundled-plugins/agent-browser/skills/agent-browser/SKILL.md +28 -11
- package/src/bundled-plugins/reviewer/skills/code-review.ts +22 -2
- package/src/channels/adapters/discord-bot-edit.ts +49 -0
- package/src/channels/adapters/discord-bot.ts +5 -0
- package/src/channels/adapters/discord-edit.ts +51 -0
- package/src/channels/adapters/discord.ts +4 -0
- package/src/channels/adapters/slack-bot-edit.ts +58 -0
- package/src/channels/adapters/slack-bot.ts +18 -6
- package/src/channels/adapters/slack-edit.ts +47 -0
- package/src/channels/adapters/slack.ts +5 -0
- package/src/channels/adapters/teams-edit.ts +61 -0
- package/src/channels/adapters/teams.ts +4 -0
- package/src/channels/adapters/telegram-bot-edit.ts +51 -0
- package/src/channels/adapters/telegram-bot.ts +5 -0
- package/src/channels/adapters/webex-bot.ts +5 -0
- package/src/channels/adapters/webex-edit.ts +47 -0
- package/src/channels/adapters/webex.ts +5 -0
- package/src/channels/continuation-willingness.ts +227 -35
- package/src/channels/router.ts +180 -14
- package/src/channels/schema.ts +24 -0
- package/src/channels/types.ts +36 -0
- package/src/server/command-runner.ts +2 -1
- package/src/server/index.ts +2 -1
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { Type } from '@mariozechner/pi-ai'
|
|
2
|
+
import { defineTool } from '@mariozechner/pi-coding-agent'
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
containsKimiToolDelimiter,
|
|
6
|
+
isNoReplySignal,
|
|
7
|
+
isUpstreamEmptyResponseSentinel,
|
|
8
|
+
stripThinkBlocks,
|
|
9
|
+
type ChannelRouter,
|
|
10
|
+
} from '@/channels/router'
|
|
11
|
+
import { ADAPTER_IDS, type AdapterId } from '@/channels/schema'
|
|
12
|
+
import type { EditMessageResult } from '@/channels/types'
|
|
13
|
+
|
|
14
|
+
import { type ChannelToolLogger, consoleChannelLogger, formatChannelToolFailure } from './channel-log'
|
|
15
|
+
import { renderOutboundEcho, TOOL_RESULT_PREFIX } from './channel-reply'
|
|
16
|
+
import { fenceToolResult } from './runtime-notice'
|
|
17
|
+
|
|
18
|
+
export type CreateChannelEditToolOptions = {
|
|
19
|
+
router: ChannelRouter
|
|
20
|
+
logger?: ChannelToolLogger
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function createChannelEditTool({ router, logger = consoleChannelLogger }: CreateChannelEditToolOptions) {
|
|
24
|
+
return defineTool({
|
|
25
|
+
name: 'channel_edit',
|
|
26
|
+
label: 'Channel Edit',
|
|
27
|
+
description:
|
|
28
|
+
'Edit the text of a message you already posted to an external messenger channel, replacing its body in place. ' +
|
|
29
|
+
'Pass the `message_id` a prior `channel_send` returned (Slack thread ts, Discord message id), plus the same ' +
|
|
30
|
+
'adapter/workspace/chat you sent it to, and the new `text`. Use this to fix a typo, correct a wrong answer, ' +
|
|
31
|
+
'or append a result to a status message you posted earlier — not to carry on a conversation (send a new ' +
|
|
32
|
+
'message for that). Editing is supported on "slack-bot", "slack", "discord-bot", "discord", "telegram-bot", ' +
|
|
33
|
+
'"webex", "webex-bot", and "teams" (chats/DMs only — Teams team-channel posts cannot be edited); an adapter ' +
|
|
34
|
+
'that does not implement editing at all returns ' +
|
|
35
|
+
'{ ok: false, error, code: "not-supported" }. You can only edit messages YOU authored; editing someone ' +
|
|
36
|
+
'else\'s returns code "permission-denied". On success returns { ok: true }; a missing/deleted target returns ' +
|
|
37
|
+
'code "not-found". Code "adapter-unavailable" is distinct from "not-supported": the adapter DOES support ' +
|
|
38
|
+
'editing but is not currently running (e.g. it failed to start on an expired token) — re-authenticating and ' +
|
|
39
|
+
'retrying may fix it, so do not treat it as permanently unsupported.',
|
|
40
|
+
parameters: Type.Object({
|
|
41
|
+
adapter: Type.Union(
|
|
42
|
+
ADAPTER_IDS.map((a) => Type.Literal(a)),
|
|
43
|
+
{
|
|
44
|
+
description:
|
|
45
|
+
'Adapter id. Editing is supported on "slack-bot", "slack", "discord-bot", "discord", "telegram-bot", ' +
|
|
46
|
+
'"webex", "webex-bot", and "teams" (Teams chats/DMs only — not team-channel posts).',
|
|
47
|
+
},
|
|
48
|
+
),
|
|
49
|
+
workspace: Type.String({
|
|
50
|
+
description:
|
|
51
|
+
'Discord guild id or Slack team id (e.g. "T0ACME"); use "@dm" for direct-message channels on either platform.',
|
|
52
|
+
minLength: 1,
|
|
53
|
+
}),
|
|
54
|
+
chat: Type.String({
|
|
55
|
+
description:
|
|
56
|
+
'Channel id the message lives in. Discord channel id (numeric snowflake) or Slack channel id (e.g. "C0CHANNEL", "D0DMID").',
|
|
57
|
+
minLength: 1,
|
|
58
|
+
}),
|
|
59
|
+
thread: Type.Optional(
|
|
60
|
+
Type.String({
|
|
61
|
+
description:
|
|
62
|
+
'Optional thread id the message lives in. Context only — the edit is addressed by `message_id`, so this is not required.',
|
|
63
|
+
minLength: 1,
|
|
64
|
+
}),
|
|
65
|
+
),
|
|
66
|
+
message_id: Type.String({
|
|
67
|
+
description:
|
|
68
|
+
'Platform-native id of the message to edit, as returned by a prior `channel_send` (Slack thread ts, Discord message id).',
|
|
69
|
+
minLength: 1,
|
|
70
|
+
}),
|
|
71
|
+
text: Type.String({
|
|
72
|
+
description: 'The new message body that fully replaces the current text.',
|
|
73
|
+
minLength: 1,
|
|
74
|
+
}),
|
|
75
|
+
}),
|
|
76
|
+
|
|
77
|
+
async execute(_toolCallId, params) {
|
|
78
|
+
const adapter = params.adapter as AdapterId
|
|
79
|
+
type DenyCode = NonNullable<(EditMessageResult & { ok: false })['code']>
|
|
80
|
+
const deny = (error: string, code?: DenyCode) => {
|
|
81
|
+
logger.warn(formatChannelToolFailure('channel_edit', error))
|
|
82
|
+
const details: { ok: false; error: string; code?: DenyCode } =
|
|
83
|
+
code !== undefined ? { ok: false, error, code } : { ok: false, error }
|
|
84
|
+
return {
|
|
85
|
+
content: [{ type: 'text' as const, text: `${TOOL_RESULT_PREFIX}channel_edit denied: ${error}` }],
|
|
86
|
+
details,
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const contentError = editTextGuardError(params.text)
|
|
91
|
+
if (contentError) return deny(contentError)
|
|
92
|
+
|
|
93
|
+
// Strip leaked `<think>` blocks before the edit reaches the router/adapter,
|
|
94
|
+
// exactly as the send path normalizes text. A replacement that is ONLY a
|
|
95
|
+
// think block leaves nothing to post, so deny rather than blank the message.
|
|
96
|
+
const text = stripThinkBlocks(params.text)
|
|
97
|
+
if (text === '') return deny('the replacement text is empty after removing reasoning blocks')
|
|
98
|
+
|
|
99
|
+
const result = await router.editMessage({
|
|
100
|
+
adapter,
|
|
101
|
+
workspace: params.workspace,
|
|
102
|
+
chat: params.chat,
|
|
103
|
+
...(params.thread !== undefined ? { thread: params.thread } : {}),
|
|
104
|
+
messageId: params.message_id,
|
|
105
|
+
text,
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
if (!result.ok) {
|
|
109
|
+
return deny(`${adapter}:${params.workspace}/${params.chat}: ${result.error}`, result.code)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const echo = renderOutboundEcho(text, undefined)
|
|
113
|
+
const receipt = `edited ${adapter}:${params.workspace}/${params.chat} message ${params.message_id}: ${echo}`
|
|
114
|
+
return {
|
|
115
|
+
content: [{ type: 'text' as const, text: fenceToolResult(receipt) }],
|
|
116
|
+
details: { ok: true },
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
})
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Same content guards channel_send runs, minus the send-only flood/dup/resolve
|
|
123
|
+
// logic: an edit is a mutation of an existing post, so the only shared risk is
|
|
124
|
+
// forwarding a control signal (NO_REPLY), the upstream empty-response sentinel
|
|
125
|
+
// (leaks thinking/signature), or raw provider tool-call delimiters as the new
|
|
126
|
+
// body. Blocked BEFORE the router call so the string never reaches the chat.
|
|
127
|
+
function editTextGuardError(text: string): string {
|
|
128
|
+
if (isNoReplySignal(text)) {
|
|
129
|
+
return '`NO_REPLY` is the silent-turn signal, not a message body. To edit a message, pass the replacement text.'
|
|
130
|
+
}
|
|
131
|
+
if (isUpstreamEmptyResponseSentinel(text)) {
|
|
132
|
+
return (
|
|
133
|
+
'refusing to forward an upstream `(Empty response: ...)` sentinel; ' +
|
|
134
|
+
"that string is a provider-SDK debug dump containing the model's thinking content and signature, not a message body."
|
|
135
|
+
)
|
|
136
|
+
}
|
|
137
|
+
if (containsKimiToolDelimiter(text)) {
|
|
138
|
+
return (
|
|
139
|
+
'refusing to forward raw provider tool-call control tokens; these are chat-template ' +
|
|
140
|
+
'delimiters that should have been parsed into a real tool call upstream.'
|
|
141
|
+
)
|
|
142
|
+
}
|
|
143
|
+
return ''
|
|
144
|
+
}
|
|
@@ -19,6 +19,8 @@ export type ChannelFetchAttachmentOrigin = {
|
|
|
19
19
|
thread: string | null
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
type FetchAttachmentDetails = { ok: boolean; error?: string; path?: string; mimetype?: string; size?: number }
|
|
23
|
+
|
|
22
24
|
export type CreateChannelFetchAttachmentToolOptions = {
|
|
23
25
|
router: ChannelRouter
|
|
24
26
|
origin: ChannelFetchAttachmentOrigin
|
|
@@ -75,7 +77,6 @@ export function createChannelFetchAttachmentTool({
|
|
|
75
77
|
}),
|
|
76
78
|
|
|
77
79
|
async execute(_toolCallId, params) {
|
|
78
|
-
type Details = { ok: boolean; error?: string; path?: string; mimetype?: string; size?: number }
|
|
79
80
|
const found = router.lookupInboundAttachment({
|
|
80
81
|
adapter,
|
|
81
82
|
workspace: origin.workspace,
|
|
@@ -112,7 +113,7 @@ export function createChannelFetchAttachmentTool({
|
|
|
112
113
|
if (!result.ok) {
|
|
113
114
|
logger.warn(formatChannelToolFailure('channel_fetch_attachment', `${adapter}: ${result.error}`))
|
|
114
115
|
const text = `channel_fetch_attachment error: ${result.error}`
|
|
115
|
-
const details:
|
|
116
|
+
const details: FetchAttachmentDetails = { ok: false, error: result.error }
|
|
116
117
|
return { content: [{ type: 'text' as const, text }], details }
|
|
117
118
|
}
|
|
118
119
|
|
|
@@ -128,13 +129,13 @@ export function createChannelFetchAttachmentTool({
|
|
|
128
129
|
const message = err instanceof Error ? err.message : String(err)
|
|
129
130
|
logger.warn(formatChannelToolFailure('channel_fetch_attachment', `${adapter}: write failed: ${message}`))
|
|
130
131
|
const text = `channel_fetch_attachment error: write failed: ${message}`
|
|
131
|
-
const details:
|
|
132
|
+
const details: FetchAttachmentDetails = { ok: false, error: `write failed: ${message}` }
|
|
132
133
|
return { content: [{ type: 'text' as const, text }], details }
|
|
133
134
|
}
|
|
134
135
|
|
|
135
136
|
const mimetypePart = result.mimetype !== undefined ? ` (${result.mimetype})` : ''
|
|
136
137
|
const text = `saved ${result.size} bytes to ${targetPath}${mimetypePart}`
|
|
137
|
-
const details:
|
|
138
|
+
const details: FetchAttachmentDetails = {
|
|
138
139
|
ok: true,
|
|
139
140
|
path: targetPath,
|
|
140
141
|
...(result.mimetype !== undefined ? { mimetype: result.mimetype } : {}),
|
|
@@ -171,7 +172,7 @@ function toPosix(p: string): string {
|
|
|
171
172
|
}
|
|
172
173
|
|
|
173
174
|
function errorResult(message: string) {
|
|
174
|
-
const details = { ok: false, error: message }
|
|
175
|
+
const details: FetchAttachmentDetails = { ok: false, error: message }
|
|
175
176
|
return { content: [{ type: 'text' as const, text: `channel_fetch_attachment error: ${message}` }], details }
|
|
176
177
|
}
|
|
177
178
|
|
|
@@ -47,6 +47,11 @@ export type CurlImpersonateRequest = {
|
|
|
47
47
|
// bail early instead of streaming gigabytes.
|
|
48
48
|
maxBytes?: number
|
|
49
49
|
timeoutSeconds?: number
|
|
50
|
+
// Read/write cookies to this jar (curl `-b`+`-c`, same path). Enables a
|
|
51
|
+
// two-request sequence sharing one jar. Cookie plumbing does NOT alter the
|
|
52
|
+
// TLS/HTTP-2/header-order impersonation, so it is exempt from the no-`-H`
|
|
53
|
+
// doctrine above.
|
|
54
|
+
cookieJarPath?: string
|
|
50
55
|
signal?: AbortSignal
|
|
51
56
|
}
|
|
52
57
|
|
|
@@ -56,6 +61,10 @@ export type CurlImpersonateResponse = {
|
|
|
56
61
|
httpStatus: number
|
|
57
62
|
contentType: string
|
|
58
63
|
bytesIn: number
|
|
64
|
+
// Cookie names the server set via Set-Cookie on this response (from curl
|
|
65
|
+
// %{header_json}); lets the policy layer detect a bot-manager 403 without
|
|
66
|
+
// parsing the jar file format.
|
|
67
|
+
setCookieNames: string[]
|
|
59
68
|
}
|
|
60
69
|
|
|
61
70
|
// Specific curl exit codes we map to typed errors. The full list is in
|
|
@@ -113,7 +122,10 @@ export async function curlImpersonate(req: CurlImpersonateRequest): Promise<Curl
|
|
|
113
122
|
// response and replay it; last-match alone fails if the attacker can
|
|
114
123
|
// append text after curl's write-out (they can't, but defense in depth).
|
|
115
124
|
const sentinel = generateSentinel()
|
|
116
|
-
|
|
125
|
+
// %{header_json} is the LAST field so the existing status/url/type/size
|
|
126
|
+
// positions stay byte-compatible with the old parser. It is a single-line
|
|
127
|
+
// JSON object of response headers; we read only Set-Cookie names from it.
|
|
128
|
+
const writeOutTemplate = `${sentinel}%{http_code}\n%{url_effective}\n%{content_type}\n%{size_download}\n%{header_json}\n`
|
|
117
129
|
|
|
118
130
|
const cmd: string[] = [
|
|
119
131
|
curlBinary,
|
|
@@ -156,6 +168,10 @@ export async function curlImpersonate(req: CurlImpersonateRequest): Promise<Curl
|
|
|
156
168
|
cmd.push('--max-filesize', String(req.maxBytes))
|
|
157
169
|
}
|
|
158
170
|
|
|
171
|
+
if (req.cookieJarPath !== undefined) {
|
|
172
|
+
cmd.push('-b', req.cookieJarPath, '-c', req.cookieJarPath)
|
|
173
|
+
}
|
|
174
|
+
|
|
159
175
|
if (req.formFields) {
|
|
160
176
|
for (const field of req.formFields) {
|
|
161
177
|
cmd.push('--data-urlencode', `${field.name}=${field.value}`)
|
|
@@ -248,6 +264,7 @@ function parseCurlOutput(buf: ArrayBuffer, sentinel: string, stderr: string): Cu
|
|
|
248
264
|
const finalUrl = (meta[1] ?? '').trim()
|
|
249
265
|
const contentType = (meta[2] ?? '').trim().toLowerCase()
|
|
250
266
|
const declaredBytes = Number(meta[3]?.trim() ?? '0') || bodyBytes.byteLength
|
|
267
|
+
const setCookieNames = parseSetCookieNames(meta[4] ?? '')
|
|
251
268
|
|
|
252
269
|
const body = new TextDecoder('utf-8', { fatal: false }).decode(bodyBytes)
|
|
253
270
|
|
|
@@ -257,7 +274,29 @@ function parseCurlOutput(buf: ArrayBuffer, sentinel: string, stderr: string): Cu
|
|
|
257
274
|
httpStatus,
|
|
258
275
|
contentType,
|
|
259
276
|
bytesIn: declaredBytes,
|
|
277
|
+
setCookieNames,
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function parseSetCookieNames(headerJson: string): string[] {
|
|
282
|
+
const trimmed = headerJson.trim()
|
|
283
|
+
if (!trimmed) return []
|
|
284
|
+
let parsed: unknown
|
|
285
|
+
try {
|
|
286
|
+
parsed = JSON.parse(trimmed)
|
|
287
|
+
} catch {
|
|
288
|
+
return []
|
|
289
|
+
}
|
|
290
|
+
if (typeof parsed !== 'object' || parsed === null) return []
|
|
291
|
+
const setCookie = (parsed as Record<string, unknown>)['set-cookie']
|
|
292
|
+
const values = Array.isArray(setCookie) ? setCookie : setCookie === undefined ? [] : [setCookie]
|
|
293
|
+
const names: string[] = []
|
|
294
|
+
for (const value of values) {
|
|
295
|
+
if (typeof value !== 'string') continue
|
|
296
|
+
const name = value.split('=', 1)[0]?.trim()
|
|
297
|
+
if (name) names.push(name)
|
|
260
298
|
}
|
|
299
|
+
return names
|
|
261
300
|
}
|
|
262
301
|
|
|
263
302
|
function lastIndexOfBytes(haystack: Uint8Array, needle: Uint8Array): number {
|
|
@@ -21,21 +21,49 @@
|
|
|
21
21
|
// translates non-2xx into a tool-level error message that's useful to the
|
|
22
22
|
// model.
|
|
23
23
|
|
|
24
|
+
import { mkdtemp, rm } from 'node:fs/promises'
|
|
25
|
+
import { tmpdir } from 'node:os'
|
|
26
|
+
import { join } from 'node:path'
|
|
27
|
+
|
|
24
28
|
import {
|
|
25
29
|
CurlImpersonateError,
|
|
26
30
|
curlImpersonate,
|
|
31
|
+
type CurlImpersonateResponse,
|
|
27
32
|
isCurlExitFilesizeExceeded,
|
|
28
33
|
isCurlExitTimeout,
|
|
29
34
|
isCurlImpersonateAvailable,
|
|
30
35
|
} from '../curl-impersonate'
|
|
31
36
|
import { MAX_RESPONSE_BYTES } from './types'
|
|
32
37
|
|
|
38
|
+
export type AntibotWarmup = 'auto' | 'off'
|
|
39
|
+
|
|
40
|
+
export type AntibotWarmupInfo = {
|
|
41
|
+
attempted: boolean
|
|
42
|
+
triggered: boolean
|
|
43
|
+
initialStatus?: number
|
|
44
|
+
initialSetCookieNames?: string[]
|
|
45
|
+
replayStatus?: number
|
|
46
|
+
}
|
|
47
|
+
|
|
33
48
|
export type FetchResult = {
|
|
34
49
|
body: string
|
|
35
50
|
contentType: string
|
|
36
51
|
finalUrl: string
|
|
37
52
|
httpStatus: number
|
|
38
53
|
bytesIn: number
|
|
54
|
+
antibotWarmup?: AntibotWarmupInfo
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Cookie names Akamai Bot Manager sets in the Set-Cookie of its 403 challenge.
|
|
58
|
+
// A 403 carrying one of these is a state-bootstrap handshake (absorb cookies,
|
|
59
|
+
// replay once), NOT an authorization failure — that distinction is what makes
|
|
60
|
+
// the auto-retry safe. Scoped to Akamai on purpose: Cloudflare/DataDome use
|
|
61
|
+
// different challenge semantics (JS/redirects/CAPTCHA) that a blind replay
|
|
62
|
+
// would not satisfy.
|
|
63
|
+
const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
|
|
64
|
+
|
|
65
|
+
function isAntibotWarmup403(response: CurlImpersonateResponse): boolean {
|
|
66
|
+
return response.httpStatus === 403 && response.setCookieNames.some((name) => AKAMAI_BOTMANAGER_COOKIE.test(name))
|
|
39
67
|
}
|
|
40
68
|
|
|
41
69
|
export class WebFetchError extends Error {
|
|
@@ -76,10 +104,11 @@ export async function fetchWithLimits(
|
|
|
76
104
|
url: string,
|
|
77
105
|
timeoutSeconds: number,
|
|
78
106
|
parentSignal?: AbortSignal,
|
|
107
|
+
antibotWarmup: AntibotWarmup = 'auto',
|
|
79
108
|
): Promise<FetchResult> {
|
|
80
109
|
const useImpersonate = !forceFallbackForTest && (await isCurlImpersonateAvailable())
|
|
81
110
|
if (useImpersonate) {
|
|
82
|
-
return fetchWithCurlImpersonate(url, timeoutSeconds, parentSignal)
|
|
111
|
+
return fetchWithCurlImpersonate(url, timeoutSeconds, antibotWarmup, parentSignal)
|
|
83
112
|
}
|
|
84
113
|
return fetchWithBunFetch(url, timeoutSeconds, parentSignal)
|
|
85
114
|
}
|
|
@@ -87,15 +116,66 @@ export async function fetchWithLimits(
|
|
|
87
116
|
async function fetchWithCurlImpersonate(
|
|
88
117
|
url: string,
|
|
89
118
|
timeoutSeconds: number,
|
|
119
|
+
antibotWarmup: AntibotWarmup,
|
|
90
120
|
parentSignal?: AbortSignal,
|
|
91
121
|
): Promise<FetchResult> {
|
|
92
|
-
|
|
122
|
+
if (antibotWarmup === 'off') {
|
|
123
|
+
const response = await runCurl(url, timeoutSeconds, undefined, parentSignal)
|
|
124
|
+
return toFetchResult(url, response)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Warmup needs a jar shared across both requests: request 1 (`-c`) captures
|
|
128
|
+
// the bot-manager cookies the 403 sets; request 2 (`-b`) replays them.
|
|
129
|
+
const jarDir = await mkdtemp(join(tmpdir(), 'typeclaw-webfetch-jar-'))
|
|
130
|
+
const jarPath = join(jarDir, 'cookies.txt')
|
|
131
|
+
const startedAt = Date.now()
|
|
93
132
|
try {
|
|
94
|
-
|
|
133
|
+
const first = await runCurl(url, timeoutSeconds, jarPath, parentSignal)
|
|
134
|
+
if (!isAntibotWarmup403(first)) {
|
|
135
|
+
return toFetchResult(url, first, { attempted: true, triggered: false })
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// `timeoutSeconds` is the budget for the WHOLE web_fetch call, not per
|
|
139
|
+
// attempt: charge the replay only the time the first request left, so a
|
|
140
|
+
// warmed fetch can't run for ~2x the requested timeout. If the first
|
|
141
|
+
// request already spent the budget, surface its 403 instead of forcing a
|
|
142
|
+
// zero-time replay that would just time out.
|
|
143
|
+
const remaining = timeoutSeconds - (Date.now() - startedAt) / 1000
|
|
144
|
+
if (remaining < 1) {
|
|
145
|
+
return toFetchResult(url, first, {
|
|
146
|
+
attempted: true,
|
|
147
|
+
triggered: false,
|
|
148
|
+
initialStatus: first.httpStatus,
|
|
149
|
+
initialSetCookieNames: first.setCookieNames,
|
|
150
|
+
})
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const replay = await runCurl(url, Math.floor(remaining), jarPath, parentSignal)
|
|
154
|
+
return toFetchResult(url, replay, {
|
|
155
|
+
attempted: true,
|
|
156
|
+
triggered: true,
|
|
157
|
+
initialStatus: first.httpStatus,
|
|
158
|
+
initialSetCookieNames: first.setCookieNames,
|
|
159
|
+
replayStatus: replay.httpStatus,
|
|
160
|
+
})
|
|
161
|
+
} finally {
|
|
162
|
+
await rm(jarDir, { recursive: true, force: true })
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async function runCurl(
|
|
167
|
+
url: string,
|
|
168
|
+
timeoutSeconds: number,
|
|
169
|
+
cookieJarPath: string | undefined,
|
|
170
|
+
parentSignal?: AbortSignal,
|
|
171
|
+
): Promise<CurlImpersonateResponse> {
|
|
172
|
+
try {
|
|
173
|
+
return await curlImpersonate({
|
|
95
174
|
url,
|
|
96
175
|
method: 'GET',
|
|
97
176
|
timeoutSeconds,
|
|
98
177
|
maxBytes: MAX_RESPONSE_BYTES,
|
|
178
|
+
cookieJarPath,
|
|
99
179
|
signal: parentSignal,
|
|
100
180
|
})
|
|
101
181
|
} catch (error) {
|
|
@@ -114,7 +194,9 @@ async function fetchWithCurlImpersonate(
|
|
|
114
194
|
const message = error instanceof Error ? error.message : String(error)
|
|
115
195
|
throw new WebFetchError(`Fetch failed: ${message}`)
|
|
116
196
|
}
|
|
197
|
+
}
|
|
117
198
|
|
|
199
|
+
function toFetchResult(url: string, response: CurlImpersonateResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
|
|
118
200
|
if (response.httpStatus < 200 || response.httpStatus >= 300) {
|
|
119
201
|
throw new WebFetchError(`Fetch failed: HTTP ${response.httpStatus}`)
|
|
120
202
|
}
|
|
@@ -132,6 +214,7 @@ async function fetchWithCurlImpersonate(
|
|
|
132
214
|
finalUrl: response.finalUrl || url,
|
|
133
215
|
httpStatus: response.httpStatus,
|
|
134
216
|
bytesIn: bodyByteLength,
|
|
217
|
+
...(antibotWarmup ? { antibotWarmup } : {}),
|
|
135
218
|
}
|
|
136
219
|
}
|
|
137
220
|
|
|
@@ -27,8 +27,11 @@ export const webFetchTool = defineTool({
|
|
|
27
27
|
'If `spawn_subagent` is available to you, PREFER delegating to the `scout` subagent by default: spawn it whenever you expect more than one fetch, an "across multiple sources" task, or any search-then-fetch loop. Scout runs the noisy fetching in its own context window and returns a distilled, citation-backed answer, keeping bulky page bodies out of yours. Only call this tool directly for a single known URL whose content you will cite immediately — or whenever you cannot spawn subagents (e.g. you are yourself a subagent), in which case fetch here. ' +
|
|
28
28
|
'Outbound requests impersonate Chrome 136 at the TLS, HTTP/2, and header layers ' +
|
|
29
29
|
'(via curl-impersonate), which helps with TLS/header fingerprint gates on sites behind Cloudflare/Akamai. ' +
|
|
30
|
-
'
|
|
31
|
-
'
|
|
30
|
+
'For Akamai Bot Manager specifically, it auto-warms the session: a first-request 403 that sets bot-manager ' +
|
|
31
|
+
'cookies (_abck/bm_sz/…) is absorbed and replayed once, which returns 200 on many Akamai-fronted sites ' +
|
|
32
|
+
'(disable with `antibotWarmup: "off"`). It still does NOT solve JavaScript challenges, behavioural ' +
|
|
33
|
+
'fingerprinting (mouse/scroll/timing), interactive CAPTCHAs, or IP-reputation blocks — a 403 from those ' +
|
|
34
|
+
'layers is expected and unrecoverable from this tool. ' +
|
|
32
35
|
'Strategy guide:\n' +
|
|
33
36
|
'- "readability": extract article content as markdown (blogs, docs, news). Default for HTML.\n' +
|
|
34
37
|
'- "jq": query JSON APIs (npm registry, GitHub API). Pass `query` (e.g. ".items[].name").\n' +
|
|
@@ -62,6 +65,15 @@ export const webFetchTool = defineTool({
|
|
|
62
65
|
maximum: MAX_TIMEOUT_SECONDS,
|
|
63
66
|
}),
|
|
64
67
|
),
|
|
68
|
+
antibotWarmup: Type.Optional(
|
|
69
|
+
Type.Union([Type.Literal('auto'), Type.Literal('off')], {
|
|
70
|
+
description:
|
|
71
|
+
'Akamai Bot Manager warmup. "auto" (default): if the first request is a 403 that sets bot-manager cookies ' +
|
|
72
|
+
'(_abck/bm_sz/…), absorb them and replay once — many Akamai-fronted sites (e.g. Coupang) only serve content ' +
|
|
73
|
+
'on the second, cookie-bearing request. "off": disable the replay; a bot-manager 403 then surfaces as the ' +
|
|
74
|
+
'normal HTTP 403 error (use to confirm the block without the warmup).',
|
|
75
|
+
}),
|
|
76
|
+
),
|
|
65
77
|
}),
|
|
66
78
|
|
|
67
79
|
async execute(_toolCallId, params, signal) {
|
|
@@ -80,7 +92,7 @@ export const webFetchTool = defineTool({
|
|
|
80
92
|
|
|
81
93
|
let response
|
|
82
94
|
try {
|
|
83
|
-
response = await fetchWithLimits(normalizedUrl, timeout, signal)
|
|
95
|
+
response = await fetchWithLimits(normalizedUrl, timeout, signal, params.antibotWarmup ?? 'auto')
|
|
84
96
|
} catch (error) {
|
|
85
97
|
const message = error instanceof Error ? error.message : String(error)
|
|
86
98
|
return errorResult(normalizedUrl, message, { startedAt, finalUrl: normalizedUrl })
|
|
@@ -141,6 +153,16 @@ export const webFetchTool = defineTool({
|
|
|
141
153
|
bytesOut: byteLength(capped.text),
|
|
142
154
|
truncated: capped.truncated,
|
|
143
155
|
durationMs: Date.now() - startedAt,
|
|
156
|
+
...(response.antibotWarmup?.triggered
|
|
157
|
+
? {
|
|
158
|
+
antibotWarmup: {
|
|
159
|
+
triggered: true,
|
|
160
|
+
initialStatus: response.antibotWarmup.initialStatus,
|
|
161
|
+
initialSetCookieNames: response.antibotWarmup.initialSetCookieNames,
|
|
162
|
+
replayStatus: response.antibotWarmup.replayStatus,
|
|
163
|
+
},
|
|
164
|
+
}
|
|
165
|
+
: {}),
|
|
144
166
|
}
|
|
145
167
|
|
|
146
168
|
return {
|
|
@@ -153,6 +175,7 @@ export const webFetchTool = defineTool({
|
|
|
153
175
|
type WebFetchParams = {
|
|
154
176
|
url: string
|
|
155
177
|
strategy?: CompactionStrategy
|
|
178
|
+
antibotWarmup?: 'auto' | 'off'
|
|
156
179
|
query?: string
|
|
157
180
|
selector?: string
|
|
158
181
|
pattern?: string
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
export type CompactionStrategy = 'readability' | 'jq' | 'selector' | 'grep' | 'snapshot' | 'raw'
|
|
2
2
|
|
|
3
|
+
export type AntibotWarmupDetails = {
|
|
4
|
+
triggered: boolean
|
|
5
|
+
initialStatus?: number
|
|
6
|
+
initialSetCookieNames?: string[]
|
|
7
|
+
replayStatus?: number
|
|
8
|
+
}
|
|
9
|
+
|
|
3
10
|
export type WebFetchDetails = {
|
|
4
11
|
url: string
|
|
5
12
|
finalUrl: string
|
|
@@ -11,6 +18,7 @@ export type WebFetchDetails = {
|
|
|
11
18
|
bytesOut: number
|
|
12
19
|
truncated: boolean
|
|
13
20
|
durationMs: number
|
|
21
|
+
antibotWarmup?: AntibotWarmupDetails
|
|
14
22
|
error?: boolean
|
|
15
23
|
message?: string
|
|
16
24
|
}
|
package/src/agent/turn-runner.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { ModelRef } from '@/config/providers'
|
|
2
2
|
|
|
3
3
|
import type { AgentSession } from './index'
|
|
4
|
-
import { detectProviderError, subscribeProviderErrors } from './provider-error'
|
|
4
|
+
import { detectProviderError, isRetryableSameRef, subscribeProviderErrors } from './provider-error'
|
|
5
|
+
import { RETRIES_PER_REF, retryTurnOnPersistentSession } from './retry-same-ref'
|
|
5
6
|
import { modelThrottleCircuit, type ThrottleCircuit } from './throttle-circuit'
|
|
6
7
|
|
|
7
8
|
export type PersistentTurnAttempt = {
|
|
@@ -73,44 +74,57 @@ export async function promptPersistentTurnWithFallback(opts: {
|
|
|
73
74
|
if (softError === undefined) softError = new Error(err.message)
|
|
74
75
|
})
|
|
75
76
|
try {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
77
|
+
// Same-ref retry loop: replay this ref on a transient failure before
|
|
78
|
+
// advancing the chain. `retry === 0` is the first, prompt()-driven attempt;
|
|
79
|
+
// later iterations resume via agent.continue() (no user-message re-append).
|
|
80
|
+
let outcome: AttemptOutcome | undefined
|
|
81
|
+
for (let retry = 0; ; retry++) {
|
|
82
|
+
softError = undefined
|
|
83
|
+
let outcomeThisAttempt: AttemptOutcome | undefined
|
|
84
|
+
try {
|
|
85
|
+
if (retry === 0) {
|
|
86
|
+
await opts.session.prompt(opts.text)
|
|
87
|
+
} else {
|
|
88
|
+
const retried = await retryTurnOnPersistentSession(opts.session, { attempt: retry - 1 })
|
|
89
|
+
// The safe continue-recipe couldn't apply: keep the PREVIOUS failure
|
|
90
|
+
// outcome (never cleared) and let it drive the advance/return below.
|
|
91
|
+
if (!retried) break
|
|
92
|
+
}
|
|
93
|
+
outcomeThisAttempt = classifySoftOutcome(softError, opts)
|
|
94
|
+
} catch (err) {
|
|
95
|
+
outcomeThisAttempt = { kind: 'hard', error: err instanceof Error ? err : new Error(String(err)) }
|
|
95
96
|
}
|
|
96
|
-
|
|
97
|
-
|
|
97
|
+
outcome = outcomeThisAttempt
|
|
98
|
+
if (outcome === undefined) break
|
|
99
|
+
// Only keep retrying while the failure is same-ref retryable, the turn
|
|
100
|
+
// stayed idempotent (no output/tool-exec yet), and budget remains.
|
|
101
|
+
const mayRetry =
|
|
102
|
+
retry < RETRIES_PER_REF &&
|
|
103
|
+
isRetryableSameRef(outcome.error.message) &&
|
|
104
|
+
!activity.producedAssistantOutput &&
|
|
105
|
+
!activity.startedToolExecution
|
|
106
|
+
if (!mayRetry) break
|
|
98
107
|
}
|
|
99
|
-
|
|
100
|
-
if (
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
108
|
+
|
|
109
|
+
if (outcome === undefined) {
|
|
110
|
+
attempts.push({ ref, outcome: 'success' })
|
|
111
|
+
circuit.recordSuccess({ profile: opts.profile, ref })
|
|
112
|
+
return { success: true, refUsed: ref, attempts }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Ref abandoned after exhausting same-ref retries. Record throttle ONCE
|
|
116
|
+
// here (not per internal retry) so a single turn can't self-trip the
|
|
117
|
+
// circuit breaker.
|
|
118
|
+
const attempt: PersistentTurnAttempt = { ref, outcome: outcome.kind, errorMessage: outcome.error.message }
|
|
119
|
+
attempts.push(attempt)
|
|
120
|
+
lastError = outcome.error
|
|
121
|
+
if (opts.shouldFailover(outcome.error)) circuit.recordThrottle({ profile: opts.profile, ref })
|
|
122
|
+
if (isLast || !canAdvance(outcome.error, activity, opts.shouldFailover)) {
|
|
123
|
+
if (outcome.kind === 'hard') throw outcome.error
|
|
124
|
+
return { success: false, refUsed: ref, attempts, lastError }
|
|
110
125
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
return { success: true, refUsed: ref, attempts }
|
|
126
|
+
opts.onAttemptFailed?.(attempt)
|
|
127
|
+
continue
|
|
114
128
|
} finally {
|
|
115
129
|
unsubProvider()
|
|
116
130
|
unsubActivity()
|
|
@@ -119,6 +133,21 @@ export async function promptPersistentTurnWithFallback(opts: {
|
|
|
119
133
|
return { success: false, refUsed: opts.refs[opts.refs.length - 1]!, attempts, lastError }
|
|
120
134
|
}
|
|
121
135
|
|
|
136
|
+
type AttemptOutcome = { kind: 'hard' | 'soft'; error: Error }
|
|
137
|
+
|
|
138
|
+
// Classify a completed (non-throwing) attempt: a captured soft error, else a
|
|
139
|
+
// leaf soft error (subagent path), else success (undefined). Mirrors the two
|
|
140
|
+
// soft-error sources the loop handled inline before the retry refactor.
|
|
141
|
+
function classifySoftOutcome(
|
|
142
|
+
softError: Error | undefined,
|
|
143
|
+
opts: { session: AgentSession; detectSoftErrorFromLeaf?: boolean },
|
|
144
|
+
): AttemptOutcome | undefined {
|
|
145
|
+
if (softError !== undefined) return { kind: 'soft', error: softError }
|
|
146
|
+
const leafSoftError = opts.detectSoftErrorFromLeaf ? detectLeafSoftError(opts.session) : undefined
|
|
147
|
+
if (leafSoftError !== undefined) return { kind: 'soft', error: leafSoftError }
|
|
148
|
+
return undefined
|
|
149
|
+
}
|
|
150
|
+
|
|
122
151
|
function canAdvance(error: Error, activity: AttemptActivity, shouldFailover: (err: Error) => boolean): boolean {
|
|
123
152
|
return shouldFailover(error) && !activity.producedAssistantOutput && !activity.startedToolExecution
|
|
124
153
|
}
|