typeclaw 0.43.1 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/auth.schema.json +92 -0
- package/package.json +7 -6
- package/secrets.schema.json +92 -0
- package/src/agent/index.ts +25 -2
- package/src/agent/model-fallback.ts +59 -40
- package/src/agent/multimodal/look-at.ts +29 -10
- package/src/agent/plugin-tools.ts +9 -4
- package/src/agent/provider-error.ts +71 -18
- package/src/agent/retry-same-ref.ts +139 -0
- package/src/agent/session-origin.ts +1 -0
- package/src/agent/subagents.ts +6 -2
- package/src/agent/tools/channel-edit.ts +144 -0
- package/src/agent/tools/channel-fetch-attachment.ts +51 -8
- package/src/agent/tools/channel-reply.ts +41 -18
- package/src/agent/tools/channel-send.ts +82 -13
- package/src/agent/tools/curl-impersonate.ts +40 -1
- package/src/agent/tools/webfetch/fetch.ts +86 -3
- package/src/agent/tools/webfetch/tool.ts +26 -3
- package/src/agent/tools/webfetch/types.ts +8 -0
- package/src/agent/turn-runner.ts +65 -36
- package/src/bundled-plugins/agent-browser/skills/agent-browser/SKILL.md +28 -11
- package/src/bundled-plugins/github-cli-auth/gh-review-detect.ts +101 -1
- package/src/bundled-plugins/github-cli-auth/review-recorder.ts +44 -3
- package/src/bundled-plugins/guard/policies/managed-config.ts +20 -4
- package/src/bundled-plugins/memory/README.md +9 -9
- package/src/bundled-plugins/memory/index.ts +40 -9
- package/src/bundled-plugins/reviewer/skills/code-review.ts +22 -2
- package/src/channels/adapters/discord-bot-edit.ts +49 -0
- package/src/channels/adapters/discord-bot.ts +5 -0
- package/src/channels/adapters/discord-edit.ts +51 -0
- package/src/channels/adapters/discord.ts +4 -0
- package/src/channels/adapters/github/inbound.ts +20 -14
- package/src/channels/adapters/github/review-thread-resolver.ts +38 -4
- package/src/channels/adapters/slack-bot-edit.ts +58 -0
- package/src/channels/adapters/slack-bot.ts +18 -6
- package/src/channels/adapters/slack-edit.ts +47 -0
- package/src/channels/adapters/slack.ts +5 -0
- package/src/channels/adapters/teams-classify.ts +144 -0
- package/src/channels/adapters/teams-edit.ts +61 -0
- package/src/channels/adapters/teams-id-token.ts +112 -0
- package/src/channels/adapters/teams-key.ts +32 -0
- package/src/channels/adapters/teams.ts +523 -0
- package/src/channels/adapters/telegram-bot-edit.ts +51 -0
- package/src/channels/adapters/telegram-bot.ts +5 -0
- package/src/channels/adapters/webex-bot.ts +5 -0
- package/src/channels/adapters/webex-edit.ts +47 -0
- package/src/channels/adapters/webex.ts +5 -0
- package/src/channels/continuation-willingness.ts +227 -35
- package/src/channels/github-review-turn-ledger.ts +43 -0
- package/src/channels/index.ts +1 -1
- package/src/channels/manager.ts +54 -1
- package/src/channels/router.ts +482 -53
- package/src/channels/schema.ts +33 -0
- package/src/channels/types.ts +36 -0
- package/src/cli/channel.ts +92 -4
- package/src/cli/hostd.ts +41 -0
- package/src/cli/init.ts +58 -1
- package/src/config/channels-mutation.ts +9 -1
- package/src/cron/index.ts +16 -4
- package/src/cron/reloadable.ts +25 -7
- package/src/cron/scheduler.ts +4 -0
- package/src/cron/schema.ts +143 -35
- package/src/doctor/channel-checks.ts +27 -0
- package/src/hostd/daemon.ts +23 -3
- package/src/hostd/protocol.ts +2 -0
- package/src/hostd/teams-renewal-manager.ts +212 -0
- package/src/init/checkpoint.ts +1 -1
- package/src/init/index.ts +68 -1
- package/src/init/run-owner-claim.ts +1 -0
- package/src/init/teams-auth.ts +111 -0
- package/src/inspect/label.ts +4 -0
- package/src/permissions/match-rule.ts +11 -1
- package/src/permissions/resolve.ts +1 -0
- package/src/role-claim/match-rule.ts +2 -1
- package/src/run/index.ts +35 -8
- package/src/run/llm-fetch-observer.ts +27 -5
- package/src/sandbox/build.ts +12 -0
- package/src/sandbox/policy.ts +13 -0
- package/src/sandbox/writable-zones.ts +46 -10
- package/src/secrets/schema.ts +32 -0
- package/src/secrets/teams-renewal.ts +229 -0
- package/src/secrets/teams-store.ts +108 -0
- package/src/server/command-runner.ts +2 -1
- package/src/server/index.ts +2 -1
- package/typeclaw.schema.json +163 -3
|
@@ -210,15 +210,22 @@ export function createChannelReplyTool({
|
|
|
210
210
|
// flag fixes). Resolve-failure blocks the reply so the agent never posts
|
|
211
211
|
// a "looks resolved" ack next to a still-open thread; the router enforces
|
|
212
212
|
// that only the bot's own threads can be resolved.
|
|
213
|
+
let resolveMissNotice: string | null = null
|
|
213
214
|
if (params.resolve_review_thread === true) {
|
|
214
|
-
const
|
|
215
|
-
if (
|
|
216
|
-
logger.warn(formatChannelToolFailure('channel_reply',
|
|
215
|
+
const resolve = await resolveReviewThreadBeforeReply(router, origin)
|
|
216
|
+
if (resolve.kind === 'block') {
|
|
217
|
+
logger.warn(formatChannelToolFailure('channel_reply', resolve.error))
|
|
217
218
|
return {
|
|
218
|
-
content: [{ type: 'text' as const, text: `channel_reply denied: ${
|
|
219
|
-
details: { ok: false, error:
|
|
219
|
+
content: [{ type: 'text' as const, text: `channel_reply denied: ${resolve.error}` }],
|
|
220
|
+
details: { ok: false, error: resolve.error },
|
|
220
221
|
}
|
|
221
222
|
}
|
|
223
|
+
// `no-match` stays non-blocking (the thread may be genuinely gone) but
|
|
224
|
+
// the resolve did NOT run, so tell the model instead of posting a clean
|
|
225
|
+
// receipt that hides the miss. Mirrors channel_send.
|
|
226
|
+
if (resolve.kind === 'no-match') {
|
|
227
|
+
resolveMissNotice = resolveMissHint(origin.thread)
|
|
228
|
+
}
|
|
222
229
|
}
|
|
223
230
|
|
|
224
231
|
const result = await router.send({
|
|
@@ -287,8 +294,9 @@ export function createChannelReplyTool({
|
|
|
287
294
|
// intentionally weaker and is safe ONLY because denials carry no echoed
|
|
288
295
|
// prose; the success result does, and the weak prefix let Kimi loop.
|
|
289
296
|
const warnNote = falseReceiptNotice !== null ? fenceRuntimeNotice(falseReceiptNotice) : ''
|
|
297
|
+
const missNote = resolveMissNotice ?? ''
|
|
290
298
|
return {
|
|
291
|
-
content: [{ type: 'text' as const, text: `${fenceToolResult(receipt)}${hint}${warnNote}` }],
|
|
299
|
+
content: [{ type: 'text' as const, text: `${fenceToolResult(receipt)}${hint}${warnNote}${missNote}` }],
|
|
292
300
|
details,
|
|
293
301
|
}
|
|
294
302
|
}
|
|
@@ -324,21 +332,25 @@ function missingReviewThreadResolveChoiceError(input: {
|
|
|
324
332
|
)
|
|
325
333
|
}
|
|
326
334
|
|
|
327
|
-
//
|
|
328
|
-
//
|
|
329
|
-
//
|
|
330
|
-
//
|
|
331
|
-
//
|
|
332
|
-
|
|
335
|
+
// `block` when the resolve should stop the reply, `no-match` when the thread is
|
|
336
|
+
// gone (non-blocking — the reply posts but the caller warns), `resolved` on
|
|
337
|
+
// success. Every hard failure — wrong author, permission denial, HTTP 404 on a
|
|
338
|
+
// misdirected lookup, transient API error — blocks, so the agent never claims a
|
|
339
|
+
// thread is settled when the resolve did not actually run.
|
|
340
|
+
type ResolveOutcome = { kind: 'resolved' } | { kind: 'no-match' } | { kind: 'block'; error: string }
|
|
341
|
+
|
|
333
342
|
async function resolveReviewThreadBeforeReply(
|
|
334
343
|
router: ChannelRouter,
|
|
335
344
|
origin: ChannelReplyOrigin,
|
|
336
|
-
): Promise<
|
|
345
|
+
): Promise<ResolveOutcome> {
|
|
337
346
|
if (origin.adapter !== 'github') {
|
|
338
|
-
return 'resolve_review_thread is only supported on github sessions.'
|
|
347
|
+
return { kind: 'block', error: 'resolve_review_thread is only supported on github sessions.' }
|
|
339
348
|
}
|
|
340
349
|
if (origin.thread === null) {
|
|
341
|
-
return
|
|
350
|
+
return {
|
|
351
|
+
kind: 'block',
|
|
352
|
+
error: 'resolve_review_thread requires replying inside a review thread (no thread on this origin).',
|
|
353
|
+
}
|
|
342
354
|
}
|
|
343
355
|
const result = await router.resolveReviewThread({
|
|
344
356
|
adapter: origin.adapter,
|
|
@@ -346,9 +358,20 @@ async function resolveReviewThreadBeforeReply(
|
|
|
346
358
|
chat: origin.chat,
|
|
347
359
|
rootCommentId: origin.thread,
|
|
348
360
|
})
|
|
349
|
-
if (result.ok) return
|
|
350
|
-
if (result.code === 'no-match') return
|
|
351
|
-
return `could not resolve review thread: ${result.error}`
|
|
361
|
+
if (result.ok) return { kind: 'resolved' }
|
|
362
|
+
if (result.code === 'no-match') return { kind: 'no-match' }
|
|
363
|
+
return { kind: 'block', error: `could not resolve review thread: ${result.error}` }
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// The model asked to resolve but no thread was rooted at this comment. Fenced
|
|
367
|
+
// as a runtime notice (not chat prose) so a persona-rich model reads it as
|
|
368
|
+
// tool feedback and re-targets, rather than replying to it in-character.
|
|
369
|
+
function resolveMissHint(thread: string | null): string {
|
|
370
|
+
return fenceRuntimeNotice(
|
|
371
|
+
`you set resolve_review_thread but no unresolved review thread is rooted at comment ${JSON.stringify(thread)} — ` +
|
|
372
|
+
`your reply posted, but that thread was not resolved (it may be already gone, or the thread id is stale). ` +
|
|
373
|
+
`If a thread should still close, resolve it on the correct thread.`,
|
|
374
|
+
)
|
|
352
375
|
}
|
|
353
376
|
|
|
354
377
|
// Tool results reach the model as USER-role messages (OpenAI / Anthropic
|
|
@@ -116,7 +116,8 @@ export function createChannelSendTool({
|
|
|
116
116
|
'GitHub review threads ONLY — ignored on every other adapter and on a github send that has no `thread`. ' +
|
|
117
117
|
'Set `true` to close out a review thread you authored once you have confirmed the new commits address your concern: pass the thread\'s root comment id as `thread`, an acknowledgement (e.g. "addressed in <sha> — resolving") as `text`, and this flag. ' +
|
|
118
118
|
'This is the post-push close-out path: a `pull_request.synchronize` recheck lists your unresolved threads, and you call this once per addressed thread. ' +
|
|
119
|
-
"Safe by default — the runtime resolves BEFORE posting and ONLY if the thread's root comment is yours, refusing (and blocking the send) on a human reviewer's thread.
|
|
119
|
+
"Safe by default — the runtime resolves BEFORE posting and ONLY if the thread's root comment is yours, refusing (and blocking the send) on a human reviewer's thread. " +
|
|
120
|
+
'REQUIRED on a github PR review-thread send that has both a `thread` and `text`: pass `false` to post your reply while keeping the thread open (not addressed, partial fix, disagreement) — omitting it there is denied. Outside that scope (no `thread`, no `text`, or a non-github adapter) it may be left unset.',
|
|
120
121
|
}),
|
|
121
122
|
),
|
|
122
123
|
}),
|
|
@@ -162,6 +163,28 @@ export function createChannelSendTool({
|
|
|
162
163
|
}
|
|
163
164
|
}
|
|
164
165
|
|
|
166
|
+
// Required-choice guard (mirrors channel_reply): a github PR review-thread
|
|
167
|
+
// send with text must make an explicit resolve_review_thread choice. The
|
|
168
|
+
// `pull_request.synchronize` recheck routes the post-push close-out HERE
|
|
169
|
+
// via channel_send, and the model kept acknowledging "addressed" without
|
|
170
|
+
// the flag, stranding the thread — channel_reply forced the choice but
|
|
171
|
+
// channel_send did not, so the most-used resolve path was the least
|
|
172
|
+
// guarded. A send is always terminal, so there is no continue exemption.
|
|
173
|
+
const missingResolveChoice = missingReviewThreadResolveChoiceError({
|
|
174
|
+
adapter,
|
|
175
|
+
chat: params.chat,
|
|
176
|
+
thread: params.thread ?? null,
|
|
177
|
+
text: bodyText,
|
|
178
|
+
resolveReviewThread: params.resolve_review_thread,
|
|
179
|
+
})
|
|
180
|
+
if (missingResolveChoice) {
|
|
181
|
+
logger.warn(formatChannelToolFailure('channel_send', missingResolveChoice))
|
|
182
|
+
return {
|
|
183
|
+
content: [{ type: 'text' as const, text: `channel_send denied: ${missingResolveChoice}` }],
|
|
184
|
+
details: { ok: false, error: missingResolveChoice },
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
165
188
|
const wantsResolve = params.resolve_review_thread === true
|
|
166
189
|
const falseReceipt = checkFalseReceipt({
|
|
167
190
|
sessionId,
|
|
@@ -207,21 +230,32 @@ export function createChannelSendTool({
|
|
|
207
230
|
// block the acknowledgement so the bot never posts "addressed — resolving"
|
|
208
231
|
// next to a still-open thread. The router enforces that only the bot's own
|
|
209
232
|
// threads can be resolved.
|
|
233
|
+
let resolveMissNotice: string | null = null
|
|
210
234
|
if (wantsResolve) {
|
|
211
|
-
const
|
|
235
|
+
const resolve = await resolveReviewThreadBeforeSend(router, {
|
|
212
236
|
adapter,
|
|
213
237
|
workspace: params.workspace,
|
|
214
238
|
chat: params.chat,
|
|
215
239
|
thread: params.thread ?? null,
|
|
216
240
|
})
|
|
217
|
-
if (
|
|
218
|
-
logger.warn(formatChannelToolFailure('channel_send',
|
|
241
|
+
if (resolve.kind === 'block') {
|
|
242
|
+
logger.warn(formatChannelToolFailure('channel_send', resolve.error))
|
|
219
243
|
return {
|
|
220
|
-
content: [{ type: 'text' as const, text: `channel_send denied: ${
|
|
221
|
-
details: { ok: false, error:
|
|
244
|
+
content: [{ type: 'text' as const, text: `channel_send denied: ${resolve.error}` }],
|
|
245
|
+
details: { ok: false, error: resolve.error },
|
|
222
246
|
}
|
|
223
247
|
}
|
|
224
|
-
|
|
248
|
+
// `no-match`: the thread listed cleanly but nothing is rooted at this
|
|
249
|
+
// comment (gone, or a mispaired id from the bare-id follow-up list). It
|
|
250
|
+
// stays non-blocking so a genuinely-deleted thread's ack still posts,
|
|
251
|
+
// but the resolve did NOT happen — surface it so the model re-targets,
|
|
252
|
+
// and skip the ledger so a phantom resolution can't suppress this
|
|
253
|
+
// thread on a later sweep.
|
|
254
|
+
if (resolve.kind === 'no-match') {
|
|
255
|
+
resolveMissNotice = resolveMissHint(params.thread ?? null)
|
|
256
|
+
} else {
|
|
257
|
+
recordResolvedThreadFromSend(sessionId, params.workspace, params.chat, params.thread ?? null)
|
|
258
|
+
}
|
|
225
259
|
}
|
|
226
260
|
|
|
227
261
|
const result = await router.send({
|
|
@@ -275,6 +309,8 @@ export function createChannelSendTool({
|
|
|
275
309
|
|
|
276
310
|
if (falseReceiptNotice !== null) hints.push(fenceRuntimeNotice(falseReceiptNotice))
|
|
277
311
|
|
|
312
|
+
if (resolveMissNotice !== null) hints.push(resolveMissNotice)
|
|
313
|
+
|
|
278
314
|
return {
|
|
279
315
|
content: [{ type: 'text' as const, text: `${fenceToolResult(receipt)}${hints.join('')}` }],
|
|
280
316
|
details,
|
|
@@ -288,15 +324,37 @@ export function createChannelSendTool({
|
|
|
288
324
|
})
|
|
289
325
|
}
|
|
290
326
|
|
|
327
|
+
function missingReviewThreadResolveChoiceError(input: {
|
|
328
|
+
adapter: AdapterId
|
|
329
|
+
chat: string
|
|
330
|
+
thread: string | null
|
|
331
|
+
text: string | undefined
|
|
332
|
+
resolveReviewThread: boolean | undefined
|
|
333
|
+
}): string {
|
|
334
|
+
const isGithubPrReviewThread = input.adapter === 'github' && /^pr:\d+$/.test(input.chat) && input.thread !== null
|
|
335
|
+
const hasText = input.text !== undefined && input.text.trim() !== ''
|
|
336
|
+
if (!isGithubPrReviewThread || !hasText || input.resolveReviewThread !== undefined) {
|
|
337
|
+
return ''
|
|
338
|
+
}
|
|
339
|
+
return (
|
|
340
|
+
'This is a github PR review-thread send with text, so `resolve_review_thread` is required: ' +
|
|
341
|
+
'set it to `true` when the concern is fixed and this bot-authored thread should be closed (the runtime ' +
|
|
342
|
+
'resolves before posting and only on your own thread), or `false` to leave it open. You omitted it; ' +
|
|
343
|
+
're-call channel_send with an explicit boolean.'
|
|
344
|
+
)
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
type ResolveOutcome = { kind: 'resolved' } | { kind: 'no-match' } | { kind: 'block'; error: string }
|
|
348
|
+
|
|
291
349
|
async function resolveReviewThreadBeforeSend(
|
|
292
350
|
router: ChannelRouter,
|
|
293
351
|
target: { adapter: AdapterId; workspace: string; chat: string; thread: string | null },
|
|
294
|
-
): Promise<
|
|
352
|
+
): Promise<ResolveOutcome> {
|
|
295
353
|
if (target.adapter !== 'github') {
|
|
296
|
-
return 'resolve_review_thread is only supported on github sends.'
|
|
354
|
+
return { kind: 'block', error: 'resolve_review_thread is only supported on github sends.' }
|
|
297
355
|
}
|
|
298
356
|
if (target.thread === null) {
|
|
299
|
-
return 'resolve_review_thread requires a `thread` (the review thread root comment id).'
|
|
357
|
+
return { kind: 'block', error: 'resolve_review_thread requires a `thread` (the review thread root comment id).' }
|
|
300
358
|
}
|
|
301
359
|
const result = await router.resolveReviewThread({
|
|
302
360
|
adapter: target.adapter,
|
|
@@ -304,9 +362,20 @@ async function resolveReviewThreadBeforeSend(
|
|
|
304
362
|
chat: target.chat,
|
|
305
363
|
rootCommentId: target.thread,
|
|
306
364
|
})
|
|
307
|
-
if (result.ok) return
|
|
308
|
-
if (result.code === 'no-match') return
|
|
309
|
-
return `could not resolve review thread: ${result.error}`
|
|
365
|
+
if (result.ok) return { kind: 'resolved' }
|
|
366
|
+
if (result.code === 'no-match') return { kind: 'no-match' }
|
|
367
|
+
return { kind: 'block', error: `could not resolve review thread: ${result.error}` }
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// The model asked to resolve but no thread was rooted at this comment. Fenced
|
|
371
|
+
// as a runtime notice (not chat prose) so a persona-rich model reads it as
|
|
372
|
+
// tool feedback and re-targets, rather than replying to it in-character.
|
|
373
|
+
function resolveMissHint(thread: string | null): string {
|
|
374
|
+
return fenceRuntimeNotice(
|
|
375
|
+
`you set resolve_review_thread but no unresolved review thread is rooted at comment ${JSON.stringify(thread)} — ` +
|
|
376
|
+
`your reply posted, but that thread was not resolved (it may be already gone, or you passed the wrong root ` +
|
|
377
|
+
`comment id). If a different thread should close, re-call channel_send with the correct \`thread\`.`,
|
|
378
|
+
)
|
|
310
379
|
}
|
|
311
380
|
|
|
312
381
|
function recordResolvedThreadFromSend(sessionId: string, workspace: string, chat: string, thread: string | null): void {
|
|
@@ -47,6 +47,11 @@ export type CurlImpersonateRequest = {
|
|
|
47
47
|
// bail early instead of streaming gigabytes.
|
|
48
48
|
maxBytes?: number
|
|
49
49
|
timeoutSeconds?: number
|
|
50
|
+
// Read/write cookies to this jar (curl `-b`+`-c`, same path). Enables a
|
|
51
|
+
// two-request sequence sharing one jar. Cookie plumbing does NOT alter the
|
|
52
|
+
// TLS/HTTP-2/header-order impersonation, so it is exempt from the no-`-H`
|
|
53
|
+
// doctrine above.
|
|
54
|
+
cookieJarPath?: string
|
|
50
55
|
signal?: AbortSignal
|
|
51
56
|
}
|
|
52
57
|
|
|
@@ -56,6 +61,10 @@ export type CurlImpersonateResponse = {
|
|
|
56
61
|
httpStatus: number
|
|
57
62
|
contentType: string
|
|
58
63
|
bytesIn: number
|
|
64
|
+
// Cookie names the server set via Set-Cookie on this response (from curl
|
|
65
|
+
// %{header_json}); lets the policy layer detect a bot-manager 403 without
|
|
66
|
+
// parsing the jar file format.
|
|
67
|
+
setCookieNames: string[]
|
|
59
68
|
}
|
|
60
69
|
|
|
61
70
|
// Specific curl exit codes we map to typed errors. The full list is in
|
|
@@ -113,7 +122,10 @@ export async function curlImpersonate(req: CurlImpersonateRequest): Promise<Curl
|
|
|
113
122
|
// response and replay it; last-match alone fails if the attacker can
|
|
114
123
|
// append text after curl's write-out (they can't, but defense in depth).
|
|
115
124
|
const sentinel = generateSentinel()
|
|
116
|
-
|
|
125
|
+
// %{header_json} is the LAST field so the existing status/url/type/size
|
|
126
|
+
// positions stay byte-compatible with the old parser. It is a single-line
|
|
127
|
+
// JSON object of response headers; we read only Set-Cookie names from it.
|
|
128
|
+
const writeOutTemplate = `${sentinel}%{http_code}\n%{url_effective}\n%{content_type}\n%{size_download}\n%{header_json}\n`
|
|
117
129
|
|
|
118
130
|
const cmd: string[] = [
|
|
119
131
|
curlBinary,
|
|
@@ -156,6 +168,10 @@ export async function curlImpersonate(req: CurlImpersonateRequest): Promise<Curl
|
|
|
156
168
|
cmd.push('--max-filesize', String(req.maxBytes))
|
|
157
169
|
}
|
|
158
170
|
|
|
171
|
+
if (req.cookieJarPath !== undefined) {
|
|
172
|
+
cmd.push('-b', req.cookieJarPath, '-c', req.cookieJarPath)
|
|
173
|
+
}
|
|
174
|
+
|
|
159
175
|
if (req.formFields) {
|
|
160
176
|
for (const field of req.formFields) {
|
|
161
177
|
cmd.push('--data-urlencode', `${field.name}=${field.value}`)
|
|
@@ -248,6 +264,7 @@ function parseCurlOutput(buf: ArrayBuffer, sentinel: string, stderr: string): Cu
|
|
|
248
264
|
const finalUrl = (meta[1] ?? '').trim()
|
|
249
265
|
const contentType = (meta[2] ?? '').trim().toLowerCase()
|
|
250
266
|
const declaredBytes = Number(meta[3]?.trim() ?? '0') || bodyBytes.byteLength
|
|
267
|
+
const setCookieNames = parseSetCookieNames(meta[4] ?? '')
|
|
251
268
|
|
|
252
269
|
const body = new TextDecoder('utf-8', { fatal: false }).decode(bodyBytes)
|
|
253
270
|
|
|
@@ -257,7 +274,29 @@ function parseCurlOutput(buf: ArrayBuffer, sentinel: string, stderr: string): Cu
|
|
|
257
274
|
httpStatus,
|
|
258
275
|
contentType,
|
|
259
276
|
bytesIn: declaredBytes,
|
|
277
|
+
setCookieNames,
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function parseSetCookieNames(headerJson: string): string[] {
|
|
282
|
+
const trimmed = headerJson.trim()
|
|
283
|
+
if (!trimmed) return []
|
|
284
|
+
let parsed: unknown
|
|
285
|
+
try {
|
|
286
|
+
parsed = JSON.parse(trimmed)
|
|
287
|
+
} catch {
|
|
288
|
+
return []
|
|
289
|
+
}
|
|
290
|
+
if (typeof parsed !== 'object' || parsed === null) return []
|
|
291
|
+
const setCookie = (parsed as Record<string, unknown>)['set-cookie']
|
|
292
|
+
const values = Array.isArray(setCookie) ? setCookie : setCookie === undefined ? [] : [setCookie]
|
|
293
|
+
const names: string[] = []
|
|
294
|
+
for (const value of values) {
|
|
295
|
+
if (typeof value !== 'string') continue
|
|
296
|
+
const name = value.split('=', 1)[0]?.trim()
|
|
297
|
+
if (name) names.push(name)
|
|
260
298
|
}
|
|
299
|
+
return names
|
|
261
300
|
}
|
|
262
301
|
|
|
263
302
|
function lastIndexOfBytes(haystack: Uint8Array, needle: Uint8Array): number {
|
|
@@ -21,21 +21,49 @@
|
|
|
21
21
|
// translates non-2xx into a tool-level error message that's useful to the
|
|
22
22
|
// model.
|
|
23
23
|
|
|
24
|
+
import { mkdtemp, rm } from 'node:fs/promises'
|
|
25
|
+
import { tmpdir } from 'node:os'
|
|
26
|
+
import { join } from 'node:path'
|
|
27
|
+
|
|
24
28
|
import {
|
|
25
29
|
CurlImpersonateError,
|
|
26
30
|
curlImpersonate,
|
|
31
|
+
type CurlImpersonateResponse,
|
|
27
32
|
isCurlExitFilesizeExceeded,
|
|
28
33
|
isCurlExitTimeout,
|
|
29
34
|
isCurlImpersonateAvailable,
|
|
30
35
|
} from '../curl-impersonate'
|
|
31
36
|
import { MAX_RESPONSE_BYTES } from './types'
|
|
32
37
|
|
|
38
|
+
export type AntibotWarmup = 'auto' | 'off'
|
|
39
|
+
|
|
40
|
+
export type AntibotWarmupInfo = {
|
|
41
|
+
attempted: boolean
|
|
42
|
+
triggered: boolean
|
|
43
|
+
initialStatus?: number
|
|
44
|
+
initialSetCookieNames?: string[]
|
|
45
|
+
replayStatus?: number
|
|
46
|
+
}
|
|
47
|
+
|
|
33
48
|
export type FetchResult = {
|
|
34
49
|
body: string
|
|
35
50
|
contentType: string
|
|
36
51
|
finalUrl: string
|
|
37
52
|
httpStatus: number
|
|
38
53
|
bytesIn: number
|
|
54
|
+
antibotWarmup?: AntibotWarmupInfo
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Cookie names Akamai Bot Manager sets in the Set-Cookie of its 403 challenge.
|
|
58
|
+
// A 403 carrying one of these is a state-bootstrap handshake (absorb cookies,
|
|
59
|
+
// replay once), NOT an authorization failure — that distinction is what makes
|
|
60
|
+
// the auto-retry safe. Scoped to Akamai on purpose: Cloudflare/DataDome use
|
|
61
|
+
// different challenge semantics (JS/redirects/CAPTCHA) that a blind replay
|
|
62
|
+
// would not satisfy.
|
|
63
|
+
const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
|
|
64
|
+
|
|
65
|
+
function isAntibotWarmup403(response: CurlImpersonateResponse): boolean {
|
|
66
|
+
return response.httpStatus === 403 && response.setCookieNames.some((name) => AKAMAI_BOTMANAGER_COOKIE.test(name))
|
|
39
67
|
}
|
|
40
68
|
|
|
41
69
|
export class WebFetchError extends Error {
|
|
@@ -76,10 +104,11 @@ export async function fetchWithLimits(
|
|
|
76
104
|
url: string,
|
|
77
105
|
timeoutSeconds: number,
|
|
78
106
|
parentSignal?: AbortSignal,
|
|
107
|
+
antibotWarmup: AntibotWarmup = 'auto',
|
|
79
108
|
): Promise<FetchResult> {
|
|
80
109
|
const useImpersonate = !forceFallbackForTest && (await isCurlImpersonateAvailable())
|
|
81
110
|
if (useImpersonate) {
|
|
82
|
-
return fetchWithCurlImpersonate(url, timeoutSeconds, parentSignal)
|
|
111
|
+
return fetchWithCurlImpersonate(url, timeoutSeconds, antibotWarmup, parentSignal)
|
|
83
112
|
}
|
|
84
113
|
return fetchWithBunFetch(url, timeoutSeconds, parentSignal)
|
|
85
114
|
}
|
|
@@ -87,15 +116,66 @@ export async function fetchWithLimits(
|
|
|
87
116
|
async function fetchWithCurlImpersonate(
|
|
88
117
|
url: string,
|
|
89
118
|
timeoutSeconds: number,
|
|
119
|
+
antibotWarmup: AntibotWarmup,
|
|
90
120
|
parentSignal?: AbortSignal,
|
|
91
121
|
): Promise<FetchResult> {
|
|
92
|
-
|
|
122
|
+
if (antibotWarmup === 'off') {
|
|
123
|
+
const response = await runCurl(url, timeoutSeconds, undefined, parentSignal)
|
|
124
|
+
return toFetchResult(url, response)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Warmup needs a jar shared across both requests: request 1 (`-c`) captures
|
|
128
|
+
// the bot-manager cookies the 403 sets; request 2 (`-b`) replays them.
|
|
129
|
+
const jarDir = await mkdtemp(join(tmpdir(), 'typeclaw-webfetch-jar-'))
|
|
130
|
+
const jarPath = join(jarDir, 'cookies.txt')
|
|
131
|
+
const startedAt = Date.now()
|
|
93
132
|
try {
|
|
94
|
-
|
|
133
|
+
const first = await runCurl(url, timeoutSeconds, jarPath, parentSignal)
|
|
134
|
+
if (!isAntibotWarmup403(first)) {
|
|
135
|
+
return toFetchResult(url, first, { attempted: true, triggered: false })
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// `timeoutSeconds` is the budget for the WHOLE web_fetch call, not per
|
|
139
|
+
// attempt: charge the replay only the time the first request left, so a
|
|
140
|
+
// warmed fetch can't run for ~2x the requested timeout. If the first
|
|
141
|
+
// request already spent the budget, surface its 403 instead of forcing a
|
|
142
|
+
// zero-time replay that would just time out.
|
|
143
|
+
const remaining = timeoutSeconds - (Date.now() - startedAt) / 1000
|
|
144
|
+
if (remaining < 1) {
|
|
145
|
+
return toFetchResult(url, first, {
|
|
146
|
+
attempted: true,
|
|
147
|
+
triggered: false,
|
|
148
|
+
initialStatus: first.httpStatus,
|
|
149
|
+
initialSetCookieNames: first.setCookieNames,
|
|
150
|
+
})
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const replay = await runCurl(url, Math.floor(remaining), jarPath, parentSignal)
|
|
154
|
+
return toFetchResult(url, replay, {
|
|
155
|
+
attempted: true,
|
|
156
|
+
triggered: true,
|
|
157
|
+
initialStatus: first.httpStatus,
|
|
158
|
+
initialSetCookieNames: first.setCookieNames,
|
|
159
|
+
replayStatus: replay.httpStatus,
|
|
160
|
+
})
|
|
161
|
+
} finally {
|
|
162
|
+
await rm(jarDir, { recursive: true, force: true })
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async function runCurl(
|
|
167
|
+
url: string,
|
|
168
|
+
timeoutSeconds: number,
|
|
169
|
+
cookieJarPath: string | undefined,
|
|
170
|
+
parentSignal?: AbortSignal,
|
|
171
|
+
): Promise<CurlImpersonateResponse> {
|
|
172
|
+
try {
|
|
173
|
+
return await curlImpersonate({
|
|
95
174
|
url,
|
|
96
175
|
method: 'GET',
|
|
97
176
|
timeoutSeconds,
|
|
98
177
|
maxBytes: MAX_RESPONSE_BYTES,
|
|
178
|
+
cookieJarPath,
|
|
99
179
|
signal: parentSignal,
|
|
100
180
|
})
|
|
101
181
|
} catch (error) {
|
|
@@ -114,7 +194,9 @@ async function fetchWithCurlImpersonate(
|
|
|
114
194
|
const message = error instanceof Error ? error.message : String(error)
|
|
115
195
|
throw new WebFetchError(`Fetch failed: ${message}`)
|
|
116
196
|
}
|
|
197
|
+
}
|
|
117
198
|
|
|
199
|
+
function toFetchResult(url: string, response: CurlImpersonateResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
|
|
118
200
|
if (response.httpStatus < 200 || response.httpStatus >= 300) {
|
|
119
201
|
throw new WebFetchError(`Fetch failed: HTTP ${response.httpStatus}`)
|
|
120
202
|
}
|
|
@@ -132,6 +214,7 @@ async function fetchWithCurlImpersonate(
|
|
|
132
214
|
finalUrl: response.finalUrl || url,
|
|
133
215
|
httpStatus: response.httpStatus,
|
|
134
216
|
bytesIn: bodyByteLength,
|
|
217
|
+
...(antibotWarmup ? { antibotWarmup } : {}),
|
|
135
218
|
}
|
|
136
219
|
}
|
|
137
220
|
|
|
@@ -27,8 +27,11 @@ export const webFetchTool = defineTool({
|
|
|
27
27
|
'If `spawn_subagent` is available to you, PREFER delegating to the `scout` subagent by default: spawn it whenever you expect more than one fetch, an "across multiple sources" task, or any search-then-fetch loop. Scout runs the noisy fetching in its own context window and returns a distilled, citation-backed answer, keeping bulky page bodies out of yours. Only call this tool directly for a single known URL whose content you will cite immediately — or whenever you cannot spawn subagents (e.g. you are yourself a subagent), in which case fetch here. ' +
|
|
28
28
|
'Outbound requests impersonate Chrome 136 at the TLS, HTTP/2, and header layers ' +
|
|
29
29
|
'(via curl-impersonate), which helps with TLS/header fingerprint gates on sites behind Cloudflare/Akamai. ' +
|
|
30
|
-
'
|
|
31
|
-
'
|
|
30
|
+
'For Akamai Bot Manager specifically, it auto-warms the session: a first-request 403 that sets bot-manager ' +
|
|
31
|
+
'cookies (_abck/bm_sz/…) is absorbed and replayed once, which returns 200 on many Akamai-fronted sites ' +
|
|
32
|
+
'(disable with `antibotWarmup: "off"`). It still does NOT solve JavaScript challenges, behavioural ' +
|
|
33
|
+
'fingerprinting (mouse/scroll/timing), interactive CAPTCHAs, or IP-reputation blocks — a 403 from those ' +
|
|
34
|
+
'layers is expected and unrecoverable from this tool. ' +
|
|
32
35
|
'Strategy guide:\n' +
|
|
33
36
|
'- "readability": extract article content as markdown (blogs, docs, news). Default for HTML.\n' +
|
|
34
37
|
'- "jq": query JSON APIs (npm registry, GitHub API). Pass `query` (e.g. ".items[].name").\n' +
|
|
@@ -62,6 +65,15 @@ export const webFetchTool = defineTool({
|
|
|
62
65
|
maximum: MAX_TIMEOUT_SECONDS,
|
|
63
66
|
}),
|
|
64
67
|
),
|
|
68
|
+
antibotWarmup: Type.Optional(
|
|
69
|
+
Type.Union([Type.Literal('auto'), Type.Literal('off')], {
|
|
70
|
+
description:
|
|
71
|
+
'Akamai Bot Manager warmup. "auto" (default): if the first request is a 403 that sets bot-manager cookies ' +
|
|
72
|
+
'(_abck/bm_sz/…), absorb them and replay once — many Akamai-fronted sites (e.g. Coupang) only serve content ' +
|
|
73
|
+
'on the second, cookie-bearing request. "off": disable the replay; a bot-manager 403 then surfaces as the ' +
|
|
74
|
+
'normal HTTP 403 error (use to confirm the block without the warmup).',
|
|
75
|
+
}),
|
|
76
|
+
),
|
|
65
77
|
}),
|
|
66
78
|
|
|
67
79
|
async execute(_toolCallId, params, signal) {
|
|
@@ -80,7 +92,7 @@ export const webFetchTool = defineTool({
|
|
|
80
92
|
|
|
81
93
|
let response
|
|
82
94
|
try {
|
|
83
|
-
response = await fetchWithLimits(normalizedUrl, timeout, signal)
|
|
95
|
+
response = await fetchWithLimits(normalizedUrl, timeout, signal, params.antibotWarmup ?? 'auto')
|
|
84
96
|
} catch (error) {
|
|
85
97
|
const message = error instanceof Error ? error.message : String(error)
|
|
86
98
|
return errorResult(normalizedUrl, message, { startedAt, finalUrl: normalizedUrl })
|
|
@@ -141,6 +153,16 @@ export const webFetchTool = defineTool({
|
|
|
141
153
|
bytesOut: byteLength(capped.text),
|
|
142
154
|
truncated: capped.truncated,
|
|
143
155
|
durationMs: Date.now() - startedAt,
|
|
156
|
+
...(response.antibotWarmup?.triggered
|
|
157
|
+
? {
|
|
158
|
+
antibotWarmup: {
|
|
159
|
+
triggered: true,
|
|
160
|
+
initialStatus: response.antibotWarmup.initialStatus,
|
|
161
|
+
initialSetCookieNames: response.antibotWarmup.initialSetCookieNames,
|
|
162
|
+
replayStatus: response.antibotWarmup.replayStatus,
|
|
163
|
+
},
|
|
164
|
+
}
|
|
165
|
+
: {}),
|
|
144
166
|
}
|
|
145
167
|
|
|
146
168
|
return {
|
|
@@ -153,6 +175,7 @@ export const webFetchTool = defineTool({
|
|
|
153
175
|
type WebFetchParams = {
|
|
154
176
|
url: string
|
|
155
177
|
strategy?: CompactionStrategy
|
|
178
|
+
antibotWarmup?: 'auto' | 'off'
|
|
156
179
|
query?: string
|
|
157
180
|
selector?: string
|
|
158
181
|
pattern?: string
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
export type CompactionStrategy = 'readability' | 'jq' | 'selector' | 'grep' | 'snapshot' | 'raw'
|
|
2
2
|
|
|
3
|
+
export type AntibotWarmupDetails = {
|
|
4
|
+
triggered: boolean
|
|
5
|
+
initialStatus?: number
|
|
6
|
+
initialSetCookieNames?: string[]
|
|
7
|
+
replayStatus?: number
|
|
8
|
+
}
|
|
9
|
+
|
|
3
10
|
export type WebFetchDetails = {
|
|
4
11
|
url: string
|
|
5
12
|
finalUrl: string
|
|
@@ -11,6 +18,7 @@ export type WebFetchDetails = {
|
|
|
11
18
|
bytesOut: number
|
|
12
19
|
truncated: boolean
|
|
13
20
|
durationMs: number
|
|
21
|
+
antibotWarmup?: AntibotWarmupDetails
|
|
14
22
|
error?: boolean
|
|
15
23
|
message?: string
|
|
16
24
|
}
|