typeclaw 0.43.1 → 0.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +1 -1
  2. package/auth.schema.json +92 -0
  3. package/package.json +7 -6
  4. package/secrets.schema.json +92 -0
  5. package/src/agent/index.ts +25 -2
  6. package/src/agent/model-fallback.ts +59 -40
  7. package/src/agent/multimodal/look-at.ts +29 -10
  8. package/src/agent/plugin-tools.ts +9 -4
  9. package/src/agent/provider-error.ts +71 -18
  10. package/src/agent/retry-same-ref.ts +139 -0
  11. package/src/agent/session-origin.ts +1 -0
  12. package/src/agent/subagents.ts +6 -2
  13. package/src/agent/tools/channel-edit.ts +144 -0
  14. package/src/agent/tools/channel-fetch-attachment.ts +51 -8
  15. package/src/agent/tools/channel-reply.ts +41 -18
  16. package/src/agent/tools/channel-send.ts +82 -13
  17. package/src/agent/tools/curl-impersonate.ts +40 -1
  18. package/src/agent/tools/webfetch/fetch.ts +86 -3
  19. package/src/agent/tools/webfetch/tool.ts +26 -3
  20. package/src/agent/tools/webfetch/types.ts +8 -0
  21. package/src/agent/turn-runner.ts +65 -36
  22. package/src/bundled-plugins/agent-browser/skills/agent-browser/SKILL.md +28 -11
  23. package/src/bundled-plugins/github-cli-auth/gh-review-detect.ts +101 -1
  24. package/src/bundled-plugins/github-cli-auth/review-recorder.ts +44 -3
  25. package/src/bundled-plugins/guard/policies/managed-config.ts +20 -4
  26. package/src/bundled-plugins/memory/README.md +9 -9
  27. package/src/bundled-plugins/memory/index.ts +40 -9
  28. package/src/bundled-plugins/reviewer/skills/code-review.ts +22 -2
  29. package/src/channels/adapters/discord-bot-edit.ts +49 -0
  30. package/src/channels/adapters/discord-bot.ts +5 -0
  31. package/src/channels/adapters/discord-edit.ts +51 -0
  32. package/src/channels/adapters/discord.ts +4 -0
  33. package/src/channels/adapters/github/inbound.ts +20 -14
  34. package/src/channels/adapters/github/review-thread-resolver.ts +38 -4
  35. package/src/channels/adapters/slack-bot-edit.ts +58 -0
  36. package/src/channels/adapters/slack-bot.ts +18 -6
  37. package/src/channels/adapters/slack-edit.ts +47 -0
  38. package/src/channels/adapters/slack.ts +5 -0
  39. package/src/channels/adapters/teams-classify.ts +144 -0
  40. package/src/channels/adapters/teams-edit.ts +61 -0
  41. package/src/channels/adapters/teams-id-token.ts +112 -0
  42. package/src/channels/adapters/teams-key.ts +32 -0
  43. package/src/channels/adapters/teams.ts +523 -0
  44. package/src/channels/adapters/telegram-bot-edit.ts +51 -0
  45. package/src/channels/adapters/telegram-bot.ts +5 -0
  46. package/src/channels/adapters/webex-bot.ts +5 -0
  47. package/src/channels/adapters/webex-edit.ts +47 -0
  48. package/src/channels/adapters/webex.ts +5 -0
  49. package/src/channels/continuation-willingness.ts +227 -35
  50. package/src/channels/github-review-turn-ledger.ts +43 -0
  51. package/src/channels/index.ts +1 -1
  52. package/src/channels/manager.ts +54 -1
  53. package/src/channels/router.ts +482 -53
  54. package/src/channels/schema.ts +33 -0
  55. package/src/channels/types.ts +36 -0
  56. package/src/cli/channel.ts +92 -4
  57. package/src/cli/hostd.ts +41 -0
  58. package/src/cli/init.ts +58 -1
  59. package/src/config/channels-mutation.ts +9 -1
  60. package/src/cron/index.ts +16 -4
  61. package/src/cron/reloadable.ts +25 -7
  62. package/src/cron/scheduler.ts +4 -0
  63. package/src/cron/schema.ts +143 -35
  64. package/src/doctor/channel-checks.ts +27 -0
  65. package/src/hostd/daemon.ts +23 -3
  66. package/src/hostd/protocol.ts +2 -0
  67. package/src/hostd/teams-renewal-manager.ts +212 -0
  68. package/src/init/checkpoint.ts +1 -1
  69. package/src/init/index.ts +68 -1
  70. package/src/init/run-owner-claim.ts +1 -0
  71. package/src/init/teams-auth.ts +111 -0
  72. package/src/inspect/label.ts +4 -0
  73. package/src/permissions/match-rule.ts +11 -1
  74. package/src/permissions/resolve.ts +1 -0
  75. package/src/role-claim/match-rule.ts +2 -1
  76. package/src/run/index.ts +35 -8
  77. package/src/run/llm-fetch-observer.ts +27 -5
  78. package/src/sandbox/build.ts +12 -0
  79. package/src/sandbox/policy.ts +13 -0
  80. package/src/sandbox/writable-zones.ts +46 -10
  81. package/src/secrets/schema.ts +32 -0
  82. package/src/secrets/teams-renewal.ts +229 -0
  83. package/src/secrets/teams-store.ts +108 -0
  84. package/src/server/command-runner.ts +2 -1
  85. package/src/server/index.ts +2 -1
  86. package/typeclaw.schema.json +163 -3
@@ -210,15 +210,22 @@ export function createChannelReplyTool({
210
210
  // flag fixes). Resolve-failure blocks the reply so the agent never posts
211
211
  // a "looks resolved" ack next to a still-open thread; the router enforces
212
212
  // that only the bot's own threads can be resolved.
213
+ let resolveMissNotice: string | null = null
213
214
  if (params.resolve_review_thread === true) {
214
- const resolveError = await resolveReviewThreadBeforeReply(router, origin)
215
- if (resolveError !== null) {
216
- logger.warn(formatChannelToolFailure('channel_reply', resolveError))
215
+ const resolve = await resolveReviewThreadBeforeReply(router, origin)
216
+ if (resolve.kind === 'block') {
217
+ logger.warn(formatChannelToolFailure('channel_reply', resolve.error))
217
218
  return {
218
- content: [{ type: 'text' as const, text: `channel_reply denied: ${resolveError}` }],
219
- details: { ok: false, error: resolveError },
219
+ content: [{ type: 'text' as const, text: `channel_reply denied: ${resolve.error}` }],
220
+ details: { ok: false, error: resolve.error },
220
221
  }
221
222
  }
223
+ // `no-match` stays non-blocking (the thread may be genuinely gone) but
224
+ // the resolve did NOT run, so tell the model instead of posting a clean
225
+ // receipt that hides the miss. Mirrors channel_send.
226
+ if (resolve.kind === 'no-match') {
227
+ resolveMissNotice = resolveMissHint(origin.thread)
228
+ }
222
229
  }
223
230
 
224
231
  const result = await router.send({
@@ -287,8 +294,9 @@ export function createChannelReplyTool({
287
294
  // intentionally weaker and is safe ONLY because denials carry no echoed
288
295
  // prose; the success result does, and the weak prefix let Kimi loop.
289
296
  const warnNote = falseReceiptNotice !== null ? fenceRuntimeNotice(falseReceiptNotice) : ''
297
+ const missNote = resolveMissNotice ?? ''
290
298
  return {
291
- content: [{ type: 'text' as const, text: `${fenceToolResult(receipt)}${hint}${warnNote}` }],
299
+ content: [{ type: 'text' as const, text: `${fenceToolResult(receipt)}${hint}${warnNote}${missNote}` }],
292
300
  details,
293
301
  }
294
302
  }
@@ -324,21 +332,25 @@ function missingReviewThreadResolveChoiceError(input: {
324
332
  )
325
333
  }
326
334
 
327
- // Returns an error string when the resolve should block the reply, or null
328
- // when it's safe to proceed. Only `no-match` (the thread is already gone, so
329
- // there's nothing to close) joins success as non-blocking; every hard failure
330
- // — wrong author, permission denial, HTTP 404 on a misdirected lookup,
331
- // transient API error — blocks, so the agent never claims a thread is settled
332
- // when the resolve did not actually run.
335
+ // `block` when the resolve should stop the reply, `no-match` when the thread is
336
+ // gone (non-blocking — the reply posts but the caller warns), `resolved` on
337
+ // success. Every hard failure — wrong author, permission denial, HTTP 404 on a
338
+ // misdirected lookup, transient API error — blocks, so the agent never claims a
339
+ // thread is settled when the resolve did not actually run.
340
+ type ResolveOutcome = { kind: 'resolved' } | { kind: 'no-match' } | { kind: 'block'; error: string }
341
+
333
342
  async function resolveReviewThreadBeforeReply(
334
343
  router: ChannelRouter,
335
344
  origin: ChannelReplyOrigin,
336
- ): Promise<string | null> {
345
+ ): Promise<ResolveOutcome> {
337
346
  if (origin.adapter !== 'github') {
338
- return 'resolve_review_thread is only supported on github sessions.'
347
+ return { kind: 'block', error: 'resolve_review_thread is only supported on github sessions.' }
339
348
  }
340
349
  if (origin.thread === null) {
341
- return 'resolve_review_thread requires replying inside a review thread (no thread on this origin).'
350
+ return {
351
+ kind: 'block',
352
+ error: 'resolve_review_thread requires replying inside a review thread (no thread on this origin).',
353
+ }
342
354
  }
343
355
  const result = await router.resolveReviewThread({
344
356
  adapter: origin.adapter,
@@ -346,9 +358,20 @@ async function resolveReviewThreadBeforeReply(
346
358
  chat: origin.chat,
347
359
  rootCommentId: origin.thread,
348
360
  })
349
- if (result.ok) return null
350
- if (result.code === 'no-match') return null
351
- return `could not resolve review thread: ${result.error}`
361
+ if (result.ok) return { kind: 'resolved' }
362
+ if (result.code === 'no-match') return { kind: 'no-match' }
363
+ return { kind: 'block', error: `could not resolve review thread: ${result.error}` }
364
+ }
365
+
366
+ // The model asked to resolve but no thread was rooted at this comment. Fenced
367
+ // as a runtime notice (not chat prose) so a persona-rich model reads it as
368
+ // tool feedback and re-targets, rather than replying to it in-character.
369
+ function resolveMissHint(thread: string | null): string {
370
+ return fenceRuntimeNotice(
371
+ `you set resolve_review_thread but no unresolved review thread is rooted at comment ${JSON.stringify(thread)} — ` +
372
+ `your reply posted, but that thread was not resolved (it may be already gone, or the thread id is stale). ` +
373
+ `If a thread should still close, resolve it on the correct thread.`,
374
+ )
352
375
  }
353
376
 
354
377
  // Tool results reach the model as USER-role messages (OpenAI / Anthropic
@@ -116,7 +116,8 @@ export function createChannelSendTool({
116
116
  'GitHub review threads ONLY — ignored on every other adapter and on a github send that has no `thread`. ' +
117
117
  'Set `true` to close out a review thread you authored once you have confirmed the new commits address your concern: pass the thread\'s root comment id as `thread`, an acknowledgement (e.g. "addressed in <sha> — resolving") as `text`, and this flag. ' +
118
118
  'This is the post-push close-out path: a `pull_request.synchronize` recheck lists your unresolved threads, and you call this once per addressed thread. ' +
119
- "Safe by default — the runtime resolves BEFORE posting and ONLY if the thread's root comment is yours, refusing (and blocking the send) on a human reviewer's thread. Leave it unset to keep the thread open (not addressed, partial fix, disagreement).",
119
+ "Safe by default — the runtime resolves BEFORE posting and ONLY if the thread's root comment is yours, refusing (and blocking the send) on a human reviewer's thread. " +
120
+ 'REQUIRED on a github PR review-thread send that has both a `thread` and `text`: pass `false` to post your reply while keeping the thread open (not addressed, partial fix, disagreement) — omitting it there is denied. Outside that scope (no `thread`, no `text`, or a non-github adapter) it may be left unset.',
120
121
  }),
121
122
  ),
122
123
  }),
@@ -162,6 +163,28 @@ export function createChannelSendTool({
162
163
  }
163
164
  }
164
165
 
166
+ // Required-choice guard (mirrors channel_reply): a github PR review-thread
167
+ // send with text must make an explicit resolve_review_thread choice. The
168
+ // `pull_request.synchronize` recheck routes the post-push close-out HERE
169
+ // via channel_send, and the model kept acknowledging "addressed" without
170
+ // the flag, stranding the thread — channel_reply forced the choice but
171
+ // channel_send did not, so the most-used resolve path was the least
172
+ // guarded. A send is always terminal, so there is no continue exemption.
173
+ const missingResolveChoice = missingReviewThreadResolveChoiceError({
174
+ adapter,
175
+ chat: params.chat,
176
+ thread: params.thread ?? null,
177
+ text: bodyText,
178
+ resolveReviewThread: params.resolve_review_thread,
179
+ })
180
+ if (missingResolveChoice) {
181
+ logger.warn(formatChannelToolFailure('channel_send', missingResolveChoice))
182
+ return {
183
+ content: [{ type: 'text' as const, text: `channel_send denied: ${missingResolveChoice}` }],
184
+ details: { ok: false, error: missingResolveChoice },
185
+ }
186
+ }
187
+
165
188
  const wantsResolve = params.resolve_review_thread === true
166
189
  const falseReceipt = checkFalseReceipt({
167
190
  sessionId,
@@ -207,21 +230,32 @@ export function createChannelSendTool({
207
230
  // block the acknowledgement so the bot never posts "addressed — resolving"
208
231
  // next to a still-open thread. The router enforces that only the bot's own
209
232
  // threads can be resolved.
233
+ let resolveMissNotice: string | null = null
210
234
  if (wantsResolve) {
211
- const resolveError = await resolveReviewThreadBeforeSend(router, {
235
+ const resolve = await resolveReviewThreadBeforeSend(router, {
212
236
  adapter,
213
237
  workspace: params.workspace,
214
238
  chat: params.chat,
215
239
  thread: params.thread ?? null,
216
240
  })
217
- if (resolveError !== null) {
218
- logger.warn(formatChannelToolFailure('channel_send', resolveError))
241
+ if (resolve.kind === 'block') {
242
+ logger.warn(formatChannelToolFailure('channel_send', resolve.error))
219
243
  return {
220
- content: [{ type: 'text' as const, text: `channel_send denied: ${resolveError}` }],
221
- details: { ok: false, error: resolveError },
244
+ content: [{ type: 'text' as const, text: `channel_send denied: ${resolve.error}` }],
245
+ details: { ok: false, error: resolve.error },
222
246
  }
223
247
  }
224
- recordResolvedThreadFromSend(sessionId, params.workspace, params.chat, params.thread ?? null)
248
+ // `no-match`: the thread listed cleanly but nothing is rooted at this
249
+ // comment (gone, or a mispaired id from the bare-id follow-up list). It
250
+ // stays non-blocking so a genuinely-deleted thread's ack still posts,
251
+ // but the resolve did NOT happen — surface it so the model re-targets,
252
+ // and skip the ledger so a phantom resolution can't suppress this
253
+ // thread on a later sweep.
254
+ if (resolve.kind === 'no-match') {
255
+ resolveMissNotice = resolveMissHint(params.thread ?? null)
256
+ } else {
257
+ recordResolvedThreadFromSend(sessionId, params.workspace, params.chat, params.thread ?? null)
258
+ }
225
259
  }
226
260
 
227
261
  const result = await router.send({
@@ -275,6 +309,8 @@ export function createChannelSendTool({
275
309
 
276
310
  if (falseReceiptNotice !== null) hints.push(fenceRuntimeNotice(falseReceiptNotice))
277
311
 
312
+ if (resolveMissNotice !== null) hints.push(resolveMissNotice)
313
+
278
314
  return {
279
315
  content: [{ type: 'text' as const, text: `${fenceToolResult(receipt)}${hints.join('')}` }],
280
316
  details,
@@ -288,15 +324,37 @@ export function createChannelSendTool({
288
324
  })
289
325
  }
290
326
 
327
+ function missingReviewThreadResolveChoiceError(input: {
328
+ adapter: AdapterId
329
+ chat: string
330
+ thread: string | null
331
+ text: string | undefined
332
+ resolveReviewThread: boolean | undefined
333
+ }): string {
334
+ const isGithubPrReviewThread = input.adapter === 'github' && /^pr:\d+$/.test(input.chat) && input.thread !== null
335
+ const hasText = input.text !== undefined && input.text.trim() !== ''
336
+ if (!isGithubPrReviewThread || !hasText || input.resolveReviewThread !== undefined) {
337
+ return ''
338
+ }
339
+ return (
340
+ 'This is a github PR review-thread send with text, so `resolve_review_thread` is required: ' +
341
+ 'set it to `true` when the concern is fixed and this bot-authored thread should be closed (the runtime ' +
342
+ 'resolves before posting and only on your own thread), or `false` to leave it open. You omitted it; ' +
343
+ 're-call channel_send with an explicit boolean.'
344
+ )
345
+ }
346
+
347
+ type ResolveOutcome = { kind: 'resolved' } | { kind: 'no-match' } | { kind: 'block'; error: string }
348
+
291
349
  async function resolveReviewThreadBeforeSend(
292
350
  router: ChannelRouter,
293
351
  target: { adapter: AdapterId; workspace: string; chat: string; thread: string | null },
294
- ): Promise<string | null> {
352
+ ): Promise<ResolveOutcome> {
295
353
  if (target.adapter !== 'github') {
296
- return 'resolve_review_thread is only supported on github sends.'
354
+ return { kind: 'block', error: 'resolve_review_thread is only supported on github sends.' }
297
355
  }
298
356
  if (target.thread === null) {
299
- return 'resolve_review_thread requires a `thread` (the review thread root comment id).'
357
+ return { kind: 'block', error: 'resolve_review_thread requires a `thread` (the review thread root comment id).' }
300
358
  }
301
359
  const result = await router.resolveReviewThread({
302
360
  adapter: target.adapter,
@@ -304,9 +362,20 @@ async function resolveReviewThreadBeforeSend(
304
362
  chat: target.chat,
305
363
  rootCommentId: target.thread,
306
364
  })
307
- if (result.ok) return null
308
- if (result.code === 'no-match') return null
309
- return `could not resolve review thread: ${result.error}`
365
+ if (result.ok) return { kind: 'resolved' }
366
+ if (result.code === 'no-match') return { kind: 'no-match' }
367
+ return { kind: 'block', error: `could not resolve review thread: ${result.error}` }
368
+ }
369
+
370
+ // The model asked to resolve but no thread was rooted at this comment. Fenced
371
+ // as a runtime notice (not chat prose) so a persona-rich model reads it as
372
+ // tool feedback and re-targets, rather than replying to it in-character.
373
+ function resolveMissHint(thread: string | null): string {
374
+ return fenceRuntimeNotice(
375
+ `you set resolve_review_thread but no unresolved review thread is rooted at comment ${JSON.stringify(thread)} — ` +
376
+ `your reply posted, but that thread was not resolved (it may be already gone, or you passed the wrong root ` +
377
+ `comment id). If a different thread should close, re-call channel_send with the correct \`thread\`.`,
378
+ )
310
379
  }
311
380
 
312
381
  function recordResolvedThreadFromSend(sessionId: string, workspace: string, chat: string, thread: string | null): void {
@@ -47,6 +47,11 @@ export type CurlImpersonateRequest = {
47
47
  // bail early instead of streaming gigabytes.
48
48
  maxBytes?: number
49
49
  timeoutSeconds?: number
50
+ // Read/write cookies to this jar (curl `-b`+`-c`, same path). Enables a
51
+ // two-request sequence sharing one jar. Cookie plumbing does NOT alter the
52
+ // TLS/HTTP-2/header-order impersonation, so it is exempt from the no-`-H`
53
+ // doctrine above.
54
+ cookieJarPath?: string
50
55
  signal?: AbortSignal
51
56
  }
52
57
 
@@ -56,6 +61,10 @@ export type CurlImpersonateResponse = {
56
61
  httpStatus: number
57
62
  contentType: string
58
63
  bytesIn: number
64
+ // Cookie names the server set via Set-Cookie on this response (from curl
65
+ // %{header_json}); lets the policy layer detect a bot-manager 403 without
66
+ // parsing the jar file format.
67
+ setCookieNames: string[]
59
68
  }
60
69
 
61
70
  // Specific curl exit codes we map to typed errors. The full list is in
@@ -113,7 +122,10 @@ export async function curlImpersonate(req: CurlImpersonateRequest): Promise<Curl
113
122
  // response and replay it; last-match alone fails if the attacker can
114
123
  // append text after curl's write-out (they can't, but defense in depth).
115
124
  const sentinel = generateSentinel()
116
- const writeOutTemplate = `${sentinel}%{http_code}\n%{url_effective}\n%{content_type}\n%{size_download}\n`
125
+ // %{header_json} is the LAST field so the existing status/url/type/size
126
+ // positions stay byte-compatible with the old parser. It is a single-line
127
+ // JSON object of response headers; we read only Set-Cookie names from it.
128
+ const writeOutTemplate = `${sentinel}%{http_code}\n%{url_effective}\n%{content_type}\n%{size_download}\n%{header_json}\n`
117
129
 
118
130
  const cmd: string[] = [
119
131
  curlBinary,
@@ -156,6 +168,10 @@ export async function curlImpersonate(req: CurlImpersonateRequest): Promise<Curl
156
168
  cmd.push('--max-filesize', String(req.maxBytes))
157
169
  }
158
170
 
171
+ if (req.cookieJarPath !== undefined) {
172
+ cmd.push('-b', req.cookieJarPath, '-c', req.cookieJarPath)
173
+ }
174
+
159
175
  if (req.formFields) {
160
176
  for (const field of req.formFields) {
161
177
  cmd.push('--data-urlencode', `${field.name}=${field.value}`)
@@ -248,6 +264,7 @@ function parseCurlOutput(buf: ArrayBuffer, sentinel: string, stderr: string): Cu
248
264
  const finalUrl = (meta[1] ?? '').trim()
249
265
  const contentType = (meta[2] ?? '').trim().toLowerCase()
250
266
  const declaredBytes = Number(meta[3]?.trim() ?? '0') || bodyBytes.byteLength
267
+ const setCookieNames = parseSetCookieNames(meta[4] ?? '')
251
268
 
252
269
  const body = new TextDecoder('utf-8', { fatal: false }).decode(bodyBytes)
253
270
 
@@ -257,7 +274,29 @@ function parseCurlOutput(buf: ArrayBuffer, sentinel: string, stderr: string): Cu
257
274
  httpStatus,
258
275
  contentType,
259
276
  bytesIn: declaredBytes,
277
+ setCookieNames,
278
+ }
279
+ }
280
+
281
+ function parseSetCookieNames(headerJson: string): string[] {
282
+ const trimmed = headerJson.trim()
283
+ if (!trimmed) return []
284
+ let parsed: unknown
285
+ try {
286
+ parsed = JSON.parse(trimmed)
287
+ } catch {
288
+ return []
289
+ }
290
+ if (typeof parsed !== 'object' || parsed === null) return []
291
+ const setCookie = (parsed as Record<string, unknown>)['set-cookie']
292
+ const values = Array.isArray(setCookie) ? setCookie : setCookie === undefined ? [] : [setCookie]
293
+ const names: string[] = []
294
+ for (const value of values) {
295
+ if (typeof value !== 'string') continue
296
+ const name = value.split('=', 1)[0]?.trim()
297
+ if (name) names.push(name)
260
298
  }
299
+ return names
261
300
  }
262
301
 
263
302
  function lastIndexOfBytes(haystack: Uint8Array, needle: Uint8Array): number {
@@ -21,21 +21,49 @@
21
21
  // translates non-2xx into a tool-level error message that's useful to the
22
22
  // model.
23
23
 
24
+ import { mkdtemp, rm } from 'node:fs/promises'
25
+ import { tmpdir } from 'node:os'
26
+ import { join } from 'node:path'
27
+
24
28
  import {
25
29
  CurlImpersonateError,
26
30
  curlImpersonate,
31
+ type CurlImpersonateResponse,
27
32
  isCurlExitFilesizeExceeded,
28
33
  isCurlExitTimeout,
29
34
  isCurlImpersonateAvailable,
30
35
  } from '../curl-impersonate'
31
36
  import { MAX_RESPONSE_BYTES } from './types'
32
37
 
38
+ export type AntibotWarmup = 'auto' | 'off'
39
+
40
+ export type AntibotWarmupInfo = {
41
+ attempted: boolean
42
+ triggered: boolean
43
+ initialStatus?: number
44
+ initialSetCookieNames?: string[]
45
+ replayStatus?: number
46
+ }
47
+
33
48
  export type FetchResult = {
34
49
  body: string
35
50
  contentType: string
36
51
  finalUrl: string
37
52
  httpStatus: number
38
53
  bytesIn: number
54
+ antibotWarmup?: AntibotWarmupInfo
55
+ }
56
+
57
+ // Cookie names Akamai Bot Manager sets in the Set-Cookie of its 403 challenge.
58
+ // A 403 carrying one of these is a state-bootstrap handshake (absorb cookies,
59
+ // replay once), NOT an authorization failure — that distinction is what makes
60
+ // the auto-retry safe. Scoped to Akamai on purpose: Cloudflare/DataDome use
61
+ // different challenge semantics (JS/redirects/CAPTCHA) that a blind replay
62
+ // would not satisfy.
63
+ const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
64
+
65
+ function isAntibotWarmup403(response: CurlImpersonateResponse): boolean {
66
+ return response.httpStatus === 403 && response.setCookieNames.some((name) => AKAMAI_BOTMANAGER_COOKIE.test(name))
39
67
  }
40
68
 
41
69
  export class WebFetchError extends Error {
@@ -76,10 +104,11 @@ export async function fetchWithLimits(
76
104
  url: string,
77
105
  timeoutSeconds: number,
78
106
  parentSignal?: AbortSignal,
107
+ antibotWarmup: AntibotWarmup = 'auto',
79
108
  ): Promise<FetchResult> {
80
109
  const useImpersonate = !forceFallbackForTest && (await isCurlImpersonateAvailable())
81
110
  if (useImpersonate) {
82
- return fetchWithCurlImpersonate(url, timeoutSeconds, parentSignal)
111
+ return fetchWithCurlImpersonate(url, timeoutSeconds, antibotWarmup, parentSignal)
83
112
  }
84
113
  return fetchWithBunFetch(url, timeoutSeconds, parentSignal)
85
114
  }
@@ -87,15 +116,66 @@ export async function fetchWithLimits(
87
116
  async function fetchWithCurlImpersonate(
88
117
  url: string,
89
118
  timeoutSeconds: number,
119
+ antibotWarmup: AntibotWarmup,
90
120
  parentSignal?: AbortSignal,
91
121
  ): Promise<FetchResult> {
92
- let response
122
+ if (antibotWarmup === 'off') {
123
+ const response = await runCurl(url, timeoutSeconds, undefined, parentSignal)
124
+ return toFetchResult(url, response)
125
+ }
126
+
127
+ // Warmup needs a jar shared across both requests: request 1 (`-c`) captures
128
+ // the bot-manager cookies the 403 sets; request 2 (`-b`) replays them.
129
+ const jarDir = await mkdtemp(join(tmpdir(), 'typeclaw-webfetch-jar-'))
130
+ const jarPath = join(jarDir, 'cookies.txt')
131
+ const startedAt = Date.now()
93
132
  try {
94
- response = await curlImpersonate({
133
+ const first = await runCurl(url, timeoutSeconds, jarPath, parentSignal)
134
+ if (!isAntibotWarmup403(first)) {
135
+ return toFetchResult(url, first, { attempted: true, triggered: false })
136
+ }
137
+
138
+ // `timeoutSeconds` is the budget for the WHOLE web_fetch call, not per
139
+ // attempt: charge the replay only the time the first request left, so a
140
+ // warmed fetch can't run for ~2x the requested timeout. If the first
141
+ // request already spent the budget, surface its 403 instead of forcing a
142
+ // zero-time replay that would just time out.
143
+ const remaining = timeoutSeconds - (Date.now() - startedAt) / 1000
144
+ if (remaining < 1) {
145
+ return toFetchResult(url, first, {
146
+ attempted: true,
147
+ triggered: false,
148
+ initialStatus: first.httpStatus,
149
+ initialSetCookieNames: first.setCookieNames,
150
+ })
151
+ }
152
+
153
+ const replay = await runCurl(url, Math.floor(remaining), jarPath, parentSignal)
154
+ return toFetchResult(url, replay, {
155
+ attempted: true,
156
+ triggered: true,
157
+ initialStatus: first.httpStatus,
158
+ initialSetCookieNames: first.setCookieNames,
159
+ replayStatus: replay.httpStatus,
160
+ })
161
+ } finally {
162
+ await rm(jarDir, { recursive: true, force: true })
163
+ }
164
+ }
165
+
166
+ async function runCurl(
167
+ url: string,
168
+ timeoutSeconds: number,
169
+ cookieJarPath: string | undefined,
170
+ parentSignal?: AbortSignal,
171
+ ): Promise<CurlImpersonateResponse> {
172
+ try {
173
+ return await curlImpersonate({
95
174
  url,
96
175
  method: 'GET',
97
176
  timeoutSeconds,
98
177
  maxBytes: MAX_RESPONSE_BYTES,
178
+ cookieJarPath,
99
179
  signal: parentSignal,
100
180
  })
101
181
  } catch (error) {
@@ -114,7 +194,9 @@ async function fetchWithCurlImpersonate(
114
194
  const message = error instanceof Error ? error.message : String(error)
115
195
  throw new WebFetchError(`Fetch failed: ${message}`)
116
196
  }
197
+ }
117
198
 
199
+ function toFetchResult(url: string, response: CurlImpersonateResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
118
200
  if (response.httpStatus < 200 || response.httpStatus >= 300) {
119
201
  throw new WebFetchError(`Fetch failed: HTTP ${response.httpStatus}`)
120
202
  }
@@ -132,6 +214,7 @@ async function fetchWithCurlImpersonate(
132
214
  finalUrl: response.finalUrl || url,
133
215
  httpStatus: response.httpStatus,
134
216
  bytesIn: bodyByteLength,
217
+ ...(antibotWarmup ? { antibotWarmup } : {}),
135
218
  }
136
219
  }
137
220
 
@@ -27,8 +27,11 @@ export const webFetchTool = defineTool({
27
27
  'If `spawn_subagent` is available to you, PREFER delegating to the `scout` subagent by default: spawn it whenever you expect more than one fetch, an "across multiple sources" task, or any search-then-fetch loop. Scout runs the noisy fetching in its own context window and returns a distilled, citation-backed answer, keeping bulky page bodies out of yours. Only call this tool directly for a single known URL whose content you will cite immediately — or whenever you cannot spawn subagents (e.g. you are yourself a subagent), in which case fetch here. ' +
28
28
  'Outbound requests impersonate Chrome 136 at the TLS, HTTP/2, and header layers ' +
29
29
  '(via curl-impersonate), which helps with TLS/header fingerprint gates on sites behind Cloudflare/Akamai. ' +
30
- 'It does NOT solve JavaScript challenges, behavioural fingerprinting (mouse/scroll/timing), interactive CAPTCHAs, ' +
31
- 'or IP-reputation blocks — a 403 from those layers is expected and unrecoverable from this tool. ' +
30
+ 'For Akamai Bot Manager specifically, it auto-warms the session: a first-request 403 that sets bot-manager ' +
31
+ 'cookies (_abck/bm_sz/…) is absorbed and replayed once, which returns 200 on many Akamai-fronted sites ' +
32
+ '(disable with `antibotWarmup: "off"`). It still does NOT solve JavaScript challenges, behavioural ' +
33
+ 'fingerprinting (mouse/scroll/timing), interactive CAPTCHAs, or IP-reputation blocks — a 403 from those ' +
34
+ 'layers is expected and unrecoverable from this tool. ' +
32
35
  'Strategy guide:\n' +
33
36
  '- "readability": extract article content as markdown (blogs, docs, news). Default for HTML.\n' +
34
37
  '- "jq": query JSON APIs (npm registry, GitHub API). Pass `query` (e.g. ".items[].name").\n' +
@@ -62,6 +65,15 @@ export const webFetchTool = defineTool({
62
65
  maximum: MAX_TIMEOUT_SECONDS,
63
66
  }),
64
67
  ),
68
+ antibotWarmup: Type.Optional(
69
+ Type.Union([Type.Literal('auto'), Type.Literal('off')], {
70
+ description:
71
+ 'Akamai Bot Manager warmup. "auto" (default): if the first request is a 403 that sets bot-manager cookies ' +
72
+ '(_abck/bm_sz/…), absorb them and replay once — many Akamai-fronted sites (e.g. Coupang) only serve content ' +
73
+ 'on the second, cookie-bearing request. "off": disable the replay; a bot-manager 403 then surfaces as the ' +
74
+ 'normal HTTP 403 error (use to confirm the block without the warmup).',
75
+ }),
76
+ ),
65
77
  }),
66
78
 
67
79
  async execute(_toolCallId, params, signal) {
@@ -80,7 +92,7 @@ export const webFetchTool = defineTool({
80
92
 
81
93
  let response
82
94
  try {
83
- response = await fetchWithLimits(normalizedUrl, timeout, signal)
95
+ response = await fetchWithLimits(normalizedUrl, timeout, signal, params.antibotWarmup ?? 'auto')
84
96
  } catch (error) {
85
97
  const message = error instanceof Error ? error.message : String(error)
86
98
  return errorResult(normalizedUrl, message, { startedAt, finalUrl: normalizedUrl })
@@ -141,6 +153,16 @@ export const webFetchTool = defineTool({
141
153
  bytesOut: byteLength(capped.text),
142
154
  truncated: capped.truncated,
143
155
  durationMs: Date.now() - startedAt,
156
+ ...(response.antibotWarmup?.triggered
157
+ ? {
158
+ antibotWarmup: {
159
+ triggered: true,
160
+ initialStatus: response.antibotWarmup.initialStatus,
161
+ initialSetCookieNames: response.antibotWarmup.initialSetCookieNames,
162
+ replayStatus: response.antibotWarmup.replayStatus,
163
+ },
164
+ }
165
+ : {}),
144
166
  }
145
167
 
146
168
  return {
@@ -153,6 +175,7 @@ export const webFetchTool = defineTool({
153
175
  type WebFetchParams = {
154
176
  url: string
155
177
  strategy?: CompactionStrategy
178
+ antibotWarmup?: 'auto' | 'off'
156
179
  query?: string
157
180
  selector?: string
158
181
  pattern?: string
@@ -1,5 +1,12 @@
1
1
  export type CompactionStrategy = 'readability' | 'jq' | 'selector' | 'grep' | 'snapshot' | 'raw'
2
2
 
3
+ export type AntibotWarmupDetails = {
4
+ triggered: boolean
5
+ initialStatus?: number
6
+ initialSetCookieNames?: string[]
7
+ replayStatus?: number
8
+ }
9
+
3
10
  export type WebFetchDetails = {
4
11
  url: string
5
12
  finalUrl: string
@@ -11,6 +18,7 @@ export type WebFetchDetails = {
11
18
  bytesOut: number
12
19
  truncated: boolean
13
20
  durationMs: number
21
+ antibotWarmup?: AntibotWarmupDetails
14
22
  error?: boolean
15
23
  message?: string
16
24
  }