typeclaw 0.45.0 → 0.46.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +1 -1
  2. package/package.json +3 -2
  3. package/src/agent/git-nudge.ts +2 -1
  4. package/src/agent/index.ts +67 -25
  5. package/src/agent/live-subagents.ts +18 -0
  6. package/src/agent/loop-guard.ts +208 -10
  7. package/src/agent/multimodal/index.ts +1 -1
  8. package/src/agent/multimodal/look-at.ts +78 -44
  9. package/src/agent/multimodal/looker.ts +151 -39
  10. package/src/agent/network/response-body.ts +26 -0
  11. package/src/agent/network/safe-http.ts +198 -0
  12. package/src/agent/plugin-tools.ts +438 -162
  13. package/src/agent/provider-error.ts +62 -4
  14. package/src/agent/restart/index.ts +61 -40
  15. package/src/agent/restart-handoff/index.ts +73 -4
  16. package/src/agent/retry-same-ref.ts +66 -0
  17. package/src/agent/reviewer-bash-policy.ts +42 -1
  18. package/src/agent/session-origin.ts +2 -0
  19. package/src/agent/subagent-drain.ts +117 -7
  20. package/src/agent/subagents.ts +77 -18
  21. package/src/agent/throttle-circuit.ts +25 -5
  22. package/src/agent/tool-error-remediation.ts +72 -0
  23. package/src/agent/tool-file-safety.ts +1050 -0
  24. package/src/agent/tools/channel-fetch-attachment.ts +10 -3
  25. package/src/agent/tools/channel-reply.ts +15 -1
  26. package/src/agent/tools/channel-send.ts +17 -1
  27. package/src/agent/tools/ddg.ts +16 -2
  28. package/src/agent/tools/post-github-review.ts +199 -0
  29. package/src/agent/tools/spawn-subagent.ts +21 -18
  30. package/src/agent/tools/webfetch/fetch.ts +143 -183
  31. package/src/agent/tools/webfetch/tool.ts +2 -3
  32. package/src/agent/tools/websearch.ts +23 -2
  33. package/src/agent/tools-without-local-file-operands.ts +27 -0
  34. package/src/agent/turn-runner.ts +109 -15
  35. package/src/bundled-plugins/backup/index.ts +2 -1
  36. package/src/bundled-plugins/backup/runner.ts +2 -1
  37. package/src/bundled-plugins/doc-render/render.ts +93 -12
  38. package/src/bundled-plugins/doc-render/skills/typeclaw-render-pdf/SKILL.md +13 -13
  39. package/src/bundled-plugins/github-cli-auth/effective-approval.ts +16 -6
  40. package/src/bundled-plugins/github-cli-auth/gh-command.ts +555 -15
  41. package/src/bundled-plugins/github-cli-auth/index.ts +131 -135
  42. package/src/bundled-plugins/github-cli-auth/review-checkout.ts +138 -0
  43. package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +2 -1
  44. package/src/bundled-plugins/memory/README.md +23 -11
  45. package/src/bundled-plugins/memory/append-tool.ts +15 -8
  46. package/src/bundled-plugins/memory/citations.ts +22 -9
  47. package/src/bundled-plugins/memory/dreaming.ts +13 -8
  48. package/src/bundled-plugins/memory/find-entry-tool.ts +4 -0
  49. package/src/bundled-plugins/memory/index.ts +0 -4
  50. package/src/bundled-plugins/memory/load-memory.ts +8 -3
  51. package/src/bundled-plugins/memory/provenance-index.ts +717 -0
  52. package/src/bundled-plugins/memory/provenance-sanitize.ts +40 -0
  53. package/src/bundled-plugins/memory/references/store-reference-tool.ts +4 -0
  54. package/src/bundled-plugins/memory/search-tool.ts +163 -60
  55. package/src/bundled-plugins/memory/stream-events.ts +2 -0
  56. package/src/bundled-plugins/memory/vector/hybrid.ts +126 -13
  57. package/src/bundled-plugins/researcher/write-report.ts +6 -3
  58. package/src/bundled-plugins/reviewer/index.ts +2 -2
  59. package/src/bundled-plugins/reviewer/reviewer.ts +10 -3
  60. package/src/bundled-plugins/reviewer/skills/code-review.ts +24 -9
  61. package/src/bundled-plugins/scout/scout.ts +8 -0
  62. package/src/bundled-plugins/security/index.ts +4 -4
  63. package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +1 -0
  64. package/src/bundled-plugins/security/policies/private-surface-read.ts +220 -29
  65. package/src/bundled-plugins/security/policies/secret-exfil-read.ts +1 -0
  66. package/src/bundled-plugins/security/policies/ssrf.ts +29 -9
  67. package/src/channels/adapters/discord-bot-channel-resolver.ts +26 -14
  68. package/src/channels/adapters/discord-bot-thread-room.ts +44 -12
  69. package/src/channels/adapters/discord-bot.ts +91 -10
  70. package/src/channels/adapters/discord-channel-resolver.ts +91 -23
  71. package/src/channels/adapters/discord-id.ts +4 -0
  72. package/src/channels/adapters/discord-resolver-cache.ts +41 -0
  73. package/src/channels/adapters/discord.ts +57 -7
  74. package/src/channels/adapters/github/auth-app.ts +41 -12
  75. package/src/channels/adapters/github/index.ts +23 -11
  76. package/src/channels/adapters/github/review-submitter.ts +302 -0
  77. package/src/channels/adapters/kakaotalk-channel-resolver.ts +12 -1
  78. package/src/channels/adapters/kakaotalk-fetch-attachment.ts +8 -4
  79. package/src/channels/adapters/kakaotalk-typing.ts +127 -0
  80. package/src/channels/adapters/kakaotalk.ts +19 -0
  81. package/src/channels/adapters/slack-attachment-download.ts +71 -0
  82. package/src/channels/adapters/slack-bot.ts +19 -8
  83. package/src/channels/adapters/slack-classify.ts +21 -2
  84. package/src/channels/adapters/slack.ts +59 -7
  85. package/src/channels/adapters/telegram-bot.ts +18 -5
  86. package/src/channels/adapters/webex-bot.ts +8 -3
  87. package/src/channels/adapters/webex.ts +8 -3
  88. package/src/channels/fetch-attachment.ts +43 -0
  89. package/src/{bundled-plugins/github-cli-auth/approve-idempotency.ts → channels/github-review-verdict-coordinator.ts} +25 -1
  90. package/src/channels/github-token-bridge.ts +10 -3
  91. package/src/channels/manager.ts +9 -0
  92. package/src/channels/router.ts +568 -12
  93. package/src/channels/types.ts +28 -1
  94. package/src/cli/channel.ts +1 -1
  95. package/src/cli/init.ts +1 -1
  96. package/src/cli/model.ts +57 -7
  97. package/src/compose/stop.ts +17 -5
  98. package/src/config/config.ts +6 -6
  99. package/src/config/models-mutation.ts +13 -1
  100. package/src/config/providers.ts +176 -0
  101. package/src/container/start.ts +49 -10
  102. package/src/doctor/checks.ts +3 -1
  103. package/src/doctor/commit.ts +2 -1
  104. package/src/doctor/file-ownership.ts +52 -0
  105. package/src/dreams/git.ts +2 -1
  106. package/src/git/hookless.ts +5 -0
  107. package/src/git/reconcile-ignored.ts +18 -5
  108. package/src/git/secret-history.ts +371 -0
  109. package/src/git/system-commit.ts +12 -6
  110. package/src/init/dockerfile.ts +86 -5
  111. package/src/init/index.ts +17 -5
  112. package/src/init/models-dev.ts +4 -0
  113. package/src/init/validate-api-key.ts +2 -0
  114. package/src/mcp/tools.ts +16 -3
  115. package/src/permissions/builtins.ts +3 -1
  116. package/src/plugin/context.ts +2 -0
  117. package/src/plugin/index.ts +1 -0
  118. package/src/plugin/manager.ts +4 -0
  119. package/src/plugin/types.ts +21 -0
  120. package/src/run/index.ts +40 -0
  121. package/src/run/llm-fetch-observer.ts +188 -13
  122. package/src/sandbox/build.ts +37 -15
  123. package/src/sandbox/canonical-secrets.ts +18 -0
  124. package/src/sandbox/env-exposure.ts +80 -0
  125. package/src/sandbox/errors.ts +13 -6
  126. package/src/sandbox/hidden-paths.ts +102 -44
  127. package/src/sandbox/index.ts +22 -4
  128. package/src/sandbox/package-install.ts +3 -27
  129. package/src/sandbox/policy.ts +11 -25
  130. package/src/sandbox/privileged-runtime.ts +255 -0
  131. package/src/sandbox/session-tmp.ts +8 -2
  132. package/src/sandbox/writable-zones.ts +204 -140
  133. package/src/secrets/index.ts +8 -0
  134. package/src/secrets/refresh-provider-oauth.ts +128 -0
  135. package/src/server/command-runner.ts +48 -4
  136. package/src/server/index.ts +4 -0
  137. package/src/skills/typeclaw-channel-github/SKILL.md +49 -56
  138. package/src/skills/typeclaw-claude-code/SKILL.md +24 -60
  139. package/src/skills/typeclaw-claude-code/references/auth-flow.md +17 -158
  140. package/src/skills/typeclaw-codex-cli/SKILL.md +19 -49
  141. package/src/skills/typeclaw-codex-cli/references/auth-flow.md +17 -154
  142. package/src/skills/typeclaw-config/SKILL.md +5 -5
  143. package/src/skills/typeclaw-config/references/recommended-mounts.md +24 -24
  144. package/src/skills/typeclaw-github-contributing/SKILL.md +23 -26
  145. package/src/skills/typeclaw-memory/SKILL.md +9 -3
  146. package/src/skills/typeclaw-permissions/SKILL.md +19 -15
  147. package/typeclaw.schema.json +20 -0
@@ -1,38 +1,13 @@
1
- // WebFetch's HTTP transport.
2
- //
3
- // Production path (container, curl-impersonate available): we shell out to
4
- // `curl_chrome136` so outbound requests carry Chrome 136's TLS handshake
5
- // (JA3/JA4), HTTP/2 SETTINGS frame, and full header set. This is what gets
6
- // us past the modern bot-detection stacks on Cloudflare/Akamai-protected
7
- // sites (Reuters, MarketWatch, etc.) when the agent is running from the
8
- // user's home network — the IP is already residential, so impersonating
9
- // the browser is the only remaining missing piece. See AGENTS.md §"Web
10
- // search" and src/agent/tools/curl-impersonate.ts for the full story.
11
- //
12
- // Test/dev fallback (curl_chrome136 not on PATH): we transparently fall
13
- // back to Bun's native `fetch()` with a static User-Agent. This keeps unit
14
- // tests on developer macOS machines working without forcing every contributor
15
- // to install curl-impersonate locally. Production runs always have the binary
16
- // because the typeclaw Dockerfile pins it.
17
- //
18
- // Best-effort doctrine: this transport does NOT guarantee the fetch succeeds.
19
- // Bot-detected sites can still serve 403/CAPTCHA pages. We surface what we
20
- // got (status, body, final URL) and let the caller decide. The web_fetch tool
21
- // translates non-2xx into a tool-level error message that's useful to the
22
- // model.
23
-
24
- import { mkdtemp, rm } from 'node:fs/promises'
25
- import { tmpdir } from 'node:os'
26
- import { join } from 'node:path'
1
+ import { CookieJar } from 'tough-cookie'
27
2
 
28
3
  import {
29
- CurlImpersonateError,
30
- curlImpersonate,
31
- type CurlImpersonateResponse,
32
- isCurlExitFilesizeExceeded,
33
- isCurlExitTimeout,
34
- isCurlImpersonateAvailable,
35
- } from '../curl-impersonate'
4
+ defaultPublicHttpDependencies,
5
+ headerValue,
6
+ requestPublicHttpUrl,
7
+ type PublicHttpDependencies,
8
+ type PublicHttpResponse,
9
+ } from '@/agent/network/safe-http'
10
+
36
11
  import { MAX_RESPONSE_BYTES } from './types'
37
12
 
38
13
  export type AntibotWarmup = 'auto' | 'off'
@@ -54,17 +29,9 @@ export type FetchResult = {
54
29
  antibotWarmup?: AntibotWarmupInfo
55
30
  }
56
31
 
57
- // Cookie names Akamai Bot Manager sets in the Set-Cookie of its 403 challenge.
58
- // A 403 carrying one of these is a state-bootstrap handshake (absorb cookies,
59
- // replay once), NOT an authorization failure — that distinction is what makes
60
- // the auto-retry safe. Scoped to Akamai on purpose: Cloudflare/DataDome use
61
- // different challenge semantics (JS/redirects/CAPTCHA) that a blind replay
62
- // would not satisfy.
63
- const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
32
+ export type WebFetchNetworkDependencies = PublicHttpDependencies
64
33
 
65
- function isAntibotWarmup403(response: CurlImpersonateResponse): boolean {
66
- return response.httpStatus === 403 && response.setCookieNames.some((name) => AKAMAI_BOTMANAGER_COOKIE.test(name))
67
- }
34
+ const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
68
35
 
69
36
  export class WebFetchError extends Error {
70
37
  constructor(message: string) {
@@ -73,7 +40,7 @@ export class WebFetchError extends Error {
73
40
  }
74
41
  }
75
42
 
76
- const FALLBACK_HEADERS: Record<string, string> = {
43
+ const REQUEST_HEADERS: Record<string, string> = {
77
44
  'User-Agent': 'typeclaw/0 (+https://github.com/code-yeongyu/typeclaw)',
78
45
  Accept: 'text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1',
79
46
  'Accept-Language': 'en-US,en;q=0.9',
@@ -90,10 +57,6 @@ export function normalizeUrl(input: string): string {
90
57
  return `https://${trimmed}`
91
58
  }
92
59
 
93
- // Test-only seam: forces fetchWithLimits to use the native-fetch fallback
94
- // even when curl-impersonate is detected. Used by fetch.test.ts to keep its
95
- // existing mocked-fetch contract working without the test having to install
96
- // a fake curl binary. Production code never calls this.
97
60
  let forceFallbackForTest = false
98
61
 
99
62
  export function _setForceFallbackForTest(value: boolean): void {
@@ -105,175 +68,172 @@ export async function fetchWithLimits(
105
68
  timeoutSeconds: number,
106
69
  parentSignal?: AbortSignal,
107
70
  antibotWarmup: AntibotWarmup = 'auto',
71
+ network: WebFetchNetworkDependencies = forceFallbackForTest ? testFetchDependencies : defaultPublicHttpDependencies,
108
72
  ): Promise<FetchResult> {
109
- const useImpersonate = !forceFallbackForTest && (await isCurlImpersonateAvailable())
110
- if (useImpersonate) {
111
- return fetchWithCurlImpersonate(url, timeoutSeconds, antibotWarmup, parentSignal)
112
- }
113
- return fetchWithBunFetch(url, timeoutSeconds, parentSignal)
114
- }
115
-
116
- async function fetchWithCurlImpersonate(
117
- url: string,
118
- timeoutSeconds: number,
119
- antibotWarmup: AntibotWarmup,
120
- parentSignal?: AbortSignal,
121
- ): Promise<FetchResult> {
122
- if (antibotWarmup === 'off') {
123
- const response = await runCurl(url, timeoutSeconds, undefined, parentSignal)
124
- return toFetchResult(url, response)
125
- }
126
-
127
- // Warmup needs a jar shared across both requests: request 1 (`-c`) captures
128
- // the bot-manager cookies the 403 sets; request 2 (`-b`) replays them.
129
- const jarDir = await mkdtemp(join(tmpdir(), 'typeclaw-webfetch-jar-'))
130
- const jarPath = join(jarDir, 'cookies.txt')
73
+ const controller = new AbortController()
74
+ const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
75
+ const onAbort = () => controller.abort(parentSignal?.reason)
76
+ parentSignal?.addEventListener('abort', onAbort, { once: true })
131
77
  const startedAt = Date.now()
132
78
  try {
133
- const first = await runCurl(url, timeoutSeconds, jarPath, parentSignal)
134
- if (!isAntibotWarmup403(first)) {
135
- return toFetchResult(url, first, { attempted: true, triggered: false })
79
+ const cookieJar = new WarmupCookieJar()
80
+ const visitedUrls = new Set<string>()
81
+ const first = await fetchOnce(url, controller.signal, network, REQUEST_HEADERS, cookieJar, visitedUrls)
82
+ const cookieNames = cookieJar.names()
83
+ const warmup =
84
+ antibotWarmup === 'auto' &&
85
+ first.statusCode === 403 &&
86
+ [...visitedUrls].some((visitedUrl) => cookieJar.hasMatching(visitedUrl, AKAMAI_BOTMANAGER_COOKIE))
87
+ if (!warmup) {
88
+ return toFetchResult(first, { attempted: antibotWarmup === 'auto', triggered: false })
136
89
  }
137
90
 
138
- // `timeoutSeconds` is the budget for the WHOLE web_fetch call, not per
139
- // attempt: charge the replay only the time the first request left, so a
140
- // warmed fetch can't run for ~2x the requested timeout. If the first
141
- // request already spent the budget, surface its 403 instead of forcing a
142
- // zero-time replay that would just time out.
143
- const remaining = timeoutSeconds - (Date.now() - startedAt) / 1000
144
- if (remaining < 1) {
145
- return toFetchResult(url, first, {
91
+ const remainingMs = timeoutSeconds * 1000 - (Date.now() - startedAt)
92
+ if (remainingMs < 1_000) {
93
+ return toFetchResult(first, {
146
94
  attempted: true,
147
95
  triggered: false,
148
- initialStatus: first.httpStatus,
149
- initialSetCookieNames: first.setCookieNames,
96
+ initialStatus: first.statusCode,
97
+ initialSetCookieNames: cookieNames,
150
98
  })
151
99
  }
152
-
153
- const replay = await runCurl(url, Math.floor(remaining), jarPath, parentSignal)
154
- return toFetchResult(url, replay, {
100
+ const replay = await fetchOnce(url, controller.signal, network, REQUEST_HEADERS, cookieJar)
101
+ return toFetchResult(replay, {
155
102
  attempted: true,
156
103
  triggered: true,
157
- initialStatus: first.httpStatus,
158
- initialSetCookieNames: first.setCookieNames,
159
- replayStatus: replay.httpStatus,
104
+ initialStatus: first.statusCode,
105
+ initialSetCookieNames: cookieNames,
106
+ replayStatus: replay.statusCode,
160
107
  })
108
+ } catch (error) {
109
+ if (controller.signal.aborted) {
110
+ if (controller.signal.reason instanceof Error && controller.signal.reason.message === 'timeout') {
111
+ throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
112
+ }
113
+ throw new WebFetchError('Request aborted')
114
+ }
115
+ if (error instanceof WebFetchError) throw error
116
+ throw new WebFetchError(`Fetch failed: ${error instanceof Error ? error.message : String(error)}`)
161
117
  } finally {
162
- await rm(jarDir, { recursive: true, force: true })
118
+ clearTimeout(timeout)
119
+ parentSignal?.removeEventListener('abort', onAbort)
163
120
  }
164
121
  }
165
122
 
166
- async function runCurl(
123
+ const testFetchDependencies: WebFetchNetworkDependencies = {
124
+ resolveAddresses: async () => [{ address: '93.184.216.34', family: 4 }],
125
+ async request(options) {
126
+ const url = `${options.protocol}//${options.headers.Host}${options.path}`
127
+ const response = await fetch(url, { headers: options.headers, signal: options.signal, redirect: 'manual' })
128
+ return {
129
+ statusCode: response.status,
130
+ headers: Object.fromEntries(response.headers.entries()),
131
+ body: response.body ?? emptyBody(),
132
+ cancel: () => {
133
+ void response.body?.cancel()
134
+ },
135
+ }
136
+ },
137
+ }
138
+
139
+ async function* emptyBody(): AsyncIterable<Uint8Array> {}
140
+
141
+ type BufferedResponse = {
142
+ body: Uint8Array
143
+ headers: PublicHttpResponse['headers']
144
+ statusCode: number
145
+ finalUrl: string
146
+ }
147
+
148
+ async function fetchOnce(
167
149
  url: string,
168
- timeoutSeconds: number,
169
- cookieJarPath: string | undefined,
170
- parentSignal?: AbortSignal,
171
- ): Promise<CurlImpersonateResponse> {
150
+ signal: AbortSignal,
151
+ network: WebFetchNetworkDependencies,
152
+ headers: Record<string, string>,
153
+ cookieJar?: WarmupCookieJar,
154
+ visitedUrls?: Set<string>,
155
+ ): Promise<BufferedResponse> {
156
+ const { response, finalUrl } = await requestPublicHttpUrl(url, {
157
+ signal,
158
+ headers,
159
+ dependencies: network,
160
+ headersForUrl(currentUrl): Record<string, string> {
161
+ visitedUrls?.add(currentUrl)
162
+ const cookie = cookieJar?.header(currentUrl)
163
+ if (cookie === undefined) return {}
164
+ return { Cookie: cookie }
165
+ },
166
+ onResponse(response, responseUrl) {
167
+ cookieJar?.store(response.headers, responseUrl)
168
+ },
169
+ })
172
170
  try {
173
- return await curlImpersonate({
174
- url,
175
- method: 'GET',
176
- timeoutSeconds,
177
- maxBytes: MAX_RESPONSE_BYTES,
178
- cookieJarPath,
179
- signal: parentSignal,
180
- })
181
- } catch (error) {
182
- if (parentSignal?.aborted) {
183
- throw new WebFetchError('Request aborted')
171
+ const declared = Number(headerValue(response.headers, 'content-length') ?? '')
172
+ if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) {
173
+ throw tooLarge(declared)
184
174
  }
185
- if (error instanceof CurlImpersonateError) {
186
- if (isCurlExitTimeout(error)) {
187
- throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
188
- }
189
- if (isCurlExitFilesizeExceeded(error)) {
190
- throw new WebFetchError(`Response too large (exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`)
191
- }
192
- throw new WebFetchError(`Fetch failed: ${error.message}`)
175
+ const chunks: Uint8Array[] = []
176
+ let total = 0
177
+ for await (const chunk of response.body) {
178
+ total += chunk.byteLength
179
+ if (total > MAX_RESPONSE_BYTES) throw tooLarge(total)
180
+ chunks.push(chunk)
193
181
  }
194
- const message = error instanceof Error ? error.message : String(error)
195
- throw new WebFetchError(`Fetch failed: ${message}`)
182
+ return { body: Buffer.concat(chunks), headers: response.headers, statusCode: response.statusCode, finalUrl }
183
+ } finally {
184
+ response.cancel()
196
185
  }
197
186
  }
198
187
 
199
- function toFetchResult(url: string, response: CurlImpersonateResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
200
- if (response.httpStatus < 200 || response.httpStatus >= 300) {
201
- throw new WebFetchError(`Fetch failed: HTTP ${response.httpStatus}`)
188
+ function toFetchResult(response: BufferedResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
189
+ if (response.statusCode < 200 || response.statusCode >= 300) {
190
+ throw new WebFetchError(`Fetch failed: HTTP ${response.statusCode}`)
202
191
  }
203
-
204
- const bodyByteLength = new TextEncoder().encode(response.body).byteLength
205
- if (bodyByteLength > MAX_RESPONSE_BYTES) {
206
- throw new WebFetchError(
207
- `Response too large (${formatBytes(bodyByteLength)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
208
- )
209
- }
210
-
211
192
  return {
212
- body: response.body,
213
- contentType: response.contentType,
214
- finalUrl: response.finalUrl || url,
215
- httpStatus: response.httpStatus,
216
- bytesIn: bodyByteLength,
217
- ...(antibotWarmup ? { antibotWarmup } : {}),
193
+ body: new TextDecoder('utf-8', { fatal: false }).decode(response.body),
194
+ contentType: headerValue(response.headers, 'content-type') ?? '',
195
+ finalUrl: response.finalUrl,
196
+ httpStatus: response.statusCode,
197
+ bytesIn: response.body.byteLength,
198
+ ...(antibotWarmup !== undefined ? { antibotWarmup } : {}),
218
199
  }
219
200
  }
220
201
 
221
- async function fetchWithBunFetch(
222
- url: string,
223
- timeoutSeconds: number,
224
- parentSignal?: AbortSignal,
225
- ): Promise<FetchResult> {
226
- const controller = new AbortController()
227
- const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
228
- const onAbort = () => controller.abort(parentSignal?.reason)
229
- parentSignal?.addEventListener('abort', onAbort, { once: true })
202
+ function setCookieValues(headers: PublicHttpResponse['headers']): string[] {
203
+ const value = headers['set-cookie']
204
+ if (Array.isArray(value)) return value
205
+ return value === undefined ? [] : [value]
206
+ }
230
207
 
231
- try {
232
- const response = await fetch(url, { headers: FALLBACK_HEADERS, signal: controller.signal, redirect: 'follow' })
233
- if (!response.ok) {
234
- throw new WebFetchError(`Fetch failed: HTTP ${response.status} ${response.statusText}`)
235
- }
208
+ class WarmupCookieJar {
209
+ readonly #jar = new CookieJar()
210
+ readonly #names = new Set<string>()
236
211
 
237
- const contentLengthHeader = response.headers.get('content-length')
238
- if (contentLengthHeader) {
239
- const declared = Number(contentLengthHeader)
240
- if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) {
241
- throw new WebFetchError(
242
- `Response too large (${formatBytes(declared)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
243
- )
244
- }
212
+ store(headers: PublicHttpResponse['headers'], responseUrl: string): void {
213
+ for (const value of setCookieValues(headers)) {
214
+ const cookie = this.#jar.setCookieSync(value, responseUrl, { ignoreError: true })
215
+ if (cookie !== undefined) this.#names.add(cookie.key)
245
216
  }
217
+ }
246
218
 
247
- const buffer = await response.arrayBuffer()
248
- if (buffer.byteLength > MAX_RESPONSE_BYTES) {
249
- throw new WebFetchError(
250
- `Response too large (${formatBytes(buffer.byteLength)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
251
- )
252
- }
219
+ names(): string[] {
220
+ return [...this.#names]
221
+ }
253
222
 
254
- const body = new TextDecoder('utf-8', { fatal: false }).decode(buffer)
255
- return {
256
- body,
257
- contentType: response.headers.get('content-type') ?? '',
258
- finalUrl: response.url || url,
259
- httpStatus: response.status,
260
- bytesIn: buffer.byteLength,
261
- }
262
- } catch (error) {
263
- if (
264
- controller.signal.aborted &&
265
- controller.signal.reason instanceof Error &&
266
- controller.signal.reason.message === 'timeout'
267
- ) {
268
- throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
269
- }
270
- if (error instanceof WebFetchError) throw error
271
- const message = error instanceof Error ? error.message : String(error)
272
- throw new WebFetchError(`Fetch failed: ${message}`)
273
- } finally {
274
- clearTimeout(timeout)
275
- parentSignal?.removeEventListener('abort', onAbort)
223
+ hasMatching(rawUrl: string, namePattern: RegExp): boolean {
224
+ return this.#jar.getCookiesSync(rawUrl).some((cookie) => namePattern.test(cookie.key))
276
225
  }
226
+
227
+ header(rawUrl: string): string | undefined {
228
+ const header = this.#jar.getCookieStringSync(rawUrl)
229
+ return header === '' ? undefined : header
230
+ }
231
+ }
232
+
233
+ function tooLarge(bytes: number): WebFetchError {
234
+ return new WebFetchError(
235
+ `Response too large (${formatBytes(bytes)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
236
+ )
277
237
  }
278
238
 
279
239
  export function parseMimeType(contentType: string): string {
@@ -25,8 +25,7 @@ export const webFetchTool = defineTool({
25
25
  'Fetch a single HTTP(S) URL and return the body, optionally compacted by a strategy. ' +
26
26
  'Use this when the user references a specific URL or when web_search surfaced a result you need to read in full. ' +
27
27
  'If `spawn_subagent` is available to you, PREFER delegating to the `scout` subagent by default: spawn it whenever you expect more than one fetch, an "across multiple sources" task, or any search-then-fetch loop. Scout runs the noisy fetching in its own context window and returns a distilled, citation-backed answer, keeping bulky page bodies out of yours. Only call this tool directly for a single known URL whose content you will cite immediately — or whenever you cannot spawn subagents (e.g. you are yourself a subagent), in which case fetch here. ' +
28
- 'Outbound requests impersonate Chrome 136 at the TLS, HTTP/2, and header layers ' +
29
- '(via curl-impersonate), which helps with TLS/header fingerprint gates on sites behind Cloudflare/Akamai. ' +
28
+ 'Outbound requests use a DNS-rebinding-safe transport that validates and pins every redirect hop. ' +
30
29
  'For Akamai Bot Manager specifically, it auto-warms the session: a first-request 403 that sets bot-manager ' +
31
30
  'cookies (_abck/bm_sz/…) is absorbed and replayed once, which returns 200 on many Akamai-fronted sites ' +
32
31
  '(disable with `antibotWarmup: "off"`). It still does NOT solve JavaScript challenges, behavioural ' +
@@ -40,7 +39,7 @@ export const webFetchTool = defineTool({
40
39
  '- "snapshot": indented semantic tree of the page (forms, headings, links).\n' +
41
40
  '- "raw": no processing.\n' +
42
41
  'If `strategy` is omitted, it is inferred from content-type. JSON responses require explicit `strategy: "jq"` (or "raw"). ' +
43
- 'No SSRF protection is applied; do not use on untrusted user-supplied URLs without an outer guard.',
42
+ 'Private, mixed-address, and non-HTTP(S) targets are rejected.',
44
43
  parameters: Type.Object({
45
44
  url: Type.String({
46
45
  description: 'URL to fetch (http:// or https://). Bare hostnames are rewritten to https://.',
@@ -1,6 +1,7 @@
1
1
  import { Type } from '@mariozechner/pi-ai'
2
2
  import { defineTool } from '@mariozechner/pi-coding-agent'
3
3
 
4
+ import { CurlImpersonateError, isCurlExitTimeout } from './curl-impersonate'
4
5
  import { ddgSearch, DdgCaptchaError, type DdgResult } from './ddg'
5
6
  import { wikipediaSearch, type WikipediaResult } from './wikipedia'
6
7
 
@@ -51,8 +52,17 @@ export const webSearchTool = defineTool({
51
52
  source === 'wikipedia' ? await wikipediaSearch(query, limit, signal) : await ddgSearch(query, limit, signal)
52
53
  return successResult(query, source, results)
53
54
  } catch (error) {
54
- if (error instanceof DdgCaptchaError) {
55
- return errorResult(error.message)
55
+ // Both a CAPTCHA and a connection timeout are transient failures that
56
+ // ddgSearch already retried under a bounded backoff. The model has no
57
+ // visibility into those spent retries, so any message that hints "try
58
+ // again" (the old CAPTCHA text did) invites it to loop the tool with
59
+ // varied queries — the exact churn this change removes. Give both the same
60
+ // terminal "do not retry now" contract so an exhausted transient failure
61
+ // ends the turn with partial results instead.
62
+ if (isExhaustedTransientSearchError(error)) {
63
+ return errorResult(
64
+ `Search unavailable for "${query}" after retries (${transientReason(error)}). Do not retry this tool now — report what you already have and note the search was unreachable.`,
65
+ )
56
66
  }
57
67
  const message = error instanceof Error ? error.message : String(error)
58
68
  return errorResult(`Search failed: ${message}`)
@@ -60,6 +70,17 @@ export const webSearchTool = defineTool({
60
70
  },
61
71
  })
62
72
 
73
+ function isExhaustedTransientSearchError(error: unknown): boolean {
74
+ if (error instanceof DdgCaptchaError) return true
75
+ if (error instanceof CurlImpersonateError && isCurlExitTimeout(error)) return true
76
+ return false
77
+ }
78
+
79
+ function transientReason(error: unknown): string {
80
+ if (error instanceof DdgCaptchaError) return 'rate-limited'
81
+ return 'network timeout'
82
+ }
83
+
63
84
  function clampLimit(value: number | undefined): number {
64
85
  if (value === undefined) return DEFAULT_LIMIT
65
86
  return Math.min(Math.max(1, Math.floor(value)), MAX_LIMIT)
@@ -0,0 +1,27 @@
1
+ // First-party system tools whose EVERY argument is a control token or a remote
2
+ // identifier (channel/workspace/message/thread/cursor ids, subagent names, task
3
+ // ids, role capabilities, reload/stream scopes) — none is ever dereferenced as a
4
+ // local path. Listing the whole tool is more robust than enumerating each
5
+ // operand: these schemas gain new id/enum fields often, and every such field
6
+ // hits the same false positives (a Slack `ts` like "1699999999.000100" matches
7
+ // the `word.ext` rule; a value equal to an agent-root dir like "memory" hits the
8
+ // fs-existence probe; a base64 cursor carries a "/"). The security cost is zero
9
+ // TODAY because none reads a local file; the risk is future drift, so this set
10
+ // is the single fence, shared by BOTH enforcement points — the file-operand
11
+ // scanner (tool-file-safety) and the private-surface-read guard. It lives in
12
+ // this leaf module (no imports) so those two modules, which import each other,
13
+ // can both read it without a circular-init TDZ. Add a tool here ONLY after
14
+ // confirming it dereferences no argument as a local path.
15
+ export const TOOLS_WITHOUT_LOCAL_FILE_OPERANDS: ReadonlySet<string> = new Set([
16
+ 'channel_read',
17
+ 'channel_history',
18
+ 'channel_edit',
19
+ 'channel_react',
20
+ 'look_at_channel_attachment',
21
+ 'stream_snapshot',
22
+ 'grant_role',
23
+ 'spawn_subagent',
24
+ 'subagent_cancel',
25
+ 'subagent_output',
26
+ 'reload',
27
+ ])