typeclaw 0.45.1 → 0.46.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/README.md +1 -1
  2. package/package.json +3 -2
  3. package/src/agent/git-nudge.ts +2 -1
  4. package/src/agent/index.ts +44 -30
  5. package/src/agent/live-subagents.ts +18 -0
  6. package/src/agent/multimodal/index.ts +1 -1
  7. package/src/agent/multimodal/look-at.ts +78 -44
  8. package/src/agent/multimodal/looker.ts +161 -39
  9. package/src/agent/network/response-body.ts +26 -0
  10. package/src/agent/network/safe-http.ts +207 -0
  11. package/src/agent/plugin-tools.ts +377 -157
  12. package/src/agent/provider-error.ts +52 -1
  13. package/src/agent/restart/index.ts +61 -40
  14. package/src/agent/restart-handoff/index.ts +73 -4
  15. package/src/agent/reviewer-bash-policy.ts +42 -1
  16. package/src/agent/session-origin.ts +2 -0
  17. package/src/agent/subagent-drain.ts +117 -7
  18. package/src/agent/subagents.ts +49 -16
  19. package/src/agent/throttle-circuit.ts +25 -5
  20. package/src/agent/tool-error-remediation.ts +72 -0
  21. package/src/agent/tool-file-safety.ts +1310 -0
  22. package/src/agent/tools/channel-fetch-attachment.ts +10 -3
  23. package/src/agent/tools/ddg.ts +16 -2
  24. package/src/agent/tools/post-github-review.ts +199 -0
  25. package/src/agent/tools/spawn-subagent.ts +21 -18
  26. package/src/agent/tools/webfetch/fetch.ts +163 -183
  27. package/src/agent/tools/webfetch/tool.ts +2 -3
  28. package/src/agent/tools/websearch.ts +23 -2
  29. package/src/agent/tools-without-local-file-operands.ts +30 -0
  30. package/src/agent/turn-runner.ts +86 -6
  31. package/src/bundled-plugins/backup/README.md +23 -10
  32. package/src/bundled-plugins/backup/git-auth.ts +6 -4
  33. package/src/bundled-plugins/backup/index.ts +85 -44
  34. package/src/bundled-plugins/backup/runner.ts +85 -1
  35. package/src/bundled-plugins/doc-render/render.ts +93 -12
  36. package/src/bundled-plugins/doc-render/skills/typeclaw-render-pdf/SKILL.md +13 -13
  37. package/src/bundled-plugins/github-cli-auth/effective-approval.ts +16 -6
  38. package/src/bundled-plugins/github-cli-auth/gh-command.ts +555 -15
  39. package/src/bundled-plugins/github-cli-auth/git-askpass.ts +74 -26
  40. package/src/bundled-plugins/github-cli-auth/git-command.ts +224 -36
  41. package/src/bundled-plugins/github-cli-auth/graphql-auth-nudge.ts +9 -9
  42. package/src/bundled-plugins/github-cli-auth/index.ts +207 -132
  43. package/src/bundled-plugins/github-cli-auth/review-checkout.ts +138 -0
  44. package/src/bundled-plugins/github-cli-auth/token-class.ts +4 -6
  45. package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +2 -1
  46. package/src/bundled-plugins/memory/README.md +32 -11
  47. package/src/bundled-plugins/memory/append-tool.ts +15 -8
  48. package/src/bundled-plugins/memory/citations.ts +22 -9
  49. package/src/bundled-plugins/memory/dreaming.ts +13 -8
  50. package/src/bundled-plugins/memory/find-entry-tool.ts +4 -0
  51. package/src/bundled-plugins/memory/index.ts +0 -4
  52. package/src/bundled-plugins/memory/load-memory.ts +8 -3
  53. package/src/bundled-plugins/memory/provenance-index.ts +717 -0
  54. package/src/bundled-plugins/memory/provenance-sanitize.ts +40 -0
  55. package/src/bundled-plugins/memory/references/store-reference-tool.ts +4 -0
  56. package/src/bundled-plugins/memory/search-tool.ts +163 -60
  57. package/src/bundled-plugins/memory/stream-events.ts +2 -0
  58. package/src/bundled-plugins/memory/vector/embed-threads.ts +36 -0
  59. package/src/bundled-plugins/memory/vector/embedder.ts +19 -1
  60. package/src/bundled-plugins/memory/vector/hybrid.ts +126 -13
  61. package/src/bundled-plugins/researcher/write-report.ts +8 -5
  62. package/src/bundled-plugins/reviewer/index.ts +2 -2
  63. package/src/bundled-plugins/reviewer/reviewer.ts +10 -3
  64. package/src/bundled-plugins/reviewer/skills/code-review.ts +19 -9
  65. package/src/bundled-plugins/scout/scout.ts +8 -0
  66. package/src/bundled-plugins/security/index.ts +4 -4
  67. package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +1 -0
  68. package/src/bundled-plugins/security/policies/private-surface-read.ts +787 -36
  69. package/src/bundled-plugins/security/policies/secret-exfil-read.ts +1 -0
  70. package/src/bundled-plugins/security/policies/ssrf.ts +147 -42
  71. package/src/channels/adapters/discord-bot-channel-resolver.ts +26 -14
  72. package/src/channels/adapters/discord-bot-thread-room.ts +44 -12
  73. package/src/channels/adapters/discord-bot.ts +66 -6
  74. package/src/channels/adapters/discord-channel-resolver.ts +91 -23
  75. package/src/channels/adapters/discord-id.ts +4 -0
  76. package/src/channels/adapters/discord-resolver-cache.ts +41 -0
  77. package/src/channels/adapters/discord.ts +57 -7
  78. package/src/channels/adapters/github/auth-app.ts +41 -12
  79. package/src/channels/adapters/github/inbound.ts +11 -0
  80. package/src/channels/adapters/github/index.ts +71 -65
  81. package/src/channels/adapters/github/reconcile-cooldown-store.ts +178 -0
  82. package/src/channels/adapters/github/reconcile-open-prs.ts +31 -0
  83. package/src/channels/adapters/github/review-submitter.ts +302 -0
  84. package/src/channels/adapters/kakaotalk-channel-resolver.ts +5 -3
  85. package/src/channels/adapters/kakaotalk-fetch-attachment.ts +8 -4
  86. package/src/channels/adapters/kakaotalk-typing.ts +17 -15
  87. package/src/channels/adapters/slack-attachment-download.ts +71 -0
  88. package/src/channels/adapters/slack-bot-edit.ts +85 -20
  89. package/src/channels/adapters/slack-bot.ts +26 -10
  90. package/src/channels/adapters/slack-classify.ts +21 -2
  91. package/src/channels/adapters/slack-format.ts +245 -0
  92. package/src/channels/adapters/slack.ts +59 -7
  93. package/src/channels/adapters/telegram-bot.ts +18 -5
  94. package/src/channels/adapters/webex-bot.ts +8 -3
  95. package/src/channels/adapters/webex.ts +8 -3
  96. package/src/channels/fetch-attachment.ts +43 -0
  97. package/src/{bundled-plugins/github-cli-auth/approve-idempotency.ts → channels/github-review-verdict-coordinator.ts} +25 -1
  98. package/src/channels/github-token-bridge.ts +12 -6
  99. package/src/channels/manager.ts +9 -0
  100. package/src/channels/router.ts +248 -6
  101. package/src/channels/types.ts +28 -1
  102. package/src/cli/channel.ts +1 -1
  103. package/src/cli/init.ts +1 -1
  104. package/src/cli/model.ts +57 -7
  105. package/src/compose/stop.ts +17 -5
  106. package/src/config/config.ts +6 -6
  107. package/src/config/models-mutation.ts +13 -1
  108. package/src/config/providers.ts +176 -0
  109. package/src/container/docker-app.ts +42 -13
  110. package/src/container/start.ts +62 -10
  111. package/src/cron/bridge.ts +9 -2
  112. package/src/doctor/checks.ts +8 -3
  113. package/src/doctor/commit.ts +2 -1
  114. package/src/doctor/file-ownership.ts +52 -0
  115. package/src/dreams/git.ts +2 -1
  116. package/src/git/bound-gc-config.ts +82 -0
  117. package/src/git/hookless.ts +5 -0
  118. package/src/git/reconcile-ignored.ts +18 -5
  119. package/src/git/system-commit.ts +12 -6
  120. package/src/hostd/spawn.ts +133 -47
  121. package/src/init/dockerfile.ts +99 -5
  122. package/src/init/index.ts +17 -5
  123. package/src/init/models-dev.ts +4 -0
  124. package/src/init/validate-api-key.ts +2 -0
  125. package/src/inspect/replay.ts +21 -9
  126. package/src/inspect/session-list.ts +71 -33
  127. package/src/mcp/tools.ts +17 -3
  128. package/src/network/internal-destinations.ts +267 -0
  129. package/src/permissions/builtins.ts +3 -1
  130. package/src/plugin/context.ts +2 -0
  131. package/src/plugin/index.ts +1 -0
  132. package/src/plugin/manager.ts +4 -0
  133. package/src/plugin/types.ts +22 -0
  134. package/src/run/index.ts +40 -0
  135. package/src/run/llm-fetch-observer.ts +41 -4
  136. package/src/sandbox/build.ts +37 -15
  137. package/src/sandbox/canonical-secrets.ts +18 -0
  138. package/src/sandbox/env-exposure.ts +83 -0
  139. package/src/sandbox/errors.ts +13 -6
  140. package/src/sandbox/hidden-paths.ts +102 -44
  141. package/src/sandbox/index.ts +22 -4
  142. package/src/sandbox/package-install.ts +3 -27
  143. package/src/sandbox/policy.ts +16 -28
  144. package/src/sandbox/privileged-runtime.ts +256 -0
  145. package/src/sandbox/session-tmp.ts +8 -2
  146. package/src/sandbox/writable-zones.ts +204 -140
  147. package/src/secrets/index.ts +8 -0
  148. package/src/secrets/refresh-provider-oauth.ts +128 -0
  149. package/src/server/command-runner.ts +48 -4
  150. package/src/server/index.ts +4 -0
  151. package/src/skills/typeclaw-channel-github/SKILL.md +49 -56
  152. package/src/skills/typeclaw-claude-code/SKILL.md +24 -60
  153. package/src/skills/typeclaw-claude-code/references/auth-flow.md +17 -158
  154. package/src/skills/typeclaw-codex-cli/SKILL.md +19 -49
  155. package/src/skills/typeclaw-codex-cli/references/auth-flow.md +17 -154
  156. package/src/skills/typeclaw-config/SKILL.md +27 -11
  157. package/src/skills/typeclaw-config/references/recommended-mounts.md +24 -24
  158. package/src/skills/typeclaw-github-contributing/SKILL.md +23 -26
  159. package/src/skills/typeclaw-memory/SKILL.md +9 -3
  160. package/src/skills/typeclaw-permissions/SKILL.md +36 -19
  161. package/src/skills/typeclaw-plugins/SKILL.md +37 -0
  162. package/src/usage/scan.ts +11 -2
  163. package/typeclaw.schema.json +25 -0
@@ -1,38 +1,14 @@
1
- // WebFetch's HTTP transport.
2
- //
3
- // Production path (container, curl-impersonate available): we shell out to
4
- // `curl_chrome136` so outbound requests carry Chrome 136's TLS handshake
5
- // (JA3/JA4), HTTP/2 SETTINGS frame, and full header set. This is what gets
6
- // us past the modern bot-detection stacks on Cloudflare/Akamai-protected
7
- // sites (Reuters, MarketWatch, etc.) when the agent is running from the
8
- // user's home network — the IP is already residential, so impersonating
9
- // the browser is the only remaining missing piece. See AGENTS.md §"Web
10
- // search" and src/agent/tools/curl-impersonate.ts for the full story.
11
- //
12
- // Test/dev fallback (curl_chrome136 not on PATH): we transparently fall
13
- // back to Bun's native `fetch()` with a static User-Agent. This keeps unit
14
- // tests on developer macOS machines working without forcing every contributor
15
- // to install curl-impersonate locally. Production runs always have the binary
16
- // because the typeclaw Dockerfile pins it.
17
- //
18
- // Best-effort doctrine: this transport does NOT guarantee the fetch succeeds.
19
- // Bot-detected sites can still serve 403/CAPTCHA pages. We surface what we
20
- // got (status, body, final URL) and let the caller decide. The web_fetch tool
21
- // translates non-2xx into a tool-level error message that's useful to the
22
- // model.
23
-
24
- import { mkdtemp, rm } from 'node:fs/promises'
25
- import { tmpdir } from 'node:os'
26
- import { join } from 'node:path'
1
+ import { CookieJar } from 'tough-cookie'
27
2
 
28
3
  import {
29
- CurlImpersonateError,
30
- curlImpersonate,
31
- type CurlImpersonateResponse,
32
- isCurlExitFilesizeExceeded,
33
- isCurlExitTimeout,
34
- isCurlImpersonateAvailable,
35
- } from '../curl-impersonate'
4
+ defaultPublicHttpDependencies,
5
+ headerValue,
6
+ requestPublicHttpUrl,
7
+ type PublicHttpDependencies,
8
+ type PublicHttpResponse,
9
+ } from '@/agent/network/safe-http'
10
+ import type { InternalDestinationPolicy } from '@/network/internal-destinations'
11
+
36
12
  import { MAX_RESPONSE_BYTES } from './types'
37
13
 
38
14
  export type AntibotWarmup = 'auto' | 'off'
@@ -54,17 +30,9 @@ export type FetchResult = {
54
30
  antibotWarmup?: AntibotWarmupInfo
55
31
  }
56
32
 
57
- // Cookie names Akamai Bot Manager sets in the Set-Cookie of its 403 challenge.
58
- // A 403 carrying one of these is a state-bootstrap handshake (absorb cookies,
59
- // replay once), NOT an authorization failure — that distinction is what makes
60
- // the auto-retry safe. Scoped to Akamai on purpose: Cloudflare/DataDome use
61
- // different challenge semantics (JS/redirects/CAPTCHA) that a blind replay
62
- // would not satisfy.
63
- const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
33
+ export type WebFetchNetworkDependencies = PublicHttpDependencies
64
34
 
65
- function isAntibotWarmup403(response: CurlImpersonateResponse): boolean {
66
- return response.httpStatus === 403 && response.setCookieNames.some((name) => AKAMAI_BOTMANAGER_COOKIE.test(name))
67
- }
35
+ const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
68
36
 
69
37
  export class WebFetchError extends Error {
70
38
  constructor(message: string) {
@@ -73,7 +41,7 @@ export class WebFetchError extends Error {
73
41
  }
74
42
  }
75
43
 
76
- const FALLBACK_HEADERS: Record<string, string> = {
44
+ const REQUEST_HEADERS: Record<string, string> = {
77
45
  'User-Agent': 'typeclaw/0 (+https://github.com/code-yeongyu/typeclaw)',
78
46
  Accept: 'text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1',
79
47
  'Accept-Language': 'en-US,en;q=0.9',
@@ -90,10 +58,6 @@ export function normalizeUrl(input: string): string {
90
58
  return `https://${trimmed}`
91
59
  }
92
60
 
93
- // Test-only seam: forces fetchWithLimits to use the native-fetch fallback
94
- // even when curl-impersonate is detected. Used by fetch.test.ts to keep its
95
- // existing mocked-fetch contract working without the test having to install
96
- // a fake curl binary. Production code never calls this.
97
61
  let forceFallbackForTest = false
98
62
 
99
63
  export function _setForceFallbackForTest(value: boolean): void {
@@ -105,177 +69,193 @@ export async function fetchWithLimits(
105
69
  timeoutSeconds: number,
106
70
  parentSignal?: AbortSignal,
107
71
  antibotWarmup: AntibotWarmup = 'auto',
72
+ network: WebFetchNetworkDependencies = forceFallbackForTest ? testFetchDependencies : defaultPublicHttpDependencies,
73
+ internalDestinationPolicy?: InternalDestinationPolicy,
108
74
  ): Promise<FetchResult> {
109
- const useImpersonate = !forceFallbackForTest && (await isCurlImpersonateAvailable())
110
- if (useImpersonate) {
111
- return fetchWithCurlImpersonate(url, timeoutSeconds, antibotWarmup, parentSignal)
112
- }
113
- return fetchWithBunFetch(url, timeoutSeconds, parentSignal)
114
- }
115
-
116
- async function fetchWithCurlImpersonate(
117
- url: string,
118
- timeoutSeconds: number,
119
- antibotWarmup: AntibotWarmup,
120
- parentSignal?: AbortSignal,
121
- ): Promise<FetchResult> {
122
- if (antibotWarmup === 'off') {
123
- const response = await runCurl(url, timeoutSeconds, undefined, parentSignal)
124
- return toFetchResult(url, response)
125
- }
126
-
127
- // Warmup needs a jar shared across both requests: request 1 (`-c`) captures
128
- // the bot-manager cookies the 403 sets; request 2 (`-b`) replays them.
129
- const jarDir = await mkdtemp(join(tmpdir(), 'typeclaw-webfetch-jar-'))
130
- const jarPath = join(jarDir, 'cookies.txt')
75
+ const controller = new AbortController()
76
+ const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
77
+ const onAbort = () => controller.abort(parentSignal?.reason)
78
+ parentSignal?.addEventListener('abort', onAbort, { once: true })
131
79
  const startedAt = Date.now()
132
80
  try {
133
- const first = await runCurl(url, timeoutSeconds, jarPath, parentSignal)
134
- if (!isAntibotWarmup403(first)) {
135
- return toFetchResult(url, first, { attempted: true, triggered: false })
81
+ const cookieJar = new WarmupCookieJar()
82
+ const visitedUrls = new Set<string>()
83
+ const first = await fetchOnce(
84
+ url,
85
+ controller.signal,
86
+ network,
87
+ REQUEST_HEADERS,
88
+ cookieJar,
89
+ visitedUrls,
90
+ internalDestinationPolicy,
91
+ )
92
+ const cookieNames = cookieJar.names()
93
+ const warmup =
94
+ antibotWarmup === 'auto' &&
95
+ first.statusCode === 403 &&
96
+ [...visitedUrls].some((visitedUrl) => cookieJar.hasMatching(visitedUrl, AKAMAI_BOTMANAGER_COOKIE))
97
+ if (!warmup) {
98
+ return toFetchResult(first, { attempted: antibotWarmup === 'auto', triggered: false })
136
99
  }
137
100
 
138
- // `timeoutSeconds` is the budget for the WHOLE web_fetch call, not per
139
- // attempt: charge the replay only the time the first request left, so a
140
- // warmed fetch can't run for ~2x the requested timeout. If the first
141
- // request already spent the budget, surface its 403 instead of forcing a
142
- // zero-time replay that would just time out.
143
- const remaining = timeoutSeconds - (Date.now() - startedAt) / 1000
144
- if (remaining < 1) {
145
- return toFetchResult(url, first, {
101
+ const remainingMs = timeoutSeconds * 1000 - (Date.now() - startedAt)
102
+ if (remainingMs < 1_000) {
103
+ return toFetchResult(first, {
146
104
  attempted: true,
147
105
  triggered: false,
148
- initialStatus: first.httpStatus,
149
- initialSetCookieNames: first.setCookieNames,
106
+ initialStatus: first.statusCode,
107
+ initialSetCookieNames: cookieNames,
150
108
  })
151
109
  }
152
-
153
- const replay = await runCurl(url, Math.floor(remaining), jarPath, parentSignal)
154
- return toFetchResult(url, replay, {
110
+ const replay = await fetchOnce(
111
+ url,
112
+ controller.signal,
113
+ network,
114
+ REQUEST_HEADERS,
115
+ cookieJar,
116
+ undefined,
117
+ internalDestinationPolicy,
118
+ )
119
+ return toFetchResult(replay, {
155
120
  attempted: true,
156
121
  triggered: true,
157
- initialStatus: first.httpStatus,
158
- initialSetCookieNames: first.setCookieNames,
159
- replayStatus: replay.httpStatus,
122
+ initialStatus: first.statusCode,
123
+ initialSetCookieNames: cookieNames,
124
+ replayStatus: replay.statusCode,
160
125
  })
126
+ } catch (error) {
127
+ if (controller.signal.aborted) {
128
+ if (controller.signal.reason instanceof Error && controller.signal.reason.message === 'timeout') {
129
+ throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
130
+ }
131
+ throw new WebFetchError('Request aborted')
132
+ }
133
+ if (error instanceof WebFetchError) throw error
134
+ throw new WebFetchError(`Fetch failed: ${error instanceof Error ? error.message : String(error)}`)
161
135
  } finally {
162
- await rm(jarDir, { recursive: true, force: true })
136
+ clearTimeout(timeout)
137
+ parentSignal?.removeEventListener('abort', onAbort)
163
138
  }
164
139
  }
165
140
 
166
- async function runCurl(
141
+ const testFetchDependencies: WebFetchNetworkDependencies = {
142
+ resolveAddresses: async () => [{ address: '93.184.216.34', family: 4 }],
143
+ async request(options) {
144
+ const url = `${options.protocol}//${options.headers.Host}${options.path}`
145
+ const response = await fetch(url, { headers: options.headers, signal: options.signal, redirect: 'manual' })
146
+ return {
147
+ statusCode: response.status,
148
+ headers: Object.fromEntries(response.headers.entries()),
149
+ body: response.body ?? emptyBody(),
150
+ cancel: () => {
151
+ void response.body?.cancel()
152
+ },
153
+ }
154
+ },
155
+ }
156
+
157
+ async function* emptyBody(): AsyncIterable<Uint8Array> {}
158
+
159
+ type BufferedResponse = {
160
+ body: Uint8Array
161
+ headers: PublicHttpResponse['headers']
162
+ statusCode: number
163
+ finalUrl: string
164
+ }
165
+
166
+ async function fetchOnce(
167
167
  url: string,
168
- timeoutSeconds: number,
169
- cookieJarPath: string | undefined,
170
- parentSignal?: AbortSignal,
171
- ): Promise<CurlImpersonateResponse> {
168
+ signal: AbortSignal,
169
+ network: WebFetchNetworkDependencies,
170
+ headers: Record<string, string>,
171
+ cookieJar?: WarmupCookieJar,
172
+ visitedUrls?: Set<string>,
173
+ internalDestinationPolicy?: InternalDestinationPolicy,
174
+ ): Promise<BufferedResponse> {
175
+ const { response, finalUrl } = await requestPublicHttpUrl(url, {
176
+ signal,
177
+ headers,
178
+ dependencies: network,
179
+ ...(internalDestinationPolicy === undefined ? {} : { internalDestinationPolicy }),
180
+ headersForUrl(currentUrl): Record<string, string> {
181
+ visitedUrls?.add(currentUrl)
182
+ const cookie = cookieJar?.header(currentUrl)
183
+ if (cookie === undefined) return {}
184
+ return { Cookie: cookie }
185
+ },
186
+ onResponse(response, responseUrl) {
187
+ cookieJar?.store(response.headers, responseUrl)
188
+ },
189
+ })
172
190
  try {
173
- return await curlImpersonate({
174
- url,
175
- method: 'GET',
176
- timeoutSeconds,
177
- maxBytes: MAX_RESPONSE_BYTES,
178
- cookieJarPath,
179
- signal: parentSignal,
180
- })
181
- } catch (error) {
182
- if (parentSignal?.aborted) {
183
- throw new WebFetchError('Request aborted')
191
+ const declared = Number(headerValue(response.headers, 'content-length') ?? '')
192
+ if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) {
193
+ throw tooLarge(declared)
184
194
  }
185
- if (error instanceof CurlImpersonateError) {
186
- if (isCurlExitTimeout(error)) {
187
- throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
188
- }
189
- if (isCurlExitFilesizeExceeded(error)) {
190
- throw new WebFetchError(`Response too large (exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`)
191
- }
192
- throw new WebFetchError(`Fetch failed: ${error.message}`)
195
+ const chunks: Uint8Array[] = []
196
+ let total = 0
197
+ for await (const chunk of response.body) {
198
+ total += chunk.byteLength
199
+ if (total > MAX_RESPONSE_BYTES) throw tooLarge(total)
200
+ chunks.push(chunk)
193
201
  }
194
- const message = error instanceof Error ? error.message : String(error)
195
- throw new WebFetchError(`Fetch failed: ${message}`)
202
+ return { body: Buffer.concat(chunks), headers: response.headers, statusCode: response.statusCode, finalUrl }
203
+ } finally {
204
+ response.cancel()
196
205
  }
197
206
  }
198
207
 
199
- function toFetchResult(url: string, response: CurlImpersonateResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
200
- if (response.httpStatus < 200 || response.httpStatus >= 300) {
201
- throw new WebFetchError(`Fetch failed: HTTP ${response.httpStatus}`)
202
- }
203
-
204
- const bodyByteLength = new TextEncoder().encode(response.body).byteLength
205
- if (bodyByteLength > MAX_RESPONSE_BYTES) {
206
- throw new WebFetchError(
207
- `Response too large (${formatBytes(bodyByteLength)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
208
- )
208
+ function toFetchResult(response: BufferedResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
209
+ if (response.statusCode < 200 || response.statusCode >= 300) {
210
+ throw new WebFetchError(`Fetch failed: HTTP ${response.statusCode}`)
209
211
  }
210
-
211
212
  return {
212
- body: response.body,
213
- contentType: response.contentType,
214
- finalUrl: response.finalUrl || url,
215
- httpStatus: response.httpStatus,
216
- bytesIn: bodyByteLength,
217
- ...(antibotWarmup ? { antibotWarmup } : {}),
213
+ body: new TextDecoder('utf-8', { fatal: false }).decode(response.body),
214
+ contentType: headerValue(response.headers, 'content-type') ?? '',
215
+ finalUrl: response.finalUrl,
216
+ httpStatus: response.statusCode,
217
+ bytesIn: response.body.byteLength,
218
+ ...(antibotWarmup !== undefined ? { antibotWarmup } : {}),
218
219
  }
219
220
  }
220
221
 
221
- async function fetchWithBunFetch(
222
- url: string,
223
- timeoutSeconds: number,
224
- parentSignal?: AbortSignal,
225
- ): Promise<FetchResult> {
226
- const controller = new AbortController()
227
- const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
228
- const onAbort = () => controller.abort(parentSignal?.reason)
229
- parentSignal?.addEventListener('abort', onAbort, { once: true })
222
+ function setCookieValues(headers: PublicHttpResponse['headers']): string[] {
223
+ const value = headers['set-cookie']
224
+ if (Array.isArray(value)) return value
225
+ return value === undefined ? [] : [value]
226
+ }
230
227
 
231
- try {
232
- const response = await fetch(url, { headers: FALLBACK_HEADERS, signal: controller.signal, redirect: 'follow' })
233
- if (!response.ok) {
234
- throw new WebFetchError(`Fetch failed: HTTP ${response.status} ${response.statusText}`)
235
- }
228
+ class WarmupCookieJar {
229
+ readonly #jar = new CookieJar()
230
+ readonly #names = new Set<string>()
236
231
 
237
- const contentLengthHeader = response.headers.get('content-length')
238
- if (contentLengthHeader) {
239
- const declared = Number(contentLengthHeader)
240
- if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) {
241
- throw new WebFetchError(
242
- `Response too large (${formatBytes(declared)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
243
- )
244
- }
232
+ store(headers: PublicHttpResponse['headers'], responseUrl: string): void {
233
+ for (const value of setCookieValues(headers)) {
234
+ const cookie = this.#jar.setCookieSync(value, responseUrl, { ignoreError: true })
235
+ if (cookie !== undefined) this.#names.add(cookie.key)
245
236
  }
237
+ }
246
238
 
247
- const buffer = await response.arrayBuffer()
248
- if (buffer.byteLength > MAX_RESPONSE_BYTES) {
249
- throw new WebFetchError(
250
- `Response too large (${formatBytes(buffer.byteLength)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
251
- )
252
- }
239
+ names(): string[] {
240
+ return [...this.#names]
241
+ }
253
242
 
254
- const body = new TextDecoder('utf-8', { fatal: false }).decode(buffer)
255
- return {
256
- body,
257
- contentType: response.headers.get('content-type') ?? '',
258
- finalUrl: response.url || url,
259
- httpStatus: response.status,
260
- bytesIn: buffer.byteLength,
261
- }
262
- } catch (error) {
263
- if (
264
- controller.signal.aborted &&
265
- controller.signal.reason instanceof Error &&
266
- controller.signal.reason.message === 'timeout'
267
- ) {
268
- throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
269
- }
270
- if (error instanceof WebFetchError) throw error
271
- const message = error instanceof Error ? error.message : String(error)
272
- throw new WebFetchError(`Fetch failed: ${message}`)
273
- } finally {
274
- clearTimeout(timeout)
275
- parentSignal?.removeEventListener('abort', onAbort)
243
+ hasMatching(rawUrl: string, namePattern: RegExp): boolean {
244
+ return this.#jar.getCookiesSync(rawUrl).some((cookie) => namePattern.test(cookie.key))
245
+ }
246
+
247
+ header(rawUrl: string): string | undefined {
248
+ const header = this.#jar.getCookieStringSync(rawUrl)
249
+ return header === '' ? undefined : header
276
250
  }
277
251
  }
278
252
 
253
+ function tooLarge(bytes: number): WebFetchError {
254
+ return new WebFetchError(
255
+ `Response too large (${formatBytes(bytes)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
256
+ )
257
+ }
258
+
279
259
  export function parseMimeType(contentType: string): string {
280
260
  const semi = contentType.indexOf(';')
281
261
  return (semi >= 0 ? contentType.slice(0, semi) : contentType).trim().toLowerCase()
@@ -25,8 +25,7 @@ export const webFetchTool = defineTool({
25
25
  'Fetch a single HTTP(S) URL and return the body, optionally compacted by a strategy. ' +
26
26
  'Use this when the user references a specific URL or when web_search surfaced a result you need to read in full. ' +
27
27
  'If `spawn_subagent` is available to you, PREFER delegating to the `scout` subagent by default: spawn it whenever you expect more than one fetch, an "across multiple sources" task, or any search-then-fetch loop. Scout runs the noisy fetching in its own context window and returns a distilled, citation-backed answer, keeping bulky page bodies out of yours. Only call this tool directly for a single known URL whose content you will cite immediately — or whenever you cannot spawn subagents (e.g. you are yourself a subagent), in which case fetch here. ' +
28
- 'Outbound requests impersonate Chrome 136 at the TLS, HTTP/2, and header layers ' +
29
- '(via curl-impersonate), which helps with TLS/header fingerprint gates on sites behind Cloudflare/Akamai. ' +
28
+ 'Outbound requests use a DNS-rebinding-safe transport that validates and pins every redirect hop. ' +
30
29
  'For Akamai Bot Manager specifically, it auto-warms the session: a first-request 403 that sets bot-manager ' +
31
30
  'cookies (_abck/bm_sz/…) is absorbed and replayed once, which returns 200 on many Akamai-fronted sites ' +
32
31
  '(disable with `antibotWarmup: "off"`). It still does NOT solve JavaScript challenges, behavioural ' +
@@ -40,7 +39,7 @@ export const webFetchTool = defineTool({
40
39
  '- "snapshot": indented semantic tree of the page (forms, headings, links).\n' +
41
40
  '- "raw": no processing.\n' +
42
41
  'If `strategy` is omitted, it is inferred from content-type. JSON responses require explicit `strategy: "jq"` (or "raw"). ' +
43
- 'No SSRF protection is applied; do not use on untrusted user-supplied URLs without an outer guard.',
42
+ 'Private, mixed-address, and non-HTTP(S) targets are rejected.',
44
43
  parameters: Type.Object({
45
44
  url: Type.String({
46
45
  description: 'URL to fetch (http:// or https://). Bare hostnames are rewritten to https://.',
@@ -1,6 +1,7 @@
1
1
  import { Type } from '@mariozechner/pi-ai'
2
2
  import { defineTool } from '@mariozechner/pi-coding-agent'
3
3
 
4
+ import { CurlImpersonateError, isCurlExitTimeout } from './curl-impersonate'
4
5
  import { ddgSearch, DdgCaptchaError, type DdgResult } from './ddg'
5
6
  import { wikipediaSearch, type WikipediaResult } from './wikipedia'
6
7
 
@@ -51,8 +52,17 @@ export const webSearchTool = defineTool({
51
52
  source === 'wikipedia' ? await wikipediaSearch(query, limit, signal) : await ddgSearch(query, limit, signal)
52
53
  return successResult(query, source, results)
53
54
  } catch (error) {
54
- if (error instanceof DdgCaptchaError) {
55
- return errorResult(error.message)
55
+ // Both a CAPTCHA and a connection timeout are transient failures that
56
+ // ddgSearch already retried under a bounded backoff. The model has no
57
+ // visibility into those spent retries, so any message that hints "try
58
+ // again" (the old CAPTCHA text did) invites it to loop the tool with
59
+ // varied queries — the exact churn this change removes. Give both the same
60
+ // terminal "do not retry now" contract so an exhausted transient failure
61
+ // ends the turn with partial results instead.
62
+ if (isExhaustedTransientSearchError(error)) {
63
+ return errorResult(
64
+ `Search unavailable for "${query}" after retries (${transientReason(error)}). Do not retry this tool now — report what you already have and note the search was unreachable.`,
65
+ )
56
66
  }
57
67
  const message = error instanceof Error ? error.message : String(error)
58
68
  return errorResult(`Search failed: ${message}`)
@@ -60,6 +70,17 @@ export const webSearchTool = defineTool({
60
70
  },
61
71
  })
62
72
 
73
+ function isExhaustedTransientSearchError(error: unknown): boolean {
74
+ if (error instanceof DdgCaptchaError) return true
75
+ if (error instanceof CurlImpersonateError && isCurlExitTimeout(error)) return true
76
+ return false
77
+ }
78
+
79
+ function transientReason(error: unknown): string {
80
+ if (error instanceof DdgCaptchaError) return 'rate-limited'
81
+ return 'network timeout'
82
+ }
83
+
63
84
  function clampLimit(value: number | undefined): number {
64
85
  if (value === undefined) return DEFAULT_LIMIT
65
86
  return Math.min(Math.max(1, Math.floor(value)), MAX_LIMIT)
@@ -0,0 +1,30 @@
1
+ // First-party system tools with no caller-supplied local filesystem operand.
2
+ // Their arguments are control tokens, remote identifiers (channel/workspace/
3
+ // message/thread/cursor ids, subagent names, task ids, role capabilities,
4
+ // reload/stream scopes), or sanitized output metadata — none is dereferenced as
5
+ // a caller-selected local path. Listing the whole tool is more robust than enumerating each
6
+ // operand: these schemas gain new id/enum fields often, and every such field
7
+ // hits the same conservative inference rules (a Slack `ts` like
8
+ // "1699999999.000100" looks like a dotted filename; a value equal to an agent-root dir like "memory" hits the
9
+ // fs-existence probe; a base64 cursor carries a "/"). The security cost is zero
10
+ // TODAY because none reads a local file; the risk is future drift, so this set
11
+ // is the single fence, shared by BOTH enforcement points — the file-operand
12
+ // scanner (tool-file-safety) and the private-surface-read guard. It lives in
13
+ // this leaf module (no imports) so those two modules, which import each other,
14
+ // can both read it without a circular-init TDZ. Add a tool here ONLY after
15
+ // confirming it dereferences no argument as a local path.
16
+ export const TOOLS_WITHOUT_LOCAL_FILE_OPERANDS: ReadonlySet<string> = new Set([
17
+ 'channel_read',
18
+ 'channel_history',
19
+ 'channel_edit',
20
+ 'channel_react',
21
+ 'channel_fetch_attachment',
22
+ 'look_at_channel_attachment',
23
+ 'post_github_review',
24
+ 'stream_snapshot',
25
+ 'grant_role',
26
+ 'spawn_subagent',
27
+ 'subagent_cancel',
28
+ 'subagent_output',
29
+ 'reload',
30
+ ])
@@ -1,7 +1,12 @@
1
- import type { ModelRef } from '@/config/providers'
1
+ import { providerForModelRef, type ModelRef } from '@/config/providers'
2
2
 
3
3
  import type { AgentSession } from './index'
4
- import { detectProviderError, isRetryableSameRef, subscribeProviderErrors } from './provider-error'
4
+ import {
5
+ detectProviderError,
6
+ isObserverTtfbTimeout,
7
+ isRetryableSameRef,
8
+ subscribeProviderErrors,
9
+ } from './provider-error'
5
10
  import { RETRIES_PER_REF, retryTurnAfterCompletedToolResult, retryTurnOnPersistentSession } from './retry-same-ref'
6
11
  import { modelThrottleCircuit, type ThrottleCircuit } from './throttle-circuit'
7
12
 
@@ -23,6 +28,12 @@ type AttemptActivity = {
23
28
  startedToolExecution: boolean
24
29
  }
25
30
 
31
+ // SDK-level attempts already contain pi-ai's internal fetch recovery. Bounding
32
+ // only repeated no-header outcomes limits dead air without timing out healthy
33
+ // reasoning or tool execution.
34
+ const MAX_NO_PROGRESS_ATTEMPTS = 3
35
+ const MAX_CUMULATIVE_NO_PROGRESS_MS = 60_000
36
+
26
37
  export async function promptPersistentTurnWithFallback(opts: {
27
38
  refs: ModelRef[]
28
39
  currentModelRef: ModelRef
@@ -39,11 +50,21 @@ export async function promptPersistentTurnWithFallback(opts: {
39
50
  onRetryBackoffStart?: () => void
40
51
  beforeAttempt?: (ref: ModelRef) => void
41
52
  onAttemptFailed?: (attempt: PersistentTurnAttempt) => void
53
+ now?: () => number
42
54
  }): Promise<PersistentTurnResult> {
43
55
  if (opts.refs.length === 0) throw new Error('promptPersistentTurnWithFallback: refs[] must be non-empty')
44
56
  const attempts: PersistentTurnAttempt[] = []
45
57
  let lastError: Error | undefined
46
58
  const circuit = opts.circuit ?? modelThrottleCircuit
59
+ // Monotonic clock for the no-progress budget: it only ever measures elapsed
60
+ // (now() - attemptStartedAt), and performance.now() can't jump backward on an
61
+ // NTP/system-clock correction the way Date.now() can, which would otherwise
62
+ // prematurely trip or stall the 60s cap.
63
+ const now = opts.now ?? (() => performance.now())
64
+ const hasNonCodexFallback = opts.refs.some((ref) => !isCodexRef(ref))
65
+ let noHeaderAttempts = 0
66
+ let cumulativeNoProgressMs = 0
67
+ let noProgressEnvelopeExhausted = false
47
68
  // The model the session currently has loaded; only call setModel when the
48
69
  // chosen ref differs from it. Tracked locally so successive setModel calls
49
70
  // within a single turn stay coherent.
@@ -55,9 +76,13 @@ export async function promptPersistentTurnWithFallback(opts: {
55
76
  // re-test the primary.
56
77
  for (let i = 0; i < opts.refs.length; i++) {
57
78
  const ref = opts.refs[i]!
58
- const isLast = i === opts.refs.length - 1
59
- // Skip a ref whose circuit is open, but never skip the last one — there is
60
- // nowhere left to fall, so we try it regardless and let the result speak.
79
+ const codexProviderOpen = isCodexRef(ref) && circuit.isProviderOpen({ profile: opts.profile, ref })
80
+ if (codexProviderOpen && hasNonCodexFallback) continue
81
+ const isLast = !opts.refs
82
+ .slice(i + 1)
83
+ .some((candidate) => isRefUsable(candidate, opts.profile, circuit, hasNonCodexFallback))
84
+ // The last usable ref remains a safety probe. Codex refs stop being usable
85
+ // together only when the chain has transport-diverse credentials to use.
61
86
  if (!isLast && circuit.isOpen({ profile: opts.profile, ref })) continue
62
87
  if (ref !== loadedRef) {
63
88
  await opts.setModelForRef(ref)
@@ -84,6 +109,7 @@ export async function promptPersistentTurnWithFallback(opts: {
84
109
  let retryAfterCompletedToolResult = false
85
110
  for (let retry = 0; ; retry++) {
86
111
  softError = undefined
112
+ const attemptStartedAt = now()
87
113
  let outcomeThisAttempt: AttemptOutcome | undefined
88
114
  try {
89
115
  if (retry === 0) {
@@ -107,6 +133,15 @@ export async function promptPersistentTurnWithFallback(opts: {
107
133
  }
108
134
  outcome = outcomeThisAttempt
109
135
  if (outcome === undefined) break
136
+ if (isObserverTtfbTimeout(outcome.error.message)) {
137
+ noHeaderAttempts++
138
+ cumulativeNoProgressMs += Math.max(0, now() - attemptStartedAt)
139
+ noProgressEnvelopeExhausted =
140
+ noHeaderAttempts >= MAX_NO_PROGRESS_ATTEMPTS || cumulativeNoProgressMs >= MAX_CUMULATIVE_NO_PROGRESS_MS
141
+ }
142
+ // A fallback reached before the cap may still recover; once the cap is
143
+ // reached, no same-ref replay or later ref may start.
144
+ if (noProgressEnvelopeExhausted) break
110
145
  // Retry within budget only when the failure is same-ref retryable and
111
146
  // either no visible/tool activity occurred, or the caller explicitly
112
147
  // authorizes the strict post-tool-result resume recipe.
@@ -133,7 +168,32 @@ export async function promptPersistentTurnWithFallback(opts: {
133
168
  attempts.push(attempt)
134
169
  lastError = outcome.error
135
170
  if (opts.shouldFailover(outcome.error)) circuit.recordThrottle({ profile: opts.profile, ref })
136
- if (isLast || !canAdvance(outcome.error, activity, opts.shouldFailover)) {
171
+ if (isCodexRef(ref) && isObserverTtfbTimeout(outcome.error.message)) {
172
+ circuit.recordProviderTrip({ profile: opts.profile, ref })
173
+ }
174
+ // A codex-only chain surfaces after one attempt once its breaker is open
175
+ // (no transport-diverse ref to fall to). isRefUsable can't express this —
176
+ // with no non-codex fallback every codex ref reads as "usable" — so it stays
177
+ // an explicit condition.
178
+ const codexOnlyProviderOpen =
179
+ !hasNonCodexFallback && isCodexRef(ref) && circuit.isProviderOpen({ profile: opts.profile, ref })
180
+ // `isLast` was computed BEFORE this attempt, but recordProviderTrip above
181
+ // may have just opened the codex breaker and made every trailing candidate
182
+ // unusable (all remaining refs are codex the loop would now skip). Re-derive
183
+ // "is there a ref left worth advancing to" from the post-trip circuit state,
184
+ // so we finish on the CURRENT outcome (throw hard / return soft) instead of
185
+ // falling through the loop and returning success:false for a ref that was
186
+ // skipped and never attempted.
187
+ const noUsableCandidateRemains = !opts.refs
188
+ .slice(i + 1)
189
+ .some((next) => isRefUsable(next, opts.profile, circuit, hasNonCodexFallback))
190
+ if (
191
+ noProgressEnvelopeExhausted ||
192
+ isLast ||
193
+ codexOnlyProviderOpen ||
194
+ noUsableCandidateRemains ||
195
+ !canAdvance(outcome.error, activity, opts.shouldFailover)
196
+ ) {
137
197
  if (outcome.kind === 'hard') throw outcome.error
138
198
  return { success: false, refUsed: ref, attempts, lastError }
139
199
  }
@@ -166,6 +226,26 @@ function canAdvance(error: Error, activity: AttemptActivity, shouldFailover: (er
166
226
  return shouldFailover(error) && !activity.producedAssistantOutput && !activity.startedToolExecution
167
227
  }
168
228
 
229
+ function isCodexRef(ref: ModelRef): boolean {
230
+ return providerForModelRef(ref) === 'openai-codex'
231
+ }
232
+
233
+ // A ref is still worth advancing to iff the loop wouldn't skip it at the top: a
234
+ // codex ref is dropped only once the codex provider breaker is open AND the chain
235
+ // has a transport-diverse (non-codex) fallback to use instead. Shared by the
236
+ // `isLast` safety-probe check and the post-trip terminal decision so both read
237
+ // the SAME circuit state and can't diverge.
238
+ function isRefUsable(
239
+ ref: ModelRef,
240
+ profile: string | undefined,
241
+ circuit: ThrottleCircuit,
242
+ hasNonCodexFallback: boolean,
243
+ ): boolean {
244
+ if (!isCodexRef(ref)) return true
245
+ if (!hasNonCodexFallback) return true
246
+ return !circuit.isProviderOpen({ profile, ref })
247
+ }
248
+
169
249
  function subscribeAttemptActivity(session: AgentSession, activity: AttemptActivity): () => void {
170
250
  const subscribe = (session as { subscribe?: unknown }).subscribe
171
251
  if (typeof subscribe !== 'function') return () => {}