typeclaw 0.45.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -2
- package/src/agent/git-nudge.ts +2 -1
- package/src/agent/index.ts +67 -25
- package/src/agent/live-subagents.ts +18 -0
- package/src/agent/loop-guard.ts +208 -10
- package/src/agent/multimodal/index.ts +1 -1
- package/src/agent/multimodal/look-at.ts +78 -44
- package/src/agent/multimodal/looker.ts +151 -39
- package/src/agent/network/response-body.ts +26 -0
- package/src/agent/network/safe-http.ts +198 -0
- package/src/agent/plugin-tools.ts +438 -162
- package/src/agent/provider-error.ts +62 -4
- package/src/agent/restart/index.ts +61 -40
- package/src/agent/restart-handoff/index.ts +73 -4
- package/src/agent/retry-same-ref.ts +66 -0
- package/src/agent/reviewer-bash-policy.ts +42 -1
- package/src/agent/session-origin.ts +2 -0
- package/src/agent/subagent-drain.ts +117 -7
- package/src/agent/subagents.ts +77 -18
- package/src/agent/throttle-circuit.ts +25 -5
- package/src/agent/tool-error-remediation.ts +72 -0
- package/src/agent/tool-file-safety.ts +1050 -0
- package/src/agent/tools/channel-fetch-attachment.ts +10 -3
- package/src/agent/tools/channel-reply.ts +15 -1
- package/src/agent/tools/channel-send.ts +17 -1
- package/src/agent/tools/ddg.ts +16 -2
- package/src/agent/tools/post-github-review.ts +199 -0
- package/src/agent/tools/spawn-subagent.ts +21 -18
- package/src/agent/tools/webfetch/fetch.ts +143 -183
- package/src/agent/tools/webfetch/tool.ts +2 -3
- package/src/agent/tools/websearch.ts +23 -2
- package/src/agent/tools-without-local-file-operands.ts +27 -0
- package/src/agent/turn-runner.ts +109 -15
- package/src/bundled-plugins/backup/index.ts +2 -1
- package/src/bundled-plugins/backup/runner.ts +2 -1
- package/src/bundled-plugins/doc-render/render.ts +93 -12
- package/src/bundled-plugins/doc-render/skills/typeclaw-render-pdf/SKILL.md +13 -13
- package/src/bundled-plugins/github-cli-auth/effective-approval.ts +16 -6
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +555 -15
- package/src/bundled-plugins/github-cli-auth/index.ts +131 -135
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +138 -0
- package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +2 -1
- package/src/bundled-plugins/memory/README.md +23 -11
- package/src/bundled-plugins/memory/append-tool.ts +15 -8
- package/src/bundled-plugins/memory/citations.ts +22 -9
- package/src/bundled-plugins/memory/dreaming.ts +13 -8
- package/src/bundled-plugins/memory/find-entry-tool.ts +4 -0
- package/src/bundled-plugins/memory/index.ts +0 -4
- package/src/bundled-plugins/memory/load-memory.ts +8 -3
- package/src/bundled-plugins/memory/provenance-index.ts +717 -0
- package/src/bundled-plugins/memory/provenance-sanitize.ts +40 -0
- package/src/bundled-plugins/memory/references/store-reference-tool.ts +4 -0
- package/src/bundled-plugins/memory/search-tool.ts +163 -60
- package/src/bundled-plugins/memory/stream-events.ts +2 -0
- package/src/bundled-plugins/memory/vector/hybrid.ts +126 -13
- package/src/bundled-plugins/researcher/write-report.ts +6 -3
- package/src/bundled-plugins/reviewer/index.ts +2 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +10 -3
- package/src/bundled-plugins/reviewer/skills/code-review.ts +24 -9
- package/src/bundled-plugins/scout/scout.ts +8 -0
- package/src/bundled-plugins/security/index.ts +4 -4
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +1 -0
- package/src/bundled-plugins/security/policies/private-surface-read.ts +220 -29
- package/src/bundled-plugins/security/policies/secret-exfil-read.ts +1 -0
- package/src/bundled-plugins/security/policies/ssrf.ts +29 -9
- package/src/channels/adapters/discord-bot-channel-resolver.ts +26 -14
- package/src/channels/adapters/discord-bot-thread-room.ts +44 -12
- package/src/channels/adapters/discord-bot.ts +91 -10
- package/src/channels/adapters/discord-channel-resolver.ts +91 -23
- package/src/channels/adapters/discord-id.ts +4 -0
- package/src/channels/adapters/discord-resolver-cache.ts +41 -0
- package/src/channels/adapters/discord.ts +57 -7
- package/src/channels/adapters/github/auth-app.ts +41 -12
- package/src/channels/adapters/github/index.ts +23 -11
- package/src/channels/adapters/github/review-submitter.ts +302 -0
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +12 -1
- package/src/channels/adapters/kakaotalk-fetch-attachment.ts +8 -4
- package/src/channels/adapters/kakaotalk-typing.ts +127 -0
- package/src/channels/adapters/kakaotalk.ts +19 -0
- package/src/channels/adapters/slack-attachment-download.ts +71 -0
- package/src/channels/adapters/slack-bot.ts +19 -8
- package/src/channels/adapters/slack-classify.ts +21 -2
- package/src/channels/adapters/slack.ts +59 -7
- package/src/channels/adapters/telegram-bot.ts +18 -5
- package/src/channels/adapters/webex-bot.ts +8 -3
- package/src/channels/adapters/webex.ts +8 -3
- package/src/channels/fetch-attachment.ts +43 -0
- package/src/{bundled-plugins/github-cli-auth/approve-idempotency.ts → channels/github-review-verdict-coordinator.ts} +25 -1
- package/src/channels/github-token-bridge.ts +10 -3
- package/src/channels/manager.ts +9 -0
- package/src/channels/router.ts +568 -12
- package/src/channels/types.ts +28 -1
- package/src/cli/channel.ts +1 -1
- package/src/cli/init.ts +1 -1
- package/src/cli/model.ts +57 -7
- package/src/compose/stop.ts +17 -5
- package/src/config/config.ts +6 -6
- package/src/config/models-mutation.ts +13 -1
- package/src/config/providers.ts +176 -0
- package/src/container/start.ts +49 -10
- package/src/doctor/checks.ts +3 -1
- package/src/doctor/commit.ts +2 -1
- package/src/doctor/file-ownership.ts +52 -0
- package/src/dreams/git.ts +2 -1
- package/src/git/hookless.ts +5 -0
- package/src/git/reconcile-ignored.ts +18 -5
- package/src/git/secret-history.ts +371 -0
- package/src/git/system-commit.ts +12 -6
- package/src/init/dockerfile.ts +86 -5
- package/src/init/index.ts +17 -5
- package/src/init/models-dev.ts +4 -0
- package/src/init/validate-api-key.ts +2 -0
- package/src/mcp/tools.ts +16 -3
- package/src/permissions/builtins.ts +3 -1
- package/src/plugin/context.ts +2 -0
- package/src/plugin/index.ts +1 -0
- package/src/plugin/manager.ts +4 -0
- package/src/plugin/types.ts +21 -0
- package/src/run/index.ts +40 -0
- package/src/run/llm-fetch-observer.ts +188 -13
- package/src/sandbox/build.ts +37 -15
- package/src/sandbox/canonical-secrets.ts +18 -0
- package/src/sandbox/env-exposure.ts +80 -0
- package/src/sandbox/errors.ts +13 -6
- package/src/sandbox/hidden-paths.ts +102 -44
- package/src/sandbox/index.ts +22 -4
- package/src/sandbox/package-install.ts +3 -27
- package/src/sandbox/policy.ts +11 -25
- package/src/sandbox/privileged-runtime.ts +255 -0
- package/src/sandbox/session-tmp.ts +8 -2
- package/src/sandbox/writable-zones.ts +204 -140
- package/src/secrets/index.ts +8 -0
- package/src/secrets/refresh-provider-oauth.ts +128 -0
- package/src/server/command-runner.ts +48 -4
- package/src/server/index.ts +4 -0
- package/src/skills/typeclaw-channel-github/SKILL.md +49 -56
- package/src/skills/typeclaw-claude-code/SKILL.md +24 -60
- package/src/skills/typeclaw-claude-code/references/auth-flow.md +17 -158
- package/src/skills/typeclaw-codex-cli/SKILL.md +19 -49
- package/src/skills/typeclaw-codex-cli/references/auth-flow.md +17 -154
- package/src/skills/typeclaw-config/SKILL.md +5 -5
- package/src/skills/typeclaw-config/references/recommended-mounts.md +24 -24
- package/src/skills/typeclaw-github-contributing/SKILL.md +23 -26
- package/src/skills/typeclaw-memory/SKILL.md +9 -3
- package/src/skills/typeclaw-permissions/SKILL.md +19 -15
- package/typeclaw.schema.json +20 -0
|
@@ -1,38 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
// Production path (container, curl-impersonate available): we shell out to
|
|
4
|
-
// `curl_chrome136` so outbound requests carry Chrome 136's TLS handshake
|
|
5
|
-
// (JA3/JA4), HTTP/2 SETTINGS frame, and full header set. This is what gets
|
|
6
|
-
// us past the modern bot-detection stacks on Cloudflare/Akamai-protected
|
|
7
|
-
// sites (Reuters, MarketWatch, etc.) when the agent is running from the
|
|
8
|
-
// user's home network — the IP is already residential, so impersonating
|
|
9
|
-
// the browser is the only remaining missing piece. See AGENTS.md §"Web
|
|
10
|
-
// search" and src/agent/tools/curl-impersonate.ts for the full story.
|
|
11
|
-
//
|
|
12
|
-
// Test/dev fallback (curl_chrome136 not on PATH): we transparently fall
|
|
13
|
-
// back to Bun's native `fetch()` with a static User-Agent. This keeps unit
|
|
14
|
-
// tests on developer macOS machines working without forcing every contributor
|
|
15
|
-
// to install curl-impersonate locally. Production runs always have the binary
|
|
16
|
-
// because the typeclaw Dockerfile pins it.
|
|
17
|
-
//
|
|
18
|
-
// Best-effort doctrine: this transport does NOT guarantee the fetch succeeds.
|
|
19
|
-
// Bot-detected sites can still serve 403/CAPTCHA pages. We surface what we
|
|
20
|
-
// got (status, body, final URL) and let the caller decide. The web_fetch tool
|
|
21
|
-
// translates non-2xx into a tool-level error message that's useful to the
|
|
22
|
-
// model.
|
|
23
|
-
|
|
24
|
-
import { mkdtemp, rm } from 'node:fs/promises'
|
|
25
|
-
import { tmpdir } from 'node:os'
|
|
26
|
-
import { join } from 'node:path'
|
|
1
|
+
import { CookieJar } from 'tough-cookie'
|
|
27
2
|
|
|
28
3
|
import {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
4
|
+
defaultPublicHttpDependencies,
|
|
5
|
+
headerValue,
|
|
6
|
+
requestPublicHttpUrl,
|
|
7
|
+
type PublicHttpDependencies,
|
|
8
|
+
type PublicHttpResponse,
|
|
9
|
+
} from '@/agent/network/safe-http'
|
|
10
|
+
|
|
36
11
|
import { MAX_RESPONSE_BYTES } from './types'
|
|
37
12
|
|
|
38
13
|
export type AntibotWarmup = 'auto' | 'off'
|
|
@@ -54,17 +29,9 @@ export type FetchResult = {
|
|
|
54
29
|
antibotWarmup?: AntibotWarmupInfo
|
|
55
30
|
}
|
|
56
31
|
|
|
57
|
-
|
|
58
|
-
// A 403 carrying one of these is a state-bootstrap handshake (absorb cookies,
|
|
59
|
-
// replay once), NOT an authorization failure — that distinction is what makes
|
|
60
|
-
// the auto-retry safe. Scoped to Akamai on purpose: Cloudflare/DataDome use
|
|
61
|
-
// different challenge semantics (JS/redirects/CAPTCHA) that a blind replay
|
|
62
|
-
// would not satisfy.
|
|
63
|
-
const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
|
|
32
|
+
export type WebFetchNetworkDependencies = PublicHttpDependencies
|
|
64
33
|
|
|
65
|
-
|
|
66
|
-
return response.httpStatus === 403 && response.setCookieNames.some((name) => AKAMAI_BOTMANAGER_COOKIE.test(name))
|
|
67
|
-
}
|
|
34
|
+
const AKAMAI_BOTMANAGER_COOKIE = /^(_abck|bm_sz|bm_s|bm_ss|bm_so|ak_bmsc)$/i
|
|
68
35
|
|
|
69
36
|
export class WebFetchError extends Error {
|
|
70
37
|
constructor(message: string) {
|
|
@@ -73,7 +40,7 @@ export class WebFetchError extends Error {
|
|
|
73
40
|
}
|
|
74
41
|
}
|
|
75
42
|
|
|
76
|
-
const
|
|
43
|
+
const REQUEST_HEADERS: Record<string, string> = {
|
|
77
44
|
'User-Agent': 'typeclaw/0 (+https://github.com/code-yeongyu/typeclaw)',
|
|
78
45
|
Accept: 'text/html,application/xhtml+xml,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1',
|
|
79
46
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
@@ -90,10 +57,6 @@ export function normalizeUrl(input: string): string {
|
|
|
90
57
|
return `https://${trimmed}`
|
|
91
58
|
}
|
|
92
59
|
|
|
93
|
-
// Test-only seam: forces fetchWithLimits to use the native-fetch fallback
|
|
94
|
-
// even when curl-impersonate is detected. Used by fetch.test.ts to keep its
|
|
95
|
-
// existing mocked-fetch contract working without the test having to install
|
|
96
|
-
// a fake curl binary. Production code never calls this.
|
|
97
60
|
let forceFallbackForTest = false
|
|
98
61
|
|
|
99
62
|
export function _setForceFallbackForTest(value: boolean): void {
|
|
@@ -105,175 +68,172 @@ export async function fetchWithLimits(
|
|
|
105
68
|
timeoutSeconds: number,
|
|
106
69
|
parentSignal?: AbortSignal,
|
|
107
70
|
antibotWarmup: AntibotWarmup = 'auto',
|
|
71
|
+
network: WebFetchNetworkDependencies = forceFallbackForTest ? testFetchDependencies : defaultPublicHttpDependencies,
|
|
108
72
|
): Promise<FetchResult> {
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
return fetchWithBunFetch(url, timeoutSeconds, parentSignal)
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
async function fetchWithCurlImpersonate(
|
|
117
|
-
url: string,
|
|
118
|
-
timeoutSeconds: number,
|
|
119
|
-
antibotWarmup: AntibotWarmup,
|
|
120
|
-
parentSignal?: AbortSignal,
|
|
121
|
-
): Promise<FetchResult> {
|
|
122
|
-
if (antibotWarmup === 'off') {
|
|
123
|
-
const response = await runCurl(url, timeoutSeconds, undefined, parentSignal)
|
|
124
|
-
return toFetchResult(url, response)
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// Warmup needs a jar shared across both requests: request 1 (`-c`) captures
|
|
128
|
-
// the bot-manager cookies the 403 sets; request 2 (`-b`) replays them.
|
|
129
|
-
const jarDir = await mkdtemp(join(tmpdir(), 'typeclaw-webfetch-jar-'))
|
|
130
|
-
const jarPath = join(jarDir, 'cookies.txt')
|
|
73
|
+
const controller = new AbortController()
|
|
74
|
+
const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
|
|
75
|
+
const onAbort = () => controller.abort(parentSignal?.reason)
|
|
76
|
+
parentSignal?.addEventListener('abort', onAbort, { once: true })
|
|
131
77
|
const startedAt = Date.now()
|
|
132
78
|
try {
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
79
|
+
const cookieJar = new WarmupCookieJar()
|
|
80
|
+
const visitedUrls = new Set<string>()
|
|
81
|
+
const first = await fetchOnce(url, controller.signal, network, REQUEST_HEADERS, cookieJar, visitedUrls)
|
|
82
|
+
const cookieNames = cookieJar.names()
|
|
83
|
+
const warmup =
|
|
84
|
+
antibotWarmup === 'auto' &&
|
|
85
|
+
first.statusCode === 403 &&
|
|
86
|
+
[...visitedUrls].some((visitedUrl) => cookieJar.hasMatching(visitedUrl, AKAMAI_BOTMANAGER_COOKIE))
|
|
87
|
+
if (!warmup) {
|
|
88
|
+
return toFetchResult(first, { attempted: antibotWarmup === 'auto', triggered: false })
|
|
136
89
|
}
|
|
137
90
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
// request already spent the budget, surface its 403 instead of forcing a
|
|
142
|
-
// zero-time replay that would just time out.
|
|
143
|
-
const remaining = timeoutSeconds - (Date.now() - startedAt) / 1000
|
|
144
|
-
if (remaining < 1) {
|
|
145
|
-
return toFetchResult(url, first, {
|
|
91
|
+
const remainingMs = timeoutSeconds * 1000 - (Date.now() - startedAt)
|
|
92
|
+
if (remainingMs < 1_000) {
|
|
93
|
+
return toFetchResult(first, {
|
|
146
94
|
attempted: true,
|
|
147
95
|
triggered: false,
|
|
148
|
-
initialStatus: first.
|
|
149
|
-
initialSetCookieNames:
|
|
96
|
+
initialStatus: first.statusCode,
|
|
97
|
+
initialSetCookieNames: cookieNames,
|
|
150
98
|
})
|
|
151
99
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
return toFetchResult(url, replay, {
|
|
100
|
+
const replay = await fetchOnce(url, controller.signal, network, REQUEST_HEADERS, cookieJar)
|
|
101
|
+
return toFetchResult(replay, {
|
|
155
102
|
attempted: true,
|
|
156
103
|
triggered: true,
|
|
157
|
-
initialStatus: first.
|
|
158
|
-
initialSetCookieNames:
|
|
159
|
-
replayStatus: replay.
|
|
104
|
+
initialStatus: first.statusCode,
|
|
105
|
+
initialSetCookieNames: cookieNames,
|
|
106
|
+
replayStatus: replay.statusCode,
|
|
160
107
|
})
|
|
108
|
+
} catch (error) {
|
|
109
|
+
if (controller.signal.aborted) {
|
|
110
|
+
if (controller.signal.reason instanceof Error && controller.signal.reason.message === 'timeout') {
|
|
111
|
+
throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
|
|
112
|
+
}
|
|
113
|
+
throw new WebFetchError('Request aborted')
|
|
114
|
+
}
|
|
115
|
+
if (error instanceof WebFetchError) throw error
|
|
116
|
+
throw new WebFetchError(`Fetch failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
161
117
|
} finally {
|
|
162
|
-
|
|
118
|
+
clearTimeout(timeout)
|
|
119
|
+
parentSignal?.removeEventListener('abort', onAbort)
|
|
163
120
|
}
|
|
164
121
|
}
|
|
165
122
|
|
|
166
|
-
|
|
123
|
+
const testFetchDependencies: WebFetchNetworkDependencies = {
|
|
124
|
+
resolveAddresses: async () => [{ address: '93.184.216.34', family: 4 }],
|
|
125
|
+
async request(options) {
|
|
126
|
+
const url = `${options.protocol}//${options.headers.Host}${options.path}`
|
|
127
|
+
const response = await fetch(url, { headers: options.headers, signal: options.signal, redirect: 'manual' })
|
|
128
|
+
return {
|
|
129
|
+
statusCode: response.status,
|
|
130
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
131
|
+
body: response.body ?? emptyBody(),
|
|
132
|
+
cancel: () => {
|
|
133
|
+
void response.body?.cancel()
|
|
134
|
+
},
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function* emptyBody(): AsyncIterable<Uint8Array> {}
|
|
140
|
+
|
|
141
|
+
type BufferedResponse = {
|
|
142
|
+
body: Uint8Array
|
|
143
|
+
headers: PublicHttpResponse['headers']
|
|
144
|
+
statusCode: number
|
|
145
|
+
finalUrl: string
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function fetchOnce(
|
|
167
149
|
url: string,
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
150
|
+
signal: AbortSignal,
|
|
151
|
+
network: WebFetchNetworkDependencies,
|
|
152
|
+
headers: Record<string, string>,
|
|
153
|
+
cookieJar?: WarmupCookieJar,
|
|
154
|
+
visitedUrls?: Set<string>,
|
|
155
|
+
): Promise<BufferedResponse> {
|
|
156
|
+
const { response, finalUrl } = await requestPublicHttpUrl(url, {
|
|
157
|
+
signal,
|
|
158
|
+
headers,
|
|
159
|
+
dependencies: network,
|
|
160
|
+
headersForUrl(currentUrl): Record<string, string> {
|
|
161
|
+
visitedUrls?.add(currentUrl)
|
|
162
|
+
const cookie = cookieJar?.header(currentUrl)
|
|
163
|
+
if (cookie === undefined) return {}
|
|
164
|
+
return { Cookie: cookie }
|
|
165
|
+
},
|
|
166
|
+
onResponse(response, responseUrl) {
|
|
167
|
+
cookieJar?.store(response.headers, responseUrl)
|
|
168
|
+
},
|
|
169
|
+
})
|
|
172
170
|
try {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
timeoutSeconds,
|
|
177
|
-
maxBytes: MAX_RESPONSE_BYTES,
|
|
178
|
-
cookieJarPath,
|
|
179
|
-
signal: parentSignal,
|
|
180
|
-
})
|
|
181
|
-
} catch (error) {
|
|
182
|
-
if (parentSignal?.aborted) {
|
|
183
|
-
throw new WebFetchError('Request aborted')
|
|
171
|
+
const declared = Number(headerValue(response.headers, 'content-length') ?? '')
|
|
172
|
+
if (Number.isFinite(declared) && declared > MAX_RESPONSE_BYTES) {
|
|
173
|
+
throw tooLarge(declared)
|
|
184
174
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
if (
|
|
190
|
-
|
|
191
|
-
}
|
|
192
|
-
throw new WebFetchError(`Fetch failed: ${error.message}`)
|
|
175
|
+
const chunks: Uint8Array[] = []
|
|
176
|
+
let total = 0
|
|
177
|
+
for await (const chunk of response.body) {
|
|
178
|
+
total += chunk.byteLength
|
|
179
|
+
if (total > MAX_RESPONSE_BYTES) throw tooLarge(total)
|
|
180
|
+
chunks.push(chunk)
|
|
193
181
|
}
|
|
194
|
-
|
|
195
|
-
|
|
182
|
+
return { body: Buffer.concat(chunks), headers: response.headers, statusCode: response.statusCode, finalUrl }
|
|
183
|
+
} finally {
|
|
184
|
+
response.cancel()
|
|
196
185
|
}
|
|
197
186
|
}
|
|
198
187
|
|
|
199
|
-
function toFetchResult(
|
|
200
|
-
if (response.
|
|
201
|
-
throw new WebFetchError(`Fetch failed: HTTP ${response.
|
|
188
|
+
function toFetchResult(response: BufferedResponse, antibotWarmup?: AntibotWarmupInfo): FetchResult {
|
|
189
|
+
if (response.statusCode < 200 || response.statusCode >= 300) {
|
|
190
|
+
throw new WebFetchError(`Fetch failed: HTTP ${response.statusCode}`)
|
|
202
191
|
}
|
|
203
|
-
|
|
204
|
-
const bodyByteLength = new TextEncoder().encode(response.body).byteLength
|
|
205
|
-
if (bodyByteLength > MAX_RESPONSE_BYTES) {
|
|
206
|
-
throw new WebFetchError(
|
|
207
|
-
`Response too large (${formatBytes(bodyByteLength)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
|
|
208
|
-
)
|
|
209
|
-
}
|
|
210
|
-
|
|
211
192
|
return {
|
|
212
|
-
body: response.body,
|
|
213
|
-
contentType: response.
|
|
214
|
-
finalUrl: response.finalUrl
|
|
215
|
-
httpStatus: response.
|
|
216
|
-
bytesIn:
|
|
217
|
-
...(antibotWarmup ? { antibotWarmup } : {}),
|
|
193
|
+
body: new TextDecoder('utf-8', { fatal: false }).decode(response.body),
|
|
194
|
+
contentType: headerValue(response.headers, 'content-type') ?? '',
|
|
195
|
+
finalUrl: response.finalUrl,
|
|
196
|
+
httpStatus: response.statusCode,
|
|
197
|
+
bytesIn: response.body.byteLength,
|
|
198
|
+
...(antibotWarmup !== undefined ? { antibotWarmup } : {}),
|
|
218
199
|
}
|
|
219
200
|
}
|
|
220
201
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
const controller = new AbortController()
|
|
227
|
-
const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
|
|
228
|
-
const onAbort = () => controller.abort(parentSignal?.reason)
|
|
229
|
-
parentSignal?.addEventListener('abort', onAbort, { once: true })
|
|
202
|
+
function setCookieValues(headers: PublicHttpResponse['headers']): string[] {
|
|
203
|
+
const value = headers['set-cookie']
|
|
204
|
+
if (Array.isArray(value)) return value
|
|
205
|
+
return value === undefined ? [] : [value]
|
|
206
|
+
}
|
|
230
207
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
throw new WebFetchError(`Fetch failed: HTTP ${response.status} ${response.statusText}`)
|
|
235
|
-
}
|
|
208
|
+
class WarmupCookieJar {
|
|
209
|
+
readonly #jar = new CookieJar()
|
|
210
|
+
readonly #names = new Set<string>()
|
|
236
211
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const
|
|
240
|
-
if (
|
|
241
|
-
throw new WebFetchError(
|
|
242
|
-
`Response too large (${formatBytes(declared)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
|
|
243
|
-
)
|
|
244
|
-
}
|
|
212
|
+
store(headers: PublicHttpResponse['headers'], responseUrl: string): void {
|
|
213
|
+
for (const value of setCookieValues(headers)) {
|
|
214
|
+
const cookie = this.#jar.setCookieSync(value, responseUrl, { ignoreError: true })
|
|
215
|
+
if (cookie !== undefined) this.#names.add(cookie.key)
|
|
245
216
|
}
|
|
217
|
+
}
|
|
246
218
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
`Response too large (${formatBytes(buffer.byteLength)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
|
|
251
|
-
)
|
|
252
|
-
}
|
|
219
|
+
names(): string[] {
|
|
220
|
+
return [...this.#names]
|
|
221
|
+
}
|
|
253
222
|
|
|
254
|
-
|
|
255
|
-
return
|
|
256
|
-
body,
|
|
257
|
-
contentType: response.headers.get('content-type') ?? '',
|
|
258
|
-
finalUrl: response.url || url,
|
|
259
|
-
httpStatus: response.status,
|
|
260
|
-
bytesIn: buffer.byteLength,
|
|
261
|
-
}
|
|
262
|
-
} catch (error) {
|
|
263
|
-
if (
|
|
264
|
-
controller.signal.aborted &&
|
|
265
|
-
controller.signal.reason instanceof Error &&
|
|
266
|
-
controller.signal.reason.message === 'timeout'
|
|
267
|
-
) {
|
|
268
|
-
throw new WebFetchError(`Request timed out after ${timeoutSeconds}s`)
|
|
269
|
-
}
|
|
270
|
-
if (error instanceof WebFetchError) throw error
|
|
271
|
-
const message = error instanceof Error ? error.message : String(error)
|
|
272
|
-
throw new WebFetchError(`Fetch failed: ${message}`)
|
|
273
|
-
} finally {
|
|
274
|
-
clearTimeout(timeout)
|
|
275
|
-
parentSignal?.removeEventListener('abort', onAbort)
|
|
223
|
+
hasMatching(rawUrl: string, namePattern: RegExp): boolean {
|
|
224
|
+
return this.#jar.getCookiesSync(rawUrl).some((cookie) => namePattern.test(cookie.key))
|
|
276
225
|
}
|
|
226
|
+
|
|
227
|
+
header(rawUrl: string): string | undefined {
|
|
228
|
+
const header = this.#jar.getCookieStringSync(rawUrl)
|
|
229
|
+
return header === '' ? undefined : header
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function tooLarge(bytes: number): WebFetchError {
|
|
234
|
+
return new WebFetchError(
|
|
235
|
+
`Response too large (${formatBytes(bytes)} exceeds ${formatBytes(MAX_RESPONSE_BYTES)} limit)`,
|
|
236
|
+
)
|
|
277
237
|
}
|
|
278
238
|
|
|
279
239
|
export function parseMimeType(contentType: string): string {
|
|
@@ -25,8 +25,7 @@ export const webFetchTool = defineTool({
|
|
|
25
25
|
'Fetch a single HTTP(S) URL and return the body, optionally compacted by a strategy. ' +
|
|
26
26
|
'Use this when the user references a specific URL or when web_search surfaced a result you need to read in full. ' +
|
|
27
27
|
'If `spawn_subagent` is available to you, PREFER delegating to the `scout` subagent by default: spawn it whenever you expect more than one fetch, an "across multiple sources" task, or any search-then-fetch loop. Scout runs the noisy fetching in its own context window and returns a distilled, citation-backed answer, keeping bulky page bodies out of yours. Only call this tool directly for a single known URL whose content you will cite immediately — or whenever you cannot spawn subagents (e.g. you are yourself a subagent), in which case fetch here. ' +
|
|
28
|
-
'Outbound requests
|
|
29
|
-
'(via curl-impersonate), which helps with TLS/header fingerprint gates on sites behind Cloudflare/Akamai. ' +
|
|
28
|
+
'Outbound requests use a DNS-rebinding-safe transport that validates and pins every redirect hop. ' +
|
|
30
29
|
'For Akamai Bot Manager specifically, it auto-warms the session: a first-request 403 that sets bot-manager ' +
|
|
31
30
|
'cookies (_abck/bm_sz/…) is absorbed and replayed once, which returns 200 on many Akamai-fronted sites ' +
|
|
32
31
|
'(disable with `antibotWarmup: "off"`). It still does NOT solve JavaScript challenges, behavioural ' +
|
|
@@ -40,7 +39,7 @@ export const webFetchTool = defineTool({
|
|
|
40
39
|
'- "snapshot": indented semantic tree of the page (forms, headings, links).\n' +
|
|
41
40
|
'- "raw": no processing.\n' +
|
|
42
41
|
'If `strategy` is omitted, it is inferred from content-type. JSON responses require explicit `strategy: "jq"` (or "raw"). ' +
|
|
43
|
-
'
|
|
42
|
+
'Private, mixed-address, and non-HTTP(S) targets are rejected.',
|
|
44
43
|
parameters: Type.Object({
|
|
45
44
|
url: Type.String({
|
|
46
45
|
description: 'URL to fetch (http:// or https://). Bare hostnames are rewritten to https://.',
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Type } from '@mariozechner/pi-ai'
|
|
2
2
|
import { defineTool } from '@mariozechner/pi-coding-agent'
|
|
3
3
|
|
|
4
|
+
import { CurlImpersonateError, isCurlExitTimeout } from './curl-impersonate'
|
|
4
5
|
import { ddgSearch, DdgCaptchaError, type DdgResult } from './ddg'
|
|
5
6
|
import { wikipediaSearch, type WikipediaResult } from './wikipedia'
|
|
6
7
|
|
|
@@ -51,8 +52,17 @@ export const webSearchTool = defineTool({
|
|
|
51
52
|
source === 'wikipedia' ? await wikipediaSearch(query, limit, signal) : await ddgSearch(query, limit, signal)
|
|
52
53
|
return successResult(query, source, results)
|
|
53
54
|
} catch (error) {
|
|
54
|
-
|
|
55
|
-
|
|
55
|
+
// Both a CAPTCHA and a connection timeout are transient failures that
|
|
56
|
+
// ddgSearch already retried under a bounded backoff. The model has no
|
|
57
|
+
// visibility into those spent retries, so any message that hints "try
|
|
58
|
+
// again" (the old CAPTCHA text did) invites it to loop the tool with
|
|
59
|
+
// varied queries — the exact churn this change removes. Give both the same
|
|
60
|
+
// terminal "do not retry now" contract so an exhausted transient failure
|
|
61
|
+
// ends the turn with partial results instead.
|
|
62
|
+
if (isExhaustedTransientSearchError(error)) {
|
|
63
|
+
return errorResult(
|
|
64
|
+
`Search unavailable for "${query}" after retries (${transientReason(error)}). Do not retry this tool now — report what you already have and note the search was unreachable.`,
|
|
65
|
+
)
|
|
56
66
|
}
|
|
57
67
|
const message = error instanceof Error ? error.message : String(error)
|
|
58
68
|
return errorResult(`Search failed: ${message}`)
|
|
@@ -60,6 +70,17 @@ export const webSearchTool = defineTool({
|
|
|
60
70
|
},
|
|
61
71
|
})
|
|
62
72
|
|
|
73
|
+
function isExhaustedTransientSearchError(error: unknown): boolean {
|
|
74
|
+
if (error instanceof DdgCaptchaError) return true
|
|
75
|
+
if (error instanceof CurlImpersonateError && isCurlExitTimeout(error)) return true
|
|
76
|
+
return false
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function transientReason(error: unknown): string {
|
|
80
|
+
if (error instanceof DdgCaptchaError) return 'rate-limited'
|
|
81
|
+
return 'network timeout'
|
|
82
|
+
}
|
|
83
|
+
|
|
63
84
|
function clampLimit(value: number | undefined): number {
|
|
64
85
|
if (value === undefined) return DEFAULT_LIMIT
|
|
65
86
|
return Math.min(Math.max(1, Math.floor(value)), MAX_LIMIT)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// First-party system tools whose EVERY argument is a control token or a remote
|
|
2
|
+
// identifier (channel/workspace/message/thread/cursor ids, subagent names, task
|
|
3
|
+
// ids, role capabilities, reload/stream scopes) — none is ever dereferenced as a
|
|
4
|
+
// local path. Listing the whole tool is more robust than enumerating each
|
|
5
|
+
// operand: these schemas gain new id/enum fields often, and every such field
|
|
6
|
+
// hits the same false positives (a Slack `ts` like "1699999999.000100" matches
|
|
7
|
+
// the `word.ext` rule; a value equal to an agent-root dir like "memory" hits the
|
|
8
|
+
// fs-existence probe; a base64 cursor carries a "/"). The security cost is zero
|
|
9
|
+
// TODAY because none reads a local file; the risk is future drift, so this set
|
|
10
|
+
// is the single fence, shared by BOTH enforcement points — the file-operand
|
|
11
|
+
// scanner (tool-file-safety) and the private-surface-read guard. It lives in
|
|
12
|
+
// this leaf module (no imports) so those two modules, which import each other,
|
|
13
|
+
// can both read it without a circular-init TDZ. Add a tool here ONLY after
|
|
14
|
+
// confirming it dereferences no argument as a local path.
|
|
15
|
+
export const TOOLS_WITHOUT_LOCAL_FILE_OPERANDS: ReadonlySet<string> = new Set([
|
|
16
|
+
'channel_read',
|
|
17
|
+
'channel_history',
|
|
18
|
+
'channel_edit',
|
|
19
|
+
'channel_react',
|
|
20
|
+
'look_at_channel_attachment',
|
|
21
|
+
'stream_snapshot',
|
|
22
|
+
'grant_role',
|
|
23
|
+
'spawn_subagent',
|
|
24
|
+
'subagent_cancel',
|
|
25
|
+
'subagent_output',
|
|
26
|
+
'reload',
|
|
27
|
+
])
|