typeclaw 0.45.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -2
- package/src/agent/git-nudge.ts +2 -1
- package/src/agent/index.ts +67 -25
- package/src/agent/live-subagents.ts +18 -0
- package/src/agent/loop-guard.ts +208 -10
- package/src/agent/multimodal/index.ts +1 -1
- package/src/agent/multimodal/look-at.ts +78 -44
- package/src/agent/multimodal/looker.ts +151 -39
- package/src/agent/network/response-body.ts +26 -0
- package/src/agent/network/safe-http.ts +198 -0
- package/src/agent/plugin-tools.ts +438 -162
- package/src/agent/provider-error.ts +62 -4
- package/src/agent/restart/index.ts +61 -40
- package/src/agent/restart-handoff/index.ts +73 -4
- package/src/agent/retry-same-ref.ts +66 -0
- package/src/agent/reviewer-bash-policy.ts +42 -1
- package/src/agent/session-origin.ts +2 -0
- package/src/agent/subagent-drain.ts +117 -7
- package/src/agent/subagents.ts +77 -18
- package/src/agent/throttle-circuit.ts +25 -5
- package/src/agent/tool-error-remediation.ts +72 -0
- package/src/agent/tool-file-safety.ts +1050 -0
- package/src/agent/tools/channel-fetch-attachment.ts +10 -3
- package/src/agent/tools/channel-reply.ts +15 -1
- package/src/agent/tools/channel-send.ts +17 -1
- package/src/agent/tools/ddg.ts +16 -2
- package/src/agent/tools/post-github-review.ts +199 -0
- package/src/agent/tools/spawn-subagent.ts +21 -18
- package/src/agent/tools/webfetch/fetch.ts +143 -183
- package/src/agent/tools/webfetch/tool.ts +2 -3
- package/src/agent/tools/websearch.ts +23 -2
- package/src/agent/tools-without-local-file-operands.ts +27 -0
- package/src/agent/turn-runner.ts +109 -15
- package/src/bundled-plugins/backup/index.ts +2 -1
- package/src/bundled-plugins/backup/runner.ts +2 -1
- package/src/bundled-plugins/doc-render/render.ts +93 -12
- package/src/bundled-plugins/doc-render/skills/typeclaw-render-pdf/SKILL.md +13 -13
- package/src/bundled-plugins/github-cli-auth/effective-approval.ts +16 -6
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +555 -15
- package/src/bundled-plugins/github-cli-auth/index.ts +131 -135
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +138 -0
- package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +2 -1
- package/src/bundled-plugins/memory/README.md +23 -11
- package/src/bundled-plugins/memory/append-tool.ts +15 -8
- package/src/bundled-plugins/memory/citations.ts +22 -9
- package/src/bundled-plugins/memory/dreaming.ts +13 -8
- package/src/bundled-plugins/memory/find-entry-tool.ts +4 -0
- package/src/bundled-plugins/memory/index.ts +0 -4
- package/src/bundled-plugins/memory/load-memory.ts +8 -3
- package/src/bundled-plugins/memory/provenance-index.ts +717 -0
- package/src/bundled-plugins/memory/provenance-sanitize.ts +40 -0
- package/src/bundled-plugins/memory/references/store-reference-tool.ts +4 -0
- package/src/bundled-plugins/memory/search-tool.ts +163 -60
- package/src/bundled-plugins/memory/stream-events.ts +2 -0
- package/src/bundled-plugins/memory/vector/hybrid.ts +126 -13
- package/src/bundled-plugins/researcher/write-report.ts +6 -3
- package/src/bundled-plugins/reviewer/index.ts +2 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +10 -3
- package/src/bundled-plugins/reviewer/skills/code-review.ts +24 -9
- package/src/bundled-plugins/scout/scout.ts +8 -0
- package/src/bundled-plugins/security/index.ts +4 -4
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +1 -0
- package/src/bundled-plugins/security/policies/private-surface-read.ts +220 -29
- package/src/bundled-plugins/security/policies/secret-exfil-read.ts +1 -0
- package/src/bundled-plugins/security/policies/ssrf.ts +29 -9
- package/src/channels/adapters/discord-bot-channel-resolver.ts +26 -14
- package/src/channels/adapters/discord-bot-thread-room.ts +44 -12
- package/src/channels/adapters/discord-bot.ts +91 -10
- package/src/channels/adapters/discord-channel-resolver.ts +91 -23
- package/src/channels/adapters/discord-id.ts +4 -0
- package/src/channels/adapters/discord-resolver-cache.ts +41 -0
- package/src/channels/adapters/discord.ts +57 -7
- package/src/channels/adapters/github/auth-app.ts +41 -12
- package/src/channels/adapters/github/index.ts +23 -11
- package/src/channels/adapters/github/review-submitter.ts +302 -0
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +12 -1
- package/src/channels/adapters/kakaotalk-fetch-attachment.ts +8 -4
- package/src/channels/adapters/kakaotalk-typing.ts +127 -0
- package/src/channels/adapters/kakaotalk.ts +19 -0
- package/src/channels/adapters/slack-attachment-download.ts +71 -0
- package/src/channels/adapters/slack-bot.ts +19 -8
- package/src/channels/adapters/slack-classify.ts +21 -2
- package/src/channels/adapters/slack.ts +59 -7
- package/src/channels/adapters/telegram-bot.ts +18 -5
- package/src/channels/adapters/webex-bot.ts +8 -3
- package/src/channels/adapters/webex.ts +8 -3
- package/src/channels/fetch-attachment.ts +43 -0
- package/src/{bundled-plugins/github-cli-auth/approve-idempotency.ts → channels/github-review-verdict-coordinator.ts} +25 -1
- package/src/channels/github-token-bridge.ts +10 -3
- package/src/channels/manager.ts +9 -0
- package/src/channels/router.ts +568 -12
- package/src/channels/types.ts +28 -1
- package/src/cli/channel.ts +1 -1
- package/src/cli/init.ts +1 -1
- package/src/cli/model.ts +57 -7
- package/src/compose/stop.ts +17 -5
- package/src/config/config.ts +6 -6
- package/src/config/models-mutation.ts +13 -1
- package/src/config/providers.ts +176 -0
- package/src/container/start.ts +49 -10
- package/src/doctor/checks.ts +3 -1
- package/src/doctor/commit.ts +2 -1
- package/src/doctor/file-ownership.ts +52 -0
- package/src/dreams/git.ts +2 -1
- package/src/git/hookless.ts +5 -0
- package/src/git/reconcile-ignored.ts +18 -5
- package/src/git/secret-history.ts +371 -0
- package/src/git/system-commit.ts +12 -6
- package/src/init/dockerfile.ts +86 -5
- package/src/init/index.ts +17 -5
- package/src/init/models-dev.ts +4 -0
- package/src/init/validate-api-key.ts +2 -0
- package/src/mcp/tools.ts +16 -3
- package/src/permissions/builtins.ts +3 -1
- package/src/plugin/context.ts +2 -0
- package/src/plugin/index.ts +1 -0
- package/src/plugin/manager.ts +4 -0
- package/src/plugin/types.ts +21 -0
- package/src/run/index.ts +40 -0
- package/src/run/llm-fetch-observer.ts +188 -13
- package/src/sandbox/build.ts +37 -15
- package/src/sandbox/canonical-secrets.ts +18 -0
- package/src/sandbox/env-exposure.ts +80 -0
- package/src/sandbox/errors.ts +13 -6
- package/src/sandbox/hidden-paths.ts +102 -44
- package/src/sandbox/index.ts +22 -4
- package/src/sandbox/package-install.ts +3 -27
- package/src/sandbox/policy.ts +11 -25
- package/src/sandbox/privileged-runtime.ts +255 -0
- package/src/sandbox/session-tmp.ts +8 -2
- package/src/sandbox/writable-zones.ts +204 -140
- package/src/secrets/index.ts +8 -0
- package/src/secrets/refresh-provider-oauth.ts +128 -0
- package/src/server/command-runner.ts +48 -4
- package/src/server/index.ts +4 -0
- package/src/skills/typeclaw-channel-github/SKILL.md +49 -56
- package/src/skills/typeclaw-claude-code/SKILL.md +24 -60
- package/src/skills/typeclaw-claude-code/references/auth-flow.md +17 -158
- package/src/skills/typeclaw-codex-cli/SKILL.md +19 -49
- package/src/skills/typeclaw-codex-cli/references/auth-flow.md +17 -154
- package/src/skills/typeclaw-config/SKILL.md +5 -5
- package/src/skills/typeclaw-config/references/recommended-mounts.md +24 -24
- package/src/skills/typeclaw-github-contributing/SKILL.md +23 -26
- package/src/skills/typeclaw-memory/SKILL.md +9 -3
- package/src/skills/typeclaw-permissions/SKILL.md +19 -15
- package/typeclaw.schema.json +20 -0
|
@@ -5,15 +5,23 @@ import type { ImageContent } from '@mariozechner/pi-ai'
|
|
|
5
5
|
import { defineTool } from '@mariozechner/pi-coding-agent'
|
|
6
6
|
|
|
7
7
|
import { createSessionWithDispose, type SessionOrigin } from '@/agent'
|
|
8
|
+
import { readResponseBodyBounded } from '@/agent/network/response-body'
|
|
8
9
|
import type { ChannelRouter } from '@/channels/router'
|
|
9
10
|
import type { AdapterId } from '@/channels/schema'
|
|
10
11
|
import { getConfig, resolveModel, resolveProfile } from '@/config'
|
|
11
12
|
import { providerForModelRef } from '@/config/providers'
|
|
13
|
+
import type { PermissionService } from '@/permissions'
|
|
12
14
|
import { LLM_FETCH_OBSERVER_TIMEOUTS, type LlmFetchObservedRequestInit } from '@/run/llm-fetch-observer'
|
|
13
15
|
import { SecretsBackend } from '@/secrets'
|
|
14
16
|
|
|
15
17
|
import { promptWithSameRefRetryOnly } from '../retry-same-ref'
|
|
16
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
buildMultimodalLookerSystemPrompt,
|
|
20
|
+
LOOK_AT_MAX_BYTES,
|
|
21
|
+
LOOK_AT_MAX_IMAGES,
|
|
22
|
+
resolveImagesBounded,
|
|
23
|
+
type ImageInput,
|
|
24
|
+
} from './looker'
|
|
17
25
|
|
|
18
26
|
type ImageParam = { url: string } | { path: string } | { data: string; mimeType: string }
|
|
19
27
|
|
|
@@ -46,52 +54,62 @@ export type ChannelLookAtOrigin = {
|
|
|
46
54
|
// Output is the subagent's text response. The subagent itself decides whether
|
|
47
55
|
// to answer the user's question (when `prompt` is supplied) or describe the
|
|
48
56
|
// image (when `prompt` is omitted) via its dynamic system prompt.
|
|
49
|
-
export
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
Type.
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
const
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
})
|
|
57
|
+
export function createLookAtTool(permissions?: PermissionService) {
|
|
58
|
+
return defineTool({
|
|
59
|
+
name: 'look_at',
|
|
60
|
+
label: 'Look at images',
|
|
61
|
+
description:
|
|
62
|
+
'Route image(s) through a vision-capable subagent and get a text result. ' +
|
|
63
|
+
'Use this when you need to see an image: a screenshot the user shared, a diagram in a doc, a photo, a chart, etc. ' +
|
|
64
|
+
'Each image is specified by ONE of `url` (https://...), `path` (absolute filesystem path), or `data`+`mimeType` (base64). ' +
|
|
65
|
+
'The optional `prompt` is a question to ask about the image(s); without it, the subagent returns a faithful description. ' +
|
|
66
|
+
'The image bytes never enter your context — only the resulting text comes back.',
|
|
67
|
+
parameters: Type.Object({
|
|
68
|
+
images: Type.Array(
|
|
69
|
+
Type.Object({
|
|
70
|
+
url: Type.Optional(Type.String({ description: 'https:// URL to fetch the image from.' })),
|
|
71
|
+
path: Type.Optional(
|
|
72
|
+
Type.String({ description: 'Absolute filesystem path (inside /agent or a mounted dir).' }),
|
|
73
|
+
),
|
|
74
|
+
data: Type.Optional(Type.String({ description: 'Base64-encoded image bytes (pair with mimeType).' })),
|
|
75
|
+
mimeType: Type.Optional(Type.String({ description: 'MIME type when using `data` (e.g. "image/png").' })),
|
|
76
|
+
}),
|
|
77
|
+
{ minItems: 1, maxItems: LOOK_AT_MAX_IMAGES, description: 'One or more images to look at.' },
|
|
78
|
+
),
|
|
79
|
+
prompt: Type.Optional(
|
|
80
|
+
Type.String({
|
|
81
|
+
description:
|
|
82
|
+
'Optional question to ask about the image(s). When omitted, the subagent returns a faithful description.',
|
|
83
|
+
}),
|
|
84
|
+
),
|
|
85
|
+
}),
|
|
86
|
+
|
|
87
|
+
async execute(_toolCallId, params, signal) {
|
|
88
|
+
const args = params as LookAtArgs
|
|
89
|
+
try {
|
|
90
|
+
const imageInputs = args.images.map(toImageInput)
|
|
91
|
+
const resolved = await resolveImagesBounded(imageInputs, signal)
|
|
92
|
+
const imageContents: ImageContent[] = resolved.map((r) => toImageContent(r.data, r.mimeType))
|
|
93
|
+
return await runLookAtImages(imageContents, args.prompt, permissions)
|
|
94
|
+
} catch (error) {
|
|
95
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
96
|
+
return errorResult(message, { count: args.images.length, prompt: args.prompt })
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
})
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export const lookAtTool = createLookAtTool()
|
|
89
103
|
|
|
90
104
|
// Channel attachments intentionally use a separate tool instead of adding
|
|
91
105
|
// channel-only dependencies to the global look_at implementation. This keeps
|
|
92
106
|
// non-channel sessions' image source validation unchanged while channel
|
|
93
107
|
// sessions get router-validated attachment_id lookup.
|
|
94
|
-
export function createChannelLookAtTool(
|
|
108
|
+
export function createChannelLookAtTool(
|
|
109
|
+
router: ChannelRouter,
|
|
110
|
+
origin: ChannelLookAtOrigin,
|
|
111
|
+
permissions?: PermissionService,
|
|
112
|
+
) {
|
|
95
113
|
return defineTool({
|
|
96
114
|
name: 'look_at_channel_attachment',
|
|
97
115
|
label: 'Look at channel attachment',
|
|
@@ -130,12 +148,14 @@ export function createChannelLookAtTool(router: ChannelRouter, origin: ChannelLo
|
|
|
130
148
|
}
|
|
131
149
|
const result = await router.fetchAttachment(origin.adapter, {
|
|
132
150
|
ref: found.ref,
|
|
151
|
+
maxBytes: LOOK_AT_MAX_BYTES,
|
|
133
152
|
...(found.filename !== undefined ? { filename: found.filename } : {}),
|
|
134
153
|
})
|
|
135
154
|
if (!result.ok) return errorResult(result.error, { count: 0, prompt: params.prompt })
|
|
136
155
|
return await runLookAtImages(
|
|
137
156
|
[toImageContent(result.buffer.toString('base64'), result.mimetype ?? 'image/jpeg')],
|
|
138
157
|
params.prompt,
|
|
158
|
+
permissions,
|
|
139
159
|
)
|
|
140
160
|
},
|
|
141
161
|
})
|
|
@@ -145,7 +165,11 @@ function toImageContent(data: string, mimeType: string): ImageContent {
|
|
|
145
165
|
return { type: 'image', data, mimeType }
|
|
146
166
|
}
|
|
147
167
|
|
|
148
|
-
async function runLookAtImages(
|
|
168
|
+
async function runLookAtImages(
|
|
169
|
+
imageContents: ImageContent[],
|
|
170
|
+
prompt: string | undefined,
|
|
171
|
+
permissions?: PermissionService,
|
|
172
|
+
) {
|
|
149
173
|
// Exception for text-only GLM Coding Plan models: the subagent below would
|
|
150
174
|
// resolve to the same image-blind model and silently fail. GLM-4.6V vision is
|
|
151
175
|
// reachable on the coding-plan endpoint with the same key (POST
|
|
@@ -176,6 +200,7 @@ async function runLookAtImages(imageContents: ImageContent[], prompt: string | u
|
|
|
176
200
|
const { session, dispose } = await createSessionWithDispose({
|
|
177
201
|
systemPromptOverride: systemPrompt,
|
|
178
202
|
origin,
|
|
203
|
+
...(permissions !== undefined ? { permissions } : {}),
|
|
179
204
|
profile: 'vision',
|
|
180
205
|
// Both knobs are required to fully disarm the subagent's tool surface:
|
|
181
206
|
// `customTools: []` blocks typeclaw's system tools (web_search/web_fetch/
|
|
@@ -208,6 +233,7 @@ const GLM_VISION_PROVIDER = 'zai-coding'
|
|
|
208
233
|
const GLM_VISION_MODEL = 'glm-4.6v'
|
|
209
234
|
const GLM_VISION_ENDPOINT = 'https://api.z.ai/api/coding/paas/v4/chat/completions'
|
|
210
235
|
const GLM_VISION_MAX_TOKENS = 32768
|
|
236
|
+
export const GLM_VISION_MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
|
211
237
|
|
|
212
238
|
// Vision requests carry a base64 image body the server must decode before it can
|
|
213
239
|
// send response headers, so the observer's 15s default TTFB (tuned for ~1s text
|
|
@@ -254,10 +280,18 @@ async function analyzeWithGlmCodingPlanVision(imageContents: ImageContent[], pro
|
|
|
254
280
|
}
|
|
255
281
|
|
|
256
282
|
if (!response.ok) {
|
|
283
|
+
await response.body?.cancel().catch(() => undefined)
|
|
257
284
|
return errorResult(`GLM vision request failed: HTTP ${response.status}`, details)
|
|
258
285
|
}
|
|
259
286
|
|
|
260
|
-
|
|
287
|
+
let body: unknown
|
|
288
|
+
try {
|
|
289
|
+
const bytes = await readResponseBodyBounded(response, GLM_VISION_MAX_RESPONSE_BYTES)
|
|
290
|
+
body = JSON.parse(bytes.toString('utf8'))
|
|
291
|
+
} catch (error) {
|
|
292
|
+
return errorResult(`GLM vision response failed: ${error instanceof Error ? error.message : String(error)}`, details)
|
|
293
|
+
}
|
|
294
|
+
const text = extractGlmVisionText(body)
|
|
261
295
|
if (text === null) {
|
|
262
296
|
return errorResult('GLM vision returned no text response', details)
|
|
263
297
|
}
|
|
@@ -1,8 +1,19 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { open } from 'node:fs/promises'
|
|
2
2
|
import { extname, isAbsolute } from 'node:path'
|
|
3
3
|
|
|
4
4
|
import { z } from 'zod'
|
|
5
5
|
|
|
6
|
+
import {
|
|
7
|
+
createPublicSocketLookup,
|
|
8
|
+
defaultPublicHttpDependencies,
|
|
9
|
+
headerValue,
|
|
10
|
+
requestPublicHttpUrl,
|
|
11
|
+
type PublicHttpAddress,
|
|
12
|
+
type PublicHttpDependencies,
|
|
13
|
+
type PublicHttpRequestOptions,
|
|
14
|
+
type PublicHttpResponse,
|
|
15
|
+
} from '@/agent/network/safe-http'
|
|
16
|
+
|
|
6
17
|
const SUPPORTED_MIME_TYPES = {
|
|
7
18
|
'.png': 'image/png',
|
|
8
19
|
'.jpg': 'image/jpeg',
|
|
@@ -18,6 +29,10 @@ const SUPPORTED_MIME_TYPES = {
|
|
|
18
29
|
// 30 s is generous for a single HTTP image fetch over a slow link.
|
|
19
30
|
export const URL_FETCH_TIMEOUT_MS = 30_000
|
|
20
31
|
export const URL_FETCH_MAX_BYTES = 20 * 1024 * 1024
|
|
32
|
+
export const LOOK_AT_MAX_IMAGES = 16
|
|
33
|
+
export const LOOK_AT_MAX_BYTES = URL_FETCH_MAX_BYTES
|
|
34
|
+
export const LOOK_AT_MAX_CONCURRENCY = 4
|
|
35
|
+
export const LOOK_AT_MAX_REDIRECTS = 5
|
|
21
36
|
|
|
22
37
|
type Mime = (typeof SUPPORTED_MIME_TYPES)[keyof typeof SUPPORTED_MIME_TYPES]
|
|
23
38
|
|
|
@@ -71,6 +86,57 @@ export type ResolvedImage = {
|
|
|
71
86
|
mimeType: string
|
|
72
87
|
}
|
|
73
88
|
|
|
89
|
+
export type LookAtAddress = PublicHttpAddress
|
|
90
|
+
export type LookAtRequestOptions = PublicHttpRequestOptions
|
|
91
|
+
export type LookAtHttpResponse = PublicHttpResponse
|
|
92
|
+
export type LookAtNetworkDependencies = PublicHttpDependencies
|
|
93
|
+
|
|
94
|
+
type ImageByteBudget = {
|
|
95
|
+
consume(bytes: number): void
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export async function resolveImagesBounded(
|
|
99
|
+
inputs: ImageInput[],
|
|
100
|
+
signal?: AbortSignal,
|
|
101
|
+
network: LookAtNetworkDependencies = defaultLookAtNetwork,
|
|
102
|
+
): Promise<ResolvedImage[]> {
|
|
103
|
+
if (inputs.length > LOOK_AT_MAX_IMAGES) {
|
|
104
|
+
throw new Error(`look_at: image count exceeds limit (${inputs.length} > ${LOOK_AT_MAX_IMAGES})`)
|
|
105
|
+
}
|
|
106
|
+
let consumed = 0
|
|
107
|
+
const budget: ImageByteBudget = {
|
|
108
|
+
consume(bytes) {
|
|
109
|
+
consumed += bytes
|
|
110
|
+
if (consumed > LOOK_AT_MAX_BYTES) {
|
|
111
|
+
throw new Error(`look_at: images exceed aggregate byte limit (${consumed} > ${LOOK_AT_MAX_BYTES})`)
|
|
112
|
+
}
|
|
113
|
+
},
|
|
114
|
+
}
|
|
115
|
+
const results = Array.from<ResolvedImage>({ length: inputs.length })
|
|
116
|
+
const siblingController = new AbortController()
|
|
117
|
+
const workerSignal =
|
|
118
|
+
signal === undefined ? siblingController.signal : AbortSignal.any([signal, siblingController.signal])
|
|
119
|
+
let next = 0
|
|
120
|
+
let firstError: unknown
|
|
121
|
+
let failed = false
|
|
122
|
+
const worker = async (): Promise<void> => {
|
|
123
|
+
try {
|
|
124
|
+
while (next < inputs.length) {
|
|
125
|
+
const index = next++
|
|
126
|
+
results[index] = await resolveImage(inputs[index] as ImageInput, workerSignal, budget, network)
|
|
127
|
+
}
|
|
128
|
+
} catch (error) {
|
|
129
|
+
if (!failed) firstError = error
|
|
130
|
+
failed = true
|
|
131
|
+
siblingController.abort(error)
|
|
132
|
+
throw error
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
await Promise.allSettled(Array.from({ length: Math.min(inputs.length, LOOK_AT_MAX_CONCURRENCY) }, worker))
|
|
136
|
+
if (failed) throw firstError
|
|
137
|
+
return results
|
|
138
|
+
}
|
|
139
|
+
|
|
74
140
|
// Materializes an ImageInput into the base64-encoded form pi-ai expects.
|
|
75
141
|
// - `url`: passthrough; pi-ai's image content does not accept URLs, so we fetch
|
|
76
142
|
// the bytes and base64-encode here (lazy; only when the tool is invoked).
|
|
@@ -78,20 +144,27 @@ export type ResolvedImage = {
|
|
|
78
144
|
// resolvable against the caller's cwd (callers should normalize ahead of
|
|
79
145
|
// time; this function rejects relative paths to avoid ambiguity).
|
|
80
146
|
// - `base64`: passthrough.
|
|
81
|
-
export async function resolveImage(
|
|
147
|
+
export async function resolveImage(
|
|
148
|
+
input: ImageInput,
|
|
149
|
+
signal?: AbortSignal,
|
|
150
|
+
aggregateBudget?: ImageByteBudget,
|
|
151
|
+
network: LookAtNetworkDependencies = defaultLookAtNetwork,
|
|
152
|
+
): Promise<ResolvedImage> {
|
|
82
153
|
if (input.kind === 'base64') {
|
|
83
154
|
if (!input.mimeType.startsWith('image/')) {
|
|
84
155
|
throw new Error(`look_at: base64 mimeType must be image/* (got "${input.mimeType}")`)
|
|
85
156
|
}
|
|
157
|
+
const decodedBytes = decodedBase64Size(input.data)
|
|
158
|
+
if (decodedBytes > URL_FETCH_MAX_BYTES) {
|
|
159
|
+
throw new Error(`look_at: base64 image too large (${decodedBytes} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
|
|
160
|
+
}
|
|
161
|
+
aggregateBudget?.consume(decodedBytes)
|
|
86
162
|
return { data: input.data, mimeType: input.mimeType }
|
|
87
163
|
}
|
|
88
164
|
if (input.kind === 'file') {
|
|
89
165
|
if (!isAbsolute(input.path)) {
|
|
90
166
|
throw new Error(`look_at: file path must be absolute (got "${input.path}")`)
|
|
91
167
|
}
|
|
92
|
-
if (!existsSync(input.path)) {
|
|
93
|
-
throw new Error(`look_at: file not found at ${input.path}`)
|
|
94
|
-
}
|
|
95
168
|
const ext = extname(input.path).toLowerCase() as keyof typeof SUPPORTED_MIME_TYPES
|
|
96
169
|
const mimeType = (SUPPORTED_MIME_TYPES[ext] ?? null) as Mime | null
|
|
97
170
|
if (mimeType === null) {
|
|
@@ -99,47 +172,86 @@ export async function resolveImage(input: ImageInput, signal?: AbortSignal): Pro
|
|
|
99
172
|
`look_at: unsupported image extension "${ext}" for ${input.path} (supported: ${Object.keys(SUPPORTED_MIME_TYPES).join(', ')})`,
|
|
100
173
|
)
|
|
101
174
|
}
|
|
102
|
-
const
|
|
103
|
-
|
|
175
|
+
const handle = await open(input.path, 'r').catch(() => {
|
|
176
|
+
throw new Error(`look_at: file not found at ${input.path}`)
|
|
177
|
+
})
|
|
178
|
+
try {
|
|
179
|
+
const size = (await handle.stat()).size
|
|
180
|
+
if (size > URL_FETCH_MAX_BYTES) {
|
|
181
|
+
throw new Error(`look_at: file too large (${size} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
|
|
182
|
+
}
|
|
183
|
+
aggregateBudget?.consume(size)
|
|
184
|
+
const bytes = Buffer.alloc(size + 1)
|
|
185
|
+
let offset = 0
|
|
186
|
+
while (offset < bytes.byteLength) {
|
|
187
|
+
const { bytesRead } = await handle.read(bytes, offset, bytes.byteLength - offset, null)
|
|
188
|
+
if (bytesRead === 0) break
|
|
189
|
+
offset += bytesRead
|
|
190
|
+
}
|
|
191
|
+
const finalSize = (await handle.stat()).size
|
|
192
|
+
if (offset !== size || finalSize !== size) {
|
|
193
|
+
throw new Error(`look_at: file changed while being read: ${input.path}`)
|
|
194
|
+
}
|
|
195
|
+
return { data: bytes.subarray(0, offset).toString('base64'), mimeType }
|
|
196
|
+
} finally {
|
|
197
|
+
await handle.close()
|
|
198
|
+
}
|
|
104
199
|
}
|
|
105
200
|
// URL branch: independent timeout + size cap on top of any caller-provided
|
|
106
201
|
// signal. The two abort signals are merged so the tool's overall abort wins
|
|
107
202
|
// over our timeout AND vice versa.
|
|
108
203
|
const timeoutSignal = AbortSignal.timeout(URL_FETCH_TIMEOUT_MS)
|
|
109
204
|
const mergedSignal = signal !== undefined ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal
|
|
110
|
-
const res = await
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
205
|
+
const { response: res, finalUrl } = await requestPublicHttpUrl(input.url, {
|
|
206
|
+
signal: mergedSignal,
|
|
207
|
+
headers: { Accept: 'image/*' },
|
|
208
|
+
maxRedirects: LOOK_AT_MAX_REDIRECTS,
|
|
209
|
+
dependencies: network,
|
|
210
|
+
})
|
|
211
|
+
try {
|
|
212
|
+
if (res.statusCode < 200 || res.statusCode >= 300) {
|
|
213
|
+
throw new Error(`look_at: failed to fetch ${finalUrl}: HTTP ${res.statusCode}`)
|
|
214
|
+
}
|
|
215
|
+
const mimeType = headerValue(res.headers, 'content-type')?.split(';')[0]?.trim() ?? 'application/octet-stream'
|
|
216
|
+
if (!mimeType.startsWith('image/')) {
|
|
217
|
+
throw new Error(`look_at: ${input.url} did not return an image content-type (got "${mimeType}")`)
|
|
218
|
+
}
|
|
219
|
+
// Streaming size check: arrayBuffer() would read the whole body before we
|
|
220
|
+
// could enforce a cap. Read chunk-by-chunk and abort once we cross the
|
|
221
|
+
// limit. Content-Length is checked first when present, but absent or lying
|
|
222
|
+
// headers fall through to the streaming check.
|
|
223
|
+
const declared = Number(headerValue(res.headers, 'content-length') ?? '')
|
|
224
|
+
if (Number.isFinite(declared) && declared > URL_FETCH_MAX_BYTES) {
|
|
225
|
+
throw new Error(`look_at: ${input.url} response too large (${declared} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
|
|
226
|
+
}
|
|
227
|
+
const chunks: Uint8Array[] = []
|
|
228
|
+
let total = 0
|
|
229
|
+
for await (const value of res.body) {
|
|
230
|
+
total += value.byteLength
|
|
231
|
+
if (total > URL_FETCH_MAX_BYTES) {
|
|
232
|
+
throw new Error(`look_at: ${input.url} response exceeded ${URL_FETCH_MAX_BYTES}-byte cap`)
|
|
233
|
+
}
|
|
234
|
+
aggregateBudget?.consume(value.byteLength)
|
|
235
|
+
chunks.push(value)
|
|
236
|
+
}
|
|
237
|
+
const buf = Buffer.concat(chunks)
|
|
238
|
+
return { data: buf.toString('base64'), mimeType }
|
|
239
|
+
} finally {
|
|
240
|
+
res.cancel()
|
|
125
241
|
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const defaultLookAtNetwork = defaultPublicHttpDependencies
|
|
245
|
+
|
|
246
|
+
export { createPublicSocketLookup }
|
|
247
|
+
|
|
248
|
+
function decodedBase64Size(data: string): number {
|
|
249
|
+
if (data.length > Math.ceil(URL_FETCH_MAX_BYTES / 3) * 4) {
|
|
250
|
+
throw new Error(`look_at: base64 image too large (encoded length exceeds ${URL_FETCH_MAX_BYTES}-byte cap)`)
|
|
129
251
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
while (true) {
|
|
133
|
-
const { done, value } = await reader.read()
|
|
134
|
-
if (done) break
|
|
135
|
-
if (value === undefined) continue
|
|
136
|
-
total += value.byteLength
|
|
137
|
-
if (total > URL_FETCH_MAX_BYTES) {
|
|
138
|
-
await reader.cancel()
|
|
139
|
-
throw new Error(`look_at: ${input.url} response exceeded ${URL_FETCH_MAX_BYTES}-byte cap`)
|
|
140
|
-
}
|
|
141
|
-
chunks.push(value)
|
|
252
|
+
if (data.length % 4 !== 0 || !/^[A-Za-z\d+/]*={0,2}$/.test(data)) {
|
|
253
|
+
throw new Error('look_at: invalid base64 image data')
|
|
142
254
|
}
|
|
143
|
-
const
|
|
144
|
-
return
|
|
255
|
+
const padding = data.endsWith('==') ? 2 : data.endsWith('=') ? 1 : 0
|
|
256
|
+
return (data.length / 4) * 3 - padding
|
|
145
257
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export async function readResponseBodyBounded(response: Response, maxBytes: number): Promise<Buffer> {
|
|
2
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) throw new Error(`invalid response byte limit: ${maxBytes}`)
|
|
3
|
+
if (response.body === null) return Buffer.alloc(0)
|
|
4
|
+
const reader = response.body.getReader()
|
|
5
|
+
const chunks: Uint8Array[] = []
|
|
6
|
+
let total = 0
|
|
7
|
+
try {
|
|
8
|
+
const declared = Number(response.headers.get('content-length') ?? '')
|
|
9
|
+
if (Number.isFinite(declared) && declared > maxBytes) throw responseTooLarge(declared, maxBytes)
|
|
10
|
+
while (true) {
|
|
11
|
+
const next = await reader.read()
|
|
12
|
+
if (next.done) break
|
|
13
|
+
total += next.value.byteLength
|
|
14
|
+
if (total > maxBytes) throw responseTooLarge(total, maxBytes)
|
|
15
|
+
chunks.push(next.value)
|
|
16
|
+
}
|
|
17
|
+
return Buffer.concat(chunks)
|
|
18
|
+
} finally {
|
|
19
|
+
await reader.cancel().catch(() => undefined)
|
|
20
|
+
reader.releaseLock()
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function responseTooLarge(size: number, maxBytes: number): Error {
|
|
25
|
+
return new Error(`response is too large (${size} bytes > ${maxBytes} byte limit)`)
|
|
26
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { lookup as dnsLookup } from 'node:dns/promises'
|
|
2
|
+
import { request as httpRequest, type IncomingHttpHeaders } from 'node:http'
|
|
3
|
+
import { request as httpsRequest } from 'node:https'
|
|
4
|
+
import { isIP, type LookupFunction } from 'node:net'
|
|
5
|
+
|
|
6
|
+
import { classifyIpAddress, classifyUrl } from '@/bundled-plugins/security/policies/ssrf'
|
|
7
|
+
|
|
8
|
+
export type PublicHttpAddress = { address: string; family: 4 | 6 }
|
|
9
|
+
|
|
10
|
+
export type PublicHttpRequestOptions = {
|
|
11
|
+
protocol: 'http:' | 'https:'
|
|
12
|
+
hostname: string
|
|
13
|
+
port?: number
|
|
14
|
+
path: string
|
|
15
|
+
method: 'GET'
|
|
16
|
+
headers: Record<string, string>
|
|
17
|
+
servername?: string
|
|
18
|
+
signal: AbortSignal
|
|
19
|
+
lookup: LookupFunction
|
|
20
|
+
autoSelectFamily: false
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type PublicHttpResponse = {
|
|
24
|
+
statusCode: number
|
|
25
|
+
headers: IncomingHttpHeaders
|
|
26
|
+
body: AsyncIterable<Uint8Array>
|
|
27
|
+
cancel(): void
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export type PublicHttpDependencies = {
|
|
31
|
+
resolveAddresses(hostname: string): Promise<readonly PublicHttpAddress[]>
|
|
32
|
+
request(options: PublicHttpRequestOptions): Promise<PublicHttpResponse>
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export type PublicHttpResult = { response: PublicHttpResponse; finalUrl: string }
|
|
36
|
+
|
|
37
|
+
export const DEFAULT_PUBLIC_HTTP_MAX_REDIRECTS = 5
|
|
38
|
+
|
|
39
|
+
export async function requestPublicHttpUrl(
|
|
40
|
+
rawUrl: string,
|
|
41
|
+
options: {
|
|
42
|
+
signal: AbortSignal
|
|
43
|
+
headers?: Record<string, string>
|
|
44
|
+
headersForUrl?: (url: string) => Record<string, string>
|
|
45
|
+
onResponse?: (response: PublicHttpResponse, url: string) => void
|
|
46
|
+
maxRedirects?: number
|
|
47
|
+
dependencies?: PublicHttpDependencies
|
|
48
|
+
},
|
|
49
|
+
): Promise<PublicHttpResult> {
|
|
50
|
+
const dependencies = options.dependencies ?? defaultPublicHttpDependencies
|
|
51
|
+
const maxRedirects = options.maxRedirects ?? DEFAULT_PUBLIC_HTTP_MAX_REDIRECTS
|
|
52
|
+
let current = requirePublicHttpUrl(rawUrl)
|
|
53
|
+
const initialOrigin = new URL(current).origin
|
|
54
|
+
for (let redirects = 0; redirects <= maxRedirects; redirects++) {
|
|
55
|
+
const parsed = new URL(current)
|
|
56
|
+
const hostname = unbracketHostname(parsed.hostname)
|
|
57
|
+
const response = await dependencies.request({
|
|
58
|
+
protocol: parsed.protocol as 'http:' | 'https:',
|
|
59
|
+
hostname,
|
|
60
|
+
...(parsed.port === '' ? {} : { port: Number(parsed.port) }),
|
|
61
|
+
path: `${parsed.pathname}${parsed.search}`,
|
|
62
|
+
method: 'GET',
|
|
63
|
+
headers: {
|
|
64
|
+
...headersForHop(options.headers ?? {}, initialOrigin, parsed.origin),
|
|
65
|
+
...options.headersForUrl?.(current),
|
|
66
|
+
Host: parsed.host,
|
|
67
|
+
},
|
|
68
|
+
...(parsed.protocol === 'https:' ? { servername: hostname } : {}),
|
|
69
|
+
signal: options.signal,
|
|
70
|
+
lookup: createPublicSocketLookup(dependencies.resolveAddresses),
|
|
71
|
+
autoSelectFamily: false,
|
|
72
|
+
})
|
|
73
|
+
options.onResponse?.(response, current)
|
|
74
|
+
if (!isRedirectStatus(response.statusCode)) return { response, finalUrl: current }
|
|
75
|
+
try {
|
|
76
|
+
const location = headerValue(response.headers, 'location')
|
|
77
|
+
if (location === null) throw new Error(`redirect from ${current} omitted the Location header`)
|
|
78
|
+
if (redirects === maxRedirects) throw new Error(`redirect limit exceeded (${maxRedirects})`)
|
|
79
|
+
current = requirePublicHttpUrl(new URL(location, current).toString())
|
|
80
|
+
} finally {
|
|
81
|
+
response.cancel()
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
throw new Error('redirect limit exceeded')
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function headersForHop(
|
|
88
|
+
headers: Record<string, string>,
|
|
89
|
+
initialOrigin: string,
|
|
90
|
+
currentOrigin: string,
|
|
91
|
+
): Record<string, string> {
|
|
92
|
+
if (initialOrigin === currentOrigin) return headers
|
|
93
|
+
return Object.fromEntries(
|
|
94
|
+
Object.entries(headers).filter(([name]) => !/^(authorization|cookie|proxy-authorization)$/i.test(name)),
|
|
95
|
+
)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function createPublicSocketLookup(resolveAddresses: PublicHttpDependencies['resolveAddresses']): LookupFunction {
|
|
99
|
+
return (hostname, options, callback) => {
|
|
100
|
+
void resolveAddresses(hostname).then(
|
|
101
|
+
(addresses) => {
|
|
102
|
+
if (addresses.length === 0) {
|
|
103
|
+
callback(new Error(`DNS lookup returned no addresses for ${hostname}`), '', 0)
|
|
104
|
+
return
|
|
105
|
+
}
|
|
106
|
+
for (const candidate of addresses) {
|
|
107
|
+
const actualFamily = isIP(candidate.address)
|
|
108
|
+
if (actualFamily !== candidate.family || (actualFamily !== 4 && actualFamily !== 6)) {
|
|
109
|
+
callback(new Error(`DNS lookup returned an invalid address for ${hostname}`), '', 0)
|
|
110
|
+
return
|
|
111
|
+
}
|
|
112
|
+
const classification = classifyIpAddress(candidate.address)
|
|
113
|
+
if (classification.blocked) {
|
|
114
|
+
callback(
|
|
115
|
+
new Error(
|
|
116
|
+
`DNS lookup rejected non-public address for ${hostname}: ${classification.reason ?? candidate.address}`,
|
|
117
|
+
),
|
|
118
|
+
'',
|
|
119
|
+
0,
|
|
120
|
+
)
|
|
121
|
+
return
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const requestedFamily = options.family === 4 || options.family === 6 ? options.family : undefined
|
|
125
|
+
const chosen = addresses.find(
|
|
126
|
+
(candidate) => requestedFamily === undefined || candidate.family === requestedFamily,
|
|
127
|
+
)
|
|
128
|
+
if (chosen === undefined) {
|
|
129
|
+
callback(new Error(`DNS lookup returned no IPv${requestedFamily} address for ${hostname}`), '', 0)
|
|
130
|
+
return
|
|
131
|
+
}
|
|
132
|
+
if (options.all === true) callback(null, [chosen])
|
|
133
|
+
else callback(null, chosen.address, chosen.family)
|
|
134
|
+
},
|
|
135
|
+
(error: unknown) => callback(error instanceof Error ? error : new Error(String(error)), '', 0),
|
|
136
|
+
)
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export function headerValue(headers: IncomingHttpHeaders, name: string): string | null {
|
|
141
|
+
const value = headers[name]
|
|
142
|
+
if (Array.isArray(value)) return value[0] ?? null
|
|
143
|
+
return value ?? null
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export const defaultPublicHttpDependencies: PublicHttpDependencies = {
|
|
147
|
+
async resolveAddresses(hostname) {
|
|
148
|
+
const addresses = await dnsLookup(hostname, { all: true, verbatim: true })
|
|
149
|
+
return addresses.map((candidate) => {
|
|
150
|
+
if (candidate.family !== 4 && candidate.family !== 6) {
|
|
151
|
+
throw new Error(`DNS lookup returned unsupported family ${candidate.family} for ${hostname}`)
|
|
152
|
+
}
|
|
153
|
+
return { address: candidate.address, family: candidate.family }
|
|
154
|
+
})
|
|
155
|
+
},
|
|
156
|
+
async request(options) {
|
|
157
|
+
return await new Promise((resolve, reject) => {
|
|
158
|
+
const request = options.protocol === 'https:' ? httpsRequest : httpRequest
|
|
159
|
+
const outgoing = request(options, (response) => {
|
|
160
|
+
resolve({
|
|
161
|
+
statusCode: response.statusCode ?? 0,
|
|
162
|
+
headers: response.headers,
|
|
163
|
+
body: response,
|
|
164
|
+
cancel: () => response.destroy(),
|
|
165
|
+
})
|
|
166
|
+
})
|
|
167
|
+
outgoing.once('error', reject)
|
|
168
|
+
outgoing.end()
|
|
169
|
+
})
|
|
170
|
+
},
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function requirePublicHttpUrl(rawUrl: string): string {
|
|
174
|
+
let parsed: URL
|
|
175
|
+
try {
|
|
176
|
+
parsed = new URL(rawUrl)
|
|
177
|
+
} catch {
|
|
178
|
+
throw new Error(`URL must be a valid public HTTP(S) URL: ${rawUrl}`)
|
|
179
|
+
}
|
|
180
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
181
|
+
throw new Error(`URL must use public HTTP(S), not ${parsed.protocol}`)
|
|
182
|
+
}
|
|
183
|
+
const classification = classifyUrl(parsed.toString())
|
|
184
|
+
if (classification.blocked) {
|
|
185
|
+
throw new Error(
|
|
186
|
+
`SSRF policy rejected non-public URL (${classification.category ?? 'internal'}): ${classification.reason ?? rawUrl}`,
|
|
187
|
+
)
|
|
188
|
+
}
|
|
189
|
+
return parsed.toString()
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function unbracketHostname(hostname: string): string {
|
|
193
|
+
return hostname.startsWith('[') && hostname.endsWith(']') ? hostname.slice(1, -1) : hostname
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function isRedirectStatus(status: number): boolean {
|
|
197
|
+
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308
|
|
198
|
+
}
|