typeclaw 0.45.1 → 0.46.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/README.md +1 -1
  2. package/package.json +3 -2
  3. package/src/agent/git-nudge.ts +2 -1
  4. package/src/agent/index.ts +44 -30
  5. package/src/agent/live-subagents.ts +18 -0
  6. package/src/agent/multimodal/index.ts +1 -1
  7. package/src/agent/multimodal/look-at.ts +78 -44
  8. package/src/agent/multimodal/looker.ts +161 -39
  9. package/src/agent/network/response-body.ts +26 -0
  10. package/src/agent/network/safe-http.ts +207 -0
  11. package/src/agent/plugin-tools.ts +377 -157
  12. package/src/agent/provider-error.ts +52 -1
  13. package/src/agent/restart/index.ts +61 -40
  14. package/src/agent/restart-handoff/index.ts +73 -4
  15. package/src/agent/reviewer-bash-policy.ts +42 -1
  16. package/src/agent/session-origin.ts +2 -0
  17. package/src/agent/subagent-drain.ts +117 -7
  18. package/src/agent/subagents.ts +49 -16
  19. package/src/agent/throttle-circuit.ts +25 -5
  20. package/src/agent/tool-error-remediation.ts +72 -0
  21. package/src/agent/tool-file-safety.ts +1310 -0
  22. package/src/agent/tools/channel-fetch-attachment.ts +10 -3
  23. package/src/agent/tools/ddg.ts +16 -2
  24. package/src/agent/tools/post-github-review.ts +199 -0
  25. package/src/agent/tools/spawn-subagent.ts +21 -18
  26. package/src/agent/tools/webfetch/fetch.ts +163 -183
  27. package/src/agent/tools/webfetch/tool.ts +2 -3
  28. package/src/agent/tools/websearch.ts +23 -2
  29. package/src/agent/tools-without-local-file-operands.ts +30 -0
  30. package/src/agent/turn-runner.ts +86 -6
  31. package/src/bundled-plugins/backup/README.md +23 -10
  32. package/src/bundled-plugins/backup/git-auth.ts +6 -4
  33. package/src/bundled-plugins/backup/index.ts +85 -44
  34. package/src/bundled-plugins/backup/runner.ts +85 -1
  35. package/src/bundled-plugins/doc-render/render.ts +93 -12
  36. package/src/bundled-plugins/doc-render/skills/typeclaw-render-pdf/SKILL.md +13 -13
  37. package/src/bundled-plugins/github-cli-auth/effective-approval.ts +16 -6
  38. package/src/bundled-plugins/github-cli-auth/gh-command.ts +555 -15
  39. package/src/bundled-plugins/github-cli-auth/git-askpass.ts +74 -26
  40. package/src/bundled-plugins/github-cli-auth/git-command.ts +224 -36
  41. package/src/bundled-plugins/github-cli-auth/graphql-auth-nudge.ts +9 -9
  42. package/src/bundled-plugins/github-cli-auth/index.ts +207 -132
  43. package/src/bundled-plugins/github-cli-auth/review-checkout.ts +138 -0
  44. package/src/bundled-plugins/github-cli-auth/token-class.ts +4 -6
  45. package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +2 -1
  46. package/src/bundled-plugins/memory/README.md +32 -11
  47. package/src/bundled-plugins/memory/append-tool.ts +15 -8
  48. package/src/bundled-plugins/memory/citations.ts +22 -9
  49. package/src/bundled-plugins/memory/dreaming.ts +13 -8
  50. package/src/bundled-plugins/memory/find-entry-tool.ts +4 -0
  51. package/src/bundled-plugins/memory/index.ts +0 -4
  52. package/src/bundled-plugins/memory/load-memory.ts +8 -3
  53. package/src/bundled-plugins/memory/provenance-index.ts +717 -0
  54. package/src/bundled-plugins/memory/provenance-sanitize.ts +40 -0
  55. package/src/bundled-plugins/memory/references/store-reference-tool.ts +4 -0
  56. package/src/bundled-plugins/memory/search-tool.ts +163 -60
  57. package/src/bundled-plugins/memory/stream-events.ts +2 -0
  58. package/src/bundled-plugins/memory/vector/embed-threads.ts +36 -0
  59. package/src/bundled-plugins/memory/vector/embedder.ts +19 -1
  60. package/src/bundled-plugins/memory/vector/hybrid.ts +126 -13
  61. package/src/bundled-plugins/researcher/write-report.ts +8 -5
  62. package/src/bundled-plugins/reviewer/index.ts +2 -2
  63. package/src/bundled-plugins/reviewer/reviewer.ts +10 -3
  64. package/src/bundled-plugins/reviewer/skills/code-review.ts +19 -9
  65. package/src/bundled-plugins/scout/scout.ts +8 -0
  66. package/src/bundled-plugins/security/index.ts +4 -4
  67. package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +1 -0
  68. package/src/bundled-plugins/security/policies/private-surface-read.ts +787 -36
  69. package/src/bundled-plugins/security/policies/secret-exfil-read.ts +1 -0
  70. package/src/bundled-plugins/security/policies/ssrf.ts +147 -42
  71. package/src/channels/adapters/discord-bot-channel-resolver.ts +26 -14
  72. package/src/channels/adapters/discord-bot-thread-room.ts +44 -12
  73. package/src/channels/adapters/discord-bot.ts +66 -6
  74. package/src/channels/adapters/discord-channel-resolver.ts +91 -23
  75. package/src/channels/adapters/discord-id.ts +4 -0
  76. package/src/channels/adapters/discord-resolver-cache.ts +41 -0
  77. package/src/channels/adapters/discord.ts +57 -7
  78. package/src/channels/adapters/github/auth-app.ts +41 -12
  79. package/src/channels/adapters/github/inbound.ts +11 -0
  80. package/src/channels/adapters/github/index.ts +71 -65
  81. package/src/channels/adapters/github/reconcile-cooldown-store.ts +178 -0
  82. package/src/channels/adapters/github/reconcile-open-prs.ts +31 -0
  83. package/src/channels/adapters/github/review-submitter.ts +302 -0
  84. package/src/channels/adapters/kakaotalk-channel-resolver.ts +5 -3
  85. package/src/channels/adapters/kakaotalk-fetch-attachment.ts +8 -4
  86. package/src/channels/adapters/kakaotalk-typing.ts +17 -15
  87. package/src/channels/adapters/slack-attachment-download.ts +71 -0
  88. package/src/channels/adapters/slack-bot-edit.ts +85 -20
  89. package/src/channels/adapters/slack-bot.ts +26 -10
  90. package/src/channels/adapters/slack-classify.ts +21 -2
  91. package/src/channels/adapters/slack-format.ts +245 -0
  92. package/src/channels/adapters/slack.ts +59 -7
  93. package/src/channels/adapters/telegram-bot.ts +18 -5
  94. package/src/channels/adapters/webex-bot.ts +8 -3
  95. package/src/channels/adapters/webex.ts +8 -3
  96. package/src/channels/fetch-attachment.ts +43 -0
  97. package/src/{bundled-plugins/github-cli-auth/approve-idempotency.ts → channels/github-review-verdict-coordinator.ts} +25 -1
  98. package/src/channels/github-token-bridge.ts +12 -6
  99. package/src/channels/manager.ts +9 -0
  100. package/src/channels/router.ts +248 -6
  101. package/src/channels/types.ts +28 -1
  102. package/src/cli/channel.ts +1 -1
  103. package/src/cli/init.ts +1 -1
  104. package/src/cli/model.ts +57 -7
  105. package/src/compose/stop.ts +17 -5
  106. package/src/config/config.ts +6 -6
  107. package/src/config/models-mutation.ts +13 -1
  108. package/src/config/providers.ts +176 -0
  109. package/src/container/docker-app.ts +42 -13
  110. package/src/container/start.ts +62 -10
  111. package/src/cron/bridge.ts +9 -2
  112. package/src/doctor/checks.ts +8 -3
  113. package/src/doctor/commit.ts +2 -1
  114. package/src/doctor/file-ownership.ts +52 -0
  115. package/src/dreams/git.ts +2 -1
  116. package/src/git/bound-gc-config.ts +82 -0
  117. package/src/git/hookless.ts +5 -0
  118. package/src/git/reconcile-ignored.ts +18 -5
  119. package/src/git/system-commit.ts +12 -6
  120. package/src/hostd/spawn.ts +133 -47
  121. package/src/init/dockerfile.ts +99 -5
  122. package/src/init/index.ts +17 -5
  123. package/src/init/models-dev.ts +4 -0
  124. package/src/init/validate-api-key.ts +2 -0
  125. package/src/inspect/replay.ts +21 -9
  126. package/src/inspect/session-list.ts +71 -33
  127. package/src/mcp/tools.ts +17 -3
  128. package/src/network/internal-destinations.ts +267 -0
  129. package/src/permissions/builtins.ts +3 -1
  130. package/src/plugin/context.ts +2 -0
  131. package/src/plugin/index.ts +1 -0
  132. package/src/plugin/manager.ts +4 -0
  133. package/src/plugin/types.ts +22 -0
  134. package/src/run/index.ts +40 -0
  135. package/src/run/llm-fetch-observer.ts +41 -4
  136. package/src/sandbox/build.ts +37 -15
  137. package/src/sandbox/canonical-secrets.ts +18 -0
  138. package/src/sandbox/env-exposure.ts +83 -0
  139. package/src/sandbox/errors.ts +13 -6
  140. package/src/sandbox/hidden-paths.ts +102 -44
  141. package/src/sandbox/index.ts +22 -4
  142. package/src/sandbox/package-install.ts +3 -27
  143. package/src/sandbox/policy.ts +16 -28
  144. package/src/sandbox/privileged-runtime.ts +256 -0
  145. package/src/sandbox/session-tmp.ts +8 -2
  146. package/src/sandbox/writable-zones.ts +204 -140
  147. package/src/secrets/index.ts +8 -0
  148. package/src/secrets/refresh-provider-oauth.ts +128 -0
  149. package/src/server/command-runner.ts +48 -4
  150. package/src/server/index.ts +4 -0
  151. package/src/skills/typeclaw-channel-github/SKILL.md +49 -56
  152. package/src/skills/typeclaw-claude-code/SKILL.md +24 -60
  153. package/src/skills/typeclaw-claude-code/references/auth-flow.md +17 -158
  154. package/src/skills/typeclaw-codex-cli/SKILL.md +19 -49
  155. package/src/skills/typeclaw-codex-cli/references/auth-flow.md +17 -154
  156. package/src/skills/typeclaw-config/SKILL.md +27 -11
  157. package/src/skills/typeclaw-config/references/recommended-mounts.md +24 -24
  158. package/src/skills/typeclaw-github-contributing/SKILL.md +23 -26
  159. package/src/skills/typeclaw-memory/SKILL.md +9 -3
  160. package/src/skills/typeclaw-permissions/SKILL.md +36 -19
  161. package/src/skills/typeclaw-plugins/SKILL.md +37 -0
  162. package/src/usage/scan.ts +11 -2
  163. package/typeclaw.schema.json +25 -0
@@ -1,8 +1,20 @@
1
- import { existsSync, readFileSync } from 'node:fs'
1
+ import { open } from 'node:fs/promises'
2
2
  import { extname, isAbsolute } from 'node:path'
3
3
 
4
4
  import { z } from 'zod'
5
5
 
6
+ import {
7
+ createPublicSocketLookup,
8
+ defaultPublicHttpDependencies,
9
+ headerValue,
10
+ requestPublicHttpUrl,
11
+ type PublicHttpAddress,
12
+ type PublicHttpDependencies,
13
+ type PublicHttpRequestOptions,
14
+ type PublicHttpResponse,
15
+ } from '@/agent/network/safe-http'
16
+ import type { InternalDestinationPolicy } from '@/network/internal-destinations'
17
+
6
18
  const SUPPORTED_MIME_TYPES = {
7
19
  '.png': 'image/png',
8
20
  '.jpg': 'image/jpeg',
@@ -18,6 +30,10 @@ const SUPPORTED_MIME_TYPES = {
18
30
  // 30 s is generous for a single HTTP image fetch over a slow link.
19
31
  export const URL_FETCH_TIMEOUT_MS = 30_000
20
32
  export const URL_FETCH_MAX_BYTES = 20 * 1024 * 1024
33
+ export const LOOK_AT_MAX_IMAGES = 16
34
+ export const LOOK_AT_MAX_BYTES = URL_FETCH_MAX_BYTES
35
+ export const LOOK_AT_MAX_CONCURRENCY = 4
36
+ export const LOOK_AT_MAX_REDIRECTS = 5
21
37
 
22
38
  type Mime = (typeof SUPPORTED_MIME_TYPES)[keyof typeof SUPPORTED_MIME_TYPES]
23
39
 
@@ -71,6 +87,64 @@ export type ResolvedImage = {
71
87
  mimeType: string
72
88
  }
73
89
 
90
+ export type LookAtAddress = PublicHttpAddress
91
+ export type LookAtRequestOptions = PublicHttpRequestOptions
92
+ export type LookAtHttpResponse = PublicHttpResponse
93
+ export type LookAtNetworkDependencies = PublicHttpDependencies
94
+
95
+ type ImageByteBudget = {
96
+ consume(bytes: number): void
97
+ }
98
+
99
+ export async function resolveImagesBounded(
100
+ inputs: ImageInput[],
101
+ signal?: AbortSignal,
102
+ network: LookAtNetworkDependencies = defaultLookAtNetwork,
103
+ internalDestinationPolicy?: InternalDestinationPolicy,
104
+ ): Promise<ResolvedImage[]> {
105
+ if (inputs.length > LOOK_AT_MAX_IMAGES) {
106
+ throw new Error(`look_at: image count exceeds limit (${inputs.length} > ${LOOK_AT_MAX_IMAGES})`)
107
+ }
108
+ let consumed = 0
109
+ const budget: ImageByteBudget = {
110
+ consume(bytes) {
111
+ consumed += bytes
112
+ if (consumed > LOOK_AT_MAX_BYTES) {
113
+ throw new Error(`look_at: images exceed aggregate byte limit (${consumed} > ${LOOK_AT_MAX_BYTES})`)
114
+ }
115
+ },
116
+ }
117
+ const results = Array.from<ResolvedImage>({ length: inputs.length })
118
+ const siblingController = new AbortController()
119
+ const workerSignal =
120
+ signal === undefined ? siblingController.signal : AbortSignal.any([signal, siblingController.signal])
121
+ let next = 0
122
+ let firstError: unknown
123
+ let failed = false
124
+ const worker = async (): Promise<void> => {
125
+ try {
126
+ while (next < inputs.length) {
127
+ const index = next++
128
+ results[index] = await resolveImage(
129
+ inputs[index] as ImageInput,
130
+ workerSignal,
131
+ budget,
132
+ network,
133
+ internalDestinationPolicy,
134
+ )
135
+ }
136
+ } catch (error) {
137
+ if (!failed) firstError = error
138
+ failed = true
139
+ siblingController.abort(error)
140
+ throw error
141
+ }
142
+ }
143
+ await Promise.allSettled(Array.from({ length: Math.min(inputs.length, LOOK_AT_MAX_CONCURRENCY) }, worker))
144
+ if (failed) throw firstError
145
+ return results
146
+ }
147
+
74
148
  // Materializes an ImageInput into the base64-encoded form pi-ai expects.
75
149
  // - `url`: passthrough; pi-ai's image content does not accept URLs, so we fetch
76
150
  // the bytes and base64-encode here (lazy; only when the tool is invoked).
@@ -78,20 +152,28 @@ export type ResolvedImage = {
78
152
  // resolvable against the caller's cwd (callers should normalize ahead of
79
153
  // time; this function rejects relative paths to avoid ambiguity).
80
154
  // - `base64`: passthrough.
81
- export async function resolveImage(input: ImageInput, signal?: AbortSignal): Promise<ResolvedImage> {
155
+ export async function resolveImage(
156
+ input: ImageInput,
157
+ signal?: AbortSignal,
158
+ aggregateBudget?: ImageByteBudget,
159
+ network: LookAtNetworkDependencies = defaultLookAtNetwork,
160
+ internalDestinationPolicy?: InternalDestinationPolicy,
161
+ ): Promise<ResolvedImage> {
82
162
  if (input.kind === 'base64') {
83
163
  if (!input.mimeType.startsWith('image/')) {
84
164
  throw new Error(`look_at: base64 mimeType must be image/* (got "${input.mimeType}")`)
85
165
  }
166
+ const decodedBytes = decodedBase64Size(input.data)
167
+ if (decodedBytes > URL_FETCH_MAX_BYTES) {
168
+ throw new Error(`look_at: base64 image too large (${decodedBytes} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
169
+ }
170
+ aggregateBudget?.consume(decodedBytes)
86
171
  return { data: input.data, mimeType: input.mimeType }
87
172
  }
88
173
  if (input.kind === 'file') {
89
174
  if (!isAbsolute(input.path)) {
90
175
  throw new Error(`look_at: file path must be absolute (got "${input.path}")`)
91
176
  }
92
- if (!existsSync(input.path)) {
93
- throw new Error(`look_at: file not found at ${input.path}`)
94
- }
95
177
  const ext = extname(input.path).toLowerCase() as keyof typeof SUPPORTED_MIME_TYPES
96
178
  const mimeType = (SUPPORTED_MIME_TYPES[ext] ?? null) as Mime | null
97
179
  if (mimeType === null) {
@@ -99,47 +181,87 @@ export async function resolveImage(input: ImageInput, signal?: AbortSignal): Pro
99
181
  `look_at: unsupported image extension "${ext}" for ${input.path} (supported: ${Object.keys(SUPPORTED_MIME_TYPES).join(', ')})`,
100
182
  )
101
183
  }
102
- const bytes = readFileSync(input.path)
103
- return { data: bytes.toString('base64'), mimeType }
184
+ const handle = await open(input.path, 'r').catch(() => {
185
+ throw new Error(`look_at: file not found at ${input.path}`)
186
+ })
187
+ try {
188
+ const size = (await handle.stat()).size
189
+ if (size > URL_FETCH_MAX_BYTES) {
190
+ throw new Error(`look_at: file too large (${size} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
191
+ }
192
+ aggregateBudget?.consume(size)
193
+ const bytes = Buffer.alloc(size + 1)
194
+ let offset = 0
195
+ while (offset < bytes.byteLength) {
196
+ const { bytesRead } = await handle.read(bytes, offset, bytes.byteLength - offset, null)
197
+ if (bytesRead === 0) break
198
+ offset += bytesRead
199
+ }
200
+ const finalSize = (await handle.stat()).size
201
+ if (offset !== size || finalSize !== size) {
202
+ throw new Error(`look_at: file changed while being read: ${input.path}`)
203
+ }
204
+ return { data: bytes.subarray(0, offset).toString('base64'), mimeType }
205
+ } finally {
206
+ await handle.close()
207
+ }
104
208
  }
105
209
  // URL branch: independent timeout + size cap on top of any caller-provided
106
210
  // signal. The two abort signals are merged so the tool's overall abort wins
107
211
  // over our timeout AND vice versa.
108
212
  const timeoutSignal = AbortSignal.timeout(URL_FETCH_TIMEOUT_MS)
109
213
  const mergedSignal = signal !== undefined ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal
110
- const res = await fetch(input.url, { signal: mergedSignal })
111
- if (!res.ok) {
112
- throw new Error(`look_at: failed to fetch ${input.url}: HTTP ${res.status}`)
113
- }
114
- const mimeType = res.headers.get('content-type')?.split(';')[0]?.trim() ?? 'application/octet-stream'
115
- if (!mimeType.startsWith('image/')) {
116
- throw new Error(`look_at: ${input.url} did not return an image content-type (got "${mimeType}")`)
117
- }
118
- // Streaming size check: arrayBuffer() would read the whole body before we
119
- // could enforce a cap. Read chunk-by-chunk and abort once we cross the
120
- // limit. Content-Length is checked first when present, but absent or lying
121
- // headers fall through to the streaming check.
122
- const declared = Number(res.headers.get('content-length') ?? '')
123
- if (Number.isFinite(declared) && declared > URL_FETCH_MAX_BYTES) {
124
- throw new Error(`look_at: ${input.url} response too large (${declared} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
214
+ const { response: res, finalUrl } = await requestPublicHttpUrl(input.url, {
215
+ signal: mergedSignal,
216
+ headers: { Accept: 'image/*' },
217
+ maxRedirects: LOOK_AT_MAX_REDIRECTS,
218
+ dependencies: network,
219
+ ...(internalDestinationPolicy === undefined ? {} : { internalDestinationPolicy }),
220
+ })
221
+ try {
222
+ if (res.statusCode < 200 || res.statusCode >= 300) {
223
+ throw new Error(`look_at: failed to fetch ${finalUrl}: HTTP ${res.statusCode}`)
224
+ }
225
+ const mimeType = headerValue(res.headers, 'content-type')?.split(';')[0]?.trim() ?? 'application/octet-stream'
226
+ if (!mimeType.startsWith('image/')) {
227
+ throw new Error(`look_at: ${input.url} did not return an image content-type (got "${mimeType}")`)
228
+ }
229
+ // Streaming size check: arrayBuffer() would read the whole body before we
230
+ // could enforce a cap. Read chunk-by-chunk and abort once we cross the
231
+ // limit. Content-Length is checked first when present, but absent or lying
232
+ // headers fall through to the streaming check.
233
+ const declared = Number(headerValue(res.headers, 'content-length') ?? '')
234
+ if (Number.isFinite(declared) && declared > URL_FETCH_MAX_BYTES) {
235
+ throw new Error(`look_at: ${input.url} response too large (${declared} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
236
+ }
237
+ const chunks: Uint8Array[] = []
238
+ let total = 0
239
+ for await (const value of res.body) {
240
+ total += value.byteLength
241
+ if (total > URL_FETCH_MAX_BYTES) {
242
+ throw new Error(`look_at: ${input.url} response exceeded ${URL_FETCH_MAX_BYTES}-byte cap`)
243
+ }
244
+ aggregateBudget?.consume(value.byteLength)
245
+ chunks.push(value)
246
+ }
247
+ const buf = Buffer.concat(chunks)
248
+ return { data: buf.toString('base64'), mimeType }
249
+ } finally {
250
+ res.cancel()
125
251
  }
126
- const reader = res.body?.getReader()
127
- if (reader === undefined) {
128
- throw new Error(`look_at: ${input.url} returned an empty body`)
252
+ }
253
+
254
+ const defaultLookAtNetwork = defaultPublicHttpDependencies
255
+
256
+ export { createPublicSocketLookup }
257
+
258
+ function decodedBase64Size(data: string): number {
259
+ if (data.length > Math.ceil(URL_FETCH_MAX_BYTES / 3) * 4) {
260
+ throw new Error(`look_at: base64 image too large (encoded length exceeds ${URL_FETCH_MAX_BYTES}-byte cap)`)
129
261
  }
130
- const chunks: Uint8Array[] = []
131
- let total = 0
132
- while (true) {
133
- const { done, value } = await reader.read()
134
- if (done) break
135
- if (value === undefined) continue
136
- total += value.byteLength
137
- if (total > URL_FETCH_MAX_BYTES) {
138
- await reader.cancel()
139
- throw new Error(`look_at: ${input.url} response exceeded ${URL_FETCH_MAX_BYTES}-byte cap`)
140
- }
141
- chunks.push(value)
262
+ if (data.length % 4 !== 0 || !/^[A-Za-z\d+/]*={0,2}$/.test(data)) {
263
+ throw new Error('look_at: invalid base64 image data')
142
264
  }
143
- const buf = Buffer.concat(chunks)
144
- return { data: buf.toString('base64'), mimeType }
265
+ const padding = data.endsWith('==') ? 2 : data.endsWith('=') ? 1 : 0
266
+ return (data.length / 4) * 3 - padding
145
267
  }
@@ -0,0 +1,26 @@
1
+ export async function readResponseBodyBounded(response: Response, maxBytes: number): Promise<Buffer> {
2
+ if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) throw new Error(`invalid response byte limit: ${maxBytes}`)
3
+ if (response.body === null) return Buffer.alloc(0)
4
+ const reader = response.body.getReader()
5
+ const chunks: Uint8Array[] = []
6
+ let total = 0
7
+ try {
8
+ const declared = Number(response.headers.get('content-length') ?? '')
9
+ if (Number.isFinite(declared) && declared > maxBytes) throw responseTooLarge(declared, maxBytes)
10
+ while (true) {
11
+ const next = await reader.read()
12
+ if (next.done) break
13
+ total += next.value.byteLength
14
+ if (total > maxBytes) throw responseTooLarge(total, maxBytes)
15
+ chunks.push(next.value)
16
+ }
17
+ return Buffer.concat(chunks)
18
+ } finally {
19
+ await reader.cancel().catch(() => undefined)
20
+ reader.releaseLock()
21
+ }
22
+ }
23
+
24
+ function responseTooLarge(size: number, maxBytes: number): Error {
25
+ return new Error(`response is too large (${size} bytes > ${maxBytes} byte limit)`)
26
+ }
@@ -0,0 +1,207 @@
1
+ import { lookup as dnsLookup } from 'node:dns/promises'
2
+ import { request as httpRequest, type IncomingHttpHeaders } from 'node:http'
3
+ import { request as httpsRequest } from 'node:https'
4
+ import { isIP, type LookupFunction } from 'node:net'
5
+
6
+ import { allowsInternalDestination, classifyIpAddress, classifyUrl } from '@/bundled-plugins/security/policies/ssrf'
7
+ import { getBootInternalDestinationPolicy, type InternalDestinationPolicy } from '@/network/internal-destinations'
8
+
9
+ export type PublicHttpAddress = { address: string; family: 4 | 6 }
10
+
11
+ export type PublicHttpRequestOptions = {
12
+ protocol: 'http:' | 'https:'
13
+ hostname: string
14
+ port?: number
15
+ path: string
16
+ method: 'GET'
17
+ headers: Record<string, string>
18
+ servername?: string
19
+ signal: AbortSignal
20
+ lookup: LookupFunction
21
+ autoSelectFamily: false
22
+ }
23
+
24
+ export type PublicHttpResponse = {
25
+ statusCode: number
26
+ headers: IncomingHttpHeaders
27
+ body: AsyncIterable<Uint8Array>
28
+ cancel(): void
29
+ }
30
+
31
+ export type PublicHttpDependencies = {
32
+ resolveAddresses(hostname: string): Promise<readonly PublicHttpAddress[]>
33
+ request(options: PublicHttpRequestOptions): Promise<PublicHttpResponse>
34
+ }
35
+
36
+ export type PublicHttpResult = { response: PublicHttpResponse; finalUrl: string }
37
+
38
+ export const DEFAULT_PUBLIC_HTTP_MAX_REDIRECTS = 5
39
+
40
+ export async function requestPublicHttpUrl(
41
+ rawUrl: string,
42
+ options: {
43
+ signal: AbortSignal
44
+ headers?: Record<string, string>
45
+ headersForUrl?: (url: string) => Record<string, string>
46
+ onResponse?: (response: PublicHttpResponse, url: string) => void
47
+ maxRedirects?: number
48
+ dependencies?: PublicHttpDependencies
49
+ internalDestinationPolicy?: InternalDestinationPolicy
50
+ },
51
+ ): Promise<PublicHttpResult> {
52
+ const dependencies = options.dependencies ?? defaultPublicHttpDependencies
53
+ const maxRedirects = options.maxRedirects ?? DEFAULT_PUBLIC_HTTP_MAX_REDIRECTS
54
+ const internalDestinationPolicy = options.internalDestinationPolicy ?? getBootInternalDestinationPolicy()
55
+ let current = requirePublicHttpUrl(rawUrl, internalDestinationPolicy)
56
+ const initialOrigin = new URL(current).origin
57
+ for (let redirects = 0; redirects <= maxRedirects; redirects++) {
58
+ const parsed = new URL(current)
59
+ const hostname = unbracketHostname(parsed.hostname)
60
+ const response = await dependencies.request({
61
+ protocol: parsed.protocol as 'http:' | 'https:',
62
+ hostname,
63
+ ...(parsed.port === '' ? {} : { port: Number(parsed.port) }),
64
+ path: `${parsed.pathname}${parsed.search}`,
65
+ method: 'GET',
66
+ headers: {
67
+ ...headersForHop(options.headers ?? {}, initialOrigin, parsed.origin),
68
+ ...options.headersForUrl?.(current),
69
+ Host: parsed.host,
70
+ },
71
+ ...(parsed.protocol === 'https:' ? { servername: hostname } : {}),
72
+ signal: options.signal,
73
+ lookup: createPublicSocketLookup(dependencies.resolveAddresses, internalDestinationPolicy),
74
+ autoSelectFamily: false,
75
+ })
76
+ options.onResponse?.(response, current)
77
+ if (!isRedirectStatus(response.statusCode)) return { response, finalUrl: current }
78
+ try {
79
+ const location = headerValue(response.headers, 'location')
80
+ if (location === null) throw new Error(`redirect from ${current} omitted the Location header`)
81
+ if (redirects === maxRedirects) throw new Error(`redirect limit exceeded (${maxRedirects})`)
82
+ current = requirePublicHttpUrl(new URL(location, current).toString(), internalDestinationPolicy)
83
+ } finally {
84
+ response.cancel()
85
+ }
86
+ }
87
+ throw new Error('redirect limit exceeded')
88
+ }
89
+
90
+ function headersForHop(
91
+ headers: Record<string, string>,
92
+ initialOrigin: string,
93
+ currentOrigin: string,
94
+ ): Record<string, string> {
95
+ if (initialOrigin === currentOrigin) return headers
96
+ return Object.fromEntries(
97
+ Object.entries(headers).filter(([name]) => !/^(authorization|cookie|proxy-authorization)$/i.test(name)),
98
+ )
99
+ }
100
+
101
+ export function createPublicSocketLookup(
102
+ resolveAddresses: PublicHttpDependencies['resolveAddresses'],
103
+ internalDestinationPolicy: InternalDestinationPolicy = getBootInternalDestinationPolicy(),
104
+ ): LookupFunction {
105
+ return (hostname, options, callback) => {
106
+ void resolveAddresses(hostname).then(
107
+ (addresses) => {
108
+ if (addresses.length === 0) {
109
+ callback(new Error(`DNS lookup returned no addresses for ${hostname}`), '', 0)
110
+ return
111
+ }
112
+ for (const candidate of addresses) {
113
+ const actualFamily = isIP(candidate.address)
114
+ if (actualFamily !== candidate.family || (actualFamily !== 4 && actualFamily !== 6)) {
115
+ callback(new Error(`DNS lookup returned an invalid address for ${hostname}`), '', 0)
116
+ return
117
+ }
118
+ const classification = classifyIpAddress(candidate.address)
119
+ if (
120
+ classification.blocked &&
121
+ !allowsInternalDestination(hostname, candidate.address, internalDestinationPolicy)
122
+ ) {
123
+ callback(
124
+ new Error(
125
+ `DNS lookup rejected non-public address for ${hostname}: ${classification.reason ?? candidate.address}`,
126
+ ),
127
+ '',
128
+ 0,
129
+ )
130
+ return
131
+ }
132
+ }
133
+ const requestedFamily = options.family === 4 || options.family === 6 ? options.family : undefined
134
+ const chosen = addresses.find(
135
+ (candidate) => requestedFamily === undefined || candidate.family === requestedFamily,
136
+ )
137
+ if (chosen === undefined) {
138
+ callback(new Error(`DNS lookup returned no IPv${requestedFamily} address for ${hostname}`), '', 0)
139
+ return
140
+ }
141
+ if (options.all === true) callback(null, [chosen])
142
+ else callback(null, chosen.address, chosen.family)
143
+ },
144
+ (error: unknown) => callback(error instanceof Error ? error : new Error(String(error)), '', 0),
145
+ )
146
+ }
147
+ }
148
+
149
+ export function headerValue(headers: IncomingHttpHeaders, name: string): string | null {
150
+ const value = headers[name]
151
+ if (Array.isArray(value)) return value[0] ?? null
152
+ return value ?? null
153
+ }
154
+
155
+ export const defaultPublicHttpDependencies: PublicHttpDependencies = {
156
+ async resolveAddresses(hostname) {
157
+ const addresses = await dnsLookup(hostname, { all: true, verbatim: true })
158
+ return addresses.map((candidate) => {
159
+ if (candidate.family !== 4 && candidate.family !== 6) {
160
+ throw new Error(`DNS lookup returned unsupported family ${candidate.family} for ${hostname}`)
161
+ }
162
+ return { address: candidate.address, family: candidate.family }
163
+ })
164
+ },
165
+ async request(options) {
166
+ return await new Promise((resolve, reject) => {
167
+ const request = options.protocol === 'https:' ? httpsRequest : httpRequest
168
+ const outgoing = request(options, (response) => {
169
+ resolve({
170
+ statusCode: response.statusCode ?? 0,
171
+ headers: response.headers,
172
+ body: response,
173
+ cancel: () => response.destroy(),
174
+ })
175
+ })
176
+ outgoing.once('error', reject)
177
+ outgoing.end()
178
+ })
179
+ },
180
+ }
181
+
182
+ function requirePublicHttpUrl(rawUrl: string, internalDestinationPolicy: InternalDestinationPolicy): string {
183
+ let parsed: URL
184
+ try {
185
+ parsed = new URL(rawUrl)
186
+ } catch {
187
+ throw new Error(`URL must be a valid public HTTP(S) URL: ${rawUrl}`)
188
+ }
189
+ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
190
+ throw new Error(`URL must use public HTTP(S), not ${parsed.protocol}`)
191
+ }
192
+ const classification = classifyUrl(parsed.toString(), internalDestinationPolicy)
193
+ if (classification.blocked) {
194
+ throw new Error(
195
+ `SSRF policy rejected non-public URL (${classification.category ?? 'internal'}): ${classification.reason ?? rawUrl}`,
196
+ )
197
+ }
198
+ return parsed.toString()
199
+ }
200
+
201
+ function unbracketHostname(hostname: string): string {
202
+ return hostname.startsWith('[') && hostname.endsWith(']') ? hostname.slice(1, -1) : hostname
203
+ }
204
+
205
+ function isRedirectStatus(status: number): boolean {
206
+ return status === 301 || status === 302 || status === 303 || status === 307 || status === 308
207
+ }