picocode-core 0.9.171 → 0.9.172

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "picocode-core",
-  "version": "0.9.171",
+  "version": "0.9.172",
   "description": "The agent runtime behind pico: sessions, tools, subagents, MCP, memory, and model access",
   "type": "module",
   "license": "MIT",
@@ -29,6 +29,7 @@
     "diff": "^7.0.0",
     "globby": "^16.2.4",
     "proper-lockfile": "^4.1.2",
+    "sharp": "^0.34.5",
     "unpdf": "^1.8.1"
   }
 }

package/src/agent.js

@@ -1,8 +1,8 @@
-import { readFile } from 'node:fs/promises'
+import { prepareImages } from './images.js'
 import { execFile } from 'node:child_process'
 import { promisify } from 'node:util'
 import { compose, scope, model, noToolsCalled, Inherit, getText } from '@prsm/ai'
-import { commitLabel, elementLabel, fileLabel, mediaTypeFor, selectionLabel } from './attachments.js'
+import { commitLabel, elementLabel, fileLabel, selectionLabel } from './attachments.js'
 
 const exec = promisify(execFile)
 // a whole patch rides along only while it is small; a bigger commit
@@ -38,31 +38,16 @@ async function hydratePart(part) {
   if (part.type === 'selection') return { type: 'text', text: selectionLabel(part) }
   if (part.type === 'commit') return hydrateCommit(part)
   if (part.type === 'element') return { type: 'text', text: elementLabel(part) }
-  if (part.type !== 'image' || part.source?.kind !== 'path') return part
-  const mediaType = part.source.mediaType || mediaTypeFor(part.source.path)
-  if (!mediaType) return { type: 'text', text: `[image unavailable: ${part.source.path}]` }
-  try {
-    const data = await readFile(part.source.path)
-    // an empty file is not an image; sent as one it fails every request
-    // that carries this history from then on
-    if (!data.length) return { type: 'text', text: `[image unavailable: ${part.source.path} is empty]` }
-    return {
-      type: 'image',
-      source: { kind: 'base64', mediaType, data: data.toString('base64') },
-    }
-  } catch {
-    return { type: 'text', text: `[image unavailable: ${part.source.path}]` }
-  }
+  return part
 }
 
-export function hydrateImages(history) {
-  return Promise.all(
-    history.map(async (message) =>
-      Array.isArray(message.content)
-        ? { ...message, content: await Promise.all(message.content.map(hydratePart)) }
-        : message,
-    ),
-  )
+export async function hydrateImages(history) {
+  const hydrated = await Promise.all(history.map(async (message) =>
+    Array.isArray(message.content)
+      ? { ...message, content: await Promise.all(message.content.map(hydratePart)) }
+      : message,
+  ))
+  return prepareImages(hydrated)
 }
 
 // reasoning models with large contexts can sit minutes before the first
@@ -172,13 +157,17 @@ export async function runTurn({ history, tools, recorder, modelName, effort, aut
   const step = compose(
     scope(
       { inherit: Inherit.Conversation, system, tools, until: noToolsCalled(), stream },
-      (ctx) =>
-        model({
+      async (ctx) => {
+        const out = await model({
           model: modelName,
           ...(effort && { effort }),
           ...(auth?.apiKey && { apiKey: auth.apiKey }),
           ...(auth?.headers && { headers: auth.headers }),
-        })({ ...ctx, abortSignal: internal.signal }),
+        })({ ...ctx, history: await hydrateImages(ctx.history), abortSignal: internal.signal })
+        // Prepared bytes and omission notices belong only to the request, not
+        // the conversation returned by the provider or subsequent tool rounds.
+        return { ...out, history: [...ctx.history, ...out.history.slice(ctx.history.length)] }
+      },
     ),
   )
 
@@ -190,7 +179,7 @@ export async function runTurn({ history, tools, recorder, modelName, effort, aut
 
   arm()
   try {
-    const out = await step({ history: await hydrateImages(history), tools: [] })
+    const out = await step({ history, tools: [] })
     const interrupted = !!signal?.aborted || stalled
     if (interrupted) {
       return { messages: partialMessages(), usage: usageSeen, lastPromptTokens, interrupted, stalled }

package/src/images.js

@@ -0,0 +1,127 @@
+import { open } from 'node:fs/promises'
+import { constants } from 'node:fs'
+import sharp from 'sharp'
+
+// Conservative application budgets, not claims about every provider's limits.
+// OpenAI documents 32px patches and model/detail-specific resizing budgets:
+// https://developers.openai.com/api/docs/guides/images-vision
+// Claude documents 32MB requests and a 2000px side limit
+// above 20 images: https://platform.claude.com/docs/en/build-with-claude/vision
+// Apply the same envelope across models, including Codex (whose endpoint
+// must not be relied on to perform the documented API's automatic resizing).
+export const IMAGE_LIMITS = Object.freeze({
+  side: 2000, patches: 1536, count: 20, requestPatches: 12000,
+  bytes: 4_000_000, requestBase64Bytes: 20_000_000,
+  inputBytes: 64_000_000, inputPixels: 100_000_000,
+})
+const patches = (w, h) => Math.ceil(w / 32) * Math.ceil(h / 32)
+
+export function imageUnavailable(part, reason) {
+  const label = part.source?.kind === 'path' ? part.source.path : 'attached image'
+  return { type: 'text', text: `[image unavailable: ${label}; ${reason}. Original unchanged. Use a smaller crop or paginated screenshot, or attach the image again.]` }
+}
+
+export function imageDimensions(width, height) {
+  let scale = Math.min(1, IMAGE_LIMITS.side / width, IMAGE_LIMITS.side / height,
+    Math.sqrt(IMAGE_LIMITS.patches * 32 * 32 / (width * height)))
+  let w = Math.max(1, Math.floor(width * scale))
+  let h = Math.max(1, Math.floor(height * scale))
+  // Rounding UP to patches matters, especially for long, narrow screenshots.
+  while (patches(w, h) > IMAGE_LIMITS.patches) {
+    scale *= 0.99
+    w = Math.max(1, Math.floor(width * scale))
+    h = Math.max(1, Math.floor(height * scale))
+  }
+  return { width: w, height: h }
+}
+
+async function imageBytes(source) {
+  if (source?.kind === 'path') {
+    const file = await open(source.path, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0))
+    try {
+      const stat = await file.stat()
+      if (!stat.isFile()) throw new Error('not a regular image file')
+      if (stat.size > IMAGE_LIMITS.inputBytes) throw new Error('source exceeds the 64MB safety limit')
+      // Bounded even if the file grows after stat.
+      const data = Buffer.alloc(Math.min(stat.size + 1, IMAGE_LIMITS.inputBytes + 1))
+      let size = 0
+      while (size < data.length) {
+        const { bytesRead } = await file.read(data, size, data.length - size, null)
+        if (!bytesRead) break
+        size += bytesRead
+      }
+      if (size > stat.size) throw new Error('image changed while reading; retry')
+      return data.subarray(0, size)
+    } finally { await file.close() }
+  }
+  let encoded = source?.kind === 'base64' ? source.data : null
+  if (source?.kind === 'url') {
+    encoded = /^data:image\/[\w.+-]+;base64,([\s\S]*)$/i.exec(source.url)?.[1]
+  }
+  if (typeof encoded !== 'string') throw new Error('unvalidated remote or unsupported image source; download it to a local file first')
+  if (encoded.length > Math.ceil(IMAGE_LIMITS.inputBytes / 3) * 4) throw new Error('source exceeds the 64MB safety limit')
+  encoded = encoded.replace(/\s/g, '')
+  if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded) || encoded.length % 4 === 1) throw new Error('invalid base64 image')
+  const bytes = Buffer.from(encoded, 'base64')
+  if (bytes.toString('base64').replace(/=+$/, '') !== encoded.replace(/=+$/, '')) throw new Error('invalid base64 image')
+  return bytes
+}
+
+export async function prepareImage(part) {
+  try {
+    const bytes = await imageBytes(part.source)
+    if (!bytes.length) throw new Error('image is empty')
+    const options = { limitInputPixels: IMAGE_LIMITS.inputPixels, failOn: 'warning' }
+    const metadata = await sharp(bytes, options).metadata()
+    if (!['png', 'jpeg', 'webp', 'gif'].includes(metadata.format)) throw new Error('unsupported image format; use PNG, JPEG, WebP or GIF')
+    if (!metadata.width || !metadata.height) throw new Error('invalid image dimensions')
+    const rotated = [5, 6, 7, 8].includes(metadata.orientation)
+    const dimensions = imageDimensions(rotated ? metadata.height : metadata.width, rotated ? metadata.width : metadata.height)
+    // Decode fully: metadata alone accepts truncated/corrupt pixel data. Always
+    // emit a static, correctly labelled image; animation uses its first frame.
+    const image = sharp(bytes, options).rotate().resize({ ...dimensions, fit: 'fill' })
+    let result = await image.png().toBuffer({ resolveWithObject: true })
+    let mediaType = 'image/png'
+    if (result.data.length > IMAGE_LIMITS.bytes) {
+      result = await sharp(result.data).flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer({ resolveWithObject: true })
+      mediaType = 'image/jpeg'
+    }
+    if (result.data.length > IMAGE_LIMITS.bytes) throw new Error('encoded image exceeds the 4MB safety limit')
+    return { part: { type: 'image', source: { kind: 'base64', mediaType, data: result.data.toString('base64') } }, patches: patches(result.info.width, result.info.height) }
+  } catch (error) {
+    // Do not expose decoder internals or base64 payloads to the model.
+    const reason = error.code === 'ENOENT' ? 'file is missing' :
+      /^(image |source |invalid |unsupported |unvalidated |not a regular|encoded image)/.test(error.message) ? error.message : 'cannot read or decode image safely'
+    return { part: imageUnavailable(part, reason), patches: 0 }
+  }
+}
+
+// Sequential and newest-first: bound decoder memory, and retain the images
+// relevant to the current task rather than letting old history exhaust budgets.
+export async function prepareImages(history) {
+  const out = history.map(message => Array.isArray(message.content) ? { ...message, content: [...message.content] } : message)
+  let count = 0, totalPatches = 0, totalBytes = 0
+  for (let m = out.length - 1; m >= 0; m--) {
+    const content = out[m].content
+    if (!Array.isArray(content)) continue
+    for (let p = content.length - 1; p >= 0; p--) {
+      const original = content[p]
+      if (original.type !== 'image') continue
+      if (count >= IMAGE_LIMITS.count) {
+        content[p] = imageUnavailable(original, 'request image count safety budget reached; newest images retained')
+        continue
+      }
+      const prepared = await prepareImage(original)
+      const size = prepared.part.source?.data.length || 0
+      if (totalPatches + prepared.patches > IMAGE_LIMITS.requestPatches || totalBytes + size > IMAGE_LIMITS.requestBase64Bytes) {
+        content[p] = imageUnavailable(original, 'request image safety budget reached; newer images take priority')
+        continue
+      }
+      content[p] = prepared.part
+      if (prepared.part.type === 'image') count++
+      totalPatches += prepared.patches
+      totalBytes += size
+    }
+  }
+  return out
+}