picocode-core 0.9.170 → 0.9.172

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "picocode-core",
-  "version": "0.9.170",
+  "version": "0.9.172",
   "description": "The agent runtime behind pico: sessions, tools, subagents, MCP, memory, and model access",
   "type": "module",
   "license": "MIT",
@@ -29,6 +29,7 @@
     "diff": "^7.0.0",
     "globby": "^16.2.4",
     "proper-lockfile": "^4.1.2",
+    "sharp": "^0.34.5",
     "unpdf": "^1.8.1"
   }
 }

package/src/agent.js

@@ -1,8 +1,8 @@
-import { readFile } from 'node:fs/promises'
+import { prepareImages } from './images.js'
 import { execFile } from 'node:child_process'
 import { promisify } from 'node:util'
 import { compose, scope, model, noToolsCalled, Inherit, getText } from '@prsm/ai'
-import { commitLabel, elementLabel, fileLabel, mediaTypeFor, selectionLabel } from './attachments.js'
+import { commitLabel, elementLabel, fileLabel, selectionLabel } from './attachments.js'
 
 const exec = promisify(execFile)
 // a whole patch rides along only while it is small; a bigger commit
@@ -38,31 +38,16 @@ async function hydratePart(part) {
   if (part.type === 'selection') return { type: 'text', text: selectionLabel(part) }
   if (part.type === 'commit') return hydrateCommit(part)
   if (part.type === 'element') return { type: 'text', text: elementLabel(part) }
-  if (part.type !== 'image' || part.source?.kind !== 'path') return part
-  const mediaType = part.source.mediaType || mediaTypeFor(part.source.path)
-  if (!mediaType) return { type: 'text', text: `[image unavailable: ${part.source.path}]` }
-  try {
-    const data = await readFile(part.source.path)
-    // an empty file is not an image; sent as one it fails every request
-    // that carries this history from then on
-    if (!data.length) return { type: 'text', text: `[image unavailable: ${part.source.path} is empty]` }
-    return {
-      type: 'image',
-      source: { kind: 'base64', mediaType, data: data.toString('base64') },
-    }
-  } catch {
-    return { type: 'text', text: `[image unavailable: ${part.source.path}]` }
-  }
+  return part
 }
 
-export function hydrateImages(history) {
-  return Promise.all(
-    history.map(async (message) =>
-      Array.isArray(message.content)
-        ? { ...message, content: await Promise.all(message.content.map(hydratePart)) }
-        : message,
-    ),
-  )
+export async function hydrateImages(history) {
+  const hydrated = await Promise.all(history.map(async (message) =>
+    Array.isArray(message.content)
+      ? { ...message, content: await Promise.all(message.content.map(hydratePart)) }
+      : message,
+  ))
+  return prepareImages(hydrated)
 }
 
 // reasoning models with large contexts can sit minutes before the first
@@ -143,7 +128,7 @@ export async function runTurn({ history, tools, recorder, modelName, effort, aut
       // a tool may legitimately run for minutes (test suites, slow fetches)
       // and emits nothing while it does; the watchdog guards the provider
       // stream, so pause it until the tool finishes. tools carry their own
-      // timeouts (bash 120s default, web tools via dredge)
+      // timeouts (bash 120s default)
       clearTimeout(watchdog)
       recorder.currentCall = event.call
       onStream?.(event)
@@ -172,13 +157,17 @@ export async function runTurn({ history, tools, recorder, modelName, effort, aut
   const step = compose(
     scope(
       { inherit: Inherit.Conversation, system, tools, until: noToolsCalled(), stream },
-      (ctx) =>
-        model({
+      async (ctx) => {
+        const out = await model({
           model: modelName,
           ...(effort && { effort }),
           ...(auth?.apiKey && { apiKey: auth.apiKey }),
           ...(auth?.headers && { headers: auth.headers }),
-        })({ ...ctx, abortSignal: internal.signal }),
+        })({ ...ctx, history: await hydrateImages(ctx.history), abortSignal: internal.signal })
+        // Prepared bytes and omission notices belong only to the request, not
+        // the conversation returned by the provider or subsequent tool rounds.
+        return { ...out, history: [...ctx.history, ...out.history.slice(ctx.history.length)] }
+      },
     ),
   )
 
@@ -190,7 +179,7 @@ export async function runTurn({ history, tools, recorder, modelName, effort, aut
 
   arm()
   try {
-    const out = await step({ history: await hydrateImages(history), tools: [] })
+    const out = await step({ history, tools: [] })
     const interrupted = !!signal?.aborted || stalled
     if (interrupted) {
       return { messages: partialMessages(), usage: usageSeen, lastPromptTokens, interrupted, stalled }

package/src/controller.js

@@ -53,7 +53,7 @@ export const SESSION_COLORS = {
   gray: '#9ca3af',
 }
 
-const WORKER_TOOLS = ['read', 'write', 'edit', 'bash', 'glob', 'grep', 'shell_output', 'shell_kill', 'web_search', 'web_fetch']
+const WORKER_TOOLS = ['read', 'write', 'edit', 'bash', 'glob', 'grep', 'shell_output', 'shell_kill']
 const AGENT_TOOLS = ['agent_plan', 'agent_start', 'agent_list', 'agent_collect', 'agent_cancel']
 const CONTEXT_COLORS = ['#67b7ff', '#c792ea', '#f7c66a', '#f78c6c', '#6be795']
 
@@ -202,7 +202,9 @@ export function createController({ boot }) {
       const sessionId = state.session?.id
       if (!sessionId) throw new Error('worker requires an active session')
       const scratchpad = ensureDir(agentScratchDir(boot.root, sessionId, agent.id))
-      const requestedTools = agent.tools?.length ? agent.tools.filter((name) => WORKER_TOOLS.includes(name)) : WORKER_TOOLS
+      const mcpTools = boot.mcp.tools()
+      const availableNames = [...WORKER_TOOLS, ...mcpTools.map((tool) => tool.name)]
+      const requestedTools = agent.tools?.length ? agent.tools.filter((name) => availableNames.includes(name)) : availableNames
       const { tools, recorder } = createToolset({
         cwd: boot.cwd,
         env: { ...boot.env, PICO_SCRATCHPAD: scratchpad },
@@ -213,10 +215,10 @@ export function createController({ boot }) {
         shells: boot.shells,
         sessionId,
         sessionFile: state.session?.file,
-        dredge: boot.dredge,
         signal,
         maxToolCalls: 30,
         allowNames: requestedTools,
+        mcpTools,
       })
       return runTurn({
         history: [{ role: 'user', content: agent.prompt }],
@@ -277,6 +279,7 @@ export function createController({ boot }) {
 
       const runWorker = async ({ history, role, tools: enabled = true, onStream }) => {
         const scratchpad = ensureDir(agentScratchDir(boot.root, sessionId, `deliberation-${id}-${role}`))
+        const mcpTools = enabled ? boot.mcp.tools() : []
         const toolset = createToolset({
           cwd: boot.cwd,
           env: { ...boot.env, PICO_SCRATCHPAD: scratchpad },
@@ -284,10 +287,10 @@ export function createController({ boot }) {
           shells: boot.shells,
           sessionId,
           sessionFile: state.session?.file,
-          dredge: boot.dredge,
           signal,
           maxToolCalls: 30,
-          allowNames: enabled ? WORKER_TOOLS : [],
+          allowNames: enabled ? [...WORKER_TOOLS, ...mcpTools.map((tool) => tool.name)] : [],
+          mcpTools,
         })
         return runTurn({
           history,
@@ -622,7 +625,6 @@ export function createController({ boot }) {
       deliberations: boot.deliberationModel || (boot.proposerModel && boot.reviewerModel) ? deliberations : null,
       onAgentsCollected: discardCollectedAgentNotes,
       askUser,
-      dredge: boot.dredge,
       mcpTools: boot.mcp.tools(),
       userTools: userToolScan.tools,
       signal: controller.signal,
@@ -1216,7 +1218,6 @@ export function createController({ boot }) {
       shells: boot.shells,
       wakeups: boot.wakeups,
       memory: boot.memory,
-      dredge: boot.dredge,
       ...extra,
     })
   }

package/src/images.js

@@ -0,0 +1,127 @@
+import { open } from 'node:fs/promises'
+import { constants } from 'node:fs'
+import sharp from 'sharp'
+
+// Conservative application budgets, not claims about every provider's limits.
+// OpenAI documents 32px patches and model/detail-specific resizing budgets:
+// https://developers.openai.com/api/docs/guides/images-vision
+// Claude documents 32MB requests and a 2000px side limit
+// above 20 images: https://platform.claude.com/docs/en/build-with-claude/vision
+// Apply the same envelope across models, including Codex (whose endpoint
+// must not be relied on to perform the documented API's automatic resizing).
+export const IMAGE_LIMITS = Object.freeze({
+  side: 2000, patches: 1536, count: 20, requestPatches: 12000,
+  bytes: 4_000_000, requestBase64Bytes: 20_000_000,
+  inputBytes: 64_000_000, inputPixels: 100_000_000,
+})
+const patches = (w, h) => Math.ceil(w / 32) * Math.ceil(h / 32)
+
+export function imageUnavailable(part, reason) {
+  const label = part.source?.kind === 'path' ? part.source.path : 'attached image'
+  return { type: 'text', text: `[image unavailable: ${label}; ${reason}. Original unchanged. Use a smaller crop or paginated screenshot, or attach the image again.]` }
+}
+
+export function imageDimensions(width, height) {
+  let scale = Math.min(1, IMAGE_LIMITS.side / width, IMAGE_LIMITS.side / height,
+    Math.sqrt(IMAGE_LIMITS.patches * 32 * 32 / (width * height)))
+  let w = Math.max(1, Math.floor(width * scale))
+  let h = Math.max(1, Math.floor(height * scale))
+  // Rounding UP to patches matters, especially for long, narrow screenshots.
+  while (patches(w, h) > IMAGE_LIMITS.patches) {
+    scale *= 0.99
+    w = Math.max(1, Math.floor(width * scale))
+    h = Math.max(1, Math.floor(height * scale))
+  }
+  return { width: w, height: h }
+}
+
+async function imageBytes(source) {
+  if (source?.kind === 'path') {
+    const file = await open(source.path, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0))
+    try {
+      const stat = await file.stat()
+      if (!stat.isFile()) throw new Error('not a regular image file')
+      if (stat.size > IMAGE_LIMITS.inputBytes) throw new Error('source exceeds the 64MB safety limit')
+      // Bounded even if the file grows after stat.
+      const data = Buffer.alloc(Math.min(stat.size + 1, IMAGE_LIMITS.inputBytes + 1))
+      let size = 0
+      while (size < data.length) {
+        const { bytesRead } = await file.read(data, size, data.length - size, null)
+        if (!bytesRead) break
+        size += bytesRead
+      }
+      if (size > stat.size) throw new Error('image changed while reading; retry')
+      return data.subarray(0, size)
+    } finally { await file.close() }
+  }
+  let encoded = source?.kind === 'base64' ? source.data : null
+  if (source?.kind === 'url') {
+    encoded = /^data:image\/[\w.+-]+;base64,([\s\S]*)$/i.exec(source.url)?.[1]
+  }
+  if (typeof encoded !== 'string') throw new Error('unvalidated remote or unsupported image source; download it to a local file first')
+  if (encoded.length > Math.ceil(IMAGE_LIMITS.inputBytes / 3) * 4) throw new Error('source exceeds the 64MB safety limit')
+  encoded = encoded.replace(/\s/g, '')
+  if (!/^[A-Za-z0-9+/]*={0,2}$/.test(encoded) || encoded.length % 4 === 1) throw new Error('invalid base64 image')
+  const bytes = Buffer.from(encoded, 'base64')
+  if (bytes.toString('base64').replace(/=+$/, '') !== encoded.replace(/=+$/, '')) throw new Error('invalid base64 image')
+  return bytes
+}
+
+export async function prepareImage(part) {
+  try {
+    const bytes = await imageBytes(part.source)
+    if (!bytes.length) throw new Error('image is empty')
+    const options = { limitInputPixels: IMAGE_LIMITS.inputPixels, failOn: 'warning' }
+    const metadata = await sharp(bytes, options).metadata()
+    if (!['png', 'jpeg', 'webp', 'gif'].includes(metadata.format)) throw new Error('unsupported image format; use PNG, JPEG, WebP or GIF')
+    if (!metadata.width || !metadata.height) throw new Error('invalid image dimensions')
+    const rotated = [5, 6, 7, 8].includes(metadata.orientation)
+    const dimensions = imageDimensions(rotated ? metadata.height : metadata.width, rotated ? metadata.width : metadata.height)
+    // Decode fully: metadata alone accepts truncated/corrupt pixel data. Always
+    // emit a static, correctly labelled image; animation uses its first frame.
+    const image = sharp(bytes, options).rotate().resize({ ...dimensions, fit: 'fill' })
+    let result = await image.png().toBuffer({ resolveWithObject: true })
+    let mediaType = 'image/png'
+    if (result.data.length > IMAGE_LIMITS.bytes) {
+      result = await sharp(result.data).flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer({ resolveWithObject: true })
+      mediaType = 'image/jpeg'
+    }
+    if (result.data.length > IMAGE_LIMITS.bytes) throw new Error('encoded image exceeds the 4MB safety limit')
+    return { part: { type: 'image', source: { kind: 'base64', mediaType, data: result.data.toString('base64') } }, patches: patches(result.info.width, result.info.height) }
+  } catch (error) {
+    // Do not expose decoder internals or base64 payloads to the model.
+    const reason = error.code === 'ENOENT' ? 'file is missing' :
+      /^(image |source |invalid |unsupported |unvalidated |not a regular|encoded image)/.test(error.message) ? error.message : 'cannot read or decode image safely'
+    return { part: imageUnavailable(part, reason), patches: 0 }
+  }
+}
+
+// Sequential and newest-first: bound decoder memory, and retain the images
+// relevant to the current task rather than letting old history exhaust budgets.
+export async function prepareImages(history) {
+  const out = history.map(message => Array.isArray(message.content) ? { ...message, content: [...message.content] } : message)
+  let count = 0, totalPatches = 0, totalBytes = 0
+  for (let m = out.length - 1; m >= 0; m--) {
+    const content = out[m].content
+    if (!Array.isArray(content)) continue
+    for (let p = content.length - 1; p >= 0; p--) {
+      const original = content[p]
+      if (original.type !== 'image') continue
+      if (count >= IMAGE_LIMITS.count) {
+        content[p] = imageUnavailable(original, 'request image count safety budget reached; newest images retained')
+        continue
+      }
+      const prepared = await prepareImage(original)
+      const size = prepared.part.source?.data.length || 0
+      if (totalPatches + prepared.patches > IMAGE_LIMITS.requestPatches || totalBytes + size > IMAGE_LIMITS.requestBase64Bytes) {
+        content[p] = imageUnavailable(original, 'request image safety budget reached; newer images take priority')
+        continue
+      }
+      content[p] = prepared.part
+      if (prepared.part.type === 'image') count++
+      totalPatches += prepared.patches
+      totalBytes += size
+    }
+  }
+  return out
+}

package/src/tools/index.js

@@ -5,10 +5,9 @@ import { createEdit } from './edit.js'
 import { createBash } from './bash.js'
 import { createGlob } from './glob.js'
 import { createGrep } from './grep.js'
-import { createWebTools } from './web.js'
 import { createView } from './view.js'
 
-export function createToolset({ cwd, env, tracker, skills, shells, sessionId, sessionFile, wakeups, memory, agents, deliberations, onAgentsCollected, askUser, dredge, mcpTools = [], userTools = [], hostTools = [], signal, maxToolCalls, maxAgentStarts, requireAgentPlan = false, allowNames, onToolUpdate, viewer }) {
+export function createToolset({ cwd, env, tracker, skills, shells, sessionId, sessionFile, wakeups, memory, agents, deliberations, onAgentsCollected, askUser, mcpTools = [], userTools = [], hostTools = [], signal, maxToolCalls, maxAgentStarts, requireAgentPlan = false, allowNames, onToolUpdate, viewer }) {
   const recorder = createRecorder(onToolUpdate)
   let agentStarts = 0
   let plannedAgentStarts = requireAgentPlan ? null : maxAgentStarts
@@ -25,10 +24,6 @@ export function createToolset({ cwd, env, tracker, skills, shells, sessionId, se
 
   if (viewer) local.push(createView({ ...deps, viewer }))
 
-  if (dredge) {
-    local.push(...createWebTools({ dredge, recorder, signal }))
-  }
-
   if (shells) {
     local.push(
       {
@@ -283,7 +278,7 @@ export function createToolset({ cwd, env, tracker, skills, shells, sessionId, se
     })
   }
 
-  const describedToolNames = new Set(['read', 'write', 'edit', 'bash', 'glob', 'grep', 'web_search', 'schedule_wakeup'])
+  const describedToolNames = new Set(['read', 'write', 'edit', 'bash', 'glob', 'grep', 'schedule_wakeup'])
   const describedTools = new Set(local.filter((tool) => describedToolNames.has(tool.name)).map((tool) => tool.name))
   const byName = new Map()
   for (const tool of [...local, ...hostTools, ...userTools, ...mcpTools]) {

package/src/tools/web.js

@@ -1,86 +0,0 @@
-import { describeParam } from './recorder.js'
-const DEFAULT_SLICE_CHARS = 24000
-const MAX_SLICE_CHARS = 100000
-
-export function resolveDredge(config = {}, env = process.env) {
-  const url = env.DREDGE_URL || config.dredge?.url || null
-  if (!url) return null
-  return {
-    url: url.replace(/\/+$/, ''),
-    apiKey: env.DREDGE_API_KEY || config.dredge?.apiKey || null,
-  }
-}
-
-const CALL_TIMEOUT_MS = 120000
-
-async function call(dredge, path, params, signal) {
-  const query = new URLSearchParams(params)
-  // bounded above dredge's own 90s queue ceiling so a wedged server becomes
-  // a tool error instead of a hung turn
-  const signals = [AbortSignal.timeout(CALL_TIMEOUT_MS), ...(signal ? [signal] : [])]
-  const response = await fetch(`${dredge.url}${path}?${query}`, {
-    signal: AbortSignal.any(signals),
-    headers: dredge.apiKey ? { authorization: `Bearer ${dredge.apiKey}` } : {},
-  })
-  const body = await response.json().catch(() => null)
-  if (!body) throw new Error(`dredge returned http ${response.status} with no usable body`)
-  return body
-}
-
-export function createWebTools({ dredge, recorder, signal }) {
-  return [
-    {
-      name: 'web_search',
-      description:
-        'Search the web. Google-style operators work: site:, filetype:pdf, quoted phrases. Returns ranked results; pass a result url to web_fetch to read it.',
-      schema: {
-        q: { type: 'string', description: 'the search query' },
-        description: describeParam,
-      },
-      execute: async ({ q }) => {
-        recorder.extra({ title: q })
-        const body = await call(dredge, '/search', { q }, signal)
-        if (!body.ok) throw new Error(body.error?.message || 'search failed')
-        const results = (body.results || []).map((r) => ({
-          title: r.title,
-          url: r.url,
-          snippet: r.snippet,
-          source: r.source,
-        }))
-        if (results.length === 0) {
-          const backends = (body.backends || []).map((b) => `${b.name}: ${b.status}`).join(', ')
-          return { results, note: backends ? `no results · backends: ${backends}` : 'no results' }
-        }
-        return { results }
-      },
-    },
-    {
-      name: 'web_fetch',
-      description:
-        'Fetch a url and read it as clean markdown (html, pdf, docx, and textual formats like json). Long documents arrive in slices: the result says which slice you have (e.g. "slice 1 of 12") and next_cursor continues from there. Every slice you fetch permanently occupies conversation context, so only walk cursors for content you actually need, and raise maxChars only when the task genuinely needs a bigger window.',
-      schema: {
-        description: describeParam,
-        url: { type: 'string', description: 'the url to fetch' },
-        cursor: { type: 'string', description: 'pagination cursor from a previous web_fetch of the same url', optional: true },
-        maxChars: { type: 'number', description: `slice size in characters, default ${DEFAULT_SLICE_CHARS}, max ${MAX_SLICE_CHARS}`, optional: true },
-      },
-      execute: async ({ url, cursor, maxChars }) => {
-        recorder.extra({ title: url.replace(/^https?:\/\//, '').slice(0, 80) })
-        const slice = Math.min(MAX_SLICE_CHARS, Math.max(1000, maxChars || DEFAULT_SLICE_CHARS))
-        const body = await call(dredge, '/fetch', { url, maxChars: slice, ...(cursor && { cursor }) }, signal)
-        if (!body.ok) {
-          const { code, message, retryable } = body.error || {}
-          throw new Error(`${code || 'fetch failed'}: ${message || url}${retryable ? ' (retryable)' : ''}`)
-        }
-        const { markdown, metadata, pagination } = body.doc
-        return {
-          markdown,
-          title: metadata?.title || null,
-          finalUrl: metadata?.final_url || url,
-          ...(pagination?.total_chunks > 1 && { slice: `${pagination.chunk_index + 1} of ${pagination.total_chunks}` }),
-          ...(pagination?.next_cursor && { next_cursor: pagination.next_cursor }),
-        }
-      },
-    },
-  ]
-}