picocode-core 0.9.129 → 0.9.130

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "picocode-core",
-  "version": "0.9.129",
+  "version": "0.9.130",
   "description": "The agent runtime behind pico: sessions, tools, subagents, MCP, memory, and model access",
   "type": "module",
   "license": "MIT",

package/src/catalog.js

@@ -63,6 +63,7 @@ export function extractModels(providers, providerIds) {
         desc: m.description || m.name || id,
         price: m.cost && m.cost.input != null ? { in: m.cost.input, out: m.cost.output } : null,
         effort: !!m.reasoning,
+        vision: !m.modalities || m.modalities.input?.includes('image') === true,
         // the measured fill number is input tokens, so prefer the input limit
         // as the denominator where the provider distinguishes it
         context: m.limit?.input || m.limit?.context || null,
@@ -81,6 +82,7 @@ export function adhocModel(name, providerIds) {
     desc: 'not in catalog',
     price: null,
     effort: false,
+    vision: true,
     context: null,
   }
 }

package/src/controller.js

@@ -363,8 +363,10 @@ export function createController({ boot }) {
   }
 
   async function executeTurn(text) {
-    const { content } = finalizeUserContent(text, state.attachments)
-    persist(makeEvent('message', { message: { role: 'user', content } }))
+    const { content, used } = finalizeUserContent(text, state.attachments)
+    const viewed = used.length > 0 && used.every((placeholder) => deliveredImages.has(placeholder))
+    for (const placeholder of used) deliveredImages.delete(placeholder)
+    persist(makeEvent('message', { message: { role: 'user', content }, ...(viewed ? { origin: 'view' } : {}) }))
     ensureSession()
     reDerive()
     await runAgentTurn()
@@ -577,6 +579,7 @@ export function createController({ boot }) {
       requireAgentPlan: !!researchAgentLimit,
       allowNames: researchAgentLimit ? AGENT_TOOLS : undefined,
       onToolUpdate,
+      viewer: state.model.vision === false ? null : { deliver: deliverImage },
     })
 
     sendAfterToolTriggered = false
@@ -807,6 +810,17 @@ export function createController({ boot }) {
     return placeholder
   }
 
+  // an image the model asked to see rides in as an expedited user message,
+  // which the tool-completion path sends right after the current call
+  const deliveredImages = new Set()
+  function deliverImage(path, label) {
+    const placeholder = attachImage(path)
+    if (!placeholder) return false
+    deliveredImages.add(placeholder)
+    set({ expedited: [...state.expedited, `${label}\n${placeholder}`] })
+    return true
+  }
+
   function attachFile(path) {
     if (!existsSync(path)) return null
     const placeholder = `[File #${++state.imageCount}]`

package/src/derive.js

@@ -103,7 +103,7 @@ function foldMessage(state, event) {
           .map((p) => (p.type === 'text' ? p.text : p.type === 'file' ? `[file: ${p.path}]` : `[image: ${String(p.source?.path || '').split('/').pop() || 'attached'}]`))
           .join('')
       : String(message.content)
-    state.transcript.push({ ...base, kind: 'user', text, content: message.content })
+    state.transcript.push({ ...base, kind: 'user', text, content: message.content, ...(event.data.origin ? { origin: event.data.origin } : {}) })
     return
   }
   if (message.role === 'assistant') {

package/src/tools/index.js

@@ -6,8 +6,9 @@ import { createBash } from './bash.js'
 import { createGlob } from './glob.js'
 import { createGrep } from './grep.js'
 import { createWebTools } from './web.js'
+import { createView } from './view.js'
 
-export function createToolset({ cwd, env, tracker, skills, shells, sessionId, sessionFile, wakeups, memory, agents, deliberations, onAgentsCollected, askUser, dredge, mcpTools = [], userTools = [], signal, maxToolCalls, maxAgentStarts, requireAgentPlan = false, allowNames, onToolUpdate }) {
+export function createToolset({ cwd, env, tracker, skills, shells, sessionId, sessionFile, wakeups, memory, agents, deliberations, onAgentsCollected, askUser, dredge, mcpTools = [], userTools = [], signal, maxToolCalls, maxAgentStarts, requireAgentPlan = false, allowNames, onToolUpdate, viewer }) {
   const recorder = createRecorder(onToolUpdate)
   let agentStarts = 0
   let plannedAgentStarts = requireAgentPlan ? null : maxAgentStarts
@@ -22,6 +23,8 @@ export function createToolset({ cwd, env, tracker, skills, shells, sessionId, se
     createGrep(deps),
   ]
 
+  if (viewer) local.push(createView({ ...deps, viewer }))
+
   if (dredge) {
     local.push(...createWebTools({ dredge, recorder, signal }))
   }

package/src/tools/read.js

@@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises'
 import { resolve } from 'node:path'
 import { execFile } from 'node:child_process'
 import { extractText } from 'unpdf'
+import { mediaTypeFor } from '../attachments.js'
 
 const MAX_LINES = 2000
 const MAX_LINE_LENGTH = 2000
@@ -28,6 +29,26 @@ async function unpdfPages(buffer) {
   return (Array.isArray(text) ? text : [text]).map((page) => String(page).split('\n'))
 }
 
+// the embedded images per page, from poppler when installed, so the model
+// knows what view can show it
+function pdfImages(full) {
+  return new Promise((resolvePromise) => {
+    execFile('pdfimages', ['-list', full], { maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => {
+      if (error) return resolvePromise(null)
+      const counts = new Map()
+      for (const line of stdout.split('\n')) {
+        const m = line.match(/^\s*(\d+)\s+\d+\s+image\s+(\d+)\s+(\d+)/)
+        if (!m) continue
+        const page = Number(m[1])
+        const list = counts.get(page) ?? []
+        list.push({ index: list.length, width: Number(m[2]), height: Number(m[3]) })
+        counts.set(page, list)
+      }
+      resolvePromise([...counts].map(([page, images]) => ({ page, images })))
+    })
+  })
+}
+
 async function pdfLines(full, buffer) {
   const pages = (await pdftotext(full)) ?? (await unpdfPages(buffer))
   const lines = []
@@ -62,6 +83,7 @@ export function createRead({ cwd, recorder, tracker }) {
       recorder.extra({ title: path })
       const buf = await readFile(full)
       const lines = isPdf(buf) ? await pdfLines(full, buf) : null
+      if (!lines && mediaTypeFor(full)) return { note: `${path} is an image; use view to look at it` }
       if (!lines && isBinary(buf)) throw new Error(`${path} is a binary file`)
       const source = lines ?? buf.toString('utf-8').split('\n')
 
@@ -77,6 +99,10 @@ export function createRead({ cwd, recorder, tracker }) {
       if (start + count < source.length) {
         result.note = `showing lines ${start + 1}-${start + sliced.length} of ${source.length}`
       }
+      if (lines) {
+        const images = await pdfImages(full)
+        if (images?.length) result.images = { note: 'use view with the path and page (and image index) to see one', pages: images }
+      }
       const context = tracker.check(full)
       if (context.length) result.context_from_agents_md = context
       return result

package/src/tools/view.js

@@ -0,0 +1,70 @@
+import { execFile } from 'node:child_process'
+import { mkdir, readdir } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { describeParam } from './recorder.js'
+import { mediaTypeFor } from '../attachments.js'
+
+const run = (cmd, args) =>
+  new Promise((resolvePromise, reject) => {
+    execFile(cmd, args, { maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => (error ? reject(error) : resolvePromise(stdout)))
+  })
+
+const isPdf = (path) => /\.pdf$/i.test(path)
+
+async function scratch(sessionId) {
+  const dir = join(tmpdir(), 'pico-view', String(sessionId || 'session'))
+  await mkdir(dir, { recursive: true })
+  return dir
+}
+
+// a page rendered by poppler, or one of the images embedded on it
+async function renderPage(full, page, dir) {
+  const prefix = join(dir, `page-${Date.now()}`)
+  await run('pdftoppm', ['-png', '-r', '110', '-f', String(page), '-l', String(page), '-singlefile', full, prefix])
+  return `${prefix}.png`
+}
+
+async function extractImage(full, page, index, dir) {
+  const prefix = join(dir, `img-${Date.now()}`)
+  await run('pdfimages', ['-png', '-f', String(page), '-l', String(page), full, prefix])
+  const files = (await readdir(dir)).filter((f) => f.startsWith(`${prefix.slice(dir.length + 1)}-`)).sort()
+  const file = files[index]
+  if (!file) throw new Error(`page ${page} has ${files.length} embedded image${files.length === 1 ? '' : 's'}; image index ${index} does not exist`)
+  return join(dir, file)
+}
+
+// the model api cannot carry an image inside a tool result, so the tool
+// hands the image to the viewer, which delivers it as the next user message
+// right after this call
+export function createView({ cwd, sessionId, recorder, viewer }) {
+  return {
+    name: 'view',
+    description: 'Look at an image. Pass an image file path, or a pdf path with a page number to see that page rendered (add image to pick one embedded image on that page, 0-based, as listed by read). The image arrives in the next user message.',
+    schema: {
+      description: describeParam,
+      path: { type: 'string', description: 'image or pdf path, relative to the working directory or absolute' },
+      page: { type: 'number', description: 'pdf page to render, 1-based', optional: true },
+      image: { type: 'number', description: 'embedded image index on that page, 0-based; omit to render the whole page', optional: true },
+    },
+    execute: async ({ path, page, image }) => {
+      const full = resolve(cwd, path)
+      recorder.extra({ title: page != null ? `${path} page ${page}${image != null ? ` image ${image}` : ''}` : path })
+      if (!existsSync(full)) throw new Error(`${path} does not exist`)
+      let file = full
+      let label = `[view: ${path}]`
+      if (isPdf(full)) {
+        if (page == null) throw new Error('a pdf needs a page number')
+        const dir = await scratch(sessionId)
+        file = image != null ? await extractImage(full, page, image, dir) : await renderPage(full, page, dir)
+        label = `[view: ${path}, page ${page}${image != null ? `, image ${image}` : ''}]`
+      } else if (!mediaTypeFor(full)) {
+        throw new Error(`${path} is not an image or pdf`)
+      }
+      const delivered = viewer.deliver(file, label)
+      if (!delivered) throw new Error('could not attach the image')
+      return { ok: true, note: 'the image follows in the next user message' }
+    },
+  }
+}