typeclaw 0.1.5 → 0.1.6

package/src/agent/multimodal/index.ts

@@ -0,0 +1,12 @@
+export { lookAtTool } from './look-at'
+export {
+  buildMultimodalLookerSystemPrompt,
+  imageInputSchema,
+  multimodalLookerPayloadSchema,
+  resolveImage,
+  URL_FETCH_MAX_BYTES,
+  URL_FETCH_TIMEOUT_MS,
+  type ImageInput,
+  type MultimodalLookerPayload,
+  type ResolvedImage,
+} from './looker'

package/src/agent/multimodal/look-at.ts

@@ -0,0 +1,185 @@
+import { Type } from '@mariozechner/pi-ai'
+import type { ImageContent } from '@mariozechner/pi-ai'
+import { defineTool } from '@mariozechner/pi-coding-agent'
+
+import { createSessionWithDispose, type SessionOrigin } from '@/agent'
+
+import { buildMultimodalLookerSystemPrompt, resolveImage, type ImageInput } from './looker'
+
+type ImageParam = { url: string } | { path: string } | { data: string; mimeType: string }
+
+type LookAtArgs = {
+  images: ImageParam[]
+  prompt?: string
+}
+
+type LookAtDetails = {
+  count: number
+  prompt?: string
+  text?: string
+  error?: string
+}
+
+// Routes an image-bearing turn to a vision-capable subagent so the main
+// session never sees the bytes. Saves main-agent context: when `models.default`
+// is text-only, this is the only way to get vision; when `models.default` IS
+// vision-capable, it still buys cheaper main-agent inference because the
+// image payload (which can be many KB after base64) only enters the vision
+// model's context.
+//
+// Output is the subagent's text response. The subagent itself decides whether
+// to answer the user's question (when `prompt` is supplied) or describe the
+// image (when `prompt` is omitted) via its dynamic system prompt.
+export const lookAtTool = defineTool({
+  name: 'look_at',
+  label: 'Look at images',
+  description:
+    'Route image(s) through a vision-capable subagent and get a text result. ' +
+    'Use this when you need to see an image: a screenshot the user shared, a diagram in a doc, a photo, a chart, etc. ' +
+    'Each image is specified by ONE of `url` (https://...), `path` (absolute filesystem path), or `data`+`mimeType` (base64). ' +
+    'The optional `prompt` is a question to ask about the image(s); without it, the subagent returns a faithful description. ' +
+    'The image bytes never enter your context — only the resulting text comes back.',
+  parameters: Type.Object({
+    images: Type.Array(
+      Type.Object({
+        url: Type.Optional(Type.String({ description: 'https:// URL to fetch the image from.' })),
+        path: Type.Optional(Type.String({ description: 'Absolute filesystem path (inside /agent or a mounted dir).' })),
+        data: Type.Optional(Type.String({ description: 'Base64-encoded image bytes (pair with mimeType).' })),
+        mimeType: Type.Optional(Type.String({ description: 'MIME type when using `data` (e.g. "image/png").' })),
+      }),
+      { minItems: 1, description: 'One or more images to look at.' },
+    ),
+    prompt: Type.Optional(
+      Type.String({
+        description:
+          'Optional question to ask about the image(s). When omitted, the subagent returns a faithful description.',
+      }),
+    ),
+  }),
+
+  async execute(_toolCallId, params, signal) {
+    const args = params as LookAtArgs
+    try {
+      const imageInputs = args.images.map(toImageInput)
+      const resolved = await Promise.all(imageInputs.map((i) => resolveImage(i, signal)))
+      const imageContents: ImageContent[] = resolved.map((r) => ({
+        type: 'image' as const,
+        data: r.data,
+        mimeType: r.mimeType,
+      }))
+
+      const systemPrompt = buildMultimodalLookerSystemPrompt(args.prompt)
+      const userText =
+        args.prompt !== undefined && args.prompt.trim() !== ''
+          ? args.prompt.trim()
+          : 'Please describe the attached image(s).'
+
+      const origin: SessionOrigin = {
+        kind: 'subagent',
+        subagent: 'multimodal-looker',
+        parentSessionId: '<look-at-tool>',
+      }
+
+      const { session, dispose } = await createSessionWithDispose({
+        systemPromptOverride: systemPrompt,
+        origin,
+        profile: 'vision',
+        // Both knobs are required to fully disarm the subagent's tool surface:
+        // `customTools: []` blocks typeclaw's system tools (websearch/webfetch/
+        // look_at/restart/...) — without it, the look_at tool would recurse
+        // into itself. `tools: []` blocks pi-coding-agent's defaults
+        // (read/bash/edit/write) — without it, a vision model could be talked
+        // into running shell commands or editing files inside its short-lived
+        // session. The looker should only describe images, not act.
+        tools: [],
+        customTools: [],
+      })
+
+      try {
+        await session.prompt(userText, { images: imageContents })
+        const text = extractLastAssistantText(session.messages)
+        if (text === null) {
+          return errorResult('multimodal-looker returned no text response', {
+            count: resolved.length,
+            prompt: args.prompt,
+          })
+        }
+        return successResult(text, { count: resolved.length, prompt: args.prompt })
+      } finally {
+        session.dispose()
+        await dispose()
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error)
+      return errorResult(message, { count: args.images.length, prompt: args.prompt })
+    }
+  },
+})
+
+function toImageInput(p: ImageParam): ImageInput {
+  const hasUrl = 'url' in p && p.url !== undefined && p.url !== ''
+  const hasPath = 'path' in p && p.path !== undefined && p.path !== ''
+  const hasData = 'data' in p && p.data !== undefined && p.data !== ''
+  const hasMime = 'mimeType' in p && p.mimeType !== undefined && p.mimeType !== ''
+
+  // `data` and `mimeType` are paired — accept both as one source. `mimeType`
+  // alone with no `data` is rejected as an incomplete base64 spec.
+  const sources: string[] = []
+  if (hasUrl) sources.push('url')
+  if (hasPath) sources.push('path')
+  if (hasData || hasMime) sources.push('data+mimeType')
+
+  if (sources.length === 0) {
+    throw new Error('look_at: each image must specify exactly one of `url`, `path`, or `data`+`mimeType`')
+  }
+  if (sources.length > 1) {
+    throw new Error(
+      `look_at: each image must specify exactly one of \`url\`, \`path\`, or \`data\`+\`mimeType\` (got: ${sources.join(', ')})`,
+    )
+  }
+  if (hasUrl) return { kind: 'url', url: (p as { url: string }).url }
+  if (hasPath) return { kind: 'file', path: (p as { path: string }).path }
+  if (hasData && hasMime) {
+    return { kind: 'base64', data: (p as { data: string }).data, mimeType: (p as { mimeType: string }).mimeType }
+  }
+  throw new Error('look_at: base64 image requires both `data` and `mimeType`')
+}
+
+// Pulls the most recent assistant turn's text content. The subagent's reply
+// shows up here once `session.prompt()` resolves. Tool calls in the assistant
+// message are ignored — multimodal-looker's session has no tools wired in
+// (`tools: []` + `customTools: []` at session creation), so in practice this
+// is pure text plus optional thinking blocks (which we skip).
+function extractLastAssistantText(messages: ReadonlyArray<unknown>): string | null {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i] as { role?: unknown; content?: unknown } | undefined
+    if (msg === undefined || msg.role !== 'assistant') continue
+    const content = msg.content
+    if (!Array.isArray(content)) continue
+    const texts: string[] = []
+    for (const part of content) {
+      if (part !== null && typeof part === 'object' && (part as { type?: unknown }).type === 'text') {
+        const t = (part as { text?: unknown }).text
+        if (typeof t === 'string') texts.push(t)
+      }
+    }
+    if (texts.length > 0) return texts.join('\n').trim()
+  }
+  return null
+}
+
+function successResult(text: string, partial: Omit<LookAtDetails, 'text' | 'error'>) {
+  const details: LookAtDetails = { ...partial, text }
+  return {
+    content: [{ type: 'text' as const, text }],
+    details,
+  }
+}
+
+function errorResult(message: string, partial: Omit<LookAtDetails, 'text' | 'error'>) {
+  const details: LookAtDetails = { ...partial, error: message }
+  return {
+    content: [{ type: 'text' as const, text: `look_at failed: ${message}` }],
+    details,
+  }
+}

package/src/agent/multimodal/looker.ts

@@ -0,0 +1,145 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { extname, isAbsolute } from 'node:path'
+
+import { z } from 'zod'
+
+const SUPPORTED_MIME_TYPES = {
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.webp': 'image/webp',
+} as const
+
+// Caps on URL-fetched images. The agent chooses URLs autonomously, so a
+// malicious or accidentally-large response could otherwise hang the tool
+// (no timeout) or fill memory (no size cap). 20 MB is well above any
+// reasonable screenshot/photo and well below container memory budgets;
+// 30 s is generous for a single HTTP image fetch over a slow link.
+export const URL_FETCH_TIMEOUT_MS = 30_000
+export const URL_FETCH_MAX_BYTES = 20 * 1024 * 1024
+
+type Mime = (typeof SUPPORTED_MIME_TYPES)[keyof typeof SUPPORTED_MIME_TYPES]
+
+export type ImageInput =
+  | { kind: 'url'; url: string }
+  | { kind: 'file'; path: string }
+  | { kind: 'base64'; data: string; mimeType: string }
+
+export const imageInputSchema = z.union([
+  z.object({ kind: z.literal('url'), url: z.string().url() }),
+  z.object({ kind: z.literal('file'), path: z.string().min(1) }),
+  z.object({ kind: z.literal('base64'), data: z.string().min(1), mimeType: z.string().min(1) }),
+])
+
+export const multimodalLookerPayloadSchema = z.object({
+  images: z.array(imageInputSchema).min(1),
+  prompt: z.string().min(1).optional(),
+})
+
+export type MultimodalLookerPayload = z.infer<typeof multimodalLookerPayloadSchema>
+
+// System prompt is built per-invocation so the agent sees the exact task. With
+// `prompt`: focused Q&A. Without: open-ended description. Tone the same in
+// both branches so callers can plug either form into the same downstream
+// pipeline (the look_at tool just relays the resulting text).
+export function buildMultimodalLookerSystemPrompt(prompt: string | undefined): string {
+  const base =
+    'You are a multimodal vision subagent. The user message contains one or more images attached to a short instruction.'
+  if (prompt !== undefined && prompt.trim() !== '') {
+    return [
+      base,
+      '',
+      'Your job is to ANSWER the question below using ONLY what is visible in the attached image(s). Be precise, concrete, and faithful to the visual content. If the image does not contain enough information to answer, say so explicitly.',
+      '',
+      `Question: ${prompt.trim()}`,
+      '',
+      'Reply with the answer directly. No preamble, no acknowledgement of the task, no markdown headings.',
+    ].join('\n')
+  }
+  return [
+    base,
+    '',
+    "Your job is to DESCRIBE the attached image(s) faithfully and in detail. Cover: subject(s), composition, colors, text content (transcribed verbatim if legible), notable visual details, and anything that would help a downstream reader who cannot see the image. Do not speculate beyond what's visible.",
+    '',
+    'Reply with the description directly. No preamble, no markdown headings, no bullet list unless multiple images.',
+  ].join('\n')
+}
+
+export type ResolvedImage = {
+  data: string
+  mimeType: string
+}
+
+// Materializes an ImageInput into the base64-encoded form pi-ai expects.
+// - `url`: passthrough; pi-ai's image content does not accept URLs, so we fetch
+//   the bytes and base64-encode here (lazy; only when the tool is invoked).
+// - `file`: read from disk, infer MIME from extension. Path must be absolute or
+//   resolvable against the caller's cwd (callers should normalize ahead of
+//   time; this function rejects relative paths to avoid ambiguity).
+// - `base64`: passthrough.
+export async function resolveImage(input: ImageInput, signal?: AbortSignal): Promise<ResolvedImage> {
+  if (input.kind === 'base64') {
+    if (!input.mimeType.startsWith('image/')) {
+      throw new Error(`look_at: base64 mimeType must be image/* (got "${input.mimeType}")`)
+    }
+    return { data: input.data, mimeType: input.mimeType }
+  }
+  if (input.kind === 'file') {
+    if (!isAbsolute(input.path)) {
+      throw new Error(`look_at: file path must be absolute (got "${input.path}")`)
+    }
+    if (!existsSync(input.path)) {
+      throw new Error(`look_at: file not found at ${input.path}`)
+    }
+    const ext = extname(input.path).toLowerCase() as keyof typeof SUPPORTED_MIME_TYPES
+    const mimeType = (SUPPORTED_MIME_TYPES[ext] ?? null) as Mime | null
+    if (mimeType === null) {
+      throw new Error(
+        `look_at: unsupported image extension "${ext}" for ${input.path} (supported: ${Object.keys(SUPPORTED_MIME_TYPES).join(', ')})`,
+      )
+    }
+    const bytes = readFileSync(input.path)
+    return { data: bytes.toString('base64'), mimeType }
+  }
+  // URL branch: independent timeout + size cap on top of any caller-provided
+  // signal. The two abort signals are merged so the tool's overall abort wins
+  // over our timeout AND vice versa.
+  const timeoutSignal = AbortSignal.timeout(URL_FETCH_TIMEOUT_MS)
+  const mergedSignal = signal !== undefined ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal
+  const res = await fetch(input.url, { signal: mergedSignal })
+  if (!res.ok) {
+    throw new Error(`look_at: failed to fetch ${input.url}: HTTP ${res.status}`)
+  }
+  const mimeType = res.headers.get('content-type')?.split(';')[0]?.trim() ?? 'application/octet-stream'
+  if (!mimeType.startsWith('image/')) {
+    throw new Error(`look_at: ${input.url} did not return an image content-type (got "${mimeType}")`)
+  }
+  // Streaming size check: arrayBuffer() would read the whole body before we
+  // could enforce a cap. Read chunk-by-chunk and abort once we cross the
+  // limit. Content-Length is checked first when present, but absent or lying
+  // headers fall through to the streaming check.
+  const declared = Number(res.headers.get('content-length') ?? '')
+  if (Number.isFinite(declared) && declared > URL_FETCH_MAX_BYTES) {
+    throw new Error(`look_at: ${input.url} response too large (${declared} bytes > ${URL_FETCH_MAX_BYTES} cap)`)
+  }
+  const reader = res.body?.getReader()
+  if (reader === undefined) {
+    throw new Error(`look_at: ${input.url} returned an empty body`)
+  }
+  const chunks: Uint8Array[] = []
+  let total = 0
+  while (true) {
+    const { done, value } = await reader.read()
+    if (done) break
+    if (value === undefined) continue
+    total += value.byteLength
+    if (total > URL_FETCH_MAX_BYTES) {
+      await reader.cancel()
+      throw new Error(`look_at: ${input.url} response exceeded ${URL_FETCH_MAX_BYTES}-byte cap`)
+    }
+    chunks.push(value)
+  }
+  const buf = Buffer.concat(chunks)
+  return { data: buf.toString('base64'), mimeType }
+}

package/src/agent/plugin-tools.ts

@@ -30,6 +30,8 @@ import type {
   ToolResult,
 } from '@/plugin'
 
+import type { SessionOrigin } from './session-origin'
+
 type AnyAgentTool =
   | typeof piReadTool
   | typeof piBashTool
@@ -73,16 +75,37 @@ export type WrapToolOptions = {
   sessionId: string
   logger: PluginLogger
   hooks: HookBus
+  // Called at tool-execute time (not at wrap time) so channel sessions whose
+  // origin mutates per turn surface the current-turn `lastInboundAuthorId`
+  // to `tool.before`. Sessions with a fixed origin can pass `() => origin`.
+  getOrigin?: () => SessionOrigin | undefined
 }
 
 export type WrapSystemToolOptions = {
   agentDir: string
   sessionId: string
   hooks: HookBus
+  getOrigin?: () => SessionOrigin | undefined
 }
 
+// Zod 4 emits a top-level `"$schema": "https://json-schema.org/draft/2020-12/schema"`
+// pointer on every converted schema. Ajv v8 (used by pi-ai's runtime tool-argument
+// validator and by ModelRegistry's models.json validator) is configured for
+// Draft 7 and rejects unknown `$schema` URIs with:
+//
+//   no schema with key or ref "https://json-schema.org/draft/2020-12/schema"
+//
+// That error is raised before the tool's execute is even invoked, so the model
+// sees the failure as a tool-call result and reacts by retrying or falling back
+// to other tools. In the memory-logger / dreaming subagents this meant the
+// `find_entry` tool was permanently broken: the subagent kept falling back to
+// `read(offset=1, limit=2000)` and chunked through entire multi-hundred-KB
+// transcripts on every channel turn. Stripping `$schema` is the minimal,
+// converter-version-independent fix; it leaves the actual JSON-schema body
+// untouched and lets Ajv use its default draft.
 export function zodToToolParameters(schema: z.ZodType<unknown>): TSchema {
-  const json = z.toJSONSchema(schema, { io: 'input', reused: 'inline' })
+  const json = z.toJSONSchema(schema, { io: 'input', reused: 'inline' }) as Record<string, unknown>
+  delete json.$schema
   return json as unknown as TSchema
 }
 
@@ -101,11 +124,13 @@ export function wrapPluginTool(tool: Tool<any>, opts: WrapToolOptions): ToolDefi
       }
 
       const mutableArgs = validated.data as Record<string, unknown>
+      const liveOrigin = opts.getOrigin?.()
       const before: ToolBeforeEvent = {
         tool: opts.toolName,
         sessionId: opts.sessionId,
         callId: toolCallId,
         args: mutableArgs,
+        ...(liveOrigin !== undefined ? { origin: liveOrigin } : {}),
       }
       const blockResult = await opts.hooks.runToolBefore(before)
       if (blockResult !== undefined) {
@@ -151,11 +176,13 @@ export function wrapSystemTool<TParams extends TSchema, TDetails = unknown, TSta
     parameters: withGuardAcknowledgements(tool.name, tool.parameters),
     async execute(toolCallId, params, signal, onUpdate, ctx) {
       const mutableArgs = params as Record<string, unknown>
+      const liveOrigin = opts.getOrigin?.()
       const blockResult = await opts.hooks.runToolBefore({
         tool: tool.name,
         sessionId: opts.sessionId,
         callId: toolCallId,
         args: mutableArgs,
+        ...(liveOrigin !== undefined ? { origin: liveOrigin } : {}),
       })
       if (blockResult !== undefined) {
         throw new Error(`blocked: ${blockResult.reason}`)
@@ -198,11 +225,13 @@ export function wrapSystemAgentTool<TParams extends TSchema, TDetails = unknown>
     parameters: withGuardAcknowledgements(tool.name, tool.parameters),
     async execute(toolCallId, params, signal, onUpdate) {
       const mutableArgs = params as Record<string, unknown>
+      const liveOrigin = opts.getOrigin?.()
       const blockResult = await opts.hooks.runToolBefore({
         tool: tool.name,
         sessionId: opts.sessionId,
         callId: toolCallId,
         args: mutableArgs,
+        ...(liveOrigin !== undefined ? { origin: liveOrigin } : {}),
       })
       if (blockResult !== undefined) {
         throw new Error(`blocked: ${blockResult.reason}`)