teapot-coding-agent 0.19.9 → 0.20.1

package/README.md

@@ -238,7 +238,7 @@ GET  /api/agents/:id/file?path=         fetch text file
 PUT  /api/agents/:id/file?path=         save { content, baseContent? } (409 on conflict)
 GET  /api/agents/:id/raw?path=          raw media bytes
 GET  /api/agents/:id/tree               file tree
-POST /api/agents/:id/prompt             send { text, start? } → { promptId }
+POST /api/agents/:id/prompt             send { text, start?, images?: [{url}] } → { promptId }
 POST /api/agents/:id/prompt/cancel      withdraw { promptId } (409 once delivered)
 POST /api/agents/:id/goal               { text?, status?, verify? }
 POST /api/agents/:id/start|stop|fork|load

package/dist/agent/agent.js

@@ -744,7 +744,7 @@ export class Agent {
      * the running loop. The very first prompt on a fresh boot also triggers the
      * lazy session restore (before the mailbox is filled, so no duplicates).
      */
-    enqueuePrompt(text, source = "user") {
+    enqueuePrompt(text, source = "user", images) {
         // a prompt during a tool park (wait_children) takes over instantly:
         // unpark fixes the DISPLAY, aborting the park signal actually ends the
         // wait — the run chain resumes with this prompt drained at the boundary
@@ -770,10 +770,11 @@ export class Agent {
         const promptId = source === "user" ? `u${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}` : "";
         void this.ensureReady()
             .then(() => {
-            this.pendingPrompts.push({ source, text, ...(promptId ? { id: promptId } : {}) });
+            this.pendingPrompts.push({ source, text, images, ...(promptId ? { id: promptId } : {}) });
             return this.log.append("prompt", this.currentSession, this.currentBranch, {
                 source,
                 text,
+                ...(images?.length ? { images: images.map((i) => ({ url: i.url, name: i.name })) } : {}),
                 ...(promptId ? { promptId } : {}),
             });
         })
@@ -808,7 +809,21 @@ export class Agent {
      */
     drainPendingPrompts() {
         for (const p of this.pendingPrompts.splice(0)) {
-            this.messages.push({ role: "user", content: p.text });
+            // multimodal: images ride as OpenAI content parts (text first, then
+            // images); plain prompts keep the string form for cache friendliness
+            if (p.images?.length) {
+                this.messages.push({
+                    role: "user",
+                    content: p.text,
+                    content_parts: [
+                        { type: "text", text: p.text },
+                        ...p.images.map((i) => ({ type: "image_url", image_url: { url: i.url } })),
+                    ],
+                });
+            }
+            else {
+                this.messages.push({ role: "user", content: p.text });
+            }
             if (p.source === "user") {
                 const id = p.id ?? `u${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`;
                 this.deliveredPrompts.push(id);
@@ -2109,19 +2124,48 @@ function rebuildMessagesFrom(list) {
     // synthesizer below must not duplicate them
     const loggedResults = new Set(list.filter((e) => e.type === "tool_result").map((e) => String(e.data.callId ?? "")));
     const bufferedUsers = [];
+    // image attachments paired with buffered prompts (multimodal replay)
+    const bufferedParts = [];
     const flushUsers = () => {
         if (openCalls.size === 0) {
-            for (const text of bufferedUsers.splice(0))
-                msgs.push({ role: "user", content: text });
+            const texts = bufferedUsers.splice(0);
+            const parts = bufferedParts.splice(0);
+            texts.forEach((text, i) => {
+                const imgs = parts[i] ?? [];
+                msgs.push(imgs.length
+                    ? {
+                        role: "user",
+                        content: text,
+                        content_parts: [
+                            { type: "text", text },
+                            ...imgs.map((im) => ({ type: "image_url", image_url: { url: im.url } })),
+                        ],
+                    }
+                    : { role: "user", content: text });
+            });
         }
     };
     for (const e of list) {
         const d = e.data;
         if (e.type === "prompt" && typeof d.text === "string") {
+            // restore image attachments so replays stay byte-identical with the
+            // original request (prefix caches stay warm across restarts)
+            const text = d.text;
+            const imgs = Array.isArray(d.images) ? d.images : [];
+            const mk = () => imgs.length
+                ? {
+                    role: "user",
+                    content: text,
+                    content_parts: [
+                        { type: "text", text },
+                        ...imgs.map((i) => ({ type: "image_url", image_url: { url: i.url } })),
+                    ],
+                }
+                : { role: "user", content: text };
             if (openCalls.size > 0)
-                bufferedUsers.push(d.text);
+                bufferedUsers.push(text), bufferedParts.push(imgs);
             else
-                msgs.push({ role: "user", content: d.text });
+                msgs.push(mk());
         }
         else if (e.type === "message") {
             // final summaries are operator-facing only — the live loop never puts
@@ -2229,7 +2273,22 @@ function rebuildMessagesFrom(list) {
         }
     }
     // prompts that were still waiting on a hole-filled tail land here
-    for (const text of bufferedUsers.splice(0))
-        msgs.push({ role: "user", content: text });
+    {
+        const texts = bufferedUsers.splice(0);
+        const parts = bufferedParts.splice(0);
+        texts.forEach((text, i) => {
+            const imgs = parts[i] ?? [];
+            msgs.push(imgs.length
+                ? {
+                    role: "user",
+                    content: text,
+                    content_parts: [
+                        { type: "text", text },
+                        ...imgs.map((im) => ({ type: "image_url", image_url: { url: im.url } })),
+                    ],
+                }
+                : { role: "user", content: text });
+        });
+    }
     return msgs;
 }

package/dist/agent/llm.js

@@ -80,13 +80,21 @@ function looksMangled(name) {
 }
 function sanitize(messages) {
     return messages.map((m) => {
+        // multimodal user message → OpenAI content-parts array (text + images);
+        // providers without vision reject it, but those models were never a fit
+        if (m.content_parts?.length) {
+            const parts = m.content_parts.map((p) => p.type === "image_url"
+                ? { type: "image_url", image_url: { url: p.image_url.url } }
+                : { type: "text", text: p.text ?? "" });
+            return { role: m.role, content: parts };
+        }
         if ((m.role === "user" || m.role === "assistant") && !m.content) {
             return { ...m, content: m.tool_calls?.length ? "(tool call)" : "(no content)" };
         }
         if (m.role === "tool" && typeof m.content !== "string") {
             return { ...m, content: String(m.content ?? "(no output)") };
         }
-        return m;
+        return { ...m };
     });
 }
 export async function chat(cfg, messages, tools, signal, onDelta) {

package/dist/model-meta.js

@@ -38,6 +38,12 @@ export async function fetchModelList(baseUrl, apiKey) {
                     completion: Number(m.pricing.completion ?? 0),
                 }
                 : undefined,
+            modalities: Array.isArray(m.architecture?.input_modalities) && m.architecture.input_modalities.length
+                ? {
+                    input: m.architecture.input_modalities,
+                    output: m.architecture.output_modalities ?? ["text"],
+                }
+                : undefined,
         }))
             .filter((m) => m.id);
         cache.set(root, { at: Date.now(), list });

package/dist/server/api.js

@@ -38,6 +38,12 @@ async function listModels(baseUrl, apiKey) {
                 completion: Number(m.pricing.completion ?? 0),
             }
             : undefined,
+        modalities: Array.isArray(m.architecture?.input_modalities) && m.architecture.input_modalities.length
+            ? {
+                input: m.architecture.input_modalities,
+                output: m.architecture.output_modalities ?? ["text"],
+            }
+            : undefined,
     }))
         .filter((m) => m.id)
         .sort((a, b) => a.id.localeCompare(b.id));
@@ -607,12 +613,21 @@ export function buildApp(master) {
         if (!a)
             return c.json({ error: "not found" }, 404);
         const body = await c.req.json();
-        if (!body.text?.trim())
-            return c.json({ error: "text required" }, 400);
+        if (!body.text?.trim() && !body.images?.length)
+            return c.json({ error: "text or images required" }, 400);
+        // validate images: data URLs or http(s), size-capped (a 20MB base64 blob
+        // in the log would hurt every future replay of this session)
+        const images = (body.images ?? []).filter((im) => {
+            const u = String(im?.url ?? "");
+            if (u.startsWith("data:image/"))
+                return u.length < 12_000_000; // ~9MB binary
+            return /^https?:\/\//.test(u);
+        });
+        const text = body.text ?? "";
         // returns immediately: the prompt is logged + broadcast now, delivered to
         // the model at the next turn boundary (never blocks on a running agent).
         // promptId ties the UI's pending echo to the later prompt-delivered note.
-        const promptId = a.enqueuePrompt(body.text, "user");
+        const promptId = a.enqueuePrompt(text || "(see attached image)", "user", images.length ? images : undefined);
         if (body.start !== false && a.status !== "running")
             a.start("prompt");
         return c.json({ ok: true, promptId, queued: a.snapshot().pendingPrompts });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teapot-coding-agent",
-  "version": "0.19.9",
+  "version": "0.20.1",
   "description": "A lightweight, always-on multi-agent harness for AI coding agents",
   "type": "module",
   "license": "AGPL-3.0-or-later",