teapot-coding-agent 0.37.72 → 0.37.73

package/dist/agent/agent.js

@@ -1,6 +1,7 @@
 import { promises as fs, existsSync, readFileSync, readdirSync, statSync } from "node:fs";
 import path from "node:path";
 import { EventLog, readEvents } from "../log/events.js";
+import { storeImage, resolveImages } from "../log/image-store.js";
 import { chat, chatStream } from "./llm.js";
 import { auditPrompt, parseAuditVerdict } from "./goal-audit.js";
 import { executeTool, toolSpecs, currentSkills, safeJoin, isContextOverflow, isRequestNodeLimit, hasRunningBgShells, hasRunningBgShellsFor, killBgShells } from "./tools.js";
@@ -569,6 +570,15 @@ export class Agent {
     get goalFile() {
         return path.join(this.opts.sessionDir, "goal.md");
     }
+    /**
+     * Image bytes live in `blobs/` INSIDE this session, beside `chat.jsonl`.
+     *
+     * So `rm -r` on a session reclaims everything it ever read — which is the whole point when one
+     * session's log is what grew to hundreds of megabytes.
+     */
+    get imagesDir() {
+        return path.join(this.opts.sessionDir, "blobs");
+    }
     get memoryFile() {
         const perSession = path.join(this.opts.sessionDir, "memory.md");
         const parent = path.dirname(this.opts.sessionDir);
@@ -681,7 +691,7 @@ export class Agent {
         if (!lineage.length)
             return;
         const last = lineage[lineage.length - 1];
-        const msgs = rebuildMessagesFrom(lineage);
+        const msgs = await rebuildMessagesFrom(lineage, this.imagesDir);
         for (const e of own) {
             if (this.sessionTitle === undefined && e.type === "prompt" && e.data?.source === "user") {
                 this.sessionTitle = sessionTitleFrom(e.data?.text) ?? undefined;
@@ -795,7 +805,7 @@ export class Agent {
             throw new Error("prompt is not on this agent's current lineage");
         const kept = lineage.slice(0, tIdx);
         const dropped = lineage.slice(tIdx);
-        const msgs = rebuildMessagesFrom(kept);
+        const msgs = await rebuildMessagesFrom(kept, this.imagesDir);
         msgs.push({ role: "user", content: text });
         const newBranch = `br${this.branchCount()}${Date.now().toString(36).slice(-4)}`;
         await this.quiesceForHistoryEdit();
@@ -1070,12 +1080,25 @@ export class Agent {
         const entry = { source, text, images, ...(promptId ? { id: promptId } : {}) };
         this.pendingPrompts.push(entry);
         void this.ensureReady()
-            .then(() => this.log.append("prompt", this.currentSession, this.currentBranch, {
-            source,
-            text,
-            ...(images?.length ? { images: images.map((i) => ({ url: i.url, name: i.name })) } : {}),
-            ...(promptId ? { promptId } : {}),
-        }))
+            .then(async () => {
+            /**
+             * A user's attachment is stored the same way a tool's image is: bytes in `blobs/`, a reference
+             * in the log. It used to keep the raw `data:` URL, which put up to 12MB of base64 into
+             * `chat.jsonl` — the same inflation the tool path just stopped doing.
+             */
+            const stored = images?.length
+                ? await Promise.all(images.map(async (i) => ({
+                    url: await (async () => (await storeImage(this.imagesDir, i.url)).ref)(),
+                    name: i.name,
+                })))
+                : undefined;
+            return this.log.append("prompt", this.currentSession, this.currentBranch, {
+                source,
+                text,
+                ...(stored?.length ? { images: stored } : {}),
+                ...(promptId ? { promptId } : {}),
+            });
+        })
             .then(() => {
             if (source === "user")
                 this.opts.onFirstUserPrompt?.(this.currentSession);
@@ -1843,12 +1866,25 @@ export class Agent {
                 this.stats.toolCalls++;
                 this.consecutiveIdleRounds = 0;
                 const capped = result.result.slice(0, 8000);
+                // data URL → `blobs/<hash>` (idempotent, so re-reading the same picture costs one file)
+                const imageRef = async (u) => (await storeImage(this.imagesDir, u)).ref;
                 await this.log.append("tool_result", this.currentSession, this.currentBranch, {
                     callId: call.id,
                     name: call.function.name,
                     ok: result.ok,
                     durationMs: Date.now() - t0,
                     result: capped,
+                    /**
+                     * The IMAGE itself belongs in the log, or a restored session cannot rebuild the request.
+                     *
+                     * MEASURED (v0.37.69, `github-bug-research-1f48715f` seq 11280): the `tool_result` row held
+                     * only the TEXT — "the image itself is attached to this message, so you are looking at it" —
+                     * and the picture existed nowhere. On restore the model reads that sentence, finds no image,
+                     * and has to invent what it saw.
+                     *
+                     * It is stored as a `blobs/<sha256>` reference, not inline: see `src/log/blobs.ts`.
+                     */
+                    ...(result.image ? { images: [{ url: await imageRef(result.image.url), name: result.image.name }] } : {}),
                 });
                 /**
                  * A tool may hand the model an IMAGE (read_file on a picture).
@@ -2837,7 +2873,7 @@ function lineageOf(events) {
         lineage.shift();
     return lineage;
 }
-function rebuildMessagesFrom(list) {
+async function rebuildMessagesFrom(list, imagesDir) {
     const msgs = [];
     const META_TOOLS = new Set([
         "finish", "report_progress", "set_goal", "get_goal",
@@ -2877,7 +2913,10 @@ function rebuildMessagesFrom(list) {
         const d = e.data;
         if (e.type === "prompt" && typeof d.text === "string") {
             const text = d.text;
-            const imgs = Array.isArray(d.images) ? d.images : [];
+            const imgsRaw = Array.isArray(d.images) ? d.images : [];
+            const imgs = imgsRaw.length
+                ? (await resolveImages(imagesDir, imgsRaw.map((i) => i.url))).map((url) => ({ url }))
+                : [];
             const mk = () => imgs.length
                 ? {
                     role: "user",
@@ -2929,11 +2968,36 @@ function rebuildMessagesFrom(list) {
             }
         }
         else if (e.type === "tool_result") {
-            msgs.push({
-                role: "tool",
-                tool_call_id: String(d.callId ?? ""),
-                content: `${d.ok === false ? "(failed) " : ""}${typeof d.result === "string" ? d.result : ""}`,
-            });
+            {
+                const text = `${d.ok === false ? "(failed) " : ""}${typeof d.result === "string" ? d.result : ""}`;
+                const toolImgsRaw = Array.isArray(d.images) ? d.images : [];
+                // `blobs/<hash>` → a data URL again, because only the wire needs the bytes inline
+                const toolImgs = toolImgsRaw.length
+                    ? (await resolveImages(imagesDir, toolImgsRaw.map((i) => i.url))).map((url) => ({ url }))
+                    : [];
+                /**
+                 * Rebuild the image part on restore, exactly as it was sent live.
+                 *
+                 * Without this a restored session silently degrades: the text still claims the image is
+                 * attached, so the model is not merely missing data, it is told something false about what it
+                 * can see.
+                 */
+                msgs.push(toolImgs.length
+                    ? {
+                        role: "tool",
+                        tool_call_id: String(d.callId ?? ""),
+                        content: text,
+                        content_parts: [
+                            { type: "text", text },
+                            ...toolImgs.map((i) => ({ type: "image_url", image_url: { url: i.url } })),
+                        ],
+                    }
+                    : {
+                        role: "tool",
+                        tool_call_id: String(d.callId ?? ""),
+                        content: text,
+                    });
+            }
             openCalls.delete(String(d.callId ?? ""));
             flushUsers();
         }

package/dist/log/image-store.js

@@ -0,0 +1,118 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+/**
+ * Image bytes live in a `blobs/` directory INSIDE the session, beside `chat.jsonl`.
+ *
+ * Three reasons, in order of how much they matter:
+ *
+ * 1. **Deleting a session must delete its images.** The operator's data dir reached 533MB of logs with
+ *    a 578KB JPEG per read; being able to `rm -r` one session and reclaim every byte is the point. A
+ *    shared store would leave orphans that no session owns and nobody dares delete.
+ * 2. base64 is 4/3 the size and inflates `chat.jsonl`, which is re-read on every replay. MEASURED: the
+ *    median tool result is 229 bytes, so one picture would dwarf a thousand ordinary rows.
+ * 3. JSON is a text format. Megabytes of encoded binary inside it make every reader of the log — replay,
+ *    grep, the timeline, a person with `less` — pay for data almost none of them want.
+ *
+ * NOT `assets/`: that is the WALLPAPER store, with its own resize cache, URL shape and lifetime (a
+ * wallpaper outlives every session). Two lifecycles in one directory is how "safe to delete" stops
+ * being true.
+ *
+ * Names are `sha256` of the content, so a write is idempotent and two different images cannot share a
+ * filename.
+ */
+const BLOBS_DIR = "blobs";
+/**
+ * Write a data URL to `images/` and return the reference to store in the log — `assets/<sha256>.<ext>`.
+ *
+ * Anything already addressed by a URL (`http(s):`, or a reference we wrote earlier) passes through
+ * untouched: it is not ours to copy, and re-encoding would lose the original bytes.
+ */
+export async function storeImage(imagesDir, dataUrl) {
+    if (!dataUrl.startsWith("data:"))
+        return { ref: dataUrl, bytes: 0 };
+    const comma = dataUrl.indexOf(",");
+    const header = dataUrl.slice(5, comma);
+    if (comma < 0 || !header.includes("base64"))
+        throw new Error("image data URL is not base64");
+    const mime = header.split(";")[0] || "application/octet-stream";
+    const buf = Buffer.from(dataUrl.slice(comma + 1), "base64");
+    const name = `${crypto.createHash("sha256").update(buf).digest("hex")}${extFor(mime)}`;
+    await fs.mkdir(imagesDir, { recursive: true });
+    const dest = path.join(imagesDir, name);
+    // content-addressed, so an existing file already holds these bytes
+    try {
+        await fs.writeFile(dest, buf, { flag: "wx" });
+    }
+    catch (err) {
+        if (err.code !== "EEXIST")
+            throw err;
+    }
+    return { ref: `${BLOBS_DIR}/${name}`, bytes: buf.length };
+}
+/**
+ * Resolve a stored reference back to something the model can receive.
+ *
+ * `assets/<name>` is read from disk and encoded as a data URL at REQUEST time — the only point where
+ * the bytes have to be inline. A plain `data:` or `http(s):` value (an older session) is returned
+ * as-is, so logs written before this existed keep working.
+ */
+export async function resolveImage(imagesDir, ref) {
+    if (!ref.startsWith(`${BLOBS_DIR}/`))
+        return ref;
+    const name = ref.slice(BLOBS_DIR.length + 1);
+    // a stored name is a basename; refuse anything that tries to climb out of the directory
+    if (!name || name !== path.basename(name))
+        return null;
+    try {
+        const buf = await fs.readFile(path.join(imagesDir, name));
+        return `data:${mimeForExt(path.extname(name))};base64,${buf.toString("base64")}`;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Resolve every image of one stored event, in order, dropping any whose bytes are gone.
+ *
+ * A missing file is DROPPED rather than faked: the text beside it stays, but an unreadable reference
+ * must not become an invented picture or a request the API rejects.
+ */
+export async function resolveImages(imagesDir, refs) {
+    const out = [];
+    for (const r of refs) {
+        const resolved = await resolveImage(imagesDir, r);
+        if (resolved)
+            out.push(resolved);
+    }
+    return out;
+}
+function extFor(mime) {
+    switch (mime) {
+        case "image/png":
+            return ".png";
+        case "image/jpeg":
+            return ".jpg";
+        case "image/gif":
+            return ".gif";
+        case "image/webp":
+            return ".webp";
+        default:
+            return ".bin";
+    }
+}
+function mimeForExt(ext) {
+    switch (ext) {
+        case ".png":
+            return "image/png";
+        case ".jpg":
+        case ".jpeg":
+            return "image/jpeg";
+        case ".gif":
+            return "image/gif";
+        case ".webp":
+            return "image/webp";
+        default:
+            return "application/octet-stream";
+    }
+}

package/dist/server/api.js

@@ -830,6 +830,8 @@ export function buildApp(master) {
         await a.load();
         return c.json({ ok: true, agent: a.snapshot() });
     });
+    // 64MB encoded (~48MB of image); a vision model rejects anything near this anyway
+    const MAX_PROMPT_IMAGE_CHARS = 64 * 1024 * 1024;
     app.post("/api/agents/:id/prompt", async (c) => {
         const a = master.agents.get(c.req.param("id"));
         if (!a)
@@ -841,10 +843,29 @@ export function buildApp(master) {
             return c.json({ error: "invalid JSON" }, 400);
         if (!body.text?.trim() && !body.images?.length)
             return c.json({ error: "text or images required" }, 400);
-        const images = (body.images ?? []).filter((im) => {
+        /**
+         * Accept any image the model could take; the cap is a SANITY bound, not a storage limit.
+         *
+         * The bytes now go to `blobs/` beside `chat.jsonl` instead of into it, so the old "keep the log
+         * small" reason for a tight cutoff is gone. What remains is refusing something absurd before it is
+         * base64-decoded — and reporting WHICH image was refused rather than silently dropping it, because
+         * a prompt that quietly loses its attachment is indistinguishable from a model that ignored it.
+         */
+        const offered = body.images ?? [];
+        const tooBig = offered.find((im) => {
+            const u = String(im?.url ?? "");
+            return u.startsWith("data:image/") && u.length > MAX_PROMPT_IMAGE_CHARS;
+        });
+        if (tooBig) {
+            return c.json({
+                error: `${tooBig.name || "an image"} is too large (${Math.round(String(tooBig.url).length / 1024 / 1024)}MB ` +
+                    `encoded, limit ${MAX_PROMPT_IMAGE_CHARS / 1024 / 1024}MB)`,
+            }, 413);
+        }
+        const images = offered.filter((im) => {
             const u = String(im?.url ?? "");
             if (u.startsWith("data:image/"))
-                return u.length < 12_000_000;
+                return true;
             return /^https?:\/\//.test(u);
         });
         const text = body.text ?? "";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "teapot-coding-agent",
-  "version": "0.37.72",
+  "version": "0.37.73",
   "description": "A lightweight, always-on multi-agent harness for AI coding agents",
   "type": "module",
   "license": "AGPL-3.0-or-later",