talon-agent 1.47.1 → 1.47.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "1.47.1",
+  "version": "1.47.2",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/src/core/engine/gateway-actions/native.ts

@@ -25,7 +25,7 @@ import {
   writeFile,
 } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
+import { dirname, extname, join } from "node:path";
 import { getMeshService } from "../../mesh/index.js";
 import {
   clearTeleport,
@@ -39,7 +39,12 @@ import {
 } from "../../../util/exec-output.js";
 import type { SharedActionHandlers } from "./types.js";
 
-type Result = { ok: boolean; text: string };
+type Result = {
+  ok: boolean;
+  text: string;
+  /** Set for image files so the tool result carries a viewable image block. */
+  image?: { data: string; mimeType: string };
+};
 
 /**
  * ripgrep binary, resolved from PATH (env-overridable for tests). NOT a
@@ -70,6 +75,21 @@ const TELEPORT_TIMEOUT_HINT =
   "background it in-shell instead: `cmd > /tmp/out.log 2>&1 &`, then poll the log " +
   "with read, or bound the command itself: `logcat -d`, `timeout 30 …`, `head -n 200`.)";
 const MAX_READ_LINES = 2_000;
+/**
+ * Image extensions the model can actually view, mapped to their MIME type.
+ * Reading one returns an image content block instead of the file's raw bytes
+ * decoded as (garbage) UTF-8 — so `read`ing a photo/screenshot/design shows
+ * the picture, not mojibake.
+ */
+const IMAGE_MIME: Record<string, string> = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+};
+/** Anthropic caps a single image around 5MB; refuse larger with a downscale hint. */
+const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
 /** Caps for the pure-JS glob/search fallbacks (rg unavailable). */
 const MAX_JS_RESULTS = 2_000;
 const MAX_JS_FILE_BYTES = 2 * 1024 * 1024;
@@ -437,23 +457,58 @@ async function read(
 ): Promise<Result> {
   const p = str(path);
   if (!p) return { ok: false, text: "A file path is required." };
+  const active = await getTeleport(chatId);
+  const where = active ? active.deviceName : "local";
+
+  // Image files: return a viewable image block, not the raw bytes decoded as
+  // UTF-8. Without this, `read`ing a photo/screenshot/design hands the model
+  // mojibake and it can't see the picture at all.
+  const mime = IMAGE_MIME[extname(p).toLowerCase()];
+  if (mime) {
+    let bytes: Buffer;
+    if (active) {
+      const res = await getMeshService().readFileBytes(active.deviceId, p);
+      if ("error" in res) return { ok: false, text: res.error };
+      bytes = res.data;
+    } else {
+      try {
+        bytes = await readFile(p);
+      } catch (err) {
+        return {
+          ok: false,
+          text: `Cannot read ${p}: ${(err as Error).message}`,
+        };
+      }
+    }
+    if (bytes.length > MAX_IMAGE_BYTES) {
+      return {
+        ok: false,
+        text:
+          `${p} [${where}] is ${(bytes.length / 1_048_576).toFixed(1)}MB — over the ` +
+          `${MAX_IMAGE_BYTES / 1_048_576}MB image limit. Downscale it first ` +
+          `(e.g. \`convert '${p}' -resize 1568x /tmp/small.jpg\`) and read that.`,
+      };
+    }
+    return {
+      ok: true,
+      text: `${p} [${where}] — image (${mime}, ${bytes.length} bytes)`,
+      image: { data: bytes.toString("base64"), mimeType: mime },
+    };
+  }
+
   const start = num(offset) ?? 0;
   const max = Math.min(num(limit) ?? MAX_READ_LINES, MAX_READ_LINES);
-  const active = await getTeleport(chatId);
   let content: string;
-  let where: string;
   if (active) {
     const res = await getMeshService().readFileBytes(active.deviceId, p);
     if ("error" in res) return { ok: false, text: res.error };
     content = res.data.toString("utf8");
-    where = active.deviceName;
   } else {
     try {
       content = await readFile(p, "utf8");
     } catch (err) {
       return { ok: false, text: `Cannot read ${p}: ${(err as Error).message}` };
     }
-    where = "local";
   }
   const lines = content.split("\n");
   const slice = lines.slice(start, start + max);

package/src/core/tools/bridge.ts

@@ -125,14 +125,39 @@ export function createBridge(
 
 /** Wrap a bridge result into the MCP content format. */
 export function textResult(result: unknown): {
-  content: Array<{ type: "text"; text: string }>;
+  content: Array<
+    | { type: "text"; text: string }
+    | { type: "image"; data: string; mimeType: string }
+  >;
   isError?: boolean;
 } {
-  const r = result as { ok?: boolean; text?: string; error?: string };
+  const r = result as {
+    ok?: boolean;
+    text?: string;
+    error?: string;
+    image?: { data?: unknown; mimeType?: unknown };
+  };
+  const content: Array<
+    | { type: "text"; text: string }
+    | { type: "image"; data: string; mimeType: string }
+  > = [{ type: "text" as const, text: r.text ?? JSON.stringify(result) }];
+  // A result may carry an image (e.g. `read` on a photo) — surface it as an
+  // MCP image block so the model actually sees the picture, not base64 text.
+  if (
+    r &&
+    typeof r === "object" &&
+    r.image &&
+    typeof r.image.data === "string" &&
+    typeof r.image.mimeType === "string"
+  ) {
+    content.push({
+      type: "image" as const,
+      data: r.image.data,
+      mimeType: r.image.mimeType,
+    });
+  }
   return {
-    content: [
-      { type: "text" as const, text: r.text ?? JSON.stringify(result) },
-    ],
+    content,
     // Gateway results carry ok:false on failure — mark those as tool errors
     // so the model treats the message as a failure, not a success payload.
     ...(r && typeof r === "object" && r.ok === false ? { isError: true } : {}),

package/src/core/types.ts

@@ -233,6 +233,12 @@ export type ActionResult = {
    * tool calls (react/edit/delete) that target this message.
    */
   message_id?: number | string;
+  /**
+   * Set by tools that return a viewable image (e.g. `read` on a photo). The
+   * MCP layer turns this into an image content block so the model sees the
+   * picture instead of base64 text.
+   */
+  image?: { data: string; mimeType: string };
   [key: string]: unknown;
 };