talon-agent 1.47.0 → 1.47.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "1.47.0",
+  "version": "1.47.2",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",
@@ -110,6 +110,7 @@
     "croner": "^10.0.1",
     "cross-spawn": "^7.0.6",
     "discord.js": "^14.16.3",
+    "file-type": "^22.0.1",
     "grammy": "^1.42.0",
     "liquidjs": "^10.27.0",
     "marked": "^18.0.0",

package/src/core/engine/gateway-actions/fetch-url.ts

@@ -3,12 +3,35 @@
  * binary content (validated by magic bytes) into the uploads workspace.
  */
 
+import { randomUUID } from "node:crypto";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
+import {
+  readBodyLimited,
+  ResponseTooLargeError,
+} from "../../../util/http-body.js";
+import {
+  advertisedBinaryKind,
+  decodeText,
+  detectBinaryType,
+  extractText,
+  isHtmlContent,
+  isTextContent,
+  matchesBinaryKind,
+} from "../../../util/web-content.js";
 import { dirs } from "../../../util/paths.js";
-import { extractText } from "./shared.js";
 import type { SharedActionHandlers } from "./types.js";
 
+const MAX_RESPONSE_MB = 50;
+const MAX_RESPONSE_BYTES = MAX_RESPONSE_MB * 1024 * 1024;
+const MAX_TEXT_CHARS = 50_000;
+
+/** Cap returned text, marking the cut so truncation is never silent. */
+function capText(text: string): string {
+  if (text.length <= MAX_TEXT_CHARS) return text;
+  return `${text.slice(0, MAX_TEXT_CHARS)}\n\n[Content truncated at ${MAX_TEXT_CHARS} characters]`;
+}
+
 export const fetchUrlHandlers: SharedActionHandlers = {
   fetch_url: async (body) => {
     const url = String(body.url ?? "");
@@ -32,84 +55,75 @@ export const fetchUrlHandlers: SharedActionHandlers = {
 
       // Reject oversized responses before downloading the body.
       // The Content-Length header is advisory but saves bandwidth when present.
-      const MAX_BYTES = 20 * 1024 * 1024; // 20 MB
       const contentLength = resp.headers.get("content-length");
-      if (contentLength && Number(contentLength) > MAX_BYTES) {
+      if (contentLength && Number(contentLength) > MAX_RESPONSE_BYTES) {
         return {
           ok: false,
-          error: `File too large (${(Number(contentLength) / 1024 / 1024).toFixed(0)}MB, max 20MB)`,
+          error: `File too large (${(Number(contentLength) / 1024 / 1024).toFixed(0)}MB, max ${MAX_RESPONSE_MB}MB)`,
         };
       }
 
-      // Binary content: download and save to workspace
       const mimeType = ct.split(";")[0].trim().toLowerCase();
-      const isText =
-        mimeType.startsWith("text/") || mimeType === "application/json";
-      if (!isText) {
-        const buffer = Buffer.from(await resp.arrayBuffer());
-        if (buffer.length > MAX_BYTES)
-          return { ok: false, error: "File too large (max 20MB)" };
-        if (buffer.length === 0)
-          return { ok: false, error: "Empty response (0 bytes)" };
-
-        // Validate magic bytes — prevent saving HTML error pages as images
-        // (servers can return error pages with image content-type headers)
-        const magic = buffer.subarray(0, 16);
-        const isRealImage =
-          (magic[0] === 0xff && magic[1] === 0xd8) || // JPEG
-          (magic[0] === 0x89 &&
-            magic[1] === 0x50 &&
-            magic[2] === 0x4e &&
-            magic[3] === 0x47) || // PNG
-          (magic[0] === 0x47 && magic[1] === 0x49 && magic[2] === 0x46) || // GIF
-          (magic[0] === 0x52 &&
-            magic[1] === 0x49 &&
-            magic[2] === 0x46 &&
-            magic[3] === 0x46 &&
-            magic[8] === 0x57 &&
-            magic[9] === 0x45 &&
-            magic[10] === 0x42 &&
-            magic[11] === 0x50); // WebP
-
-        // If content-type says image but bytes say otherwise, treat as text
-        if (ct.startsWith("image/") && !isRealImage) {
-          const text = extractText(buffer.toString("utf-8"), 500);
+      let buffer: Buffer;
+      try {
+        buffer = await readBodyLimited(resp, MAX_RESPONSE_BYTES);
+      } catch (err) {
+        if (err instanceof ResponseTooLargeError) {
           return {
             ok: false,
-            error: `Server returned an error page instead of an image. Content: ${text}`,
+            error: `Response too large (max ${MAX_RESPONSE_MB}MB)`,
           };
         }
+        throw err;
+      }
+
+      if (isTextContent(mimeType, buffer)) {
+        const trimmed = decodeText(buffer, ct).trim();
+        if (!trimmed)
+          return { ok: true, text: "(Page has no readable content)" };
 
-        const ext = isRealImage
-          ? magic[0] === 0xff
-            ? "jpg"
-            : magic[0] === 0x89
-              ? "png"
-              : magic[0] === 0x47
-                ? "gif"
-                : "webp"
-          : ct.includes("pdf")
-            ? "pdf"
-            : ct.includes("zip")
-              ? "zip"
-              : "bin";
-        const uploadsDir = dirs.uploads;
-        if (!existsSync(uploadsDir)) mkdirSync(uploadsDir, { recursive: true });
-        const filePath = resolve(uploadsDir, `${Date.now()}-fetched.${ext}`);
-        writeFileSync(filePath, buffer);
-        const typeLabel = isRealImage
-          ? "image"
-          : (ct.split("/")[1]?.split(";")[0] ?? "file");
+        // extractText is a DOM extractor — running it on JSON/XML/JavaScript/
+        // plain text strips small payloads like {"status":"ok"} to nothing,
+        // so only HTML (declared or sniffed) goes through it.
+        if (!isHtmlContent(mimeType, trimmed)) {
+          return { ok: true, text: capText(trimmed) };
+        }
+        const text = extractText(trimmed, Number.POSITIVE_INFINITY);
+        if (text.length < 20)
+          return { ok: true, text: "(Page has no readable content)" };
+        return { ok: true, text: capText(text) };
+      }
+
+      if (buffer.length === 0)
+        return { ok: false, error: "Empty response (0 bytes)" };
+
+      const detected = await detectBinaryType(buffer);
+      const advertised = advertisedBinaryKind(mimeType);
+
+      // Do not save an error page or arbitrary bytes under a trusted-looking
+      // image/PDF/ZIP extension merely because the server advertised one.
+      if (advertised && !matchesBinaryKind(advertised, detected, buffer)) {
+        const text = extractText(decodeText(buffer, ct), 500);
         return {
-          ok: true,
-          text: `Downloaded ${typeLabel} (${(buffer.length / 1024).toFixed(0)}KB) to: ${filePath}\nRead it with the Read tool or send it with send(type="file", file_path="${filePath}").`,
+          ok: false,
+          error: `Server returned invalid ${advertised} content.${text ? ` Content: ${text}` : ""}`,
         };
       }
-      const raw = await resp.text();
-      const text = extractText(raw);
-      if (text.length < 20)
-        return { ok: true, text: "(Page has no readable content)" };
-      return { ok: true, text };
+
+      const uploadsDir = dirs.uploads;
+      if (!existsSync(uploadsDir)) mkdirSync(uploadsDir, { recursive: true });
+      const filePath = resolve(
+        uploadsDir,
+        `${Date.now()}-${randomUUID().slice(0, 8)}-fetched.${detected?.ext ?? "bin"}`,
+      );
+      writeFileSync(filePath, buffer);
+      const typeLabel = detected?.mime.startsWith("image/")
+        ? "image"
+        : (detected?.ext ?? ct.split("/")[1]?.split(";")[0] ?? "file");
+      return {
+        ok: true,
+        text: `Downloaded ${typeLabel} (${(buffer.length / 1024).toFixed(0)}KB) to: ${filePath}\nRead it with the Read tool or send it with send(type="file", file_path="${filePath}").`,
+      };
     } catch (err) {
       return {
         ok: false,

package/src/core/engine/gateway-actions/native.ts

@@ -25,7 +25,7 @@ import {
   writeFile,
 } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
+import { dirname, extname, join } from "node:path";
 import { getMeshService } from "../../mesh/index.js";
 import {
   clearTeleport,
@@ -39,7 +39,12 @@ import {
 } from "../../../util/exec-output.js";
 import type { SharedActionHandlers } from "./types.js";
 
-type Result = { ok: boolean; text: string };
+type Result = {
+  ok: boolean;
+  text: string;
+  /** Set for image files so the tool result carries a viewable image block. */
+  image?: { data: string; mimeType: string };
+};
 
 /**
  * ripgrep binary, resolved from PATH (env-overridable for tests). NOT a
@@ -70,6 +75,21 @@ const TELEPORT_TIMEOUT_HINT =
   "background it in-shell instead: `cmd > /tmp/out.log 2>&1 &`, then poll the log " +
   "with read, or bound the command itself: `logcat -d`, `timeout 30 …`, `head -n 200`.)";
 const MAX_READ_LINES = 2_000;
+/**
+ * Image extensions the model can actually view, mapped to their MIME type.
+ * Reading one returns an image content block instead of the file's raw bytes
+ * decoded as (garbage) UTF-8 — so `read`ing a photo/screenshot/design shows
+ * the picture, not mojibake.
+ */
+const IMAGE_MIME: Record<string, string> = {
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".gif": "image/gif",
+  ".webp": "image/webp",
+};
+/** Anthropic caps a single image around 5MB; refuse larger with a downscale hint. */
+const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
 /** Caps for the pure-JS glob/search fallbacks (rg unavailable). */
 const MAX_JS_RESULTS = 2_000;
 const MAX_JS_FILE_BYTES = 2 * 1024 * 1024;
@@ -437,23 +457,58 @@ async function read(
 ): Promise<Result> {
   const p = str(path);
   if (!p) return { ok: false, text: "A file path is required." };
+  const active = await getTeleport(chatId);
+  const where = active ? active.deviceName : "local";
+
+  // Image files: return a viewable image block, not the raw bytes decoded as
+  // UTF-8. Without this, `read`ing a photo/screenshot/design hands the model
+  // mojibake and it can't see the picture at all.
+  const mime = IMAGE_MIME[extname(p).toLowerCase()];
+  if (mime) {
+    let bytes: Buffer;
+    if (active) {
+      const res = await getMeshService().readFileBytes(active.deviceId, p);
+      if ("error" in res) return { ok: false, text: res.error };
+      bytes = res.data;
+    } else {
+      try {
+        bytes = await readFile(p);
+      } catch (err) {
+        return {
+          ok: false,
+          text: `Cannot read ${p}: ${(err as Error).message}`,
+        };
+      }
+    }
+    if (bytes.length > MAX_IMAGE_BYTES) {
+      return {
+        ok: false,
+        text:
+          `${p} [${where}] is ${(bytes.length / 1_048_576).toFixed(1)}MB — over the ` +
+          `${MAX_IMAGE_BYTES / 1_048_576}MB image limit. Downscale it first ` +
+          `(e.g. \`convert '${p}' -resize 1568x /tmp/small.jpg\`) and read that.`,
+      };
+    }
+    return {
+      ok: true,
+      text: `${p} [${where}] — image (${mime}, ${bytes.length} bytes)`,
+      image: { data: bytes.toString("base64"), mimeType: mime },
+    };
+  }
+
   const start = num(offset) ?? 0;
   const max = Math.min(num(limit) ?? MAX_READ_LINES, MAX_READ_LINES);
-  const active = await getTeleport(chatId);
   let content: string;
-  let where: string;
   if (active) {
     const res = await getMeshService().readFileBytes(active.deviceId, p);
     if ("error" in res) return { ok: false, text: res.error };
     content = res.data.toString("utf8");
-    where = active.deviceName;
   } else {
     try {
       content = await readFile(p, "utf8");
     } catch (err) {
       return { ok: false, text: `Cannot read ${p}: ${(err as Error).message}` };
     }
-    where = "local";
   }
   const lines = content.split("\n");
   const slice = lines.slice(start, start + max);

package/src/core/engine/gateway-actions/shared.ts

@@ -1,9 +1,8 @@
 /**
- * Helpers shared across shared-action domains: due-date parsing, HTML text
- * extraction, and per-job model-override validation.
+ * Helpers shared across shared-action domains: due-date parsing and per-job
+ * model-override validation.
  */
 
-import * as cheerio from "cheerio";
 import { resolveExplicitModelRef } from "../../models/active-model.js";
 import {
   getBackendForChat,
@@ -20,16 +19,6 @@ export function parseDueDate(value: unknown): number | undefined {
   return Number.isFinite(ms) ? ms : undefined;
 }
 
-/** Extract readable text from HTML using cheerio (proper DOM parser). */
-export function extractText(html: string, maxLength = 8000): string {
-  const $ = cheerio.load(html);
-  // Remove non-content elements
-  $("script, style, noscript, iframe, svg, nav, footer, header").remove();
-  // Get text content, normalize whitespace
-  const text = $("body").text().replace(/\s+/g, " ").trim();
-  return text.slice(0, maxLength);
-}
-
 /**
  * Validate a per-job model override so create_cron_job / trigger_create can
  * reject a bad override up front — the agent retries or tells the user —

package/src/core/tools/bridge.ts

@@ -125,14 +125,39 @@ export function createBridge(
 
 /** Wrap a bridge result into the MCP content format. */
 export function textResult(result: unknown): {
-  content: Array<{ type: "text"; text: string }>;
+  content: Array<
+    | { type: "text"; text: string }
+    | { type: "image"; data: string; mimeType: string }
+  >;
   isError?: boolean;
 } {
-  const r = result as { ok?: boolean; text?: string; error?: string };
+  const r = result as {
+    ok?: boolean;
+    text?: string;
+    error?: string;
+    image?: { data?: unknown; mimeType?: unknown };
+  };
+  const content: Array<
+    | { type: "text"; text: string }
+    | { type: "image"; data: string; mimeType: string }
+  > = [{ type: "text" as const, text: r.text ?? JSON.stringify(result) }];
+  // A result may carry an image (e.g. `read` on a photo) — surface it as an
+  // MCP image block so the model actually sees the picture, not base64 text.
+  if (
+    r &&
+    typeof r === "object" &&
+    r.image &&
+    typeof r.image.data === "string" &&
+    typeof r.image.mimeType === "string"
+  ) {
+    content.push({
+      type: "image" as const,
+      data: r.image.data,
+      mimeType: r.image.mimeType,
+    });
+  }
   return {
-    content: [
-      { type: "text" as const, text: r.text ?? JSON.stringify(result) },
-    ],
+    content,
     // Gateway results carry ok:false on failure — mark those as tool errors
     // so the model treats the message as a failure, not a success payload.
     ...(r && typeof r === "object" && r.ok === false ? { isError: true } : {}),

package/src/core/types.ts

@@ -233,6 +233,12 @@ export type ActionResult = {
    * tool calls (react/edit/delete) that target this message.
    */
   message_id?: number | string;
+  /**
+   * Set by tools that return a viewable image (e.g. `read` on a photo). The
+   * MCP layer turns this into an image content block so the model sees the
+   * picture instead of base64 text.
+   */
+  image?: { data: string; mimeType: string };
   [key: string]: unknown;
 };
 

package/src/util/http-body.ts

@@ -0,0 +1,51 @@
+/**
+ * Bounded reading of fetch response bodies. Content-Length is optional and
+ * advisory, so the only trustworthy size limit is one enforced while the
+ * body streams in.
+ */
+
+export class ResponseTooLargeError extends Error {}
+
+/**
+ * Read a response incrementally so a missing or dishonest Content-Length
+ * header cannot buffer an unbounded body in memory.
+ */
+export async function readBodyLimited(
+  response: Response,
+  maxBytes: number,
+): Promise<Buffer> {
+  if (!response.body) {
+    const buffer = Buffer.from(await response.arrayBuffer());
+    if (buffer.length > maxBytes) throw new ResponseTooLargeError();
+    return buffer;
+  }
+
+  const reader = response.body.getReader();
+  const chunks: Buffer[] = [];
+  let totalBytes = 0;
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (!value?.byteLength) continue;
+
+      totalBytes += value.byteLength;
+      if (totalBytes > maxBytes) {
+        try {
+          await reader.cancel("response exceeds download limit");
+        } catch {
+          // Cancellation is cleanup only; preserve the useful size error.
+        }
+        throw new ResponseTooLargeError();
+      }
+      chunks.push(
+        Buffer.from(value.buffer, value.byteOffset, value.byteLength),
+      );
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  return Buffer.concat(chunks, totalBytes);
+}

package/src/util/web-content.ts

@@ -0,0 +1,115 @@
+/**
+ * Interpretation of fetched web content: charset decoding, text/binary
+ * classification, HTML text extraction, and magic-byte validation of binary
+ * payloads (via file-type) so server-declared Content-Type headers are never
+ * trusted on their own.
+ */
+
+import * as cheerio from "cheerio";
+import { fileTypeFromBuffer } from "file-type";
+
+/** Binary categories whose content Talon validates against magic bytes. */
+export type BinaryKind = "image" | "pdf" | "zip";
+
+export interface DetectedBinary {
+  ext: string;
+  mime: string;
+}
+
+function isTextMimeType(mimeType: string): boolean {
+  return (
+    mimeType.startsWith("text/") ||
+    mimeType === "application/json" ||
+    mimeType.endsWith("+json") ||
+    mimeType === "application/xml" ||
+    mimeType.endsWith("+xml") ||
+    mimeType === "application/javascript" ||
+    mimeType === "application/x-javascript"
+  );
+}
+
+function looksLikeStructuredText(buffer: Buffer): boolean {
+  if (buffer.subarray(0, 512).includes(0)) return false;
+  const sample = buffer.subarray(0, 512).toString("utf-8").trimStart();
+  return (
+    sample.startsWith("{") ||
+    sample.startsWith("[") ||
+    sample.startsWith("<?xml") ||
+    /^<!doctype\s+html/i.test(sample) ||
+    /^<[a-z][\w:-]*(?:\s|>)/i.test(sample)
+  );
+}
+
+/**
+ * Whether a response should be treated as readable text. Servers frequently
+ * omit Content-Type or fall back to application/octet-stream, so structured
+ * text is also sniffed from the body when the header says nothing useful.
+ */
+export function isTextContent(mimeType: string, buffer: Buffer): boolean {
+  if (isTextMimeType(mimeType)) return true;
+  const isGenericMime = !mimeType || mimeType === "application/octet-stream";
+  return isGenericMime && looksLikeStructuredText(buffer);
+}
+
+/**
+ * Whether textual content is an HTML document (declared or sniffed) and thus
+ * benefits from DOM text extraction rather than being returned verbatim.
+ */
+export function isHtmlContent(mimeType: string, text: string): boolean {
+  return (
+    mimeType.includes("html") || /^<!doctype\s+html|^<html[\s>]/i.test(text)
+  );
+}
+
+/** Decode a body honoring the declared response charset, defaulting to UTF-8. */
+export function decodeText(buffer: Buffer, contentType: string): string {
+  const charset = /charset\s*=\s*["']?([^;\s"']+)/i.exec(contentType)?.[1];
+  try {
+    return new TextDecoder(charset || "utf-8").decode(buffer);
+  } catch {
+    return buffer.toString("utf-8");
+  }
+}
+
+/** Extract readable text from HTML using cheerio (proper DOM parser). */
+export function extractText(html: string, maxLength = 8000): string {
+  const $ = cheerio.load(html);
+  // Remove non-content elements
+  $("script, style, noscript, iframe, svg, nav, footer, header").remove();
+  // Get text content, normalize whitespace
+  const text = $("body").text().replace(/\s+/g, " ").trim();
+  return text.slice(0, maxLength);
+}
+
+/** Identify a binary payload from its magic bytes. */
+export async function detectBinaryType(
+  buffer: Buffer,
+): Promise<DetectedBinary | null> {
+  const detected = await fileTypeFromBuffer(buffer);
+  return detected ? { ext: detected.ext, mime: detected.mime } : null;
+}
+
+/** The binary kind a Content-Type header promises, if it promises one. */
+export function advertisedBinaryKind(mimeType: string): BinaryKind | null {
+  if (mimeType.startsWith("image/")) return "image";
+  if (mimeType.includes("pdf")) return "pdf";
+  if (mimeType.includes("zip")) return "zip";
+  return null;
+}
+
+/** Whether the actual bytes satisfy what the Content-Type header promised. */
+export function matchesBinaryKind(
+  kind: BinaryKind,
+  detected: DetectedBinary | null,
+  buffer: Buffer,
+): boolean {
+  switch (kind) {
+    case "image":
+      return detected?.mime.startsWith("image/") ?? false;
+    case "pdf":
+      return detected?.ext === "pdf";
+    case "zip":
+      // The whole zip family (docx, jar, epub, …) shares the PK signature.
+      return buffer[0] === 0x50 && buffer[1] === 0x4b;
+  }
+}