talon-agent 3.34.1 → 3.36.0

package/src/frontend/native/routes/host.ts

@@ -1,6 +1,9 @@
 import type { IncomingMessage, ServerResponse } from "node:http";
+import type { Readable } from "node:stream";
+import type { AttachmentRef } from "./params.js";
 import type {
   BackendOption,
+  ClientAttachment,
   BridgeEvent,
   BridgeStatus,
   ClientChat,
@@ -18,7 +21,13 @@ import type {
 import type { ConfigSnapshot } from "../settings.js";
 
 /** Optional attachment references carried alongside a sent message. */
-type SendOptions = {
+export type SendOptions = {
+  /**
+   * Every file attached to the message, as the client's references to its
+   * own uploads. `imagePath` and `attachmentPath` below are the single-file
+   * shape older clients send; the handler folds them into the same list.
+   */
+  attachments?: AttachmentRef[];
   /** Relative bridge path to render inline (e.g. `/media?id=…`). */
   imagePath?: string;
   /** Absolute on-disk path handed to the model so it can read the file. */
@@ -41,12 +50,16 @@ export type BridgeServerHandlers = {
   search(query: string, chatId?: string): SearchResult[];
   /** Fire-and-forget: streams its results back through `broadcast`. */
   send(id: string, text: string, opts?: SendOptions): void;
-  /** Persist an uploaded image and return its render path + on-disk path. */
+  /**
+   * Stream an uploaded file to disk and return its wire description. The body
+   * is consumed as it arrives (never buffered whole), so the size ceiling is
+   * disk, not memory.
+   */
   upload(
     filename: string,
     contentType: string,
-    bytes: Buffer,
-  ): Promise<{ imagePath: string; path: string }>;
+    body: Readable,
+  ): Promise<ClientAttachment>;
   listModels(
     id?: string,
   ):
@@ -139,7 +152,6 @@ export type RouteHost = {
   fingerprint: () => string | null;
   json: (res: ServerResponse, code: number, body: unknown) => void;
   readJson: (req: IncomingMessage) => Promise<Record<string, unknown>>;
-  readRaw: (req: IncomingMessage, max: number) => Promise<Buffer>;
   corsHeaders: () => Record<string, string>;
   streamFile: (
     res: ServerResponse,

package/src/frontend/native/routes/params.ts

@@ -27,3 +27,24 @@ export function asPositiveInt(v: string | null): number | undefined {
   const n = Number(v);
   return Number.isInteger(n) && n > 0 ? n : undefined;
 }
+
+/** A client's reference to a file it already uploaded. */
+export type AttachmentRef = { url?: string; path?: string };
+
+/**
+ * Coerce the `attachments` array on a `/send` body to reference shape. Only
+ * the identifying fields are read: name, size and type are whatever the
+ * daemon recorded when it wrote the file, never what the client claims now.
+ */
+export function asAttachmentRefs(v: unknown): AttachmentRef[] {
+  if (!Array.isArray(v)) return [];
+  const refs: AttachmentRef[] = [];
+  for (const item of v) {
+    if (typeof item !== "object" || item === null) continue;
+    const row = item as Record<string, unknown>;
+    const url = asString(row.url);
+    const path = asString(row.path);
+    if (url || path) refs.push({ url, path });
+  }
+  return refs;
+}

package/src/frontend/native/runtime.ts

@@ -12,7 +12,12 @@ import type { TalonConfig } from "../../util/config.js";
 import type { Gateway } from "../../core/engine/gateway.js";
 import { getMeshService, type MeshService } from "../../core/mesh/index.js";
 import { NativeChats } from "./chats.js";
-import type { BridgeEvent, ClientToolCall, ContextInfo } from "./protocol.js";
+import type {
+  BridgeEvent,
+  ClientAttachment,
+  ClientToolCall,
+  ContextInfo,
+} from "./protocol.js";
 
 /** One tool call observed during a running turn, kept for mid-turn replay. */
 export type LiveToolEntry = {
@@ -21,11 +26,10 @@ export type LiveToolEntry = {
   done?: boolean;
 };
 
-/** A chat's single queued follow-up, with its attachment paths intact. */
+/** A chat's single queued follow-up, with its attachments intact. */
 export type QueuedEntry = {
   text: string;
-  imagePath?: string;
-  attachmentPath?: string;
+  attachments: ClientAttachment[];
 };
 
 export type NativeRuntime = {
@@ -51,6 +55,13 @@ export type NativeRuntime = {
    * frontends behave).
    */
   readonly media: Map<string, string>;
+  /**
+   * Uploads this daemon run has accepted, keyed by their media id. `/send`
+   * resolves a client's attachment references through here rather than
+   * trusting the paths in the request body, so a message can only ever point
+   * the model at a file this daemon itself wrote to the uploads dir.
+   */
+  readonly uploads: Map<string, ClientAttachment>;
   /**
    * Live context-window fill per chat, refreshed at the end of each turn.
    * Cached (not computed inline) so the sync `toClientChat` projection stays
@@ -99,6 +110,7 @@ export function createNativeRuntime(
     chats: new NativeChats(),
     mesh: getMeshService(),
     media: new Map(),
+    uploads: new Map(),
     contextByChat: new Map(),
     liveTurns: new Map(),
     queuedByChat: new Map(),

package/src/frontend/native/server.ts

@@ -24,9 +24,9 @@ import { createServer as createTlsServer } from "node:https";
 import { createHash, timingSafeEqual } from "node:crypto";
 import { createReadStream } from "node:fs";
 import { stat } from "node:fs/promises";
-import { extname } from "node:path";
 import { log, logError, logDebug, logWarn } from "../../util/log.js";
 import { formatFingerprint, type BridgeTlsIdentity } from "./tls.js";
+import { contentTypeFor } from "./media.js";
 import { type BridgeEvent } from "./protocol.js";
 import { buildRoutes } from "./routes/index.js";
 import type { BridgeServerHandlers, RouteHost } from "./routes/host.js";
@@ -38,7 +38,7 @@ import {
   type RouteHandler,
 } from "./routes/table.js";
 
-export type { BridgeServerHandlers } from "./routes/host.js";
+export type { BridgeServerHandlers, SendOptions } from "./routes/host.js";
 export { BRIDGE_ROUTE_AUTH, type BridgeRouteKey } from "./routes/table.js";
 
 const SSE_PING_MS = 25_000;
@@ -352,7 +352,6 @@ export class BridgeServer {
       fingerprint: () => this.getFingerprint(),
       json: (res, code, body) => this.json(res, code, body),
       readJson: (req) => this.readJson(req),
-      readRaw: (req, max) => this.readRaw(req, max),
       corsHeaders: () => this.corsHeaders(),
       streamFile: (res, file) => this.streamFile(res, file),
       serveMedia: (res, id) => this.serveMedia(res, id),
@@ -591,17 +590,6 @@ export class BridgeServer {
   }
 
   /** Read a raw request body (binary-safe) up to `max` bytes. */
-  private async readRaw(req: IncomingMessage, max: number): Promise<Buffer> {
-    const chunks: Buffer[] = [];
-    let total = 0;
-    for await (const chunk of req) {
-      total += (chunk as Buffer).length;
-      if (total > max) throw new Error("Upload too large");
-      chunks.push(chunk as Buffer);
-    }
-    return Buffer.concat(chunks);
-  }
-
   private async readJson(
     req: IncomingMessage,
   ): Promise<Record<string, unknown>> {
@@ -621,24 +609,3 @@ export class BridgeServer {
     return parsed as Record<string, unknown>;
   }
 }
-
-/** Minimal image content-type map for the media endpoint. */
-function contentTypeFor(filePath: string): string {
-  switch (extname(filePath).toLowerCase()) {
-    case ".png":
-      return "image/png";
-    case ".jpg":
-    case ".jpeg":
-      return "image/jpeg";
-    case ".gif":
-      return "image/gif";
-    case ".webp":
-      return "image/webp";
-    case ".bmp":
-      return "image/bmp";
-    case ".svg":
-      return "image/svg+xml";
-    default:
-      return "application/octet-stream";
-  }
-}

package/src/frontend/native/status.ts

@@ -12,7 +12,9 @@ export function bridgeStatus(runtime: NativeRuntime): BridgeStatus {
   return {
     app: "talon-bridge",
     protocol: BRIDGE_PROTOCOL_VERSION,
-    capabilities: ["mesh", "mesh-commands", "plugins-skills"],
+    // "attachments": /upload streams any file type and /send takes an
+    // `attachments` list, so a client can stage several files per message.
+    capabilities: ["mesh", "mesh-commands", "plugins-skills", "attachments"],
     botName: runtime.botName,
     backend: config.backend,
     model: resolveModel(config.model)?.displayName ?? config.model,

package/src/frontend/native/turn.ts

@@ -16,7 +16,7 @@ import { recordTurnMeta } from "./turn-meta.js";
 import type { ChatEntry } from "./chats.js";
 import { refreshContext } from "./context.js";
 import { emitAssistant, emitUser } from "./emit.js";
-import type { BridgeEvent } from "./protocol.js";
+import type { BridgeEvent, ClientAttachment } from "./protocol.js";
 import { takeQueued } from "./queue.js";
 import type { LiveToolEntry, NativeRuntime } from "./runtime.js";
 import { summarizeToolResult } from "./tool-result.js";
@@ -24,7 +24,38 @@ import { summarizeToolResult } from "./tool-result.js";
 /** The tool calls of one running turn, in call order. */
 type TurnTools = Map<string, LiveToolEntry>;
 
-export type StartTurnOptions = { imagePath?: string; attachmentPath?: string };
+export type StartTurnOptions = { attachments?: ClientAttachment[] };
+
+/** Human-readable size for the attachment lines handed to the model. */
+function formatSize(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  const units = ["KB", "MB", "GB", "TB"];
+  let value = bytes / 1024;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit += 1;
+  }
+  return `${value < 10 ? value.toFixed(1) : Math.round(value)} ${units[unit]}`;
+}
+
+/**
+ * Point the model at the files the user attached. Each gets its own line with
+ * an absolute path so the model can read it with its own tools; images keep
+ * the `[Attached image: …]` wording backends already recognise.
+ */
+export function attachmentPrompt(
+  text: string,
+  attachments: ClientAttachment[],
+): string {
+  if (!attachments.length) return text;
+  const lines = attachments.map((a) =>
+    a.image
+      ? `[Attached image: ${a.path}]`
+      : `[Attached file: ${a.name} (${a.mimeType}, ${formatSize(a.size)}) at ${a.path}]`,
+  );
+  return `${text ? `${text}\n\n` : ""}${lines.join("\n")}`;
+}
 
 /** True when a turn is currently running for a chat (used to decide queue). */
 export function isBusy(runtime: NativeRuntime, chatId: string): boolean {
@@ -247,7 +278,7 @@ async function runTurn(
   entry: ChatEntry,
   text: string,
   messageId: number,
-  attachmentPath?: string,
+  attachments: ClientAttachment[],
 ): Promise<void> {
   const start = Date.now();
   // Tool calls observed during this turn, in call order — recorded into the
@@ -256,10 +287,7 @@ async function runTurn(
   // the activity so far (cleared in the `finally`).
   const turnTools: TurnTools = new Map();
   runtime.liveTurns.set(entry.id, turnTools);
-  // Point the model at an attached image so it can read the file itself.
-  const prompt = attachmentPath
-    ? `${text ? `${text}\n\n` : ""}[Attached image: ${attachmentPath}]`
-    : text;
+  const prompt = attachmentPrompt(text, attachments);
   try {
     const result = await execute({
       chatId: entry.id,
@@ -287,8 +315,7 @@ async function runTurn(
     if (queued) {
       setImmediate(() =>
         startTurn(runtime, entry, queued.text, {
-          imagePath: queued.imagePath,
-          attachmentPath: queued.attachmentPath,
+          attachments: queued.attachments,
         }),
       );
     }
@@ -303,16 +330,11 @@ export function startTurn(
   text: string,
   options?: StartTurnOptions,
 ): void {
-  const messageId = emitUser(
-    runtime,
-    entry,
-    text,
-    options?.imagePath,
-    options?.attachmentPath,
-  );
+  const attachments = options?.attachments ?? [];
+  const messageId = emitUser(runtime, entry, text, attachments);
   runtime.broadcast({ kind: "turn_start", chatId: entry.id });
   runtime.broadcast({ kind: "typing", chatId: entry.id, on: true });
-  void runTurn(runtime, entry, text, messageId, options?.attachmentPath);
+  void runTurn(runtime, entry, text, messageId, attachments);
 }
 
 /** Best-effort interrupt of a chat's in-flight turn. `true` if one was