u-foo 3.0.2 → 3.0.4

package/src/code/imageIngest.js

@@ -0,0 +1,367 @@
+"use strict";
+
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { execFileSync } = require("child_process");
+const { mediaTypeFromPath, sniffMediaType, MAX_IMAGE_BYTES } = require("./tools/readImage");
+
+const IMAGE_EXT_RE = /\.(png|jpe?g|gif|webp)$/i;
+const FILE_URL_RE = /^file:\/\//i;
+
+function uploadsDir(workspaceRoot = "", sessionId = "") {
+  const root = path.resolve(String(workspaceRoot || process.cwd()));
+  const sid = String(sessionId || "session").trim().replace(/[^a-zA-Z0-9._-]+/g, "_") || "session";
+  return path.join(root, ".ufoo", "agent", "ucode", "uploads", sid);
+}
+
+function safeBaseName(filePath = "") {
+  const base = path.basename(String(filePath || "image.png"));
+  const cleaned = base.replace(/[^\w.\-()+ ]+/g, "_").replace(/\s+/g, " ").trim();
+  if (!cleaned) return "image.png";
+  if (!IMAGE_EXT_RE.test(cleaned)) return `${cleaned}.png`;
+  return cleaned.slice(0, 120);
+}
+
+function decodeFileUrl(value = "") {
+  const text = String(value || "").trim();
+  if (!FILE_URL_RE.test(text)) return text;
+  try {
+    const parsed = new URL(text);
+    if (parsed.protocol !== "file:") return text;
+    return decodeURIComponent(parsed.pathname || "");
+  } catch {
+    return text.replace(FILE_URL_RE, "");
+  }
+}
+
+function looksLikeImagePath(candidate = "") {
+  const text = decodeFileUrl(String(candidate || "").trim().replace(/^['"]|['"]$/g, ""));
+  if (!text || !IMAGE_EXT_RE.test(text)) return false;
+  if (text.startsWith("/") || /^[A-Za-z]:[\\/]/.test(text) || text.startsWith("~")) return true;
+  // Relative paths ending in image ext (drag from cwd listings)
+  if (!/\s/.test(text) && IMAGE_EXT_RE.test(text)) return true;
+  return false;
+}
+
+function expandHome(filePath = "") {
+  const text = String(filePath || "");
+  if (text.startsWith("~/")) return path.join(os.homedir(), text.slice(2));
+  return text;
+}
+
+/**
+ * Extract image file paths from terminal paste / drag-drop text.
+ * Supports file://, quoted paths with spaces, and bare absolute paths.
+ */
+function extractImagePathsFromPaste(text = "") {
+  const raw = String(text || "");
+  if (!raw.trim()) return [];
+  const found = [];
+  const seen = new Set();
+
+  function pushPath(candidate) {
+    let next = decodeFileUrl(String(candidate || "").trim());
+    next = next.replace(/^['"]|['"]$/g, "");
+    if (!looksLikeImagePath(next)) return;
+    next = expandHome(next);
+    const key = path.resolve(next);
+    if (seen.has(key)) return;
+    seen.add(key);
+    found.push(next);
+  }
+
+  // Quoted paths (possibly with spaces)
+  const quoted = /["']([^"']+\.(?:png|jpe?g|gif|webp))["']/gi;
+  let match;
+  while ((match = quoted.exec(raw))) {
+    pushPath(match[1]);
+  }
+
+  // file:// URLs
+  const fileUrls = /file:\/\/[^\s"'<>]+/gi;
+  while ((match = fileUrls.exec(raw))) {
+    pushPath(match[0]);
+  }
+
+  // Bare tokens / lines
+  for (const line of raw.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    if (looksLikeImagePath(trimmed)) {
+      pushPath(trimmed);
+      continue;
+    }
+    for (const token of trimmed.split(/\s+/)) {
+      if (looksLikeImagePath(token)) pushPath(token);
+    }
+  }
+
+  return found;
+}
+
+function stripExtractedPathsFromText(text = "", paths = []) {
+  let out = String(text || "");
+  for (const p of paths) {
+    const variants = [
+      `"${p}"`,
+      `'${p}'`,
+      p,
+      p.startsWith("/") ? `file://${p}` : "",
+      p.startsWith("/") ? `file://${encodeURI(p)}` : "",
+    ].filter(Boolean);
+    for (const v of variants) {
+      out = out.split(v).join(" ");
+    }
+  }
+  return out
+    .replace(/[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .replace(/[ \t]{2,}/g, " ")
+    .replace(/\s+"/g, " ")
+    .replace(/"\s+/g, " ")
+    .replace(/\s+'/g, " ")
+    .replace(/'\s+/g, " ")
+    .trim();
+}
+
+function formatImageLogLabel({ relPath = "", fileName = "", path: pathText = "" } = {}) {
+  const name = String(fileName || "").trim()
+    || path.basename(String(relPath || pathText || "").trim())
+    || "image";
+  return `[image: ${name}]`;
+}
+
+function formatUserLogWithAttachments(userText = "", attachments = []) {
+  const labels = (Array.isArray(attachments) ? attachments : [])
+    .map((item) => formatImageLogLabel(item))
+    .filter(Boolean);
+  const body = String(userText || "").trim();
+  if (labels.length === 0) return body;
+  if (!body) return labels.join(" ");
+  return `${labels.join(" ")} ${body}`;
+}
+
+function buildAttachedImagesPromptPrefix(attachments = []) {
+  const list = Array.isArray(attachments) ? attachments : [];
+  if (list.length === 0) return "";
+  const lines = [
+    "[Attached images — call read_image on each path]",
+    ...list.map((item) => `- ${item.relPath || item.path || ""}`).filter((line) => line !== "- "),
+    "",
+  ];
+  return lines.join("\n");
+}
+
+function ingestImageFile({
+  sourcePath = "",
+  workspaceRoot = process.cwd(),
+  sessionId = "",
+  buffer = null,
+  preferredName = "",
+} = {}) {
+  const root = path.resolve(String(workspaceRoot || process.cwd()));
+  let data = buffer;
+  let fromPath = String(sourcePath || "").trim();
+
+  if (!data) {
+    if (!fromPath) {
+      return { ok: false, error: "sourcePath or buffer required" };
+    }
+    fromPath = expandHome(decodeFileUrl(fromPath));
+    try {
+      const stat = fs.statSync(fromPath);
+      if (!stat.isFile()) return { ok: false, error: `not a file: ${fromPath}` };
+      if (stat.size > MAX_IMAGE_BYTES) {
+        return {
+          ok: false,
+          error: `image too large (${stat.size} bytes); max ${MAX_IMAGE_BYTES}`,
+        };
+      }
+      data = fs.readFileSync(fromPath);
+    } catch (err) {
+      return { ok: false, error: err && err.message ? err.message : "read failed" };
+    }
+  }
+
+  if (!Buffer.isBuffer(data)) {
+    return { ok: false, error: "image buffer required" };
+  }
+  if (data.length > MAX_IMAGE_BYTES) {
+    return {
+      ok: false,
+      error: `image too large (${data.length} bytes); max ${MAX_IMAGE_BYTES}`,
+    };
+  }
+
+  const sniffed = sniffMediaType(data);
+  const fromName = mediaTypeFromPath(preferredName || fromPath);
+  const mediaType = sniffed || fromName;
+  if (!mediaType) {
+    return { ok: false, error: "unsupported image type (use png, jpeg, gif, or webp)" };
+  }
+
+  const ext = mediaType === "image/jpeg"
+    ? ".jpg"
+    : mediaType === "image/gif"
+      ? ".gif"
+      : mediaType === "image/webp"
+        ? ".webp"
+        : ".png";
+
+  let base = safeBaseName(preferredName || fromPath || `clipboard${ext}`);
+  if (!IMAGE_EXT_RE.test(base)) base = `${base}${ext}`;
+  // Normalize extension to sniffed type
+  base = `${path.basename(base, path.extname(base))}${ext}`;
+
+  const dir = uploadsDir(root, sessionId);
+  fs.mkdirSync(dir, { recursive: true });
+  const stamp = Date.now().toString(36);
+  const destName = `${stamp}-${base}`;
+  const absPath = path.join(dir, destName);
+  fs.writeFileSync(absPath, data);
+
+  const relPath = path.relative(root, absPath).split(path.sep).join("/");
+  return {
+    ok: true,
+    relPath,
+    absPath,
+    fileName: base,
+    mediaType,
+    bytes: data.length,
+  };
+}
+
+function tryIngestClipboardImage({
+  workspaceRoot = process.cwd(),
+  sessionId = "",
+  platform = process.platform,
+  execFile = execFileSync,
+} = {}) {
+  if (platform !== "darwin") {
+    return { ok: false, error: "clipboard image ingest is only supported on macOS" };
+  }
+
+  const tmpPath = path.join(
+    os.tmpdir(),
+    `ufoo-clipboard-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}.png`,
+  );
+  // AppleScript: write clipboard PNGf to a temp file.
+  const script = [
+    `set outPath to POSIX file ${JSON.stringify(tmpPath)}`,
+    "try",
+    "  set pngData to the clipboard as «class PNGf»",
+    "  set fileRef to open for access outPath with write permission",
+    "  set eof of fileRef to 0",
+    "  write pngData to fileRef",
+    "  close access fileRef",
+    '  return "ok"',
+    "on error errMsg number errNum",
+    "  try",
+    "    close access outPath",
+    "  end try",
+    '  return "err:" & errMsg',
+    "end try",
+  ].join("\n");
+
+  let resultText = "";
+  try {
+    resultText = String(execFile("osascript", ["-e", script], {
+      encoding: "utf8",
+      timeout: 5000,
+      maxBuffer: 1024 * 1024,
+    }) || "").trim();
+  } catch (err) {
+    try { fs.unlinkSync(tmpPath); } catch { /* ignore */ }
+    return {
+      ok: false,
+      error: err && err.message ? err.message : "clipboard read failed",
+    };
+  }
+
+  if (!resultText.startsWith("ok")) {
+    try { fs.unlinkSync(tmpPath); } catch { /* ignore */ }
+    return {
+      ok: false,
+      error: resultText.replace(/^err:/, "").trim() || "no PNG image on clipboard",
+    };
+  }
+
+  try {
+    const ingested = ingestImageFile({
+      sourcePath: tmpPath,
+      workspaceRoot,
+      sessionId,
+      preferredName: `clipboard-${Date.now().toString(36)}.png`,
+    });
+    return ingested;
+  } finally {
+    try { fs.unlinkSync(tmpPath); } catch { /* ignore */ }
+  }
+}
+
+/**
+ * Handle a paste chunk: ingest image paths and/or macOS clipboard bitmap.
+ * Returns text to insert into the editor (paths removed) plus attachments.
+ */
+function handleImagePaste(text = "", {
+  workspaceRoot = process.cwd(),
+  sessionId = "",
+  tryClipboard = true,
+  platform = process.platform,
+  execFile = execFileSync,
+} = {}) {
+  const raw = String(text || "");
+  const paths = extractImagePathsFromPaste(raw);
+  const attachments = [];
+  const errors = [];
+
+  for (const sourcePath of paths) {
+    const ingested = ingestImageFile({ sourcePath, workspaceRoot, sessionId });
+    if (ingested.ok) attachments.push(ingested);
+    else errors.push(ingested.error || "ingest failed");
+  }
+
+  let remaining = stripExtractedPathsFromText(raw, paths);
+
+  // If paste had no usable text/paths, try clipboard PNG (Cmd+V of a screenshot).
+  const trimmedRemaining = remaining.trim();
+  const looksEmptyOrBinary = !trimmedRemaining
+    || /[\x00-\x08\x0e-\x1f]/.test(raw)
+    || (Buffer.byteLength(raw, "utf8") > 200 && paths.length === 0 && !/\s/.test(raw.slice(0, 40)));
+
+  if (tryClipboard && attachments.length === 0 && looksEmptyOrBinary) {
+    const clip = tryIngestClipboardImage({
+      workspaceRoot,
+      sessionId,
+      platform,
+      execFile,
+    });
+    if (clip.ok) {
+      attachments.push(clip);
+      remaining = "";
+    } else if (paths.length === 0 && !trimmedRemaining) {
+      errors.push(clip.error || "clipboard ingest failed");
+    }
+  }
+
+  return {
+    text: remaining,
+    attachments,
+    errors,
+  };
+}
+
+module.exports = {
+  IMAGE_EXT_RE,
+  uploadsDir,
+  safeBaseName,
+  extractImagePathsFromPaste,
+  stripExtractedPathsFromText,
+  formatImageLogLabel,
+  formatUserLogWithAttachments,
+  buildAttachedImagesPromptPrefix,
+  ingestImageFile,
+  tryIngestClipboardImage,
+  handleImagePaste,
+};

package/src/code/nativeRunner.js

@@ -27,6 +27,8 @@ const {
   clearUserPrompts,
   formatUserReminderMessage,
   ensurePendingUserPrompts,
+  shouldAutoContinuePlan,
+  buildPlanAutoContinueReminder,
 } = require("./context/userNudge");
 const {
   runAskUserTool,
@@ -50,12 +52,14 @@ const {
 } = require("./protocol");
 const { stableStringify } = require("./context/stableJson");
 const { getReadToolDescription } = require("../agents/prompts/native/toolDescriptions/read");
+const { getReadImageToolDescription } = require("../agents/prompts/native/toolDescriptions/readImage");
 const { getWriteToolDescription } = require("../agents/prompts/native/toolDescriptions/write");
 const { getEditToolDescription } = require("../agents/prompts/native/toolDescriptions/edit");
 const { getBashToolDescription } = require("../agents/prompts/native/toolDescriptions/bash");
 
 const CORE_TOOL_NAMES = new Set([
   "read",
+  "read_image",
   "write",
   "edit",
   "bash",
@@ -64,7 +68,14 @@ const CORE_TOOL_NAMES = new Set([
   "task_run",
   "ask_user",
 ]);
-const EXECUTABLE_GRAPH_TOOLS = new Set(["read", "write", "edit", "bash", "artifact_read"]);
+const EXECUTABLE_GRAPH_TOOLS = new Set([
+  "read",
+  "read_image",
+  "write",
+  "edit",
+  "bash",
+  "artifact_read",
+]);
 const CONTROL_PLANE_TOOLS = new Set(["plan_graph", "task_run"]);
 const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
 const DEFAULT_ANTHROPIC_BASE_URL = "https://api.anthropic.com/v1";
@@ -76,6 +87,8 @@ const DEFAULT_KIMI_MODEL = "k3";
 const DEFAULT_MAX_NATIVE_TOOL_CALLS = 100;
 const DEFAULT_MAX_NATIVE_TOOL_ERRORS = 20;
 const DEFAULT_NATIVE_TIMEOUT_MS = 43200000; // 12 hours
+/** Max text-only auto-continues while a plan is waiting on a task (per user submit). */
+const DEFAULT_MAX_PLAN_AUTO_CONTINUES = 24;
 // Anthropic Messages rejects max_tokens above the model's real cap (64K on
 // current models), so the transports use different defaults. Override either
 // via UFOO_UCODE_MAX_TOKENS (positive integer).
@@ -392,6 +405,23 @@ function buildCoreToolSpecs() {
         },
       },
     },
+    {
+      type: "function",
+      function: {
+        name: "read_image",
+        description: getReadImageToolDescription(),
+        parameters: {
+          type: "object",
+          properties: {
+            path: {
+              type: "string",
+              description: "Workspace-relative path to a png, jpeg, gif, or webp image.",
+            },
+          },
+          required: ["path"],
+        },
+      },
+    },
     {
       type: "function",
       function: {
@@ -1733,6 +1763,9 @@ async function runNativeLoop({
   // materialize Provider messages yet. STRICT via UFOO_UCODE_PROTOCOL_STRICT=1.
   let activeLedger = null;
   let lastProtocolLedger = null;
+  let planAutoContinues = 0;
+  let lastAutoContinueWaitingId = "";
+  let consecutiveEmptyAutoContinues = 0;
 
   if (resume) {
     await withFaultPoint("before_provider_resume", () => {});
@@ -1749,6 +1782,29 @@ async function runNativeLoop({
     messages.push({ role: "user", content });
   }
 
+  function tryInjectPlanAutoContinue() {
+    if (!shouldAutoContinuePlan(executionState)) return false;
+    if (planAutoContinues >= DEFAULT_MAX_PLAN_AUTO_CONTINUES) return false;
+    const waitingId = String(
+      (executionState.planGraph && executionState.planGraph.waitingFor
+        && executionState.planGraph.waitingFor.id) || ""
+    ).trim();
+    if (
+      consecutiveEmptyAutoContinues >= 2
+      && waitingId
+      && waitingId === lastAutoContinueWaitingId
+    ) {
+      return false;
+    }
+    const reminder = buildPlanAutoContinueReminder(executionState);
+    if (!reminder) return false;
+    messages.push({ role: "user", content: reminder });
+    planAutoContinues += 1;
+    lastAutoContinueWaitingId = waitingId;
+    consecutiveEmptyAutoContinues += 1;
+    return true;
+  }
+
   while (true) {
     guards.ensureActive();
 
@@ -1820,6 +1876,9 @@ async function runNativeLoop({
       if (!aggregated.trim() && text) {
         aggregated = text;
       }
+      if (tryInjectPlanAutoContinue()) {
+        continue;
+      }
       return {
         text: aggregated,
         streamed,
@@ -1831,6 +1890,9 @@ async function runNativeLoop({
       };
     }
 
+    // A tool-using turn resets the empty auto-continue streak (progress possible).
+    consecutiveEmptyAutoContinues = 0;
+
     const pendingCalls = transport.prepareToolCalls({ messages, turnResult, toolCalls });
     if (!pendingCalls) {
       return {

package/src/code/providers/anthropicMessagesTransport.js

@@ -1,6 +1,12 @@
 "use strict";
 
 const { assertTransport } = require("./transportContract");
+const {
+  extractVisionPayload,
+  stripVisionBase64,
+  visionSummaryText,
+  toAnthropicImageBlock,
+} = require("./visionBlocks");
 
 /**
  * Anthropic Messages API transport adapter.
@@ -70,11 +76,32 @@ function createAnthropicMessagesTransport(deps = {}) {
       }));
     },
     appendToolResult({ collected, call, toolResult }) {
+      const vision = extractVisionPayload(toolResult);
+      const isError = Boolean(!toolResult || toolResult.ok === false);
+      if (vision) {
+        const textPayload = stripVisionBase64(toolResult);
+        collected.push({
+          type: "tool_result",
+          tool_use_id: String(call.source.id || ""),
+          content: [
+            {
+              type: "text",
+              text: clipText(
+                `${visionSummaryText(vision, toolResult)}\n${toJsonString(textPayload)}`,
+                12000,
+              ),
+            },
+            toAnthropicImageBlock(vision),
+          ],
+          is_error: isError,
+        });
+        return;
+      }
       collected.push({
         type: "tool_result",
         tool_use_id: String(call.source.id || ""),
         content: clipText(toJsonString(toolResult), 12000),
-        is_error: Boolean(!toolResult || toolResult.ok === false),
+        is_error: isError,
       });
     },
     flushToolResults({ messages, collected }) {

package/src/code/providers/index.js

@@ -5,4 +5,5 @@ module.exports = {
   ...require("./openaiChatTransport"),
   ...require("./anthropicMessagesTransport"),
   ...require("./modelsCatalog"),
+  ...require("./visionBlocks"),
 };

package/src/code/providers/openaiChatTransport.js

@@ -2,6 +2,12 @@
 
 const { randomUUID } = require("crypto");
 const { assertTransport } = require("./transportContract");
+const {
+  extractVisionPayload,
+  stripVisionBase64,
+  visionSummaryText,
+  toOpenAiImagePart,
+} = require("./visionBlocks");
 
 /**
  * OpenAI-compatible chat-completions transport adapter.
@@ -82,11 +88,23 @@ function createOpenAiChatTransport(deps = {}) {
       }));
     },
     appendToolResult({ messages, call, toolResult }) {
+      const vision = extractVisionPayload(toolResult);
+      const payload = vision ? stripVisionBase64(toolResult) : toolResult;
       messages.push({
         role: "tool",
         tool_call_id: call.source.id,
-        content: clipText(toJsonString(toolResult), 12000),
+        content: clipText(toJsonString(payload), 12000),
       });
+      if (vision) {
+        const imagePart = toOpenAiImagePart(vision);
+        messages.push({
+          role: "user",
+          content: [
+            { type: "text", text: visionSummaryText(vision, toolResult) },
+            imagePart,
+          ],
+        });
+      }
     },
   };
 

package/src/code/providers/visionBlocks.js

@@ -0,0 +1,110 @@
+"use strict";
+
+/**
+ * Helpers for expanding read_image tool results into provider vision blocks.
+ */
+
+function extractVisionPayload(toolResult = null) {
+  if (!toolResult || typeof toolResult !== "object") return null;
+  const base64 = String(toolResult.base64 || "").trim();
+  const mediaType = String(toolResult.mediaType || "").trim().toLowerCase();
+  if (!base64 || !mediaType.startsWith("image/")) return null;
+  if (toolResult.ok === false) return null;
+  return {
+    path: String(toolResult.path || "").trim(),
+    mediaType,
+    bytes: Number.isFinite(toolResult.bytes) ? toolResult.bytes : null,
+    base64,
+    artifactId: String(toolResult.artifactId || "").trim(),
+  };
+}
+
+function isVisionToolResult(toolResult = null) {
+  return Boolean(extractVisionPayload(toolResult));
+}
+
+function stripVisionBase64(value) {
+  if (Array.isArray(value)) {
+    return value.map((item) => stripVisionBase64(item));
+  }
+  if (!value || typeof value !== "object") return value;
+  const out = {};
+  for (const [key, entry] of Object.entries(value)) {
+    if (key === "base64") continue;
+    out[key] = stripVisionBase64(entry);
+  }
+  return out;
+}
+
+function visionSummaryText(vision = null, toolResult = null) {
+  const pathText = (vision && vision.path) || (toolResult && toolResult.path) || "image";
+  const mediaType = (vision && vision.mediaType) || (toolResult && toolResult.mediaType) || "image/*";
+  const bytes = (vision && vision.bytes) != null
+    ? vision.bytes
+    : (toolResult && toolResult.bytes);
+  const parts = [
+    `Image loaded for vision: ${pathText}`,
+    `mediaType=${mediaType}`,
+  ];
+  if (Number.isFinite(bytes)) parts.push(`bytes=${bytes}`);
+  if (vision && vision.artifactId) parts.push(`artifactId=${vision.artifactId}`);
+  parts.push("Visual content is attached for this model call only; call read_image again later if needed.");
+  return parts.join(" | ");
+}
+
+function toAnthropicImageBlock(vision = null) {
+  if (!vision) return null;
+  return {
+    type: "image",
+    source: {
+      type: "base64",
+      media_type: vision.mediaType,
+      data: vision.base64,
+    },
+  };
+}
+
+function toOpenAiImagePart(vision = null) {
+  if (!vision) return null;
+  return {
+    type: "image_url",
+    image_url: {
+      url: `data:${vision.mediaType};base64,${vision.base64}`,
+    },
+  };
+}
+
+function degradeVisionContent(content) {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return content;
+  const texts = [];
+  for (const block of content) {
+    if (!block || typeof block !== "object") continue;
+    const type = String(block.type || "").trim().toLowerCase();
+    if (type === "text" && block.text) {
+      texts.push(String(block.text));
+      continue;
+    }
+    if (type === "image" || type === "image_url") {
+      const pathHint = block.path
+        || (block.source && block.source.path)
+        || "";
+      texts.push(pathHint ? `[image: ${pathHint}]` : "[image]");
+      continue;
+    }
+    if (type === "tool_result" && Array.isArray(block.content)) {
+      texts.push(degradeVisionContent(block.content));
+    }
+  }
+  return texts.filter(Boolean).join("\n") || "[multimodal content]";
+}
+
+module.exports = {
+  extractVisionPayload,
+  isVisionToolResult,
+  stripVisionBase64,
+  visionSummaryText,
+  toAnthropicImageBlock,
+  toOpenAiImagePart,
+  degradeVisionContent,
+};