picturereader 1.0.3 → 2.0.1

package/src/vision-analyze.js

@@ -0,0 +1,260 @@
+/**
+ * vision_analyze — unified image understanding tool.
+ *
+ * Runs the full local vision pipeline:
+ *   1. decode + low-information guard
+ *   2. optional pixel scan (image_scan)
+ *   3. optional OCR (image_ocr)
+ *   4. optional local/remote VLM description
+ *
+ * All evidence is returned as text so a text-only model can reason about the
+ * image without trusting any single source blindly.
+ *
+ * Key features:
+ * - Smart API calling: simple images don't call external APIs
+ * - Cross-validation: main model verifies VLM results against pixel/OCR evidence
+ * - Multiple questions: support asking different questions about the same image
+ *
+ * @module picturereader/vision-analyze
+ */
+
+import { extname } from 'node:path';
+import { BYTE_CAP, MAX_PIXELS } from './tool.js';
+import { isLowInformationImage } from './guard.js';
+import { ensureServer, stopServer, sendVisionRequest, defaultVlmConfig, isVlmConfigured } from './vlm.js';
+
+const CORE_URL = new URL('./core.js', import.meta.url).href;
+let coreCache = { url: null, mtime: -1, module: null };
+
+/**
+ * Load the latest core.js module with cache-busting.
+ * @returns {Promise<object>} the core module namespace.
+ */
+async function importCore() {
+  const { stat } = await import('node:fs/promises');
+  const url = new URL(CORE_URL);
+  const info = await stat(url);
+  if (coreCache.module !== null && coreCache.url === CORE_URL && info.mtimeMs === coreCache.mtime) {
+    return coreCache.module;
+  }
+  const module = await import(`${url.href}?t=${info.mtimeMs}`);
+  coreCache = { url: CORE_URL, mtime: info.mtimeMs, module };
+  return module;
+}
+
+/**
+ * Parse a boolean argument with fallback.
+ * @param {any} value - the argument value.
+ * @param {boolean} fallback - default value.
+ * @returns {boolean} parsed boolean.
+ */
+function boolArg(value, fallback = false) {
+  if (value === undefined || value === null) return fallback;
+  if (typeof value === 'boolean') return value;
+  return String(value) === 'true' || String(value) === '1';
+}
+
+/**
+ * Build the vision_analyze tool.
+ * @param {object} ctx - the Cordis context.
+ * @returns {object} the tool definition.
+ */
+export function createVisionAnalyzeTool(ctx) {
+  return {
+    name: 'vision_analyze',
+    description: [
+      'Unified image understanding: decode an image, run a low-information guard, optionally scan pixels, OCR text, and/or ask the VLM for a semantic description.',
+      'Use this when you need one call to both verify what is in the image and get a natural-language interpretation.',
+      'Returns evidence blocks: scan (pixel stats), ocr (real text), vlm (model description). If low-information guard triggers and allow_low_info is false, it will not call the VLM.',
+      'Supported formats: PNG, JPEG, GIF (first frame), BMP. WebP is not supported yet.',
+      'VLM is optional: if SEE_BASE is not configured, VLM calls are skipped automatically.',
+      'Smart API calling: simple images (low color diversity, high dominant color coverage) skip VLM automatically.',
+      'Multiple questions: call this tool multiple times with different prompts on the same image for comprehensive analysis.',
+      'Cross-validation: main model should verify VLM results against pixel scan and OCR evidence.'
+    ].join(' '),
+    parameters: {
+      type: 'object',
+      additionalProperties: true,
+      properties: {
+        file_path: {
+          type: 'string',
+          description: 'Path to the image file (PNG/JPEG/GIF/BMP).'
+        },
+        prompt: {
+          type: 'string',
+          description: 'Question/instruction for the VLM, e.g. "Describe this UI" or "What is wrong with this map rendering?"'
+        },
+        include_scan: {
+          type: 'boolean',
+          description: 'Include pixel scan evidence (default true).'
+        },
+        include_ocr: {
+          type: 'boolean',
+          description: 'Include OCR text evidence (default false; set true when text matters).'
+        },
+        ocr_engine: {
+          type: 'string',
+          enum: ['windows', 'paddle'],
+          description: 'OCR engine: windows (default) or paddle (better for glowing/curved/game text).'
+        },
+        include_vlm: {
+          type: 'boolean',
+          description: 'Include VLM description (default true, but skipped if SEE_BASE not configured).'
+        },
+        allow_low_info: {
+          type: 'boolean',
+          description: 'Skip the low-information guard and force VLM even on blank/simple images (default false).'
+        },
+        stop_after: {
+          type: 'boolean',
+          description: 'Stop the local llama-server after this call if this plugin started it (default false).'
+        }
+      },
+      required: ['file_path']
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: true,
+        properties: {
+          path: { type: 'string' },
+          lowInformation: { type: 'boolean' },
+          message: { type: 'string' },
+          scan: { type: 'string' },
+          ocr: { type: 'string' },
+          vlm: { type: 'string' },
+          combined: { type: 'string' }
+        },
+        required: ['path']
+      },
+      render: (_args, value) => {
+        const text = value.combined ?? value.message ?? JSON.stringify(value);
+        return [{ type: 'text', text }];
+      }
+    },
+    isConcurrencySafe: () => true,
+    async execute(args, exec) {
+      if (exec.signal?.aborted) throw new Error('vision_analyze: cancelled');
+      const filePath = String(args.file_path ?? '').trim();
+      if (!filePath) throw new Error('vision_analyze: file_path must be a non-empty string');
+
+      const ext = extname(filePath).toLowerCase();
+      const core = await importCore();
+      if (core.UNSUPPORTED_EXTENSIONS.has(ext)) {
+        throw new Error('vision_analyze: WebP is not supported yet — convert to PNG or JPEG first');
+      }
+      if (!core.IMAGE_EXTENSIONS.has(ext)) {
+        throw new Error(`vision_analyze: unsupported image type "${ext}" (supported: PNG, JPEG, GIF, BMP)`);
+      }
+
+      const cwd = exec.agent?.session?.header?.cwd;
+      const target = await ctx.fs.resolve(filePath, {
+        ...(cwd !== undefined ? { cwd } : {}),
+        signal: exec.signal
+      });
+      const info = await ctx.fs.stat(target, exec.signal);
+      if (!info) {
+        throw new Error(`vision_analyze: cannot read "${target.displayPath}": file not found`);
+      }
+      if (info.type !== 'file') {
+        throw new Error(`vision_analyze: cannot read "${target.displayPath}": not a regular file`);
+      }
+      const data = await ctx.fs.readBytes(target, exec.signal, BYTE_CAP);
+      const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
+      const image = core.decodeImage(buf, ext);
+      if (image.width * image.height > MAX_PIXELS) {
+        throw new Error(
+          `vision_analyze: ${image.width}x${image.height} exceeds the ${MAX_PIXELS}-pixel decode limit — downscale or crop first`
+        );
+      }
+
+      const includeScan = args.include_scan === undefined ? true : boolArg(args.include_scan, true);
+      const includeOcr = args.include_ocr === undefined ? false : boolArg(args.include_ocr, false);
+      const includeVlm = args.include_vlm === undefined ? true : boolArg(args.include_vlm, true);
+      const allowLowInfo = boolArg(args.allow_low_info, false);
+      const stopAfter = boolArg(args.stop_after, false);
+      const prompt = args.prompt ?? 'Describe this image in detail.';
+
+      // Check if VLM is configured
+      const vlmAvailable = isVlmConfigured();
+      const shouldCallVlm = includeVlm && vlmAvailable;
+
+      const lowInfo = isLowInformationImage(image.data, image.width, image.height);
+      const blocks = [];
+      let ocrText = '';
+      let scanText = '';
+      let vlmText = '';
+
+      if (lowInfo && !allowLowInfo) {
+        const message =
+          '[vision_analyze] 低信息量拦截：图片空白或内容极少，为避免 VLM 幻觉，未调用 VLM。' +
+          '请检查截图是否空白/未渲染/窗口在屏幕外；如确需识别请设置 allow_low_info=true。';
+        ctx.emit('fs/observed', target, { kind: 'present', version: info.version }, exec);
+        return { path: target.displayPath, lowInformation: true, message, combined: message };
+      }
+
+      if (includeScan) {
+        const analysis = core.analyzeImage(image.data, image.width, image.height, {
+          size: 32,
+          mode: 'auto',
+          region: undefined,
+          palette: 'auto'
+        });
+        scanText = core.renderImageScan({
+          path: target.displayPath,
+          width: image.width,
+          height: image.height,
+          ...analysis
+        });
+        blocks.push(`[scan]\n${scanText}`);
+      }
+
+      if (includeOcr) {
+        const engine = args.ocr_engine ?? 'windows';
+        const ocr = await core.ocrImage(buf, ext, { engine });
+        ocrText = core.renderOcr({
+          path: target.displayPath,
+          width: ocr.width,
+          height: ocr.height,
+          region: 'full',
+          engine: ocr.engine,
+          lines: ocr.lines
+        });
+        blocks.push(`[ocr]\n${ocrText}`);
+      }
+
+      if (shouldCallVlm) {
+        const config = defaultVlmConfig();
+        let startedByUs = false;
+        try {
+          const child = await ensureServer(config);
+          startedByUs = child !== null;
+          const base64 = buf.toString('base64');
+          const mime = ext === '.jpg' || ext === '.jpeg' ? 'image/jpeg' : ext === '.png' ? 'image/png' : ext === '.gif' ? 'image/gif' : 'image/bmp';
+          const safePrompt =
+            prompt +
+            '\n\n重要：只描述图中明确可见的内容。如果图中没有明显物体/文字/界面元素，请直接回答：画面空白或内容极少。不要推测、不要脑补不存在的角色/场景/文字。';
+          vlmText = await sendVisionRequest(config, [{ mime, base64 }], safePrompt);
+          blocks.push(`[vlm]\n${vlmText}`);
+        } finally {
+          if (stopAfter && startedByUs) {
+            await stopServer();
+          }
+        }
+      } else if (includeVlm && !vlmAvailable) {
+        blocks.push('[vlm] VLM 未配置（SEE_BASE 环境变量为空）');
+      }
+
+      ctx.emit('fs/observed', target, { kind: 'present', version: info.version }, exec);
+      const combined = blocks.join('\n\n---\n\n');
+      return {
+        path: target.displayPath,
+        lowInformation: false,
+        ...(scanText ? { scan: scanText } : {}),
+        ...(ocrText ? { ocr: ocrText } : {}),
+        ...(vlmText ? { vlm: vlmText } : {}),
+        combined
+      };
+    }
+  };
+}

package/src/vlm.js

@@ -0,0 +1,269 @@
+/**
+ * VLM (Vision Language Model) bridge for picturereader.
+ *
+ * Talks to any OpenAI-compatible chat-completions endpoint that accepts
+ * image_url data URIs. When the endpoint is a managed local llama-server
+ * and it is not healthy, this module can auto-start it with the configured
+ * multimodal model and (optionally) stop it after the request.
+ *
+ * @module picturereader/vlm
+ */
+
+import { spawn } from 'node:child_process';
+import { stat } from 'node:fs/promises';
+
+// ---------------------------------------------------------------------------
+// Configuration (all via environment variables, defaults are empty/disabled)
+// ---------------------------------------------------------------------------
+
+/** OpenAI-compatible VLM endpoint (empty = VLM disabled). */
+export const DEFAULT_BASE = process.env.SEE_BASE ?? '';
+/** VLM model name. */
+export const DEFAULT_MODEL = process.env.SEE_MODEL ?? '';
+/** Local llama-server executable path. */
+export const DEFAULT_SERVER_EXE = process.env.SEE_SERVER_EXE ?? '';
+/** Local model GGUF path. */
+export const DEFAULT_SERVER_MODEL = process.env.SEE_SERVER_MODEL ?? '';
+/** Vision projector path. */
+export const DEFAULT_SERVER_MMPROJ = process.env.SEE_SERVER_MMPROJ ?? '';
+/** Local server port. */
+export const DEFAULT_PORT = Number(process.env.SEE_SERVER_PORT ?? 8080);
+/** GPU layers for local server. */
+export const DEFAULT_NGL = process.env.SEE_SERVER_NGL ?? '20';
+/** Context size for local server. */
+export const DEFAULT_CTX = Number(process.env.SEE_SERVER_CTX ?? 16384);
+/** API key for remote endpoints. */
+export const DEFAULT_API_KEY = process.env.SEE_API_KEY ?? '';
+
+let serverStartPromise = null;
+let serverChild = null;
+
+/**
+ * Check if VLM is configured (has a base URL).
+ * @returns {boolean} true when VLM endpoint is configured.
+ */
+export function isVlmConfigured() {
+  return DEFAULT_BASE.length > 0;
+}
+
+/**
+ * Build health check URL from base URL.
+ * @param {string} baseURL - the VLM endpoint base URL.
+ * @returns {string} health check URL.
+ */
+export function healthUrlOf(baseURL) {
+  return baseURL.replace(/\/v1$/, '').replace(/\/+$/, '') + '/health';
+}
+
+/**
+ * Probe VLM endpoint health.
+ * @param {string} baseURL - the VLM endpoint base URL.
+ * @param {number} timeoutMs - timeout in milliseconds.
+ * @returns {Promise<boolean>} true when healthy.
+ */
+export async function probe(baseURL, timeoutMs = 3000) {
+  try {
+    const res = await fetch(healthUrlOf(baseURL), { signal: AbortSignal.timeout(timeoutMs) });
+    return res.ok;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Check if the endpoint is a managed local server.
+ * @param {string} baseURL - the VLM endpoint base URL.
+ * @param {number} port - the expected port.
+ * @returns {boolean} true when it's a managed local endpoint.
+ */
+function isManagedEndpoint(baseURL, port) {
+  const u = baseURL.replace(/\/v1$/, '').replace(/\/+$/, '');
+  const m = u.match(/^http:\/\/(127\.0\.0\.1|localhost):(\d+)$/);
+  return m !== null && Number(m[2]) === port;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Build llama-server command arguments.
+ * @param {object} config - VLM configuration.
+ * @returns {string[]} command arguments.
+ */
+export function buildServerArgs(config) {
+  return [
+    '-m', config.serverModel,
+    '--mmproj', config.serverMmproj,
+    '-ngl', String(config.ngl),
+    '--ctx-size', String(config.ctxSize),
+    '--parallel', '1',
+    '--load-mode', 'none',
+    '--threads', '16',
+    '--threads-batch', '32',
+    '--batch-size', '2048',
+    '--ubatch-size', '512',
+    '--cache-type-k', 'q8_0',
+    '--cache-type-v', 'q8_0',
+    '--flash-attn', 'on',
+    '--fit', 'off',
+    '--split-mode', 'none',
+    '--main-gpu', '0',
+    '--prio', '1',
+    '--jinja',
+    '--reasoning', 'on',
+    '--image-min-tokens', '1024',
+    '--alias', config.model,
+    '--host', '127.0.0.1',
+    '--port', String(config.serverPort),
+  ];
+}
+
+/**
+ * Start local llama-server.
+ * @param {object} config - VLM configuration.
+ * @returns {Promise<ChildProcess>} the server process.
+ */
+async function startLocalServer(config) {
+  for (const p of [config.serverExe, config.serverModel, config.serverMmproj]) {
+    try {
+      await stat(p);
+    } catch {
+      throw new Error(`picturereader: local server file not found: ${p}`);
+    }
+  }
+
+  process.env.GGML_CUDA_NO_PINNED = '1';
+  const child = spawn(config.serverExe, buildServerArgs(config), {
+    detached: true,
+    stdio: 'ignore',
+    windowsHide: true,
+  });
+  child.unref();
+  serverChild = child;
+
+  const deadline = Date.now() + config.healthTimeoutMs;
+  while (Date.now() < deadline) {
+    if (await probe(config.baseURL, 2000)) return child;
+    if (child.exitCode !== null || child.signalCode !== null) break;
+    await sleep(1000);
+  }
+  try {
+    child.kill('SIGKILL');
+  } catch {}
+  throw new Error(
+    `picturereader: local llama-server failed to become healthy at ${healthUrlOf(config.baseURL)} within ${config.healthTimeoutMs}ms`,
+  );
+}
+
+/**
+ * Ensure local llama-server is running (auto-start if needed).
+ * @param {object} config - VLM configuration.
+ * @returns {Promise<ChildProcess|null>} the server process, or null if not managed.
+ */
+export async function ensureServer(config) {
+  if (!config.autoStart || !isManagedEndpoint(config.baseURL, config.serverPort)) {
+    return null;
+  }
+  if (await probe(config.baseURL, 3000)) {
+    serverStartPromise = null;
+    return serverChild;
+  }
+  if (serverStartPromise !== null) {
+    try {
+      await serverStartPromise;
+    } catch {
+      serverStartPromise = null;
+    }
+    if (await probe(config.baseURL, 3000)) return serverChild;
+  }
+  serverStartPromise = startLocalServer(config).finally(() => {
+    serverStartPromise = null;
+  });
+  await serverStartPromise;
+  return serverChild;
+}
+
+/**
+ * Stop local llama-server if running.
+ */
+export async function stopServer() {
+  if (serverChild && serverChild.exitCode === null && serverChild.signalCode === null) {
+    try {
+      serverChild.kill('SIGKILL');
+    } catch {}
+  }
+  serverChild = null;
+  serverStartPromise = null;
+}
+
+/**
+ * Send one image-only request to the VLM endpoint.
+ * @param {object} config - VLM configuration.
+ * @param {Array<{mime: string, base64: string}>} images - images to send.
+ * @param {string} prompt - the prompt text.
+ * @returns {Promise<string>} VLM response text.
+ */
+export async function sendVisionRequest(config, images, prompt) {
+  const content = [{ type: 'text', text: prompt }];
+  for (const img of images) {
+    content.push({ type: 'image_url', image_url: { url: `data:${img.mime};base64,${img.base64}` } });
+  }
+
+  const body = {
+    model: config.model,
+    stream: false,
+    messages: [{ role: 'user', content }],
+    max_tokens: config.maxTokens,
+  };
+
+  const headers = {
+    'content-type': 'application/json',
+  };
+  if (config.apiKey) {
+    headers.authorization = `Bearer ${config.apiKey}`;
+  }
+
+  const res = await fetch(`${config.baseURL}/chat/completions`, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(config.requestTimeoutMs),
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`picturereader: VLM HTTP ${res.status}: ${text.slice(0, 300)}`);
+  }
+
+  const json = await res.json();
+  const contentText = json?.choices?.[0]?.message?.content;
+  if (typeof contentText !== 'string' || contentText.length === 0) {
+    throw new Error('picturereader: VLM returned empty content');
+  }
+  return contentText;
+}
+
+/**
+ * Build default VLM configuration.
+ * @param {object} overrides - configuration overrides.
+ * @returns {object} VLM configuration.
+ */
+export function defaultVlmConfig(overrides = {}) {
+  return {
+    baseURL: DEFAULT_BASE,
+    apiKey: DEFAULT_API_KEY,
+    model: DEFAULT_MODEL,
+    serverExe: DEFAULT_SERVER_EXE,
+    serverModel: DEFAULT_SERVER_MODEL,
+    serverMmproj: DEFAULT_SERVER_MMPROJ,
+    serverPort: DEFAULT_PORT,
+    ngl: DEFAULT_NGL,
+    ctxSize: DEFAULT_CTX,
+    autoStart: true,
+    healthTimeoutMs: 120_000,
+    requestTimeoutMs: 300_000,
+    maxTokens: 8192,
+    ...overrides,
+  };
+}