skyloom 1.22.0 → 1.23.0

package/src/gateway/gateway.ts

@@ -16,8 +16,11 @@ import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { getLogger } from '../core/logger';
 import { createSystemContext } from '../core/factory';
 import { buildAdapters } from './registry';
-import { describeMedia } from './types';
+import { describeMedia, parseReply } from './types';
+import { isSendableSrc } from './helpers';
+import { describeImages } from './vision';
 import type { ChannelAdapter, InboundMessage, RawRequest } from './types';
+import type { LoadedMedia } from './helpers';
 
 const log = getLogger('gateway');
 
@@ -29,11 +32,44 @@ async function readBody(req: IncomingMessage): Promise<Buffer> {
 }
 
 /** Run an agent turn for an inbound message and collect the final text reply. */
-/** Build the agent prompt from an inbound message (text + media description). */
-function buildPrompt(msg: InboundMessage): string {
+/** Build the agent prompt: text + media description + any vision result. */
+function buildPrompt(msg: InboundMessage, canSendMedia: boolean, visionText?: string | null): string {
+  const parts: string[] = [];
   const mediaDesc = describeMedia(msg.media);
-  if (!mediaDesc) return msg.text;
-  return msg.text ? `${msg.text}\n\n(用户还发送了媒体: ${mediaDesc})` : `用户发送了媒体: ${mediaDesc}`;
+  if (msg.text) parts.push(msg.text);
+  if (mediaDesc) parts.push(`(用户发送了媒体: ${mediaDesc})`);
+  if (visionText) parts.push(`(图片内容识别: ${visionText})`);
+  if (canSendMedia) {
+    parts.push('(若需回发图片或文件,在回复中用 Markdown 图片 ![说明](路径或URL) 或 [[file:路径或URL]] 表示,路径可为本地文件或 http(s) 链接。)');
+  }
+  return parts.join('\n\n') || msg.text;
+}
+
+/** Download inbound images and run vision over them. Returns null if disabled. */
+async function visionForMessage(
+  ctx: ReturnType<typeof createSystemContext>,
+  adapter: ChannelAdapter,
+  msg: InboundMessage,
+): Promise<string | null> {
+  const chCfg = ((ctx.config as any).channels || {})[adapter.id] || {};
+  const llmCfg = (ctx.config as any).llm || {};
+  if (chCfg.vision === false) return null;
+  const model = chCfg.visionModel || llmCfg.vision_model || llmCfg.visionModel;
+  if (!model) return null; // vision is opt-in: requires a configured model
+  const images = (msg.media || []).filter((m) => m.kind === 'image');
+  if (!images.length || !adapter.fetchMedia) return null;
+
+  const loaded: LoadedMedia[] = [];
+  for (const att of images.slice(0, 4)) {
+    try {
+      const got = await adapter.fetchMedia(att, msg);
+      if (got) loaded.push({ data: got.data, filename: att.filename || 'image', contentType: got.contentType });
+    } catch (e) {
+      log.warn('vision_fetch_failed', { channel: adapter.id, error: String(e) });
+    }
+  }
+  if (!loaded.length) return null;
+  return describeImages(loaded, { model });
 }
 
 /** Resolve the agent for a channel message. */
@@ -52,16 +88,18 @@ async function dispatch(
   const agent = resolveAgent(ctx, adapter);
   if (!agent) throw new Error('no agent available');
   await agent.init();
-  const prompt = buildPrompt(msg);
+  const visionText = await visionForMessage(ctx, adapter, msg);
+  const prompt = buildPrompt(msg, !!adapter.sendMedia, visionText);
 
   // Streaming path: stream content chunks straight to the adapter (e.g. a Feishu
   // card patched as text arrives). Falls back to collect-then-send otherwise.
   const cfgStreaming = ((ctx.config as any).channels || {})[adapter.id]?.streaming !== false;
   if (adapter.sendStreaming && cfgStreaming) {
+    let full = '';
     async function* contentChunks(): AsyncGenerator<string> {
       try {
         for await (const ev of agent.chatStream(prompt)) {
-          if ((ev as any).type === 'content') yield (ev as any).text as string;
+          if ((ev as any).type === 'content') { const t = (ev as any).text as string; full += t; yield t; }
         }
       } catch (e) {
         log.warn('gateway_agent_failed', { channel: adapter.id, error: String(e) });
@@ -69,6 +107,8 @@ async function dispatch(
       }
     }
     await adapter.sendStreaming(msg.replyTo, contentChunks());
+    // After streaming the text, deliver any media the agent referenced.
+    await deliverMedia(adapter, msg, full);
     return;
   }
 
@@ -81,7 +121,36 @@ async function dispatch(
     log.warn('gateway_agent_failed', { channel: adapter.id, error: String(e) });
     text = `[出错了] ${String(e)}`;
   }
-  await adapter.send(msg.replyTo, text.trim() || '(无回复)');
+  // Non-streaming: split out media so the text message is clean.
+  if (adapter.sendMedia) {
+    const parsed = parseReply(text);
+    await adapter.send(msg.replyTo, parsed.text || '(无回复)');
+    await deliverMedia(adapter, msg, text, parsed.media);
+  } else {
+    await adapter.send(msg.replyTo, text.trim() || '(无回复)');
+  }
+}
+
+/** Upload+send any media the agent referenced in its reply. Best-effort. */
+async function deliverMedia(
+  adapter: ChannelAdapter,
+  msg: InboundMessage,
+  fullText: string,
+  pre?: ReturnType<typeof parseReply>['media'],
+): Promise<void> {
+  if (!adapter.sendMedia) return;
+  const media = pre ?? parseReply(fullText).media;
+  for (const item of media) {
+    if (!isSendableSrc(item.src)) {
+      log.warn('gateway_media_unsendable', { channel: adapter.id, src: item.src });
+      continue;
+    }
+    try {
+      await adapter.sendMedia(msg.replyTo, item);
+    } catch (e) {
+      log.warn('gateway_send_media_failed', { channel: adapter.id, src: item.src, error: String(e) });
+    }
+  }
 }
 
 export interface GatewayOptions {

package/src/gateway/helpers.ts

@@ -5,6 +5,8 @@
  */
 
 import axios from 'axios';
+import * as fs from 'fs';
+import * as path from 'path';
 
 /**
  * Resolve a secret/config value. Accepts a literal string, or an env-ref object
@@ -58,6 +60,64 @@ export async function getJson(
   return res.data;
 }
 
+/** A loaded binary plus its filename, ready to upload. */
+export interface LoadedMedia {
+  data: Buffer;
+  filename: string;
+  contentType?: string;
+}
+
+/**
+ * Load media bytes from a local filesystem path or an http(s) URL. Local paths
+ * are read directly; remote URLs are fetched (capped at 30 MiB to avoid
+ * pulling something huge into memory). Throws if the source can't be loaded.
+ */
+export async function loadMedia(src: string): Promise<LoadedMedia> {
+  if (/^https?:\/\//i.test(src)) {
+    const res = await axios.get(src, {
+      responseType: 'arraybuffer',
+      timeout: 30000,
+      maxContentLength: 30 * 1024 * 1024,
+      validateStatus: (s) => s >= 200 && s < 300,
+    });
+    const urlName = path.basename(new URL(src).pathname) || 'file';
+    const ct = res.headers['content-type'];
+    return {
+      data: Buffer.from(res.data),
+      filename: urlName,
+      contentType: typeof ct === 'string' ? ct : undefined,
+    };
+  }
+  const data = fs.readFileSync(src); // throws ENOENT if missing — caller handles
+  return { data, filename: path.basename(src) };
+}
+
+/** Is this a sendable media source (http(s) URL or an existing local file)? */
+export function isSendableSrc(src: string): boolean {
+  if (/^https?:\/\//i.test(src)) return true;
+  try { return fs.existsSync(src) && fs.statSync(src).isFile(); } catch { return false; }
+}
+
+/** POST multipart/form-data (Node 18+ FormData/Blob), return parsed JSON. */
+export async function postMultipart(
+  url: string,
+  fields: Record<string, string | { data: Buffer; filename: string; contentType?: string }>,
+  opts?: { headers?: Record<string, string>; timeoutMs?: number },
+): Promise<any> {
+  const form = new FormData();
+  for (const [k, v] of Object.entries(fields)) {
+    if (typeof v === 'string') form.append(k, v);
+    else form.append(k, new Blob([v.data], v.contentType ? { type: v.contentType } : undefined), v.filename);
+  }
+  const res = await axios.post(url, form, {
+    headers: { ...(opts?.headers || {}) },
+    timeout: opts?.timeoutMs ?? 30000,
+    maxBodyLength: Infinity,
+    validateStatus: (s) => s >= 200 && s < 300,
+  });
+  return res.data;
+}
+
 /**
  * A small token cache: fetch an access token via `fetcher`, cache it until it
  * is near expiry, and refresh transparently. Channels (Feishu/WeCom) all need

package/src/gateway/types.ts

@@ -43,6 +43,49 @@ export interface MediaAttachment {
   url?: string;
 }
 
+/** An outbound media item the agent wants to send (parsed from its reply). */
+export interface OutboundMedia {
+  kind: 'image' | 'file';
+  /** Local filesystem path or http(s) URL to the binary. */
+  src: string;
+  /** Optional caption / alt text. */
+  alt?: string;
+}
+
+/** The result of splitting an agent reply into plain text + outbound media. */
+export interface ParsedReply {
+  text: string;
+  media: OutboundMedia[];
+}
+
+/**
+ * Parse media directives out of an agent's reply so channels can upload+send
+ * them. Recognized forms (stripped from the returned text):
+ *   - Markdown image:  ![alt](src)
+ *   - Explicit image:  [[image:src]]  or  [[image:src|alt]]
+ *   - Explicit file:   [[file:src]]   or  [[file:src|alt]]
+ * `src` is a local path or http(s) URL. Only http(s) and existing local files
+ * are treated as media; anything else is left in the text untouched.
+ */
+export function parseReply(reply: string): ParsedReply {
+  const media: OutboundMedia[] = [];
+  let text = reply;
+
+  // [[image:src|alt]] / [[file:src|alt]]
+  text = text.replace(/\[\[(image|file):([^\]|]+)(?:\|([^\]]*))?\]\]/gi, (_m, kind, src, alt) => {
+    media.push({ kind: kind.toLowerCase() as 'image' | 'file', src: String(src).trim(), alt: alt ? String(alt).trim() : undefined });
+    return '';
+  });
+
+  // Markdown images: ![alt](src)
+  text = text.replace(/!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g, (_m, alt, src) => {
+    media.push({ kind: 'image', src: String(src).trim(), alt: alt ? String(alt).trim() : undefined });
+    return '';
+  });
+
+  return { text: text.replace(/\n{3,}/g, '\n\n').trim(), media };
+}
+
 /** Render a media list into a compact, model-readable description line. */
 export function describeMedia(media: MediaAttachment[] | undefined): string {
   if (!media || media.length === 0) return '';
@@ -119,6 +162,21 @@ export interface ChannelAdapter {
    * should throttle their own updates and tolerate an empty/aborted stream.
    */
   sendStreaming?(target: ReplyTarget, chunks: AsyncIterable<string>): Promise<void>;
+
+  /**
+   * Optional: upload and send an image or file. When an adapter implements this,
+   * the gateway extracts media directives from the agent's reply (parseReply)
+   * and delivers them after the text. Adapters without it simply keep the
+   * media reference in the text.
+   */
+  sendMedia?(target: ReplyTarget, item: OutboundMedia): Promise<void>;
+
+  /**
+   * Optional: download an inbound media attachment's bytes so the gateway can
+   * run vision over an image. `att` is one entry from InboundMessage.media.
+   * Returns the binary or null if it can't be fetched.
+   */
+  fetchMedia?(att: MediaAttachment, msg: InboundMessage): Promise<{ data: Buffer; contentType?: string } | null>;
 }
 
 /** Factory signature: build an adapter from its config block (or null if disabled/misconfigured). */

package/src/gateway/vision.ts

@@ -0,0 +1,78 @@
+/**
+ * Vision describe — turn an inbound image into a text description so the agent
+ * can "see" what the user sent, without rewiring the core text-only LLM loop.
+ *
+ * Self-contained on purpose: a single OpenAI-compatible chat/completions call
+ * with an image_url (base64 data URL) content block. The model + key are
+ * resolved from config.channels.<id>.visionModel / config.llm.vision_model
+ * (default gpt-4o-mini), falling back to env keys the same way the rest of
+ * Skyloom does. If no key/model is available, vision is skipped silently and the
+ * gateway just uses the media description line.
+ */
+
+import axios from 'axios';
+import { getLogger } from '../core/logger';
+import type { LoadedMedia } from './helpers';
+
+const log = getLogger('gateway-vision');
+
+/** OpenAI-compatible base URL for a provider inferred from the model id. */
+function baseUrlFor(model: string): string {
+  const l = model.toLowerCase();
+  if (l.includes('claude')) return 'https://api.anthropic.com/v1'; // not OpenAI-shaped; skipped below
+  if (l.includes('gemini')) return 'https://generativelanguage.googleapis.com/v1beta/openai';
+  if (l.includes('grok') || l.includes('xai')) return 'https://api.x.ai/v1';
+  if (l.includes('qwen') || l.includes('dashscope')) return 'https://dashscope.aliyuncs.com/compatible-mode/v1';
+  return 'https://api.openai.com/v1';
+}
+
+/** Resolve an API key for the vision model from env (best-effort). */
+function keyFor(model: string, env: NodeJS.ProcessEnv): string | undefined {
+  const l = model.toLowerCase();
+  const candidates = l.includes('gemini') ? ['GEMINI_API_KEY', 'GOOGLE_API_KEY']
+    : l.includes('grok') || l.includes('xai') ? ['XAI_API_KEY']
+    : l.includes('qwen') || l.includes('dashscope') ? ['DASHSCOPE_API_KEY', 'QWEN_API_KEY']
+    : ['OPENAI_API_KEY'];
+  for (const c of candidates) if (env[c]) return env[c];
+  return undefined;
+}
+
+export interface VisionOptions {
+  model?: string;
+  env?: NodeJS.ProcessEnv;
+  prompt?: string;
+}
+
+/**
+ * Describe one or more images. Returns a description string, or null if vision
+ * is unavailable (no key/model) or fails — callers fall back to the media line.
+ */
+export async function describeImages(images: LoadedMedia[], opts: VisionOptions = {}): Promise<string | null> {
+  if (!images.length) return null;
+  const env = opts.env || process.env;
+  const model = opts.model || 'gpt-4o-mini';
+  // Anthropic isn't OpenAI-chat-shaped here; skip to keep this helper simple.
+  if (model.toLowerCase().includes('claude')) return null;
+  const key = keyFor(model, env);
+  if (!key) return null;
+
+  const prompt = opts.prompt || '请用中文简洁描述这些图片的内容(关键物体、文字、场景);如果含可读文字请转写出来。';
+  const content: any[] = [{ type: 'text', text: prompt }];
+  for (const img of images.slice(0, 4)) {
+    const mime = img.contentType || 'image/png';
+    content.push({ type: 'image_url', image_url: { url: `data:${mime};base64,${img.data.toString('base64')}` } });
+  }
+
+  try {
+    const res = await axios.post(
+      `${baseUrlFor(model)}/chat/completions`,
+      { model, messages: [{ role: 'user', content }], max_tokens: 500, temperature: 0.2 },
+      { headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` }, timeout: 30000, validateStatus: (s) => s >= 200 && s < 300 },
+    );
+    const text = res.data?.choices?.[0]?.message?.content;
+    return typeof text === 'string' && text.trim() ? text.trim() : null;
+  } catch (e) {
+    log.warn('vision_describe_failed', { model, error: String(e).slice(0, 160) });
+    return null;
+  }
+}

package/tests/gateway.test.ts

@@ -1,7 +1,9 @@
 import { describe, it, expect } from "vitest";
 import * as crypto from "crypto";
 import { resolveSecret, TokenCache } from "../src/gateway/helpers";
-import { describeMedia } from "../src/gateway/types";
+import { describeMedia, parseReply } from "../src/gateway/types";
+import { isSendableSrc } from "../src/gateway/helpers";
+import { describeImages } from "../src/gateway/vision";
 import { buildAdapters, SUPPORTED_CHANNELS } from "../src/gateway/registry";
 import { decryptFeishu, createFeishuAdapter } from "../src/gateway/channels/feishu";
 import { wecomSignature, decryptWecom, createWecomAdapter } from "../src/gateway/channels/wecom";
@@ -81,6 +83,87 @@ describe("gateway · media", () => {
   });
 });
 
+describe("gateway · parseReply (outbound media)", () => {
+  it("extracts a markdown image and strips it from the text", () => {
+    const r = parseReply("看这张图 ![猫](https://x.com/cat.png) 好看吧");
+    expect(r.media).toEqual([{ kind: "image", src: "https://x.com/cat.png", alt: "猫" }]);
+    expect(r.text).toContain("看这张图");
+    expect(r.text).not.toContain("![");
+  });
+
+  it("extracts [[image:...]] and [[file:...|alt]] directives", () => {
+    const r = parseReply("结果:\n[[image:/tmp/out.png]]\n[[file:/tmp/report.pdf|季度报告]]");
+    expect(r.media).toEqual([
+      { kind: "image", src: "/tmp/out.png", alt: undefined },
+      { kind: "file", src: "/tmp/report.pdf", alt: "季度报告" },
+    ]);
+    expect(r.text).toBe("结果:");
+  });
+
+  it("leaves text without media untouched", () => {
+    const r = parseReply("就是一段普通文字");
+    expect(r.media).toHaveLength(0);
+    expect(r.text).toBe("就是一段普通文字");
+  });
+
+  it("handles multiple images in one reply", () => {
+    const r = parseReply("![a](http://h/1.png) 和 ![b](http://h/2.png)");
+    expect(r.media.map((m) => m.src)).toEqual(["http://h/1.png", "http://h/2.png"]);
+  });
+});
+
+describe("gateway · isSendableSrc", () => {
+  it("accepts http(s) URLs, rejects bare non-existent paths", () => {
+    expect(isSendableSrc("https://x.com/a.png")).toBe(true);
+    expect(isSendableSrc("http://x.com/a.png")).toBe(true);
+    expect(isSendableSrc("/no/such/file/xyz.png")).toBe(false);
+    expect(isSendableSrc("not a path")).toBe(false);
+  });
+});
+
+describe("gateway · sendMedia capability", () => {
+  it("all three adapters expose sendMedia", () => {
+    const f = createFeishuAdapter({ appId: "a", appSecret: "s" }, {})!;
+    const w = createWecomAdapter({ corpId: "c", corpSecret: "s", token: "t", encodingAesKey: "k".repeat(43), agentId: 1 }, {})!;
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    expect(typeof f.sendMedia).toBe("function");
+    expect(typeof w.sendMedia).toBe("function");
+    expect(typeof q.sendMedia).toBe("function");
+  });
+
+  it("qq sendMedia rejects a non-URL source", async () => {
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    await expect(q.sendMedia!({ channel: "qq", kind: "group", groupOpenid: "g" }, { kind: "image", src: "/local/file.png" }))
+      .rejects.toThrow(/http\(s\) URL/);
+  });
+});
+
+describe("gateway · vision (multimodal read)", () => {
+  it("describeImages returns null with no images", async () => {
+    expect(await describeImages([], { model: "gpt-4o-mini", env: {} })).toBeNull();
+  });
+  it("returns null when no API key is available (skips silently)", async () => {
+    const img = { data: Buffer.from("x"), filename: "a.png", contentType: "image/png" };
+    expect(await describeImages([img], { model: "gpt-4o-mini", env: {} })).toBeNull();
+  });
+  it("skips Anthropic models (not OpenAI-chat-shaped here)", async () => {
+    const img = { data: Buffer.from("x"), filename: "a.png" };
+    expect(await describeImages([img], { model: "claude-sonnet-4-6", env: { ANTHROPIC_API_KEY: "k" } })).toBeNull();
+  });
+  it("all three adapters expose fetchMedia", () => {
+    const f = createFeishuAdapter({ appId: "a", appSecret: "s" }, {})!;
+    const w = createWecomAdapter({ corpId: "c", corpSecret: "s", token: "t", encodingAesKey: "k".repeat(43), agentId: 1 }, {})!;
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    expect(typeof f.fetchMedia).toBe("function");
+    expect(typeof w.fetchMedia).toBe("function");
+    expect(typeof q.fetchMedia).toBe("function");
+  });
+  it("qq fetchMedia returns null without a url", async () => {
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    expect(await q.fetchMedia!({ kind: "image" } as any, {} as any)).toBeNull();
+  });
+});
+
 describe("gateway · registry", () => {
   it("lists the three supported channels", () => {
     expect(SUPPORTED_CHANNELS.sort()).toEqual(["feishu", "qq", "wecom"]);