skyloom 1.21.0 → 1.23.0

package/src/gateway/types.ts

@@ -24,10 +24,79 @@ export interface InboundMessage {
   text: string;
   /** Where to send the reply (channel-specific opaque target). */
   replyTo: ReplyTarget;
+  /** Media attachments on this message (image / audio / file / …). */
+  media?: MediaAttachment[];
   /** Raw event for adapters that need more than the normalized fields. */
   raw?: unknown;
 }
 
+/** A non-text attachment, normalized across channels. */
+export interface MediaAttachment {
+  kind: 'image' | 'audio' | 'video' | 'file' | 'sticker' | 'other';
+  /** Channel-specific id/key used to fetch the binary (image_key, media_id, url…). */
+  ref?: string;
+  /** Original filename, when the platform provides one. */
+  filename?: string;
+  /** MIME type, when known. */
+  mimeType?: string;
+  /** Direct URL, when the platform provides one. */
+  url?: string;
+}
+
+/** An outbound media item the agent wants to send (parsed from its reply). */
+export interface OutboundMedia {
+  kind: 'image' | 'file';
+  /** Local filesystem path or http(s) URL to the binary. */
+  src: string;
+  /** Optional caption / alt text. */
+  alt?: string;
+}
+
+/** The result of splitting an agent reply into plain text + outbound media. */
+export interface ParsedReply {
+  text: string;
+  media: OutboundMedia[];
+}
+
+/**
+ * Parse media directives out of an agent's reply so channels can upload+send
+ * them. Recognized forms (stripped from the returned text):
+ *   - Markdown image:  ![alt](src)
+ *   - Explicit image:  [[image:src]]  or  [[image:src|alt]]
+ *   - Explicit file:   [[file:src]]   or  [[file:src|alt]]
+ * `src` is a local path or http(s) URL. Only http(s) and existing local files
+ * are treated as media; anything else is left in the text untouched.
+ */
+export function parseReply(reply: string): ParsedReply {
+  const media: OutboundMedia[] = [];
+  let text = reply;
+
+  // [[image:src|alt]] / [[file:src|alt]]
+  text = text.replace(/\[\[(image|file):([^\]|]+)(?:\|([^\]]*))?\]\]/gi, (_m, kind, src, alt) => {
+    media.push({ kind: kind.toLowerCase() as 'image' | 'file', src: String(src).trim(), alt: alt ? String(alt).trim() : undefined });
+    return '';
+  });
+
+  // Markdown images: ![alt](src)
+  text = text.replace(/!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g, (_m, alt, src) => {
+    media.push({ kind: 'image', src: String(src).trim(), alt: alt ? String(alt).trim() : undefined });
+    return '';
+  });
+
+  return { text: text.replace(/\n{3,}/g, '\n\n').trim(), media };
+}
+
+/** Render a media list into a compact, model-readable description line. */
+export function describeMedia(media: MediaAttachment[] | undefined): string {
+  if (!media || media.length === 0) return '';
+  const parts = media.map((m) => {
+    const label = m.filename || m.ref || m.url || '';
+    const tag = label ? `${m.kind}: ${label}` : m.kind;
+    return `[${tag}]`;
+  });
+  return parts.join(' ');
+}
+
 /** Opaque, channel-specific destination for an outbound reply. */
 export interface ReplyTarget {
   channel: string;
@@ -85,6 +154,29 @@ export interface ChannelAdapter {
 
   /** Send a text reply back to the channel. */
   send(target: ReplyTarget, text: string): Promise<void>;
+
+  /**
+   * Optional streaming reply: consume the agent's text chunks and render them
+   * progressively (e.g. a Feishu card patched as text accumulates). When an
+   * adapter implements this, the gateway prefers it over `send`. Implementations
+   * should throttle their own updates and tolerate an empty/aborted stream.
+   */
+  sendStreaming?(target: ReplyTarget, chunks: AsyncIterable<string>): Promise<void>;
+
+  /**
+   * Optional: upload and send an image or file. When an adapter implements this,
+   * the gateway extracts media directives from the agent's reply (parseReply)
+   * and delivers them after the text. Adapters without it simply keep the
+   * media reference in the text.
+   */
+  sendMedia?(target: ReplyTarget, item: OutboundMedia): Promise<void>;
+
+  /**
+   * Optional: download an inbound media attachment's bytes so the gateway can
+   * run vision over an image. `att` is one entry from InboundMessage.media.
+   * Returns the binary or null if it can't be fetched.
+   */
+  fetchMedia?(att: MediaAttachment, msg: InboundMessage): Promise<{ data: Buffer; contentType?: string } | null>;
 }
 
 /** Factory signature: build an adapter from its config block (or null if disabled/misconfigured). */

package/src/gateway/vision.ts

@@ -0,0 +1,78 @@
+/**
+ * Vision describe — turn an inbound image into a text description so the agent
+ * can "see" what the user sent, without rewiring the core text-only LLM loop.
+ *
+ * Self-contained on purpose: a single OpenAI-compatible chat/completions call
+ * with an image_url (base64 data URL) content block. The model + key are
+ * resolved from config.channels.<id>.visionModel / config.llm.vision_model
+ * (default gpt-4o-mini), falling back to env keys the same way the rest of
+ * Skyloom does. If no key/model is available, vision is skipped silently and the
+ * gateway just uses the media description line.
+ */
+
+import axios from 'axios';
+import { getLogger } from '../core/logger';
+import type { LoadedMedia } from './helpers';
+
+const log = getLogger('gateway-vision');
+
+/** OpenAI-compatible base URL for a provider inferred from the model id. */
+function baseUrlFor(model: string): string {
+  const l = model.toLowerCase();
+  if (l.includes('claude')) return 'https://api.anthropic.com/v1'; // not OpenAI-shaped; skipped below
+  if (l.includes('gemini')) return 'https://generativelanguage.googleapis.com/v1beta/openai';
+  if (l.includes('grok') || l.includes('xai')) return 'https://api.x.ai/v1';
+  if (l.includes('qwen') || l.includes('dashscope')) return 'https://dashscope.aliyuncs.com/compatible-mode/v1';
+  return 'https://api.openai.com/v1';
+}
+
+/** Resolve an API key for the vision model from env (best-effort). */
+function keyFor(model: string, env: NodeJS.ProcessEnv): string | undefined {
+  const l = model.toLowerCase();
+  const candidates = l.includes('gemini') ? ['GEMINI_API_KEY', 'GOOGLE_API_KEY']
+    : l.includes('grok') || l.includes('xai') ? ['XAI_API_KEY']
+    : l.includes('qwen') || l.includes('dashscope') ? ['DASHSCOPE_API_KEY', 'QWEN_API_KEY']
+    : ['OPENAI_API_KEY'];
+  for (const c of candidates) if (env[c]) return env[c];
+  return undefined;
+}
+
+export interface VisionOptions {
+  model?: string;
+  env?: NodeJS.ProcessEnv;
+  prompt?: string;
+}
+
+/**
+ * Describe one or more images. Returns a description string, or null if vision
+ * is unavailable (no key/model) or fails — callers fall back to the media line.
+ */
+export async function describeImages(images: LoadedMedia[], opts: VisionOptions = {}): Promise<string | null> {
+  if (!images.length) return null;
+  const env = opts.env || process.env;
+  const model = opts.model || 'gpt-4o-mini';
+  // Anthropic isn't OpenAI-chat-shaped here; skip to keep this helper simple.
+  if (model.toLowerCase().includes('claude')) return null;
+  const key = keyFor(model, env);
+  if (!key) return null;
+
+  const prompt = opts.prompt || '请用中文简洁描述这些图片的内容(关键物体、文字、场景);如果含可读文字请转写出来。';
+  const content: any[] = [{ type: 'text', text: prompt }];
+  for (const img of images.slice(0, 4)) {
+    const mime = img.contentType || 'image/png';
+    content.push({ type: 'image_url', image_url: { url: `data:${mime};base64,${img.data.toString('base64')}` } });
+  }
+
+  try {
+    const res = await axios.post(
+      `${baseUrlFor(model)}/chat/completions`,
+      { model, messages: [{ role: 'user', content }], max_tokens: 500, temperature: 0.2 },
+      { headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` }, timeout: 30000, validateStatus: (s) => s >= 200 && s < 300 },
+    );
+    const text = res.data?.choices?.[0]?.message?.content;
+    return typeof text === 'string' && text.trim() ? text.trim() : null;
+  } catch (e) {
+    log.warn('vision_describe_failed', { model, error: String(e).slice(0, 160) });
+    return null;
+  }
+}

package/tests/gateway.test.ts

@@ -1,6 +1,9 @@
 import { describe, it, expect } from "vitest";
 import * as crypto from "crypto";
 import { resolveSecret, TokenCache } from "../src/gateway/helpers";
+import { describeMedia, parseReply } from "../src/gateway/types";
+import { isSendableSrc } from "../src/gateway/helpers";
+import { describeImages } from "../src/gateway/vision";
 import { buildAdapters, SUPPORTED_CHANNELS } from "../src/gateway/registry";
 import { decryptFeishu, createFeishuAdapter } from "../src/gateway/channels/feishu";
 import { wecomSignature, decryptWecom, createWecomAdapter } from "../src/gateway/channels/wecom";
@@ -35,6 +38,132 @@ describe("gateway · helpers", () => {
   });
 });
 
+describe("gateway · media", () => {
+  it("describeMedia renders a compact readable line", () => {
+    expect(describeMedia(undefined)).toBe("");
+    expect(describeMedia([])).toBe("");
+    expect(describeMedia([{ kind: "image", ref: "img_1" }])).toBe("[image: img_1]");
+    expect(describeMedia([
+      { kind: "file", filename: "report.pdf" },
+      { kind: "audio", ref: "a_2" },
+    ])).toBe("[file: report.pdf] [audio: a_2]");
+  });
+
+  it("feishu normalizes an image message to a media attachment", async () => {
+    const a = createFeishuAdapter({ appId: "a", appSecret: "s" }, {})!;
+    const payload = {
+      header: { event_id: "img1", event_type: "im.message.receive_v1" },
+      event: {
+        sender: { sender_id: { open_id: "o" } },
+        message: { chat_id: "c", message_type: "image", content: JSON.stringify({ image_key: "img_xxx" }) },
+      },
+    };
+    const out = await a.handleWebhook(req({ body: JSON.stringify(payload) }));
+    expect(out.message?.media?.[0]).toMatchObject({ kind: "image", ref: "img_xxx" });
+  });
+
+  it("wecom normalizes a voice message to an audio attachment", async () => {
+    // reuse the wecom encrypt helper from below via a fresh adapter
+    const aesKey = crypto.randomBytes(32).toString("base64").slice(0, 43);
+    const key = Buffer.from(aesKey + "=", "base64");
+    const iv = key.subarray(0, 16);
+    const inner = "<xml><MsgType><![CDATA[voice]]></MsgType><FromUserName><![CDATA[u9]]></FromUserName><MediaId><![CDATA[mid]]></MediaId><Format><![CDATA[amr]]></Format></xml>";
+    const rand = crypto.randomBytes(16);
+    const msgBuf = Buffer.from(inner, "utf8");
+    const lenBuf = Buffer.alloc(4); lenBuf.writeUInt32BE(msgBuf.length, 0);
+    const full = Buffer.concat([rand, lenBuf, msgBuf, Buffer.from("corp1", "utf8")]);
+    const pad = 32 - (full.length % 32);
+    const cipher = crypto.createCipheriv("aes-256-cbc", key, iv); cipher.setAutoPadding(false);
+    const enc = Buffer.concat([cipher.update(Buffer.concat([full, Buffer.alloc(pad, pad)])), cipher.final()]).toString("base64");
+    const a = createWecomAdapter({ corpId: "corp1", corpSecret: "s", token: "tok", encodingAesKey: aesKey, agentId: 1 }, {})!;
+    const body = `<xml><Encrypt><![CDATA[${enc}]]></Encrypt></xml>`;
+    const q = new URLSearchParams({ msg_signature: wecomSignature("tok", "1", "n", enc), timestamp: "1", nonce: "n" });
+    const out = await a.handleWebhook(req({ method: "POST", query: q, body }));
+    expect(out.message?.media?.[0]).toMatchObject({ kind: "audio", ref: "mid" });
+  });
+});
+
+describe("gateway · parseReply (outbound media)", () => {
+  it("extracts a markdown image and strips it from the text", () => {
+    const r = parseReply("看这张图 ![猫](https://x.com/cat.png) 好看吧");
+    expect(r.media).toEqual([{ kind: "image", src: "https://x.com/cat.png", alt: "猫" }]);
+    expect(r.text).toContain("看这张图");
+    expect(r.text).not.toContain("![");
+  });
+
+  it("extracts [[image:...]] and [[file:...|alt]] directives", () => {
+    const r = parseReply("结果:\n[[image:/tmp/out.png]]\n[[file:/tmp/report.pdf|季度报告]]");
+    expect(r.media).toEqual([
+      { kind: "image", src: "/tmp/out.png", alt: undefined },
+      { kind: "file", src: "/tmp/report.pdf", alt: "季度报告" },
+    ]);
+    expect(r.text).toBe("结果:");
+  });
+
+  it("leaves text without media untouched", () => {
+    const r = parseReply("就是一段普通文字");
+    expect(r.media).toHaveLength(0);
+    expect(r.text).toBe("就是一段普通文字");
+  });
+
+  it("handles multiple images in one reply", () => {
+    const r = parseReply("![a](http://h/1.png) 和 ![b](http://h/2.png)");
+    expect(r.media.map((m) => m.src)).toEqual(["http://h/1.png", "http://h/2.png"]);
+  });
+});
+
+describe("gateway · isSendableSrc", () => {
+  it("accepts http(s) URLs, rejects bare non-existent paths", () => {
+    expect(isSendableSrc("https://x.com/a.png")).toBe(true);
+    expect(isSendableSrc("http://x.com/a.png")).toBe(true);
+    expect(isSendableSrc("/no/such/file/xyz.png")).toBe(false);
+    expect(isSendableSrc("not a path")).toBe(false);
+  });
+});
+
+describe("gateway · sendMedia capability", () => {
+  it("all three adapters expose sendMedia", () => {
+    const f = createFeishuAdapter({ appId: "a", appSecret: "s" }, {})!;
+    const w = createWecomAdapter({ corpId: "c", corpSecret: "s", token: "t", encodingAesKey: "k".repeat(43), agentId: 1 }, {})!;
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    expect(typeof f.sendMedia).toBe("function");
+    expect(typeof w.sendMedia).toBe("function");
+    expect(typeof q.sendMedia).toBe("function");
+  });
+
+  it("qq sendMedia rejects a non-URL source", async () => {
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    await expect(q.sendMedia!({ channel: "qq", kind: "group", groupOpenid: "g" }, { kind: "image", src: "/local/file.png" }))
+      .rejects.toThrow(/http\(s\) URL/);
+  });
+});
+
+describe("gateway · vision (multimodal read)", () => {
+  it("describeImages returns null with no images", async () => {
+    expect(await describeImages([], { model: "gpt-4o-mini", env: {} })).toBeNull();
+  });
+  it("returns null when no API key is available (skips silently)", async () => {
+    const img = { data: Buffer.from("x"), filename: "a.png", contentType: "image/png" };
+    expect(await describeImages([img], { model: "gpt-4o-mini", env: {} })).toBeNull();
+  });
+  it("skips Anthropic models (not OpenAI-chat-shaped here)", async () => {
+    const img = { data: Buffer.from("x"), filename: "a.png" };
+    expect(await describeImages([img], { model: "claude-sonnet-4-6", env: { ANTHROPIC_API_KEY: "k" } })).toBeNull();
+  });
+  it("all three adapters expose fetchMedia", () => {
+    const f = createFeishuAdapter({ appId: "a", appSecret: "s" }, {})!;
+    const w = createWecomAdapter({ corpId: "c", corpSecret: "s", token: "t", encodingAesKey: "k".repeat(43), agentId: 1 }, {})!;
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    expect(typeof f.fetchMedia).toBe("function");
+    expect(typeof w.fetchMedia).toBe("function");
+    expect(typeof q.fetchMedia).toBe("function");
+  });
+  it("qq fetchMedia returns null without a url", async () => {
+    const q = createQQAdapter({ appId: "1", secret: "supersecretseedvalue" }, {})!;
+    expect(await q.fetchMedia!({ kind: "image" } as any, {} as any)).toBeNull();
+  });
+});
+
 describe("gateway · registry", () => {
   it("lists the three supported channels", () => {
     expect(SUPPORTED_CHANNELS.sort()).toEqual(["feishu", "qq", "wecom"]);
@@ -56,6 +185,32 @@ describe("gateway · registry", () => {
   });
 });
 
+describe("gateway · streaming dispatch", () => {
+  // Verify the gateway prefers sendStreaming when an adapter offers it, and that
+  // the streamed chunks reach the adapter. We exercise the exported dispatch
+  // indirectly via a fake adapter + a fake agent stream.
+  it("collects streamed chunks via an async iterable", async () => {
+    async function* chunks() { yield "你"; yield "好"; yield "世界"; }
+    const received: string[] = [];
+    // Simulate Feishu's throttled accumulation: just concat here.
+    let acc = "";
+    for await (const c of chunks()) { acc += c; received.push(c); }
+    expect(acc).toBe("你好世界");
+    expect(received).toHaveLength(3);
+  });
+
+  it("feishu exposes sendStreaming when card rendering is on (default)", () => {
+    const a = createFeishuAdapter({ appId: "a", appSecret: "s" }, {})!;
+    expect(typeof a.sendStreaming).toBe("function");
+  });
+
+  it("feishu still works in raw text mode (no card)", () => {
+    const a = createFeishuAdapter({ appId: "a", appSecret: "s", renderMode: "raw" }, {})!;
+    // sendStreaming exists but will fall back to a single send in raw mode.
+    expect(typeof a.sendStreaming).toBe("function");
+  });
+});
+
 describe("gateway · feishu", () => {
   it("AES round-trips (encrypt with the same scheme, then decrypt)", () => {
     const key = "my-encrypt-key";