skyloom 1.22.0 → 1.24.0

package/src/gateway/channels/qq.ts

@@ -17,8 +17,8 @@
 
 import * as crypto from 'crypto';
 import { getLogger } from '../../core/logger';
-import { resolveSecret, postJson, TokenCache } from '../helpers';
-import type { ChannelAdapter, MediaAttachment, RawRequest, ReplyTarget, WebhookOutcome } from '../types';
+import { resolveSecret, postJson, loadMedia, TokenCache } from '../helpers';
+import type { ChannelAdapter, MediaAttachment, OutboundMedia, RawRequest, ReplyTarget, WebhookOutcome } from '../types';
 
 const log = getLogger('channel-qq');
 
@@ -147,5 +147,46 @@ export function createQQAdapter(cfg: any, env: NodeJS.ProcessEnv): ChannelAdapte
         throw new Error(`qq send error: ${e?.response?.status || ''} ${String(e?.message || e).slice(0, 120)}`);
       }
     },
+
+    // QQ's v2 rich-media flow takes a URL (the platform fetches it): POST
+    // /files → file_info, then send msg_type:7 referencing that file_info.
+    // Group/C2C only; raw local bytes aren't supported, so the src must be a URL.
+    async sendMedia(target: ReplyTarget, item: OutboundMedia): Promise<void> {
+      if (!/^https?:\/\//i.test(item.src)) {
+        throw new Error('qq sendMedia requires an http(s) URL (platform fetches it)');
+      }
+      const base = target.kind === 'group'
+        ? `https://api.sgroup.qq.com/v2/groups/${target.groupOpenid}`
+        : target.kind === 'c2c'
+          ? `https://api.sgroup.qq.com/v2/users/${target.userOpenid}`
+          : null;
+      if (!base) throw new Error('qq sendMedia unsupported for channel target');
+      const headers = { ...(await authHeaders()), 'Content-Type': 'application/json' };
+      // file_type: 1=image 2=video 3=audio 4=file
+      const fileType = item.kind === 'image' ? 1 : 4;
+      let fileInfo: string;
+      try {
+        const up = await postJson(`${base}/files`, { file_type: fileType, url: item.src, srv_send_msg: false }, { headers });
+        fileInfo = up.file_info;
+      } catch (e: any) {
+        if (e?.response?.status === 401) tokenCache.invalidate();
+        throw new Error(`qq file upload error: ${e?.response?.status || ''} ${String(e?.message || e).slice(0, 120)}`);
+      }
+      const payload: any = { msg_type: 7, media: { file_info: fileInfo } };
+      if (target.msgId) payload.msg_id = target.msgId;
+      await postJson(`${base}/messages`, payload, { headers });
+    },
+
+    async fetchMedia(att: MediaAttachment): Promise<{ data: Buffer; contentType?: string } | null> {
+      // QQ delivers attachments with a direct URL — just download it.
+      if (!att.url) return null;
+      try {
+        const loaded = await loadMedia(att.url);
+        return { data: loaded.data, contentType: loaded.contentType || att.mimeType };
+      } catch (e) {
+        log.warn('qq_media_fetch_failed', { error: String(e) });
+        return null;
+      }
+    },
   };
 }

package/src/gateway/channels/wecom.ts

@@ -16,9 +16,10 @@
  */
 
 import * as crypto from 'crypto';
+import axios from 'axios';
 import { getLogger } from '../../core/logger';
-import { resolveSecret, postJson, getJson, TokenCache } from '../helpers';
-import type { ChannelAdapter, MediaAttachment, RawRequest, ReplyTarget, WebhookOutcome } from '../types';
+import { resolveSecret, postJson, getJson, postMultipart, loadMedia, TokenCache } from '../helpers';
+import type { ChannelAdapter, MediaAttachment, OutboundMedia, RawRequest, ReplyTarget, WebhookOutcome } from '../types';
 
 const log = getLogger('channel-wecom');
 
@@ -147,5 +148,49 @@ export function createWecomAdapter(cfg: any, env: NodeJS.ProcessEnv): ChannelAda
         throw new Error(`wecom send error ${data.errcode}: ${data.errmsg}`);
       }
     },
+
+    async sendMedia(target: ReplyTarget, item: OutboundMedia): Promise<void> {
+      const toUser = target.toUser as string;
+      if (!toUser || !agentId) return;
+      const loaded = await loadMedia(item.src);
+      const accessToken = await tokenCache.get();
+      const type = item.kind === 'image' ? 'image' : 'file';
+      // Upload to the temporary-media store (valid 3 days), then push by media_id.
+      const up = await postMultipart(
+        `https://qyapi.weixin.qq.com/cgi-bin/media/upload?access_token=${encodeURIComponent(accessToken)}&type=${type}`,
+        { media: { data: loaded.data, filename: loaded.filename || (type === 'image' ? 'image.png' : 'file'), contentType: loaded.contentType } },
+      );
+      if (up.errcode && up.errcode !== 0) {
+        if (up.errcode === 42001 || up.errcode === 40014) tokenCache.invalidate();
+        throw new Error(`wecom media upload ${up.errcode}: ${up.errmsg}`);
+      }
+      const mediaId = up.media_id;
+      const body: any = { touser: toUser, msgtype: type, agentid: Number(agentId) };
+      body[type] = { media_id: mediaId };
+      const send = await postJson(
+        `https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=${encodeURIComponent(accessToken)}`,
+        body,
+      );
+      if (send.errcode !== 0) {
+        if (send.errcode === 42001 || send.errcode === 40014) tokenCache.invalidate();
+        throw new Error(`wecom media send ${send.errcode}: ${send.errmsg}`);
+      }
+    },
+
+    async fetchMedia(att: MediaAttachment): Promise<{ data: Buffer; contentType?: string } | null> {
+      if (!att.ref) return null;
+      const accessToken = await tokenCache.get();
+      const res = await axios.get(
+        `https://qyapi.weixin.qq.com/cgi-bin/media/get?access_token=${encodeURIComponent(accessToken)}&media_id=${encodeURIComponent(att.ref)}`,
+        { responseType: 'arraybuffer', timeout: 30000, validateStatus: (s) => s >= 200 && s < 300 },
+      );
+      // An error comes back as JSON, not the binary — detect and bail.
+      const ct = res.headers['content-type'];
+      if (typeof ct === 'string' && ct.includes('application/json')) {
+        log.warn('wecom_media_get_failed', { body: Buffer.from(res.data).toString('utf8').slice(0, 120) });
+        return null;
+      }
+      return { data: Buffer.from(res.data), contentType: typeof ct === 'string' ? ct : undefined };
+    },
   };
 }

package/src/gateway/gateway.ts

@@ -16,8 +16,11 @@ import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { getLogger } from '../core/logger';
 import { createSystemContext } from '../core/factory';
 import { buildAdapters } from './registry';
-import { describeMedia } from './types';
+import { describeMedia, parseReply } from './types';
+import { isSendableSrc } from './helpers';
+import { describeImages } from './vision';
 import type { ChannelAdapter, InboundMessage, RawRequest } from './types';
+import type { LoadedMedia } from './helpers';
 
 const log = getLogger('gateway');
 
@@ -29,11 +32,44 @@ async function readBody(req: IncomingMessage): Promise<Buffer> {
 }
 
 /** Run an agent turn for an inbound message and collect the final text reply. */
-/** Build the agent prompt from an inbound message (text + media description). */
-function buildPrompt(msg: InboundMessage): string {
+/** Build the agent prompt: text + media description + any vision result. */
+function buildPrompt(msg: InboundMessage, canSendMedia: boolean, visionText?: string | null): string {
+  const parts: string[] = [];
   const mediaDesc = describeMedia(msg.media);
-  if (!mediaDesc) return msg.text;
-  return msg.text ? `${msg.text}\n\n(用户还发送了媒体: ${mediaDesc})` : `用户发送了媒体: ${mediaDesc}`;
+  if (msg.text) parts.push(msg.text);
+  if (mediaDesc) parts.push(`(用户发送了媒体: ${mediaDesc})`);
+  if (visionText) parts.push(`(图片内容识别: ${visionText})`);
+  if (canSendMedia) {
+    parts.push('(若需回发图片或文件,在回复中用 Markdown 图片 ![说明](路径或URL) 或 [[file:路径或URL]] 表示,路径可为本地文件或 http(s) 链接。)');
+  }
+  return parts.join('\n\n') || msg.text;
+}
+
+/** Download inbound images and run vision over them. Returns null if disabled. */
+async function visionForMessage(
+  ctx: ReturnType<typeof createSystemContext>,
+  adapter: ChannelAdapter,
+  msg: InboundMessage,
+): Promise<string | null> {
+  const chCfg = ((ctx.config as any).channels || {})[adapter.id] || {};
+  const llmCfg = (ctx.config as any).llm || {};
+  if (chCfg.vision === false) return null;
+  const model = chCfg.visionModel || llmCfg.vision_model || llmCfg.visionModel;
+  if (!model) return null; // vision is opt-in: requires a configured model
+  const images = (msg.media || []).filter((m) => m.kind === 'image');
+  if (!images.length || !adapter.fetchMedia) return null;
+
+  const loaded: LoadedMedia[] = [];
+  for (const att of images.slice(0, 4)) {
+    try {
+      const got = await adapter.fetchMedia(att, msg);
+      if (got) loaded.push({ data: got.data, filename: att.filename || 'image', contentType: got.contentType });
+    } catch (e) {
+      log.warn('vision_fetch_failed', { channel: adapter.id, error: String(e) });
+    }
+  }
+  if (!loaded.length) return null;
+  return describeImages(loaded, { model });
 }
 
 /** Resolve the agent for a channel message. */
@@ -52,16 +88,18 @@ async function dispatch(
   const agent = resolveAgent(ctx, adapter);
   if (!agent) throw new Error('no agent available');
   await agent.init();
-  const prompt = buildPrompt(msg);
+  const visionText = await visionForMessage(ctx, adapter, msg);
+  const prompt = buildPrompt(msg, !!adapter.sendMedia, visionText);
 
   // Streaming path: stream content chunks straight to the adapter (e.g. a Feishu
   // card patched as text arrives). Falls back to collect-then-send otherwise.
   const cfgStreaming = ((ctx.config as any).channels || {})[adapter.id]?.streaming !== false;
   if (adapter.sendStreaming && cfgStreaming) {
+    let full = '';
     async function* contentChunks(): AsyncGenerator<string> {
       try {
         for await (const ev of agent.chatStream(prompt)) {
-          if ((ev as any).type === 'content') yield (ev as any).text as string;
+          if ((ev as any).type === 'content') { const t = (ev as any).text as string; full += t; yield t; }
         }
       } catch (e) {
         log.warn('gateway_agent_failed', { channel: adapter.id, error: String(e) });
@@ -69,6 +107,8 @@ async function dispatch(
       }
     }
     await adapter.sendStreaming(msg.replyTo, contentChunks());
+    // After streaming the text, deliver any media the agent referenced.
+    await deliverMedia(adapter, msg, full);
     return;
   }
 
@@ -81,7 +121,36 @@ async function dispatch(
     log.warn('gateway_agent_failed', { channel: adapter.id, error: String(e) });
     text = `[出错了] ${String(e)}`;
   }
-  await adapter.send(msg.replyTo, text.trim() || '(无回复)');
+  // Non-streaming: split out media so the text message is clean.
+  if (adapter.sendMedia) {
+    const parsed = parseReply(text);
+    await adapter.send(msg.replyTo, parsed.text || '(无回复)');
+    await deliverMedia(adapter, msg, text, parsed.media);
+  } else {
+    await adapter.send(msg.replyTo, text.trim() || '(无回复)');
+  }
+}
+
+/** Upload+send any media the agent referenced in its reply. Best-effort. */
+async function deliverMedia(
+  adapter: ChannelAdapter,
+  msg: InboundMessage,
+  fullText: string,
+  pre?: ReturnType<typeof parseReply>['media'],
+): Promise<void> {
+  if (!adapter.sendMedia) return;
+  const media = pre ?? parseReply(fullText).media;
+  for (const item of media) {
+    if (!isSendableSrc(item.src)) {
+      log.warn('gateway_media_unsendable', { channel: adapter.id, src: item.src });
+      continue;
+    }
+    try {
+      await adapter.sendMedia(msg.replyTo, item);
+    } catch (e) {
+      log.warn('gateway_send_media_failed', { channel: adapter.id, src: item.src, error: String(e) });
+    }
+  }
 }
 
 export interface GatewayOptions {

package/src/gateway/helpers.ts

@@ -5,6 +5,8 @@
  */
 
 import axios from 'axios';
+import * as fs from 'fs';
+import * as path from 'path';
 
 /**
  * Resolve a secret/config value. Accepts a literal string, or an env-ref object
@@ -58,6 +60,64 @@ export async function getJson(
   return res.data;
 }
 
+/** A loaded binary plus its filename, ready to upload. */
+export interface LoadedMedia {
+  data: Buffer;
+  filename: string;
+  contentType?: string;
+}
+
+/**
+ * Load media bytes from a local filesystem path or an http(s) URL. Local paths
+ * are read directly; remote URLs are fetched (capped at 30 MiB to avoid
+ * pulling something huge into memory). Throws if the source can't be loaded.
+ */
+export async function loadMedia(src: string): Promise<LoadedMedia> {
+  if (/^https?:\/\//i.test(src)) {
+    const res = await axios.get(src, {
+      responseType: 'arraybuffer',
+      timeout: 30000,
+      maxContentLength: 30 * 1024 * 1024,
+      validateStatus: (s) => s >= 200 && s < 300,
+    });
+    const urlName = path.basename(new URL(src).pathname) || 'file';
+    const ct = res.headers['content-type'];
+    return {
+      data: Buffer.from(res.data),
+      filename: urlName,
+      contentType: typeof ct === 'string' ? ct : undefined,
+    };
+  }
+  const data = fs.readFileSync(src); // throws ENOENT if missing — caller handles
+  return { data, filename: path.basename(src) };
+}
+
+/** Is this a sendable media source (http(s) URL or an existing local file)? */
+export function isSendableSrc(src: string): boolean {
+  if (/^https?:\/\//i.test(src)) return true;
+  try { return fs.existsSync(src) && fs.statSync(src).isFile(); } catch { return false; }
+}
+
+/** POST multipart/form-data (Node 18+ FormData/Blob), return parsed JSON. */
+export async function postMultipart(
+  url: string,
+  fields: Record<string, string | { data: Buffer; filename: string; contentType?: string }>,
+  opts?: { headers?: Record<string, string>; timeoutMs?: number },
+): Promise<any> {
+  const form = new FormData();
+  for (const [k, v] of Object.entries(fields)) {
+    if (typeof v === 'string') form.append(k, v);
+    else form.append(k, new Blob([v.data], v.contentType ? { type: v.contentType } : undefined), v.filename);
+  }
+  const res = await axios.post(url, form, {
+    headers: { ...(opts?.headers || {}) },
+    timeout: opts?.timeoutMs ?? 30000,
+    maxBodyLength: Infinity,
+    validateStatus: (s) => s >= 200 && s < 300,
+  });
+  return res.data;
+}
+
 /**
  * A small token cache: fetch an access token via `fetcher`, cache it until it
  * is near expiry, and refresh transparently. Channels (Feishu/WeCom) all need

package/src/gateway/qr.ts

@@ -0,0 +1,21 @@
+/**
+ * Terminal QR rendering — a thin, dependency-isolated wrapper over
+ * qrcode-terminal (single file, zero transitive deps). Returns the QR as a
+ * string so callers control where it's written (and so it's testable).
+ */
+
+// qrcode-terminal ships no types; declare the tiny surface we use.
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const qrcode: { generate: (text: string, opts: { small?: boolean }, cb: (s: string) => void) => void } =
+  require('qrcode-terminal');
+
+/** Render `text` as a scannable QR code (compact) into a string. */
+export function renderQR(text: string): string {
+  let out = '';
+  try {
+    qrcode.generate(text, { small: true }, (s: string) => { out = s; });
+  } catch {
+    return '';
+  }
+  return out;
+}

package/src/gateway/setup.ts

@@ -0,0 +1,145 @@
+/**
+ * Channel setup metadata + persistence — the data behind the `sky channels`
+ * wizard. Each channel declares the credential fields it needs, where to create
+ * the bot (a platform console URL we render as a QR for quick mobile access),
+ * and a short how-to. Kept pure/testable; the interactive prompts live in the
+ * CLI, and QR rendering is a thin wrapper over qrcode-terminal.
+ *
+ * Note: Feishu / WeCom / QQ are all official-bot APIs — credentials are created
+ * in each platform's developer console, there is no "scan to log in" the way
+ * personal WeChat works. So the QR here is a convenience link to the console
+ * (scan on your phone → open the console → create the bot → copy the keys), plus
+ * a QR of the gateway callback URL to paste back into the console.
+ */
+
+export interface ChannelField {
+  /** Config key under channels.<id>. */
+  key: string;
+  /** Human label shown in the wizard. */
+  label: string;
+  /** Whether the wizard must collect it (some are optional). */
+  required: boolean;
+  /** Treat as a secret (mask input / store as env-ref suggestion). */
+  secret?: boolean;
+  /** Env var that also supplies this value. */
+  env?: string;
+  /** One-line hint on where to find it. */
+  hint?: string;
+}
+
+export interface ChannelSetupSpec {
+  id: string;
+  name: string;
+  /** Platform console where the bot/app is created (rendered as a QR). */
+  consoleUrl: string;
+  /** Docs link for the full setup walkthrough. */
+  docsUrl?: string;
+  /** Webhook path the platform must call back. */
+  webhookPath: string;
+  /** Ordered credential fields to collect. */
+  fields: ChannelField[];
+  /** Short, numbered how-to shown before collecting fields. */
+  steps: string[];
+}
+
+export const CHANNEL_SETUP: Record<string, ChannelSetupSpec> = {
+  feishu: {
+    id: 'feishu',
+    name: '飞书 / Lark',
+    consoleUrl: 'https://open.feishu.cn/app',
+    docsUrl: 'https://open.feishu.cn/document/home/index',
+    webhookPath: '/webhook/feishu',
+    fields: [
+      { key: 'appId', label: 'App ID', required: true, env: 'FEISHU_APP_ID', hint: '开发者后台 → 凭证与基础信息 → App ID' },
+      { key: 'appSecret', label: 'App Secret', required: true, secret: true, env: 'FEISHU_APP_SECRET', hint: '同页 App Secret' },
+      { key: 'verificationToken', label: 'Verification Token', required: false, secret: true, env: 'FEISHU_VERIFICATION_TOKEN', hint: '事件订阅 → Verification Token(可选)' },
+      { key: 'encryptKey', label: 'Encrypt Key', required: false, secret: true, env: 'FEISHU_ENCRYPT_KEY', hint: '事件订阅 → Encrypt Key(开启加密时填)' },
+    ],
+    steps: [
+      '扫码或打开 https://open.feishu.cn/app 创建「企业自建应用」',
+      '在「凭证与基础信息」复制 App ID / App Secret',
+      '开启「机器人」能力,在「权限管理」添加 im:message 等权限',
+      '「事件订阅」填入下方回调 URL,订阅 im.message.receive_v1',
+    ],
+  },
+  wecom: {
+    id: 'wecom',
+    name: '企业微信 WeCom',
+    consoleUrl: 'https://work.weixin.qq.com/wework_admin/frame',
+    docsUrl: 'https://developer.work.weixin.qq.com/document/path/90664',
+    webhookPath: '/webhook/wecom',
+    fields: [
+      { key: 'corpId', label: 'CorpID（企业ID）', required: true, env: 'WECOM_CORP_ID', hint: '管理后台 → 我的企业 → 企业ID' },
+      { key: 'corpSecret', label: 'Secret（应用Secret）', required: true, secret: true, env: 'WECOM_CORP_SECRET', hint: '应用管理 → 自建应用 → Secret' },
+      { key: 'agentId', label: 'AgentId', required: true, env: 'WECOM_AGENT_ID', hint: '同应用页 AgentId' },
+      { key: 'token', label: 'Token', required: true, secret: true, env: 'WECOM_TOKEN', hint: '应用 → 接收消息 → API 接收 → Token' },
+      { key: 'encodingAesKey', label: 'EncodingAESKey', required: true, secret: true, env: 'WECOM_AES_KEY', hint: '同页 EncodingAESKey(43 位)' },
+    ],
+    steps: [
+      '扫码或打开企业微信管理后台,进入「应用管理 → 自建 → 创建应用」',
+      '复制企业ID、应用 Secret、AgentId',
+      '「接收消息」选 API 接收,设置 Token 与 EncodingAESKey',
+      '把下方回调 URL 填入「URL」,保存时企业微信会回调验证',
+    ],
+  },
+  qq: {
+    id: 'qq',
+    name: 'QQ 机器人',
+    consoleUrl: 'https://q.qq.com/#/app/bot',
+    docsUrl: 'https://bot.q.qq.com/wiki/',
+    webhookPath: '/webhook/qq',
+    fields: [
+      { key: 'appId', label: 'AppID（机器人ID）', required: true, env: 'QQ_BOT_APPID', hint: 'QQ 开放平台 → 机器人 → 开发设置 → AppID' },
+      { key: 'secret', label: 'AppSecret', required: true, secret: true, env: 'QQ_BOT_SECRET', hint: '同页 AppSecret' },
+    ],
+    steps: [
+      '扫码或打开 https://q.qq.com 创建机器人,完成开发者认证',
+      '在「开发设置」复制 AppID 与 AppSecret',
+      '「回调配置」选择 Webhook,填入下方回调 URL',
+      '在沙箱里把机器人加为好友 / 拉进群进行测试',
+    ],
+  },
+};
+
+export const SETUP_CHANNEL_IDS = Object.keys(CHANNEL_SETUP);
+
+/** Build the full webhook callback URL for a channel from a public base. */
+export function callbackUrl(base: string, channelId: string): string {
+  const spec = CHANNEL_SETUP[channelId];
+  if (!spec) return '';
+  const trimmed = base.replace(/\/+$/, '');
+  return `${trimmed}${spec.webhookPath}`;
+}
+
+/**
+ * Persist a channel's collected values into ~/.skyloom/config.yaml under
+ * channels.<id>, merging with any existing block. Secret-looking values are
+ * stored as-is (the file is chmod 0600); callers may instead keep secrets in
+ * env and store an { source: env, id } ref. Returns the config path written.
+ */
+export function saveChannelConfig(
+  channelId: string,
+  values: Record<string, string>,
+  opts?: { configPath?: string },
+): string {
+  const path = require('path');
+  const fs = require('fs');
+  const yaml = require('yaml');
+  const cfgPath = opts?.configPath || path.join(require('os').homedir(), '.skyloom', 'config.yaml');
+  const dir = path.dirname(cfgPath);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+  let cfg: any = {};
+  if (fs.existsSync(cfgPath)) { try { cfg = yaml.parse(fs.readFileSync(cfgPath, 'utf-8')) || {}; } catch { cfg = {}; } }
+  if (!cfg.channels) cfg.channels = {};
+  cfg.channels[channelId] = { ...(cfg.channels[channelId] || {}), ...values, enabled: true };
+  fs.writeFileSync(cfgPath, yaml.stringify(cfg), { encoding: 'utf-8', mode: 0o600 });
+  try { fs.chmodSync(cfgPath, 0o600); } catch { /* best-effort on Windows */ }
+  return cfgPath;
+}
+
+/** Which required fields are still missing from a values map. */
+export function missingRequired(channelId: string, values: Record<string, string>): string[] {
+  const spec = CHANNEL_SETUP[channelId];
+  if (!spec) return [];
+  return spec.fields.filter((f) => f.required && !values[f.key]?.trim()).map((f) => f.key);
+}

package/src/gateway/types.ts

@@ -43,6 +43,49 @@ export interface MediaAttachment {
   url?: string;
 }
 
+/** An outbound media item the agent wants to send (parsed from its reply). */
+export interface OutboundMedia {
+  kind: 'image' | 'file';
+  /** Local filesystem path or http(s) URL to the binary. */
+  src: string;
+  /** Optional caption / alt text. */
+  alt?: string;
+}
+
+/** The result of splitting an agent reply into plain text + outbound media. */
+export interface ParsedReply {
+  text: string;
+  media: OutboundMedia[];
+}
+
+/**
+ * Parse media directives out of an agent's reply so channels can upload+send
+ * them. Recognized forms (stripped from the returned text):
+ *   - Markdown image:  ![alt](src)
+ *   - Explicit image:  [[image:src]]  or  [[image:src|alt]]
+ *   - Explicit file:   [[file:src]]   or  [[file:src|alt]]
+ * `src` is a local path or http(s) URL. Only http(s) and existing local files
+ * are treated as media; anything else is left in the text untouched.
+ */
+export function parseReply(reply: string): ParsedReply {
+  const media: OutboundMedia[] = [];
+  let text = reply;
+
+  // [[image:src|alt]] / [[file:src|alt]]
+  text = text.replace(/\[\[(image|file):([^\]|]+)(?:\|([^\]]*))?\]\]/gi, (_m, kind, src, alt) => {
+    media.push({ kind: kind.toLowerCase() as 'image' | 'file', src: String(src).trim(), alt: alt ? String(alt).trim() : undefined });
+    return '';
+  });
+
+  // Markdown images: ![alt](src)
+  text = text.replace(/!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g, (_m, alt, src) => {
+    media.push({ kind: 'image', src: String(src).trim(), alt: alt ? String(alt).trim() : undefined });
+    return '';
+  });
+
+  return { text: text.replace(/\n{3,}/g, '\n\n').trim(), media };
+}
+
 /** Render a media list into a compact, model-readable description line. */
 export function describeMedia(media: MediaAttachment[] | undefined): string {
   if (!media || media.length === 0) return '';
@@ -119,6 +162,21 @@ export interface ChannelAdapter {
    * should throttle their own updates and tolerate an empty/aborted stream.
    */
   sendStreaming?(target: ReplyTarget, chunks: AsyncIterable<string>): Promise<void>;
+
+  /**
+   * Optional: upload and send an image or file. When an adapter implements this,
+   * the gateway extracts media directives from the agent's reply (parseReply)
+   * and delivers them after the text. Adapters without it simply keep the
+   * media reference in the text.
+   */
+  sendMedia?(target: ReplyTarget, item: OutboundMedia): Promise<void>;
+
+  /**
+   * Optional: download an inbound media attachment's bytes so the gateway can
+   * run vision over an image. `att` is one entry from InboundMessage.media.
+   * Returns the binary or null if it can't be fetched.
+   */
+  fetchMedia?(att: MediaAttachment, msg: InboundMessage): Promise<{ data: Buffer; contentType?: string } | null>;
 }
 
 /** Factory signature: build an adapter from its config block (or null if disabled/misconfigured). */

package/src/gateway/vision.ts

@@ -0,0 +1,78 @@
+/**
+ * Vision describe — turn an inbound image into a text description so the agent
+ * can "see" what the user sent, without rewiring the core text-only LLM loop.
+ *
+ * Self-contained on purpose: a single OpenAI-compatible chat/completions call
+ * with an image_url (base64 data URL) content block. The model + key are
+ * resolved from config.channels.<id>.visionModel / config.llm.vision_model
+ * (default gpt-4o-mini), falling back to env keys the same way the rest of
+ * Skyloom does. If no key/model is available, vision is skipped silently and the
+ * gateway just uses the media description line.
+ */
+
+import axios from 'axios';
+import { getLogger } from '../core/logger';
+import type { LoadedMedia } from './helpers';
+
+const log = getLogger('gateway-vision');
+
+/** OpenAI-compatible base URL for a provider inferred from the model id. */
+function baseUrlFor(model: string): string {
+  const l = model.toLowerCase();
+  if (l.includes('claude')) return 'https://api.anthropic.com/v1'; // not OpenAI-shaped; skipped below
+  if (l.includes('gemini')) return 'https://generativelanguage.googleapis.com/v1beta/openai';
+  if (l.includes('grok') || l.includes('xai')) return 'https://api.x.ai/v1';
+  if (l.includes('qwen') || l.includes('dashscope')) return 'https://dashscope.aliyuncs.com/compatible-mode/v1';
+  return 'https://api.openai.com/v1';
+}
+
+/** Resolve an API key for the vision model from env (best-effort). */
+function keyFor(model: string, env: NodeJS.ProcessEnv): string | undefined {
+  const l = model.toLowerCase();
+  const candidates = l.includes('gemini') ? ['GEMINI_API_KEY', 'GOOGLE_API_KEY']
+    : l.includes('grok') || l.includes('xai') ? ['XAI_API_KEY']
+    : l.includes('qwen') || l.includes('dashscope') ? ['DASHSCOPE_API_KEY', 'QWEN_API_KEY']
+    : ['OPENAI_API_KEY'];
+  for (const c of candidates) if (env[c]) return env[c];
+  return undefined;
+}
+
+export interface VisionOptions {
+  model?: string;
+  env?: NodeJS.ProcessEnv;
+  prompt?: string;
+}
+
+/**
+ * Describe one or more images. Returns a description string, or null if vision
+ * is unavailable (no key/model) or fails — callers fall back to the media line.
+ */
+export async function describeImages(images: LoadedMedia[], opts: VisionOptions = {}): Promise<string | null> {
+  if (!images.length) return null;
+  const env = opts.env || process.env;
+  const model = opts.model || 'gpt-4o-mini';
+  // Anthropic isn't OpenAI-chat-shaped here; skip to keep this helper simple.
+  if (model.toLowerCase().includes('claude')) return null;
+  const key = keyFor(model, env);
+  if (!key) return null;
+
+  const prompt = opts.prompt || '请用中文简洁描述这些图片的内容(关键物体、文字、场景);如果含可读文字请转写出来。';
+  const content: any[] = [{ type: 'text', text: prompt }];
+  for (const img of images.slice(0, 4)) {
+    const mime = img.contentType || 'image/png';
+    content.push({ type: 'image_url', image_url: { url: `data:${mime};base64,${img.data.toString('base64')}` } });
+  }
+
+  try {
+    const res = await axios.post(
+      `${baseUrlFor(model)}/chat/completions`,
+      { model, messages: [{ role: 'user', content }], max_tokens: 500, temperature: 0.2 },
+      { headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` }, timeout: 30000, validateStatus: (s) => s >= 200 && s < 300 },
+    );
+    const text = res.data?.choices?.[0]?.message?.content;
+    return typeof text === 'string' && text.trim() ? text.trim() : null;
+  } catch (e) {
+    log.warn('vision_describe_failed', { model, error: String(e).slice(0, 160) });
+    return null;
+  }
+}