weflow-cli 1.10.0 → 1.11.1

package/scripts/semantic_search.py

@@ -182,50 +182,76 @@ def get_name_map(contact_db, contact_key, contact_salt):
     return name_map
 
 
-def collect_chat_messages(conn, name_map, days=90):
-    """Collect chat messages for indexing."""
-    c = conn.cursor()
+def collect_chat_messages(conns, name_map, days=90, per_talker=500):
+    """Collect chat messages for indexing. `conns` 是**分片连接列表**，不是单个连接。
+
+    微信 4.x 的消息按分片存，而且**每片要自己派生的钥匙**才打得开（`nt_decrypt.
+    derive_database_key`）。本机实测四片：`message_3.db` 里是近两天的消息，而配置指向的
+    `message_0.db` 最新一条是 44 天前的。这里原来只开 `ntDbPath` 一个文件、还用裸的全局钥匙 ——
+    于是索引的聊天半边会**静默地收进旧数据**（症状与 issue #10 的"只读到半个月前"同形）。
+    索引是要花钱嵌的，收错数据比报错更贵。
+
+    三处随之改动，各有理由：
+      · 会话清单从所有分片取并集 —— 新会话的表可能只存在于新分片里；
+      · 不再 `[:100]` 截断：本机 122 个会话，而且那 100 个是**无排序取前 100**（等于随机）；
+      · id 里带 `rowid`：实测一张表里 12,106 条消息只落在 12,058 个不同秒上，只按 `talker:ts`
+        去重会静默顶掉同秒的多条。
+    """
     now = datetime.now(TZ)
     start_ts = int((now - timedelta(days=days)).timestamp())
 
-    try:
-        c.execute("SELECT user_name FROM Name2Id WHERE is_session = 1")
-        sessions = [r[0] for r in c.fetchall()]
-    except:
-        c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'Msg_%'")
-        sessions = [r[0] for r in c.fetchall()]
-
-    items = []
-    for talker in sessions[:100]:  # Limit to avoid too many
-        tbl = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
+    sessions = []
+    for conn in conns:
+        c = conn.cursor()
         try:
-            c.execute(f'SELECT COUNT(*) FROM sqlite_master WHERE name="{tbl}"')
-            if c.fetchone()[0] == 0:
+            c.execute("SELECT user_name FROM Name2Id WHERE is_session = 1")
+            rows = [r[0] for r in c.fetchall()]
+        except Exception:
+            try:
+                c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'Msg_%'")
+                rows = [r[0] for r in c.fetchall()]
+            except Exception:
+                rows = []
+        for name in rows:
+            if name not in sessions:
+                sessions.append(name)
+
+    items, seen = [], set()
+    for talker in sessions:
+        tbl = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
+        for conn in conns:
+            c = conn.cursor()
+            try:
+                c.execute('SELECT COUNT(*) FROM sqlite_master WHERE name=?', (tbl,))
+                if c.fetchone()[0] == 0:
+                    continue
+                c.execute(f'''
+                    SELECT rowid, create_time, real_sender_id, message_content
+                    FROM "{tbl}"
+                    WHERE create_time >= ?
+                    ORDER BY create_time DESC
+                    LIMIT ?
+                ''', (start_ts, per_talker))
+                rows = c.fetchall()
+            except Exception:
                 continue
-
-            c.execute(f'''
-                SELECT create_time, real_sender_id, message_content
-                FROM "{tbl}"
-                WHERE create_time >= ?
-                ORDER BY create_time DESC
-                LIMIT 500
-            ''', (start_ts,))
-
-            for ts, sender, content in c.fetchall():
+            for rid, ts, sender, content in rows:
                 if not content or not isinstance(content, str) or len(content) < 10:
                     continue
+                item_id = f"chat:{talker}:{ts}:{rid}"
+                if item_id in seen:
+                    continue
+                seen.add(item_id)
                 sender_name = name_map.get(sender, sender) if sender else name_map.get(talker, talker)
                 dt = datetime.fromtimestamp(ts, tz=TZ)
                 items.append({
-                    "id": f"chat:{talker}:{ts}",
+                    "id": item_id,
                     "type": "chat",
                     "talker": name_map.get(talker, talker),
                     "sender": sender_name,
                     "time": dt.strftime('%Y-%m-%d %H:%M'),
                     "text": content[:500],
                 })
-        except:
-            pass
 
     return items
 
@@ -325,11 +351,25 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
 
     # Collect items
     items = []
+    shards_failed = []
     try:
-        conn = open_db(nt_db, nt_key, nt_salt)
-        chat_items = collect_chat_messages(conn, name_map, days=days)
-        items.extend(chat_items)
-        conn.close()
+        import nt_decrypt  # 懒加载：纯逻辑路径（关键词/重排/测试）不该被它拖住
+        passphrase = decrypt_lock(config.get('favPassphrase') or config.get('decryptKey') or '')
+        pairs, failures = nt_decrypt.connect_message_shards_detailed(
+            nt_db, nt_key, nt_salt, passphrase)
+        shards_failed = [f['name'] for f in failures]
+        if shards_failed:
+            print(f"[WARN] {len(shards_failed)} 个分片读不了: {', '.join(shards_failed)}",
+                  file=sys.stderr)
+        try:
+            chat_items = collect_chat_messages([c for _p, c in pairs], name_map, days=days)
+            items.extend(chat_items)
+        finally:
+            for _p, c in pairs:
+                try:
+                    c.close()
+                except Exception:
+                    pass
     except Exception as e:
         print(f"[WARN] 聊天消息收集失败: {e}", file=sys.stderr)
 
@@ -342,7 +382,8 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
 
     if not new_items:
         return {"status": "up_to_date", "total": len(items),
-                "chatDays": days, "articleDays": article_days}
+                "chatDays": days, "articleDays": article_days,
+                "shardsFailed": shards_failed}
 
     # Generate embeddings
     print("生成 embeddings...", file=sys.stderr)
@@ -381,6 +422,9 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
         "new": len(new_items),
         "chatDays": days,
         "articleDays": article_days,
+        # 分片读不全时**要报出来**：`shardsFailed` 是"全读了"与"读了能读到的"之差，
+        # 正是覆盖度报告存在的理由（nt_decrypt.connect_message_shards_detailed 的原话）。
+        "shardsFailed": shards_failed,
     }
 
 
@@ -501,7 +545,9 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
     try:
         answers, _usage = client.decide(state, questions)
     except Exception as error:
-        print('  [WARN] 重排失败（%s），保持原检索顺序：%s' % (type(error).__name__, error))
+        # `file=sys.stderr`：`--json` 时 stdout 是机器在解析的那一份（同文件其它 WARN 早就这样）
+        print('  [WARN] 重排失败（%s），保持原检索顺序：%s' % (type(error).__name__, error),
+              file=sys.stderr)
         return results
 
     scored = []
@@ -514,7 +560,7 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
         scored.append((value, i, item))
 
     if all(value is None for value, _i, _item in scored):
-        print('  [WARN] 重排没有返回任何可用分数，保持原检索顺序')
+        print('  [WARN] 重排没有返回任何可用分数，保持原检索顺序', file=sys.stderr)
         return results
 
     # 没拿到分数的按原序垫在最后，而不是当成 0 分——"未知"和"不相关"不是一回事。
@@ -523,7 +569,7 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
     top = scored[0]
     if best is not None and best != '候选%d' % top[1]:
         print('  [WARN] 重排自检不一致：逐条打分最高的是候选%d，单选却答 %s'
-              % (top[1], best))
+              % (top[1], best), file=sys.stderr)
 
     ordered = []
     for value, i, item in scored:

package/src/services/exportService.ts

@@ -1,5 +1,5 @@
 import { join, basename, dirname } from 'path'
-import { existsSync, mkdirSync, writeFileSync } from 'fs'
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
 import { execFile } from 'child_process'
 import { promisify } from 'util'
 import { chatService } from './chatService.js'
@@ -15,10 +15,67 @@ export type ExportContract = 'raw' | 'weflow-v1'
 export interface ExportResult { success: boolean; path?: string; count?: number; error?: string }
 
 const execFileAsync = promisify(execFile)
+
+/**
+ * md 导出落盘的媒体清单：`<output>/<talker>_assets/manifest.json`（`local_id:create_time` → 条目）。
+ *
+ * 这是"其余格式也能带图/带文件"的那一半：**md 导出抽取媒体，json/txt/excel 消费同一份清单** —— 匹配
+ * 逻辑（缓存缩略图、资源 md5、原图那档、`msg/file` 里的文档本体）只存在于 Python 那条路上，在这里重写
+ * 一遍必然与它漂移。读不到就返回空表：那说明还没跑过 md/`--media` 导出，行为与从前一致。
+ */
+export interface MediaRecord {
+  /** 相对输出目录的路径；`kind: 'file'` 时指向 `files/` 下的文档。 */
+  path: string
+  kind: 'image' | 'file'
+  /** 文档的原始文件名（图片没有）。 */
+  name?: string
+}
+
+/**
+ * 从清单里取这条消息的媒体条目。
+ *
+ * 键是 `local_id:create_time`：**local_id 会在分片之间重复**（同一会话的表可能存在于多个分片，
+ * 各片自己编号），只按 local_id 查会让 json/txt/excel 引用到别的消息的图。两条消息硬要同 id 同秒
+ * 才算撞上，那个概率可以忽略；回退查裸 local_id 是为了兼容更早写出的清单。
+ */
+export function mediaFor(manifest: Record<string, MediaRecord>, message: { localId?: unknown; createTime?: unknown }): MediaRecord | undefined {
+  const id = String(message.localId)
+  return manifest[`${id}:${message.createTime}`] ?? manifest[id]
+}
+
+export function readMediaManifest(outputDir: string, talker: string): Record<string, MediaRecord> {
+  try {
+    const file = join(outputDir, `${talker}_assets`, 'manifest.json')
+    if (!existsSync(file)) return {}
+    const data = JSON.parse(readFileSync(file, 'utf8'))
+    const map = data && data.messages
+    if (!map || typeof map !== 'object') return {}
+    // v1 的值是裸字符串（只有图片），v2 起是带 `kind` 的对象。升级不该让磁盘上已有的清单失效 ——
+    // 用户不会因为升了个版本就重跑一次导出。
+    const normalized: Record<string, MediaRecord> = {}
+    for (const [key, value] of Object.entries(map as Record<string, unknown>)) {
+      if (typeof value === 'string') {
+        normalized[key] = { path: value, kind: 'image' }
+      } else if (value && typeof value === 'object' && typeof (value as MediaRecord).path === 'string') {
+        const record = value as MediaRecord
+        normalized[key] = { ...record, kind: record.kind === 'file' ? 'file' : 'image' }
+      }
+    }
+    return normalized
+  } catch {
+    return {}
+  }
+}
 function getPackageRoot(): string {
   return resolvePackageRoot(import.meta.url)
 }
 
+/** txt/excel 共用的文字标签：文档要带上原名，光一个相对路径看不出那是什么东西。 */
+function mediaLabel(record: MediaRecord | undefined): string {
+  if (!record) return ''
+  return record.kind === 'file' ? `${record.name || record.path} (${record.path})` : record.path
+}
+
 export class ExportService {
   async exportJson(talker: string, outputDir: string, limit = 0, from?: number, to?: number, contract: ExportContract = 'raw'): Promise<ExportResult> {
     try {
@@ -29,13 +86,26 @@ export class ExportService {
 
       const dir = this.ensureOutputDir(outputDir)
       const filePath = join(dir, `${talker}_messages.json`)
+      // 有清单就带上媒体相对路径（清单由 markdown 导出产生，见 readMediaManifest）
+      const mediaMap = readMediaManifest(dir, talker)
+      const withMedia = Object.keys(mediaMap).length
+        ? messages.map(m => {
+          const media = mediaFor(mediaMap, m)
+          // `media` 保持"相对路径字符串"这个老形状（已有消费方按字符串读）；`mediaKind`/`mediaName`
+          // 是新增的 —— 不带这两项，读 JSON 的人只能靠扩展名猜这是图片还是文档，而导出明明知道。
+          if (!media) return m
+          return media.kind === 'file'
+            ? { ...m, media: media.path, mediaKind: media.kind, mediaName: media.name }
+            : { ...m, media: media.path, mediaKind: media.kind }
+        })
+        : messages
       const payload = contract === 'weflow-v1'
-        ? createWeFlowEnvelope(messages, new Date().toISOString(), {
+        ? createWeFlowEnvelope(withMedia, new Date().toISOString(), {
           requestedFrom: from,
           requestedTo: to,
           requestedLimit: limit,
         })
-        : messages
+        : withMedia
       writeFileSync(filePath, JSON.stringify(payload, null, 2), 'utf8')
       return { success: true, path: filePath, count: messages.length }
     } catch {
@@ -50,10 +120,15 @@ export class ExportService {
         return { success: false, error: '未找到消息' }
       }
 
+      const mediaMap = readMediaManifest(this.ensureOutputDir(outputDir), talker)
       const lines = messages.map(m => {
         const time = new Date(m.createTime * 1000).toLocaleString('zh-CN')
         const sender = m.isSend ? '我' : (m.senderUsername || talker)
-        const content = m.parsedContent || m.rawContent || ''
+        // 有清单就把 `[图片]` / `[文件] 名字` 换成能直接打开的引用，没有就保持原样
+        const media = mediaFor(mediaMap, m)
+        const content = media
+          ? (media.kind === 'file' ? `[文件: ${mediaLabel(media)}]` : `[图片: ${media.path}]`)
+          : (m.parsedContent || m.rawContent || '')
         return `[${time}] ${sender}: ${content}`
       })
 
@@ -66,7 +141,12 @@ export class ExportService {
     }
   }
 
-  async exportHtml(talker: string, outputDir: string, limit = 0, date = '', from?: number, to?: number, quiet = false, options: { fullImages?: boolean } = {}): Promise<ExportResult> {
+  async exportHtml(talker: string, outputDir: string, limit = 0, date = '', from?: number, to?: number, quiet = false, options: { fullImages?: boolean; format?: 'html' | 'md'; assetsOnly?: boolean } = {}): Promise<ExportResult> {
+    const format = options.format ?? 'html'
+    if (format === 'md' && (from !== undefined || to !== undefined)) {
+      // 不静默降级成「导出全部」：范围是调用方明确要的，做不到就说出来。
+      return { success: false, error: 'markdown 导出暂不支持 --from/--to：请用 --date，或整段导出' }
+    }
     try {
       if (from !== undefined || to !== undefined) {
         return this.exportHtmlBasic(talker, outputDir, limit, from, to)
@@ -83,6 +163,9 @@ export class ExportService {
       const passphrase = configService.get('favPassphrase') || cfg.decryptKey || ''
 
       if (!db || !key || !salt) {
+        if (format === 'md') {
+          return { success: false, error: 'markdown 导出需要已配置的数据库与密钥（这条路没有简版回退）' }
+        }
         // Fallback: basic HTML
         let fallbackFrom: number | undefined = from
         let fallbackTo: number | undefined = to
@@ -114,11 +197,16 @@ export class ExportService {
         }
       } catch {}
 
-      const args: string[] = [
-        script,
+      const args: string[] = [script]
+      if (format === 'md') {
+        // 图片落盘 + 清单。资产目录按 **talker** 命名，好让 json/txt/excel 导出能确定性找到它
+        // （它们的文件前缀是 talker，而 md 文件名用的是显示名，不显式指定就对不上）。
+        args.push('--format', 'md', '--assets-dir', join(outputDir || './output', `${talker}_assets`))
+        if (options.assetsOnly) args.push('--assets-only')
+      } else {
         // Paginate: a whole year of chat in one file makes the browser crawl.
-        '--per-page', '100',
-      ]
+        args.push('--per-page', '100')
+      }
       const ownWxid = String(cfg.wxid || '').trim()
       const exportCacheDir = cacheDir && existsSync(cacheDir) ? cacheDir : undefined
       const exportAccountDir = accountDir && existsSync(accountDir) ? accountDir : undefined
@@ -189,11 +277,36 @@ export class ExportService {
       if (date) {
         return { success: false, error: `指定日期导出失败: ${failure}` }
       }
-      // Fallback to basic HTML
+      if (format === 'md') {
+        // markdown 没有「简版」：回退成 HTML 会让调用方以为拿到了 .md。失败就是失败。
+        return { success: false, error: failure }
+      }
+      // Fallback to basic HTML：**这条回退不带内嵌图片**，调用方得知道 ——
+      // 2026-10-13 实测：Python 因一个 AttributeError 退出后，这里静默回退，
+      // 而 CLI 照样打印「✓ 导出成功」，人只会以为导出正常。
+      if (!quiet) console.error('  ⚠ 已回退到简版 HTML（不含内嵌图片）')
       return this.exportHtmlBasic(talker, outputDir, limit, from, to)
     }
   }
 
+  /**
+   * 只抽媒体（图片与文档落盘 + 清单），不产出正文 —— `export … json|txt|excel --media` 先跑这一步。
+   * 与 markdown 导出共用同一条 Python 通路与同一个资产目录，所以清单只有一份。
+   *
+   * `limit`/`date` 必须跟着调用方走：抽取原本固定按"整个会话"跑，而消费它的 json/txt/excel 是按
+   * `--limit` 切片的 —— `--limit 200` 于是把整段历史的媒体都抽出来（文档还要拷成文件），
+   * 实测 67 份里只有 33 份落在导出的 200 条里。
+   */
+  async extractMedia(talker: string, outputDir: string, quiet = false, options: { fullImages?: boolean } = {}, limit = 0, date = ''): Promise<ExportResult> {
+    return this.exportHtml(talker, outputDir, limit, date, undefined, undefined, quiet,
+      { ...options, format: 'md', assetsOnly: true })
+  }
+
+  /** markdown + 图片落盘到 `<output>/<talker>_assets/`（同一条 Python 通路，换个输出形态）。 */
+  async exportMarkdown(talker: string, outputDir: string, limit = 0, date = '', from?: number, to?: number, quiet = false, options: { fullImages?: boolean } = {}): Promise<ExportResult> {
+    return this.exportHtml(talker, outputDir, limit, date, from, to, quiet, { ...options, format: 'md' })
+  }
+
   private async exportHtmlBasic(talker: string, outputDir: string, limit = 0, from?: number, to?: number): Promise<ExportResult> {
     const messages = await chatService.getMessagesInRange(talker, limit, from, to)
     if (messages.length === 0) {
@@ -228,16 +341,20 @@ export class ExportService {
         { header: '发送者', key: 'sender', width: 15 },
         { header: '类型', key: 'type', width: 10 },
         { header: '内容', key: 'content', width: 80 },
-        { header: '消息ID', key: 'msgId', width: 15 }
+        { header: '消息ID', key: 'msgId', width: 15 },
+        // 有清单时填相对路径（清单由 markdown 导出产生）
+        { header: '媒体', key: 'media', width: 40 }
       ]
 
+      const mediaMap = readMediaManifest(this.ensureOutputDir(outputDir), talker)
       for (const m of messages) {
         sheet.addRow({
           time: new Date(m.createTime * 1000).toLocaleString('zh-CN'),
           sender: m.isSend ? '我' : (m.senderUsername || talker),
           type: this.getMessageTypeName(m.localType),
           content: m.parsedContent || m.rawContent || '',
-          msgId: m.localId
+          msgId: m.localId,
+          media: mediaLabel(mediaFor(mediaMap, m))
         })
       }