weflow-cli 1.10.0 → 1.11.0

package/scripts/export_chat_html.py

@@ -15,7 +15,9 @@ import base64
 import urllib.request
 import urllib.error
 import concurrent.futures
+import shutil
 import struct
+import xml.etree.ElementTree as ET
 import json
 from pathlib import Path
 
@@ -94,6 +96,25 @@ COVER_STATE = {
     'thumb_fetched': 0, 'thumb_cached': 0, 'thumb_skipped': 0,
 }
 
+# 落盘时用的扩展名：按**字节**判（`detect_mime_from_bytes`），不按来源猜 —— 实测同一会话 556 张图里
+# 87 个是 GIF、27 个是 PNG，一律叫 .jpg 会让编辑器按扩展名解码、把动画表情显示成坏图。
+_IMAGE_EXT_BY_MIME = {'image/jpeg': '.jpg', 'image/png': '.png',
+                      'image/gif': '.gif', 'image/webp': '.webp'}
+
+# 文档本体（appmsg `type=6`）从账号目录的 `msg/file/` 拷进 assets。原件是用户自己的文件：不重编码、
+# 扩展名保持原样（编辑器/系统按扩展名决定用什么打开）。上限只防"一个几百 MB 的压缩包把导出变成磁盘炸弹"，
+# 超限的**不静默丢**：计入 `FILE_ASSET_STATE['unavailable']` 并写进清单顶部计数。
+MAX_ASSET_FILE_BYTES = 64 * 1024 * 1024
+_FILE_ASSET_SUBDIR = 'files'
+
+# 文档那半的计数（图片那半的计数在写盘时现算）。放在模块级是为了不改 `write_assets` 的返回形状 ——
+# 它已经有两个调用方（markdown 导出与 `--media`），与 COVER_STATE/VOICE_STATE 是同一套写法。
+#
+# `exported` 与 `written` 是两件事：`written` 只数**这次新拷**的，`exported` 数"这条消息的文档确实在
+# assets 里"。分开是因为第二次跑同一个目录时新拷贝是 0 —— 只报 `written` 会打印成"0 document(s)"，
+# 而实际上 67 份都在，那是给自己制造假象（2026-10-13 真的这么打印过一次）。
+FILE_ASSET_STATE = {'exported': 0, 'written': 0, 'unavailable': 0}
+
 # Off by default: full-resolution originals are 5-24MB / 3000-5700px each, and
 # downscaling ~500 of them costs minutes (PIL decode dominates). The cache's
 # own thumbnails need no processing at all and are what an export normally
@@ -488,6 +509,71 @@ def scan_account_media(account_dir, own_wxid='', talker=''):
     return image_map
 
 
+def normalize_asset_name(name):
+    """把消息里的 `title` 与磁盘文件名归一成同一个键。
+
+    微信写盘时会给自己加一份副本后缀：消息里是 `晚餐聚餐安排（学术研讨会后）.docx`，磁盘上是
+    `晚餐聚餐安排（学术研讨会后）(1).docx`（实测）。半角全角都出现过，所以两种括号都剥。
+    """
+    stem, ext = os.path.splitext(str(name or ''))
+    return (re.sub(r'[\s]*[（(]\d+[)）]$', '', stem).strip().lower() + ext.lower())
+
+
+def scan_account_file_index(account_dir):
+    """`msg/file/**` 的**只 stat 索引**：`{(归一化名, 字节数): [路径]}`。
+
+    为什么不照图片那套按 talker 缩范围：`msg/file` 按月分目录（`msg/file/2024-04/…`），目录里没有
+    talker 的痕迹，一条会话的文件散在所有月份里。好在它只 `stat` 不读内容 —— 真机 1,677 个文件
+    不到一秒。键带上字节数是因为同名而内容不同的情形确实存在；真正的消歧靠 md5，但那要读盘，
+    所以只留给**多候选**那一档（实测 1,855 条文件消息里 154 条落到这档，全部认下）。
+    """
+    index = {}
+    root = os.path.join(account_dir, 'msg', 'file')
+    if not os.path.isdir(root):
+        return index
+    for current_root, _, files in os.walk(root):
+        for fname in files:
+            path = os.path.join(current_root, fname)
+            try:
+                size = os.path.getsize(path)
+            except OSError:
+                continue
+            index.setdefault((normalize_asset_name(fname), size), []).append(path)
+    return index
+
+
+def resolve_account_file(info, file_index):
+    """把一条文件消息对到本地那份文件上；对不上返回 None（不猜，D-014）。
+
+    先按 `(归一化名, 字节数)` 查：真机 1,855 条里 1,489 条唯一命中；多候选才读盘比 md5。整体命中
+    88.6%，剩下的 11.4% 是本地确实没有那份文件（未下载 / 已清理），与图片那半同一种情况。
+    """
+    if not info or not file_index:
+        return None
+    name = normalize_asset_name(info.get('title', ''))
+    size = info.get('bytes') or 0
+    candidates = file_index.get((name, size)) if size else None
+    if candidates is None:
+        # `totallen` 缺失（少数消息没有）时退化成按名字查一遍：只遍历键，不读盘。
+        candidates = [path for (key_name, _size), paths in file_index.items()
+                      if key_name == name for path in paths]
+    if not candidates:
+        return None
+    if len(candidates) == 1:
+        return candidates[0]
+    want = (info.get('md5') or '').lower()
+    if want:
+        for path in candidates:
+            try:
+                with open(path, 'rb') as fh:
+                    if hashlib.md5(fh.read()).hexdigest() == want:
+                        return path
+            except OSError:
+                continue
+        return None
+    return candidates[0]
+
+
 def clean_account_wxid(value):
     value = str(value or '').strip()
     parts = value.rsplit('_', 1)
@@ -1610,7 +1696,49 @@ def sender_matches_account(sender_user_name, own_wxid):
     return False
 
 
-def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display_name='', resource_map=None, own_wxid=''):
+def parse_file_appmsg(content):
+    """顶层 appmsg `type=6`（文件）的元数据，或 None。
+
+    取的是 `<appmsg>` 的**直接子元素**，不是"整段里第一个同名标签"。原因：引用消息（type 57）把被引用
+    的原文整个塞进 `<refermsg>`，而 `<refermsg>` 就在 `<appmsg>` **内部** —— 那份原文里的 `<type>6</type>`
+    /`<title>`/`<appattach>` 与这条消息自己的字段处在同一段文本里，"切到 appmsg 块"和"取第一个匹配"
+    都分不开，只有按层级取才分得开。2026-10-13 首次测量用正则满文档搜，7,083 条"像文件"里绝大多数
+    就是这种引用原文。
+    """
+    if not content:
+        return None
+    match = re.search(r'<appmsg[\s>].*?</appmsg>', str(content), re.S)
+    if not match:
+        return None
+    try:
+        element = ET.fromstring('<root>' + match.group(0) + '</root>').find('appmsg')
+    except ET.ParseError:
+        # 真机 7,878 行里有 2 行解析不了（转义残缺）。返回 None = 当"不是文件消息"：少带出一个文件，
+        # 好过把某份引用原文的名字当成本条文件的名字。
+        return None
+    if element is None:
+        return None
+    fields = {child.tag: (child.text or '').strip() for child in element}
+    if fields.get('type') != '6':
+        return None
+    title = fields.get('title', '')
+    if not title:
+        return None
+    attachment = element.find('appattach')
+    attach = {child.tag: (child.text or '').strip() for child in attachment} if attachment is not None else {}
+    try:
+        total = int(fields.get('totallen') or attach.get('totallen') or '0')
+    except ValueError:
+        total = 0
+    return {
+        'title': title,
+        'ext': (fields.get('fileext') or attach.get('fileext') or '').lower(),
+        'md5': (fields.get('md5') or attach.get('md5') or '').lower(),
+        'bytes': total,
+    }
+
+
+def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display_name='', resource_map=None, own_wxid='', file_index=None):
     """Format a single message for HTML display.
 
     Args:
@@ -1620,6 +1748,8 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
         image_map: {local_id: (base64_data, mime_type)} from NT cache scan
         sender_map: {sender_id: user_name} from Name2Id table
         display_name: human-readable name for the target talker
+        file_index: {(归一化名, 字节数): [路径]} from scan_account_file_index, for
+            resolving a document attachment to the copy on disk
     """
     is_group = '@chatroom' in str(talker or '')
     local_id = row[0] or 0
@@ -1730,6 +1860,12 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
     if '\x00' in content or sum(ord(char) < 32 and char not in '\n\r\t' for char in content) > 2:
         content = ''
 
+    # 文档本体：消息里只有名字与 md5，文件本身在账号目录的 `msg/file/` 下。这里只**解析并定位**，
+    # 不读也不拷 —— 拷贝发生在写 assets 那一步（HTML 那条路根本不落盘：它把图片内嵌成 base64，
+    # 一个几 MB 的 PDF 内嵌进网页既没人打得开、也把文件撑大）。
+    file_info = parse_file_appmsg(content) if local_type == 49 else None
+    file_hit = resolve_account_file(file_info, file_index)
+
     # Determine display content
     display = ''
     image_b64 = None
@@ -1920,19 +2056,24 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
         'local_type': local_type,
         'display': display,
         'image_b64': image_b64,
+        # 文档本体的落点（`path` 为 None = 本地没有那份文件）。写 assets 时才拷。
+        'file': ({'title': file_info['title'], 'ext': file_info['ext'],
+                  'bytes': file_info['bytes'], 'md5': file_info['md5'],
+                  'path': file_hit} if file_info else None),
         # Built here rather than accumulated in module state: format_message
         # runs twice (the prefetch dry sweep and the real pass) and the dry
         # pass is discarded, so a counter would double-count.
         'media': build_media_record(
             local_type, image_b64, image_map, local_id, create_time, content,
-            resource_md5s, metadata_content, bool(server_id)),
+            resource_md5s, metadata_content, bool(server_id),
+            file_info, file_hit),
     }
 
 
 MEDIA_KIND_BY_TYPE = {
     3: 'image', 34: 'voice', 43: 'video-thumb', 47: 'emoji', 49: 'link-cover',
 }
-MEDIA_KINDS = {'image', 'emoji', 'voice', 'video-thumb', 'link-cover'}
+MEDIA_KINDS = {'image', 'emoji', 'voice', 'video-thumb', 'link-cover', 'file'}
 
 
 def build_media_report(items, scope, talker, per_page):
@@ -1980,7 +2121,8 @@ def build_media_report(items, scope, talker, per_page):
 
 
 def build_media_record(local_type, image_b64, image_map, local_id, create_time,
-                       content, resource_md5s, metadata_content, has_server_id):
+                       content, resource_md5s, metadata_content, has_server_id,
+                       file_info=None, file_hit=None):
     """What happened to this message's media, or None for a non-media message.
 
     Derived from the same `resolve_media` the page used, so the report cannot
@@ -1995,6 +2137,19 @@ def build_media_record(local_type, image_b64, image_map, local_id, create_time,
         'mime': None, 'bytes': None, 'sha256': None, 'reason': None,
     }
 
+    if file_info is not None:
+        # 文档本体（appmsg type 6）走的是另一条路：没有缩略图、也不在图片索引里，能不能导出只看
+        # 本地 `msg/file/` 有没有那份文件。不并入图片那条链，否则报告会把"文件没下载"写成"没有可靠标识"。
+        record['kind'] = 'file'
+        record['bytes'] = file_info.get('bytes') or None
+        if file_hit:
+            record['status'] = 'exported'
+            record['mediaKey'] = 'file'
+        else:
+            record['status'] = 'missing'
+            record['reason'] = 'file-not-in-local-storage'
+        return record
+
     if image_b64:
         hit = resolve_media(image_map, local_id, create_time, metadata_content,
                             resource_md5s)
@@ -2293,6 +2448,141 @@ p {{ margin:8px 0; font-size:14px; }}
 </html>'''
 
 
+def _asset_file_name(title, used_names, source, md5):
+    """落盘的文档文件名：保持原样（只清洗非法字符），重名才加 md5 前 8 位。
+
+    不改名是刻意的 —— 那是用户自己的文件名，读者要靠它认东西；微信自己也会加 `(1)`。但两个不同来源
+    的文件可能同名，那种情况必须区分，否则后来者会覆盖先到者、清单指错文件。
+    """
+    safe = sanitize_filename(title).strip().strip('.') or 'file'
+    if used_names.get(safe) in (None, source):
+        return safe
+    stem, ext = os.path.splitext(safe)
+    return '%s_%s%s' % (stem, (md5 or 'copy')[:8], ext)
+
+
+def write_assets(out_dir, file_prefix, formatted, assets_dir=''):
+    """把媒体**落盘**并把 `manifest.json` 写好；返回 `(assets 目录名, manifest, 新写文件数)`。
+
+    HTML 走的是"内嵌 base64 ⇒ 单文件可分享"，而 markdown 与其它格式的用法是喂给编辑器/别的工具 ——
+    那里 `![](data:image/…)` 多数不被认，图片必须是**文件 + 相对路径**。所以这里写文件，并留一份清单
+    （键 `local_id:create_time` → `{path, kind, …}`）：**抽取只做一遍**，json/txt/excel 只是消费它。
+
+    图片与文档本体都进同一个清单：图片来自内存里的 base64，文档是**从 `msg/file/` 拷过来**的（路径在
+    `format_message` 阶段就定好了）。清单 v2 起值是对象（带 `kind`），读取端要兼容 v1 的裸字符串。
+    """
+    import base64
+    import hashlib
+    os.makedirs(out_dir, exist_ok=True)
+    assets = assets_dir or os.path.join(out_dir, file_prefix + '_assets')
+    os.makedirs(assets, exist_ok=True)
+
+    manifest = {}
+    written = 0
+    for item in formatted:
+        blob = item.get('image_b64')
+        if not blob:
+            continue
+        data = base64.b64decode(blob)
+        # 扩展名按**字节**判：实测同一会话 556 张图里 87 个是 GIF、27 个是 PNG，一律叫 .jpg 会让
+        # 编辑器按扩展名解码、把动画表情显示成坏图。
+        mime = detect_mime_from_bytes(data[:32]) or 'image/jpeg'
+        name = 'img_%s_%s%s' % (item.get('local_id'),
+                               hashlib.md5(blob[:64].encode()).hexdigest()[:8],
+                               _IMAGE_EXT_BY_MIME.get(mime, '.jpg'))
+        path = os.path.join(assets, name)
+        if not os.path.exists(path):
+            with open(path, 'wb') as fh:
+                fh.write(data)
+            written += 1
+        # 键带时间戳：**同一个 local_id 会在分片之间重复**（每条会话的表可能存在于多个分片，
+        # id 各片自己编号）—— 只按 local_id 做键，json/txt/excel 会拿别人的图。
+        manifest['%s:%s' % (item.get('local_id'), item.get('create_time'))] = {
+            'path': os.path.relpath(path, out_dir).replace('\\', '/'),
+            'kind': 'image',
+        }
+
+    file_exported = 0
+    file_written = 0
+    file_unavailable = 0
+    files_dir = os.path.join(assets, _FILE_ASSET_SUBDIR)
+    used_names = {}
+    for item in formatted:
+        info = item.get('file')
+        if not info:
+            continue
+        source = info.get('path')
+        size = 0
+        if source:
+            try:
+                size = os.path.getsize(source)
+            except OSError:
+                source = None
+        # 超限与"本地没有"归到同一档计数（都是"这条消息的文档没能带出来"），但不静默：计数写进清单顶部。
+        if not source or size > MAX_ASSET_FILE_BYTES:
+            file_unavailable += 1
+            continue
+        name = _asset_file_name(info.get('title') or os.path.basename(source), used_names, source,
+                                info.get('md5'))
+        target = os.path.join(files_dir, name)
+        if used_names.get(name) != source:
+            os.makedirs(files_dir, exist_ok=True)
+            if not os.path.exists(target):
+                with open(target, 'wb') as dst, open(source, 'rb') as src:
+                    shutil.copyfileobj(src, dst)
+                file_written += 1
+        used_names[name] = source
+        file_exported += 1
+        manifest['%s:%s' % (item.get('local_id'), item.get('create_time'))] = {
+            'path': os.path.relpath(target, out_dir).replace('\\', '/'),
+            'kind': 'file',
+            'name': info.get('title') or name,
+            'bytes': size,
+        }
+
+    FILE_ASSET_STATE['exported'] = file_exported
+    FILE_ASSET_STATE['written'] = file_written
+    FILE_ASSET_STATE['unavailable'] = file_unavailable
+    with open(os.path.join(assets, 'manifest.json'), 'w', encoding='utf-8') as fh:
+        json.dump({'schema': 'weflow-media-manifest/v2', 'messages': manifest,
+                   'documents': {'exported': file_exported, 'unavailable': file_unavailable}},
+                  fh, ensure_ascii=False, indent=2)
+    return os.path.basename(os.path.normpath(assets)), manifest, written + file_written
+
+
+def write_markdown(out_dir, file_prefix, formatted, assets_dir=''):
+    """markdown 正文 + 媒体落盘（落盘那半在 `write_assets`，两者共用同一份清单）。"""
+    assets_name, manifest, written = write_assets(out_dir, file_prefix, formatted, assets_dir)
+
+    lines = ['# %s' % file_prefix, '']
+    for item in formatted:
+        who = '我' if (item.get('is_self') or item.get('is_send')) else (item.get('sender') or '')
+        # 这个文件是 `import datetime`（模块），且它的时间戳一律是**本地时区**（见 render 那几处），
+        # 所以这里也照它的写法，不带 tz —— 导出页里显示的时间与本文件其它时间戳保持一致。
+        stamp = datetime.datetime.fromtimestamp(item.get('create_time') or 0).strftime('%Y-%m-%d %H:%M')
+        body = plain_fragment(item.get('display') or '', limit=100000) or ''
+        record = manifest.get('%s:%s' % (item.get('local_id'), item.get('create_time')))
+        if record:
+            # v1 的值是裸字符串；v2 起是对象。两种都要认（磁盘上可能还留着上一版的清单）。
+            if isinstance(record, dict):
+                rel = record.get('path') or ''
+                kind = record.get('kind') or 'image'
+                label = record.get('name') or ''
+            else:
+                rel, kind, label = record, 'image', ''
+            if rel and kind == 'file':
+                # 文档不是图片：`![](…)` 会被当成坏图，要写成链接，链接文字用原始文件名。
+                keep = '' if (not body or body.startswith('[文件]')) else body + chr(10) + chr(10)
+                body = keep + '[%s](%s)' % (label or os.path.basename(rel), rel)
+            elif rel:
+                body = ('' if (not body or body == '[图片]') else body + chr(10) + chr(10)) + '![图片](%s)' % rel
+        lines.extend(['**%s** %s' % (who, stamp), '', body.strip() or '（空）', ''])
+
+    md_path = os.path.join(out_dir, file_prefix + '.md')
+    with open(md_path, 'w', encoding='utf-8') as fh:
+        fh.write(chr(10).join(lines))
+    return md_path, len(manifest), written, assets_name
+
 def main():
     import argparse
     parser = argparse.ArgumentParser(description='Export WeChat NT chat as HTML')
@@ -2302,6 +2592,12 @@ def main():
     parser.add_argument('--talker', default=os.environ.get('WEFLOW_TALKER'), required=not os.environ.get('WEFLOW_TALKER'), help='Talker username')
     parser.add_argument('--name', default=os.environ.get('WEFLOW_EXPORT_NAME', ''), help='Display name')
     parser.add_argument('--out', default=os.environ.get('WEFLOW_EXPORT_OUTPUT', './output'), help='Output directory')
+    parser.add_argument('--format', choices=('html', 'md'), default='html',
+                        help='html=内嵌图片的多分页页面（默认）；md=markdown + 图片落盘到 assets/')
+    parser.add_argument('--assets-only', action='store_true',
+                        help='只抽取媒体到 assets/ 并写清单，不写 markdown（给 json/txt/excel 的 --media 用）')
+    parser.add_argument('--assets-dir', default='',
+                        help='md 模式下图片落到哪个目录（默认 <out>/<前缀>_assets；同时写 manifest.json）')
     parser.add_argument('--parts', type=int, default=5, help='Number of parts to split into')
     parser.add_argument('--wx-dir', default='', help='Traditional WeChat data dir (FileStorage fallback)')
     parser.add_argument('--cache-dir', default=os.environ.get('WEFLOW_EXPORT_CACHE_DIR', ''), help='NT cache directory for image thumbnails')
@@ -2347,6 +2643,14 @@ def main():
                 value = shrink_embedded(value, max_side=720, force=True)
             image_map[key] = value
     print(f"  Found {len(image_map)} media entries for embedding")
+
+    # 文档本体的索引：只 stat，不读内容（1,677 个文件的走一遍不到一秒）。没有它，`appmsg type=6`
+    # 的消息就只能渲染成 `[文件] 名字` —— 正是反馈里"其余格式不支持导出文档"的那一半。
+    file_index = {}
+    if args.account_dir:
+        file_index = scan_account_file_index(args.account_dir)
+        if file_index:
+            print(f"  Account file index: {len(file_index)} key(s)")
     _phase("media scan")
 
     # Custom stickers: derive the local cache key from the account seed.
@@ -2462,7 +2766,7 @@ def main():
     PREFETCH['sink'] = sink
     try:
         for row in messages:
-            format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid)
+            format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid, file_index)
     finally:
         PREFETCH['sink'] = None
     if sink:
@@ -2483,7 +2787,7 @@ def main():
     for i, row in enumerate(messages):
         if i % progress_step == 0:
             print(f"  Formatting {i}/{len(messages)}...", flush=True)
-        result = format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid)
+        result = format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid, file_index)
         if result.get('image_b64'):
             img_hit_count += 1
             if result.get('local_type') == 49:
@@ -2513,6 +2817,42 @@ def main():
     # Use display name for filename if provided, otherwise fallback to wxid
     file_prefix = sanitize_filename(display_name) if display_name else args.talker.replace('@', '_').replace('/', '_')
 
+    if args.assets_only:
+        # 只抽媒体：其它格式的 `--media` 会先跑这一步，再各自消费清单。
+        assets_name, manifest, asset_new = write_assets(args.out, file_prefix, formatted, args.assets_dir)
+        print(f"Assets: {len(manifest)} media item(s), {FILE_ASSET_STATE['exported']} document(s) "
+              f"({FILE_ASSET_STATE['unavailable']} unavailable, {FILE_ASSET_STATE['written']} new) "
+              f"({asset_new} new) -> {assets_name}")
+        print(json.dumps({"success": True, "total": len(formatted),
+                          "files": [os.path.join(args.out, assets_name)], "format": "assets",
+                          "assets": len(manifest), "assetsNew": asset_new,
+                          "documents": FILE_ASSET_STATE['exported'],
+                          "documentsNew": FILE_ASSET_STATE['written'],
+                          "documentsUnavailable": FILE_ASSET_STATE['unavailable'],
+                          "assetsDir": assets_name}, ensure_ascii=False))
+        conn.close()
+        return
+    if args.format == 'md':
+        md_path, asset_total, asset_new, assets_name = write_markdown(args.out, file_prefix, formatted, args.assets_dir)
+        if media_items:
+            media_report_path = os.path.join(args.out, f'{file_prefix}_media.json')
+            with open(media_report_path, 'w', encoding='utf-8') as handle:
+                json.dump(build_media_report(media_items, display_name, args.talker, len(formatted)),
+                          handle, ensure_ascii=False, indent=2)
+        print(f"Markdown: {len(formatted)} message(s), {asset_total} media item(s), "
+              f"{FILE_ASSET_STATE['exported']} document(s) ({FILE_ASSET_STATE['unavailable']} unavailable, "
+              f"{FILE_ASSET_STATE['written']} new) ({asset_new} new) -> {os.path.basename(md_path)}")
+        print(json.dumps({"success": True, "total": len(formatted), "parts": 1,
+                          "files": [md_path], "format": "md",
+                          "assets": asset_total, "assetsNew": asset_new,
+                          "documents": FILE_ASSET_STATE['exported'],
+                          "documentsNew": FILE_ASSET_STATE['written'],
+                          "documentsUnavailable": FILE_ASSET_STATE['unavailable'],
+                          "assetsDir": assets_name},
+                         ensure_ascii=False))
+        conn.close()
+        return
+
     # Media coverage report. Written beside the HTML parts, under the same
     # prefix, so it is always found next to what it describes.
     media_report = None

package/scripts/semantic_search.py

@@ -182,50 +182,76 @@ def get_name_map(contact_db, contact_key, contact_salt):
     return name_map
 
 
-def collect_chat_messages(conn, name_map, days=90):
-    """Collect chat messages for indexing."""
-    c = conn.cursor()
+def collect_chat_messages(conns, name_map, days=90, per_talker=500):
+    """Collect chat messages for indexing. `conns` 是**分片连接列表**，不是单个连接。
+
+    微信 4.x 的消息按分片存，而且**每片要自己派生的钥匙**才打得开（`nt_decrypt.
+    derive_database_key`）。本机实测四片：`message_3.db` 里是近两天的消息，而配置指向的
+    `message_0.db` 最新一条是 44 天前的。这里原来只开 `ntDbPath` 一个文件、还用裸的全局钥匙 ——
+    于是索引的聊天半边会**静默地收进旧数据**（症状与 issue #10 的"只读到半个月前"同形）。
+    索引是要花钱嵌的，收错数据比报错更贵。
+
+    三处随之改动，各有理由：
+      · 会话清单从所有分片取并集 —— 新会话的表可能只存在于新分片里；
+      · 不再 `[:100]` 截断：本机 122 个会话，而且那 100 个是**无排序取前 100**（等于随机）；
+      · id 里带 `rowid`：实测一张表里 12,106 条消息只落在 12,058 个不同秒上，只按 `talker:ts`
+        去重会静默顶掉同秒的多条。
+    """
     now = datetime.now(TZ)
     start_ts = int((now - timedelta(days=days)).timestamp())
 
-    try:
-        c.execute("SELECT user_name FROM Name2Id WHERE is_session = 1")
-        sessions = [r[0] for r in c.fetchall()]
-    except:
-        c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'Msg_%'")
-        sessions = [r[0] for r in c.fetchall()]
-
-    items = []
-    for talker in sessions[:100]:  # Limit to avoid too many
-        tbl = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
+    sessions = []
+    for conn in conns:
+        c = conn.cursor()
         try:
-            c.execute(f'SELECT COUNT(*) FROM sqlite_master WHERE name="{tbl}"')
-            if c.fetchone()[0] == 0:
+            c.execute("SELECT user_name FROM Name2Id WHERE is_session = 1")
+            rows = [r[0] for r in c.fetchall()]
+        except Exception:
+            try:
+                c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'Msg_%'")
+                rows = [r[0] for r in c.fetchall()]
+            except Exception:
+                rows = []
+        for name in rows:
+            if name not in sessions:
+                sessions.append(name)
+
+    items, seen = [], set()
+    for talker in sessions:
+        tbl = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
+        for conn in conns:
+            c = conn.cursor()
+            try:
+                c.execute('SELECT COUNT(*) FROM sqlite_master WHERE name=?', (tbl,))
+                if c.fetchone()[0] == 0:
+                    continue
+                c.execute(f'''
+                    SELECT rowid, create_time, real_sender_id, message_content
+                    FROM "{tbl}"
+                    WHERE create_time >= ?
+                    ORDER BY create_time DESC
+                    LIMIT ?
+                ''', (start_ts, per_talker))
+                rows = c.fetchall()
+            except Exception:
                 continue
-
-            c.execute(f'''
-                SELECT create_time, real_sender_id, message_content
-                FROM "{tbl}"
-                WHERE create_time >= ?
-                ORDER BY create_time DESC
-                LIMIT 500
-            ''', (start_ts,))
-
-            for ts, sender, content in c.fetchall():
+            for rid, ts, sender, content in rows:
                 if not content or not isinstance(content, str) or len(content) < 10:
                     continue
+                item_id = f"chat:{talker}:{ts}:{rid}"
+                if item_id in seen:
+                    continue
+                seen.add(item_id)
                 sender_name = name_map.get(sender, sender) if sender else name_map.get(talker, talker)
                 dt = datetime.fromtimestamp(ts, tz=TZ)
                 items.append({
-                    "id": f"chat:{talker}:{ts}",
+                    "id": item_id,
                     "type": "chat",
                     "talker": name_map.get(talker, talker),
                     "sender": sender_name,
                     "time": dt.strftime('%Y-%m-%d %H:%M'),
                     "text": content[:500],
                 })
-        except:
-            pass
 
     return items
 
@@ -325,11 +351,25 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
 
     # Collect items
     items = []
+    shards_failed = []
     try:
-        conn = open_db(nt_db, nt_key, nt_salt)
-        chat_items = collect_chat_messages(conn, name_map, days=days)
-        items.extend(chat_items)
-        conn.close()
+        import nt_decrypt  # 懒加载：纯逻辑路径（关键词/重排/测试）不该被它拖住
+        passphrase = decrypt_lock(config.get('favPassphrase') or config.get('decryptKey') or '')
+        pairs, failures = nt_decrypt.connect_message_shards_detailed(
+            nt_db, nt_key, nt_salt, passphrase)
+        shards_failed = [f['name'] for f in failures]
+        if shards_failed:
+            print(f"[WARN] {len(shards_failed)} 个分片读不了: {', '.join(shards_failed)}",
+                  file=sys.stderr)
+        try:
+            chat_items = collect_chat_messages([c for _p, c in pairs], name_map, days=days)
+            items.extend(chat_items)
+        finally:
+            for _p, c in pairs:
+                try:
+                    c.close()
+                except Exception:
+                    pass
     except Exception as e:
         print(f"[WARN] 聊天消息收集失败: {e}", file=sys.stderr)
 
@@ -342,7 +382,8 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
 
     if not new_items:
         return {"status": "up_to_date", "total": len(items),
-                "chatDays": days, "articleDays": article_days}
+                "chatDays": days, "articleDays": article_days,
+                "shardsFailed": shards_failed}
 
     # Generate embeddings
     print("生成 embeddings...", file=sys.stderr)
@@ -381,6 +422,9 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
         "new": len(new_items),
         "chatDays": days,
         "articleDays": article_days,
+        # 分片读不全时**要报出来**：`shardsFailed` 是"全读了"与"读了能读到的"之差，
+        # 正是覆盖度报告存在的理由（nt_decrypt.connect_message_shards_detailed 的原话）。
+        "shardsFailed": shards_failed,
     }
 
 
@@ -501,7 +545,9 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
     try:
         answers, _usage = client.decide(state, questions)
     except Exception as error:
-        print('  [WARN] 重排失败（%s），保持原检索顺序：%s' % (type(error).__name__, error))
+        # `file=sys.stderr`：`--json` 时 stdout 是机器在解析的那一份（同文件其它 WARN 早就这样）
+        print('  [WARN] 重排失败（%s），保持原检索顺序：%s' % (type(error).__name__, error),
+              file=sys.stderr)
         return results
 
     scored = []
@@ -514,7 +560,7 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
         scored.append((value, i, item))
 
     if all(value is None for value, _i, _item in scored):
-        print('  [WARN] 重排没有返回任何可用分数，保持原检索顺序')
+        print('  [WARN] 重排没有返回任何可用分数，保持原检索顺序', file=sys.stderr)
         return results
 
     # 没拿到分数的按原序垫在最后，而不是当成 0 分——"未知"和"不相关"不是一回事。
@@ -523,7 +569,7 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
     top = scored[0]
     if best is not None and best != '候选%d' % top[1]:
         print('  [WARN] 重排自检不一致：逐条打分最高的是候选%d，单选却答 %s'
-              % (top[1], best))
+              % (top[1], best), file=sys.stderr)
 
     ordered = []
     for value, i, item in scored: