weflow-cli 1.16.0 → 1.19.0

package/scripts/imported_chats.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""读 `weflow-cli import qchat` 落下的存储（output/imports/qq/sessions/）。
+
+**单一读者**：chat_notes 与 semantic_search 都从这里取导入会话。两边各解析一遍格式，
+漂了就是"知识卡看得见、搜索看不见"这类静默分叉——`nt_common` 存在的同款理由。
+
+存储形状（写入方是 `src/services/qchatImport.ts`，字段语义对齐 weflow-message/v1）：
+每个会话一个 JSON：schema/session{id,name,type}/importedAt/sourceFiles/messages[]，
+消息字段 serverId(=qchat messageId，全局唯一)/createTime(unix 秒)/isSend(可缺省=
+未知)/senderUsername(QQ 号，敏感)/senderName/messageType(13 稳定名)/parsedContent/
+rawContent。
+
+路径：默认 `output/imports/qq/sessions`（cwd 相对，与 chat_notes 的 OUTPUT_ROOT 同一
+约定）；`WEFLOW_IMPORT_ROOT` 指到别处时用 `<root>/sessions`（测试用，TS 写入方读同
+一个变量）。
+"""
+import json
+import os
+from pathlib import Path
+
+SCHEMA = 'weflow-import/qq-v1'
+
+
+def sessions_dir(root=None):
+    if root is None:
+        root = os.environ.get('WEFLOW_IMPORT_ROOT')
+    if root:
+        return Path(root) / 'sessions'
+    return Path('output/imports/qq/sessions')
+
+
+def sessions(root=None):
+    """全部已导入会话（按导入时间新→旧）。坏文件/非本 schema 的文件跳过不挡路。
+
+    返回的是存储原文（dict），消息已按 createTime 升序。
+    """
+    d = sessions_dir(root)
+    if not d.is_dir():
+        return []
+    out = []
+    for p in sorted(d.glob('*.json')):
+        try:
+            data = json.loads(p.read_text(encoding='utf-8'))
+        except Exception:
+            continue
+        if not isinstance(data, dict):
+            continue
+        if data.get('schema') != SCHEMA or not isinstance(data.get('messages'), list):
+            continue
+        out.append(data)
+    out.sort(key=lambda s: s.get('importedAt') or '', reverse=True)
+    return out
+
+
+def session_messages(sess, days=None, now=None):
+    """一个会话的消息，可选按"最近 days 天"过滤。
+
+    时间语义与 chat_notes/semantic_search 的窗口一致：unix 秒、含当天。
+    """
+    from datetime import datetime, timedelta, timezone
+
+    msgs = [m for m in sess.get('messages', []) if isinstance(m, dict)]
+    if days is None:
+        return msgs
+    tz = timezone(timedelta(hours=8))
+    now = now or datetime.now(tz)
+    cutoff = int((now - timedelta(days=days)).timestamp())
+    return [m for m in msgs if (m.get('createTime') or 0) >= cutoff]

package/scripts/mcp_bridge.py

@@ -24,6 +24,7 @@ except ImportError:
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from _utils import load_config, decrypt_lock, parse_frontmatter
+from nt_common import decode_content
 
 OUTPUT_ROOT = 'output'
 TZ = timezone(timedelta(hours=8))
@@ -196,7 +197,8 @@ def cmd_search_messages(args):
             ''', (start_ts, end_ts))
 
             for ts, sender, content in c.fetchall():
-                if not content or not isinstance(content, str):
+                content = decode_content(content)   # 压缩正文（issue #12）
+                if not content:
                     continue
                 if keyword and keyword.lower() not in content.lower():
                     continue

package/scripts/media_index.py

@@ -0,0 +1,150 @@
+"""媒体精确匹配：用微信自己的 `hardlink.db` 把 md5 直接对到本地文件，不猜目录。
+
+在此之前，找一张图/一个文件只能**扫描整个 msg 目录**再比对，慢且会漏（同 md5 的
+转发件、被改名的副本都不认）。`hardlink.db` 是微信自己的索引：三张表分别记
+image / file / video 的 `md5 → (dir1, dir2, file_name)`，`dir2id` 把这两个 id 映射成
+目录名，路径因此是**确定的**。
+
+实测出来的路径规律（本机 2026-10-26，逐条对过真实文件）：
+
+    msg/attach/<dir2id[dir2]>/<dir2id[dir1]>/Img/<file_name>   图片
+    msg/file/<dir2id[dir1]>/<file_name>                        文件
+    msg/video/<dir2id[dir1]>/<file_name>                       视频
+
+用法：
+    python scripts/media_index.py --md5 <md5前若干位>
+    python scripts/media_index.py --list --type file --limit 20
+    python scripts/media_index.py --json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import sys
+
+SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from _utils import load_config, decrypt_lock  # noqa: E402
+from nt_common import derive_database_key  # noqa: E402
+from export_chat_html import connect  # noqa: E402
+
+#: (表名, 相对 `msg/` 的路径模板)。
+#:
+#: **图片的 dir1/dir2 顺序与另两张表相反**：实测 image 行是 `dir1=会话hash, dir2=月份`
+#: （落在 `attach/<会话>/<月份>/Img/`），而 file/video 行是 `dir1=月份, dir2=0`。
+#: 按统一顺序拼会把图片全指到 `attach/<月份>/<会话>/` 下一个也不存在（2026-10-26 实测）。
+TABLES = {
+    'image': ('image_hardlink_info_v4', 'attach/{d1}/{d2}/Img/{name}'),
+    'file': ('file_hardlink_info_v4', 'file/{d1}/{name}'),
+    'video': ('video_hardlink_info_v4', 'video/{d1}/{name}'),
+}
+
+
+def hardlink_path(cfg):
+    """hardlink.db 与消息库同属 db_storage，用 `ntDbPath` 反推。"""
+    msg = pathlib.Path(cfg.get('ntDbPath') or '')
+    if not msg.name:
+        raise SystemExit('配置里没有 ntDbPath')
+    return msg.parent.parent / 'hardlink' / 'hardlink.db'
+
+
+def account_root(cfg):
+    """`msg/` 的父目录（媒体按这种结构摆放）。"""
+    return pathlib.Path(cfg.get('ntDbPath') or '').parent.parent.parent
+
+
+def open_db(cfg):
+    path = hardlink_path(cfg)
+    if not path.is_file():
+        raise SystemExit(f'找不到 {path}')
+    key = decrypt_lock(cfg.get('ntKey', '') or '')
+    salt = decrypt_lock(cfg.get('ntSalt', '') or '')
+    pw = decrypt_lock(cfg.get('favPassphrase', '') or cfg.get('decryptKey', '') or '')
+    k, s = derive_database_key(str(path), key, salt, pw)
+    return connect(str(path), k, s)
+
+
+def load_rows(conn, kind):
+    table = TABLES[kind][0]
+    cur = conn.cursor()
+    dirs = {rid: name for rid, name in cur.execute('SELECT rowid, username FROM dir2id')}
+    rows = []
+    for md5, typ, name, size, d1, d2 in cur.execute(
+            'SELECT md5, type, file_name, file_size, dir1, dir2 FROM "%s"' % table):
+        rows.append({'md5': md5 or '', 'type': typ, 'file': name or '',
+                     'size': size or 0, 'dir1': dirs.get(d1, ''), 'dir2': dirs.get(d2, ''),
+                     'kind': kind})
+    return rows
+
+
+def resolve(cfg, row):
+    """一行 → 本地绝对路径（不保证文件还在：微信会清理）。"""
+    tmpl = TABLES[row['kind']][1]
+    rel = tmpl.format(d1=row['dir1'], d2=row['dir2'], name=row['file'])
+    return account_root(cfg) / 'msg' / rel
+
+
+def main():
+    # 手跑时 stdout 是本地代码页（本机 GBK），下面的 ✓/✗ 在 GBK 里编不出来会直接抛
+    # UnicodeEncodeError。走 bridge 时父进程已设 PYTHONIOENCODING，两条路表现才会一致。
+    try:
+        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+    except Exception:
+        pass
+    ap = argparse.ArgumentParser(description='媒体精确匹配（读 hardlink.db，本地只读）')
+    ap.add_argument('--md5', default='', help='按 md5 前缀查（不区分大小写）')
+    ap.add_argument('--type', default='all', choices=['all', 'image', 'file', 'video'])
+    ap.add_argument('--list', action='store_true', help='列出行（不给 --md5 时等价于列出）')
+    ap.add_argument('--limit', type=int, default=20)
+    ap.add_argument('--missing-only', action='store_true', help='只列文件已不在本地的')
+    ap.add_argument('--json', action='store_true')
+    args = ap.parse_args()
+
+    cfg = load_config()
+    conn, _cur = open_db(cfg)
+    try:
+        kinds = list(TABLES) if args.type == 'all' else [args.type]
+        rows = []
+        for kind in kinds:
+            rows.extend(load_rows(conn, kind))
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+    if args.md5:
+        needle = args.md5.lower()
+        rows = [r for r in rows if r['md5'].lower().startswith(needle)]
+
+    for r in rows:
+        p = resolve(cfg, r)
+        r['path'] = str(p)
+        r['exists'] = p.is_file()
+    if args.missing_only:
+        rows = [r for r in rows if not r['exists']]
+
+    total = len(rows)
+    rows = rows[:args.limit]
+
+    if args.json:
+        print(json.dumps({'success': True, 'total': total, 'rows': rows},
+                         ensure_ascii=False, indent=2))
+        return
+    if not rows:
+        print('没有匹配的记录')
+        return
+    present = sum(1 for r in rows if r['exists'])
+    print(f'匹配 {total} 条（显示 {len(rows)} 条，其中文件仍在本地的 {present} 条）\n')
+    for r in rows:
+        mark = '✓' if r['exists'] else '✗'
+        print(f"{mark} {r['kind']:<5} {r['md5'][:12]}  {r['file'][:44]:<44} {r['size']:>9} 字节")
+        print(f"    {r['path']}")
+    if present < len(rows):
+        print(f'\n（✗ 表示 hardlink.db 有记录但文件已被微信清理，不是路径算错）')
+
+
+if __name__ == '__main__':
+    main()

package/scripts/nt_common.py

@@ -80,3 +80,35 @@ def table_columns(cursor, table):
     except Exception:
         return set()
     return {row[1] for row in rows}
+
+
+#: zstd frame magic。`message_content` 在 4.x 里**可能被压缩**（issue #12）。
+ZSTD_MAGIC = bytes([0x28, 0xB5, 0x2F, 0xFD])
+
+
+def decode_content(value):
+    """`message_content` → 文本。**所有读消息的脚本都该走这里。**
+
+    三种形态都要接：`str`（老格式，直接返回）、zstd frame（4.x 压缩后存的）、
+    以及普通 bytes。解不出就回空串——调用方拿到的永远是字符串，不需要自己判断类型。
+
+    为什么要一个公共函数：这条规则以前散在五个脚本里，其中四个写的是
+    `if not isinstance(content, str): continue`，于是**压缩正文被整条跳过**——
+    消息不进索引、不进待办、不进年度统计，而且不报错（issue #12 同源）。
+    """
+    if isinstance(value, str):
+        return value
+    if not value:
+        return ''
+    try:
+        data = bytes(value)
+    except Exception:
+        return ''
+    if data[:4] == ZSTD_MAGIC:
+        try:
+            import zstandard
+            return zstandard.ZstdDecompressor().decompress(data).decode('utf-8', 'ignore')
+        except Exception:
+            # 声称是 zstd 但内容坏了：回退成按字节解码，不抛也不丢整条消息。
+            return data.decode('utf-8', 'ignore')
+    return data.decode('utf-8', 'ignore')

package/scripts/nt_decrypt.py

@@ -989,19 +989,13 @@ QUOTE_CLIP = 120
 
 ZSTD_MAGIC = bytes([0x28, 0xB5, 0x2F, 0xFD])
 def _decode_content(raw):
-    """消息内容 → 文本。BLOB 先按 zstd 解压（公众号消息同一套机制）。解不出回空串。"""
-    if isinstance(raw, str):
-        return raw
-    if not raw:
-        return ''
-    data = bytes(raw)
-    if data[:4] == ZSTD_MAGIC:        # zstd frame header
-        try:
-            import zstandard
-            return zstandard.ZstdDecompressor().decompress(data).decode('utf-8', 'ignore')
-        except Exception:
-            return ''
-    return data.decode('utf-8', 'ignore')
+    """消息内容 → 文本。实现在 `nt_common.decode_content`（各脚本共用一份）。
+
+    保留这个名字是为了不动 `non_text_display` 等既有调用点；行为与公共函数一致：
+    str 原样、zstd frame 解压、其它 bytes 按 UTF-8 解码，解不出回空串。
+    """
+    from nt_common import decode_content
+    return decode_content(raw)
 
 
 def _xml_block(xml, tag):
@@ -1117,7 +1111,15 @@ def _message_dict(row, sender_id_map, name_map, own_wxid, is_group=False):
 
     # Parse message_content - TEXT column
     raw_content = row.get('message_content')
-    content = raw_content if isinstance(raw_content, str) else ""
+    # **普通文本也可能是 zstd 压缩的 BLOB**（issue #12）。这里原来只认 `str`，
+    # 于是 `local_type == 1` 的压缩正文被置空、再交给 `non_text_display()`，
+    # 最终输出 `[未识别的消息类型 1]` —— 消息还在，正文没了，下游按
+    # `parsedContent` 做的关键词 / @所有人 筛选就漏掉这些通知。
+    # `_decode_content()` 已经处理三种形态（str / zstd frame / 普通 bytes），复用它即可。
+    if local_type == 1:
+        content = _decode_content(raw_content)
+    else:
+        content = raw_content if isinstance(raw_content, str) else ""
 
     # `content`/`rawContent` stay exactly as stored; only `parsedContent` - the
     # field every consumer reads first - gets the display-ready form.
@@ -1140,7 +1142,9 @@ def _message_dict(row, sender_id_map, name_map, own_wxid, is_group=False):
         "senderDisplay": sender_display,
         "content": content,
         "rawContent": content,
-        "parsedContent": display[:200] if local_type == 1 else display[:200],
+        # 文本不截断：末尾的 `@所有人` 一类关键词正是下游筛选要用的，截掉等于静默漏消息
+        # （issue #12）。非文本仍保留 200 字的展示上限——它们是标签和标题，不是正文。
+        "parsedContent": display if local_type == 1 else display[:200],
     }
 
 
@@ -1275,6 +1279,120 @@ def get_messages(conns, talker, limit=100, offset=0, name_map=None, own_wxid=Non
     return {"messages": collected}
 
 
+def cursor_probe(db_path, key_hex, salt_hex, passphrase='',
+                 tables_per_shard=5, rows_per_table=500):
+    """Measure what a stable incremental cursor would need on this database.
+
+    D-027 keeps `stableCursor: false` until every message backend can honour
+    the same semantics. This probe answers the NT backend's **data-side** half:
+    whether every message table carries the anchor columns, and whether sampled
+    rows actually hold a nonzero `server_id` (the reader's identity falls back
+    to localId+createTime without one, and `local_id` is per-shard, so those
+    rows cannot be resumed across shards). Duplicate server ids are collected
+    across the sample too - an id that appears in two places cannot key a
+    cursor, whatever the column definition says.
+
+    Sampling is bounded and says so (`exhaustive: false`): `tables_per_shard`
+    tables per shard (sorted names, deterministic) read at both rowid ends,
+    `rows_per_table` rows each. A full scan of every shard would cost minutes
+    on a real account; the schema check IS exhaustive - `PRAGMA table_info` is
+    cheap, so every Msg_* table's anchor columns are checked, not sampled.
+
+    Read-only: no write, no network, nothing leaves stdout.
+    """
+    pairs, failures = connect_message_shards_detailed(
+        db_path, key_hex, salt_hex, passphrase)
+    shard_items = []
+    totals = {'tables': 0, 'tablesSampled': 0, 'rowsSampled': 0,
+              'serverIdMissing': 0}
+    anchor_missing_total = 0
+    # server_id(str) -> {(shard, table)} it was sampled in. A set per id keeps
+    # the two rowid ends of one small table from double-counting the same row
+    # as a "duplicate" - both ends see it, but it is one location.
+    id_locations = {}
+
+    for failure in failures:
+        shard_items.append({
+            'name': failure['name'], 'opened': False, 'reason': failure['reason'],
+            'tables': None,
+        })
+
+    for path, conn in pairs:
+        item = {'name': os.path.basename(path), 'opened': True, 'reason': None}
+        try:
+            tables = sorted(msg_tables(conn))
+            item['tables'] = len(tables)
+            totals['tables'] += len(tables)
+
+            # Schema half: exhaustive over every message table.
+            anchor_missing = []
+            for table in tables:
+                cols = table_columns(conn.cursor(), table)
+                missing = [c for c in MESSAGE_ANCHOR_COLUMNS if c not in cols]
+                if missing and missing not in anchor_missing:
+                    anchor_missing.append(missing)
+            anchor_missing_total += len(anchor_missing)
+            item['anchorColumnsMissing'] = anchor_missing
+
+            # Data half: bounded sample, both rowid ends per table. A table
+            # shorter than the bound hands the same rows to both ends - they
+            # are one sample, not two, so rowids are deduplicated per table
+            # (and the same id read at both ends must not look "duplicated").
+            sample_targets = tables[:tables_per_shard]
+            item['tablesSampled'] = len(sample_targets)
+            totals['tablesSampled'] += len(sample_targets)
+            sampled = 0
+            missing_ids = 0
+            for table in sample_targets:
+                c = conn.cursor()
+                seen_rowids = set()
+                table_ids = set()
+                for order in ('ASC', 'DESC'):
+                    c.execute(
+                        'SELECT rowid, server_id FROM "%s" ORDER BY rowid %s LIMIT ?'
+                        % (table, order), (rows_per_table,))
+                    for rowid, server_id in c.fetchall():
+                        if rowid in seen_rowids:
+                            continue
+                        seen_rowids.add(rowid)
+                        sampled += 1
+                        if server_id is None or server_id == 0:
+                            missing_ids += 1
+                        else:
+                            table_ids.add(str(int(server_id)))
+                for sid in table_ids:
+                    id_locations.setdefault(sid, set()).add((item['name'], table))
+            item['rowsSampled'] = sampled
+            item['serverIdMissing'] = missing_ids
+            totals['rowsSampled'] += sampled
+            totals['serverIdMissing'] += missing_ids
+        except Exception as exc:
+            item['reason'] = 'READ_FAILED: %s' % str(exc).split('\n')[0][:120]
+        finally:
+            shard_items.append(item)
+            conn.close()
+
+    duplicates = {sid: locs for sid, locs in id_locations.items() if len(locs) > 1}
+    return {
+        'shards': shard_items,
+        'schemaAnchorMissingTables': anchor_missing_total,
+        'sample': {
+            'tablesPerShard': tables_per_shard,
+            'rowsPerTable': rows_per_table,
+            'exhaustive': False,
+            'tablesTotal': totals['tables'],
+            'tablesSampled': totals['tablesSampled'],
+            'rowsSampled': totals['rowsSampled'],
+            'serverIdMissing': totals['serverIdMissing'],
+            'duplicateServerIds': len(duplicates),
+            'duplicateServerIdSamples': [
+                {'serverId': sid, 'locations': sorted('%s/%s' % loc for loc in locs)}
+                for sid, locs in sorted(duplicates.items())[:20]
+            ],
+        },
+    }
+
+
 def get_contacts(conns, limit=200, keyword=None):
     """Get contacts from NT database, merged across every shard.
 
@@ -1627,10 +1745,23 @@ def main():
     fav_list_parser.add_argument('--key', default=os.environ.get('WEFLOW_NT_KEY'), required=not os.environ.get('WEFLOW_NT_KEY'), help='Key hex (64 chars)')
     fav_list_parser.add_argument('--limit', type=int, default=100)
     fav_list_parser.add_argument('--offset', type=int, default=0)
-    fav_list_parser.add_argument('--keyword', default=os.environ.get('WEFLOW_QUERY_KEYWORD'), help='Search keyword')
+    fav_list_parser.add_argument('--keyword', default=os.environ.get('WEFLOW_QUERY_KEYWORD'), help='Filter by keyword')
     fav_list_parser.add_argument('--type', type=int, dest='fav_type',
                                  help='Filter by type: 1=text 2=image 4=video 5=article 14=chatrecord')
 
+    # cursor-probe command: the data-side half of D-027's stable-cursor answer.
+    cursor_parser = sub.add_parser(
+        'cursor-probe',
+        help='Probe what a stable incremental cursor would need here (read-only, bounded sample)')
+    cursor_parser.add_argument('--db', default=os.environ.get('WEFLOW_DB_PATH'), required=not os.environ.get('WEFLOW_DB_PATH'), help='Path to NT database')
+    cursor_parser.add_argument('--key', default=os.environ.get('WEFLOW_NT_KEY'), required=not os.environ.get('WEFLOW_NT_KEY'), help='Key hex (64 chars)')
+    cursor_parser.add_argument('--salt', default=os.environ.get('WEFLOW_NT_SALT'), required=not os.environ.get('WEFLOW_NT_SALT'), help='Salt hex (32 chars)')
+    cursor_parser.add_argument('--passphrase', default=os.environ.get('WEFLOW_NT_PASSPHRASE'), help='Shared passphrase for deriving per-shard keys')
+    cursor_parser.add_argument('--tables-per-shard', type=int, default=5,
+                               help='Message tables sampled per shard for the identity half (schema half is exhaustive)')
+    cursor_parser.add_argument('--rows-per-table', type=int, default=500,
+                               help='Rows read at each rowid end per sampled table')
+
     args = parser.parse_args()
 
     if args.command == 'scan':
@@ -1812,6 +1943,13 @@ def main():
                                args.keyword, getattr(args, 'fav_type', None))
         print(json.dumps(result, ensure_ascii=True, default=str))
 
+    elif args.command == 'cursor-probe':
+        result = cursor_probe(args.db, args.key, args.salt,
+                              getattr(args, 'passphrase', '') or '',
+                              tables_per_shard=args.tables_per_shard,
+                              rows_per_table=args.rows_per_table)
+        print(json.dumps(result, ensure_ascii=True))
+
     else:
         parser.print_help()