weflow-cli 1.10.0 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +141 -0
- package/OPERATIONS.md +40 -2
- package/README.md +2 -2
- package/bin/weflow-cli.ts +23 -1
- package/dist/bin/weflow-cli.js +26 -1
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/src/services/exportService.d.ts +43 -0
- package/dist/src/services/exportService.d.ts.map +1 -1
- package/dist/src/services/exportService.js +116 -11
- package/dist/src/services/exportService.js.map +1 -1
- package/docs/FUNCTIONAL-IMPROVEMENT-SPEC.md +21 -0
- package/docs/PROJECT_STATE.md +3 -1
- package/docs/RELEASING.md +46 -0
- package/package.json +1 -1
- package/scripts/export_chat_html.py +450 -8
- package/scripts/semantic_search.py +82 -36
- package/src/services/exportService.ts +129 -12
|
@@ -14,8 +14,11 @@ import re
|
|
|
14
14
|
import base64
|
|
15
15
|
import urllib.request
|
|
16
16
|
import urllib.error
|
|
17
|
+
import urllib.parse
|
|
17
18
|
import concurrent.futures
|
|
19
|
+
import shutil
|
|
18
20
|
import struct
|
|
21
|
+
import xml.etree.ElementTree as ET
|
|
19
22
|
import json
|
|
20
23
|
from pathlib import Path
|
|
21
24
|
|
|
@@ -94,6 +97,25 @@ COVER_STATE = {
|
|
|
94
97
|
'thumb_fetched': 0, 'thumb_cached': 0, 'thumb_skipped': 0,
|
|
95
98
|
}
|
|
96
99
|
|
|
100
|
+
# 落盘时用的扩展名:按**字节**判(`detect_mime_from_bytes`),不按来源猜 —— 实测同一会话 556 张图里
|
|
101
|
+
# 87 个是 GIF、27 个是 PNG,一律叫 .jpg 会让编辑器按扩展名解码、把动画表情显示成坏图。
|
|
102
|
+
_IMAGE_EXT_BY_MIME = {'image/jpeg': '.jpg', 'image/png': '.png',
|
|
103
|
+
'image/gif': '.gif', 'image/webp': '.webp'}
|
|
104
|
+
|
|
105
|
+
# 文档本体(appmsg `type=6`)从账号目录的 `msg/file/` 拷进 assets。原件是用户自己的文件:不重编码、
|
|
106
|
+
# 扩展名保持原样(编辑器/系统按扩展名决定用什么打开)。上限只防"一个几百 MB 的压缩包把导出变成磁盘炸弹",
|
|
107
|
+
# 超限的**不静默丢**:计入 `FILE_ASSET_STATE['unavailable']` 并写进清单顶部计数。
|
|
108
|
+
MAX_ASSET_FILE_BYTES = 64 * 1024 * 1024
|
|
109
|
+
_FILE_ASSET_SUBDIR = 'files'
|
|
110
|
+
|
|
111
|
+
# 文档那半的计数(图片那半的计数在写盘时现算)。放在模块级是为了不改 `write_assets` 的返回形状 ——
|
|
112
|
+
# 它已经有两个调用方(markdown 导出与 `--media`),与 COVER_STATE/VOICE_STATE 是同一套写法。
|
|
113
|
+
#
|
|
114
|
+
# `exported` 与 `written` 是两件事:`written` 只数**这次新拷**的,`exported` 数"这条消息的文档确实在
|
|
115
|
+
# assets 里"。分开是因为第二次跑同一个目录时新拷贝是 0 —— 只报 `written` 会打印成"0 document(s)",
|
|
116
|
+
# 而实际上 67 份都在,那是给自己制造假象(2026-10-13 真的这么打印过一次)。
|
|
117
|
+
FILE_ASSET_STATE = {'exported': 0, 'written': 0, 'unavailable': 0}
|
|
118
|
+
|
|
97
119
|
# Off by default: full-resolution originals are 5-24MB / 3000-5700px each, and
|
|
98
120
|
# downscaling ~500 of them costs minutes (PIL decode dominates). The cache's
|
|
99
121
|
# own thumbnails need no processing at all and are what an export normally
|
|
@@ -488,6 +510,75 @@ def scan_account_media(account_dir, own_wxid='', talker=''):
|
|
|
488
510
|
return image_map
|
|
489
511
|
|
|
490
512
|
|
|
513
|
+
def normalize_asset_name(name):
|
|
514
|
+
"""把消息里的 `title` 与磁盘文件名归一成同一个键。
|
|
515
|
+
|
|
516
|
+
微信写盘时会给自己加一份副本后缀:消息里是 `晚餐聚餐安排(学术研讨会后).docx`,磁盘上是
|
|
517
|
+
`晚餐聚餐安排(学术研讨会后)(1).docx`(实测)。半角全角都出现过,所以两种括号都剥。
|
|
518
|
+
"""
|
|
519
|
+
stem, ext = os.path.splitext(str(name or ''))
|
|
520
|
+
return (re.sub(r'[\s]*[((]\d+[))]$', '', stem).strip().lower() + ext.lower())
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def scan_account_file_index(account_dir):
|
|
524
|
+
"""`msg/file/**` 的**只 stat 索引**:`{(归一化名, 字节数): [路径]}`。
|
|
525
|
+
|
|
526
|
+
为什么不照图片那套按 talker 缩范围:`msg/file` 按月分目录(`msg/file/2024-04/…`),目录里没有
|
|
527
|
+
talker 的痕迹,一条会话的文件散在所有月份里。好在它只 `stat` 不读内容 —— 真机 1,677 个文件
|
|
528
|
+
不到一秒。键带上字节数是因为同名而内容不同的情形确实存在;真正的消歧靠 md5,但那要读盘,
|
|
529
|
+
所以只留给**多候选**那一档(实测 1,855 条文件消息里 154 条落到这档,全部认下)。
|
|
530
|
+
"""
|
|
531
|
+
index = {}
|
|
532
|
+
root = os.path.join(account_dir, 'msg', 'file')
|
|
533
|
+
if not os.path.isdir(root):
|
|
534
|
+
return index
|
|
535
|
+
for current_root, _, files in os.walk(root):
|
|
536
|
+
for fname in files:
|
|
537
|
+
path = os.path.join(current_root, fname)
|
|
538
|
+
try:
|
|
539
|
+
size = os.path.getsize(path)
|
|
540
|
+
except OSError:
|
|
541
|
+
continue
|
|
542
|
+
index.setdefault((normalize_asset_name(fname), size), []).append(path)
|
|
543
|
+
# 排序是为了**可复现**:多候选且消息没带 md5 时取 `[0]`,而 `os.walk` 的顺序不保证稳定 ——
|
|
544
|
+
# 不排的话,同一次导出在两台机器上可能落到不同名字的副本上。
|
|
545
|
+
for paths in index.values():
|
|
546
|
+
paths.sort()
|
|
547
|
+
return index
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def resolve_account_file(info, file_index):
|
|
551
|
+
"""把一条文件消息对到本地那份文件上;对不上返回 None(不猜,D-014)。
|
|
552
|
+
|
|
553
|
+
先按 `(归一化名, 字节数)` 查:真机 1,855 条里 1,489 条唯一命中;多候选才读盘比 md5。整体命中
|
|
554
|
+
88.6%,剩下的 11.4% 是本地确实没有那份文件(未下载 / 已清理),与图片那半同一种情况。
|
|
555
|
+
"""
|
|
556
|
+
if not info or not file_index:
|
|
557
|
+
return None
|
|
558
|
+
name = normalize_asset_name(info.get('title', ''))
|
|
559
|
+
size = info.get('bytes') or 0
|
|
560
|
+
candidates = file_index.get((name, size)) if size else None
|
|
561
|
+
if candidates is None:
|
|
562
|
+
# `totallen` 缺失(少数消息没有)时退化成按名字查一遍:只遍历键,不读盘。
|
|
563
|
+
candidates = [path for (key_name, _size), paths in file_index.items()
|
|
564
|
+
if key_name == name for path in paths]
|
|
565
|
+
if not candidates:
|
|
566
|
+
return None
|
|
567
|
+
if len(candidates) == 1:
|
|
568
|
+
return candidates[0]
|
|
569
|
+
want = (info.get('md5') or '').lower()
|
|
570
|
+
if want:
|
|
571
|
+
for path in candidates:
|
|
572
|
+
try:
|
|
573
|
+
with open(path, 'rb') as fh:
|
|
574
|
+
if hashlib.md5(fh.read()).hexdigest() == want:
|
|
575
|
+
return path
|
|
576
|
+
except OSError:
|
|
577
|
+
continue
|
|
578
|
+
return None
|
|
579
|
+
return candidates[0]
|
|
580
|
+
|
|
581
|
+
|
|
491
582
|
def clean_account_wxid(value):
|
|
492
583
|
value = str(value or '').strip()
|
|
493
584
|
parts = value.rsplit('_', 1)
|
|
@@ -1610,7 +1701,49 @@ def sender_matches_account(sender_user_name, own_wxid):
|
|
|
1610
1701
|
return False
|
|
1611
1702
|
|
|
1612
1703
|
|
|
1613
|
-
def
|
|
1704
|
+
def parse_file_appmsg(content):
|
|
1705
|
+
"""顶层 appmsg `type=6`(文件)的元数据,或 None。
|
|
1706
|
+
|
|
1707
|
+
取的是 `<appmsg>` 的**直接子元素**,不是"整段里第一个同名标签"。原因:引用消息(type 57)把被引用
|
|
1708
|
+
的原文整个塞进 `<refermsg>`,而 `<refermsg>` 就在 `<appmsg>` **内部** —— 那份原文里的 `<type>6</type>`
|
|
1709
|
+
/`<title>`/`<appattach>` 与这条消息自己的字段处在同一段文本里,"切到 appmsg 块"和"取第一个匹配"
|
|
1710
|
+
都分不开,只有按层级取才分得开。2026-10-13 首次测量用正则满文档搜,7,083 条"像文件"里绝大多数
|
|
1711
|
+
就是这种引用原文。
|
|
1712
|
+
"""
|
|
1713
|
+
if not content:
|
|
1714
|
+
return None
|
|
1715
|
+
match = re.search(r'<appmsg[\s>].*?</appmsg>', str(content), re.S)
|
|
1716
|
+
if not match:
|
|
1717
|
+
return None
|
|
1718
|
+
try:
|
|
1719
|
+
element = ET.fromstring('<root>' + match.group(0) + '</root>').find('appmsg')
|
|
1720
|
+
except ET.ParseError:
|
|
1721
|
+
# 真机 7,878 行里有 2 行解析不了(转义残缺)。返回 None = 当"不是文件消息":少带出一个文件,
|
|
1722
|
+
# 好过把某份引用原文的名字当成本条文件的名字。
|
|
1723
|
+
return None
|
|
1724
|
+
if element is None:
|
|
1725
|
+
return None
|
|
1726
|
+
fields = {child.tag: (child.text or '').strip() for child in element}
|
|
1727
|
+
if fields.get('type') != '6':
|
|
1728
|
+
return None
|
|
1729
|
+
title = fields.get('title', '')
|
|
1730
|
+
if not title:
|
|
1731
|
+
return None
|
|
1732
|
+
attachment = element.find('appattach')
|
|
1733
|
+
attach = {child.tag: (child.text or '').strip() for child in attachment} if attachment is not None else {}
|
|
1734
|
+
try:
|
|
1735
|
+
total = int(fields.get('totallen') or attach.get('totallen') or '0')
|
|
1736
|
+
except ValueError:
|
|
1737
|
+
total = 0
|
|
1738
|
+
return {
|
|
1739
|
+
'title': title,
|
|
1740
|
+
'ext': (fields.get('fileext') or attach.get('fileext') or '').lower(),
|
|
1741
|
+
'md5': (fields.get('md5') or attach.get('md5') or '').lower(),
|
|
1742
|
+
'bytes': total,
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
|
|
1746
|
+
def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display_name='', resource_map=None, own_wxid='', file_index=None):
|
|
1614
1747
|
"""Format a single message for HTML display.
|
|
1615
1748
|
|
|
1616
1749
|
Args:
|
|
@@ -1620,6 +1753,8 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1620
1753
|
image_map: {local_id: (base64_data, mime_type)} from NT cache scan
|
|
1621
1754
|
sender_map: {sender_id: user_name} from Name2Id table
|
|
1622
1755
|
display_name: human-readable name for the target talker
|
|
1756
|
+
file_index: {(归一化名, 字节数): [路径]} from scan_account_file_index, for
|
|
1757
|
+
resolving a document attachment to the copy on disk
|
|
1623
1758
|
"""
|
|
1624
1759
|
is_group = '@chatroom' in str(talker or '')
|
|
1625
1760
|
local_id = row[0] or 0
|
|
@@ -1730,6 +1865,12 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1730
1865
|
if '\x00' in content or sum(ord(char) < 32 and char not in '\n\r\t' for char in content) > 2:
|
|
1731
1866
|
content = ''
|
|
1732
1867
|
|
|
1868
|
+
# 文档本体:消息里只有名字与 md5,文件本身在账号目录的 `msg/file/` 下。这里只**解析并定位**,
|
|
1869
|
+
# 不读也不拷 —— 拷贝发生在写 assets 那一步(HTML 那条路根本不落盘:它把图片内嵌成 base64,
|
|
1870
|
+
# 一个几 MB 的 PDF 内嵌进网页既没人打得开、也把文件撑大)。
|
|
1871
|
+
file_info = parse_file_appmsg(content) if local_type == 49 else None
|
|
1872
|
+
file_hit = resolve_account_file(file_info, file_index)
|
|
1873
|
+
|
|
1733
1874
|
# Determine display content
|
|
1734
1875
|
display = ''
|
|
1735
1876
|
image_b64 = None
|
|
@@ -1920,19 +2061,24 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1920
2061
|
'local_type': local_type,
|
|
1921
2062
|
'display': display,
|
|
1922
2063
|
'image_b64': image_b64,
|
|
2064
|
+
# 文档本体的落点(`path` 为 None = 本地没有那份文件)。写 assets 时才拷。
|
|
2065
|
+
'file': ({'title': file_info['title'], 'ext': file_info['ext'],
|
|
2066
|
+
'bytes': file_info['bytes'], 'md5': file_info['md5'],
|
|
2067
|
+
'path': file_hit} if file_info else None),
|
|
1923
2068
|
# Built here rather than accumulated in module state: format_message
|
|
1924
2069
|
# runs twice (the prefetch dry sweep and the real pass) and the dry
|
|
1925
2070
|
# pass is discarded, so a counter would double-count.
|
|
1926
2071
|
'media': build_media_record(
|
|
1927
2072
|
local_type, image_b64, image_map, local_id, create_time, content,
|
|
1928
|
-
resource_md5s, metadata_content, bool(server_id)
|
|
2073
|
+
resource_md5s, metadata_content, bool(server_id),
|
|
2074
|
+
file_info, file_hit),
|
|
1929
2075
|
}
|
|
1930
2076
|
|
|
1931
2077
|
|
|
1932
2078
|
MEDIA_KIND_BY_TYPE = {
|
|
1933
2079
|
3: 'image', 34: 'voice', 43: 'video-thumb', 47: 'emoji', 49: 'link-cover',
|
|
1934
2080
|
}
|
|
1935
|
-
MEDIA_KINDS = {'image', 'emoji', 'voice', 'video-thumb', 'link-cover'}
|
|
2081
|
+
MEDIA_KINDS = {'image', 'emoji', 'voice', 'video-thumb', 'link-cover', 'file'}
|
|
1936
2082
|
|
|
1937
2083
|
|
|
1938
2084
|
def build_media_report(items, scope, talker, per_page):
|
|
@@ -1980,7 +2126,8 @@ def build_media_report(items, scope, talker, per_page):
|
|
|
1980
2126
|
|
|
1981
2127
|
|
|
1982
2128
|
def build_media_record(local_type, image_b64, image_map, local_id, create_time,
|
|
1983
|
-
content, resource_md5s, metadata_content, has_server_id
|
|
2129
|
+
content, resource_md5s, metadata_content, has_server_id,
|
|
2130
|
+
file_info=None, file_hit=None):
|
|
1984
2131
|
"""What happened to this message's media, or None for a non-media message.
|
|
1985
2132
|
|
|
1986
2133
|
Derived from the same `resolve_media` the page used, so the report cannot
|
|
@@ -1995,6 +2142,19 @@ def build_media_record(local_type, image_b64, image_map, local_id, create_time,
|
|
|
1995
2142
|
'mime': None, 'bytes': None, 'sha256': None, 'reason': None,
|
|
1996
2143
|
}
|
|
1997
2144
|
|
|
2145
|
+
if file_info is not None:
|
|
2146
|
+
# 文档本体(appmsg type 6)走的是另一条路:没有缩略图、也不在图片索引里,能不能导出只看
|
|
2147
|
+
# 本地 `msg/file/` 有没有那份文件。不并入图片那条链,否则报告会把"文件没下载"写成"没有可靠标识"。
|
|
2148
|
+
record['kind'] = 'file'
|
|
2149
|
+
record['bytes'] = file_info.get('bytes') or None
|
|
2150
|
+
if file_hit:
|
|
2151
|
+
record['status'] = 'exported'
|
|
2152
|
+
record['mediaKey'] = 'file'
|
|
2153
|
+
else:
|
|
2154
|
+
record['status'] = 'missing'
|
|
2155
|
+
record['reason'] = 'file-not-in-local-storage'
|
|
2156
|
+
return record
|
|
2157
|
+
|
|
1998
2158
|
if image_b64:
|
|
1999
2159
|
hit = resolve_media(image_map, local_id, create_time, metadata_content,
|
|
2000
2160
|
resource_md5s)
|
|
@@ -2293,6 +2453,233 @@ p {{ margin:8px 0; font-size:14px; }}
|
|
|
2293
2453
|
</html>'''
|
|
2294
2454
|
|
|
2295
2455
|
|
|
2456
|
+
_WINDOWS_RESERVED_NAMES = frozenset(
|
|
2457
|
+
['con', 'prn', 'aux', 'nul']
|
|
2458
|
+
+ ['com%d' % index for index in range(1, 10)]
|
|
2459
|
+
+ ['lpt%d' % index for index in range(1, 10)])
|
|
2460
|
+
|
|
2461
|
+
|
|
2462
|
+
def _clean_asset_component(text):
|
|
2463
|
+
"""把一段文件名清洗到"任何 Windows 上都能创建"。
|
|
2464
|
+
|
|
2465
|
+
在 `sanitize_filename` 之外多管一件会**静默改结果**的事:首尾的空白与点(Windows 会把 `a .` 存成 `a`,
|
|
2466
|
+
于是清单里的名字和磁盘上的名字对不上)。
|
|
2467
|
+
"""
|
|
2468
|
+
return sanitize_filename(text).strip().strip('.')
|
|
2469
|
+
|
|
2470
|
+
|
|
2471
|
+
def _asset_file_name(title, used_names, source, md5):
|
|
2472
|
+
"""落盘的文档文件名:保持原样(只清洗非法字符),重名才加 md5 前 8 位。
|
|
2473
|
+
|
|
2474
|
+
不改名是刻意的 —— 那是用户自己的文件名,读者要靠它认东西;微信自己也会加 `(1)`。但两个不同来源
|
|
2475
|
+
的文件可能同名,那种情况必须区分,否则后来者会覆盖先到者、清单指错文件。
|
|
2476
|
+
|
|
2477
|
+
**截断要在扩展名之前**:真机实测过一个超长标题,80 字符上限把 `.pdf` 一起切掉了,落盘成没有扩展名的
|
|
2478
|
+
文件 —— 系统与编辑器都不知道该用什么打开它。
|
|
2479
|
+
"""
|
|
2480
|
+
stem, ext = os.path.splitext(str(title or ''))
|
|
2481
|
+
# 扩展名单独保留(也过一遍清洗),但不参与"给主干剩下的额度";异常长的"扩展名"当没扩展名处理
|
|
2482
|
+
ext = _clean_asset_component(ext).replace('.', '')
|
|
2483
|
+
if len(ext) > 16:
|
|
2484
|
+
stem, ext = str(title or ''), ''
|
|
2485
|
+
stem = _clean_asset_component(stem)
|
|
2486
|
+
room = max(1, 80 - (len(ext) + 1 if ext else 0))
|
|
2487
|
+
base = stem[:room] or 'file'
|
|
2488
|
+
if base.lower() in _WINDOWS_RESERVED_NAMES:
|
|
2489
|
+
# `CON.pdf` 这类在 Windows 上打不开(会被当成设备名)—— 加个前缀就正常了
|
|
2490
|
+
base = '_' + base
|
|
2491
|
+
safe = base + ('.' + ext if ext else '')
|
|
2492
|
+
if used_names.get(safe) in (None, source):
|
|
2493
|
+
return safe
|
|
2494
|
+
stem, ext = os.path.splitext(safe)
|
|
2495
|
+
return '%s_%s%s' % (stem, (md5 or 'copy')[:8], ext)
|
|
2496
|
+
|
|
2497
|
+
|
|
2498
|
+
def _load_existing_manifest(manifest_path, out_dir):
|
|
2499
|
+
"""读回同目录里已有的清单条目(v1 的裸字符串也认),丢掉**已经不在磁盘上**的那些。
|
|
2500
|
+
|
|
2501
|
+
为什么不是直接覆盖:清单住在 `<会话>_assets/` 这个**共享**目录里,而导出可以分批做
|
|
2502
|
+
(`--limit 200` 一次、整段再一次;md 一次、`json --media` 再一次)。覆盖会让窄窗口那次的清单把宽窗口
|
|
2503
|
+
那次的条目**删掉**,而 md 正文里的链接还在 —— 之后任何"读清单带媒体"的导出都会少一批,
|
|
2504
|
+
且两边都不报错。2026-10-13 真机就是这么撞上的:整段导出的 585 条被 `--limit 600` 那次的 150 条盖掉。
|
|
2505
|
+
"""
|
|
2506
|
+
try:
|
|
2507
|
+
with open(manifest_path, 'r', encoding='utf-8') as handle:
|
|
2508
|
+
data = json.load(handle)
|
|
2509
|
+
except (OSError, ValueError):
|
|
2510
|
+
return {}
|
|
2511
|
+
messages = data.get('messages') if isinstance(data, dict) else None
|
|
2512
|
+
if not isinstance(messages, dict):
|
|
2513
|
+
return {}
|
|
2514
|
+
kept = {}
|
|
2515
|
+
for key, value in messages.items():
|
|
2516
|
+
if isinstance(value, str):
|
|
2517
|
+
record = {'path': value, 'kind': 'image'}
|
|
2518
|
+
elif isinstance(value, dict) and isinstance(value.get('path'), str):
|
|
2519
|
+
record = value
|
|
2520
|
+
else:
|
|
2521
|
+
continue
|
|
2522
|
+
if os.path.isfile(os.path.join(out_dir, record['path'].replace('/', os.sep))):
|
|
2523
|
+
kept[key] = record
|
|
2524
|
+
return kept
|
|
2525
|
+
|
|
2526
|
+
|
|
2527
|
+
def write_assets(out_dir, file_prefix, formatted, assets_dir=''):
|
|
2528
|
+
"""把媒体**落盘**并把 `manifest.json` 写好;返回 `(assets 目录名, manifest, 新写文件数)`。
|
|
2529
|
+
|
|
2530
|
+
HTML 走的是"内嵌 base64 ⇒ 单文件可分享",而 markdown 与其它格式的用法是喂给编辑器/别的工具 ——
|
|
2531
|
+
那里 `` 多数不被认,图片必须是**文件 + 相对路径**。所以这里写文件,并留一份清单
|
|
2532
|
+
(键 `local_id:create_time` → `{path, kind, …}`):**抽取只做一遍**,json/txt/excel 只是消费它。
|
|
2533
|
+
|
|
2534
|
+
图片与文档本体都进同一个清单:图片来自内存里的 base64,文档是**从 `msg/file/` 拷过来**的(路径在
|
|
2535
|
+
`format_message` 阶段就定好了)。清单 v2 起值是对象(带 `kind`),读取端要兼容 v1 的裸字符串。
|
|
2536
|
+
"""
|
|
2537
|
+
import base64
|
|
2538
|
+
import hashlib
|
|
2539
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
2540
|
+
assets = assets_dir or os.path.join(out_dir, file_prefix + '_assets')
|
|
2541
|
+
os.makedirs(assets, exist_ok=True)
|
|
2542
|
+
|
|
2543
|
+
# 与已有清单**合并**:分批导出时,窄窗口那一次不该把宽窗口那次的条目删掉(见 `_load_existing_manifest`)。
|
|
2544
|
+
manifest_path = os.path.join(assets, 'manifest.json')
|
|
2545
|
+
manifest = _load_existing_manifest(manifest_path, out_dir)
|
|
2546
|
+
written = 0
|
|
2547
|
+
for item in formatted:
|
|
2548
|
+
blob = item.get('image_b64')
|
|
2549
|
+
if not blob:
|
|
2550
|
+
continue
|
|
2551
|
+
data = base64.b64decode(blob)
|
|
2552
|
+
# 扩展名按**字节**判:实测同一会话 556 张图里 87 个是 GIF、27 个是 PNG,一律叫 .jpg 会让
|
|
2553
|
+
# 编辑器按扩展名解码、把动画表情显示成坏图。
|
|
2554
|
+
mime = detect_mime_from_bytes(data[:32]) or 'image/jpeg'
|
|
2555
|
+
name = 'img_%s_%s%s' % (item.get('local_id'),
|
|
2556
|
+
hashlib.md5(blob[:64].encode()).hexdigest()[:8],
|
|
2557
|
+
_IMAGE_EXT_BY_MIME.get(mime, '.jpg'))
|
|
2558
|
+
path = os.path.join(assets, name)
|
|
2559
|
+
if not os.path.exists(path):
|
|
2560
|
+
with open(path, 'wb') as fh:
|
|
2561
|
+
fh.write(data)
|
|
2562
|
+
written += 1
|
|
2563
|
+
# 键带时间戳:**同一个 local_id 会在分片之间重复**(每条会话的表可能存在于多个分片,
|
|
2564
|
+
# id 各片自己编号)—— 只按 local_id 做键,json/txt/excel 会拿别人的图。
|
|
2565
|
+
manifest['%s:%s' % (item.get('local_id'), item.get('create_time'))] = {
|
|
2566
|
+
'path': os.path.relpath(path, out_dir).replace('\\', '/'),
|
|
2567
|
+
'kind': 'image',
|
|
2568
|
+
}
|
|
2569
|
+
|
|
2570
|
+
file_written = 0
|
|
2571
|
+
file_unavailable = 0
|
|
2572
|
+
files_dir = os.path.join(assets, _FILE_ASSET_SUBDIR)
|
|
2573
|
+
used_names = {}
|
|
2574
|
+
for item in formatted:
|
|
2575
|
+
info = item.get('file')
|
|
2576
|
+
if not info:
|
|
2577
|
+
continue
|
|
2578
|
+
source = info.get('path')
|
|
2579
|
+
size = 0
|
|
2580
|
+
if source:
|
|
2581
|
+
try:
|
|
2582
|
+
size = os.path.getsize(source)
|
|
2583
|
+
except OSError:
|
|
2584
|
+
source = None
|
|
2585
|
+
# 超限与"本地没有"归到同一档计数(都是"这条消息的文档没能带出来"),但不静默:计数写进清单顶部。
|
|
2586
|
+
if not source or size > MAX_ASSET_FILE_BYTES:
|
|
2587
|
+
file_unavailable += 1
|
|
2588
|
+
continue
|
|
2589
|
+
name = _asset_file_name(info.get('title') or os.path.basename(source), used_names, source,
|
|
2590
|
+
info.get('md5'))
|
|
2591
|
+
target = os.path.join(files_dir, name)
|
|
2592
|
+
if used_names.get(name) != source:
|
|
2593
|
+
try:
|
|
2594
|
+
os.makedirs(files_dir, exist_ok=True)
|
|
2595
|
+
if not os.path.exists(target):
|
|
2596
|
+
with open(target, 'wb') as dst, open(source, 'rb') as src:
|
|
2597
|
+
shutil.copyfileobj(src, dst)
|
|
2598
|
+
file_written += 1
|
|
2599
|
+
except OSError:
|
|
2600
|
+
# 一个文件拷不进去(名字太怪、路径过长、目标被占用)不能让**整次导出**在最后一步崩掉 ——
|
|
2601
|
+
# 前面几分钟的解析与图片抽取都在,代价只是这一份文档缺失,且它会计入 unavailable。
|
|
2602
|
+
file_unavailable += 1
|
|
2603
|
+
continue
|
|
2604
|
+
used_names[name] = source
|
|
2605
|
+
manifest['%s:%s' % (item.get('local_id'), item.get('create_time'))] = {
|
|
2606
|
+
'path': os.path.relpath(target, out_dir).replace('\\', '/'),
|
|
2607
|
+
'kind': 'file',
|
|
2608
|
+
'name': info.get('title') or name,
|
|
2609
|
+
'bytes': size,
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
# `exported` 数的是**合并后清单里真实存在的文档条目**(第二遍跑同一目录时它不为 0,而新拷贝是 0);
|
|
2613
|
+
# `copied` 才是"这次新拷了几个"。两个都写出来,免得读清单的人把后者当前者。
|
|
2614
|
+
file_exported = sum(1 for record in manifest.values() if record.get('kind') == 'file')
|
|
2615
|
+
FILE_ASSET_STATE['exported'] = file_exported
|
|
2616
|
+
FILE_ASSET_STATE['written'] = file_written
|
|
2617
|
+
FILE_ASSET_STATE['unavailable'] = file_unavailable
|
|
2618
|
+
with open(manifest_path, 'w', encoding='utf-8') as fh:
|
|
2619
|
+
json.dump({'schema': 'weflow-media-manifest/v2', 'messages': manifest,
|
|
2620
|
+
'documents': {'exported': file_exported, 'copied': file_written,
|
|
2621
|
+
'unavailable': file_unavailable}},
|
|
2622
|
+
fh, ensure_ascii=False, indent=2)
|
|
2623
|
+
return os.path.basename(os.path.normpath(assets)), manifest, written + file_written
|
|
2624
|
+
|
|
2625
|
+
|
|
2626
|
+
def _markdown_link_target(rel):
|
|
2627
|
+
"""把相对路径转成 markdown 链接目标:只编码会**截断链接**的那些字符。
|
|
2628
|
+
|
|
2629
|
+
真机实测(2026-10-13,重导咸鱼梦想家时核对产物才发现):文件名叫
|
|
2630
|
+
`…研究生学位(毕业)论文开题报告(2).pdf`,写出来是 `[名字](…报告(2).pdf)` —— 任何 markdown 渲染器
|
|
2631
|
+
都在第一个半角 `)` 处结束目标,链接指向 `…报告(2`,点开是空的。带空格的文件名同理(目标在空白处结束)。
|
|
2632
|
+
所以按段编码这几个字符(括号、空白、`<`、`>`、`[`、`]`、`#`、`?`、`%`、引号、反斜杠),
|
|
2633
|
+
**中文保持原样**:本地预览器都按 UTF-8 解路径,编码成 `%E5%…` 反而让人看不懂。
|
|
2634
|
+
"""
|
|
2635
|
+
out = []
|
|
2636
|
+
for ch in str(rel or '').replace('\\', '/'):
|
|
2637
|
+
if ch.isspace() or ch in '()[]<>#"\'%?\\':
|
|
2638
|
+
out.append('%%%02X' % ord(ch))
|
|
2639
|
+
else:
|
|
2640
|
+
out.append(ch)
|
|
2641
|
+
return ''.join(out)
|
|
2642
|
+
|
|
2643
|
+
|
|
2644
|
+
def write_markdown(out_dir, file_prefix, formatted, assets_dir=''):
|
|
2645
|
+
"""markdown 正文 + 媒体落盘(落盘那半在 `write_assets`,两者共用同一份清单)。"""
|
|
2646
|
+
assets_name, manifest, written = write_assets(out_dir, file_prefix, formatted, assets_dir)
|
|
2647
|
+
|
|
2648
|
+
lines = ['# %s' % file_prefix, '']
|
|
2649
|
+
for item in formatted:
|
|
2650
|
+
who = '我' if (item.get('is_self') or item.get('is_send')) else (item.get('sender') or '')
|
|
2651
|
+
# 这个文件是 `import datetime`(模块),且它的时间戳一律是**本地时区**(见 render 那几处),
|
|
2652
|
+
# 所以这里也照它的写法,不带 tz —— 导出页里显示的时间与本文件其它时间戳保持一致。
|
|
2653
|
+
stamp = datetime.datetime.fromtimestamp(item.get('create_time') or 0).strftime('%Y-%m-%d %H:%M')
|
|
2654
|
+
body = plain_fragment(item.get('display') or '', limit=100000) or ''
|
|
2655
|
+
record = manifest.get('%s:%s' % (item.get('local_id'), item.get('create_time')))
|
|
2656
|
+
if record:
|
|
2657
|
+
# v1 的值是裸字符串;v2 起是对象。两种都要认(磁盘上可能还留着上一版的清单)。
|
|
2658
|
+
if isinstance(record, dict):
|
|
2659
|
+
rel = record.get('path') or ''
|
|
2660
|
+
kind = record.get('kind') or 'image'
|
|
2661
|
+
label = record.get('name') or ''
|
|
2662
|
+
else:
|
|
2663
|
+
rel, kind, label = record, 'image', ''
|
|
2664
|
+
if rel and kind == 'file':
|
|
2665
|
+
# 文档不是图片:`` 会被当成坏图,要写成链接,链接文字用原始文件名。
|
|
2666
|
+
keep = '' if (not body or body.startswith('[文件]')) else body + chr(10) + chr(10)
|
|
2667
|
+
# 链接文字用原始文件名;`[`/`]` 换成实体(`[草稿] 方案.docx` 会把链接本身的结构打断),
|
|
2668
|
+
# 控制字符去掉(标题里可能带换行/制表符,落盘名已去掉,文字也得跟上,否则渲染出来断开)
|
|
2669
|
+
text = ''.join(ch for ch in (label or os.path.basename(rel))
|
|
2670
|
+
if ord(ch) >= 32 and ord(ch) != 127)
|
|
2671
|
+
text = text.replace('[', '[').replace(']', ']')
|
|
2672
|
+
body = keep + '[%s](%s)' % (text, _markdown_link_target(rel))
|
|
2673
|
+
elif rel:
|
|
2674
|
+
body = ('' if (not body or body == '[图片]') else body + chr(10) + chr(10)) \
|
|
2675
|
+
+ '' % _markdown_link_target(rel)
|
|
2676
|
+
lines.extend(['**%s** %s' % (who, stamp), '', body.strip() or '(空)', ''])
|
|
2677
|
+
|
|
2678
|
+
md_path = os.path.join(out_dir, file_prefix + '.md')
|
|
2679
|
+
with open(md_path, 'w', encoding='utf-8') as fh:
|
|
2680
|
+
fh.write(chr(10).join(lines))
|
|
2681
|
+
return md_path, len(manifest), written, assets_name
|
|
2682
|
+
|
|
2296
2683
|
def main():
|
|
2297
2684
|
import argparse
|
|
2298
2685
|
parser = argparse.ArgumentParser(description='Export WeChat NT chat as HTML')
|
|
@@ -2302,6 +2689,12 @@ def main():
|
|
|
2302
2689
|
parser.add_argument('--talker', default=os.environ.get('WEFLOW_TALKER'), required=not os.environ.get('WEFLOW_TALKER'), help='Talker username')
|
|
2303
2690
|
parser.add_argument('--name', default=os.environ.get('WEFLOW_EXPORT_NAME', ''), help='Display name')
|
|
2304
2691
|
parser.add_argument('--out', default=os.environ.get('WEFLOW_EXPORT_OUTPUT', './output'), help='Output directory')
|
|
2692
|
+
parser.add_argument('--format', choices=('html', 'md'), default='html',
|
|
2693
|
+
help='html=内嵌图片的多分页页面(默认);md=markdown + 图片落盘到 assets/')
|
|
2694
|
+
parser.add_argument('--assets-only', action='store_true',
|
|
2695
|
+
help='只抽取媒体到 assets/ 并写清单,不写 markdown(给 json/txt/excel 的 --media 用)')
|
|
2696
|
+
parser.add_argument('--assets-dir', default='',
|
|
2697
|
+
help='md 模式下图片落到哪个目录(默认 <out>/<前缀>_assets;同时写 manifest.json)')
|
|
2305
2698
|
parser.add_argument('--parts', type=int, default=5, help='Number of parts to split into')
|
|
2306
2699
|
parser.add_argument('--wx-dir', default='', help='Traditional WeChat data dir (FileStorage fallback)')
|
|
2307
2700
|
parser.add_argument('--cache-dir', default=os.environ.get('WEFLOW_EXPORT_CACHE_DIR', ''), help='NT cache directory for image thumbnails')
|
|
@@ -2347,6 +2740,14 @@ def main():
|
|
|
2347
2740
|
value = shrink_embedded(value, max_side=720, force=True)
|
|
2348
2741
|
image_map[key] = value
|
|
2349
2742
|
print(f" Found {len(image_map)} media entries for embedding")
|
|
2743
|
+
|
|
2744
|
+
# 文档本体的索引:只 stat,不读内容(1,677 个文件的走一遍不到一秒)。没有它,`appmsg type=6`
|
|
2745
|
+
# 的消息就只能渲染成 `[文件] 名字` —— 正是反馈里"其余格式不支持导出文档"的那一半。
|
|
2746
|
+
file_index = {}
|
|
2747
|
+
if args.account_dir:
|
|
2748
|
+
file_index = scan_account_file_index(args.account_dir)
|
|
2749
|
+
if file_index:
|
|
2750
|
+
print(f" Account file index: {len(file_index)} key(s)")
|
|
2350
2751
|
_phase("media scan")
|
|
2351
2752
|
|
|
2352
2753
|
# Custom stickers: derive the local cache key from the account seed.
|
|
@@ -2462,7 +2863,7 @@ def main():
|
|
|
2462
2863
|
PREFETCH['sink'] = sink
|
|
2463
2864
|
try:
|
|
2464
2865
|
for row in messages:
|
|
2465
|
-
format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid)
|
|
2866
|
+
format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid, file_index)
|
|
2466
2867
|
finally:
|
|
2467
2868
|
PREFETCH['sink'] = None
|
|
2468
2869
|
if sink:
|
|
@@ -2483,7 +2884,7 @@ def main():
|
|
|
2483
2884
|
for i, row in enumerate(messages):
|
|
2484
2885
|
if i % progress_step == 0:
|
|
2485
2886
|
print(f" Formatting {i}/{len(messages)}...", flush=True)
|
|
2486
|
-
result = format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid)
|
|
2887
|
+
result = format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid, file_index)
|
|
2487
2888
|
if result.get('image_b64'):
|
|
2488
2889
|
img_hit_count += 1
|
|
2489
2890
|
if result.get('local_type') == 49:
|
|
@@ -2513,6 +2914,42 @@ def main():
|
|
|
2513
2914
|
# Use display name for filename if provided, otherwise fallback to wxid
|
|
2514
2915
|
file_prefix = sanitize_filename(display_name) if display_name else args.talker.replace('@', '_').replace('/', '_')
|
|
2515
2916
|
|
|
2917
|
+
if args.assets_only:
|
|
2918
|
+
# 只抽媒体:其它格式的 `--media` 会先跑这一步,再各自消费清单。
|
|
2919
|
+
assets_name, manifest, asset_new = write_assets(args.out, file_prefix, formatted, args.assets_dir)
|
|
2920
|
+
print(f"Assets: {len(manifest)} media item(s), {FILE_ASSET_STATE['exported']} document(s) "
|
|
2921
|
+
f"({FILE_ASSET_STATE['unavailable']} unavailable, {FILE_ASSET_STATE['written']} new) "
|
|
2922
|
+
f"({asset_new} new) -> {assets_name}")
|
|
2923
|
+
print(json.dumps({"success": True, "total": len(formatted),
|
|
2924
|
+
"files": [os.path.join(args.out, assets_name)], "format": "assets",
|
|
2925
|
+
"assets": len(manifest), "assetsNew": asset_new,
|
|
2926
|
+
"documents": FILE_ASSET_STATE['exported'],
|
|
2927
|
+
"documentsNew": FILE_ASSET_STATE['written'],
|
|
2928
|
+
"documentsUnavailable": FILE_ASSET_STATE['unavailable'],
|
|
2929
|
+
"assetsDir": assets_name}, ensure_ascii=False))
|
|
2930
|
+
conn.close()
|
|
2931
|
+
return
|
|
2932
|
+
if args.format == 'md':
|
|
2933
|
+
md_path, asset_total, asset_new, assets_name = write_markdown(args.out, file_prefix, formatted, args.assets_dir)
|
|
2934
|
+
if media_items:
|
|
2935
|
+
media_report_path = os.path.join(args.out, f'{file_prefix}_media.json')
|
|
2936
|
+
with open(media_report_path, 'w', encoding='utf-8') as handle:
|
|
2937
|
+
json.dump(build_media_report(media_items, display_name, args.talker, len(formatted)),
|
|
2938
|
+
handle, ensure_ascii=False, indent=2)
|
|
2939
|
+
print(f"Markdown: {len(formatted)} message(s), {asset_total} media item(s), "
|
|
2940
|
+
f"{FILE_ASSET_STATE['exported']} document(s) ({FILE_ASSET_STATE['unavailable']} unavailable, "
|
|
2941
|
+
f"{FILE_ASSET_STATE['written']} new) ({asset_new} new) -> {os.path.basename(md_path)}")
|
|
2942
|
+
print(json.dumps({"success": True, "total": len(formatted), "parts": 1,
|
|
2943
|
+
"files": [md_path], "format": "md",
|
|
2944
|
+
"assets": asset_total, "assetsNew": asset_new,
|
|
2945
|
+
"documents": FILE_ASSET_STATE['exported'],
|
|
2946
|
+
"documentsNew": FILE_ASSET_STATE['written'],
|
|
2947
|
+
"documentsUnavailable": FILE_ASSET_STATE['unavailable'],
|
|
2948
|
+
"assetsDir": assets_name},
|
|
2949
|
+
ensure_ascii=False))
|
|
2950
|
+
conn.close()
|
|
2951
|
+
return
|
|
2952
|
+
|
|
2516
2953
|
# Media coverage report. Written beside the HTML parts, under the same
|
|
2517
2954
|
# prefix, so it is always found next to what it describes.
|
|
2518
2955
|
media_report = None
|
|
@@ -2584,8 +3021,13 @@ def _phase(label):
|
|
|
2584
3021
|
|
|
2585
3022
|
|
|
2586
3023
|
def sanitize_filename(name: str) -> str:
|
|
2587
|
-
"""Remove characters unsafe for filenames.
|
|
2588
|
-
|
|
3024
|
+
"""Remove characters unsafe for filenames.
|
|
3025
|
+
|
|
3026
|
+
控制字符也要去掉:它们不在"非法字符"那张表里,但 Windows 创建时会直接
|
|
3027
|
+
`OSError: Invalid argument`(2026-10-13 实测到一个带制表符的标题把整个导出搞崩)。
|
|
3028
|
+
"""
|
|
3029
|
+
kept = ''.join(ch for ch in str(name or '') if ord(ch) >= 32 and ord(ch) != 127)
|
|
3030
|
+
return re.sub(r'[\\/:*?"<>|]', '_', kept)[:80]
|
|
2589
3031
|
|
|
2590
3032
|
|
|
2591
3033
|
if __name__ == '__main__':
|