weflow-cli 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +105 -0
- package/CHANGELOG.md +121 -0
- package/CONTRIBUTING.md +45 -0
- package/LICENSE +21 -21
- package/OPERATIONS.md +272 -0
- package/README.en.md +281 -0
- package/README.md +334 -149
- package/SECURITY.md +47 -0
- package/bin/weflow-cli.ts +4050 -581
- package/dist/bin/weflow-cli.js +4310 -588
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/mcp-server/index.js +178 -236
- package/dist/mcp-server/index.js.map +1 -1
- package/dist/src/core/dbPathService.d.ts +15 -1
- package/dist/src/core/dbPathService.d.ts.map +1 -1
- package/dist/src/core/dbPathService.js +177 -19
- package/dist/src/core/dbPathService.js.map +1 -1
- package/dist/src/core/keyService.d.ts +8 -0
- package/dist/src/core/keyService.d.ts.map +1 -1
- package/dist/src/core/keyService.js +127 -22
- package/dist/src/core/keyService.js.map +1 -1
- package/dist/src/core/ntCore.d.ts +59 -1
- package/dist/src/core/ntCore.d.ts.map +1 -1
- package/dist/src/core/ntCore.js +201 -65
- package/dist/src/core/ntCore.js.map +1 -1
- package/dist/src/core/sqlcipherCore.d.ts.map +1 -1
- package/dist/src/core/sqlcipherCore.js +18 -1
- package/dist/src/core/sqlcipherCore.js.map +1 -1
- package/dist/src/core/wcdbCore.d.ts.map +1 -1
- package/dist/src/core/wcdbCore.js +3 -6
- package/dist/src/core/wcdbCore.js.map +1 -1
- package/dist/src/services/assistantDaemon.d.ts +19 -0
- package/dist/src/services/assistantDaemon.d.ts.map +1 -0
- package/dist/src/services/assistantDaemon.js +117 -0
- package/dist/src/services/assistantDaemon.js.map +1 -0
- package/dist/src/services/assistantMemory.d.ts +36 -0
- package/dist/src/services/assistantMemory.d.ts.map +1 -0
- package/dist/src/services/assistantMemory.js +182 -0
- package/dist/src/services/assistantMemory.js.map +1 -0
- package/dist/src/services/assistantPrivacy.d.ts +25 -0
- package/dist/src/services/assistantPrivacy.d.ts.map +1 -0
- package/dist/src/services/assistantPrivacy.js +77 -0
- package/dist/src/services/assistantPrivacy.js.map +1 -0
- package/dist/src/services/assistantRouting.d.ts +18 -0
- package/dist/src/services/assistantRouting.d.ts.map +1 -0
- package/dist/src/services/assistantRouting.js +68 -0
- package/dist/src/services/assistantRouting.js.map +1 -0
- package/dist/src/services/assistantService.d.ts +37 -0
- package/dist/src/services/assistantService.d.ts.map +1 -0
- package/dist/src/services/assistantService.js +284 -0
- package/dist/src/services/assistantService.js.map +1 -0
- package/dist/src/services/assistantTools.d.ts +24 -0
- package/dist/src/services/assistantTools.d.ts.map +1 -0
- package/dist/src/services/assistantTools.js +582 -0
- package/dist/src/services/assistantTools.js.map +1 -0
- package/dist/src/services/chatService.d.ts +44 -0
- package/dist/src/services/chatService.d.ts.map +1 -1
- package/dist/src/services/chatService.js +105 -4
- package/dist/src/services/chatService.js.map +1 -1
- package/dist/src/services/configService.d.ts +55 -0
- package/dist/src/services/configService.d.ts.map +1 -1
- package/dist/src/services/configService.js +147 -6
- package/dist/src/services/configService.js.map +1 -1
- package/dist/src/services/evidenceService.d.ts +30 -0
- package/dist/src/services/evidenceService.d.ts.map +1 -0
- package/dist/src/services/evidenceService.js +98 -0
- package/dist/src/services/evidenceService.js.map +1 -0
- package/dist/src/services/exportService.d.ts +13 -20
- package/dist/src/services/exportService.d.ts.map +1 -1
- package/dist/src/services/exportService.js +113 -47
- package/dist/src/services/exportService.js.map +1 -1
- package/dist/src/services/initKeyService.d.ts +19 -0
- package/dist/src/services/initKeyService.d.ts.map +1 -0
- package/dist/src/services/initKeyService.js +80 -0
- package/dist/src/services/initKeyService.js.map +1 -0
- package/dist/src/services/messageContract.d.ts +23 -0
- package/dist/src/services/messageContract.d.ts.map +1 -0
- package/dist/src/services/messageContract.js +49 -0
- package/dist/src/services/messageContract.js.map +1 -0
- package/dist/src/services/messageQuery.d.ts +7 -0
- package/dist/src/services/messageQuery.d.ts.map +1 -0
- package/dist/src/services/messageQuery.js +30 -0
- package/dist/src/services/messageQuery.js.map +1 -0
- package/dist/src/services/wechat-formatter.js +14 -14
- package/dist/src/services/wechatMessageService.d.ts.map +1 -1
- package/dist/src/services/wechatMessageService.js +31 -3
- package/dist/src/services/wechatMessageService.js.map +1 -1
- package/dist/src/services/whitelistService.d.ts +52 -4
- package/dist/src/services/whitelistService.d.ts.map +1 -1
- package/dist/src/services/whitelistService.js +183 -17
- package/dist/src/services/whitelistService.js.map +1 -1
- package/dist/src/types.d.ts +20 -0
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/utils/dateRange.d.ts +15 -0
- package/dist/src/utils/dateRange.d.ts.map +1 -0
- package/dist/src/utils/dateRange.js +48 -0
- package/dist/src/utils/dateRange.js.map +1 -0
- package/dist/src/utils/mcpSecurity.d.ts +5 -0
- package/dist/src/utils/mcpSecurity.d.ts.map +1 -0
- package/dist/src/utils/mcpSecurity.js +39 -0
- package/dist/src/utils/mcpSecurity.js.map +1 -0
- package/dist/src/utils/packageRoot.d.ts +2 -0
- package/dist/src/utils/packageRoot.d.ts.map +1 -0
- package/dist/src/utils/packageRoot.js +16 -0
- package/dist/src/utils/packageRoot.js.map +1 -0
- package/dist/src/utils/python.d.ts +2 -0
- package/dist/src/utils/python.d.ts.map +1 -0
- package/dist/src/utils/python.js +72 -0
- package/dist/src/utils/python.js.map +1 -0
- package/dist/src/utils/pythonProcessEnv.d.ts +3 -0
- package/dist/src/utils/pythonProcessEnv.d.ts.map +1 -0
- package/dist/src/utils/pythonProcessEnv.js +46 -0
- package/dist/src/utils/pythonProcessEnv.js.map +1 -0
- package/dist/src/utils/talkerUtils.d.ts +4 -1
- package/dist/src/utils/talkerUtils.d.ts.map +1 -1
- package/dist/src/utils/talkerUtils.js +9 -9
- package/dist/src/utils/talkerUtils.js.map +1 -1
- package/dist/src/utils/wechatEmoji.d.ts +5 -0
- package/dist/src/utils/wechatEmoji.d.ts.map +1 -0
- package/dist/src/utils/wechatEmoji.js +35 -0
- package/dist/src/utils/wechatEmoji.js.map +1 -0
- package/docs/AI_INTERFACE.md +189 -0
- package/docs/BRANCHES.md +87 -0
- package/docs/DATA_CONTRACT.md +74 -0
- package/docs/DECISIONS.md +270 -0
- package/docs/DEEPSEEK_V4_FLASH/346/226/275/345/267/245/346/226/207/346/241/243.md +234 -0
- package/docs/EVIDENCE_GUIDE.md +96 -0
- package/docs/MCP.md +95 -0
- package/docs/PARTNERS.md +41 -0
- package/docs/PROJECT_STATE.md +96 -0
- package/docs/RELEASING.md +124 -0
- package/docs/ROADMAP.md +191 -0
- package/docs/SETUP.md +103 -0
- package/docs/images/weflow-architecture-gpt-image-2.png +0 -0
- package/docs/images/weflow-architecture.png +0 -0
- package/docs/images/weflow-architecture.svg +65 -0
- package/mcp-server/index.ts +169 -269
- package/package.json +81 -72
- package/requirements-3x.txt +4 -0
- package/requirements-voice.txt +15 -0
- package/requirements.txt +9 -0
- package/resources/js/marked.min.js +69 -0
- package/resources/wechat-emoji/666.png +0 -0
- package/resources/wechat-emoji/Awesome.png +0 -0
- package/resources/wechat-emoji/Concerned.png +0 -0
- package/resources/wechat-emoji/Cry.png +0 -0
- package/resources/wechat-emoji/Emm.png +0 -0
- package/resources/wechat-emoji/Facepalm.png +0 -0
- package/resources/wechat-emoji/Grin.png +0 -0
- package/resources/wechat-emoji/OK.png +0 -0
- package/resources/wechat-emoji/Respect.png +0 -0
- package/resources/wechat-emoji/Sick.png +0 -0
- package/resources/wechat-emoji/Sleep.png +0 -0
- package/resources/wechat-emoji/Smile.png +0 -0
- package/resources/wechat-emoji//344/272/262/344/272/262.png +0 -0
- package/resources/wechat-emoji//344/276/277/344/276/277.png +0 -0
- package/resources/wechat-emoji//345/201/267/347/254/221.png +0 -0
- package/resources/wechat-emoji//345/202/262/346/205/242.png +0 -0
- package/resources/wechat-emoji//345/206/215/350/247/201.png +0 -0
- package/resources/wechat-emoji//345/207/213/350/260/242.png +0 -0
- package/resources/wechat-emoji//345/212/240/346/262/271.png +0 -0
- package/resources/wechat-emoji//345/213/276/345/274/225.png +0 -0
- package/resources/wechat-emoji//345/217/221/345/221/206.png +0 -0
- package/resources/wechat-emoji//345/217/221/346/200/222.png +0 -0
- package/resources/wechat-emoji//345/217/221/346/212/226.png +0 -0
- package/resources/wechat-emoji//345/217/257/346/200/234.png +0 -0
- package/resources/wechat-emoji//345/217/263/345/223/274/345/223/274.png +0 -0
- package/resources/wechat-emoji//345/217/271/346/260/224.png +0 -0
- package/resources/wechat-emoji//345/220/203/347/223/234.png +0 -0
- package/resources/wechat-emoji//345/220/210/345/215/201.png +0 -0
- package/resources/wechat-emoji//345/220/220.png +0 -0
- package/resources/wechat-emoji//345/221/262/347/211/231.png +0 -0
- package/resources/wechat-emoji//345/222/222/351/252/202.png +0 -0
- package/resources/wechat-emoji//345/222/226/345/225/241.png +0 -0
- package/resources/wechat-emoji//345/223/207.png +0 -0
- package/resources/wechat-emoji//345/225/244/351/205/222.png +0 -0
- package/resources/wechat-emoji//345/230/230.png +0 -0
- package/resources/wechat-emoji//345/230/264/345/224/207.png +0 -0
- package/resources/wechat-emoji//345/230/277/345/223/210.png +0 -0
- package/resources/wechat-emoji//345/233/247.png +0 -0
- package/resources/wechat-emoji//345/233/260.png +0 -0
- package/resources/wechat-emoji//345/235/217/347/254/221.png +0 -0
- package/resources/wechat-emoji//345/244/247/345/223/255.png +0 -0
- package/resources/wechat-emoji//345/244/251/345/225/212.png +0 -0
- package/resources/wechat-emoji//345/244/252/351/230/263.png +0 -0
- package/resources/wechat-emoji//345/244/261/346/234/233.png +0 -0
- package/resources/wechat-emoji//345/245/270/347/254/221.png +0 -0
- package/resources/wechat-emoji//345/245/275/347/232/204.png +0 -0
- package/resources/wechat-emoji//345/247/224/345/261/210.png +0 -0
- package/resources/wechat-emoji//345/256/263/347/276/236.png +0 -0
- package/resources/wechat-emoji//345/260/264/345/260/254.png +0 -0
- package/resources/wechat-emoji//345/272/206/347/245/235.png +0 -0
- package/resources/wechat-emoji//345/274/261.png +0 -0
- package/resources/wechat-emoji//345/274/272.png +0 -0
- package/resources/wechat-emoji//345/276/227/346/204/217.png +0 -0
- package/resources/wechat-emoji//345/276/256/347/254/221.png +0 -0
- package/resources/wechat-emoji//345/277/203/347/242/216.png +0 -0
- package/resources/wechat-emoji//345/277/253/345/223/255/344/272/206.png +0 -0
- package/resources/wechat-emoji//346/201/220/346/203/247.png +0 -0
- package/resources/wechat-emoji//346/202/240/351/227/262.png +0 -0
- package/resources/wechat-emoji//346/203/212/346/201/220.png +0 -0
- package/resources/wechat-emoji//346/203/212/350/256/266.png +0 -0
- package/resources/wechat-emoji//346/204/211/345/277/253.png +0 -0
- package/resources/wechat-emoji//346/206/250/347/254/221.png +0 -0
- package/resources/wechat-emoji//346/211/223/350/204/270.png +0 -0
- package/resources/wechat-emoji//346/212/223/347/213/202.png +0 -0
- package/resources/wechat-emoji//346/212/240/351/274/273.png +0 -0
- package/resources/wechat-emoji//346/212/261/346/213/263.png +0 -0
- package/resources/wechat-emoji//346/213/245/346/212/261.png +0 -0
- package/resources/wechat-emoji//346/213/263/345/244/264.png +0 -0
- package/resources/wechat-emoji//346/215/202/350/204/270.png +0 -0
- package/resources/wechat-emoji//346/217/241/346/211/213.png +0 -0
- package/resources/wechat-emoji//346/222/207/345/230/264.png +0 -0
- package/resources/wechat-emoji//346/223/246/346/261/227.png +0 -0
- package/resources/wechat-emoji//346/225/262/346/211/223.png +0 -0
- package/resources/wechat-emoji//346/227/240/350/257/255.png +0 -0
- package/resources/wechat-emoji//346/227/272/346/237/264.png +0 -0
- package/resources/wechat-emoji//346/231/225.png +0 -0
- package/resources/wechat-emoji//346/234/210/344/272/256.png +0 -0
- package/resources/wechat-emoji//346/234/272/346/231/272.png +0 -0
- package/resources/wechat-emoji//346/261/227.png +0 -0
- package/resources/wechat-emoji//346/265/201/346/263/252.png +0 -0
- package/resources/wechat-emoji//347/202/270/345/274/271.png +0 -0
- package/resources/wechat-emoji//347/203/237/350/212/261.png +0 -0
- package/resources/wechat-emoji//347/210/206/347/253/271.png +0 -0
- package/resources/wechat-emoji//347/210/261/345/277/203.png +0 -0
- package/resources/wechat-emoji//347/214/252/345/244/264.png +0 -0
- package/resources/wechat-emoji//347/216/253/347/221/260.png +0 -0
- package/resources/wechat-emoji//347/224/237/347/227/205.png +0 -0
- package/resources/wechat-emoji//347/226/221/351/227/256.png +0 -0
- package/resources/wechat-emoji//347/231/274.png +0 -0
- package/resources/wechat-emoji//347/231/275/347/234/274.png +0 -0
- package/resources/wechat-emoji//347/232/261/347/234/211.png +0 -0
- package/resources/wechat-emoji//347/235/241.png +0 -0
- package/resources/wechat-emoji//347/240/264/346/266/225/344/270/272/347/254/221.png +0 -0
- package/resources/wechat-emoji//347/244/274/347/211/251.png +0 -0
- package/resources/wechat-emoji//347/244/276/344/274/232/347/244/276/344/274/232.png +0 -0
- package/resources/wechat-emoji//347/246/217.png +0 -0
- package/resources/wechat-emoji//347/254/221/350/204/270.png +0 -0
- package/resources/wechat-emoji//347/272/242/345/214/205.png +0 -0
- package/resources/wechat-emoji//347/277/273/347/231/275/347/234/274.png +0 -0
- package/resources/wechat-emoji//350/200/266.png +0 -0
- package/resources/wechat-emoji//350/203/234/345/210/251.png +0 -0
- package/resources/wechat-emoji//350/204/270/347/272/242.png +0 -0
- package/resources/wechat-emoji//350/211/262.png +0 -0
- package/resources/wechat-emoji//350/213/246/346/266/251.png +0 -0
- package/resources/wechat-emoji//350/217/234/345/210/200.png +0 -0
- package/resources/wechat-emoji//350/233/213/347/263/225.png +0 -0
- package/resources/wechat-emoji//350/241/260.png +0 -0
- package/resources/wechat-emoji//350/243/202/345/274/200.png +0 -0
- package/resources/wechat-emoji//350/256/251/346/210/221/347/234/213/347/234/213.png +0 -0
- package/resources/wechat-emoji//350/260/203/347/232/256.png +0 -0
- package/resources/wechat-emoji//350/267/263/350/267/263.png +0 -0
- package/resources/wechat-emoji//350/275/254/345/234/210.png +0 -0
- package/resources/wechat-emoji//351/204/231/350/247/206.png +0 -0
- package/resources/wechat-emoji//351/227/255/345/230/264.png +0 -0
- package/resources/wechat-emoji//351/230/264/351/231/251.png +0 -0
- package/resources/wechat-emoji//351/232/276/350/277/207.png +0 -0
- package/resources/wechat-emoji//351/252/267/351/253/205.png +0 -0
- package/resources/wechat-emoji//351/274/223/346/216/214.png +0 -0
- package/scripts/_batch_link_sources.py +73 -73
- package/scripts/_utils.py +29 -0
- package/scripts/annual_report.py +796 -796
- package/scripts/auto_tag.py +128 -128
- package/scripts/biz_daily.py +210 -53
- package/scripts/chat_report.py +10 -0
- package/scripts/chat_stats.py +702 -702
- package/scripts/classify_daily.py +17 -11
- package/scripts/create_reading_notes.py +370 -370
- package/scripts/daily_stats.py +132 -0
- package/scripts/enrich_backlinks.py +163 -163
- package/scripts/export_chat_html.py +1704 -107
- package/scripts/extract_todos.py +445 -404
- package/scripts/fav_server.py +119 -12
- package/scripts/fix_topics.py +111 -111
- package/scripts/generate_ai_report.py +1 -1
- package/scripts/generate_html.py +597 -277
- package/scripts/inspection_records.py +354 -0
- package/scripts/mcp_bridge.py +398 -398
- package/scripts/nt_decrypt.py +702 -48
- package/scripts/pipeline.py +27 -19
- package/scripts/promote_all.py +312 -310
- package/scripts/promote_ideas.py +299 -297
- package/scripts/rag_chat.py +194 -191
- package/scripts/semantic_search.py +435 -431
- package/scripts/sync_fav.py +201 -146
- package/scripts/sync_weread.py +192 -192
- package/scripts/vault_rag.py +140 -139
- package/scripts/vault_search.py +141 -141
- package/scripts/wechat_emoji.py +132 -0
- package/scripts/wechat_emoticon.py +433 -0
- package/scripts/wechat_image.py +258 -0
- package/scripts/wechat_voice.py +327 -0
- package/src/core/dbPathService.ts +163 -18
- package/src/core/keyService.ts +137 -25
- package/src/core/ntCore.ts +215 -62
- package/src/core/sqlcipherCore.ts +17 -1
- package/src/core/wcdbCore.ts +3 -7
- package/src/services/assistantDaemon.ts +111 -0
- package/src/services/assistantMemory.ts +193 -0
- package/src/services/assistantPrivacy.ts +87 -0
- package/src/services/assistantRouting.ts +92 -0
- package/src/services/assistantService.ts +295 -0
- package/src/services/assistantTools.ts +549 -0
- package/src/services/chatService.ts +133 -4
- package/src/services/configService.ts +194 -6
- package/src/services/evidenceService.ts +129 -0
- package/src/services/exportService.ts +113 -48
- package/src/services/initKeyService.ts +83 -0
- package/src/services/messageContract.ts +77 -0
- package/src/services/messageQuery.ts +38 -0
- package/src/services/wechat-formatter.ts +348 -348
- package/src/services/wechatMessageService.ts +25 -3
- package/src/services/wereadService.ts +308 -308
- package/src/services/whitelistService.ts +209 -22
- package/src/types.ts +20 -0
- package/src/utils/dateRange.ts +58 -0
- package/src/utils/mcpSecurity.ts +39 -0
- package/src/utils/packageRoot.ts +14 -0
- package/src/utils/python.ts +73 -0
- package/src/utils/pythonProcessEnv.ts +45 -0
- package/src/utils/talkerUtils.ts +14 -2
- package/dist/src/utils/errors.d.ts +0 -25
- package/dist/src/utils/errors.d.ts.map +0 -1
- package/dist/src/utils/errors.js +0 -43
- package/dist/src/utils/errors.js.map +0 -1
- package/dist/src/utils/pythonRunner.d.ts +0 -27
- package/dist/src/utils/pythonRunner.d.ts.map +0 -1
- package/dist/src/utils/pythonRunner.js +0 -62
- package/dist/src/utils/pythonRunner.js.map +0 -1
- package/src/utils/errors.ts +0 -42
- package/src/utils/pythonRunner.ts +0 -81
|
@@ -5,6 +5,7 @@ Splits large conversations into multiple parts.
|
|
|
5
5
|
Embeds cached image thumbnails from NT cache directory.
|
|
6
6
|
"""
|
|
7
7
|
import sys
|
|
8
|
+
import time
|
|
8
9
|
import os
|
|
9
10
|
import hashlib
|
|
10
11
|
import datetime
|
|
@@ -13,6 +14,9 @@ import re
|
|
|
13
14
|
import base64
|
|
14
15
|
import urllib.request
|
|
15
16
|
import urllib.error
|
|
17
|
+
import concurrent.futures
|
|
18
|
+
import struct
|
|
19
|
+
import json
|
|
16
20
|
from pathlib import Path
|
|
17
21
|
|
|
18
22
|
try:
|
|
@@ -27,7 +31,97 @@ MSG_TYPES = {
|
|
|
27
31
|
43: 'video', 47: 'emoji', 48: 'location', 49: 'link',
|
|
28
32
|
50: 'voip', 10000: 'system', 10002: 'quote',
|
|
29
33
|
}
|
|
30
|
-
MAX_EMBED_SIZE =
|
|
34
|
+
MAX_EMBED_SIZE = 8 * 1024 * 1024 # Bound self-contained HTML growth per image.
|
|
35
|
+
V2_MAGIC = b'\x07\x08V2\x08\x07'
|
|
36
|
+
V2_CIPHERTEXT_START = 0x0F
|
|
37
|
+
BUILTIN_EMOJI_DIR = os.path.join(os.path.dirname(__file__), '..', 'resources', 'wechat-emoji')
|
|
38
|
+
# Labels used by WeChat's built-in default emoji. These messages may only
|
|
39
|
+
# retain a PUA/signature marker in the export, so their CDN media is not
|
|
40
|
+
# recoverable from the message row itself.
|
|
41
|
+
#
|
|
42
|
+
# The table is derived from the artwork actually present in BUILTIN_EMOJI_DIR
|
|
43
|
+
# rather than hand-maintained: adding a PNG named after the face is enough.
|
|
44
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
45
|
+
try:
|
|
46
|
+
import wechat_emoji
|
|
47
|
+
BUILTIN_EMOJI_MAP = {f'[{name}]': name for name in wechat_emoji.IMAGE_FACES}
|
|
48
|
+
_WECHAT_EMOJI = True
|
|
49
|
+
import wechat_emoji as _face_index
|
|
50
|
+
except Exception:
|
|
51
|
+
BUILTIN_EMOJI_MAP = {}
|
|
52
|
+
_WECHAT_EMOJI = False
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
import wechat_emoticon
|
|
56
|
+
_WECHAT_EMOTICON = True
|
|
57
|
+
except Exception:
|
|
58
|
+
_WECHAT_EMOTICON = False
|
|
59
|
+
|
|
60
|
+
try:
|
|
61
|
+
import wechat_image
|
|
62
|
+
_WECHAT_IMAGE = True
|
|
63
|
+
except Exception:
|
|
64
|
+
_WECHAT_IMAGE = False
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
import wechat_voice
|
|
68
|
+
_WECHAT_VOICE = True
|
|
69
|
+
except Exception:
|
|
70
|
+
_WECHAT_VOICE = False
|
|
71
|
+
|
|
72
|
+
# Remote media is the single slowest step of an export: each miss costs a page
|
|
73
|
+
# fetch plus an image download, and inline thumbnails alone run to several
|
|
74
|
+
# hundred per conversation. Bound it per run and keep results on disk, misses
|
|
75
|
+
# included, so re-exports are instant. Set by main().
|
|
76
|
+
COVER_STATE = {
|
|
77
|
+
'dir': '', 'budget': 0, 'fetched': 0, 'cached': 0, 'skipped': 0,
|
|
78
|
+
'thumb_budget': 0, 'thumb_fetched': 0, 'thumb_cached': 0, 'thumb_skipped': 0,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
# Off by default: full-resolution originals are 5-24MB / 3000-5700px each, and
|
|
82
|
+
# downscaling ~500 of them costs minutes (PIL decode dominates). The cache's
|
|
83
|
+
# own thumbnails need no processing at all and are what an export normally
|
|
84
|
+
# wants. Turn on for maximum fidelity at the cost of a much slower export.
|
|
85
|
+
FULL_IMAGES = os.environ.get('WEFLOW_FULL_IMAGES', '') == '1'
|
|
86
|
+
|
|
87
|
+
# og:image lives in <head>; WeChat article pages are multi-MB, so reading the
|
|
88
|
+
# whole thing to find it wastes seconds per article.
|
|
89
|
+
COVER_HEAD_BYTES = 64 * 1024
|
|
90
|
+
|
|
91
|
+
# Base64 length above which an account-index original is re-encoded before
|
|
92
|
+
# embedding. Below it the file is already display-sized and PIL would only
|
|
93
|
+
# cost time, which matters across thousands of media entries.
|
|
94
|
+
EMBED_SHRINK_THRESHOLD = 400 * 1024
|
|
95
|
+
|
|
96
|
+
# Two separate budgets: share-page covers are rare and expensive, inline
|
|
97
|
+
# appmsg thumbnails are common and cheap. Sharing one pool lets a run of
|
|
98
|
+
# thumbnails starve the covers that actually change how a page looks.
|
|
99
|
+
COVER_FETCH_LIMIT = 60
|
|
100
|
+
THUMB_FETCH_LIMIT = 300
|
|
101
|
+
|
|
102
|
+
# Set up by main(); custom stickers decrypted from WeChat's local cache.
|
|
103
|
+
STICKER_STATE = {'key': b'', 'dirs': [], 'cache_dir': ''}
|
|
104
|
+
|
|
105
|
+
# {wxid: remark/nickname}, loaded by main(). Group rows name their sender only
|
|
106
|
+
# by wxid, so without this a group transcript is unreadable. Empty when the
|
|
107
|
+
# contact database is unavailable, in which case ids are shown as-is.
|
|
108
|
+
CONTACT_NAMES = {}
|
|
109
|
+
|
|
110
|
+
# Voice messages: {local_id: silk bytes} for this conversation, plus the
|
|
111
|
+
# transcript cache. The export only ever *reads* transcripts - recognising
|
|
112
|
+
# speech runs ~1.4x the audio's own length on CPU, so it is a separate,
|
|
113
|
+
# resumable pass (`scripts/wechat_voice.py`) rather than part of an export.
|
|
114
|
+
VOICE_STATE = {'map': {}, 'cache': None}
|
|
115
|
+
|
|
116
|
+
# Remote media is discovered one message at a time, but a conversation needs
|
|
117
|
+
# it from hundreds of messages at once - and a single message rarely needs
|
|
118
|
+
# more than one URL, so nothing is ever fetched in parallel. main() therefore
|
|
119
|
+
# runs a throwaway formatting pass that only records the URLs (0.1s, no
|
|
120
|
+
# network), fetches them concurrently, then formats for real against a warm
|
|
121
|
+
# cache. Set to a list during that pass; None otherwise.
|
|
122
|
+
PREFETCH = {'sink': None, 'active': False}
|
|
123
|
+
PREFETCH_WORKERS = 24
|
|
124
|
+
PREFETCH_MAX_URLS = 800
|
|
31
125
|
|
|
32
126
|
|
|
33
127
|
def connect(db_path, key_hex, salt_hex):
|
|
@@ -38,21 +132,35 @@ def connect(db_path, key_hex, salt_hex):
|
|
|
38
132
|
return conn, c
|
|
39
133
|
|
|
40
134
|
|
|
41
|
-
def fetch_messages(conn, talker):
|
|
135
|
+
def fetch_messages(conn, talker, date=''):
|
|
42
136
|
"""Fetch all messages for a talker, ordered by time ascending."""
|
|
43
137
|
tbl = 'Msg_' + hashlib.md5(talker.encode()).hexdigest()
|
|
44
138
|
c = conn.cursor()
|
|
45
139
|
|
|
140
|
+
c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (tbl,))
|
|
141
|
+
if not c.fetchone():
|
|
142
|
+
return []
|
|
143
|
+
|
|
46
144
|
c.execute(f"SELECT COUNT(*) FROM \"{tbl}\"")
|
|
47
145
|
total = c.fetchone()[0]
|
|
48
146
|
print(f"Total messages: {total}")
|
|
49
147
|
|
|
148
|
+
date_filter = ''
|
|
149
|
+
date_params = []
|
|
150
|
+
if date:
|
|
151
|
+
try:
|
|
152
|
+
day = datetime.datetime.strptime(date, '%Y-%m-%d')
|
|
153
|
+
except ValueError:
|
|
154
|
+
raise ValueError('date must use YYYY-MM-DD')
|
|
155
|
+
date_filter = ' WHERE create_time >= ? AND create_time < ?'
|
|
156
|
+
date_params = [int(day.timestamp()), int((day + datetime.timedelta(days=1)).timestamp())]
|
|
157
|
+
|
|
50
158
|
c.execute(f'''
|
|
51
159
|
SELECT local_id, server_id, local_type, sort_seq, real_sender_id,
|
|
52
160
|
create_time, status, source, message_content, compress_content
|
|
53
|
-
FROM "{tbl}"
|
|
161
|
+
FROM "{tbl}"{date_filter}
|
|
54
162
|
ORDER BY create_time ASC
|
|
55
|
-
''')
|
|
163
|
+
''', date_params)
|
|
56
164
|
|
|
57
165
|
messages = []
|
|
58
166
|
batch = 0
|
|
@@ -68,6 +176,52 @@ def fetch_messages(conn, talker):
|
|
|
68
176
|
return messages
|
|
69
177
|
|
|
70
178
|
|
|
179
|
+
def discover_message_shards(db_path):
|
|
180
|
+
"""Return all NT message shards alongside the configured database."""
|
|
181
|
+
db = Path(db_path)
|
|
182
|
+
candidates = sorted(db.parent.glob('message_*.db'))
|
|
183
|
+
return [str(path) for path in candidates if path.name.lower() not in {
|
|
184
|
+
'message_fts.db', 'message_resource.db'
|
|
185
|
+
}]
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def derive_database_key(path, fallback_key, fallback_salt, passphrase=''):
|
|
189
|
+
"""Derive a shard-specific SQLCipher key from the shared NT passphrase."""
|
|
190
|
+
if not passphrase:
|
|
191
|
+
return fallback_key, fallback_salt
|
|
192
|
+
try:
|
|
193
|
+
with open(path, 'rb') as fh:
|
|
194
|
+
salt = fh.read(16)
|
|
195
|
+
if len(salt) != 16:
|
|
196
|
+
return fallback_key, fallback_salt
|
|
197
|
+
raw_passphrase = bytes.fromhex(passphrase)
|
|
198
|
+
key = hashlib.pbkdf2_hmac('sha512', raw_passphrase, salt, 256000, 32).hex()
|
|
199
|
+
return key, salt.hex()
|
|
200
|
+
except (OSError, ValueError):
|
|
201
|
+
return fallback_key, fallback_salt
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def fetch_messages_from_shards(db_path, key_hex, salt_hex, talker, date='', passphrase=''):
|
|
205
|
+
"""Read and merge messages from every NT message shard."""
|
|
206
|
+
shards = discover_message_shards(db_path)
|
|
207
|
+
if not shards:
|
|
208
|
+
shards = [db_path]
|
|
209
|
+
messages = []
|
|
210
|
+
for shard in shards:
|
|
211
|
+
shard_conn = None
|
|
212
|
+
try:
|
|
213
|
+
shard_key, shard_salt = derive_database_key(shard, key_hex, salt_hex, passphrase)
|
|
214
|
+
shard_conn, _ = connect(shard, shard_key, shard_salt)
|
|
215
|
+
messages.extend(fetch_messages(shard_conn, talker, date))
|
|
216
|
+
except Exception:
|
|
217
|
+
continue
|
|
218
|
+
finally:
|
|
219
|
+
if shard_conn is not None:
|
|
220
|
+
shard_conn.close()
|
|
221
|
+
messages.sort(key=lambda row: (int(row[5] or 0), int(row[0] or 0)))
|
|
222
|
+
return messages
|
|
223
|
+
|
|
224
|
+
|
|
71
225
|
def build_sender_map(conn, talker):
|
|
72
226
|
"""Map sender_id -> display name using Name2Id table and contact DB."""
|
|
73
227
|
sender_map = {}
|
|
@@ -89,7 +243,24 @@ def build_sender_map(conn, talker):
|
|
|
89
243
|
return sender_map
|
|
90
244
|
|
|
91
245
|
|
|
92
|
-
def
|
|
246
|
+
def build_sender_map_from_shards(db_path, key_hex, salt_hex, talker, passphrase=''):
|
|
247
|
+
"""Merge sender mappings from every message shard."""
|
|
248
|
+
sender_map = {}
|
|
249
|
+
for shard in discover_message_shards(db_path):
|
|
250
|
+
shard_conn = None
|
|
251
|
+
try:
|
|
252
|
+
shard_key, shard_salt = derive_database_key(shard, key_hex, salt_hex, passphrase)
|
|
253
|
+
shard_conn, _ = connect(shard, shard_key, shard_salt)
|
|
254
|
+
sender_map.update(build_sender_map(shard_conn, talker))
|
|
255
|
+
except Exception:
|
|
256
|
+
continue
|
|
257
|
+
finally:
|
|
258
|
+
if shard_conn is not None:
|
|
259
|
+
shard_conn.close()
|
|
260
|
+
return sender_map
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def scan_nt_cache(nt_cache_dir, talker, account_dir='', own_wxid=''):
|
|
93
264
|
"""Scan NT cache directory for image thumbnails and temp images.
|
|
94
265
|
|
|
95
266
|
NT cache structure:
|
|
@@ -131,7 +302,13 @@ def scan_nt_cache(nt_cache_dir, talker):
|
|
|
131
302
|
with open(fpath, 'rb') as fh:
|
|
132
303
|
data = fh.read()
|
|
133
304
|
if len(data) < MAX_EMBED_SIZE:
|
|
305
|
+
if _WECHAT_IMAGE and FULL_IMAGES:
|
|
306
|
+
data, mime = wechat_image.shrink(data, mime, max_side=480)
|
|
134
307
|
image_map[local_id] = (base64.b64encode(data).decode(), mime)
|
|
308
|
+
cache_time = parse_cache_timestamp(fname)
|
|
309
|
+
if cache_time:
|
|
310
|
+
image_map[f'pair:{local_id}:{cache_time}'] = image_map[local_id]
|
|
311
|
+
image_map[f'time:{cache_time}'] = image_map[local_id]
|
|
135
312
|
except:
|
|
136
313
|
pass
|
|
137
314
|
|
|
@@ -152,18 +329,435 @@ def scan_nt_cache(nt_cache_dir, talker):
|
|
|
152
329
|
with open(fpath, 'rb') as fh:
|
|
153
330
|
data = fh.read()
|
|
154
331
|
if len(data) < MAX_EMBED_SIZE:
|
|
155
|
-
|
|
332
|
+
if _WECHAT_IMAGE and FULL_IMAGES:
|
|
333
|
+
data, _m = wechat_image.shrink(data, 'image/jpeg', max_side=480)
|
|
334
|
+
image_map[local_id] = (base64.b64encode(data).decode(), _m if _WECHAT_IMAGE else 'image/jpeg')
|
|
335
|
+
cache_time = parse_cache_timestamp(fname)
|
|
336
|
+
if cache_time:
|
|
337
|
+
image_map[f'pair:{local_id}:{cache_time}'] = image_map[local_id]
|
|
338
|
+
image_map[f'time:{cache_time}'] = image_map[local_id]
|
|
156
339
|
except:
|
|
157
340
|
pass
|
|
158
341
|
|
|
342
|
+
for root, _, files in os.walk(msg_dir):
|
|
343
|
+
if root == img_temp_dir:
|
|
344
|
+
continue
|
|
345
|
+
for fname in files:
|
|
346
|
+
parts = fname.split('_', 1)
|
|
347
|
+
local_id = int(parts[0]) if parts and parts[0].isdigit() else None
|
|
348
|
+
file_md5 = extract_media_md5(fname)
|
|
349
|
+
if local_id is None and not file_md5:
|
|
350
|
+
continue
|
|
351
|
+
cache_time = parse_cache_timestamp(fname)
|
|
352
|
+
if local_id is not None and (local_id in image_map or f'time:{cache_time}' in image_map):
|
|
353
|
+
continue
|
|
354
|
+
if file_md5 and f'md5:{file_md5}' in image_map:
|
|
355
|
+
continue
|
|
356
|
+
fpath = os.path.join(root, fname)
|
|
357
|
+
try:
|
|
358
|
+
if os.path.getsize(fpath) >= MAX_EMBED_SIZE:
|
|
359
|
+
continue
|
|
360
|
+
mime = detect_mime(fpath)
|
|
361
|
+
if not mime:
|
|
362
|
+
continue
|
|
363
|
+
with open(fpath, 'rb') as fh:
|
|
364
|
+
data = fh.read()
|
|
365
|
+
if data:
|
|
366
|
+
if _WECHAT_IMAGE and FULL_IMAGES:
|
|
367
|
+
data, mime = wechat_image.shrink(data, mime, max_side=480)
|
|
368
|
+
if not data:
|
|
369
|
+
continue
|
|
370
|
+
image = (base64.b64encode(data).decode(), mime)
|
|
371
|
+
if local_id is not None:
|
|
372
|
+
image_map[local_id] = image
|
|
373
|
+
if cache_time:
|
|
374
|
+
if local_id is not None:
|
|
375
|
+
image_map[f'pair:{local_id}:{cache_time}'] = image
|
|
376
|
+
image_map[f'time:{cache_time}'] = image
|
|
377
|
+
if file_md5:
|
|
378
|
+
image_map[f'md5:{file_md5}'] = image
|
|
379
|
+
except OSError:
|
|
380
|
+
continue
|
|
381
|
+
|
|
382
|
+
# The account media index is where full-resolution originals live; without
|
|
383
|
+
# it an export uses only this conversation's own cache thumbnails.
|
|
384
|
+
if FULL_IMAGES and account_dir and os.path.isdir(account_dir):
|
|
385
|
+
for key, image in scan_account_media(account_dir, own_wxid, talker).items():
|
|
386
|
+
if key.startswith('md5:'):
|
|
387
|
+
image_map.setdefault(key, image)
|
|
388
|
+
|
|
389
|
+
return image_map
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def scan_account_media(account_dir, own_wxid='', talker=''):
|
|
393
|
+
"""Index image resources stored outside a conversation cache directory.
|
|
394
|
+
|
|
395
|
+
Scoped to `talker` whenever it is known. Walking the whole account means
|
|
396
|
+
stat-ing and probing ~20k files (and 255-way XOR on every non-image), which
|
|
397
|
+
dominated export time; a conversation only ever needs its own media.
|
|
398
|
+
"""
|
|
399
|
+
image_map = {}
|
|
400
|
+
v2_key = resolve_v2_media_key(account_dir, own_wxid)
|
|
401
|
+
if talker:
|
|
402
|
+
talker_md5 = hashlib.md5(talker.encode()).hexdigest()
|
|
403
|
+
roots = [os.path.join(account_dir, 'msg', 'attach', talker_md5)]
|
|
404
|
+
cache_root = os.path.join(account_dir, 'cache')
|
|
405
|
+
if os.path.isdir(cache_root):
|
|
406
|
+
for month in sorted(os.listdir(cache_root)):
|
|
407
|
+
d = os.path.join(cache_root, month, 'Message', talker_md5)
|
|
408
|
+
if os.path.isdir(d):
|
|
409
|
+
roots.append(d)
|
|
410
|
+
else:
|
|
411
|
+
roots = [
|
|
412
|
+
os.path.join(account_dir, 'cache'),
|
|
413
|
+
os.path.join(account_dir, 'msg'),
|
|
414
|
+
os.path.join(account_dir, 'resource'),
|
|
415
|
+
os.path.join(account_dir, 'business'),
|
|
416
|
+
os.path.join(account_dir, 'temp'),
|
|
417
|
+
]
|
|
418
|
+
seen = set()
|
|
419
|
+
for root in roots:
|
|
420
|
+
if not os.path.isdir(root):
|
|
421
|
+
continue
|
|
422
|
+
for current_root, _, files in os.walk(root):
|
|
423
|
+
for fname in files:
|
|
424
|
+
path = os.path.join(current_root, fname)
|
|
425
|
+
try:
|
|
426
|
+
stat = os.stat(path)
|
|
427
|
+
if stat.st_size <= 16 or stat.st_size > MAX_EMBED_SIZE:
|
|
428
|
+
continue
|
|
429
|
+
real_path = os.path.realpath(path)
|
|
430
|
+
if real_path in seen:
|
|
431
|
+
continue
|
|
432
|
+
seen.add(real_path)
|
|
433
|
+
with open(path, 'rb') as fh:
|
|
434
|
+
header = fh.read(64)
|
|
435
|
+
mime = detect_mime_from_bytes(header[:16])
|
|
436
|
+
if not mime:
|
|
437
|
+
decoded = decode_wechat_media(header, path, v2_key)
|
|
438
|
+
if decoded:
|
|
439
|
+
data, mime = decoded
|
|
440
|
+
else:
|
|
441
|
+
with open(path, 'rb') as fh:
|
|
442
|
+
data = fh.read(MAX_EMBED_SIZE + 1)
|
|
443
|
+
if not mime or len(data) > MAX_EMBED_SIZE:
|
|
444
|
+
continue
|
|
445
|
+
image = (base64.b64encode(data).decode(), mime)
|
|
446
|
+
for media_md5 in extract_media_md5s(fname):
|
|
447
|
+
image_map.setdefault(f'md5:{media_md5}', image)
|
|
448
|
+
content_md5 = hashlib.md5(data).hexdigest()
|
|
449
|
+
image_map.setdefault(f'md5:{content_md5}', image)
|
|
450
|
+
except (OSError, ValueError):
|
|
451
|
+
continue
|
|
159
452
|
return image_map
|
|
160
453
|
|
|
161
454
|
|
|
455
|
+
def clean_account_wxid(value):
|
|
456
|
+
value = str(value or '').strip()
|
|
457
|
+
parts = value.rsplit('_', 1)
|
|
458
|
+
if len(parts) == 2 and len(parts[1]) == 4 and parts[1].isalnum():
|
|
459
|
+
return parts[0]
|
|
460
|
+
return value
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def resolve_v2_media_key(account_dir, own_wxid='', kvcomm_dir=''):
|
|
464
|
+
"""Derive and verify the local WeChat V2 image key without persisting it."""
|
|
465
|
+
if not account_dir:
|
|
466
|
+
return None
|
|
467
|
+
if not kvcomm_dir:
|
|
468
|
+
appdata = os.environ.get('APPDATA', '')
|
|
469
|
+
kvcomm_dir = os.path.join(appdata, 'Tencent', 'xwechat', 'net', 'kvcomm')
|
|
470
|
+
try:
|
|
471
|
+
codes = sorted({
|
|
472
|
+
int(match.group(1))
|
|
473
|
+
for name in os.listdir(kvcomm_dir)
|
|
474
|
+
if (match := re.fullmatch(r'key_(\d+)_.+\.statistic', name, re.IGNORECASE))
|
|
475
|
+
})
|
|
476
|
+
except (OSError, ValueError):
|
|
477
|
+
return None
|
|
478
|
+
if not codes:
|
|
479
|
+
return None
|
|
480
|
+
|
|
481
|
+
templates = []
|
|
482
|
+
for root in ('msg', 'cache', 'resource'):
|
|
483
|
+
search_root = os.path.join(account_dir, root)
|
|
484
|
+
if not os.path.isdir(search_root):
|
|
485
|
+
continue
|
|
486
|
+
for current_root, _, files in os.walk(search_root):
|
|
487
|
+
for name in files:
|
|
488
|
+
if not name.lower().endswith('_t.dat'):
|
|
489
|
+
continue
|
|
490
|
+
path = os.path.join(current_root, name)
|
|
491
|
+
try:
|
|
492
|
+
with open(path, 'rb') as stream:
|
|
493
|
+
header = stream.read(V2_CIPHERTEXT_START + 16)
|
|
494
|
+
if header.startswith(V2_MAGIC) and len(header) >= V2_CIPHERTEXT_START + 16:
|
|
495
|
+
templates.append(header[V2_CIPHERTEXT_START:V2_CIPHERTEXT_START + 16])
|
|
496
|
+
except OSError:
|
|
497
|
+
continue
|
|
498
|
+
if len(templates) >= 32:
|
|
499
|
+
break
|
|
500
|
+
if len(templates) >= 32:
|
|
501
|
+
break
|
|
502
|
+
if len(templates) >= 32:
|
|
503
|
+
break
|
|
504
|
+
if not templates:
|
|
505
|
+
return None
|
|
506
|
+
|
|
507
|
+
wxids = list(dict.fromkeys(filter(None, (
|
|
508
|
+
clean_account_wxid(own_wxid),
|
|
509
|
+
clean_account_wxid(Path(account_dir).name),
|
|
510
|
+
))))
|
|
511
|
+
try:
|
|
512
|
+
from Crypto.Cipher import AES
|
|
513
|
+
except ImportError:
|
|
514
|
+
return None
|
|
515
|
+
for wxid in wxids:
|
|
516
|
+
for code in codes:
|
|
517
|
+
aes_key = hashlib.md5(f'{code}{wxid}'.encode()).hexdigest()[:16].encode('ascii')
|
|
518
|
+
try:
|
|
519
|
+
plaintext = AES.new(aes_key, AES.MODE_ECB).decrypt(templates[0])
|
|
520
|
+
except (TypeError, ValueError):
|
|
521
|
+
continue
|
|
522
|
+
if detect_mime_from_bytes(plaintext) or plaintext.startswith((b'wxgf', b'WXGF')):
|
|
523
|
+
return code & 0xff, aes_key
|
|
524
|
+
return None
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def decode_wechat_v2(filepath, xor_key, aes_key):
|
|
528
|
+
try:
|
|
529
|
+
from Crypto.Cipher import AES
|
|
530
|
+
from Crypto.Util import Padding
|
|
531
|
+
with open(filepath, 'rb') as stream:
|
|
532
|
+
data = stream.read(MAX_EMBED_SIZE + 1)
|
|
533
|
+
if len(data) > MAX_EMBED_SIZE or not data.startswith(V2_MAGIC):
|
|
534
|
+
return None
|
|
535
|
+
signature, aes_size, xor_size = struct.unpack('<6sLLx', data[:V2_CIPHERTEXT_START])
|
|
536
|
+
if signature != V2_MAGIC:
|
|
537
|
+
return None
|
|
538
|
+
encrypted_size = aes_size + 16 - aes_size % 16
|
|
539
|
+
encrypted = data[V2_CIPHERTEXT_START:V2_CIPHERTEXT_START + encrypted_size]
|
|
540
|
+
decrypted = Padding.unpad(AES.new(aes_key, AES.MODE_ECB).decrypt(encrypted), 16)
|
|
541
|
+
remainder = data[V2_CIPHERTEXT_START + encrypted_size:]
|
|
542
|
+
if xor_size:
|
|
543
|
+
if xor_size > len(remainder):
|
|
544
|
+
return None
|
|
545
|
+
raw = remainder[:-xor_size]
|
|
546
|
+
tail = bytes(value ^ xor_key for value in remainder[-xor_size:])
|
|
547
|
+
else:
|
|
548
|
+
raw, tail = remainder, b''
|
|
549
|
+
output = decrypted + raw + tail
|
|
550
|
+
mime = detect_mime_from_bytes(output[:16])
|
|
551
|
+
return (output, mime) if mime else None
|
|
552
|
+
except (OSError, ValueError, struct.error):
|
|
553
|
+
return None
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def decode_wechat_media(data, filepath=None, v2_key=None):
|
|
557
|
+
"""Decode common XOR-obfuscated WeChat image cache payloads."""
|
|
558
|
+
if not data or len(data) < 16:
|
|
559
|
+
return None
|
|
560
|
+
if data.startswith(V2_MAGIC) and filepath and v2_key:
|
|
561
|
+
# No downscaling here: this is called once per file while building the
|
|
562
|
+
# account media index (tens of thousands of files), so doing image work
|
|
563
|
+
# at this layer makes the index build take minutes. Shrinking happens
|
|
564
|
+
# in get_cached_image(), which only runs for images actually embedded.
|
|
565
|
+
return decode_wechat_v2(filepath, *v2_key)
|
|
566
|
+
for key in range(1, 256):
|
|
567
|
+
decoded = bytes(value ^ key for value in data[: min(len(data), 64)])
|
|
568
|
+
mime = detect_mime_from_bytes(decoded)
|
|
569
|
+
if mime:
|
|
570
|
+
if filepath:
|
|
571
|
+
try:
|
|
572
|
+
with open(filepath, 'rb') as fh:
|
|
573
|
+
raw = fh.read(MAX_EMBED_SIZE + 1)
|
|
574
|
+
if len(raw) > MAX_EMBED_SIZE:
|
|
575
|
+
return None
|
|
576
|
+
except OSError:
|
|
577
|
+
return None
|
|
578
|
+
else:
|
|
579
|
+
raw = data
|
|
580
|
+
full = bytes(value ^ key for value in raw)
|
|
581
|
+
return full, mime
|
|
582
|
+
return None
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def parse_cache_timestamp(filename):
|
|
586
|
+
parts = filename.split('_', 2)
|
|
587
|
+
if len(parts) < 2 or not parts[1].isdigit():
|
|
588
|
+
return 0
|
|
589
|
+
timestamp = int(parts[1])
|
|
590
|
+
if timestamp > 10_000_000_000:
|
|
591
|
+
timestamp //= 1000
|
|
592
|
+
return timestamp
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def extract_media_md5(value):
|
|
596
|
+
values = extract_media_md5s(value)
|
|
597
|
+
return values[0] if values else ''
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def extract_media_md5s(value):
|
|
601
|
+
matches = re.findall(r'(?<![0-9a-f])([0-9a-f]{32})(?![0-9a-f])', str(value or ''), re.IGNORECASE)
|
|
602
|
+
return list(dict.fromkeys(match.lower() for match in matches))
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def extract_blob_md5s(value, known_md5s=None):
|
|
606
|
+
if value is None:
|
|
607
|
+
return []
|
|
608
|
+
if isinstance(value, memoryview):
|
|
609
|
+
data = value.tobytes()
|
|
610
|
+
elif isinstance(value, (bytes, bytearray)):
|
|
611
|
+
data = bytes(value)
|
|
612
|
+
else:
|
|
613
|
+
data = str(value).encode('utf-8', errors='ignore')
|
|
614
|
+
matches = re.findall(rb'(?i)([0-9a-f]{32})(?:[._][thbc])?\.dat', data)
|
|
615
|
+
if not matches:
|
|
616
|
+
matches = re.findall(rb'(?i)(?<![0-9a-f])([0-9a-f]{32})(?![0-9a-f])', data)
|
|
617
|
+
result = [item.decode('ascii').lower() for item in matches]
|
|
618
|
+
# MessageResourceInfo commonly stores MD5 values as raw 16-byte fields.
|
|
619
|
+
# Index those candidates; get_cached_image will retain only candidates
|
|
620
|
+
# that resolve to an actual local media file.
|
|
621
|
+
for offset in range(0, max(0, len(data) - 15)):
|
|
622
|
+
candidate = data[offset:offset + 16]
|
|
623
|
+
candidate_hex = candidate.hex()
|
|
624
|
+
if (candidate not in (b'\x00' * 16, b'\xff' * 16)
|
|
625
|
+
and (known_md5s is None or candidate_hex in known_md5s)):
|
|
626
|
+
result.append(candidate_hex)
|
|
627
|
+
return list(dict.fromkeys(result))
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def load_resource_media_map(account_dir, key_hex, salt_hex, messages, image_map=None, passphrase=''):
|
|
631
|
+
"""Load message-resource MD5s keyed by message IDs when available."""
|
|
632
|
+
if not account_dir or not messages:
|
|
633
|
+
return {}
|
|
634
|
+
resource_db = os.path.join(account_dir, 'db_storage', 'message', 'message_resource.db')
|
|
635
|
+
if not os.path.isfile(resource_db):
|
|
636
|
+
return {}
|
|
637
|
+
server_ids = {int(row[1] or 0) for row in messages if row[1]}
|
|
638
|
+
result = {}
|
|
639
|
+
known_md5s = {
|
|
640
|
+
key[4:].lower() for key in (image_map or {})
|
|
641
|
+
if isinstance(key, str) and key.startswith('md5:')
|
|
642
|
+
}
|
|
643
|
+
conn = None
|
|
644
|
+
try:
|
|
645
|
+
resource_key, resource_salt = derive_database_key(resource_db, key_hex, salt_hex, passphrase)
|
|
646
|
+
conn, cursor = connect(resource_db, resource_key, resource_salt)
|
|
647
|
+
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND lower(name)=lower('MessageResourceInfo')")
|
|
648
|
+
table = cursor.fetchone()
|
|
649
|
+
if not table:
|
|
650
|
+
conn.close()
|
|
651
|
+
return result
|
|
652
|
+
cursor.execute('SELECT message_svr_id, message_local_id, packed_info FROM "MessageResourceInfo"')
|
|
653
|
+
for server_id, local_id, packed_info in cursor.fetchall():
|
|
654
|
+
sid = int(server_id or 0)
|
|
655
|
+
# Local IDs can collide across conversations and database shards.
|
|
656
|
+
if not sid or sid not in server_ids:
|
|
657
|
+
continue
|
|
658
|
+
md5s = extract_blob_md5s(packed_info, known_md5s)
|
|
659
|
+
if md5s:
|
|
660
|
+
result.setdefault(f'server:{sid}', []).extend(md5s)
|
|
661
|
+
conn.close()
|
|
662
|
+
except Exception:
|
|
663
|
+
try:
|
|
664
|
+
conn.close()
|
|
665
|
+
except Exception:
|
|
666
|
+
pass
|
|
667
|
+
return {key: list(dict.fromkeys(values)) for key, values in result.items()}
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
_SHRINK_MEMO = {}
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def shrink_embedded(image, max_side=720, force=False):
|
|
674
|
+
"""Downscale an (base64, mime) pair on the way out of the media index.
|
|
675
|
+
|
|
676
|
+
The index deliberately keeps originals: it is built over the whole account
|
|
677
|
+
and only a fraction of it ever gets embedded, so image work belongs here
|
|
678
|
+
rather than there.
|
|
679
|
+
|
|
680
|
+
`force` skips the opt-in gate, for media that has no smaller alternative -
|
|
681
|
+
an original pulled in from the account index because the conversation
|
|
682
|
+
cache never held it at all.
|
|
683
|
+
"""
|
|
684
|
+
if not image or not _WECHAT_IMAGE:
|
|
685
|
+
return image
|
|
686
|
+
if not force and not FULL_IMAGES:
|
|
687
|
+
return image
|
|
688
|
+
memo_key = image[0][:64] + ':' + str(len(image[0]))
|
|
689
|
+
if memo_key in _SHRINK_MEMO:
|
|
690
|
+
return _SHRINK_MEMO[memo_key]
|
|
691
|
+
|
|
692
|
+
# Disk cache: shrinking a multi-MB photo costs ~0.15s, and the same media
|
|
693
|
+
# comes back on every export. Key is the source digest, so this is safe to
|
|
694
|
+
# share across conversations.
|
|
695
|
+
cache_dir = COVER_STATE.get('imgshrink') or ''
|
|
696
|
+
disk_key = hashlib.md5(image[0][:4096].encode()).hexdigest() + str(len(image[0]))
|
|
697
|
+
disk_path = os.path.join(cache_dir, disk_key) if cache_dir else ''
|
|
698
|
+
if disk_path and os.path.isfile(disk_path):
|
|
699
|
+
try:
|
|
700
|
+
with open(disk_path, 'rb') as fh:
|
|
701
|
+
blob = fh.read()
|
|
702
|
+
nl = blob.find(b'\n')
|
|
703
|
+
if nl > 0:
|
|
704
|
+
result = (blob[:nl].decode(), blob[nl + 1:].decode())
|
|
705
|
+
_SHRINK_MEMO[memo_key] = result
|
|
706
|
+
return result
|
|
707
|
+
except OSError:
|
|
708
|
+
pass
|
|
709
|
+
|
|
710
|
+
try:
|
|
711
|
+
raw = base64.b64decode(image[0])
|
|
712
|
+
smaller, mime = wechat_image.shrink(raw, image[1], max_side=max_side)
|
|
713
|
+
result = image if smaller is raw else (base64.b64encode(smaller).decode(), mime)
|
|
714
|
+
_SHRINK_MEMO[memo_key] = result
|
|
715
|
+
if disk_path and smaller is not raw:
|
|
716
|
+
try:
|
|
717
|
+
os.makedirs(cache_dir, exist_ok=True)
|
|
718
|
+
with open(disk_path, 'wb') as fh:
|
|
719
|
+
fh.write(result[0].encode() + b'\n' + result[1].encode())
|
|
720
|
+
except OSError:
|
|
721
|
+
pass
|
|
722
|
+
return result
|
|
723
|
+
except Exception:
|
|
724
|
+
return image
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
def get_cached_image(image_map, local_id, create_time, content='', resource_md5s=None, server_id=0):
|
|
728
|
+
"""Best local media for a message: by content md5, then by identity.
|
|
729
|
+
|
|
730
|
+
md5 is the strongest key, but it is not always derivable - a group row can
|
|
731
|
+
lack the resource mapping that carries it. The conversation cache also
|
|
732
|
+
indexes by `(local_id, create_time)`, which is identity-aware and was
|
|
733
|
+
already being stored; reading it recovers media the md5 path alone misses.
|
|
734
|
+
|
|
735
|
+
Deliberately *not* falling back to a bare `local_id`: that id restarts in
|
|
736
|
+
every message shard, so matching on it alone attaches one conversation's
|
|
737
|
+
image to another conversation's message.
|
|
738
|
+
"""
|
|
739
|
+
if not image_map:
|
|
740
|
+
return None
|
|
741
|
+
for media_md5 in resource_md5s or []:
|
|
742
|
+
if image_map.get(f'md5:{media_md5}'):
|
|
743
|
+
return shrink_embedded(image_map[f'md5:{media_md5}'])
|
|
744
|
+
for media_md5 in extract_media_md5s(content):
|
|
745
|
+
if image_map.get(f'md5:{media_md5}'):
|
|
746
|
+
return shrink_embedded(image_map[f'md5:{media_md5}'])
|
|
747
|
+
try:
|
|
748
|
+
pair_key = f'pair:{int(local_id)}:{int(create_time)}'
|
|
749
|
+
except (TypeError, ValueError):
|
|
750
|
+
return None
|
|
751
|
+
if image_map.get(pair_key):
|
|
752
|
+
return shrink_embedded(image_map[pair_key])
|
|
753
|
+
return None
|
|
754
|
+
|
|
755
|
+
|
|
162
756
|
def detect_mime(filepath):
|
|
163
757
|
"""Detect MIME type from file header."""
|
|
164
758
|
try:
|
|
165
759
|
with open(filepath, 'rb') as f:
|
|
166
|
-
header = f.read(
|
|
760
|
+
header = f.read(12)
|
|
167
761
|
if header[:2] == b'\xff\xd8':
|
|
168
762
|
return 'image/jpeg'
|
|
169
763
|
if header[:4] == b'\x89PNG':
|
|
@@ -194,9 +788,10 @@ def find_thumbnail(create_time, msg_local_id, wx_dir):
|
|
|
194
788
|
fpath = os.path.join(img_dir, f)
|
|
195
789
|
if not os.path.isfile(fpath):
|
|
196
790
|
continue
|
|
791
|
+
if not f.startswith(f'{msg_local_id}_'):
|
|
792
|
+
continue
|
|
197
793
|
fstat = os.stat(fpath)
|
|
198
|
-
|
|
199
|
-
if time_diff < 300 and os.path.getsize(fpath) < MAX_EMBED_SIZE:
|
|
794
|
+
if os.path.getsize(fpath) < MAX_EMBED_SIZE:
|
|
200
795
|
with open(fpath, 'rb') as fh:
|
|
201
796
|
data = fh.read()
|
|
202
797
|
if len(data) < MAX_EMBED_SIZE:
|
|
@@ -206,28 +801,145 @@ def find_thumbnail(create_time, msg_local_id, wx_dir):
|
|
|
206
801
|
return None
|
|
207
802
|
|
|
208
803
|
|
|
209
|
-
def
|
|
210
|
-
|
|
211
|
-
if
|
|
804
|
+
def _decrypt_aes_cbc(payload, key_hex):
|
|
805
|
+
key_hex = re.sub(r'[^0-9a-f]', '', str(key_hex or ''), flags=re.IGNORECASE)
|
|
806
|
+
if len(key_hex) != 32:
|
|
212
807
|
return None
|
|
213
808
|
try:
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
return None
|
|
222
|
-
mime = resp.headers.get('Content-Type', 'image/jpeg').split(';')[0].strip()
|
|
223
|
-
if mime not in ('image/jpeg', 'image/png', 'image/gif', 'image/webp'):
|
|
224
|
-
# Try detecting from data
|
|
225
|
-
mime = detect_mime_from_bytes(data) or 'image/jpeg'
|
|
226
|
-
return (base64.b64encode(data).decode(), mime)
|
|
809
|
+
from Crypto.Cipher import AES
|
|
810
|
+
key = bytes.fromhex(key_hex)
|
|
811
|
+
decrypted = AES.new(key, AES.MODE_CBC, key).decrypt(payload)
|
|
812
|
+
padding = decrypted[-1] if decrypted else 0
|
|
813
|
+
if 0 < padding <= AES.block_size and decrypted.endswith(bytes([padding]) * padding):
|
|
814
|
+
decrypted = decrypted[:-padding]
|
|
815
|
+
return decrypted
|
|
227
816
|
except Exception:
|
|
228
817
|
return None
|
|
229
818
|
|
|
230
819
|
|
|
820
|
+
def _valid_media(data):
|
|
821
|
+
if not data:
|
|
822
|
+
return None
|
|
823
|
+
mime = detect_mime_from_bytes(data[:16])
|
|
824
|
+
if mime:
|
|
825
|
+
return data, mime
|
|
826
|
+
return None
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
# Writes a file with no '\n', which the (b64, mime) format can never produce.
|
|
830
|
+
NEGATIVE_MARKER = b'!'
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _cache_media(cache_file, payload):
|
|
834
|
+
"""Store a downloaded (b64, mime) pair, or a negative marker when None.
|
|
835
|
+
|
|
836
|
+
Misses are cached too. The bulk of these URLs are dead WeChat CDN links,
|
|
837
|
+
so without a negative cache every re-export re-attempted all of them.
|
|
838
|
+
"""
|
|
839
|
+
if not cache_file:
|
|
840
|
+
return
|
|
841
|
+
try:
|
|
842
|
+
os.makedirs(os.path.dirname(cache_file), exist_ok=True)
|
|
843
|
+
if payload is None:
|
|
844
|
+
with open(cache_file, 'wb') as fh:
|
|
845
|
+
fh.write(NEGATIVE_MARKER)
|
|
846
|
+
else:
|
|
847
|
+
with open(cache_file, 'wb') as fh:
|
|
848
|
+
fh.write(payload[0].encode() + b'\n' + payload[1].encode())
|
|
849
|
+
except OSError:
|
|
850
|
+
pass
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def _read_cache_media(cache_file):
|
|
854
|
+
"""Read a cached download.
|
|
855
|
+
|
|
856
|
+
Returns a (b64, mime) pair, None for a cached miss, or 'unknown' when the
|
|
857
|
+
path is absent or unreadable and a fetch should be attempted.
|
|
858
|
+
"""
|
|
859
|
+
if not cache_file or not os.path.isfile(cache_file):
|
|
860
|
+
return 'unknown'
|
|
861
|
+
try:
|
|
862
|
+
with open(cache_file, 'rb') as fh:
|
|
863
|
+
blob = fh.read()
|
|
864
|
+
except OSError:
|
|
865
|
+
return 'unknown'
|
|
866
|
+
if blob == NEGATIVE_MARKER:
|
|
867
|
+
return None
|
|
868
|
+
sep = blob.find(b'\n')
|
|
869
|
+
if sep <= 0:
|
|
870
|
+
return 'unknown'
|
|
871
|
+
return (blob[:sep].decode(), blob[sep + 1:].decode())
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
875
|
+
"""Download image from URL and return (base64_data, mime_type) or None.
|
|
876
|
+
|
|
877
|
+
Cached on disk and budgeted per run: this is the hot path for every appmsg
|
|
878
|
+
thumbnail and sticker fallback, and it accounted for the entire runtime of
|
|
879
|
+
a link-heavy export before either guard existed.
|
|
880
|
+
|
|
881
|
+
`budgeted=False` is for calls made *by* the cover fetchers, which have
|
|
882
|
+
already spent their own budget for this URL.
|
|
883
|
+
"""
|
|
884
|
+
if not url or not url.startswith(('http://', 'https://')):
|
|
885
|
+
return None
|
|
886
|
+
|
|
887
|
+
sink = PREFETCH['sink']
|
|
888
|
+
if sink is not None:
|
|
889
|
+
# Dry pass: record the request, spend no budget, touch no network.
|
|
890
|
+
sink.append(('img', url, aes_key))
|
|
891
|
+
return None
|
|
892
|
+
|
|
893
|
+
cache_dir = COVER_STATE.get('dir') or ''
|
|
894
|
+
# The AES key changes the bytes, so it belongs in the cache identity.
|
|
895
|
+
slug = hashlib.md5(f'{url}\x00{aes_key}'.encode()).hexdigest()
|
|
896
|
+
cache_file = os.path.join(cache_dir, slug + '.b64') if cache_dir else ''
|
|
897
|
+
|
|
898
|
+
cached = _read_cache_media(cache_file)
|
|
899
|
+
if cached != 'unknown':
|
|
900
|
+
if budgeted:
|
|
901
|
+
COVER_STATE['thumb_cached'] += 1
|
|
902
|
+
return cached
|
|
903
|
+
|
|
904
|
+
if budgeted and not PREFETCH['active']:
|
|
905
|
+
if COVER_STATE.get('thumb_budget', 0) <= 0:
|
|
906
|
+
COVER_STATE['thumb_skipped'] += 1
|
|
907
|
+
return None
|
|
908
|
+
COVER_STATE['thumb_budget'] -= 1
|
|
909
|
+
COVER_STATE['thumb_fetched'] += 1
|
|
910
|
+
|
|
911
|
+
for _ in range(2):
|
|
912
|
+
try:
|
|
913
|
+
req = urllib.request.Request(url, headers={
|
|
914
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
915
|
+
'Referer': 'https://mp.weixin.qq.com/',
|
|
916
|
+
})
|
|
917
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
918
|
+
data = resp.read(MAX_EMBED_SIZE + 1)
|
|
919
|
+
# Too large and unparseable both mean "nothing usable here"; record
|
|
920
|
+
# it so the next export does not pay for the same dead URL.
|
|
921
|
+
if len(data) > MAX_EMBED_SIZE:
|
|
922
|
+
_cache_media(cache_file, None)
|
|
923
|
+
return None
|
|
924
|
+
candidates = [data]
|
|
925
|
+
decrypted = _decrypt_aes_cbc(data, aes_key)
|
|
926
|
+
if decrypted:
|
|
927
|
+
candidates.insert(0, decrypted)
|
|
928
|
+
detected = next((hit for hit in map(_valid_media, candidates) if hit), None)
|
|
929
|
+
if detected:
|
|
930
|
+
data, mime = detected
|
|
931
|
+
if _WECHAT_IMAGE:
|
|
932
|
+
# Covers arrive full-size; the reader shows them at 240px.
|
|
933
|
+
data, mime = wechat_image.shrink(data, mime, max_side=480)
|
|
934
|
+
result = (base64.b64encode(data).decode(), mime)
|
|
935
|
+
_cache_media(cache_file, result)
|
|
936
|
+
return result
|
|
937
|
+
except Exception:
|
|
938
|
+
continue
|
|
939
|
+
_cache_media(cache_file, None)
|
|
940
|
+
return None
|
|
941
|
+
|
|
942
|
+
|
|
231
943
|
def detect_mime_from_bytes(header_bytes):
|
|
232
944
|
"""Detect MIME type from byte header."""
|
|
233
945
|
if header_bytes[:2] == b'\xff\xd8':
|
|
@@ -245,32 +957,556 @@ def extract_appmsg_image(content):
|
|
|
245
957
|
"""Extract image URL from appmsg XML content."""
|
|
246
958
|
if not content:
|
|
247
959
|
return None
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
m = re.search(rf'<{tag}>([^<]+)</{tag}>', content)
|
|
960
|
+
for tag in ('encrypturl', 'thumburl', 'cdnthumburl', 'appthumburl'):
|
|
961
|
+
m = re.search(rf'<{tag}\b[^>]*>([\s\S]*?)</{tag}>', content, re.IGNORECASE)
|
|
251
962
|
if m:
|
|
252
963
|
url = m.group(1).strip()
|
|
253
|
-
|
|
964
|
+
url = url.replace('<![CDATA[', '').replace(']]>', '').strip()
|
|
965
|
+
url = decode_xml(url).replace('\\/', '/').replace('*#*', ':').strip()
|
|
966
|
+
if url.startswith(('http://', 'https://')) and not is_share_page_url(url):
|
|
967
|
+
return url
|
|
968
|
+
for tag in ('encrypturl', 'thumburl', 'cdnthumburl', 'appthumburl'):
|
|
969
|
+
m = re.search(rf'\b{tag}\s*=\s*["\']([^"\']+)', content, re.IGNORECASE)
|
|
970
|
+
if m:
|
|
971
|
+
url = decode_xml(m.group(1).strip()).replace('\\/', '/').replace('*#*', ':').strip()
|
|
972
|
+
if url.startswith(('http://', 'https://')) and not is_share_page_url(url):
|
|
254
973
|
return url
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
974
|
+
for raw_url in re.findall(r'https?://[^\s<>"\']+', str(content), re.IGNORECASE):
|
|
975
|
+
url = (decode_xml(raw_url).replace('\\/', '/').replace('\\u0026', '&')
|
|
976
|
+
.replace('*#*', ':').strip(' \t\r\n\\\'"'))
|
|
977
|
+
# App-card URLs (notably b23.tv/Bilibili share links) are page links,
|
|
978
|
+
# not image resources. Never emit them as a broken <img> source.
|
|
979
|
+
if url.startswith(('http://', 'https://')) and not is_share_page_url(url):
|
|
260
980
|
return url
|
|
261
981
|
return None
|
|
262
982
|
|
|
263
983
|
|
|
984
|
+
def is_share_page_url(url):
|
|
985
|
+
return bool(re.match(
|
|
986
|
+
r'https?://(?:www\.)?(?:b23\.tv|bilibili\.com|pan\.quark\.cn|'
|
|
987
|
+
r'y\.music\.163\.com|music\.163\.com|mp\.weixin\.qq\.com|'
|
|
988
|
+
r'schoai\.cn|share\.traecontent\.cn|hycx-gd\.cn|'
|
|
989
|
+
r'campusgateway\.51job\.com|tieba\.baidu\.com)(?:/|$)',
|
|
990
|
+
str(url or ''), re.IGNORECASE,
|
|
991
|
+
))
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def resolve_emoticon_seed(configured, account_dir, out_dir):
|
|
995
|
+
"""The account's sticker seed, discovered from memory if unconfigured.
|
|
996
|
+
|
|
997
|
+
The seed is a per-account constant that only exists in WeChat's process
|
|
998
|
+
memory. Nothing used to populate it, so `STICKER_STATE['key']` stayed
|
|
999
|
+
empty, local sticker decryption never ran, and every custom sticker
|
|
1000
|
+
degraded to a `[表情]` placeholder.
|
|
1001
|
+
|
|
1002
|
+
Returns (seed, discovered). The scan costs a few seconds and is only
|
|
1003
|
+
needed once: the result is memoised under the output directory, and
|
|
1004
|
+
persisting it via `weflow-cli config set emoticonSeed` skips even that.
|
|
1005
|
+
"""
|
|
1006
|
+
if configured:
|
|
1007
|
+
return str(configured), False
|
|
1008
|
+
if not account_dir or not _WECHAT_EMOTICON:
|
|
1009
|
+
return '', False
|
|
1010
|
+
|
|
1011
|
+
cache_dir = os.path.join(out_dir, '.sticker-cache')
|
|
1012
|
+
memo = os.path.join(cache_dir, 'seed')
|
|
1013
|
+
try:
|
|
1014
|
+
with open(memo, 'r', encoding='utf-8') as fh:
|
|
1015
|
+
memoised = fh.read().strip()
|
|
1016
|
+
if memoised:
|
|
1017
|
+
return memoised, False
|
|
1018
|
+
except OSError:
|
|
1019
|
+
pass
|
|
1020
|
+
|
|
1021
|
+
dirs = wechat_emoticon.sticker_cache_dirs(account_dir)
|
|
1022
|
+
sample = wechat_emoticon.any_sticker_file(dirs)
|
|
1023
|
+
if not sample:
|
|
1024
|
+
return '', False
|
|
1025
|
+
wxid = wechat_emoticon.account_wxid(os.path.basename(os.path.normpath(account_dir)))
|
|
1026
|
+
seed = wechat_emoticon.find_seed(wxid, sample)
|
|
1027
|
+
if not seed:
|
|
1028
|
+
return '', False
|
|
1029
|
+
try:
|
|
1030
|
+
os.makedirs(cache_dir, exist_ok=True)
|
|
1031
|
+
with open(memo, 'w', encoding='utf-8') as fh:
|
|
1032
|
+
fh.write(str(seed))
|
|
1033
|
+
except OSError:
|
|
1034
|
+
pass
|
|
1035
|
+
return str(seed), True
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def _page_cache_file(page_url):
|
|
1039
|
+
"""Cache path for a resolved share-page cover.
|
|
1040
|
+
|
|
1041
|
+
Namespaced apart from download_image_as_base64's entries so a page URL and
|
|
1042
|
+
an image URL can never collide on the same digest.
|
|
1043
|
+
"""
|
|
1044
|
+
cache_dir = COVER_STATE.get('dir') or ''
|
|
1045
|
+
if not cache_dir:
|
|
1046
|
+
return ''
|
|
1047
|
+
return os.path.join(cache_dir, 'page-' + hashlib.md5(page_url.encode()).hexdigest() + '.b64')
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def download_bilibili_cover(page_url, timeout=10):
|
|
1051
|
+
"""Resolve a Bilibili share page and embed its og:image cover.
|
|
1052
|
+
|
|
1053
|
+
Guarded like download_page_og_image: Bilibili shares are common in chat,
|
|
1054
|
+
and without a cache + shared budget every re-export re-fetched all of
|
|
1055
|
+
them, which alone accounted for minutes of an export.
|
|
1056
|
+
"""
|
|
1057
|
+
if not page_url or not re.match(
|
|
1058
|
+
r'https?://(?:www\.)?(?:b23\.tv|bilibili\.com)(?:/|$)',
|
|
1059
|
+
page_url, re.IGNORECASE):
|
|
1060
|
+
return None
|
|
1061
|
+
|
|
1062
|
+
sink = PREFETCH['sink']
|
|
1063
|
+
if sink is not None:
|
|
1064
|
+
sink.append(('page', page_url, ''))
|
|
1065
|
+
return None
|
|
1066
|
+
|
|
1067
|
+
cache_file = _page_cache_file(page_url)
|
|
1068
|
+
cached = _read_cache_media(cache_file)
|
|
1069
|
+
if cached != 'unknown':
|
|
1070
|
+
COVER_STATE['cached'] += 1
|
|
1071
|
+
return cached
|
|
1072
|
+
if not PREFETCH['active']:
|
|
1073
|
+
# The prefetch pass has PREFETCH_MAX_URLS as its own bound and runs
|
|
1074
|
+
# concurrently, so charging it here would only make a large
|
|
1075
|
+
# conversation hit the cap early and fall back to serial fetching.
|
|
1076
|
+
if COVER_STATE.get('budget', 0) <= 0:
|
|
1077
|
+
COVER_STATE['skipped'] += 1
|
|
1078
|
+
return None
|
|
1079
|
+
COVER_STATE['budget'] -= 1
|
|
1080
|
+
COVER_STATE['fetched'] += 1
|
|
1081
|
+
|
|
1082
|
+
try:
|
|
1083
|
+
req = urllib.request.Request(page_url, headers={
|
|
1084
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
1085
|
+
'Referer': 'https://www.bilibili.com/',
|
|
1086
|
+
})
|
|
1087
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
1088
|
+
html = resp.read(COVER_HEAD_BYTES).decode('utf-8', errors='ignore')
|
|
1089
|
+
final_url = resp.geturl()
|
|
1090
|
+
match = re.search(
|
|
1091
|
+
r'<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\']+)',
|
|
1092
|
+
html, re.IGNORECASE,
|
|
1093
|
+
)
|
|
1094
|
+
cover_url = decode_xml(match.group(1)).replace('\\/', '/').strip() if match else ''
|
|
1095
|
+
if not cover_url:
|
|
1096
|
+
bvid_match = re.search(r'/(BV[0-9A-Za-z]+)(?:/|\?|$)', final_url, re.IGNORECASE)
|
|
1097
|
+
if not bvid_match:
|
|
1098
|
+
return None
|
|
1099
|
+
api_url = 'https://api.bilibili.com/x/web-interface/view?bvid=' + bvid_match.group(1)
|
|
1100
|
+
api_req = urllib.request.Request(api_url, headers={
|
|
1101
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
|
|
1102
|
+
'Referer': final_url,
|
|
1103
|
+
})
|
|
1104
|
+
with urllib.request.urlopen(api_req, timeout=timeout) as api_resp:
|
|
1105
|
+
payload = json.loads(api_resp.read(1024 * 1024).decode('utf-8'))
|
|
1106
|
+
cover_url = str((payload.get('data') or {}).get('pic') or '')
|
|
1107
|
+
if not cover_url:
|
|
1108
|
+
_cache_media(cache_file, None)
|
|
1109
|
+
return None
|
|
1110
|
+
cover = download_image_as_base64(cover_url, timeout=timeout, budgeted=False)
|
|
1111
|
+
_cache_media(cache_file, cover)
|
|
1112
|
+
return cover
|
|
1113
|
+
except Exception:
|
|
1114
|
+
_cache_media(cache_file, None)
|
|
1115
|
+
return None
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def download_page_og_image(page_url, timeout=10):
|
|
1119
|
+
"""Fetch a share page's og:image and embed the resolved cover.
|
|
1120
|
+
|
|
1121
|
+
Guarded by a disk cache and a per-run budget: without them a conversation
|
|
1122
|
+
full of links spends minutes on network round-trips, and every re-export
|
|
1123
|
+
repeats the whole cost.
|
|
1124
|
+
"""
|
|
1125
|
+
if not page_url or not page_url.startswith(('http://', 'https://')):
|
|
1126
|
+
return None
|
|
1127
|
+
|
|
1128
|
+
sink = PREFETCH['sink']
|
|
1129
|
+
if sink is not None:
|
|
1130
|
+
sink.append(('page', page_url, ''))
|
|
1131
|
+
return None
|
|
1132
|
+
|
|
1133
|
+
cache_file = _page_cache_file(page_url)
|
|
1134
|
+
cached = _read_cache_media(cache_file)
|
|
1135
|
+
if cached != 'unknown':
|
|
1136
|
+
COVER_STATE['cached'] += 1
|
|
1137
|
+
return cached
|
|
1138
|
+
|
|
1139
|
+
if not PREFETCH['active']:
|
|
1140
|
+
# The prefetch pass has PREFETCH_MAX_URLS as its own bound and runs
|
|
1141
|
+
# concurrently, so charging it here would only make a large
|
|
1142
|
+
# conversation hit the cap early and fall back to serial fetching.
|
|
1143
|
+
if COVER_STATE.get('budget', 0) <= 0:
|
|
1144
|
+
COVER_STATE['skipped'] += 1
|
|
1145
|
+
return None
|
|
1146
|
+
COVER_STATE['budget'] -= 1
|
|
1147
|
+
COVER_STATE['fetched'] += 1
|
|
1148
|
+
|
|
1149
|
+
try:
|
|
1150
|
+
req = urllib.request.Request(page_url, headers={
|
|
1151
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
1152
|
+
'Referer': page_url,
|
|
1153
|
+
})
|
|
1154
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
1155
|
+
html = resp.read(COVER_HEAD_BYTES).decode('utf-8', errors='ignore')
|
|
1156
|
+
patterns = (
|
|
1157
|
+
r'<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\']+)',
|
|
1158
|
+
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:image',
|
|
1159
|
+
r'<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\']+)',
|
|
1160
|
+
)
|
|
1161
|
+
cover_url = ''
|
|
1162
|
+
for pattern in patterns:
|
|
1163
|
+
match = re.search(pattern, html, re.IGNORECASE)
|
|
1164
|
+
if match:
|
|
1165
|
+
cover_url = decode_xml(match.group(1)).replace('\\/', '/').strip()
|
|
1166
|
+
break
|
|
1167
|
+
cover = download_image_as_base64(cover_url, timeout=timeout, budgeted=False) if cover_url else None
|
|
1168
|
+
_cache_media(cache_file, cover)
|
|
1169
|
+
return cover
|
|
1170
|
+
except Exception:
|
|
1171
|
+
_cache_media(cache_file, None)
|
|
1172
|
+
return None
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
def prefetch_remote(records):
|
|
1176
|
+
"""Fetch recorded remote media concurrently, filling the disk cache.
|
|
1177
|
+
|
|
1178
|
+
`records` is the PREFETCH sink: (kind, url, aes_key) triples. Everything
|
|
1179
|
+
lands in the same cache `download_image_as_base64` reads, so the real
|
|
1180
|
+
formatting pass becomes a sequence of cache hits. Returns a short summary
|
|
1181
|
+
for the progress line.
|
|
1182
|
+
"""
|
|
1183
|
+
seen = set()
|
|
1184
|
+
jobs = []
|
|
1185
|
+
for kind, url, aes_key in records:
|
|
1186
|
+
key = (kind, url, aes_key)
|
|
1187
|
+
if key in seen:
|
|
1188
|
+
continue
|
|
1189
|
+
seen.add(key)
|
|
1190
|
+
jobs.append(key)
|
|
1191
|
+
if not jobs:
|
|
1192
|
+
return 0, 0
|
|
1193
|
+
|
|
1194
|
+
dropped = 0
|
|
1195
|
+
if len(jobs) > PREFETCH_MAX_URLS:
|
|
1196
|
+
dropped = len(jobs) - PREFETCH_MAX_URLS
|
|
1197
|
+
jobs = jobs[:PREFETCH_MAX_URLS]
|
|
1198
|
+
|
|
1199
|
+
def run(job):
|
|
1200
|
+
kind, url, aes_key = job
|
|
1201
|
+
try:
|
|
1202
|
+
if kind == 'page':
|
|
1203
|
+
if download_bilibili_cover(url):
|
|
1204
|
+
return True
|
|
1205
|
+
return download_page_og_image(url) is not None
|
|
1206
|
+
return download_image_as_base64(url, aes_key) is not None
|
|
1207
|
+
except Exception:
|
|
1208
|
+
return False
|
|
1209
|
+
|
|
1210
|
+
done = 0
|
|
1211
|
+
PREFETCH['active'] = True
|
|
1212
|
+
try:
|
|
1213
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=PREFETCH_WORKERS) as pool:
|
|
1214
|
+
for ok in pool.map(run, jobs):
|
|
1215
|
+
if ok:
|
|
1216
|
+
done += 1
|
|
1217
|
+
finally:
|
|
1218
|
+
PREFETCH['active'] = False
|
|
1219
|
+
return done, dropped
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
def extract_xml_attr_url(content, name):
|
|
1223
|
+
match = re.search(rf'\b{name}\s*=\s*["\']([^"\']+)', str(content or ''), re.IGNORECASE)
|
|
1224
|
+
if not match:
|
|
1225
|
+
return None
|
|
1226
|
+
url = decode_xml(match.group(1)).replace('\\/', '/').replace('*#*', ':').strip()
|
|
1227
|
+
return url if url.startswith(('http://', 'https://')) else None
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def extract_xml_attr_value(content, name):
|
|
1231
|
+
match = re.search(rf'\b{name}\s*=\s*["\']([^"\']*)', str(content or ''), re.IGNORECASE)
|
|
1232
|
+
return decode_xml(match.group(1).strip()) if match else ''
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
def render_contact_card(content):
|
|
1236
|
+
nickname = extract_xml_attr_value(content, 'nickname') or '公众号名片'
|
|
1237
|
+
username = extract_xml_attr_value(content, 'username')
|
|
1238
|
+
avatar_url = extract_xml_attr_url(content, 'brandIconUrl')
|
|
1239
|
+
avatar = download_image_as_base64(avatar_url) if avatar_url else None
|
|
1240
|
+
parts = []
|
|
1241
|
+
if avatar:
|
|
1242
|
+
b64, mime = avatar
|
|
1243
|
+
parts.append(f'<img class="msg-app-thumb" src="data:{mime};base64,{b64}" loading="lazy" />')
|
|
1244
|
+
parts.append(f'<span class="msg-app-title">{escape_html(nickname)}</span>')
|
|
1245
|
+
if username:
|
|
1246
|
+
parts.append(f'<div class="msg-app-desc">{escape_html(username)}</div>')
|
|
1247
|
+
return '<div class="msg-app">' + ''.join(parts) + '</div>'
|
|
1248
|
+
|
|
1249
|
+
|
|
1250
|
+
def load_contact_names(db_path, key_hex, salt_hex):
|
|
1251
|
+
"""{username: best available name} from the contact database.
|
|
1252
|
+
|
|
1253
|
+
A group message only identifies its sender by wxid, which is unusable in a
|
|
1254
|
+
transcript. Prefer the remark (what the user calls them) over the account
|
|
1255
|
+
nickname over the alias, matching how the 1:1 path already resolves the
|
|
1256
|
+
conversation partner.
|
|
1257
|
+
"""
|
|
1258
|
+
names = {}
|
|
1259
|
+
if not db_path or not key_hex or not salt_hex or not os.path.isfile(db_path):
|
|
1260
|
+
return names
|
|
1261
|
+
conn = None
|
|
1262
|
+
try:
|
|
1263
|
+
conn, cursor = connect(db_path, key_hex, salt_hex)
|
|
1264
|
+
cursor.execute('SELECT username, remark, nick_name, alias FROM contact')
|
|
1265
|
+
for username, remark, nick_name, alias in cursor.fetchall():
|
|
1266
|
+
if not username:
|
|
1267
|
+
continue
|
|
1268
|
+
names[username] = remark or nick_name or alias or username
|
|
1269
|
+
except Exception:
|
|
1270
|
+
return names
|
|
1271
|
+
finally:
|
|
1272
|
+
if conn is not None:
|
|
1273
|
+
conn.close()
|
|
1274
|
+
return names
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
def contact_name(wxid):
|
|
1278
|
+
"""Display name for a wxid, falling back to the wxid itself.
|
|
1279
|
+
|
|
1280
|
+
An unresolved id is kept rather than blanked: it is still unique, and a
|
|
1281
|
+
transcript that silently drops the speaker is worse than one showing one.
|
|
1282
|
+
"""
|
|
1283
|
+
return CONTACT_NAMES.get(wxid) or wxid
|
|
1284
|
+
|
|
1285
|
+
|
|
1286
|
+
def split_group_speaker(content, sender_map, own_wxid=''):
|
|
1287
|
+
"""(speaker id, content) with the group sender prefix removed.
|
|
1288
|
+
|
|
1289
|
+
Group rows generally prefix the content with the speaker's id. Only an id
|
|
1290
|
+
the sender map actually knows is accepted, so a message that merely starts
|
|
1291
|
+
with `note: ...` is not mistaken for one. Returns (None, content) when
|
|
1292
|
+
there is no prefix to strip.
|
|
1293
|
+
"""
|
|
1294
|
+
match = re.match(r'^([A-Za-z0-9_@.-]{5,64})\s*[:：]\s', str(content or ''))
|
|
1295
|
+
if not match:
|
|
1296
|
+
return None, content
|
|
1297
|
+
candidate = match.group(1)
|
|
1298
|
+
if candidate not in set((sender_map or {}).values()):
|
|
1299
|
+
return None, content
|
|
1300
|
+
return candidate, content[match.end():]
|
|
1301
|
+
|
|
1302
|
+
|
|
1303
|
+
def image_is_blank(b64_data, threshold=0.985):
|
|
1304
|
+
"""True for an embedded payload that carries no visible artwork.
|
|
1305
|
+
|
|
1306
|
+
Sticker sources occasionally hand back an all-white frame - a truncated
|
|
1307
|
+
local download, or a CDN placeholder. Embedding it renders as an empty
|
|
1308
|
+
square, which reads as a broken export rather than as the placeholder it
|
|
1309
|
+
is, so callers prefer the text fallback.
|
|
1310
|
+
"""
|
|
1311
|
+
if not b64_data or not _WECHAT_IMAGE:
|
|
1312
|
+
return False
|
|
1313
|
+
try:
|
|
1314
|
+
from collections import Counter
|
|
1315
|
+
import io as _io
|
|
1316
|
+
from PIL import Image
|
|
1317
|
+
with Image.open(_io.BytesIO(base64.b64decode(b64_data))) as image:
|
|
1318
|
+
# An animation's first frame can be blank while the animation is
|
|
1319
|
+
# not; judging it on frame 1 would drop a perfectly good sticker.
|
|
1320
|
+
if getattr(image, 'is_animated', False):
|
|
1321
|
+
return False
|
|
1322
|
+
pixels = list(image.convert('RGB').resize((32, 32)).getdata())
|
|
1323
|
+
if not pixels:
|
|
1324
|
+
return True
|
|
1325
|
+
return Counter(pixels).most_common(1)[0][1] / len(pixels) > threshold
|
|
1326
|
+
except Exception:
|
|
1327
|
+
return False
|
|
1328
|
+
|
|
1329
|
+
|
|
1330
|
+
def render_voice(local_id, content):
|
|
1331
|
+
"""`[语音 6″]` plus its transcript when one has been cached.
|
|
1332
|
+
|
|
1333
|
+
Browsers cannot play SILK and ffmpeg cannot decode it, so there is no audio
|
|
1334
|
+
element to offer - the transcript is the only way a voice message can carry
|
|
1335
|
+
meaning in an export.
|
|
1336
|
+
"""
|
|
1337
|
+
length = (extract_xml_attr_value(content, 'voicelength')
|
|
1338
|
+
or extract_xml_attr_value(content, 'length'))
|
|
1339
|
+
seconds = f' {round(int(length) / 1000)}″' if length.isdigit() and int(length) > 0 else ''
|
|
1340
|
+
label = f'<span class="msg-media">[语音{seconds}]</span>'
|
|
1341
|
+
blob = VOICE_STATE['map'].get(local_id)
|
|
1342
|
+
cache = VOICE_STATE['cache']
|
|
1343
|
+
if blob is None or cache is None:
|
|
1344
|
+
return label
|
|
1345
|
+
text = cache.get(wechat_voice.voice_key(blob))
|
|
1346
|
+
if not text:
|
|
1347
|
+
return label
|
|
1348
|
+
# Labelled, not presented as the words themselves. Recognition of dialect
|
|
1349
|
+
# speech is approximate: an unlabelled transcript reads as a quote, and a
|
|
1350
|
+
# confidently wrong quote is worse than an obvious placeholder.
|
|
1351
|
+
return (f'{label}<div class="msg-voice-text">'
|
|
1352
|
+
f'<span class="voice-tag">机器转写·粤语欠准</span>{escape_html(text)}</div>')
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def render_location(content):
|
|
1356
|
+
"""Readable label for a type-48 location row instead of its raw XML."""
|
|
1357
|
+
label = extract_xml_attr_value(content, 'poiname') or extract_xml_attr_value(content, 'label')
|
|
1358
|
+
return f'[位置] {escape_html(label)}' if label else '<span class="msg-media">[位置]</span>'
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def extract_xml_text(content, tag):
|
|
1362
|
+
"""Extract plain or CDATA-wrapped text from one XML element."""
|
|
1363
|
+
if not content:
|
|
1364
|
+
return ''
|
|
1365
|
+
match = re.search(rf'<{tag}\b[^>]*>([\s\S]*?)</{tag}>', content, re.IGNORECASE)
|
|
1366
|
+
if not match:
|
|
1367
|
+
return ''
|
|
1368
|
+
value = match.group(1).strip()
|
|
1369
|
+
cdata = re.fullmatch(r'<!\[CDATA\[([\s\S]*)\]\]>', value)
|
|
1370
|
+
return decode_xml((cdata.group(1) if cdata else value).strip())
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
def extract_media_aes_key(content):
|
|
1374
|
+
if not content:
|
|
1375
|
+
return ''
|
|
1376
|
+
for name in ('aeskey', 'aes_key', 'encryptaeskey'):
|
|
1377
|
+
match = re.search(rf'\b{name}\s*=\s*["\']([0-9a-f]{{32}})', content, re.IGNORECASE)
|
|
1378
|
+
if match:
|
|
1379
|
+
return match.group(1)
|
|
1380
|
+
match = re.search(rf'<{name}\b[^>]*>\s*([0-9a-f]{{32}})\s*</{name}>', content, re.IGNORECASE)
|
|
1381
|
+
if match:
|
|
1382
|
+
return match.group(1)
|
|
1383
|
+
return ''
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
def plain_fragment(text, limit=200):
|
|
1387
|
+
"""Tag-stripped, whitespace-collapsed preview of an XML fragment."""
|
|
1388
|
+
if not text:
|
|
1389
|
+
return ''
|
|
1390
|
+
text = re.sub(r'<\?xml[^>]*\?>', ' ', str(text))
|
|
1391
|
+
text = re.sub(r'<[^>]*>', ' ', text)
|
|
1392
|
+
text = re.sub(r'\s+', ' ', decode_xml(text)).strip()
|
|
1393
|
+
return text[:limit]
|
|
1394
|
+
|
|
1395
|
+
|
|
1396
|
+
def readable_fragment(text, limit=300):
|
|
1397
|
+
"""`text` with any embedded document reduced to prose.
|
|
1398
|
+
|
|
1399
|
+
Quoted replies (appmsg type 57) put a whole escaped message inside <des>,
|
|
1400
|
+
so rendering it verbatim fills the bubble with markup. Plain text passes
|
|
1401
|
+
through unchanged.
|
|
1402
|
+
"""
|
|
1403
|
+
if not text:
|
|
1404
|
+
return ''
|
|
1405
|
+
if '<' in text or '<' in text:
|
|
1406
|
+
return plain_fragment(decode_xml(text), limit)
|
|
1407
|
+
return text[:limit]
|
|
1408
|
+
|
|
1409
|
+
|
|
1410
|
+
def render_group_template(content):
|
|
1411
|
+
"""Readable text for a `sysmsgtemplate` group notice, or ''.
|
|
1412
|
+
|
|
1413
|
+
The row carries a sentence template with `$name$` placeholders plus a
|
|
1414
|
+
`link_list` that maps each placeholder to member nicknames. Stripping tags
|
|
1415
|
+
without substituting left nothing but the chatroom id.
|
|
1416
|
+
"""
|
|
1417
|
+
template = extract_xml_text(content, 'template')
|
|
1418
|
+
if not template:
|
|
1419
|
+
return ''
|
|
1420
|
+
for placeholder in set(re.findall(r'\$(\w+)\$', template)):
|
|
1421
|
+
link = re.search(rf'<link\s+name="{placeholder}"[\s\S]*?</link>', content, re.IGNORECASE)
|
|
1422
|
+
if not link:
|
|
1423
|
+
continue
|
|
1424
|
+
nicknames = []
|
|
1425
|
+
for raw in re.findall(r'<nickname>([\s\S]*?)</nickname>', link.group(0), re.IGNORECASE):
|
|
1426
|
+
value = raw.strip()
|
|
1427
|
+
cdata = re.fullmatch(r'<!\[CDATA\[([\s\S]*)\]\]>', value)
|
|
1428
|
+
text = (cdata.group(1) if cdata else value).strip()
|
|
1429
|
+
if text:
|
|
1430
|
+
nicknames.append(decode_xml(text))
|
|
1431
|
+
separator = extract_xml_text(link.group(0), 'separator') or '、'
|
|
1432
|
+
template = template.replace(f'${placeholder}$', separator.join(nicknames))
|
|
1433
|
+
# Any placeholder with no link_list entry is left dangling; drop it rather
|
|
1434
|
+
# than print `$username$` into the transcript.
|
|
1435
|
+
return re.sub(r'\$\w+\$', '', template).strip()
|
|
1436
|
+
|
|
1437
|
+
|
|
1438
|
+
def render_system_message(content, names=None):
|
|
1439
|
+
"""Readable text for a type-10000 system row.
|
|
1440
|
+
|
|
1441
|
+
These rows carry XML, but it is WeChat's own display markup
|
|
1442
|
+
(`<img src="SystemMessages_HongbaoIcon.png"/>`, `<_wc_custom_link_ ...>`)
|
|
1443
|
+
rather than a document worth showing. Escaping it verbatim put a wall of
|
|
1444
|
+
`<sysmsg ...>` in the bubble; a revoke notice read as XML instead of
|
|
1445
|
+
saying who revoked what.
|
|
1446
|
+
"""
|
|
1447
|
+
if not content:
|
|
1448
|
+
return ''
|
|
1449
|
+
if '<' not in content:
|
|
1450
|
+
return escape_html(content)
|
|
1451
|
+
if '<sysmsgtemplate' in content:
|
|
1452
|
+
# A join/invite notice: a template plus a member list, which has to be
|
|
1453
|
+
# substituted before any of the generic extraction below can help.
|
|
1454
|
+
return escape_html(render_group_template(content)) or escape_html(plain_fragment(content, 200))
|
|
1455
|
+
revoke = re.search(r'<revokemsg\b[\s\S]*?</revokemsg>', content, re.IGNORECASE)
|
|
1456
|
+
scope = revoke.group(0) if revoke else content
|
|
1457
|
+
text = ''
|
|
1458
|
+
for tag in ('content', 'title', 'text'):
|
|
1459
|
+
text = extract_xml_text(scope, tag)
|
|
1460
|
+
if text:
|
|
1461
|
+
break
|
|
1462
|
+
if not text:
|
|
1463
|
+
text = plain_fragment(scope, 200)
|
|
1464
|
+
for wxid, name in (names or {}).items():
|
|
1465
|
+
# The row keeps `$wxid_...$` for the client to expand at render time.
|
|
1466
|
+
if wxid and name:
|
|
1467
|
+
text = text.replace(f'${wxid}$', name)
|
|
1468
|
+
return escape_html(text)
|
|
1469
|
+
|
|
1470
|
+
|
|
264
1471
|
def escape_html(text):
|
|
265
1472
|
if not text:
|
|
266
1473
|
return ''
|
|
267
|
-
|
|
1474
|
+
text = ''.join(char for char in str(text) if char in '\n\r\t' or ord(char) >= 32)
|
|
1475
|
+
return (text
|
|
268
1476
|
.replace('&', '&')
|
|
269
1477
|
.replace('<', '<')
|
|
270
1478
|
.replace('>', '>')
|
|
271
1479
|
.replace('"', '"'))
|
|
272
1480
|
|
|
273
1481
|
|
|
1482
|
+
def load_builtin_emoji(name):
|
|
1483
|
+
"""One bundled face as (base64, mime), or None."""
|
|
1484
|
+
path = wechat_emoji.IMAGE_FACES.get(name) if _WECHAT_EMOJI else None
|
|
1485
|
+
if not path:
|
|
1486
|
+
return None
|
|
1487
|
+
try:
|
|
1488
|
+
with open(path, 'rb') as stream:
|
|
1489
|
+
data = stream.read(MAX_EMBED_SIZE + 1)
|
|
1490
|
+
if len(data) > MAX_EMBED_SIZE:
|
|
1491
|
+
return None
|
|
1492
|
+
return base64.b64encode(data).decode(), 'image/png'
|
|
1493
|
+
except OSError:
|
|
1494
|
+
return None
|
|
1495
|
+
|
|
1496
|
+
|
|
1497
|
+
def render_builtin_emoji(content, label=None):
|
|
1498
|
+
"""Render every built-in face in `content`, in place.
|
|
1499
|
+
|
|
1500
|
+
Faces become <span class="wxface wxf-…">; the artwork itself is emitted
|
|
1501
|
+
once per page by face_css(). The previous approach appended one <img> per
|
|
1502
|
+
message and only ever matched the first label, so a sentence containing
|
|
1503
|
+
several emoji rendered at most one - and repeated the base64 for it.
|
|
1504
|
+
"""
|
|
1505
|
+
if not _WECHAT_EMOJI:
|
|
1506
|
+
return escape_html(content)
|
|
1507
|
+
return wechat_emoji.render_faces(escape_html(str(content or '')))
|
|
1508
|
+
|
|
1509
|
+
|
|
274
1510
|
def parse_source(source_text):
|
|
275
1511
|
"""Parse source field to extract sender and content."""
|
|
276
1512
|
sender = ''
|
|
@@ -297,7 +1533,19 @@ def parse_source(source_text):
|
|
|
297
1533
|
return sender, content
|
|
298
1534
|
|
|
299
1535
|
|
|
300
|
-
def
|
|
1536
|
+
def sender_matches_account(sender_user_name, own_wxid):
|
|
1537
|
+
"""Match the sender against the configured account, including NT suffixes."""
|
|
1538
|
+
if not sender_user_name or not own_wxid:
|
|
1539
|
+
return False
|
|
1540
|
+
if sender_user_name == own_wxid or sender_user_name.startswith(own_wxid + '_'):
|
|
1541
|
+
return True
|
|
1542
|
+
account_base = own_wxid.rsplit('_', 1)
|
|
1543
|
+
if len(account_base) == 2 and len(account_base[1]) == 4 and account_base[1].isalnum():
|
|
1544
|
+
return sender_user_name == account_base[0]
|
|
1545
|
+
return False
|
|
1546
|
+
|
|
1547
|
+
|
|
1548
|
+
def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display_name='', resource_map=None, own_wxid=''):
|
|
301
1549
|
"""Format a single message for HTML display.
|
|
302
1550
|
|
|
303
1551
|
Args:
|
|
@@ -308,47 +1556,120 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
308
1556
|
sender_map: {sender_id: user_name} from Name2Id table
|
|
309
1557
|
display_name: human-readable name for the target talker
|
|
310
1558
|
"""
|
|
1559
|
+
is_group = '@chatroom' in str(talker or '')
|
|
311
1560
|
local_id = row[0] or 0
|
|
1561
|
+
server_id = row[1] or 0
|
|
312
1562
|
local_type = row[2] or 0
|
|
1563
|
+
if local_type > 0xffffffff:
|
|
1564
|
+
local_type &= 0xffffffff
|
|
313
1565
|
real_sender_id = row[4] or 0
|
|
314
1566
|
create_time = row[5] or 0
|
|
315
1567
|
source = row[7]
|
|
316
1568
|
message_content = row[8]
|
|
1569
|
+
compressed_content = row[9]
|
|
1570
|
+
resource_md5s = list((resource_map or {}).get(f'server:{int(server_id)}', [])) if server_id else []
|
|
317
1571
|
|
|
318
1572
|
# Resolve sender name
|
|
319
1573
|
sender_user_name = (sender_map or {}).get(real_sender_id, '')
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
if is_self:
|
|
324
|
-
|
|
325
|
-
elif display_name:
|
|
326
|
-
sender_display = display_name
|
|
327
|
-
elif sender_user_name:
|
|
328
|
-
sender_display = sender_user_name
|
|
329
|
-
else:
|
|
330
|
-
sender_display = talker
|
|
1574
|
+
# Do not infer "self" from a missing mapping. NT shards can have incomplete
|
|
1575
|
+
# Name2Id rows, and that would otherwise mark every unresolved message as sent.
|
|
1576
|
+
is_self = sender_matches_account(sender_user_name, own_wxid)
|
|
1577
|
+
if not is_self and sender_user_name and not own_wxid:
|
|
1578
|
+
is_self = sender_user_name != talker
|
|
331
1579
|
|
|
332
1580
|
# Get content
|
|
333
1581
|
content = ''
|
|
334
1582
|
if isinstance(message_content, str) and message_content:
|
|
335
1583
|
content = message_content
|
|
336
1584
|
elif isinstance(message_content, bytes):
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
1585
|
+
content = decode_message_content(message_content)
|
|
1586
|
+
|
|
1587
|
+
if not content and isinstance(compressed_content, bytes):
|
|
1588
|
+
content = decode_message_content(compressed_content)
|
|
1589
|
+
|
|
1590
|
+
source_text = decode_message_content(source) if isinstance(source, (bytes, bytearray, memoryview)) else str(source or '')
|
|
1591
|
+
if not content and source_text:
|
|
1592
|
+
_, content = parse_source(source_text)
|
|
1593
|
+
# Some NT rows store the complete emoji XML entity-escaped in the message
|
|
1594
|
+
# column (for example ``<msg>...</msg>``). Normalize it before
|
|
1595
|
+
# detecting media metadata so it follows the same path as raw XML.
|
|
1596
|
+
if '<' in content.lower():
|
|
1597
|
+
normalized_content = content
|
|
1598
|
+
for _ in range(2):
|
|
1599
|
+
candidate = decode_xml(normalized_content)
|
|
1600
|
+
if candidate == normalized_content:
|
|
1601
|
+
break
|
|
1602
|
+
normalized_content = candidate
|
|
1603
|
+
if re.search(r'<(?:msg|emoji)\b', normalized_content, re.IGNORECASE):
|
|
1604
|
+
content = normalized_content
|
|
1605
|
+
# NT emoji metadata may be split between source XML and message_content.
|
|
1606
|
+
# Prefer the representation that actually carries media identity/URLs;
|
|
1607
|
+
# source can contain only PUA/signature fields for the same message.
|
|
1608
|
+
metadata_parts = []
|
|
1609
|
+
if source_text:
|
|
1610
|
+
metadata_parts.append(source_text)
|
|
1611
|
+
if content and content != source_text:
|
|
1612
|
+
metadata_parts.append(content)
|
|
1613
|
+
metadata_content = '\n'.join(metadata_parts)
|
|
1614
|
+
is_emoji_xml = bool(re.search(r'<(?:msg\s*>)?\s*<emoji\b|<emoji\b', metadata_content, re.IGNORECASE))
|
|
1615
|
+
is_contact_card = bool(
|
|
1616
|
+
re.search(r'<msg\b[^>]*(?:nickname|username)=', metadata_content, re.IGNORECASE)
|
|
1617
|
+
and re.search(r'\bbrandIconUrl=', metadata_content, re.IGNORECASE)
|
|
1618
|
+
)
|
|
1619
|
+
has_builtin_signature = bool(
|
|
1620
|
+
re.search(r'<signature\b[^>]*>[^<]+</signature>', metadata_content, re.IGNORECASE)
|
|
1621
|
+
and _face_index.has_face(content)
|
|
1622
|
+
)
|
|
1623
|
+
builtin_emoji_label = _face_index.find_face(content)
|
|
1624
|
+
|
|
1625
|
+
# A group row's `display_name` is the group, not the speaker, so using it
|
|
1626
|
+
# put the same name on every bubble and nobody could tell who said what.
|
|
1627
|
+
# The speaker is in the content prefix; fall back to the sender map.
|
|
1628
|
+
group_speaker = None
|
|
1629
|
+
if is_group:
|
|
1630
|
+
group_speaker, content = split_group_speaker(content, sender_map)
|
|
1631
|
+
|
|
1632
|
+
if local_type == 10000:
|
|
1633
|
+
# Revoke notices and group templates are emitted by the system, and
|
|
1634
|
+
# their real_sender_id resolves to nobody. Anything else here reads as
|
|
1635
|
+
# if a person had said it.
|
|
1636
|
+
sender_display = '系统'
|
|
1637
|
+
elif is_self:
|
|
1638
|
+
sender_display = '我'
|
|
1639
|
+
elif group_speaker:
|
|
1640
|
+
sender_display = ('我' if sender_matches_account(group_speaker, own_wxid)
|
|
1641
|
+
else contact_name(group_speaker))
|
|
1642
|
+
elif is_group and sender_user_name:
|
|
1643
|
+
sender_display = contact_name(sender_user_name)
|
|
1644
|
+
elif display_name and not is_group:
|
|
1645
|
+
# `display_name` is the conversation. For a group that is the group
|
|
1646
|
+
# itself, so using it as a fallback labelled unrelated rows with the
|
|
1647
|
+
# group name - three system rows in one export read as "江南试验群".
|
|
1648
|
+
sender_display = display_name
|
|
1649
|
+
elif sender_user_name:
|
|
1650
|
+
sender_display = contact_name(sender_user_name)
|
|
1651
|
+
else:
|
|
1652
|
+
sender_display = '未知发送者'
|
|
341
1653
|
|
|
342
|
-
if
|
|
343
|
-
|
|
1654
|
+
if '\x00' in content or sum(ord(char) < 32 and char not in '\n\r\t' for char in content) > 2:
|
|
1655
|
+
content = ''
|
|
344
1656
|
|
|
345
1657
|
# Determine display content
|
|
346
1658
|
display = ''
|
|
347
1659
|
image_b64 = None
|
|
348
1660
|
|
|
349
|
-
if
|
|
1661
|
+
if is_contact_card:
|
|
1662
|
+
display = render_contact_card(metadata_content)
|
|
1663
|
+
elif local_type == 1 and '<' not in content:
|
|
1664
|
+
# Tested against `content`, not `metadata_content`: the latter always
|
|
1665
|
+
# carries the row's `<msgsource>` block, so it always contains '<'.
|
|
1666
|
+
# Any text message with sender metadata therefore failed this branch
|
|
1667
|
+
# and fell through to the emoji one, which rendered it as `[表情]` plus
|
|
1668
|
+
# whatever URL its catch-all found - a meeting invite became a broken
|
|
1669
|
+
# image whose src was `https://meeting.tencent.com/dm/...`.
|
|
350
1670
|
# Text
|
|
351
|
-
|
|
1671
|
+
builtin_label = _face_index.find_face(content)
|
|
1672
|
+
display = render_builtin_emoji(content, builtin_label) if builtin_label else escape_html(content)
|
|
352
1673
|
elif local_type == 3:
|
|
353
1674
|
# Image - try cache map first, then traditional FileStorage
|
|
354
1675
|
display = '<span class="msg-media">[图片]</span>'
|
|
@@ -356,58 +1677,140 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
356
1677
|
mime = 'image/jpeg'
|
|
357
1678
|
|
|
358
1679
|
# Priority 1: NT cache thumbnails
|
|
359
|
-
|
|
360
|
-
|
|
1680
|
+
cached = get_cached_image(image_map, local_id, create_time, metadata_content, resource_md5s)
|
|
1681
|
+
if cached:
|
|
1682
|
+
img_data, mime = cached
|
|
361
1683
|
# Priority 2: Traditional FileStorage
|
|
362
1684
|
else:
|
|
363
1685
|
result = find_thumbnail(create_time, local_id, wx_dir)
|
|
364
1686
|
if result:
|
|
365
1687
|
img_data, mime = result
|
|
1688
|
+
else:
|
|
1689
|
+
thumb_url = extract_appmsg_image(content)
|
|
1690
|
+
downloaded = download_image_as_base64(thumb_url, extract_media_aes_key(content)) if thumb_url else None
|
|
1691
|
+
if downloaded:
|
|
1692
|
+
img_data, mime = downloaded
|
|
366
1693
|
|
|
367
1694
|
if img_data:
|
|
368
1695
|
image_b64 = img_data
|
|
369
1696
|
display += f'<br><img src="data:{mime};base64,{img_data}" loading="lazy" />'
|
|
370
|
-
elif
|
|
1697
|
+
elif local_type not in MSG_TYPES and get_cached_image(image_map, local_id, create_time, content, resource_md5s):
|
|
371
1698
|
# Some image messages use encoded types (e.g. 21474836529 = images in appmsg)
|
|
372
1699
|
# Check image_map for any message type
|
|
373
|
-
img_data, mime = image_map
|
|
1700
|
+
img_data, mime = get_cached_image(image_map, local_id, create_time, content, resource_md5s)
|
|
374
1701
|
if img_data:
|
|
375
1702
|
image_b64 = img_data
|
|
376
1703
|
display = f'<span class="msg-media">[图片]</span><br><img src="data:{mime};base64,{img_data}" loading="lazy" />'
|
|
377
1704
|
elif local_type == 34:
|
|
378
|
-
display =
|
|
1705
|
+
display = render_voice(local_id, metadata_content or content)
|
|
379
1706
|
elif local_type == 43:
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
1707
|
+
# A video row has no frame of its own, but a poster image can exist
|
|
1708
|
+
# under the same md5, so show it when the index has one.
|
|
1709
|
+
seconds = extract_xml_attr_value(metadata_content, 'playlength')
|
|
1710
|
+
label = f'[视频 {seconds}″]' if seconds.isdigit() and seconds != '0' else '[视频]'
|
|
1711
|
+
display = f'<span class="msg-media">{label}</span>'
|
|
1712
|
+
cached = get_cached_image(image_map, local_id, create_time, metadata_content, resource_md5s)
|
|
1713
|
+
if cached:
|
|
1714
|
+
img_data, mime = cached
|
|
1715
|
+
image_b64 = img_data
|
|
1716
|
+
display += f'<br><img src="data:{mime};base64,{img_data}" loading="lazy" />'
|
|
1717
|
+
elif local_type == 48:
|
|
1718
|
+
display = render_location(metadata_content or content)
|
|
1719
|
+
elif is_emoji_xml or (local_type in (1, 47) and ('<' in metadata_content or local_type == 47)):
|
|
1720
|
+
emoji_label = content if content.startswith('[') and content.endswith(']') else '[表情]'
|
|
1721
|
+
cached = get_cached_image(image_map, local_id, create_time, metadata_content, resource_md5s)
|
|
1722
|
+
if cached:
|
|
1723
|
+
img_data, mime = cached
|
|
1724
|
+
image_b64 = img_data
|
|
1725
|
+
display = f'<span class="msg-media">{escape_html(emoji_label)}</span><br><img src="data:{mime};base64,{img_data}" loading="lazy" />'
|
|
1726
|
+
else:
|
|
1727
|
+
# Local first. WeChat's own sticker cache is offline, instant, and
|
|
1728
|
+
# normally resolves; the CDN paths need a network round-trip each
|
|
1729
|
+
# and usually have nothing left to serve for older stickers.
|
|
1730
|
+
if _WECHAT_EMOTICON and STICKER_STATE['key'] and metadata_content:
|
|
1731
|
+
sticker_md5 = extract_xml_attr_value(metadata_content, 'md5') or ''
|
|
1732
|
+
data, mime = wechat_emoticon.load_sticker(
|
|
1733
|
+
STICKER_STATE['dirs'], sticker_md5, STICKER_STATE['key'],
|
|
1734
|
+
STICKER_STATE['cache_dir'])
|
|
1735
|
+
if data:
|
|
1736
|
+
img_data = base64.b64encode(data).decode()
|
|
1737
|
+
image_b64 = img_data
|
|
1738
|
+
display = f'<span class="msg-media">{escape_html(emoji_label)}</span><br><img src="data:{mime};base64,{img_data}" loading="lazy" />'
|
|
1739
|
+
thumb_url = ''
|
|
1740
|
+
if not image_b64:
|
|
1741
|
+
thumb_url = extract_appmsg_image(metadata_content)
|
|
1742
|
+
downloaded = download_image_as_base64(thumb_url, extract_media_aes_key(metadata_content)) if thumb_url else None
|
|
1743
|
+
if not downloaded:
|
|
1744
|
+
fallback_url = extract_xml_attr_url(metadata_content, 'thumburl')
|
|
1745
|
+
if fallback_url and fallback_url != thumb_url:
|
|
1746
|
+
downloaded = download_image_as_base64(fallback_url)
|
|
1747
|
+
if downloaded:
|
|
1748
|
+
img_data, mime = downloaded
|
|
1749
|
+
image_b64 = img_data
|
|
1750
|
+
display = f'<span class="msg-media">{escape_html(emoji_label)}</span><br><img src="data:{mime};base64,{img_data}" loading="lazy" />'
|
|
1751
|
+
if image_b64 and image_is_blank(image_b64):
|
|
1752
|
+
# A blank square is indistinguishable from a broken page; fall
|
|
1753
|
+
# back to the label so the message still reads as an emoji.
|
|
1754
|
+
image_b64 = None
|
|
1755
|
+
img_data = None
|
|
1756
|
+
display = f'<span class="msg-media">{escape_html(emoji_label)}</span>'
|
|
1757
|
+
if image_b64:
|
|
1758
|
+
pass
|
|
1759
|
+
elif has_builtin_signature and builtin_emoji_label:
|
|
1760
|
+
display = render_builtin_emoji(content, builtin_emoji_label)
|
|
1761
|
+
else:
|
|
1762
|
+
# Forwarded/default emoji messages can contain a complete
|
|
1763
|
+
# appmsg XML wrapper but no recoverable local media. Keep the
|
|
1764
|
+
# export readable instead of dumping the XML into the bubble.
|
|
1765
|
+
if is_emoji_xml:
|
|
1766
|
+
title = extract_xml_text(content, 'title')
|
|
1767
|
+
display = escape_html(title or emoji_label)
|
|
1768
|
+
elif is_share_page_url(content.strip()):
|
|
1769
|
+
# A webpage URL carried by an emoji-like row is still a
|
|
1770
|
+
# link, not an image. Preserve the original content.
|
|
1771
|
+
display = escape_html(content)
|
|
1772
|
+
else:
|
|
1773
|
+
display = escape_html(content) if content else '<span class="msg-media">[表情]</span>'
|
|
1774
|
+
elif local_type == 49 and not is_emoji_xml:
|
|
384
1775
|
# App message (link/file/article)
|
|
385
1776
|
if content:
|
|
386
1777
|
# Try to parse XML for title/desc
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
elif title_m:
|
|
1778
|
+
title = readable_fragment(extract_xml_text(content, 'title'))
|
|
1779
|
+
desc = readable_fragment(extract_xml_text(content, 'des'))
|
|
1780
|
+
url = extract_xml_text(content, 'url')
|
|
1781
|
+
app_type = extract_xml_text(content, 'type')
|
|
1782
|
+
|
|
1783
|
+
if app_type == '6' and re.search(r'\.\w+$', title):
|
|
1784
|
+
display = f'<span class="msg-file">[文件] {escape_html(title)}</span>'
|
|
1785
|
+
elif title:
|
|
396
1786
|
parts = []
|
|
1787
|
+
builtin_title = _face_index.find_face(title)
|
|
1788
|
+
title_html = (render_builtin_emoji(title, builtin_title)
|
|
1789
|
+
if builtin_title else escape_html(title))
|
|
397
1790
|
# Extract and embed article thumbnail image
|
|
398
1791
|
thumb_url = extract_appmsg_image(content)
|
|
399
1792
|
if thumb_url:
|
|
400
|
-
img_data = download_image_as_base64(thumb_url)
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
1793
|
+
img_data = download_image_as_base64(thumb_url, extract_media_aes_key(content))
|
|
1794
|
+
else:
|
|
1795
|
+
page_url = extract_xml_text(content, 'url')
|
|
1796
|
+
img_data = download_bilibili_cover(page_url)
|
|
1797
|
+
if not img_data:
|
|
1798
|
+
img_data = download_page_og_image(page_url)
|
|
1799
|
+
if img_data:
|
|
1800
|
+
b64, mime = img_data
|
|
1801
|
+
image_b64 = b64
|
|
1802
|
+
parts.append(f'<img class="msg-app-thumb" src="data:{mime};base64,{b64}" loading="lazy" />')
|
|
1803
|
+
# No remote-src fallback. The candidate URL came from a
|
|
1804
|
+
# catch-all that accepts any URL in the row, which is usually a
|
|
1805
|
+
# web page link (`meeting.tencent.com`, `github.com`,
|
|
1806
|
+
# `support.weixin.qq.com`) rather than an image; emitting it
|
|
1807
|
+
# produced a broken-image icon in 56 of 57 sampled cases.
|
|
1808
|
+
if url.startswith(('http://', 'https://')):
|
|
1809
|
+
parts.append(f'<a class="msg-link" href="{escape_html(url)}" target="_blank">{title_html}</a>')
|
|
407
1810
|
else:
|
|
408
|
-
parts.append(f'<span class="msg-app-title">{
|
|
409
|
-
if
|
|
410
|
-
parts.append(f'<div class="msg-app-desc">{escape_html(
|
|
1811
|
+
parts.append(f'<span class="msg-app-title">{title_html}</span>')
|
|
1812
|
+
if desc:
|
|
1813
|
+
parts.append(f'<div class="msg-app-desc">{escape_html(desc)}</div>')
|
|
411
1814
|
display = '<div class="msg-app">' + ''.join(parts) + '</div>'
|
|
412
1815
|
else:
|
|
413
1816
|
display = '<span class="msg-media">[链接/文件]</span>'
|
|
@@ -416,7 +1819,8 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
416
1819
|
elif local_type == 50:
|
|
417
1820
|
display = '<span class="msg-media">[语音通话]</span>'
|
|
418
1821
|
elif local_type == 10000:
|
|
419
|
-
|
|
1822
|
+
names = {talker: display_name or talker, own_wxid: '我'}
|
|
1823
|
+
display = f'<span class="msg-sys">{render_system_message(content, names)}</span>'
|
|
420
1824
|
elif local_type == 10002:
|
|
421
1825
|
display = escape_html(content) if content else '<span class="msg-media">[引用]</span>'
|
|
422
1826
|
else:
|
|
@@ -438,6 +1842,20 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
438
1842
|
}
|
|
439
1843
|
|
|
440
1844
|
|
|
1845
|
+
def decode_message_content(value):
|
|
1846
|
+
"""Decode NT message content, which may be Zstandard compressed."""
|
|
1847
|
+
if not value:
|
|
1848
|
+
return ''
|
|
1849
|
+
try:
|
|
1850
|
+
import zstandard
|
|
1851
|
+
value = zstandard.ZstdDecompressor().decompress(value)
|
|
1852
|
+
except Exception:
|
|
1853
|
+
pass
|
|
1854
|
+
if isinstance(value, bytes):
|
|
1855
|
+
return value.decode('utf-8', errors='ignore')
|
|
1856
|
+
return str(value)
|
|
1857
|
+
|
|
1858
|
+
|
|
441
1859
|
def decode_xml(text):
|
|
442
1860
|
"""Decode XML entities."""
|
|
443
1861
|
return (text
|
|
@@ -448,9 +1866,15 @@ def decode_xml(text):
|
|
|
448
1866
|
.replace(''', "'"))
|
|
449
1867
|
|
|
450
1868
|
|
|
451
|
-
def build_html_page(talker, messages_part, part_num, total_parts, display_name):
|
|
452
|
-
"""Build a single HTML page for a part.
|
|
453
|
-
|
|
1869
|
+
def build_html_page(talker, messages_part, part_num, total_parts, display_name, file_prefix=''):
|
|
1870
|
+
"""Build a single HTML page for a part.
|
|
1871
|
+
|
|
1872
|
+
`file_prefix` must be the prefix the parts were actually written under.
|
|
1873
|
+
Deriving it from the talker instead produced links to
|
|
1874
|
+
`wxid_..._part2.html` next to files named after the remark, so every
|
|
1875
|
+
navigation link 404'd.
|
|
1876
|
+
"""
|
|
1877
|
+
talker_safe = file_prefix or talker.replace('@', '_').replace('/', '_')
|
|
454
1878
|
rows = []
|
|
455
1879
|
for m in messages_part:
|
|
456
1880
|
dt = datetime.datetime.fromtimestamp(m['create_time'])
|
|
@@ -474,6 +1898,10 @@ def build_html_page(talker, messages_part, part_num, total_parts, display_name):
|
|
|
474
1898
|
from_time = datetime.datetime.fromtimestamp(messages_part[0]['create_time']).strftime('%Y-%m-%d %H:%M')
|
|
475
1899
|
to_time = datetime.datetime.fromtimestamp(messages_part[-1]['create_time']).strftime('%Y-%m-%d %H:%M')
|
|
476
1900
|
|
|
1901
|
+
# Emit artwork only for the faces this page uses; shipping the whole
|
|
1902
|
+
# set in every part would add ~1MB to each.
|
|
1903
|
+
face_rules = wechat_emoji.face_css(''.join(rows)) if _WECHAT_EMOJI else ''
|
|
1904
|
+
|
|
477
1905
|
return f'''<!DOCTYPE html>
|
|
478
1906
|
<html lang="zh-CN">
|
|
479
1907
|
<head>
|
|
@@ -536,7 +1964,14 @@ body {{
|
|
|
536
1964
|
.msg-sender {{ font-size: 11px; color: #999; margin-bottom: 3px; }}
|
|
537
1965
|
.msg-content {{ font-size: 15px; line-height: 1.5; white-space: pre-wrap; word-break: break-word; }}
|
|
538
1966
|
.msg-content img {{ max-width: 240px; max-height: 240px; border-radius: 4px; margin-top: 6px; display: block; }}
|
|
1967
|
+
.wxface {{
|
|
1968
|
+
display: inline-block; width: 22px; height: 22px; vertical-align: -5px;
|
|
1969
|
+
background-size: 22px 22px; background-repeat: no-repeat; margin: 0 1px;
|
|
1970
|
+
}}
|
|
1971
|
+
{face_rules}
|
|
539
1972
|
.msg-media {{ color: #888; font-size: 14px; }}
|
|
1973
|
+
.msg-voice-text {{ margin-top: 4px; padding: 6px 9px; background: rgba(0,0,0,0.045); border-left: 3px solid #bbb; border-radius: 3px; font-size: 14px; line-height: 1.5; color: #444; }}
|
|
1974
|
+
.voice-tag {{ display: inline-block; margin-right: 6px; padding: 1px 5px; border-radius: 3px; background: #fdf0e0; color: #b5762a; font-size: 11px; vertical-align: 1px; white-space: nowrap; }}
|
|
540
1975
|
.msg-sys {{ color: #bbb; font-size: 13px; }}
|
|
541
1976
|
.msg-file {{ color: #07c160; font-weight: 500; }}
|
|
542
1977
|
.msg-app {{ margin: 0; }}
|
|
@@ -613,7 +2048,8 @@ body {{
|
|
|
613
2048
|
</div>
|
|
614
2049
|
<div class="footer">
|
|
615
2050
|
<p>Exported by WeFlow CLI · {datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}</p>
|
|
616
|
-
<p class="hint">💡
|
|
2051
|
+
<p class="hint">💡 图片与表情来自本机微信数据（会话缓存 + 账号媒体索引）。本机从未下载过的图片、以及未缓存封面的视频，无法显示。</p>
|
|
2052
|
+
<p class="hint">⚠️ 语音下方文字为<b>本机机器转写</b>，未经校对。粤语/方言的识别准确率有限，可能出现通顺但并非原话的内容——<b>不可作为原话引用</b>；需要确证时请以录音为准。</p>
|
|
617
2053
|
</div>
|
|
618
2054
|
<script>
|
|
619
2055
|
function searchMessages(query) {{
|
|
@@ -647,59 +2083,206 @@ function searchMessages(query) {{
|
|
|
647
2083
|
</html>'''
|
|
648
2084
|
|
|
649
2085
|
|
|
2086
|
+
def build_empty_html(talker, date, display_name):
|
|
2087
|
+
"""Build a date-scoped empty page without falling back to older messages."""
|
|
2088
|
+
name = escape_html(display_name or talker)
|
|
2089
|
+
date_text = escape_html(date or '指定日期')
|
|
2090
|
+
return f'''<!DOCTYPE html>
|
|
2091
|
+
<html lang="zh-CN">
|
|
2092
|
+
<head>
|
|
2093
|
+
<meta charset="UTF-8">
|
|
2094
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
2095
|
+
<title>聊天记录 - {name} - {date_text}</title>
|
|
2096
|
+
<style>
|
|
2097
|
+
body {{ margin:0; padding:48px 20px; background:#ededed; font-family:-apple-system,BlinkMacSystemFont,"Microsoft YaHei",sans-serif; color:#666; }}
|
|
2098
|
+
.empty {{ max-width:720px; margin:0 auto; padding:40px 20px; text-align:center; background:#fff; border-radius:12px; box-shadow:0 1px 3px rgba(0,0,0,.08); }}
|
|
2099
|
+
h2 {{ margin:0 0 12px; color:#333; font-size:18px; }}
|
|
2100
|
+
p {{ margin:8px 0; font-size:14px; }}
|
|
2101
|
+
</style>
|
|
2102
|
+
</head>
|
|
2103
|
+
<body><main class="empty"><h2>{name}</h2><p>{date_text}没有消息记录</p><p>未加载其他日期的历史消息。</p></main></body>
|
|
2104
|
+
</html>'''
|
|
2105
|
+
|
|
2106
|
+
|
|
650
2107
|
def main():
|
|
651
2108
|
import argparse
|
|
652
2109
|
parser = argparse.ArgumentParser(description='Export WeChat NT chat as HTML')
|
|
653
|
-
parser.add_argument('--db', required=
|
|
654
|
-
parser.add_argument('--key', required=
|
|
655
|
-
parser.add_argument('--salt', required=
|
|
656
|
-
parser.add_argument('--talker', required=
|
|
657
|
-
parser.add_argument('--name', default='', help='Display name')
|
|
658
|
-
parser.add_argument('--out', default='./output', help='Output directory')
|
|
2110
|
+
parser.add_argument('--db', default=os.environ.get('WEFLOW_DB_PATH'), required=not os.environ.get('WEFLOW_DB_PATH'), help='Path to NT database (message_0.db)')
|
|
2111
|
+
parser.add_argument('--key', default=os.environ.get('WEFLOW_NT_KEY'), required=not os.environ.get('WEFLOW_NT_KEY'), help='Key hex (64 chars)')
|
|
2112
|
+
parser.add_argument('--salt', default=os.environ.get('WEFLOW_NT_SALT'), required=not os.environ.get('WEFLOW_NT_SALT'), help='Salt hex (32 chars)')
|
|
2113
|
+
parser.add_argument('--talker', default=os.environ.get('WEFLOW_TALKER'), required=not os.environ.get('WEFLOW_TALKER'), help='Talker username')
|
|
2114
|
+
parser.add_argument('--name', default=os.environ.get('WEFLOW_EXPORT_NAME', ''), help='Display name')
|
|
2115
|
+
parser.add_argument('--out', default=os.environ.get('WEFLOW_EXPORT_OUTPUT', './output'), help='Output directory')
|
|
659
2116
|
parser.add_argument('--parts', type=int, default=5, help='Number of parts to split into')
|
|
660
2117
|
parser.add_argument('--wx-dir', default='', help='Traditional WeChat data dir (FileStorage fallback)')
|
|
661
|
-
parser.add_argument('--cache-dir', default='', help='NT cache directory for image thumbnails')
|
|
2118
|
+
parser.add_argument('--cache-dir', default=os.environ.get('WEFLOW_EXPORT_CACHE_DIR', ''), help='NT cache directory for image thumbnails')
|
|
2119
|
+
parser.add_argument('--account-dir', default=os.environ.get('WEFLOW_EXPORT_ACCOUNT_DIR', ''), help='Account data directory for media resources')
|
|
662
2120
|
parser.add_argument('--single', action='store_true', help='Generate a single HTML file (no splitting)')
|
|
2121
|
+
parser.add_argument('--per-page', type=int, default=0,
|
|
2122
|
+
help='Messages per file; overrides --parts. Keeps a long history snappy to open')
|
|
2123
|
+
parser.add_argument('--date', default=os.environ.get('WEFLOW_EXPORT_DATE', ''), help='Only export messages from local date YYYY-MM-DD')
|
|
2124
|
+
parser.add_argument('--emoticon-seed', default=os.environ.get('WEFLOW_EMOTICON_SEED', ''),
|
|
2125
|
+
help='Account seed; decrypts custom stickers from the local cache')
|
|
2126
|
+
parser.add_argument('--passphrase', default=os.environ.get('WEFLOW_NT_PASSPHRASE', ''), help='Shared NT passphrase for deriving shard keys')
|
|
2127
|
+
parser.add_argument('--own-wxid', default=os.environ.get('WEFLOW_OWN_WXID', ''), help='Configured account identifier for self-message detection')
|
|
2128
|
+
parser.add_argument('--contact-db', default=os.environ.get('WEFLOW_CONTACT_DB_PATH', ''), help='Contact database, for resolving group senders to names')
|
|
2129
|
+
parser.add_argument('--contact-key', default=os.environ.get('WEFLOW_CONTACT_KEY', ''), help='Contact database key hex')
|
|
2130
|
+
parser.add_argument('--contact-salt', default=os.environ.get('WEFLOW_CONTACT_SALT', ''), help='Contact database salt hex')
|
|
663
2131
|
args = parser.parse_args()
|
|
664
2132
|
|
|
665
2133
|
os.makedirs(args.out, exist_ok=True)
|
|
666
2134
|
|
|
667
2135
|
# Scan NT cache for image thumbnails
|
|
668
2136
|
image_map = {}
|
|
2137
|
+
_phase("start")
|
|
669
2138
|
if args.cache_dir:
|
|
670
2139
|
print(f"Scanning NT cache: {args.cache_dir}")
|
|
671
|
-
image_map = scan_nt_cache(args.cache_dir, args.talker)
|
|
672
|
-
|
|
2140
|
+
image_map = scan_nt_cache(args.cache_dir, args.talker, args.account_dir, args.own_wxid)
|
|
2141
|
+
else:
|
|
2142
|
+
image_map = {}
|
|
2143
|
+
# The account media index holds everything the conversation cache does not.
|
|
2144
|
+
# A conversation cache only keeps recent months, so without this a group
|
|
2145
|
+
# photo from last year resolved 0/1444 - every image and video came out as
|
|
2146
|
+
# a bare `[图片]`. Both indexes are merged rather than chosen between.
|
|
2147
|
+
if args.account_dir:
|
|
2148
|
+
account_media = scan_account_media(args.account_dir, args.own_wxid, args.talker)
|
|
2149
|
+
print(f" Account media index: {len(account_media)} entries")
|
|
2150
|
+
for key, value in account_media.items():
|
|
2151
|
+
if key in image_map:
|
|
2152
|
+
continue
|
|
2153
|
+
# Most entries are already thumbnail-sized (median 6KB); only the
|
|
2154
|
+
# few large originals are worth re-encoding.
|
|
2155
|
+
if len(value[0]) > EMBED_SHRINK_THRESHOLD:
|
|
2156
|
+
value = shrink_embedded(value, max_side=720, force=True)
|
|
2157
|
+
image_map[key] = value
|
|
2158
|
+
print(f" Found {len(image_map)} media entries for embedding")
|
|
2159
|
+
_phase("media scan")
|
|
2160
|
+
|
|
2161
|
+
# Custom stickers: derive the local cache key from the account seed.
|
|
2162
|
+
if args.account_dir and _WECHAT_EMOTICON:
|
|
2163
|
+
STICKER_STATE['dirs'] = wechat_emoticon.sticker_cache_dirs(args.account_dir)
|
|
2164
|
+
STICKER_STATE['cache_dir'] = os.path.join(args.out, '.sticker-cache')
|
|
2165
|
+
seed, discovered = resolve_emoticon_seed(args.emoticon_seed, args.account_dir, args.out)
|
|
2166
|
+
if seed:
|
|
2167
|
+
wxid = wechat_emoticon.account_wxid(os.path.basename(os.path.normpath(args.account_dir)))
|
|
2168
|
+
STICKER_STATE['key'] = wechat_emoticon.derive_key(seed, wxid)
|
|
2169
|
+
source = 'discovered' if discovered else 'configured'
|
|
2170
|
+
print(f"Stickers: {len(STICKER_STATE['dirs'])} cache dir(s), seed {source}")
|
|
2171
|
+
if discovered:
|
|
2172
|
+
print(f" 建议固化以避免每次扫描: weflow-cli config set emoticonSeed {seed}")
|
|
2173
|
+
else:
|
|
2174
|
+
print("Stickers: 未找到 seed（需微信正在运行），自定义表情包将显示为 [表情]")
|
|
2175
|
+
|
|
2176
|
+
# Remote fetches: cache on disk and bound the per-run budget, otherwise
|
|
2177
|
+
# a link-heavy conversation spends minutes on network round-trips.
|
|
2178
|
+
COVER_STATE['dir'] = os.path.join(args.out, '.cover-cache')
|
|
2179
|
+
COVER_STATE['imgshrink'] = os.path.join(args.out, '.imgshrink-cache')
|
|
2180
|
+
COVER_STATE['budget'] = COVER_FETCH_LIMIT
|
|
2181
|
+
COVER_STATE['thumb_budget'] = THUMB_FETCH_LIMIT
|
|
2182
|
+
|
|
2183
|
+
# Contact names make a group transcript readable; the ids alone do not.
|
|
2184
|
+
CONTACT_NAMES.update(load_contact_names(args.contact_db, args.contact_key, args.contact_salt))
|
|
2185
|
+
if CONTACT_NAMES:
|
|
2186
|
+
print(f"Contacts: {len(CONTACT_NAMES)} name(s)")
|
|
2187
|
+
|
|
2188
|
+
# Voice: load the payloads so a cached transcript can be shown. Anything
|
|
2189
|
+
# not yet transcribed simply renders as `[语音 N″]`.
|
|
2190
|
+
if _WECHAT_VOICE:
|
|
2191
|
+
VOICE_STATE['cache'] = wechat_voice.TranscriptCache(os.path.join(args.out, '.voice-cache'))
|
|
2192
|
+
try:
|
|
2193
|
+
VOICE_STATE['map'] = wechat_voice.load_voice_map(
|
|
2194
|
+
args.db, args.key, args.salt, args.passphrase, args.talker)
|
|
2195
|
+
except Exception:
|
|
2196
|
+
VOICE_STATE['map'] = {}
|
|
2197
|
+
if VOICE_STATE['map']:
|
|
2198
|
+
done = sum(
|
|
2199
|
+
1 for blob in VOICE_STATE['map'].values()
|
|
2200
|
+
if VOICE_STATE['cache'].get(wechat_voice.voice_key(blob)) is not None
|
|
2201
|
+
)
|
|
2202
|
+
print(f"Voice: {len(VOICE_STATE['map'])} clip(s), {done} transcribed")
|
|
2203
|
+
|
|
2204
|
+
# The caller resolves the display name from the session list, which only
|
|
2205
|
+
# covers recent conversations. A less recent chat therefore exported as
|
|
2206
|
+
# `wxid_..._part1.html` even though its remark was known all along. Fall
|
|
2207
|
+
# back to the contact database so files are named after the person.
|
|
2208
|
+
if not args.name and args.talker:
|
|
2209
|
+
args.name = CONTACT_NAMES.get(args.talker, '')
|
|
2210
|
+
if args.name:
|
|
2211
|
+
print(f"Name: {args.name} (from contacts)")
|
|
673
2212
|
|
|
674
2213
|
# Connect
|
|
675
2214
|
print(f"Connecting to {args.db}...")
|
|
2215
|
+
_phase("cover init")
|
|
676
2216
|
conn, c = connect(args.db, args.key, args.salt)
|
|
677
2217
|
|
|
678
2218
|
# Fetch messages
|
|
679
2219
|
print(f"Fetching messages for {args.talker}...")
|
|
680
|
-
|
|
2220
|
+
_phase("before fetch")
|
|
2221
|
+
messages = fetch_messages_from_shards(args.db, args.key, args.salt, args.talker, args.date, args.passphrase)
|
|
681
2222
|
|
|
682
2223
|
if not messages:
|
|
2224
|
+
if args.date:
|
|
2225
|
+
file_prefix = sanitize_filename(args.name or args.talker)
|
|
2226
|
+
filepath = os.path.join(args.out, f'{file_prefix}.html')
|
|
2227
|
+
with open(filepath, 'w', encoding='utf-8') as f:
|
|
2228
|
+
f.write(build_empty_html(args.talker, args.date, args.name or args.talker))
|
|
2229
|
+
print(f"No messages found for {args.date}; wrote empty date-scoped page")
|
|
2230
|
+
print(json.dumps({"success": True, "total": 0, "parts": 1, "files": [filepath]}))
|
|
2231
|
+
conn.close()
|
|
2232
|
+
return
|
|
683
2233
|
print("No messages found!")
|
|
684
2234
|
conn.close()
|
|
685
2235
|
sys.exit(1)
|
|
686
2236
|
|
|
687
|
-
|
|
2237
|
+
resource_map = load_resource_media_map(
|
|
2238
|
+
args.account_dir, args.key, args.salt, messages, image_map, args.passphrase
|
|
2239
|
+
)
|
|
2240
|
+
if resource_map:
|
|
2241
|
+
print(f" Found resource mappings for {len(resource_map)} message keys")
|
|
2242
|
+
_phase("resource map")
|
|
2243
|
+
|
|
2244
|
+
# Build sender name map from every message shard because each shard can
|
|
2245
|
+
# contain a different Name2Id mapping.
|
|
688
2246
|
print(f"Building sender name map...")
|
|
689
|
-
sender_map =
|
|
2247
|
+
sender_map = build_sender_map_from_shards(
|
|
2248
|
+
args.db, args.key, args.salt, args.talker, args.passphrase
|
|
2249
|
+
)
|
|
690
2250
|
print(f" Found {len(sender_map)} sender(s): {list(sender_map.values())}")
|
|
691
2251
|
|
|
692
2252
|
# Format messages
|
|
693
2253
|
display_name = args.name or args.talker
|
|
694
|
-
|
|
2254
|
+
_phase("before formatting")
|
|
695
2255
|
wx_dir = args.wx_dir or ''
|
|
2256
|
+
|
|
2257
|
+
# Remote media is the dominant cost and is discovered one message at a
|
|
2258
|
+
# time, so doing it inline serialises hundreds of round-trips. Sweep the
|
|
2259
|
+
# conversation once to record what it needs (local-only, ~0.1s), fetch it
|
|
2260
|
+
# all concurrently, then format against a warm cache.
|
|
2261
|
+
sink = []
|
|
2262
|
+
PREFETCH['sink'] = sink
|
|
2263
|
+
try:
|
|
2264
|
+
for row in messages:
|
|
2265
|
+
format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid)
|
|
2266
|
+
finally:
|
|
2267
|
+
PREFETCH['sink'] = None
|
|
2268
|
+
if sink:
|
|
2269
|
+
print(f"Prefetching {len(sink)} remote media reference(s)...", flush=True)
|
|
2270
|
+
_phase("prefetch scan")
|
|
2271
|
+
ok, dropped = prefetch_remote(sink)
|
|
2272
|
+
print(f" Resolved {ok} remote item(s)" + (f", skipped {dropped} (cap)" if dropped else ""), flush=True)
|
|
2273
|
+
_phase("prefetch fetch")
|
|
2274
|
+
# Budgets were not charged during the prefetch, so anything it missed
|
|
2275
|
+
# is still bounded by the full per-run allowance here.
|
|
2276
|
+
|
|
2277
|
+
print(f"Formatting {len(messages)} messages...")
|
|
696
2278
|
formatted = []
|
|
697
2279
|
img_hit_count = 0
|
|
698
2280
|
article_img_count = 0
|
|
2281
|
+
progress_step = max(1, len(messages) // 20)
|
|
699
2282
|
for i, row in enumerate(messages):
|
|
700
|
-
if i %
|
|
701
|
-
print(f" Formatting {i}/{len(messages)}...")
|
|
702
|
-
result = format_message(row, args.talker, wx_dir, image_map, sender_map, display_name)
|
|
2283
|
+
if i % progress_step == 0:
|
|
2284
|
+
print(f" Formatting {i}/{len(messages)}...", flush=True)
|
|
2285
|
+
result = format_message(row, args.talker, wx_dir, image_map, sender_map, display_name, resource_map, args.own_wxid)
|
|
703
2286
|
if result.get('image_b64'):
|
|
704
2287
|
img_hit_count += 1
|
|
705
2288
|
if result.get('local_type') == 49:
|
|
@@ -709,12 +2292,15 @@ def main():
|
|
|
709
2292
|
|
|
710
2293
|
# Split into parts (or single file)
|
|
711
2294
|
total = len(formatted)
|
|
712
|
-
if args.
|
|
2295
|
+
if args.per_page and args.per_page > 0:
|
|
2296
|
+
parts = max(1, (total + args.per_page - 1) // args.per_page)
|
|
2297
|
+
elif args.single:
|
|
713
2298
|
parts = 1
|
|
714
2299
|
else:
|
|
715
2300
|
parts = min(args.parts, total)
|
|
716
2301
|
per_part = (total + parts - 1) // parts
|
|
717
2302
|
|
|
2303
|
+
_phase(f"formatting {len(messages)} msgs")
|
|
718
2304
|
print(f"Splitting into {parts} part(s) (~{per_part} messages each)...")
|
|
719
2305
|
|
|
720
2306
|
# Use display name for filename if provided, otherwise fallback to wxid
|
|
@@ -729,8 +2315,8 @@ def main():
|
|
|
729
2315
|
if not chunk:
|
|
730
2316
|
break
|
|
731
2317
|
|
|
732
|
-
html = build_html_page(args.talker, chunk, i + 1, parts, display_name)
|
|
733
|
-
if
|
|
2318
|
+
html = build_html_page(args.talker, chunk, i + 1, parts, display_name, file_prefix)
|
|
2319
|
+
if parts == 1:
|
|
734
2320
|
filename = f"{file_prefix}.html"
|
|
735
2321
|
else:
|
|
736
2322
|
filename = f"{file_prefix}_part{i+1}.html"
|
|
@@ -757,6 +2343,17 @@ def main():
|
|
|
757
2343
|
}))
|
|
758
2344
|
|
|
759
2345
|
|
|
2346
|
+
_PHASE_T0 = time.time()
|
|
2347
|
+
|
|
2348
|
+
|
|
2349
|
+
def _phase(label):
|
|
2350
|
+
global _PHASE_T0
|
|
2351
|
+
now = time.time()
|
|
2352
|
+
if os.environ.get("WEFLOW_DEBUG"):
|
|
2353
|
+
print(f" [t+{now - _PHASE_T0:6.1f}s] {label}", flush=True)
|
|
2354
|
+
_PHASE_T0 = now
|
|
2355
|
+
|
|
2356
|
+
|
|
760
2357
|
def sanitize_filename(name: str) -> str:
|
|
761
2358
|
"""Remove characters unsafe for filenames."""
|
|
762
2359
|
return re.sub(r'[\\/:*?"<>|]', '_', name)[:80]
|