weflow-cli 1.6.3 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/OPERATIONS.md +36 -1
- package/README.en.md +1 -1
- package/README.md +1 -1
- package/bin/weflow-cli.ts +312 -0
- package/dist/bin/weflow-cli.js +352 -0
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/src/core/ntCore.d.ts +26 -0
- package/dist/src/core/ntCore.d.ts.map +1 -1
- package/dist/src/core/ntCore.js +21 -0
- package/dist/src/core/ntCore.js.map +1 -1
- package/dist/src/services/chatService.d.ts +13 -0
- package/dist/src/services/chatService.d.ts.map +1 -1
- package/dist/src/services/chatService.js +20 -0
- package/dist/src/services/chatService.js.map +1 -1
- package/dist/src/services/messageQuery.d.ts +35 -0
- package/dist/src/services/messageQuery.d.ts.map +1 -1
- package/dist/src/services/messageQuery.js +61 -0
- package/dist/src/services/messageQuery.js.map +1 -1
- package/dist/src/services/syncService.d.ts +121 -0
- package/dist/src/services/syncService.d.ts.map +1 -0
- package/dist/src/services/syncService.js +244 -0
- package/dist/src/services/syncService.js.map +1 -0
- package/dist/src/services/syncState.d.ts +113 -0
- package/dist/src/services/syncState.d.ts.map +1 -0
- package/dist/src/services/syncState.js +214 -0
- package/dist/src/services/syncState.js.map +1 -0
- package/docs/DECISIONS.md +60 -20
- package/docs/FUNCTIONAL-IMPROVEMENT-SPEC.md +338 -0
- package/docs/HEALTH-CHECK.md +6 -5
- package/docs/MCP.md +1 -1
- package/docs/NPM-PUBLISH.md +12 -4
- package/docs/PROJECT_STATE.md +8 -6
- package/docs/SETUP.md +1 -1
- package/docs/SYNC_CONTRACT.md +152 -0
- package/package.json +3 -3
- package/scripts/export_chat_html.py +242 -26
- package/scripts/nt_decrypt.py +123 -15
- package/src/core/ntCore.ts +42 -0
- package/src/services/chatService.ts +26 -0
- package/src/services/messageQuery.ts +97 -0
- package/src/services/syncService.ts +335 -0
- package/src/services/syncState.ts +275 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# 同步与任务状态契约
|
|
2
|
+
|
|
3
|
+
本文冻结两个内部状态 schema:`weflow-sync/v1`(跨运行的同步检查点)与 `weflow-job/v1`
|
|
4
|
+
(单次运行的记录)。二者都在本机 `~/.weflow-cli/` 下,**不对外暴露**。
|
|
5
|
+
|
|
6
|
+
它们是 [FUNCTIONAL-IMPROVEMENT-SPEC.md](FUNCTIONAL-IMPROVEMENT-SPEC.md) §4 与 §5 的落地,
|
|
7
|
+
边界受 [DECISIONS.md](DECISIONS.md) 约束(尤其 D-014、D-019、D-025、D-027)。
|
|
8
|
+
|
|
9
|
+
## 为什么是两个文件
|
|
10
|
+
|
|
11
|
+
生命周期不同,不是猜测性抽象:
|
|
12
|
+
|
|
13
|
+
| 文件 | 生命周期 | 记录的是 |
|
|
14
|
+
| --- | --- | --- |
|
|
15
|
+
| `sync/<source>-<name>-<hash>.json` | **跨运行**,每次同步覆盖 | 已覆盖到哪、分片读没读全 |
|
|
16
|
+
| `jobs/<jobId>.json` | **单次运行** | 这次运行的起止、状态、结果 |
|
|
17
|
+
|
|
18
|
+
一次运行写一个 job 记录;同步检查点记录"到目前为止覆盖范围"这一长期事实。
|
|
19
|
+
|
|
20
|
+
## `weflow-sync/v1`
|
|
21
|
+
|
|
22
|
+
路径:`~/.weflow-cli/sync/<source>-<sanitize(talker)>-<sha256(talker)[0:8]>.json`
|
|
23
|
+
|
|
24
|
+
**按会话分文件**,不是一个全局文件。理由:全局文件要求跨会话读改写并合并,而分文件
|
|
25
|
+
让每次写入都是整文件原子替换;同时文件名里的 hash 后缀让 `wxid_a@chatroom` 与
|
|
26
|
+
`wxid_a_chatroom` 落到不同文件,而显示名不会出现在目录列表里。
|
|
27
|
+
|
|
28
|
+
```json
|
|
29
|
+
{
|
|
30
|
+
"schema": "weflow-sync/v1",
|
|
31
|
+
"schemaVersion": 1,
|
|
32
|
+
"source": "wechat-nt",
|
|
33
|
+
"scope": "<显示名,未知时等于 talker>",
|
|
34
|
+
"talker": "<会话标识,不是路径>",
|
|
35
|
+
"lastSuccessfulRun": "2026-09-19T09:00:00+08:00",
|
|
36
|
+
"lastAttempt": "2026-09-19T09:02:10+08:00",
|
|
37
|
+
"coveredFrom": "2026-09-01T00:00:00+08:00",
|
|
38
|
+
"coveredTo": "2026-09-19T08:59:59+08:00",
|
|
39
|
+
"checkpoint": { "newestCreateTime": 1789000000, "overlapSeconds": 300 },
|
|
40
|
+
"recordsRead": 1200,
|
|
41
|
+
"recordsDeduplicated": 14,
|
|
42
|
+
"recordsReturned": 1186,
|
|
43
|
+
"shardsRead": 4,
|
|
44
|
+
"shardsFailed": 0,
|
|
45
|
+
"shards": {
|
|
46
|
+
"scanned": 4, "opened": 4, "failed": 0,
|
|
47
|
+
"items": [
|
|
48
|
+
{ "name": "message_0.db", "opened": true, "hasTalkerTable": true,
|
|
49
|
+
"rowsForTalker": 8123, "reason": null }
|
|
50
|
+
]
|
|
51
|
+
},
|
|
52
|
+
"coverage": "complete",
|
|
53
|
+
"mayHaveMore": false,
|
|
54
|
+
"warnings": [],
|
|
55
|
+
"updatedAt": "2026-09-19T09:02:10+08:00"
|
|
56
|
+
}
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### 两个时间戳必须分开
|
|
60
|
+
|
|
61
|
+
| 字段 | 何时推进 |
|
|
62
|
+
| --- | --- |
|
|
63
|
+
| `lastAttempt` | **每次尝试都推进**,无论结果 |
|
|
64
|
+
| `lastSuccessfulRun` | **仅在 `coverage !== 'partial'` 时推进** |
|
|
65
|
+
|
|
66
|
+
只有一个的话,"跑了但部分完成"会和"跑成功了"混为一谈——而这正是方案 §5.2 明令禁止的
|
|
67
|
+
(部分完成不能返回成功状态)。
|
|
68
|
+
|
|
69
|
+
### `coverage` 三档
|
|
70
|
+
|
|
71
|
+
| 值 | 含义 |
|
|
72
|
+
| --- | --- |
|
|
73
|
+
| `complete` | 扫到的分片全部打开、打开的全读成功,且 `mayHaveMore` 为 false |
|
|
74
|
+
| `unverified` | 有分片**打不开**,但打开的都读成功。已读部分是可信的,未打开的分片是否含本会话消息**无法判断** |
|
|
75
|
+
| `partial` | 某个**已打开**的分片读失败,或 `mayHaveMore` 为 true,或分页在到达窗口起点前就短页结束 |
|
|
76
|
+
|
|
77
|
+
`unverified` 不是失败:`connect_message_shards` 一直把打不开的分片当作可跳过的(这是有意设计),
|
|
78
|
+
CLI 无法知道那个分片里有没有本会话的消息。如实标成 `unverified` + `warnings` 比直接失败更有用,
|
|
79
|
+
也让自动化不至于卡死。
|
|
80
|
+
|
|
81
|
+
`partial` 则**必须失败**(`code: 'SYNC_PARTIAL'`,退出码 1):状态文件照写(方案 §5.2 要求
|
|
82
|
+
部分完成不得删除已生成的可用结果),但 `lastSuccessfulRun` 不推进。
|
|
83
|
+
|
|
84
|
+
### 身份
|
|
85
|
+
|
|
86
|
+
按方案 §4.2,`serverId` 非零时优先;**`localId` 绝不单独作为全局身份**(D-014 同此约束)。
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
talker|s:<serverId> serverId 非零
|
|
90
|
+
talker|l:<localId>|t:<createTime> 否则
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
**与方案 §4.2 的一处有意偏离**:键里**不含 `shard`**。消息载荷里没有 shard 字段
|
|
94
|
+
(`nt_decrypt._message_dict` 不产出它),加进去要动 `export json` 的消息契约,而 D-016/D-027
|
|
95
|
+
把那条线封住了。它对本契约要解决的**重叠去重**也不必要——同一条消息重读两次,得到的
|
|
96
|
+
`serverId` 或 `(localId, createTime)` 本来就一样。只有跨分片的 `(localId, createTime)` 撞车
|
|
97
|
+
才需要它,而 `talker` + `createTime` 已让这种撞车极其遥远。**真观测到再加,不预先加。**
|
|
98
|
+
|
|
99
|
+
### 明确不做的事
|
|
100
|
+
|
|
101
|
+
**`mayHaveMore` 不是稳定游标。** 按 D-027,稳定增量游标必须在所有相关后端都支持后才能
|
|
102
|
+
对外宣称;在此之前只提供重叠时间窗 + 本地去重,由下游决定重复容忍度。
|
|
103
|
+
|
|
104
|
+
**不得写入**:数据库密钥、`x'<key><salt>'`、聊天正文、绝对路径(D-001 / D-002 / D-025)。
|
|
105
|
+
状态服务在写盘前统一过一遍脱敏,不指望每个调用方都记得。
|
|
106
|
+
|
|
107
|
+
## `weflow-job/v1`
|
|
108
|
+
|
|
109
|
+
路径:`~/.weflow-cli/jobs/<jobId>.json`。`jobId` 形如 `job_20260919_0001`。
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
{
|
|
113
|
+
"schema": "weflow-job/v1",
|
|
114
|
+
"jobId": "job_20260919_0001",
|
|
115
|
+
"kind": "sync.messages",
|
|
116
|
+
"state": "running",
|
|
117
|
+
"createdAt": "2026-09-19T09:00:00+08:00",
|
|
118
|
+
"updatedAt": "2026-09-19T09:02:10+08:00",
|
|
119
|
+
"window": { "from": 1788000000, "to": 1789000000 },
|
|
120
|
+
"progress": { "completed": 800, "total": 1200 },
|
|
121
|
+
"resumeable": true,
|
|
122
|
+
"result": { "records": 800, "recordsDeduplicated": 14, "shardsFailed": 0 },
|
|
123
|
+
"error": null
|
|
124
|
+
}
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
状态机:`planned → waiting_confirmation → running → paused`,终态 `completed | partial | failed | cancelled`。
|
|
128
|
+
本轮只实现 `running | completed | partial | failed`,其余是预留(不实现的不写进状态机文档当既有能力)。
|
|
129
|
+
|
|
130
|
+
`partial` 是本项目新增的一档,**不算成功**。`error` 只放既有的大写错误码
|
|
131
|
+
(`NOT_INITIALIZED`、`CONFIRMATION_REQUIRED`、`INVALID_ARGUMENT`、`SYNC_PARTIAL`、
|
|
132
|
+
`DATABASE_CONNECTION_FAILED` 等),不写中文长句供机器解析。
|
|
133
|
+
|
|
134
|
+
**幂等**:同一范围重复提交应返回已有任务或明确新建,不得静默重复写入。本轮用
|
|
135
|
+
`kind + talker + 窗口` 生成幂等提示,命中未完成的同键任务时返回该任务而非新建。
|
|
136
|
+
|
|
137
|
+
> 本轮 `sync` 是 job 记录的唯一写入者。若日后认为不值得单独一个文件,
|
|
138
|
+
> **回退路径**:删掉 job 存储、把那个 `lastJob` 对象并入各会话的 `weflow-sync/v1` 文件。
|
|
139
|
+
> 因为此刻还没有第二个消费者,这是一次单文件改动,无兼容性影响。
|
|
140
|
+
|
|
141
|
+
## 版本升级策略
|
|
142
|
+
|
|
143
|
+
- `schema` 变化 = 不兼容变更,需要迁移说明。
|
|
144
|
+
- 只增字段 = 兼容变更;读方必须容忍未知字段(与 `DATA_CONTRACT.md` 对下游的要求一致)。
|
|
145
|
+
- 读到未知 `schema` 时**不得覆盖**:报明确错误并保留原文件。
|
|
146
|
+
|
|
147
|
+
## 相关文档
|
|
148
|
+
|
|
149
|
+
- [功能增强技术方案](FUNCTIONAL-IMPROVEMENT-SPEC.md)
|
|
150
|
+
- [消息数据契约](DATA_CONTRACT.md)
|
|
151
|
+
- [技术决策记录](DECISIONS.md)
|
|
152
|
+
- [项目当前状态](PROJECT_STATE.md)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "weflow-cli",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.0",
|
|
4
4
|
"description": "让 AI 读懂你的微信 — 公众号日报·聊天导出·知识库搜索·MCP Server",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
},
|
|
44
44
|
"scripts": {
|
|
45
45
|
"build": "tsc",
|
|
46
|
-
"test": "node --import tsx --test test/**/*.test.ts",
|
|
46
|
+
"test": "node --import tsx --test \"test/**/*.test.ts\"",
|
|
47
47
|
"dev": "npx tsx bin/weflow-cli.ts",
|
|
48
48
|
"start": "node cli.cjs",
|
|
49
49
|
"prepublishOnly": "npm run build"
|
|
@@ -78,6 +78,6 @@
|
|
|
78
78
|
"typescript": "^5.7.0"
|
|
79
79
|
},
|
|
80
80
|
"engines": {
|
|
81
|
-
"node": ">=
|
|
81
|
+
"node": ">=22.13.0"
|
|
82
82
|
}
|
|
83
83
|
}
|
|
@@ -72,10 +72,21 @@ except Exception:
|
|
|
72
72
|
# Remote media is the single slowest step of an export: each miss costs a page
|
|
73
73
|
# fetch plus an image download, and inline thumbnails alone run to several
|
|
74
74
|
# hundred per conversation. Bound it per run and keep results on disk, misses
|
|
75
|
-
# included, so re-exports are instant.
|
|
75
|
+
# included, so re-exports are instant.
|
|
76
|
+
#
|
|
77
|
+
# Two separate pools: share-page covers are rare and expensive, inline appmsg
|
|
78
|
+
# thumbnails are common and cheap. Sharing one would let a run of thumbnails
|
|
79
|
+
# starve the covers that actually change how a page looks.
|
|
80
|
+
COVER_FETCH_LIMIT = 60
|
|
81
|
+
THUMB_FETCH_LIMIT = 300
|
|
82
|
+
|
|
83
|
+
# Seeded with the full budgets rather than 0: main() re-sets them, but anything
|
|
84
|
+
# that reaches the fetchers without going through main() used to find a budget
|
|
85
|
+
# of 0 and silently skip every remote fetch, returning None with no error.
|
|
76
86
|
COVER_STATE = {
|
|
77
|
-
'dir': '', 'budget':
|
|
78
|
-
'thumb_budget':
|
|
87
|
+
'dir': '', 'budget': COVER_FETCH_LIMIT, 'fetched': 0, 'cached': 0, 'skipped': 0,
|
|
88
|
+
'thumb_budget': THUMB_FETCH_LIMIT,
|
|
89
|
+
'thumb_fetched': 0, 'thumb_cached': 0, 'thumb_skipped': 0,
|
|
79
90
|
}
|
|
80
91
|
|
|
81
92
|
# Off by default: full-resolution originals are 5-24MB / 3000-5700px each, and
|
|
@@ -93,12 +104,6 @@ COVER_HEAD_BYTES = 64 * 1024
|
|
|
93
104
|
# cost time, which matters across thousands of media entries.
|
|
94
105
|
EMBED_SHRINK_THRESHOLD = 400 * 1024
|
|
95
106
|
|
|
96
|
-
# Two separate budgets: share-page covers are rare and expensive, inline
|
|
97
|
-
# appmsg thumbnails are common and cheap. Sharing one pool lets a run of
|
|
98
|
-
# thumbnails starve the covers that actually change how a page looks.
|
|
99
|
-
COVER_FETCH_LIMIT = 60
|
|
100
|
-
THUMB_FETCH_LIMIT = 300
|
|
101
|
-
|
|
102
107
|
# Set up by main(); custom stickers decrypted from WeChat's local cache.
|
|
103
108
|
STICKER_STATE = {'key': b'', 'dirs': [], 'cache_dir': ''}
|
|
104
109
|
|
|
@@ -119,6 +124,12 @@ VOICE_STATE = {'map': {}, 'cache': None}
|
|
|
119
124
|
# runs a throwaway formatting pass that only records the URLs (0.1s, no
|
|
120
125
|
# network), fetches them concurrently, then formats for real against a warm
|
|
121
126
|
# cache. Set to a list during that pass; None otherwise.
|
|
127
|
+
# Why the most recent remote fetch produced nothing. The skip counter alone
|
|
128
|
+
# said "not fetched" without saying why, so "no budget left" and "the URL is
|
|
129
|
+
# dead" were indistinguishable in the export - and both looked like a bare
|
|
130
|
+
# placeholder to the reader.
|
|
131
|
+
MEDIA_STATE = {'remote_reason': None}
|
|
132
|
+
|
|
122
133
|
PREFETCH = {'sink': None, 'active': False}
|
|
123
134
|
PREFETCH_WORKERS = 24
|
|
124
135
|
PREFETCH_MAX_URLS = 800
|
|
@@ -386,6 +397,29 @@ def scan_nt_cache(nt_cache_dir, talker, account_dir='', own_wxid=''):
|
|
|
386
397
|
if key.startswith('md5:'):
|
|
387
398
|
image_map.setdefault(key, image)
|
|
388
399
|
|
|
400
|
+
# `unique:<local_id>`: an id that resolves to exactly one distinct image in
|
|
401
|
+
# this conversation is safe to match on its own. The id restarts across
|
|
402
|
+
# shards, but uniqueness inside one conversation's scan rules that out -
|
|
403
|
+
# which is what separates this from the bare-local_id match that
|
|
404
|
+
# get_cached_image deliberately refuses. Deduped by content, so one picture
|
|
405
|
+
# cached at several timestamps still counts as unique.
|
|
406
|
+
local_candidates = {}
|
|
407
|
+
for key, image in image_map.items():
|
|
408
|
+
if isinstance(key, str) and key.startswith('pair:'):
|
|
409
|
+
parts = key.split(':', 2)
|
|
410
|
+
if len(parts) != 3:
|
|
411
|
+
continue
|
|
412
|
+
local_id = parts[1]
|
|
413
|
+
elif isinstance(key, int):
|
|
414
|
+
local_id = str(key)
|
|
415
|
+
else:
|
|
416
|
+
continue
|
|
417
|
+
digest = hashlib.sha256(image[0].encode('ascii')).hexdigest()
|
|
418
|
+
local_candidates.setdefault(local_id, {})[digest] = image
|
|
419
|
+
for local_id, candidates in local_candidates.items():
|
|
420
|
+
if len(candidates) == 1:
|
|
421
|
+
image_map[f'unique:{local_id}'] = next(iter(candidates.values()))
|
|
422
|
+
|
|
389
423
|
return image_map
|
|
390
424
|
|
|
391
425
|
|
|
@@ -724,35 +758,57 @@ def shrink_embedded(image, max_side=720, force=False):
|
|
|
724
758
|
return image
|
|
725
759
|
|
|
726
760
|
|
|
727
|
-
def
|
|
728
|
-
"""Best local media for a message
|
|
761
|
+
def resolve_media(image_map, local_id, create_time, content='', resource_md5s=None, server_id=0):
|
|
762
|
+
"""Best local media for a message, with the identity that matched.
|
|
729
763
|
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
764
|
+
Returns `(base64, mime, media_key)` or None. `media_key` is the image_map
|
|
765
|
+
key that produced the hit, which is what the coverage report records - so
|
|
766
|
+
a report can never claim a match the page itself did not get, because both
|
|
767
|
+
come from this one function.
|
|
734
768
|
|
|
735
769
|
Deliberately *not* falling back to a bare `local_id`: that id restarts in
|
|
736
770
|
every message shard, so matching on it alone attaches one conversation's
|
|
737
|
-
image to another conversation's message.
|
|
771
|
+
image to another conversation's message. `unique:<local_id>` is the sound
|
|
772
|
+
version of that idea - scan_nt_cache only emits it when the id maps to a
|
|
773
|
+
single distinct image, so uniqueness has already ruled the collision out.
|
|
738
774
|
"""
|
|
739
775
|
if not image_map:
|
|
740
776
|
return None
|
|
741
777
|
for media_md5 in resource_md5s or []:
|
|
742
|
-
|
|
743
|
-
|
|
778
|
+
key = f'md5:{media_md5}'
|
|
779
|
+
if image_map.get(key):
|
|
780
|
+
return shrink_embedded(image_map[key]) + (key,)
|
|
744
781
|
for media_md5 in extract_media_md5s(content):
|
|
745
|
-
|
|
746
|
-
|
|
782
|
+
key = f'md5:{media_md5}'
|
|
783
|
+
if image_map.get(key):
|
|
784
|
+
return shrink_embedded(image_map[key]) + (key,)
|
|
747
785
|
try:
|
|
748
786
|
pair_key = f'pair:{int(local_id)}:{int(create_time)}'
|
|
749
787
|
except (TypeError, ValueError):
|
|
750
788
|
return None
|
|
751
789
|
if image_map.get(pair_key):
|
|
752
|
-
return shrink_embedded(image_map[pair_key])
|
|
790
|
+
return shrink_embedded(image_map[pair_key]) + (pair_key,)
|
|
791
|
+
try:
|
|
792
|
+
unique_key = f'unique:{int(local_id)}'
|
|
793
|
+
except (TypeError, ValueError):
|
|
794
|
+
return None
|
|
795
|
+
if image_map.get(unique_key):
|
|
796
|
+
return shrink_embedded(image_map[unique_key]) + (unique_key,)
|
|
753
797
|
return None
|
|
754
798
|
|
|
755
799
|
|
|
800
|
+
def get_cached_image(image_map, local_id, create_time, content='', resource_md5s=None, server_id=0):
|
|
801
|
+
"""`resolve_media` without the matched key, for every pre-existing caller.
|
|
802
|
+
|
|
803
|
+
md5 is the strongest key, but it is not always derivable - a group row can
|
|
804
|
+
lack the resource mapping that carries it. The conversation cache also
|
|
805
|
+
indexes by `(local_id, create_time)`, which is identity-aware and was
|
|
806
|
+
already being stored; reading it recovers media the md5 path alone misses.
|
|
807
|
+
"""
|
|
808
|
+
hit = resolve_media(image_map, local_id, create_time, content, resource_md5s, server_id)
|
|
809
|
+
return (hit[0], hit[1]) if hit else None
|
|
810
|
+
|
|
811
|
+
|
|
756
812
|
def detect_mime(filepath):
|
|
757
813
|
"""Detect MIME type from file header."""
|
|
758
814
|
try:
|
|
@@ -882,12 +938,14 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
882
938
|
already spent their own budget for this URL.
|
|
883
939
|
"""
|
|
884
940
|
if not url or not url.startswith(('http://', 'https://')):
|
|
941
|
+
MEDIA_STATE['remote_reason'] = 'not-http'
|
|
885
942
|
return None
|
|
886
943
|
|
|
887
944
|
sink = PREFETCH['sink']
|
|
888
945
|
if sink is not None:
|
|
889
946
|
# Dry pass: record the request, spend no budget, touch no network.
|
|
890
947
|
sink.append(('img', url, aes_key))
|
|
948
|
+
MEDIA_STATE['remote_reason'] = 'deferred-to-prefetch'
|
|
891
949
|
return None
|
|
892
950
|
|
|
893
951
|
cache_dir = COVER_STATE.get('dir') or ''
|
|
@@ -899,11 +957,13 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
899
957
|
if cached != 'unknown':
|
|
900
958
|
if budgeted:
|
|
901
959
|
COVER_STATE['thumb_cached'] += 1
|
|
960
|
+
MEDIA_STATE['remote_reason'] = None
|
|
902
961
|
return cached
|
|
903
962
|
|
|
904
963
|
if budgeted and not PREFETCH['active']:
|
|
905
964
|
if COVER_STATE.get('thumb_budget', 0) <= 0:
|
|
906
965
|
COVER_STATE['thumb_skipped'] += 1
|
|
966
|
+
MEDIA_STATE['remote_reason'] = 'budget-exhausted'
|
|
907
967
|
return None
|
|
908
968
|
COVER_STATE['thumb_budget'] -= 1
|
|
909
969
|
COVER_STATE['thumb_fetched'] += 1
|
|
@@ -920,6 +980,7 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
920
980
|
# it so the next export does not pay for the same dead URL.
|
|
921
981
|
if len(data) > MAX_EMBED_SIZE:
|
|
922
982
|
_cache_media(cache_file, None)
|
|
983
|
+
MEDIA_STATE['remote_reason'] = 'too-large'
|
|
923
984
|
return None
|
|
924
985
|
candidates = [data]
|
|
925
986
|
decrypted = _decrypt_aes_cbc(data, aes_key)
|
|
@@ -933,10 +994,12 @@ def download_image_as_base64(url, aes_key='', timeout=10, budgeted=True):
|
|
|
933
994
|
data, mime = wechat_image.shrink(data, mime, max_side=480)
|
|
934
995
|
result = (base64.b64encode(data).decode(), mime)
|
|
935
996
|
_cache_media(cache_file, result)
|
|
997
|
+
MEDIA_STATE['remote_reason'] = None
|
|
936
998
|
return result
|
|
937
999
|
except Exception:
|
|
938
1000
|
continue
|
|
939
1001
|
_cache_media(cache_file, None)
|
|
1002
|
+
MEDIA_STATE['remote_reason'] = 'download-failed'
|
|
940
1003
|
return None
|
|
941
1004
|
|
|
942
1005
|
|
|
@@ -1560,8 +1623,14 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1560
1623
|
local_id = row[0] or 0
|
|
1561
1624
|
server_id = row[1] or 0
|
|
1562
1625
|
local_type = row[2] or 0
|
|
1626
|
+
# WeChat stores some forwards as high-bit variants of type 49. The mask
|
|
1627
|
+
# below turns those into plain 49, which is a known type - so the cached
|
|
1628
|
+
# image branch further down can only recognise them if the pre-mask value
|
|
1629
|
+
# is kept. Dropping this made that branch unreachable.
|
|
1630
|
+
raw_local_type = local_type
|
|
1563
1631
|
if local_type > 0xffffffff:
|
|
1564
1632
|
local_type &= 0xffffffff
|
|
1633
|
+
is_encoded_media_type = raw_local_type > 0xffffffff and local_type == 49
|
|
1565
1634
|
real_sender_id = row[4] or 0
|
|
1566
1635
|
create_time = row[5] or 0
|
|
1567
1636
|
source = row[7]
|
|
@@ -1611,6 +1680,11 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1611
1680
|
if content and content != source_text:
|
|
1612
1681
|
metadata_parts.append(content)
|
|
1613
1682
|
metadata_content = '\n'.join(metadata_parts)
|
|
1683
|
+
# A type-49 row carrying a title or url is a link/article card, not a bare
|
|
1684
|
+
# image. Without this guard the encoded-media fallback below would hide the
|
|
1685
|
+
# card (and its title/link) behind a cached thumbnail.
|
|
1686
|
+
has_app_link_metadata = local_type == 49 and bool(
|
|
1687
|
+
extract_xml_text(content, 'title') or extract_xml_text(content, 'url'))
|
|
1614
1688
|
is_emoji_xml = bool(re.search(r'<(?:msg\s*>)?\s*<emoji\b|<emoji\b', metadata_content, re.IGNORECASE))
|
|
1615
1689
|
is_contact_card = bool(
|
|
1616
1690
|
re.search(r'<msg\b[^>]*(?:nickname|username)=', metadata_content, re.IGNORECASE)
|
|
@@ -1694,7 +1768,9 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1694
1768
|
if img_data:
|
|
1695
1769
|
image_b64 = img_data
|
|
1696
1770
|
display += f'<br><img src="data:{mime};base64,{img_data}" loading="lazy" />'
|
|
1697
|
-
elif local_type not in MSG_TYPES
|
|
1771
|
+
elif (local_type not in MSG_TYPES or is_encoded_media_type) \
|
|
1772
|
+
and not has_app_link_metadata \
|
|
1773
|
+
and get_cached_image(image_map, local_id, create_time, content, resource_md5s):
|
|
1698
1774
|
# Some image messages use encoded types (e.g. 21474836529 = images in appmsg)
|
|
1699
1775
|
# Check image_map for any message type
|
|
1700
1776
|
img_data, mime = get_cached_image(image_map, local_id, create_time, content, resource_md5s)
|
|
@@ -1787,11 +1863,14 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1787
1863
|
builtin_title = _face_index.find_face(title)
|
|
1788
1864
|
title_html = (render_builtin_emoji(title, builtin_title)
|
|
1789
1865
|
if builtin_title else escape_html(title))
|
|
1790
|
-
#
|
|
1866
|
+
# Try the conversation cache before the network: a cached
|
|
1867
|
+
# thumbnail is already on disk and costs nothing, and without
|
|
1868
|
+
# this the article branch downloaded covers it already had.
|
|
1791
1869
|
thumb_url = extract_appmsg_image(content)
|
|
1792
|
-
|
|
1870
|
+
img_data = get_cached_image(image_map, local_id, create_time, metadata_content, resource_md5s)
|
|
1871
|
+
if not img_data and thumb_url:
|
|
1793
1872
|
img_data = download_image_as_base64(thumb_url, extract_media_aes_key(content))
|
|
1794
|
-
|
|
1873
|
+
if not img_data:
|
|
1795
1874
|
page_url = extract_xml_text(content, 'url')
|
|
1796
1875
|
img_data = download_bilibili_cover(page_url)
|
|
1797
1876
|
if not img_data:
|
|
@@ -1839,9 +1918,117 @@ def format_message(row, talker, wx_dir, image_map=None, sender_map=None, display
|
|
|
1839
1918
|
'local_type': local_type,
|
|
1840
1919
|
'display': display,
|
|
1841
1920
|
'image_b64': image_b64,
|
|
1921
|
+
# Built here rather than accumulated in module state: format_message
|
|
1922
|
+
# runs twice (the prefetch dry sweep and the real pass) and the dry
|
|
1923
|
+
# pass is discarded, so a counter would double-count.
|
|
1924
|
+
'media': build_media_record(
|
|
1925
|
+
local_type, image_b64, image_map, local_id, create_time, content,
|
|
1926
|
+
resource_md5s, metadata_content, bool(server_id)),
|
|
1842
1927
|
}
|
|
1843
1928
|
|
|
1844
1929
|
|
|
1930
|
+
MEDIA_KIND_BY_TYPE = {
|
|
1931
|
+
3: 'image', 34: 'voice', 43: 'video-thumb', 47: 'emoji', 49: 'link-cover',
|
|
1932
|
+
}
|
|
1933
|
+
MEDIA_KINDS = {'image', 'emoji', 'voice', 'video-thumb', 'link-cover'}
|
|
1934
|
+
|
|
1935
|
+
|
|
1936
|
+
def build_media_report(items, scope, talker, per_page):
|
|
1937
|
+
"""Coverage summary plus one entry per media item.
|
|
1938
|
+
|
|
1939
|
+
Written beside the HTML rather than into it: the page is for reading, this
|
|
1940
|
+
is for answering "what is missing and why" without opening the export.
|
|
1941
|
+
Paths, keys and message bodies are not included (D-001/D-002/D-025).
|
|
1942
|
+
"""
|
|
1943
|
+
counts = {}
|
|
1944
|
+
for item in items:
|
|
1945
|
+
counts[item['status']] = counts.get(item['status'], 0) + 1
|
|
1946
|
+
counts['total'] = len(items)
|
|
1947
|
+
return {
|
|
1948
|
+
'schema': 'weflow-media-report/v1',
|
|
1949
|
+
'source': 'weflow-cli',
|
|
1950
|
+
'generatedAt': datetime.datetime.now().astimezone().isoformat(timespec='seconds'),
|
|
1951
|
+
'scope': scope,
|
|
1952
|
+
'counts': counts,
|
|
1953
|
+
'budget': {
|
|
1954
|
+
'coversFetched': COVER_STATE.get('fetched', 0),
|
|
1955
|
+
'coversCached': COVER_STATE.get('cached', 0),
|
|
1956
|
+
'coversSkipped': COVER_STATE.get('skipped', 0),
|
|
1957
|
+
'thumbsFetched': COVER_STATE.get('thumb_fetched', 0),
|
|
1958
|
+
'thumbsCached': COVER_STATE.get('thumb_cached', 0),
|
|
1959
|
+
'thumbsSkipped': COVER_STATE.get('thumb_skipped', 0),
|
|
1960
|
+
},
|
|
1961
|
+
'items': [
|
|
1962
|
+
{
|
|
1963
|
+
'mediaKey': item['mediaKey'],
|
|
1964
|
+
'kind': item['kind'],
|
|
1965
|
+
'status': item['status'],
|
|
1966
|
+
'mime': item['mime'],
|
|
1967
|
+
'bytes': item['bytes'],
|
|
1968
|
+
'sha256': item['sha256'],
|
|
1969
|
+
'reason': item['reason'],
|
|
1970
|
+
'locator': {
|
|
1971
|
+
'part': (item['index'] // per_page + 1) if per_page else 1,
|
|
1972
|
+
'createTime': None,
|
|
1973
|
+
},
|
|
1974
|
+
}
|
|
1975
|
+
for item in items
|
|
1976
|
+
],
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
|
|
1980
|
+
def build_media_record(local_type, image_b64, image_map, local_id, create_time,
|
|
1981
|
+
content, resource_md5s, metadata_content, has_server_id):
|
|
1982
|
+
"""What happened to this message's media, or None for a non-media message.
|
|
1983
|
+
|
|
1984
|
+
Derived from the same `resolve_media` the page used, so the report cannot
|
|
1985
|
+
claim a match the page did not get (or vice versa). A media-free message
|
|
1986
|
+
returns None: this is a media manifest, not a message manifest.
|
|
1987
|
+
"""
|
|
1988
|
+
if local_type not in MEDIA_KIND_BY_TYPE:
|
|
1989
|
+
return None
|
|
1990
|
+
kind = MEDIA_KIND_BY_TYPE[local_type]
|
|
1991
|
+
record = {
|
|
1992
|
+
'mediaKey': None, 'kind': kind, 'status': None,
|
|
1993
|
+
'mime': None, 'bytes': None, 'sha256': None, 'reason': None,
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
if image_b64:
|
|
1997
|
+
hit = resolve_media(image_map, local_id, create_time, metadata_content,
|
|
1998
|
+
resource_md5s)
|
|
1999
|
+
if hit:
|
|
2000
|
+
# Matched a local key: report which one, so "why this image" is
|
|
2001
|
+
# answerable without re-running the export.
|
|
2002
|
+
record['mediaKey'] = hit[2]
|
|
2003
|
+
record['status'] = 'embedded'
|
|
2004
|
+
else:
|
|
2005
|
+
# Bytes came from somewhere other than the conversation index.
|
|
2006
|
+
if MEDIA_STATE['remote_reason'] is None:
|
|
2007
|
+
record['status'] = 'remote-fetched'
|
|
2008
|
+
record['mediaKey'] = 'remote'
|
|
2009
|
+
else:
|
|
2010
|
+
record['status'] = 'embedded'
|
|
2011
|
+
record['mediaKey'] = 'unindexed-local'
|
|
2012
|
+
raw = base64.b64decode(image_b64)
|
|
2013
|
+
record['bytes'] = len(raw)
|
|
2014
|
+
record['sha256'] = hashlib.sha256(raw).hexdigest()
|
|
2015
|
+
return record
|
|
2016
|
+
|
|
2017
|
+
record['status'] = 'missing'
|
|
2018
|
+
if not has_server_id and not extract_media_md5s(metadata_content) and not resource_md5s:
|
|
2019
|
+
# D-014: with no reliable identity there is nothing safe to match on,
|
|
2020
|
+
# and guessing would attach someone else's media.
|
|
2021
|
+
record['reason'] = 'no-reliable-identity'
|
|
2022
|
+
elif kind == 'voice':
|
|
2023
|
+
record['status'] = 'unsupported'
|
|
2024
|
+
record['reason'] = 'voice-not-in-media-index'
|
|
2025
|
+
elif MEDIA_STATE['remote_reason']:
|
|
2026
|
+
record['reason'] = MEDIA_STATE['remote_reason']
|
|
2027
|
+
else:
|
|
2028
|
+
record['reason'] = 'not-in-local-cache'
|
|
2029
|
+
return record
|
|
2030
|
+
|
|
2031
|
+
|
|
1845
2032
|
def decode_message_content(value):
|
|
1846
2033
|
"""Decode NT message content, which may be Zstandard compressed."""
|
|
1847
2034
|
if not value:
|
|
@@ -2285,6 +2472,7 @@ def main():
|
|
|
2285
2472
|
|
|
2286
2473
|
print(f"Formatting {len(messages)} messages...")
|
|
2287
2474
|
formatted = []
|
|
2475
|
+
media_items = []
|
|
2288
2476
|
img_hit_count = 0
|
|
2289
2477
|
article_img_count = 0
|
|
2290
2478
|
progress_step = max(1, len(messages) // 20)
|
|
@@ -2296,6 +2484,12 @@ def main():
|
|
|
2296
2484
|
img_hit_count += 1
|
|
2297
2485
|
if result.get('local_type') == 49:
|
|
2298
2486
|
article_img_count += 1
|
|
2487
|
+
record = result.get('media')
|
|
2488
|
+
if record:
|
|
2489
|
+
# Index kept so the report can point back at the page part.
|
|
2490
|
+
record = dict(record)
|
|
2491
|
+
record['index'] = i
|
|
2492
|
+
media_items.append(record)
|
|
2299
2493
|
formatted.append(result)
|
|
2300
2494
|
print(f" Messages with embedded images: {img_hit_count} (including {article_img_count} article thumbnails)")
|
|
2301
2495
|
|
|
@@ -2315,6 +2509,20 @@ def main():
|
|
|
2315
2509
|
# Use display name for filename if provided, otherwise fallback to wxid
|
|
2316
2510
|
file_prefix = sanitize_filename(display_name) if display_name else args.talker.replace('@', '_').replace('/', '_')
|
|
2317
2511
|
|
|
2512
|
+
# Media coverage report. Written beside the HTML parts, under the same
|
|
2513
|
+
# prefix, so it is always found next to what it describes.
|
|
2514
|
+
media_report = None
|
|
2515
|
+
media_report_path = None
|
|
2516
|
+
if media_items:
|
|
2517
|
+
media_report = build_media_report(media_items, display_name, args.talker, per_part)
|
|
2518
|
+
media_report_path = os.path.join(args.out, f'{file_prefix}_media.json')
|
|
2519
|
+
with open(media_report_path, 'w', encoding='utf-8') as handle:
|
|
2520
|
+
json.dump(media_report, handle, ensure_ascii=False, indent=2)
|
|
2521
|
+
summary = media_report['counts']
|
|
2522
|
+
missing = summary.get('missing', 0) + summary.get('unsupported', 0)
|
|
2523
|
+
print(f"Media report: {len(media_items)} item(s), {missing} unavailable "
|
|
2524
|
+
f"-> {os.path.basename(media_report_path)}")
|
|
2525
|
+
|
|
2318
2526
|
html_files = []
|
|
2319
2527
|
for i in range(parts):
|
|
2320
2528
|
start = i * per_part
|
|
@@ -2343,12 +2551,20 @@ def main():
|
|
|
2343
2551
|
print(f"\nDone! {len(html_files)} HTML files written to {args.out}")
|
|
2344
2552
|
print(f"Total: {total} messages")
|
|
2345
2553
|
|
|
2346
|
-
# Print JSON summary for CLI integration
|
|
2554
|
+
# Print JSON summary for CLI integration.
|
|
2555
|
+
# The four original keys are untouched; `media` is purely additive and the
|
|
2556
|
+
# one existing consumer (src/services/exportService.ts) ignores it.
|
|
2347
2557
|
print(json.dumps({
|
|
2348
2558
|
"success": True,
|
|
2349
2559
|
"total": total,
|
|
2350
2560
|
"parts": len(html_files),
|
|
2351
2561
|
"files": html_files,
|
|
2562
|
+
"media": ({
|
|
2563
|
+
"schema": "weflow-media-report/v1",
|
|
2564
|
+
"records": len(media_items),
|
|
2565
|
+
"status": media_report['counts'],
|
|
2566
|
+
"report": os.path.basename(media_report_path) if media_report_path else None,
|
|
2567
|
+
} if media_report else None),
|
|
2352
2568
|
}))
|
|
2353
2569
|
|
|
2354
2570
|
|