weflow-cli 1.8.2 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/ARCHITECTURE.md +62 -26
  2. package/CHANGELOG.md +1290 -0
  3. package/OPERATIONS.md +144 -11
  4. package/README.en.md +20 -4
  5. package/README.md +30 -5
  6. package/bin/weflow-cli-electron.cjs +2 -0
  7. package/bin/weflow-cli.ts +996 -118
  8. package/dist/bin/weflow-cli.js +1038 -146
  9. package/dist/bin/weflow-cli.js.map +1 -1
  10. package/dist/mcp-server/http.d.ts +14 -0
  11. package/dist/mcp-server/http.d.ts.map +1 -0
  12. package/dist/mcp-server/http.js +84 -0
  13. package/dist/mcp-server/http.js.map +1 -0
  14. package/dist/mcp-server/httpConfig.d.ts +50 -0
  15. package/dist/mcp-server/httpConfig.d.ts.map +1 -0
  16. package/dist/mcp-server/httpConfig.js +104 -0
  17. package/dist/mcp-server/httpConfig.js.map +1 -0
  18. package/dist/mcp-server/index.d.ts +13 -1
  19. package/dist/mcp-server/index.d.ts.map +1 -1
  20. package/dist/mcp-server/index.js +58 -374
  21. package/dist/mcp-server/index.js.map +1 -1
  22. package/dist/src/core/dbPathService.d.ts +12 -0
  23. package/dist/src/core/dbPathService.d.ts.map +1 -1
  24. package/dist/src/core/dbPathService.js +20 -3
  25. package/dist/src/core/dbPathService.js.map +1 -1
  26. package/dist/src/panel/server.d.ts.map +1 -1
  27. package/dist/src/panel/server.js +61 -1
  28. package/dist/src/panel/server.js.map +1 -1
  29. package/dist/src/services/assistantEval.d.ts +69 -0
  30. package/dist/src/services/assistantEval.d.ts.map +1 -1
  31. package/dist/src/services/assistantEval.js +270 -8
  32. package/dist/src/services/assistantEval.js.map +1 -1
  33. package/dist/src/services/assistantScenes.d.ts +81 -0
  34. package/dist/src/services/assistantScenes.d.ts.map +1 -0
  35. package/dist/src/services/assistantScenes.js +322 -0
  36. package/dist/src/services/assistantScenes.js.map +1 -0
  37. package/dist/src/services/assistantService.d.ts +33 -4
  38. package/dist/src/services/assistantService.d.ts.map +1 -1
  39. package/dist/src/services/assistantService.js +102 -6
  40. package/dist/src/services/assistantService.js.map +1 -1
  41. package/dist/src/services/assistantSkills.d.ts +94 -0
  42. package/dist/src/services/assistantSkills.d.ts.map +1 -0
  43. package/dist/src/services/assistantSkills.js +330 -0
  44. package/dist/src/services/assistantSkills.js.map +1 -0
  45. package/dist/src/services/assistantTools.d.ts +34 -1
  46. package/dist/src/services/assistantTools.d.ts.map +1 -1
  47. package/dist/src/services/assistantTools.js +931 -88
  48. package/dist/src/services/assistantTools.js.map +1 -1
  49. package/dist/src/services/assistantTrace.d.ts +4 -0
  50. package/dist/src/services/assistantTrace.d.ts.map +1 -1
  51. package/dist/src/services/assistantTrace.js +9 -0
  52. package/dist/src/services/assistantTrace.js.map +1 -1
  53. package/dist/src/services/configService.d.ts +9 -0
  54. package/dist/src/services/configService.d.ts.map +1 -1
  55. package/dist/src/services/configService.js +5 -2
  56. package/dist/src/services/configService.js.map +1 -1
  57. package/dist/src/services/messageQuery.d.ts +18 -0
  58. package/dist/src/services/messageQuery.d.ts.map +1 -1
  59. package/dist/src/services/messageQuery.js +29 -0
  60. package/dist/src/services/messageQuery.js.map +1 -1
  61. package/dist/src/services/outboundSend.d.ts +53 -0
  62. package/dist/src/services/outboundSend.d.ts.map +1 -0
  63. package/dist/src/services/outboundSend.js +90 -0
  64. package/dist/src/services/outboundSend.js.map +1 -0
  65. package/dist/src/services/pythonBridge.d.ts.map +1 -1
  66. package/dist/src/services/pythonBridge.js +1 -0
  67. package/dist/src/services/pythonBridge.js.map +1 -1
  68. package/dist/src/services/wechatClient.d.ts +125 -0
  69. package/dist/src/services/wechatClient.d.ts.map +1 -0
  70. package/dist/src/services/wechatClient.js +344 -0
  71. package/dist/src/services/wechatClient.js.map +1 -0
  72. package/dist/src/services/wechatMedia.d.ts +44 -0
  73. package/dist/src/services/wechatMedia.d.ts.map +1 -0
  74. package/dist/src/services/wechatMedia.js +155 -0
  75. package/dist/src/services/wechatMedia.js.map +1 -0
  76. package/dist/src/services/wechatMessageService.d.ts +11 -0
  77. package/dist/src/services/wechatMessageService.d.ts.map +1 -1
  78. package/dist/src/services/wechatMessageService.js +55 -11
  79. package/dist/src/services/wechatMessageService.js.map +1 -1
  80. package/dist/src/types.d.ts +4 -0
  81. package/dist/src/types.d.ts.map +1 -1
  82. package/dist/src/utils/python.js +2 -2
  83. package/dist/src/utils/python.js.map +1 -1
  84. package/docs/BRANCHES.md +37 -0
  85. package/docs/CONTACT_DB_SCHEMA.md +1085 -0
  86. package/docs/DECISIONS.md +1624 -0
  87. package/docs/EXTENDING.md +98 -10
  88. package/docs/FUNCTIONAL-IMPROVEMENT-SPEC.md +6 -0
  89. package/docs/HEALTH-CHECK.md +12 -7
  90. package/docs/JEV/345/272/224/347/224/250/346/226/275/345/267/245/346/226/207/346/241/243.md +2 -2
  91. package/docs/MCP.md +58 -4
  92. package/docs/PROJECT_STATE.md +180 -11
  93. package/docs/RELEASING.md +115 -1
  94. package/docs/images/weflow-architecture-gpt-image-2.png +0 -0
  95. package/logo.png +0 -0
  96. package/mcp-server/http.ts +108 -0
  97. package/mcp-server/httpConfig.ts +126 -0
  98. package/mcp-server/index.ts +359 -681
  99. package/package.json +5 -3
  100. package/resources/js/graph3d/NOTICE.txt +35 -0
  101. package/resources/js/graph3d/d3-binarytree.min.js +2 -0
  102. package/resources/js/graph3d/d3-dispatch.min.js +2 -0
  103. package/resources/js/graph3d/d3-force-3d.min.js +2 -0
  104. package/resources/js/graph3d/d3-octree.min.js +2 -0
  105. package/resources/js/graph3d/d3-quadtree.min.js +2 -0
  106. package/resources/js/graph3d/d3-timer.min.js +2 -0
  107. package/resources/js/graph3d/layout.mjs +63 -0
  108. package/resources/js/graph3d/three.min.js +6 -0
  109. package/resources/js/graph3d/viewer2d.js +448 -0
  110. package/resources/panel/ball-position.cjs +48 -2
  111. package/resources/panel/index.html +18 -4
  112. package/resources/panel/main.cjs +137 -5
  113. package/resources/panel/mascot-base.png +0 -0
  114. package/resources/panel/mascot-blink-1.png +0 -0
  115. package/resources/panel/mascot-blink-2.png +0 -0
  116. package/resources/panel/mascot-iris.png +0 -0
  117. package/resources/panel/mascot-lift-down-1.png +0 -0
  118. package/resources/panel/mascot-lift-down-2.png +0 -0
  119. package/resources/panel/mascot-lift-down-3.png +0 -0
  120. package/resources/panel/mascot-lift-down-4.png +0 -0
  121. package/resources/panel/mascot-lift-down-5.png +0 -0
  122. package/resources/panel/mascot-lift-down-6.png +0 -0
  123. package/resources/panel/mascot-lift-up-1.png +0 -0
  124. package/resources/panel/mascot-lift-up-2.png +0 -0
  125. package/resources/panel/mascot-lift-up-3.png +0 -0
  126. package/resources/panel/mascot-lift-up-4.png +0 -0
  127. package/resources/panel/mascot-lift-up-5.png +0 -0
  128. package/resources/panel/mascot-lift-up-6.png +0 -0
  129. package/resources/panel/mascot-peek.png +0 -0
  130. package/resources/panel/mascot-tickle-1.png +0 -0
  131. package/resources/panel/mascot-tickle-2.png +0 -0
  132. package/resources/panel/mascot-tickle-3.png +0 -0
  133. package/resources/panel/mascot-tickle-4.png +0 -0
  134. package/resources/panel/panel.css +205 -59
  135. package/resources/panel/preload.cjs +26 -0
  136. package/resources/panel/renderer.js +646 -17
  137. package/scripts/_utils.py +1203 -931
  138. package/scripts/backfill_articles.py +202 -21
  139. package/scripts/biz_daily.py +26 -34
  140. package/scripts/bonds.py +331 -0
  141. package/scripts/chat_notes.py +561 -470
  142. package/scripts/chat_stats.py +8 -7
  143. package/scripts/check_readme_commands.py +426 -0
  144. package/scripts/compile_wiki.py +187 -59
  145. package/scripts/contact_schema.py +339 -0
  146. package/scripts/data/wechat_emoji.json +225 -0
  147. package/scripts/export_chat_html.py +3 -1
  148. package/scripts/fav_notes.py +4 -1
  149. package/scripts/graph_2d.py +151 -0
  150. package/scripts/graph_3d.py +646 -0
  151. package/scripts/health_check.py +163 -26
  152. package/scripts/mcp_bridge.py +6 -7
  153. package/scripts/nt_decrypt.py +3 -3
  154. package/scripts/panel_frames.py +683 -0
  155. package/scripts/pipeline.py +21 -10
  156. package/scripts/quality_eval.py +5 -0
  157. package/scripts/rag_chat.py +33 -16
  158. package/scripts/reply_debt.py +26 -6
  159. package/scripts/semantic_search.py +130 -22
  160. package/scripts/vault_rag.py +42 -16
  161. package/scripts/vault_search.py +44 -8
  162. package/scripts/wechat_emoticon.py +6 -0
  163. package/scripts/wechat_voice.py +9 -0
  164. package/scripts/wiki_lint.py +124 -24
  165. package/src/core/dbPathService.ts +22 -4
  166. package/src/panel/server.ts +58 -1
  167. package/src/services/assistantEval.ts +323 -12
  168. package/src/services/assistantScenes.ts +342 -0
  169. package/src/services/assistantService.ts +107 -6
  170. package/src/services/assistantSkills.ts +364 -0
  171. package/src/services/assistantTools.ts +2452 -1640
  172. package/src/services/assistantTrace.ts +9 -0
  173. package/src/services/configService.ts +415 -403
  174. package/src/services/messageQuery.ts +29 -0
  175. package/src/services/outboundSend.ts +126 -0
  176. package/src/services/pythonBridge.ts +1 -0
  177. package/src/services/wechatClient.ts +380 -0
  178. package/src/services/wechatMedia.ts +166 -0
  179. package/src/services/wechatMessageService.ts +62 -11
  180. package/src/types.ts +4 -0
  181. package/src/utils/python.ts +2 -2
  182. package/dist/src/utils/wechatEmoji.d.ts +0 -5
  183. package/dist/src/utils/wechatEmoji.d.ts.map +0 -1
  184. package/dist/src/utils/wechatEmoji.js +0 -35
  185. package/dist/src/utils/wechatEmoji.js.map +0 -1
@@ -4,6 +4,7 @@
4
4
  用法:
5
5
  python scripts/backfill_articles.py --since 2026-03-01 --until 2026-08-31 --dry-run --json
6
6
  python scripts/backfill_articles.py --since 2026-03-01 --until 2026-08-31 --workers 12 --yes
7
+ python scripts/backfill_articles.py --since 2025-09-05 --until 2026-03-02 --topic AI --yes
7
8
 
8
9
  ## 为什么不是直接跑 `pipeline run --date`
9
10
 
@@ -26,6 +27,17 @@
26
27
  **图片型文章抓不到正文。** 实测 3/4、7/4 两天:27/40、44/60 有正文,其余是图文消息
27
28
  (内容全在图片里),清洗掉图片与链接后不足 100 字,与 `biz_daily` 一样被跳过。这不是
28
29
  抓取失败,是那类文章本来就没有文字。所以"2 万篇"这个量级是有正文的部分,不是总量。
30
+
31
+ ## `--topic`:只回填一部分主题(先筛后抓)
32
+
33
+ `--topic AI` 用的就是本脚本自己判主题的那套 `_guess_topic`(只看标题与号名、**不看正文**),
34
+ 所以它筛出来的 AI 集合,与全量跑完之后落在 `AI/` 目录里的那批**逐个相同** —— 不是另立一套
35
+ 口径,只是省掉了其余主题的抓取(实测 2025-09-05~2026-03-02 那段:全量 10,880 篇里 AI 占
36
+ 1,759 篇,抓取量约 1/6)。
37
+
38
+ **代价必须写出来:那一天就不再是完整的一天。** 所以 `.articles.json` 里会多一个 `topicFilter`
39
+ (README 里也写明),`day_done` 据此在之后跑**全量**时**不会**跳过这一天。少了这一步,那天
40
+ 其余五个主题的文章会被永久漏掉、而且不报错 —— 文件在、json 在、"已回填"看起来处处成立。
29
41
  """
30
42
  import argparse
31
43
  import hashlib
@@ -121,6 +133,32 @@ def apply_topics(articles):
121
133
  return articles
122
134
 
123
135
 
136
+ def parse_topics(values):
137
+ """把 `--topic` 的取值展开成一个列表:**可重复,也可逗号分隔**。
138
+
139
+ 两种都收的理由是仓库里另一条命令(`article_notes.py`)的 `--topic` 就是 `action='append'`
140
+ 且写明"可重复或逗号分隔"。只认一种的话,照另一边习惯写就会**静默只做一个主题**。
141
+
142
+ `values` 传字符串也当单个处理:`for chunk in 'AI'` 会**逐字符**迭代,而每个字符都不在分类法里
143
+ —— 于是"整批被拒"看起来跟"参数拼错"一模一样,很难查。这里把它挡住。
144
+ """
145
+ if isinstance(values, str) or values is None:
146
+ values = [values or '']
147
+ return [t.strip() for chunk in values for t in str(chunk).split(',') if t.strip()]
148
+
149
+
150
+ def select_topics(articles, topics):
151
+ """只留下选中主题的文章;`topics` 为空/None 表示全要(原样返回)。
152
+
153
+ 必须在 `apply_topics` **之后**调:主题是那个函数的输出。顺序对"选中哪些"没有影响
154
+ (`_guess_topic` 逐篇独立、只看标题与号名),所以筛出来的集合与全量跑完落在对应主题
155
+ 目录里的那批逐个相同。
156
+ """
157
+ if not topics:
158
+ return articles
159
+ return [a for a in articles if a.get('topic') in topics]
160
+
161
+
124
162
  def fetch_bodies(articles, workers: int, log=None):
125
163
  """并发把这批文章的正文抓回来,写进 `fetched_md`。
126
164
 
@@ -151,13 +189,66 @@ def fetch_bodies(articles, workers: int, log=None):
151
189
  return {'attempted': len(todo), 'ok': ok}
152
190
 
153
191
 
154
- def write_day(articles, day, out_root):
192
+ def _read_index(day_dir) -> dict:
193
+ """读这一天已有的 `.articles.json`(没有就空表)。
194
+
195
+ **读不了的时候不是安静地返回空表**:那等于把已有条目从索引里抹掉,而这件事不会报错
196
+ —— 正是这一整段要防的事。所以把原文件改名留成 `.json.bad` 再继续,至少还能人工救。
197
+ """
198
+ path = Path(day_dir) / '.articles.json'
199
+ if not path.exists():
200
+ return {}
201
+ try:
202
+ payload = json.loads(path.read_text(encoding='utf-8'))
203
+ return payload if isinstance(payload, dict) else {}
204
+ except Exception as exc:
205
+ try:
206
+ path.rename(path.with_name('.articles.json.bad'))
207
+ except OSError:
208
+ pass
209
+ print(' 警告:%s 读不了(%s),已留成 .articles.json.bad,旧索引未参与合并'
210
+ % (path.name, exc))
211
+ return {}
212
+
213
+
214
+ def _merge_articles(kept, fresh):
215
+ """合并两组 `.articles.json` 条目,按「来源-标题」去重(那是它在盘上的文件名)。
216
+
217
+ 顺序按 `time` 再 `title` 排,是为了让同一天的索引**稳定**:分两次回填的那天,
218
+ 第二次不该把先前的条目挤到别处去,否则每次跑都产生一份看着像"改了"的索引。
219
+ 没有 `time` 的老条目(理论上不会有)排在前面,不参与排序。
220
+ """
221
+ merged = {}
222
+ for entry in list(kept) + list(fresh):
223
+ if not isinstance(entry, dict):
224
+ continue
225
+ merged[(str(entry.get('source', '')), str(entry.get('title', '')))] = entry
226
+ return sorted(merged.values(), key=lambda e: (str(e.get('time', '')), str(e.get('title', ''))))
227
+
228
+
229
+ def write_day(articles, day, out_root, topic_filter=None, truncated=False):
155
230
  """落盘:`<日期>/<主题>/<来源>-<标题>.md` + `.articles.json` + README。
156
231
 
157
232
  格式与 `biz_daily` 一致——`create_reading_notes` 与 `compile_wiki` 都按那个形状读。
158
233
  正文那段写 `## AI 摘要`(内容用文章自带的 digest):**本脚本不调模型**,所以这里
159
234
  放的只能是原文已有的东西,不能假装是生成的摘要。
235
+
236
+ 传了 `topic_filter`(`--topic`)时,这里会**再筛一次**并在元数据里留下 `topicFilter`:
237
+
238
+ - 筛在**这里**做,是为了让标记不可能撒谎。如果筛只发生在调用方(`main`),那么有人
239
+ 漏调那一步时,`write_day` 会把六个主题全写出去、同时盖上"只回填了 AI"的章 ——
240
+ 一个自称只回填了一半、实际写满了的目录,比不写标记更坏。
241
+ - README 里也写明这一天是哪几个主题。**这两处不是装饰**:一天只回填了一半却看起来像
242
+ 完整的一天,是这份产出唯一会"静默变少"的方式(见 `day_done`)。
243
+ - 传了 `truncated`(`--limit-per-day` 截过)时同理,多一个 `truncated` 标记。
244
+ - **索引是与这一天已有的内容合并的,不是覆盖**:先 `--topic AI` 再来 `--topic 学术`,
245
+ 先前那个主题的条目必须留着。覆盖会让**索引比盘上少几条**而 md 还在 —— 不报错,
246
+ 而"索引与盘不一致"正是这一支反复吃的那个形状(见 `_merge_articles`、`_read_index`)。
247
+ `topicFilter` 取并集;只要这次是全量、或这天本来就是完整的,标记就**消失**。
248
+
249
+ `truncated` 只影响元数据(它记录的是"调用方截过",不宜在这里重算——截在调用方发生)。
160
250
  """
251
+ articles = select_topics(articles, topic_filter)
161
252
  groups, fallbacks = _group_by_topic(articles)
162
253
  out_dir = Path(out_root) / day
163
254
  written, skipped = [], []
@@ -201,13 +292,50 @@ def write_day(articles, day, out_root):
201
292
  written.append(article)
202
293
  out_dir.mkdir(parents=True, exist_ok=True)
203
294
  serializable = [_serializable_article(a, day) for a in written]
295
+ # **索引描述的是这一天盘上全部的文章,不是这一轮写的那批。** 分两次、用不同主题回填同一天
296
+ # 时(`--topic AI` 之后再来 `--topic 学术`),只写本轮的话,先前那个主题的条目会从索引里
297
+ # 消失而 md 还留在盘上——索引少一条、盘上多一篇,不报错。所以先把不属于本轮的旧条目接过来。
298
+ prior = _read_index(out_dir)
299
+ kept = [e for e in prior.get('articles', [])
300
+ if e.get('topic') not in set(topic_filter or ())]
301
+ stored_filter = prior.get('topicFilter')
302
+ # 「**没有**旧索引」与「旧索引是**完整**的一天」是两件事,第一版把它们混成了一个 `None`:
303
+ # 于是第一次做过滤回填时标记根本没写,那天看起来像完整的一天 —— 正是要防的那件事。
304
+ # 区分它们的是"旧索引里有没有东西",不是"有没有那个键"。
305
+ if topic_filter is None:
306
+ scope_filter = None # 全量跑过 → 这天完整
307
+ elif not prior.get('articles'):
308
+ scope_filter = sorted(topic_filter) # 第一次写这天,且只写了一部分
309
+ elif stored_filter is None:
310
+ scope_filter = None # 本来就完整,这次只是重做其中一部分
311
+ else:
312
+ scope_filter = sorted(set(stored_filter) | set(topic_filter))
313
+ if scope_filter and set(scope_filter) >= set(TOPICS):
314
+ # 六个主题都做过一遍了 —— 这一天"能做的都做了",与全量跑过没有区别。
315
+ # 不撤标记的话它**永远**被当成没做完:以后随便跑一次全量都会把整个窗口重抓一遍。
316
+ scope_filter = None
317
+ payload = {'date': day, 'generated_at': datetime.now(TZ).strftime('%Y-%m-%d %H:%M:%S'),
318
+ 'backfilled': True, 'articles': _merge_articles(kept, serializable)}
319
+ # 只在**确实是一部分**的时候才加这两个键:全量回填写出来的文件与以前逐字节相同,
320
+ # 而 `day_done` 把"没有 topicFilter"读作"完整的一天"。缺了它就等于把半天的产出
321
+ # 冒充成整天——这正是这条路径唯一能静默丢东西的地方。
322
+ if scope_filter:
323
+ payload['topicFilter'] = scope_filter
324
+ if truncated or prior.get('truncated'):
325
+ # `truncated` 一旦记上就不撤销:这一次没截,不代表上一次截掉的那些补回来了。
326
+ payload['truncated'] = True
204
327
  (out_dir / '.articles.json').write_text(
205
- json.dumps({'date': day, 'generated_at': datetime.now(TZ).strftime('%Y-%m-%d %H:%M:%S'),
206
- 'backfilled': True, 'articles': serializable},
207
- ensure_ascii=False, indent=2), encoding='utf-8')
328
+ json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
329
+ if scope_filter:
330
+ scope = ('**只回填了 %s 主题** —— 这一天并非全部文章,其余主题还没拉。\n'
331
+ % '、'.join(scope_filter))
332
+ else:
333
+ scope = ''
334
+ # README 说的是**这一天**有几篇(合并后的),不是这一轮写了几篇 —— 分两次回填的那天,
335
+ # 前者才是读它的人想要的数。
208
336
  (out_dir / 'README.md').write_text(
209
337
  '# 公众号文章 — %s(历史回填)\n\n共 %d 篇。正文由 `backfill_articles.py` 抓取,'
210
- '**未下载图片、未调用模型**。\n' % (day, len(written)), encoding='utf-8')
338
+ '**未下载图片、未调用模型**。\n%s' % (day, len(payload['articles']), scope), encoding='utf-8')
211
339
  return {'written': len(written), 'skipped': len(skipped),
212
340
  'fallbackTopics': fallbacks, 'skipSample': skipped[:3]}
213
341
 
@@ -277,15 +405,40 @@ def days_with_articles(out_root) -> list:
277
405
  return days
278
406
 
279
407
 
280
- def day_done(out_root, day) -> bool:
281
- """这一天已经回填过了吗(增量:免得重跑把同样的文章再抓一遍)。"""
408
+ def day_done(out_root, day, topics=None) -> bool:
409
+ """这一天**我们要的那些主题**已经回填过了吗(增量:免得重跑把同样的文章再抓一遍)。
410
+
411
+ `topics=None` 表示"全量"。**判据不只是"写过"**,还要看写的是不是完整的一天:
412
+
413
+ - 只回填过 AI 的一天,`.articles.json` 里带着 `topicFilter`。此后跑**全量**时必须
414
+ **不能**被跳过 —— 否则那天其余五个主题的文章会被永久漏掉,而且不报错:文件在、
415
+ `.articles.json` 在、"已回填"看起来成立。这就是 `days_with_articles` 那条注释里
416
+ 记的同一类事故(拿一个不完整的判据当"做完了")。
417
+ - 反向:`--topic AI` 之后又跑 `--topic AI,学术`,存过的是子集,也要重做;
418
+ 而存过的**覆盖**了这次要的(或那天本来就完整),才算做完。
419
+ - 被 `--limit-per-day` 截过的一天**从来不算做完**:截掉的那些既没抓、也看不出来。
420
+ """
282
421
  path = Path(out_root) / day / '.articles.json'
283
422
  if not path.exists():
284
423
  return False
285
424
  try:
286
- return bool(json.loads(path.read_text(encoding='utf-8')).get('articles'))
425
+ payload = json.loads(path.read_text(encoding='utf-8'))
287
426
  except Exception:
288
427
  return False
428
+ if not payload.get('articles'):
429
+ return False
430
+ if payload.get('truncated'):
431
+ return False
432
+ stored = payload.get('topicFilter')
433
+ if stored is None:
434
+ return True # 完整的一天,所有主题都在
435
+ if set(stored) >= set(TOPICS):
436
+ # 六个主题在标记里都在 = 这天完整(`write_day` 现在会直接把标记撤掉,但**旧代码写的**
437
+ # 天数带着这个形态留在盘上 —— 读的判据也认它,免得那几天永远"没做完")。
438
+ return True
439
+ if topics is None:
440
+ return False # 只要过一部分,全量的活还没干
441
+ return set(stored) >= set(topics)
289
442
 
290
443
 
291
444
  def main():
@@ -300,7 +453,13 @@ def main():
300
453
  parser.add_argument('--vault-sync', action='store_true',
301
454
  help='只做这件事:把已回填的天补拷进 Vault 的 Sources(本地,不抓取、不调模型)')
302
455
  parser.add_argument('--workers', type=int, default=8, help='抓正文的并发数(默认 8)')
303
- parser.add_argument('--limit-per-day', type=int, default=0, help='每天最多抓几篇(0=不限)')
456
+ parser.add_argument('--limit-per-day', type=int, default=0,
457
+ help='每天最多抓几篇(0=不限)。设了就不是完整的一天:'
458
+ '截掉的那些既没抓也看不出来,所以增量判断会重做这天')
459
+ parser.add_argument('--topic', action='append', default=[], metavar='T',
460
+ help='只回填这些主题(可重复或逗号分隔,合法值 %s),不写=全部。'
461
+ '只回填过一部分主题的一天,之后跑全量时不会被跳过。'
462
+ % '/'.join(TOPICS))
304
463
  parser.add_argument('--refresh', action='store_true', help='已回填过的天也重做(默认跳过)')
305
464
  parser.add_argument('--max-days', type=int, default=0, help='最多处理几天(0=不限,用于试跑)')
306
465
  parser.add_argument('--dry-run', action='store_true', help='只报到要抓多少,不抓不写')
@@ -319,6 +478,17 @@ def main():
319
478
  print('--until 不能早于 --since'); return 1
320
479
  if args.workers < 1:
321
480
  print('--workers 要 ≥ 1'); return 1
481
+ picked = parse_topics(args.topic)
482
+ unknown = [t for t in picked if t not in TOPICS]
483
+ if unknown:
484
+ # 拼错一个词(比如 `--topic ai` 小写)会静默筛出 0 篇、然后报一句"没有要处理的"——
485
+ # 看着像"这天本来就没文章"。所以这里当场报错并列出合法值,放在碰库之前。
486
+ message = ('--topic 里有不在分类法里的主题:%s\n合法值:%s'
487
+ % ('、'.join(unknown), '、'.join(TOPICS)))
488
+ print(json.dumps({'success': False, 'error': message}, ensure_ascii=False)
489
+ if args.json else message)
490
+ return 1
491
+ topics = sorted(set(picked)) or None
322
492
 
323
493
  if args.vault_sync:
324
494
  # **补拷模式**:不抓取、不调模型,只把已经躺在 biz-daily 的那几天补进 Vault。
@@ -328,6 +498,8 @@ def main():
328
498
  # 重复跑是**安全**的:`copy_to_vault` 逐 md 覆盖、不删别的东西,而它现在还会洗残留,
329
499
  # 所以"再补一次"同时也是"把库里那一层重新洗一遍"(2026-09-28 就是这么把 1,755 篇
330
500
  # 漏网的补上的 —— 那天被 `backfilled` 标记挡在门外的 12 天)。
501
+ if topics:
502
+ print('提示:--vault-sync 只把**已回填的** md 补拷进 Vault,--topic 对这一步不起作用。')
331
503
  days = days_with_articles(args.out)
332
504
  if args.max_days:
333
505
  days = days[:args.max_days]
@@ -374,28 +546,35 @@ def main():
374
546
 
375
547
  todo = []
376
548
  for day in days:
377
- if not args.refresh and day_done(args.out, day):
549
+ if not args.refresh and day_done(args.out, day, topics):
378
550
  continue
379
551
  articles = collect_articles(cursor, users, name_map, day)
552
+ # 筛选在 `collect_articles` **之后**:主题是它内部 `apply_topics` 的输出,
553
+ # 先筛就无从筛起(理由与等价性写在 `select_topics` 上)。
554
+ articles = select_topics(articles, topics)
555
+ # 截断要在**筛完之后**判断:`--limit-per-day 5 --topic AI` 该解作"每天 5 篇 AI"。
556
+ truncated = bool(args.limit_per_day) and len(articles) > args.limit_per_day
380
557
  if args.limit_per_day:
381
558
  articles = articles[:args.limit_per_day]
382
559
  if articles:
383
- todo.append((day, articles))
560
+ todo.append((day, articles, truncated))
384
561
 
385
562
  if args.dry_run:
386
- total = sum(len(items) for _, items in todo)
387
- need = sum(1 for _, items in todo for a in items
563
+ total = sum(len(items) for _, items, _ in todo)
564
+ need = sum(1 for _, items, _ in todo for a in items
388
565
  if len(body_without_media(a.get('local_text'))) < MIN_BODY_CHARS and a.get('url'))
389
566
  payload = {
390
567
  'success': True, 'dryRun': True, 'action': 'backfill-articles',
568
+ 'topics': topics or '全部',
391
569
  'days': len(todo), 'articles': total, 'needFetch': need,
392
570
  'workers': args.workers, 'invokesAI': False, 'downloadsImages': False,
393
571
  'readsLocalData': True,
394
572
  'note': '只抓正文,不下图片、不调用模型;抓完还要跑 vault notes 与 article-notes',
395
573
  }
396
574
  print(json.dumps(payload, ensure_ascii=False, indent=2) if args.json
397
- else '%d 天、%d 篇(其中 %d 篇要联网抓正文),并发 %d'
398
- % (len(todo), total, need, args.workers))
575
+ else '%d 天、%d 篇(其中 %d 篇要联网抓正文),并发 %d,主题 %s'
576
+ % (len(todo), total, need, args.workers,
577
+ '全部' if not topics else '、'.join(topics)))
399
578
  conn.close()
400
579
  return 0
401
580
 
@@ -404,23 +583,25 @@ def main():
404
583
  conn.close()
405
584
  return 0
406
585
  if not args.yes:
407
- print('会抓 %d 天、%d 篇的正文(不调模型)。加 --yes 确认。'
408
- % (len(todo), sum(len(i) for _, i in todo)))
586
+ print('会抓 %d 天、%d 篇的正文(不调模型,主题 %s)。加 --yes 确认。'
587
+ % (len(todo), sum(len(i) for _, i, _ in todo),
588
+ '全部' if not topics else '、'.join(topics)))
409
589
  conn.close()
410
590
  return 1
411
591
 
412
592
  report = []
413
- for index, (day, articles) in enumerate(todo, 1):
414
- print('\n[%d/%d] %s — 库里 %d 篇' % (index, len(todo), day, len(articles)))
593
+ for index, (day, articles, truncated) in enumerate(todo, 1):
594
+ print('\n[%d/%d] %s — 选中 %d 篇' % (index, len(todo), day, len(articles)))
415
595
  fetch = fetch_bodies(articles, args.workers, log=print)
416
- result = write_day(articles, day, args.out)
596
+ result = write_day(articles, day, args.out, topic_filter=topics, truncated=truncated)
417
597
  print(' 写入 %d 篇,跳过 %d 篇(正文太短),主题兜底 %d 篇'
418
598
  % (result['written'], result['skipped'], result['fallbackTopics']))
419
599
  if args.vault_copy:
420
600
  sync = copy_to_vault(day, args.out, args.vault_root)
421
601
  print(' 入 Vault 的 Sources: %d 个 md' % sync['copied'])
422
602
  report.append({'date': day, 'inDb': len(articles), 'fetched': fetch['ok'],
423
- 'written': result['written'], 'skippedThin': result['skipped']})
603
+ 'written': result['written'], 'skippedThin': result['skipped'],
604
+ 'truncated': truncated})
424
605
  conn.close()
425
606
 
426
607
  total_written = sum(r['written'] for r in report)
@@ -212,19 +212,29 @@ def _group_by_topic(articles):
212
212
  return groups, fallbacks
213
213
 
214
214
 
215
- def _classify_with_jev(client, title, body, topics):
215
+ def _classify_with_jev(client, title, body, topics, noise=None):
216
216
  """让 Jev 判断这一篇;**问不出来就返回 None**,由调用方逐篇退回老路。
217
217
 
218
218
  这里刻意吞掉异常(包括 `JevError` 之外的意外),因为"分类这一篇失败"
219
219
  不该让整天的日报挂掉。但异常类型会打出来——一个真的写错了的行号不该
220
220
  在日志里长得跟"网络抖了一下"一样。
221
+
222
+ `noise` 可选(见 `_classify_articles_parallel`):把**同一条**失败折叠成一次输出。
223
+ 一天上百篇全因同一个原因失败时(实测:判断层按地区不可用,HTTP 451),打上百行
224
+ 等于没说;而**没见过的异常类型或没见过的消息仍然每次照打**——折叠的是重复,不是未知。
221
225
  """
222
226
  if client is None:
223
227
  return None
224
228
  try:
225
229
  return client.decide_article(title, body, topics)
226
230
  except Exception as error:
227
- print(f' [WARN] Jev 分类失败({type(error).__name__}),本篇退回 LLM 解析:{error}')
231
+ signature = (type(error).__name__, str(error)[:200])
232
+ if noise is not None and signature in noise['seen']:
233
+ noise['repeat'] += 1
234
+ else:
235
+ if noise is not None:
236
+ noise['seen'].add(signature)
237
+ print(f' [WARN] Jev 分类失败({type(error).__name__}),本篇退回 LLM 解析:{error}')
228
238
  return None
229
239
 
230
240
 
@@ -284,9 +294,12 @@ def _classify_articles_parallel(articles, client, topics, workers=JEV_WORKERS):
284
294
 
285
295
  started = time.time()
286
296
  decisions = {}
297
+ # 同一条失败折叠成一次输出(见 `_classify_with_jev`):一天上百篇同一个原因失败时,
298
+ # 逐篇刷同一行等于没说;但没见过的异常/消息仍然照打。
299
+ noise = {'seen': set(), 'repeat': 0}
287
300
  with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
288
301
  futures = {
289
- pool.submit(_classify_with_jev, client, title, body, topics): index
302
+ pool.submit(_classify_with_jev, client, title, body, topics, noise): index
290
303
  for index, title, body in todo
291
304
  }
292
305
  for future in as_completed(futures):
@@ -296,6 +309,8 @@ def _classify_articles_parallel(articles, client, topics, workers=JEV_WORKERS):
296
309
  # 先报个数:并发之后逐篇日志没有意义了,但这三件事必须看得见。
297
310
  print(f' 分类完成 {ok}/{len(todo)} 篇,耗时 {time.time() - started:.1f}s'
298
311
  f'({workers} 并发;串行约需 {len(todo) * 0.97:.0f}s)')
312
+ if noise['repeat']:
313
+ print(f' [WARN] 另有 {noise["repeat"]} 篇因**同一条**失败被折叠(原因已在上面打过一次)')
299
314
  return decisions
300
315
 
301
316
 
@@ -473,37 +488,14 @@ def get_db_keys(config):
473
488
  contact_salt = config.get('contactSalt', '')
474
489
  contact_key = decrypt_lock(contact_key_enc) if contact_key_enc else ''
475
490
 
476
- # biz_message_0.db (每库独立密钥)
477
- biz_db = os.path.join(msg_dir, 'biz_message_0.db')
478
- biz_key_enc = config.get('bizKey', '')
479
- biz_salt = config.get('bizSalt', '')
480
- pass_enc = config.get('favPassphrase', '')
481
- passphrase = decrypt_lock(pass_enc) if pass_enc else ''
482
- if passphrase and os.path.exists(biz_db):
483
- with open(biz_db, 'rb') as fh:
484
- biz_salt = fh.read(16).hex()
485
- biz_key_enc = ''
486
- if biz_key_enc and biz_salt:
487
- biz_key = decrypt_lock(biz_key_enc)
488
- else:
489
- # 从全库共用 passphrase 派生 (微信 4.x: PBKDF2-HMAC-SHA512, 256000 轮, 32 字节)
490
- pass_enc = config.get('favPassphrase', '')
491
- passphrase = decrypt_lock(pass_enc) if pass_enc else ''
492
- biz_key = ''
493
- if passphrase and os.path.exists(biz_db):
494
- with open(biz_db, 'rb') as fh:
495
- biz_salt = fh.read(16).hex()
496
- biz_key = hashlib.pbkdf2_hmac(
497
- 'sha512', bytes.fromhex(passphrase),
498
- bytes.fromhex(biz_salt), 256000, dklen=32
499
- ).hex()
500
- if not biz_key:
501
- # 只指向 passphrase 一条路: 早先还建议 `config set bizKey`, 但该键不在
502
- # CLI 的可写白名单里, 照做必然报 INVALID_CONFIG_KEY。
503
- raise SystemExit(
504
- '缺少公众号数据库密钥: 请运行 weflow-cli fav set-key --passphrase <64位hex> '
505
- '配置全库 passphrase (自动派生各库密钥)'
506
- )
491
+ # biz_message_0.db(订阅号库):钥匙派生统一在 `_utils.biz_message_db()`。
492
+ # 这块原来自己算了两遍(先按 bizKey/bizSalt,又被下面按 passphrase 派生的一遍覆盖),
493
+ # 两遍并存就是分叉留下的痕迹 —— 见那个函数的注释。
494
+ from _utils import biz_message_db
495
+ try:
496
+ biz_db, biz_key, biz_salt = biz_message_db(config)
497
+ except RuntimeError as exc:
498
+ raise SystemExit(str(exc))
507
499
 
508
500
  return {
509
501
  'biz_db': biz_db, 'biz_key': biz_key, 'biz_salt': biz_salt,