weflow-cli 1.12.0 → 1.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +228 -1
- package/OPERATIONS.md +255 -19
- package/README.md +10 -0
- package/bin/weflow-cli.ts +260 -51
- package/dist/bin/weflow-cli.js +255 -32
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/src/core/dbPathService.d.ts.map +1 -1
- package/dist/src/core/dbPathService.js +5 -0
- package/dist/src/core/dbPathService.js.map +1 -1
- package/dist/src/core/ntCore.d.ts +12 -1
- package/dist/src/core/ntCore.d.ts.map +1 -1
- package/dist/src/core/ntCore.js +23 -3
- package/dist/src/core/ntCore.js.map +1 -1
- package/dist/src/core/sqlcipherCore.d.ts +8 -0
- package/dist/src/core/sqlcipherCore.d.ts.map +1 -1
- package/dist/src/core/sqlcipherCore.js +46 -11
- package/dist/src/core/sqlcipherCore.js.map +1 -1
- package/dist/src/services/configService.d.ts +3 -0
- package/dist/src/services/configService.d.ts.map +1 -1
- package/dist/src/services/configService.js +6 -3
- package/dist/src/services/configService.js.map +1 -1
- package/dist/src/services/diagnosticReport.d.ts +80 -0
- package/dist/src/services/diagnosticReport.d.ts.map +1 -0
- package/dist/src/services/diagnosticReport.js +191 -0
- package/dist/src/services/diagnosticReport.js.map +1 -0
- package/dist/src/services/wechatClient.d.ts +17 -3
- package/dist/src/services/wechatClient.d.ts.map +1 -1
- package/dist/src/services/wechatClient.js +68 -23
- package/dist/src/services/wechatClient.js.map +1 -1
- package/docs/DECISIONS.md +347 -1
- package/docs/NEXT_ACTIONS.md +7 -1
- package/docs/POSITIONING.md +68 -0
- package/docs/PROJECT_STATE.md +29 -8
- package/package.json +88 -86
- package/scripts/_utils.py +120 -4
- package/scripts/biz_daily.py +118 -11
- package/scripts/daily_task.ps1 +132 -0
- package/scripts/export_chat_html.py +12 -1
- package/scripts/install_tasks.cmd +32 -0
- package/scripts/install_tasks.ps1 +91 -0
- package/scripts/jev_client.py +335 -3
- package/scripts/nt_decrypt.py +79 -14
- package/scripts/pipeline.py +21 -0
- package/scripts/repro_customer_init.cmd +86 -0
- package/scripts/semantic_search.py +286 -24
- package/src/core/dbPathService.ts +5 -0
- package/src/core/ntCore.ts +34 -3
- package/src/core/sqlcipherCore.ts +47 -11
- package/src/services/configService.ts +12 -3
- package/src/services/diagnosticReport.ts +237 -0
- package/src/services/wechatClient.ts +428 -380
package/package.json
CHANGED
|
@@ -1,86 +1,88 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "weflow-cli",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "让 AI 读懂你的微信 — 公众号日报·聊天导出·知识库搜索·MCP Server",
|
|
5
|
-
"type": "module",
|
|
6
|
-
"bin": {
|
|
7
|
-
"weflow-cli": "cli.cjs"
|
|
8
|
-
},
|
|
9
|
-
"files": [
|
|
10
|
-
"cli.cjs",
|
|
11
|
-
"dist/",
|
|
12
|
-
"scripts/**/*.py",
|
|
13
|
-
"scripts
|
|
14
|
-
"
|
|
15
|
-
"
|
|
16
|
-
"
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
"
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"
|
|
25
|
-
"
|
|
26
|
-
"
|
|
27
|
-
"
|
|
28
|
-
"
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
-
"
|
|
39
|
-
"
|
|
40
|
-
"
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
52
|
-
"
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"
|
|
60
|
-
"
|
|
61
|
-
"
|
|
62
|
-
"
|
|
63
|
-
"
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"
|
|
70
|
-
"
|
|
71
|
-
"
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "weflow-cli",
|
|
3
|
+
"version": "1.13.1",
|
|
4
|
+
"description": "让 AI 读懂你的微信 — 公众号日报·聊天导出·知识库搜索·MCP Server",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"weflow-cli": "cli.cjs"
|
|
8
|
+
},
|
|
9
|
+
"files": [
|
|
10
|
+
"cli.cjs",
|
|
11
|
+
"dist/",
|
|
12
|
+
"scripts/**/*.py",
|
|
13
|
+
"scripts/**/*.ps1",
|
|
14
|
+
"scripts/**/*.cmd",
|
|
15
|
+
"scripts/data/",
|
|
16
|
+
"mcp-server/",
|
|
17
|
+
"bin/",
|
|
18
|
+
"src/",
|
|
19
|
+
"resources/",
|
|
20
|
+
"docs/images/",
|
|
21
|
+
"docs/*.md",
|
|
22
|
+
"logo.png",
|
|
23
|
+
"README.md",
|
|
24
|
+
"OPERATIONS.md",
|
|
25
|
+
"ARCHITECTURE.md",
|
|
26
|
+
"CONTRIBUTING.md",
|
|
27
|
+
"SECURITY.md",
|
|
28
|
+
"CHANGELOG.md",
|
|
29
|
+
"requirements*.txt",
|
|
30
|
+
"LICENSE"
|
|
31
|
+
],
|
|
32
|
+
"keywords": [
|
|
33
|
+
"wechat",
|
|
34
|
+
"weixin",
|
|
35
|
+
"mcp",
|
|
36
|
+
"rss",
|
|
37
|
+
"daily",
|
|
38
|
+
"公众号",
|
|
39
|
+
"聊天记录",
|
|
40
|
+
"ai",
|
|
41
|
+
"deepseek",
|
|
42
|
+
"claude"
|
|
43
|
+
],
|
|
44
|
+
"repository": {
|
|
45
|
+
"type": "git",
|
|
46
|
+
"url": "git+https://github.com/zhuobichen/weflow-cli.git"
|
|
47
|
+
},
|
|
48
|
+
"scripts": {
|
|
49
|
+
"build": "tsc",
|
|
50
|
+
"test": "node --import tsx --test \"test/**/*.test.ts\"",
|
|
51
|
+
"dev": "npx tsx bin/weflow-cli.ts",
|
|
52
|
+
"start": "node cli.cjs",
|
|
53
|
+
"prepublishOnly": "npm run build",
|
|
54
|
+
"eval:assistant": "node --import tsx scripts/assistant_eval.ts"
|
|
55
|
+
},
|
|
56
|
+
"dependencies": {
|
|
57
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
58
|
+
"@wenyan-md/core": "^3.0.10",
|
|
59
|
+
"chalk": "^5.4.1",
|
|
60
|
+
"commander": "^13.1.0",
|
|
61
|
+
"css-tree": "^3.2.1",
|
|
62
|
+
"exceljs": "^4.4.0",
|
|
63
|
+
"form-data-encoder": "^4.1.0",
|
|
64
|
+
"formdata-node": "^6.0.3",
|
|
65
|
+
"front-matter": "^4.0.2",
|
|
66
|
+
"fzstd": "^0.1.1",
|
|
67
|
+
"highlight.js": "^11.11.1",
|
|
68
|
+
"inquirer": "^13.4.3",
|
|
69
|
+
"jsdom": "^27.4.0",
|
|
70
|
+
"koffi": "^2.9.0",
|
|
71
|
+
"marked": "^18.0.4",
|
|
72
|
+
"marked-highlight": "^2.2.4",
|
|
73
|
+
"qrcode-terminal": "^0.12.0"
|
|
74
|
+
},
|
|
75
|
+
"optionalDependencies": {
|
|
76
|
+
"lz4": "^0.6.5"
|
|
77
|
+
},
|
|
78
|
+
"devDependencies": {
|
|
79
|
+
"@types/node": "^22.0.0",
|
|
80
|
+
"@types/qrcode-terminal": "^0.12.2",
|
|
81
|
+
"electron": "^42.0.1",
|
|
82
|
+
"tsx": "^4.19.0",
|
|
83
|
+
"typescript": "^5.7.0"
|
|
84
|
+
},
|
|
85
|
+
"engines": {
|
|
86
|
+
"node": ">=22.13.0"
|
|
87
|
+
}
|
|
88
|
+
}
|
package/scripts/_utils.py
CHANGED
|
@@ -648,6 +648,30 @@ def load_config():
|
|
|
648
648
|
return json.load(f)
|
|
649
649
|
|
|
650
650
|
|
|
651
|
+
# ---- 「不说话地失败」的最小护栏 ---------------------------------------------
|
|
652
|
+
# 这个文件里有一批函数**故意不抛**(跨机器拷配置时,解不开的密文该表现为"没配",
|
|
653
|
+
# 而不是让整条命令崩)。但"不抛"不等于"不说":`except: return ''` 会让调用方
|
|
654
|
+
# 与用户都以为"本来就没有",而真实原因是**解密失败/配置坏了**。
|
|
655
|
+
# 所以这里只做一件事:同一条原因**说一次**,然后照旧返回空值。
|
|
656
|
+
# 只说一次是必须的——这些函数每次请求/每次重排都会被调用,刷屏等于没警告。
|
|
657
|
+
_WARNED_ONCE = set()
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def warn_once(tag, message):
|
|
661
|
+
"""同一条原因只往 stderr 说一次(tag 相同即视为同一条)。"""
|
|
662
|
+
if tag in _WARNED_ONCE:
|
|
663
|
+
return
|
|
664
|
+
_WARNED_ONCE.add(tag)
|
|
665
|
+
import sys as _sys
|
|
666
|
+
print('[WARN] ' + message, file=_sys.stderr)
|
|
667
|
+
_sys.stderr.flush()
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def reset_warn_once():
|
|
671
|
+
"""测试用:清空去重表。"""
|
|
672
|
+
_WARNED_ONCE.clear()
|
|
673
|
+
|
|
674
|
+
|
|
651
675
|
# 配置里"不想在日报里看到的主题"的键名。值形如 `新闻,投资,学术`。
|
|
652
676
|
EXCLUDED_TOPICS_KEY = 'dailyExcludeTopics'
|
|
653
677
|
|
|
@@ -876,11 +900,15 @@ def get_dashscope_key(config=None) -> str:
|
|
|
876
900
|
if config is None:
|
|
877
901
|
try:
|
|
878
902
|
config = load_config()
|
|
879
|
-
except Exception:
|
|
903
|
+
except Exception as exc:
|
|
904
|
+
warn_once('load_config', '读不到配置文件 %s(%s: %s)—— 本次当作空配置,需要配置的功能会退化成默认行为'
|
|
905
|
+
% (CONFIG_PATH, type(exc).__name__, exc))
|
|
880
906
|
return ''
|
|
881
907
|
try:
|
|
882
908
|
return decrypt_lock(config.get('dashscopeApiKey', ''))
|
|
883
|
-
except Exception:
|
|
909
|
+
except Exception as exc:
|
|
910
|
+
warn_once('dashscopeApiKey', '配置项 dashscopeApiKey 解不开(%s: %s)—— 当作"没配",语义检索会退成关键词'
|
|
911
|
+
% (type(exc).__name__, exc))
|
|
884
912
|
return ''
|
|
885
913
|
|
|
886
914
|
|
|
@@ -895,14 +923,102 @@ def get_typesafe_key(config=None) -> str:
|
|
|
895
923
|
if config is None:
|
|
896
924
|
try:
|
|
897
925
|
config = load_config()
|
|
898
|
-
except Exception:
|
|
926
|
+
except Exception as exc:
|
|
927
|
+
warn_once('load_config', '读不到配置文件 %s(%s: %s)—— 本次当作空配置,需要配置的功能会退化成默认行为'
|
|
928
|
+
% (CONFIG_PATH, type(exc).__name__, exc))
|
|
899
929
|
return ''
|
|
900
930
|
try:
|
|
901
931
|
return decrypt_lock(config.get('typesafeApiKey', ''))
|
|
902
|
-
except Exception:
|
|
932
|
+
except Exception as exc:
|
|
933
|
+
warn_once('typesafeApiKey', '配置项 typesafeApiKey 解不开(%s: %s)—— 当作"没配",决策模型退回 LLM 解析'
|
|
934
|
+
% (type(exc).__name__, exc))
|
|
903
935
|
return ''
|
|
904
936
|
|
|
905
937
|
|
|
938
|
+
def get_openrouter_key(config=None) -> str:
|
|
939
|
+
"""读取 OpenRouter 的 key(决策模型的第二条通路,D-120),自动解密 lock: 前缀。
|
|
940
|
+
|
|
941
|
+
与 `get_typesafe_key` 逐字同一条规矩:**跨机器拷配置时表现为"没配",不是崩**。
|
|
942
|
+
解密失败回空串之后,`create_client()` 会返回 None,调用方按老路退回 ——
|
|
943
|
+
这是"这条路没配好"的**安全**失效方向。
|
|
944
|
+
"""
|
|
945
|
+
if config is None:
|
|
946
|
+
try:
|
|
947
|
+
config = load_config()
|
|
948
|
+
except Exception as exc:
|
|
949
|
+
warn_once('load_config', '读不到配置文件 %s(%s: %s)—— 本次当作空配置,需要配置的功能会退化成默认行为'
|
|
950
|
+
% (CONFIG_PATH, type(exc).__name__, exc))
|
|
951
|
+
return ''
|
|
952
|
+
try:
|
|
953
|
+
return decrypt_lock(config.get('openrouterApiKey', ''))
|
|
954
|
+
except Exception as exc:
|
|
955
|
+
warn_once('openrouterApiKey', '配置项 openrouterApiKey 解不开(%s: %s)—— 当作"没配",OpenRouter 那条路会停用'
|
|
956
|
+
% (type(exc).__name__, exc))
|
|
957
|
+
return ''
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def get_judge_provider(config=None) -> str:
|
|
961
|
+
"""决策模型走哪条通路:`native`(默认)、`chat`(国内对话模型)或 `openrouter`。
|
|
962
|
+
|
|
963
|
+
**默认必须是 native**:这个开关存在的意义是"本机原生被地区封锁时换条路",
|
|
964
|
+
而不是替所有用户改行为。取值不认识时也回 native —— 打错的配置项不该把判断整层关掉。
|
|
965
|
+
"""
|
|
966
|
+
if config is None:
|
|
967
|
+
try:
|
|
968
|
+
config = load_config()
|
|
969
|
+
except Exception:
|
|
970
|
+
return 'native'
|
|
971
|
+
value = str((config or {}).get('judgeProvider') or '').strip().lower()
|
|
972
|
+
return value if value in ('native', 'chat', 'openrouter') else 'native'
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def get_chat_endpoint(config=None):
|
|
976
|
+
"""`judgeProvider=chat` 时该往哪个 (base_url, model) 发 —— 与"中转站"同一条口径。
|
|
977
|
+
|
|
978
|
+
故意只做一件事:**委托 `_custom_endpoint()`**。那个函数是 `aiBaseUrl`/`aiModel` 的权威读法
|
|
979
|
+
(base 与 model 成对出现),判断器另写一份就会在"只配了 base 没配 model"这类组合上分叉,
|
|
980
|
+
而分叉的表现是"发去了错的端点"——不报错、只是判断质量变了。没配 `aiBaseUrl` 时返回 (None, None)。
|
|
981
|
+
"""
|
|
982
|
+
try:
|
|
983
|
+
return _custom_endpoint()
|
|
984
|
+
except Exception:
|
|
985
|
+
return None, None
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def get_ai_engine(config=None) -> str:
|
|
989
|
+
"""配置里选的对话引擎:`local` / `deepseek`(默认)/ `claude` / `ollama`。
|
|
990
|
+
|
|
991
|
+
**决策模型的 `chat` 通路必须读它**:`judgeProvider=chat` 的语义是"用**配置里的**对话模型",
|
|
992
|
+
而"配置里的"到底指谁只有 `aiEngine` 说得清。2026-10-18 之前那条路无条件用 `deepseekApiKey`,
|
|
993
|
+
于是 `aiEngine=local` 的用户一开 `judgeProvider=chat` 就会把文章正文发去 DeepSeek ——
|
|
994
|
+
这不是"多花点钱",是**用户明确选了不出境之后又被送出境**。
|
|
995
|
+
"""
|
|
996
|
+
if config is None:
|
|
997
|
+
try:
|
|
998
|
+
config = load_config()
|
|
999
|
+
except Exception as exc:
|
|
1000
|
+
warn_once('load_config', '读不到配置文件 %s(%s: %s)—— 本次当作空配置,需要配置的功能会退化成默认行为'
|
|
1001
|
+
% (CONFIG_PATH, type(exc).__name__, exc))
|
|
1002
|
+
return 'deepseek'
|
|
1003
|
+
return str((config or {}).get('aiEngine') or 'deepseek').strip().lower()
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
def get_openrouter_model(config=None) -> str:
|
|
1007
|
+
"""OpenRouter 上用的模型 id(默认 `typesafe/jev-router`,即 TypeSafe 发布的决策模型)。
|
|
1008
|
+
|
|
1009
|
+
⚠️ **地区限制是逐模型的**(实测 2026-10-17):`typesafe/jev-router` 与 `:free` 模型能通,
|
|
1010
|
+
付费通用模型(`openai/gpt-4o-mini`)在本地区返回 403。换模型必须重测,见 D-120。
|
|
1011
|
+
"""
|
|
1012
|
+
if config is None:
|
|
1013
|
+
try:
|
|
1014
|
+
config = load_config()
|
|
1015
|
+
except Exception as exc:
|
|
1016
|
+
warn_once('load_config', '读不到配置文件 %s(%s: %s)—— 本次当作空配置,需要配置的功能会退化成默认行为'
|
|
1017
|
+
% (CONFIG_PATH, type(exc).__name__, exc))
|
|
1018
|
+
return ''
|
|
1019
|
+
return str((config or {}).get('openrouterJudgeModel') or '').strip()
|
|
1020
|
+
|
|
1021
|
+
|
|
906
1022
|
# ======================================================================
|
|
907
1023
|
# Markdown / Frontmatter
|
|
908
1024
|
# ======================================================================
|
package/scripts/biz_daily.py
CHANGED
|
@@ -64,6 +64,7 @@ FETCH_TIMEOUT = 15
|
|
|
64
64
|
DEEPSEEK_TIMEOUT = 60
|
|
65
65
|
FETCH_DELAY_MIN = 8 # 最小抓取间隔 (秒)
|
|
66
66
|
FETCH_DELAY_MAX = 12 # 最大抓取间隔 (秒)
|
|
67
|
+
PREFETCH_WORKERS = 8 # `--prefetch-bodies` 不带数字时的并发数(预热正文缓存,见 prefetch_bodies)
|
|
67
68
|
|
|
68
69
|
# TOPICS 现在从 _utils 导入(见文件头的 import)——分类法只有一份定义,
|
|
69
70
|
# 因为"散文写的枚举"已经漂过一次:这里曾硬编码五类,而 TOPICS 是六类。
|
|
@@ -141,6 +142,58 @@ def _tags_for_write(article, topic):
|
|
|
141
142
|
return article.get('tags', [topic])
|
|
142
143
|
|
|
143
144
|
|
|
145
|
+
def _article_key(entry):
|
|
146
|
+
"""同一篇文章的合并键:url 优先(微信文章 url 唯一),没有 url 时退回标题+时间。"""
|
|
147
|
+
url = str(entry.get('url') or '').strip()
|
|
148
|
+
if url:
|
|
149
|
+
return url
|
|
150
|
+
title = str(entry.get('title') or '').strip()
|
|
151
|
+
if not title:
|
|
152
|
+
return ''
|
|
153
|
+
return 'no-url:%s|%s' % (title, entry.get('time') or '')
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def merge_articles_json(json_path, date_str, serializable, decision_model=None):
|
|
157
|
+
"""把这一批**并进** `.articles.json`,返回落盘的全部条目(按时间排序)。
|
|
158
|
+
|
|
159
|
+
为什么必须合并:同一天可能跑两次(午报 13 点 + 日终 24 点)。第二次进来时
|
|
160
|
+
`.run_state.json` 已经把上午处理过的文章跳过了,所以这批只有下午新到达的那些;
|
|
161
|
+
若覆盖写,上午那批的标题/摘要/主题/relevance 就从这个文件里消失。而 README 与
|
|
162
|
+
`index.html` 是**从磁盘上的 md 重建**的 —— 于是"页面是全天、结构化数据只有半天",
|
|
163
|
+
下游(AI 报告、daily-stats、知识库统计)静默少一半,没有任何地方会报错。
|
|
164
|
+
`.image_map.json` 早就是合并写的(`existing_map.update(...)`),这里只是把它缺的那半补上。
|
|
165
|
+
"""
|
|
166
|
+
merged = {}
|
|
167
|
+
previous_model = None
|
|
168
|
+
if json_path.exists():
|
|
169
|
+
try:
|
|
170
|
+
old = json.loads(json_path.read_text(encoding='utf-8'))
|
|
171
|
+
previous_model = old.get('decisionModel')
|
|
172
|
+
for entry in old.get('articles') or []:
|
|
173
|
+
key = _article_key(entry)
|
|
174
|
+
if key:
|
|
175
|
+
merged[key] = entry
|
|
176
|
+
except Exception:
|
|
177
|
+
# 旧文件坏了不该让今天失败:从空开始(与 `.image_map.json` 同一态度)
|
|
178
|
+
print(' [WARN] 旧的 .articles.json 读不了,这次从空开始(旧条目不参与合并)')
|
|
179
|
+
for entry in serializable:
|
|
180
|
+
key = _article_key(entry)
|
|
181
|
+
if key:
|
|
182
|
+
merged[key] = entry
|
|
183
|
+
articles = sorted(merged.values(), key=lambda e: (e.get('time') or '', e.get('title') or ''))
|
|
184
|
+
payload = {
|
|
185
|
+
'generated_at': datetime.now(timezone(timedelta(hours=8))).strftime('%Y-%m-%d %H:%M:%S'),
|
|
186
|
+
'date': date_str,
|
|
187
|
+
# 本次没判断(`--no-ai`)时不要用 None 盖掉上一次记下的模型名:那会让
|
|
188
|
+
# "这天是谁判的"在重跑后凭空消失。
|
|
189
|
+
'decisionModel': decision_model if decision_model is not None else previous_model,
|
|
190
|
+
'articles': articles,
|
|
191
|
+
}
|
|
192
|
+
json_path.parent.mkdir(parents=True, exist_ok=True)
|
|
193
|
+
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
194
|
+
return articles
|
|
195
|
+
|
|
196
|
+
|
|
144
197
|
def _serializable_article(article, date_str):
|
|
145
198
|
"""一篇文章 -> `.articles.json` 里的那条记录。
|
|
146
199
|
|
|
@@ -666,6 +719,43 @@ def fetch_article_cached(url: str, use_cache: bool = True) -> tuple[str | None,
|
|
|
666
719
|
return body, False
|
|
667
720
|
|
|
668
721
|
|
|
722
|
+
def prefetch_bodies(articles, workers=PREFETCH_WORKERS, log=None):
|
|
723
|
+
"""**并发**把这一批文章的正文抓进缓存(`output/.cache/fetch/`),只抓不写盘。
|
|
724
|
+
|
|
725
|
+
为什么值得有:Phase 1 是**串行**的,而且未命中缓存时每篇要 `sleep` 8~12 秒(节流是为了
|
|
726
|
+
少打扰上游)——450 篇就是约 75 分钟。而**命中缓存既不联网也不 sleep**(见
|
|
727
|
+
`fetch_article_cached`),所以先把正文并发抓下来,Phase 1 就变成秒级。
|
|
728
|
+
对上游的**请求总次数完全相同**,区别只是把同样多的请求压进更短的时间窗 ——
|
|
729
|
+
这是一个显式取舍,所以默认**关**,由 `--prefetch-bodies` 打开。
|
|
730
|
+
|
|
731
|
+
失败**不抛也不报错退出**:Phase 1 还会再试一次(那里才是"这篇到底有没有正文"的判据)。
|
|
732
|
+
返回 `{'requested', 'ok'}`。
|
|
733
|
+
"""
|
|
734
|
+
urls = []
|
|
735
|
+
for a in articles:
|
|
736
|
+
url = a.get('url')
|
|
737
|
+
if url and url not in urls:
|
|
738
|
+
urls.append(url)
|
|
739
|
+
if not urls:
|
|
740
|
+
return {'requested': 0, 'ok': 0}
|
|
741
|
+
|
|
742
|
+
def _one(url):
|
|
743
|
+
try:
|
|
744
|
+
body, _cached = fetch_article_cached(url)
|
|
745
|
+
return bool(body and body.strip())
|
|
746
|
+
except Exception: # noqa: BLE001 单篇炸了不能拖掉整批
|
|
747
|
+
return False
|
|
748
|
+
|
|
749
|
+
done = ok = 0
|
|
750
|
+
with ThreadPoolExecutor(max_workers=max(1, int(workers))) as pool:
|
|
751
|
+
for hit in pool.map(_one, urls):
|
|
752
|
+
done += 1
|
|
753
|
+
ok += 1 if hit else 0
|
|
754
|
+
if log and (done % 50 == 0 or done == len(urls)):
|
|
755
|
+
log(' 预热 %d/%d(成功 %d)' % (done, len(urls), ok))
|
|
756
|
+
return {'requested': len(urls), 'ok': ok}
|
|
757
|
+
|
|
758
|
+
|
|
669
759
|
def _download_one_image(url: str, local_path: Path) -> bool:
|
|
670
760
|
"""取一张图。成功返回 True;失败**不抛**——一张图挂了不该影响这一篇的其它图。"""
|
|
671
761
|
try:
|
|
@@ -846,6 +936,12 @@ def main():
|
|
|
846
936
|
parser.add_argument('--no-ai', action='store_true', help='关闭摘要、分类和日报简报的 AI 调用')
|
|
847
937
|
parser.add_argument('--no-fetch-cache', action='store_true',
|
|
848
938
|
help='忽略正文抓取缓存,强制重新抓取(默认命中缓存,用于续跑与重跑)')
|
|
939
|
+
parser.add_argument('--prefetch-bodies', nargs='?', type=int, const=PREFETCH_WORKERS, default=0,
|
|
940
|
+
metavar='N',
|
|
941
|
+
help='Phase 1 之前先**并发**把这一天的正文抓进缓存(N=并发数,默认 %d)——'
|
|
942
|
+
'Phase 1 于是全部命中缓存、不再逐篇 sleep(450 篇约 75 分钟 → 秒级)。'
|
|
943
|
+
'对上游的请求次数不变,只是压进更短的时间窗,所以默认关(见 prefetch_bodies)'
|
|
944
|
+
% PREFETCH_WORKERS)
|
|
849
945
|
parser.add_argument('--no-summary', action='store_true',
|
|
850
946
|
help='只要判断、不要生成:完全跳过 LLM 调用(摘要/标签/概念/简报),'
|
|
851
947
|
'主题与相关度仍由 Jev 判断;因此**不需要 DeepSeek key**')
|
|
@@ -1003,6 +1099,17 @@ def main():
|
|
|
1003
1099
|
# Output folder already created above (with dedup state)
|
|
1004
1100
|
print(f'输出目录: {out_dir}\n')
|
|
1005
1101
|
|
|
1102
|
+
# ====== Phase 0: 并发预热正文缓存(可选,`--prefetch-bodies`)======
|
|
1103
|
+
if args.prefetch_bodies and not args.no_fetch_cache:
|
|
1104
|
+
print(f'=== Phase 0: 并发预热正文缓存({args.prefetch_bodies} 并发)===\n')
|
|
1105
|
+
prefetch = prefetch_bodies(articles, args.prefetch_bodies, log=print)
|
|
1106
|
+
print(f'预热完成:{prefetch["ok"]}/{prefetch["requested"]} 篇进了缓存'
|
|
1107
|
+
f'(没进的 Phase 1 会再试一次)\n')
|
|
1108
|
+
elif args.prefetch_bodies:
|
|
1109
|
+
# 说清楚为什么没做:`--no-fetch-cache` 让 Phase 1 绕过缓存,预热等于白抓一遍。
|
|
1110
|
+
print('[WARN] 同时给了 --no-fetch-cache 与 --prefetch-bodies:'
|
|
1111
|
+
'前者让 Phase 1 绕过缓存,预热没有意义,已跳过\n')
|
|
1112
|
+
|
|
1006
1113
|
# ====== Phase 1: Fetch all articles ======
|
|
1007
1114
|
print(f'=== Phase 1: 抓取 {len(articles)} 篇文章 ===\n')
|
|
1008
1115
|
cache_hits = 0
|
|
@@ -1218,19 +1325,14 @@ def main():
|
|
|
1218
1325
|
f'已归入兜底类「{DEFAULT_TOPIC}」(不是判断结果)')
|
|
1219
1326
|
|
|
1220
1327
|
# --- 写入结构化 JSON:一次提取,多次复用(供 AI 报告等下游使用) ---
|
|
1328
|
+
# **合并写**:同一天跑两次(午报 + 日终)时,这一批只有新到达的文章,
|
|
1329
|
+
# 覆盖写会把上一批从文件里抹掉(详解见 `merge_articles_json` 的 docstring)。
|
|
1221
1330
|
serializable = [_serializable_article(a, date_str) for a in articles]
|
|
1222
1331
|
|
|
1223
1332
|
json_path = out_dir / '.articles.json'
|
|
1224
1333
|
try:
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
'generated_at': datetime.now(timezone(timedelta(hours=8))).strftime('%Y-%m-%d %H:%M:%S'),
|
|
1228
|
-
'date': date_str,
|
|
1229
|
-
# 只增字段:老读者不认识它会忽略;但"这次是谁判的"从此有据可查。
|
|
1230
|
-
'decisionModel': decision_model,
|
|
1231
|
-
'articles': serializable,
|
|
1232
|
-
}, f, ensure_ascii=False, indent=2)
|
|
1233
|
-
print(f' ✓ 结构化数据: {json_path} ({len(serializable)} 篇)')
|
|
1334
|
+
merged_articles = merge_articles_json(json_path, date_str, serializable, decision_model)
|
|
1335
|
+
print(f' ✓ 结构化数据: {json_path} ({len(merged_articles)} 篇,本次新增 {len(serializable)} 篇)')
|
|
1234
1336
|
except Exception as e:
|
|
1235
1337
|
print(f' [WARN] JSON 写入失败: {e}')
|
|
1236
1338
|
|
|
@@ -1303,8 +1405,13 @@ def main():
|
|
|
1303
1405
|
if map_file.exists():
|
|
1304
1406
|
try:
|
|
1305
1407
|
existing_map = json.loads(map_file.read_text(encoding='utf-8'))
|
|
1306
|
-
except:
|
|
1307
|
-
|
|
1408
|
+
except Exception as exc:
|
|
1409
|
+
# 读不出来的旧映射表会让**之前几天的图片**从 HTML 里消失,
|
|
1410
|
+
# 而这一篇看起来还是成功的。宁可丢这一批映射,也要说清是什么原因。
|
|
1411
|
+
print(' [WARN] 旧的 .image_map.json 读不了(%s: %s)—— 这次从空表开始,'
|
|
1412
|
+
'旧的图片映射会丢(HTML 里那些图可能显示不出来)'
|
|
1413
|
+
% (type(exc).__name__, exc), file=sys.stderr)
|
|
1414
|
+
existing_map = {}
|
|
1308
1415
|
existing_map.update(url_mapping)
|
|
1309
1416
|
map_file.write_text(json.dumps(existing_map, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
1310
1417
|
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# Runs a daily report pass, with everything written to logs/daily-<date>.log.
|
|
2
|
+
#
|
|
3
|
+
# Two modes, because a WeChat official-account day only fills up gradually:
|
|
4
|
+
#
|
|
5
|
+
# daily_task.ps1 noon -> today so far (run at 13:00; half a day of push messages)
|
|
6
|
+
# daily_task.ps1 -> yesterday, complete (run at 00:05; same as before)
|
|
7
|
+
#
|
|
8
|
+
# The second pass is incremental by design: `.run_state.json` makes biz_daily skip the
|
|
9
|
+
# articles the noon pass already handled, so 00:05 only fetches/summarises what arrived
|
|
10
|
+
# in the afternoon, while README/index.html are rebuilt from every md on disk.
|
|
11
|
+
#
|
|
12
|
+
# Why this wrapper exists instead of putting the command straight into the scheduled task:
|
|
13
|
+
#
|
|
14
|
+
# 1. The task must not depend on a build artifact. It calls the python layer
|
|
15
|
+
# (scripts/pipeline.py) directly, which is the same source the repo builds from.
|
|
16
|
+
# 2. A task whose output goes nowhere answers "it failed" and nothing else. One log
|
|
17
|
+
# file per run, so "why was yesterday short?" has an answer.
|
|
18
|
+
# 3. Redirecting stdout to a FILE changes python's stdout encoding (it stops being the
|
|
19
|
+
# console and becomes the locale codepage). A title containing a character outside
|
|
20
|
+
# that codepage would then raise UnicodeEncodeError *after* the day was already
|
|
21
|
+
# written, i.e. the run looks like a failure while the artifacts are fine. Hence
|
|
22
|
+
# PYTHONIOENCODING/PYTHONUTF8 below.
|
|
23
|
+
#
|
|
24
|
+
# Extra arguments are appended, so `daily_task.ps1 --help` is a safe smoke test (it
|
|
25
|
+
# prints the pipeline's help into that day's log and exits without touching anything).
|
|
26
|
+
$ErrorActionPreference = 'Stop'
|
|
27
|
+
|
|
28
|
+
$root = Split-Path -Parent $PSScriptRoot
|
|
29
|
+
$logDir = Join-Path $root 'logs'
|
|
30
|
+
if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir | Out-Null }
|
|
31
|
+
|
|
32
|
+
# Mode is the first positional word; anything else is forwarded to pipeline.py.
|
|
33
|
+
$mode = 'day'
|
|
34
|
+
$rest = @($args)
|
|
35
|
+
if ($args.Count -gt 0 -and ($args[0] -eq 'noon' -or $args[0] -eq 'day')) {
|
|
36
|
+
$mode = $args[0]
|
|
37
|
+
$rest = if ($args.Count -gt 1) { @($args[1..($args.Count - 1)]) } else { @() }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
$today = (Get-Date).ToString('yyyy-MM-dd')
|
|
41
|
+
$yesterday = (Get-Date).AddDays(-1).ToString('yyyy-MM-dd')
|
|
42
|
+
$targetDay = if ($mode -eq 'noon') { $today } else { $yesterday }
|
|
43
|
+
|
|
44
|
+
$stamp = Get-Date -Format 'yyyy-MM-dd'
|
|
45
|
+
$suffix = if ($mode -eq 'noon') { '-noon' } else { '' }
|
|
46
|
+
$out = Join-Path $logDir ('daily-{0}{1}.log' -f $stamp, $suffix)
|
|
47
|
+
$err = Join-Path $logDir ('daily-{0}{1}.err.log' -f $stamp, $suffix)
|
|
48
|
+
|
|
49
|
+
$env:PYTHONIOENCODING = 'utf-8'
|
|
50
|
+
$env:PYTHONUTF8 = '1'
|
|
51
|
+
|
|
52
|
+
$python = (Get-Command python.exe).Source
|
|
53
|
+
$script = Join-Path $PSScriptRoot 'pipeline.py'
|
|
54
|
+
$modeArgs = if ($mode -eq 'noon') { @('--date', $today) } else { @('--yesterday') }
|
|
55
|
+
$allArgs = @($script) + $modeArgs + @('--prefetch-bodies') + $rest
|
|
56
|
+
|
|
57
|
+
# Idempotent "make sure the target day is as complete as it can be right now", not
|
|
58
|
+
# "run at 08:20".
|
|
59
|
+
#
|
|
60
|
+
# Why: this machine slept from 2026-10-17 02:09 to 2026-10-18 15:26 and BOTH 08:20
|
|
61
|
+
# triggers were counted as missed while StartWhenAvailable=On did not start the task
|
|
62
|
+
# on wake (NumberOfMissedRuns=2, no run). A schedule that only fires if the machine
|
|
63
|
+
# happens to be awake at one instant is not a schedule. With the guard below the task
|
|
64
|
+
# can be triggered cheaply and often (schedule + at logon): if the target is already
|
|
65
|
+
# done this exits in milliseconds, otherwise it produces/extends the day.
|
|
66
|
+
#
|
|
67
|
+
# The day-mode guard deliberately does NOT trust index.html alone: the noon pass also
|
|
68
|
+
# writes one, so "index.html exists" would make the 00:05 pass skip the afternoon and
|
|
69
|
+
# leave the day half-done behind a complete-looking page. `.articles.json`'s
|
|
70
|
+
# generated_at tells us when the last pass actually ran; anything during that day
|
|
71
|
+
# (before 23:00) is a mid-day artifact and must be extended.
|
|
72
|
+
#
|
|
73
|
+
# Note the day pass runs at 00:05 **the next day**, so a finished day is one whose
|
|
74
|
+
# generated_at is later than the day itself - not one stamped at 23:xx. Testing
|
|
75
|
+
# `hour -ge 23` alone would have made every 00:05 run redo the whole day (observed
|
|
76
|
+
# while verifying this guard: generated_at 00:20 was judged incomplete).
|
|
77
|
+
function Test-DayComplete([string]$dir, [string]$targetDay) {
|
|
78
|
+
$idx = Join-Path $dir 'index.html'
|
|
79
|
+
if (-not (Test-Path $idx)) { return $false }
|
|
80
|
+
$art = Join-Path $dir '.articles.json'
|
|
81
|
+
if (-not (Test-Path $art)) { return $true } # old artifact without json: trust it
|
|
82
|
+
try {
|
|
83
|
+
$j = Get-Content $art -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
84
|
+
$gen = [datetime]::ParseExact([string]$j.generated_at, 'yyyy-MM-dd HH:mm:ss', $null)
|
|
85
|
+
$genDay = $gen.ToString('yyyy-MM-dd')
|
|
86
|
+
# finished = generated on a later day (the usual 00:05 pass), or late the same night
|
|
87
|
+
return (($genDay -gt $targetDay) -or (($genDay -eq $targetDay) -and ($gen.Hour -ge 23)))
|
|
88
|
+
} catch { return $true }
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
# Skipped when arguments are passed, so `daily_task.ps1 --help` still works as a
|
|
92
|
+
# smoke test with no side effects.
|
|
93
|
+
if ($rest.Count -eq 0) {
|
|
94
|
+
$dayDir = Join-Path $root ('output\biz-daily\' + $targetDay)
|
|
95
|
+
$done = if ($mode -eq 'noon') { Test-Path (Join-Path $dayDir '.noon_done') } else { Test-DayComplete $dayDir $targetDay }
|
|
96
|
+
if ($done) {
|
|
97
|
+
$note = '===== {0} [{1}] {2} already complete - skip =====' -f `
|
|
98
|
+
(Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $mode, $targetDay
|
|
99
|
+
Add-Content -Path (Join-Path $logDir 'daily-task.log') -Value $note -Encoding UTF8
|
|
100
|
+
exit 0
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
# Start-Process OVERWRITES its redirect targets, so a same-day rerun would erase the log
|
|
105
|
+
# of the run that just failed - exactly the one worth keeping. Rotate any existing pair to
|
|
106
|
+
# daily-<date>.run<N>[.err].log first, so every attempt stays on disk. N is the first free
|
|
107
|
+
# index, which also works when several reruns happen in one day.
|
|
108
|
+
if (Test-Path $out) {
|
|
109
|
+
$n = 1
|
|
110
|
+
while (Test-Path (Join-Path $logDir ('daily-{0}{1}.run{2}.log' -f $stamp, $suffix, $n))) { $n++ }
|
|
111
|
+
Move-Item -Path $out -Destination (Join-Path $logDir ('daily-{0}{1}.run{2}.log' -f $stamp, $suffix, $n))
|
|
112
|
+
if (Test-Path $err) {
|
|
113
|
+
Move-Item -Path $err -Destination (Join-Path $logDir ('daily-{0}{1}.run{2}.err.log' -f $stamp, $suffix, $n))
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
# Start-Process writes the raw bytes through: no PowerShell error-record wrapping, and
|
|
118
|
+
# no shell quoting to get wrong.
|
|
119
|
+
$proc = Start-Process -FilePath $python -ArgumentList $allArgs -NoNewWindow -Wait -PassThru `
|
|
120
|
+
-RedirectStandardOutput $out -RedirectStandardError $err
|
|
121
|
+
|
|
122
|
+
if ($proc.ExitCode -eq 0 -and $rest.Count -eq 0) {
|
|
123
|
+
# Marker for the noon pass only; the day pass is judged by generated_at above.
|
|
124
|
+
if ($mode -eq 'noon') {
|
|
125
|
+
$dayDir = Join-Path $root ('output\biz-daily\' + $targetDay)
|
|
126
|
+
if (Test-Path $dayDir) {
|
|
127
|
+
Set-Content -Path (Join-Path $dayDir '.noon_done') -Value (Get-Date -Format 'yyyy-MM-dd HH:mm:ss') -Encoding UTF8
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
exit $proc.ExitCode
|