weflow-cli 1.17.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +3 -0
- package/bin/weflow-cli.ts +167 -0
- package/dist/bin/weflow-cli.js +199 -0
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/src/services/diagnosticReport.d.ts +6 -0
- package/dist/src/services/diagnosticReport.d.ts.map +1 -1
- package/dist/src/services/diagnosticReport.js +16 -0
- package/dist/src/services/diagnosticReport.js.map +1 -1
- package/dist/src/services/wechatCompat.d.ts +29 -0
- package/dist/src/services/wechatCompat.d.ts.map +1 -0
- package/dist/src/services/wechatCompat.js +65 -0
- package/dist/src/services/wechatCompat.js.map +1 -0
- package/docs/DATA_CONTRACT.md +25 -25
- package/docs/DECISIONS.md +26 -0
- package/docs/FUNCTIONAL-IMPROVEMENT-SPEC.md +365 -365
- package/docs/HEALTH-CHECK.md +104 -104
- package/docs/JEV/345/272/224/347/224/250/346/226/275/345/267/245/346/226/207/346/241/243.md +310 -310
- package/docs/NEXT_ACTIONS.md +236 -236
- package/docs/NPM-PUBLISH.md +130 -130
- package/docs/NT_DATABASES.md +47 -0
- package/docs/POSITIONING.md +68 -68
- package/docs/PROJECT_STATE.md +2 -1
- package/docs/RELEASING.md +4 -0
- package/package.json +1 -1
- package/requirements-3x.txt +4 -4
- package/requirements-voice.txt +15 -15
- package/resources/js/graph3d/NOTICE.txt +35 -35
- package/resources/js/graph3d/d3-binarytree.min.js +2 -2
- package/resources/js/graph3d/d3-dispatch.min.js +2 -2
- package/resources/js/graph3d/d3-force-3d.min.js +2 -2
- package/resources/js/graph3d/d3-octree.min.js +2 -2
- package/resources/js/graph3d/d3-quadtree.min.js +2 -2
- package/resources/js/graph3d/d3-timer.min.js +2 -2
- package/resources/js/graph3d/three.min.js +6 -6
- package/resources/panel/quick-menu.cjs +197 -197
- package/scripts/_utils.py +1338 -1338
- package/scripts/annual_report.py +3 -1
- package/scripts/article_notes.py +423 -423
- package/scripts/contact_schema.py +339 -339
- package/scripts/create_reading_notes.py +11 -11
- package/scripts/daily_task.ps1 +213 -213
- package/scripts/extract_todos.py +3 -1
- package/scripts/fav_notes.py +321 -321
- package/scripts/health_check.py +473 -473
- package/scripts/imported_chats.py +68 -68
- package/scripts/install_tasks.ps1 +105 -105
- package/scripts/leads.py +179 -179
- package/scripts/mcp_bridge.py +3 -1
- package/scripts/media_index.py +150 -0
- package/scripts/network.py +206 -206
- package/scripts/nt_common.py +32 -0
- package/scripts/nt_decrypt.py +19 -15
- package/scripts/opinion_timeline.py +138 -138
- package/scripts/replay.py +803 -803
- package/scripts/run_hidden.vbs +24 -24
- package/scripts/semantic_search.py +5 -1
- package/scripts/user_notes.py +315 -315
- package/scripts/voice_transcribe.py +139 -139
- package/scripts/watch_issues.py +212 -212
- package/scripts/watch_mail.py +131 -131
- package/scripts/wechat_avatar.py +185 -0
- package/scripts/wechat_ledger.py +190 -0
- package/src/services/cursorProbe.ts +222 -222
- package/src/services/diagnosticReport.ts +22 -0
- package/src/services/qchatImport.ts +327 -327
- package/src/services/setupVenv.ts +238 -238
- package/src/services/syncService.ts +363 -363
- package/src/services/syncState.ts +275 -275
- package/src/services/wechatCompat.ts +67 -0
- package/src/services/wechatMedia.ts +166 -166
package/scripts/fav_notes.py
CHANGED
|
@@ -1,321 +1,321 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""
|
|
3
|
-
收藏知识卡 — 把微信收藏里的文章变成知识卡,喂给 wiki compile。
|
|
4
|
-
|
|
5
|
-
用法:
|
|
6
|
-
python scripts/fav_notes.py --dry-run --json # 看代价(只读本地;有哪些要联网抓也报出来)
|
|
7
|
-
python scripts/fav_notes.py --limit 10 --yes # 先试水 10 条
|
|
8
|
-
python scripts/compile_wiki.py --source output/fav-notes
|
|
9
|
-
|
|
10
|
-
## 为什么收藏值得单独一条来源
|
|
11
|
-
|
|
12
|
-
日报那条线吃的是"你今天读了什么",而**收藏是你明确留下的**——两者不是一回事:
|
|
13
|
-
收藏里常有日报窗口之外的文章,而且"我主动存过"本身就是兴趣的最强信号。
|
|
14
|
-
(另外这条线自带一个好处:收藏的文章多半日报已经抓过,`fetch_article_cached` 会命中缓存,
|
|
15
|
-
所以联网这一步常常是零成本。)
|
|
16
|
-
|
|
17
|
-
## 与另外两条线的关系
|
|
18
|
-
|
|
19
|
-
同一形状、同一消费者(`wiki_compile` 的 `--source`):文章线 `article_notes.py`、
|
|
20
|
-
对话线 `chat_notes.py`、收藏线就是本脚本。共用的件在 `_utils.py`。
|
|
21
|
-
|
|
22
|
-
**一处不同,必须写清**:文章的卡里"摘要"是**照抄**原笔记的,而收藏记录里**没有摘要**
|
|
23
|
-
(只有标题/来源/链接)——所以这条线的摘要**是模型写的**。卡片里会标出来
|
|
24
|
-
(frontmatter 的 `summary_by: model`),免得它看起来像抄来的原文。
|
|
25
|
-
"""
|
|
26
|
-
import argparse
|
|
27
|
-
import json
|
|
28
|
-
import os
|
|
29
|
-
import sys
|
|
30
|
-
from datetime import datetime, timezone, timedelta
|
|
31
|
-
from pathlib import Path
|
|
32
|
-
|
|
33
|
-
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
34
|
-
from _utils import (SEPARATOR, call_deepseek, decrypt_lock, get_api_key, load_config, # noqa: E402
|
|
35
|
-
note_path, plain, strip_wx_ads, write_with_frontmatter)
|
|
36
|
-
|
|
37
|
-
TZ = timezone(timedelta(hours=8))
|
|
38
|
-
OUTPUT_ROOT = 'output/fav-notes'
|
|
39
|
-
DEFAULT_LIMIT = 10
|
|
40
|
-
MAX_ARTICLE_CHARS = 3000
|
|
41
|
-
MIN_ARTICLE_CHARS = 200
|
|
42
|
-
LINK_CHARS = 6 * 1024 * 1024 # 单篇正文上限(缓存来的 markdown 一般远小于它)
|
|
43
|
-
|
|
44
|
-
FAV_PROMPT = """你是知识库的**编译者,不是作者**。下面是一篇用户**收藏**的公众号文章。
|
|
45
|
-
|
|
46
|
-
请只依据它,返回一个 JSON 对象(不要解释、不要代码块围栏):
|
|
47
|
-
{{"summary": "这篇文章在说什么(2-3 句,只写文章里有的)",
|
|
48
|
-
"concepts": [{{"name": "概念名(2-12 字,不要带书名号引号方括号)", "desc": "这篇文章关于它说了什么(一句话,具体)"}}]}}
|
|
49
|
-
|
|
50
|
-
要求:
|
|
51
|
-
1. **只写文章里有的**。没讲到的不许补;拿不准的宁可不写。
|
|
52
|
-
2. `concepts` 2-6 条、按重要度排序;**概念要具体**("MCP 协议"可以,"技术"不行),
|
|
53
|
-
也不是分类词("AI"、"新闻"这种主题标签不算概念)。
|
|
54
|
-
3. 每条 desc 是"这篇文章关于它的说法",不是百科定义。
|
|
55
|
-
4. 全部中文,不要 markdown 语法(不要 #、*、-)。
|
|
56
|
-
|
|
57
|
-
文章:{title}(来源:{source})
|
|
58
|
-
|
|
59
|
-
正文:
|
|
60
|
-
{body}
|
|
61
|
-
"""
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def read_favorites(limit: int):
|
|
65
|
-
"""读收藏列表(本地、只读)。返回 (条目列表, 错误原因)。"""
|
|
66
|
-
import nt_decrypt
|
|
67
|
-
config = load_config()
|
|
68
|
-
fav_db = config.get('favDbPath', '')
|
|
69
|
-
if not fav_db:
|
|
70
|
-
return [], '配置里没有 favDbPath(收藏库还没连上)'
|
|
71
|
-
got = nt_decrypt.get_favorites(fav_db, decrypt_lock(config.get('favKey', '')), limit=limit)
|
|
72
|
-
if isinstance(got, dict) and got.get('error'):
|
|
73
|
-
return [], str(got['error'])
|
|
74
|
-
items = got.get('favorites') if isinstance(got, dict) else got
|
|
75
|
-
return list(items or []), None
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
def local_text(item: dict) -> str:
|
|
79
|
-
"""收藏**自带**的正文(笔记/文本类有,文章类没有)。**不碰网络。**
|
|
80
|
-
|
|
81
|
-
与 `favorite_text` 分开,是为了让 `--dry-run` 能说清"哪几条要联网抓"而**不去抓**:
|
|
82
|
-
预览是只读的(它的说明里写着"只读本地"),自己去打网络就名不副实了——我第一版
|
|
83
|
-
就是这么写的,跑 dry-run 时看见它在抓文章才发现。
|
|
84
|
-
"""
|
|
85
|
-
for key in ('desc', 'content', 'summary'):
|
|
86
|
-
text = strip_wx_ads(str(item.get(key) or '')).strip()
|
|
87
|
-
if len(text) >= MIN_ARTICLE_CHARS:
|
|
88
|
-
return text
|
|
89
|
-
return ''
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
def classify_source(item: dict) -> str:
|
|
93
|
-
"""这条收藏的正文从哪来(**只看本地信息**):local / fetch / none。"""
|
|
94
|
-
if local_text(item):
|
|
95
|
-
return 'local'
|
|
96
|
-
return 'fetch' if str(item.get('link') or '').strip() else 'none'
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def favorite_text(item: dict, fetch=None):
|
|
100
|
-
"""一条收藏的正文:本地文本优先,没有(或太短)就抓链接。返回 (正文, 来源说明)。
|
|
101
|
-
|
|
102
|
-
收藏分两种:笔记/文本类**自带内容**(`desc`/`content`),文章类**基本**只有 `link`——
|
|
103
|
-
"基本"是量出来的:本机 1,234 篇文章里有 **7 条**的网页描述(`weburlitem/pagedesc`)超过
|
|
104
|
-
`MIN_ARTICLE_CHARS`,会被当成"自带正文"(那是**摘要不是正文**)。门槛 200 字挡住了短噪声
|
|
105
|
-
(例如本机 20 条 `type=1` 的 `desc` 只有 49 字、且是路径式字符串 ⇒ 全部落 none)。
|
|
106
|
-
抓取复用日报那条线的 `fetch_article_cached`——不重写第二份(那套微信 UA 绕 WAF、
|
|
107
|
-
gzip、重试是踩出来的),而且**带缓存**:日报抓过的文章这里零网络。
|
|
108
|
-
|
|
109
|
-
`fetch` 是**可注入的口子**:默认走 `biz_daily.fetch_article_cached`,测试传桩进来——
|
|
110
|
-
否则单元测试会真去联网(我第一版就是这么写的,跑测试时才发现)。
|
|
111
|
-
"""
|
|
112
|
-
for key in ('desc', 'content', 'summary'):
|
|
113
|
-
text = strip_wx_ads(str(item.get(key) or '')).strip()
|
|
114
|
-
if len(text) >= MIN_ARTICLE_CHARS:
|
|
115
|
-
return text, 'local'
|
|
116
|
-
link = str(item.get('link') or '').strip()
|
|
117
|
-
if not link:
|
|
118
|
-
return '', 'no-text'
|
|
119
|
-
if fetch is None:
|
|
120
|
-
try:
|
|
121
|
-
import biz_daily
|
|
122
|
-
fetch = biz_daily.fetch_article_cached
|
|
123
|
-
except Exception as error:
|
|
124
|
-
return '', 'fetch-failed: %s' % type(error).__name__
|
|
125
|
-
try:
|
|
126
|
-
got = fetch(link)
|
|
127
|
-
except Exception as error:
|
|
128
|
-
return '', 'fetch-failed: %s' % type(error).__name__
|
|
129
|
-
if not got:
|
|
130
|
-
return '', 'fetch-failed'
|
|
131
|
-
text, cached = got
|
|
132
|
-
if not text:
|
|
133
|
-
return '', 'fetch-failed'
|
|
134
|
-
return strip_wx_ads(str(text))[:LINK_CHARS], ('cache' if cached else 'network')
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def parse_note(raw: str):
|
|
138
|
-
"""解出 `{summary, concepts}`。解不出来返回 None(**不猜**)。"""
|
|
139
|
-
if not isinstance(raw, str) or not raw.strip():
|
|
140
|
-
return None
|
|
141
|
-
text = raw.strip()
|
|
142
|
-
start, end = text.find('{'), text.rfind('}')
|
|
143
|
-
if start < 0 or end <= start:
|
|
144
|
-
return None
|
|
145
|
-
try:
|
|
146
|
-
data = json.loads(text[start:end + 1])
|
|
147
|
-
except Exception:
|
|
148
|
-
return None
|
|
149
|
-
if not isinstance(data, dict) or not isinstance(data.get('concepts'), list):
|
|
150
|
-
return None
|
|
151
|
-
summary = plain(data.get('summary')).strip()
|
|
152
|
-
if not summary:
|
|
153
|
-
return None
|
|
154
|
-
concepts = []
|
|
155
|
-
for item in data['concepts']:
|
|
156
|
-
if not isinstance(item, dict):
|
|
157
|
-
continue
|
|
158
|
-
name = plain(item.get('name')).strip().strip('[]')
|
|
159
|
-
desc = plain(item.get('desc')).strip()
|
|
160
|
-
if name and desc:
|
|
161
|
-
concepts.append({'name': name, 'desc': desc})
|
|
162
|
-
if len(concepts) >= 6:
|
|
163
|
-
break
|
|
164
|
-
return {'summary': summary, 'concepts': concepts}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def build_card(item: dict, note: dict, body_len: int, saved: str, updated: str):
|
|
168
|
-
"""卡片 → (frontmatter, body)。摘要这次**是模型写的**,所以标出来。"""
|
|
169
|
-
title = str(item.get('title') or '无标题').strip()
|
|
170
|
-
source = str(item.get('source_name') or '').strip()
|
|
171
|
-
frontmatter = {
|
|
172
|
-
'title': title,
|
|
173
|
-
'type': 'fav-card',
|
|
174
|
-
'source': source or '收藏',
|
|
175
|
-
'topic': '收藏',
|
|
176
|
-
'tags': ['收藏'],
|
|
177
|
-
# **收藏时间**不是发布时间——别把它叫 published,那是另一回事
|
|
178
|
-
'saved': saved,
|
|
179
|
-
'url': str(item.get('link') or ''),
|
|
180
|
-
'summary_by': 'model', # 这条线的摘要不是抄的,标出来
|
|
181
|
-
'bodyChars': body_len,
|
|
182
|
-
'updated': updated,
|
|
183
|
-
}
|
|
184
|
-
parts = ['# %s\n' % title, '## AI 摘要\n', note['summary'] + '\n']
|
|
185
|
-
if note['concepts']:
|
|
186
|
-
parts.append('## 概念\n')
|
|
187
|
-
for concept in note['concepts']:
|
|
188
|
-
parts.append('- [[%s]] %s %s' % (concept['name'], SEPARATOR, concept['desc']))
|
|
189
|
-
parts.append('')
|
|
190
|
-
return frontmatter, '\n'.join(parts)
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
def already_carded(item: dict, out_dir: str) -> bool:
|
|
194
|
-
"""这条收藏已经有卡了吗(按标题落成的文件名)。"""
|
|
195
|
-
title = str(item.get('title') or '').strip()
|
|
196
|
-
return bool(title) and note_path(out_dir, title).exists()
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
def exit_code(written: int) -> int:
|
|
200
|
-
"""部分成功算成功——与另外两条线同一个约定。"""
|
|
201
|
-
return 0 if written else 1
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def main():
|
|
205
|
-
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
|
206
|
-
parser = argparse.ArgumentParser(description='收藏知识卡(把收藏变成喂给 wiki compile 的卡)')
|
|
207
|
-
parser.add_argument('--limit', type=int, default=DEFAULT_LIMIT,
|
|
208
|
-
help='最多处理几条收藏(默认 %d)' % DEFAULT_LIMIT)
|
|
209
|
-
parser.add_argument('--out', default=OUTPUT_ROOT, help='卡片输出目录')
|
|
210
|
-
parser.add_argument('--refresh', action='store_true', help='已有卡的也重做')
|
|
211
|
-
parser.add_argument('--dry-run', action='store_true', help='只报要发多少给模型,不调用')
|
|
212
|
-
parser.add_argument('--yes', action='store_true', help='确认调用云端模型')
|
|
213
|
-
parser.add_argument('--json', action='store_true')
|
|
214
|
-
args = parser.parse_args()
|
|
215
|
-
|
|
216
|
-
if args.limit < 1:
|
|
217
|
-
print(json.dumps({'success': False, 'error': '--limit 要 ≥ 1'}))
|
|
218
|
-
return 1
|
|
219
|
-
|
|
220
|
-
favorites, error = read_favorites(args.limit)
|
|
221
|
-
if error:
|
|
222
|
-
message = '读收藏失败:%s' % error
|
|
223
|
-
print(json.dumps({'success': False, 'error': message}, ensure_ascii=False)
|
|
224
|
-
if args.json else message)
|
|
225
|
-
return 1
|
|
226
|
-
|
|
227
|
-
runnable, thin, existing = [], [], []
|
|
228
|
-
for item in favorites:
|
|
229
|
-
if args.refresh or not already_carded(item, args.out):
|
|
230
|
-
runnable.append(item)
|
|
231
|
-
else:
|
|
232
|
-
existing.append(item)
|
|
233
|
-
|
|
234
|
-
if args.dry_run:
|
|
235
|
-
# **只看本地信息**:预览是只读的,不替用户去打网络(抓取留给真跑那一步)
|
|
236
|
-
counts = {'local': 0, 'fetch': 0, 'none': 0}
|
|
237
|
-
for item in runnable:
|
|
238
|
-
counts[classify_source(item)] += 1
|
|
239
|
-
preview = {
|
|
240
|
-
'success': True, 'dryRun': True, 'action': 'fav-notes',
|
|
241
|
-
'favorites': len(favorites), 'calls': len(runnable),
|
|
242
|
-
'alreadyCarded': len(existing), 'noText': counts['none'],
|
|
243
|
-
'textSource': {'local': counts['local'], 'needsFetch': counts['fetch']},
|
|
244
|
-
'model': 'DeepSeek(生成)', 'readsLocalData': True, 'fetchArticleUrls': False,
|
|
245
|
-
'invokesAI': False,
|
|
246
|
-
'note': '每条一次调用;正文优先用收藏自带的文本,文章类走日报那条线的抓取(带缓存)'
|
|
247
|
-
'——抓取发生在真跑时,预览不联网',
|
|
248
|
-
}
|
|
249
|
-
if args.json:
|
|
250
|
-
print(json.dumps(preview, ensure_ascii=False, indent=2))
|
|
251
|
-
else:
|
|
252
|
-
print('收藏 %d 条,会调用模型 %d 次(已有卡 %d 条跳过)'
|
|
253
|
-
% (len(favorites), len(runnable), len(existing)))
|
|
254
|
-
print('正文:自带 %d 条、要联网抓 %d 条、取不到 %d 条(预览不联网,抓取在 --yes 之后)'
|
|
255
|
-
% (counts['local'], counts['fetch'], counts['none']))
|
|
256
|
-
return 0
|
|
257
|
-
|
|
258
|
-
if not runnable:
|
|
259
|
-
message = '没有要处理的收藏(%d 条都已有卡)' % len(favorites)
|
|
260
|
-
print(json.dumps({'success': False, 'error': message}, ensure_ascii=False)
|
|
261
|
-
if args.json else message, file=sys.stdout if args.json else sys.stderr)
|
|
262
|
-
return 1
|
|
263
|
-
|
|
264
|
-
if not args.yes:
|
|
265
|
-
print('会把 %d 条收藏的正文发给 DeepSeek(每条一次调用)。加 --yes 确认。' % len(runnable),
|
|
266
|
-
file=sys.stderr)
|
|
267
|
-
return 1
|
|
268
|
-
|
|
269
|
-
config = load_config()
|
|
270
|
-
api_key = get_api_key(config)
|
|
271
|
-
if not api_key:
|
|
272
|
-
print('没有配置 deepseekApiKey(weflow-cli config set deepseekApiKey "...")', file=sys.stderr)
|
|
273
|
-
return 1
|
|
274
|
-
|
|
275
|
-
updated = datetime.now(TZ).strftime('%Y-%m-%d %H:%M')
|
|
276
|
-
written, failed = [], []
|
|
277
|
-
for item in runnable:
|
|
278
|
-
text, how = favorite_text(item)
|
|
279
|
-
title = str(item.get('title') or '无标题').strip()
|
|
280
|
-
if not text:
|
|
281
|
-
failed.append({'title': title[:30], 'reason': '取不到正文(%s)' % how})
|
|
282
|
-
continue
|
|
283
|
-
saved = ''
|
|
284
|
-
if item.get('update_time'):
|
|
285
|
-
saved = datetime.fromtimestamp(int(item['update_time']), tz=TZ).strftime('%Y-%m-%d')
|
|
286
|
-
prompt = FAV_PROMPT.format(title=title, source=item.get('source_name') or '未知',
|
|
287
|
-
body=text[:MAX_ARTICLE_CHARS])
|
|
288
|
-
try:
|
|
289
|
-
raw = call_deepseek(prompt, api_key, max_tokens=2000, timeout=120)
|
|
290
|
-
except Exception as error:
|
|
291
|
-
failed.append({'title': title[:30], 'reason': '调用失败:%s' % error})
|
|
292
|
-
continue
|
|
293
|
-
note = parse_note(raw)
|
|
294
|
-
if note is None:
|
|
295
|
-
failed.append({'title': title[:30], 'reason': '模型没返回可用 JSON',
|
|
296
|
-
'rawTail': str(raw)[-200:] if raw else '(空)'})
|
|
297
|
-
continue
|
|
298
|
-
target = note_path(args.out, title)
|
|
299
|
-
target.parent.mkdir(parents=True, exist_ok=True)
|
|
300
|
-
frontmatter, body = build_card(item, note, len(text), saved, updated)
|
|
301
|
-
write_with_frontmatter(str(target), frontmatter, body)
|
|
302
|
-
written.append({'title': title[:30], 'file': str(target),
|
|
303
|
-
'concepts': len(note['concepts']), 'textFrom': how})
|
|
304
|
-
|
|
305
|
-
result = {'success': True, 'action': 'fav-notes', 'model': 'DeepSeek(生成)',
|
|
306
|
-
'sendsNothing': True, 'written': written, 'failed': failed,
|
|
307
|
-
'alreadyCarded': len(existing), 'favorites': len(favorites), 'calls': len(runnable)}
|
|
308
|
-
if args.json:
|
|
309
|
-
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
310
|
-
else:
|
|
311
|
-
for item in written:
|
|
312
|
-
print('✓ %s(%d 个概念,正文来自 %s)' % (item['title'], item['concepts'], item['textFrom']))
|
|
313
|
-
for item in failed:
|
|
314
|
-
print('✗ %s:%s' % (item['title'], item['reason']), file=sys.stderr)
|
|
315
|
-
print('\n共 %d 张卡;接着跑 python scripts/compile_wiki.py --source %s'
|
|
316
|
-
% (len(written), args.out))
|
|
317
|
-
return exit_code(len(written))
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
if __name__ == '__main__':
|
|
321
|
-
sys.exit(main())
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
收藏知识卡 — 把微信收藏里的文章变成知识卡,喂给 wiki compile。
|
|
4
|
+
|
|
5
|
+
用法:
|
|
6
|
+
python scripts/fav_notes.py --dry-run --json # 看代价(只读本地;有哪些要联网抓也报出来)
|
|
7
|
+
python scripts/fav_notes.py --limit 10 --yes # 先试水 10 条
|
|
8
|
+
python scripts/compile_wiki.py --source output/fav-notes
|
|
9
|
+
|
|
10
|
+
## 为什么收藏值得单独一条来源
|
|
11
|
+
|
|
12
|
+
日报那条线吃的是"你今天读了什么",而**收藏是你明确留下的**——两者不是一回事:
|
|
13
|
+
收藏里常有日报窗口之外的文章,而且"我主动存过"本身就是兴趣的最强信号。
|
|
14
|
+
(另外这条线自带一个好处:收藏的文章多半日报已经抓过,`fetch_article_cached` 会命中缓存,
|
|
15
|
+
所以联网这一步常常是零成本。)
|
|
16
|
+
|
|
17
|
+
## 与另外两条线的关系
|
|
18
|
+
|
|
19
|
+
同一形状、同一消费者(`wiki_compile` 的 `--source`):文章线 `article_notes.py`、
|
|
20
|
+
对话线 `chat_notes.py`、收藏线就是本脚本。共用的件在 `_utils.py`。
|
|
21
|
+
|
|
22
|
+
**一处不同,必须写清**:文章的卡里"摘要"是**照抄**原笔记的,而收藏记录里**没有摘要**
|
|
23
|
+
(只有标题/来源/链接)——所以这条线的摘要**是模型写的**。卡片里会标出来
|
|
24
|
+
(frontmatter 的 `summary_by: model`),免得它看起来像抄来的原文。
|
|
25
|
+
"""
|
|
26
|
+
import argparse
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import sys
|
|
30
|
+
from datetime import datetime, timezone, timedelta
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
|
|
33
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
34
|
+
from _utils import (SEPARATOR, call_deepseek, decrypt_lock, get_api_key, load_config, # noqa: E402
|
|
35
|
+
note_path, plain, strip_wx_ads, write_with_frontmatter)
|
|
36
|
+
|
|
37
|
+
TZ = timezone(timedelta(hours=8))
|
|
38
|
+
OUTPUT_ROOT = 'output/fav-notes'
|
|
39
|
+
DEFAULT_LIMIT = 10
|
|
40
|
+
MAX_ARTICLE_CHARS = 3000
|
|
41
|
+
MIN_ARTICLE_CHARS = 200
|
|
42
|
+
LINK_CHARS = 6 * 1024 * 1024 # 单篇正文上限(缓存来的 markdown 一般远小于它)
|
|
43
|
+
|
|
44
|
+
FAV_PROMPT = """你是知识库的**编译者,不是作者**。下面是一篇用户**收藏**的公众号文章。
|
|
45
|
+
|
|
46
|
+
请只依据它,返回一个 JSON 对象(不要解释、不要代码块围栏):
|
|
47
|
+
{{"summary": "这篇文章在说什么(2-3 句,只写文章里有的)",
|
|
48
|
+
"concepts": [{{"name": "概念名(2-12 字,不要带书名号引号方括号)", "desc": "这篇文章关于它说了什么(一句话,具体)"}}]}}
|
|
49
|
+
|
|
50
|
+
要求:
|
|
51
|
+
1. **只写文章里有的**。没讲到的不许补;拿不准的宁可不写。
|
|
52
|
+
2. `concepts` 2-6 条、按重要度排序;**概念要具体**("MCP 协议"可以,"技术"不行),
|
|
53
|
+
也不是分类词("AI"、"新闻"这种主题标签不算概念)。
|
|
54
|
+
3. 每条 desc 是"这篇文章关于它的说法",不是百科定义。
|
|
55
|
+
4. 全部中文,不要 markdown 语法(不要 #、*、-)。
|
|
56
|
+
|
|
57
|
+
文章:{title}(来源:{source})
|
|
58
|
+
|
|
59
|
+
正文:
|
|
60
|
+
{body}
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def read_favorites(limit: int):
|
|
65
|
+
"""读收藏列表(本地、只读)。返回 (条目列表, 错误原因)。"""
|
|
66
|
+
import nt_decrypt
|
|
67
|
+
config = load_config()
|
|
68
|
+
fav_db = config.get('favDbPath', '')
|
|
69
|
+
if not fav_db:
|
|
70
|
+
return [], '配置里没有 favDbPath(收藏库还没连上)'
|
|
71
|
+
got = nt_decrypt.get_favorites(fav_db, decrypt_lock(config.get('favKey', '')), limit=limit)
|
|
72
|
+
if isinstance(got, dict) and got.get('error'):
|
|
73
|
+
return [], str(got['error'])
|
|
74
|
+
items = got.get('favorites') if isinstance(got, dict) else got
|
|
75
|
+
return list(items or []), None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def local_text(item: dict) -> str:
|
|
79
|
+
"""收藏**自带**的正文(笔记/文本类有,文章类没有)。**不碰网络。**
|
|
80
|
+
|
|
81
|
+
与 `favorite_text` 分开,是为了让 `--dry-run` 能说清"哪几条要联网抓"而**不去抓**:
|
|
82
|
+
预览是只读的(它的说明里写着"只读本地"),自己去打网络就名不副实了——我第一版
|
|
83
|
+
就是这么写的,跑 dry-run 时看见它在抓文章才发现。
|
|
84
|
+
"""
|
|
85
|
+
for key in ('desc', 'content', 'summary'):
|
|
86
|
+
text = strip_wx_ads(str(item.get(key) or '')).strip()
|
|
87
|
+
if len(text) >= MIN_ARTICLE_CHARS:
|
|
88
|
+
return text
|
|
89
|
+
return ''
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def classify_source(item: dict) -> str:
|
|
93
|
+
"""这条收藏的正文从哪来(**只看本地信息**):local / fetch / none。"""
|
|
94
|
+
if local_text(item):
|
|
95
|
+
return 'local'
|
|
96
|
+
return 'fetch' if str(item.get('link') or '').strip() else 'none'
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def favorite_text(item: dict, fetch=None):
|
|
100
|
+
"""一条收藏的正文:本地文本优先,没有(或太短)就抓链接。返回 (正文, 来源说明)。
|
|
101
|
+
|
|
102
|
+
收藏分两种:笔记/文本类**自带内容**(`desc`/`content`),文章类**基本**只有 `link`——
|
|
103
|
+
"基本"是量出来的:本机 1,234 篇文章里有 **7 条**的网页描述(`weburlitem/pagedesc`)超过
|
|
104
|
+
`MIN_ARTICLE_CHARS`,会被当成"自带正文"(那是**摘要不是正文**)。门槛 200 字挡住了短噪声
|
|
105
|
+
(例如本机 20 条 `type=1` 的 `desc` 只有 49 字、且是路径式字符串 ⇒ 全部落 none)。
|
|
106
|
+
抓取复用日报那条线的 `fetch_article_cached`——不重写第二份(那套微信 UA 绕 WAF、
|
|
107
|
+
gzip、重试是踩出来的),而且**带缓存**:日报抓过的文章这里零网络。
|
|
108
|
+
|
|
109
|
+
`fetch` 是**可注入的口子**:默认走 `biz_daily.fetch_article_cached`,测试传桩进来——
|
|
110
|
+
否则单元测试会真去联网(我第一版就是这么写的,跑测试时才发现)。
|
|
111
|
+
"""
|
|
112
|
+
for key in ('desc', 'content', 'summary'):
|
|
113
|
+
text = strip_wx_ads(str(item.get(key) or '')).strip()
|
|
114
|
+
if len(text) >= MIN_ARTICLE_CHARS:
|
|
115
|
+
return text, 'local'
|
|
116
|
+
link = str(item.get('link') or '').strip()
|
|
117
|
+
if not link:
|
|
118
|
+
return '', 'no-text'
|
|
119
|
+
if fetch is None:
|
|
120
|
+
try:
|
|
121
|
+
import biz_daily
|
|
122
|
+
fetch = biz_daily.fetch_article_cached
|
|
123
|
+
except Exception as error:
|
|
124
|
+
return '', 'fetch-failed: %s' % type(error).__name__
|
|
125
|
+
try:
|
|
126
|
+
got = fetch(link)
|
|
127
|
+
except Exception as error:
|
|
128
|
+
return '', 'fetch-failed: %s' % type(error).__name__
|
|
129
|
+
if not got:
|
|
130
|
+
return '', 'fetch-failed'
|
|
131
|
+
text, cached = got
|
|
132
|
+
if not text:
|
|
133
|
+
return '', 'fetch-failed'
|
|
134
|
+
return strip_wx_ads(str(text))[:LINK_CHARS], ('cache' if cached else 'network')
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def parse_note(raw: str):
|
|
138
|
+
"""解出 `{summary, concepts}`。解不出来返回 None(**不猜**)。"""
|
|
139
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
140
|
+
return None
|
|
141
|
+
text = raw.strip()
|
|
142
|
+
start, end = text.find('{'), text.rfind('}')
|
|
143
|
+
if start < 0 or end <= start:
|
|
144
|
+
return None
|
|
145
|
+
try:
|
|
146
|
+
data = json.loads(text[start:end + 1])
|
|
147
|
+
except Exception:
|
|
148
|
+
return None
|
|
149
|
+
if not isinstance(data, dict) or not isinstance(data.get('concepts'), list):
|
|
150
|
+
return None
|
|
151
|
+
summary = plain(data.get('summary')).strip()
|
|
152
|
+
if not summary:
|
|
153
|
+
return None
|
|
154
|
+
concepts = []
|
|
155
|
+
for item in data['concepts']:
|
|
156
|
+
if not isinstance(item, dict):
|
|
157
|
+
continue
|
|
158
|
+
name = plain(item.get('name')).strip().strip('[]')
|
|
159
|
+
desc = plain(item.get('desc')).strip()
|
|
160
|
+
if name and desc:
|
|
161
|
+
concepts.append({'name': name, 'desc': desc})
|
|
162
|
+
if len(concepts) >= 6:
|
|
163
|
+
break
|
|
164
|
+
return {'summary': summary, 'concepts': concepts}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def build_card(item: dict, note: dict, body_len: int, saved: str, updated: str):
|
|
168
|
+
"""卡片 → (frontmatter, body)。摘要这次**是模型写的**,所以标出来。"""
|
|
169
|
+
title = str(item.get('title') or '无标题').strip()
|
|
170
|
+
source = str(item.get('source_name') or '').strip()
|
|
171
|
+
frontmatter = {
|
|
172
|
+
'title': title,
|
|
173
|
+
'type': 'fav-card',
|
|
174
|
+
'source': source or '收藏',
|
|
175
|
+
'topic': '收藏',
|
|
176
|
+
'tags': ['收藏'],
|
|
177
|
+
# **收藏时间**不是发布时间——别把它叫 published,那是另一回事
|
|
178
|
+
'saved': saved,
|
|
179
|
+
'url': str(item.get('link') or ''),
|
|
180
|
+
'summary_by': 'model', # 这条线的摘要不是抄的,标出来
|
|
181
|
+
'bodyChars': body_len,
|
|
182
|
+
'updated': updated,
|
|
183
|
+
}
|
|
184
|
+
parts = ['# %s\n' % title, '## AI 摘要\n', note['summary'] + '\n']
|
|
185
|
+
if note['concepts']:
|
|
186
|
+
parts.append('## 概念\n')
|
|
187
|
+
for concept in note['concepts']:
|
|
188
|
+
parts.append('- [[%s]] %s %s' % (concept['name'], SEPARATOR, concept['desc']))
|
|
189
|
+
parts.append('')
|
|
190
|
+
return frontmatter, '\n'.join(parts)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def already_carded(item: dict, out_dir: str) -> bool:
|
|
194
|
+
"""这条收藏已经有卡了吗(按标题落成的文件名)。"""
|
|
195
|
+
title = str(item.get('title') or '').strip()
|
|
196
|
+
return bool(title) and note_path(out_dir, title).exists()
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def exit_code(written: int) -> int:
|
|
200
|
+
"""部分成功算成功——与另外两条线同一个约定。"""
|
|
201
|
+
return 0 if written else 1
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def main():
|
|
205
|
+
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
|
206
|
+
parser = argparse.ArgumentParser(description='收藏知识卡(把收藏变成喂给 wiki compile 的卡)')
|
|
207
|
+
parser.add_argument('--limit', type=int, default=DEFAULT_LIMIT,
|
|
208
|
+
help='最多处理几条收藏(默认 %d)' % DEFAULT_LIMIT)
|
|
209
|
+
parser.add_argument('--out', default=OUTPUT_ROOT, help='卡片输出目录')
|
|
210
|
+
parser.add_argument('--refresh', action='store_true', help='已有卡的也重做')
|
|
211
|
+
parser.add_argument('--dry-run', action='store_true', help='只报要发多少给模型,不调用')
|
|
212
|
+
parser.add_argument('--yes', action='store_true', help='确认调用云端模型')
|
|
213
|
+
parser.add_argument('--json', action='store_true')
|
|
214
|
+
args = parser.parse_args()
|
|
215
|
+
|
|
216
|
+
if args.limit < 1:
|
|
217
|
+
print(json.dumps({'success': False, 'error': '--limit 要 ≥ 1'}))
|
|
218
|
+
return 1
|
|
219
|
+
|
|
220
|
+
favorites, error = read_favorites(args.limit)
|
|
221
|
+
if error:
|
|
222
|
+
message = '读收藏失败:%s' % error
|
|
223
|
+
print(json.dumps({'success': False, 'error': message}, ensure_ascii=False)
|
|
224
|
+
if args.json else message)
|
|
225
|
+
return 1
|
|
226
|
+
|
|
227
|
+
runnable, thin, existing = [], [], []
|
|
228
|
+
for item in favorites:
|
|
229
|
+
if args.refresh or not already_carded(item, args.out):
|
|
230
|
+
runnable.append(item)
|
|
231
|
+
else:
|
|
232
|
+
existing.append(item)
|
|
233
|
+
|
|
234
|
+
if args.dry_run:
|
|
235
|
+
# **只看本地信息**:预览是只读的,不替用户去打网络(抓取留给真跑那一步)
|
|
236
|
+
counts = {'local': 0, 'fetch': 0, 'none': 0}
|
|
237
|
+
for item in runnable:
|
|
238
|
+
counts[classify_source(item)] += 1
|
|
239
|
+
preview = {
|
|
240
|
+
'success': True, 'dryRun': True, 'action': 'fav-notes',
|
|
241
|
+
'favorites': len(favorites), 'calls': len(runnable),
|
|
242
|
+
'alreadyCarded': len(existing), 'noText': counts['none'],
|
|
243
|
+
'textSource': {'local': counts['local'], 'needsFetch': counts['fetch']},
|
|
244
|
+
'model': 'DeepSeek(生成)', 'readsLocalData': True, 'fetchArticleUrls': False,
|
|
245
|
+
'invokesAI': False,
|
|
246
|
+
'note': '每条一次调用;正文优先用收藏自带的文本,文章类走日报那条线的抓取(带缓存)'
|
|
247
|
+
'——抓取发生在真跑时,预览不联网',
|
|
248
|
+
}
|
|
249
|
+
if args.json:
|
|
250
|
+
print(json.dumps(preview, ensure_ascii=False, indent=2))
|
|
251
|
+
else:
|
|
252
|
+
print('收藏 %d 条,会调用模型 %d 次(已有卡 %d 条跳过)'
|
|
253
|
+
% (len(favorites), len(runnable), len(existing)))
|
|
254
|
+
print('正文:自带 %d 条、要联网抓 %d 条、取不到 %d 条(预览不联网,抓取在 --yes 之后)'
|
|
255
|
+
% (counts['local'], counts['fetch'], counts['none']))
|
|
256
|
+
return 0
|
|
257
|
+
|
|
258
|
+
if not runnable:
|
|
259
|
+
message = '没有要处理的收藏(%d 条都已有卡)' % len(favorites)
|
|
260
|
+
print(json.dumps({'success': False, 'error': message}, ensure_ascii=False)
|
|
261
|
+
if args.json else message, file=sys.stdout if args.json else sys.stderr)
|
|
262
|
+
return 1
|
|
263
|
+
|
|
264
|
+
if not args.yes:
|
|
265
|
+
print('会把 %d 条收藏的正文发给 DeepSeek(每条一次调用)。加 --yes 确认。' % len(runnable),
|
|
266
|
+
file=sys.stderr)
|
|
267
|
+
return 1
|
|
268
|
+
|
|
269
|
+
config = load_config()
|
|
270
|
+
api_key = get_api_key(config)
|
|
271
|
+
if not api_key:
|
|
272
|
+
print('没有配置 deepseekApiKey(weflow-cli config set deepseekApiKey "...")', file=sys.stderr)
|
|
273
|
+
return 1
|
|
274
|
+
|
|
275
|
+
updated = datetime.now(TZ).strftime('%Y-%m-%d %H:%M')
|
|
276
|
+
written, failed = [], []
|
|
277
|
+
for item in runnable:
|
|
278
|
+
text, how = favorite_text(item)
|
|
279
|
+
title = str(item.get('title') or '无标题').strip()
|
|
280
|
+
if not text:
|
|
281
|
+
failed.append({'title': title[:30], 'reason': '取不到正文(%s)' % how})
|
|
282
|
+
continue
|
|
283
|
+
saved = ''
|
|
284
|
+
if item.get('update_time'):
|
|
285
|
+
saved = datetime.fromtimestamp(int(item['update_time']), tz=TZ).strftime('%Y-%m-%d')
|
|
286
|
+
prompt = FAV_PROMPT.format(title=title, source=item.get('source_name') or '未知',
|
|
287
|
+
body=text[:MAX_ARTICLE_CHARS])
|
|
288
|
+
try:
|
|
289
|
+
raw = call_deepseek(prompt, api_key, max_tokens=2000, timeout=120)
|
|
290
|
+
except Exception as error:
|
|
291
|
+
failed.append({'title': title[:30], 'reason': '调用失败:%s' % error})
|
|
292
|
+
continue
|
|
293
|
+
note = parse_note(raw)
|
|
294
|
+
if note is None:
|
|
295
|
+
failed.append({'title': title[:30], 'reason': '模型没返回可用 JSON',
|
|
296
|
+
'rawTail': str(raw)[-200:] if raw else '(空)'})
|
|
297
|
+
continue
|
|
298
|
+
target = note_path(args.out, title)
|
|
299
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
300
|
+
frontmatter, body = build_card(item, note, len(text), saved, updated)
|
|
301
|
+
write_with_frontmatter(str(target), frontmatter, body)
|
|
302
|
+
written.append({'title': title[:30], 'file': str(target),
|
|
303
|
+
'concepts': len(note['concepts']), 'textFrom': how})
|
|
304
|
+
|
|
305
|
+
result = {'success': True, 'action': 'fav-notes', 'model': 'DeepSeek(生成)',
|
|
306
|
+
'sendsNothing': True, 'written': written, 'failed': failed,
|
|
307
|
+
'alreadyCarded': len(existing), 'favorites': len(favorites), 'calls': len(runnable)}
|
|
308
|
+
if args.json:
|
|
309
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
310
|
+
else:
|
|
311
|
+
for item in written:
|
|
312
|
+
print('✓ %s(%d 个概念,正文来自 %s)' % (item['title'], item['concepts'], item['textFrom']))
|
|
313
|
+
for item in failed:
|
|
314
|
+
print('✗ %s:%s' % (item['title'], item['reason']), file=sys.stderr)
|
|
315
|
+
print('\n共 %d 张卡;接着跑 python scripts/compile_wiki.py --source %s'
|
|
316
|
+
% (len(written), args.out))
|
|
317
|
+
return exit_code(len(written))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
if __name__ == '__main__':
|
|
321
|
+
sys.exit(main())
|