weflow-cli 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +62 -26
- package/CHANGELOG.md +1051 -0
- package/OPERATIONS.md +99 -13
- package/README.en.md +14 -5
- package/README.md +23 -8
- package/bin/weflow-cli.ts +364 -40
- package/dist/bin/weflow-cli.js +381 -69
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/mcp-server/index.d.ts.map +1 -1
- package/dist/mcp-server/index.js +19 -1
- package/dist/mcp-server/index.js.map +1 -1
- package/dist/src/core/dbPathService.d.ts +12 -0
- package/dist/src/core/dbPathService.d.ts.map +1 -1
- package/dist/src/core/dbPathService.js +20 -3
- package/dist/src/core/dbPathService.js.map +1 -1
- package/dist/src/panel/server.d.ts.map +1 -1
- package/dist/src/panel/server.js +35 -0
- package/dist/src/panel/server.js.map +1 -1
- package/dist/src/services/assistantService.d.ts +19 -0
- package/dist/src/services/assistantService.d.ts.map +1 -1
- package/dist/src/services/assistantService.js +20 -0
- package/dist/src/services/assistantService.js.map +1 -1
- package/dist/src/services/assistantTools.d.ts +27 -1
- package/dist/src/services/assistantTools.d.ts.map +1 -1
- package/dist/src/services/assistantTools.js +169 -35
- package/dist/src/services/assistantTools.js.map +1 -1
- package/dist/src/services/configService.d.ts +2 -0
- package/dist/src/services/configService.d.ts.map +1 -1
- package/dist/src/services/configService.js +3 -2
- package/dist/src/services/configService.js.map +1 -1
- package/dist/src/services/exportService.d.ts +43 -0
- package/dist/src/services/exportService.d.ts.map +1 -1
- package/dist/src/services/exportService.js +116 -11
- package/dist/src/services/exportService.js.map +1 -1
- package/dist/src/services/messageQuery.d.ts +18 -0
- package/dist/src/services/messageQuery.d.ts.map +1 -1
- package/dist/src/services/messageQuery.js +29 -0
- package/dist/src/services/messageQuery.js.map +1 -1
- package/dist/src/services/wechatClient.d.ts +125 -0
- package/dist/src/services/wechatClient.d.ts.map +1 -0
- package/dist/src/services/wechatClient.js +344 -0
- package/dist/src/services/wechatClient.js.map +1 -0
- package/dist/src/services/wechatMedia.d.ts +44 -0
- package/dist/src/services/wechatMedia.d.ts.map +1 -0
- package/dist/src/services/wechatMedia.js +155 -0
- package/dist/src/services/wechatMedia.js.map +1 -0
- package/dist/src/services/wechatMessageService.d.ts +11 -0
- package/dist/src/services/wechatMessageService.d.ts.map +1 -1
- package/dist/src/services/wechatMessageService.js +55 -11
- package/dist/src/services/wechatMessageService.js.map +1 -1
- package/dist/src/types.d.ts +4 -0
- package/dist/src/types.d.ts.map +1 -1
- package/docs/CONTACT_DB_SCHEMA.md +1085 -0
- package/docs/DECISIONS.md +1274 -0
- package/docs/EXTENDING.md +69 -7
- package/docs/FUNCTIONAL-IMPROVEMENT-SPEC.md +27 -0
- package/docs/HEALTH-CHECK.md +12 -7
- package/docs/JEV/345/272/224/347/224/250/346/226/275/345/267/245/346/226/207/346/241/243.md +2 -2
- package/docs/MCP.md +2 -2
- package/docs/PROJECT_STATE.md +64 -5
- package/docs/RELEASING.md +128 -1
- package/docs/images/weflow-architecture-gpt-image-2.png +0 -0
- package/logo.png +0 -0
- package/mcp-server/index.ts +18 -3
- package/package.json +5 -3
- package/resources/js/graph3d/NOTICE.txt +35 -0
- package/resources/js/graph3d/d3-binarytree.min.js +2 -0
- package/resources/js/graph3d/d3-dispatch.min.js +2 -0
- package/resources/js/graph3d/d3-force-3d.min.js +2 -0
- package/resources/js/graph3d/d3-octree.min.js +2 -0
- package/resources/js/graph3d/d3-quadtree.min.js +2 -0
- package/resources/js/graph3d/d3-timer.min.js +2 -0
- package/resources/js/graph3d/layout.mjs +63 -0
- package/resources/js/graph3d/three.min.js +6 -0
- package/resources/js/graph3d/viewer2d.js +448 -0
- package/resources/panel/index.html +12 -0
- package/resources/panel/mascot-blink-1.png +0 -0
- package/resources/panel/mascot-blink-2.png +0 -0
- package/resources/panel/mascot-lift-down-1.png +0 -0
- package/resources/panel/mascot-lift-down-2.png +0 -0
- package/resources/panel/mascot-lift-down-3.png +0 -0
- package/resources/panel/mascot-lift-down-4.png +0 -0
- package/resources/panel/mascot-lift-down-5.png +0 -0
- package/resources/panel/mascot-lift-down-6.png +0 -0
- package/resources/panel/mascot-lift-up-1.png +0 -0
- package/resources/panel/mascot-lift-up-2.png +0 -0
- package/resources/panel/mascot-lift-up-3.png +0 -0
- package/resources/panel/mascot-lift-up-4.png +0 -0
- package/resources/panel/mascot-lift-up-5.png +0 -0
- package/resources/panel/mascot-lift-up-6.png +0 -0
- package/resources/panel/mascot-tickle-1.png +0 -0
- package/resources/panel/mascot-tickle-2.png +0 -0
- package/resources/panel/mascot-tickle-3.png +0 -0
- package/resources/panel/mascot-tickle-4.png +0 -0
- package/resources/panel/panel.css +111 -0
- package/resources/panel/renderer.js +480 -8
- package/scripts/_utils.py +1203 -977
- package/scripts/biz_daily.py +25 -10
- package/scripts/chat_notes.py +15 -5
- package/scripts/check_readme_commands.py +426 -0
- package/scripts/compile_wiki.py +163 -57
- package/scripts/contact_schema.py +339 -0
- package/scripts/data/wechat_emoji.json +225 -0
- package/scripts/export_chat_html.py +346 -6
- package/scripts/fav_notes.py +4 -1
- package/scripts/graph_2d.py +151 -0
- package/scripts/graph_3d.py +646 -0
- package/scripts/health_check.py +163 -26
- package/scripts/nt_decrypt.py +3 -3
- package/scripts/panel_frames.py +683 -0
- package/scripts/pipeline.py +21 -10
- package/scripts/quality_eval.py +5 -0
- package/scripts/rag_chat.py +33 -16
- package/scripts/reply_debt.py +26 -6
- package/scripts/semantic_search.py +212 -58
- package/scripts/vault_rag.py +42 -16
- package/scripts/vault_search.py +44 -8
- package/scripts/wechat_emoticon.py +6 -0
- package/scripts/wiki_lint.py +94 -17
- package/src/core/dbPathService.ts +22 -4
- package/src/panel/server.ts +33 -0
- package/src/services/assistantService.ts +26 -0
- package/src/services/assistantTools.ts +2452 -2319
- package/src/services/configService.ts +5 -2
- package/src/services/exportService.ts +129 -12
- package/src/services/messageQuery.ts +29 -0
- package/src/services/wechatClient.ts +380 -0
- package/src/services/wechatMedia.ts +166 -0
- package/src/services/wechatMessageService.ts +62 -11
- package/src/types.ts +4 -0
- package/dist/src/utils/wechatEmoji.d.ts +0 -5
- package/dist/src/utils/wechatEmoji.d.ts.map +0 -1
- package/dist/src/utils/wechatEmoji.js +0 -35
- package/dist/src/utils/wechatEmoji.js.map +0 -1
|
@@ -12,10 +12,13 @@
|
|
|
12
12
|
# 搜索(有索引用向量,否则关键词 fallback)
|
|
13
13
|
python scripts/semantic_search.py search "有人推荐过遥感的工具吗" --top-k 10
|
|
14
14
|
|
|
15
|
+
# 只看一条知识库(默认 wiki=文章线;`all` 才两条都看)
|
|
16
|
+
python scripts/semantic_search.py search "项目排期" --line chat
|
|
17
|
+
|
|
15
18
|
# 增量更新(只处理新数据)
|
|
16
19
|
python scripts/semantic_search.py update
|
|
17
20
|
|
|
18
|
-
输出: JSON
|
|
21
|
+
输出: JSON 格式(`search` 这一支是一个对象:`line` / `results` / `indexCounts` / 有时 `note`)
|
|
19
22
|
"""
|
|
20
23
|
|
|
21
24
|
import sys, os, json, hashlib, re, argparse
|
|
@@ -59,7 +62,7 @@ except:
|
|
|
59
62
|
pass
|
|
60
63
|
|
|
61
64
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
62
|
-
from _utils import load_config, decrypt_lock, get_dashscope_key
|
|
65
|
+
from _utils import load_config, decrypt_lock, get_dashscope_key, LINE_LABELS # noqa: E402
|
|
63
66
|
from jev_client import create_client
|
|
64
67
|
|
|
65
68
|
OUTPUT_ROOT = 'output'
|
|
@@ -70,6 +73,39 @@ EMBEDDING_DIM = 1024 # 阿里云 text-embedding-v4
|
|
|
70
73
|
BATCH_SIZE = 10 # 阿里云 text-embedding-v4 单次最多 10 条
|
|
71
74
|
TZ = timezone(timedelta(hours=8))
|
|
72
75
|
|
|
76
|
+
# 两条知识库线在**索引里**的 `type` 值。索引是一份,这是**故意的**:两条线在同一次嵌入里
|
|
77
|
+
# 一起算,线靠记录上的 `type` 分(`article` / `chat`)——所以"按线检索"就是给相似度加一个
|
|
78
|
+
# 掩码,不需要第二份索引、也不用重新嵌入。
|
|
79
|
+
#
|
|
80
|
+
# 对外(`--line`、图谱、体检)一律说 `wiki` / `chat`,索引内部叫 `article` / `chat`,
|
|
81
|
+
# **翻译只在这里做一次**。散着写的话改词只会改到一半,而两条线的名字漂了不会报错。
|
|
82
|
+
LINE_TYPES = {'wiki': ('article',), 'chat': ('chat',)}
|
|
83
|
+
LINE_CHOICES = ('all', 'wiki', 'chat')
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def line_types(line: str):
|
|
87
|
+
"""线 → 索引里允许的 `type` 集合;`all` 返回 None(不过滤)。
|
|
88
|
+
|
|
89
|
+
认不出的线**报错**,不悄悄当 all:`--line wik` 静默搜两条线的话,读的人会以为
|
|
90
|
+
自己限定住了,而那正是"分开"没生效的样子。
|
|
91
|
+
"""
|
|
92
|
+
if line == 'all':
|
|
93
|
+
return None
|
|
94
|
+
if line not in LINE_TYPES:
|
|
95
|
+
raise ValueError('未知的线:%s(可选 %s)' % (line, '、'.join(LINE_CHOICES)))
|
|
96
|
+
return LINE_TYPES[line]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def line_label(line: str) -> str:
|
|
100
|
+
return LINE_LABELS.get(line, '两条线') if line != 'all' else '两条线'
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def add_line_arg(parser):
|
|
104
|
+
"""三条线共用的 `--line`(默认**文章线**:两条线分开之后,不传就是只看文章那条,
|
|
105
|
+
要合起来看显式给 `all`)。"""
|
|
106
|
+
parser.add_argument('--line', choices=list(LINE_CHOICES), default='wiki',
|
|
107
|
+
help='只看一条知识库:wiki=文章线(默认)、chat=聊天线、all=两条都看')
|
|
108
|
+
|
|
73
109
|
|
|
74
110
|
def json_output(data):
|
|
75
111
|
print(json.dumps(data, ensure_ascii=False, indent=2))
|
|
@@ -146,50 +182,76 @@ def get_name_map(contact_db, contact_key, contact_salt):
|
|
|
146
182
|
return name_map
|
|
147
183
|
|
|
148
184
|
|
|
149
|
-
def collect_chat_messages(
|
|
150
|
-
"""Collect chat messages for indexing.
|
|
151
|
-
|
|
185
|
+
def collect_chat_messages(conns, name_map, days=90, per_talker=500):
|
|
186
|
+
"""Collect chat messages for indexing. `conns` 是**分片连接列表**,不是单个连接。
|
|
187
|
+
|
|
188
|
+
微信 4.x 的消息按分片存,而且**每片要自己派生的钥匙**才打得开(`nt_decrypt.
|
|
189
|
+
derive_database_key`)。本机实测四片:`message_3.db` 里是近两天的消息,而配置指向的
|
|
190
|
+
`message_0.db` 最新一条是 44 天前的。这里原来只开 `ntDbPath` 一个文件、还用裸的全局钥匙 ——
|
|
191
|
+
于是索引的聊天半边会**静默地收进旧数据**(症状与 issue #10 的"只读到半个月前"同形)。
|
|
192
|
+
索引是要花钱嵌的,收错数据比报错更贵。
|
|
193
|
+
|
|
194
|
+
三处随之改动,各有理由:
|
|
195
|
+
· 会话清单从所有分片取并集 —— 新会话的表可能只存在于新分片里;
|
|
196
|
+
· 不再 `[:100]` 截断:本机 122 个会话,而且那 100 个是**无排序取前 100**(等于随机);
|
|
197
|
+
· id 里带 `rowid`:实测一张表里 12,106 条消息只落在 12,058 个不同秒上,只按 `talker:ts`
|
|
198
|
+
去重会静默顶掉同秒的多条。
|
|
199
|
+
"""
|
|
152
200
|
now = datetime.now(TZ)
|
|
153
201
|
start_ts = int((now - timedelta(days=days)).timestamp())
|
|
154
202
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
except:
|
|
159
|
-
c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'Msg_%'")
|
|
160
|
-
sessions = [r[0] for r in c.fetchall()]
|
|
161
|
-
|
|
162
|
-
items = []
|
|
163
|
-
for talker in sessions[:100]: # Limit to avoid too many
|
|
164
|
-
tbl = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
|
|
203
|
+
sessions = []
|
|
204
|
+
for conn in conns:
|
|
205
|
+
c = conn.cursor()
|
|
165
206
|
try:
|
|
166
|
-
c.execute(
|
|
167
|
-
|
|
207
|
+
c.execute("SELECT user_name FROM Name2Id WHERE is_session = 1")
|
|
208
|
+
rows = [r[0] for r in c.fetchall()]
|
|
209
|
+
except Exception:
|
|
210
|
+
try:
|
|
211
|
+
c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'Msg_%'")
|
|
212
|
+
rows = [r[0] for r in c.fetchall()]
|
|
213
|
+
except Exception:
|
|
214
|
+
rows = []
|
|
215
|
+
for name in rows:
|
|
216
|
+
if name not in sessions:
|
|
217
|
+
sessions.append(name)
|
|
218
|
+
|
|
219
|
+
items, seen = [], set()
|
|
220
|
+
for talker in sessions:
|
|
221
|
+
tbl = f"Msg_{hashlib.md5(talker.encode()).hexdigest()}"
|
|
222
|
+
for conn in conns:
|
|
223
|
+
c = conn.cursor()
|
|
224
|
+
try:
|
|
225
|
+
c.execute('SELECT COUNT(*) FROM sqlite_master WHERE name=?', (tbl,))
|
|
226
|
+
if c.fetchone()[0] == 0:
|
|
227
|
+
continue
|
|
228
|
+
c.execute(f'''
|
|
229
|
+
SELECT rowid, create_time, real_sender_id, message_content
|
|
230
|
+
FROM "{tbl}"
|
|
231
|
+
WHERE create_time >= ?
|
|
232
|
+
ORDER BY create_time DESC
|
|
233
|
+
LIMIT ?
|
|
234
|
+
''', (start_ts, per_talker))
|
|
235
|
+
rows = c.fetchall()
|
|
236
|
+
except Exception:
|
|
168
237
|
continue
|
|
169
|
-
|
|
170
|
-
c.execute(f'''
|
|
171
|
-
SELECT create_time, real_sender_id, message_content
|
|
172
|
-
FROM "{tbl}"
|
|
173
|
-
WHERE create_time >= ?
|
|
174
|
-
ORDER BY create_time DESC
|
|
175
|
-
LIMIT 500
|
|
176
|
-
''', (start_ts,))
|
|
177
|
-
|
|
178
|
-
for ts, sender, content in c.fetchall():
|
|
238
|
+
for rid, ts, sender, content in rows:
|
|
179
239
|
if not content or not isinstance(content, str) or len(content) < 10:
|
|
180
240
|
continue
|
|
241
|
+
item_id = f"chat:{talker}:{ts}:{rid}"
|
|
242
|
+
if item_id in seen:
|
|
243
|
+
continue
|
|
244
|
+
seen.add(item_id)
|
|
181
245
|
sender_name = name_map.get(sender, sender) if sender else name_map.get(talker, talker)
|
|
182
246
|
dt = datetime.fromtimestamp(ts, tz=TZ)
|
|
183
247
|
items.append({
|
|
184
|
-
"id":
|
|
248
|
+
"id": item_id,
|
|
185
249
|
"type": "chat",
|
|
186
250
|
"talker": name_map.get(talker, talker),
|
|
187
251
|
"sender": sender_name,
|
|
188
252
|
"time": dt.strftime('%Y-%m-%d %H:%M'),
|
|
189
253
|
"text": content[:500],
|
|
190
254
|
})
|
|
191
|
-
except:
|
|
192
|
-
pass
|
|
193
255
|
|
|
194
256
|
return items
|
|
195
257
|
|
|
@@ -289,11 +351,25 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
|
|
|
289
351
|
|
|
290
352
|
# Collect items
|
|
291
353
|
items = []
|
|
354
|
+
shards_failed = []
|
|
292
355
|
try:
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
356
|
+
import nt_decrypt # 懒加载:纯逻辑路径(关键词/重排/测试)不该被它拖住
|
|
357
|
+
passphrase = decrypt_lock(config.get('favPassphrase') or config.get('decryptKey') or '')
|
|
358
|
+
pairs, failures = nt_decrypt.connect_message_shards_detailed(
|
|
359
|
+
nt_db, nt_key, nt_salt, passphrase)
|
|
360
|
+
shards_failed = [f['name'] for f in failures]
|
|
361
|
+
if shards_failed:
|
|
362
|
+
print(f"[WARN] {len(shards_failed)} 个分片读不了: {', '.join(shards_failed)}",
|
|
363
|
+
file=sys.stderr)
|
|
364
|
+
try:
|
|
365
|
+
chat_items = collect_chat_messages([c for _p, c in pairs], name_map, days=days)
|
|
366
|
+
items.extend(chat_items)
|
|
367
|
+
finally:
|
|
368
|
+
for _p, c in pairs:
|
|
369
|
+
try:
|
|
370
|
+
c.close()
|
|
371
|
+
except Exception:
|
|
372
|
+
pass
|
|
297
373
|
except Exception as e:
|
|
298
374
|
print(f"[WARN] 聊天消息收集失败: {e}", file=sys.stderr)
|
|
299
375
|
|
|
@@ -306,7 +382,8 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
|
|
|
306
382
|
|
|
307
383
|
if not new_items:
|
|
308
384
|
return {"status": "up_to_date", "total": len(items),
|
|
309
|
-
"chatDays": days, "articleDays": article_days
|
|
385
|
+
"chatDays": days, "articleDays": article_days,
|
|
386
|
+
"shardsFailed": shards_failed}
|
|
310
387
|
|
|
311
388
|
# Generate embeddings
|
|
312
389
|
print("生成 embeddings...", file=sys.stderr)
|
|
@@ -345,22 +422,31 @@ def build_index(api_key: str, full: bool = False, days: int = 90, article_days:
|
|
|
345
422
|
"new": len(new_items),
|
|
346
423
|
"chatDays": days,
|
|
347
424
|
"articleDays": article_days,
|
|
425
|
+
# 分片读不全时**要报出来**:`shardsFailed` 是"全读了"与"读了能读到的"之差,
|
|
426
|
+
# 正是覆盖度报告存在的理由(nt_decrypt.connect_message_shards_detailed 的原话)。
|
|
427
|
+
"shardsFailed": shards_failed,
|
|
348
428
|
}
|
|
349
429
|
|
|
350
430
|
|
|
351
|
-
def keyword_search(query: str, top_k: int = 10, root=None):
|
|
431
|
+
def keyword_search(query: str, top_k: int = 10, root=None, line: str = 'all'):
|
|
352
432
|
"""Simple keyword fallback: scan articles + messages for keyword matches.
|
|
353
433
|
|
|
354
434
|
`root` 只为可测而参数化(默认就是日报目录)。它原先硬编码成相对路径,
|
|
355
435
|
于是这条路径只能靠 chdir 才能测,也就一直没被测过——而它正是检索不可用时
|
|
356
436
|
真正跑着的那条(本机索引从未构建过)。
|
|
437
|
+
|
|
438
|
+
`line` 在这里**必须真的把两条路分开**:这一段的两个来源正好一人一条线
|
|
439
|
+
(日报文章 = 文章线、`mcp_bridge` 的聊天消息 = 聊天线)。
|
|
357
440
|
"""
|
|
358
441
|
results = []
|
|
359
442
|
keywords = query.lower().split()
|
|
443
|
+
types = line_types(line)
|
|
444
|
+
want_article = types is None or 'article' in types
|
|
445
|
+
want_chat = types is None or 'chat' in types
|
|
360
446
|
|
|
361
447
|
# Scan biz-daily articles
|
|
362
448
|
biz_dir = Path(root or 'output/biz-daily')
|
|
363
|
-
if biz_dir.exists():
|
|
449
|
+
if want_article and biz_dir.exists():
|
|
364
450
|
for md_file in sorted(biz_dir.rglob('*.md'), reverse=True):
|
|
365
451
|
# **只收真文章。** 日报目录里还躺着 <天>/README.md、<天>/行动建议.md
|
|
366
452
|
# 这类**生成产物**,它们会被当成检索结果返回(实测搜到过
|
|
@@ -387,19 +473,20 @@ def keyword_search(query: str, top_k: int = 10, root=None):
|
|
|
387
473
|
})
|
|
388
474
|
|
|
389
475
|
# Scan chat messages from mcp_bridge
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
476
|
+
if want_chat:
|
|
477
|
+
try:
|
|
478
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
479
|
+
from mcp_bridge import search_messages as _search_msg
|
|
480
|
+
msg_results = _search_msg(query, 50)
|
|
481
|
+
for r in msg_results.get('results', [])[:top_k]:
|
|
482
|
+
results.append({
|
|
483
|
+
'title': f"[{r.get('time','')}] {r.get('talker','')} > {r.get('sender','')}",
|
|
484
|
+
'source': '微信聊天',
|
|
485
|
+
'score': len(query),
|
|
486
|
+
'text': r.get('content', '')[:200],
|
|
487
|
+
})
|
|
488
|
+
except:
|
|
489
|
+
pass
|
|
403
490
|
|
|
404
491
|
results.sort(key=lambda x: -x['score'])
|
|
405
492
|
return results[:top_k]
|
|
@@ -458,7 +545,9 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
|
|
|
458
545
|
try:
|
|
459
546
|
answers, _usage = client.decide(state, questions)
|
|
460
547
|
except Exception as error:
|
|
461
|
-
|
|
548
|
+
# `file=sys.stderr`:`--json` 时 stdout 是机器在解析的那一份(同文件其它 WARN 早就这样)
|
|
549
|
+
print(' [WARN] 重排失败(%s),保持原检索顺序:%s' % (type(error).__name__, error),
|
|
550
|
+
file=sys.stderr)
|
|
462
551
|
return results
|
|
463
552
|
|
|
464
553
|
scored = []
|
|
@@ -471,7 +560,7 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
|
|
|
471
560
|
scored.append((value, i, item))
|
|
472
561
|
|
|
473
562
|
if all(value is None for value, _i, _item in scored):
|
|
474
|
-
print(' [WARN] 重排没有返回任何可用分数,保持原检索顺序')
|
|
563
|
+
print(' [WARN] 重排没有返回任何可用分数,保持原检索顺序', file=sys.stderr)
|
|
475
564
|
return results
|
|
476
565
|
|
|
477
566
|
# 没拿到分数的按原序垫在最后,而不是当成 0 分——"未知"和"不相关"不是一回事。
|
|
@@ -480,7 +569,7 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
|
|
|
480
569
|
top = scored[0]
|
|
481
570
|
if best is not None and best != '候选%d' % top[1]:
|
|
482
571
|
print(' [WARN] 重排自检不一致:逐条打分最高的是候选%d,单选却答 %s'
|
|
483
|
-
% (top[1], best))
|
|
572
|
+
% (top[1], best), file=sys.stderr)
|
|
484
573
|
|
|
485
574
|
ordered = []
|
|
486
575
|
for value, i, item in scored:
|
|
@@ -492,8 +581,19 @@ def rerank(query, results, client=None, pool_size=RERANK_CANDIDATES):
|
|
|
492
581
|
return ordered
|
|
493
582
|
|
|
494
583
|
|
|
495
|
-
def search(query: str, api_key: str, top_k: int = 10, rerank_results: bool = True
|
|
496
|
-
|
|
584
|
+
def search(query: str, api_key: str, top_k: int = 10, rerank_results: bool = True,
|
|
585
|
+
line: str = 'all'):
|
|
586
|
+
"""Semantic search with keyword fallback.
|
|
587
|
+
|
|
588
|
+
`line` 在**取候选之前**就把不属于这条线的记录挡掉(把它们的相似度压成 -inf 再
|
|
589
|
+
argsort)。反过来的做法——先取 top-k 再过滤——看着也行,但那会把条数算错:
|
|
590
|
+
`--top-k 10 --line chat` 可能只回 3 条,而"为什么只有 3 条"没有任何地方会说。
|
|
591
|
+
|
|
592
|
+
**函数这一层的默认是 `all`(不缩小范围)**,用户看到的默认(文章线)在 CLI 那一层
|
|
593
|
+
(`add_line_arg`)。这样某个调用方忘了传 `line` 时结果是"多搜了一条线",而不是
|
|
594
|
+
"静默少搜一半"——后者在这个仓库里是最贵的那类 bug。
|
|
595
|
+
"""
|
|
596
|
+
types = line_types(line) # 认不出的线在这里就报错,不静默当 all
|
|
497
597
|
# Try embedding-based search first
|
|
498
598
|
if VECTORS_FILE.exists() and META_FILE.exists():
|
|
499
599
|
try:
|
|
@@ -505,11 +605,21 @@ def search(query: str, api_key: str, top_k: int = 10, rerank_results: bool = Tru
|
|
|
505
605
|
query_vec = np.array(query_embeddings[0], dtype=np.float32)
|
|
506
606
|
query_vec = query_vec / np.linalg.norm(query_vec)
|
|
507
607
|
similarities = np.dot(vectors, query_vec)
|
|
608
|
+
if types is not None:
|
|
609
|
+
in_scope = np.array([str(m.get('type', '')) in types for m in meta], dtype=bool)
|
|
610
|
+
if not in_scope.any():
|
|
611
|
+
# 一条都没有:**说出来**,别让它长成"搜了但没有结果"
|
|
612
|
+
return []
|
|
613
|
+
similarities = np.where(in_scope, similarities, -np.inf)
|
|
508
614
|
# 先取一个比重排池更大的候选集,重排才有可以换的空间。
|
|
509
615
|
pool_size = max(top_k, RERANK_CANDIDATES) if rerank_results else top_k
|
|
616
|
+
if types is not None:
|
|
617
|
+
pool_size = min(pool_size, int(in_scope.sum()))
|
|
510
618
|
top_indices = np.argsort(similarities)[::-1][:pool_size]
|
|
511
619
|
results = []
|
|
512
620
|
for idx in top_indices:
|
|
621
|
+
if not np.isfinite(similarities[idx]):
|
|
622
|
+
continue # 掩码挡掉的(-inf)不许混进结果
|
|
513
623
|
item = meta[idx]
|
|
514
624
|
results.append({**item, 'score': float(similarities[idx])})
|
|
515
625
|
if rerank_results:
|
|
@@ -519,13 +629,39 @@ def search(query: str, api_key: str, top_k: int = 10, rerank_results: bool = Tru
|
|
|
519
629
|
pass
|
|
520
630
|
|
|
521
631
|
# Fallback: keyword search
|
|
522
|
-
results = keyword_search(query, max(top_k, RERANK_CANDIDATES) if rerank_results else top_k
|
|
632
|
+
results = keyword_search(query, max(top_k, RERANK_CANDIDATES) if rerank_results else top_k,
|
|
633
|
+
line=line)
|
|
523
634
|
if rerank_results:
|
|
524
635
|
# 关键词打分比余弦更粗,重排在这里的收益反而更大。
|
|
525
636
|
results = rerank(query, results, create_client())
|
|
526
637
|
return results[:top_k]
|
|
527
638
|
|
|
528
639
|
|
|
640
|
+
def line_counts() -> dict:
|
|
641
|
+
"""索引里每条线各有多少条记录(`all` 也算一份总数)。
|
|
642
|
+
|
|
643
|
+
**只读 meta**(不解锁向量文件、不调任何服务、不花一分钱),所以"这一趟为什么是空的"
|
|
644
|
+
可以在返回之前就答上来:`--line chat` 空成一片时,先得能分清是"没命中"还是
|
|
645
|
+
"这条线在索引里根本没有记录"——后者要重新建索引,前者不用。
|
|
646
|
+
"""
|
|
647
|
+
counts = {line: 0 for line in LINE_CHOICES}
|
|
648
|
+
if not META_FILE.exists():
|
|
649
|
+
return counts
|
|
650
|
+
try:
|
|
651
|
+
meta = json.loads(META_FILE.read_text(encoding='utf-8'))
|
|
652
|
+
except Exception:
|
|
653
|
+
return counts
|
|
654
|
+
for item in meta:
|
|
655
|
+
t = str(item.get('type', ''))
|
|
656
|
+
for line, types in LINE_TYPES.items():
|
|
657
|
+
if t in types:
|
|
658
|
+
counts[line] += 1
|
|
659
|
+
# `all` 是**索引里的总条数**,不是两条线相加:以后要是多出第三种 type,
|
|
660
|
+
# 它会被 `all` 带上、而不属于任何一条线,两条线相加就少算了。
|
|
661
|
+
counts['all'] = len(meta)
|
|
662
|
+
return counts
|
|
663
|
+
|
|
664
|
+
|
|
529
665
|
# ====== Main ======
|
|
530
666
|
|
|
531
667
|
def add_window_args(parser):
|
|
@@ -563,6 +699,7 @@ def main():
|
|
|
563
699
|
p.add_argument('--top-k', type=int, default=10)
|
|
564
700
|
p.add_argument('--no-rerank', action='store_true',
|
|
565
701
|
help='不调用决策模型重排,只按向量/关键词相似度返回(回退到引入重排之前)')
|
|
702
|
+
add_line_arg(p)
|
|
566
703
|
|
|
567
704
|
args = parser.parse_args()
|
|
568
705
|
# 手动跑时的第二道闸门(CLI 那条路已经按 1-36500 校验过):0 天等于建一个空库,
|
|
@@ -582,8 +719,25 @@ def main():
|
|
|
582
719
|
if not query:
|
|
583
720
|
result = {"error": "missing search query"}
|
|
584
721
|
else:
|
|
585
|
-
|
|
586
|
-
|
|
722
|
+
# **输出带上是哪条线**(`--line` 默认文章线)。少了这一层,`--line chat`
|
|
723
|
+
# 空成一片时读的人分不清"没命中"和"这条线根本没进索引"——而两者的下一步
|
|
724
|
+
# (换个词 / 重建索引)是两回事。
|
|
725
|
+
rows = search(query, api_key, top_k=args.top_k,
|
|
726
|
+
rerank_results=not args.no_rerank, line=args.line)
|
|
727
|
+
counts = line_counts()
|
|
728
|
+
result = {
|
|
729
|
+
'success': True,
|
|
730
|
+
'line': args.line,
|
|
731
|
+
'lineLabel': line_label(args.line),
|
|
732
|
+
'results': rows,
|
|
733
|
+
'indexCounts': counts,
|
|
734
|
+
}
|
|
735
|
+
if not rows and counts['all'] == 0:
|
|
736
|
+
result['note'] = ('语义索引还没建(output/.semantic_index 不存在)——'
|
|
737
|
+
'先跑 weflow-cli search-index')
|
|
738
|
+
elif not rows and args.line != 'all' and counts[args.line] == 0:
|
|
739
|
+
result['note'] = ('索引里一条「%s」的记录都没有(建索引那天的窗口可能没覆盖到),'
|
|
740
|
+
'先看 --line all' % line_label(args.line))
|
|
587
741
|
else:
|
|
588
742
|
result = {"error": f"未知命令: {args.command}"}
|
|
589
743
|
|
package/scripts/vault_rag.py
CHANGED
|
@@ -16,17 +16,35 @@ DEFAULT_VAULT = os.path.join(PROJECT_ROOT, 'output', 'wechat-vault')
|
|
|
16
16
|
DEFAULT_BIZ = os.path.join(PROJECT_ROOT, 'output', 'biz-daily')
|
|
17
17
|
|
|
18
18
|
sys.path.insert(0, SCRIPTS_DIR)
|
|
19
|
-
from _utils import CONCEPT_DIRS, call_deepseek
|
|
19
|
+
from _utils import CONCEPT_DIRS, LINE_LABELS, call_deepseek, line_for_relative_concepts
|
|
20
20
|
|
|
21
|
+
LINE_CHOICES = ('all', 'wiki', 'chat')
|
|
21
22
|
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
|
|
24
|
+
def line_label(line: str) -> str:
|
|
25
|
+
return '两条线' if line == 'all' else LINE_LABELS.get(line, line)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def collect_context(vault: str, biz_daily: str, question: str, top_k: int,
|
|
29
|
+
line: str = 'all') -> list[dict]:
|
|
30
|
+
"""从 Vault + biz-daily 检索相关上下文。
|
|
31
|
+
|
|
32
|
+
`line` 把**材料也分开**:概念页按目录分线,而用户的笔记层与日报文章都属于**文章线**
|
|
33
|
+
(`--line chat` 只给聊天线的概念页)。不这么分的话,"只看聊天知识库"仍然会从阅读笔记里
|
|
34
|
+
捞结果进来——那正是"两条线分开"要避免的事。
|
|
35
|
+
|
|
36
|
+
**函数默认 `all`(不缩小范围)**,用户看到的默认(文章线)在 CLI 那一层:忘了传 `line`
|
|
37
|
+
的调用方会多搜一条线,而不是静默少搜一半。
|
|
38
|
+
"""
|
|
24
39
|
results = []
|
|
25
40
|
search_terms = question.lower().split()
|
|
41
|
+
want = (lambda which: line == 'all' or line == which)
|
|
26
42
|
|
|
27
|
-
# 1.
|
|
43
|
+
# 1. 概念页(权重最高)——不传 `--line` 时两个目录都读:文章知识库与聊天知识库是分开的
|
|
28
44
|
# (`_utils.CONCEPT_DIRS`),只读一个的话分出去的那一半会静默地搜不到。
|
|
29
45
|
for relative in CONCEPT_DIRS:
|
|
46
|
+
if not want(line_for_relative_concepts(relative)):
|
|
47
|
+
continue
|
|
30
48
|
concepts_dir = Path(vault) / relative
|
|
31
49
|
if not concepts_dir.is_dir():
|
|
32
50
|
continue
|
|
@@ -41,7 +59,8 @@ def collect_context(vault: str, biz_daily: str, question: str, top_k: int) -> li
|
|
|
41
59
|
|
|
42
60
|
# 2. 阅读笔记 —— **`Notes/` 这个目录在本仓库里从来不存**(2026-09-27 实测:
|
|
43
61
|
# Vault 里没有它),所以这一段一直搜的是空气。笔记实际住在 `002_Literature`。
|
|
44
|
-
|
|
62
|
+
# 这是**用户自己的**层(文章线),所以 `--line chat` 时整段跳过。
|
|
63
|
+
for relative in (('002_Literature', '003_Ideas', '008_MOC') if want('wiki') else ()):
|
|
45
64
|
notes_dir = Path(vault) / relative
|
|
46
65
|
if not notes_dir.is_dir():
|
|
47
66
|
continue
|
|
@@ -57,9 +76,9 @@ def collect_context(vault: str, biz_daily: str, question: str, top_k: int) -> li
|
|
|
57
76
|
if score > 0:
|
|
58
77
|
results.append({'source': 'note', 'title': md.stem[:60], 'content': text, 'score': score})
|
|
59
78
|
|
|
60
|
-
# 3.
|
|
61
|
-
biz_path = Path(biz_daily)
|
|
62
|
-
for date_dir in sorted(biz_path.glob('20*'), reverse=True)[:7]: # 最近 7 天
|
|
79
|
+
# 3. 最新文章(日报 = 文章线,`--line chat` 时跳过)
|
|
80
|
+
biz_path = Path(biz_daily) if want('wiki') else None
|
|
81
|
+
for date_dir in (sorted(biz_path.glob('20*'), reverse=True)[:7] if biz_path else []): # 最近 7 天
|
|
63
82
|
for md in date_dir.rglob('*.md'):
|
|
64
83
|
if md.name == 'README.md' or 'README' in md.name:
|
|
65
84
|
continue
|
|
@@ -75,7 +94,7 @@ def collect_context(vault: str, biz_daily: str, question: str, top_k: int) -> li
|
|
|
75
94
|
return results[:top_k]
|
|
76
95
|
|
|
77
96
|
|
|
78
|
-
def build_prompt(question: str, context: list[dict]) -> str:
|
|
97
|
+
def build_prompt(question: str, context: list[dict], line: str = 'all') -> str:
|
|
79
98
|
sources_text = ''
|
|
80
99
|
for i, ctx in enumerate(context):
|
|
81
100
|
label = {'concept': '概念', 'note': '笔记', 'article': '文章'}.get(ctx['source'], ctx['source'])
|
|
@@ -83,6 +102,9 @@ def build_prompt(question: str, context: list[dict]) -> str:
|
|
|
83
102
|
|
|
84
103
|
return f'''你是个人知识助手,基于用户笔记库回答问题。
|
|
85
104
|
|
|
105
|
+
**这一趟只检索了:{line_label(line)}**(另外的范围里可能有相关内容)。没有段落时按
|
|
106
|
+
"**这个范围里**没有相关信息"说,别说成"知识库里没有"。
|
|
107
|
+
|
|
86
108
|
知识库内容:
|
|
87
109
|
{sources_text}
|
|
88
110
|
|
|
@@ -99,6 +121,8 @@ def main():
|
|
|
99
121
|
parser.add_argument('--vault', default=DEFAULT_VAULT, help='Vault 路径')
|
|
100
122
|
parser.add_argument('--biz-daily', default=DEFAULT_BIZ, help='biz-daily 路径')
|
|
101
123
|
parser.add_argument('--api-key', default='', help='DeepSeek API key')
|
|
124
|
+
parser.add_argument('--line', choices=list(LINE_CHOICES), default='wiki',
|
|
125
|
+
help='只看一条知识库:wiki=文章线(默认)、chat=聊天线、all=两条都看')
|
|
102
126
|
parser.add_argument('--json', action='store_true', help='JSON 输出')
|
|
103
127
|
args = parser.parse_args()
|
|
104
128
|
|
|
@@ -113,21 +137,23 @@ def main():
|
|
|
113
137
|
sys.exit(1)
|
|
114
138
|
|
|
115
139
|
top_k = int(args.top_k)
|
|
116
|
-
context = collect_context(args.vault, args.biz_daily, question, top_k)
|
|
140
|
+
context = collect_context(args.vault, args.biz_daily, question, top_k, line=args.line)
|
|
117
141
|
|
|
118
142
|
if args.json:
|
|
119
|
-
output = {'question': question, '
|
|
143
|
+
output = {'question': question, 'line': args.line, 'lineLabel': line_label(args.line),
|
|
144
|
+
'sources': [{'source': c['source'], 'title': c['title'], 'score': c['score']} for c in context]}
|
|
120
145
|
if context:
|
|
121
|
-
prompt = build_prompt(question, context)
|
|
146
|
+
prompt = build_prompt(question, context, args.line)
|
|
122
147
|
output['answer'] = call_deepseek(prompt, api_key, max_tokens=800)
|
|
123
148
|
else:
|
|
124
|
-
|
|
149
|
+
# **说清是哪个范围里没有**:`知识库中没有相关信息` 会被读成"整库都没有"
|
|
150
|
+
output['answer'] = '%s里没有相关信息' % line_label(args.line)
|
|
125
151
|
print(json.dumps(output, ensure_ascii=False, indent=2))
|
|
126
152
|
return
|
|
127
153
|
|
|
128
|
-
print(f'🔍 "{question}"
|
|
154
|
+
print(f'🔍 "{question}"(范围:{line_label(args.line)})\n')
|
|
129
155
|
if not context:
|
|
130
|
-
print('
|
|
156
|
+
print('%s里没有相关信息(`--line all` 两条一起看)' % line_label(args.line))
|
|
131
157
|
return
|
|
132
158
|
|
|
133
159
|
print(f'检索到 {len(context)} 条相关内容:\n')
|
|
@@ -135,7 +161,7 @@ def main():
|
|
|
135
161
|
icon = {'concept': '🧠', 'note': '📝', 'article': '📄'}.get(c['source'], '📎')
|
|
136
162
|
print(f' {i+1}. {icon} [{c["source"]}] {c["title"]}')
|
|
137
163
|
|
|
138
|
-
prompt = build_prompt(question, context)
|
|
164
|
+
prompt = build_prompt(question, context, args.line)
|
|
139
165
|
print(f'\n{"="*50}')
|
|
140
166
|
answer = call_deepseek(prompt, api_key, max_tokens=800)
|
|
141
167
|
print(answer)
|