weflow-cli 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +3 -2
- package/CHANGELOG.md +1949 -1
- package/OPERATIONS.md +774 -1
- package/README.en.md +93 -7
- package/README.md +31 -6
- package/bin/weflow-cli.ts +934 -27
- package/dist/bin/weflow-cli.js +952 -27
- package/dist/bin/weflow-cli.js.map +1 -1
- package/dist/mcp-server/index.js +39 -13
- package/dist/mcp-server/index.js.map +1 -1
- package/dist/src/core/appMsgFormat.d.ts +37 -0
- package/dist/src/core/appMsgFormat.d.ts.map +1 -0
- package/dist/src/core/appMsgFormat.js +106 -0
- package/dist/src/core/appMsgFormat.js.map +1 -0
- package/dist/src/core/ntCore.d.ts +19 -2
- package/dist/src/core/ntCore.d.ts.map +1 -1
- package/dist/src/core/ntCore.js +11 -1
- package/dist/src/core/ntCore.js.map +1 -1
- package/dist/src/core/sqlcipherCore.d.ts +0 -12
- package/dist/src/core/sqlcipherCore.d.ts.map +1 -1
- package/dist/src/core/sqlcipherCore.js +5 -64
- package/dist/src/core/sqlcipherCore.js.map +1 -1
- package/dist/src/core/wcdbCore.d.ts +1 -1
- package/dist/src/core/wcdbCore.d.ts.map +1 -1
- package/dist/src/core/wcdbCore.js +5 -1
- package/dist/src/core/wcdbCore.js.map +1 -1
- package/dist/src/panel/client.d.ts +25 -0
- package/dist/src/panel/client.d.ts.map +1 -0
- package/dist/src/panel/client.js +34 -0
- package/dist/src/panel/client.js.map +1 -0
- package/dist/src/panel/endpoint.d.ts +45 -0
- package/dist/src/panel/endpoint.d.ts.map +1 -0
- package/dist/src/panel/endpoint.js +132 -0
- package/dist/src/panel/endpoint.js.map +1 -0
- package/dist/src/panel/launch.d.ts +50 -0
- package/dist/src/panel/launch.d.ts.map +1 -0
- package/dist/src/panel/launch.js +74 -0
- package/dist/src/panel/launch.js.map +1 -0
- package/dist/src/panel/server.d.ts +23 -0
- package/dist/src/panel/server.d.ts.map +1 -0
- package/dist/src/panel/server.js +413 -0
- package/dist/src/panel/server.js.map +1 -0
- package/dist/src/panel/userId.d.ts +37 -0
- package/dist/src/panel/userId.d.ts.map +1 -0
- package/dist/src/panel/userId.js +56 -0
- package/dist/src/panel/userId.js.map +1 -0
- package/dist/src/services/assistantDaemon.d.ts +20 -3
- package/dist/src/services/assistantDaemon.d.ts.map +1 -1
- package/dist/src/services/assistantDaemon.js +46 -9
- package/dist/src/services/assistantDaemon.js.map +1 -1
- package/dist/src/services/assistantEval.d.ts +141 -0
- package/dist/src/services/assistantEval.d.ts.map +1 -0
- package/dist/src/services/assistantEval.js +497 -0
- package/dist/src/services/assistantEval.js.map +1 -0
- package/dist/src/services/assistantMemory.d.ts +106 -4
- package/dist/src/services/assistantMemory.d.ts.map +1 -1
- package/dist/src/services/assistantMemory.js +396 -61
- package/dist/src/services/assistantMemory.js.map +1 -1
- package/dist/src/services/assistantPrivacy.d.ts +15 -2
- package/dist/src/services/assistantPrivacy.d.ts.map +1 -1
- package/dist/src/services/assistantPrivacy.js +18 -4
- package/dist/src/services/assistantPrivacy.js.map +1 -1
- package/dist/src/services/assistantRouter.d.ts +41 -0
- package/dist/src/services/assistantRouter.d.ts.map +1 -0
- package/dist/src/services/assistantRouter.js +151 -0
- package/dist/src/services/assistantRouter.js.map +1 -0
- package/dist/src/services/assistantService.d.ts +209 -3
- package/dist/src/services/assistantService.d.ts.map +1 -1
- package/dist/src/services/assistantService.js +590 -75
- package/dist/src/services/assistantService.js.map +1 -1
- package/dist/src/services/assistantTools.d.ts +205 -1
- package/dist/src/services/assistantTools.d.ts.map +1 -1
- package/dist/src/services/assistantTools.js +1089 -58
- package/dist/src/services/assistantTools.js.map +1 -1
- package/dist/src/services/assistantTrace.d.ts +47 -0
- package/dist/src/services/assistantTrace.d.ts.map +1 -0
- package/dist/src/services/assistantTrace.js +150 -0
- package/dist/src/services/assistantTrace.js.map +1 -0
- package/dist/src/services/chatService.d.ts +1 -1
- package/dist/src/services/chatService.d.ts.map +1 -1
- package/dist/src/services/chatService.js +5 -2
- package/dist/src/services/chatService.js.map +1 -1
- package/dist/src/services/configService.d.ts +9 -0
- package/dist/src/services/configService.d.ts.map +1 -1
- package/dist/src/services/configService.js +15 -8
- package/dist/src/services/configService.js.map +1 -1
- package/dist/src/services/pythonBridge.d.ts +41 -0
- package/dist/src/services/pythonBridge.d.ts.map +1 -0
- package/dist/src/services/pythonBridge.js +119 -0
- package/dist/src/services/pythonBridge.js.map +1 -0
- package/dist/src/services/syncService.d.ts +8 -2
- package/dist/src/services/syncService.d.ts.map +1 -1
- package/dist/src/services/syncService.js +26 -7
- package/dist/src/services/syncService.js.map +1 -1
- package/dist/src/services/wechatMessageService.d.ts +25 -0
- package/dist/src/services/wechatMessageService.d.ts.map +1 -1
- package/dist/src/services/wechatMessageService.js +69 -7
- package/dist/src/services/wechatMessageService.js.map +1 -1
- package/dist/src/utils/atomicWrite.d.ts +4 -0
- package/dist/src/utils/atomicWrite.d.ts.map +1 -0
- package/dist/src/utils/atomicWrite.js +60 -0
- package/dist/src/utils/atomicWrite.js.map +1 -0
- package/dist/src/utils/dateRange.d.ts +15 -0
- package/dist/src/utils/dateRange.d.ts.map +1 -1
- package/dist/src/utils/dateRange.js +47 -0
- package/dist/src/utils/dateRange.js.map +1 -1
- package/dist/src/utils/text.d.ts +18 -0
- package/dist/src/utils/text.d.ts.map +1 -0
- package/dist/src/utils/text.js +20 -0
- package/dist/src/utils/text.js.map +1 -0
- package/docs/AI_INTERFACE.md +2 -0
- package/docs/DECISIONS.md +1424 -0
- package/docs/EXTENDING.md +187 -0
- package/docs/JEV/345/272/224/347/224/250/346/226/275/345/267/245/346/226/207/346/241/243.md +310 -0
- package/docs/MCP.md +23 -4
- package/docs/PROJECT_STATE.md +299 -98
- package/docs/SETUP.md +22 -0
- package/docs/SYNC_CONTRACT.md +31 -3
- package/mcp-server/index.ts +681 -657
- package/package.json +84 -83
- package/resources/panel/ball-position.cjs +153 -0
- package/resources/panel/index.html +46 -0
- package/resources/panel/main.cjs +447 -0
- package/resources/panel/mascot-focus.png +0 -0
- package/resources/panel/mascot-happy.png +0 -0
- package/resources/panel/mascot-sorry.png +0 -0
- package/resources/panel/mascot-tired.png +0 -0
- package/resources/panel/mascot.png +0 -0
- package/resources/panel/package.json +7 -0
- package/resources/panel/panel.css +407 -0
- package/resources/panel/preload.cjs +72 -0
- package/resources/panel/quick-menu.cjs +197 -0
- package/resources/panel/renderer.js +441 -0
- package/resources/panel/tray-menu.cjs +40 -0
- package/resources/panel/tray.png +0 -0
- package/scripts/_utils.py +403 -54
- package/scripts/annual_report.py +4 -2
- package/scripts/article_notes.py +423 -0
- package/scripts/backfill_articles.py +438 -0
- package/scripts/biz_daily.py +590 -105
- package/scripts/chat_notes.py +470 -0
- package/scripts/classify_daily.py +10 -18
- package/scripts/compile_wiki.py +1059 -46
- package/scripts/create_reading_notes.py +199 -14
- package/scripts/decide.py +253 -0
- package/scripts/draft_reply.py +671 -0
- package/scripts/export_chat_html.py +35 -33
- package/scripts/extract_todos.py +31 -5
- package/scripts/fav_notes.py +318 -0
- package/scripts/fix_topics.py +2 -1
- package/scripts/generate_ai_report.py +214 -9
- package/scripts/generate_html.py +18 -3
- package/scripts/generate_review.py +2 -2
- package/scripts/jev_client.py +294 -0
- package/scripts/jev_probe.py +440 -0
- package/scripts/nt_common.py +82 -0
- package/scripts/nt_decrypt.py +1755 -1460
- package/scripts/pipeline.py +27 -2
- package/scripts/promote_ideas.py +19 -4
- package/scripts/quality_eval.py +707 -0
- package/scripts/rag_chat.py +4 -4
- package/scripts/read_image.py +187 -0
- package/scripts/reply_debt.py +488 -0
- package/scripts/route_cards.py +453 -0
- package/scripts/semantic_search.py +208 -49
- package/scripts/user_notes.py +315 -0
- package/scripts/vault_rag.py +13 -7
- package/scripts/vault_search.py +21 -4
- package/scripts/wiki_lint.py +397 -0
- package/src/core/appMsgFormat.ts +117 -0
- package/src/core/ntCore.ts +27 -2
- package/src/core/sqlcipherCore.ts +5 -61
- package/src/core/wcdbCore.ts +6 -2
- package/src/panel/client.ts +49 -0
- package/src/panel/endpoint.ts +144 -0
- package/src/panel/launch.ts +101 -0
- package/src/panel/server.ts +424 -0
- package/src/panel/userId.ts +72 -0
- package/src/services/assistantDaemon.ts +57 -9
- package/src/services/assistantEval.ts +611 -0
- package/src/services/assistantMemory.ts +413 -57
- package/src/services/assistantPrivacy.ts +18 -4
- package/src/services/assistantRouter.ts +191 -0
- package/src/services/assistantService.ts +671 -74
- package/src/services/assistantTools.ts +1118 -57
- package/src/services/assistantTrace.ts +175 -0
- package/src/services/chatService.ts +6 -2
- package/src/services/configService.ts +403 -385
- package/src/services/pythonBridge.ts +139 -0
- package/src/services/syncService.ts +37 -9
- package/src/services/wechatMessageService.ts +73 -7
- package/src/utils/atomicWrite.ts +56 -0
- package/src/utils/dateRange.ts +46 -0
- package/src/utils/text.ts +20 -0
package/scripts/_utils.py
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
2
|
weflow-cli 公共工具函数 — 供 biz_daily / classify_daily / chat_report 等共用。
|
|
3
3
|
"""
|
|
4
|
-
import os, json, base64, socket, urllib.request, hashlib
|
|
5
|
-
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
4
|
+
import os, json, base64, socket, urllib.request, hashlib
|
|
6
5
|
from functools import wraps
|
|
7
6
|
|
|
8
7
|
|
|
@@ -340,68 +339,237 @@ def clear_cache(cache_subdir: str = ''):
|
|
|
340
339
|
fpath = os.path.join(target, fname)
|
|
341
340
|
if os.path.isfile(fpath):
|
|
342
341
|
os.unlink(fpath)
|
|
343
|
-
|
|
344
|
-
|
|
345
342
|
# ======================================================================
|
|
346
|
-
#
|
|
343
|
+
# Config Decrypt
|
|
347
344
|
# ======================================================================
|
|
348
345
|
|
|
349
|
-
def parallel_map(func, items, max_workers: int = 3, delay: float = 0.1,
|
|
350
|
-
desc: str = '') -> list:
|
|
351
|
-
"""并发执行函数,自动控制并发数和速率。
|
|
352
|
-
|
|
353
|
-
Args:
|
|
354
|
-
func: 接受单个 item 的函数,返回结果
|
|
355
|
-
items: 待处理的列表
|
|
356
|
-
max_workers: 最大并发数(默认 3,避免 API 限流)
|
|
357
|
-
delay: 每个任务提交后的延迟(秒),用于速率控制
|
|
358
|
-
desc: 进度描述(用于日志输出)
|
|
359
|
-
|
|
360
|
-
Returns:
|
|
361
|
-
与 items 顺序对应的结果列表
|
|
362
|
-
"""
|
|
363
|
-
results = [None] * len(items)
|
|
364
|
-
errors = [None] * len(items)
|
|
365
|
-
|
|
366
|
-
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
367
|
-
future_to_idx = {}
|
|
368
|
-
for idx, item in enumerate(items):
|
|
369
|
-
future = executor.submit(func, item)
|
|
370
|
-
future_to_idx[future] = idx
|
|
371
|
-
if delay > 0:
|
|
372
|
-
time.sleep(delay)
|
|
373
|
-
|
|
374
|
-
done_count = 0
|
|
375
|
-
total = len(items)
|
|
376
|
-
for future in as_completed(future_to_idx):
|
|
377
|
-
idx = future_to_idx[future]
|
|
378
|
-
try:
|
|
379
|
-
results[idx] = future.result()
|
|
380
|
-
except Exception as e:
|
|
381
|
-
errors[idx] = e
|
|
382
|
-
if desc:
|
|
383
|
-
print(f' [{desc}] 第 {idx+1}/{total} 项失败: {e}')
|
|
384
|
-
done_count += 1
|
|
385
|
-
if desc and done_count % 10 == 0:
|
|
386
|
-
print(f' [{desc}] 进度: {done_count}/{total}')
|
|
387
|
-
|
|
388
|
-
# 报告错误统计
|
|
389
|
-
fail_count = sum(1 for e in errors if e is not None)
|
|
390
|
-
if fail_count > 0 and desc:
|
|
391
|
-
print(f' [{desc}] 完成: {total - fail_count} 成功, {fail_count} 失败')
|
|
392
|
-
|
|
393
|
-
return results
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
# ======================================================================
|
|
397
|
-
# Config Decrypt
|
|
398
346
|
# ======================================================================
|
|
347
|
+
# 主题分类法 —— **唯一一处定义**
|
|
348
|
+
#
|
|
349
|
+
# 列表与定义都收在这里,因为「散文写的枚举会漂」在这个仓库里已经真的发生过:
|
|
350
|
+
# `biz_daily.TOPIC_PROMPT` 的第一行按 TOPICS 生成了六类,但同一段提示词后面两处
|
|
351
|
+
# 提醒是硬编码的五类(少了政治),判断规则里也没有政治那一栏——而政治在语料里占 26%。
|
|
352
|
+
# 两条路径(LLM 提示词 / 决策模型的判据)现在读同一张表,所以它们不可能再对不上。
|
|
353
|
+
#
|
|
354
|
+
# **顺序有意义**:`generate_ai_report` 按这个顺序给报告分栏。
|
|
355
|
+
TOPICS = ['AI', '学术', '新闻', '文学', '投资', '政治']
|
|
356
|
+
|
|
357
|
+
# 聊天卡拷进 Vault `Sources/Chat/` 时的文件名前缀。**必须共享**:`chat_notes` 用它写文件、
|
|
358
|
+
# `compile_wiki` 用它把概念页的"来源"链接指过去——两处各写一份就是"同一条信息写两处",
|
|
359
|
+
# 而这个仓库已经因为这类分叉吃过亏。
|
|
360
|
+
# 为什么要有这个前缀:聊天卡的名字**就是会话名**(`白马非马`、`老表亲戚群`),而概念页
|
|
361
|
+
# 也可能叫同一个名字;两个文件同名时 `[[白马非马]]` 在 Obsidian 里是二义的。
|
|
362
|
+
CHAT_CARD_PREFIX = '会话-'
|
|
363
|
+
|
|
364
|
+
# 知识页住在哪几个目录(**Vault 相对路径**)。**分成两个是有意的**:用户明确要求
|
|
365
|
+
# 聊天知识库与公众号文章知识库分开(2026-09-27),所以是"一条线一个目录"而不是混在一起。
|
|
366
|
+
# 顺序即展示顺序:文章在前。
|
|
367
|
+
#
|
|
368
|
+
# 读的一方(`wiki_lint` / `vault_search` / `vault_rag` / 助手的 `search_knowledge`)
|
|
369
|
+
# 都要把**两个都读**,否则新分出去的那一半会静默地搜不到——而这个仓库已经因为
|
|
370
|
+
# "同一个事实写在两处"栽过好几次,所以 TS 那边有一份平行的列表,有测试钉住两者一致。
|
|
371
|
+
CONCEPT_DIRS = ('Wiki/Concepts', 'Chat/Concepts')
|
|
372
|
+
|
|
373
|
+
# 一篇文章**没能被分类**时落到哪一类。
|
|
374
|
+
#
|
|
375
|
+
# 收成一处是因为它曾经在写入路径上有两个不同的答案、相隔七行:
|
|
376
|
+
# `.articles.json` 那条路默认 `''`(空主题),而分组/落 md 那条路默认 `'学术'`。
|
|
377
|
+
# 于是一篇文章在 md 里写 `topic: 学术`、在 json 里写 `topic: ""`——
|
|
378
|
+
# **同一次运行、同一篇文章,两个产物给出不同的主题**,而且都不报错。
|
|
379
|
+
#
|
|
380
|
+
# 真实后果(2026-09-04 的产出,178 篇):md 全落在 `学术/` 目录、frontmatter 写
|
|
381
|
+
# `topic: 学术`(其中两篇是「OpenAI 深夜发布 GPT-6」「专为高管准备的 AI 助手」,
|
|
382
|
+
# 明显不该是学术),而 json 里同一批全是 `""` —— 报告那条路读 json,
|
|
383
|
+
# 判据 `topic != FOCUS_TOPIC and relevance != '高'` 就把它们整批静默排除了。
|
|
384
|
+
#
|
|
385
|
+
# 什么情况下会走到这里:`daily --no-ai`、或者没配 API key 时,Phase 2(分类)整段
|
|
386
|
+
# 不跑,文章 dict 里**根本没有 `topic` 键**,而 Phase 3 照常写文件。
|
|
387
|
+
#
|
|
388
|
+
# 这个值是**兜底不是判断**:落到这里的文章其实是"未分类"。之所以不新造一个
|
|
389
|
+
# `未分类` 类别,是因为它要额外维护一个目录、一处 HTML 分栏、以及下游对
|
|
390
|
+
# 主题字面量的比较;改用 `未分类` 属于改既有契约,得单独决策。
|
|
391
|
+
DEFAULT_TOPIC = '学术'
|
|
392
|
+
|
|
393
|
+
# 每一类的判据。既生成提示词里的判断规则,也直接作为 Jev 的 choice criteria。
|
|
394
|
+
TOPIC_CRITERIA = {
|
|
395
|
+
'AI': 'AI大模型/Agent/编程/开源/科技产品/工具教程',
|
|
396
|
+
'投资': '股票基金/融资/经济分析/商业市场',
|
|
397
|
+
'新闻': '时事政策/社会热点/娱乐/招聘促销/会议通知',
|
|
398
|
+
'文学': '散文小说/美食旅游/生活随笔/历史文化',
|
|
399
|
+
'学术': '科研论文/期刊文章/实验室成果/学术会议/高校研究',
|
|
400
|
+
'政治': '党政理论/政策解读/领导人讲话/官方评论',
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
# 相关度的三个档位。与 `TOPICS` 一样是**下游按字面量比较**的封闭词表:
|
|
404
|
+
# `generate_ai_report.py`、`enrich_backlinks.py`、`generate_html.py` 都在比
|
|
405
|
+
# `'高'`/`'中'`/`'低'` 这三个字。定义收在这里,`jev_client` 与 `biz_daily`
|
|
406
|
+
# 都从这里取——原先 `jev_client` 一份、`biz_daily` 的成员校验里再写一份。
|
|
407
|
+
#
|
|
408
|
+
# **顺序即语义**:`jev_client.RELEVANCE_LEVELS` 是给模型看的判据文本,按下标与
|
|
409
|
+
# 这个列表一一对应(有测试钉住),所以不要重排。
|
|
410
|
+
RELEVANCE_NAMES = ['低', '中', '高']
|
|
411
|
+
|
|
412
|
+
# 一篇文章**没能被判相关度**时落到哪一档。和 `DEFAULT_TOPIC` 是同一类东西:
|
|
413
|
+
# 兜底不是判断。之所以要具名,是因为"全库 2199/2201 篇都是「中」"这件事的答案
|
|
414
|
+
# 就在这里——分类失败的每条路径最终都落到这个值,而 `'中'` 读起来是
|
|
415
|
+
# 「有启发性」(正面评价),不是「没判断」。值不变更(改档位会动到下游的
|
|
416
|
+
# 收录判据),只是把它写在一处,好让它可被搜索、可被讨论。
|
|
417
|
+
DEFAULT_RELEVANCE = '中'
|
|
418
|
+
|
|
399
419
|
|
|
400
420
|
def load_config():
|
|
401
421
|
with open(CONFIG_PATH, 'r', encoding='utf-8') as f:
|
|
402
422
|
return json.load(f)
|
|
403
423
|
|
|
404
424
|
|
|
425
|
+
# 配置里"不想在日报里看到的主题"的键名。值形如 `新闻,投资,学术`。
|
|
426
|
+
EXCLUDED_TOPICS_KEY = 'dailyExcludeTopics'
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def excluded_topics(config=None, explicit='', protected=()):
|
|
430
|
+
"""用户不想在日报里看到的主题集合。`explicit`(命令行)覆盖配置。
|
|
431
|
+
|
|
432
|
+
**这是展示层的开关,不是抓取层的**:正文照常抓取、照常归档,只是不出现在
|
|
433
|
+
日报里。理由是实测:拉之前只有来源+标题+摘要,按它判类型**不可靠**(与来源级
|
|
434
|
+
比对一致率仅 60%、误伤 48/217),而**排除是不可逆的**——没抓取就没归档。
|
|
435
|
+
放在展示层还换来一件事:改主意不用重抓,改个配置重新生成报告即可。
|
|
436
|
+
|
|
437
|
+
写错的主题名会被忽略并**打一行 WARN**(静默忽略会让人以为过滤生效了)。
|
|
438
|
+
|
|
439
|
+
`protected` 是不许被排除的主题(呼叫方传自己的「焦点主题」)。它比「未知主题名」
|
|
440
|
+
更值得拦一下:把焦点主题排掉,报告要么没有主体、要么直接报「未找到文章」
|
|
441
|
+
**指错方向**(那句话说去跑 biz_daily,可文章其实在)。同样 WARN 后忽略。
|
|
442
|
+
"""
|
|
443
|
+
raw = explicit or (config or {}).get(EXCLUDED_TOPICS_KEY, '') or ''
|
|
444
|
+
names = [t.strip() for t in str(raw).replace(',', ',').split(',') if t.strip()]
|
|
445
|
+
out = set()
|
|
446
|
+
for name in names:
|
|
447
|
+
if name in TOPICS:
|
|
448
|
+
if name in protected:
|
|
449
|
+
print('[WARN] %s 里的「%s」是这份报告的主体,不能排除,已忽略'
|
|
450
|
+
% (EXCLUDED_TOPICS_KEY, name))
|
|
451
|
+
continue
|
|
452
|
+
out.add(name)
|
|
453
|
+
else:
|
|
454
|
+
print('[WARN] %s 里的「%s」不是已知主题,已忽略(可选:%s)'
|
|
455
|
+
% (EXCLUDED_TOPICS_KEY, name, '/'.join(TOPICS)))
|
|
456
|
+
return out
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
# === 来源级先验:"哪个号稳定发哪一类"从真实判断里长出来 ===
|
|
460
|
+
#
|
|
461
|
+
# 为什么不写死一张表:拉取之前只有来源+标题+摘要,按它判类型实测只有 60% 一致率、
|
|
462
|
+
# 误伤 48/217(见 excluded_topics 的说明)。但**抓回来之后**每次都有 Jev 按正文判的
|
|
463
|
+
# 主题,日积月累就看得出某个号一贯发什么。这张表因此是长出来的,不是我编的。
|
|
464
|
+
#
|
|
465
|
+
# 它现在只用来**报数**,不用来跳过任何东西:跳过是不可逆的。
|
|
466
|
+
SOURCE_TOPICS_FILE = 'source_topics.json'
|
|
467
|
+
SOURCE_PRIOR_MIN_SAMPLES = 8 # 少于这么多篇不下结论
|
|
468
|
+
SOURCE_PRIOR_SHARE = 0.8 # 某一类占比到这个数才算"稳定地只发这一路"
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def source_topics_path(path=None):
|
|
472
|
+
"""先验表的落盘位置。默认与 config.json 同目录(家目录)。"""
|
|
473
|
+
return path or os.path.join(os.path.dirname(CONFIG_PATH), SOURCE_TOPICS_FILE)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def load_source_topics(path=None):
|
|
477
|
+
"""读先验表:`{来源: {主题: 次数}}`。
|
|
478
|
+
|
|
479
|
+
读不出、坏掉、不是字典——一律返回 `{}`。这是**辅助**数据,坏了不该让日报挂掉。
|
|
480
|
+
"""
|
|
481
|
+
try:
|
|
482
|
+
with open(source_topics_path(path), encoding='utf-8') as f:
|
|
483
|
+
data = json.load(f)
|
|
484
|
+
except (OSError, ValueError):
|
|
485
|
+
return {}
|
|
486
|
+
if not isinstance(data, dict):
|
|
487
|
+
return {}
|
|
488
|
+
out = {}
|
|
489
|
+
for source, counts in data.items():
|
|
490
|
+
if isinstance(source, str) and source.strip() and isinstance(counts, dict):
|
|
491
|
+
out[source] = counts
|
|
492
|
+
return out
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def record_source_topics(pairs, path=None):
|
|
496
|
+
"""把这一批 `(来源, 主题)` 累加进先验表,返回摘要 dict。
|
|
497
|
+
|
|
498
|
+
**只增不减**(与 frontmatter 的 only-increase 同一个纪律):累计次数是你事后判断
|
|
499
|
+
"这个号到底稳不稳"的唯一依据,抹掉就回不来了。
|
|
500
|
+
来源为空、主题不在 TOPICS 里的都不计——计了会污染分母,让占比算错。
|
|
501
|
+
"""
|
|
502
|
+
import tempfile as _tmp # 模块顶没导入它:本文件里它是函数内导入的
|
|
503
|
+
store = load_source_topics(path)
|
|
504
|
+
added = skipped = 0
|
|
505
|
+
for source, topic in pairs:
|
|
506
|
+
source = (source or '').strip()
|
|
507
|
+
topic = (topic or '').strip()
|
|
508
|
+
if not source or topic not in TOPICS:
|
|
509
|
+
skipped += 1
|
|
510
|
+
continue
|
|
511
|
+
bucket = store.setdefault(source, {})
|
|
512
|
+
try:
|
|
513
|
+
bucket[topic] = int(bucket.get(topic, 0)) + 1
|
|
514
|
+
except (TypeError, ValueError):
|
|
515
|
+
bucket[topic] = 1 # 值被改坏了,从 1 重新数,别让它把整个文件废掉
|
|
516
|
+
added += 1
|
|
517
|
+
target = source_topics_path(path)
|
|
518
|
+
try:
|
|
519
|
+
os.makedirs(os.path.dirname(target) or '.', exist_ok=True)
|
|
520
|
+
fd, tmp_path = _tmp.mkstemp(suffix='.json', dir=os.path.dirname(target) or '.')
|
|
521
|
+
try:
|
|
522
|
+
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
|
523
|
+
json.dump(store, f, ensure_ascii=False, indent=1, sort_keys=True)
|
|
524
|
+
os.replace(tmp_path, target)
|
|
525
|
+
except Exception:
|
|
526
|
+
if os.path.exists(tmp_path):
|
|
527
|
+
os.unlink(tmp_path)
|
|
528
|
+
raise
|
|
529
|
+
except OSError as exc:
|
|
530
|
+
# 写不进去(权限、磁盘)不能让日报失败——它只是记个账。
|
|
531
|
+
return {'sources': len(store), 'added': added, 'skipped': skipped,
|
|
532
|
+
'error': str(exc)}
|
|
533
|
+
return {'sources': len(store), 'added': added, 'skipped': skipped, 'error': ''}
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def stable_source_topic(counts, min_samples=SOURCE_PRIOR_MIN_SAMPLES,
|
|
537
|
+
share=SOURCE_PRIOR_SHARE):
|
|
538
|
+
"""某个来源是不是已经**稳定地只发某一类**。
|
|
539
|
+
|
|
540
|
+
返回 `(主题, 占比, 样本数)`;还判不了返回 `None`。
|
|
541
|
+
**`None` 是"还不知道",不是"没有主题"**——调用方必须把这两种情况分开,
|
|
542
|
+
别把 `None` 当成某个默认主题用(那正好是"来源判不准"最坏的那种错)。
|
|
543
|
+
"""
|
|
544
|
+
clean = {}
|
|
545
|
+
for topic, count in (counts or {}).items():
|
|
546
|
+
try:
|
|
547
|
+
value = int(count)
|
|
548
|
+
except (TypeError, ValueError):
|
|
549
|
+
continue
|
|
550
|
+
if value > 0:
|
|
551
|
+
clean[topic] = value
|
|
552
|
+
total = sum(clean.values())
|
|
553
|
+
if total < min_samples:
|
|
554
|
+
return None
|
|
555
|
+
topic, top = max(clean.items(), key=lambda kv: kv[1])
|
|
556
|
+
ratio = top / total
|
|
557
|
+
if ratio < share:
|
|
558
|
+
return None
|
|
559
|
+
return (topic, ratio, total)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def source_prior_candidates(store, exclude, **kwargs):
|
|
563
|
+
"""先验里"已经稳到能判"且落在排除集里的来源,按样本数从多到少。
|
|
564
|
+
|
|
565
|
+
只报出来给人看,**不据此跳过任何东西**:跳过是不可逆的,而这张表还在长。"""
|
|
566
|
+
rows = []
|
|
567
|
+
for source, counts in (store or {}).items():
|
|
568
|
+
stable = stable_source_topic(counts, **kwargs)
|
|
569
|
+
if stable and stable[0] in (exclude or ()):
|
|
570
|
+
rows.append((source, stable[0], stable[1], stable[2]))
|
|
571
|
+
return sorted(rows, key=lambda row: (-row[3], row[0]))
|
|
572
|
+
|
|
405
573
|
def decrypt_lock(locked_str: str) -> str:
|
|
406
574
|
if not locked_str or not locked_str.startswith('lock:'):
|
|
407
575
|
return locked_str
|
|
@@ -440,6 +608,43 @@ def get_api_key(config=None) -> str:
|
|
|
440
608
|
return decrypt_lock(config.get('deepseekApiKey', ''))
|
|
441
609
|
|
|
442
610
|
|
|
611
|
+
def get_dashscope_key(config=None) -> str:
|
|
612
|
+
"""读取阿里云百炼(DashScope)的 embedding key,自动解密 lock: 前缀。
|
|
613
|
+
|
|
614
|
+
和 `get_api_key` / `get_typesafe_key` 同一形状:**不要直接 config.get**——
|
|
615
|
+
这个键进了 `ENCRYPTED_KEYS`,磁盘上是密文。解密失败回空串而不是抛,
|
|
616
|
+
理由同 get_typesafe_key(跨机器拷配置时它该表现为"没配",不是崩)。
|
|
617
|
+
"""
|
|
618
|
+
if config is None:
|
|
619
|
+
try:
|
|
620
|
+
config = load_config()
|
|
621
|
+
except Exception:
|
|
622
|
+
return ''
|
|
623
|
+
try:
|
|
624
|
+
return decrypt_lock(config.get('dashscopeApiKey', ''))
|
|
625
|
+
except Exception:
|
|
626
|
+
return ''
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def get_typesafe_key(config=None) -> str:
|
|
630
|
+
"""读取 TypeSafe (Jev) 决策模型的 key,自动解密 lock: 前缀。
|
|
631
|
+
|
|
632
|
+
解密失败**回空串而不是抛**:TS 侧 lockDecrypt() 失败就是静默回 '',而
|
|
633
|
+
decrypt_lock() 是抛的。两份 config.json 一跨机器拷贝,同一份密文在
|
|
634
|
+
TS 侧表现为"没配 key"、在这里表现为崩——那会让日报整个挂掉,而它本该
|
|
635
|
+
只是退回到 LLM 解析路径。
|
|
636
|
+
"""
|
|
637
|
+
if config is None:
|
|
638
|
+
try:
|
|
639
|
+
config = load_config()
|
|
640
|
+
except Exception:
|
|
641
|
+
return ''
|
|
642
|
+
try:
|
|
643
|
+
return decrypt_lock(config.get('typesafeApiKey', ''))
|
|
644
|
+
except Exception:
|
|
645
|
+
return ''
|
|
646
|
+
|
|
647
|
+
|
|
443
648
|
# ======================================================================
|
|
444
649
|
# Markdown / Frontmatter
|
|
445
650
|
# ======================================================================
|
|
@@ -558,3 +763,147 @@ def generate_action_suggestion(title: str, source: str, topic: str,
|
|
|
558
763
|
)
|
|
559
764
|
engine = create_engine(engine_type, api_key)
|
|
560
765
|
return engine.chat(prompt, max_tokens=max_tokens)
|
|
766
|
+
|
|
767
|
+
# ---------------------------------------------------------------- 知识卡片的公共件
|
|
768
|
+
#
|
|
769
|
+
# `chat_notes.py`(对话线)与 `article_notes.py`(文章线)产出的是**同一种卡片**:
|
|
770
|
+
# 带 frontmatter + `## AI 摘要` + `[[概念]] — 说明` 的 markdown,都喂给 `compile_wiki`。
|
|
771
|
+
# 所以这几样放在这里,一份实现两边用——两处各写一份,早晚有一处会漏。
|
|
772
|
+
|
|
773
|
+
SEPARATOR = '—' # 与 `compile_wiki` 认的破折号一致(那条描述要能被它读回描述)
|
|
774
|
+
|
|
775
|
+
# 文件名里不许出现的字符(Windows 一套 + 控制字符)
|
|
776
|
+
_ILLEGAL_IN_NAME = set('\\/:*?"<>|') | {chr(code) for code in range(0, 32)}
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
def plain(text) -> str:
|
|
780
|
+
"""自由文本里的 `[[…]]` 一律拆掉。
|
|
781
|
+
|
|
782
|
+
纪律是「wikilink 只由我们的代码渲染,不由模型写」:`compile_wiki.scan_articles` 收正文里
|
|
783
|
+
**所有** wikilink,它不会问这是谁写的——模型在摘要里随手一个 `[[X]]`,就会凭空长出一个概念。
|
|
784
|
+
"""
|
|
785
|
+
out, index = [], 0
|
|
786
|
+
source = str(text or '')
|
|
787
|
+
while True:
|
|
788
|
+
start = source.find('[[', index)
|
|
789
|
+
if start < 0:
|
|
790
|
+
out.append(source[index:])
|
|
791
|
+
break
|
|
792
|
+
end = source.find(']]', start)
|
|
793
|
+
if end < 0:
|
|
794
|
+
out.append(source[index:])
|
|
795
|
+
break
|
|
796
|
+
out.append(source[index:start])
|
|
797
|
+
out.append(source[start + 2:end])
|
|
798
|
+
index = end + 2
|
|
799
|
+
return ''.join(out).replace('[[', '').replace(']]', '')
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def note_path(out_root, name: str, fallback: str = '未命名'):
|
|
803
|
+
"""一张卡一个文件。名字要过文件名安全:会话/文章名里有 `/` 会写到别的目录去。"""
|
|
804
|
+
from pathlib import Path as _Path
|
|
805
|
+
cleaned = ''.join('_' if ch in _ILLEGAL_IN_NAME else ch for ch in str(name))
|
|
806
|
+
cleaned = ' '.join(cleaned.split())[:80].strip().strip('.')
|
|
807
|
+
return _Path(out_root) / ('%s.md' % (cleaned or fallback))
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
# ---------------------------------------------------------------- 微信正文里的界面残留
|
|
811
|
+
|
|
812
|
+
# 微信文章正文会带**阅读器自己的界面文字**("在小说阅读器读本章"那一串),以及文末的
|
|
813
|
+
# 原创标记与公众号名重复。实测:库里 1633 篇笔记,**1484 篇(91%)**的摘要里混着它
|
|
814
|
+
# (先按"整份文件里出现某条短语"数是 330,那是数漏了——短语有好几种写法,按摘要层数才对)。
|
|
815
|
+
# 这份清理在真库上跑过:1484 篇 → 清理后 **0 篇**仍带残留。
|
|
816
|
+
#
|
|
817
|
+
# `classify_daily.py` 里本来有一份 `AD_PATTERNS`,但它只用在日报那条路上,
|
|
818
|
+
# **写笔记那条路(`create_reading_notes.py`)一次都没清洗**——而且对着真串比对,
|
|
819
|
+
# 那份表还漏了一条:真串是「在**公众号**小说中沉浸阅读」,表里写的是
|
|
820
|
+
# 「在小说阅读器**中**沉浸阅读」。所以这里一次列全,各处共用。
|
|
821
|
+
_AD_PHRASES = (
|
|
822
|
+
'在小说阅读器读本章',
|
|
823
|
+
'在小说阅读器中沉浸阅读',
|
|
824
|
+
'在公众号小说中沉浸阅读',
|
|
825
|
+
'去阅读',
|
|
826
|
+
'Scan to Follow',
|
|
827
|
+
'轻触阅读原文',
|
|
828
|
+
'预览时标签不可点',
|
|
829
|
+
'继续滑动看下一个',
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
_UNDERSCORE_RUN = None # 惰性编译,避免在 import 期做什么重活
|
|
833
|
+
_JS_LINK = None
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def strip_wx_ads(text: str) -> str:
|
|
837
|
+
"""去掉微信正文里那几样界面残留。
|
|
838
|
+
|
|
839
|
+
处理顺序有讲究:先删短语,再删下划线长串,最后收拾"原创 + 公众号名重复 N 遍"。
|
|
840
|
+
只动**已知的**几样——这是给摘要与提示词做清洁,不是改写正文;
|
|
841
|
+
拿不准的一律留着(宁可留一点噪音,也不要把用户的话删掉)。
|
|
842
|
+
"""
|
|
843
|
+
import re as _re
|
|
844
|
+
global _UNDERSCORE_RUN
|
|
845
|
+
if _UNDERSCORE_RUN is None:
|
|
846
|
+
_UNDERSCORE_RUN = _re.compile(r'_{4,}')
|
|
847
|
+
out = str(text or '')
|
|
848
|
+
for phrase in _AD_PHRASES:
|
|
849
|
+
out = out.replace(phrase, ' ')
|
|
850
|
+
out = _UNDERSCORE_RUN.sub(' ', out)
|
|
851
|
+
global _JS_LINK
|
|
852
|
+
if _JS_LINK is None:
|
|
853
|
+
_JS_LINK = _re.compile(r'\[.*?\]\(javascript:void\(0\);?\)')
|
|
854
|
+
out = _JS_LINK.sub(' ', out)
|
|
855
|
+
# 「原创 开源星探 开源星探 开源星探」:公众号名在文末被重复粘了几遍,压成一个
|
|
856
|
+
out = _re.sub(r'原创\s+(\S{2,12})(?:\s+\1\b){1,}\s*', ' ', out)
|
|
857
|
+
out = _re.sub(r'原创\s+[一-龥A-Za-z0-9]{2,12}\s*$', '', out)
|
|
858
|
+
out = _re.sub(r'\b(\S{2,12})(?:\s+\1\b)+', r'\1', out)
|
|
859
|
+
out = _re.sub(r'[ \t]{2,}', ' ', out)
|
|
860
|
+
out = _re.sub(r'\n{3,}', '\n\n', out)
|
|
861
|
+
# 短语被删掉之后可能留下孤零零的空白或行首标点
|
|
862
|
+
out = _re.sub(r'[ \t]+([,。、;:!?])', r'\1', out)
|
|
863
|
+
return out.strip()
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
_FENCED_BLOCK = None
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def strip_wx_ads_document(text: str) -> str:
|
|
870
|
+
"""整篇原文用的清洗:**frontmatter 与围栏代码块一个字不动**,其余同 `strip_wx_ads`。
|
|
871
|
+
|
|
872
|
+
删短语的规则**完全复用** `strip_wx_ads`,不另写一套 —— 两处各写一份,迟早会对
|
|
873
|
+
"什么算残留"给出不同答案(本仓库因为这类分叉吃过亏)。
|
|
874
|
+
|
|
875
|
+
两处防护各有实测理由:
|
|
876
|
+
|
|
877
|
+
- **frontmatter**:`去阅读` 是**短词**,完全可能出现在标题里("如何去阅读一本书")。
|
|
878
|
+
标题被改就等于改文件名对齐关系 —— 笔记靠 `日期-标题` 找原文,改了就对不上了。
|
|
879
|
+
实测当前 35,060 篇里 0 篇的 frontmatter 含短语,所以这一条是**防未来**,不是修现在。
|
|
880
|
+
- **围栏代码块**:`strip_wx_ads` 末尾那几条空白规整(`[ \\t]{2,}` → ' '、`\\n{3,}` → `\\n\\n`)
|
|
881
|
+
对摘要是无损的,对**原文**却会吃掉代码缩进 —— 实测 20 篇带围栏、其中 10 篇围栏内容会变
|
|
882
|
+
(HTML/JS 示例)。原文是素材,缩进就是内容。
|
|
883
|
+
"""
|
|
884
|
+
if text.startswith('---'):
|
|
885
|
+
end = text.find('\n---', 3)
|
|
886
|
+
front, body = (text[:end + 4], text[end + 4:]) if end != -1 else ('', text)
|
|
887
|
+
else:
|
|
888
|
+
front, body = '', text
|
|
889
|
+
# `strip_wx_ads` 末尾带 `.strip()`,会把正文首尾空白吃掉。frontmatter 与正文之间那两行
|
|
890
|
+
# 空行是正文的一部分,所以这里把首尾空白单独摘出来、洗完再放回去。
|
|
891
|
+
lead = body[:len(body) - len(body.lstrip())]
|
|
892
|
+
tail = body[len(body.rstrip()):]
|
|
893
|
+
core = body.strip()
|
|
894
|
+
global _FENCED_BLOCK
|
|
895
|
+
if _FENCED_BLOCK is None:
|
|
896
|
+
import re as _re
|
|
897
|
+
_FENCED_BLOCK = _re.compile(r'^```.*?^```', _re.S | _re.M)
|
|
898
|
+
# 占位符用 chr(0) 拼,不写字面反斜杠:本仓有"JSON 层吃掉转义"的前科,能避开就避开。
|
|
899
|
+
mark = chr(0) + 'FENCE' + chr(0)
|
|
900
|
+
blocks = []
|
|
901
|
+
|
|
902
|
+
def stash(match):
|
|
903
|
+
blocks.append(match.group(0))
|
|
904
|
+
return mark + str(len(blocks) - 1) + mark
|
|
905
|
+
|
|
906
|
+
cleaned = strip_wx_ads(_FENCED_BLOCK.sub(stash, core))
|
|
907
|
+
for index, block in enumerate(blocks):
|
|
908
|
+
cleaned = cleaned.replace(mark + str(index) + mark, block)
|
|
909
|
+
return front + lead + cleaned + tail
|
package/scripts/annual_report.py
CHANGED
|
@@ -21,7 +21,7 @@ except ImportError:
|
|
|
21
21
|
sys.exit(1)
|
|
22
22
|
|
|
23
23
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
24
|
-
from _utils import load_config, decrypt_lock, parse_frontmatter, call_deepseek
|
|
24
|
+
from _utils import load_config, decrypt_lock, parse_frontmatter, call_deepseek, get_api_key
|
|
25
25
|
|
|
26
26
|
TZ = timezone(timedelta(hours=8))
|
|
27
27
|
OUTPUT_ROOT = 'output'
|
|
@@ -733,7 +733,9 @@ def main():
|
|
|
733
733
|
|
|
734
734
|
year = args.year
|
|
735
735
|
config = load_config()
|
|
736
|
-
|
|
736
|
+
# 一律走 get_api_key:它会解密 lock: 前缀,而直接 config.get 拿到的是密文。
|
|
737
|
+
# 原先这四处直接读,恰好因为本机那把 key 是明文而没暴露。
|
|
738
|
+
api_key = args.api_key or os.environ.get('DEEPSEEK_API_KEY', '') or get_api_key(config)
|
|
737
739
|
|
|
738
740
|
print(f'📊 正在生成 {year} 年度报告...')
|
|
739
741
|
|