weflow-cli 1.5.0 → 1.6.1

package/scripts/biz_daily.py

@@ -27,7 +27,7 @@ from pathlib import Path
 
 # 公共工具
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from _utils import call_deepseek, load_config, decrypt_lock, write_with_frontmatter, format_wikilinks
+from _utils import call_deepseek, load_config, decrypt_lock, get_api_key, write_with_frontmatter, format_wikilinks
 
 try:
     from sqlcipher3 import dbapi2 as sqlcipher
@@ -56,7 +56,31 @@ DEEPSEEK_TIMEOUT = 60
 FETCH_DELAY_MIN = 8   # 最小抓取间隔 (秒)
 FETCH_DELAY_MAX = 12  # 最大抓取间隔 (秒)
 
-TOPICS = ['AI', '学术', '新闻', '文学', '投资']
+TOPICS = ['AI', '学术', '新闻', '文学', '投资', '政治']
+
+
+def load_source_categories(config: dict) -> dict[str, str]:
+    """Load the explicit official-account category map from CLI config."""
+    raw = config.get('dailySourceCategories', '')
+    if not raw:
+        return {}
+    try:
+        data = json.loads(raw)
+    except (TypeError, json.JSONDecodeError):
+        print('[WARN] dailySourceCategories 不是有效 JSON，忽略来源类别配置')
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    return {
+        str(name).strip().casefold(): str(category).strip()
+        for name, category in data.items()
+        if str(name).strip() and str(category).strip() in TOPICS
+    }
+
+
+def get_source_category(categories: dict[str, str], username: str, display_name: str) -> str:
+    """Match a category by official-account ID first, then display name."""
+    return categories.get(username.casefold()) or categories.get(display_name.casefold()) or ''
 def _guess_topic(article: dict) -> str:
     """Keyword-based topic guess when AI classification fails."""
     title = article.get('title', '')
@@ -90,36 +114,40 @@ def _guess_topic(article: dict) -> str:
     return '新闻'
 
 
-TOPIC_PROMPT = f"""对文章分类、摘要、打标签，并评估与读者的相关度。
+TOPIC_PROMPT = f"""对文章分类、深度摘要、打标签，并评估与读者的相关度。
 
 【读者定位】环境科学研究生，研究方向是计算机与环境的交叉领域（环境模型、大气污染模拟、遥感反演、环境大数据分析、LCA等），关注AI工具如何提升科研效率。
 
-【主题】必须且只能是：{' / '.join(TOPICS)} 中的一个词，不要写其他任何文字。
+【主题】必须且只能是：{' / '.join(TOPICS)} 中的一个词。
 
 判断规则：
-- AI：涉及AI大模型/Agent/编程/开源/科技产品/工具教程 → 归AI
-- 投资：涉及股票基金/融资/经济分析/商业市场 → 归投资
-- 新闻：时事政策/社会热点/娱乐圈/招聘促销/会议通知 → 归新闻
-- 文学：散文小说/美食旅游/生活随笔/历史文化 → 归文学
-- 学术：严格科研论文/学术期刊/实验室研究/学位论文 → 归学术（公众号文章极少属此类）
+- AI：AI大模型/Agent/编程/开源/科技产品/工具教程
+- 投资：股票基金/融资/经济分析/商业市场
+- 新闻：时事政策/社会热点/娱乐/招聘促销/会议通知
+- 文学：散文小说/美食旅游/生活随笔/历史文化
+- 学术：科研论文/期刊文章/实验室成果/学术会议/高校研究（环境/气候/地理/海洋/生态/化学/材料/生物医学等）
 
-【相关度】判断这篇文章对上述读者的实用价值：
+【相关度】对上述读者的实用价值：
 - 高：可直接用于科研（新工具/新方法/数据源/代码库）
-- 中：有启发性，需转化后使用（思路/趋势/跨领域技术）
-- 低：信息性阅读，无直接行动价值（纯新闻/娱乐/文学）
+- 中：有启发性（思路/趋势/跨领域技术）
+- 低：信息性阅读（纯新闻/娱乐/文学）
+
+**分类关键**：
+- 【主题】只写一个词（AI/学术/新闻/文学/投资），不要写其他文字
+- 【相关度】只写一个词（高/中/低）
+- 科研论文、期刊文章优先归学术；AI技术/工具/产品归AI
 
-**关键**：
-- 【主题】这一行只写一个词：AI 或 学术 或 新闻 或 文学 或 投资
-- 【相关度】只写：高 / 中 / 低
-- 科技报道、开发者工具、AI产品即使提到Nature/Science也归AI或新闻，不归学术
-- 不确定时选最可能的非学术分类
+**摘要要求**：
+- 【摘要】写一段完整的深度摘要（150-300字），不要只写一两句
+- 格式：【核心观点】一句话概括中心思想。【关键细节】列出3-5个具体要点（工具/方法/数据/结论/人物/事件等），每个要点一句话
+- 摘要不需要包含分类/相关度/标签信息，那些由上面的字段处理
 
 返回格式（严格）：
 【主题】AI
 【相关度】高
 【标签】tag1, tag2, tag3
-【摘要】2-4句中文总结
-【概念】概念名|说明, 概念名|说明"""
+【摘要】【核心观点】一句话。【关键细节】1. 要点一；2. 要点二；3. 要点三
+【概念】概念名|一句话说明, 概念名|一句话说明"""
 
 # ====== Helpers ======
 
@@ -141,16 +169,35 @@ def get_db_keys(config):
     contact_salt = config.get('contactSalt', '')
     contact_key = decrypt_lock(contact_key_enc) if contact_key_enc else ''
 
-    # biz_message_0.db (separate key from message/contact db)
+    # biz_message_0.db (每库独立密钥)
     biz_db = os.path.join(msg_dir, 'biz_message_0.db')
-    # Try config first, then fallback to known key from scan
     biz_key_enc = config.get('bizKey', '')
     biz_salt = config.get('bizSalt', '')
+    pass_enc = config.get('favPassphrase', '')
+    passphrase = decrypt_lock(pass_enc) if pass_enc else ''
+    if passphrase and os.path.exists(biz_db):
+        with open(biz_db, 'rb') as fh:
+            biz_salt = fh.read(16).hex()
+        biz_key_enc = ''
     if biz_key_enc and biz_salt:
         biz_key = decrypt_lock(biz_key_enc)
     else:
-        print('[ERROR] 缺少 biz_message_0.db 密钥，请运行: python scripts/nt_decrypt.py scan --json')
-        sys.exit(1)
+        # 从全库共用 passphrase 派生 (微信 4.x: PBKDF2-HMAC-SHA512, 256000 轮, 32 字节)
+        pass_enc = config.get('favPassphrase', '')
+        passphrase = decrypt_lock(pass_enc) if pass_enc else ''
+        biz_key = ''
+        if passphrase and os.path.exists(biz_db):
+            with open(biz_db, 'rb') as fh:
+                biz_salt = fh.read(16).hex()
+            biz_key = hashlib.pbkdf2_hmac(
+                'sha512', bytes.fromhex(passphrase),
+                bytes.fromhex(biz_salt), 256000, dklen=32
+            ).hex()
+        if not biz_key:
+            raise SystemExit(
+                '缺少公众号数据库密钥: 请运行 weflow-cli fav set-key --passphrase <64位hex> '
+                '配置全库 passphrase (自动派生各库密钥), 或 weflow-cli config set bizKey <raw key>'
+            )
 
     return {
         'biz_db': biz_db, 'biz_key': biz_key, 'biz_salt': biz_salt,
@@ -317,6 +364,8 @@ def download_images_to_local(markdown: str, images_dir: Path) -> tuple[str, dict
 def extract_article_info(content_bytes: bytes) -> dict:
     """Extract title, digest, URL, full text from protobuf message_content."""
     import zstandard as zstd
+    if isinstance(content_bytes, str):
+        content_bytes = content_bytes.encode('utf-8', errors='ignore')
     dctx = zstd.ZstdDecompressor()
     try:
         text = dctx.decompress(content_bytes).decode('utf-8', errors='ignore')
@@ -384,6 +433,19 @@ def sanitize_filename(name: str) -> str:
     return re.sub(r'[\\/:*?"<>|]', '_', name)[:80]
 
 
+def parse_source_filters(values) -> list[str]:
+    """Normalize repeated or comma-separated official-account filters."""
+    filters = []
+    seen = set()
+    for value in values or []:
+        for item in re.split(r'[,;\n]+', str(value or '')):
+            normalized = item.strip().strip('"\'')
+            if normalized and normalized.casefold() not in seen:
+                seen.add(normalized.casefold())
+                filters.append(normalized)
+    return filters
+
+
 # ====== Main ======
 
 def main():
@@ -394,7 +456,11 @@ def main():
     parser.add_argument('--date', help='日期 YYYY-MM-DD, 默认今天')
     parser.add_argument('--dry-run', action='store_true', help='仅预览，不抓取')
     parser.add_argument('--limit', type=int, default=0, help='最多处理篇数 (0=不限制)')
-    parser.add_argument('--api-key', help='DeepSeek API key (或设环境变量 DEEPSEEK_API_KEY)')
+    parser.add_argument('--api-key', help='AI API key (或设环境变量 DEEPSEEK_API_KEY)')
+    parser.add_argument('--engine', default='deepseek', help='AI 引擎: local/deepseek/claude/ollama')
+    parser.add_argument('--no-ai', action='store_true', help='关闭摘要、分类和日报简报的 AI 调用')
+    parser.add_argument('--source', action='append', default=[], metavar='NAME',
+                        help='仅处理指定公众号，可重复或用逗号分隔；默认读取配置 dailySources')
     args = parser.parse_args()
 
     # Date
@@ -409,16 +475,27 @@ def main():
 
     # Load config & DB keys
     config = load_config()
+    if str(config.get('dailyAiEnabled', 'true')).strip().lower() == 'false':
+        args.no_ai = True
     keys = get_db_keys(config)
-
-    # API key
-    api_key = args.api_key or os.environ.get('DEEPSEEK_API_KEY', '') or config.get('deepseekApiKey', '')
-    if not args.dry_run and not api_key:
-        print('[ERROR] 缺少 DeepSeek API key。请通过 --api-key、环境变量 DEEPSEEK_API_KEY 或 ~/.weflow-cli/config.json 中的 deepseekApiKey 提供')
+    source_filters = parse_source_filters(args.source or [config.get('dailySources', '')])
+    source_categories = load_source_categories(config)
+
+    # API key — only required for cloud engines
+    engine = args.engine or 'deepseek'
+    api_key = args.api_key or os.environ.get('DEEPSEEK_API_KEY', '') or get_api_key(config)
+    if not args.dry_run and engine in ('deepseek', 'claude') and not api_key:
+        print(f'[ERROR] --engine {engine} 需要 API key。请通过 --api-key、环境变量或配置文件提供')
         sys.exit(1)
+    # Auto-detect local engine if no api_key and engine is deepseek
+    if not api_key and engine == 'deepseek':
+        engine = 'local'
+        print(f'  无 API key，切换到本地引擎')
 
     print(f'=== 公众号日报 {date_str} ===')
     print(f'Biz DB: {keys["biz_db"]}')
+    if source_filters:
+        print(f'公众号筛选: {", ".join(source_filters)}')
 
     # Connect biz db
     raw_key = f"x'{keys['biz_key']}{keys['biz_salt']}'"
@@ -443,6 +520,16 @@ def main():
     # Find today's articles
     c.execute("SELECT user_name FROM Name2Id WHERE user_name LIKE 'gh_%'")
     biz_users = [row[0] for row in c.fetchall()]
+    if source_filters:
+        filter_keys = {item.casefold() for item in source_filters}
+        biz_users = [
+            user for user in biz_users
+            if user.casefold() in filter_keys
+            or str(name_map.get(user, user)).casefold() in filter_keys
+        ]
+        print(f'匹配公众号: {len(biz_users)} 个')
+        if not biz_users:
+            print('[WARN] 没有匹配的公众号；请检查昵称或 gh_ ID')
 
     articles = []
     for user in biz_users:
@@ -465,6 +552,9 @@ def main():
                 articles.append({
                         'account': user,
                         'account_name': name_map.get(user, user),
+                        'source_category': get_source_category(
+                            source_categories, user, name_map.get(user, user)
+                        ) if source_filters else '',
                         'title': info['title'],
                         'digest': info['digest'],
                         'url': info['url'],
@@ -536,26 +626,49 @@ def main():
         elif a.get('local_text'):
             print(f'  无URL, 使用本地缓存')
 
-    # ====== Phase 2: DeepSeek summary + topic classification ======
-    if api_key:
-        print(f'\n=== Phase 2: AI 摘要 + 主题分类 ===\n')
+    # ====== Phase 2: AI summary + topic classification ======
+    if not args.no_ai and (api_key or engine != 'deepseek'):
+        print(f'\n=== Phase 2: AI 摘要 + 主题分类 (engine={engine}) ===\n')
+        from _utils import call_ai
         for i, a in enumerate(articles):
             t, n, ti = a['time'], a['account_name'], a['title']
             content = a.get('fetched_md') or a.get('local_text', '')
             if content and len(content.strip()) > 50:
                 try:
-                    prompt = TOPIC_PROMPT + f'\n\n标题：{a["title"]}\n来源：{a["account_name"]}\n\n内容：\n{content[:4000]}'
-                    response = call_deepseek(prompt, api_key, max_tokens=600)
+                    category_hint = a.get('source_category', '')
+                    if category_hint:
+                        summary_prompt = f'''请只为下面这篇公众号文章生成一段 50-300 字的中文摘要。
+来源类别已经确定为「{category_hint}」，不要重新判断或改写文章分类，不要输出主题、标签、相关度或概念字段。
+
+标题：{a["title"]}
+来源：{a["account_name"]}
+
+正文：
+{content[:4000]}'''
+                        response = call_ai(summary_prompt, engine, api_key, max_tokens=1000)
+                        a['topic'] = category_hint
+                        a['tags'] = [category_hint]
+                        a['summary'] = response.strip()[:1000]
+                        a['relevance'] = '中'
+                        a['concepts'] = []
+                        print(f'[{i+1}/{len(articles)}] [{t}] {n} - [{category_hint}] 固定来源类别，仅生成摘要')
+                        time.sleep(0.3)
+                        continue
+                    prompt = TOPIC_PROMPT + f'\n\n标题：{a["title"]}\n来源：{a["account_name"]}'
+                    prompt += f'\n\n内容：\n{content[:4000]}'
+                    response = call_ai(prompt, engine, api_key, max_tokens=2000)
 
                     # Parse response: 【主题】xxx 【相关度】xxx 【标签】xxx 【摘要】xxx 【概念】xxx
                     topic_match = re.search(r'【主题】\s*(.+)', response)
                     relevance_match = re.search(r'【相关度】\s*(.+)', response)
                     tags_match = re.search(r'【标签】\s*(.+)', response)
-                    # Stop summary at next 【tag or end
-                    summary_match = re.search(r'【摘要】\s*(.+?)(?=\n【|$)', response, re.DOTALL)
+                    # Capture summary - stop at next top-level 【tag (主题/相关度/标签/概念) or end
+                    summary_match = re.search(r'【摘要】\s*(.+?)(?=\n【(?:主题|相关度|标签|概念)】|$)', response, re.DOTALL)
                     concepts_match = re.search(r'【概念】\s*(.+)', response, re.DOTALL)
 
-                    if topic_match:
+                    if category_hint:
+                        a['topic'] = category_hint
+                    elif topic_match:
                         raw_topic = topic_match.group(1).strip()
                         if raw_topic in TOPICS:
                             a['topic'] = raw_topic
@@ -589,7 +702,9 @@ def main():
                         a['relevance'] = '中'
 
                     # Parse tags: comma-separated, clean up
-                    if tags_match:
+                    if category_hint:
+                        a['tags'] = [category_hint]
+                    elif tags_match:
                         a['tags'] = [t.strip() for t in tags_match.group(1).split(',') if t.strip()]
                     else:
                         a['tags'] = [a['topic']]
@@ -617,16 +732,16 @@ def main():
                     time.sleep(0.3)
                 except Exception as e:
                     a['summary'] = a.get('digest', '') or content[:300]
-                    a['topic'] = '学术'
-                    a['tags'] = ['学术']
+                    a['topic'] = a.get('source_category') or '学术'
+                    a['tags'] = [a['topic']]
                     a['concepts'] = []
                     print(f'[{i+1}/{len(articles)}] [{t}] {n} - ERR: {e}')
             elif content:
                 a['summary'] = content[:400]
-                a['topic'] = '学术'
+                a['topic'] = a.get('source_category') or '学术'
             else:
                 a['summary'] = a.get('digest', '(无内容)')
-                a['topic'] = '学术'
+                a['topic'] = a.get('source_category') or '学术'
 
         # Print topic distribution
         from collections import Counter
@@ -787,7 +902,7 @@ def main():
 
     # Generate AI briefing from all article files on disk
     briefing = ''
-    if api_key and all_articles:
+    if all_articles:
         topic_counts = {}
         for a in all_articles:
             t = a['topic']
@@ -821,13 +936,13 @@ def main():
                                 break
                     if summary:
                         highlights.append(f"[{topic}] {title} | {summary}")
-                except:
+                except Exception:
                     pass
             if len(highlights) >= 25:
                 break
 
         highlight_text = '\n'.join(highlights[:25])
-        if highlight_text:
+        if highlight_text and not args.no_ai:
             try:
                 briefing_prompt = f"""你是公众号日报助手。基于今天 {len(all_articles)} 篇文章（{topic_summary}），生成一段200字以内的今日简报。
 
@@ -840,13 +955,37 @@ def main():
 {highlight_text}
 
 请直接输出简报内容（不要标题）。"""
-                briefing = call_deepseek(briefing_prompt, api_key, max_tokens=500, timeout=90).strip()
+                # Use call_ai to support local/deepseek/claude engines
+                from _utils import call_ai
+                engine_type = 'deepseek' if api_key else 'local'
+                briefing = call_ai(briefing_prompt, engine_type, api_key, max_tokens=500).strip()
                 if briefing:
                     print(f'\n  ✓ 简报生成完成')
                 else:
-                    print(f'\n  [WARN] 简报 API 返回空，跳过')
+                    print(f'\n  [WARN] 简报 API 返回空，使用摘要拼接')
+                    briefing = ''
             except Exception as e:
-                print(f'\n  [WARN] 简报生成失败: {e}')
+                print(f'\n  [WARN] 简报生成失败: {e}，使用摘要拼接')
+                briefing = ''
+
+        # Fallback: build briefing from highlights if AI failed
+        if not briefing and highlights:
+            topic_groups = {}
+            for h in highlights[:15]:
+                parts = h.split(' | ', 1)
+                title_part = parts[0].replace('[', '').replace(']', ' - ')
+                summary_part = parts[1] if len(parts) > 1 else ''
+                topic = h.split(']')[0].replace('[', '')
+                if topic not in topic_groups:
+                    topic_groups[topic] = []
+                if len(topic_groups[topic]) < 3:
+                    topic_groups[topic].append(f"{title_part.split(' - ', 1)[-1]}：{summary_part[:60]}")
+            parts = []
+            for topic in TOPICS:
+                if topic in topic_groups:
+                    items = topic_groups[topic]
+                    parts.append(f"{topic}：{'；'.join(items)}")
+            briefing = '今日共收录' + str(len(all_articles)) + '篇文章。' + '。'.join(parts) if parts else f'今日共收录{len(all_articles)}篇文章，涵盖{topic_summary}。'
 
     index_lines = [
         f'# 公众号日报 — {date_str}',
@@ -882,12 +1021,30 @@ def main():
         if count:
             print(f'  {t}/: {count} 篇')
 
-    # Save run state for incremental dedup
+    # Save run state for incremental dedup (atomically)
+    # Only mark articles that were actually written to disk
+    written_urls = set()
+    for topic in TOPICS:
+        topic_dir = out_dir / topic
+        if not topic_dir.is_dir():
+            continue
+        for md_file in topic_dir.glob('*.md'):
+            try:
+                content = md_file.read_text(encoding='utf-8')
+                for line in content.split('\n'):
+                    if line.strip().startswith('url:'):
+                        url = line.split(':', 1)[1].strip().strip('"').strip("'")
+                        if url:
+                            written_urls.add(url)
+                        break
+            except Exception:
+                pass
     for a in topic_groups.values():
-        for art in a:
-            fp = hashlib.sha256(art.get('url', '').encode()).hexdigest()[:16]
-            if art.get('url'):
-                processed[fp] = art['title'][:50]
+        if isinstance(a, list):
+            for art in a:
+                if isinstance(art, dict) and art.get('url') and art['url'] in written_urls:
+                    fp = hashlib.sha256(art['url'].encode()).hexdigest()[:16]
+                    processed[fp] = art.get('title', '')[:50]
     with open(state_file, 'w', encoding='utf-8') as f:
         json.dump(processed, f, ensure_ascii=False, indent=2)
 

package/scripts/chat_report.py

@@ -152,6 +152,16 @@ def main():
     p.add_argument('--output', '-o', default=OUTPUT_ROOT)
     args = p.parse_args()
 
+    env_talkers = os.environ.get('WEFLOW_REPORT_TALKERS', '')
+    if env_talkers and not args.talker:
+        try:
+            parsed_talkers = json.loads(env_talkers)
+            if isinstance(parsed_talkers, list):
+                args.talker = [str(item) for item in parsed_talkers if str(item).strip()]
+        except (TypeError, ValueError, json.JSONDecodeError):
+            print('[ERROR] invalid report talker selection')
+            sys.exit(1)
+
     # Time range
     tz = timezone(timedelta(hours=8))
     if args.month: