weflow-cli 1.5.0

package/scripts/pipeline.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+端到端流水线 — biz_daily → classify_daily → wiki compile → AI report 一键串联。
+
+默认策略：
+- AI 推理优先走本地（Ollama/LM Studio/Claude Code 本地服务），不需要 API key
+- 如果没有本地服务，再显式指定 --engine deepseek/claude
+
+用法:
+  python scripts/pipeline.py                     # 全部步骤，默认走本地推理
+  python scripts/pipeline.py --engine deepseek --api-key <key>
+  python scripts/pipeline.py --engine claude --api-key <key>
+  python scripts/pipeline.py --date 2026-06-09 --skip-classify
+"""
+import sys, os, subprocess, time, shutil
+from pathlib import Path
+
+SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
+SOURCE_ROOT = 'output/biz-daily'
+PYTHON = sys.executable
+
+
+def run_step(name: str, args: list[str]) -> bool:
+    """运行一个步骤，返回是否成功。"""
+    print(f'\n{"="*50}')
+    print(f'  Step: {name}')
+    print(f'{"="*50}')
+    result = subprocess.run([PYTHON] + args, cwd=os.path.dirname(SCRIPTS_DIR))
+    if result.returncode != 0:
+        print(f'\n[FAIL] {name} 失败 (exit={result.returncode})')
+        return False
+    return True
+
+
+def main():
+    sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+    import argparse
+    parser = argparse.ArgumentParser(description='端到端公众号日报流水线')
+    parser.add_argument('--api-key', help='云端 API key（--engine 为 deepseek/claude 时需要）')
+    parser.add_argument('--engine', default='local',
+                        help='AI 引擎: local(自动检测·默认) / deepseek / claude / ollama')
+    parser.add_argument('--date', help='日期 YYYY-MM-DD')
+    parser.add_argument('--interest', default='AI', help='兴趣主题（默认 AI）')
+    parser.add_argument('--wiki-limit', type=int, default=20, help='概念编译数量（默认 20）')
+    parser.add_argument('--skip-classify', action='store_true', help='跳过后处理')
+    parser.add_argument('--skip-wiki', action='store_true', help='跳过概念编译')
+    parser.add_argument('--skip-vault', action='store_true', help='跳过 Vault 同步')
+    parser.add_argument('--skip-html', action='store_true', help='跳过 HTML 生成')
+    parser.add_argument('--skip-ai-report', action='store_true', help='跳过 AI 深度阅读报告')
+    parser.add_argument('--ai-report-range', type=int, default=1, help='AI 报告覆盖最近 N 天（默认 1=仅当天）')
+    args = parser.parse_args()
+
+    sys.path.insert(0, SCRIPTS_DIR)
+    from _utils import load_config
+    config = load_config()
+
+    # api_key：本地/ollama 引擎不需要；云端需要
+    api_key = args.api_key or ''
+    if args.engine in ('deepseek', 'claude') and not api_key:
+        api_key = os.environ.get('DEEPSEEK_API_KEY', '') or config.get('deepseekApiKey', '')
+        if not api_key:
+            print(f'[ERROR] --engine {args.engine} 需要 API key。请通过 --api-key、'
+                  f'环境变量 DEEPSEEK_API_KEY 或 ~/.weflow-cli/config.json 提供')
+            sys.exit(1)
+
+    started = time.time()
+
+    # Step 1: biz_daily
+    step1_args = [os.path.join(SCRIPTS_DIR, 'biz_daily.py')]
+    if api_key:
+        step1_args += ['--api-key', api_key]
+    if args.date:
+        step1_args += ['--date', args.date]
+    if not run_step('biz_daily — 抓取+摘要', step1_args):
+        sys.exit(1)
+
+    # Step 2: classify_daily（可选）
+    if not args.skip_classify:
+        step2_args = [
+            os.path.join(SCRIPTS_DIR, 'classify_daily.py'),
+            '--api-key', api_key if api_key else 'local',
+            '--interest', args.interest,
+        ]
+        if args.date:
+            step2_args.insert(1, args.date)
+        run_step('classify_daily — 后处理', step2_args)
+
+    # Step 3: Vault sync（可选）
+    if not args.skip_vault:
+        date_str = args.date or time.strftime('%Y-%m-%d')
+        source_dir = os.path.join(SOURCE_ROOT, date_str)
+        vault_dir = os.path.join(os.path.dirname(SCRIPTS_DIR), 'output', 'wechat-vault',
+                                 'Sources', 'WeChat', date_str)
+        if os.path.exists(source_dir):
+            if os.path.exists(vault_dir):
+                shutil.rmtree(vault_dir)
+            shutil.copytree(source_dir, vault_dir)
+            file_count = sum(1 for _ in Path(vault_dir).rglob('*.md'))
+            print(f'\n  Vault 同步: {file_count} 个文件 → {vault_dir}')
+
+    # Step 4: wiki compile（可选）
+    if not args.skip_wiki:
+        step3_args = [
+            os.path.join(SCRIPTS_DIR, 'compile_wiki.py'),
+            '--api-key', api_key if api_key else 'local',
+            '--limit', str(args.wiki_limit),
+        ]
+        run_step('wiki compile — 概念编译', step3_args)
+
+    # Step 5: HTML 生成（可选）
+    if not args.skip_html:
+        date_str = args.date or time.strftime('%Y-%m-%d')
+        run_step('generate_html',
+                 [os.path.join(SCRIPTS_DIR, 'generate_html.py'), '--date', date_str])
+
+    # Step 6: 双向链接增强
+    date_str = args.date or time.strftime('%Y-%m-%d')
+    run_step('enrich_backlinks — 双向链接',
+             [os.path.join(SCRIPTS_DIR, 'enrich_backlinks.py'), '--date', date_str])
+
+    # Step 7: 阅读笔记生成
+    run_step('create_reading_notes — 阅读笔记',
+             [os.path.join(SCRIPTS_DIR, 'create_reading_notes.py'), '--date', date_str])
+
+    # Step 8: AI 深度阅读报告
+    if not args.skip_ai_report:
+        step8_args = [
+            os.path.join(SCRIPTS_DIR, 'generate_ai_report.py'),
+            '--engine', args.engine,
+            '--range', str(args.ai_report_range),
+        ]
+        if api_key:
+            step8_args += ['--api-key', api_key]
+        if args.date and args.ai_report_range == 1:
+            step8_args += ['--date', args.date]
+        run_step('generate_ai_report — AI 阅读日报', step8_args)
+
+    elapsed = time.time() - started
+    print(f'\n{"="*50}')
+    print(f'  流水线完成！耗时 {elapsed/60:.1f} 分钟')
+    print(f'{"="*50}')
+
+
+if __name__ == '__main__':
+    main()

package/scripts/promote_all.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+"""
+全自动知识升级 — 003_Ideas → 004_Permanent → 005_Reference → 006_Projects
+
+用法:
+  python scripts/promote_all.py --api-key <key>
+"""
+
+import sys, os, re
+from pathlib import Path
+from collections import Counter, defaultdict
+from datetime import datetime
+
+SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.dirname(SCRIPTS_DIR)
+DEFAULT_VAULT = os.path.join(PROJECT_ROOT, 'output', 'wechat-vault')
+
+sys.path.insert(0, SCRIPTS_DIR)
+from _utils import call_deepseek
+
+
+def read_frontmatter(path: Path) -> dict:
+    c = path.read_text(encoding='utf-8')
+    if not c.startswith('---'):
+        return {}
+    end = c.find('---', 3)
+    if end == -1:
+        return {}
+    fm = {}
+    for line in c[3:end].split('\n'):
+        if ':' in line:
+            k, _, v = line.partition(':')
+            fm[k.strip()] = v.strip().strip('"').strip("'")
+    return fm
+
+
+def collect_articles(vault_path: str) -> list[dict]:
+    lit = Path(vault_path) / '002_Literature' / 'WeChat'
+    articles = []
+    for d in lit.iterdir():
+        if not d.is_dir(): continue
+        for f in d.glob('*.md'):
+            fm = read_frontmatter(f)
+            articles.append({
+                'title': fm.get('title', f.stem),
+                'source': fm.get('source', ''),
+                'date': d.name,
+                'topic': fm.get('hasTopic', '').replace('[[','').replace(']]',''),
+                'path': f,
+            })
+    return articles
+
+
+def generate_permanent(api_key, vault_path, articles):
+    """004_Permanent: 将 Best Ideas 扩展为原子永久笔记。"""
+    perm_dir = Path(vault_path) / '004_Permanent'
+    perm_dir.mkdir(parents=True, exist_ok=True)
+
+    # 按主题取 articles 样本
+    by_topic = defaultdict(list)
+    for a in articles:
+        t = a['topic'] or '综合'
+        by_topic[t].append(a)
+
+    created = 0
+    for topic, arts in by_topic.items():
+        if len(arts) < 10: continue
+        if (perm_dir / f'Perm-{topic}.md').exists(): continue
+
+        samples = sorted(arts, key=lambda a: len(a['title']))[-10:]
+        titles = '\n'.join(f'- {a["title"][:50]}' for a in samples)
+
+        prompt = f'''基于以下 {topic} 主题文章，写一篇 300 字的原子笔记，提炼该领域的核心知识。
+格式：1个核心概念 → 3个关键洞察 → 1个行动建议。
+
+文章样本：
+{titles}
+
+只返回笔记内容，用 Markdown 格式。'''
+
+        try:
+            text = call_deepseek(prompt, api_key, max_tokens=600)
+        except Exception as e:
+            print(f'  [WARN] {topic}: {e}')
+            continue
+
+        content = f'''---
+title: "{topic} 核心知识"
+created: {datetime.now().strftime('%Y-%m-%d')}
+type: permanent-note
+hasTopic: [[{topic}]]
+tags: [permanent, {topic.lower()}, atomic]
+sources: {len(arts)} articles
+---
+
+# {topic} 核心知识
+
+{text}
+
+---
+
+## 来源文章
+'''
+        for a in arts[:15]:
+            content += f'- [[{a["date"]}/{a["path"].name}|{a["title"][:50]}]]\n'
+
+        (perm_dir / f'Perm-{topic}.md').write_text(content, encoding='utf-8')
+        created += 1
+        print(f'  [{topic}] → Perm-{topic}.md')
+    return created
+
+
+def generate_reference(vault_path, articles):
+    """005_Reference: 自动提取工具和方法论。"""
+    ref_dir = Path(vault_path) / '005_Reference'
+
+    # 关键词扫描
+    tool_keywords = ['工具', '开源', 'GitHub', 'Star', 'CLI', 'API', '框架', '平台', '模型',
+                     '插件', 'Python', 'R包', '库', 'Agent', 'Skill', 'MCP', 'RAG']
+    method_keywords = ['方法', '算法', '流程', '技术', '策略', '方案', '分析', '优化',
+                       '评估', '验证', '建模', '预测', '训练', '调优', '部署']
+
+    tools = defaultdict(list)
+    methods = defaultdict(list)
+
+    for a in articles:
+        title = a['title']
+        for kw in tool_keywords:
+            if kw in title:
+                tools[kw].append(a)
+                break
+        for kw in method_keywords:
+            if kw in title:
+                methods[kw].append(a)
+                break
+
+    created = 0
+
+    # Tools index
+    if tools:
+        (ref_dir / 'Tools').mkdir(parents=True, exist_ok=True)
+        content = f'''---
+title: "工具索引"
+created: {datetime.now().strftime('%Y-%m-%d')}
+type: reference
+tags: [reference, tools]
+---
+
+# 工具索引
+
+> 自动从 {len(articles)} 篇文章中提取
+
+'''
+        for kw in sorted(tools, key=lambda k: len(tools[k]), reverse=True):
+            if len(tools[kw]) < 3: continue
+            content += f'\n## {kw} ({len(tools[kw])} 篇)\n'
+            for a in tools[kw][:5]:
+                content += f'- [[{a["date"]}/{a["path"].name}|{a["title"][:60]}]]\n'
+
+        (ref_dir / 'Tools' / '工具索引.md').write_text(content, encoding='utf-8')
+        created += 1
+        print(f'  Tools: {len(tools)} categories')
+
+    # Methods index
+    if methods:
+        (ref_dir / 'Methods').mkdir(parents=True, exist_ok=True)
+        content = f'''---
+title: "方法论索引"
+created: {datetime.now().strftime('%Y-%m-%d')}
+type: reference
+tags: [reference, methods]
+---
+
+# 方法论索引
+
+> 自动从 {len(articles)} 篇文章中提取
+
+'''
+        for kw in sorted(methods, key=lambda k: len(methods[k]), reverse=True):
+            if len(methods[kw]) < 3: continue
+            content += f'\n## {kw} ({len(methods[kw])} 篇)\n'
+            for a in methods[kw][:5]:
+                content += f'- [[{a["date"]}/{a["path"].name}|{a["title"][:60]}]]\n'
+
+        (ref_dir / 'Methods' / '方法论索引.md').write_text(content, encoding='utf-8')
+        created += 1
+        print(f'  Methods: {len(methods)} categories')
+
+    return created
+
+
+def generate_projects(api_key, vault_path, articles):
+    """006_Projects: AI 生成项目提案。"""
+    proj_dir = Path(vault_path) / '006_Projects'
+    proj_dir.mkdir(parents=True, exist_ok=True)
+
+    # 取最大的主题生成项目
+    by_topic = defaultdict(list)
+    for a in articles:
+        t = a['topic'] or '综合'
+        by_topic[t].append(a)
+
+    # 只对文章数 > 50 的主题生成项目
+    created = 0
+    for topic, arts in sorted(by_topic.items(), key=lambda x: len(x[1]), reverse=True):
+        if len(arts) < 50: continue
+        if (proj_dir / f'Proj-{topic}.md').exists(): continue
+
+        samples = sorted(arts, key=lambda a: len(a['title']))[-8:]
+        titles = '\n'.join(f'- {a["title"][:50]}' for a in samples)
+
+        prompt = f'''为 {topic} 领域生成一个研究项目提案（150字），格式：
+## 项目名称
+## 目标
+## 关键里程碑 (3个)
+## 所需资源
+
+基于文章样本：
+{titles}
+
+只返回提案内容。'''
+
+        try:
+            text = call_deepseek(prompt, api_key, max_tokens=400)
+        except Exception as e:
+            print(f'  [WARN] {topic}: {e}')
+            continue
+
+        content = f'''---
+title: "{topic} 研究项目"
+created: {datetime.now().strftime('%Y-%m-%d')}
+type: project
+hasTopic: [[{topic}]]
+tags: [project, {topic.lower()}]
+status: proposed
+article_count: {len(arts)}
+---
+
+# {topic} 研究项目
+
+{text}
+
+---
+
+## 📊 数据基础
+
+- **文章数**: {len(arts)} 篇
+- **覆盖日期**: {len(set(a["date"] for a in arts))} 天
+
+## 📚 相关文献
+
+'''
+        for a in arts[:20]:
+            content += f'- [[{a["date"]}/{a["path"].name}|{a["title"][:50]}]]\n'
+
+        content += f'''
+
+## 🔗 相关资源
+
+- [[MOC-{topic}]] — 主题导航
+- [[Perm-{topic}]] — 核心知识
+- [[idea-{topic}-研究想法]] — 研究想法
+'''
+
+        (proj_dir / f'Proj-{topic}.md').write_text(content, encoding='utf-8')
+        created += 1
+        print(f'  [{topic}] → Proj-{topic}.md ({len(arts)} articles)')
+
+    return created
+
+
+def main():
+    sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+    import argparse
+    p = argparse.ArgumentParser(description='全自动知识升级管道')
+    p.add_argument('--vault', default=DEFAULT_VAULT)
+    p.add_argument('--api-key', default='')
+    p.add_argument('--skip-ai', action='store_true')
+    args = p.parse_args()
+
+    api_key = args.api_key or os.environ.get('DEEPSEEK_API_KEY', '')
+
+    print('📊 收集数据...')
+    articles = collect_articles(args.vault)
+    print(f'  {len(articles)} 篇文章')
+
+    # 004: Permanent Notes (AI)
+    if not args.skip_ai and api_key:
+        print('\n📝 生成 004_Permanent 原子笔记...')
+        p_count = generate_permanent(api_key, args.vault, articles)
+    else:
+        p_count = 0
+
+    # 005: Reference (keyword-based, no AI needed)
+    print('\n📚 生成 005_Reference 工具/方法论索引...')
+    r_count = generate_reference(args.vault, articles)
+
+    # 006: Projects (AI)
+    if not args.skip_ai and api_key:
+        print('\n🎯 生成 006_Projects 项目提案...')
+        j_count = generate_projects(api_key, args.vault, articles)
+    else:
+        j_count = 0
+
+    total = p_count + r_count + j_count
+    print(f'\n✓ 全自动完成: {total} 个文件 (004:{p_count} + 005:{r_count} + 006:{j_count})')
+
+
+if __name__ == '__main__':
+    main()