source-kb 0.2.2__py3-none-any.whl

skills/kb-sync/scripts/scan_repos.py

@@ -0,0 +1,687 @@
+#!/usr/bin/env python3
+"""仓库变动扫描脚本 - 重构版
+
+从 kb-project.yaml 读取配置，使用 core 模块进行 git 操作和文件分类。
+
+用法:
+    python scan_repos.py --init                          # 初始化状态
+    python scan_repos.py                                 # 扫描变动
+    python scan_repos.py --kb my-project                  # 只扫描指定知识库
+    python scan_repos.py --module user-service          # 只扫描指定模块
+    python scan_repos.py --update-state --kb my-project --module user-service --commit <hash>  # 更新基线
+    python scan_repos.py --batch-update                  # 批量更新基线
+    python scan_repos.py --force                         # 强制扫描
+    python scan_repos.py --config <path>                 # 指定配置文件
+"""
+
+import sys
+import json
+import argparse
+from pathlib import Path
+from typing import Optional
+from datetime import datetime
+
+# 确保 stdout 使用 UTF-8 编码
+sys.stdout.reconfigure(encoding="utf-8")
+
+# 添加项目根目录到 sys.path
+SCRIPT_DIR = Path(__file__).resolve().parent
+PROJECT_ROOT = SCRIPT_DIR.parent.parent.parent  # source-kb/
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.append(str(PROJECT_ROOT))
+
+try:
+    from core.config import load_config
+    from core.git import (
+        ensure_repo, get_head_commit,
+        diff_files, diff_stat, FileLock
+    )
+    from core.preset import classify_file, get_affected_docs, load_preset
+except ImportError as e:
+    print(f"错误: 无法导入 core 模块: {e}", file=sys.stderr)
+    print("请确保在 source-kb/ 目录下运行此脚本", file=sys.stderr)
+    sys.exit(1)
+
+
+# ============================================================================
+# 配置和状态管理
+# ============================================================================
+
+def get_state_file(config: dict) -> Path:
+    """获取状态文件路径
+    
+    状态文件放在 kb-project.yaml 同目录下的 .source-kb/sync-state.json
+    
+    Args:
+        config: 完整配置字典
+        
+    Returns:
+        状态文件路径
+    """
+    # 从配置中获取任意一个 knowledge_dir，推导出项目根目录
+    kbs = config.get("knowledge_bases", {})
+    if not kbs:
+        raise ValueError("配置中没有知识库")
+    
+    # 使用第一个知识库的 knowledge_dir 的父目录
+    first_kb = next(iter(kbs.values()))
+    kb_dir = Path(first_kb["knowledge_dir"])
+    project_root = kb_dir.parent
+    
+    state_dir = project_root / ".source-kb"
+    state_dir.mkdir(parents=True, exist_ok=True)
+    
+    return state_dir / "sync-state.json"
+
+
+def get_report_file(config: dict) -> Path:
+    """获取报告文件路径
+    
+    报告文件放在 .source-kb/sync-report.json
+    
+    Args:
+        config: 完整配置字典
+        
+    Returns:
+        报告文件路径
+    """
+    state_file = get_state_file(config)
+    return state_file.parent / "sync-report.json"
+
+
+def load_state(config: dict) -> dict:
+    """加载状态文件
+    
+    Args:
+        config: 完整配置字典
+        
+    Returns:
+        状态字典，格式: {kb_name: {module_name: commit_hash}}
+    """
+    state_file = get_state_file(config)
+    
+    if not state_file.exists():
+        return {}
+    
+    try:
+        with open(state_file, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"警告: 读取状态文件失败: {e}", file=sys.stderr)
+        return {}
+
+
+def save_state(config: dict, state: dict):
+    """保存状态文件
+    
+    Args:
+        config: 完整配置字典
+        state: 状态字典
+    """
+    state_file = get_state_file(config)
+    
+    try:
+        with open(state_file, "w", encoding="utf-8") as f:
+            json.dump(state, f, indent=2, ensure_ascii=False)
+    except Exception as e:
+        print(f"错误: 保存状态文件失败: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def save_report(config: dict, report: dict):
+    """保存扫描报告
+    
+    Args:
+        config: 完整配置字典
+        report: 报告字典
+    """
+    report_file = get_report_file(config)
+    
+    try:
+        with open(report_file, "w", encoding="utf-8") as f:
+            json.dump(report, f, indent=2, ensure_ascii=False)
+        print(f"\n报告已保存: {report_file}")
+    except Exception as e:
+        print(f"警告: 保存报告文件失败: {e}", file=sys.stderr)
+
+
+# ============================================================================
+# 模块和分类器加载
+# ============================================================================
+
+def get_modules(kb_config: dict) -> list[dict]:
+    """从 kb 配置提取模块列表
+    
+    multi-repo: 从 source.repos 读取
+    monorepo: 从 source.modules 读取
+    
+    Args:
+      fig: 知识库配置
+        
+    Returns:
+        模块列表，格式: [{"name": "xxx", "branch": "main", "url": "...", "local": "...", "type": "service"}]
+    """
+    source = kb_config.get("source", {})
+    structure = source.get("structure")
+    
+    if structure == "multi-repo":
+        repos = source.get("repos", [])
+        return [
+            {
+                "name": repo.get("name", "unknown"),
+                "branch": repo.get("branch", "main"),
+                "url": repo.get("url"),
+                "local": repo.get("local"),
+                "type": repo.get("type", "service")
+            }
+            for repo in repos
+        ]
+    
+    elif structure == "monorepo":
+        modules = source.get("modules", [])
+        base_url = source.get("url")
+        base_local = source.get("local")
+        base_branch = source.get("branch", "main")
+        
+        return [
+            {
+                "name": module.get("name", "unknown"),
+                "branch": base_branch,
+                "url": base_url,
+                "local": base_local,
+                "type": module.get("type", "service"),
+                "path": module.get("path")  # monorepo 特有
+            }
+            for module in modules
+        ]
+    
+    else:
+        raise ValueError(f"不支持的 source.structure: {structure}")
+
+
+def classify_files_to_docs(preset: dict, file_paths: list[str]) -> list[str]:
+    """对文件列表进行分类，返回受影响的文档文件名列表
+
+    使用 core.preset 的 classify_file + get_affected_docs。
+
+    Args:
+        preset: preset 配置字典
+        file_paths: 文件相对路径列表
+
+    Returns:
+        受影响的文档文件名列表（去重）
+    """
+    all_categories = []
+    for fp in file_paths:
+        cats = classify_file(preset, fp)
+        all_categories.extend(cats)
+
+    if not all_categories:
+        return []
+
+    return sorted(get_affected_docs(preset, all_categories))
+
+
+# ============================================================================
+# 扫描逻辑
+# ============================================================================
+
+def scan_module(
+    kb_name: str,
+    module_config: dict,
+    old_commit: Optional[str],
+    cache_dir: Path,
+    preset: dict
+) -> dict:
+    """扫描单个模块的变动
+
+    Args:
+        kb_name: 知识库名称
+        module_config: 模块配置
+        old_commit: 旧 commit（None 表示初始化）
+        cache_dir: 缓存目录
+        preset: preset 配置字典
+
+    Returns:
+        变动信息字典
+    """
+    module_name = module_config["name"]
+    branch = module_config.get("branch", "main")
+    
+    # 确保本地有可用的仓库
+    try:
+        repo_path = ensure_repo(
+            url=module_config.get("url"),
+            local=module_config.get("local"),
+            cache_dir=cache_dir,
+            module_name=module_name,
+            branch=branch
+        )
+    except Exception as e:
+        return {
+            "error": str(e),
+            "module": module_name
+        }
+    
+    # 获取当前 commit
+    ref = f"origin/{branch}"
+    new_commit = get_head_commit(repo_path, ref)
+    
+    if new_commit is None:
+        return {
+            "error": f"无法获取 {ref} 的 commit",
+            "module": module_name
+        }
+    
+    # 如果没有旧 commit，返回初始化信息
+    if old_commit is None:
+        return {
+            "module": module_name,
+            "status": "initialized",
+            "commit": new_commit,
+            "branch": branch
+        }
+    
+    # 如果 commit 没有变化
+    if old_commit == new_commit:
+        return {
+            "module": module_name,
+            "status": "unchanged",
+            "commit": new_commit
+        }
+    
+    # 获取变动文件列表
+    try:
+        changes = diff_files(repo_path, old_commit, new_commit)
+        stat = diff_stat(repo_path, old_commit, new_commit)
+    except Exception as e:
+        return {
+            "error": f"获取 diff 失败: {e}",
+            "module": module_name,
+            "old_commit": old_commit,
+            "new_commit": new_commit
+        }
+    
+    # 文件分类
+    changed_files = [c["file"] for c in changes]
+    affected_docs = classify_files_to_docs(preset, changed_files)
+    
+    return {
+        "module": module_name,
+        "status": "changed",
+        "old_commit": old_commit,
+        "new_commit": new_commit,
+        "branch": branch,
+        "changes": changes,
+        "stat": stat,
+        "affected_docs": affected_docs,
+        "file_count": len(changed_files)
+    }
+
+
+def scan(
+    config: dict,
+    filter_kb: Optional[str] = None,
+    filter_module: Optional[str] = None,
+    force: bool = False
+) -> dict:
+    """扫描变动
+    
+    Args:
+        config: load_config() 返回的完整配置
+        filter_kb: 只扫描指定知识库
+        filter_module: 只扫描指定模块
+        force: 强制扫描（忽略状态文件）
+        
+    Returns:
+        扫描结果，格式: {kb_name: {module_name: {变动信息}}}
+    """
+    # 加载状态
+    state = {} if force else load_state(config)
+
+    results = {}
+
+    # 遍历知识库
+    kbs = config.get("knowledge_bases", {})
+    for kb_name, kb_config in kbs.items():
+        # 过滤知识库
+        if filter_kb and kb_name != filter_kb:
+            continue
+        
+        print(f"\n扫描知识库: {kb_name}")
+        
+        # 获取知识库级锁
+        kb_dir = Path(kb_config["knowledge_dir"])
+        lock = FileLock(kb_dir / ".kb-lock")
+        
+        with lock:
+            # 获取缓存目录（从 kb 配置读取，默认 .source-cache）
+            source = kb_config.get("source", {})
+            cache_dir = Path(source.get("cache_dir", "./.source-cache"))
+            if not cache_dir.is_absolute():
+                cache_dir = PROJECT_ROOT / cache_dir
+            cache_dir.mkdir(parents=True, exist_ok=True)
+
+            # 加载 preset 配置
+            preset_name = kb_config.get("preset")
+            try:
+                preset = load_preset(preset_name)
+            except Exception as e:
+                print(f"  错误: 加载 preset 失败: {e}", file=sys.stderr)
+                results[kb_name] = {"error": f"加载 preset 失败: {e}"}
+                continue
+            
+            # 获取模块列表
+            try:
+                modules = get_modules(kb_config)
+            except Exception as e:
+                print(f"  错误: 获取模块列表失败: {e}", file=sys.stderr)
+                results[kb_name] = {"error": f"获取模块列表失败: {e}"}
+                continue
+            
+            kb_results = {}
+            
+            # 遍历模块
+            for module_config in modules:
+                module_name = module_config["name"]
+                
+                # 过滤模块
+                if filter_module and module_name != filter_module:
+                    continue
+                
+                print(f"  模块: {module_name}")
+                
+                # 获取旧 commit
+                old_commit = state.get(kb_name, {}).get(module_name)
+                
+                # 扫描模块
+                result = scan_module(
+                    kb_name,
+                    module_config,
+                    old_commit,
+                    cache_dir,
+                    preset
+                )
+                
+                kb_results[module_name] = result
+                
+                # 打印结果
+                if "error" in result:
+                    print(f"    错误: {result['error']}")
+                elif result["status"] == "initialized":
+                    print(f"    初始化: {result['commit'][:8]}")
+                elif result["status"] == "unchanged":
+                    print(f"    无变动: {result['commit'][:8]}")
+                elif result["status"] == "changed":
+                    print(f"    变动: {result['old_commit'][:8]} -> {result['new_commit'][:8]}")
+                    print(f"    文件: {result['file_count']} 个")
+                    print(f"    影响文档: {', '.join(result['affected_docs']) if result['affected_docs'] else '无'}")
+            
+            results[kb_name] = kb_results
+    
+    return results
+
+
+# ============================================================================
+# CLI 命令
+# ============================================================================
+
+def cmd_init(config: dict):
+    """初始化状态文件"""
+    print("初始化状态...")
+    
+    state = {}
+
+    kbs = config.get("knowledge_bases", {})
+    for kb_name, kb_config in kbs.items():
+        print(f"\n知识库: {kb_name}")
+
+        # 从 kb 配置读取缓存目录（与 scan() 一致）
+        source = kb_config.get("source", {})
+        cache_dir = Path(source.get("cache_dir", "./.source-cache"))
+        if not cache_dir.is_absolute():
+            cache_dir = PROJECT_ROOT / cache_dir
+        cache_dir.mkdir(parents=True, exist_ok=True)
+
+        modules = get_modules(kb_config)
+        kb_state = {}
+        
+        for module_config in modules:
+            module_name = module_config["name"]
+            branch = module_config.get("branch", "main")
+            
+            print(f"  模块: {module_name}")
+            
+            try:
+                repo_path = ensure_repo(
+                    url=module_config.get("url"),
+                    local=module_config.get("local"),
+                    cache_dir=cache_dir,
+                    module_name=module_name,
+                    branch=branch
+                )
+                
+                ref = f"origin/{branch}"
+                commit = get_head_commit(repo_path, ref)
+                
+                if commit:
+                    kb_state[module_name] = commit
+                    print(f"    初始化: {commit[:8]}")
+                else:
+                    print(f"    错误: 无法获取 commit")
+            
+            except Exception as e:
+                print(f"    错误: {e}")
+        
+        state[kb_name] = kb_state
+    
+    save_state(config, state)
+    print(f"\n状态已保存: {get_state_file(config)}")
+
+
+def cmd_scan(config: dict, args):
+    """扫描变动"""
+    results = scan(
+        config,
+        filter_kb=args.kb,
+        filter_module=args.module,
+        force=args.force
+    )
+    
+    # 保存报告
+    report = {
+        "timestamp": datetime.now().isoformat(),
+        "results": results
+    }
+    save_report(config, report)
+    
+    # 统计
+    total_changed = 0
+    total_unchanged = 0
+    total_errors = 0
+    
+    for kb_results in results.values():
+        if isinstance(kb_results, dict) and "error" not in kb_results:
+            for module_result in kb_results.values():
+                if "error" in module_result:
+                    total_errors += 1
+                elif module_result.get("status") == "changed":
+                    total_changed += 1
+                elif module_result.get("status") == "unchanged":
+                    total_unchanged += 1
+    
+    print(f"\n总结:")
+    print(f"  变动: {total_changed} 个模块")
+    print(f"  无变动: {total_unchanged} 个模块")
+    if total_errors > 0:
+        print(f"  错误: {total_errors} 个模块")
+
+    # 结构化状态输出
+    print(json.dumps({"status": "ok", "changed": total_changed, "unchanged": total_unchanged, "errors": total_errors}, ensure_ascii=False), file=sys.stderr)
+
+
+def cmd_update_state(config: dict, args):
+    """更新状态文件中的基线 commit"""
+    if not args.kb or not args.module or not args.commit:
+        print("错误: --update-state 需要 --kb <知识库> --module <模块> <commit>", file=sys.stderr)
+        sys.exit(1)
+    
+    state = load_state(config)
+    
+    if args.kb not in state:
+        state[args.kb] = {}
+    
+    state[args.kb][args.module] = args.commit
+    save_state(config, state)
+    
+    print(f"已更新: {args.kb}/{args.module} -> {args.commit[:8]}")
+
+
+def cmd_batch_update(config: dict):
+    """批量更新基线（将所有模块更新到最新 commit）"""
+    print("批量更新基线...")
+    
+    state = load_state(config)
+    kbs = config.get("knowledge_bases", {})
+    for kb_name, kb_config in kbs.items():
+        print(f"\n知识库: {kb_name}")
+
+        # 从 kb 配置读取缓存目录（与 scan() 一致）
+        source = kb_config.get("source", {})
+        cache_dir = Path(source.get("cache_dir", "./.source-cache"))
+        if not cache_dir.is_absolute():
+            cache_dir = PROJECT_ROOT / cache_dir
+
+        modules = get_modules(kb_config)
+        
+        if kb_name not in state:
+            state[kb_name] = {}
+        
+        for module_config in modules:
+            module_name = module_config["name"]
+            branch = module_config.get("branch", "main")
+            
+            print(f"  模块: {module_name}")
+            
+            try:
+                repo_path = ensure_repo(
+                    url=module_config.get("url"),
+                    local=module_config.get("local"),
+                    cache_dir=cache_dir,
+                    module_name=module_name,
+                    branch=branch
+                )
+                
+                ref = f"origin/{branch}"
+                commit = get_head_commit(repo_path, ref)
+                
+                if commit:
+                    old_commit = state[kb_name].get(module_name)
+                    state[kb_name][module_name] = commit
+                    
+                    if old_commit:
+                        print(f"    更新: {old_commit[:8]} -> {commit[:8]}")
+                    else:
+                        print(f"    初始化: {commit[:8]}")
+                else:
+                    print(f"    错误: 无法获取 commit")
+            
+            except Exception as e:
+                print(f"    错误: {e}")
+    
+    save_state(config, state)
+    print(f"\n状态已保存: {get_state_file(config)}")
+
+
+# ============================================================================
+# 主函数
+# ============================================================================
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="扫描仓库变动",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__
+    )
+    
+    parser.add_argument(
+        "--config",
+        type=str,
+        help="指定 kb-project.yaml 路径（可选，默认自动查找）"
+    )
+    
+    parser.add_argument(
+        "--init",
+        action="store_true",
+        help="初始化状态文件"
+    )
+    
+    parser.add_argument(
+        "--kb",
+        type=str,
+        help="只扫描指定知识库"
+    )
+    
+    parser.add_argument(
+        "--module",
+        type=str,
+        help="只扫描指定模块"
+    )
+    
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="强制扫描（忽略状态文件）"
+    )
+    
+    parser.add_argument(        "--update-state",
+        action="store_true",
+        help="更新状态文件中的基线 commit"
+    )
+    
+    parser.add_argument(
+        "--batch-update",
+        action="store_true",
+        help="批量更新基线（将所有模块更新到最新 commit）"
+    )
+    
+    parser.add_argument(
+        "--commit",
+        type=str,
+        help="commit hash（用于 --update-state）"
+    )
+    
+    args = parser.parse_args()
+    
+    # 加载配置
+    try:
+        if args.config:
+            config = load_config(Path(args.config))
+        else:
+            config = load_config()
+    except Exception as e:
+        print(f"错误: 加载配置失败: {e}", file=sys.stderr)
+        sys.exit(1)
+    
+    # 执行命令
+    try:
+        if args.init:
+            cmd_init(config)
+        elif args.update_state:
+            cmd_update_state(config, args)
+        elif args.batch_update:
+            cmd_batch_update(config)
+        else:
+            cmd_scan(config, args)
+    except KeyboardInterrupt:
+        print("\n\n已取消", file=sys.stderr)
+        sys.exit(130)
+    except Exception as e:
+        print(f"\n错误: {e}", file=sys.stderr)
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()