paperfit-cli 1.0.0

package/scripts/test_cv_detector.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+"""
+CV Detector 测试脚本
+
+用于验证 cv_detector.py 对 VTO Benchmark 样本的检测能力。
+
+用法:
+    python scripts/test_cv_detector.py
+
+输出:
+    - 各类别缺陷检测成功率
+    - 漏报/误报分析
+    - 检测性能指标
+"""
+
+import json
+import sys
+from pathlib import Path
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+sys.path.insert(0, str(Path(__file__).parent))
+from cv_detector import BatchCVDetector, CVDefectDetector
+
+
+@dataclass
+class GroundTruth:
+    """基准真相"""
+    category: str
+    defect_id: str
+    page: int
+    expected: bool  # 是否应该被检测到
+
+
+# 基准真相定义
+# 根据 inject_defects.py 注入的缺陷
+# 注意：我们检测的是"视觉表现"而非"注入原因"
+# 例如：B2-float-width（1.5 线宽）的视觉表现是 D1-overfull-hbox
+GROUND_TRUTH = {
+    "cat_a": [
+        GroundTruth("A", "A1-widow-orphan", page=0, expected=True),  # 每页都有短行
+        GroundTruth("A", "A2-trailing-whitespace", page=3, expected=True),  # 末页留白
+    ],
+    "cat_b": [
+        # B1 和 B2 的视觉表现已在 Category D 中通过溢出检测捕获
+        # 这里只检查是否有 D 类溢出（来自 B2 的 1.5 线宽）
+        GroundTruth("D", "D1-overfull-hbox", page=1, expected=True),  # B2 导致的溢出
+    ],
+    "cat_d": [
+        GroundTruth("D", "D1-overfull-hbox", page=0, expected=True),  # 长单词溢出
+        # D3 URL 溢出与 D1 在视觉上不可区分，不单独检查
+    ],
+}
+
+
+def run_detection(pages_dir: str) -> dict:
+    """运行批量检测"""
+    detector = BatchCVDetector(pages_dir)
+    return detector.run_batch()
+
+
+def evaluate_detection(report: dict, ground_truth: List[GroundTruth]) -> dict:
+    """评估检测结果
+
+    检测策略：只要某个缺陷类型在任何页面被检测到，就视为该缺陷类别被检出
+    """
+    tp = 0  # True Positive
+    fp = 0  # False Positive
+    fn = 0  # False Negative
+
+    # 收集所有检测到的缺陷类型（不区分页码）
+    detected_types = set()
+    for page_result in report["page_results"]:
+        for defect in page_result["detections"]:
+            detected_types.add((defect["category"], defect["defect_id"]))
+
+    # 检查每个基准真相
+    for gt in ground_truth:
+        # 检查该缺陷类型是否在任何页面被检测到
+        is_detected = (gt.category, gt.defect_id) in detected_types
+
+        if gt.expected and is_detected:
+            tp += 1
+        elif gt.expected and not is_detected:
+            fn += 1
+            print(f"  [漏报] {gt.defect_id}")
+        elif not gt.expected and is_detected:
+            fp += 1
+
+    # 计算指标
+    precision = tp / (tp + fp) if (tp + fp) > 0 else 0
+    recall = tp / (tp + fn) if (tp + fn) > 0 else 0
+    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0
+
+    return {
+        "true_positives": tp,
+        "false_positives": fp,
+        "false_negatives": fn,
+        "precision": precision,
+        "recall": recall,
+        "f1_score": f1,
+    }
+
+
+def print_detection_summary(report: dict):
+    """打印检测摘要"""
+    print(f"\nPages analyzed: {report['pages_analyzed']}")
+    print(f"Total detections: {report['total_detections']}")
+    print(f"Category breakdown: {report['category_breakdown']}")
+
+    for page_result in report["page_results"]:
+        if page_result["detections"]:
+            page_num = page_result["page_number"]
+            count = page_result["detection_count"]
+            print(f"\nPage {page_num}: {count} detection(s)")
+            for d in page_result["detections"]:
+                print(f"  - [{d['category']}] {d['defect_id']}: {d['description']}")
+
+
+def main():
+    """主函数"""
+    samples_dir = Path(__file__).parent.parent / "data" / "benchmarks" / "samples"
+
+    print("=" * 60)
+    print("CV Detector 测试报告")
+    print("=" * 60)
+
+    all_results = {}
+
+    # 测试 Category A
+    print("\n### Category A: 空间利用缺陷")
+    cat_a_dir = samples_dir / "pages_a"
+    if cat_a_dir.exists():
+        report_a = run_detection(str(cat_a_dir))
+        print_detection_summary(report_a)
+        eval_a = evaluate_detection(report_a, GROUND_TRUTH["cat_a"])
+        all_results["cat_a"] = eval_a
+        print(f"\n评估指标:")
+        print(f"  Precision: {eval_a['precision']:.2%}")
+        print(f"  Recall: {eval_a['recall']:.2%}")
+        print(f"  F1 Score: {eval_a['f1_score']:.2%}")
+    else:
+        print(f"  [跳过] {cat_a_dir} 不存在")
+
+    # 测试 Category B
+    print("\n### Category B: 浮动体缺陷")
+    cat_b_dir = samples_dir / "pages_b"
+    if cat_b_dir.exists():
+        report_b = run_detection(str(cat_b_dir))
+        print_detection_summary(report_b)
+        eval_b = evaluate_detection(report_b, GROUND_TRUTH["cat_b"])
+        all_results["cat_b"] = eval_b
+        print(f"\n评估指标:")
+        print(f"  Precision: {eval_b['precision']:.2%}")
+        print(f"  Recall: {eval_b['recall']:.2%}")
+        print(f"  F1 Score: {eval_b['f1_score']:.2%}")
+    else:
+        print(f"  [跳过] {cat_b_dir} 不存在")
+
+    # 测试 Category D
+    print("\n### Category D: 溢出缺陷")
+    cat_d_dir = samples_dir / "pages_d"
+    if cat_d_dir.exists():
+        report_d = run_detection(str(cat_d_dir))
+        print_detection_summary(report_d)
+        eval_d = evaluate_detection(report_d, GROUND_TRUTH["cat_d"])
+        all_results["cat_d"] = eval_d
+        print(f"\n评估指标:")
+        print(f"  Precision: {eval_d['precision']:.2%}")
+        print(f"  Recall: {eval_d['recall']:.2%}")
+        print(f"  F1 Score: {eval_d['f1_score']:.2%}")
+    else:
+        print(f"  [跳过] {cat_d_dir} 不存在")
+
+    # 总体汇总
+    print("\n" + "=" * 60)
+    print("总体汇总")
+    print("=" * 60)
+
+    total_tp = sum(r["true_positives"] for r in all_results.values())
+    total_fp = sum(r["false_positives"] for r in all_results.values())
+    total_fn = sum(r["false_negatives"] for r in all_results.values())
+
+    overall_precision = total_tp / (total_tp + total_fp) if (total_tp + total_fp) > 0 else 0
+    overall_recall = total_tp / (total_tp + total_fn) if (total_tp + total_fn) > 0 else 0
+    overall_f1 = 2 * overall_precision * overall_recall / (overall_precision + overall_recall) if (overall_precision + overall_recall) > 0 else 0
+
+    print(f"\n总检测数: {total_tp + total_fp} (TP={total_tp}, FP={total_fp}, FN={total_fn})")
+    print(f"总体 Precision: {overall_precision:.2%}")
+    print(f"总体 Recall: {overall_recall:.2%}")
+    print(f"总体 F1 Score: {overall_f1:.2%}")
+
+    # 保存结果
+    output_file = samples_dir / "cv_detection_results.json"
+    output_data = {
+        "category_results": all_results,
+        "overall": {
+            "true_positives": total_tp,
+            "false_positives": total_fp,
+            "false_negatives": total_fn,
+            "precision": overall_precision,
+            "recall": overall_recall,
+            "f1_score": overall_f1,
+        }
+    }
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(output_data, f, indent=2, ensure_ascii=False)
+    print(f"\n结果已保存到：{output_file}")
+
+    return 0 if overall_f1 >= 0.7 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/test_integration.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""
+PaperFit 系统集成测试
+
+验证 commands、agents 和 skills 之间的集成关系。
+"""
+
+import json
+import re
+from pathlib import Path
+from typing import Dict, List, Set, Tuple
+
+
+class IntegrationTester:
+    """PaperFit 集成测试器"""
+
+    def __init__(self, root_dir: str = "."):
+        self.root = Path(root_dir)
+        self.results: List[Tuple[str, bool, str]] = []
+
+    def test_command_agent_links(self) -> bool:
+        """测试 commands 到 agents 的引用链路"""
+        print("\n" + "=" * 60)
+        print("测试：Commands → Agents 引用链路")
+        print("=" * 60)
+
+        commands_dir = self.root / ".claude" / "commands"
+        agents_dir = self.root / "agents"
+
+        # 获取所有 agent 文件名
+        available_agents = {f.name for f in agents_dir.glob("*.md")}
+        print(f"\n可用 Agents: {sorted(available_agents)}")
+
+        # 检查每个命令文件
+        for cmd_file in commands_dir.glob("*.md"):
+            content = cmd_file.read_text()
+
+            # 提取引用的 agents
+            agent_refs = set(re.findall(r'agents/([\w-]+\.md)', content))
+
+            if agent_refs:
+                missing = agent_refs - available_agents
+                if missing:
+                    self.results.append((
+                        f"commands/{cmd_file.name}",
+                        False,
+                        f"引用不存在的 agent: {missing}"
+                    ))
+                else:
+                    self.results.append((
+                        f"commands/{cmd_file.name}",
+                        True,
+                        f"引用 agents: {sorted(agent_refs)}"
+                    ))
+            else:
+                # 有些命令可能不引用 agent（如 show-status）
+                self.results.append((
+                    f"commands/{cmd_file.name}",
+                    True,
+                    "未引用特定 agent（可能是查询类命令）"
+                ))
+
+        return all(r[1] for r in self.results if r[0].startswith("commands/"))
+
+    def test_agent_skill_links(self) -> bool:
+        """测试 agents 到 skills 的引用链路"""
+        print("\n" + "=" * 60)
+        print("测试：Agents → Skills 引用链路")
+        print("=" * 60)
+
+        skills_dir = self.root / "skills"
+
+        # 获取所有 skill 目录名
+        available_skills = {
+            d.name for d in skills_dir.iterdir()
+            if d.is_dir() and (d / "SKILL.md").exists()
+        }
+        print(f"\n可用 Skills: {sorted(available_skills)}")
+
+        # 检查每个 agent 文件
+        agents_dir = self.root / "agents"
+        for agent_file in agents_dir.glob("*.md"):
+            content = agent_file.read_text()
+
+            # 提取引用的 skills
+            skill_refs = set(re.findall(r'[\w-]+', content))
+            matched_skills = skill_refs & available_skills
+
+            if matched_skills:
+                self.results.append((
+                    f"agents/{agent_file.name}",
+                    True,
+                    f"引用 skills: {sorted(matched_skills)}"
+                ))
+            else:
+                self.results.append((
+                    f"agents/{agent_file.name}",
+                    True,
+                    "未显式引用特定 skill（可能是调度类 agent）"
+                ))
+
+        return True  # 即使没有引用 skill 也不算错误
+
+    def test_vto_taxonomy_coverage(self) -> bool:
+        """测试 VTO 分类体系的完整性"""
+        print("\n" + "=" * 60)
+        print("测试：VTO 分类体系覆盖")
+        print("=" * 60)
+
+        taxonomy_file = self.root / "config" / "vto_taxonomy.yaml"
+
+        if not taxonomy_file.exists():
+            self.results.append((
+                "config/vto_taxonomy.yaml",
+                False,
+                "文件不存在"
+            ))
+            return False
+
+        content = taxonomy_file.read_text()
+
+        # 检查是否包含所有 Category
+        required_categories = ["Category A", "Category B", "Category C", "Category D", "Category E"]
+        missing_categories = [cat for cat in required_categories if cat not in content]
+
+        if missing_categories:
+            self.results.append((
+                "config/vto_taxonomy.yaml",
+                False,
+                f"缺少分类: {missing_categories}"
+            ))
+        else:
+            self.results.append((
+                "config/vto_taxonomy.yaml",
+                True,
+                "包含全部 5 个 VTO 分类"
+            ))
+
+        return len(missing_categories) == 0
+
+    def test_state_manager_integration(self) -> bool:
+        """测试 state_manager.py 的集成"""
+        print("\n" + "=" * 60)
+        print("测试：State Manager 集成")
+        print("=" * 60)
+
+        state_manager = self.root / "scripts" / "state_manager.py"
+
+        if not state_manager.exists():
+            self.results.append((
+                "scripts/state_manager.py",
+                False,
+                "文件不存在"
+            ))
+            return False
+
+        content = state_manager.read_text()
+
+        # 检查关键功能
+        checks = {
+            "_backup": "备份功能",
+            "_backup_directory": "目录备份功能",
+            "case_dir": "Case 目录支持",
+            "data/state.json": "状态文件路径"
+        }
+
+        missing = [name for pattern, name in checks.items() if pattern not in content]
+
+        if missing:
+            self.results.append((
+                "scripts/state_manager.py",
+                False,
+                f"缺少功能: {missing}"
+            ))
+        else:
+            self.results.append((
+                "scripts/state_manager.py",
+                True,
+                "所有关键功能已实现"
+            ))
+
+        return len(missing) == 0
+
+    def test_compile_script(self) -> bool:
+        """测试编译脚本"""
+        print("\n" + "=" * 60)
+        print("测试：编译脚本")
+        print("=" * 60)
+
+        compile_script = self.root / "scripts" / "compile.sh"
+
+        if not compile_script.exists():
+            self.results.append((
+                "scripts/compile.sh",
+                False,
+                "文件不存在"
+            ))
+            return False
+
+        content = compile_script.read_text()
+
+        # 检查是否包含必要的编译命令
+        if "latexmk" in content or "pdflatex" in content:
+            self.results.append((
+                "scripts/compile.sh",
+                True,
+                "包含 LaTeX 编译命令"
+            ))
+        else:
+            self.results.append((
+                "scripts/compile.sh",
+                False,
+                "缺少 LaTeX 编译命令"
+            ))
+
+        return True
+
+    def generate_report(self) -> str:
+        """生成测试报告"""
+        passed = sum(1 for r in self.results if r[1])
+        failed = sum(1 for r in self.results if not r[1])
+
+        report = "\n" + "=" * 60 + "\n"
+        report += "PaperFit 集成测试报告\n"
+        report + "=" * 60 + "\n\n"
+
+        report += f"通过：{passed}\n"
+        report += f"失败：{failed}\n\n"
+
+        if failed > 0:
+            report += "失败详情:\n"
+            for path, success, msg in self.results:
+                if not success:
+                    report += f"  ✗ {path}: {msg}\n"
+            report += "\n"
+
+        report += "=" * 60 + "\n"
+        report += "系统组件摘要\n"
+        report + "=" * 60 + "\n\n"
+
+        # Commands
+        commands_dir = self.root / ".claude" / "commands"
+        cmd_files = list(commands_dir.glob("*.md")) if commands_dir.exists() else []
+        report += f"Commands: {len(cmd_files)} 个\n"
+        for cmd in cmd_files:
+            content = cmd.read_text()
+            cmd_name = re.search(r'# /(\S+)', content)
+            if cmd_name:
+                report += f"  - /{cmd_name.group(1)}\n"
+
+        # Agents
+        agents_dir = self.root / "agents"
+        agent_files = list(agents_dir.glob("*.md")) if agents_dir.exists() else []
+        report += f"\nAgents: {len(agent_files)} 个\n"
+        for agent in agent_files:
+            report += f"  - {agent.name}\n"
+
+        # Skills
+        skills_dir = self.root / "skills"
+        skill_dirs = [d for d in skills_dir.iterdir() if d.is_dir() and (d / "SKILL.md").exists()] if skills_dir.exists() else []
+        report += f"\nSkills: {len(skill_dirs)} 个\n"
+        for skill in skill_dirs:
+            report += f"  - {skill.name}/\n"
+
+        report += "\n" + "=" * 60 + "\n"
+
+        return report
+
+    def run_all_tests(self) -> bool:
+        """运行所有集成测试"""
+        self.test_command_agent_links()
+        self.test_agent_skill_links()
+        self.test_vto_taxonomy_coverage()
+        self.test_state_manager_integration()
+        self.test_compile_script()
+
+        report = self.generate_report()
+        print(report)
+
+        failed = sum(1 for r in self.results if not r[1])
+        return failed == 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    root = sys.argv[1] if len(sys.argv) > 1 else "."
+    tester = IntegrationTester(root)
+    success = tester.run_all_tests()
+    sys.exit(0 if success else 1)