isage-benchmark-agent 0.1.0.1__cp311-none-any.whl

sage/benchmark/benchmark_agent/scripts/experiments/exp_main_planning.py

@@ -0,0 +1,315 @@
+#!/usr/bin/env python3
+"""
+Section 5.2.2: Task Planning Experiment (RQ2)
+
+研究问题: 现有方法将复杂任务分解为执行步骤的能力如何?
+
+测试方法:
+- planner.simple       : 简单贪心匹配 (Baseline)
+- planner.hierarchical : 层次化分解 (HuggingGPT, ICML'23)
+- planner.llm_based    : LLM 生成计划 (CoT Prompting)
+- planner.react        : 交错执行 (ReAct, Yao et al., ICLR'23)
+- planner.tot          : 树搜索 (Tree-of-Thoughts, NeurIPS'23)
+
+目标指标:
+- Primary: Plan Success Rate ≥ 90%
+- Secondary: Step Accuracy, Tool Coverage
+- Tertiary: Average Plan Length
+
+Usage:
+    python exp_main_planning.py
+    python exp_main_planning.py --max-samples 100
+    python exp_main_planning.py --skip-llm
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+
+from .exp_utils import (
+    ExperimentResult,
+    ExperimentSummary,
+    create_progress_bar,
+    load_benchmark_data,
+    print_metrics_detail,
+    print_result_row,
+    print_section_header,
+    print_subsection_header,
+    save_results,
+    setup_experiment_env,
+)
+
+
+def compute_step_accuracy(predicted_steps: list[str], reference_steps: list[str]) -> float:
+    """计算步骤级别的准确率。"""
+    if not reference_steps:
+        return 1.0 if not predicted_steps else 0.0
+
+    correct = sum(1 for p, r in zip(predicted_steps, reference_steps) if p == r)
+    return correct / len(reference_steps)
+
+
+def compute_tool_coverage(predicted_steps: list[str], reference_steps: list[str]) -> float:
+    """计算工具覆盖率。"""
+    if not reference_steps:
+        return 1.0
+
+    ref_set = set(reference_steps)
+    pred_set = set(predicted_steps)
+    return len(pred_set & ref_set) / len(ref_set)
+
+
+def is_plan_success(
+    predicted_steps: list[str], reference_steps: list[str], threshold: float = 0.8
+) -> bool:
+    """判断计划是否成功 (步骤准确率 >= threshold)。"""
+    if not reference_steps:
+        return not predicted_steps
+
+    step_acc = compute_step_accuracy(predicted_steps, reference_steps)
+    return step_acc >= threshold
+
+
+def run_planning_experiment(
+    max_samples: int = 100,
+    skip_llm: bool = False,
+    verbose: bool = True,
+) -> ExperimentSummary:
+    """
+    运行 Task Planning 实验。
+
+    Args:
+        max_samples: 最大测试样本数
+        skip_llm: 是否跳过 LLM-based 方法
+        verbose: 是否打印详细信息
+
+    Returns:
+        ExperimentSummary 对象
+    """
+    setup_experiment_env(verbose=verbose)
+
+    print_section_header("Section 5.2.2: Task Planning (RQ2)")
+    print("   Target: Plan Success Rate ≥ 90%")
+    print(f"   Max samples: {max_samples}")
+
+    # 加载数据
+    samples = load_benchmark_data("planning", split="test", max_samples=max_samples)
+    if not samples:
+        print("  ❌ No planning data available")
+        return ExperimentSummary(section="5_2_main", challenge="planning")
+
+    print(f"   Loaded {len(samples)} samples")
+
+    # 获取策略注册表
+    try:
+        from sage.benchmark.benchmark_agent import get_adapter_registry
+
+        registry = get_adapter_registry()
+    except ImportError as e:
+        print(f"  ❌ Failed to import adapter registry: {e}")
+        return ExperimentSummary(section="5_2_main", challenge="planning")
+
+    # 定义测试策略 - 包含主流 SOTA 方法
+    strategies = [
+        ("planner.simple", "Simple (Greedy)"),
+        ("planner.hierarchical", "Hierarchical (HuggingGPT)"),
+        ("planner.llm_based", "LLM-based (CoT)"),
+        ("planner.react", "ReAct (Yao et al.)"),
+        ("planner.tot", "Tree-of-Thoughts"),
+    ]
+
+    # 跳过 LLM 策略
+    LLM_STRATEGIES = {"planner.llm_based", "planner.react", "planner.tot"}
+    if skip_llm:
+        strategies = [(name, display) for name, display in strategies if name not in LLM_STRATEGIES]
+        print("  ⚠️  Skipping LLM-based strategies")
+
+    results = []
+    target = 0.90
+
+    for strategy_name, display_name in strategies:
+        print_subsection_header(f"Testing: {display_name}")
+
+        try:
+            planner = registry.get(strategy_name)
+        except Exception as e:
+            print(f"    ⚠️  Failed to create planner: {e}")
+            continue
+
+        # 运行评测
+        start_time = time.time()
+        success_count = 0
+        step_accuracies = []
+        tool_coverages = []
+        plan_lengths = []
+
+        with create_progress_bar(len(samples), desc=f"  {display_name}") as pbar:
+            for sample in samples:
+                try:
+                    task_description = sample.get("task", sample.get("instruction", ""))
+                    available_tools = sample.get("available_tools", sample.get("tools", []))
+                    reference_plan = sample.get("ground_truth", sample.get("expected_plan", []))
+
+                    # 标准化 reference
+                    if isinstance(reference_plan, list) and reference_plan:
+                        if isinstance(reference_plan[0], dict):
+                            ref_steps = [
+                                s.get("tool_id", s.get("tool", "")) for s in reference_plan
+                            ]
+                        else:
+                            ref_steps = reference_plan
+                    else:
+                        ref_steps = []
+
+                    # 调用规划器
+                    plan_result = planner.plan(task_description, available_tools=available_tools)
+
+                    # 提取预测步骤
+                    if hasattr(plan_result, "tool_sequence"):
+                        pred_steps = plan_result.tool_sequence
+                    elif hasattr(plan_result, "steps"):
+                        pred_steps = [
+                            s.tool_id if hasattr(s, "tool_id") else str(s)
+                            for s in plan_result.steps
+                        ]
+                    elif isinstance(plan_result, list):
+                        pred_steps = [
+                            s.get("tool_id", str(s)) if isinstance(s, dict) else str(s)
+                            for s in plan_result
+                        ]
+                    else:
+                        pred_steps = []
+
+                    # 计算指标
+                    if is_plan_success(pred_steps, ref_steps):
+                        success_count += 1
+
+                    step_acc = compute_step_accuracy(pred_steps, ref_steps)
+                    tool_cov = compute_tool_coverage(pred_steps, ref_steps)
+
+                    step_accuracies.append(step_acc)
+                    tool_coverages.append(tool_cov)
+                    plan_lengths.append(len(pred_steps))
+
+                except Exception as e:
+                    if verbose:
+                        print(f"    Error: {e}")
+                    step_accuracies.append(0.0)
+                    tool_coverages.append(0.0)
+                    plan_lengths.append(0)
+
+                pbar.update(1)
+
+        elapsed = time.time() - start_time
+
+        # 计算汇总指标
+        n = len(samples)
+        success_rate = success_count / n if n > 0 else 0
+        avg_step_acc = sum(step_accuracies) / n if n > 0 else 0
+        avg_tool_cov = sum(tool_coverages) / n if n > 0 else 0
+        avg_plan_len = sum(plan_lengths) / n if n > 0 else 0
+
+        exp_result = ExperimentResult(
+            challenge="planning",
+            strategy=strategy_name,
+            metrics={
+                "plan_success_rate": success_rate,
+                "step_accuracy": avg_step_acc,
+                "tool_coverage": avg_tool_cov,
+            },
+            metadata={
+                "total_samples": n,
+                "success_count": success_count,
+                "avg_plan_length": avg_plan_len,
+                "latency_ms": elapsed * 1000 / n if n > 0 else 0,
+            },
+            passed=success_rate >= target,
+            target=target,
+        )
+        results.append(exp_result)
+
+        # 打印结果
+        print_result_row(
+            display_name, {"plan_success_rate": success_rate}, exp_result.passed, target
+        )
+        if verbose:
+            print_metrics_detail(exp_result.metrics)
+
+    # 找出最佳策略
+    best_result = max(results, key=lambda r: r.metrics["plan_success_rate"]) if results else None
+
+    summary = ExperimentSummary(
+        section="5_2_main",
+        challenge="planning",
+        results=results,
+        best_strategy=best_result.strategy if best_result else None,
+        best_metric=best_result.metrics["plan_success_rate"] if best_result else None,
+        target_met=any(r.passed for r in results),
+    )
+
+    # 保存结果
+    output_file = save_results(summary.to_dict(), "5_2_main", "planning")
+    print(f"\n  Results saved to: {output_file}")
+
+    # 生成图表
+    if results:
+        from figure_generator import (
+            generate_detailed_table,
+            get_figures_dir,
+            get_tables_dir,
+            plot_challenge_comparison,
+        )
+
+        figures_dir = get_figures_dir()
+        tables_dir = get_tables_dir()
+
+        # 生成对比图
+        plot_challenge_comparison(
+            [{"strategy": r.strategy.split(".")[-1], "metrics": r.metrics} for r in results],
+            challenge="planning",
+            metrics=["plan_success_rate", "step_accuracy", "tool_coverage"],
+            target=target,
+            output_path=figures_dir / "fig2_main_planning_comparison.pdf",
+            title="Task Planning: Strategy Comparison",
+        )
+
+        # 生成表格
+        generate_detailed_table(
+            [{"strategy": r.strategy.split(".")[-1], "metrics": r.metrics} for r in results],
+            challenge="planning",
+            metrics=["plan_success_rate", "step_accuracy", "tool_coverage"],
+            output_path=tables_dir / "table_planning_detailed.tex",
+        )
+
+        print(f"  Figure saved to: {figures_dir / 'fig2_main_planning_comparison.pdf'}")
+
+    return summary
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Section 5.2.2: Task Planning Experiment")
+    parser.add_argument("--max-samples", type=int, default=100, help="Maximum samples to test")
+    parser.add_argument("--skip-llm", action="store_true", help="Skip LLM-based methods")
+    parser.add_argument("--verbose", action="store_true", default=True, help="Verbose output")
+    args = parser.parse_args()
+
+    summary = run_planning_experiment(
+        max_samples=args.max_samples,
+        skip_llm=args.skip_llm,
+        verbose=args.verbose,
+    )
+
+    # 打印总结
+    print("\n" + "=" * 70)
+    print("📊 Summary")
+    print("=" * 70)
+    print(f"  Best strategy: {summary.best_strategy}")
+    print(
+        f"  Best success rate: {summary.best_metric * 100:.1f}%" if summary.best_metric else "  N/A"
+    )
+    print(f"  Target met: {'✅ YES' if summary.target_met else '❌ NO'}")
+
+
+if __name__ == "__main__":
+    main()

sage/benchmark/benchmark_agent/scripts/experiments/exp_main_selection.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""
+Section 5.2.3: Tool Selection Experiment (RQ3)
+
+研究问题: 现有方法从大规模工具库中选择正确工具的能力如何?
+
+测试方法:
+- selector.keyword    : BM25 关键词匹配
+- selector.embedding  : Dense Retrieval 语义匹配
+- selector.hybrid     : 40% BM25 + 60% Dense 融合
+- selector.gorilla    : Embedding 检索 + LLM 重排序
+- selector.dfsdt      : LLM 逐个评分 (ToolLLM 方法)
+
+目标指标:
+- Primary: Top-K Accuracy ≥ 95% (K=5)
+- Secondary: MRR, Recall@K, Precision@K
+- Tertiary: Latency (ms)
+
+Usage:
+    python exp_main_selection.py
+    python exp_main_selection.py --max-samples 100 --top-k 5
+    python exp_main_selection.py --skip-llm
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+
+from .exp_utils import (
+    ExperimentResult,
+    ExperimentSummary,
+    create_progress_bar,
+    load_benchmark_data,
+    print_metrics_detail,
+    print_result_row,
+    print_section_header,
+    print_subsection_header,
+    save_results,
+    setup_experiment_env,
+)
+
+
+def compute_mrr(predictions: list[list[str]], references: list[list[str]]) -> float:
+    """计算 Mean Reciprocal Rank。"""
+    rr_sum = 0.0
+    for pred, ref in zip(predictions, references):
+        ref_set = set(ref)
+        for i, p in enumerate(pred):
+            if p in ref_set:
+                rr_sum += 1.0 / (i + 1)
+                break
+    return rr_sum / len(predictions) if predictions else 0.0
+
+
+def compute_top_k_accuracy(
+    predictions: list[list[str]], references: list[list[str]], k: int
+) -> float:
+    """计算 Top-K Accuracy。"""
+    hits = 0
+    for pred, ref in zip(predictions, references):
+        pred_top_k = set(pred[:k])
+        ref_set = set(ref)
+        if pred_top_k & ref_set:
+            hits += 1
+    return hits / len(predictions) if predictions else 0.0
+
+
+def compute_recall_at_k(predictions: list[list[str]], references: list[list[str]], k: int) -> float:
+    """计算 Recall@K。"""
+    recalls = []
+    for pred, ref in zip(predictions, references):
+        pred_top_k = set(pred[:k])
+        ref_set = set(ref)
+        if ref_set:
+            recalls.append(len(pred_top_k & ref_set) / len(ref_set))
+        else:
+            recalls.append(0.0)
+    return sum(recalls) / len(recalls) if recalls else 0.0
+
+
+def compute_precision_at_k(
+    predictions: list[list[str]], references: list[list[str]], k: int
+) -> float:
+    """计算 Precision@K。"""
+    precisions = []
+    for pred, ref in zip(predictions, references):
+        pred_top_k = set(pred[:k])
+        ref_set = set(ref)
+        if pred_top_k:
+            precisions.append(len(pred_top_k & ref_set) / len(pred_top_k))
+        else:
+            precisions.append(0.0)
+    return sum(precisions) / len(precisions) if precisions else 0.0
+
+
+def normalize_ground_truth(ground_truth: object) -> list[str]:
+    """将地面真实标签统一为字符串列表。"""
+
+    if ground_truth is None:
+        return []
+
+    if isinstance(ground_truth, str):
+        return [ground_truth]
+
+    if isinstance(ground_truth, list):
+        return [str(item) for item in ground_truth]
+
+    if isinstance(ground_truth, dict):
+        for key in ("top_k", "tool_ids", "tools", "ids"):
+            value = ground_truth.get(key)
+            if isinstance(value, list):
+                return [str(item) for item in value]
+            if isinstance(value, str):
+                return [value]
+        # fall back to any string-like values
+        values = [str(value) for value in ground_truth.values() if value]
+        if values:
+            return values
+
+    return [str(ground_truth)]
+
+
+def run_selection_experiment(
+    max_samples: int = 100,
+    top_k: int = 5,
+    skip_llm: bool = False,
+    verbose: bool = True,
+) -> ExperimentSummary:
+    """
+    运行 Tool Selection 实验。
+
+    Args:
+        max_samples: 最大测试样本数
+        top_k: Top-K 参数
+        skip_llm: 是否跳过 LLM-based 方法
+        verbose: 是否打印详细信息
+
+    Returns:
+        ExperimentSummary 对象
+    """
+    setup_experiment_env(verbose=verbose)
+
+    print_section_header("Section 5.2.3: Tool Selection (RQ3)")
+    print(f"   Target: Top-{top_k} Accuracy ≥ 95%")
+    print(f"   Max samples: {max_samples}")
+
+    # 加载数据
+    samples = load_benchmark_data("selection", split="test", max_samples=max_samples)
+    if not samples:
+        print("  ❌ No selection data available")
+        return ExperimentSummary(section="5_2_main", challenge="selection")
+
+    print(f"   Loaded {len(samples)} samples")
+
+    # 获取策略注册表
+    try:
+        from sage.benchmark.benchmark_agent import get_adapter_registry
+
+        registry = get_adapter_registry()
+    except ImportError as e:
+        print(f"  ❌ Failed to import adapter registry: {e}")
+        return ExperimentSummary(section="5_2_main", challenge="selection")
+
+    # 定义测试策略
+    strategies = [
+        ("selector.keyword", "Keyword (BM25)"),
+        ("selector.embedding", "Embedding"),
+        ("selector.hybrid", "Hybrid"),
+        ("selector.gorilla", "Gorilla"),
+        ("selector.dfsdt", "DFSDT"),
+    ]
+
+    # 跳过 LLM 策略
+    LLM_STRATEGIES = {"selector.gorilla", "selector.dfsdt"}
+    if skip_llm:
+        strategies = [(name, display) for name, display in strategies if name not in LLM_STRATEGIES]
+        print("  ⚠️  Skipping LLM-based strategies")
+
+    results = []
+    target = 0.95
+
+    for strategy_name, display_name in strategies:
+        print_subsection_header(f"Testing: {display_name}")
+
+        try:
+            selector = registry.get(strategy_name)
+        except Exception as e:
+            print(f"    ⚠️  Failed to create selector: {e}")
+            continue
+
+        # 运行评测
+        start_time = time.time()
+        all_predictions = []
+        all_references = []
+
+        with create_progress_bar(len(samples), desc=f"  {display_name}") as pbar:
+            for sample in samples:
+                try:
+                    query = sample.get("instruction", sample.get("query", ""))
+                    candidate_tools = sample.get("candidate_tools", [])
+                    ground_truth = sample.get("ground_truth", sample.get("expected_tools", []))
+
+                    # 调用选择器
+                    predictions = selector.select(
+                        query, candidate_tools=candidate_tools, top_k=top_k
+                    )
+
+                    # 提取工具 ID
+                    if predictions and hasattr(predictions[0], "tool_id"):
+                        pred_ids = [p.tool_id for p in predictions]
+                    elif predictions and isinstance(predictions[0], dict):
+                        pred_ids = [p.get("tool_id", p.get("id", str(p))) for p in predictions]
+                    else:
+                        pred_ids = [str(p) for p in predictions] if predictions else []
+
+                    # 标准化 ground truth
+                    ref_ids = normalize_ground_truth(ground_truth)
+
+                    all_predictions.append(pred_ids)
+                    all_references.append(ref_ids)
+
+                except Exception as e:
+                    if verbose:
+                        print(f"    Error: {e}")
+                    all_predictions.append([])
+                    all_references.append(sample.get("ground_truth", []))
+
+                pbar.update(1)
+
+        elapsed = time.time() - start_time
+
+        # 计算指标
+        n = len(samples)
+        top_k_acc = compute_top_k_accuracy(all_predictions, all_references, top_k)
+        mrr = compute_mrr(all_predictions, all_references)
+        recall_k = compute_recall_at_k(all_predictions, all_references, top_k)
+        precision_k = compute_precision_at_k(all_predictions, all_references, top_k)
+
+        exp_result = ExperimentResult(
+            challenge="selection",
+            strategy=strategy_name,
+            metrics={
+                "top_k_accuracy": top_k_acc,
+                "mrr": mrr,
+                f"recall@{top_k}": recall_k,
+                f"precision@{top_k}": precision_k,
+            },
+            metadata={
+                "total_samples": n,
+                "top_k": top_k,
+                "latency_ms": elapsed * 1000 / n if n > 0 else 0,
+            },
+            passed=top_k_acc >= target,
+            target=target,
+        )
+        results.append(exp_result)
+
+        # 打印结果
+        print_result_row(display_name, {"top_k_accuracy": top_k_acc}, exp_result.passed, target)
+        if verbose:
+            print_metrics_detail(exp_result.metrics)
+
+    # 找出最佳策略
+    best_result = max(results, key=lambda r: r.metrics["top_k_accuracy"]) if results else None
+
+    summary = ExperimentSummary(
+        section="5_2_main",
+        challenge="selection",
+        results=results,
+        best_strategy=best_result.strategy if best_result else None,
+        best_metric=best_result.metrics["top_k_accuracy"] if best_result else None,
+        target_met=any(r.passed for r in results),
+    )
+
+    # 保存结果
+    output_file = save_results(summary.to_dict(), "5_2_main", "selection")
+    print(f"\n  Results saved to: {output_file}")
+
+    # 生成图表
+    if results:
+        from figure_generator import (
+            generate_detailed_table,
+            get_figures_dir,
+            get_tables_dir,
+            plot_challenge_comparison,
+        )
+
+        figures_dir = get_figures_dir()
+        tables_dir = get_tables_dir()
+
+        # 生成对比图
+        plot_challenge_comparison(
+            [{"strategy": r.strategy.split(".")[-1], "metrics": r.metrics} for r in results],
+            challenge="selection",
+            metrics=["top_k_accuracy", "mrr"],
+            target=target,
+            output_path=figures_dir / "fig3_main_selection_comparison.pdf",
+            title=f"Tool Selection: Strategy Comparison (Top-{top_k})",
+        )
+
+        # 生成表格
+        generate_detailed_table(
+            [{"strategy": r.strategy.split(".")[-1], "metrics": r.metrics} for r in results],
+            challenge="selection",
+            metrics=["top_k_accuracy", "mrr", f"recall@{top_k}", f"precision@{top_k}"],
+            output_path=tables_dir / "table_selection_detailed.tex",
+        )
+
+        print(f"  Figure saved to: {figures_dir / 'fig3_main_selection_comparison.pdf'}")
+
+    return summary
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Section 5.2.3: Tool Selection Experiment")
+    parser.add_argument("--max-samples", type=int, default=100, help="Maximum samples to test")
+    parser.add_argument("--top-k", type=int, default=5, help="Top-K parameter")
+    parser.add_argument("--skip-llm", action="store_true", help="Skip LLM-based methods")
+    parser.add_argument("--verbose", action="store_true", default=True, help="Verbose output")
+    args = parser.parse_args()
+
+    summary = run_selection_experiment(
+        max_samples=args.max_samples,
+        top_k=args.top_k,
+        skip_llm=args.skip_llm,
+        verbose=args.verbose,
+    )
+
+    # 打印总结
+    print("\n" + "=" * 70)
+    print("📊 Summary")
+    print("=" * 70)
+    print(f"  Best strategy: {summary.best_strategy}")
+    print(
+        f"  Best Top-K accuracy: {summary.best_metric * 100:.1f}%"
+        if summary.best_metric
+        else "  N/A"
+    )
+    print(f"  Target met: {'✅ YES' if summary.target_met else '❌ NO'}")
+
+
+if __name__ == "__main__":
+    main()