local-deep-research 0.3.12__py3-none-any.whl → 0.4.0__py3-none-any.whl

local_deep_research/benchmarks/benchmark_functions.py

@@ -0,0 +1,353 @@
+"""
+API functions for benchmarking.
+
+This module provides functions for running benchmarks programmatically.
+"""
+
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+from ..benchmarks import (
+    calculate_metrics,
+    generate_report,
+    run_benchmark,
+    run_browsecomp_benchmark,
+    run_simpleqa_benchmark,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate_simpleqa(
+    num_examples: int = 100,
+    search_iterations: int = 3,
+    questions_per_iteration: int = 3,
+    search_tool: str = "searxng",
+    human_evaluation: bool = False,
+    evaluation_model: Optional[str] = None,
+    evaluation_provider: Optional[str] = None,
+    output_dir: str = "benchmark_results",
+    search_model: Optional[str] = None,
+    search_provider: Optional[str] = None,
+    endpoint_url: Optional[str] = None,
+    search_strategy: str = "source_based",
+) -> Dict[str, Any]:
+    """
+    Run SimpleQA benchmark evaluation.
+
+    Args:
+        num_examples: Number of examples to evaluate
+        search_iterations: Number of search iterations per query
+        questions_per_iteration: Number of questions per iteration
+        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
+        human_evaluation: Whether to use human evaluation
+        evaluation_model: Optional custom model for evaluation
+        evaluation_provider: Optional custom provider for evaluation
+        output_dir: Directory to save results
+        search_model: Optional model to use for the search system
+        search_provider: Optional provider to use for the search system
+        endpoint_url: Optional endpoint URL for OpenRouter or other API services
+        search_strategy: Search strategy to use (default: 'source_based')
+
+    Returns:
+        Dictionary with benchmark results
+    """
+    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")
+
+    # Set up search configuration
+    search_config = {
+        "iterations": search_iterations,
+        "questions_per_iteration": questions_per_iteration,
+        "search_tool": search_tool,
+        "search_strategy": search_strategy,
+    }
+
+    # Add model configurations if provided
+    if search_model:
+        search_config["model_name"] = search_model
+    if search_provider:
+        search_config["provider"] = search_provider
+    if endpoint_url:
+        search_config["openai_endpoint_url"] = endpoint_url
+
+    # Check environment variables for additional configuration
+    if env_model := os.environ.get("LDR_SEARCH_MODEL"):
+        search_config["model_name"] = env_model
+    if env_provider := os.environ.get("LDR_SEARCH_PROVIDER"):
+        search_config["provider"] = env_provider
+    if env_url := os.environ.get("LDR_ENDPOINT_URL"):
+        search_config["openai_endpoint_url"] = env_url
+
+    # Set up evaluation configuration if needed
+    evaluation_config = None
+    if evaluation_model or evaluation_provider:
+        evaluation_config = {
+            "temperature": 0  # Always use zero temperature for evaluation
+        }
+        if evaluation_model:
+            evaluation_config["model_name"] = evaluation_model
+        if evaluation_provider:
+            evaluation_config["provider"] = evaluation_provider
+            # Add OpenRouter URL if using openai_endpoint
+            if evaluation_provider == "openai_endpoint":
+                evaluation_config["openai_endpoint_url"] = (
+                    "https://openrouter.ai/api/v1"
+                )
+
+    # Run the benchmark
+    results = run_simpleqa_benchmark(
+        num_examples=num_examples,
+        output_dir=output_dir,
+        search_config=search_config,
+        evaluation_config=evaluation_config,
+        human_evaluation=human_evaluation,
+    )
+
+    return results
+
+
+def evaluate_browsecomp(
+    num_examples: int = 100,
+    search_iterations: int = 3,
+    questions_per_iteration: int = 3,
+    search_tool: str = "searxng",
+    human_evaluation: bool = False,
+    evaluation_model: Optional[str] = None,
+    evaluation_provider: Optional[str] = None,
+    output_dir: str = "benchmark_results",
+    search_model: Optional[str] = None,
+    search_provider: Optional[str] = None,
+    endpoint_url: Optional[str] = None,
+    search_strategy: str = "source_based",
+) -> Dict[str, Any]:
+    """
+    Run BrowseComp benchmark evaluation.
+
+    Args:
+        num_examples: Number of examples to evaluate
+        search_iterations: Number of search iterations per query
+        questions_per_iteration: Number of questions per iteration
+        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
+        human_evaluation: Whether to use human evaluation
+        evaluation_model: Optional custom model for evaluation
+        evaluation_provider: Optional custom provider for evaluation
+        output_dir: Directory to save results
+        search_model: Optional model to use for the search system
+        search_provider: Optional provider to use for the search system
+        endpoint_url: Optional endpoint URL for OpenRouter or other API services
+        search_strategy: Search strategy to use (default: 'source_based')
+
+    Returns:
+        Dictionary with benchmark results
+    """
+    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")
+
+    # Set up search configuration
+    search_config = {
+        "iterations": search_iterations,
+        "questions_per_iteration": questions_per_iteration,
+        "search_tool": search_tool,
+        "search_strategy": search_strategy,
+    }
+
+    # Add model configurations if provided
+    if search_model:
+        search_config["model_name"] = search_model
+    if search_provider:
+        search_config["provider"] = search_provider
+    if endpoint_url:
+        search_config["openai_endpoint_url"] = endpoint_url
+
+    # Check environment variables for additional configuration
+    if env_model := os.environ.get("LDR_SEARCH_MODEL"):
+        search_config["model_name"] = env_model
+    if env_provider := os.environ.get("LDR_SEARCH_PROVIDER"):
+        search_config["provider"] = env_provider
+    if env_url := os.environ.get("LDR_ENDPOINT_URL"):
+        search_config["openai_endpoint_url"] = env_url
+
+    # Set up evaluation configuration if needed
+    evaluation_config = None
+    if evaluation_model or evaluation_provider:
+        evaluation_config = {
+            "temperature": 0  # Always use zero temperature for evaluation
+        }
+        if evaluation_model:
+            evaluation_config["model_name"] = evaluation_model
+        if evaluation_provider:
+            evaluation_config["provider"] = evaluation_provider
+            # Add OpenRouter URL if using openai_endpoint
+            if evaluation_provider == "openai_endpoint":
+                evaluation_config["openai_endpoint_url"] = (
+                    "https://openrouter.ai/api/v1"
+                )
+
+    # Run the benchmark
+    results = run_browsecomp_benchmark(
+        num_examples=num_examples,
+        output_dir=output_dir,
+        search_config=search_config,
+        evaluation_config=evaluation_config,
+        human_evaluation=human_evaluation,
+    )
+
+    return results
+
+
+def get_available_benchmarks() -> List[Dict[str, str]]:
+    """
+    Get information about available benchmarks.
+
+    Returns:
+        List of dictionaries with benchmark information
+    """
+    return [
+        {
+            "id": "simpleqa",
+            "name": "SimpleQA",
+            "description": "Benchmark for factual question answering",
+            "recommended_examples": 100,
+        },
+        {
+            "id": "browsecomp",
+            "name": "BrowseComp",
+            "description": "Benchmark for web browsing comprehension",
+            "recommended_examples": 100,
+        },
+    ]
+
+
+def compare_configurations(
+    dataset_type: str = "simpleqa",
+    num_examples: int = 20,
+    configurations: List[Dict[str, Any]] = None,
+    output_dir: str = "benchmark_comparisons",
+) -> Dict[str, Any]:
+    """
+    Compare multiple search configurations on the same benchmark.
+
+    Args:
+        dataset_type: Type of dataset to use
+        num_examples: Number of examples to evaluate
+        configurations: List of search configurations to compare
+        output_dir: Directory to save results
+
+    Returns:
+        Dictionary with comparison results
+    """
+    if not configurations:
+        # Default configurations to compare
+        configurations = [
+            {
+                "name": "Base Config",
+                "search_tool": "searxng",
+                "iterations": 1,
+                "questions_per_iteration": 3,
+            },
+            {
+                "name": "More Iterations",
+                "search_tool": "searxng",
+                "iterations": 3,
+                "questions_per_iteration": 3,
+            },
+            {
+                "name": "More Questions",
+                "search_tool": "searxng",
+                "iterations": 1,
+                "questions_per_iteration": 5,
+            },
+        ]
+
+    # Create output directory
+    import os
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Run benchmarks for each configuration
+    results = []
+    for config in configurations:
+        config_name = config.pop("name", f"Config-{len(results)}")
+
+        logger.info(f"Running benchmark with configuration: {config_name}")
+
+        search_config = {
+            "iterations": config.pop("iterations", 1),
+            "questions_per_iteration": config.pop("questions_per_iteration", 3),
+            "search_tool": config.pop("search_tool", "searxng"),
+        }
+
+        # Add any remaining config items
+        for key, value in config.items():
+            search_config[key] = value
+
+        # Run benchmark with this configuration
+        benchmark_result = run_benchmark(
+            dataset_type=dataset_type,
+            num_examples=num_examples,
+            output_dir=os.path.join(output_dir, config_name.replace(" ", "_")),
+            search_config=search_config,
+            run_evaluation=True,
+        )
+
+        # Add configuration name to results
+        benchmark_result["configuration_name"] = config_name
+        benchmark_result["search_config"] = search_config
+
+        results.append(benchmark_result)
+
+    # Generate comparison report
+    import time
+
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    report_file = os.path.join(output_dir, f"comparison_{dataset_type}_{timestamp}.md")
+
+    with open(report_file, "w") as f:
+        f.write(f"# Configuration Comparison - {dataset_type.capitalize()}\n\n")
+
+        # Write summary table
+        f.write("## Summary\n\n")
+        f.write("| Configuration | Accuracy | Avg. Time | Examples |\n")
+        f.write("|---------------|----------|-----------|----------|\n")
+
+        for result in results:
+            accuracy = result.get("metrics", {}).get("accuracy", 0)
+            avg_time = result.get("metrics", {}).get("average_processing_time", 0)
+            examples = result.get("total_examples", 0)
+
+            f.write(
+                f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"
+            )
+
+        f.write("\n## Configuration Details\n\n")
+
+        for result in results:
+            f.write(f"### {result['configuration_name']}\n\n")
+
+            config = result.get("search_config", {})
+            f.write("```\n")
+            for key, value in config.items():
+                f.write(f"{key}: {value}\n")
+            f.write("```\n\n")
+
+    logger.info(f"Comparison report saved to {report_file}")
+
+    return {
+        "status": "complete",
+        "dataset_type": dataset_type,
+        "configurations_tested": len(configurations),
+        "report_path": report_file,
+        "results": results,
+    }
+
+
+# Export the API functions
+__all__ = [
+    "evaluate_simpleqa",
+    "evaluate_browsecomp",
+    "get_available_benchmarks",
+    "compare_configurations",
+    "run_benchmark",  # For advanced users
+    "calculate_metrics",
+    "generate_report",
+]

local_deep_research/benchmarks/cli/__init__.py

@@ -0,0 +1,16 @@
+"""
+CLI module for benchmark functionality.
+
+This package provides command-line interface tools for
+running benchmarks and optimization tasks.
+"""
+
+from .benchmark_commands import main as benchmark_main
+from .benchmark_commands import (
+    setup_benchmark_parser,
+)
+
+__all__ = [
+    "benchmark_main",
+    "setup_benchmark_parser",
+]

local_deep_research/benchmarks/cli/benchmark_commands.py

@@ -0,0 +1,338 @@
+"""
+Command-line interface for benchmarking.
+
+This module provides CLI commands for running benchmarks.
+"""
+
+import argparse
+import logging
+
+from .. import (
+    get_available_datasets,
+    run_browsecomp_benchmark,
+    run_simpleqa_benchmark,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def setup_benchmark_parser(subparsers):
+    """
+    Set up the benchmark CLI commands.
+
+    Args:
+        subparsers: argparse subparsers object to add commands to
+    """
+    # Common benchmark arguments
+    benchmark_parent = argparse.ArgumentParser(add_help=False)
+    benchmark_parent.add_argument(
+        "--examples",
+        type=int,
+        default=100,
+        help="Number of examples to run (default: 100)",
+    )
+    benchmark_parent.add_argument(
+        "--iterations",
+        type=int,
+        default=3,
+        help="Number of search iterations (default: 3)",
+    )
+    benchmark_parent.add_argument(
+        "--questions", type=int, default=3, help="Questions per iteration (default: 3)"
+    )
+    benchmark_parent.add_argument(
+        "--search-tool",
+        type=str,
+        default="searxng",
+        help="Search tool to use (default: searxng)",
+    )
+    benchmark_parent.add_argument(
+        "--output-dir",
+        type=str,
+        default="data/benchmark_results",
+        help="Directory to save results (default: data/benchmark_results)",
+    )
+    benchmark_parent.add_argument(
+        "--human-eval",
+        action="store_true",
+        help="Use human evaluation instead of automated",
+    )
+    benchmark_parent.add_argument(
+        "--eval-model", type=str, help="Model to use for evaluation"
+    )
+    benchmark_parent.add_argument(
+        "--eval-provider", type=str, help="Provider to use for evaluation"
+    )
+    benchmark_parent.add_argument(
+        "--custom-dataset", type=str, help="Path to custom dataset"
+    )
+    benchmark_parent.add_argument(
+        "--no-eval", action="store_true", help="Skip evaluation phase"
+    )
+
+    # Add model configuration options for the search system
+    benchmark_parent.add_argument(
+        "--search-model", type=str, help="Model to use for the search system"
+    )
+    benchmark_parent.add_argument(
+        "--search-provider", type=str, help="Provider to use for the search system"
+    )
+    benchmark_parent.add_argument(
+        "--endpoint-url", type=str, help="Endpoint URL for OpenRouter or other API services"
+    )
+    benchmark_parent.add_argument(
+        "--search-strategy", type=str, default="source_based",
+        choices=["source_based", "standard", "rapid", "parallel", "iterdrag"],
+        help="Search strategy to use (default: source_based)"
+    )
+
+    # SimpleQA benchmark command
+    simpleqa_parser = subparsers.add_parser(
+        "simpleqa", parents=[benchmark_parent], help="Run SimpleQA benchmark"
+    )
+    simpleqa_parser.set_defaults(func=run_simpleqa_cli)
+
+    # BrowseComp benchmark command
+    browsecomp_parser = subparsers.add_parser(
+        "browsecomp", parents=[benchmark_parent], help="Run BrowseComp benchmark"
+    )
+    browsecomp_parser.set_defaults(func=run_browsecomp_cli)
+
+    # List available benchmarks command
+    list_parser = subparsers.add_parser("list", help="List available benchmarks")
+    list_parser.set_defaults(func=list_benchmarks_cli)
+
+    # Compare configurations command
+    compare_parser = subparsers.add_parser(
+        "compare", help="Compare multiple search configurations"
+    )
+    compare_parser.add_argument(
+        "--dataset",
+        type=str,
+        default="simpleqa",
+        choices=["simpleqa", "browsecomp"],
+        help="Dataset to use for comparison",
+    )
+    compare_parser.add_argument(
+        "--examples",
+        type=int,
+        default=20,
+        help="Number of examples for each configuration (default: 20)",
+    )
+    compare_parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="data/benchmark_results/comparison",
+        help="Directory to save comparison results",
+    )
+    compare_parser.set_defaults(func=compare_configs_cli)
+
+
+def run_simpleqa_cli(args):
+    """
+    CLI handler for SimpleQA benchmark.
+
+    Args:
+        args: Parsed command-line arguments
+    """
+    # Set up search configuration
+    search_config = {
+        "iterations": args.iterations,
+        "questions_per_iteration": args.questions,
+        "search_tool": args.search_tool,
+    }
+
+    # Add model configuration if provided
+    if hasattr(args, "search_model") and args.search_model:
+        search_config["model_name"] = args.search_model
+    if hasattr(args, "search_provider") and args.search_provider:
+        search_config["provider"] = args.search_provider
+    if hasattr(args, "endpoint_url") and args.endpoint_url:
+        search_config["openai_endpoint_url"] = args.endpoint_url
+    if hasattr(args, "search_strategy") and args.search_strategy:
+        search_config["search_strategy"] = args.search_strategy
+
+    # Set up evaluation configuration if needed
+    evaluation_config = None
+    if args.eval_model or args.eval_provider:
+        evaluation_config = {}
+        if args.eval_model:
+            evaluation_config["model_name"] = args.eval_model
+        if args.eval_provider:
+            evaluation_config["provider"] = args.eval_provider
+
+    # Run the benchmark
+    result = run_simpleqa_benchmark(
+        num_examples=args.examples,
+        dataset_path=args.custom_dataset,
+        output_dir=args.output_dir,
+        search_config=search_config,
+        evaluation_config=evaluation_config,
+        human_evaluation=args.human_eval,
+        run_evaluation=not args.no_eval,
+    )
+
+    # Print results summary
+    if "metrics" in result:
+        print("\nSimpleQA Benchmark Results:")
+        print(f"  Accuracy: {result['metrics'].get('accuracy', 0):.3f}")
+        print(f"  Total examples: {result['total_examples']}")
+        print(f"  Correct answers: {result['metrics'].get('correct', 0)}")
+        print(
+            f"  Average time: {result['metrics'].get('average_processing_time', 0):.2f}s"
+        )
+        print(f"\nReport saved to: {result.get('report_path', 'N/A')}")
+    else:
+        print("\nSimpleQA Benchmark Completed (no evaluation)")
+        print(f"  Total examples: {result['total_examples']}")
+        print(f"  Results saved to: {result.get('results_path', 'N/A')}")
+
+
+def run_browsecomp_cli(args):
+    """
+    CLI handler for BrowseComp benchmark.
+
+    Args:
+        args: Parsed command-line arguments
+    """
+    # Set up search configuration
+    search_config = {
+        "iterations": args.iterations,
+        "questions_per_iteration": args.questions,
+        "search_tool": args.search_tool,
+    }
+
+    # Add model configuration if provided
+    if hasattr(args, "search_model") and args.search_model:
+        search_config["model_name"] = args.search_model
+    if hasattr(args, "search_provider") and args.search_provider:
+        search_config["provider"] = args.search_provider
+    if hasattr(args, "endpoint_url") and args.endpoint_url:
+        search_config["openai_endpoint_url"] = args.endpoint_url
+    if hasattr(args, "search_strategy") and args.search_strategy:
+        search_config["search_strategy"] = args.search_strategy
+
+    # Set up evaluation configuration if needed
+    evaluation_config = None
+    if args.eval_model or args.eval_provider:
+        evaluation_config = {}
+        if args.eval_model:
+            evaluation_config["model_name"] = args.eval_model
+        if args.eval_provider:
+            evaluation_config["provider"] = args.eval_provider
+
+    # Run the benchmark
+    result = run_browsecomp_benchmark(
+        num_examples=args.examples,
+        dataset_path=args.custom_dataset,
+        output_dir=args.output_dir,
+        search_config=search_config,
+        evaluation_config=evaluation_config,
+        human_evaluation=args.human_eval,
+        run_evaluation=not args.no_eval,
+    )
+
+    # Print results summary
+    if "metrics" in result:
+        print("\nBrowseComp Benchmark Results:")
+        print(f"  Accuracy: {result['metrics'].get('accuracy', 0):.3f}")
+        print(f"  Total examples: {result['total_examples']}")
+        print(f"  Correct answers: {result['metrics'].get('correct', 0)}")
+        print(
+            f"  Average time: {result['metrics'].get('average_processing_time', 0):.2f}s"
+        )
+        print(f"\nReport saved to: {result.get('report_path', 'N/A')}")
+    else:
+        print("\nBrowseComp Benchmark Completed (no evaluation)")
+        print(f"  Total examples: {result['total_examples']}")
+        print(f"  Results saved to: {result.get('results_path', 'N/A')}")
+
+
+def list_benchmarks_cli(args):
+    """
+    CLI handler for listing available benchmarks.
+
+    Args:
+        args: Parsed command-line arguments
+    """
+    datasets = get_available_datasets()
+
+    print("\nAvailable Benchmarks:")
+    for dataset in datasets:
+        print(f"  {dataset['id']}: {dataset['name']}")
+        print(f"    {dataset['description']}")
+        print(f"    URL: {dataset['url']}")
+        print()
+
+
+def compare_configs_cli(args):
+    """
+    CLI handler for comparing multiple configurations.
+
+    Args:
+        args: Parsed command-line arguments
+    """
+    # Import the compare configurations function
+    from ...api.benchmark_functions import compare_configurations
+
+    # Run the comparison
+    result = compare_configurations(
+        dataset_type=args.dataset,
+        num_examples=args.examples,
+        output_dir=args.output_dir,
+    )
+
+    # Print results summary
+    print("\nConfiguration Comparison Results:")
+    print(f"  Dataset: {args.dataset}")
+    print(f"  Configurations tested: {result['configurations_tested']}")
+    print(f"  Report saved to: {result['report_path']}")
+
+    # Print brief comparison table
+    print("\nResults Summary:")
+    print("Configuration   | Accuracy | Avg. Time")
+    print("--------------- | -------- | ---------")
+    for res in result["results"]:
+        name = res["configuration_name"]
+        acc = res.get("metrics", {}).get("accuracy", 0)
+        time = res.get("metrics", {}).get("average_processing_time", 0)
+        print(f"{name:15} | {acc:.3f}    | {time:.2f}s")
+
+
+def main():
+    """
+    Main entry point for benchmark CLI.
+    """
+    parser = argparse.ArgumentParser(
+        description="Local Deep Research Benchmarking Tool", prog="ldr-benchmark"
+    )
+
+    # Set up logging
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
+
+    # Create subparsers
+    subparsers = parser.add_subparsers(
+        dest="command", help="Command to run", required=True
+    )
+
+    # Set up commands
+    setup_benchmark_parser(subparsers)
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Set up logging
+    log_level = logging.DEBUG if args.verbose else logging.INFO
+    logging.basicConfig(
+        level=log_level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+
+    # Run command
+    if hasattr(args, "func"):
+        args.func(args)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()