local-deep-research 0.3.12__py3-none-any.whl → 0.4.1__py3-none-any.whl

local_deep_research/api/benchmark_functions.py

@@ -0,0 +1,288 @@
+"""
+API functions for benchmarking.
+
+This module provides functions for running benchmarks programmatically.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from ..benchmarks import (
+    calculate_metrics,
+    generate_report,
+    run_benchmark,
+    run_browsecomp_benchmark,
+    run_simpleqa_benchmark,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate_simpleqa(
+    num_examples: int = 100,
+    search_iterations: int = 3,
+    questions_per_iteration: int = 3,
+    search_tool: str = "searxng",
+    human_evaluation: bool = False,
+    evaluation_model: Optional[str] = None,
+    evaluation_provider: Optional[str] = None,
+    output_dir: str = "benchmark_results",
+) -> Dict[str, Any]:
+    """
+    Run SimpleQA benchmark evaluation.
+
+    Args:
+        num_examples: Number of examples to evaluate
+        search_iterations: Number of search iterations per query
+        questions_per_iteration: Number of questions per iteration
+        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
+        human_evaluation: Whether to use human evaluation
+        evaluation_model: Optional custom model for evaluation
+        evaluation_provider: Optional custom provider for evaluation
+        output_dir: Directory to save results
+
+    Returns:
+        Dictionary with benchmark results
+    """
+    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")
+
+    # Set up search configuration
+    search_config = {
+        "iterations": search_iterations,
+        "questions_per_iteration": questions_per_iteration,
+        "search_tool": search_tool,
+    }
+
+    # Set up evaluation configuration if needed
+    evaluation_config = None
+    if evaluation_model or evaluation_provider:
+        evaluation_config = {}
+        if evaluation_model:
+            evaluation_config["model_name"] = evaluation_model
+        if evaluation_provider:
+            evaluation_config["provider"] = evaluation_provider
+
+    # Run the benchmark
+    results = run_simpleqa_benchmark(
+        num_examples=num_examples,
+        output_dir=output_dir,
+        search_config=search_config,
+        evaluation_config=evaluation_config,
+        human_evaluation=human_evaluation,
+    )
+
+    return results
+
+
+def evaluate_browsecomp(
+    num_examples: int = 100,
+    search_iterations: int = 3,
+    questions_per_iteration: int = 3,
+    search_tool: str = "searxng",
+    human_evaluation: bool = False,
+    evaluation_model: Optional[str] = None,
+    evaluation_provider: Optional[str] = None,
+    output_dir: str = "benchmark_results",
+) -> Dict[str, Any]:
+    """
+    Run BrowseComp benchmark evaluation.
+
+    Args:
+        num_examples: Number of examples to evaluate
+        search_iterations: Number of search iterations per query
+        questions_per_iteration: Number of questions per iteration
+        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
+        human_evaluation: Whether to use human evaluation
+        evaluation_model: Optional custom model for evaluation
+        evaluation_provider: Optional custom provider for evaluation
+        output_dir: Directory to save results
+
+    Returns:
+        Dictionary with benchmark results
+    """
+    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")
+
+    # Set up search configuration
+    search_config = {
+        "iterations": search_iterations,
+        "questions_per_iteration": questions_per_iteration,
+        "search_tool": search_tool,
+    }
+
+    # Set up evaluation configuration if needed
+    evaluation_config = None
+    if evaluation_model or evaluation_provider:
+        evaluation_config = {}
+        if evaluation_model:
+            evaluation_config["model_name"] = evaluation_model
+        if evaluation_provider:
+            evaluation_config["provider"] = evaluation_provider
+
+    # Run the benchmark
+    results = run_browsecomp_benchmark(
+        num_examples=num_examples,
+        output_dir=output_dir,
+        search_config=search_config,
+        evaluation_config=evaluation_config,
+        human_evaluation=human_evaluation,
+    )
+
+    return results
+
+
+def get_available_benchmarks() -> List[Dict[str, str]]:
+    """
+    Get information about available benchmarks.
+
+    Returns:
+        List of dictionaries with benchmark information
+    """
+    return [
+        {
+            "id": "simpleqa",
+            "name": "SimpleQA",
+            "description": "Benchmark for factual question answering",
+            "recommended_examples": 100,
+        },
+        {
+            "id": "browsecomp",
+            "name": "BrowseComp",
+            "description": "Benchmark for web browsing comprehension",
+            "recommended_examples": 100,
+        },
+    ]
+
+
+def compare_configurations(
+    dataset_type: str = "simpleqa",
+    num_examples: int = 20,
+    configurations: List[Dict[str, Any]] = None,
+    output_dir: str = "benchmark_comparisons",
+) -> Dict[str, Any]:
+    """
+    Compare multiple search configurations on the same benchmark.
+
+    Args:
+        dataset_type: Type of dataset to use
+        num_examples: Number of examples to evaluate
+        configurations: List of search configurations to compare
+        output_dir: Directory to save results
+
+    Returns:
+        Dictionary with comparison results
+    """
+    if not configurations:
+        # Default configurations to compare
+        configurations = [
+            {
+                "name": "Base Config",
+                "search_tool": "searxng",
+                "iterations": 1,
+                "questions_per_iteration": 3,
+            },
+            {
+                "name": "More Iterations",
+                "search_tool": "searxng",
+                "iterations": 3,
+                "questions_per_iteration": 3,
+            },
+            {
+                "name": "More Questions",
+                "search_tool": "searxng",
+                "iterations": 1,
+                "questions_per_iteration": 5,
+            },
+        ]
+
+    # Create output directory
+    import os
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Run benchmarks for each configuration
+    results = []
+    for config in configurations:
+        config_name = config.pop("name", f"Config-{len(results)}")
+
+        logger.info(f"Running benchmark with configuration: {config_name}")
+
+        search_config = {
+            "iterations": config.pop("iterations", 1),
+            "questions_per_iteration": config.pop("questions_per_iteration", 3),
+            "search_tool": config.pop("search_tool", "searxng"),
+        }
+
+        # Add any remaining config items
+        for key, value in config.items():
+            search_config[key] = value
+
+        # Run benchmark with this configuration
+        benchmark_result = run_benchmark(
+            dataset_type=dataset_type,
+            num_examples=num_examples,
+            output_dir=os.path.join(output_dir, config_name.replace(" ", "_")),
+            search_config=search_config,
+            run_evaluation=True,
+        )
+
+        # Add configuration name to results
+        benchmark_result["configuration_name"] = config_name
+        benchmark_result["search_config"] = search_config
+
+        results.append(benchmark_result)
+
+    # Generate comparison report
+    import time
+
+    timestamp = time.strftime("%Y%m%d_%H%M%S")
+    report_file = os.path.join(output_dir, f"comparison_{dataset_type}_{timestamp}.md")
+
+    with open(report_file, "w") as f:
+        f.write(f"# Configuration Comparison - {dataset_type.capitalize()}\n\n")
+
+        # Write summary table
+        f.write("## Summary\n\n")
+        f.write("| Configuration | Accuracy | Avg. Time | Examples |\n")
+        f.write("|---------------|----------|-----------|----------|\n")
+
+        for result in results:
+            accuracy = result.get("metrics", {}).get("accuracy", 0)
+            avg_time = result.get("metrics", {}).get("average_processing_time", 0)
+            examples = result.get("total_examples", 0)
+
+            f.write(
+                f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"
+            )
+
+        f.write("\n## Configuration Details\n\n")
+
+        for result in results:
+            f.write(f"### {result['configuration_name']}\n\n")
+
+            config = result.get("search_config", {})
+            f.write("```\n")
+            for key, value in config.items():
+                f.write(f"{key}: {value}\n")
+            f.write("```\n\n")
+
+    logger.info(f"Comparison report saved to {report_file}")
+
+    return {
+        "status": "complete",
+        "dataset_type": dataset_type,
+        "configurations_tested": len(configurations),
+        "report_path": report_file,
+        "results": results,
+    }
+
+
+# Export the API functions
+__all__ = [
+    "evaluate_simpleqa",
+    "evaluate_browsecomp",
+    "get_available_benchmarks",
+    "compare_configurations",
+    "run_benchmark",  # For advanced users
+    "calculate_metrics",
+    "generate_report",
+]

local_deep_research/api/research_functions.py

@@ -3,17 +3,16 @@ API module for Local Deep Research.
 Provides programmatic access to search and research capabilities.
 """
 
-import logging
 from typing import Any, Callable, Dict, Optional
 
+from loguru import logger
+
 from ..config.llm_config import get_llm
 from ..config.search_config import get_search
 from ..report_generator import IntegratedReportGenerator
 from ..search_system import AdvancedSearchSystem
 from ..utilities.search_utilities import remove_think_tags
 
-logger = logging.getLogger(__name__)
-
 
 def _init_search_system(
     model_name: str | None = None,
@@ -24,6 +23,7 @@ def _init_search_system(
     search_tool: Optional[str] = None,
     iterations: int = 1,
     questions_per_iteration: int = 1,
+    search_strategy: str = "source_based",
 ) -> AdvancedSearchSystem:
     """
     Initializes the advanced search system with specified parameters. This function sets up
@@ -41,6 +41,7 @@ def _init_search_system(
         search_tool: Search engine to use (auto, wikipedia, arxiv, etc.). If None, uses default
         iterations: Number of research cycles to perform
         questions_per_iteration: Number of questions to generate per cycle
+        search_strategy: The name of the search strategy to use.
 
     Returns:
         AdvancedSearchSystem: An instance of the configured AdvancedSearchSystem.
@@ -64,7 +65,10 @@ def _init_search_system(
             )
 
     # Create search system with custom parameters
-    system = AdvancedSearchSystem(llm=llm, search=search_engine)
+    logger.info("Search strategy: {}", search_strategy)
+    system = AdvancedSearchSystem(
+        llm=llm, search=search_engine, strategy_name=search_strategy
+    )
 
     # Override default settings with user-provided values
     system.max_iterations = iterations

local_deep_research/benchmarks/README.md

@@ -0,0 +1,162 @@
+# LDR Benchmarking Framework
+
+The Local Deep Research (LDR) Benchmarking Framework allows you to evaluate and compare LDR's performance on standardized benchmarks.
+
+## Features
+
+- Run benchmarks on SimpleQA and BrowseComp datasets
+- Configure search parameters (iterations, questions per iteration, search tool)
+- Use automated evaluation with Claude 3.7 Sonnet (default) or other models
+- Generate detailed reports with metrics and examples
+- Compare multiple configurations to find optimal settings
+- API, CLI, and web interface integration
+
+## Benchmark Datasets
+
+### SimpleQA
+
+A straightforward question-answering benchmark with factual questions. This benchmark tests LDR's ability to find and synthesize factual information.
+
+### BrowseComp
+
+A web browsing comprehension benchmark with more complex questions requiring synthesis across multiple sources. This benchmark tests LDR's ability to understand and navigate complex information needs.
+
+## Usage
+
+### Programmatic API
+
+```python
+from local_deep_research.api.benchmark_functions import evaluate_simpleqa
+
+# Run SimpleQA benchmark with 20 examples
+result = evaluate_simpleqa(
+    num_examples=20,
+    search_iterations=3,
+    questions_per_iteration=3,
+    search_tool="searxng"
+)
+
+# Print accuracy
+print(f"Accuracy: {result['metrics']['accuracy']:.3f}")
+```
+
+### Command Line Interface
+
+```bash
+# Run SimpleQA benchmark
+python -m local_deep_research.cli.benchmark_commands simpleqa --examples 20 --iterations 3
+
+# Run BrowseComp benchmark
+python -m local_deep_research.cli.benchmark_commands browsecomp --examples 10 --search-tool wikipedia
+
+# Compare configurations
+python -m local_deep_research.cli.benchmark_commands compare --dataset simpleqa --examples 5
+```
+
+### Web Interface
+
+The benchmark dashboard is available at `/benchmark` in the LDR web interface. You can:
+
+1. Select a benchmark to run
+2. Configure parameters
+3. Run the benchmark
+4. View results and reports
+
+## Evaluation
+
+By default, benchmarks are evaluated using Claude 3.7 Sonnet via OpenRouter. You can customize the evaluation model:
+
+```python
+# Use a different model for evaluation
+result = evaluate_simpleqa(
+    num_examples=10,
+    evaluation_model="gpt-4o",
+    evaluation_provider="openai"
+)
+```
+
+You can also use human evaluation:
+
+```python
+# Use human evaluation
+result = evaluate_simpleqa(
+    num_examples=5,
+    human_evaluation=True
+)
+```
+
+## Configuration Comparison
+
+Compare multiple configurations to find optimal settings:
+
+```python
+from local_deep_research.api.benchmark_functions import compare_configurations
+
+# Define configurations to compare
+configurations = [
+    {
+        "name": "Base Config",
+        "search_tool": "searxng",
+        "iterations": 1,
+        "questions_per_iteration": 3
+    },
+    {
+        "name": "More Iterations",
+        "search_tool": "searxng",
+        "iterations": 3,
+        "questions_per_iteration": 3
+    },
+    {
+        "name": "Different Search Engine",
+        "search_tool": "wikipedia",
+        "iterations": 1,
+        "questions_per_iteration": 3
+    }
+]
+
+# Run comparison
+result = compare_configurations(
+    dataset_type="simpleqa",
+    num_examples=10,
+    configurations=configurations
+)
+```
+
+## Output Format
+
+Benchmark results include:
+
+- **metrics**: Accuracy, processing time, confidence scores
+- **report_path**: Path to generated report
+- **results_path**: Path to raw results file
+- **total_examples**: Number of examples processed
+- **status**: Completion status
+
+## Example Reports
+
+Reports include:
+
+- Overall accuracy and metrics
+- Configuration details
+- Example correct and incorrect answers
+- Time and date information
+
+## Integration with LDR Web App
+
+The benchmarking framework is fully integrated with the LDR web interface. You can run benchmarks and view results directly in the web app.
+
+## Adding Custom Benchmarks
+
+To add a custom benchmark:
+
+1. Create a dataset loader in `datasets.py`
+2. Add evaluation templates in `templates.py`
+3. Create benchmark runners in `runners.py`
+4. Expose the benchmark through the API in `api/benchmark_functions.py`
+
+## Performance Considerations
+
+- Running benchmarks can be resource-intensive
+- Start with a small number of examples for testing
+- Full benchmarks with 100+ examples may take several hours to complete
+- Consider using a more powerful machine for large benchmarks

local_deep_research/benchmarks/__init__.py

@@ -0,0 +1,51 @@
+"""
+Benchmarking module for Local Deep Research.
+
+This module provides tools for evaluating LDR's performance on standard benchmarks
+and for optimizing performance through parameter tuning.
+"""
+
+__version__ = "0.2.0"
+
+# Core benchmark functionality
+from .datasets import get_available_datasets, load_dataset
+from .metrics import (
+    calculate_metrics,
+    calculate_quality_metrics,
+    calculate_speed_metrics,
+    calculate_resource_metrics,
+    calculate_combined_score,
+    generate_report,
+)
+from .runners import run_benchmark, run_browsecomp_benchmark, run_simpleqa_benchmark
+
+# Optimization functionality
+from .optimization import (
+    optimize_parameters,
+    optimize_for_quality,
+    optimize_for_speed,
+    optimize_for_efficiency,
+)
+
+__all__ = [
+    # Core benchmark functionality
+    "run_benchmark",
+    "run_simpleqa_benchmark",
+    "run_browsecomp_benchmark",
+    "load_dataset",
+    "get_available_datasets",
+    "calculate_metrics",
+    "generate_report",
+
+    # Metrics for optimization
+    "calculate_quality_metrics",
+    "calculate_speed_metrics",
+    "calculate_resource_metrics",
+    "calculate_combined_score",
+
+    # Optimization functionality
+    "optimize_parameters",
+    "optimize_for_quality",
+    "optimize_for_speed",
+    "optimize_for_efficiency",
+]