local-deep-research 0.3.12__py3-none-any.whl → 0.4.0__py3-none-any.whl

local_deep_research/benchmarks/efficiency/speed_profiler.py

@@ -0,0 +1,214 @@
+"""
+Speed profiling tools for Local Deep Research.
+
+This module provides functionality for measuring execution time
+of different components and processes in the research system.
+"""
+
+import logging
+import time
+from contextlib import contextmanager
+from typing import Dict, List, Optional, Any, Callable
+
+logger = logging.getLogger(__name__)
+
+
+class SpeedProfiler:
+    """
+    Profiler for tracking execution speed of components.
+    
+    This class provides methods for timing operations and
+    collecting performance statistics for later analysis.
+    """
+    
+    def __init__(self):
+        """Initialize the profiler with empty timing data."""
+        self.timings = {}
+        self.current_timers = {}
+        self.total_start_time = None
+        self.total_end_time = None
+    
+    def start(self):
+        """Start the global profiling session."""
+        self.timings = {}
+        self.current_timers = {}
+        self.total_start_time = time.time()
+    
+    def stop(self):
+        """Stop the global profiling session."""
+        self.total_end_time = time.time()
+        
+        # Stop any timers that are still running
+        for name in list(self.current_timers.keys()):
+            self.stop_timer(name)
+    
+    def start_timer(self, name: str):
+        """
+        Start a named timer.
+        
+        Args:
+            name: Name of the timer to start
+        """
+        if name in self.current_timers:
+            logger.warning(f"Timer '{name}' is already running. Restarting.")
+        
+        self.current_timers[name] = time.time()
+    
+    def stop_timer(self, name: str):
+        """
+        Stop a named timer and record the elapsed time.
+        
+        Args:
+            name: Name of the timer to stop
+        """
+        if name not in self.current_timers:
+            logger.warning(f"Timer '{name}' was not started.")
+            return
+        
+        elapsed = time.time() - self.current_timers[name]
+        
+        if name not in self.timings:
+            self.timings[name] = {
+                "total": elapsed,
+                "count": 1,
+                "min": elapsed,
+                "max": elapsed,
+                "starts": [self.current_timers[name]],
+                "durations": [elapsed]
+            }
+        else:
+            self.timings[name]["total"] += elapsed
+            self.timings[name]["count"] += 1
+            self.timings[name]["min"] = min(self.timings[name]["min"], elapsed)
+            self.timings[name]["max"] = max(self.timings[name]["max"], elapsed)
+            self.timings[name]["starts"].append(self.current_timers[name])
+            self.timings[name]["durations"].append(elapsed)
+        
+        del self.current_timers[name]
+    
+    @contextmanager
+    def timer(self, name: str):
+        """
+        Context manager for timing a block of code.
+        
+        Args:
+            name: Name of the timer
+        
+        Example:
+            with profiler.timer("my_operation"):
+                # Code to time
+                do_something()
+        """
+        self.start_timer(name)
+        try:
+            yield
+        finally:
+            self.stop_timer(name)
+    
+    def get_timings(self) -> Dict[str, Any]:
+        """
+        Get all recorded timings.
+        
+        Returns:
+            Dictionary of timing data for all measured operations
+        """
+        result = self.timings.copy()
+        
+        # Add averages
+        for name, data in result.items():
+            if data["count"] > 0:
+                data["avg"] = data["total"] / data["count"]
+        
+        # Add total duration
+        if self.total_start_time is not None and self.total_end_time is not None:
+            result["total"] = {
+                "total": self.total_end_time - self.total_start_time,
+                "count": 1,
+                "min": self.total_end_time - self.total_start_time,
+                "max": self.total_end_time - self.total_start_time,
+                "avg": self.total_end_time - self.total_start_time,
+                "starts": [self.total_start_time],
+                "durations": [self.total_end_time - self.total_start_time]
+            }
+        
+        return result
+    
+    def get_summary(self) -> Dict[str, float]:
+        """
+        Get a summary of timing information.
+        
+        Returns:
+            Dictionary with summary statistics
+        """
+        timings = self.get_timings()
+        summary = {}
+        
+        # Total duration
+        if "total" in timings:
+            summary["total_duration"] = timings["total"]["total"]
+        elif self.total_start_time is not None and self.total_end_time is not None:
+            summary["total_duration"] = self.total_end_time - self.total_start_time
+        else:
+            summary["total_duration"] = sum(t["total"] for t in timings.values())
+        
+        # Component durations
+        for name, data in timings.items():
+            if name != "total":
+                summary[f"{name}_duration"] = data["total"]
+                summary[f"{name}_percent"] = (
+                    data["total"] / summary["total_duration"] * 100 
+                    if summary["total_duration"] > 0 else 0
+                )
+        
+        # Per-operation breakdowns
+        for name, data in timings.items():
+            if data["count"] > 0:
+                summary[f"{name}_per_operation"] = data["total"] / data["count"]
+        
+        return summary
+    
+    def print_summary(self):
+        """Print a formatted summary of timing information."""
+        summary = self.get_summary()
+        total = summary.get("total_duration", 0)
+        
+        print("\n===== SPEED PROFILE SUMMARY =====")
+        print(f"Total execution time: {total:.2f} seconds")
+        print("\n--- Component Breakdown ---")
+        
+        # Print each component's timing
+        for name, data in self.timings.items():
+            if name != "total":
+                percent = data["total"] / total * 100 if total > 0 else 0
+                print(f"{name}: {data['total']:.2f}s ({percent:.1f}%) - "
+                      f"{data['count']} calls, avg {data['total'] / data['count']:.3f}s per call")
+        
+        print("\n==============================")
+
+
+def time_function(func: Callable) -> Callable:
+    """
+    Decorator to time a function's execution.
+    
+    Args:
+        func: Function to time
+        
+    Returns:
+        Wrapped function that logs its execution time
+    
+    Example:
+        @time_function
+        def my_slow_function():
+            # Some slow code
+            pass
+    """
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        elapsed = time.time() - start_time
+        
+        logger.info(f"{func.__name__} took {elapsed:.3f} seconds")
+        
+        return result
+    
+    return wrapper

local_deep_research/benchmarks/evaluators/__init__.py

@@ -0,0 +1,18 @@
+"""
+Benchmark evaluation package.
+
+This package provides evaluators for different benchmark types and
+a composite evaluator for weighted multi-benchmark evaluation.
+"""
+
+from .base import BaseBenchmarkEvaluator
+from .browsecomp import BrowseCompEvaluator
+from .composite import CompositeBenchmarkEvaluator
+from .simpleqa import SimpleQAEvaluator
+
+__all__ = [
+    "BaseBenchmarkEvaluator",
+    "SimpleQAEvaluator",
+    "BrowseCompEvaluator",
+    "CompositeBenchmarkEvaluator",
+]

local_deep_research/benchmarks/evaluators/base.py

@@ -0,0 +1,74 @@
+"""
+Base class for benchmark evaluators.
+
+This module defines the abstract base class that all benchmark evaluators
+must implement, establishing a common interface for different benchmark types.
+"""
+
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Dict
+
+logger = logging.getLogger(__name__)
+
+
+class BaseBenchmarkEvaluator(ABC):
+    """
+    Abstract base class for benchmark evaluators.
+
+    All benchmark evaluator implementations must inherit from this class and
+    implement the evaluate method to run their specific benchmark type.
+    """
+
+    def __init__(self, name: str):
+        """
+        Initialize benchmark evaluator with a name.
+
+        Args:
+            name: Unique identifier for this benchmark type
+        """
+        self.name = name
+
+    def get_name(self) -> str:
+        """
+        Get the benchmark name.
+
+        Returns:
+            The benchmark identifier
+        """
+        return self.name
+
+    @abstractmethod
+    def evaluate(
+        self,
+        system_config: Dict[str, Any],
+        num_examples: int,
+        output_dir: str,
+    ) -> Dict[str, Any]:
+        """
+        Run benchmark evaluation with given system configuration.
+
+        Args:
+            system_config: Configuration parameters for the system under test
+            num_examples: Number of benchmark examples to evaluate
+            output_dir: Directory to save evaluation results
+
+        Returns:
+            Dictionary with evaluation metrics including quality_score (0-1)
+        """
+        pass
+
+    def _create_subdirectory(self, output_dir: str) -> str:
+        """
+        Create a benchmark-specific subdirectory for output.
+
+        Args:
+            output_dir: Parent directory for output
+
+        Returns:
+            Path to the benchmark-specific directory
+        """
+        benchmark_dir = os.path.join(output_dir, self.name)
+        os.makedirs(benchmark_dir, exist_ok=True)
+        return benchmark_dir

local_deep_research/benchmarks/evaluators/browsecomp.py

@@ -0,0 +1,83 @@
+"""
+BrowseComp benchmark evaluator.
+
+This module provides a benchmark evaluator implementation for the BrowseComp
+benchmark, which tests browsing comprehension capabilities.
+"""
+
+import logging
+from typing import Any, Dict
+
+from ..runners import run_browsecomp_benchmark
+from .base import BaseBenchmarkEvaluator
+
+logger = logging.getLogger(__name__)
+
+
+class BrowseCompEvaluator(BaseBenchmarkEvaluator):
+    """
+    Evaluator for the BrowseComp benchmark.
+
+    This evaluator runs the BrowseComp benchmark, which tests a system's ability
+    to accurately comprehend and answer questions from web browsing.
+    """
+
+    def __init__(self):
+        """Initialize the BrowseComp evaluator."""
+        super().__init__("browsecomp")
+
+    def evaluate(
+        self,
+        system_config: Dict[str, Any],
+        num_examples: int,
+        output_dir: str,
+    ) -> Dict[str, Any]:
+        """
+        Run BrowseComp benchmark and return metrics.
+
+        Args:
+            system_config: Search and LLM configuration parameters
+            num_examples: Number of benchmark examples to run
+            output_dir: Directory to save evaluation results
+
+        Returns:
+            Dictionary with metrics including quality_score based on accuracy
+        """
+        # Create benchmark-specific directory
+        benchmark_dir = self._create_subdirectory(output_dir)
+
+        # Log benchmark execution
+        logger.info(f"Running BrowseComp benchmark with {num_examples} examples")
+
+        try:
+            # Run BrowseComp benchmark
+            results = run_browsecomp_benchmark(
+                num_examples=num_examples,
+                output_dir=benchmark_dir,
+                search_config=system_config,
+                run_evaluation=True,
+            )
+
+            # Extract metrics
+            metrics = results.get("metrics", {})
+            accuracy = metrics.get("accuracy", 0.0)
+
+            # Return evaluation results with quality score
+            return {
+                "benchmark_type": self.name,
+                "accuracy": accuracy,
+                "quality_score": accuracy,  # Map accuracy directly to quality score
+                "raw_results": results,
+                "report_path": results.get("report_path"),
+            }
+
+        except Exception as e:
+            logger.error(f"Error in BrowseComp evaluation: {str(e)}")
+
+            # Return error information
+            return {
+                "benchmark_type": self.name,
+                "error": str(e),
+                "quality_score": 0.0,
+                "accuracy": 0.0,
+            }

local_deep_research/benchmarks/evaluators/composite.py

@@ -0,0 +1,121 @@
+"""
+Composite benchmark evaluator.
+
+This module provides a composite evaluator that can run multiple benchmarks
+with weighted scores to provide a comprehensive evaluation.
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+# Import specific evaluator implementations
+from .browsecomp import BrowseCompEvaluator
+from .simpleqa import SimpleQAEvaluator
+
+logger = logging.getLogger(__name__)
+
+
+class CompositeBenchmarkEvaluator:
+    """
+    Evaluator that combines multiple benchmarks with weighted scores.
+
+    This evaluator runs multiple benchmark types and combines their scores
+    according to specified weights, enabling comprehensive evaluation across
+    different metrics and tasks.
+    """
+
+    def __init__(self, benchmark_weights: Optional[Dict[str, float]] = None):
+        """
+        Initialize with benchmark weights.
+
+        Args:
+            benchmark_weights: Dictionary mapping benchmark names to weights
+                Default: {"simpleqa": 1.0}
+        """
+        # Default to SimpleQA only if no weights provided
+        self.benchmark_weights = benchmark_weights or {"simpleqa": 1.0}
+
+        # Create evaluators for available benchmarks
+        self.evaluators = {
+            "simpleqa": SimpleQAEvaluator(),
+            "browsecomp": BrowseCompEvaluator(),
+        }
+
+        # Normalize weights to sum to 1.0
+        total_weight = sum(self.benchmark_weights.values())
+        if total_weight <= 0:
+            logger.warning(
+                "Total benchmark weight is zero or negative. Using default weights."
+            )
+            self.normalized_weights = {"simpleqa": 1.0}
+        else:
+            self.normalized_weights = {
+                k: w / total_weight for k, w in self.benchmark_weights.items()
+            }
+
+        # Log the weights being used
+        logger.info(f"Using normalized benchmark weights: {self.normalized_weights}")
+
+    def evaluate(
+        self,
+        system_config: Dict[str, Any],
+        num_examples: int,
+        output_dir: str,
+    ) -> Dict[str, Any]:
+        """
+        Run all requested benchmarks and compute weighted score.
+
+        Args:
+            system_config: Configuration parameters for the system under test
+            num_examples: Number of benchmark examples to evaluate
+            output_dir: Directory to save evaluation results
+
+        Returns:
+            Dictionary with combined metrics and individual benchmark results
+        """
+        all_results = {}
+        combined_score = 0.0
+
+        # Run each benchmark with weight > 0
+        for benchmark_name, weight in self.normalized_weights.items():
+            if weight > 0 and benchmark_name in self.evaluators:
+                evaluator = self.evaluators[benchmark_name]
+
+                try:
+                    # Run benchmark evaluation
+                    result = evaluator.evaluate(
+                        system_config=system_config,
+                        num_examples=num_examples,
+                        output_dir=output_dir,
+                    )
+
+                    # Store individual results
+                    all_results[benchmark_name] = result
+
+                    # Calculate weighted contribution to combined score
+                    quality_score = result.get("quality_score", 0.0)
+                    weighted_contribution = quality_score * weight
+
+                    logger.info(
+                        f"Benchmark {benchmark_name}: score={quality_score:.4f}, "
+                        f"weight={weight:.2f}, contribution={weighted_contribution:.4f}"
+                    )
+
+                    # Add to combined score
+                    combined_score += weighted_contribution
+
+                except Exception as e:
+                    logger.error(f"Error running {benchmark_name} benchmark: {str(e)}")
+                    all_results[benchmark_name] = {
+                        "benchmark_type": benchmark_name,
+                        "error": str(e),
+                        "quality_score": 0.0,
+                    }
+
+        # Return combined results
+        return {
+            "quality_score": combined_score,
+            "benchmark_results": all_results,
+            "benchmark_weights": self.normalized_weights,
+            "combined_score": combined_score,
+        }