local-deep-research 0.3.12__py3-none-any.whl → 0.4.1__py3-none-any.whl

local_deep_research/benchmarks/metrics/visualization.py

@@ -0,0 +1,205 @@
+"""
+Visualization utilities for optimization results.
+
+This module provides functions for generating visual representations
+of benchmark and optimization results.
+"""
+
+import logging
+import os
+from typing import Dict, List, Optional, Tuple, Union
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+# Check if matplotlib is available
+try:
+    import matplotlib.pyplot as plt
+    from matplotlib.figure import Figure
+
+    MATPLOTLIB_AVAILABLE = True
+except ImportError:
+    MATPLOTLIB_AVAILABLE = False
+    logger.warning("Matplotlib not available. Visualization functions will be limited.")
+
+
+def plot_optimization_history(
+    trial_values: List[float],
+    best_values: List[float],
+    output_file: Optional[str] = None,
+    title: str = "Optimization History",
+) -> Optional[Figure]:
+    """
+    Plot the optimization history.
+
+    Args:
+        trial_values: List of objective values for each trial
+        best_values: List of best values observed up to each trial
+        output_file: Path to save the plot (if None, returns figure without saving)
+        title: Plot title
+
+    Returns:
+        Matplotlib figure or None if matplotlib is not available
+    """
+    if not MATPLOTLIB_AVAILABLE:
+        logger.warning("Matplotlib not available. Cannot create plot.")
+        return None
+
+    fig, ax = plt.subplots(figsize=(10, 6))
+    trials = list(range(1, len(trial_values) + 1))
+
+    # Plot trial values and best values
+    ax.plot(trials, trial_values, "o-", alpha=0.5, label="Trial Value")
+    ax.plot(trials, best_values, "r-", label="Best Value")
+
+    # Add labels and title
+    ax.set_xlabel("Trial Number")
+    ax.set_ylabel("Objective Value")
+    ax.set_title(title)
+    ax.grid(True, linestyle="--", alpha=0.7)
+    ax.legend()
+
+    # Save or return
+    if output_file:
+        fig.tight_layout()
+        fig.savefig(output_file, dpi=300, bbox_inches="tight")
+        logger.info(f"Saved optimization history plot to {output_file}")
+
+    return fig
+
+
+def plot_parameter_importance(
+    parameter_names: List[str],
+    importance_values: List[float],
+    output_file: Optional[str] = None,
+    title: str = "Parameter Importance",
+) -> Optional[Figure]:
+    """
+    Plot parameter importance.
+
+    Args:
+        parameter_names: List of parameter names
+        importance_values: List of importance values
+        output_file: Path to save the plot (if None, returns figure without saving)
+        title: Plot title
+
+    Returns:
+        Matplotlib figure or None if matplotlib is not available
+    """
+    if not MATPLOTLIB_AVAILABLE:
+        logger.warning("Matplotlib not available. Cannot create plot.")
+        return None
+
+    # Sort by importance
+    sorted_indices = np.argsort(importance_values)
+    sorted_names = [parameter_names[i] for i in sorted_indices]
+    sorted_values = [importance_values[i] for i in sorted_indices]
+
+    fig, ax = plt.subplots(figsize=(10, 6))
+    y_pos = range(len(sorted_names))
+    
+    # Create horizontal bar chart
+    ax.barh(y_pos, sorted_values, align="center")
+    ax.set_yticks(y_pos)
+    ax.set_yticklabels(sorted_names)
+    ax.invert_yaxis()  # Labels read top-to-bottom
+    
+    # Add labels and title
+    ax.set_xlabel("Importance")
+    ax.set_title(title)
+    ax.grid(True, linestyle="--", alpha=0.3, axis="x")
+
+    # Save or return
+    if output_file:
+        fig.tight_layout()
+        fig.savefig(output_file, dpi=300, bbox_inches="tight")
+        logger.info(f"Saved parameter importance plot to {output_file}")
+
+    return fig
+
+
+def plot_quality_vs_speed(
+    quality_scores: List[float],
+    speed_scores: List[float],
+    parameter_values: Optional[List[Dict[str, any]]] = None,
+    output_file: Optional[str] = None,
+    title: str = "Quality vs. Speed Trade-off",
+) -> Optional[Figure]:
+    """
+    Plot quality vs. speed trade-off.
+
+    Args:
+        quality_scores: List of quality scores
+        speed_scores: List of speed scores
+        parameter_values: Optional list of parameter dictionaries for each point
+        output_file: Path to save the plot (if None, returns figure without saving)
+        title: Plot title
+
+    Returns:
+        Matplotlib figure or None if matplotlib is not available
+    """
+    if not MATPLOTLIB_AVAILABLE:
+        logger.warning("Matplotlib not available. Cannot create plot.")
+        return None
+
+    fig, ax = plt.subplots(figsize=(10, 8))
+    
+    # Create scatter plot
+    scatter = ax.scatter(
+        speed_scores, 
+        quality_scores, 
+        c=np.arange(len(quality_scores)), 
+        cmap="viridis", 
+        alpha=0.7,
+        s=100
+    )
+    
+    # Add colorbar to show trial number
+    cbar = plt.colorbar(scatter)
+    cbar.set_label("Trial Number")
+    
+    # Add labels and title
+    ax.set_xlabel("Speed Score (higher = faster)")
+    ax.set_ylabel("Quality Score (higher = better)")
+    ax.set_title(title)
+    ax.grid(True, linestyle="--", alpha=0.5)
+    
+    # Add reference lines
+    ax.axhline(y=0.7, color="r", linestyle="--", alpha=0.3, label="Good Quality Threshold")
+    ax.axvline(x=0.7, color="g", linestyle="--", alpha=0.3, label="Good Speed Threshold")
+    
+    # Mark Pareto frontier
+    if len(quality_scores) > 2:
+        try:
+            # Identify Pareto frontier points
+            pareto_points = []
+            for i in range(len(quality_scores)):
+                is_pareto = True
+                for j in range(len(quality_scores)):
+                    if i != j:
+                        if quality_scores[j] >= quality_scores[i] and speed_scores[j] >= speed_scores[i]:
+                            if quality_scores[j] > quality_scores[i] or speed_scores[j] > speed_scores[i]:
+                                is_pareto = False
+                                break
+                if is_pareto:
+                    pareto_points.append((speed_scores[i], quality_scores[i]))
+            
+            # Sort pareto points by speed score
+            pareto_points.sort()
+            if pareto_points:
+                pareto_x, pareto_y = zip(*pareto_points)
+                ax.plot(pareto_x, pareto_y, "k--", label="Pareto Frontier")
+                ax.scatter(pareto_x, pareto_y, c="red", s=50, alpha=0.8)
+        except Exception as e:
+            logger.warning(f"Error calculating Pareto frontier: {e}")
+    
+    ax.legend()
+    
+    # Save or return
+    if output_file:
+        fig.tight_layout()
+        fig.savefig(output_file, dpi=300, bbox_inches="tight")
+        logger.info(f"Saved quality vs. speed plot to {output_file}")
+
+    return fig

local_deep_research/benchmarks/metrics.py

@@ -0,0 +1,11 @@
+"""
+Metrics calculation and report generation.
+
+This module is maintained for backward compatibility.
+New code should use the metrics package directly.
+"""
+
+from .metrics.calculation import calculate_metrics
+from .metrics.reporting import generate_report
+
+__all__ = ["calculate_metrics", "generate_report"]

local_deep_research/benchmarks/optimization/__init__.py

@@ -0,0 +1,32 @@
+"""
+Optimization submodule for parameter tuning in Local Deep Research.
+
+This module provides tools for finding optimal parameter configurations
+for the research system using Optuna and other optimization methods.
+"""
+
+from local_deep_research.benchmarks.optimization.api import (
+    optimize_for_efficiency,
+    optimize_for_quality,
+    optimize_for_speed,
+    optimize_parameters,
+)
+from local_deep_research.benchmarks.optimization.metrics import (
+    calculate_combined_score,
+    calculate_quality_metrics,
+    calculate_resource_metrics,
+    calculate_speed_metrics,
+)
+from local_deep_research.benchmarks.optimization.optuna_optimizer import OptunaOptimizer
+
+__all__ = [
+    "OptunaOptimizer",
+    "optimize_parameters",
+    "optimize_for_speed",
+    "optimize_for_quality",
+    "optimize_for_efficiency",
+    "calculate_quality_metrics",
+    "calculate_speed_metrics",
+    "calculate_resource_metrics",
+    "calculate_combined_score",
+]

local_deep_research/benchmarks/optimization/api.py

@@ -0,0 +1,274 @@
+"""
+API functions for optimization tasks in Local Deep Research.
+
+This module provides a simplified interface for parameter optimization
+without having to directly work with the optimizer classes.
+"""
+
+import logging
+import os
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+# No metrics imports needed here, they're used in the OptunaOptimizer
+from .optuna_optimizer import OptunaOptimizer
+
+logger = logging.getLogger(__name__)
+
+
+def optimize_parameters(
+    query: str,
+    param_space: Optional[Dict[str, Any]] = None,
+    output_dir: str = os.path.join("data", "optimization_results"),
+    model_name: Optional[str] = None,
+    provider: Optional[str] = None,
+    search_tool: Optional[str] = None,
+    temperature: float = 0.7,
+    n_trials: int = 30,
+    timeout: Optional[int] = None,
+    n_jobs: int = 1,
+    study_name: Optional[str] = None,
+    optimization_metrics: Optional[List[str]] = None,
+    metric_weights: Optional[Dict[str, float]] = None,
+    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
+    benchmark_weights: Optional[Dict[str, float]] = None,
+) -> Tuple[Dict[str, Any], float]:
+    """
+    Optimize parameters for Local Deep Research.
+
+    Args:
+        query: The research query to use for all experiments
+        param_space: Dictionary defining parameter search spaces (optional)
+        output_dir: Directory to save optimization results
+        model_name: Name of the LLM model to use
+        provider: LLM provider
+        search_tool: Search engine to use
+        temperature: LLM temperature
+        n_trials: Number of parameter combinations to try
+        timeout: Maximum seconds to run optimization (None for no limit)
+        n_jobs: Number of parallel jobs for optimization
+        study_name: Name of the Optuna study
+        optimization_metrics: List of metrics to optimize (default: ["quality", "speed"])
+        metric_weights: Dictionary of weights for each metric
+        progress_callback: Optional callback for progress updates
+        benchmark_weights: Dictionary mapping benchmark types to weights
+            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
+            If None, only SimpleQA is used with weight 1.0
+
+    Returns:
+        Tuple of (best_parameters, best_score)
+    """
+    # Create optimizer
+    optimizer = OptunaOptimizer(
+        base_query=query,
+        output_dir=output_dir,
+        model_name=model_name,
+        provider=provider,
+        search_tool=search_tool,
+        temperature=temperature,
+        n_trials=n_trials,
+        timeout=timeout,
+        n_jobs=n_jobs,
+        study_name=study_name,
+        optimization_metrics=optimization_metrics,
+        metric_weights=metric_weights,
+        progress_callback=progress_callback,
+        benchmark_weights=benchmark_weights,
+    )
+
+    # Run optimization
+    return optimizer.optimize(param_space)
+
+
+def optimize_for_speed(
+    query: str,
+    n_trials: int = 20,
+    output_dir: str = os.path.join("data", "optimization_results"),
+    model_name: Optional[str] = None,
+    provider: Optional[str] = None,
+    search_tool: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
+    benchmark_weights: Optional[Dict[str, float]] = None,
+) -> Tuple[Dict[str, Any], float]:
+    """
+    Optimize parameters with a focus on speed performance.
+
+    Args:
+        query: The research query to use for all experiments
+        n_trials: Number of parameter combinations to try
+        output_dir: Directory to save optimization results
+        model_name: Name of the LLM model to use
+        provider: LLM provider
+        search_tool: Search engine to use
+        progress_callback: Optional callback for progress updates
+        benchmark_weights: Dictionary mapping benchmark types to weights
+            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
+            If None, only SimpleQA is used with weight 1.0
+
+    Returns:
+        Tuple of (best_parameters, best_score)
+    """
+    # Focus on speed with reduced parameter space
+    param_space = {
+        "iterations": {
+            "type": "int",
+            "low": 1,
+            "high": 3,
+            "step": 1,
+        },
+        "questions_per_iteration": {
+            "type": "int",
+            "low": 1,
+            "high": 3,
+            "step": 1,
+        },
+        "search_strategy": {
+            "type": "categorical",
+            "choices": ["rapid", "parallel", "source_based"],
+        },
+    }
+
+    # Speed-focused weights
+    metric_weights = {"speed": 0.8, "quality": 0.2, "resource": 0.0}
+
+    return optimize_parameters(
+        query=query,
+        param_space=param_space,
+        output_dir=output_dir,
+        model_name=model_name,
+        provider=provider,
+        search_tool=search_tool,
+        n_trials=n_trials,
+        metric_weights=metric_weights,
+        optimization_metrics=["speed", "quality"],
+        progress_callback=progress_callback,
+        benchmark_weights=benchmark_weights,
+    )
+
+
+def optimize_for_quality(
+    query: str,
+    n_trials: int = 30,
+    output_dir: str = os.path.join("data", "optimization_results"),
+    model_name: Optional[str] = None,
+    provider: Optional[str] = None,
+    search_tool: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
+    benchmark_weights: Optional[Dict[str, float]] = None,
+) -> Tuple[Dict[str, Any], float]:
+    """
+    Optimize parameters with a focus on result quality.
+
+    Args:
+        query: The research query to use for all experiments
+        n_trials: Number of parameter combinations to try
+        output_dir: Directory to save optimization results
+        model_name: Name of the LLM model to use
+        provider: LLM provider
+        search_tool: Search engine to use
+        progress_callback: Optional callback for progress updates
+        benchmark_weights: Dictionary mapping benchmark types to weights
+            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
+            If None, only SimpleQA is used with weight 1.0
+
+    Returns:
+        Tuple of (best_parameters, best_score)
+    """
+    # Quality-focused weights
+    metric_weights = {"quality": 0.9, "speed": 0.1, "resource": 0.0}
+
+    return optimize_parameters(
+        query=query,
+        output_dir=output_dir,
+        model_name=model_name,
+        provider=provider,
+        search_tool=search_tool,
+        n_trials=n_trials,
+        metric_weights=metric_weights,
+        optimization_metrics=["quality", "speed"],
+        progress_callback=progress_callback,
+        benchmark_weights=benchmark_weights,
+    )
+
+
+def optimize_for_efficiency(
+    query: str,
+    n_trials: int = 25,
+    output_dir: str = os.path.join("data", "optimization_results"),
+    model_name: Optional[str] = None,
+    provider: Optional[str] = None,
+    search_tool: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
+    benchmark_weights: Optional[Dict[str, float]] = None,
+) -> Tuple[Dict[str, Any], float]:
+    """
+    Optimize parameters with a focus on resource efficiency.
+
+    Args:
+        query: The research query to use for all experiments
+        n_trials: Number of parameter combinations to try
+        output_dir: Directory to save optimization results
+        model_name: Name of the LLM model to use
+        provider: LLM provider
+        search_tool: Search engine to use
+        progress_callback: Optional callback for progress updates
+        benchmark_weights: Dictionary mapping benchmark types to weights
+            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
+            If None, only SimpleQA is used with weight 1.0
+
+    Returns:
+        Tuple of (best_parameters, best_score)
+    """
+    # Balance of quality, speed and resource usage
+    metric_weights = {"quality": 0.4, "speed": 0.3, "resource": 0.3}
+
+    return optimize_parameters(
+        query=query,
+        output_dir=output_dir,
+        model_name=model_name,
+        provider=provider,
+        search_tool=search_tool,
+        n_trials=n_trials,
+        metric_weights=metric_weights,
+        optimization_metrics=["quality", "speed", "resource"],
+        progress_callback=progress_callback,
+        benchmark_weights=benchmark_weights,
+    )
+
+
+def get_default_param_space() -> Dict[str, Any]:
+    """
+    Get the default parameter search space for optimization.
+
+    Returns:
+        Dictionary defining the default parameter search spaces
+    """
+    return {
+        "iterations": {
+            "type": "int",
+            "low": 1,
+            "high": 5,
+            "step": 1,
+        },
+        "questions_per_iteration": {
+            "type": "int",
+            "low": 1,
+            "high": 5,
+            "step": 1,
+        },
+        "search_strategy": {
+            "type": "categorical",
+            "choices": ["iterdrag", "standard", "rapid", "parallel", "source_based"],
+        },
+        "max_results": {
+            "type": "int",
+            "low": 10,
+            "high": 100,
+            "step": 10,
+        },
+        "max_filtered_results": {
+            "type": "int",
+            "low": 5,
+            "high": 50,
+            "step": 5,
+        },
+    }

local_deep_research/benchmarks/optimization/metrics.py

@@ -0,0 +1,20 @@
+"""
+Optimization metrics calculation.
+
+This module is maintained for backward compatibility.
+New code should use the unified metrics module.
+"""
+
+from ..metrics.calculation import (
+    calculate_combined_score,
+    calculate_quality_metrics,
+    calculate_resource_metrics,
+    calculate_speed_metrics,
+)
+
+__all__ = [
+    "calculate_quality_metrics",
+    "calculate_speed_metrics",
+    "calculate_resource_metrics",
+    "calculate_combined_score",
+]