local-deep-research 0.3.12__py3-none-any.whl → 0.4.1__py3-none-any.whl

local_deep_research/benchmarks/evaluators/simpleqa.py

@@ -0,0 +1,271 @@
+"""
+SimpleQA benchmark evaluator.
+
+This module provides a benchmark evaluator implementation for the SimpleQA
+benchmark, which tests simple question-answering capabilities.
+"""
+
+import json
+import logging
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from local_deep_research.api import quick_summary
+from ..datasets.base import DatasetRegistry
+from ..metrics import calculate_metrics, generate_report
+from ..runners import run_simpleqa_benchmark  # Keep for backward compatibility
+from .base import BaseBenchmarkEvaluator
+
+logger = logging.getLogger(__name__)
+
+
+class SimpleQAEvaluator(BaseBenchmarkEvaluator):
+    """
+    Evaluator for the SimpleQA benchmark.
+
+    This evaluator runs the SimpleQA benchmark, which tests a system's ability
+    to accurately answer straightforward factual questions.
+    """
+
+    def __init__(self):
+        """Initialize the SimpleQA evaluator."""
+        super().__init__("simpleqa")
+
+    def evaluate(
+        self,
+        system_config: Dict[str, Any],
+        num_examples: int,
+        output_dir: str,
+        use_direct_dataset: bool = True,
+    ) -> Dict[str, Any]:
+        """
+        Run SimpleQA benchmark and return metrics.
+
+        Args:
+            system_config: Search and LLM configuration parameters
+            num_examples: Number of benchmark examples to run
+            output_dir: Directory to save evaluation results
+            use_direct_dataset: Whether to use dataset classes directly (recommended)
+                                or fall back to runner functions
+
+        Returns:
+            Dictionary with metrics including quality_score based on accuracy
+        """
+        # Create benchmark-specific directory
+        benchmark_dir = self._create_subdirectory(output_dir)
+
+        # Log benchmark execution
+        logger.info(f"Running SimpleQA benchmark with {num_examples} examples")
+
+        try:
+            if use_direct_dataset:
+                # Use dataset classes directly (new approach)
+                results = self._run_with_dataset_class(
+                    system_config=system_config,
+                    num_examples=num_examples,
+                    output_dir=benchmark_dir,
+                )
+            else:
+                # Fall back to legacy runner function
+                results = run_simpleqa_benchmark(
+                    num_examples=num_examples,
+                    output_dir=benchmark_dir,
+                    search_config=system_config,
+                    run_evaluation=True,
+                )
+
+            # Extract metrics
+            metrics = results.get("metrics", {})
+            accuracy = metrics.get("accuracy", 0.0)
+
+            # Return evaluation results with quality score
+            return {
+                "benchmark_type": self.name,
+                "accuracy": accuracy,
+                "quality_score": accuracy,  # Map accuracy directly to quality score
+                "raw_results": results,
+                "report_path": results.get("report_path"),
+            }
+
+        except Exception as e:
+            logger.error(f"Error in SimpleQA evaluation: {str(e)}")
+
+            # Return error information
+            return {
+                "benchmark_type": self.name,
+                "error": str(e),
+                "quality_score": 0.0,
+                "accuracy": 0.0,
+            }
+
+    def _run_with_dataset_class(
+        self,
+        system_config: Dict[str, Any],
+        num_examples: int,
+        output_dir: str,
+    ) -> Dict[str, Any]:
+        """
+        Run SimpleQA benchmark using dataset classes directly.
+
+        This implementation directly uses the dataset classes rather than
+        going through the runner functions, allowing for more flexibility
+        and better integration with the object-oriented architecture.
+
+        Args:
+            system_config: Search and LLM configuration parameters
+            num_examples: Number of benchmark examples to run
+            output_dir: Directory to save evaluation results
+
+        Returns:
+            Dictionary with benchmark results
+        """
+        # Create a dataset instance using the registry
+        try:
+            dataset_instance = DatasetRegistry.create_dataset(
+                dataset_id="simpleqa",
+                num_examples=num_examples,
+                seed=system_config.get("seed", 42),
+            )
+
+            # Load dataset examples
+            examples = dataset_instance.load()
+            logger.info(f"Loaded {len(examples)} SimpleQA examples")
+
+            # Set up output files
+            timestamp = time.strftime("%Y%m%d_%H%M%S")
+            results_file = os.path.join(output_dir, f"simpleqa_{timestamp}_results.jsonl")
+            evaluation_file = os.path.join(output_dir, f"simpleqa_{timestamp}_evaluation.jsonl")
+            report_file = os.path.join(output_dir, f"simpleqa_{timestamp}_report.md")
+
+            # Process each example
+            results = []
+
+            for i, example in enumerate(examples):
+                # Extract question and answer using dataset methods
+                question = dataset_instance.get_question(example)
+                correct_answer = dataset_instance.get_answer(example)
+
+                logger.info(f"Processing {i + 1}/{len(examples)}: {question[:50]}...")
+
+                try:
+                    # Format query based on dataset type
+                    formatted_query = question  # Simple format for SimpleQA
+
+                    # Time the search
+                    start_time = time.time()
+
+                    # Create search config from system_config
+                    search_params = {
+                        "iterations": system_config.get("iterations", 3),
+                        "questions_per_iteration": system_config.get("questions_per_iteration", 3),
+                        "search_tool": system_config.get("search_tool", "searxng"),
+                        # Note: search_strategy is stored in the config but not passed to quick_summary
+                        # as it's not supported by the underlying API
+                    }
+
+                    # Get response from LDR
+                    from local_deep_research.api import quick_summary
+                    search_result = quick_summary(
+                        query=formatted_query,
+                        iterations=search_params.get("iterations"),
+                        questions_per_iteration=search_params.get("questions_per_iteration"),
+                        search_tool=search_params.get("search_tool"),
+                    )
+
+                    end_time = time.time()
+                    processing_time = end_time - start_time
+
+                    # Extract response
+                    response = search_result.get("summary", "")
+
+                    # Extract structured answer
+                    from ..graders import extract_answer_from_response
+                    extracted = extract_answer_from_response(response, "simpleqa")
+
+                    # Format result
+                    result = {
+                        "id": example.get("id", f"example_{i}"),
+                        "problem": question,
+                        "correct_answer": correct_answer,
+                        "response": response,
+                        "extracted_answer": extracted["extracted_answer"],
+                        "confidence": extracted["confidence"],
+                        "processing_time": processing_time,
+                        "sources": search_result.get("sources", []),
+                        "search_config": search_params,
+                    }
+
+                    # Add to results list
+                    results.append(result)
+
+                    # Write result to file
+                    with open(results_file, "a") as f:
+                        f.write(json.dumps(result) + "\n")
+
+                except Exception as e:
+                    logger.error(f"Error processing example {i + 1}: {str(e)}")
+
+                    # Create error result
+                    error_result = {
+                        "id": example.get("id", f"example_{i}"),
+                        "problem": question,
+                        "correct_answer": correct_answer,
+                        "error": str(e),
+                        "processing_time": 0,
+                    }
+
+                    # Add to results list
+                    results.append(error_result)
+
+                    # Write error result to file
+                    with open(results_file, "a") as f:
+                        f.write(json.dumps(error_result) + "\n")
+
+            # Grade results
+            from ..graders import grade_results
+            evaluation_results = grade_results(
+                results_file=results_file,
+                output_file=evaluation_file,
+                dataset_type="simpleqa",
+            )
+
+            # Calculate metrics
+            metrics = calculate_metrics(evaluation_file)
+
+            # Generate report
+            dataset_name = "SimpleQA"
+            report_path = generate_report(
+                metrics=metrics,
+                results_file=evaluation_file,
+                output_file=report_file,
+                dataset_name=dataset_name,
+                config_info={
+                    "Dataset": "SimpleQA",
+                    "Examples": len(examples),
+                    "Iterations": search_params.get("iterations", 3),
+                    "Questions per iteration": search_params.get("questions_per_iteration", 3),
+                    "Search tool": search_params.get("search_tool", "searxng"),
+                    "Search strategy": search_params.get("search_strategy", "source_based"),
+                },
+            )
+
+            # Return results
+            return {
+                "status": "complete",
+                "dataset_type": "simpleqa",
+                "results_path": results_file,
+                "evaluation_path": evaluation_file,
+                "report_path": report_path,
+                "metrics": metrics,
+                "total_examples": len(examples),
+                "accuracy": metrics.get("accuracy", 0),
+            }
+
+        except Exception as e:
+            logger.error(f"Error in direct dataset evaluation: {str(e)}")
+            return {
+                "status": "error",
+                "dataset_type": "simpleqa",
+                "error": str(e),
+            }

local_deep_research/benchmarks/graders.py

@@ -0,0 +1,410 @@
+"""
+Evaluation and grading functionality.
+
+This module provides tools for evaluating model outputs against reference answers.
+"""
+
+import json
+import logging
+import os
+import re
+from typing import Any, Callable, Dict, List, Optional
+
+from langchain.schema import HumanMessage
+
+from ..config.llm_config import get_llm
+from .templates import BROWSECOMP_GRADER_TEMPLATE, SIMPLEQA_GRADER_TEMPLATE
+
+logger = logging.getLogger(__name__)
+
+# Default evaluation configuration using Claude 3.7 Sonnet via OpenRouter
+DEFAULT_EVALUATION_CONFIG = {
+    "model_name": "anthropic/claude-3.7-sonnet",  # Correct model ID for OpenRouter
+    "provider": "openai_endpoint",  # Use OpenRouter
+    "openai_endpoint_url": "https://openrouter.ai/api/v1",  # OpenRouter URL
+    "temperature": 0,  # Zero temp for consistent evaluation
+    # Note: max_tokens removed as it's not supported by LDR's get_llm()
+}
+
+
+def get_evaluation_llm(custom_config: Optional[Dict[str, Any]] = None):
+    """
+    Get an LLM for evaluation purposes using Claude 3.7 Sonnet via OpenRouter
+    by default, which can be overridden with custom settings.
+
+    Args:
+        custom_config: Optional custom configuration that overrides defaults
+
+    Returns:
+        An LLM instance for evaluation
+    """
+    # Start with default config (Claude 3.7 Sonnet via OpenRouter)
+    config = DEFAULT_EVALUATION_CONFIG.copy()
+
+    # Override with any custom settings
+    if custom_config:
+        config.update(custom_config)
+
+    logger.info(
+        f"Getting evaluation LLM with provider={config['provider']}, model={config['model_name']}"
+    )
+
+    # Remove any parameters that LDR's get_llm doesn't support
+    # This ensures compatibility with LDR's implementation
+    ldr_supported_params = {
+        "model_name",
+        "temperature",
+        "provider",
+        "openai_endpoint_url",
+        "api_key",
+    }
+
+    filtered_config = {k: v for k, v in config.items() if k in ldr_supported_params}
+
+    # Check if we're using openai_endpoint but don't have an API key configured
+    if filtered_config.get("provider") == "openai_endpoint":
+        # Try to get API key from environment or config
+        import os
+
+        api_key = os.getenv("OPENAI_ENDPOINT_API_KEY")
+        if not api_key:
+            logger.warning(
+                "Using openai_endpoint provider but no API key found. "
+                "Set the OPENAI_ENDPOINT_API_KEY environment variable or "
+                "specify api_key in the evaluation_config."
+            )
+            # Try to fall back to LDR's config if API key not explicitly provided
+            # The get_llm function will handle this case
+
+    # Get the LLM using LDR's existing function
+    return get_llm(**filtered_config)
+
+
+def extract_answer_from_response(
+    response: str, dataset_type: str = "simpleqa"
+) -> Dict[str, str]:
+    """
+    Extract structured information from LDR's response.
+
+    Args:
+        response: Response from LDR
+        dataset_type: Type of dataset
+
+    Returns:
+        Dictionary with extracted answer and confidence
+    """
+    # Clean up citations
+    response = re.sub(r"\[\d+\]", "", response)
+
+    # Extract differently based on dataset type
+    if dataset_type.lower() == "browsecomp":
+        # Extract the final answer from structured response
+        answer_match = re.search(r"Exact Answer:\s*(.*?)(?:\n|$)", response)
+        exact_answer = answer_match.group(1).strip() if answer_match else "None"
+
+        # Extract confidence
+        confidence_match = re.search(r"Confidence:\s*(\d+)%", response)
+        confidence = confidence_match.group(1) if confidence_match else "100"
+
+        return {"extracted_answer": exact_answer, "confidence": confidence}
+
+    # For SimpleQA, return the whole response as the answer
+    return {
+        "extracted_answer": response,
+        "confidence": "100",  # SimpleQA doesn't have confidence scores
+    }
+
+
+def grade_results(
+    results_file: str,
+    output_file: str,
+    dataset_type: str = "simpleqa",
+    evaluation_config: Optional[Dict[str, Any]] = None,
+    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Grade benchmark results using LLM.
+
+    Args:
+        results_file: Path to results file
+        output_file: Path to save graded results
+        dataset_type: Type of dataset
+        evaluation_config: Optional custom config for evaluation LLM
+        progress_callback: Optional callback for progress updates
+
+    Returns:
+        List of graded results
+    """
+    # Get evaluation LLM
+    evaluation_llm = get_evaluation_llm(evaluation_config)
+
+    # Select appropriate template
+    template = (
+        BROWSECOMP_GRADER_TEMPLATE
+        if dataset_type.lower() == "browsecomp"
+        else SIMPLEQA_GRADER_TEMPLATE
+    )
+
+    # Load results
+    results = []
+    with open(results_file, "r") as f:
+        for line in f:
+            if line.strip():
+                results.append(json.loads(line))
+
+    # Remove output file if it exists
+    if os.path.exists(output_file):
+        os.remove(output_file)
+
+    graded_results = []
+    correct_count = 0
+
+    # Process each result
+    for idx, result in enumerate(results):
+        question = result.get("problem", "")
+        correct_answer = result.get("correct_answer", "")
+        response = result.get("response", "")
+
+        # Call progress callback if provided
+        if progress_callback:
+            progress_callback(
+                idx,
+                len(results),
+                {"status": "grading", "index": idx, "total": len(results)},
+            )
+
+        logger.info(f"Grading {idx + 1}/{len(results)}: {question[:50]}...")
+
+        # Format grading prompt
+        grading_prompt = template.format(
+            question=question, correct_answer=correct_answer, response=response
+        )
+
+        try:
+            # Grade using LLM
+            if hasattr(evaluation_llm, "invoke") and callable(evaluation_llm.invoke):
+                if hasattr(evaluation_llm, "chat_messages"):
+                    # Handle ChatOpenAI and similar models that use messages
+                    grading_response = evaluation_llm.invoke(
+                        [HumanMessage(content=grading_prompt)]
+                    ).content
+                else:
+                    # Handle other LLM types
+                    grading_response = evaluation_llm.invoke(grading_prompt)
+                    if hasattr(grading_response, "content"):
+                        grading_response = grading_response.content
+            else:
+                # Fallback for other LLM interfaces
+                grading_response = str(evaluation_llm(grading_prompt))
+
+            # Extract grading information using regex
+            if dataset_type.lower() == "browsecomp":
+                # BrowseComp-specific extraction
+                extracted_answer_match = re.search(
+                    r"extracted_final_answer:\s*(.*?)(?:\n|$)", grading_response
+                )
+                extracted_answer = (
+                    extracted_answer_match.group(1).strip()
+                    if extracted_answer_match
+                    else "None"
+                )
+
+                reasoning_match = re.search(
+                    r"reasoning:\s*(.*?)(?:\n\n|\ncorrect:|\Z)",
+                    grading_response,
+                    re.DOTALL,
+                )
+                reasoning = reasoning_match.group(1).strip() if reasoning_match else ""
+
+                correct_match = re.search(
+                    r"correct:\s*(yes|no)", grading_response, re.IGNORECASE
+                )
+                is_correct = (
+                    (correct_match.group(1).lower() == "yes")
+                    if correct_match
+                    else False
+                )
+
+                confidence_match = re.search(r"confidence:\s*(\d+)", grading_response)
+                confidence = confidence_match.group(1) if confidence_match else "100"
+            else:
+                # SimpleQA extraction
+                extracted_answer_match = re.search(
+                    r"Extracted Answer:\s*(.*?)(?:\n|$)", grading_response
+                )
+                extracted_answer = (
+                    extracted_answer_match.group(1).strip()
+                    if extracted_answer_match
+                    else "None"
+                )
+
+                reasoning_match = re.search(
+                    r"Reasoning:\s*(.*?)(?:\nCorrect:|\Z)", grading_response, re.DOTALL
+                )
+                reasoning = reasoning_match.group(1).strip() if reasoning_match else ""
+
+                correct_match = re.search(
+                    r"Correct:\s*(yes|no)", grading_response, re.IGNORECASE
+                )
+                is_correct = (
+                    (correct_match.group(1).lower() == "yes")
+                    if correct_match
+                    else False
+                )
+
+                confidence = "100"  # SimpleQA doesn't have confidence
+
+            if is_correct:
+                correct_count += 1
+
+            # Format graded result
+            graded_result = result.copy()
+            graded_result.update(
+                {
+                    "extracted_by_grader": extracted_answer,
+                    "reasoning": reasoning,
+                    "is_correct": is_correct,
+                    "graded_confidence": confidence,
+                    "grader_response": grading_response,
+                }
+            )
+
+            graded_results.append(graded_result)
+
+            # Write to output file
+            with open(output_file, "a") as f:
+                f.write(json.dumps(graded_result) + "\n")
+
+            # Call progress callback if provided
+            if progress_callback:
+                progress_callback(
+                    idx,
+                    len(results),
+                    {
+                        "status": "graded",
+                        "is_correct": is_correct,
+                        "result": graded_result,
+                    },
+                )
+
+        except Exception as e:
+            logger.error(f"Error grading result {idx + 1}: {str(e)}")
+
+            # Handle error
+            error_result = result.copy()
+            error_result["grading_error"] = str(e)
+
+            with open(output_file, "a") as f:
+                f.write(json.dumps(error_result) + "\n")
+
+            graded_results.append(error_result)
+
+            # Call progress callback if provided
+            if progress_callback:
+                progress_callback(
+                    idx,
+                    len(results),
+                    {"status": "error", "error": str(e), "result": error_result},
+                )
+
+    accuracy = correct_count / len(results) if results else 0
+    logger.info(f"Grading complete. Accuracy: {accuracy:.3f}")
+    logger.info(f"Correct: {correct_count}/{len(results)}")
+
+    return graded_results
+
+
+def human_evaluation(
+    results_file: str, output_file: str, interactive: bool = True
+) -> List[Dict[str, Any]]:
+    """
+    Allow for human evaluation of results.
+
+    Args:
+        results_file: Path to results file
+        output_file: Path to save human-graded results
+        interactive: Whether to run in interactive console mode
+
+    Returns:
+        List of human-graded results
+    """
+    # Load results
+    results = []
+    with open(results_file, "r") as f:
+        for line in f:
+            if line.strip():
+                results.append(json.loads(line))
+
+    # Remove output file if it exists
+    if os.path.exists(output_file):
+        os.remove(output_file)
+
+    human_graded_results = []
+    correct_count = 0
+
+    if interactive:
+        logger.info(f"Human evaluation: {len(results)} examples to grade")
+        print(f"Human evaluation: {len(results)} examples to grade")
+        print(
+            "For each example, you'll see the question, correct answer, and model's response."
+        )
+        print("You'll be asked to judge if the model's answer is correct.")
+
+    for idx, result in enumerate(results):
+        question = result.get("problem", "")
+        correct_answer = result.get("correct_answer", "")
+        response = result.get("response", "")
+        extracted_answer = result.get("extracted_answer", "")
+
+        if interactive:
+            print(f"\n\n===== Example {idx + 1}/{len(results)} =====")
+            print(f"Question: {question}")
+            print(f"\nCorrect Answer: {correct_answer}")
+            print(f"\nModel Response: {response}")
+            print(f"\nExtracted Answer: {extracted_answer}")
+
+            # Get human judgment
+            while True:
+                judgment = (
+                    input("\nIs the model's answer correct? (y/n): ").strip().lower()
+                )
+                if judgment in ["y", "n"]:
+                    break
+                print("Please enter 'y' or 'n'")
+
+            is_correct = judgment == "y"
+
+            # Get reasoning
+            reasoning = input("Please provide reasoning for your judgment: ").strip()
+        else:
+            # Non-interactive mode - placeholder for API/UI implementation
+            # In a real implementation, this would be filled by UI actions
+            is_correct = False
+            reasoning = "Non-interactive evaluation"
+
+        if is_correct:
+            correct_count += 1
+
+        # Update result with human judgment
+        human_result = result.copy()
+        human_result.update(
+            {
+                "is_correct": is_correct,
+                "reasoning": reasoning,
+                "human_evaluation": True,
+            }
+        )
+
+        human_graded_results.append(human_result)
+
+        # Write to output file
+        with open(output_file, "a") as f:
+            f.write(json.dumps(human_result) + "\n")
+
+    accuracy = correct_count / len(results) if results else 0
+    logger.info(f"Human evaluation complete. Accuracy: {accuracy:.3f}")
+    if interactive:
+        print(f"\nHuman evaluation complete. Accuracy: {accuracy:.3f}")
+        print(f"Correct: {correct_count}/{len(results)}")
+
+    return human_graded_results