local-deep-research 0.5.9__py3-none-any.whl → 0.6.0__py3-none-any.whl

local_deep_research/benchmarks/web_api/benchmark_service.py

@@ -0,0 +1,920 @@
+"""Benchmark service for handling web-based benchmark execution."""
+
+import hashlib
+import json
+import threading
+import time
+from datetime import datetime
+from typing import Dict, List, Optional, Any
+
+from loguru import logger
+
+from ..models.benchmark_models import (
+    BenchmarkRun,
+    BenchmarkResult,
+    BenchmarkStatus,
+    DatasetType,
+)
+from ..datasets import load_dataset
+from ..graders import extract_answer_from_response, grade_single_result
+from ..runners import format_query
+from ...api.research_functions import quick_summary
+from ...utilities.db_utils import get_db_session
+from ...web.services.socket_service import SocketIOService
+
+
+class BenchmarkService:
+    """Service for managing benchmark runs through the web interface."""
+
+    def __init__(self, socket_service=None):
+        self.active_runs: Dict[int, Dict] = {}
+        self.socket_service = socket_service or self._get_socket_service()
+        self.rate_limit_detected: Dict[
+            int, bool
+        ] = {}  # Track rate limiting per benchmark run
+
+    def _get_socket_service(self):
+        """Get socket service instance, handling cases where Flask app is not available."""
+        try:
+            return SocketIOService()
+        except Exception:
+            # Return a mock socket service for testing/standalone use
+            class MockSocketService:
+                def emit_to_room(self, room, event, data):
+                    pass
+
+            return MockSocketService()
+
+    def generate_config_hash(self, search_config: Dict[str, Any]) -> str:
+        """Generate a hash for search configuration compatibility checking."""
+        relevant_params = {
+            "iterations": search_config.get("iterations"),
+            "questions_per_iteration": search_config.get(
+                "questions_per_iteration"
+            ),
+            "search_tool": search_config.get("search_tool"),
+            "search_strategy": search_config.get("search_strategy"),
+            "model_name": search_config.get("model_name"),
+            "provider": search_config.get("provider"),
+        }
+        # Remove None values
+        relevant_params = {
+            k: v for k, v in relevant_params.items() if v is not None
+        }
+        config_str = json.dumps(relevant_params, sort_keys=True)
+        return hashlib.md5(config_str.encode()).hexdigest()[:8]
+
+    def generate_query_hash(self, question: str, dataset_type: str) -> str:
+        """Generate a hash for a query to enable deduplication."""
+        query_content = f"{question.strip()}|{dataset_type.lower()}"
+        return hashlib.md5(query_content.encode()).hexdigest()
+
+    def create_benchmark_run(
+        self,
+        run_name: Optional[str],
+        search_config: Dict[str, Any],
+        evaluation_config: Dict[str, Any],
+        datasets_config: Dict[str, Dict],
+    ) -> int:
+        """Create a new benchmark run in the database."""
+        session = get_db_session()
+
+        try:
+            config_hash = self.generate_config_hash(search_config)
+
+            # Calculate total examples
+            total_examples = sum(
+                config.get("count", 0) for config in datasets_config.values()
+            )
+
+            benchmark_run = BenchmarkRun(
+                run_name=run_name,
+                config_hash=config_hash,
+                query_hash_list=[],  # Will be populated as we process
+                search_config=search_config,
+                evaluation_config=evaluation_config,
+                datasets_config=datasets_config,
+                total_examples=total_examples,
+                status=BenchmarkStatus.PENDING,
+            )
+
+            session.add(benchmark_run)
+            session.commit()
+
+            logger.info(
+                f"Created benchmark run {benchmark_run.id} with config hash {config_hash}"
+            )
+            return benchmark_run.id
+
+        except Exception:
+            session.rollback()
+            logger.exception("Error creating benchmark run")
+            raise
+        finally:
+            session.close()
+
+    def get_existing_results(self, config_hash: str) -> Dict[str, Dict]:
+        """Get existing results with compatible configuration."""
+        session = get_db_session()
+
+        try:
+            # Find compatible runs
+            compatible_runs = (
+                session.query(BenchmarkRun)
+                .filter(BenchmarkRun.config_hash == config_hash)
+                .filter(BenchmarkRun.status == BenchmarkStatus.COMPLETED)
+                .all()
+            )
+
+            existing_results = {}
+            for run in compatible_runs:
+                results = (
+                    session.query(BenchmarkResult)
+                    .filter(BenchmarkResult.benchmark_run_id == run.id)
+                    .filter(
+                        BenchmarkResult.is_correct.isnot(None)
+                    )  # Only completed evaluations
+                    .all()
+                )
+
+                for result in results:
+                    existing_results[result.query_hash] = {
+                        "id": result.example_id,
+                        "dataset_type": result.dataset_type.value,
+                        "problem": result.question,
+                        "correct_answer": result.correct_answer,
+                        "response": result.response,
+                        "extracted_answer": result.extracted_answer,
+                        "confidence": result.confidence,
+                        "processing_time": result.processing_time,
+                        "sources": result.sources,
+                        "is_correct": result.is_correct,
+                        "graded_confidence": result.graded_confidence,
+                        "grader_response": result.grader_response,
+                        "query_hash": result.query_hash,
+                    }
+
+            logger.info(
+                f"Found {len(existing_results)} existing results for config hash {config_hash}"
+            )
+            return existing_results
+
+        except Exception:
+            logger.exception("Error loading existing results")
+            return {}
+        finally:
+            session.close()
+
+    def start_benchmark(self, benchmark_run_id: int) -> bool:
+        """Start a benchmark run in a background thread."""
+        try:
+            # Mark as in progress
+            self.update_benchmark_status(
+                benchmark_run_id, BenchmarkStatus.IN_PROGRESS
+            )
+
+            # Start background thread
+            thread = threading.Thread(
+                target=self._run_benchmark_thread,
+                args=(benchmark_run_id,),
+                daemon=True,
+            )
+            thread.start()
+
+            self.active_runs[benchmark_run_id] = {
+                "thread": thread,
+                "start_time": datetime.now(),
+                "status": "running",
+            }
+
+            logger.info(f"Started benchmark run {benchmark_run_id}")
+            return True
+
+        except Exception as e:
+            logger.exception(f"Error starting benchmark {benchmark_run_id}")
+            self.update_benchmark_status(
+                benchmark_run_id, BenchmarkStatus.FAILED, str(e)
+            )
+            return False
+
+    def _run_benchmark_thread(self, benchmark_run_id: int):
+        """Main benchmark execution thread."""
+        session = get_db_session()
+
+        try:
+            # Get benchmark run details
+            benchmark_run = (
+                session.query(BenchmarkRun)
+                .filter(BenchmarkRun.id == benchmark_run_id)
+                .first()
+            )
+            if not benchmark_run:
+                raise ValueError(f"Benchmark run {benchmark_run_id} not found")
+
+            # Load existing results for deduplication
+            existing_results = self.get_existing_results(
+                benchmark_run.config_hash
+            )
+
+            # Create task queue
+            task_queue = self._create_task_queue(
+                benchmark_run.datasets_config,
+                existing_results,
+                benchmark_run_id,
+            )
+
+            # Update total with new tasks only
+            benchmark_run.total_examples = len(task_queue) + len(
+                existing_results
+            )
+            benchmark_run.completed_examples = len(existing_results)
+            benchmark_run.start_time = datetime.now()
+            session.commit()
+
+            # Process tasks
+            for i, task in enumerate(task_queue):
+                try:
+                    # Process single task
+                    result = self._process_benchmark_task(
+                        task,
+                        benchmark_run.search_config,
+                        benchmark_run.evaluation_config,
+                    )
+
+                    # Save result
+                    self._save_benchmark_result(result, benchmark_run_id)
+
+                    # Update progress
+                    benchmark_run.completed_examples += 1
+                    session.commit()
+
+                    # Send real-time update
+                    self._send_progress_update(
+                        benchmark_run_id,
+                        benchmark_run.completed_examples,
+                        benchmark_run.total_examples,
+                    )
+
+                except Exception as e:
+                    logger.exception(f"Error processing task {i}")
+                    benchmark_run.failed_examples += 1
+                    session.commit()
+
+                    # Check if this is a rate limiting error
+                    error_str = str(e).lower()
+                    if (
+                        "403" in error_str
+                        or "rate limit" in error_str
+                        or "forbidden" in error_str
+                    ):
+                        self.rate_limit_detected[benchmark_run_id] = True
+                        # Send rate limit warning via WebSocket
+                        self.socket_service.emit_to_subscribers(
+                            "research_progress",
+                            benchmark_run_id,
+                            {
+                                "rate_limit_detected": True,
+                                "message": "SearXNG rate limiting detected",
+                            },
+                        )
+
+            # Mark as completed
+            benchmark_run.end_time = datetime.now()
+            benchmark_run.status = BenchmarkStatus.COMPLETED
+
+            # Calculate final accuracy
+            self._calculate_final_accuracy(benchmark_run_id)
+            session.commit()
+
+            # Send completion notification
+            self.socket_service.emit_to_subscribers(
+                "research_progress",
+                benchmark_run_id,
+                {
+                    "status": "completed",
+                    "message": "Benchmark completed successfully",
+                    "progress": 100,
+                    "benchmark_run_id": benchmark_run_id,
+                },
+            )
+
+        except Exception as e:
+            logger.exception(f"Benchmark run {benchmark_run_id} failed")
+            self.update_benchmark_status(
+                benchmark_run_id, BenchmarkStatus.FAILED, str(e)
+            )
+        finally:
+            session.close()
+            if benchmark_run_id in self.active_runs:
+                del self.active_runs[benchmark_run_id]
+
+    def _create_task_queue(
+        self,
+        datasets_config: Dict,
+        existing_results: Dict,
+        benchmark_run_id: int,
+    ) -> List[Dict]:
+        """Create list of tasks to process, excluding existing results."""
+        tasks = []
+
+        for dataset_name, config in datasets_config.items():
+            if config.get("count", 0) > 0:
+                dataset = load_dataset(
+                    dataset_type=dataset_name,
+                    num_examples=config["count"],
+                    seed=None,
+                )
+
+                for i, example in enumerate(dataset):
+                    # Extract question based on dataset type
+                    if dataset_name.lower() == "simpleqa":
+                        question = example.get("problem", "")
+                        correct_answer = example.get("answer", "")
+                    else:  # browsecomp
+                        question = example.get("problem", "")
+                        correct_answer = example.get("answer", "")
+
+                    # Generate query hash
+                    query_hash = self.generate_query_hash(
+                        question, dataset_name
+                    )
+
+                    # Skip if already processed
+                    if query_hash in existing_results:
+                        continue
+
+                    tasks.append(
+                        {
+                            "benchmark_run_id": benchmark_run_id,
+                            "example_id": example.get("id", f"example_{i}"),
+                            "dataset_type": dataset_name,
+                            "question": question,
+                            "correct_answer": correct_answer,
+                            "query_hash": query_hash,
+                            "task_index": len(tasks),
+                        }
+                    )
+
+        return tasks
+
+    def _process_benchmark_task(
+        self, task: Dict, search_config: Dict, evaluation_config: Dict
+    ) -> Dict:
+        """Process a single benchmark task."""
+        try:
+            # Generate a unique tracking ID for this benchmark task
+            import uuid
+
+            tracking_id = str(uuid.uuid4())
+
+            # Format query
+            formatted_query = format_query(
+                task["question"], task["dataset_type"]
+            )
+
+            # Run research with progress callback for WebSocket updates
+            start_time = time.time()
+
+            def benchmark_progress_callback(
+                status: str, progress: int, data: dict
+            ):
+                """Progress callback to emit detailed research progress via WebSocket"""
+                try:
+                    timestamp = datetime.now().isoformat()
+
+                    # Create research-compatible log entry
+                    log_entry = {
+                        "time": timestamp,
+                        "message": f"Example {task['example_id']}: {status}",
+                        "progress": progress,
+                        "metadata": {
+                            "phase": data.get("phase", "benchmark_processing"),
+                            "type": data.get("type", "info"),
+                            "example_id": task["example_id"],
+                            "benchmark_run_id": task["benchmark_run_id"],
+                            **data,  # Include all other data
+                        },
+                    }
+
+                    # Determine log type based on status/message content
+                    if (
+                        "complete" in status.lower()
+                        or "finished" in status.lower()
+                    ):
+                        log_entry["metadata"]["type"] = "milestone"
+                    elif (
+                        "error" in status.lower() or "failed" in status.lower()
+                    ):
+                        log_entry["metadata"]["type"] = "error"
+                    elif (
+                        "starting" in status.lower()
+                        or "begin" in status.lower()
+                    ):
+                        log_entry["metadata"]["type"] = "milestone"
+
+                    # Create progress data in research format
+                    progress_data = {
+                        "progress": progress,
+                        "message": status,
+                        "status": "in_progress",
+                        "log_entry": log_entry,
+                        "progress_log": json.dumps(
+                            [log_entry]
+                        ),  # Array format expected by socket.js
+                    }
+
+                    # Emit using research_progress format that the UI expects
+                    self.socket_service.emit_to_subscribers(
+                        "research_progress",
+                        task["benchmark_run_id"],
+                        progress_data,
+                    )
+
+                except Exception:
+                    logger.exception("Error sending benchmark progress update")
+
+            search_result = quick_summary(
+                query=formatted_query,
+                research_id=tracking_id,  # Pass the tracking ID
+                iterations=search_config.get("iterations", 8),
+                questions_per_iteration=search_config.get(
+                    "questions_per_iteration", 5
+                ),
+                search_tool=search_config.get("search_tool", "searxng"),
+                search_strategy=search_config.get(
+                    "search_strategy", "focused_iteration"
+                ),
+                progress_callback=benchmark_progress_callback,
+            )
+            processing_time = time.time() - start_time
+
+            # Extract answer
+            response = search_result.get("summary", "")
+            extracted_data = extract_answer_from_response(
+                response, task["dataset_type"]
+            )
+            extracted_answer = (
+                extracted_data.get("extracted_answer", "")
+                if isinstance(extracted_data, dict)
+                else str(extracted_data)
+            )
+
+            # Extract sources - handle both direct sources and all_links_of_system
+            sources = search_result.get("sources", [])
+            if not sources and "all_links_of_system" in search_result:
+                sources = search_result.get("all_links_of_system", [])
+
+            # Log for debugging
+            logger.debug(f"Search result keys: {list(search_result.keys())}")
+            logger.debug(f"Sources found: {len(sources)} items")
+
+            # Prepare result
+            result = {
+                **task,
+                "response": response,
+                "extracted_answer": extracted_answer,
+                "confidence": str(
+                    extracted_data.get("confidence", "100")
+                    if isinstance(extracted_data, dict)
+                    else "100"
+                ),
+                "processing_time": processing_time,
+                "sources": json.dumps(sources),  # Convert to JSON string
+                "completed_at": datetime.now(),
+                "research_id": tracking_id,  # Store the UUID in the research_id field
+            }
+
+            # Evaluate result - requires proper grading model
+            try:
+                # Check if we have a proper evaluation model configured
+                eval_provider = evaluation_config.get("provider", "").lower()
+                eval_model = evaluation_config.get("model_name", "")
+
+                if (
+                    eval_provider in ["ollama", "local"]
+                    or "gemma" in eval_model.lower()
+                ):
+                    # Local models are not reliable enough for grading
+                    result.update(
+                        {
+                            "is_correct": None,
+                            "graded_confidence": "0",
+                            "grader_response": "🔑 Evaluation requires OpenRouter API key. Set llm.openai_endpoint.api_key in database settings to use Claude 3.7 Sonnet for accurate grading via OpenRouter.",
+                            "evaluation_error": "Local models not suitable for grading",
+                        }
+                    )
+                else:
+                    # Try to evaluate with proper model
+                    result_data = {
+                        "id": task["example_id"],
+                        "problem": task["question"],
+                        "correct_answer": task["correct_answer"],
+                        "response": response,
+                        "extracted_answer": extracted_answer,
+                    }
+
+                    eval_result = grade_single_result(
+                        result_data, task["dataset_type"], evaluation_config
+                    )
+                    if eval_result and not eval_result.get("grading_error"):
+                        result.update(
+                            {
+                                "is_correct": eval_result.get(
+                                    "is_correct", False
+                                ),
+                                "graded_confidence": eval_result.get(
+                                    "graded_confidence", "0"
+                                ),
+                                "grader_response": eval_result.get(
+                                    "grader_response", ""
+                                ),
+                            }
+                        )
+                    else:
+                        error_msg = (
+                            eval_result.get(
+                                "grading_error", "Unknown evaluation error"
+                            )
+                            if eval_result
+                            else "No evaluation results returned"
+                        )
+                        result.update(
+                            {
+                                "is_correct": None,
+                                "graded_confidence": "0",
+                                "grader_response": f"🔑 Evaluation failed: {error_msg}. Set llm.openai_endpoint.api_key in database settings to use Claude 3.7 Sonnet via OpenRouter.",
+                                "evaluation_error": error_msg,
+                            }
+                        )
+
+            except Exception as e:
+                logger.exception("Evaluation error")
+                result.update(
+                    {
+                        "is_correct": None,
+                        "graded_confidence": "0",
+                        "grader_response": f"🔑 Evaluation failed: {str(e)}. Set llm.openai_endpoint.api_key in database settings to use Claude 3.7 Sonnet via OpenRouter.",
+                        "evaluation_error": str(e),
+                    }
+                )
+
+            return result
+
+        except Exception as e:
+            logger.exception("Research error")
+            return {
+                **task,
+                "research_error": str(e),
+                "completed_at": datetime.now(),
+            }
+
+    def _save_benchmark_result(self, result: Dict, benchmark_run_id: int):
+        """Save benchmark result to database."""
+        session = get_db_session()
+
+        try:
+            benchmark_result = BenchmarkResult(
+                benchmark_run_id=benchmark_run_id,
+                example_id=result["example_id"],
+                query_hash=result["query_hash"],
+                dataset_type=DatasetType(result["dataset_type"]),
+                research_id=result.get(
+                    "research_id"
+                ),  # Include the research_id (UUID)
+                question=result["question"],
+                correct_answer=result["correct_answer"],
+                response=result.get("response"),
+                extracted_answer=result.get("extracted_answer"),
+                confidence=result.get("confidence"),
+                processing_time=result.get("processing_time"),
+                sources=result.get("sources"),
+                is_correct=result.get("is_correct"),
+                graded_confidence=result.get("graded_confidence"),
+                grader_response=result.get("grader_response"),
+                completed_at=result.get("completed_at"),
+                research_error=result.get("research_error"),
+                evaluation_error=result.get("evaluation_error"),
+                task_index=result.get("task_index"),
+            )
+
+            session.add(benchmark_result)
+            session.commit()
+
+        except Exception:
+            session.rollback()
+            logger.exception("Error saving benchmark result")
+            raise
+        finally:
+            session.close()
+
+    def _send_progress_update(
+        self, benchmark_run_id: int, completed: int, total: int
+    ):
+        """Send real-time progress update via websocket."""
+        try:
+            percentage = (completed / total * 100) if total > 0 else 0
+
+            # Create log entry for milestone progress
+            log_entry = {
+                "time": datetime.now().isoformat(),
+                "message": f"Completed {completed}/{total} examples ({percentage:.1f}%)",
+                "progress": percentage,
+                "metadata": {
+                    "phase": "benchmark_progress",
+                    "type": "milestone",
+                    "completed": completed,
+                    "total": total,
+                    "benchmark_run_id": benchmark_run_id,
+                },
+            }
+
+            progress_data = {
+                "status": "in_progress",
+                "message": f"Processing examples: {completed}/{total}",
+                "progress": percentage,
+                "completed": completed,
+                "total": total,
+                "benchmark_run_id": benchmark_run_id,
+                "log_entry": log_entry,
+                "progress_log": json.dumps([log_entry]),
+            }
+
+            self.socket_service.emit_to_subscribers(
+                "research_progress", benchmark_run_id, progress_data
+            )
+
+        except Exception:
+            logger.exception("Error sending progress update")
+
+    def _calculate_final_accuracy(self, benchmark_run_id: int):
+        """Calculate and save final accuracy metrics."""
+        session = get_db_session()
+
+        try:
+            # Get all results for this run
+            results = (
+                session.query(BenchmarkResult)
+                .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id)
+                .filter(BenchmarkResult.is_correct.isnot(None))
+                .all()
+            )
+
+            if results:
+                correct_count = sum(1 for r in results if r.is_correct)
+                overall_accuracy = (correct_count / len(results)) * 100
+
+                # Calculate processing rate
+                total_time = sum(r.processing_time or 0 for r in results)
+                processing_rate = (
+                    (len(results) / (total_time / 60)) if total_time > 0 else 0
+                )
+
+                # Update benchmark run
+                benchmark_run = (
+                    session.query(BenchmarkRun)
+                    .filter(BenchmarkRun.id == benchmark_run_id)
+                    .first()
+                )
+                if benchmark_run:
+                    benchmark_run.overall_accuracy = overall_accuracy
+                    benchmark_run.processing_rate = processing_rate
+                    session.commit()
+
+        except Exception:
+            logger.exception("Error calculating final accuracy")
+        finally:
+            session.close()
+
+    def update_benchmark_status(
+        self,
+        benchmark_run_id: int,
+        status: BenchmarkStatus,
+        error_message: str = None,
+    ):
+        """Update benchmark run status."""
+        session = get_db_session()
+
+        try:
+            benchmark_run = (
+                session.query(BenchmarkRun)
+                .filter(BenchmarkRun.id == benchmark_run_id)
+                .first()
+            )
+            if benchmark_run:
+                benchmark_run.status = status
+                benchmark_run.updated_at = datetime.now()
+
+                if error_message:
+                    benchmark_run.error_message = error_message
+
+                if (
+                    status == BenchmarkStatus.IN_PROGRESS
+                    and not benchmark_run.start_time
+                ):
+                    benchmark_run.start_time = datetime.now()
+                elif (
+                    status
+                    in [BenchmarkStatus.COMPLETED, BenchmarkStatus.FAILED]
+                    and not benchmark_run.end_time
+                ):
+                    benchmark_run.end_time = datetime.now()
+
+                session.commit()
+
+        except Exception:
+            session.rollback()
+            logger.exception("Error updating benchmark status")
+        finally:
+            session.close()
+
+    def get_benchmark_status(self, benchmark_run_id: int) -> Optional[Dict]:
+        """Get current status of a benchmark run."""
+        session = get_db_session()
+
+        try:
+            benchmark_run = (
+                session.query(BenchmarkRun)
+                .filter(BenchmarkRun.id == benchmark_run_id)
+                .first()
+            )
+            if not benchmark_run:
+                return None
+
+            # Calculate running accuracy from current results AND reused results from compatible runs
+            # First get results specifically for this benchmark run
+            current_results = (
+                session.query(BenchmarkResult)
+                .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id)
+                .filter(BenchmarkResult.is_correct.isnot(None))
+                .all()
+            )
+
+            # Then get reused results from compatible benchmark runs (same config hash)
+            # Only count results up to the number we say we've "completed"
+            if benchmark_run.completed_examples > len(current_results):
+                # We have reused results, get them from compatible runs
+                reused_count_needed = benchmark_run.completed_examples - len(
+                    current_results
+                )
+
+                compatible_results = (
+                    session.query(BenchmarkResult)
+                    .join(
+                        BenchmarkRun,
+                        BenchmarkResult.benchmark_run_id == BenchmarkRun.id,
+                    )
+                    .filter(
+                        BenchmarkRun.config_hash == benchmark_run.config_hash
+                    )
+                    .filter(
+                        BenchmarkRun.id != benchmark_run_id
+                    )  # Exclude current run
+                    .filter(BenchmarkRun.status == BenchmarkStatus.COMPLETED)
+                    .filter(BenchmarkResult.is_correct.isnot(None))
+                    .order_by(BenchmarkResult.id)  # Consistent ordering
+                    .limit(reused_count_needed)
+                    .all()
+                )
+
+                # Combine current and reused results
+                results = (
+                    current_results + compatible_results[:reused_count_needed]
+                )
+            else:
+                # No reused results, just use current results
+                results = current_results
+
+            running_accuracy = None
+            simpleqa_accuracy = None
+            browsecomp_accuracy = None
+
+            if results:
+                # Overall running accuracy
+                correct_count = sum(1 for r in results if r.is_correct)
+                running_accuracy = (correct_count / len(results)) * 100
+
+                # Per-dataset accuracy
+                simpleqa_results = [
+                    r for r in results if r.dataset_type.value == "simpleqa"
+                ]
+                if simpleqa_results:
+                    simpleqa_correct = sum(
+                        1 for r in simpleqa_results if r.is_correct
+                    )
+                    simpleqa_accuracy = (
+                        simpleqa_correct / len(simpleqa_results)
+                    ) * 100
+
+                browsecomp_results = [
+                    r for r in results if r.dataset_type.value == "browsecomp"
+                ]
+                if browsecomp_results:
+                    browsecomp_correct = sum(
+                        1 for r in browsecomp_results if r.is_correct
+                    )
+                    browsecomp_accuracy = (
+                        browsecomp_correct / len(browsecomp_results)
+                    ) * 100
+
+            # Calculate time estimates and reliability metrics
+            estimated_time_remaining = None
+            total_elapsed_time = None
+            avg_time_per_example = None
+            accuracy_confidence = None
+
+            if (
+                benchmark_run.start_time
+                and benchmark_run.completed_examples > 0
+            ):
+                # Calculate elapsed time
+                current_time = datetime.now()
+                total_elapsed_time = (
+                    current_time - benchmark_run.start_time
+                ).total_seconds()
+
+                # Calculate average processing time per example
+                avg_time_per_example = (
+                    total_elapsed_time / benchmark_run.completed_examples
+                )
+
+                # Estimate remaining time
+                remaining_examples = (
+                    benchmark_run.total_examples
+                    - benchmark_run.completed_examples
+                )
+                if remaining_examples > 0:
+                    estimated_time_remaining = (
+                        avg_time_per_example * remaining_examples
+                    )
+
+            # Calculate accuracy confidence interval (95% confidence)
+            if results and len(results) >= 3:
+                import math
+
+                n = len(results)
+                p = running_accuracy / 100 if running_accuracy else 0
+                # Standard error for proportion
+                se = math.sqrt(p * (1 - p) / n)
+                # 95% confidence interval (±1.96 * SE)
+                margin_of_error = 1.96 * se * 100
+                accuracy_confidence = {
+                    "lower_bound": max(0, running_accuracy - margin_of_error),
+                    "upper_bound": min(100, running_accuracy + margin_of_error),
+                    "margin_of_error": margin_of_error,
+                    "sample_size": n,
+                }
+
+            return {
+                "id": benchmark_run.id,
+                "run_name": benchmark_run.run_name,
+                "status": benchmark_run.status.value,
+                "completed_examples": benchmark_run.completed_examples,
+                "total_examples": benchmark_run.total_examples,
+                "failed_examples": benchmark_run.failed_examples,
+                "overall_accuracy": benchmark_run.overall_accuracy
+                or running_accuracy,  # Use running accuracy if final not calculated
+                "running_accuracy": running_accuracy,  # Current running accuracy
+                "simpleqa_accuracy": simpleqa_accuracy,  # Per-dataset accuracy
+                "browsecomp_accuracy": browsecomp_accuracy,
+                "processing_rate": benchmark_run.processing_rate,
+                "estimated_time_remaining": estimated_time_remaining,  # seconds
+                "total_elapsed_time": total_elapsed_time,  # seconds
+                "avg_time_per_example": avg_time_per_example,  # seconds
+                "accuracy_confidence": accuracy_confidence,  # confidence interval
+                "created_at": benchmark_run.created_at.isoformat()
+                if benchmark_run.created_at
+                else None,
+                "start_time": benchmark_run.start_time.isoformat()
+                if benchmark_run.start_time
+                else None,
+                "end_time": benchmark_run.end_time.isoformat()
+                if benchmark_run.end_time
+                else None,
+                "error_message": benchmark_run.error_message,
+            }
+
+        except Exception:
+            logger.exception("Error getting benchmark status")
+            return None
+        finally:
+            session.close()
+
+    def cancel_benchmark(self, benchmark_run_id: int) -> bool:
+        """Cancel a running benchmark."""
+        try:
+            if benchmark_run_id in self.active_runs:
+                self.active_runs[benchmark_run_id]["status"] = "cancelled"
+
+            self.update_benchmark_status(
+                benchmark_run_id, BenchmarkStatus.CANCELLED
+            )
+            logger.info(f"Cancelled benchmark run {benchmark_run_id}")
+            return True
+
+        except Exception:
+            logger.exception(f"Error cancelling benchmark {benchmark_run_id}")
+            return False
+
+
+# Global service instance
+benchmark_service = BenchmarkService()