rag-benchmarking 1.0.0rc1__py3-none-any.whl

app/__init__.py

@@ -0,0 +1,5 @@
+__all__ = [
+    "__version__",
+]
+
+__version__ = "0.1.0"

app/api/evaluate.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import asyncio
+import functools
+from pathlib import Path
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, Field
+
+from app.api.security import get_api_key
+from app.eval.reporting import write_report_files
+from app.eval.result_store import ResultStore
+from harness.schemas import AgentTrace
+
+router = APIRouter(prefix="/v1", tags=["evaluate"])
+
+_result_store = ResultStore()
+
+
+class EvalSample(BaseModel):
+    question: str
+    contexts: list[str] = Field(default_factory=list)
+    answer: str
+    ground_truths: list[str] = Field(default_factory=list)
+
+
+class EvalRequest(BaseModel):
+    samples: list[EvalSample]
+    metrics: list[str] | None = None
+    out_json: str | None = None
+    out_md: str | None = None
+
+
+@router.post("/evaluate")
+async def post_evaluate(req: EvalRequest, _: str | None = Depends(get_api_key)) -> dict[str, Any]:
+    """Evaluate RAG samples against the requested metrics.
+
+    Routes through EvaluationRunner which handles:
+    - Deterministic metrics (source_attribution_accuracy) — no LLM needed
+    - RAGAS metrics (faithfulness, answer_relevancy, context_precision, context_recall) — needs Gemini/OpenAI
+    - Retrieval metrics (precision_at_k, recall_at_k, mrr, ndcg_at_k) — no LLM needed
+
+    RAGAS calls are offloaded to a thread-pool executor to avoid blocking
+    the FastAPI event loop.
+    """
+    from harness.runner import EvaluationRunner
+    from harness.schemas import EvalSample as HarnessEvalSample, RunConfig
+
+    loop = asyncio.get_running_loop()
+    try:
+        # Convert API samples to harness EvalSample objects
+        harness_samples = [
+            HarnessEvalSample(
+                question=s.question,
+                contexts=s.contexts,
+                answer=s.answer,
+                ground_truth=s.ground_truths[0] if s.ground_truths else None,
+            )
+            for s in req.samples
+        ]
+
+        config = RunConfig(metrics=req.metrics or ["faithfulness", "answer_relevancy"])
+        runner = EvaluationRunner(config)
+
+        result = await loop.run_in_executor(None, runner.evaluate, harness_samples)
+
+        # Convert BenchmarkReport to dict for response
+        output: dict[str, Any] = {
+            "metrics": result.metrics,
+            "skipped_metrics": result.skipped_metrics,
+            "skip_reasons": result.skip_reasons,
+            "run_id": result.run_id,
+            "n_samples": result.n_samples,
+        }
+
+        paths = write_report_files(
+            {"metrics": result.metrics, "per_sample": {}, "skipped_metrics": result.skipped_metrics},
+            out_json=Path(req.out_json) if req.out_json else None,
+            out_md=Path(req.out_md) if req.out_md else None,
+        )
+        output["written"] = paths
+        return output
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+
+
+class AgentEvalRequest(BaseModel):
+    trace: AgentTrace
+    metrics: list[str] = Field(
+        default=["source_attribution_accuracy", "agent_faithfulness", "tool_call_accuracy"]
+    )
+
+
+@router.post("/evaluate/agent")
+async def post_evaluate_agent(
+    request: AgentEvalRequest,
+    _: str | None = Depends(get_api_key),
+) -> dict[str, Any]:
+    """Evaluate an agentic RAG trace using agentic-specific metrics."""
+    import re
+
+    from app.eval.agentic_metrics import source_attribution_accuracy
+    from app.eval.agentic_llm_metrics import (
+        compute_agent_faithfulness,
+        compute_retrieval_necessity,
+        compute_tool_call_accuracy,
+    )
+
+    scores: dict[str, float] = {}
+    details: dict[str, Any] = {}
+
+    for metric in request.metrics:
+        if metric == "source_attribution_accuracy":
+            cited = re.findall(r'\[source:\s*([^\]]+)\]', request.trace.final_answer)
+            retrieved = [c.source_id for c in request.trace.retrieved_chunks]
+            r = source_attribution_accuracy(cited, retrieved)
+            scores[metric] = r["score"]
+            details[metric] = r
+        elif metric == "agent_faithfulness":
+            loop = asyncio.get_running_loop()
+            r = await loop.run_in_executor(None, compute_agent_faithfulness, request.trace)
+            scores[metric] = r["score"]
+            details[metric] = r
+        elif metric == "tool_call_accuracy":
+            loop = asyncio.get_running_loop()
+            r = await loop.run_in_executor(None, compute_tool_call_accuracy, request.trace)
+            scores[metric] = r["score"]
+            details[metric] = r
+        elif metric == "retrieval_necessity":
+            contexts = [c.content for c in request.trace.retrieved_chunks]
+            loop = asyncio.get_running_loop()
+            r = await loop.run_in_executor(
+                None,
+                functools.partial(
+                    compute_retrieval_necessity,
+                    request.trace.question,
+                    request.trace.final_answer,
+                    contexts,
+                ),
+            )
+            scores[metric] = r["score"]
+            details[metric] = r
+
+    return {
+        "scores": scores,
+        "details": details,
+        "trace_id": request.trace.trace_id,
+    }
+
+
+@router.get("/runs")
+async def list_runs(
+    limit: int = 50,
+    _: str | None = Depends(get_api_key),
+) -> list[dict[str, Any]]:
+    """List recent evaluation runs from the result store."""
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, functools.partial(_result_store.list_runs, limit))
+
+
+@router.post("/runs/compare")
+async def compare_runs(
+    run_ids: list[str],
+    _: str | None = Depends(get_api_key),
+) -> dict[str, Any]:
+    """Compare metrics across multiple named runs."""
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, functools.partial(_result_store.compare_runs, run_ids))

app/api/query.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import asyncio
+import functools
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, Field
+
+from app.engine.rag_engine import RAGEngine, RetrievedChunk
+
+router = APIRouter(prefix="/v1", tags=["query"])
+
+
+class QueryRequest(BaseModel):
+    query: str = Field(..., min_length=1)
+    top_k: int = Field(5, ge=1, le=20)
+    rerank: bool = Field(default=False)
+
+
+class QueryResponse(BaseModel):
+    answer: str
+    citations: list[RetrievedChunk]
+    timings_ms: dict[str, float] | None = None
+    tokens: dict[str, int] | None = None
+    groundedness: float | None = None
+
+
+def get_rag_engine() -> RAGEngine:
+    return RAGEngine()
+
+
+@router.post("/query", response_model=QueryResponse)
+async def post_query(
+    req: QueryRequest, engine: RAGEngine = Depends(get_rag_engine)
+) -> QueryResponse:
+    """Async query endpoint.
+
+    ``RAGEngine.query()`` calls sentence-transformers (CPU-bound / sync) and
+    blocking HTTP calls to OpenAI / Gemini.  We run it in a thread-pool
+    executor so the FastAPI event loop is never blocked.
+
+    Pattern for sync code in async FastAPI:
+        loop = asyncio.get_running_loop()
+        result = await loop.run_in_executor(None, functools.partial(sync_fn, *args))
+    """
+    loop = asyncio.get_running_loop()
+    try:
+        result = await loop.run_in_executor(
+            None,
+            functools.partial(engine.query, req.query, req.top_k, req.rerank),
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+
+    # Token usage: LLMClient.generate() stores self._last_token_usage after
+    # each call.  Guard against mock engines (test environments) where the
+    # attribute may be a Mock object rather than a dict.
+    _raw = getattr(getattr(engine, "llm", None), "_last_token_usage", None)
+    last_usage: dict[str, int] = (
+        _raw
+        if isinstance(_raw, dict)
+        else {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+    )
+
+    return QueryResponse(
+        answer=result.answer,
+        citations=result.citations,
+        timings_ms=result.timings,
+        tokens=last_usage,
+        groundedness=result.groundedness,
+    )

app/api/security.py

@@ -0,0 +1,23 @@
+from fastapi import HTTPException, Security, status
+from fastapi.security import APIKeyHeader
+
+from app.config.settings import get_settings
+
+api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
+
+
+def get_api_key(api_key_header: str = Security(api_key_header)) -> str | None:
+    settings = get_settings()
+
+    # If no API key is configured on the server, allow access (open mode)
+    # WARNING: This is for development/POC only.
+    if not settings.api_key:
+        return None
+
+    if api_key_header == settings.api_key:
+        return api_key_header
+
+    raise HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail="Could not validate credentials",
+    )

app/config/__init__.py

@@ -0,0 +1 @@
+

app/config/settings.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class AppSettings(BaseSettings):
+    """Central application settings loaded from environment and optional .env file.
+
+    Attributes
+    ----------
+    app_env: str
+        Application environment (e.g., dev, prod).
+    log_level: str
+        Logging level string (e.g., INFO, DEBUG).
+    openai_api_key: Optional[str]
+        API key for OpenAI (optional).
+    gemini_api_key: Optional[str]
+        API key for Google Gemini (optional).
+    qdrant_url: Optional[str]
+        Qdrant Cloud URL.
+    qdrant_api_key: Optional[str]
+        Qdrant Cloud API key.
+    qdrant_collection: str
+        Default Qdrant collection name.
+    """
+
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    app_env: str = Field(default="dev", alias="APP_ENV")
+    log_level: str = Field(default="INFO", alias="LOG_LEVEL")
+
+    openai_api_key: str | None = Field(default=None, alias="OPENAI_API_KEY")
+    api_key: str | None = Field(default=None, alias="API_KEY")
+    gemini_api_key: str | None = Field(default=None, alias="GEMINI_API_KEY")
+    llm_provider: str | None = Field(default=None, alias="LLM_PROVIDER")
+    openai_model: str | None = Field(default=None, alias="OPENAI_MODEL")
+    gemini_model: str | None = Field(default=None, alias="GEMINI_MODEL")
+    llm_temperature: float = Field(default=0.2, alias="LLM_TEMPERATURE")
+    llm_max_tokens: int = Field(default=512, alias="LLM_MAX_TOKENS")
+
+    qdrant_url: str | None = Field(default=None, alias="QDRANT_URL")
+    qdrant_api_key: str | None = Field(default=None, alias="QDRANT_API_KEY")
+    qdrant_collection: str = Field(default="agentic_rag_poc", alias="QDRANT_COLLECTION")
+    # Self-check configuration
+    self_check_min_groundedness: float = Field(default=0.7, alias="SELF_CHECK_MIN_GROUNDEDNESS")
+    self_check_retry: bool = Field(default=True, alias="SELF_CHECK_RETRY")
+
+    # Prompts
+    system_prompt: str = Field(
+        default=(
+            "You are a helpful assistant. Answer based only on the provided context. Cite sources."
+        ),
+        alias="SYSTEM_PROMPT",
+    )
+    user_prompt_template: str = Field(
+        default="Context:\n{context_blocks}\n\nQuestion: {query}\nAnswer:",
+        alias="USER_PROMPT_TEMPLATE",
+    )
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> AppSettings:
+    """Return a cached instance of application settings.
+
+    Returns
+    -------
+    AppSettings
+        Loaded settings object.
+    """
+
+    return AppSettings()  # type: ignore[call-arg]

app/engine/rag_engine.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from pydantic import BaseModel
+
+import app.llm.client as llm_client
+import app.retrieval.service as retrieval_service
+from app.config.settings import get_settings
+from app.utils.timing import timer
+
+logger = logging.getLogger(__name__)
+
+
+class RetrievedChunk(BaseModel):
+    text: str
+    source_id: str
+    chunk_index: int
+    score: float
+
+
+class RAGResult(BaseModel):
+    answer: str
+    citations: list[RetrievedChunk]
+    timings: dict[str, float]
+    groundedness: float | None = None
+
+
+class RAGEngine:
+    def __init__(self) -> None:
+        self.settings = get_settings()
+        self.llm = llm_client.LLMClient()
+
+    def query(self, query: str, top_k: int, rerank: bool) -> RAGResult:
+        """
+        Execute the full RAG pipeline including retrieval, reranking, generation, 
+        and optional retry.
+        """
+        timings: dict[str, float] = {}
+
+        # 1. Retrieve
+        logger.info("Starting RAG query", extra={"query": query, "top_k": top_k, "rerank": rerank})
+        with timer() as t_retr:
+            chunks = retrieval_service.retrieve_top_chunks(query, top_k=max(top_k, 10))
+        timings["retrieve"] = t_retr["elapsed_ms"]
+
+        if not chunks:
+            logger.warning("No chunks retrieved for query", extra={"query": query})
+            return RAGResult(answer="", citations=[], timings=timings)
+
+        logger.info("Retrieved chunks", extra={"count": len(chunks)})
+
+        # 2. Rerank
+        if rerank:
+            try:
+                from app.retrieval.reranker import CrossEncoderReranker
+
+                with timer() as t_rr:
+                    reranker = CrossEncoderReranker()
+                    chunks = reranker.rerank(query, chunks, top_k=len(chunks))
+                timings["rerank"] = t_rr["elapsed_ms"]
+            except Exception:
+                pass
+
+        current_chunks = chunks[:top_k]
+
+        # 3. Generate
+        with timer() as t_gen:
+            answer = self._call_llm(query, current_chunks)
+        timings["generate"] = t_gen["elapsed_ms"]
+
+        # 4. Self-Check
+        groundedness = None
+        try:
+            from app.quality.self_check import compute_groundedness
+
+            with timer() as t_sc:
+                groundedness = compute_groundedness(
+                    answer, [c.get("text", "") for c in current_chunks]
+                )
+            timings["self_check"] = t_sc["elapsed_ms"]
+        except Exception:
+            pass
+
+        # 5. Retry if needed
+        if (
+            groundedness is not None
+            and groundedness < self.settings.self_check_min_groundedness
+            and self.settings.self_check_retry
+        ):
+
+            logger.info(
+                "Groundedness below threshold, attempting retry",
+                extra={
+                    "groundedness": groundedness,
+                    "threshold": self.settings.self_check_min_groundedness,
+                },
+            )
+            try:
+                # Retry logic
+                retry_result = self._retry_workflow(query, top_k, rerank, groundedness)
+                if retry_result:
+                    logger.info(
+                        "Retry successful, adopting new answer",
+                        extra={"new_groundedness": retry_result["groundedness"]},
+                    )
+                    answer = retry_result["answer"]
+                    groundedness = retry_result["groundedness"]
+                    current_chunks = retry_result["chunks"]
+                    timings.update(retry_result["timings"])
+                else:
+                    logger.info("Retry did not improve groundedness")
+            except Exception as e:
+                logger.error("Error during retry workflow", extra={"error": str(e)})
+                pass
+
+        citations = [
+            RetrievedChunk(
+                text=c.get("text", ""),
+                source_id=c.get("source_id", ""),
+                chunk_index=int(c.get("chunk_index", 0)),
+                score=float(c.get("score", 0.0)),
+            )
+            for c in current_chunks
+        ]
+
+        return RAGResult(
+            answer=answer, citations=citations, timings=timings, groundedness=groundedness
+        )
+
+    def _call_llm(self, query: str, chunks: list[dict[str, Any]]) -> str:
+        context_blocks = "\n\n".join(
+            [f"[source: {c.get('source_id','')}]\n{c.get('text','')}" for c in chunks]
+        )
+        user_prompt = self.settings.user_prompt_template.format(
+            context_blocks=context_blocks, query=query
+        )
+        return self.llm.generate(self.settings.system_prompt, user_prompt)
+
+    def _retry_workflow(
+        self, query: str, top_k: int, rerank: bool, current_score: float
+    ) -> dict[str, Any] | None:
+        timings = {}
+
+        # Expand retrieval
+        with timer() as t_retr:
+            more_chunks = retrieval_service.retrieve_top_chunks(query, top_k=20)
+        timings["retrieve_retry"] = t_retr["elapsed_ms"]
+
+        if rerank:
+            try:
+                from app.retrieval.reranker import CrossEncoderReranker
+
+                with timer() as t_rr:
+                    reranker = CrossEncoderReranker()
+                    more_chunks = reranker.rerank(query, more_chunks, top_k=len(more_chunks))
+                timings["rerank_retry"] = t_rr["elapsed_ms"]
+            except Exception:
+                pass
+
+        more_chunks = more_chunks[:top_k]
+
+        # Generate
+        with timer() as t_gen:
+            answer = self._call_llm(query, more_chunks)
+        timings["generate_retry"] = t_gen["elapsed_ms"]
+
+        # Check
+        groundedness = None
+        try:
+            from app.quality.self_check import compute_groundedness
+
+            with timer() as t_sc:
+                groundedness = compute_groundedness(
+                    answer, [c.get("text", "") for c in more_chunks]
+                )
+            timings["self_check_retry"] = t_sc["elapsed_ms"]
+        except Exception:
+            return None
+
+        if groundedness is not None and groundedness >= current_score:
+            return {
+                "answer": answer,
+                "groundedness": groundedness,
+                "chunks": more_chunks,
+                "timings": timings,
+            }
+        return None

app/eval/agentic_llm_metrics.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from app.llm.client import LLMClient
+from harness.schemas import AgentTrace
+
+logger = logging.getLogger(__name__)
+
+
+def _call_judge(system: str, user: str) -> dict[str, Any]:
+    """Call LLM judge and parse JSON. Returns empty dict on failure."""
+    llm = LLMClient()
+    raw = llm.generate(system, user).strip()
+    try:
+        if raw.startswith("```"):
+            parts = raw.split("```")
+            raw = parts[1] if len(parts) > 1 else raw
+            if raw.startswith("json"):
+                raw = raw[4:]
+            raw = raw.strip()
+        return json.loads(raw)
+    except Exception as exc:
+        logger.warning("Judge parse failure: %s | raw: %.200s", exc, raw)
+        return {}
+
+
+_AGENT_FAITHFULNESS_SYSTEM = """You are evaluating factual consistency across an AI agent's full reasoning trace.
+
+For each reasoning step, verify every factual claim is supported by the tool output at that step or a previously retrieved source.
+A claim is NOT supported if it appears in the reasoning but not in any tool output, or contradicts the tool output.
+
+Return JSON only:
+{
+  "step_analysis": [
+    {"step_index": int, "claims": ["..."], "supported": [true|false], "faithfulness_score": float}
+  ],
+  "trace_faithfulness_score": float,
+  "worst_step": int,
+  "critical_hallucinations": ["..."]
+}"""
+
+_TOOL_ACCURACY_SYSTEM = """You are evaluating whether an AI agent made appropriate tool calls to answer a question.
+For each tool call assess: was it necessary, was the correct tool chosen, was the input well-formed?
+Score each: 0 (wrong), 0.5 (partial), 1 (correct).
+
+Return JSON only:
+{
+  "tool_evaluations": [
+    {"step_index": int, "tool_name": str, "necessary": bool, "correct_tool": bool,
+     "input_quality": float, "score": float, "reason": str}
+  ],
+  "overall_score": float
+}"""
+
+_NECESSITY_SYSTEM = """Evaluate whether retrieval was necessary to answer this question.
+Categories: NECESSARY / HELPFUL / UNNECESSARY
+
+Return JSON only:
+{
+  "necessity": "NECESSARY"|"HELPFUL"|"UNNECESSARY",
+  "parametric_answer_possible": bool,
+  "retrieval_contribution": "none"|"marginal"|"significant"|"essential",
+  "score": float,
+  "reasoning": str
+}"""
+
+
+def compute_agent_faithfulness(trace: AgentTrace) -> dict:
+    """
+    Evaluates faithfulness across the entire agent reasoning trace.
+    Returns score 1.0 when no reasoning steps present (neutral).
+    """
+    if not trace.reasoning_steps:
+        return {
+            "score": 1.0,
+            "worst_step": -1,
+            "critical_hallucinations": [],
+            "step_analysis": [],
+        }
+
+    steps_text = "\n\n".join(
+        f"Step {s.step_index}: {s.thought}\nObservation: {s.observation}"
+        for s in trace.reasoning_steps
+    )
+    sources_text = "\n\n".join(
+        f"[{c.source_id}]: {c.content}" for c in trace.retrieved_chunks
+    ) or "No explicit chunks provided."
+
+    user = f"FULL AGENT TRACE:\n{steps_text}\n\nSOURCES:\n{sources_text}"
+    data = _call_judge(_AGENT_FAITHFULNESS_SYSTEM, user)
+
+    score = float(data.get("trace_faithfulness_score", 0.0))
+    return {
+        "score": max(0.0, min(1.0, score)),
+        "worst_step": data.get("worst_step", -1),
+        "critical_hallucinations": data.get("critical_hallucinations", []),
+        "step_analysis": data.get("step_analysis", []),
+    }
+
+
+def compute_tool_call_accuracy(trace: AgentTrace) -> dict:
+    """
+    Evaluates whether the agent made appropriate tool calls.
+    Returns score 1.0 with empty evaluations when no tool calls present.
+    """
+    if not trace.tool_calls:
+        return {"score": 1.0, "tool_evaluations": []}
+
+    calls_text = "\n".join(
+        f"Step {tc.step_index}: {tc.tool_name}({tc.tool_input}) → {tc.tool_output[:200]}"
+        for tc in trace.tool_calls
+    )
+    user = (
+        f"QUESTION: {trace.question}\n\n"
+        f"TOOL CALLS:\n{calls_text}\n\n"
+        "AVAILABLE TOOLS: retrieve, web_search, code_exec, calculator"
+    )
+    data = _call_judge(_TOOL_ACCURACY_SYSTEM, user)
+    score = float(data.get("overall_score", 0.0))
+    return {
+        "score": max(0.0, min(1.0, score)),
+        "tool_evaluations": data.get("tool_evaluations", []),
+    }
+
+
+def compute_retrieval_necessity(
+    question: str,
+    answer: str,
+    contexts: list[str],
+) -> dict:
+    """
+    Evaluates whether retrieval was actually necessary for this query.
+    High score = retrieval was essential; low score = retrieval was unnecessary.
+    """
+    context_text = "\n\n".join(contexts)
+    user = (
+        f"QUESTION: {question}\n\n"
+        f"RETRIEVED CONTEXT:\n{context_text}\n\n"
+        f"FINAL ANSWER: {answer}"
+    )
+    data = _call_judge(_NECESSITY_SYSTEM, user)
+    score = float(data.get("score", 0.0))
+    return {
+        "score": max(0.0, min(1.0, score)),
+        "necessity": data.get("necessity", "UNKNOWN"),
+        "retrieval_contribution": data.get("retrieval_contribution", "UNKNOWN"),
+        "reasoning": data.get("reasoning", ""),
+    }