ragaudit-cli 0.3.0__py3-none-any.whl

ragaudit/runner/evaluate.py

@@ -0,0 +1,325 @@
+"""v2 evaluation: run questions through a configured pipeline, score them, cache the results.
+
+Per question: recall@k and reciprocal rank against the gold chunk_ids,
+answer correctness from v1's LLM judge, a failure class (pass /
+retrieval_miss / generation_miss), and v1's verdict (a correct answer
+without the gold chunk is a LUCKY_PASS).
+
+Results are cached under (question_id, config_hash, index_version), so a
+question is never re-run for the same pipeline config on the same corpus.
+Which questions run is decided by `select_questions`:
+
+- the config was never evaluated on this split: run everything (`full`);
+- the corpus (index_version) changed but the config didn't: run all new
+  questions plus a seeded random sample of N old ones (`regression`),
+  because new data mostly breaks old questions by adding distractors, and
+  a sample catches that without re-running thousands of questions;
+- nothing changed: every result comes from the cache, and any question
+  added since is run (`cached`).
+
+This module never reads the test split itself; test-split evaluation goes
+through ragaudit.runner.locked, which unlocks it and calls in here.
+"""
+
+import hashlib
+import json
+import logging
+import random
+import time
+import uuid
+from collections.abc import Callable
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from ragaudit.models.adapter import Adapter, RAGResponse
+from ragaudit.models.evaluation import EvalSummary, FailureClass, QuestionResult
+from ragaudit.models.question import Question, Split, utc_now
+from ragaudit.models.scoring import AblationResult, CorrectnessResult, Verdict
+from ragaudit.scoring.text_match import DEFAULT_MATCH_THRESHOLD, first_matching_context
+from ragaudit.scoring.verdict import compute_verdict
+from ragaudit.storage.store import Store
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_K = 5
+DEFAULT_REGRESSION_SAMPLE = 50
+SCORING_VERSION = 1
+"""Bump when scoring changes meaningfully, so cached results from the old scoring aren't reused."""
+
+CorrectnessJudge = Callable[[str, str, str], CorrectnessResult]
+"""question, gold_answer, candidate_answer -> CorrectnessResult (v1's judge_correctness, bound to a client)."""
+
+NoContextQuery = Callable[[str], str]
+"""question -> answer with no retrieval at all, for ablation."""
+
+ADAPTER_ERROR_JUDGE_MODEL = "n/a (adapter raised)"
+
+
+class Selection(BaseModel):
+    mode: str
+    questions: list[Question]
+    n_new: int = 0
+    n_sampled_old: int = 0
+
+
+def file_sha256(path: str | Path) -> str:
+    return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+
+def compute_config_hash(
+    config: dict, adapter_sha256: str, judge_model: str, k: int, match_threshold: float | None = None
+) -> str:
+    """Identity of everything that can change a result: pipeline config, adapter code, judge model, k.
+
+    match_threshold is set only for --match-by text, so id-mode hashes (and
+    their cached results) are the same as before text matching existed.
+    """
+    payload = {
+        "config": config,
+        "adapter_sha256": adapter_sha256,
+        "judge_model": judge_model,
+        "k": k,
+        "scoring_version": SCORING_VERSION,
+    }
+    if match_threshold is not None:
+        payload["match"] = {"by": "text", "threshold": match_threshold}
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
+
+
+def score_retrieval_at_k(gold_chunk_ids: list[str], retrieved_chunk_ids: list[str], k: int) -> tuple[float, float, bool]:
+    """(recall@k, reciprocal rank, every gold chunk retrieved) over the first k retrieved ids."""
+    top_k = retrieved_chunk_ids[:k]
+    gold = set(gold_chunk_ids)
+    found = gold & set(top_k)
+    recall = len(found) / len(gold) if gold else 0.0
+    first_rank = next((rank for rank, chunk_id in enumerate(top_k) if chunk_id in gold), None)
+    reciprocal_rank = 1.0 / (first_rank + 1) if first_rank is not None else 0.0
+    return recall, reciprocal_rank, bool(gold) and found == gold
+
+
+def score_retrieval_by_text_at_k(
+    gold_texts: list[str], contexts: list[str], k: int, threshold: float = DEFAULT_MATCH_THRESHOLD
+) -> tuple[float, float, bool]:
+    """Like score_retrieval_at_k, but a gold chunk counts as retrieved when one of the first k contexts overlaps its text."""
+    top_k = contexts[:k]
+    ranks = [first_matching_context(text, top_k, threshold)[0] for text in gold_texts]
+    found = [rank for rank in ranks if rank is not None]
+    recall = len(found) / len(gold_texts) if gold_texts else 0.0
+    reciprocal_rank = 1.0 / (min(found) + 1) if found else 0.0
+    return recall, reciprocal_rank, bool(gold_texts) and len(found) == len(gold_texts)
+
+
+def classify(retrieval_hit: bool, is_correct: bool) -> FailureClass:
+    if not retrieval_hit:
+        return FailureClass.RETRIEVAL_MISS
+    return FailureClass.PASS if is_correct else FailureClass.GENERATION_MISS
+
+
+def evaluate_question(
+    question: Question,
+    adapter: Adapter,
+    judge: CorrectnessJudge,
+    k: int,
+    chunk_texts: dict[str, str] | None = None,
+    match_threshold: float = DEFAULT_MATCH_THRESHOLD,
+) -> QuestionResult:
+    """Run one question through the adapter and score it. An adapter exception is a scored failure, not a crash.
+
+    With chunk_texts (chunk_id -> text), retrieval is matched by text against
+    the returned contexts (--match-by text) instead of by chunk id.
+    """
+    started = time.perf_counter()
+    try:
+        response = adapter.query(question.question)
+        error = None
+    except Exception as exc:
+        logger.warning("Adapter raised for question_id=%s: %s", question.question_id, exc)
+        response, error = RAGResponse(answer=""), str(exc)
+    latency_ms = response.latency_ms if response.latency_ms is not None else (time.perf_counter() - started) * 1000
+
+    if chunk_texts is None:
+        recall, reciprocal_rank, hit = score_retrieval_at_k(question.chunk_ids, response.retrieved_chunk_ids, k)
+    else:
+        missing = [chunk_id for chunk_id in question.chunk_ids if chunk_id not in chunk_texts]
+        if missing:
+            raise ValueError(f"chunk(s) {missing} are not in the corpus passed with --match-by text")
+        gold_texts = [chunk_texts[chunk_id] for chunk_id in question.chunk_ids]
+        recall, reciprocal_rank, hit = score_retrieval_by_text_at_k(gold_texts, response.contexts, k, match_threshold)
+    if error is None:
+        correctness = judge(question.question, question.gold_answer, response.answer)
+    else:
+        correctness = CorrectnessResult(
+            is_correct=False, rationale=f"adapter raised: {error}", judge_model=ADAPTER_ERROR_JUDGE_MODEL
+        )
+
+    return QuestionResult(
+        question_id=question.question_id,
+        split=question.split.value,
+        answer=response.answer,
+        retrieved_chunk_ids=response.retrieved_chunk_ids[:k],
+        recall_at_k=recall,
+        reciprocal_rank=reciprocal_rank,
+        retrieval_hit=hit,
+        correctness=correctness,
+        failure_class=classify(hit, correctness.is_correct),
+        verdict=compute_verdict(is_correct=correctness.is_correct, retrieval_hit=hit),
+        adapter_error=error,
+        latency_ms=round(latency_ms, 2),
+    )
+
+
+def select_questions(
+    store: Store,
+    questions: list[Question],
+    split: Split,
+    config_hash: str,
+    index_version: str,
+    regression_sample: int = DEFAULT_REGRESSION_SAMPLE,
+    seed: int = 0,
+) -> Selection:
+    """Pick which active questions this evaluation covers (see the module docstring)."""
+    last_run = store.last_eval_run(config_hash, split)
+    if last_run is None:
+        return Selection(mode="full", questions=questions, n_new=len(questions))
+
+    seen = store.result_question_ids(config_hash, split)
+    new = [q for q in questions if q.question_id not in seen]
+    old = [q for q in questions if q.question_id in seen]
+
+    if last_run.index_version == index_version:
+        return Selection(mode="cached", questions=old + new, n_new=len(new))
+
+    rng = random.Random(f"{seed}:{index_version}")
+    sampled_ids = set(rng.sample(sorted(q.question_id for q in old), min(regression_sample, len(old))))
+    sampled = [q for q in old if q.question_id in sampled_ids]
+    return Selection(mode="regression", questions=new + sampled, n_new=len(new), n_sampled_old=len(sampled))
+
+
+def summarize(
+    results: list[QuestionResult],
+    *,
+    split: Split,
+    config: dict,
+    config_hash: str,
+    index_version: str,
+    k: int,
+    selection: Selection,
+    n_active_in_split: int,
+    n_cached: int,
+) -> EvalSummary:
+    n = len(results)
+
+    def rate(count: float) -> float:
+        return round(count / n, 4) if n else 0.0
+
+    failure_counts = {fc.value: 0 for fc in FailureClass}
+    verdict_counts = {v.value: 0 for v in Verdict}
+    for result in results:
+        failure_counts[result.failure_class.value] += 1
+        verdict_counts[result.verdict.value] += 1
+
+    lucky = [r for r in results if r.verdict == Verdict.LUCKY_PASS]
+    ablated = [r for r in lucky if r.ablation is not None]
+    return EvalSummary(
+        run_id=str(uuid.uuid4()),
+        created_at=utc_now(),
+        split=split.value,
+        config_hash=config_hash,
+        config=config,
+        index_version=index_version,
+        k=k,
+        mode=selection.mode,
+        n_questions=n,
+        n_active_in_split=n_active_in_split,
+        n_new=selection.n_new,
+        n_sampled_old=selection.n_sampled_old,
+        n_cached=n_cached,
+        n_run=n - n_cached,
+        score=round(failure_counts[FailureClass.PASS.value] / n * 100, 2) if n else 0.0,
+        answer_accuracy=rate(sum(r.correctness.is_correct for r in results)),
+        recall_at_k=rate(sum(r.recall_at_k for r in results)),
+        mrr=rate(sum(r.reciprocal_rank for r in results)),
+        failure_counts=failure_counts,
+        verdict_counts=verdict_counts,
+        judge_failures=sum(r.correctness.judge_parse_failed for r in results),
+        adapter_errors=sum(r.adapter_error is not None for r in results),
+        lucky_pass_confirmed=sum(r.ablation.confirms_lucky_pass for r in ablated) if ablated else None,
+    )
+
+
+def _ablate(result: QuestionResult, question: Question, no_context_query: NoContextQuery, judge: CorrectnessJudge) -> QuestionResult:
+    answer = no_context_query(question.question)
+    is_correct = judge(question.question, question.gold_answer, answer).is_correct
+    return result.model_copy(
+        update={"ablation": AblationResult(answer=answer, is_correct=is_correct, confirms_lucky_pass=is_correct)}
+    )
+
+
+def run_evaluation(
+    store: Store,
+    questions: list[Question],
+    split: Split,
+    adapter: Adapter,
+    judge: CorrectnessJudge,
+    *,
+    config: dict,
+    config_hash: str,
+    index_version: str,
+    k: int = DEFAULT_K,
+    regression_sample: int = DEFAULT_REGRESSION_SAMPLE,
+    seed: int = 0,
+    full: bool = False,
+    no_context_query: NoContextQuery | None = None,
+    on_progress: Callable[[int, int], None] | None = None,
+    chunk_texts: dict[str, str] | None = None,
+    match_threshold: float = DEFAULT_MATCH_THRESHOLD,
+) -> tuple[EvalSummary, list[QuestionResult]]:
+    """Evaluate the selected questions (cached results reused), save the run, return summary and results.
+
+    `questions` must all be from `split`; callers pass dev questions
+    directly and test questions only via ragaudit.runner.locked. Pass
+    no_context_query to ablate LUCKY_PASS results (v1's confirmation step).
+    """
+    if any(q.split != split for q in questions):
+        raise ValueError(f"run_evaluation got questions outside the {split.value} split")
+
+    if full:
+        selection = Selection(mode="full", questions=questions, n_new=len(questions))
+    else:
+        selection = select_questions(store, questions, split, config_hash, index_version, regression_sample, seed)
+
+    results: list[QuestionResult] = []
+    n_cached = 0
+    for index, question in enumerate(selection.questions, start=1):
+        result = store.load_result(question.question_id, config_hash, index_version)
+        if result is None:
+            result = evaluate_question(question, adapter, judge, k, chunk_texts, match_threshold)
+            store.save_result(result, config_hash, index_version)
+        else:
+            n_cached += 1
+        if no_context_query is not None and result.verdict == Verdict.LUCKY_PASS and result.ablation is None:
+            result = _ablate(result, question, no_context_query, judge)
+            store.save_result(result, config_hash, index_version)
+        results.append(result)
+        if on_progress is not None:
+            on_progress(index, len(selection.questions))
+
+    summary = summarize(
+        results,
+        split=split,
+        config=config,
+        config_hash=config_hash,
+        index_version=index_version,
+        k=k,
+        selection=selection,
+        n_active_in_split=len(questions),
+        n_cached=n_cached,
+    )
+    store.save_eval_run(summary)
+    return summary, results
+
+
+def evaluate_dev(store: Store, adapter: Adapter, judge: CorrectnessJudge, **kwargs) -> tuple[EvalSummary, list[QuestionResult]]:
+    """Evaluate the active dev split."""
+    return run_evaluation(store, store.dev_questions(), Split.DEV, adapter, judge, **kwargs)

ragaudit/runner/executor.py

@@ -0,0 +1,98 @@
+"""Runs a TestSet through an Adapter and scores every case."""
+
+import logging
+from typing import Callable
+
+from ragaudit.models.adapter import Adapter, RAGResponse
+from ragaudit.models.scoring import CorrectnessResult, GroundednessResult, RetrievalResult, ScoredCase
+from ragaudit.models.testset import TestCase, TestSet
+from ragaudit.scoring.retrieval import score_retrieval
+from ragaudit.scoring.verdict import compute_verdict
+
+logger = logging.getLogger(__name__)
+
+CorrectnessJudge = Callable[[str, str, str], CorrectnessResult]
+"""question, expected_answer, candidate_answer -> CorrectnessResult"""
+
+GroundednessJudge = Callable[[str, list[str]], GroundednessResult]
+"""answer, contexts -> GroundednessResult"""
+
+RetrievalScorer = Callable[[TestCase, RAGResponse], RetrievalResult]
+"""test case, adapter response -> RetrievalResult. Defaults to matching by chunk id."""
+
+ADAPTER_ERROR_JUDGE_MODEL = "n/a (adapter raised)"
+
+
+def score_by_id(test_case: TestCase, response: RAGResponse) -> RetrievalResult:
+    return score_retrieval(test_case.source_chunk_id, response.retrieved_chunk_ids)
+
+
+def _failure_scored_case(test_case: TestCase, error: Exception, retrieval_scorer: RetrievalScorer) -> ScoredCase:
+    """Build a ScoredCase for a TestCase whose adapter.query() call raised."""
+    empty_response = RAGResponse(answer="", retrieved_chunk_ids=[], contexts=[], raw={"error": str(error)})
+    retrieval = retrieval_scorer(test_case, empty_response)
+    correctness = CorrectnessResult(
+        is_correct=False,
+        rationale=f"adapter raised: {error}",
+        judge_model=ADAPTER_ERROR_JUDGE_MODEL,
+    )
+    groundedness = GroundednessResult(
+        is_grounded=False,
+        rationale="adapter raised, no answer to judge",
+        judge_model=ADAPTER_ERROR_JUDGE_MODEL,
+    )
+    verdict = compute_verdict(is_correct=False, retrieval_hit=retrieval.hit)
+    return ScoredCase(
+        test_id=test_case.test_id,
+        response=empty_response,
+        correctness=correctness,
+        retrieval=retrieval,
+        groundedness=groundedness,
+        verdict=verdict,
+    )
+
+
+def run_test_set(
+    test_set: TestSet,
+    adapter: Adapter,
+    correctness_judge: CorrectnessJudge,
+    groundedness_judge: GroundednessJudge,
+    retrieval_scorer: RetrievalScorer = score_by_id,
+) -> list[ScoredCase]:
+    """Run every TestCase in test_set through adapter and score it.
+
+    retrieval_scorer decides retrieval hits: by chunk id (the default) or,
+    for `--match-by text`, by overlap of the returned contexts with the gold
+    chunk's text.
+
+    A TestCase whose adapter.query() call raises is recorded as a failed
+    ScoredCase (correctness/groundedness both False) instead of aborting
+    the run — one broken question must not kill a 500-case run.
+    """
+    scored_cases: list[ScoredCase] = []
+
+    for test_case in test_set.test_cases:
+        try:
+            response = adapter.query(test_case.question)
+        except Exception as exc:
+            logger.warning("Adapter raised for test_id=%s: %s", test_case.test_id, exc)
+            scored_cases.append(_failure_scored_case(test_case, exc, retrieval_scorer))
+            continue
+
+        retrieval = retrieval_scorer(test_case, response)
+        correctness = correctness_judge(test_case.question, test_case.expected_answer, response.answer)
+        groundedness = groundedness_judge(response.answer, response.contexts)
+        verdict = compute_verdict(is_correct=correctness.is_correct, retrieval_hit=retrieval.hit)
+
+        scored_cases.append(
+            ScoredCase(
+                test_id=test_case.test_id,
+                response=response,
+                correctness=correctness,
+                retrieval=retrieval,
+                groundedness=groundedness,
+                verdict=verdict,
+            )
+        )
+
+    return scored_cases

ragaudit/runner/locked.py

@@ -0,0 +1,37 @@
+"""The final evaluation path: the only code allowed to read test-split gold answers.
+
+Everything here unlocks the test split for exactly as long as it needs,
+and hands back aggregate scores rather than questions, answers or
+per-question results, so a caller (in particular `ragaudit tune`) learns
+how well a config did on the test set without ever seeing what the test
+set contains. Per-question test results are still cached in the store,
+behind the same lock, so the report can be rebuilt.
+
+The pipeline under test does see the test *questions*: it has to answer
+them. That's inherent to evaluating it.
+"""
+
+from ragaudit.models.adapter import Adapter
+from ragaudit.models.evaluation import EvalSummary
+from ragaudit.models.question import Question, Split
+from ragaudit.runner.evaluate import CorrectnessJudge, run_evaluation
+from ragaudit.storage.lock import _unlock_test_split
+from ragaudit.storage.store import Store
+
+
+def _load_test_questions(store: Store) -> list[Question]:
+    """Active test-split questions, with gold answers. Private to this module."""
+    with _unlock_test_split():
+        return store.load_questions(split=Split.TEST)
+
+
+def evaluate_test_split(store: Store, adapter: Adapter, judge: CorrectnessJudge, **kwargs) -> EvalSummary:
+    """Evaluate the active test split once and return only the aggregate summary."""
+    with _unlock_test_split():
+        summary, _ = run_evaluation(store, _load_test_questions(store), Split.TEST, adapter, judge, **kwargs)
+    return summary
+
+
+def count_test_questions(store: Store) -> int:
+    """Number of active test questions (ids only, no unlock needed)."""
+    return len(store.question_ids(Split.TEST))

ragaudit/runner/summary.py

@@ -0,0 +1,45 @@
+"""Pure aggregation of ScoredCases into a RunSummary. No I/O, no LLM."""
+
+from ragaudit.models.run import RunSummary
+from ragaudit.models.scoring import ScoredCase, Verdict
+
+
+def compute_run_summary(scored_cases: list[ScoredCase]) -> RunSummary:
+    """Aggregate a list of ScoredCases into a RunSummary."""
+    total = len(scored_cases)
+    verdict_counts: dict[Verdict, int] = {verdict: 0 for verdict in Verdict}
+    groundedness_scores: list[float] = []
+    lucky_pass_cases: list[ScoredCase] = []
+    retrieval_hits = 0
+    correct = 0
+    judge_failures = 0
+
+    for case in scored_cases:
+        verdict_counts[case.verdict] += 1
+        if case.retrieval.hit:
+            retrieval_hits += 1
+        if case.correctness.is_correct:
+            correct += 1
+        if case.groundedness.score is not None:
+            groundedness_scores.append(case.groundedness.score)
+        if case.verdict == Verdict.LUCKY_PASS:
+            lucky_pass_cases.append(case)
+        if case.correctness.judge_parse_failed or case.groundedness.judge_parse_failed:
+            judge_failures += 1
+
+    ablation_confirmed_count = None
+    if lucky_pass_cases:
+        ablation_confirmed_count = sum(
+            1 for case in lucky_pass_cases if case.ablation is not None and case.ablation.confirms_lucky_pass
+        )
+
+    return RunSummary(
+        total_cases=total,
+        verdict_counts=verdict_counts,
+        retrieval_hit_rate=(retrieval_hits / total) if total else 0.0,
+        answer_correctness_rate=(correct / total) if total else 0.0,
+        lucky_pass_rate=(verdict_counts[Verdict.LUCKY_PASS] / total) if total else 0.0,
+        mean_groundedness=(sum(groundedness_scores) / len(groundedness_scores)) if groundedness_scores else None,
+        ablation_confirmed_count=ablation_confirmed_count,
+        judge_failure_count=judge_failures,
+    )