ragaudit-cli 0.3.0__py3-none-any.whl

ragaudit/models/evaluation.py

@@ -0,0 +1,70 @@
+"""v2 per-question evaluation results and their aggregate."""
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+from .scoring import AblationResult, CorrectnessResult, Verdict
+
+
+class FailureClass(StrEnum):
+    PASS = "pass"
+    """A gold chunk was retrieved in the top k and the answer is correct."""
+    RETRIEVAL_MISS = "retrieval_miss"
+    """Gold chunk(s) not retrieved in the top k, whether or not the answer happened to be right."""
+    GENERATION_MISS = "generation_miss"
+    """Gold chunk(s) retrieved, answer still wrong."""
+
+
+class QuestionResult(BaseModel):
+    """One question run through a pipeline config and scored. Never contains the gold answer."""
+
+    question_id: str
+    split: str
+    answer: str
+    retrieved_chunk_ids: list[str] = Field(default_factory=list)
+    recall_at_k: float
+    """Share of the gold chunk_ids found in the top k (0 or 1 for single-chunk questions)."""
+    reciprocal_rank: float
+    """1 / (1 + rank of the first gold chunk in the top k), or 0 if none was retrieved."""
+    retrieval_hit: bool
+    """Every gold chunk was retrieved in the top k."""
+    correctness: CorrectnessResult
+    failure_class: FailureClass
+    verdict: Verdict
+    """v1's retrieval-contribution signal: a correct answer without retrieval is a LUCKY_PASS."""
+    ablation: AblationResult | None = None
+    adapter_error: str | None = None
+    latency_ms: float | None = None
+
+
+class EvalSummary(BaseModel):
+    """Aggregate of one evaluation: the headline score is the pass rate, in points (0-100)."""
+
+    run_id: str
+    created_at: datetime
+    split: str
+    config_hash: str
+    config: dict = Field(default_factory=dict)
+    index_version: str
+    k: int
+    mode: str
+    """full (config never evaluated), regression (index changed: new + sampled old), or cached (index unchanged)."""
+    n_questions: int
+    n_active_in_split: int
+    n_new: int = 0
+    n_sampled_old: int = 0
+    n_cached: int = 0
+    n_run: int = 0
+    score: float
+    """Pass rate x 100: retrieval hit AND correct answer. Lucky passes don't count."""
+    answer_accuracy: float
+    recall_at_k: float
+    mrr: float
+    failure_counts: dict[str, int]
+    verdict_counts: dict[str, int]
+    judge_failures: int = 0
+    adapter_errors: int = 0
+    lucky_pass_confirmed: int | None = None
+    """Of LUCKY_PASS results, how many ablation confirmed (None if ablation didn't run)."""

ragaudit/models/question.py

@@ -0,0 +1,152 @@
+"""v2 questions: a verified fact, the question that tests it, and where it came from."""
+
+import hashlib
+from datetime import datetime, timezone
+from enum import StrEnum
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+DEV_FRACTION_PERCENT = 70
+"""Share of chunks assigned to the dev split; the rest go to the locked test split."""
+
+SPLIT_SALT = "ragaudit-chunk-split-v1"
+"""Never change this: doing so would reshuffle every chunk (and its questions) between splits."""
+
+
+class QuestionType(StrEnum):
+    SINGLE = "single"
+    MULTI_HOP = "multi_hop"
+
+
+class Split(StrEnum):
+    DEV = "dev"
+    TEST = "test"
+
+
+class QuestionStatus(StrEnum):
+    ACTIVE = "active"
+    STALE = "stale"
+    """Its source chunk changed or was deleted. Never scored."""
+    REJECTED = "rejected"
+    """Failed a generation check; reject_reason says which. Never scored."""
+
+
+class RejectReason(StrEnum):
+    TOO_LEXICAL = "too_lexical"
+    RELATIVE_TIME = "relative_time"
+    UNSUPPORTED = "unsupported"
+    UNANSWERABLE = "unanswerable"
+    AMBIGUOUS = "ambiguous"
+    VERIFIER_UNPARSEABLE = "verifier_unparseable"
+    CROSS_SPLIT = "cross_split"
+    """A multi-hop question whose source chunks are not all in the same split."""
+
+
+def compute_question_id(chunk_hashes: list[str], fact: str) -> str:
+    """Stable id: hash of the source chunk hash(es) plus the fact text.
+
+    The same fact extracted from the same chunk content always gets the
+    same id, so re-running generation is idempotent.
+    """
+    payload = "|".join(chunk_hashes) + "\n" + fact.strip()
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def assign_split(chunk_id: str) -> Split:
+    """Deterministically assign a chunk to dev (70%) or test (30%) from its chunk_id alone.
+
+    The split is per chunk, not per question: if questions from one chunk
+    could land on both sides, tuning on the dev ones would teach the
+    pipeline that chunk's content and inflate the test score. It hashes
+    the chunk *id*, not its content, so editing a chunk never moves it
+    between splits, and adding chunks never moves old ones.
+    """
+    digest = hashlib.sha256(f"{SPLIT_SALT}:{chunk_id}".encode("utf-8")).hexdigest()
+    return Split.DEV if int(digest[:8], 16) % 100 < DEV_FRACTION_PERCENT else Split.TEST
+
+
+def split_for_chunks(chunk_ids: list[str]) -> Split | None:
+    """The split shared by every source chunk, or None when they disagree (a cross-split multi-hop question)."""
+    splits = {assign_split(chunk_id) for chunk_id in chunk_ids}
+    return splits.pop() if len(splits) == 1 else None
+
+
+def utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class Question(BaseModel):
+    """One generated question, linked to the chunk(s) its gold answer came from."""
+
+    question_id: str
+    chunk_ids: list[str]
+    """One entry normally; two or more for multi-hop questions."""
+    chunk_hashes: list[str]
+    """Content hash of each source chunk, parallel to chunk_ids."""
+    fact: str
+    question: str
+    gold_answer: str
+    question_type: QuestionType = QuestionType.SINGLE
+    split: Split | None
+    """Its source chunks' split. None only for questions rejected as cross_split, which belong to neither."""
+    status: QuestionStatus = QuestionStatus.ACTIVE
+    reject_reason: str | None = None
+    created_at: datetime = Field(default_factory=utc_now)
+    """Absolute UTC timestamp."""
+    source_timestamp: datetime | None = None
+    """When the source material was captured, if the corpus provides it."""
+    generator_model: str
+    verifier_model: str | None = None
+    lexical_overlap: float | None = None
+    """Fraction of the question's content tokens that appear in its source chunk(s)."""
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("created_at", "source_timestamp")
+    @classmethod
+    def _force_utc(cls, value: datetime | None) -> datetime | None:
+        if value is None:
+            return None
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc)
+
+    @model_validator(mode="after")
+    def _split_required_unless_rejected(self) -> "Question":
+        if self.split is None and self.status != QuestionStatus.REJECTED:
+            raise ValueError("only rejected questions may have no split")
+        return self
+
+    @classmethod
+    def create(
+        cls,
+        chunk_ids: list[str],
+        chunk_hashes: list[str],
+        fact: str,
+        question: str,
+        gold_answer: str,
+        generator_model: str,
+        **kwargs,
+    ) -> "Question":
+        """Build a Question with its id and split derived the one canonical way.
+
+        A question whose source chunks span both splits is always rejected
+        as cross_split, whatever status the caller asked for.
+        """
+        question_id = compute_question_id(chunk_hashes, fact)
+        split = split_for_chunks(chunk_ids)
+        if split is None:
+            kwargs["status"] = QuestionStatus.REJECTED
+            kwargs["reject_reason"] = RejectReason.CROSS_SPLIT.value
+        question_type = QuestionType.MULTI_HOP if len(chunk_ids) > 1 else QuestionType.SINGLE
+        return cls(
+            question_id=question_id,
+            chunk_ids=chunk_ids,
+            chunk_hashes=chunk_hashes,
+            fact=fact,
+            question=question,
+            gold_answer=gold_answer,
+            question_type=kwargs.pop("question_type", question_type),
+            split=split,
+            generator_model=generator_model,
+            **kwargs,
+        )

ragaudit/models/run.py

@@ -0,0 +1,44 @@
+"""Models for a full audit run and its aggregate summary."""
+
+from datetime import datetime
+
+from pydantic import BaseModel
+
+from .scoring import ScoredCase, Verdict
+
+
+class RunConfig(BaseModel):
+    """Configuration a run was executed with."""
+
+    run_id: str
+    testset_id: str
+    adapter_path: str
+    correctness_judge_model: str
+    groundedness_judge_model: str
+    run_ablation: bool = True
+    started_at: datetime
+
+
+class RunSummary(BaseModel):
+    """Aggregate stats computed after all cases are scored — the report's headline."""
+
+    total_cases: int
+    verdict_counts: dict[Verdict, int]
+    retrieval_hit_rate: float
+    answer_correctness_rate: float
+    lucky_pass_rate: float
+    """The headline metric."""
+    mean_groundedness: float | None = None
+    ablation_confirmed_count: int | None = None
+    """Of LUCKY_PASS cases, how many ablation confirmed."""
+    judge_failure_count: int = 0
+    """Cases where the correctness judge, groundedness judge, or both failed to parse."""
+
+
+class Run(BaseModel):
+    """A full audit run: config, every scored case, and the aggregate summary."""
+
+    config: RunConfig
+    finished_at: datetime | None = None
+    scored_cases: list[ScoredCase]
+    summary: RunSummary | None = None

ragaudit/models/scoring.py

@@ -0,0 +1,82 @@
+"""Per-case scoring results and the verdict that cross-references them."""
+
+from enum import StrEnum
+
+from pydantic import BaseModel
+
+from .adapter import RAGResponse
+
+
+class Verdict(StrEnum):
+    """The cross-reference of answer correctness against retrieval hit."""
+
+    TRUE_PASS = "true_pass"
+    LUCKY_PASS = "lucky_pass"
+    GENERATION_FAILURE = "generation_failure"
+    RETRIEVAL_FAILURE = "retrieval_failure"
+
+
+class CorrectnessResult(BaseModel):
+    """LLM-judged comparison of RAGResponse.answer against TestCase.expected_answer."""
+
+    is_correct: bool
+    score: float | None = None
+    """Raw judge score in [0, 1], if the judge emits one."""
+    rationale: str | None = None
+    judge_model: str
+    judge_parse_failed: bool = False
+    """True if the judge's response couldn't be parsed and is_correct fell back to False."""
+
+
+class RetrievalResult(BaseModel):
+    """Whether retrieval found the gold chunk — no LLM involved.
+
+    By id (the default): set membership of expected_chunk_id. By text: the
+    gold chunk's text overlaps one of the returned contexts (see
+    ragaudit.scoring.text_match).
+    """
+
+    hit: bool
+    expected_chunk_id: str
+    retrieved_chunk_ids: list[str]
+    rank: int | None = None
+    """Index of the first match: in retrieved_chunk_ids (by id) or in the contexts (by text)."""
+    match_by: str = "id"
+    """"id" or "text"."""
+    match_score: float | None = None
+    """By text only: the best overlap between the gold chunk and any returned context."""
+
+
+class GroundednessResult(BaseModel):
+    """Whether the answer is supported by the contexts the adapter actually returned."""
+
+    is_grounded: bool
+    score: float | None = None
+    rationale: str | None = None
+    judge_model: str
+    judge_parse_failed: bool = False
+    """True if the judge's response couldn't be parsed and is_grounded fell back to False."""
+
+
+class AblationResult(BaseModel):
+    """Re-run with retrieval disabled, to empirically confirm a candidate LUCKY_PASS.
+
+    Only populated for cases that scored correct + not-retrieved.
+    """
+
+    answer: str
+    is_correct: bool
+    confirms_lucky_pass: bool
+    """True iff the answer is still correct with no retrieval."""
+
+
+class ScoredCase(BaseModel):
+    """One TestCase run through a pipeline and fully scored."""
+
+    test_id: str
+    response: RAGResponse
+    correctness: CorrectnessResult
+    retrieval: RetrievalResult
+    groundedness: GroundednessResult
+    verdict: Verdict
+    ablation: AblationResult | None = None

ragaudit/models/testset.py

@@ -0,0 +1,30 @@
+"""Models for generated test sets: questions with known ground-truth sources."""
+
+from datetime import datetime
+
+from pydantic import BaseModel, Field
+
+
+class TestCase(BaseModel):
+    """One generated question with its ground-truth source chunk."""
+
+    test_id: str
+    question: str
+    expected_answer: str
+    source_chunk_id: str
+    source_doc_id: str
+    """Denormalized, avoids a join for report rendering."""
+    generation_model: str
+    created_at: datetime
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+
+class TestSet(BaseModel):
+    """A generated batch of TestCases tied to a specific corpus state."""
+
+    testset_id: str
+    corpus_hash: str
+    """Hash over all Document.content_hash, for reproducibility/staleness checks."""
+    created_at: datetime
+    generation_model: str
+    test_cases: list[TestCase]

ragaudit/models/tune.py

@@ -0,0 +1,56 @@
+"""v2 tuning: one trial per evaluated config, and the run's outcome."""
+
+from datetime import datetime
+
+from pydantic import BaseModel, Field
+
+from .evaluation import EvalSummary
+
+
+class Trial(BaseModel):
+    """One pipeline config evaluated on the dev split."""
+
+    index: int
+    config: dict
+    config_hash: str
+    source: str
+    """baseline, grid, random, llm, or llm_fallback (the LLM's proposal was invalid; a random one was used)."""
+    rationale: str | None = None
+    dev: EvalSummary
+    created_at: datetime
+
+
+class TuneResult(BaseModel):
+    tune_id: str
+    created_at: datetime
+    finished_at: datetime | None = None
+    strategy: str
+    search_space: dict[str, list]
+    adapter_path: str
+    judge_model: str
+    index_version: str
+    k: int
+    seed: int
+    max_trials: int
+    trials: list[Trial] = Field(default_factory=list)
+    baseline_index: int = 0
+    best_index: int = 0
+    baseline_test: EvalSummary | None = None
+    """Aggregate only: tune never sees test questions or answers."""
+    best_test: EvalSummary | None = None
+    overfit_threshold: float = 10.0
+    overfit_gap: float | None = None
+    """Best config's dev score minus its test score, in points."""
+    overfit_warning: bool = False
+    stopped_reason: str
+    """max_trials, search_space_exhausted, or max_llm_calls."""
+    llm_calls: int = 0
+    best_config_path: str | None = None
+
+    @property
+    def baseline(self) -> Trial:
+        return self.trials[self.baseline_index]
+
+    @property
+    def best(self) -> Trial:
+        return self.trials[self.best_index]

ragaudit/report/generator.py

@@ -0,0 +1,155 @@
+"""Renders a Run into a single, self-contained static HTML report.
+
+The report needs question text for its per-case table, which lives on
+TestCase, not ScoredCase — so this takes both the Run and the TestSet it
+was run against, cross-referenced by test_id.
+"""
+
+from importlib import resources
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+from ragaudit.models.run import Run
+from ragaudit.models.scoring import ScoredCase, Verdict
+from ragaudit.models.testset import TestSet
+from ragaudit.report.plain_summary import build_plain_summary
+
+TEMPLATES_DIR = Path(__file__).parent / "templates"
+
+_VERDICT_LABELS = {
+    Verdict.TRUE_PASS: "True pass",
+    Verdict.LUCKY_PASS: "Lucky pass",
+    Verdict.GENERATION_FAILURE: "Generation failure",
+    Verdict.RETRIEVAL_FAILURE: "Retrieval failure",
+}
+
+
+def _build_case_row(case: ScoredCase, question: str, expected_answer: str) -> dict:
+    return {
+        "test_id": case.test_id,
+        "question": question,
+        "expected_answer": expected_answer,
+        "answer": case.response.answer,
+        "contexts": case.response.contexts,
+        "verdict": case.verdict.value,
+        "verdict_label": _VERDICT_LABELS[case.verdict],
+        "retrieval_rank": case.retrieval.rank,
+        "judge_parse_failed": case.correctness.judge_parse_failed or case.groundedness.judge_parse_failed,
+        "ablation": (
+            {"answer": case.ablation.answer, "confirms_lucky_pass": case.ablation.confirms_lucky_pass}
+            if case.ablation is not None
+            else None
+        ),
+    }
+
+
+def _compute_headline(total_cases: int, verdict_matrix: dict) -> dict:
+    """Pick the single dominant finding to state in the headline sentence.
+
+    Lucky passes are always the headline when any exist — that is the
+    tool's core claim. Otherwise, headline the largest failure category
+    (retrieval beats generation on a tie) so a run full of failures never
+    reads as quietly clean just because nothing was "lucky." Only when
+    there are no lucky passes and no failures at all does the headline
+    turn positive.
+    """
+
+    def pct(count: int) -> float:
+        return round(count / total_cases * 100, 1) if total_cases else 0.0
+
+    lucky_pass = verdict_matrix["lucky_pass"]
+    if lucky_pass > 0:
+        return {"kind": "lucky_pass", "count": lucky_pass, "pct": pct(lucky_pass)}
+
+    failure_counts = {
+        "retrieval_failure": verdict_matrix["retrieval_failure"],
+        "generation_failure": verdict_matrix["generation_failure"],
+    }
+    dominant_kind = max(failure_counts, key=failure_counts.get)
+    dominant_count = failure_counts[dominant_kind]
+    if dominant_count > 0:
+        return {"kind": dominant_kind, "count": dominant_count, "pct": pct(dominant_count)}
+
+    return {"kind": "all_true_pass", "count": verdict_matrix["true_pass"], "pct": pct(verdict_matrix["true_pass"])}
+
+
+def _build_case_rows(run: Run, test_set: TestSet) -> list[dict]:
+    test_case_by_id = {tc.test_id: tc for tc in test_set.test_cases}
+    rows = []
+    for case in run.scored_cases:
+        test_case = test_case_by_id.get(case.test_id)
+        question = test_case.question if test_case else "(question not found in test set)"
+        expected_answer = test_case.expected_answer if test_case else ""
+        rows.append(_build_case_row(case, question, expected_answer))
+    return rows
+
+
+def _load_style_css() -> str:
+    """Load the canonical stylesheet from package data.
+
+    A silently unstyled report is worse than a crash — if the stylesheet
+    can't be loaded (e.g. it was left out of a packaging change), raise
+    rather than falling back to an empty string.
+    """
+    try:
+        css = resources.files("ragaudit.report").joinpath("templates", "style.css").read_text(encoding="utf-8")
+    except (FileNotFoundError, ModuleNotFoundError, OSError) as exc:
+        raise RuntimeError(
+            "Could not load the report stylesheet from package data "
+            "(ragaudit/report/templates/style.css). The report cannot be rendered without it."
+        ) from exc
+
+    if not css.strip():
+        raise RuntimeError("The report stylesheet (ragaudit/report/templates/style.css) is empty.")
+
+    return css
+
+
+def generate_report(run: Run, test_set: TestSet, output_path: str | Path) -> Path:
+    """Render `run` into a self-contained HTML file at `output_path`.
+
+    The canonical stylesheet (ragaudit/report/templates/style.css, shipped
+    as package data) is inlined into the output, so the resulting file has
+    no external dependencies. Returns output_path.
+    """
+    if run.summary is None:
+        raise ValueError("Run.summary must be computed (see runner.summary.compute_run_summary) before reporting")
+
+    output_path = Path(output_path)
+    summary = run.summary
+
+    env = Environment(
+        loader=FileSystemLoader(str(TEMPLATES_DIR)),
+        autoescape=select_autoescape(["html", "jinja"]),
+    )
+    template = env.get_template("report.html.jinja")
+
+    verdict_matrix = {
+        "true_pass": summary.verdict_counts.get(Verdict.TRUE_PASS, 0),
+        "lucky_pass": summary.verdict_counts.get(Verdict.LUCKY_PASS, 0),
+        "generation_failure": summary.verdict_counts.get(Verdict.GENERATION_FAILURE, 0),
+        "retrieval_failure": summary.verdict_counts.get(Verdict.RETRIEVAL_FAILURE, 0),
+    }
+    lucky_pass_count = verdict_matrix["lucky_pass"]
+
+    plain_summary = build_plain_summary(summary, run.scored_cases)
+    html = template.render(
+        plain_summary=plain_summary,
+        summary_lines=[tuple(line.split(": ", 1)) for line in plain_summary.lines()],
+        run=run,
+        summary=summary,
+        style_css=_load_style_css(),
+        rows=_build_case_rows(run, test_set),
+        verdict_matrix=verdict_matrix,
+        headline=_compute_headline(summary.total_cases, verdict_matrix),
+        lucky_pass_count=lucky_pass_count,
+        lucky_pass_pct=round(summary.lucky_pass_rate * 100, 1),
+        total_cases=summary.total_cases,
+        judge_failure_count=summary.judge_failure_count,
+        ablation_confirmed_count=summary.ablation_confirmed_count,
+    )
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(html, encoding="utf-8")
+    return output_path