ragcheck-eval 0.1.0__py3-none-any.whl

ragcheck/__init__.py

@@ -0,0 +1,18 @@
+"""RAGCheck: pytest for RAG systems."""
+
+from ragcheck.adapters.base import RAGAdapter, RAGResponse, RetrievedChunk
+from ragcheck.datasets.models import EvalDataset, EvalSample, QAPair
+from ragcheck.metrics.base import Metric, MetricResult
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "EvalDataset",
+    "EvalSample",
+    "Metric",
+    "MetricResult",
+    "QAPair",
+    "RAGAdapter",
+    "RAGResponse",
+    "RetrievedChunk",
+]

ragcheck/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Adapters that wrap arbitrary RAG pipelines for evaluation."""
+
+from ragcheck.adapters.base import RAGAdapter, RAGResponse, RetrievedChunk
+from ragcheck.adapters.function import FunctionAdapter
+from ragcheck.adapters.langchain import LangChainAdapter
+
+__all__ = ["FunctionAdapter", "LangChainAdapter", "RAGAdapter", "RAGResponse", "RetrievedChunk"]

ragcheck/adapters/base.py

@@ -0,0 +1,39 @@
+"""Core adapter interface: wrap any RAG pipeline so RAGCheck can evaluate it."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel, Field
+
+
+class RetrievedChunk(BaseModel):
+    """One chunk returned by the retrieval stage."""
+
+    content: str
+    source_id: str
+    score: float | None = None
+    metadata: dict = Field(default_factory=dict)
+
+
+class RAGResponse(BaseModel):
+    """Everything a pipeline produced for one question."""
+
+    answer: str
+    retrieved_chunks: list[RetrievedChunk]
+    latencies_ms: dict[str, float] = Field(default_factory=dict)
+    token_usage: dict[str, int] = Field(default_factory=dict)
+    refused: bool = False
+
+
+class RAGAdapter(ABC):
+    """Wrap any RAG pipeline so RAGCheck can evaluate it."""
+
+    @abstractmethod
+    def query(self, question: str) -> RAGResponse:
+        """Answer a single question, returning the answer plus retrieval trace."""
+        ...
+
+    def batch_query(self, questions: list[str]) -> list[RAGResponse]:
+        """Override for pipelines with native batching. Default: sequential."""
+        return [self.query(q) for q in questions]

ragcheck/adapters/function.py

@@ -0,0 +1,19 @@
+"""Adapter that wraps a plain Python callable."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from ragcheck.adapters.base import RAGAdapter, RAGResponse
+
+
+class FunctionAdapter(RAGAdapter):
+    """Wrap any ``(question: str) -> RAGResponse`` callable as a RAGAdapter."""
+
+    def __init__(self, fn: Callable[[str], RAGResponse]) -> None:
+        """Store the pipeline function to delegate queries to."""
+        self._fn = fn
+
+    def query(self, question: str) -> RAGResponse:
+        """Delegate the question to the wrapped function."""
+        return self._fn(question)

ragcheck/adapters/langchain.py

@@ -0,0 +1,59 @@
+"""Adapter for LangChain retriever + chain pipelines.
+
+LangChain is not a dependency of ragcheck; objects are duck-typed. The
+retriever must expose ``invoke(question) -> list[Document]`` (LangChain's
+standard retriever interface) and the chain ``invoke(inputs) -> str | Message``.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Protocol
+
+from ragcheck.adapters.base import RAGAdapter, RAGResponse, RetrievedChunk
+
+
+class _Invokable(Protocol):
+    def invoke(self, __input: Any) -> Any: ...
+
+
+class LangChainAdapter(RAGAdapter):
+    """Wrap a LangChain retriever + generation chain as a RAGAdapter.
+
+    The chain is invoked with ``{"question": ..., "context": ...}`` where
+    context is the concatenated retrieved documents. Document source ids come
+    from ``doc.metadata[source_key]`` (falling back to a positional id).
+    """
+
+    def __init__(
+        self,
+        retriever: _Invokable,
+        chain: _Invokable,
+        source_key: str = "source",
+    ) -> None:
+        """Bind a retriever and a chain; ``source_key`` names the id metadata field."""
+        self.retriever = retriever
+        self.chain = chain
+        self.source_key = source_key
+
+    def query(self, question: str) -> RAGResponse:
+        """Retrieve documents, run the chain, and normalize into a RAGResponse."""
+        t0 = time.perf_counter()
+        documents = self.retriever.invoke(question)
+        t1 = time.perf_counter()
+        chunks = [
+            RetrievedChunk(
+                content=getattr(doc, "page_content", str(doc)),
+                source_id=str(getattr(doc, "metadata", {}).get(self.source_key, f"doc_{i}")),
+            )
+            for i, doc in enumerate(documents)
+        ]
+        context = "\n\n".join(f"[{c.source_id}] {c.content}" for c in chunks)
+        raw = self.chain.invoke({"question": question, "context": context})
+        t2 = time.perf_counter()
+        answer = str(getattr(raw, "content", raw)).strip()
+        return RAGResponse(
+            answer=answer,
+            retrieved_chunks=chunks,
+            latencies_ms={"retrieval": (t1 - t0) * 1000, "generation": (t2 - t1) * 1000},
+        )

ragcheck/cache.py

@@ -0,0 +1,67 @@
+"""SQLite-backed cache for LLM judgments."""
+
+from __future__ import annotations
+
+import hashlib
+import sqlite3
+import threading
+from pathlib import Path
+
+
+def make_key(*parts: str) -> str:
+    """Build a stable SHA256 cache key from ordered string components.
+
+    Callers pass (metric_name, prompt_version, question, context, answer, ...);
+    components are length-prefixed so no two part sequences collide.
+    """
+    h = hashlib.sha256()
+    for part in parts:
+        encoded = part.encode("utf-8")
+        h.update(str(len(encoded)).encode("ascii"))
+        h.update(b":")
+        h.update(encoded)
+    return h.hexdigest()
+
+
+class JudgmentCache:
+    """Persistent key-value store for judge outputs.
+
+    Thread-safe (judged metrics run samples in parallel). Hits are counted
+    on ``hits`` / ``misses`` so the runner can report cache effectiveness
+    on re-runs.
+    """
+
+    def __init__(self, path: str | Path) -> None:
+        """Open (or create) the cache database at ``path``."""
+        self._conn = sqlite3.connect(str(path), check_same_thread=False)
+        self._lock = threading.Lock()
+        self._conn.execute(
+            "CREATE TABLE IF NOT EXISTS judgments (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
+        )
+        self._conn.commit()
+        self.hits = 0
+        self.misses = 0
+
+    def get(self, key: str) -> str | None:
+        """Return the cached value for ``key``, or None on a miss."""
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT value FROM judgments WHERE key = ?", (key,)
+            ).fetchone()
+            if row is None:
+                self.misses += 1
+                return None
+            self.hits += 1
+            return str(row[0])
+
+    def set(self, key: str, value: str) -> None:
+        """Store ``value`` under ``key``, overwriting any previous entry."""
+        with self._lock:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO judgments (key, value) VALUES (?, ?)", (key, value)
+            )
+            self._conn.commit()
+
+    def close(self) -> None:
+        """Close the underlying SQLite connection."""
+        self._conn.close()

ragcheck/cli.py

@@ -0,0 +1,210 @@
+"""RAGCheck command-line interface."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from ragcheck.config import load_config
+from ragcheck.report.cli_summary import print_summary
+from ragcheck.runner import run_eval
+
+app = typer.Typer(help="pytest for RAG systems.", no_args_is_help=True)
+console = Console()
+
+
+@app.callback()
+def main() -> None:
+    """RAGCheck: evaluate RAG pipelines."""
+    # Explicit callback keeps subcommand names (`ragcheck run ...`) even while
+    # only one command exists; typer otherwise collapses single-command apps.
+
+
+@app.command()
+def run(
+    config: Path = typer.Argument(..., help="Path to an eval config YAML."),
+    yes: bool = typer.Option(
+        False, "--yes", help="Proceed without confirmation on large LLM-judged runs."
+    ),
+) -> None:
+    """Run an evaluation and print the scorecard."""
+    eval_config = load_config(config)
+    if yes:
+        eval_config.assume_yes = True
+    report, out_path = run_eval(eval_config)
+    print_summary(report, console)
+    console.print(f"[green]Report written to[/green] {out_path}")
+    if eval_config.html:
+        console.print(f"[green]HTML report:[/green] {out_path.with_suffix('.html')}")
+
+
+@app.command()
+def compare(
+    old: Path = typer.Argument(..., help="Baseline report JSON."),
+    new: Path = typer.Argument(..., help="New report JSON."),
+    fail_if: list[str] = typer.Option(
+        [],
+        "--fail-if",
+        help="Exit 1 when a metric's delta drops below a threshold, e.g. 'faithfulness<-0.05'. "
+        "Repeatable.",
+    ),
+) -> None:
+    """Diff two eval reports; non-zero exit on threshold breach (CI-friendly)."""
+    from ragcheck.report.regression import (
+        compare_reports,
+        load_report,
+        markdown_diff,
+        parse_fail_if,
+    )
+
+    diffs = compare_reports(load_report(old), load_report(new), parse_fail_if(fail_if))
+    console.print(markdown_diff(diffs, old.name, new.name))
+    breached = [d for d in diffs if d.breached]
+    if breached:
+        names = ", ".join(d.metric_name for d in breached)
+        console.print(f"\n[red]Regression threshold breached:[/red] {names}")
+        raise typer.Exit(code=1)
+
+
+@app.command()
+def demo(
+    output: Path = typer.Option(Path("ragcheck_output"), help="Where to write the reports."),
+) -> None:
+    """Run the zero-key demo: canned pipeline, offline judge, full reports in ~5 seconds."""
+    from ragcheck.config import EvalConfig, MetricSpec
+    from ragcheck.demo import DemoPipeline, demo_dataset
+    from ragcheck.runner import evaluate
+
+    console.print(
+        "[bold]RAGCheck demo[/bold] - no API key needed.\n"
+        "[dim]Canned pipeline answers (two deliberately wrong) + a deterministic offline "
+        "judge, so you can see the workflow. Real runs use Claude/Groq judges.[/dim]\n"
+    )
+    config = EvalConfig(
+        dataset=Path("demo"),
+        adapter="ragcheck.demo:DemoPipeline",
+        metrics=[
+            MetricSpec(name="hit_rate", params={"k": 3}),
+            MetricSpec(name="mrr"),
+            MetricSpec(name="faithfulness"),
+            MetricSpec(name="answer_relevance"),
+            MetricSpec(name="refusal_calibration"),
+        ],
+        judge_provider="offline-demo",
+        output_dir=output,
+        cache_path=output / "demo_cache.sqlite",
+        run_name="demo",
+    )
+    report, out_path = evaluate(
+        DemoPipeline(), demo_dataset(), config, adapter_name="demo_pipeline"
+    )
+    print_summary(report, console)
+    faith = next(m for m in report.metrics if m.metric_name == "faithfulness")
+    refusal = next(m for m in report.metrics if m.metric_name == "refusal_calibration")
+    console.print(
+        f"\n[bold]What just happened:[/bold] faithfulness caught the planted hallucination "
+        f"({[c['claim'] for c in faith.details['failed_claims']][:1]}), and refusal "
+        f"calibration flagged the invented answer "
+        f"(false-answer rate: {refusal.details['false_answer_rate']:.2f}).\n"
+        f"[green]Open the HTML report to see the failing samples with their contexts:[/green] "
+        f"{out_path.with_suffix('.html')}\n"
+        "[dim]Next: wrap your own pipeline (docs/quickstart.md) and set "
+        "ANTHROPIC_API_KEY or GROQ_API_KEY for real judging.[/dim]"
+    )
+
+
+@app.command("generate-dataset")
+def generate_dataset_cmd(
+    corpus_dir: Path = typer.Argument(..., help="Directory of .txt/.md corpus files."),
+    n: int = typer.Option(200, help="Base QA pairs to generate (before paraphrase expansion)."),
+    unanswerable_frac: float = typer.Option(0.15, help="Fraction of n that is unanswerable."),
+    paraphrase_groups: int = typer.Option(
+        20, help="Answerable questions to expand with 4 paraphrases each."
+    ),
+    out: Path = typer.Option(Path("dataset.jsonl"), help="Output JSONL path."),
+    provider: str = typer.Option("anthropic", help="Generator provider: anthropic or groq."),
+    model: str | None = typer.Option(None, help="Generator model (provider default if omitted)."),
+    seed: int = typer.Option(13, help="Sampling seed (keep fixed for resumability)."),
+    cache_path: Path = typer.Option(
+        Path(".ragcheck_cache.sqlite"), help="Generation cache (makes re-runs resumable)."
+    ),
+) -> None:
+    """Generate an eval dataset (difficulty tiers, unanswerables, paraphrases) from a corpus."""
+    from collections import Counter
+
+    from ragcheck.cache import JudgmentCache
+    from ragcheck.datasets.synthetic import SyntheticGenerator
+    from ragcheck.judge.judge import Judge
+    from ragcheck.llm import build_client
+
+    cache = JudgmentCache(cache_path)
+    judge = Judge(build_client(provider, model), cache)
+    generator = SyntheticGenerator(judge, seed=seed)
+    dataset = generator.generate(
+        corpus_dir, n=n, unanswerable_frac=unanswerable_frac, paraphrase_groups=paraphrase_groups
+    )
+
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text("\n".join(p.model_dump_json(exclude_none=True) for p in dataset.pairs) + "\n")
+
+    tiers = Counter(p.difficulty for p in dataset.pairs if p.answerable)
+    n_groups = len({p.paraphrase_group for p in dataset.pairs if p.paraphrase_group})
+    console.print(
+        f"[green]Wrote {len(dataset.pairs)} pairs to[/green] {out}\n"
+        f"tiers: {dict(tiers)}  |  unanswerable: "
+        f"{sum(1 for p in dataset.pairs if not p.answerable)}  |  "
+        f"paraphrase groups: {n_groups}  |  "
+        f"cache: {cache.hits} hits / {cache.misses} misses"
+    )
+
+
+@app.command("validate-judge")
+def validate_judge_cmd(
+    labels: Path = typer.Argument(..., help="JSONL of {question, answer, context, human_label}."),
+    metric: str = typer.Option("faithfulness", help="LLM-judged metric to validate."),
+    provider: str = typer.Option("anthropic", help="Judge provider: anthropic or groq."),
+    model: str | None = typer.Option(None, help="Judge model (provider default if omitted)."),
+    threshold: float = typer.Option(
+        0.5, help="Per-sample metric score at or above which the judge label is 'pass'."
+    ),
+    concurrency: int = typer.Option(4, help="Parallel judge calls."),
+    cache_path: Path = typer.Option(Path(".ragcheck_cache.sqlite"), help="Judgment cache."),
+    output: Path = typer.Option(
+        Path("ragcheck_output"), help="Directory for the validation report JSON."
+    ),
+) -> None:
+    """Measure LLM-judge vs. human agreement (Cohen's kappa) on labeled samples."""
+    from rich.table import Table
+
+    from ragcheck.cache import JudgmentCache
+    from ragcheck.judge.judge import Judge
+    from ragcheck.judge.validation import load_labels, validate_judge
+    from ragcheck.llm import build_client
+
+    judge = Judge(build_client(provider, model), JudgmentCache(cache_path))
+    report = validate_judge(
+        load_labels(labels), metric, judge, threshold=threshold, concurrency=concurrency
+    )
+
+    table = Table(title=f"Judge validation - {report.metric_name}")
+    table.add_column("Stat")
+    table.add_column("Value", justify="right")
+    table.add_row("judge model", f"{report.judge_model} ({report.prompt_version})")
+    table.add_row("samples", str(report.n_samples))
+    table.add_row("agreement", f"{report.agreement:.3f}")
+    table.add_row("Cohen's kappa", f"{report.kappa:.3f}")
+    c = report.confusion
+    confusion = f"tp={c['tp']} fp={c['fp']} fn={c['fn']} tn={c['tn']}"
+    table.add_row("confusion (judge vs human)", confusion)
+    console.print(table)
+
+    output.mkdir(parents=True, exist_ok=True)
+    out_path = output / f"judge_validation_{report.metric_name}.json"
+    out_path.write_text(report.model_dump_json(indent=2))
+    console.print(f"[green]Validation report written to[/green] {out_path}")
+
+
+if __name__ == "__main__":
+    app()

ragcheck/config.py

@@ -0,0 +1,54 @@
+"""Eval run configuration: Pydantic models + YAML loading."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+
+class MetricSpec(BaseModel):
+    """A metric selection with optional constructor params."""
+
+    name: str
+    params: dict = Field(default_factory=dict)
+
+
+class EvalConfig(BaseModel):
+    """Everything needed to run one evaluation."""
+
+    dataset: Path
+    adapter: str
+    metrics: list[MetricSpec]
+    judge_provider: str = "anthropic"
+    judge_model: str | None = None
+    concurrency: int = 4
+    html: bool = True
+    judge_validation: Path | None = None  # embed this validation report, if present
+    confirm_above: int = 200  # LLM-judged runs larger than this need assume_yes
+    assume_yes: bool = False
+    output_dir: Path = Path("ragcheck_output")
+    cache_path: Path = Path(".ragcheck_cache.sqlite")
+    run_name: str | None = None
+
+    @field_validator("metrics", mode="before")
+    @classmethod
+    def _coerce_metric_specs(cls, value: object) -> object:
+        """Allow bare metric names in YAML alongside {name, params} mappings."""
+        if isinstance(value, list):
+            return [{"name": v} if isinstance(v, str) else v for v in value]
+        return value
+
+
+def load_config(path: str | Path) -> EvalConfig:
+    """Load an EvalConfig from YAML, resolving paths relative to the file."""
+    path = Path(path)
+    data = yaml.safe_load(path.read_text())
+    config = EvalConfig.model_validate(data)
+    base = path.parent
+    for field in ("dataset", "output_dir", "cache_path"):
+        value: Path = getattr(config, field)
+        if not value.is_absolute():
+            setattr(config, field, base / value)
+    return config

ragcheck/datasets/__init__.py

@@ -0,0 +1,6 @@
+"""Evaluation dataset models and loaders."""
+
+from ragcheck.datasets.loaders import load_dataset
+from ragcheck.datasets.models import EvalDataset, EvalSample, QAPair
+
+__all__ = ["EvalDataset", "EvalSample", "QAPair", "load_dataset"]

ragcheck/datasets/loaders.py

@@ -0,0 +1,35 @@
+"""Load evaluation datasets from JSONL or CSV files."""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+
+from ragcheck.datasets.models import EvalDataset, QAPair
+
+
+def load_dataset(path: str | Path) -> EvalDataset:
+    """Load a dataset from a ``.jsonl`` or ``.csv`` file.
+
+    JSONL: one QAPair object per line. CSV: columns matching QAPair fields;
+    ``relevant_source_ids`` is a semicolon-separated string.
+    """
+    path = Path(path)
+    if path.suffix == ".jsonl":
+        pairs = [
+            QAPair.model_validate(json.loads(line))
+            for line in path.read_text().splitlines()
+            if line.strip()
+        ]
+    elif path.suffix == ".csv":
+        pairs = []
+        with path.open(newline="") as f:
+            for row in csv.DictReader(f):
+                raw_ids = row.pop("relevant_source_ids", "") or ""
+                data: dict[str, object] = {k: v for k, v in row.items() if v not in (None, "")}
+                data["relevant_source_ids"] = [s for s in raw_ids.split(";") if s]
+                pairs.append(QAPair.model_validate(data))
+    else:
+        raise ValueError(f"Unsupported dataset format: {path.suffix} (use .jsonl or .csv)")
+    return EvalDataset(name=path.stem, pairs=pairs)

ragcheck/datasets/models.py

@@ -0,0 +1,32 @@
+"""Dataset data models."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from ragcheck.adapters.base import RAGResponse
+
+
+class QAPair(BaseModel):
+    """A single evaluation question with optional ground truth."""
+
+    question: str
+    ground_truth_answer: str | None = None
+    relevant_source_ids: list[str] = Field(default_factory=list)
+    answerable: bool = True
+    difficulty: str = "medium"
+    paraphrase_group: str | None = None
+
+
+class EvalDataset(BaseModel):
+    """A named collection of QA pairs."""
+
+    name: str = "dataset"
+    pairs: list[QAPair]
+
+
+class EvalSample(BaseModel):
+    """A QA pair joined with the pipeline's response - the unit metrics consume."""
+
+    qa: QAPair
+    response: RAGResponse