ragaudit-cli 0.3.0__py3-none-any.whl

ragaudit/__init__.py

@@ -0,0 +1 @@
+"""ragaudit: audits RAG pipelines by checking whether retrieval did the work."""

ragaudit/adapters/base.py

@@ -0,0 +1,109 @@
+"""The Adapter Protocol and loaders for user-supplied adapter files."""
+
+import importlib.util
+from pathlib import Path
+from types import ModuleType
+from typing import Callable
+
+from ragaudit.models.adapter import Adapter, ConfigurableAdapter, RAGResponse
+
+from .callable_adapter import CallableAdapter
+
+__all__ = [
+    "Adapter",
+    "ConfigurableAdapter",
+    "RAGResponse",
+    "AdapterLoadError",
+    "AdapterFactory",
+    "load_adapter",
+    "load_adapter_factory",
+]
+
+AdapterFactory = Callable[[dict], Adapter]
+"""config dict -> Adapter configured with it."""
+
+
+class AdapterLoadError(Exception):
+    """Raised when an adapter file can't be loaded or doesn't expose an adapter."""
+
+
+def load_adapter(path: str) -> Adapter:
+    """Load an Adapter from a Python file, for the `--adapter` CLI flag.
+
+    The file must define either a module-level `adapter` object satisfying
+    the Adapter Protocol, or a module-level `query(question: str) -> RAGResponse`
+    function, which is wrapped into an Adapter via CallableAdapter.
+    """
+    module = _import_adapter_module(path)
+
+    adapter = getattr(module, "adapter", None)
+    if adapter is not None:
+        return adapter
+
+    query_fn = getattr(module, "query", None)
+    if callable(query_fn):
+        return CallableAdapter(query_fn)
+
+    raise AdapterLoadError(
+        f"Adapter file '{path}' must define a module-level `adapter` object "
+        "or a `query(question: str) -> RAGResponse` function."
+    )
+
+
+def load_adapter_factory(path: str) -> AdapterFactory:
+    """Load an adapter file as a factory that builds a pipeline from a config dict (v2 evaluate/tune).
+
+    Checked in order:
+
+    1. a module-level `make_adapter(config: dict) -> Adapter` function;
+    2. an `adapter` object (or `query` function) whose adapter has a
+       `configure(config: dict)` method, called before returning it;
+    3. a plain v1 adapter, which only accepts an empty config.
+    """
+    module = _import_adapter_module(path)
+
+    make_adapter = getattr(module, "make_adapter", None)
+    if callable(make_adapter):
+        return lambda config: make_adapter(dict(config))
+
+    query_fn = getattr(module, "query", None)
+    if getattr(module, "adapter", None) is not None:
+        base = module.adapter
+    elif callable(query_fn):
+        base = CallableAdapter(query_fn)
+    else:
+        raise AdapterLoadError(
+            f"Adapter file '{path}' must define `make_adapter(config)`, a module-level `adapter` object, "
+            "or a `query(question: str) -> RAGResponse` function."
+        )
+
+    def factory(config: dict) -> Adapter:
+        configure = getattr(base, "configure", None)
+        if callable(configure):
+            configure(dict(config))
+            return base
+        if config:
+            raise AdapterLoadError(
+                f"Adapter file '{path}' doesn't accept a config: define `make_adapter(config)` or give the "
+                "adapter a `configure(config)` method to use --config or `ragaudit tune`."
+            )
+        return base
+
+    return factory
+
+
+def _import_adapter_module(path: str) -> ModuleType:
+    file_path = Path(path)
+    if not file_path.is_file():
+        raise AdapterLoadError(f"Adapter file not found: {path}")
+
+    spec = importlib.util.spec_from_file_location(file_path.stem, file_path)
+    if spec is None or spec.loader is None:
+        raise AdapterLoadError(f"Could not load adapter module from: {path}")
+
+    module = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(module)
+    except Exception as exc:
+        raise AdapterLoadError(f"Adapter file raised an error on import: {path}") from exc
+    return module

ragaudit/adapters/callable_adapter.py

@@ -0,0 +1,15 @@
+"""Wraps a plain callable into the Adapter Protocol."""
+
+from typing import Callable
+
+from ragaudit.models.adapter import RAGResponse
+
+
+class CallableAdapter:
+    """Adapts a plain `query(question: str) -> RAGResponse` callable to Adapter."""
+
+    def __init__(self, fn: Callable[[str], RAGResponse]) -> None:
+        self._fn = fn
+
+    def query(self, question: str) -> RAGResponse:
+        return self._fn(question)

ragaudit/cli.py

@@ -0,0 +1,540 @@
+"""The ragaudit CLI: ingest -> generate -> run -> report.
+
+This module only wires together already-implemented modules (loading,
+chunking, generation, running, ablation, scoring aggregation, storage,
+and reporting) behind a Typer app. It contains no scoring or business
+logic of its own.
+
+v2 adds `generate --corpus`, `evaluate` and `tune`; their bodies live in
+cli_v2.py. Without --corpus, `generate` runs exactly the v1 code path.
+"""
+
+import functools
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from ragaudit import cli_easy, cli_v2
+from ragaudit.adapters.base import AdapterLoadError, load_adapter
+from ragaudit.generate.pipeline import (
+    DEFAULT_LEXICAL_THRESHOLD,
+    DEFAULT_MAX_FACTS_PER_CHUNK,
+    DEFAULT_MULTI_HOP_PAIRS,
+)
+from ragaudit.generate.testset_generator import generate_test_set
+from ragaudit.ingest import chunker
+from ragaudit.ingest.loader import load_documents as load_documents_from_disk
+from ragaudit.llm.client import LLMClient, LLMClientError
+from ragaudit.models.run import Run, RunConfig
+from ragaudit.models.scoring import Verdict
+from ragaudit.report.generator import generate_report
+from ragaudit.runner.ablation import run_ablation
+from ragaudit.runner.evaluate import DEFAULT_K, DEFAULT_REGRESSION_SAMPLE
+from ragaudit.runner.executor import run_test_set, score_by_id
+from ragaudit.runner.summary import compute_run_summary
+from ragaudit.scoring.correctness import judge_correctness
+from ragaudit.scoring.groundedness import judge_groundedness
+from ragaudit.scoring.retrieval import score_retrieval_by_text
+from ragaudit.scoring.text_match import DEFAULT_MATCH_THRESHOLD, validate_match_by
+from ragaudit.storage import db as db_module
+from ragaudit.storage import repository
+from ragaudit.env import load_dotenv
+from ragaudit.scaffold import DEFAULT_ADAPTER_PATH
+from ragaudit.storage.store import DEFAULT_STORE_PATH
+from ragaudit.tune.loop import DEFAULT_MAX_TRIALS, DEFAULT_OVERFIT_THRESHOLD
+
+app = typer.Typer(help="Audits RAG pipelines by checking whether retrieval actually did the work.")
+
+DB_OPTION = typer.Option(Path("ragaudit.db"), "--db", help="SQLite database path.")
+STORE_OPTION = typer.Option(DEFAULT_STORE_PATH, "--store", help="v2 store path.")
+LLM_OPTION = typer.Option(
+    None, "--llm", help="v2 LLM provider: openai (default, RAGAUDIT_API_*) or mock (offline). Env: RAGAUDIT_LLM."
+)
+
+NO_CONTEXT_SYSTEM_PROMPT = (
+    "Answer the question using only your own internal knowledge. You have not "
+    "been given any retrieved context or documents to work from."
+)
+
+
+def _error_exit(message: str) -> None:
+    """Print a clean, colored error to stderr and exit with status 1."""
+    typer.secho(f"Error: {message}", fg=typer.colors.RED, err=True)
+    raise typer.Exit(code=1)
+
+
+def _cli_safe(func):
+    """Convert any exception that escapes a command into a clean error, never a stack trace."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except typer.Exit:
+            raise
+        except AdapterLoadError as exc:
+            _error_exit(str(exc))
+        except LLMClientError as exc:
+            _error_exit(f"LLM request failed: {exc}")
+        except KeyError as exc:
+            _error_exit(f"missing required environment variable: {exc}")
+        except Exception as exc:  # last-resort safety net
+            _error_exit(f"unexpected error: {exc}")
+
+    return wrapper
+
+
+@app.callback()
+def _load_settings() -> None:
+    # RAGAUDIT_* settings can live in ./.env (see `ragaudit init`); exported variables win.
+    load_dotenv()
+
+
+@app.command()
+@_cli_safe
+def init(
+    framework: str = typer.Option(
+        "plain", "--framework",
+        help="Adapter template: plain, langchain, llamaindex, or demo (a ready-to-run toy pipeline, no edits needed).",
+    ),
+    out: Path = typer.Option(DEFAULT_ADAPTER_PATH, "--out", help="Where to write the adapter template."),
+    force: bool = typer.Option(False, "--force", help="Overwrite existing files."),
+    docs: Path = typer.Option(
+        cli_easy.DEFAULT_DEMO_DOCS, "--docs", help="demo only: the docs folder the demo pipeline searches."
+    ),
+) -> None:
+    """Write a starter adapter (one TODO: plug in your pipeline) and a .env.example. --framework demo needs no edits."""
+    cli_easy.run_init(framework, out, force, docs)
+
+
+@app.command()
+@_cli_safe
+def doctor(
+    adapter: Path = typer.Option(DEFAULT_ADAPTER_PATH, "--adapter", help="Adapter file to check."),
+    llm: Optional[str] = LLM_OPTION,
+) -> None:
+    """Check your setup: Python, RAGAUDIT_* settings, one LLM call, and the adapter. Prints a fix for each failure."""
+    cli_easy.run_doctor(adapter, llm)
+
+
+@app.command()
+@_cli_safe
+def ingest(
+    path: Path = typer.Argument(..., help="Directory containing .txt/.md documents to ingest."),
+    db: Path = DB_OPTION,
+    chunk_size: int = typer.Option(chunker.DEFAULT_CHUNK_SIZE, "--chunk-size", help="Characters per chunk."),
+    overlap: int = typer.Option(chunker.DEFAULT_OVERLAP, "--overlap", help="Character overlap between chunks."),
+) -> None:
+    """Load documents from PATH, chunk them, and persist to the database."""
+    if not path.exists():
+        _error_exit(f"path not found: {path}")
+
+    documents = load_documents_from_disk(path)
+    if not documents:
+        _error_exit(f"no .txt/.md documents found under {path}")
+
+    all_chunks = []
+    with typer.progressbar(documents, label="Chunking documents") as progress:
+        for document in progress:
+            all_chunks.extend(chunker.chunk_document(document, chunk_size=chunk_size, overlap=overlap))
+
+    conn = db_module.get_connection(db)
+    try:
+        repository.save_documents(conn, documents)
+        repository.save_chunks(conn, all_chunks)
+    finally:
+        conn.close()
+
+    typer.echo(f"Ingested {len(documents)} document(s) into {len(all_chunks)} chunk(s). Saved to {db}")
+
+
+@app.command()
+@_cli_safe
+def generate(
+    n: Optional[int] = typer.Option(
+        None,
+        "--n",
+        help="Number of questions to generate per chunk (default 1). With --corpus: max facts per chunk (default 3).",
+    ),
+    db: Path = DB_OPTION,
+    corpus: Optional[Path] = typer.Option(
+        None,
+        "--corpus",
+        help="v2: build a verified, de-worded, dev/test-split question set from this .jsonl file or directory.",
+    ),
+    store: Path = STORE_OPTION,
+    llm: Optional[str] = LLM_OPTION,
+    generator_model: Optional[str] = typer.Option(None, "--generator-model", help="v2: generator model."),
+    verifier_model: Optional[str] = typer.Option(
+        None, "--verifier-model", help="v2: verifier model; must differ from the generator."
+    ),
+    max_chunks: Optional[int] = typer.Option(None, "--max-chunks", help="v2: process at most this many chunks."),
+    max_llm_calls: Optional[int] = typer.Option(None, "--max-llm-calls", help="v2: stop after this many LLM calls."),
+    multi_hop: bool = typer.Option(False, "--multi-hop", help="v2: also write questions that need two chunks."),
+    multi_hop_pairs: int = typer.Option(
+        DEFAULT_MULTI_HOP_PAIRS, "--multi-hop-pairs", help="v2: max chunk pairs for multi-hop per run."
+    ),
+    lexical_threshold: float = typer.Option(
+        DEFAULT_LEXICAL_THRESHOLD,
+        "--lexical-threshold",
+        help="v2: max share of question content words found in the chunk before a rewrite is rejected.",
+    ),
+    retry_processed: bool = typer.Option(
+        False, "--retry-processed", help="v2: re-process chunks that previously yielded no active question."
+    ),
+) -> None:
+    """Generate a test set from ingested chunks and persist it."""
+    if corpus is not None:
+        cli_v2.run_generate(
+            corpus=corpus,
+            store_path=store,
+            max_facts_per_chunk=n if n is not None else DEFAULT_MAX_FACTS_PER_CHUNK,
+            llm=llm,
+            generator_model=generator_model,
+            verifier_model=verifier_model,
+            max_chunks=max_chunks,
+            max_llm_calls=max_llm_calls,
+            multi_hop=multi_hop,
+            multi_hop_pairs=multi_hop_pairs,
+            lexical_threshold=lexical_threshold,
+            retry_processed=retry_processed,
+        )
+        return
+
+    n = 1 if n is None else n
+    if not db.exists():
+        _error_exit(f"database not found at {db} — run `ragaudit ingest <path>` first.")
+
+    conn = db_module.get_connection(db)
+    try:
+        documents = repository.load_documents(conn)
+        chunks = repository.load_chunks(conn)
+        if not documents or not chunks:
+            _error_exit("no ingested documents found — run `ragaudit ingest <path>` first.")
+
+        client = LLMClient()
+        typer.echo(
+            f"Generating {n} question(s) per chunk for {len(chunks)} chunk(s) using {client.model} "
+            "(this may take a while)..."
+        )
+        test_set = generate_test_set(client, documents, chunks, n_per_chunk=n)
+        repository.save_test_set(conn, test_set)
+    finally:
+        conn.close()
+
+    typer.echo(
+        f"Generated {len(test_set.test_cases)} test case(s) from {len(chunks)} chunk(s). "
+        f"Saved test set '{test_set.testset_id}' to {db}"
+    )
+
+
+MATCH_BY_OPTION = typer.Option(
+    "id",
+    "--match-by",
+    help="How a retrieval hit is decided: id (the adapter returns ragaudit's chunk ids; default) or text "
+    "(a returned context overlaps the gold chunk's text; works without re-indexing your pipeline).",
+)
+MATCH_THRESHOLD_OPTION = typer.Option(
+    DEFAULT_MATCH_THRESHOLD, "--match-threshold", help="--match-by text: minimum overlap (0-1) to count as a hit."
+)
+
+
+@app.command()
+@_cli_safe
+def evaluate(
+    adapter: Path = typer.Option(..., "--adapter", help="Adapter file: `make_adapter(config)`, `adapter`, or `query`."),
+    corpus: Optional[Path] = typer.Option(
+        None, "--corpus", help="Current corpus, to detect new/changed/deleted chunks (default: last recorded)."
+    ),
+    config: Optional[Path] = typer.Option(None, "--config", help="YAML pipeline config passed to the adapter."),
+    split: str = typer.Option("dev", "--split", help="dev (default) or test. Test goes through the locked path."),
+    k: int = typer.Option(DEFAULT_K, "--k", help="Score retrieval over the top k retrieved chunk ids."),
+    regression_sample: int = typer.Option(
+        DEFAULT_REGRESSION_SAMPLE,
+        "--regression-sample",
+        help="When only the corpus changed: old questions to re-run alongside all new ones.",
+    ),
+    seed: int = typer.Option(0, "--seed", help="Seed for the regression sample."),
+    full: bool = typer.Option(False, "--full", help="Ignore regression sampling and cover every active question."),
+    ablation: bool = typer.Option(False, "--ablation", help="Re-ask lucky passes with no context (v1 ablation)."),
+    store: Path = STORE_OPTION,
+    llm: Optional[str] = LLM_OPTION,
+    judge_model: Optional[str] = typer.Option(None, "--judge-model", help="Correctness judge model."),
+    match_by: str = MATCH_BY_OPTION,
+    match_threshold: float = MATCH_THRESHOLD_OPTION,
+) -> None:
+    """v2: evaluate a pipeline on generated questions, with caching and regression sampling."""
+    cli_v2.run_evaluate(
+        adapter_path=adapter,
+        corpus=corpus,
+        config_path=config,
+        split=split,
+        k=k,
+        regression_sample=regression_sample,
+        seed=seed,
+        store_path=store,
+        llm=llm,
+        judge_model=judge_model,
+        ablation=ablation,
+        full=full,
+        match_by=match_by,
+        match_threshold=match_threshold,
+    )
+
+
+@app.command()
+@_cli_safe
+def tune(
+    adapter: Path = typer.Option(..., "--adapter", help="Adapter file exposing `make_adapter(config)`."),
+    search_space: Path = typer.Option(..., "--search-space", help="YAML file of knobs and their allowed values."),
+    corpus: Optional[Path] = typer.Option(None, "--corpus", help="Current corpus (default: last recorded)."),
+    strategy: str = typer.Option("grid", "--strategy", help="grid, random (seeded) or llm."),
+    max_trials: int = typer.Option(DEFAULT_MAX_TRIALS, "--max-trials", help="Max configs to evaluate, baseline included."),
+    max_llm_calls: Optional[int] = typer.Option(
+        None, "--max-llm-calls", help="Cap on judge + proposer calls; room for the final test evaluation is reserved."
+    ),
+    seed: int = typer.Option(0, "--seed", help="Seed for the random strategy and fallbacks."),
+    k: int = typer.Option(DEFAULT_K, "--k", help="Score retrieval over the top k retrieved chunk ids."),
+    overfit_threshold: float = typer.Option(
+        DEFAULT_OVERFIT_THRESHOLD, "--overfit-threshold", help="Warn when best dev minus best test exceeds this (points)."
+    ),
+    out: Path = typer.Option(Path("best_config.yaml"), "--out", help="Where to write the best config (YAML)."),
+    store: Path = STORE_OPTION,
+    llm: Optional[str] = LLM_OPTION,
+    judge_model: Optional[str] = typer.Option(None, "--judge-model", help="Correctness judge model."),
+    proposer_model: Optional[str] = typer.Option(None, "--proposer-model", help="Model for --strategy llm."),
+) -> None:
+    """v2: search pipeline configs on the dev split; report baseline vs best on the locked test split."""
+    cli_v2.run_tune_command(
+        adapter_path=adapter,
+        search_space_path=search_space,
+        corpus=corpus,
+        strategy=strategy,
+        max_trials=max_trials,
+        max_llm_calls=max_llm_calls,
+        seed=seed,
+        k=k,
+        overfit_threshold=overfit_threshold,
+        out=out,
+        store_path=store,
+        llm=llm,
+        judge_model=judge_model,
+        proposer_model=proposer_model,
+    )
+
+
+def _text_retrieval_scorer(chunk_texts: dict[str, str], threshold: float):
+    def scorer(test_case, response):
+        gold_text = chunk_texts.get(test_case.source_chunk_id)
+        if gold_text is None:
+            raise ValueError(f"chunk {test_case.source_chunk_id} is not in the database; re-run `ragaudit ingest`.")
+        return score_retrieval_by_text(
+            test_case.source_chunk_id, gold_text, response.retrieved_chunk_ids, response.contexts, threshold
+        )
+
+    return scorer
+
+
+def _execute_run(
+    conn,
+    test_set,
+    adapter_path: Path,
+    client,
+    *,
+    run_ablation_flag: bool,
+    match_by: str,
+    match_threshold: float,
+) -> Run:
+    """Run, score, ablate and save one Run. Shared by `run` and `check`."""
+    loaded_adapter = load_adapter(str(adapter_path))
+
+    def correctness_judge(question: str, expected_answer: str, candidate_answer: str):
+        return judge_correctness(client, question, expected_answer, candidate_answer)
+
+    def groundedness_judge(answer: str, contexts: list[str]):
+        return judge_groundedness(client, answer, contexts)
+
+    if match_by == "text":
+        chunk_texts = {chunk.chunk_id: chunk.text for chunk in repository.load_chunks(conn)}
+        retrieval_scorer = _text_retrieval_scorer(chunk_texts, match_threshold)
+        typer.echo(f"Matching retrieval by text (threshold {match_threshold:g}), using the contexts the adapter returns.")
+    else:
+        retrieval_scorer = score_by_id
+
+    typer.echo(f"Running {len(test_set.test_cases)} test case(s) through adapter {adapter_path}...")
+    scored_cases = run_test_set(test_set, loaded_adapter, correctness_judge, groundedness_judge, retrieval_scorer)
+
+    if match_by == "text" and scored_cases and not any(case.response.contexts for case in scored_cases):
+        typer.secho(
+            "Warning: --match-by text needs the retrieved passages, but the adapter returned no `contexts` for any "
+            "question, so every case counts as a retrieval miss. Return the retrieved texts in RAGResponse.contexts.",
+            fg=typer.colors.YELLOW,
+        )
+
+    if run_ablation_flag:
+        lucky_pass_count = sum(1 for case in scored_cases if case.verdict == Verdict.LUCKY_PASS)
+        if lucky_pass_count:
+            typer.echo(f"Ablation: enabled — running against {lucky_pass_count} lucky-pass candidate(s)...")
+
+            def no_context_query(question: str) -> str:
+                return client.complete(NO_CONTEXT_SYSTEM_PROMPT, question)
+
+            def check_correctness(question: str, expected_answer: str, candidate_answer: str) -> bool:
+                return judge_correctness(client, question, expected_answer, candidate_answer).is_correct
+
+            scored_cases = run_ablation(scored_cases, test_set.test_cases, no_context_query, check_correctness)
+        else:
+            typer.echo("Ablation: enabled, but no lucky-pass candidates were found — nothing to ablate.")
+    else:
+        typer.echo("Ablation: disabled (--no-ablation).")
+
+    summary = compute_run_summary(scored_cases)
+    run = Run(
+        config=RunConfig(
+            run_id=str(uuid.uuid4()),
+            testset_id=test_set.testset_id,
+            adapter_path=str(adapter_path),
+            correctness_judge_model=client.model,
+            groundedness_judge_model=client.model,
+            run_ablation=run_ablation_flag,
+            started_at=datetime.now(timezone.utc),
+        ),
+        finished_at=datetime.now(timezone.utc),
+        scored_cases=scored_cases,
+        summary=summary,
+    )
+    repository.save_run(conn, run)
+    return run
+
+
+@app.command(name="run")
+@_cli_safe
+def run_cmd(
+    adapter: Path = typer.Option(..., "--adapter", help="Path to a Python file exposing `query` or `adapter`."),
+    db: Path = DB_OPTION,
+    testset_id: Optional[str] = typer.Option(
+        None, "--testset-id", help="Test set id to run (defaults to the most recently generated)."
+    ),
+    no_ablation: bool = typer.Option(False, "--no-ablation", help="Skip the retrieval-disabled ablation step."),
+    match_by: str = MATCH_BY_OPTION,
+    match_threshold: float = MATCH_THRESHOLD_OPTION,
+) -> None:
+    """Run the adapter over a test set, score it, ablate lucky passes, and persist the Run."""
+    try:
+        validate_match_by(match_by)
+    except ValueError as exc:
+        _error_exit(str(exc))
+    if not db.exists():
+        _error_exit(f"database not found at {db} — run `ragaudit ingest` and `ragaudit generate` first.")
+
+    conn = db_module.get_connection(db)
+    try:
+        resolved_testset_id = testset_id or repository.get_latest_testset_id(conn)
+        if resolved_testset_id is None:
+            _error_exit("no test set found — run `ragaudit generate` first.")
+
+        test_set = repository.load_test_set(conn, resolved_testset_id)
+        if test_set is None:
+            _error_exit(f"test set '{resolved_testset_id}' not found.")
+
+        try:
+            load_adapter(str(adapter))
+        except AdapterLoadError as exc:
+            _error_exit(str(exc))
+
+        run = _execute_run(
+            conn,
+            test_set,
+            adapter,
+            LLMClient(),
+            run_ablation_flag=not no_ablation,
+            match_by=match_by,
+            match_threshold=match_threshold,
+        )
+    finally:
+        conn.close()
+
+    typer.echo(
+        f"Run '{run.config.run_id}' complete: {run.summary.total_cases} case(s), "
+        f"lucky-pass rate {run.summary.lucky_pass_rate:.1%}. Saved to {db}"
+    )
+
+
+@app.command()
+@_cli_safe
+def check(
+    docs: Path = typer.Option(..., "--docs", help="Folder of .txt/.md documents your pipeline answers from."),
+    adapter: Path = typer.Option(DEFAULT_ADAPTER_PATH, "--adapter", help="Adapter file (see `ragaudit init`)."),
+    questions: int = typer.Option(
+        cli_easy.DEFAULT_CHECK_QUESTIONS, "--questions", help="How many test questions to write in total."
+    ),
+    match_by: str = typer.Option(
+        "text", "--match-by", help="text (default: compare returned contexts with the source passage) or id."
+    ),
+    match_threshold: float = MATCH_THRESHOLD_OPTION,
+    yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt."),
+    no_open: bool = typer.Option(False, "--no-open", help="Don't open the report in the browser."),
+    max_llm_calls: Optional[int] = typer.Option(
+        None, "--max-llm-calls", help="Hard cap on LLM calls (default: the estimate's upper bound)."
+    ),
+    db: Path = typer.Option(cli_easy.DEFAULT_CHECK_DB, "--db", help="Working database, recreated on every check."),
+    out: Path = typer.Option(cli_easy.DEFAULT_CHECK_REPORT, "--out", help="Where to write the HTML report."),
+    llm: Optional[str] = LLM_OPTION,
+) -> None:
+    """Test your RAG end to end: write questions from your docs, run your adapter, score it, open the report."""
+    cli_easy.run_check(
+        docs, adapter, questions, match_by, match_threshold, yes, no_open, max_llm_calls, db, out, llm
+    )
+
+
+@app.command()
+@_cli_safe
+def report(
+    out: Optional[Path] = typer.Option(
+        None, "--out", help="Output HTML file path (default report.html; generate-report.html / tune-report.html for v2 kinds)."
+    ),
+    db: Path = DB_OPTION,
+    run_id: Optional[str] = typer.Option(
+        None, "--run-id", help="Run id to report on (defaults to the most recently run). For v2 kinds, the generate run or tune id."
+    ),
+    kind: str = typer.Option(
+        "run", "--kind", help="run (v1 run report, the default), generate (v2 generate run) or tune (v2 tune run)."
+    ),
+    store: Path = STORE_OPTION,
+) -> None:
+    """Render the latest (or specified) Run into a static HTML report. v2: --kind generate|tune."""
+    if kind in ("generate", "tune"):
+        output_path = cli_v2.run_report(kind, store, out or Path(f"{kind}-report.html"), run_id)
+        typer.echo(f"Report written to {output_path}")
+        return
+    if kind != "run":
+        _error_exit(f"--kind must be run, generate or tune, got {kind!r}")
+    out = out or Path("report.html")
+    if not db.exists():
+        _error_exit(f"database not found at {db} — run `ragaudit ingest`, `generate`, and `run` first.")
+
+    conn = db_module.get_connection(db)
+    try:
+        resolved_run_id = run_id or repository.get_latest_run_id(conn)
+        if resolved_run_id is None:
+            _error_exit("no run found — run `ragaudit run --adapter <path>` first.")
+
+        run = repository.load_run(conn, resolved_run_id)
+        if run is None:
+            _error_exit(f"run '{resolved_run_id}' not found.")
+
+        test_set = repository.load_test_set(conn, run.config.testset_id)
+        if test_set is None:
+            _error_exit(f"test set '{run.config.testset_id}' referenced by run '{resolved_run_id}' not found.")
+    finally:
+        conn.close()
+
+    output_path = generate_report(run, test_set, out)
+    typer.echo(f"Report written to {output_path}")
+
+
+if __name__ == "__main__":
+    app()