ragaudit-cli 0.3.0__py3-none-any.whl

ragaudit/scaffold.py

@@ -0,0 +1,252 @@
+"""`ragaudit init`: starter files for plugging a pipeline into ragaudit.
+
+Each adapter template has exactly one TODO, at the spot where the user
+connects their own pipeline; everything around it (building the
+RAGResponse, timing, a clear error while the TODO is unfilled) is done.
+
+The `demo` template is the exception: a complete toy pipeline (keyword
+search over the docs folder, answered by the configured LLM) that works
+with no edits, for people who want to see a report before wiring up
+their own pipeline.
+"""
+
+import os
+from pathlib import Path
+
+FRAMEWORKS = ("demo", "plain", "langchain", "llamaindex")
+DEMO_DOCS_ENV = "RAGAUDIT_DEMO_DOCS"
+"""`ragaudit check` sets this so the demo adapter searches the same folder it writes questions from."""
+DEFAULT_DEMO_DOCS = Path("docs")
+DEFAULT_ADAPTER_PATH = Path("ragaudit_adapter.py")
+ENV_EXAMPLE_NAME = ".env.example"
+
+_HEADER = '''"""ragaudit adapter: connects ragaudit to your RAG pipeline ({framework}).
+
+ragaudit calls query(question) once per test question and scores what you return:
+
+- answer:   your pipeline's final answer.
+- contexts: the text of each passage your pipeline retrieved, best first.
+            `ragaudit check` (and --match-by text) decides whether retrieval
+            found the right passage by comparing these texts with the
+            passage each question was written from, so your pipeline can
+            keep its own chunking and ids.
+- retrieved_chunk_ids: only used with --match-by id, when your index was
+            built from ragaudit's own chunks. Fine to leave empty.
+
+Check it with:  ragaudit doctor --adapter {filename}
+Then run:       ragaudit check --docs ./your-docs --adapter {filename}
+"""
+
+import time
+
+from ragaudit.models.adapter import RAGResponse
+'''
+
+_PLAIN = '''
+
+def ask_my_pipeline(question: str) -> tuple[str, list[str]]:
+    """Return (answer, texts of the retrieved passages) for one question."""
+    # TODO: call your pipeline here and return its answer and retrieved passages, e.g.
+    #     result = my_rag.ask(question)
+    #     return result.answer, [source.text for source in result.sources]
+    raise NotImplementedError("{filename} is not filled in yet: connect your pipeline at the TODO in ask_my_pipeline().")
+
+
+def query(question: str) -> RAGResponse:
+    started = time.perf_counter()
+    answer, contexts = ask_my_pipeline(question)
+    return RAGResponse(
+        answer=str(answer),
+        contexts=[str(text) for text in contexts],
+        latency_ms=(time.perf_counter() - started) * 1000,
+    )
+'''
+
+_LANGCHAIN = '''
+# Your pipeline's retriever and chat model. Any LangChain retriever works
+# (vectorstore.as_retriever(), a hybrid or multi-query retriever, ...), and
+# any chat model or LLM (ChatOpenAI, ChatOllama, ...).
+# TODO: build or import your own retriever and llm here, e.g.
+#     from my_app.rag import vectorstore, llm
+#     retriever = vectorstore.as_retriever(search_kwargs={{"k": 4}})
+retriever = None
+llm = None
+
+# The prompt your pipeline uses to answer from the retrieved context. If
+# you already have a chain, copy its prompt here so ragaudit tests the
+# same thing your users get.
+PROMPT = (
+    "Answer the question using only the context below. If the context does not "
+    "contain the answer, say you don't know.\\n\\nContext:\\n{{context}}\\n\\nQuestion: {{question}}"
+)
+
+
+def query(question: str) -> RAGResponse:
+    if retriever is None or llm is None:
+        raise NotImplementedError("{filename} is not filled in yet: set `retriever` and `llm` at the TODO.")
+    started = time.perf_counter()
+    docs = retriever.invoke(question)
+    contexts = [doc.page_content for doc in docs]
+    reply = llm.invoke(PROMPT.format(context="\\n\\n".join(contexts), question=question))
+    answer = getattr(reply, "content", reply)  # chat models return a message, plain LLMs a string
+    return RAGResponse(
+        answer=str(answer),
+        contexts=contexts,
+        retrieved_chunk_ids=[str(doc.metadata["chunk_id"]) for doc in docs if "chunk_id" in doc.metadata],
+        latency_ms=(time.perf_counter() - started) * 1000,
+    )
+'''
+
+_LLAMAINDEX = '''
+# Your pipeline's query engine: index.as_query_engine(), a RetrieverQueryEngine,
+# or anything whose .query(question) returns a Response with source_nodes.
+# TODO: build or import your own query engine here, e.g.
+#     from my_app.rag import index
+#     query_engine = index.as_query_engine(similarity_top_k=4)
+query_engine = None
+
+
+def query(question: str) -> RAGResponse:
+    if query_engine is None:
+        raise NotImplementedError("{filename} is not filled in yet: set `query_engine` at the TODO.")
+    started = time.perf_counter()
+    response = query_engine.query(question)
+    nodes = [scored.node for scored in (response.source_nodes or [])]
+    return RAGResponse(
+        answer=str(response),
+        contexts=[node.get_content() for node in nodes],
+        retrieved_chunk_ids=[str(node.metadata["chunk_id"]) for node in nodes if "chunk_id" in node.metadata],
+        latency_ms=(time.perf_counter() - started) * 1000,
+    )
+'''
+
+_DEMO = r'''"""ragaudit demo adapter: a tiny, complete RAG pipeline that works with no edits.
+
+Retrieval: every .txt/.md file under the docs folder is split into
+paragraphs, and the 3 paragraphs sharing the most words with the
+question are returned. Generation: the configured ragaudit LLM
+(RAGAUDIT_API_BASE / RAGAUDIT_API_KEY / RAGAUDIT_MODEL, or --llm mock)
+answers from those paragraphs.
+
+It exists to show what a ragaudit report looks like. Keyword search is
+deliberately simple, so expect some retrieval failures and lucky passes.
+To test your own pipeline: ragaudit init --framework plain --force
+
+Run it:  ragaudit check --docs __DOCS__ --adapter __FILENAME__
+"""
+
+import os
+import re
+import time
+from pathlib import Path
+
+from ragaudit.llm.factory import make_client
+from ragaudit.models.adapter import RAGResponse
+
+USES_RAGAUDIT_LLM = True  # tells `ragaudit check` that each answer costs one call to its LLM
+TOP_K = 3
+
+# `ragaudit check --docs X` points this at X; otherwise the folder below
+# (relative to this file) is used.
+DOCS = Path(os.environ.get("__DOCS_ENV__") or Path(__file__).parent / "__DOCS__")
+
+SYSTEM_PROMPT = (
+    "Answer the question using only the context. If the context does not contain the answer, "
+    "say you don't know. Answer in one or two sentences."
+)
+STOPWORDS = {
+    "a", "an", "and", "are", "as", "at", "be", "by", "does", "for", "from", "how", "in", "is", "it",
+    "of", "on", "or", "that", "the", "this", "to", "was", "what", "when", "where", "which", "who",
+    "why", "with",
+}
+
+_paragraphs: list[str] | None = None
+_llm = None
+
+
+def words(text: str) -> set[str]:
+    return {word for word in re.findall(r"[a-z0-9]+", text.lower()) if word not in STOPWORDS}
+
+
+def paragraphs() -> list[str]:
+    """Every paragraph of every .txt/.md file under DOCS, read once."""
+    global _paragraphs
+    if _paragraphs is None:
+        if not DOCS.is_dir():
+            raise FileNotFoundError(f"docs folder not found: {DOCS}. Put your .txt/.md files there.")
+        files = sorted(p for p in DOCS.rglob("*") if p.suffix.lower() in (".txt", ".md") and p.is_file())
+        _paragraphs = [
+            " ".join(block.split())
+            for path in files
+            for block in re.split(r"\n\s*\n", path.read_text(encoding="utf-8", errors="replace"))
+            if block.strip()
+        ]
+    return _paragraphs
+
+
+def retrieve(question: str, k: int = TOP_K) -> list[str]:
+    asked = words(question)
+    scored = [(len(asked & words(text)), -i, text) for i, text in enumerate(paragraphs())]
+    return [text for _, _, text in sorted(scored, reverse=True)[:k]]  # like any search: always the top k
+
+
+def query(question: str) -> RAGResponse:
+    global _llm
+    started = time.perf_counter()
+    contexts = retrieve(question)
+    if _llm is None:
+        _llm = make_client(mock_model="mock-answerer")
+    context_block = "\n---\n".join(contexts) or "(no context retrieved)"
+    answer = _llm.complete(SYSTEM_PROMPT, f"Context:\n{context_block}\n\nQuestion: {question}")
+    return RAGResponse(answer=answer.strip(), contexts=contexts, latency_ms=(time.perf_counter() - started) * 1000)
+'''
+
+_BODIES = {"plain": _PLAIN, "langchain": _LANGCHAIN, "llamaindex": _LLAMAINDEX}
+
+ENV_EXAMPLE = """# ragaudit reads these settings from the environment, or from a .env file in
+# the folder you run it from. Copy this file to .env and keep ONE block:
+#     cp .env.example .env
+# .env holds your API key: keep it out of git.
+
+# Option A: OpenAI, or any OpenAI-compatible API (Azure, Together, Groq, vLLM, ...).
+RAGAUDIT_API_BASE=https://api.openai.com/v1
+RAGAUDIT_API_KEY=sk-your-key-here
+RAGAUDIT_MODEL=gpt-4o-mini
+
+# Option B: local Ollama. Free, and nothing leaves your machine.
+# Install Ollama, run `ollama pull llama3.1:8b`, then use these instead of Option A:
+# RAGAUDIT_API_BASE=http://localhost:11434/v1
+# RAGAUDIT_API_KEY=ollama
+# RAGAUDIT_MODEL=llama3.1:8b
+"""
+
+
+def adapter_template(framework: str, filename: str, docs: str = str(DEFAULT_DEMO_DOCS)) -> str:
+    """The adapter file's text. `docs` (demo only) is the docs folder, relative to the adapter's folder."""
+    if framework not in FRAMEWORKS:
+        raise ValueError(f"--framework must be one of {', '.join(FRAMEWORKS)}, got {framework!r}")
+    if framework == "demo":
+        return (
+            _DEMO.replace("__DOCS_ENV__", DEMO_DOCS_ENV)
+            .replace("__FILENAME__", filename)
+            .replace("__DOCS__", Path(docs).as_posix())
+        )
+    label = {"plain": "plain Python", "langchain": "LangChain", "llamaindex": "LlamaIndex"}[framework]
+    return _HEADER.format(framework=label, filename=filename) + _BODIES[framework].format(filename=filename)
+
+
+def write_starter_files(
+    framework: str, adapter_path: Path, force: bool = False, docs: Path = DEFAULT_DEMO_DOCS
+) -> list[Path]:
+    """Write the adapter template and .env.example (next to it). Refuses to overwrite unless force."""
+    env_path = adapter_path.parent / ENV_EXAMPLE_NAME
+    docs_from_adapter = os.path.relpath(docs.resolve(), adapter_path.parent.resolve())
+    content = adapter_template(framework, adapter_path.name, docs_from_adapter)
+    existing = [path for path in (adapter_path, env_path) if path.exists()]
+    if existing and not force:
+        names = ", ".join(str(path) for path in existing)
+        raise FileExistsError(f"{names} already exists; pass --force to overwrite, or --out to pick another file.")
+    adapter_path.parent.mkdir(parents=True, exist_ok=True)
+    adapter_path.write_text(content)
+    env_path.write_text(ENV_EXAMPLE)
+    return [adapter_path, env_path]

ragaudit/scoring/correctness.py

@@ -0,0 +1,94 @@
+"""LLM judge for answer correctness against the expected answer."""
+
+import json
+import logging
+from typing import Protocol
+
+from ragaudit.models.scoring import CorrectnessResult
+
+logger = logging.getLogger(__name__)
+
+SYSTEM_PROMPT = (
+    "You are judging whether a candidate answer correctly answers a question, "
+    "given a reference (expected) answer. Respond with strict JSON only, no "
+    "prose, no markdown fences, in the form: "
+    '{"is_correct": bool, "score": float between 0 and 1, "rationale": str}'
+)
+
+
+class Completer(Protocol):
+    """The subset of LLMClient this module depends on — easy to stub in tests."""
+
+    model: str
+
+    def complete(self, system: str, user: str) -> str: ...
+
+
+def _build_user_prompt(question: str, expected_answer: str, candidate_answer: str) -> str:
+    return (
+        f"Question: {question}\n"
+        f"Expected answer: {expected_answer}\n"
+        f"Candidate answer: {candidate_answer}\n\n"
+        "Is the candidate answer correct?"
+    )
+
+
+def parse_correctness_response(raw: str) -> dict | None:
+    """Parse the judge's JSON response. Returns None on any malformed input."""
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(data, dict):
+        return None
+
+    is_correct = data.get("is_correct")
+    if not isinstance(is_correct, bool):
+        return None
+
+    score = data.get("score")
+    if score is not None and not isinstance(score, (int, float)):
+        return None
+
+    rationale = data.get("rationale")
+    if rationale is not None and not isinstance(rationale, str):
+        return None
+
+    return {
+        "is_correct": is_correct,
+        "score": float(score) if score is not None else None,
+        "rationale": rationale,
+    }
+
+
+def judge_correctness(
+    client: Completer,
+    question: str,
+    expected_answer: str,
+    candidate_answer: str,
+) -> CorrectnessResult:
+    """Judge whether candidate_answer correctly answers question, vs expected_answer.
+
+    On unparseable judge output, conservatively returns is_correct=False
+    with a rationale noting the parse failure, rather than raising — one
+    bad judge call must not crash a run.
+    """
+    raw = client.complete(SYSTEM_PROMPT, _build_user_prompt(question, expected_answer, candidate_answer))
+    parsed = parse_correctness_response(raw)
+
+    if parsed is None:
+        logger.warning("Correctness judge returned unparseable JSON for question: %s", question)
+        return CorrectnessResult(
+            is_correct=False,
+            score=None,
+            rationale="judge returned unparseable JSON",
+            judge_model=client.model,
+            judge_parse_failed=True,
+        )
+
+    return CorrectnessResult(
+        is_correct=parsed["is_correct"],
+        score=parsed["score"],
+        rationale=parsed["rationale"],
+        judge_model=client.model,
+    )

ragaudit/scoring/groundedness.py

@@ -0,0 +1,86 @@
+"""LLM judge for whether an answer is grounded in the contexts it was given."""
+
+import json
+import logging
+from typing import Protocol
+
+from ragaudit.models.scoring import GroundednessResult
+
+logger = logging.getLogger(__name__)
+
+SYSTEM_PROMPT = (
+    "You are judging whether an answer is faithfully grounded in the "
+    "provided contexts — i.e. every claim in the answer is supported by "
+    "the contexts, regardless of whether the contexts are actually relevant "
+    "to the question. Respond with strict JSON only, no prose, no markdown "
+    'fences, in the form: {"is_grounded": bool, "score": float between 0 '
+    'and 1, "rationale": str}'
+)
+
+
+class Completer(Protocol):
+    """The subset of LLMClient this module depends on — easy to stub in tests."""
+
+    model: str
+
+    def complete(self, system: str, user: str) -> str: ...
+
+
+def _build_user_prompt(answer: str, contexts: list[str]) -> str:
+    joined_contexts = "\n---\n".join(contexts) if contexts else "(no contexts provided)"
+    return f"Contexts:\n{joined_contexts}\n\nAnswer:\n{answer}\n\nIs the answer grounded in the contexts?"
+
+
+def parse_groundedness_response(raw: str) -> dict | None:
+    """Parse the judge's JSON response. Returns None on any malformed input."""
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(data, dict):
+        return None
+
+    is_grounded = data.get("is_grounded")
+    if not isinstance(is_grounded, bool):
+        return None
+
+    score = data.get("score")
+    if score is not None and not isinstance(score, (int, float)):
+        return None
+
+    rationale = data.get("rationale")
+    if rationale is not None and not isinstance(rationale, str):
+        return None
+
+    return {
+        "is_grounded": is_grounded,
+        "score": float(score) if score is not None else None,
+        "rationale": rationale,
+    }
+
+
+def judge_groundedness(client: Completer, answer: str, contexts: list[str]) -> GroundednessResult:
+    """Judge whether answer is faithfully supported by contexts.
+
+    On unparseable judge output, conservatively returns is_grounded=False
+    with a rationale noting the parse failure, rather than raising.
+    """
+    raw = client.complete(SYSTEM_PROMPT, _build_user_prompt(answer, contexts))
+    parsed = parse_groundedness_response(raw)
+
+    if parsed is None:
+        logger.warning("Groundedness judge returned unparseable JSON for answer: %s", answer)
+        return GroundednessResult(
+            is_grounded=False,
+            score=None,
+            rationale="judge returned unparseable JSON",
+            judge_model=client.model,
+            judge_parse_failed=True,
+        )
+
+    return GroundednessResult(
+        is_grounded=parsed["is_grounded"],
+        score=parsed["score"],
+        rationale=parsed["rationale"],
+        judge_model=client.model,
+    )

ragaudit/scoring/retrieval.py

@@ -0,0 +1,42 @@
+"""Pure retrieval scoring — no LLM, no network. By chunk id (set membership) or by text overlap."""
+
+from ragaudit.models.scoring import RetrievalResult
+from ragaudit.scoring.text_match import first_matching_context
+
+
+def score_retrieval(expected_chunk_id: str, retrieved_chunk_ids: list[str]) -> RetrievalResult:
+    """Check whether the expected chunk appears among the retrieved chunk IDs.
+
+    rank is the first matching index in retrieved_chunk_ids, or None if the
+    expected chunk was not retrieved.
+    """
+    rank = retrieved_chunk_ids.index(expected_chunk_id) if expected_chunk_id in retrieved_chunk_ids else None
+    return RetrievalResult(
+        hit=rank is not None,
+        expected_chunk_id=expected_chunk_id,
+        retrieved_chunk_ids=retrieved_chunk_ids,
+        rank=rank,
+    )
+
+
+def score_retrieval_by_text(
+    expected_chunk_id: str,
+    gold_text: str,
+    retrieved_chunk_ids: list[str],
+    contexts: list[str],
+    threshold: float,
+) -> RetrievalResult:
+    """A hit if any returned context overlaps the gold chunk's text at or above threshold.
+
+    rank is the index of the first matching context. retrieved_chunk_ids
+    are kept as returned (they may be the pipeline's own ids).
+    """
+    rank, best = first_matching_context(gold_text, contexts, threshold)
+    return RetrievalResult(
+        hit=rank is not None,
+        expected_chunk_id=expected_chunk_id,
+        retrieved_chunk_ids=retrieved_chunk_ids,
+        rank=rank,
+        match_by="text",
+        match_score=round(best, 4),
+    )

ragaudit/scoring/text_match.py

@@ -0,0 +1,51 @@
+"""Retrieval scoring by text overlap, for pipelines that don't carry ragaudit's chunk ids.
+
+`--match-by id` (the default) counts a retrieval hit only when the adapter
+returns the gold chunk's id, so the pipeline has to be indexed against
+ragaudit's chunks. `--match-by text` instead compares the gold chunk's
+*text* with the contexts the adapter already returns, so a pipeline that
+chunks the corpus its own way works without re-indexing.
+
+The score is the overlap coefficient of word bigrams: shared bigrams
+divided by the bigram count of the smaller side. It is high when most of
+the gold passage appears inside a returned context (the pipeline's chunks
+are bigger) or most of a context appears inside the gold passage (they are
+smaller), whatever the chunk boundaries. Bigrams rather than single words
+keep two passages on the same topic from matching just because they share
+vocabulary. The denominator never drops below MIN_BIGRAMS, so a context of
+a few words can't match everything.
+"""
+
+import re
+
+DEFAULT_MATCH_THRESHOLD = 0.5
+MIN_BIGRAMS = 8
+MATCH_MODES = ("id", "text")
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+
+def _bigrams(text: str) -> set[tuple[str, str]]:
+    tokens = _TOKEN_RE.findall(text.lower())
+    return set(zip(tokens, tokens[1:]))
+
+
+def text_match_score(gold_text: str, context: str) -> float:
+    """Overlap of gold_text and context in [0, 1] (see the module docstring)."""
+    gold, ctx = _bigrams(gold_text), _bigrams(context)
+    if not gold or not ctx:
+        return 0.0
+    return len(gold & ctx) / max(min(len(gold), len(ctx)), MIN_BIGRAMS)
+
+
+def first_matching_context(gold_text: str, contexts: list[str], threshold: float) -> tuple[int | None, float]:
+    """(index of the first context scoring >= threshold or None, best score over all contexts)."""
+    scores = [text_match_score(gold_text, context) for context in contexts]
+    rank = next((i for i, score in enumerate(scores) if score >= threshold), None)
+    return rank, max(scores, default=0.0)
+
+
+def validate_match_by(match_by: str) -> str:
+    if match_by not in MATCH_MODES:
+        raise ValueError(f"--match-by must be id or text, got {match_by!r}")
+    return match_by

ragaudit/scoring/verdict.py

@@ -0,0 +1,14 @@
+"""Pure function cross-referencing answer correctness against retrieval."""
+
+from ragaudit.models.scoring import Verdict
+
+
+def compute_verdict(is_correct: bool, retrieval_hit: bool) -> Verdict:
+    """Cross-reference correctness against retrieval into a single Verdict."""
+    if is_correct and retrieval_hit:
+        return Verdict.TRUE_PASS
+    if is_correct and not retrieval_hit:
+        return Verdict.LUCKY_PASS
+    if not is_correct and retrieval_hit:
+        return Verdict.GENERATION_FAILURE
+    return Verdict.RETRIEVAL_FAILURE

ragaudit/storage/db.py

@@ -0,0 +1,81 @@
+"""SQLite connection management and schema creation.
+
+Migrations are intentionally trivial: every table is created with
+`CREATE TABLE IF NOT EXISTS`, so re-opening an existing database is a
+no-op. No migration framework, no versioning — if the schema needs to
+change later, that's a deliberate future decision, not something this
+module tries to anticipate.
+"""
+
+import sqlite3
+from pathlib import Path
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS documents (
+    doc_id TEXT PRIMARY KEY,
+    source_path TEXT NOT NULL,
+    content TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    metadata_json TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+    chunk_id TEXT PRIMARY KEY,
+    doc_id TEXT NOT NULL,
+    position INTEGER NOT NULL,
+    text TEXT NOT NULL,
+    token_count INTEGER,
+    metadata_json TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS testsets (
+    testset_id TEXT PRIMARY KEY,
+    corpus_hash TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    generation_model TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS test_cases (
+    test_id TEXT PRIMARY KEY,
+    testset_id TEXT NOT NULL,
+    position INTEGER NOT NULL,
+    question TEXT NOT NULL,
+    expected_answer TEXT NOT NULL,
+    source_chunk_id TEXT NOT NULL,
+    source_doc_id TEXT NOT NULL,
+    generation_model TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    metadata_json TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS runs (
+    run_id TEXT PRIMARY KEY,
+    testset_id TEXT NOT NULL,
+    adapter_path TEXT NOT NULL,
+    correctness_judge_model TEXT NOT NULL,
+    groundedness_judge_model TEXT NOT NULL,
+    run_ablation INTEGER NOT NULL,
+    started_at TEXT NOT NULL,
+    finished_at TEXT,
+    summary_json TEXT
+);
+
+CREATE TABLE IF NOT EXISTS scored_cases (
+    run_id TEXT NOT NULL,
+    test_id TEXT NOT NULL,
+    position INTEGER NOT NULL,
+    data_json TEXT NOT NULL,
+    PRIMARY KEY (run_id, test_id)
+);
+"""
+
+
+def get_connection(db_path: str | Path) -> sqlite3.Connection:
+    """Open a SQLite connection at db_path and ensure the schema exists.
+
+    db_path may be ":memory:" for an ephemeral in-process database.
+    """
+    connection = sqlite3.connect(str(db_path))
+    connection.executescript(SCHEMA)
+    connection.commit()
+    return connection

ragaudit/storage/lock.py

@@ -0,0 +1,36 @@
+"""The test-split lock: test gold answers are readable only inside the final evaluation path.
+
+The store consults `is_test_split_unlocked()` before returning any test-split
+question or result. The only code that unlocks it is
+`ragaudit.runner.locked`, which returns aggregate scores, never answers.
+The `ragaudit.tune` package must never import this module; a unit test
+enforces that.
+
+Python can't make this tamper-proof, so the lock exists to stop
+accidents (a tuning loop that quietly reads test answers), not a
+determined author.
+"""
+
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import Iterator
+
+_unlocked: ContextVar[bool] = ContextVar("ragaudit_test_split_unlocked", default=False)
+
+
+class LockedTestSetError(Exception):
+    """Raised when code outside the final evaluation path asks for test-split data."""
+
+
+def is_test_split_unlocked() -> bool:
+    return _unlocked.get()
+
+
+@contextmanager
+def _unlock_test_split() -> Iterator[None]:
+    """Unlock test-split reads for the duration of the block. Final evaluation path only."""
+    token = _unlocked.set(True)
+    try:
+        yield
+    finally:
+        _unlocked.reset(token)