ragaudit-cli 0.3.0__py3-none-any.whl

ragaudit/generate/testset_generator.py

@@ -0,0 +1,146 @@
+"""Generates a TestSet by prompting an LLM to write questions per chunk."""
+
+import hashlib
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Protocol
+
+from ragaudit.models.document import Chunk, Document
+from ragaudit.models.testset import TestCase, TestSet
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_QUESTIONS_PER_CHUNK = 1
+
+SYSTEM_PROMPT = (
+    "You are generating a test set for a RAG evaluation. Given a single "
+    "passage of text, write questions that can be answered ONLY using facts "
+    "stated in that passage — not from general knowledge about the topic, "
+    "and not answerable by someone who has never seen this passage but "
+    "knows the subject well.\n"
+    "\n"
+    "Phrase every question exactly as a real user would ask it. Never refer "
+    "to \"the passage\", \"the document\", \"the text\", \"this section\", "
+    "or any other document structure — the reader must not be able to tell "
+    "the question was generated from a document. Do not ask meta-questions "
+    "about ordering, sections, or headings.\n"
+    "\n"
+    "If asked for more than one question, each question must target a "
+    "different fact from the passage — no two questions may test the same "
+    "piece of information.\n"
+    "\n"
+    "Respond with strict JSON only, no prose, no markdown fences, in the "
+    'form: {"questions": [{"question": str, "answer": str}, ...]}'
+)
+
+
+class Completer(Protocol):
+    """The subset of LLMClient this module depends on — easy to stub in tests."""
+
+    model: str
+
+    def complete(self, system: str, user: str) -> str: ...
+
+
+def _build_user_prompt(chunk_text: str, n: int) -> str:
+    return f"Write {n} question(s) answerable only from this passage:\n\n{chunk_text}"
+
+
+def parse_questions_response(raw: str) -> list[dict[str, str]] | None:
+    """Parse the LLM's JSON response into a list of {question, answer} dicts.
+
+    Returns None on any malformed input (bad JSON, wrong shape, missing
+    fields) instead of raising, so callers can skip and log without
+    crashing a large run.
+    """
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        return None
+
+    if not isinstance(data, dict):
+        return None
+    questions = data.get("questions")
+    if not isinstance(questions, list):
+        return None
+
+    parsed = []
+    for item in questions:
+        if not isinstance(item, dict):
+            continue
+        question = item.get("question")
+        answer = item.get("answer")
+        if isinstance(question, str) and isinstance(answer, str) and question and answer:
+            parsed.append({"question": question, "answer": answer})
+    return parsed if parsed else None
+
+
+def _test_id(chunk_id: str, index: int) -> str:
+    payload = f"{chunk_id}:{index}"
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def generate_test_cases_for_chunk(
+    client: Completer,
+    chunk: Chunk,
+    n: int = DEFAULT_QUESTIONS_PER_CHUNK,
+) -> list[TestCase]:
+    """Prompt the LLM for questions on one chunk, returning TestCases.
+
+    Returns an empty list (logging a warning) if the LLM's response can't
+    be parsed as the expected JSON shape — never raises.
+    """
+    raw = client.complete(SYSTEM_PROMPT, _build_user_prompt(chunk.text, n))
+    parsed = parse_questions_response(raw)
+
+    if parsed is None:
+        logger.warning("Skipping chunk %s: LLM returned unparseable JSON", chunk.chunk_id)
+        return []
+
+    now = datetime.now(timezone.utc)
+    return [
+        TestCase(
+            test_id=_test_id(chunk.chunk_id, index),
+            question=item["question"],
+            expected_answer=item["answer"],
+            source_chunk_id=chunk.chunk_id,
+            source_doc_id=chunk.doc_id,
+            generation_model=client.model,
+            created_at=now,
+        )
+        for index, item in enumerate(parsed)
+    ]
+
+
+def compute_corpus_hash(documents: list[Document]) -> str:
+    """Hash over all Document.content_hash values, sorted for order-independence."""
+    payload = "|".join(sorted(doc.content_hash for doc in documents))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def generate_test_set(
+    client: Completer,
+    documents: list[Document],
+    chunks: list[Chunk],
+    n_per_chunk: int = DEFAULT_QUESTIONS_PER_CHUNK,
+    testset_id: str | None = None,
+) -> TestSet:
+    """Generate a full TestSet by prompting the LLM once per chunk.
+
+    Chunks whose LLM response can't be parsed are skipped and logged; a
+    malformed response from one chunk never aborts the rest of the run.
+    """
+    test_cases: list[TestCase] = []
+    for chunk in chunks:
+        test_cases.extend(generate_test_cases_for_chunk(client, chunk, n=n_per_chunk))
+
+    corpus_hash = compute_corpus_hash(documents)
+    created_at = datetime.now(timezone.utc)
+    return TestSet(
+        testset_id=testset_id or hashlib.sha256(f"{corpus_hash}:{created_at.isoformat()}".encode()).hexdigest(),
+        corpus_hash=corpus_hash,
+        created_at=created_at,
+        generation_model=client.model,
+        test_cases=test_cases,
+    )

ragaudit/generate/text.py

@@ -0,0 +1,75 @@
+"""Pure text helpers for v2 generation: tokenizing, lexical overlap, relative-time detection."""
+
+import re
+
+STOPWORDS = frozenset(
+    """
+    a about above after again against all also am an and any are as at be because been before being below
+    between both but by can could did do does doing down during each few for from further had has have having
+    he her here hers herself him himself his how i if in into is it its itself just me more most my myself no
+    nor not of off on once only or other our ours ourselves out over own same she should so some such than that
+    the their theirs them themselves then there these they this those through to too under until up very was we
+    were what when where which while who whom whose why will with would you your yours yourself yourselves
+    s t don shall may might must per via vs etc many much
+    """.split()
+)
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+(?:[.,][0-9]+)*")
+
+RELATIVE_TIME_RE = re.compile(
+    r"\b("
+    r"yesterday|today|tonight|tomorrow|now|nowadays|currently|current|recent|recently|lately|latest|"
+    r"these\s+days|at\s+present|at\s+the\s+moment|right\s+now|so\s+far|to\s+date|up\s+to\s+date|"
+    r"(?:this|last|next|past|previous|coming|upcoming)\s+"
+    r"(?:few\s+)?(?:day|week|weekend|month|quarter|year|decade|season|spring|summer|autumn|fall|winter)s?|"
+    r"(?:a\s+few|several|some|\d+|one|two|three|four|five|six|seven|eight|nine|ten)\s+"
+    r"(?:day|week|month|year|decade)s?\s+(?:ago|back)|"
+    r"ago|most\s+recent(?:ly)?"
+    r")\b",
+    re.IGNORECASE,
+)
+
+
+def _normalize(token: str) -> str:
+    """A crude plural fold so "calibrations" and "calibration" count as the same word."""
+    if len(token) > 3 and token.endswith("s") and not token.endswith("ss"):
+        return token[:-1]
+    return token
+
+
+def tokenize(text: str) -> list[str]:
+    return [_normalize(t) for t in _TOKEN_RE.findall(text.lower())]
+
+
+def content_tokens(text: str) -> set[str]:
+    """Distinct non-stopword tokens (stopwords are checked before the plural fold, so "does" stays out)."""
+    return {_normalize(t) for t in _TOKEN_RE.findall(text.lower()) if t not in STOPWORDS}
+
+
+def lexical_overlap(question: str, source_text: str) -> float:
+    """Fraction of the question's distinct content tokens that also appear in the source text.
+
+    High overlap means the question reuses the chunk's wording, which makes
+    keyword retrieval trivially easy and inflates retrieval scores. A
+    question with no content tokens at all scores 0.0.
+    """
+    question_tokens = content_tokens(question)
+    if not question_tokens:
+        return 0.0
+    return len(question_tokens & content_tokens(source_text)) / len(question_tokens)
+
+
+def find_relative_time(text: str) -> str | None:
+    """Return the first relative-time expression in text ("last week", "recently", ...), or None."""
+    match = RELATIVE_TIME_RE.search(text)
+    return match.group(0) if match else None
+
+
+def has_relative_time(text: str) -> bool:
+    return find_relative_time(text) is not None
+
+
+def jaccard(a: set[str], b: set[str]) -> float:
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)

ragaudit/ingest/chunker.py

@@ -0,0 +1,53 @@
+"""Splits Document content into fixed-size, overlapping Chunk objects."""
+
+import hashlib
+
+from ragaudit.models.document import Chunk, Document
+
+DEFAULT_CHUNK_SIZE = 800
+DEFAULT_OVERLAP = 100
+
+
+def _chunk_id(doc_id: str, position: int, text: str) -> str:
+    payload = f"{doc_id}:{position}:{text}"
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def chunk_document(
+    document: Document,
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+    overlap: int = DEFAULT_OVERLAP,
+) -> list[Chunk]:
+    """Split a Document's content into fixed-size chunks with overlap.
+
+    Chunking is deterministic: chunk_id is a hash of (doc_id, position,
+    text), so re-chunking the same document with the same parameters always
+    produces the same chunk_ids.
+    """
+    if chunk_size <= 0:
+        raise ValueError("chunk_size must be positive")
+    if overlap < 0 or overlap >= chunk_size:
+        raise ValueError("overlap must be non-negative and smaller than chunk_size")
+
+    text = document.content
+    if not text:
+        return []
+
+    step = chunk_size - overlap
+    chunks: list[Chunk] = []
+    position = 0
+    start = 0
+    while start < len(text):
+        piece = text[start : start + chunk_size]
+        chunks.append(
+            Chunk(
+                chunk_id=_chunk_id(document.doc_id, position, piece),
+                doc_id=document.doc_id,
+                position=position,
+                text=piece,
+            )
+        )
+        position += 1
+        start += step
+
+    return chunks

ragaudit/ingest/corpus.py

@@ -0,0 +1,120 @@
+"""Loads a v2 corpus: chunks with stable ids, content hashes and optional capture timestamps.
+
+Two input formats:
+
+- a `.jsonl` file, one chunk per line:
+  `{"chunk_id": str, "text": str, "timestamp"?: ISO-8601, "doc_id"?: str, "position"?: int}`
+- a directory of `.txt`/`.md` files, loaded and chunked with the v1 loader
+  and chunker, but with stable ids `relative/path.md#position`.
+
+v2 needs chunk ids that survive edits: a *changed* chunk (same id, new
+hash) is handled differently from a new one. v1's chunk ids hash the
+document content, so an edit gives every chunk a fresh id; that's why
+directory corpora get path-based ids here instead.
+"""
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ragaudit.ingest import chunker
+from ragaudit.ingest.loader import load_documents
+from ragaudit.models.document import Chunk
+
+TIMESTAMP_KEY = "timestamp"
+
+
+class CorpusError(Exception):
+    """Raised when a corpus file is malformed."""
+
+
+def chunk_hash(chunk: Chunk) -> str:
+    """Content hash of a chunk's text."""
+    return hashlib.sha256(chunk.text.encode("utf-8")).hexdigest()
+
+
+def parse_timestamp(value: str) -> datetime:
+    """Parse an ISO-8601 date or datetime; naive values are taken as UTC."""
+    parsed = datetime.fromisoformat(value.strip())
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+def chunk_timestamp(chunk: Chunk) -> datetime | None:
+    value = chunk.metadata.get(TIMESTAMP_KEY)
+    return parse_timestamp(value) if value else None
+
+
+def _load_jsonl(path: Path) -> list[Chunk]:
+    chunks: list[Chunk] = []
+    for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise CorpusError(f"{path}:{line_no}: invalid JSON ({exc.msg})") from exc
+        if not isinstance(record, dict):
+            raise CorpusError(f"{path}:{line_no}: expected a JSON object")
+
+        chunk_id = record.get("chunk_id")
+        text = record.get("text")
+        if not isinstance(chunk_id, str) or not chunk_id:
+            raise CorpusError(f"{path}:{line_no}: missing string field 'chunk_id'")
+        if not isinstance(text, str) or not text.strip():
+            raise CorpusError(f"{path}:{line_no}: missing non-empty string field 'text'")
+
+        metadata: dict[str, str] = {}
+        timestamp = record.get(TIMESTAMP_KEY)
+        if timestamp:
+            try:
+                metadata[TIMESTAMP_KEY] = parse_timestamp(str(timestamp)).isoformat()
+            except ValueError as exc:
+                raise CorpusError(f"{path}:{line_no}: timestamp is not ISO-8601: {timestamp!r}") from exc
+        for key, value in record.items():
+            if key not in {"chunk_id", "text", "doc_id", "position", TIMESTAMP_KEY} and isinstance(value, str):
+                metadata[key] = value
+
+        chunks.append(
+            Chunk(
+                chunk_id=chunk_id,
+                doc_id=str(record.get("doc_id") or chunk_id),
+                position=int(record.get("position", len(chunks))),
+                text=text,
+                metadata=metadata,
+            )
+        )
+    return chunks
+
+
+def _load_directory(root: Path, chunk_size: int, overlap: int) -> list[Chunk]:
+    chunks: list[Chunk] = []
+    for document in load_documents(root):
+        relative = Path(document.source_path).relative_to(root).as_posix()
+        for chunk in chunker.chunk_document(document, chunk_size=chunk_size, overlap=overlap):
+            chunks.append(
+                chunk.model_copy(update={"chunk_id": f"{relative}#{chunk.position}", "doc_id": relative})
+            )
+    return chunks
+
+
+def load_corpus(
+    path: str | Path,
+    chunk_size: int = chunker.DEFAULT_CHUNK_SIZE,
+    overlap: int = chunker.DEFAULT_OVERLAP,
+) -> list[Chunk]:
+    """Load a corpus from a .jsonl file or a directory of .txt/.md files. Chunk ids must be unique."""
+    path = Path(path)
+    if not path.exists():
+        raise CorpusError(f"corpus not found: {path}")
+
+    chunks = _load_directory(path, chunk_size, overlap) if path.is_dir() else _load_jsonl(path)
+
+    seen: set[str] = set()
+    for chunk in chunks:
+        if chunk.chunk_id in seen:
+            raise CorpusError(f"duplicate chunk_id in corpus: {chunk.chunk_id}")
+        seen.add(chunk.chunk_id)
+    return chunks

ragaudit/ingest/loader.py

@@ -0,0 +1,49 @@
+"""Walks a directory and loads .txt/.md files into Document objects."""
+
+import hashlib
+import logging
+from pathlib import Path
+
+from ragaudit.models.document import Document
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_EXTENSIONS = {".txt", ".md"}
+
+
+def _sha256(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def load_documents(root: str | Path) -> list[Document]:
+    """Walk `root` and load every supported file into a Document.
+
+    Only .txt and .md files are read. Files that can't be decoded as UTF-8
+    text, or can't be read at all, are skipped with a logged warning rather
+    than raising.
+    """
+    root_path = Path(root)
+    documents: list[Document] = []
+
+    for file_path in sorted(root_path.rglob("*")):
+        if not file_path.is_file() or file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
+            continue
+
+        try:
+            content = file_path.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, OSError) as exc:
+            logger.warning("Skipping unreadable file %s: %s", file_path, exc)
+            continue
+
+        content_hash = _sha256(content)
+        doc_id = _sha256(f"{file_path}:{content_hash}")
+        documents.append(
+            Document(
+                doc_id=doc_id,
+                source_path=str(file_path),
+                content=content,
+                content_hash=content_hash,
+            )
+        )
+
+    return documents

ragaudit/ingest/versioning.py

@@ -0,0 +1,92 @@
+"""Corpus versions and staleness: what changed since the last generate/evaluate run.
+
+`index_version` is a hash of the sorted set of current chunk hashes. Each
+run diffs the current corpus (chunk_id -> chunk_hash) against the last
+recorded one:
+
+- new chunk: no questions yet, so `generate` will write some;
+- changed chunk (same id, new hash): its old questions are marked stale,
+  and `generate` writes new ones for the new content;
+- deleted chunk: its questions are marked stale.
+
+Staleness is decided per question from every (chunk_id, chunk_hash) it
+depends on, so a multi-hop question goes stale if either chunk changes.
+If a corpus edit is reverted, the questions it made stale become active
+again, because their source content is back exactly as it was.
+"""
+
+import hashlib
+
+from pydantic import BaseModel, Field
+
+from ragaudit.ingest.corpus import chunk_hash
+from ragaudit.models.document import Chunk
+from ragaudit.storage.store import Store
+
+
+class CorpusDiff(BaseModel):
+    index_version: str
+    previous_index_version: str | None = None
+    new: list[str] = Field(default_factory=list)
+    changed: list[str] = Field(default_factory=list)
+    deleted: list[str] = Field(default_factory=list)
+    unchanged: int = 0
+    marked_stale: int = 0
+    revived: int = 0
+
+    @property
+    def index_changed(self) -> bool:
+        return self.index_version != self.previous_index_version
+
+    def counts(self) -> dict[str, int]:
+        return {
+            "new": len(self.new),
+            "changed": len(self.changed),
+            "deleted": len(self.deleted),
+            "unchanged": self.unchanged,
+            "marked_stale": self.marked_stale,
+            "revived": self.revived,
+        }
+
+
+def chunk_map(chunks: list[Chunk]) -> dict[str, str]:
+    return {chunk.chunk_id: chunk_hash(chunk) for chunk in chunks}
+
+
+def compute_index_version(chunk_hashes: list[str] | dict[str, str]) -> str:
+    """Hash of the sorted set of chunk hashes: same content, same version, whatever the order or ids."""
+    hashes = chunk_hashes.values() if isinstance(chunk_hashes, dict) else chunk_hashes
+    return hashlib.sha256("\n".join(sorted(set(hashes))).encode("utf-8")).hexdigest()
+
+
+def diff_chunk_maps(previous: dict[str, str], current: dict[str, str]) -> tuple[list[str], list[str], list[str], int]:
+    """(new ids, changed ids, deleted ids, unchanged count), ids sorted."""
+    new = sorted(cid for cid in current if cid not in previous)
+    changed = sorted(cid for cid in current if cid in previous and previous[cid] != current[cid])
+    deleted = sorted(cid for cid in previous if cid not in current)
+    unchanged = sum(1 for cid in current if previous.get(cid) == current[cid])
+    return new, changed, deleted, unchanged
+
+
+def sync_corpus(store: Store, chunks: list[Chunk]) -> CorpusDiff:
+    """Diff the corpus against the last recorded version, update question staleness, record the new version."""
+    current = chunk_map(chunks)
+    index_version = compute_index_version(current)
+    last = store.latest_corpus_version()
+    previous_version, previous_map = last if last is not None else (None, {})
+
+    new, changed, deleted, unchanged = diff_chunk_maps(previous_map, current)
+    marked_stale, revived = store.refresh_staleness(current)
+    if last is None or previous_version != index_version or previous_map != current:
+        store.record_corpus_version(index_version, current)
+
+    return CorpusDiff(
+        index_version=index_version,
+        previous_index_version=previous_version,
+        new=new,
+        changed=changed,
+        deleted=deleted,
+        unchanged=unchanged,
+        marked_stale=marked_stale,
+        revived=revived,
+    )

ragaudit/llm/client.py

@@ -0,0 +1,86 @@
+"""Thin OpenAI-compatible chat completions client over httpx."""
+
+import os
+import time
+
+import httpx
+
+DEFAULT_MAX_RETRIES = 3
+RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
+
+
+class LLMClientError(Exception):
+    """Raised when the LLM API call fails after exhausting all retries."""
+
+
+class LLMClient:
+    """Minimal OpenAI-compatible chat completions client.
+
+    Configuration defaults to environment variables:
+    - RAGAUDIT_API_BASE: base URL, e.g. https://api.openai.com/v1
+    - RAGAUDIT_API_KEY: bearer token
+    - RAGAUDIT_MODEL: model name
+    """
+
+    def __init__(
+        self,
+        api_base: str | None = None,
+        api_key: str | None = None,
+        model: str | None = None,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        timeout: float = 60.0,
+        client: httpx.Client | None = None,
+    ) -> None:
+        self.api_base = (api_base or os.environ["RAGAUDIT_API_BASE"]).rstrip("/")
+        self.api_key = api_key or os.environ["RAGAUDIT_API_KEY"]
+        self.model = model or os.environ["RAGAUDIT_MODEL"]
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self._client = client or httpx.Client(timeout=timeout)
+
+    def complete(self, system: str, user: str) -> str:
+        """Send one chat completion request and return the assistant's text.
+
+        Retries on 429 and 5xx responses (and transport errors) with
+        exponential backoff, up to max_retries attempts total.
+        """
+        url = f"{self.api_base}/chat/completions"
+        headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
+        payload = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+            "stream": False,
+        }
+
+        last_error: Exception | str | None = None
+        for attempt in range(self.max_retries):
+            is_last_attempt = attempt == self.max_retries - 1
+            try:
+                response = self._client.post(url, headers=headers, json=payload)
+            except httpx.TransportError as exc:
+                last_error = exc
+                if is_last_attempt:
+                    break
+                self._sleep_backoff(attempt)
+                continue
+
+            if response.status_code in RETRYABLE_STATUS_CODES:
+                last_error = f"status {response.status_code}: {response.text}"
+                if is_last_attempt:
+                    break
+                self._sleep_backoff(attempt)
+                continue
+
+            if response.status_code >= 400:
+                raise LLMClientError(f"LLM request failed with status {response.status_code}: {response.text}")
+
+            data = response.json()
+            return data["choices"][0]["message"]["content"]
+
+        raise LLMClientError(f"LLM request failed after {self.max_retries} attempts: {last_error}")
+
+    def _sleep_backoff(self, attempt: int) -> None:
+        time.sleep(2**attempt)