evalseed 0.1.0__py3-none-any.whl

evalseed/__init__.py

@@ -0,0 +1,31 @@
+"""evalseed — quality-filtered synthetic Q&A datasets for RAG evaluation."""
+
+from evalseed.dataset import Dataset
+from evalseed.exceptions import (
+    EvalseedError,
+    FilterError,
+    GenerationError,
+    JudgeAuthError,
+    JudgeError,
+)
+from evalseed.judges import Judge, OpenAIJudge
+from evalseed.pipeline import Pipeline
+from evalseed.schemas import FilterResult, QAPair, QAType
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Dataset",
+    "EvalseedError",
+    "FilterError",
+    "FilterResult",
+    "GenerationError",
+    "Judge",
+    "JudgeAuthError",
+    "JudgeError",
+    "OpenAIJudge",
+    "Pipeline",
+    "QAPair",
+    "QAType",
+    "__version__",
+]

evalseed/cli.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from evalseed import JudgeAuthError, OpenAIJudge, Pipeline
+from evalseed.schemas import QAType
+
+
+def _parse_types(value: str) -> list[QAType]:
+    out: list[QAType] = []
+    for raw in value.split(","):
+        raw = raw.strip()
+        if not raw:
+            continue
+        out.append(QAType(raw))
+    if not out:
+        raise argparse.ArgumentTypeError("at least one QA type required")
+    return out
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="evalseed",
+        description="Generate quality-filtered synthetic Q&A datasets for RAG evaluation.",
+    )
+    parser.add_argument("corpus", help="Path to a corpus file or directory.")
+    parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Output JSONL path. Defaults to <corpus>/output/eval.jsonl "
+        "(or <corpus_parent>/output/eval.jsonl if corpus is a file).",
+    )
+    parser.add_argument("-n", "--n-pairs", type=int, default=50, help="Target pair count.")
+    parser.add_argument(
+        "--types",
+        type=_parse_types,
+        default=[QAType.SINGLE_HOP, QAType.MULTI_HOP],
+        help="Comma-separated QA types: single_hop,multi_hop,distractor",
+    )
+    parser.add_argument("--model", default="gpt-4o-mini", help="OpenAI model id.")
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--all", action="store_true", help="Save passed AND rejected pairs.")
+
+    args = parser.parse_args(argv)
+    out_path = _resolve_out_path(args.corpus, args.out)
+    try:
+        judge = OpenAIJudge(model=args.model)
+        pipeline = Pipeline(
+            judge=judge,
+            n_pairs=args.n_pairs,
+            types=args.types,
+            seed=args.seed,
+        )
+        dataset = pipeline.generate_from_corpus(args.corpus)
+    except JudgeAuthError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    dataset.save(out_path, only_passed=not args.all)
+    print(f"wrote {out_path}")
+    return 0
+
+
+def _resolve_out_path(corpus: str, out: str | None) -> Path:
+    if out is not None:
+        return Path(out)
+    corpus_path = Path(corpus)
+    base = corpus_path if corpus_path.is_dir() else corpus_path.parent
+    return base / "output" / "eval.jsonl"
+
+
+if __name__ == "__main__":
+    sys.exit(main())

evalseed/corpus.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from evalseed.exceptions import GenerationError
+
+
+@dataclass(frozen=True)
+class Chunk:
+    """A contiguous slice of a source document."""
+
+    text: str
+    source: str
+    chunk_index: int
+
+
+_SUPPORTED_SUFFIXES = {".txt", ".md", ".markdown", ".rst"}
+
+
+def load_corpus(path: str | Path) -> list[tuple[str, str]]:
+    """Load (source_name, text) pairs from a file or directory.
+
+    Supports plain-text formats only in v0.1 (.txt, .md, .markdown, .rst).
+    PDF/HTML loaders are intentionally deferred — users can preprocess.
+    """
+    p = Path(path)
+    if not p.exists():
+        raise GenerationError(f"corpus path does not exist: {p}")
+
+    if p.is_file():
+        return [(p.name, _read_text(p))]
+
+    docs: list[tuple[str, str]] = []
+    for sub in sorted(p.rglob("*")):
+        if sub.is_file() and sub.suffix.lower() in _SUPPORTED_SUFFIXES:
+            docs.append((str(sub.relative_to(p)), _read_text(sub)))
+    if not docs:
+        raise GenerationError(
+            f"no supported files (.txt/.md/.rst) found under {p}"
+        )
+    return docs
+
+
+def _read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        return path.read_text(encoding="utf-8", errors="replace")
+
+
+def chunk_text(
+    text: str,
+    source: str,
+    target_chars: int = 1500,
+    overlap_chars: int = 150,
+) -> list[Chunk]:
+    """Paragraph-aware char-window chunker.
+
+    Greedily packs paragraphs into windows of roughly ``target_chars``,
+    with character overlap between consecutive windows. Good enough for
+    v0.1; more sophisticated chunking is a v0.2 concern.
+    """
+    if target_chars <= 0:
+        raise ValueError("target_chars must be positive")
+    if overlap_chars < 0 or overlap_chars >= target_chars:
+        raise ValueError("overlap_chars must be in [0, target_chars)")
+
+    paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
+    if not paragraphs:
+        return []
+
+    chunks: list[Chunk] = []
+    buf: list[str] = []
+    buf_len = 0
+    idx = 0
+
+    def flush() -> None:
+        nonlocal buf, buf_len, idx
+        if not buf:
+            return
+        chunk_text_value = "\n\n".join(buf)
+        chunks.append(Chunk(text=chunk_text_value, source=source, chunk_index=idx))
+        idx += 1
+        if overlap_chars and len(chunk_text_value) > overlap_chars:
+            tail = chunk_text_value[-overlap_chars:]
+            buf = [tail]
+            buf_len = len(tail)
+        else:
+            buf = []
+            buf_len = 0
+
+    for para in paragraphs:
+        if buf_len + len(para) + 2 > target_chars and buf:
+            flush()
+        buf.append(para)
+        buf_len += len(para) + 2
+
+    flush()
+    return chunks

evalseed/dataset.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from collections.abc import Iterator
+from pathlib import Path
+from typing import overload
+
+from evalseed.schemas import QAPair
+
+
+class Dataset:
+    """A collection of QA pairs with serialization and stats helpers.
+
+    Iteration yields all pairs (passed and rejected). Use ``passed`` /
+    ``rejected`` to filter, or ``save`` to write only passed pairs to disk.
+    """
+
+    def __init__(self, pairs: list[QAPair]) -> None:
+        self._pairs: list[QAPair] = list(pairs)
+
+    def __len__(self) -> int:
+        return len(self._pairs)
+
+    def __iter__(self) -> Iterator[QAPair]:
+        return iter(self._pairs)
+
+    @overload
+    def __getitem__(self, key: int) -> QAPair: ...
+    @overload
+    def __getitem__(self, key: slice) -> Dataset: ...
+    def __getitem__(self, key: int | slice) -> QAPair | Dataset:
+        if isinstance(key, slice):
+            return Dataset(self._pairs[key])
+        return self._pairs[key]
+
+    @property
+    def passed(self) -> Dataset:
+        return Dataset([p for p in self._pairs if p.passed])
+
+    @property
+    def rejected(self) -> Dataset:
+        return Dataset([p for p in self._pairs if not p.passed])
+
+    def stats(self) -> dict[str, object]:
+        total = len(self._pairs)
+        passed = sum(1 for p in self._pairs if p.passed)
+        rejection_counts: Counter[str] = Counter()
+        for pair in self._pairs:
+            for r in pair.filter_results:
+                if not r.passed:
+                    rejection_counts[r.filter_name] += 1
+        return {
+            "total": total,
+            "passed": passed,
+            "rejected": total - passed,
+            "pass_rate": passed / total if total else 0.0,
+            "rejections_by_filter": dict(rejection_counts),
+        }
+
+    def save(self, path: str | Path, only_passed: bool = True) -> None:
+        """Write pairs to a JSONL file. By default writes only passed pairs."""
+        target = Path(path)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        pairs = self.passed if only_passed else self
+        with target.open("w", encoding="utf-8") as f:
+            for pair in pairs:
+                f.write(pair.model_dump_json() + "\n")
+
+    @classmethod
+    def load(cls, path: str | Path) -> Dataset:
+        pairs: list[QAPair] = []
+        with Path(path).open(encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                pairs.append(QAPair.model_validate(json.loads(line)))
+        return cls(pairs)

evalseed/exceptions.py

@@ -0,0 +1,23 @@
+class EvalseedError(Exception):
+    """Base class for all evalseed errors."""
+
+
+class GenerationError(EvalseedError):
+    """Raised when QA pair generation fails."""
+
+
+class FilterError(EvalseedError):
+    """Raised when a filter cannot evaluate a pair."""
+
+
+class JudgeError(EvalseedError):
+    """Raised when the underlying judge LLM call fails."""
+
+
+class JudgeAuthError(JudgeError):
+    """Raised when the judge cannot authenticate (missing or invalid API key).
+
+    Distinct from JudgeError so callers that normally swallow transient
+    judge failures can let auth errors surface — retrying a bad key is
+    pointless and would silently produce empty results.
+    """

evalseed/filters/__init__.py

@@ -0,0 +1,17 @@
+from evalseed.filters.answerability import AnswerabilityFilter
+from evalseed.filters.base import Filter, PreFilter
+from evalseed.filters.difficulty import DifficultyFilter
+from evalseed.filters.faithfulness import FaithfulnessFilter
+from evalseed.filters.prefilters import LengthPreFilter, RegexPreFilter
+from evalseed.filters.triviality import TrivialityFilter
+
+__all__ = [
+    "AnswerabilityFilter",
+    "DifficultyFilter",
+    "FaithfulnessFilter",
+    "Filter",
+    "LengthPreFilter",
+    "PreFilter",
+    "RegexPreFilter",
+    "TrivialityFilter",
+]

evalseed/filters/answerability.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from evalseed.exceptions import JudgeAuthError, JudgeError
+from evalseed.filters.base import Filter
+from evalseed.schemas import FilterResult, QAPair
+
+_SYSTEM = (
+    "You evaluate whether a question is well-posed and answerable from a "
+    "specific context, with a single defensible answer. Respond ONLY with JSON."
+)
+
+_USER_TEMPLATE = """Evaluate the QUESTION against the CONTEXT.
+
+CONTEXT:
+\"\"\"
+{context}
+\"\"\"
+
+QUESTION: {question}
+
+Decide:
+1. Is the question unambiguous (a careful reader would not produce multiple equally valid answers)?
+2. Is the question answerable from the context alone (no external knowledge required)?
+3. Is the question self-contained (does not rely on pronouns or "the above" referring outside itself)?
+
+Return JSON:
+{{
+  "unambiguous": <true|false>,
+  "answerable": <true|false>,
+  "self_contained": <true|false>,
+  "reason": "<one short sentence if any check fails, else empty string>"
+}}
+"""
+
+
+class AnswerabilityFilter(Filter):
+    """Rejects ambiguous, externally-dependent, or non-self-contained questions."""
+
+    name = "answerability"
+
+    def evaluate(self, pair: QAPair) -> FilterResult:
+        try:
+            result = self.judge.judge(
+                _SYSTEM,
+                _USER_TEMPLATE.format(context=pair.context, question=pair.question),
+            )
+        except JudgeAuthError:
+            raise
+        except JudgeError as exc:
+            return self._result(passed=False, reason=f"judge error: {exc}")
+
+        unambiguous = bool(result.get("unambiguous", False))
+        answerable = bool(result.get("answerable", False))
+        self_contained = bool(result.get("self_contained", False))
+        passed = unambiguous and answerable and self_contained
+        reason = str(result.get("reason", "")).strip() or None
+        if passed:
+            return self._result(
+                passed=True,
+                unambiguous=unambiguous,
+                answerable=answerable,
+                self_contained=self_contained,
+            )
+        failed = [
+            name
+            for name, ok in (
+                ("unambiguous", unambiguous),
+                ("answerable", answerable),
+                ("self_contained", self_contained),
+            )
+            if not ok
+        ]
+        return self._result(
+            passed=False,
+            reason=reason or f"failed: {', '.join(failed)}",
+            unambiguous=unambiguous,
+            answerable=answerable,
+            self_contained=self_contained,
+        )

evalseed/filters/base.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from evalseed.judges import Judge
+from evalseed.schemas import FilterResult, QAPair
+
+
+class PreFilter(ABC):
+    """Cheap, judge-free filter — runs before any LLM calls."""
+
+    name: str
+
+    @abstractmethod
+    def evaluate(self, pair: QAPair) -> FilterResult: ...
+
+
+class Filter(ABC):
+    """LLM-judge-backed filter."""
+
+    name: str
+
+    def __init__(self, judge: Judge) -> None:
+        self.judge = judge
+
+    @abstractmethod
+    def evaluate(self, pair: QAPair) -> FilterResult: ...
+
+    def _result(
+        self,
+        passed: bool,
+        score: float | None = None,
+        reason: str | None = None,
+        **metadata: object,
+    ) -> FilterResult:
+        return FilterResult(
+            filter_name=self.name,
+            passed=passed,
+            score=score,
+            reason=reason,
+            metadata=dict(metadata),
+        )

evalseed/filters/difficulty.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from evalseed.exceptions import JudgeAuthError, JudgeError
+from evalseed.filters.base import Filter
+from evalseed.judges import Judge
+from evalseed.schemas import Difficulty, FilterResult, QAPair
+
+_SYSTEM = (
+    "You assess the reasoning effort required by a question, given its "
+    "context. You output one of: 'easy', 'medium', 'hard'. Respond ONLY with JSON."
+)
+
+_USER_TEMPLATE = """Assess the difficulty of answering QUESTION from CONTEXT.
+
+CONTEXT:
+\"\"\"
+{context}
+\"\"\"
+
+QUESTION: {question}
+
+Difficulty rubric:
+- easy: single-span lookup, no reasoning required.
+- medium: requires combining 2-3 facts, light synthesis, or simple inference.
+- hard: requires multi-step reasoning, comparison across distant spans, or non-trivial inference.
+
+Return JSON:
+{{
+  "predicted": "<easy|medium|hard>",
+  "reason": "<one short sentence>"
+}}
+"""
+
+
+_LEVELS = {Difficulty.EASY: 0, Difficulty.MEDIUM: 1, Difficulty.HARD: 2}
+
+
+class DifficultyFilter(Filter):
+    """Records judge-predicted difficulty alongside the labeled value.
+
+    By default this is a label-enrichment pass — the prediction is stored in
+    metadata and the pair always passes. A 3-level rubric like easy/medium/hard
+    is too coarse for one-step disagreements ("easy" vs "medium") to be a
+    quality signal, so disagreement-based rejection is opt-in.
+
+    Modes:
+      strict=False (default): always pass; record prediction + agreement.
+      strict=True: reject only on a TWO-step gap (easy↔hard).
+    """
+
+    name = "difficulty"
+
+    def __init__(self, judge: Judge, strict: bool = False) -> None:
+        super().__init__(judge)
+        self.strict = strict
+
+    def evaluate(self, pair: QAPair) -> FilterResult:
+        try:
+            result = self.judge.judge(
+                _SYSTEM,
+                _USER_TEMPLATE.format(context=pair.context, question=pair.question),
+            )
+        except JudgeAuthError:
+            raise
+        except JudgeError as exc:
+            return self._result(passed=False, reason=f"judge error: {exc}")
+
+        predicted_raw = str(result.get("predicted", "")).strip().lower()
+        try:
+            predicted = Difficulty(predicted_raw)
+        except ValueError:
+            return self._result(
+                passed=True,
+                reason=f"unparseable difficulty: {predicted_raw!r}",
+                predicted=predicted_raw,
+            )
+
+        if pair.difficulty is None:
+            return self._result(passed=True, predicted=predicted.value)
+
+        gap = abs(_LEVELS[pair.difficulty] - _LEVELS[predicted])
+        if not self.strict or gap < 2:
+            return self._result(
+                passed=True,
+                labeled=pair.difficulty.value,
+                predicted=predicted.value,
+                gap=gap,
+            )
+
+        return self._result(
+            passed=False,
+            reason=(
+                f"labeled difficulty {pair.difficulty.value!r} is two steps "
+                f"away from judge prediction {predicted.value!r}"
+            ),
+            labeled=pair.difficulty.value,
+            predicted=predicted.value,
+            gap=gap,
+        )

evalseed/filters/faithfulness.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from evalseed.exceptions import JudgeAuthError, JudgeError
+from evalseed.filters.base import Filter
+from evalseed.judges import Judge
+from evalseed.schemas import FilterResult, QAPair
+
+_SYSTEM = (
+    "You are a strict evaluator of factual grounding. You decide whether an "
+    "answer is fully supported by the provided context, with no information "
+    "added from outside the context. Respond ONLY with JSON."
+)
+
+_USER_TEMPLATE = """Evaluate whether the ANSWER is fully entailed by the CONTEXT for the QUESTION.
+
+CONTEXT:
+\"\"\"
+{context}
+\"\"\"
+
+QUESTION: {question}
+
+ANSWER: {answer}
+
+Return JSON with this exact schema:
+{{
+  "faithful": <true|false>,
+  "score": <float between 0 and 1>,
+  "reason": "<one short sentence>"
+}}
+
+Rules:
+- "faithful" must be false if any factual claim in the answer is not supported by the context.
+- "faithful" must be false if the answer adds quantitative details, dates, or names not in the context.
+- Paraphrase is OK as long as every claim is supported.
+"""
+
+
+class FaithfulnessFilter(Filter):
+    """Rejects pairs where the answer cannot be entailed from the context."""
+
+    name = "faithfulness"
+
+    def __init__(self, judge: Judge, threshold: float = 0.7) -> None:
+        super().__init__(judge)
+        self.threshold = threshold
+
+    def evaluate(self, pair: QAPair) -> FilterResult:
+        try:
+            result = self.judge.judge(
+                _SYSTEM,
+                _USER_TEMPLATE.format(
+                    context=pair.context,
+                    question=pair.question,
+                    answer=pair.answer,
+                ),
+            )
+        except JudgeAuthError:
+            raise
+        except JudgeError as exc:
+            return self._result(passed=False, reason=f"judge error: {exc}")
+
+        faithful = bool(result.get("faithful", False))
+        score_raw = result.get("score", 0.0)
+        try:
+            score = float(score_raw)
+        except (TypeError, ValueError):
+            score = 0.0
+        reason = str(result.get("reason", "")).strip() or None
+        passed = faithful and score >= self.threshold
+        return self._result(passed=passed, score=score, reason=reason if not passed else None)