turkish-rag-eval 0.2.0__py3-none-any.whl

turkish_rag_eval/__init__.py

@@ -0,0 +1,14 @@
+"""turkish-rag-eval: which parts of a Turkish RAG pipeline actually pay off.
+
+The package is a retrieval measurement harness. Point it at a corpus and a set
+of labelled questions and it reports, with confidence intervals, what each
+pipeline choice - chunking strategy, stemming, dense model, fusion - is worth
+and what it costs.
+
+Every published number is tagged with this version, because a benchmark whose
+results cannot be reproduced against a fixed release is not comparable.
+"""
+
+__version__ = "0.2.0"
+
+__all__ = ["__version__"]

turkish_rag_eval/abstain.py

@@ -0,0 +1,122 @@
+"""Decide when retrieval is too weak to answer, and escalate to a human.
+
+In a regulated domain the useful question is not "how accurate is the system"
+but "how accurate is it on the queries it chose to answer, and how many did it
+choose to answer". Those move against each other, so this module sweeps the
+threshold and reports the trade-off instead of picking one number and hiding
+the curve.
+
+Two confidence signals are measured, because the obvious one fails:
+
+margin  - normalised gap between the top hit and the runner-up. Intuitive,
+          and useless on RRF output: fused scores are 1/(60+rank), so the
+          top-two gap is ~2% for every query, confident or not.
+score   - the dense retriever's raw top-1 cosine similarity. An absolute
+          measure of how close anything in the corpus came to the query,
+          which is what an abstain decision actually needs.
+
+The comparison is the point. See results/abstain_curve_*.json.
+"""
+
+import argparse
+import json
+
+from .paths import ROOT
+
+RESULTS = ROOT / "results"
+SIGNALS = ("margin", "score")
+
+
+def margin_confidence(hits: list[tuple[int, float]]) -> float:
+    """Normalised top-1 margin in [0, 1]. 0 when the top two are tied."""
+    if not hits:
+        return 0.0
+    top = hits[0][1]
+    if top <= 0:
+        return 0.0
+    runner_up = hits[1][1] if len(hits) > 1 else 0.0
+    return max(0.0, (top - runner_up) / top)
+
+
+def score_confidence(hits: list[tuple[int, float]]) -> float:
+    """Raw top-1 score, clamped to [0, 1]. Meaningful for cosine similarity."""
+    if not hits:
+        return 0.0
+    return max(0.0, min(1.0, hits[0][1]))
+
+
+def sweep(records: list[dict], signal: str, steps: int = 21) -> list[dict]:
+    """Coverage / selective-accuracy curve over thresholds of one signal."""
+    curve = []
+    for i in range(steps):
+        threshold = i / (steps - 1)
+        answered = [r for r in records if r[signal] >= threshold]
+        correct = sum(1 for r in answered if r["correct"])
+        curve.append({
+            "threshold": round(threshold, 3),
+            "coverage": len(answered) / len(records) if records else 0.0,
+            "selective_accuracy": correct / len(answered) if answered else None,
+            "answered": len(answered),
+            "escalated": len(records) - len(answered),
+        })
+    return curve
+
+
+def pick_threshold(curve: list[dict], min_accuracy: float,
+                   min_answered: int = 5) -> dict | None:
+    """Highest-coverage threshold that still clears an accuracy floor.
+
+    None means no threshold reaches the floor - a real answer, not a
+    failure: the system should not run unattended at that requirement.
+    """
+    viable = [p for p in curve
+              if p["selective_accuracy"] is not None
+              and p["selective_accuracy"] >= min_accuracy
+              and p["answered"] >= min_answered]
+    return max(viable, key=lambda p: p["coverage"]) if viable else None
+
+
+def report(records: list[dict]) -> None:
+    baseline = sum(1 for r in records if r["correct"]) / len(records)
+    print(f"{len(records)} sorgu | esiksiz top-1 dogrulugu {baseline:.3f}\n")
+
+    for signal in SIGNALS:
+        curve = sweep(records, signal)
+        (RESULTS / f"abstain_curve_{signal}.json").write_text(
+            json.dumps(curve, ensure_ascii=False, indent=2), encoding="utf-8")
+
+        print(f"--- sinyal: {signal} ---")
+        print(f"{'esik':>6} {'kapsam':>8} {'dogruluk':>9} {'cevaplanan':>11} {'insana':>7}")
+        for point in curve:
+            if point["answered"] == 0:
+                continue
+            accuracy = f"{point['selective_accuracy']:.3f}"
+            print(f"{point['threshold']:>6.2f} {point['coverage']:>8.3f} "
+                  f"{accuracy:>9} {point['answered']:>11} {point['escalated']:>7}")
+
+        for floor in (0.70, 0.80, 0.90):
+            choice = pick_threshold(curve, floor)
+            if choice is None:
+                print(f"  >= {floor:.0%} dogruluk: hicbir esik saglamiyor")
+            else:
+                print(f"  >= {floor:.0%} dogruluk -> esik {choice['threshold']:.2f}: "
+                      f"{choice['coverage']:.0%} otomatik, "
+                      f"{choice['escalated']} soru insana")
+        print()
+
+
+def run(args=None) -> int:
+    path = RESULTS / "abstain_records.json"
+    if not path.exists():
+        raise SystemExit("once run_abstain.py calistirilmali")
+    report(json.loads(path.read_text(encoding="utf-8")))
+    return 0
+
+
+def main() -> int:
+    argparse.ArgumentParser(description=__doc__).parse_args()
+    return run(None)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

turkish_rag_eval/agreement.py

@@ -0,0 +1,233 @@
+"""Inter-annotator agreement for second-annotated questions.
+
+A second annotation reuses the original ``question``, ``doc_id`` and
+``doc_title`` but is written by a different ``annotator`` without looking at
+the first answer span (see CONTRIBUTING.md), and links back with
+``"second_of": "<original qid>"``. A question can end up with more than two
+annotators, so agreement is computed over every pairwise combination and
+reported as the mean per question - not just the first pair - before being
+averaged again, equally per question, into the headline numbers.
+
+Two measurements, decided in the issue #12 discussion:
+
+iou      - Jaccard similarity of the two spans' token sets. The headline
+           number: one symmetric score that, unlike F1, correctly charges the
+           extra material when one span is a superset of the other (the
+           common case of two annotators bracketing the same fact with
+           different amounts of surrounding clause).
+token_f1 - reported alongside IoU because span-extraction/QA evaluation
+           traditionally looks for it.
+kappa    - Cohen's kappa, per chunking strategy, over the binary "is this
+           chunk relevant" label each span would assign to each chunk of the
+           source document (same doc_id and body contains the span, exactly
+           the definition run_eval.is_relevant uses to score retrievers).
+           Kappa, not IoU, is used here because most chunks are irrelevant to
+           both annotators - raw or set-based overlap would be inflated by
+           that chance agreement, which kappa corrects for.
+
+All three use `turkish_text.tokenize`, so the casing / I-ı folding is
+consistent with the rest of the harness. Only the standard library is used,
+so this runs without the heavier retrieval dependencies (numpy,
+sentence-transformers) that run_eval.py needs.
+"""
+
+import argparse
+import json
+from collections import Counter
+from itertools import combinations
+from pathlib import Path
+
+from .chunking import STRATEGIES
+from .gold import ROOT, load_gold, normalise
+from .turkish_text import tokenize, turkish_lower
+
+CORPUS = ROOT / "data" / "raw" / "corpus.json"
+
+
+def is_relevant(chunk: dict, item: dict) -> bool:
+    """Same definition as run_eval.is_relevant, kept dependency-free here."""
+    if chunk["doc_id"] != item["doc_id"]:
+        return False
+    return normalise(item["answer_span"]) in normalise(chunk["body"])
+
+
+def group_by_question(items: list[dict]) -> list[list[dict]]:
+    """Every item answering the same question, keyed by normalised text.
+
+    A second annotation is required to reuse the question unchanged (see
+    validate_gold.py), so this alone finds every annotator of a question -
+    ``second_of`` does not need to be followed to group them.
+    """
+    groups: dict[str, list[dict]] = {}
+    for item in items:
+        key = normalise(turkish_lower(item["question"]))
+        groups.setdefault(key, []).append(item)
+    return [group for group in groups.values() if len(group) >= 2]
+
+
+def root_qid(group: list[dict]) -> str:
+    """The qid other items in the group point at with ``second_of``."""
+    for item in group:
+        if not isinstance(item.get("second_of"), str):
+            return item["qid"]
+    return min(item["qid"] for item in group)
+
+
+def pairwise_items(group: list[dict]) -> list[tuple[dict, dict]]:
+    """Every combination in the group with independent annotators."""
+    return [(a, b) for a, b in combinations(group, 2)
+            if a.get("annotator") != b.get("annotator")]
+
+
+def span_iou(a: str, b: str) -> float:
+    """Jaccard similarity of the two spans' token sets."""
+    tokens_a, tokens_b = set(tokenize(a)), set(tokenize(b))
+    union = tokens_a | tokens_b
+    return len(tokens_a & tokens_b) / len(union) if union else 1.0
+
+
+def span_f1(a: str, b: str) -> float:
+    """SQuAD-style token F1 between two raw answer spans."""
+    tokens_a, tokens_b = tokenize(a), tokenize(b)
+    if not tokens_a or not tokens_b:
+        return float(tokens_a == tokens_b)
+    overlap = sum((Counter(tokens_a) & Counter(tokens_b)).values())
+    if overlap == 0:
+        return 0.0
+    precision = overlap / len(tokens_a)
+    recall = overlap / len(tokens_b)
+    return 2 * precision * recall / (precision + recall)
+
+
+def cohens_kappa(labels_a: list[bool], labels_b: list[bool]) -> float | None:
+    """Chance-corrected agreement between two binary label lists."""
+    n = len(labels_a)
+    if n == 0:
+        return None
+    observed = sum(a == b for a, b in zip(labels_a, labels_b)) / n
+    rate_a, rate_b = sum(labels_a) / n, sum(labels_b) / n
+    expected = rate_a * rate_b + (1 - rate_a) * (1 - rate_b)
+    if expected >= 1.0:
+        return 1.0
+    return (observed - expected) / (1 - expected)
+
+
+def chunk_kappa(item_a: dict, item_b: dict, doc: dict) -> dict[str, float | None]:
+    result = {}
+    for name, chunker in STRATEGIES.items():
+        chunks = chunker(doc)
+        labels_a = [is_relevant(c, item_a) for c in chunks]
+        labels_b = [is_relevant(c, item_b) for c in chunks]
+        result[name] = cohens_kappa(labels_a, labels_b)
+    return result
+
+
+def evaluate_groups(groups: list[list[dict]],
+                     docs_by_id: dict[str, dict]) -> list[dict]:
+    """One row per question, averaging every annotator pair it has."""
+    rows = []
+    for group in groups:
+        pairs = pairwise_items(group)
+        if not pairs:
+            continue
+        pair_metrics = []
+        for a, b in pairs:
+            doc = docs_by_id.get(a["doc_id"])
+            pair_metrics.append({
+                "annotators": (a.get("annotator", "gold"), b.get("annotator", "gold")),
+                "iou": span_iou(a["answer_span"], b["answer_span"]),
+                "token_f1": span_f1(a["answer_span"], b["answer_span"]),
+                "kappa": chunk_kappa(a, b, doc) if doc is not None else {},
+            })
+        rows.append({
+            "qid": root_qid(group),
+            "question": group[0]["question"],
+            "annotators": sorted({m for p in pair_metrics for m in p["annotators"]}),
+            "pairs": len(pairs),
+            "mean_iou": _mean(p["iou"] for p in pair_metrics),
+            "mean_token_f1": _mean(p["token_f1"] for p in pair_metrics),
+            "mean_kappa": {
+                name: _mean_or_none(p["kappa"].get(name) for p in pair_metrics
+                                     if p["kappa"].get(name) is not None)
+                for name in STRATEGIES
+            },
+            "pair_details": pair_metrics,
+        })
+    return rows
+
+
+def summarise(rows: list[dict]) -> dict:
+    """Headline numbers: each question counts once, regardless of how many
+    annotators (and therefore pairs) it has."""
+    summary = {
+        "questions": len(rows),
+        "pairs": sum(r["pairs"] for r in rows),
+        "mean_iou": _mean(r["mean_iou"] for r in rows),
+        "mean_token_f1": _mean(r["mean_token_f1"] for r in rows),
+    }
+    for name in STRATEGIES:
+        summary[f"mean_kappa_{name}"] = _mean_or_none(
+            r["mean_kappa"][name] for r in rows if r["mean_kappa"][name] is not None)
+    return summary
+
+
+def _mean(values) -> float:
+    values = list(values)
+    return sum(values) / len(values) if values else 0.0
+
+
+def _mean_or_none(values) -> float | None:
+    values = list(values)
+    return sum(values) / len(values) if values else None
+
+
+def add_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--corpus", default=str(CORPUS))
+
+
+def run(args) -> int:
+    groups = group_by_question(load_gold())
+    if not groups:
+        print("birden fazla etiketleyicisi olan soru yok")
+        return 0
+
+    corpus_path = Path(args.corpus)
+    if corpus_path.exists():
+        docs = json.loads(corpus_path.read_text(encoding="utf-8"))
+    else:
+        print(f"uyarı: {corpus_path} yok - parça düzeyinde kappa atlanıyor")
+        docs = []
+    docs_by_id = {d["doc_id"]: d for d in docs}
+
+    rows = evaluate_groups(groups, docs_by_id)
+    summary = summarise(rows)
+
+    for row in rows:
+        kappa = ", ".join(f"{k}={v:.2f}" for k, v in row["mean_kappa"].items()
+                           if v is not None)
+        print(f"{row['qid']} ({', '.join(row['annotators'])}, {row['pairs']} çift): "
+              f"iou={row['mean_iou']:.2f} token_f1={row['mean_token_f1']:.2f}"
+              + (f", kappa: {kappa}" if kappa else ""))
+    print(f"\n{summary['questions']} soru, {summary['pairs']} çift, "
+          f"ortalama IoU={summary['mean_iou']:.3f} "
+          f"ortalama token_f1={summary['mean_token_f1']:.3f}")
+    for name in STRATEGIES:
+        kappa = summary[f"mean_kappa_{name}"]
+        print(f"  ortalama kappa ({name}): "
+              + (f"{kappa:.3f}" if kappa is not None else "n/a"))
+
+    out = ROOT / "results" / "agreement.json"
+    out.parent.mkdir(exist_ok=True)
+    out.write_text(json.dumps({"summary": summary, "questions": rows},
+                               ensure_ascii=False, indent=2), encoding="utf-8")
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    add_arguments(parser)
+    return run(parser.parse_args())
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())