ragradar-evaluate 0.1.0__py3-none-any.whl

ragradar_evaluate/__init__.py

@@ -0,0 +1,43 @@
+from ragradar_core.schema import (
+    CacheEvent,
+    ChunkRecord,
+    RunRecord,
+    TokenBudget,
+    TokenUsage,
+    ToolCallRecord,
+    Turn,
+)
+
+from ragradar_evaluate.facade import (
+    CheckResult,
+    EvalResult,
+    MetricInfo,
+    available_metrics,
+    check,
+    evaluate,
+)
+from ragradar_evaluate.policy.schema import InputQualityPolicy
+
+# NOTE: benchmark machinery (seeding, building, checking, exporting) is
+# internal — the CLI's `benchmark` commands drive it, and check() consults
+# learned thresholds automatically. It is deliberately absent here.
+
+__all__ = [
+    # User tasks
+    "check",
+    "evaluate",
+    "available_metrics",
+    # Result / config types
+    "CheckResult",
+    "EvalResult",
+    "MetricInfo",
+    "InputQualityPolicy",
+    # Re-exported schema dataclasses so users need only one import.
+    "ChunkRecord",
+    "TokenBudget",
+    "TokenUsage",
+    "Turn",
+    "CacheEvent",
+    "ToolCallRecord",
+    "RunRecord",
+]

ragradar_evaluate/benchmark/builder.py

@@ -0,0 +1,117 @@
+import json
+
+from ragradar_core import store
+from scipy import stats
+
+INPUT_FACTORS = [
+    "duplicate_ratio",
+    "top_chunk_score",
+    "high_score_truncations",
+    "token_headroom_pct",
+    "source_domain_count",
+    "low_score_chunk_ratio",
+    "mean_relevance",
+    "truncated_count",
+    "score_variance",
+]
+
+RAGAS_METRICS = ["faithfulness", "answer_relevancy"]
+
+
+def _suggest_threshold(values: list[float], ragas_scores: list[float]) -> float:
+    sorted_vals = sorted(set(values))
+    if len(sorted_vals) < 2:
+        return sorted_vals[0] if sorted_vals else 0.0
+
+    best_threshold = sorted_vals[0]
+    best_diff = 0.0
+
+    for i in range(len(sorted_vals) - 1):
+        threshold = (sorted_vals[i] + sorted_vals[i + 1]) / 2
+        below = [r for v, r in zip(values, ragas_scores) if v <= threshold]
+        above = [r for v, r in zip(values, ragas_scores) if v > threshold]
+
+        if not below or not above:
+            continue
+
+        diff = abs(sum(above) / len(above) - sum(below) / len(below))
+        if diff > best_diff:
+            best_diff = diff
+            best_threshold = threshold
+
+    return round(best_threshold, 4)
+
+
+def build(pipeline: str | None = None) -> dict:
+    runs = store.get_all_evaluated_runs(pipeline)
+
+    if len(runs) < 10:
+        raise ValueError(f"Need at least 10 evaluated runs to build benchmark, found {len(runs)}.")
+
+    parsed = []
+    for r in runs:
+        parsed.append(json.loads(r["eval_scores"]))
+
+    pipeline_key = pipeline or "__default"
+    factors_result = {}
+    batch_entries: list[tuple] = []
+
+    for factor in INPUT_FACTORS:
+        factor_values: list[float] = []
+        ragas_values: dict[str, list[float]] = {m: [] for m in RAGAS_METRICS}
+
+        for eval_data in parsed:
+            input_data = eval_data.get("input") or {}
+            output_data = eval_data.get("output") or {}
+
+            fval = input_data.get(factor)
+            if fval is None:
+                continue
+
+            has_ragas = any(output_data.get(m) is not None for m in RAGAS_METRICS)
+            if not has_ragas:
+                continue
+
+            factor_values.append(float(fval))
+            for m in RAGAS_METRICS:
+                ragas_values[m].append(float(output_data.get(m) or 0.0))
+
+        if len(factor_values) < 3:
+            continue
+
+        correlations: dict[str, float | None] = {}
+        for m in RAGAS_METRICS:
+            vals = ragas_values[m]
+            if (
+                len(vals) == len(factor_values)
+                and len(set(factor_values)) > 1
+                and len(set(vals)) > 1
+            ):
+                corr, _ = stats.pearsonr(factor_values, vals)
+                # float(): scipy stubs expose corr as numpy.float64 (_T_co), not float;
+                # the cast is the narrowing workaround — no type: ignore needed.
+                correlations[f"{m}_correlation"] = round(float(corr), 4)
+            else:
+                correlations[f"{m}_correlation"] = None
+
+        valid_corrs = [v for v in correlations.values() if v is not None]
+        primary_corr = max(valid_corrs, key=abs) if valid_corrs else 0.0
+
+        primary_ragas = ragas_values[RAGAS_METRICS[0]]
+        suggested = _suggest_threshold(factor_values, primary_ragas)
+
+        batch_entries.append((pipeline_key, factor, suggested, primary_corr, len(factor_values)))
+
+        factors_result[factor] = {
+            **correlations,
+            "suggested_threshold": suggested,
+            "sample_count": len(factor_values),
+        }
+
+    store.write_benchmark_entries_batch(batch_entries)
+
+    return {
+        "run_count": len(runs),
+        "pipeline": pipeline,
+        "factors": factors_result,
+    }

ragradar_evaluate/benchmark/checker.py

@@ -0,0 +1,79 @@
+import json
+
+from ragradar_core import store
+from ragradar_core.schema import RunRecord
+
+from ragradar_evaluate.layers import input_quality
+from ragradar_evaluate.policy.persistence import load_policy
+
+
+def check(
+    session_id: int,
+    run_seq: int,
+    pipeline: str | None = None,
+) -> dict:
+    run_row = store.get_run(session_id, run_seq)
+    if run_row is None:
+        raise ValueError(f"Run s{session_id}r{run_seq} not found.")
+
+    record = RunRecord.from_json(json.loads(run_row["run_data"]))
+    pipeline = pipeline or run_row["pipeline"] or "__default"
+    policy = load_policy(pipeline)
+
+    input_scores = input_quality.score_input_quality(record, policy)
+    benchmark = store.get_benchmark(pipeline)
+    benchmark_map = {b["factor"]: b for b in benchmark}
+
+    factors = {}
+    fail_count = 0
+
+    check_factors = [
+        ("duplicate_ratio", "higher_bad"),
+        ("top_chunk_score", "lower_bad"),
+        ("high_score_truncations", "higher_bad"),
+        ("token_headroom_pct", "lower_bad"),
+        ("source_domain_count", "higher_bad"),
+        ("low_score_chunk_ratio", "higher_bad"),
+    ]
+
+    for factor, direction in check_factors:
+        value = input_scores.get(factor) if input_scores else None
+        bench = benchmark_map.get(factor)
+        threshold = bench["threshold"] if bench else None
+
+        if value is None or threshold is None:
+            status = "ok"
+        elif direction == "lower_bad":
+            status = "fail" if value < threshold else "ok"
+        else:
+            status = "fail" if value > threshold else "ok"
+
+        if status == "fail":
+            fail_count += 1
+
+        factors[factor] = {
+            "value": value,
+            "benchmark_threshold": threshold,
+            "status": status,
+        }
+
+    # risk_score is None when the run was never evaluated or its input
+    # metrics could not be computed (0.0 strictly means "computed, no
+    # risk") — unknown risk never counts toward the verdict.
+    eval_data = store.get_eval_scores(session_id, run_seq)
+    risk = eval_data.get("risk_score") if eval_data else None
+
+    if (risk is not None and risk > 0.7) or fail_count >= 3:
+        overall = "fail"
+    elif fail_count >= 1:
+        overall = "warn"
+    else:
+        overall = "ok"
+
+    return {
+        "run_id": f"s{session_id}r{run_seq}",
+        "risk_score": risk,
+        "benchmark_available": len(benchmark) > 0,
+        "factors": factors,
+        "overall": overall,
+    }

ragradar_evaluate/benchmark/exporter.py

@@ -0,0 +1,43 @@
+import json
+from datetime import datetime
+from pathlib import Path
+
+from ragradar_core import store
+
+
+def export(pipeline: str | None = None, output_path: Path | None = None) -> Path:
+    runs = store.get_all_evaluated_runs(pipeline)
+
+    records = []
+    for r in runs:
+        if r["pipeline"] and r["pipeline"].endswith("__seeded"):
+            continue
+        run_data = json.loads(r["run_data"])
+        if not run_data.get("chunks") or not run_data.get("response"):
+            continue
+        records.append((r, run_data))
+
+    if output_path is None:
+        pipe_name = pipeline or "all"
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        exports_dir = store._ragradar_dir() / "exports"
+        exports_dir.mkdir(parents=True, exist_ok=True)
+        output_path = exports_dir / f"{pipe_name}_ragas_{timestamp}.jsonl"
+    else:
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        for row, run_data in records:
+            entry = {
+                "question": run_data["query"],
+                "answer": run_data["response"],
+                "contexts": [c["content"] for c in run_data.get("chunks", [])],
+                "ground_truth": None,
+                "run_id": f"s{row['session_id']}r{row['run_seq']}",
+                "pipeline": row["pipeline"],
+                "evaluated_at": row["evaluated_at"],
+            }
+            f.write(json.dumps(entry) + "\n")
+
+    return output_path

ragradar_evaluate/benchmark/seeder.py

@@ -0,0 +1,89 @@
+import random
+
+from ragradar_core import store
+from ragradar_core.schema import ChunkRecord, RunRecord, TokenBudget, Turn
+
+
+def seed(pipeline: str, count: int = 20) -> int:
+    """Generate synthetic run records as day-zero baseline.
+
+    Half known-good, half known-bad. Seeded runs do NOT have RAGAS scores --
+    they serve as input quality baseline only.
+    """
+    seeded_pipeline = f"{pipeline}__seeded"
+    half = count // 2
+
+    records = [_good_record(i) for i in range(half)]
+    records += [_bad_record(i) for i in range(count - half)]
+
+    session_id = store.get_or_create_session(seeded_pipeline)
+    start_seq = store.next_run_seq(session_id)
+    store.write_runs_batch(session_id, start_seq, records, seeded_pipeline)
+
+    return count
+
+
+def _good_record(idx: int) -> RunRecord:
+    chunks = [
+        ChunkRecord(
+            chunk_id=f"seed_c{idx}_{j}",
+            source_doc_id=f"seed_doc_{j % 3}",
+            content=f"Synthetic high-quality chunk content for topic {idx}, variant {j}.",
+            token_count=150,
+            retrieval_score=0.85 + random.uniform(0, 0.1),
+            rerank_score=0.90 + random.uniform(0, 0.08),
+            retrieval_path="hybrid",
+            truncated=False,
+            cache_hit=True,
+        )
+        for j in range(4)
+    ]
+    return RunRecord(
+        query=f"Synthetic good query {idx}: what is the best practice?",
+        response=f"Synthetic good response {idx}: comprehensive answer.",
+        chunks=chunks,
+        token_budget=TokenBudget(
+            total_limit=4096,
+            chunks_allocated=2400,
+            history_allocated=400,
+            system_allocated=600,
+            headroom=696,
+        ),
+        history_pre=[
+            Turn(role="user", content="context question", tokens=5),
+            Turn(role="assistant", content="context answer", tokens=10),
+        ],
+        history_post=[
+            Turn(role="user", content="context question", tokens=5),
+            Turn(role="assistant", content="context answer", tokens=10),
+        ],
+    )
+
+
+def _bad_record(idx: int) -> RunRecord:
+    chunks = [
+        ChunkRecord(
+            chunk_id=f"seed_bad_c{idx}_{j}",
+            source_doc_id=f"seed_doc_{j % 8}",
+            content=f"Low quality chunk {idx}_{j}.",
+            token_count=200,
+            retrieval_score=0.3 + random.uniform(0, 0.15),
+            rerank_score=0.25 + random.uniform(0, 0.15),
+            retrieval_path="bm25",
+            truncated=(j % 2 == 0),
+            cache_hit=False,
+        )
+        for j in range(6)
+    ]
+    return RunRecord(
+        query=f"Synthetic bad query {idx}: vague unclear question?",
+        response=f"Synthetic bad response {idx}: incomplete.",
+        chunks=chunks,
+        token_budget=TokenBudget(
+            total_limit=4096,
+            chunks_allocated=3800,
+            history_allocated=100,
+            system_allocated=100,
+            headroom=96,
+        ),
+    )