ragradar-evaluate 0.1.0__tar.gz

ragradar_evaluate-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+dist/
+build/
+*.so
+
+# uv
+.uv/
+uv.lock
+
+# ragradar runtime — never commit user run data
+.ragradar/
+
+# environment
+.env
+*.env
+.env.*
+
+# IDE
+.vscode/
+.idea/
+*.swp
+
+# OS
+.DS_Store
+Thumbs.db
+
+# test output
+.pytest_cache/
+htmlcov/
+.coverage
+
+# example output
+examples/rag_pipeline/output/
+.claude/

ragradar_evaluate-0.1.0/PKG-INFO

@@ -0,0 +1,162 @@
+Metadata-Version: 2.4
+Name: ragradar-evaluate
+Version: 0.1.0
+Summary: Evaluation layer for ragradar observability system
+Project-URL: Homepage, https://github.com/pleokarthik/RAGRadar
+Project-URL: Repository, https://github.com/pleokarthik/RAGRadar
+Project-URL: Issues, https://github.com/pleokarthik/RAGRadar/issues
+Author-email: Leo Karthik Paramasivan <pleokarthik@gmail.com>
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: click>=8.0
+Requires-Dist: ragas>=0.2
+Requires-Dist: ragradar-core<0.2.0,>=0.1.0
+Requires-Dist: rich>=13.0
+Requires-Dist: scipy>=1.13
+Provides-Extra: ollama
+Requires-Dist: ollama>=0.2; extra == 'ollama'
+Provides-Extra: transformers
+Requires-Dist: sentence-transformers>=3.0; extra == 'transformers'
+Description-Content-Type: text/markdown
+
+# ragradar-evaluate
+
+Scores captured runs. Two tasks, one discovery helper:
+
+| Task | Call | Cost |
+|---|---|---|
+| "Is this run healthy?" | `check(run_id)` | free — deterministic, no LLM, instant |
+| "Score it fully" | `evaluate(run_id)` | free input metrics + LLM-judged output metrics |
+| "What can be scored?" | `available_metrics()` | free |
+
+```
+pip install ragradar-evaluate
+```
+
+## Is this run healthy? — check()
+
+Call before paying for an LLM; put it in CI.
+
+```python
+import ragradar_capture
+from ragradar_evaluate import check
+
+run_id = ragradar_capture.capture(
+    "what is RRF?", "RRF fuses rankings.",
+    chunks=[{"chunk_id": "c1", "source_doc_id": "d1",
+             "content": "RRF combines rankings.", "token_count": 10,
+             "rerank_score": 0.9}],
+)
+result = check(run_id)
+
+print(result.verdict)      # "ok" | "warn" | "fail"
+print(result.problems)     # ["duplicate chunks: ratio 0.50 exceeds 0.20", ...]
+print(result.risk_score)   # 0.0-1.0, None if input quality couldn't be assessed
+print(result.factors)      # per-factor {value, threshold, status}
+print(result.thresholds)   # "learned" | "policy" — which standards were applied
+```
+
+`check()` compares all free input metrics against the **current
+standards**: once at least 10 evaluated runs exist for the pipeline it
+uses thresholds learned from your own history (and says so via
+`thresholds == "learned"`); before that it falls back to the policy
+defaults. A run captured without chunks gets a `warn` verdict explaining
+the missing data — never an exception.
+
+## Score it fully — evaluate()
+
+```python
+import ragradar_capture
+from ragradar_evaluate import evaluate
+
+run_id = ragradar_capture.capture(
+    "what is RRF?", "RRF fuses rankings.",
+    chunks=[{"chunk_id": "c1", "source_doc_id": "d1",
+             "content": "RRF combines rankings.", "token_count": 10,
+             "rerank_score": 0.9}],
+)
+
+# Complete eval: every metric applicable to the record.
+result = evaluate(run_id)
+
+# One atomic metric — nothing else is computed:
+result = evaluate(run_id, metrics=["duplicates"], save=False)
+
+# A chosen subset:
+result = evaluate(run_id, metrics=["relevance", "faithfulness"])
+```
+
+`target` can be an sNrN string (what `ragradar_capture.capture()` returns), a
+committed `Capture` object, or a bare `RunRecord` (then pass
+`save=False` — there's no run row to write to).
+
+### EvalResult
+
+| Field | Meaning |
+|---|---|
+| `metrics` | per-metric results: a dict of values for input families, a float for RAGAS metrics |
+| `skipped` | metric → reason: `"not requested"`, `"missing data: ..."`, or `"requires ground_truth"` |
+| `errors` | metric → error string; RAGAS-not-installed and RAGAS runtime failures land here identically — `evaluate()` never raises for judge failures |
+| `policy_violations` | policy thresholds breached by the computed values |
+| `risk_score` | `None` when input metrics weren't computed; `0.0` only ever means "computed, no risk" |
+| `run_id` / `saved` | identity and whether scores were persisted |
+
+`save=True` (default) persists via the one store path; `ragradar explain
+<run_id>` then shows the scores alongside its analysis.
+
+## available_metrics()
+
+| Metric | Layer | Cost | Requires |
+|---|---|---|---|
+| `relevance` | input | free | chunks |
+| `duplicates` | input | free | chunks |
+| `truncation` | input | free | chunks |
+| `token_efficiency` | input | free | chunks |
+| `coherence` | input | free | chunks |
+| `faithfulness` | output | llm | chunks, response |
+| `answer_relevancy` | output | llm | chunks, response |
+| `context_precision` | output | llm | chunks, response |
+| `context_recall` | output | llm | chunks, response, ground_truth |
+
+Output metrics are RAGAS LLM-as-judge calls — they cost money and need a
+configured judge. **To stay free-only**, use `check()`, or select input
+metrics explicitly: `evaluate(run_id, metrics=["relevance",
+"duplicates", "truncation", "token_efficiency", "coherence"])`.
+
+## Policy system
+
+Human-set thresholds encoding known failure modes; active from day one
+and the fallback standard for `check()`. Stored per pipeline.
+
+```bash
+ragradar-evaluate policy show
+ragradar-evaluate policy set max_duplicate_ratio 0.1
+ragradar-evaluate policy reset
+```
+
+Programmatic override: `evaluate(run_id, policy=InputQualityPolicy(...))`.
+
+## Benchmark lifecycle (CLI)
+
+Learned thresholds accumulate as you evaluate real runs — `check()`
+picks them up automatically at 10+ evaluated runs. The CLI exposes the
+machinery for inspection:
+
+```bash
+ragradar-evaluate run s2r3                 # evaluate one run (both layers)
+ragradar-evaluate run s2r3 --input-only    # free metrics only
+ragradar-evaluate run --session s2         # evaluate a whole session
+ragradar-evaluate benchmark show           # current learned thresholds
+ragradar-evaluate benchmark build          # rebuild from evaluated history
+ragradar-evaluate benchmark check s2r3     # factor-by-factor threshold check
+ragradar-evaluate benchmark export         # RAGAS-compatible JSONL dataset
+```
+
+`--input-only --output-only` together is an error (it would compute
+nothing).

ragradar_evaluate-0.1.0/README.md

@@ -0,0 +1,135 @@
+# ragradar-evaluate
+
+Scores captured runs. Two tasks, one discovery helper:
+
+| Task | Call | Cost |
+|---|---|---|
+| "Is this run healthy?" | `check(run_id)` | free — deterministic, no LLM, instant |
+| "Score it fully" | `evaluate(run_id)` | free input metrics + LLM-judged output metrics |
+| "What can be scored?" | `available_metrics()` | free |
+
+```
+pip install ragradar-evaluate
+```
+
+## Is this run healthy? — check()
+
+Call before paying for an LLM; put it in CI.
+
+```python
+import ragradar_capture
+from ragradar_evaluate import check
+
+run_id = ragradar_capture.capture(
+    "what is RRF?", "RRF fuses rankings.",
+    chunks=[{"chunk_id": "c1", "source_doc_id": "d1",
+             "content": "RRF combines rankings.", "token_count": 10,
+             "rerank_score": 0.9}],
+)
+result = check(run_id)
+
+print(result.verdict)      # "ok" | "warn" | "fail"
+print(result.problems)     # ["duplicate chunks: ratio 0.50 exceeds 0.20", ...]
+print(result.risk_score)   # 0.0-1.0, None if input quality couldn't be assessed
+print(result.factors)      # per-factor {value, threshold, status}
+print(result.thresholds)   # "learned" | "policy" — which standards were applied
+```
+
+`check()` compares all free input metrics against the **current
+standards**: once at least 10 evaluated runs exist for the pipeline it
+uses thresholds learned from your own history (and says so via
+`thresholds == "learned"`); before that it falls back to the policy
+defaults. A run captured without chunks gets a `warn` verdict explaining
+the missing data — never an exception.
+
+## Score it fully — evaluate()
+
+```python
+import ragradar_capture
+from ragradar_evaluate import evaluate
+
+run_id = ragradar_capture.capture(
+    "what is RRF?", "RRF fuses rankings.",
+    chunks=[{"chunk_id": "c1", "source_doc_id": "d1",
+             "content": "RRF combines rankings.", "token_count": 10,
+             "rerank_score": 0.9}],
+)
+
+# Complete eval: every metric applicable to the record.
+result = evaluate(run_id)
+
+# One atomic metric — nothing else is computed:
+result = evaluate(run_id, metrics=["duplicates"], save=False)
+
+# A chosen subset:
+result = evaluate(run_id, metrics=["relevance", "faithfulness"])
+```
+
+`target` can be an sNrN string (what `ragradar_capture.capture()` returns), a
+committed `Capture` object, or a bare `RunRecord` (then pass
+`save=False` — there's no run row to write to).
+
+### EvalResult
+
+| Field | Meaning |
+|---|---|
+| `metrics` | per-metric results: a dict of values for input families, a float for RAGAS metrics |
+| `skipped` | metric → reason: `"not requested"`, `"missing data: ..."`, or `"requires ground_truth"` |
+| `errors` | metric → error string; RAGAS-not-installed and RAGAS runtime failures land here identically — `evaluate()` never raises for judge failures |
+| `policy_violations` | policy thresholds breached by the computed values |
+| `risk_score` | `None` when input metrics weren't computed; `0.0` only ever means "computed, no risk" |
+| `run_id` / `saved` | identity and whether scores were persisted |
+
+`save=True` (default) persists via the one store path; `ragradar explain
+<run_id>` then shows the scores alongside its analysis.
+
+## available_metrics()
+
+| Metric | Layer | Cost | Requires |
+|---|---|---|---|
+| `relevance` | input | free | chunks |
+| `duplicates` | input | free | chunks |
+| `truncation` | input | free | chunks |
+| `token_efficiency` | input | free | chunks |
+| `coherence` | input | free | chunks |
+| `faithfulness` | output | llm | chunks, response |
+| `answer_relevancy` | output | llm | chunks, response |
+| `context_precision` | output | llm | chunks, response |
+| `context_recall` | output | llm | chunks, response, ground_truth |
+
+Output metrics are RAGAS LLM-as-judge calls — they cost money and need a
+configured judge. **To stay free-only**, use `check()`, or select input
+metrics explicitly: `evaluate(run_id, metrics=["relevance",
+"duplicates", "truncation", "token_efficiency", "coherence"])`.
+
+## Policy system
+
+Human-set thresholds encoding known failure modes; active from day one
+and the fallback standard for `check()`. Stored per pipeline.
+
+```bash
+ragradar-evaluate policy show
+ragradar-evaluate policy set max_duplicate_ratio 0.1
+ragradar-evaluate policy reset
+```
+
+Programmatic override: `evaluate(run_id, policy=InputQualityPolicy(...))`.
+
+## Benchmark lifecycle (CLI)
+
+Learned thresholds accumulate as you evaluate real runs — `check()`
+picks them up automatically at 10+ evaluated runs. The CLI exposes the
+machinery for inspection:
+
+```bash
+ragradar-evaluate run s2r3                 # evaluate one run (both layers)
+ragradar-evaluate run s2r3 --input-only    # free metrics only
+ragradar-evaluate run --session s2         # evaluate a whole session
+ragradar-evaluate benchmark show           # current learned thresholds
+ragradar-evaluate benchmark build          # rebuild from evaluated history
+ragradar-evaluate benchmark check s2r3     # factor-by-factor threshold check
+ragradar-evaluate benchmark export         # RAGAS-compatible JSONL dataset
+```
+
+`--input-only --output-only` together is an error (it would compute
+nothing).

ragradar_evaluate-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "ragradar-evaluate"
+version = "0.1.0"
+description = "Evaluation layer for ragradar observability system"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [
+    { name = "Leo Karthik Paramasivan", email = "pleokarthik@gmail.com" },
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "ragradar-core>=0.1.0,<0.2.0",
+    "ragas>=0.2",
+    "scipy>=1.13",
+    "rich>=13.0",
+    "click>=8.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/pleokarthik/RAGRadar"
+Repository = "https://github.com/pleokarthik/RAGRadar"
+Issues = "https://github.com/pleokarthik/RAGRadar/issues"
+
+[project.optional-dependencies]
+ollama = [
+    "ollama>=0.2",
+]
+transformers = [
+    "sentence-transformers>=3.0",
+]
+
+[project.scripts]
+ragradar-evaluate = "ragradar_evaluate.cli:main"
+
+[tool.uv.sources]
+ragradar-core = { workspace = true }
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ragradar_evaluate"]

ragradar_evaluate-0.1.0/src/ragradar_evaluate/__init__.py

@@ -0,0 +1,43 @@
+from ragradar_core.schema import (
+    CacheEvent,
+    ChunkRecord,
+    RunRecord,
+    TokenBudget,
+    TokenUsage,
+    ToolCallRecord,
+    Turn,
+)
+
+from ragradar_evaluate.facade import (
+    CheckResult,
+    EvalResult,
+    MetricInfo,
+    available_metrics,
+    check,
+    evaluate,
+)
+from ragradar_evaluate.policy.schema import InputQualityPolicy
+
+# NOTE: benchmark machinery (seeding, building, checking, exporting) is
+# internal — the CLI's `benchmark` commands drive it, and check() consults
+# learned thresholds automatically. It is deliberately absent here.
+
+__all__ = [
+    # User tasks
+    "check",
+    "evaluate",
+    "available_metrics",
+    # Result / config types
+    "CheckResult",
+    "EvalResult",
+    "MetricInfo",
+    "InputQualityPolicy",
+    # Re-exported schema dataclasses so users need only one import.
+    "ChunkRecord",
+    "TokenBudget",
+    "TokenUsage",
+    "Turn",
+    "CacheEvent",
+    "ToolCallRecord",
+    "RunRecord",
+]

ragradar_evaluate-0.1.0/src/ragradar_evaluate/benchmark/builder.py

@@ -0,0 +1,117 @@
+import json
+
+from ragradar_core import store
+from scipy import stats
+
+INPUT_FACTORS = [
+    "duplicate_ratio",
+    "top_chunk_score",
+    "high_score_truncations",
+    "token_headroom_pct",
+    "source_domain_count",
+    "low_score_chunk_ratio",
+    "mean_relevance",
+    "truncated_count",
+    "score_variance",
+]
+
+RAGAS_METRICS = ["faithfulness", "answer_relevancy"]
+
+
+def _suggest_threshold(values: list[float], ragas_scores: list[float]) -> float:
+    sorted_vals = sorted(set(values))
+    if len(sorted_vals) < 2:
+        return sorted_vals[0] if sorted_vals else 0.0
+
+    best_threshold = sorted_vals[0]
+    best_diff = 0.0
+
+    for i in range(len(sorted_vals) - 1):
+        threshold = (sorted_vals[i] + sorted_vals[i + 1]) / 2
+        below = [r for v, r in zip(values, ragas_scores) if v <= threshold]
+        above = [r for v, r in zip(values, ragas_scores) if v > threshold]
+
+        if not below or not above:
+            continue
+
+        diff = abs(sum(above) / len(above) - sum(below) / len(below))
+        if diff > best_diff:
+            best_diff = diff
+            best_threshold = threshold
+
+    return round(best_threshold, 4)
+
+
+def build(pipeline: str | None = None) -> dict:
+    runs = store.get_all_evaluated_runs(pipeline)
+
+    if len(runs) < 10:
+        raise ValueError(f"Need at least 10 evaluated runs to build benchmark, found {len(runs)}.")
+
+    parsed = []
+    for r in runs:
+        parsed.append(json.loads(r["eval_scores"]))
+
+    pipeline_key = pipeline or "__default"
+    factors_result = {}
+    batch_entries: list[tuple] = []
+
+    for factor in INPUT_FACTORS:
+        factor_values: list[float] = []
+        ragas_values: dict[str, list[float]] = {m: [] for m in RAGAS_METRICS}
+
+        for eval_data in parsed:
+            input_data = eval_data.get("input") or {}
+            output_data = eval_data.get("output") or {}
+
+            fval = input_data.get(factor)
+            if fval is None:
+                continue
+
+            has_ragas = any(output_data.get(m) is not None for m in RAGAS_METRICS)
+            if not has_ragas:
+                continue
+
+            factor_values.append(float(fval))
+            for m in RAGAS_METRICS:
+                ragas_values[m].append(float(output_data.get(m) or 0.0))
+
+        if len(factor_values) < 3:
+            continue
+
+        correlations: dict[str, float | None] = {}
+        for m in RAGAS_METRICS:
+            vals = ragas_values[m]
+            if (
+                len(vals) == len(factor_values)
+                and len(set(factor_values)) > 1
+                and len(set(vals)) > 1
+            ):
+                corr, _ = stats.pearsonr(factor_values, vals)
+                # float(): scipy stubs expose corr as numpy.float64 (_T_co), not float;
+                # the cast is the narrowing workaround — no type: ignore needed.
+                correlations[f"{m}_correlation"] = round(float(corr), 4)
+            else:
+                correlations[f"{m}_correlation"] = None
+
+        valid_corrs = [v for v in correlations.values() if v is not None]
+        primary_corr = max(valid_corrs, key=abs) if valid_corrs else 0.0
+
+        primary_ragas = ragas_values[RAGAS_METRICS[0]]
+        suggested = _suggest_threshold(factor_values, primary_ragas)
+
+        batch_entries.append((pipeline_key, factor, suggested, primary_corr, len(factor_values)))
+
+        factors_result[factor] = {
+            **correlations,
+            "suggested_threshold": suggested,
+            "sample_count": len(factor_values),
+        }
+
+    store.write_benchmark_entries_batch(batch_entries)
+
+    return {
+        "run_count": len(runs),
+        "pipeline": pipeline,
+        "factors": factors_result,
+    }

ragradar_evaluate-0.1.0/src/ragradar_evaluate/benchmark/checker.py

@@ -0,0 +1,79 @@
+import json
+
+from ragradar_core import store
+from ragradar_core.schema import RunRecord
+
+from ragradar_evaluate.layers import input_quality
+from ragradar_evaluate.policy.persistence import load_policy
+
+
+def check(
+    session_id: int,
+    run_seq: int,
+    pipeline: str | None = None,
+) -> dict:
+    run_row = store.get_run(session_id, run_seq)
+    if run_row is None:
+        raise ValueError(f"Run s{session_id}r{run_seq} not found.")
+
+    record = RunRecord.from_json(json.loads(run_row["run_data"]))
+    pipeline = pipeline or run_row["pipeline"] or "__default"
+    policy = load_policy(pipeline)
+
+    input_scores = input_quality.score_input_quality(record, policy)
+    benchmark = store.get_benchmark(pipeline)
+    benchmark_map = {b["factor"]: b for b in benchmark}
+
+    factors = {}
+    fail_count = 0
+
+    check_factors = [
+        ("duplicate_ratio", "higher_bad"),
+        ("top_chunk_score", "lower_bad"),
+        ("high_score_truncations", "higher_bad"),
+        ("token_headroom_pct", "lower_bad"),
+        ("source_domain_count", "higher_bad"),
+        ("low_score_chunk_ratio", "higher_bad"),
+    ]
+
+    for factor, direction in check_factors:
+        value = input_scores.get(factor) if input_scores else None
+        bench = benchmark_map.get(factor)
+        threshold = bench["threshold"] if bench else None
+
+        if value is None or threshold is None:
+            status = "ok"
+        elif direction == "lower_bad":
+            status = "fail" if value < threshold else "ok"
+        else:
+            status = "fail" if value > threshold else "ok"
+
+        if status == "fail":
+            fail_count += 1
+
+        factors[factor] = {
+            "value": value,
+            "benchmark_threshold": threshold,
+            "status": status,
+        }
+
+    # risk_score is None when the run was never evaluated or its input
+    # metrics could not be computed (0.0 strictly means "computed, no
+    # risk") — unknown risk never counts toward the verdict.
+    eval_data = store.get_eval_scores(session_id, run_seq)
+    risk = eval_data.get("risk_score") if eval_data else None
+
+    if (risk is not None and risk > 0.7) or fail_count >= 3:
+        overall = "fail"
+    elif fail_count >= 1:
+        overall = "warn"
+    else:
+        overall = "ok"
+
+    return {
+        "run_id": f"s{session_id}r{run_seq}",
+        "risk_score": risk,
+        "benchmark_available": len(benchmark) > 0,
+        "factors": factors,
+        "overall": overall,
+    }

ragradar_evaluate-0.1.0/src/ragradar_evaluate/benchmark/exporter.py

@@ -0,0 +1,43 @@
+import json
+from datetime import datetime
+from pathlib import Path
+
+from ragradar_core import store
+
+
+def export(pipeline: str | None = None, output_path: Path | None = None) -> Path:
+    runs = store.get_all_evaluated_runs(pipeline)
+
+    records = []
+    for r in runs:
+        if r["pipeline"] and r["pipeline"].endswith("__seeded"):
+            continue
+        run_data = json.loads(r["run_data"])
+        if not run_data.get("chunks") or not run_data.get("response"):
+            continue
+        records.append((r, run_data))
+
+    if output_path is None:
+        pipe_name = pipeline or "all"
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        exports_dir = store._ragradar_dir() / "exports"
+        exports_dir.mkdir(parents=True, exist_ok=True)
+        output_path = exports_dir / f"{pipe_name}_ragas_{timestamp}.jsonl"
+    else:
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        for row, run_data in records:
+            entry = {
+                "question": run_data["query"],
+                "answer": run_data["response"],
+                "contexts": [c["content"] for c in run_data.get("chunks", [])],
+                "ground_truth": None,
+                "run_id": f"s{row['session_id']}r{row['run_seq']}",
+                "pipeline": row["pipeline"],
+                "evaluated_at": row["evaluated_at"],
+            }
+            f.write(json.dumps(entry) + "\n")
+
+    return output_path