rag-eval-harness 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rag_eval_harness/__init__.py +12 -0
- rag_eval_harness/adapters/__init__.py +6 -0
- rag_eval_harness/adapters/http.py +92 -0
- rag_eval_harness/adapters/traces.py +29 -0
- rag_eval_harness/agent/__init__.py +5 -0
- rag_eval_harness/agent/agents.py +180 -0
- rag_eval_harness/agent/graph.py +171 -0
- rag_eval_harness/agent/supervisor.py +168 -0
- rag_eval_harness/agent/trace.py +71 -0
- rag_eval_harness/agent/triage.py +130 -0
- rag_eval_harness/api/__init__.py +3 -0
- rag_eval_harness/api/app.py +92 -0
- rag_eval_harness/api/auth.py +57 -0
- rag_eval_harness/api/routes.py +251 -0
- rag_eval_harness/api/schemas.py +63 -0
- rag_eval_harness/cli.py +439 -0
- rag_eval_harness/config.py +44 -0
- rag_eval_harness/engine.py +154 -0
- rag_eval_harness/evaluators/__init__.py +20 -0
- rag_eval_harness/evaluators/lexical.py +109 -0
- rag_eval_harness/evaluators/protocol.py +12 -0
- rag_eval_harness/evaluators/ragas.py +177 -0
- rag_eval_harness/evaluators/stub.py +66 -0
- rag_eval_harness/io.py +147 -0
- rag_eval_harness/py.typed +0 -0
- rag_eval_harness/redact.py +35 -0
- rag_eval_harness/regression/__init__.py +19 -0
- rag_eval_harness/regression/compare.py +237 -0
- rag_eval_harness/store/__init__.py +5 -0
- rag_eval_harness/store/models.py +79 -0
- rag_eval_harness/store/repo.py +272 -0
- rag_eval_harness/types.py +124 -0
- rag_eval_harness-0.2.0.dist-info/METADATA +161 -0
- rag_eval_harness-0.2.0.dist-info/RECORD +37 -0
- rag_eval_harness-0.2.0.dist-info/WHEEL +4 -0
- rag_eval_harness-0.2.0.dist-info/entry_points.txt +2 -0
- rag_eval_harness-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""RAG eval harness: RAGAS with a memory and a diff view."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version("rag-eval-harness")
|
|
9
|
+
except PackageNotFoundError: # pragma: no cover
|
|
10
|
+
__version__ = "0.2.0"
|
|
11
|
+
|
|
12
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from rag_eval_harness.io import parse_contexts
|
|
8
|
+
from rag_eval_harness.types import EvalRow, is_truthy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _row_error(exc: BaseException) -> str:
|
|
12
|
+
if isinstance(exc, httpx.TimeoutException):
|
|
13
|
+
return "http adapter: timeout"
|
|
14
|
+
if isinstance(exc, httpx.HTTPStatusError):
|
|
15
|
+
return f"http adapter: HTTP {exc.response.status_code}"
|
|
16
|
+
return f"http adapter: {type(exc).__name__}"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class HttpAdapter:
|
|
20
|
+
"""POST {\"question\"} to a user RAG endpoint and collect answer + contexts."""
|
|
21
|
+
|
|
22
|
+
name = "http"
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
url: str,
|
|
27
|
+
*,
|
|
28
|
+
token: str | None = None,
|
|
29
|
+
timeout: float = 30.0,
|
|
30
|
+
client: httpx.Client | None = None,
|
|
31
|
+
) -> None:
|
|
32
|
+
self.url = url
|
|
33
|
+
self.token = token
|
|
34
|
+
self.timeout = timeout
|
|
35
|
+
self._client = client
|
|
36
|
+
|
|
37
|
+
def _headers(self) -> dict[str, str]:
|
|
38
|
+
headers = {"Content-Type": "application/json", "Accept": "application/json"}
|
|
39
|
+
if self.token:
|
|
40
|
+
headers["Authorization"] = f"Bearer {self.token}"
|
|
41
|
+
return headers
|
|
42
|
+
|
|
43
|
+
def _parse_response(self, payload: Any) -> tuple[str | None, list[str], bool]:
|
|
44
|
+
if not isinstance(payload, dict):
|
|
45
|
+
raise ValueError("SUT response must be a JSON object")
|
|
46
|
+
answer = payload.get("answer", payload.get("response"))
|
|
47
|
+
contexts = payload.get("retrieved_contexts", payload.get("contexts"))
|
|
48
|
+
abstained = payload.get("abstained", payload.get("escalated"))
|
|
49
|
+
if answer is None or not str(answer).strip():
|
|
50
|
+
raise ValueError("SUT response missing answer")
|
|
51
|
+
return str(answer), parse_contexts(contexts), is_truthy(abstained)
|
|
52
|
+
|
|
53
|
+
def run(self, rows: list[EvalRow]) -> list[EvalRow]:
|
|
54
|
+
owns_client = self._client is None
|
|
55
|
+
client = self._client or httpx.Client(timeout=self.timeout)
|
|
56
|
+
try:
|
|
57
|
+
out: list[EvalRow] = []
|
|
58
|
+
for row in rows:
|
|
59
|
+
if row.error:
|
|
60
|
+
out.append(row)
|
|
61
|
+
continue
|
|
62
|
+
try:
|
|
63
|
+
response = client.post(
|
|
64
|
+
self.url,
|
|
65
|
+
json={"question": row.question},
|
|
66
|
+
headers=self._headers(),
|
|
67
|
+
)
|
|
68
|
+
response.raise_for_status()
|
|
69
|
+
answer, contexts, abstained = self._parse_response(response.json())
|
|
70
|
+
out.append(
|
|
71
|
+
EvalRow(
|
|
72
|
+
question=row.question,
|
|
73
|
+
answer=answer,
|
|
74
|
+
retrieved_contexts=contexts,
|
|
75
|
+
ground_truth=row.ground_truth,
|
|
76
|
+
abstained=abstained,
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
except Exception as exc: # noqa: BLE001 — per-row failure must not abort the run
|
|
80
|
+
out.append(
|
|
81
|
+
EvalRow(
|
|
82
|
+
question=row.question,
|
|
83
|
+
answer=row.answer,
|
|
84
|
+
retrieved_contexts=row.retrieved_contexts,
|
|
85
|
+
ground_truth=row.ground_truth,
|
|
86
|
+
error=_row_error(exc),
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
return out
|
|
90
|
+
finally:
|
|
91
|
+
if owns_client:
|
|
92
|
+
client.close()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from rag_eval_harness.types import EvalRow
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TracesAdapter:
|
|
7
|
+
"""Use precomputed answers and retrieved contexts already on each row."""
|
|
8
|
+
|
|
9
|
+
name = "traces"
|
|
10
|
+
|
|
11
|
+
def run(self, rows: list[EvalRow]) -> list[EvalRow]:
|
|
12
|
+
out: list[EvalRow] = []
|
|
13
|
+
for row in rows:
|
|
14
|
+
if row.error:
|
|
15
|
+
out.append(row)
|
|
16
|
+
continue
|
|
17
|
+
if row.answer is None or not str(row.answer).strip():
|
|
18
|
+
out.append(
|
|
19
|
+
EvalRow(
|
|
20
|
+
question=row.question,
|
|
21
|
+
answer=row.answer,
|
|
22
|
+
retrieved_contexts=row.retrieved_contexts,
|
|
23
|
+
ground_truth=row.ground_truth,
|
|
24
|
+
error="missing answer for traces adapter",
|
|
25
|
+
)
|
|
26
|
+
)
|
|
27
|
+
continue
|
|
28
|
+
out.append(row)
|
|
29
|
+
return out
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any, Literal
|
|
5
|
+
|
|
6
|
+
import pydantic_ai
|
|
7
|
+
from pydantic import BaseModel, Field
|
|
8
|
+
from pydantic_ai import Agent, ModelRetry, RunContext, UsageLimits
|
|
9
|
+
from pydantic_ai.models import Model
|
|
10
|
+
from pydantic_ai.usage import RunUsage
|
|
11
|
+
|
|
12
|
+
from rag_eval_harness.agent.trace import TraceRecorder
|
|
13
|
+
from rag_eval_harness.agent.triage import FailureGroup, RowEvidence
|
|
14
|
+
from rag_eval_harness.regression.compare import RegressionReport
|
|
15
|
+
|
|
16
|
+
# The CLI prints to a terminal and to CI logs. Pydantic AI's first-run banner would land in both.
|
|
17
|
+
pydantic_ai.BANNER_ENABLED = False
|
|
18
|
+
|
|
19
|
+
USAGE_LIMITS = UsageLimits(request_limit=4)
|
|
20
|
+
|
|
21
|
+
METRIC_GLOSSARY = """\
|
|
22
|
+
Metrics (0 to 1, higher is better; scores come from an LLM judge or from token overlap):
|
|
23
|
+
- context_recall: how much of the ground truth the retrieved contexts support.
|
|
24
|
+
- context_precision: how much of the retrieved context is relevant to the ground truth.
|
|
25
|
+
- faithfulness: how much of the answer the retrieved contexts support.
|
|
26
|
+
- answer_relevancy: how directly the answer addresses the question."""
|
|
27
|
+
|
|
28
|
+
DIAGNOSER_INSTRUCTIONS = f"""\
|
|
29
|
+
You diagnose one stage of a RAG regression. The same questions ran through a baseline
|
|
30
|
+
pipeline and a head pipeline. Mean scores for this stage dropped past the CI threshold.
|
|
31
|
+
|
|
32
|
+
You get the rows that dropped most. For each row: the question, the ground truth, both
|
|
33
|
+
answers, the contexts the baseline retrieved that the head did not ("lost"), and the
|
|
34
|
+
contexts the head retrieved that the baseline did not ("gained").
|
|
35
|
+
|
|
36
|
+
Name the one change in the pipeline that best explains the drop across rows, such as
|
|
37
|
+
fewer results returned, a different chunk size, a different ranker, or a prompt change.
|
|
38
|
+
Cite only the row indexes you were given. If the rows do not support a single cause, say
|
|
39
|
+
so and set confidence to low. Do not invent pipeline details the evidence does not show.
|
|
40
|
+
|
|
41
|
+
{METRIC_GLOSSARY}"""
|
|
42
|
+
|
|
43
|
+
SYNTHESIZER_INSTRUCTIONS = """\
|
|
44
|
+
You write the explanation attached to a failed RAG regression gate. You get the mean
|
|
45
|
+
metric table and one diagnosis per failing pipeline stage. Write for an engineer reading
|
|
46
|
+
a CI log: plain words, no hedging beyond what the diagnoses state, no new causes."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Diagnosis(BaseModel):
|
|
50
|
+
suspected_cause: str = Field(description="One or two sentences naming what changed.")
|
|
51
|
+
evidence_rows: list[int] = Field(description="Row indexes from the prompt that show it.")
|
|
52
|
+
suggested_fix: str = Field(description="One concrete change to try next.")
|
|
53
|
+
confidence: Literal["low", "medium", "high"]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class Summary(BaseModel):
|
|
57
|
+
headline: str = Field(description="One sentence a reviewer reads first.")
|
|
58
|
+
likely_cause: str = Field(description="The cause, in two sentences at most.")
|
|
59
|
+
next_steps: list[str] = Field(description="At most three, most useful first.")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class Finding:
|
|
64
|
+
group: FailureGroup
|
|
65
|
+
diagnosis: Diagnosis
|
|
66
|
+
|
|
67
|
+
def to_dict(self) -> dict[str, Any]:
|
|
68
|
+
return {**self.group.to_dict(), "diagnosis": self.diagnosis.model_dump()}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
diagnoser = Agent(
|
|
72
|
+
output_type=Diagnosis,
|
|
73
|
+
deps_type=FailureGroup,
|
|
74
|
+
instructions=DIAGNOSER_INSTRUCTIONS,
|
|
75
|
+
name="diagnoser",
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
synthesizer = Agent(
|
|
79
|
+
output_type=Summary,
|
|
80
|
+
instructions=SYNTHESIZER_INSTRUCTIONS,
|
|
81
|
+
name="synthesizer",
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@diagnoser.output_validator
|
|
86
|
+
def _cites_known_rows(ctx: RunContext[FailureGroup], output: Diagnosis) -> Diagnosis:
|
|
87
|
+
known = {row.index for row in ctx.deps.rows}
|
|
88
|
+
unknown = sorted(set(output.evidence_rows) - known)
|
|
89
|
+
if unknown:
|
|
90
|
+
raise ModelRetry(f"Rows {unknown} are not in this group. Cite only {sorted(known)}.")
|
|
91
|
+
return output
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _render_contexts(label: str, contexts: list[str]) -> list[str]:
|
|
95
|
+
if not contexts:
|
|
96
|
+
return [f"{label}: none"]
|
|
97
|
+
return [f"{label}:"] + [f" - {text}" for text in contexts]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _render_row(row: RowEvidence) -> str:
|
|
101
|
+
drops = ", ".join(
|
|
102
|
+
f"{name} {before:.3f} -> {after:.3f}" for name, (before, after) in row.drops.items()
|
|
103
|
+
)
|
|
104
|
+
lines = [
|
|
105
|
+
f"### Row {row.index}",
|
|
106
|
+
f"Question: {row.question}",
|
|
107
|
+
f"Ground truth: {row.ground_truth or '(none)'}",
|
|
108
|
+
f"Drops: {drops}",
|
|
109
|
+
f"Baseline answer: {row.baseline_answer or '(none)'}",
|
|
110
|
+
f"Head answer: {row.head_answer or '(none)'}",
|
|
111
|
+
*_render_contexts("Lost contexts", row.lost_contexts),
|
|
112
|
+
*_render_contexts("Gained contexts", row.gained_contexts),
|
|
113
|
+
]
|
|
114
|
+
return "\n".join(lines)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def render_group(group: FailureGroup) -> str:
|
|
118
|
+
metrics = "\n".join(
|
|
119
|
+
f"- {item.metric}: {item.baseline:.3f} -> {item.head:.3f} ({item.delta:+.3f})"
|
|
120
|
+
for item in group.metrics
|
|
121
|
+
)
|
|
122
|
+
rows = "\n\n".join(_render_row(row) for row in group.rows)
|
|
123
|
+
return f"Stage: {group.stage}\n\nFailing means:\n{metrics}\n\n{rows}"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def render_findings(report: RegressionReport, findings: list[Finding]) -> str:
|
|
127
|
+
table = "\n".join(
|
|
128
|
+
f"- {item.metric}: {item.baseline:.3f} -> {item.head:.3f} ({item.delta:+.3f})"
|
|
129
|
+
f"{' FAIL' if item.dropped else ''}"
|
|
130
|
+
for item in report.deltas
|
|
131
|
+
)
|
|
132
|
+
parts = [f"Threshold: {report.threshold}\n\nMeans:\n{table}"]
|
|
133
|
+
for finding in findings:
|
|
134
|
+
diagnosis = finding.diagnosis
|
|
135
|
+
parts.append(
|
|
136
|
+
f"## {finding.group.stage} diagnosis ({diagnosis.confidence} confidence)\n"
|
|
137
|
+
f"Cause: {diagnosis.suspected_cause}\n"
|
|
138
|
+
f"Evidence rows: {diagnosis.evidence_rows}\n"
|
|
139
|
+
f"Suggested fix: {diagnosis.suggested_fix}"
|
|
140
|
+
)
|
|
141
|
+
return "\n\n".join(parts)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
async def diagnose(
|
|
145
|
+
group: FailureGroup,
|
|
146
|
+
*,
|
|
147
|
+
model: Model | str,
|
|
148
|
+
recorder: TraceRecorder | None = None,
|
|
149
|
+
usage: RunUsage | None = None,
|
|
150
|
+
usage_limits: UsageLimits | None = USAGE_LIMITS,
|
|
151
|
+
) -> Diagnosis:
|
|
152
|
+
"""Run the diagnoser. A delegating agent passes its own usage so one budget covers both."""
|
|
153
|
+
recorder = recorder or TraceRecorder()
|
|
154
|
+
start = TraceRecorder.counts(usage)
|
|
155
|
+
async with recorder.span("diagnoser", stage=group.stage, delegated=usage is not None) as entry:
|
|
156
|
+
result = await diagnoser.run(
|
|
157
|
+
render_group(group),
|
|
158
|
+
deps=group,
|
|
159
|
+
model=model,
|
|
160
|
+
usage=usage,
|
|
161
|
+
usage_limits=usage_limits,
|
|
162
|
+
)
|
|
163
|
+
recorder.finish(entry, result, start=start)
|
|
164
|
+
return result.output
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
async def summarize(
|
|
168
|
+
report: RegressionReport,
|
|
169
|
+
findings: list[Finding],
|
|
170
|
+
*,
|
|
171
|
+
model: Model | str,
|
|
172
|
+
recorder: TraceRecorder | None = None,
|
|
173
|
+
) -> Summary:
|
|
174
|
+
recorder = recorder or TraceRecorder()
|
|
175
|
+
async with recorder.span("synthesizer") as entry:
|
|
176
|
+
result = await synthesizer.run(
|
|
177
|
+
render_findings(report, findings), model=model, usage_limits=USAGE_LIMITS
|
|
178
|
+
)
|
|
179
|
+
recorder.finish(entry, result)
|
|
180
|
+
return result.output
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import operator
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Annotated, Any, Literal, TypedDict
|
|
6
|
+
|
|
7
|
+
from langgraph.graph import END, START, StateGraph
|
|
8
|
+
from langgraph.types import Send
|
|
9
|
+
from pydantic_ai.models import Model
|
|
10
|
+
|
|
11
|
+
from rag_eval_harness.agent.agents import Finding, Summary, diagnose, summarize
|
|
12
|
+
from rag_eval_harness.agent.supervisor import supervise
|
|
13
|
+
from rag_eval_harness.agent.trace import TraceRecorder
|
|
14
|
+
from rag_eval_harness.agent.triage import STAGE_ORDER, FailureGroup, triage
|
|
15
|
+
from rag_eval_harness.regression.compare import RegressionReport
|
|
16
|
+
from rag_eval_harness.types import RowScore
|
|
17
|
+
|
|
18
|
+
Mode = Literal["workflow", "supervisor"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class InvestigationState(TypedDict, total=False):
|
|
22
|
+
mode: Mode
|
|
23
|
+
report: RegressionReport
|
|
24
|
+
baseline_rows: list[RowScore]
|
|
25
|
+
head_rows: list[RowScore]
|
|
26
|
+
limit: int
|
|
27
|
+
groups: list[FailureGroup]
|
|
28
|
+
findings: Annotated[list[Finding], operator.add]
|
|
29
|
+
summary: Summary
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class DiagnoseTask(TypedDict):
|
|
33
|
+
group: FailureGroup
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _by_stage(findings: list[Finding]) -> list[Finding]:
|
|
37
|
+
return sorted(findings, key=lambda finding: STAGE_ORDER.index(finding.group.stage))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def build_graph(model: Model | str, recorder: TraceRecorder | None = None):
|
|
41
|
+
"""Two paths from START, picked by state["mode"].
|
|
42
|
+
|
|
43
|
+
workflow: triage -> one diagnose per failing stage, in parallel -> summarize
|
|
44
|
+
supervisor: one agent that picks its own tools and delegates to the diagnoser
|
|
45
|
+
"""
|
|
46
|
+
recorder = recorder or TraceRecorder()
|
|
47
|
+
|
|
48
|
+
def route(state: InvestigationState) -> str:
|
|
49
|
+
return "supervise" if state["mode"] == "supervisor" else "triage"
|
|
50
|
+
|
|
51
|
+
def triage_node(state: InvestigationState) -> dict[str, Any]:
|
|
52
|
+
groups = triage(
|
|
53
|
+
state["report"], state["baseline_rows"], state["head_rows"], limit=state["limit"]
|
|
54
|
+
)
|
|
55
|
+
return {"groups": groups}
|
|
56
|
+
|
|
57
|
+
def fan_out(state: InvestigationState) -> list[Send] | str:
|
|
58
|
+
if not state["groups"]:
|
|
59
|
+
return END
|
|
60
|
+
return [Send("diagnose", {"group": group}) for group in state["groups"]]
|
|
61
|
+
|
|
62
|
+
async def diagnose_node(task: DiagnoseTask) -> dict[str, Any]:
|
|
63
|
+
diagnosis = await diagnose(task["group"], model=model, recorder=recorder)
|
|
64
|
+
return {"findings": [Finding(task["group"], diagnosis)]}
|
|
65
|
+
|
|
66
|
+
async def summarize_node(state: InvestigationState) -> dict[str, Any]:
|
|
67
|
+
summary = await summarize(
|
|
68
|
+
state["report"], _by_stage(state["findings"]), model=model, recorder=recorder
|
|
69
|
+
)
|
|
70
|
+
return {"summary": summary}
|
|
71
|
+
|
|
72
|
+
async def supervise_node(state: InvestigationState) -> dict[str, Any]:
|
|
73
|
+
summary, findings = await supervise(
|
|
74
|
+
state["report"],
|
|
75
|
+
state["baseline_rows"],
|
|
76
|
+
state["head_rows"],
|
|
77
|
+
model=model,
|
|
78
|
+
limit=state["limit"],
|
|
79
|
+
recorder=recorder,
|
|
80
|
+
)
|
|
81
|
+
return {"summary": summary, "findings": findings}
|
|
82
|
+
|
|
83
|
+
graph = StateGraph(InvestigationState)
|
|
84
|
+
graph.add_node("triage", triage_node)
|
|
85
|
+
graph.add_node("diagnose", diagnose_node)
|
|
86
|
+
graph.add_node("summarize", summarize_node)
|
|
87
|
+
graph.add_node("supervise", supervise_node)
|
|
88
|
+
graph.add_conditional_edges(START, route, ["triage", "supervise"])
|
|
89
|
+
graph.add_conditional_edges("triage", fan_out, ["diagnose", END])
|
|
90
|
+
graph.add_edge("diagnose", "summarize")
|
|
91
|
+
graph.add_edge("summarize", END)
|
|
92
|
+
graph.add_edge("supervise", END)
|
|
93
|
+
return graph.compile()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class Investigation:
|
|
98
|
+
report: RegressionReport
|
|
99
|
+
mode: Mode = "workflow"
|
|
100
|
+
findings: list[Finding] = field(default_factory=list)
|
|
101
|
+
summary: Summary | None = None
|
|
102
|
+
|
|
103
|
+
def to_dict(self) -> dict[str, Any]:
|
|
104
|
+
return {
|
|
105
|
+
"mode": self.mode,
|
|
106
|
+
"report": self.report.to_dict(),
|
|
107
|
+
"findings": [finding.to_dict() for finding in self.findings],
|
|
108
|
+
"summary": self.summary.model_dump() if self.summary else None,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
def to_markdown(self) -> str:
|
|
112
|
+
if self.report.passed:
|
|
113
|
+
return "No regression. Nothing to investigate."
|
|
114
|
+
if self.summary is None:
|
|
115
|
+
return (
|
|
116
|
+
"The gate failed, but no row dropped on the failing metrics. "
|
|
117
|
+
"Check row errors with `rag-eval diff`."
|
|
118
|
+
)
|
|
119
|
+
lines = ["## Why the gate failed", "", self.summary.headline, "", self.summary.likely_cause]
|
|
120
|
+
for finding in self.findings:
|
|
121
|
+
group, diagnosis = finding.group, finding.diagnosis
|
|
122
|
+
metrics = ", ".join(
|
|
123
|
+
f"{item.metric} {item.baseline:.3f} -> {item.head:.3f}" for item in group.metrics
|
|
124
|
+
)
|
|
125
|
+
questions = {row.index: row.question for row in group.rows}
|
|
126
|
+
lines += [
|
|
127
|
+
"",
|
|
128
|
+
f"### {group.stage}: {metrics}",
|
|
129
|
+
"",
|
|
130
|
+
f"{diagnosis.suspected_cause} (confidence: {diagnosis.confidence})",
|
|
131
|
+
"",
|
|
132
|
+
f"Try: {diagnosis.suggested_fix}",
|
|
133
|
+
]
|
|
134
|
+
if diagnosis.evidence_rows:
|
|
135
|
+
lines += ["", "Evidence:"]
|
|
136
|
+
lines += [f"- [{index}] {questions[index]}" for index in diagnosis.evidence_rows]
|
|
137
|
+
if self.summary.next_steps:
|
|
138
|
+
lines += ["", "### Next steps", ""]
|
|
139
|
+
lines += [f"{n}. {step}" for n, step in enumerate(self.summary.next_steps, start=1)]
|
|
140
|
+
return "\n".join(lines)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
async def investigate(
|
|
144
|
+
report: RegressionReport,
|
|
145
|
+
baseline_rows: list[RowScore],
|
|
146
|
+
head_rows: list[RowScore],
|
|
147
|
+
*,
|
|
148
|
+
model: Model | str,
|
|
149
|
+
limit: int = 5,
|
|
150
|
+
mode: Mode = "workflow",
|
|
151
|
+
recorder: TraceRecorder | None = None,
|
|
152
|
+
) -> Investigation:
|
|
153
|
+
"""Explain a failed gate. A passing report returns without calling a model."""
|
|
154
|
+
if report.passed:
|
|
155
|
+
return Investigation(report=report, mode=mode)
|
|
156
|
+
state = await build_graph(model, recorder).ainvoke(
|
|
157
|
+
{
|
|
158
|
+
"mode": mode,
|
|
159
|
+
"report": report,
|
|
160
|
+
"baseline_rows": baseline_rows,
|
|
161
|
+
"head_rows": head_rows,
|
|
162
|
+
"limit": limit,
|
|
163
|
+
"findings": [],
|
|
164
|
+
}
|
|
165
|
+
)
|
|
166
|
+
return Investigation(
|
|
167
|
+
report=report,
|
|
168
|
+
mode=mode,
|
|
169
|
+
findings=_by_stage(state.get("findings", [])),
|
|
170
|
+
summary=state.get("summary"),
|
|
171
|
+
)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from pydantic_ai import Agent, ModelRetry, RunContext, UsageLimits
|
|
7
|
+
from pydantic_ai.models import Model
|
|
8
|
+
|
|
9
|
+
from rag_eval_harness.agent.agents import METRIC_GLOSSARY, Finding, Summary, diagnose
|
|
10
|
+
from rag_eval_harness.agent.trace import TraceRecorder
|
|
11
|
+
from rag_eval_harness.agent.triage import STAGE_BY_METRIC, Stage, row_evidence, triage
|
|
12
|
+
from rag_eval_harness.regression.compare import RegressionReport, pair_rows, worst_row_drops
|
|
13
|
+
from rag_eval_harness.types import RowScore
|
|
14
|
+
|
|
15
|
+
# One budget for the supervisor and every diagnoser it delegates to.
|
|
16
|
+
SUPERVISOR_LIMITS = UsageLimits(request_limit=15, tool_calls_limit=20)
|
|
17
|
+
MAX_WORST_ROWS = 10
|
|
18
|
+
|
|
19
|
+
SUPERVISOR_INSTRUCTIONS = f"""\
|
|
20
|
+
You investigate a failed RAG regression gate. The same questions ran through a baseline
|
|
21
|
+
pipeline and a head pipeline, and at least one mean metric dropped past the threshold.
|
|
22
|
+
|
|
23
|
+
Use the tools to find out why. Start with failing_metrics. Look at the worst rows for each
|
|
24
|
+
failing metric and open a few with row_detail. Call diagnose_stage for a failing stage
|
|
25
|
+
when you want a careful diagnosis of its rows. Stop once the evidence supports a cause.
|
|
26
|
+
Do not invent pipeline details the tools did not show.
|
|
27
|
+
|
|
28
|
+
{METRIC_GLOSSARY}"""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class SupervisorDeps:
|
|
33
|
+
report: RegressionReport
|
|
34
|
+
baseline_rows: list[RowScore]
|
|
35
|
+
head_rows: list[RowScore]
|
|
36
|
+
model: Model | str
|
|
37
|
+
limit: int
|
|
38
|
+
recorder: TraceRecorder
|
|
39
|
+
findings: list[Finding] = field(default_factory=list)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
supervisor = Agent(
|
|
43
|
+
output_type=Summary,
|
|
44
|
+
deps_type=SupervisorDeps,
|
|
45
|
+
instructions=SUPERVISOR_INSTRUCTIONS,
|
|
46
|
+
name="supervisor",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@supervisor.tool
|
|
51
|
+
def failing_metrics(ctx: RunContext[SupervisorDeps]) -> list[dict[str, Any]]:
|
|
52
|
+
"""Mean of every metric for baseline and head, its pipeline stage, and whether it failed."""
|
|
53
|
+
return [
|
|
54
|
+
{
|
|
55
|
+
"metric": item.metric,
|
|
56
|
+
"stage": STAGE_BY_METRIC.get(item.metric, "other"),
|
|
57
|
+
"baseline": round(item.baseline, 4),
|
|
58
|
+
"head": round(item.head, 4),
|
|
59
|
+
"delta": round(item.delta, 4),
|
|
60
|
+
"failed": item.dropped,
|
|
61
|
+
}
|
|
62
|
+
for item in ctx.deps.report.deltas
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@supervisor.tool
|
|
67
|
+
def worst_rows(
|
|
68
|
+
ctx: RunContext[SupervisorDeps], metric: str, limit: int = 5
|
|
69
|
+
) -> list[dict[str, Any]]:
|
|
70
|
+
"""Rows whose score on `metric` dropped most from baseline to head, worst first."""
|
|
71
|
+
known = sorted(item.metric for item in ctx.deps.report.deltas)
|
|
72
|
+
if metric not in known:
|
|
73
|
+
raise ModelRetry(f"Unknown metric {metric!r}. Use one of {known}.")
|
|
74
|
+
drops = worst_row_drops(
|
|
75
|
+
ctx.deps.baseline_rows,
|
|
76
|
+
ctx.deps.head_rows,
|
|
77
|
+
metric=metric,
|
|
78
|
+
limit=min(max(limit, 1), MAX_WORST_ROWS),
|
|
79
|
+
)
|
|
80
|
+
return [
|
|
81
|
+
{
|
|
82
|
+
"index": item.index,
|
|
83
|
+
"question": item.question,
|
|
84
|
+
"baseline": round(item.baseline, 4),
|
|
85
|
+
"head": round(item.head, 4),
|
|
86
|
+
}
|
|
87
|
+
for item in drops
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@supervisor.tool
|
|
92
|
+
def row_detail(ctx: RunContext[SupervisorDeps], index: int) -> dict[str, Any]:
|
|
93
|
+
"""One row: question, ground truth, scores, both answers, and contexts head lost or gained."""
|
|
94
|
+
pairs = pair_rows(ctx.deps.baseline_rows, ctx.deps.head_rows)
|
|
95
|
+
if not 0 <= index < len(pairs):
|
|
96
|
+
raise ModelRetry(f"Row {index} does not exist. Use 0 to {len(pairs) - 1}.")
|
|
97
|
+
left, right = pairs[index]
|
|
98
|
+
if left is None or right is None:
|
|
99
|
+
side = "baseline" if left is None else "head"
|
|
100
|
+
raise ModelRetry(f"Row {index} has no {side} counterpart, so there is nothing to compare.")
|
|
101
|
+
scores = {
|
|
102
|
+
name: (left.metrics[name], right.metrics[name])
|
|
103
|
+
for name in sorted(set(left.metrics) & set(right.metrics))
|
|
104
|
+
}
|
|
105
|
+
evidence = row_evidence(index, left, right, scores)
|
|
106
|
+
return {
|
|
107
|
+
"index": index,
|
|
108
|
+
"question": evidence.question,
|
|
109
|
+
"ground_truth": evidence.ground_truth,
|
|
110
|
+
"scores": {name: {"baseline": b, "head": h} for name, (b, h) in scores.items()},
|
|
111
|
+
"baseline_answer": evidence.baseline_answer,
|
|
112
|
+
"head_answer": evidence.head_answer,
|
|
113
|
+
"lost_contexts": evidence.lost_contexts,
|
|
114
|
+
"gained_contexts": evidence.gained_contexts,
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@supervisor.tool
|
|
119
|
+
async def diagnose_stage(ctx: RunContext[SupervisorDeps], stage: Stage) -> dict[str, Any]:
|
|
120
|
+
"""Hand one failing stage to the diagnoser agent. Returns cause, cited rows, fix, confidence."""
|
|
121
|
+
deps = ctx.deps
|
|
122
|
+
for finding in deps.findings:
|
|
123
|
+
if finding.group.stage == stage:
|
|
124
|
+
return finding.diagnosis.model_dump()
|
|
125
|
+
groups = {
|
|
126
|
+
group.stage: group
|
|
127
|
+
for group in triage(deps.report, deps.baseline_rows, deps.head_rows, limit=deps.limit)
|
|
128
|
+
}
|
|
129
|
+
group = groups.get(stage)
|
|
130
|
+
if group is None:
|
|
131
|
+
raise ModelRetry(f"Stage {stage!r} has no failing rows. Failing stages: {sorted(groups)}.")
|
|
132
|
+
diagnosis = await diagnose(
|
|
133
|
+
group,
|
|
134
|
+
model=deps.model,
|
|
135
|
+
recorder=deps.recorder,
|
|
136
|
+
usage=ctx.usage,
|
|
137
|
+
usage_limits=ctx.usage_limits,
|
|
138
|
+
)
|
|
139
|
+
deps.findings.append(Finding(group, diagnosis))
|
|
140
|
+
return diagnosis.model_dump()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
async def supervise(
|
|
144
|
+
report: RegressionReport,
|
|
145
|
+
baseline_rows: list[RowScore],
|
|
146
|
+
head_rows: list[RowScore],
|
|
147
|
+
*,
|
|
148
|
+
model: Model | str,
|
|
149
|
+
limit: int,
|
|
150
|
+
recorder: TraceRecorder,
|
|
151
|
+
) -> tuple[Summary, list[Finding]]:
|
|
152
|
+
deps = SupervisorDeps(
|
|
153
|
+
report=report,
|
|
154
|
+
baseline_rows=baseline_rows,
|
|
155
|
+
head_rows=head_rows,
|
|
156
|
+
model=model,
|
|
157
|
+
limit=limit,
|
|
158
|
+
recorder=recorder,
|
|
159
|
+
)
|
|
160
|
+
async with recorder.span("supervisor") as entry:
|
|
161
|
+
result = await supervisor.run(
|
|
162
|
+
f"The gate failed at threshold {report.threshold}. Find out why.",
|
|
163
|
+
deps=deps,
|
|
164
|
+
model=model,
|
|
165
|
+
usage_limits=SUPERVISOR_LIMITS,
|
|
166
|
+
)
|
|
167
|
+
recorder.finish(entry, result)
|
|
168
|
+
return result.output, deps.findings
|