askback 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- askback/__init__.py +29 -0
- askback/__main__.py +8 -0
- askback/adapters/__init__.py +11 -0
- askback/adapters/base.py +189 -0
- askback/adapters/factory.py +176 -0
- askback/adapters/http.py +443 -0
- askback/adapters/in_process.py +135 -0
- askback/adapters/payload.py +207 -0
- askback/adapters/scripted.py +117 -0
- askback/adapters/socketio_adapter.py +885 -0
- askback/api/__init__.py +18 -0
- askback/api/corpus_store.py +85 -0
- askback/api/dashboard.py +444 -0
- askback/api/server.py +200 -0
- askback/api/state.py +82 -0
- askback/api/static/app.css +423 -0
- askback/api/static/app.js +98 -0
- askback/api/static/core.js +201 -0
- askback/api/static/index.html +175 -0
- askback/api/static/panels.js +651 -0
- askback/cli.py +618 -0
- askback/core/__init__.py +58 -0
- askback/core/config.py +224 -0
- askback/core/contracts.py +201 -0
- askback/core/errors.py +35 -0
- askback/core/logging.py +57 -0
- askback/core/registry.py +73 -0
- askback/core/text.py +151 -0
- askback/datasets/__init__.py +14 -0
- askback/datasets/loader.py +268 -0
- askback/datasets/schema.py +525 -0
- askback/demo/__init__.py +42 -0
- askback/demo/agent.py +195 -0
- askback/demo/suites/demo.yaml +100 -0
- askback/evaluation/__init__.py +70 -0
- askback/evaluation/base.py +214 -0
- askback/evaluation/calibration.py +405 -0
- askback/evaluation/citation_eval.py +580 -0
- askback/evaluation/clarification.py +477 -0
- askback/evaluation/engine.py +193 -0
- askback/evaluation/escalation_eval.py +109 -0
- askback/evaluation/gates.py +351 -0
- askback/evaluation/judge_cache.py +94 -0
- askback/evaluation/llm_judge.py +463 -0
- askback/evaluation/patterns.py +48 -0
- askback/evaluation/pii_eval.py +252 -0
- askback/evaluation/rag_eval.py +168 -0
- askback/evaluation/regression.py +438 -0
- askback/evaluation/results.py +588 -0
- askback/evaluation/rule_based.py +198 -0
- askback/evaluation/safety_eval.py +170 -0
- askback/providers/__init__.py +10 -0
- askback/providers/base.py +157 -0
- askback/providers/openai_compat.py +340 -0
- askback/py.typed +0 -0
- askback/reporting/__init__.py +77 -0
- askback/reporting/excel_report.py +117 -0
- askback/reporting/html_report.py +283 -0
- askback/reporting/json_report.py +27 -0
- askback/reporting/markdown_report.py +95 -0
- askback/reporting/redaction.py +185 -0
- askback/runner/__init__.py +5 -0
- askback/runner/responders.py +279 -0
- askback/runner/test_runner.py +424 -0
- askback-0.1.0.dist-info/METADATA +504 -0
- askback-0.1.0.dist-info/RECORD +71 -0
- askback-0.1.0.dist-info/WHEEL +5 -0
- askback-0.1.0.dist-info/entry_points.txt +2 -0
- askback-0.1.0.dist-info/licenses/LICENSE +176 -0
- askback-0.1.0.dist-info/licenses/NOTICE +15 -0
- askback-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
"""Run results: what one evaluated case produced, and what a whole run produced.
|
|
2
|
+
|
|
3
|
+
These structures are the wire format for JSON reports, so they round-trip
|
|
4
|
+
through `to_dict` / `from_dict`: reading a report back and writing it again
|
|
5
|
+
gives the same report. The JSON is the canonical form, and the other formats are
|
|
6
|
+
rendered from a report read back out of it, so anything `from_dict` dropped
|
|
7
|
+
would be missing from them too. A baseline is not one of these: it is a summary
|
|
8
|
+
of a run, and `regression.Baseline` owns its format.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections import defaultdict
|
|
14
|
+
from collections.abc import Mapping, Sequence
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from ..core.contracts import ChatResponse
|
|
20
|
+
from ..core.errors import DatasetError
|
|
21
|
+
from .base import Check, EvalResult, JudgeUsage
|
|
22
|
+
from .clarification import Detection, PersonaUsage, Round
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _now() -> str:
|
|
26
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(slots=True)
|
|
30
|
+
class TurnResult:
|
|
31
|
+
"""One turn of a case's conversation: what was sent and what came back."""
|
|
32
|
+
|
|
33
|
+
#: 1-based, like the loader's "turn N" in a dataset error.
|
|
34
|
+
index: int
|
|
35
|
+
user: str
|
|
36
|
+
#: False for a setup turn, which is sent and never graded.
|
|
37
|
+
evaluated: bool = True
|
|
38
|
+
reply: str = ""
|
|
39
|
+
route: str = ""
|
|
40
|
+
status: str = ""
|
|
41
|
+
latency_ms: float = 0.0
|
|
42
|
+
#: None until the turn is graded. A setup turn, or one the adapter could not
|
|
43
|
+
#: deliver, has no verdict - which is not the same thing as a passing one.
|
|
44
|
+
score: float | None = None
|
|
45
|
+
passed: bool | None = None
|
|
46
|
+
results: list[EvalResult] = field(default_factory=list)
|
|
47
|
+
error: str | None = None
|
|
48
|
+
#: Every question the agent asked on the way to `reply`, and what we answered.
|
|
49
|
+
rounds: list[Round] = field(default_factory=list)
|
|
50
|
+
#: How the final reply was classified. Written only when it is a question, was ruled
|
|
51
|
+
#: out as one, or questions were answered on the way; a plain answer says nothing.
|
|
52
|
+
detection: Detection | None = None
|
|
53
|
+
#: Set when the agent asked and we had nothing to say: a gap in the suite, not a
|
|
54
|
+
#: failure of the agent. The turn is neither graded nor passed.
|
|
55
|
+
unmeasured: str | None = None
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def persona_usage(self) -> PersonaUsage | None:
|
|
59
|
+
usages = [r.usage for r in self.rounds if r.usage is not None]
|
|
60
|
+
return sum(usages[1:], usages[0]) if usages else None
|
|
61
|
+
|
|
62
|
+
def to_dict(self) -> dict[str, Any]:
|
|
63
|
+
data: dict[str, Any] = {
|
|
64
|
+
"index": self.index,
|
|
65
|
+
"user": self.user,
|
|
66
|
+
"evaluated": self.evaluated,
|
|
67
|
+
"reply": self.reply,
|
|
68
|
+
"route": self.route,
|
|
69
|
+
"status": self.status,
|
|
70
|
+
"latency_ms": self.latency_ms,
|
|
71
|
+
"score": self.score,
|
|
72
|
+
"passed": self.passed,
|
|
73
|
+
"error": self.error,
|
|
74
|
+
"results": [r.to_dict() for r in self.results],
|
|
75
|
+
}
|
|
76
|
+
# Each only when it has something to say, so a turn with no questions in it
|
|
77
|
+
# serialises exactly as it did before clarification existed.
|
|
78
|
+
if self.rounds:
|
|
79
|
+
data["rounds"] = [r.to_dict() for r in self.rounds]
|
|
80
|
+
if self.detection is not None and (
|
|
81
|
+
self.detection.is_question or self.detection.suppressed or self.rounds
|
|
82
|
+
):
|
|
83
|
+
data["detection"] = self.detection.to_dict()
|
|
84
|
+
if self.unmeasured is not None:
|
|
85
|
+
data["unmeasured"] = self.unmeasured
|
|
86
|
+
return data
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def from_dict(cls, data: Mapping[str, Any]) -> TurnResult:
|
|
90
|
+
score, passed = data.get("score"), data.get("passed")
|
|
91
|
+
return cls(
|
|
92
|
+
index=int(data["index"]),
|
|
93
|
+
user=str(data.get("user", "")),
|
|
94
|
+
evaluated=bool(data.get("evaluated", True)),
|
|
95
|
+
reply=str(data.get("reply", "")),
|
|
96
|
+
route=str(data.get("route", "")),
|
|
97
|
+
status=str(data.get("status", "")),
|
|
98
|
+
latency_ms=float(data.get("latency_ms", 0.0)),
|
|
99
|
+
# None stays None: an ungraded turn must not come back as a failed one.
|
|
100
|
+
score=None if score is None else float(score),
|
|
101
|
+
passed=None if passed is None else bool(passed),
|
|
102
|
+
results=[EvalResult.from_dict(r) for r in data.get("results", [])],
|
|
103
|
+
error=data.get("error"),
|
|
104
|
+
rounds=[Round.from_dict(r) for r in data.get("rounds", [])],
|
|
105
|
+
detection=Detection.from_dict(data["detection"]) if data.get("detection") else None,
|
|
106
|
+
unmeasured=data.get("unmeasured"),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(slots=True)
|
|
111
|
+
class SampleResult:
|
|
112
|
+
"""One run of a case's whole conversation, in a session of its own.
|
|
113
|
+
|
|
114
|
+
`message`, `reply`, `route`, `status`, `latency_ms` and `response` describe
|
|
115
|
+
the last graded turn that ran, as they do on `CaseResult`. A sample passes
|
|
116
|
+
only if every graded turn did; one the adapter could not deliver never does.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
#: 1-based position among the case's samples.
|
|
120
|
+
index: int
|
|
121
|
+
session_id: str
|
|
122
|
+
message: str
|
|
123
|
+
score: float = 0.0
|
|
124
|
+
passed: bool = False
|
|
125
|
+
reply: str = ""
|
|
126
|
+
route: str = ""
|
|
127
|
+
status: str = ""
|
|
128
|
+
latency_ms: float = 0.0
|
|
129
|
+
response: dict[str, Any] = field(default_factory=dict)
|
|
130
|
+
results: list[EvalResult] = field(default_factory=list)
|
|
131
|
+
error: str | None = None
|
|
132
|
+
#: Every turn that was attempted. Turns after an error are absent, not blank.
|
|
133
|
+
turns: list[TurnResult] = field(default_factory=list)
|
|
134
|
+
#: Why the sample could not be measured (see `TurnResult.unmeasured`). Not an error.
|
|
135
|
+
unmeasured: str | None = None
|
|
136
|
+
|
|
137
|
+
def to_dict(self) -> dict[str, Any]:
|
|
138
|
+
data: dict[str, Any] = {
|
|
139
|
+
"index": self.index,
|
|
140
|
+
"session_id": self.session_id,
|
|
141
|
+
"score": round(self.score, 4),
|
|
142
|
+
"passed": self.passed,
|
|
143
|
+
"error": self.error,
|
|
144
|
+
"response": self.response,
|
|
145
|
+
"turns": [t.to_dict() for t in self.turns],
|
|
146
|
+
}
|
|
147
|
+
if self.unmeasured is not None:
|
|
148
|
+
data["unmeasured"] = self.unmeasured
|
|
149
|
+
return data
|
|
150
|
+
|
|
151
|
+
@classmethod
|
|
152
|
+
def from_dict(cls, data: Mapping[str, Any]) -> SampleResult:
|
|
153
|
+
return cls.from_turns(
|
|
154
|
+
[TurnResult.from_dict(t) for t in data.get("turns", [])],
|
|
155
|
+
index=int(data["index"]),
|
|
156
|
+
session_id=str(data.get("session_id", "")),
|
|
157
|
+
score=float(data.get("score", 0.0)),
|
|
158
|
+
passed=bool(data["passed"]),
|
|
159
|
+
response=dict(data.get("response") or {}),
|
|
160
|
+
error=data.get("error"),
|
|
161
|
+
unmeasured=data.get("unmeasured"),
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
@classmethod
|
|
165
|
+
def from_turns(
|
|
166
|
+
cls,
|
|
167
|
+
turns: list[TurnResult],
|
|
168
|
+
*,
|
|
169
|
+
index: int,
|
|
170
|
+
session_id: str,
|
|
171
|
+
score: float,
|
|
172
|
+
passed: bool,
|
|
173
|
+
response: dict[str, Any],
|
|
174
|
+
error: str | None,
|
|
175
|
+
unmeasured: str | None = None,
|
|
176
|
+
) -> SampleResult:
|
|
177
|
+
"""Rebuild a sample around its turns, re-deriving what `to_dict` leaves out.
|
|
178
|
+
|
|
179
|
+
The exchange a sample shows is its last graded turn that ran, and its
|
|
180
|
+
results are its graded turns' results; neither is written twice.
|
|
181
|
+
"""
|
|
182
|
+
graded = [t for t in turns if t.evaluated and t.error is None]
|
|
183
|
+
shown = graded[-1] if graded else None
|
|
184
|
+
return cls(
|
|
185
|
+
index=index,
|
|
186
|
+
session_id=session_id,
|
|
187
|
+
message=shown.user if shown else (turns[0].user if turns else ""),
|
|
188
|
+
score=score,
|
|
189
|
+
passed=passed,
|
|
190
|
+
reply=shown.reply if shown else "",
|
|
191
|
+
route=shown.route if shown else "",
|
|
192
|
+
status=shown.status if shown else "",
|
|
193
|
+
latency_ms=shown.latency_ms if shown else 0.0,
|
|
194
|
+
response=response,
|
|
195
|
+
results=[r for t in turns if t.evaluated and t.error is None for r in t.results],
|
|
196
|
+
error=error,
|
|
197
|
+
turns=turns,
|
|
198
|
+
unmeasured=unmeasured,
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
@dataclass(slots=True)
|
|
203
|
+
class CaseResult:
|
|
204
|
+
"""One test case, run and graded: one sample, or several when it declares them.
|
|
205
|
+
|
|
206
|
+
`message`, `reply`, `route`, `status`, `latency_ms`, `response` and `turns`
|
|
207
|
+
are the representative sample's - the first one that failed, else the last -
|
|
208
|
+
so a reader that knows nothing about samples or turns sees one coherent
|
|
209
|
+
exchange, and for a case run once that is simply the run. `results` is every
|
|
210
|
+
graded turn's results from every sample that ran: a critical failure in any
|
|
211
|
+
of them gates the run, however the case's K-of-N verdict came out.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
case_id: str
|
|
215
|
+
suite: str
|
|
216
|
+
message: str
|
|
217
|
+
score: float = 0.0
|
|
218
|
+
passed: bool = False
|
|
219
|
+
description: str = ""
|
|
220
|
+
risk: str = "low"
|
|
221
|
+
tags: list[str] = field(default_factory=list)
|
|
222
|
+
reply: str = ""
|
|
223
|
+
route: str = ""
|
|
224
|
+
status: str = ""
|
|
225
|
+
latency_ms: float = 0.0
|
|
226
|
+
results: list[EvalResult] = field(default_factory=list)
|
|
227
|
+
#: Set when the case could not be run at all (an exception, not a low score).
|
|
228
|
+
error: str | None = None
|
|
229
|
+
response: dict[str, Any] = field(default_factory=dict)
|
|
230
|
+
#: Every sample that ran. Fewer than `samples_planned` when the gate ended early.
|
|
231
|
+
samples: list[SampleResult] = field(default_factory=list)
|
|
232
|
+
samples_planned: int = 1
|
|
233
|
+
#: The K of the K-of-N gate this case was judged against.
|
|
234
|
+
min_passing_samples: int = 1
|
|
235
|
+
#: The representative sample's `unmeasured`: our gap, so the case neither passed nor
|
|
236
|
+
#: counts as the agent failing it.
|
|
237
|
+
unmeasured: str | None = None
|
|
238
|
+
|
|
239
|
+
@property
|
|
240
|
+
def representative(self) -> SampleResult | None:
|
|
241
|
+
return next((s for s in self.samples if not s.passed), self.samples[-1] if self.samples else None)
|
|
242
|
+
|
|
243
|
+
@property
|
|
244
|
+
def turns(self) -> list[TurnResult]:
|
|
245
|
+
sample = self.representative
|
|
246
|
+
return sample.turns if sample else []
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def passing_samples(self) -> int:
|
|
250
|
+
return sum(1 for s in self.samples if s.passed)
|
|
251
|
+
|
|
252
|
+
@property
|
|
253
|
+
def sampled(self) -> bool:
|
|
254
|
+
"""True when the case asked for more than one run, so non-determinism was measured."""
|
|
255
|
+
return self.samples_planned > 1
|
|
256
|
+
|
|
257
|
+
@property
|
|
258
|
+
def flaky(self) -> bool:
|
|
259
|
+
"""Some samples passed and some did not: the same input gave different verdicts."""
|
|
260
|
+
return 0 < self.passing_samples < len(self.samples)
|
|
261
|
+
|
|
262
|
+
@property
|
|
263
|
+
def judge_usage(self) -> JudgeUsage | None:
|
|
264
|
+
"""What the judge spent on every graded turn of every sample. None: no metered judge."""
|
|
265
|
+
usages = [r.judge for r in self.results if r.judge is not None]
|
|
266
|
+
return sum(usages[1:], usages[0]) if usages else None
|
|
267
|
+
|
|
268
|
+
@property
|
|
269
|
+
def persona_usage(self) -> PersonaUsage | None:
|
|
270
|
+
"""What the simulated user cost across every turn of every sample. None: no persona ran."""
|
|
271
|
+
usages = [u for s in self.samples for t in s.turns if (u := t.persona_usage) is not None]
|
|
272
|
+
return sum(usages[1:], usages[0]) if usages else None
|
|
273
|
+
|
|
274
|
+
@property
|
|
275
|
+
def degraded_results(self) -> list[EvalResult]:
|
|
276
|
+
"""Results graded by the fallback because the judge that was asked for failed."""
|
|
277
|
+
return [r for r in self.results if r.degraded is not None]
|
|
278
|
+
|
|
279
|
+
@property
|
|
280
|
+
def failed_checks(self) -> list[Check]:
|
|
281
|
+
return [c for r in self.results for c in r.failures]
|
|
282
|
+
|
|
283
|
+
@property
|
|
284
|
+
def critical_checks(self) -> list[Check]:
|
|
285
|
+
return [c for r in self.results for c in r.critical_failures]
|
|
286
|
+
|
|
287
|
+
def to_dict(self) -> dict[str, Any]:
|
|
288
|
+
data: dict[str, Any] = {
|
|
289
|
+
"case_id": self.case_id,
|
|
290
|
+
"suite": self.suite,
|
|
291
|
+
"message": self.message,
|
|
292
|
+
"description": self.description,
|
|
293
|
+
"risk": self.risk,
|
|
294
|
+
"tags": self.tags,
|
|
295
|
+
"score": round(self.score, 4),
|
|
296
|
+
"passed": self.passed,
|
|
297
|
+
"reply": self.reply,
|
|
298
|
+
"route": self.route,
|
|
299
|
+
"status": self.status,
|
|
300
|
+
"latency_ms": self.latency_ms,
|
|
301
|
+
"error": self.error,
|
|
302
|
+
"results": [r.to_dict() for r in self.results],
|
|
303
|
+
"response": self.response,
|
|
304
|
+
"turns": [t.to_dict() for t in self.turns],
|
|
305
|
+
}
|
|
306
|
+
if self.unmeasured is not None:
|
|
307
|
+
data["unmeasured"] = self.unmeasured
|
|
308
|
+
# Only a case that asked for sampling says anything about it, so a case
|
|
309
|
+
# run once serialises exactly as it did before samples existed.
|
|
310
|
+
if self.sampled:
|
|
311
|
+
data.update(
|
|
312
|
+
samples_planned=self.samples_planned,
|
|
313
|
+
min_passing_samples=self.min_passing_samples,
|
|
314
|
+
passing_samples=self.passing_samples,
|
|
315
|
+
flaky=self.flaky,
|
|
316
|
+
samples=[s.to_dict() for s in self.samples],
|
|
317
|
+
)
|
|
318
|
+
return data
|
|
319
|
+
|
|
320
|
+
@classmethod
|
|
321
|
+
def from_dict(cls, data: Mapping[str, Any]) -> CaseResult:
|
|
322
|
+
"""Rebuild the whole case, samples and checks included.
|
|
323
|
+
|
|
324
|
+
A case run once wrote no `samples`, so its one run is rebuilt from the
|
|
325
|
+
case-level fields, and it serialises again exactly as it was read.
|
|
326
|
+
"""
|
|
327
|
+
passed, error = bool(data["passed"]), data.get("error")
|
|
328
|
+
score = float(data.get("score", 0.0))
|
|
329
|
+
response = dict(data.get("response") or {})
|
|
330
|
+
if "samples" in data:
|
|
331
|
+
samples = [SampleResult.from_dict(s) for s in data["samples"]]
|
|
332
|
+
else:
|
|
333
|
+
samples = [
|
|
334
|
+
SampleResult.from_turns(
|
|
335
|
+
[TurnResult.from_dict(t) for t in data.get("turns", [])],
|
|
336
|
+
index=1,
|
|
337
|
+
session_id="",
|
|
338
|
+
score=score,
|
|
339
|
+
passed=passed,
|
|
340
|
+
response=response,
|
|
341
|
+
error=error,
|
|
342
|
+
unmeasured=data.get("unmeasured"),
|
|
343
|
+
)
|
|
344
|
+
]
|
|
345
|
+
return cls(
|
|
346
|
+
case_id=str(data["case_id"]),
|
|
347
|
+
suite=str(data.get("suite", "")),
|
|
348
|
+
message=str(data.get("message", "")),
|
|
349
|
+
score=score,
|
|
350
|
+
passed=passed,
|
|
351
|
+
description=str(data.get("description", "")),
|
|
352
|
+
risk=str(data.get("risk", "low")),
|
|
353
|
+
tags=[str(t) for t in data.get("tags", [])],
|
|
354
|
+
reply=str(data.get("reply", "")),
|
|
355
|
+
route=str(data.get("route", "")),
|
|
356
|
+
status=str(data.get("status", "")),
|
|
357
|
+
latency_ms=float(data.get("latency_ms", 0.0)),
|
|
358
|
+
results=[EvalResult.from_dict(r) for r in data.get("results", [])],
|
|
359
|
+
error=error,
|
|
360
|
+
response=response,
|
|
361
|
+
samples=samples,
|
|
362
|
+
samples_planned=int(data.get("samples_planned", 1)),
|
|
363
|
+
min_passing_samples=int(data.get("min_passing_samples", 1)),
|
|
364
|
+
unmeasured=data.get("unmeasured"),
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
@dataclass(slots=True)
|
|
369
|
+
class RunReport:
|
|
370
|
+
"""Every case in one execution, plus the context needed to reproduce it."""
|
|
371
|
+
|
|
372
|
+
run_id: str
|
|
373
|
+
started_at: str = field(default_factory=_now)
|
|
374
|
+
finished_at: str = ""
|
|
375
|
+
duration_s: float = 0.0
|
|
376
|
+
#: Provider, retriever, prompt version and thresholds. Two runs are only
|
|
377
|
+
#: comparable when this matches; the regression comparator says so if not.
|
|
378
|
+
context: dict[str, Any] = field(default_factory=dict)
|
|
379
|
+
cases: list[CaseResult] = field(default_factory=list)
|
|
380
|
+
#: Set when the run stopped before every case ran (judge budget exhausted).
|
|
381
|
+
#: Such a report has no verdict: its pass rate is about cases that happened to
|
|
382
|
+
#: run first, not about the suite.
|
|
383
|
+
incomplete: dict[str, Any] | None = None
|
|
384
|
+
#: How much of its text has been withheld: full | ci | metrics_only. A report
|
|
385
|
+
#: straight from the runner is `full`; `reporting.redaction` is what changes it.
|
|
386
|
+
redaction: str = "full"
|
|
387
|
+
|
|
388
|
+
# ------------------------------------------------------------- summaries #
|
|
389
|
+
|
|
390
|
+
@property
|
|
391
|
+
def judge_usage(self) -> JudgeUsage | None:
|
|
392
|
+
"""The judge's total spend across the run; None when no metered judge ran."""
|
|
393
|
+
usages = [u for c in self.cases if (u := c.judge_usage) is not None]
|
|
394
|
+
return sum(usages[1:], usages[0]) if usages else None
|
|
395
|
+
|
|
396
|
+
@property
|
|
397
|
+
def persona_usage(self) -> PersonaUsage | None:
|
|
398
|
+
"""The simulated user's total spend; None when no persona ran. Never part of the judge's."""
|
|
399
|
+
usages = [u for c in self.cases if (u := c.persona_usage) is not None]
|
|
400
|
+
return sum(usages[1:], usages[0]) if usages else None
|
|
401
|
+
|
|
402
|
+
@property
|
|
403
|
+
def degraded_results(self) -> list[tuple[CaseResult, EvalResult]]:
|
|
404
|
+
return [(c, r) for c in self.cases for r in c.degraded_results]
|
|
405
|
+
|
|
406
|
+
@property
|
|
407
|
+
def sampled(self) -> bool:
|
|
408
|
+
return any(c.sampled for c in self.cases)
|
|
409
|
+
|
|
410
|
+
@property
|
|
411
|
+
def flaky_cases(self) -> list[CaseResult]:
|
|
412
|
+
return [c for c in self.cases if c.flaky]
|
|
413
|
+
|
|
414
|
+
def summary(self) -> dict[str, Any]:
|
|
415
|
+
total = len(self.cases)
|
|
416
|
+
passed = sum(1 for c in self.cases if c.passed)
|
|
417
|
+
errors = sum(1 for c in self.cases if c.error)
|
|
418
|
+
summary: dict[str, Any] = {
|
|
419
|
+
"total": total,
|
|
420
|
+
"passed": passed,
|
|
421
|
+
"failed": total - passed,
|
|
422
|
+
"errors": errors,
|
|
423
|
+
"pass_rate": round(passed / total, 4) if total else 0.0,
|
|
424
|
+
"mean_score": round(sum(c.score for c in self.cases) / total, 4) if total else 0.0,
|
|
425
|
+
"critical_failures": sum(len(c.critical_checks) for c in self.cases),
|
|
426
|
+
"mean_latency_ms": round(
|
|
427
|
+
sum(c.latency_ms for c in self.cases) / total, 2
|
|
428
|
+
) if total else 0.0,
|
|
429
|
+
}
|
|
430
|
+
# Absent, not zero: "0 unmeasured" would claim clarification was looked for.
|
|
431
|
+
if unmeasured := sum(1 for c in self.cases if c.unmeasured):
|
|
432
|
+
summary["unmeasured"] = unmeasured
|
|
433
|
+
if (persona := self.persona_usage) is not None:
|
|
434
|
+
summary.update(
|
|
435
|
+
persona_calls=persona.calls,
|
|
436
|
+
persona_prompt_tokens=persona.prompt_tokens,
|
|
437
|
+
persona_completion_tokens=persona.completion_tokens,
|
|
438
|
+
persona_cost_usd=round(persona.cost_usd, 6),
|
|
439
|
+
)
|
|
440
|
+
# Absent, not zero, when nothing was sampled: "0 flaky" is a finding
|
|
441
|
+
# about cases that were run more than once, and none were.
|
|
442
|
+
if self.sampled:
|
|
443
|
+
summary["flaky"] = len(self.flaky_cases)
|
|
444
|
+
# Likewise absent for a free judge: zero calls would claim a metered one ran.
|
|
445
|
+
if (usage := self.judge_usage) is not None:
|
|
446
|
+
summary.update(
|
|
447
|
+
judge_calls=usage.calls,
|
|
448
|
+
judge_cache_hits=usage.cache_hits,
|
|
449
|
+
judge_prompt_tokens=usage.prompt_tokens,
|
|
450
|
+
judge_completion_tokens=usage.completion_tokens,
|
|
451
|
+
judge_cost_usd=round(usage.cost_usd, 6),
|
|
452
|
+
judge_degraded=len(self.degraded_results),
|
|
453
|
+
)
|
|
454
|
+
return summary
|
|
455
|
+
|
|
456
|
+
def by_suite(self) -> dict[str, dict[str, Any]]:
|
|
457
|
+
buckets: dict[str, list[CaseResult]] = defaultdict(list)
|
|
458
|
+
for case in self.cases:
|
|
459
|
+
buckets[case.suite].append(case)
|
|
460
|
+
return {
|
|
461
|
+
suite: {
|
|
462
|
+
"total": len(items),
|
|
463
|
+
"passed": sum(1 for c in items if c.passed),
|
|
464
|
+
"pass_rate": round(sum(1 for c in items if c.passed) / len(items), 4),
|
|
465
|
+
"mean_score": round(sum(c.score for c in items) / len(items), 4),
|
|
466
|
+
}
|
|
467
|
+
for suite, items in sorted(buckets.items())
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
def by_evaluator(self) -> dict[str, dict[str, Any]]:
|
|
471
|
+
scores: dict[str, list[float]] = defaultdict(list)
|
|
472
|
+
failures: dict[str, int] = defaultdict(int)
|
|
473
|
+
for case in self.cases:
|
|
474
|
+
for result in case.results:
|
|
475
|
+
if result.skipped:
|
|
476
|
+
continue
|
|
477
|
+
scores[result.evaluator].append(result.score)
|
|
478
|
+
failures[result.evaluator] += 0 if result.passed else 1
|
|
479
|
+
return {
|
|
480
|
+
name: {
|
|
481
|
+
"cases": len(values),
|
|
482
|
+
"mean_score": round(sum(values) / len(values), 4),
|
|
483
|
+
"failed": failures[name],
|
|
484
|
+
}
|
|
485
|
+
for name, values in sorted(scores.items())
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
def failing_checks(self) -> dict[str, int]:
|
|
489
|
+
"""Which named checks fail most often. The first thing to read in a report."""
|
|
490
|
+
counts: dict[str, int] = defaultdict(int)
|
|
491
|
+
for case in self.cases:
|
|
492
|
+
for check in case.failed_checks:
|
|
493
|
+
counts[check.name] += 1
|
|
494
|
+
return dict(sorted(counts.items(), key=lambda kv: kv[1], reverse=True))
|
|
495
|
+
|
|
496
|
+
def to_dict(self) -> dict[str, Any]:
|
|
497
|
+
data: dict[str, Any] = {
|
|
498
|
+
"run_id": self.run_id,
|
|
499
|
+
"started_at": self.started_at,
|
|
500
|
+
"finished_at": self.finished_at,
|
|
501
|
+
"duration_s": round(self.duration_s, 2),
|
|
502
|
+
"redaction": self.redaction,
|
|
503
|
+
"context": self.context,
|
|
504
|
+
"summary": self.summary(),
|
|
505
|
+
"by_suite": self.by_suite(),
|
|
506
|
+
"by_evaluator": self.by_evaluator(),
|
|
507
|
+
"failing_checks": self.failing_checks(),
|
|
508
|
+
"cases": [c.to_dict() for c in self.cases],
|
|
509
|
+
}
|
|
510
|
+
if self.incomplete is not None:
|
|
511
|
+
data["incomplete"] = self.incomplete
|
|
512
|
+
return data
|
|
513
|
+
|
|
514
|
+
@classmethod
|
|
515
|
+
def from_dict(cls, data: Mapping[str, Any]) -> RunReport:
|
|
516
|
+
"""Read a report back whole. The summaries are recomputed from its cases.
|
|
517
|
+
|
|
518
|
+
A verdict, a severity or a case id that is not there is refused, not
|
|
519
|
+
defaulted: a baseline that read as passing because a key was missing
|
|
520
|
+
would hide every regression measured against it.
|
|
521
|
+
"""
|
|
522
|
+
# A baseline has the same top-level keys and none of the checks. Read as
|
|
523
|
+
# a report it would look like a run with no critical failures.
|
|
524
|
+
if data.get("kind") == "baseline":
|
|
525
|
+
raise DatasetError(
|
|
526
|
+
"This file is a baseline summary, not a run report: it holds verdicts "
|
|
527
|
+
"only, and cannot be rendered or promoted again."
|
|
528
|
+
)
|
|
529
|
+
try:
|
|
530
|
+
return cls(
|
|
531
|
+
run_id=str(data.get("run_id", "")),
|
|
532
|
+
started_at=str(data.get("started_at", "")),
|
|
533
|
+
finished_at=str(data.get("finished_at", "")),
|
|
534
|
+
duration_s=float(data.get("duration_s", 0.0)),
|
|
535
|
+
context=dict(data.get("context", {})),
|
|
536
|
+
cases=[CaseResult.from_dict(c) for c in data.get("cases", [])],
|
|
537
|
+
incomplete=dict(data["incomplete"]) if data.get("incomplete") else None,
|
|
538
|
+
# Reports older than the redaction layer were written whole.
|
|
539
|
+
redaction=str(data.get("redaction", "full")),
|
|
540
|
+
)
|
|
541
|
+
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
|
542
|
+
raise DatasetError(
|
|
543
|
+
f"Not a readable run report ({type(exc).__name__}: {exc})."
|
|
544
|
+
) from exc
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def response_snapshot(response: ChatResponse, *, max_chunks: int = 5) -> dict[str, Any]:
|
|
548
|
+
"""A compact, JSON-safe view of a response for the report.
|
|
549
|
+
|
|
550
|
+
Unredacted, like everything the runner produces: the reply is whole and the
|
|
551
|
+
top chunks carry their text, so a local `full` report can show what the agent
|
|
552
|
+
read. Whether any of it reaches a file is decided in one place, by
|
|
553
|
+
`reporting.redaction`, which withholds chunk text from every mode but `full`.
|
|
554
|
+
"""
|
|
555
|
+
return {
|
|
556
|
+
"reply": response.reply,
|
|
557
|
+
"route": response.route,
|
|
558
|
+
"status": response.status.value,
|
|
559
|
+
"intent": response.intent.name,
|
|
560
|
+
"intent_confidence": round(response.intent.confidence, 4),
|
|
561
|
+
"citations": [c.doc_id for c in response.citations],
|
|
562
|
+
"retrieved": [
|
|
563
|
+
{"doc_id": c.doc_id, "score": c.score, "title": c.title, "text": c.text}
|
|
564
|
+
for c in response.retrieved[:max_chunks]
|
|
565
|
+
],
|
|
566
|
+
"tool_calls": [
|
|
567
|
+
{"name": t.name, "ok": t.ok, "error": t.error} for t in response.tool_calls
|
|
568
|
+
],
|
|
569
|
+
"safety": {
|
|
570
|
+
"blocked": response.safety.blocked,
|
|
571
|
+
"injection_detected": response.safety.injection_detected,
|
|
572
|
+
"risk_score": response.safety.risk_score,
|
|
573
|
+
"reasons": response.safety.reasons,
|
|
574
|
+
},
|
|
575
|
+
"latency_ms": response.latency_ms,
|
|
576
|
+
"trace_id": response.trace_id,
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def merge_reports(reports: Sequence[RunReport], run_id: str) -> RunReport:
|
|
581
|
+
"""Combine several runs into one report. Used when suites run separately."""
|
|
582
|
+
merged = RunReport(run_id=run_id)
|
|
583
|
+
for report in reports:
|
|
584
|
+
merged.cases.extend(report.cases)
|
|
585
|
+
merged.context.update(report.context)
|
|
586
|
+
merged.finished_at = _now()
|
|
587
|
+
merged.duration_s = sum(r.duration_s for r in reports)
|
|
588
|
+
return merged
|