askback 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. askback/__init__.py +29 -0
  2. askback/__main__.py +8 -0
  3. askback/adapters/__init__.py +11 -0
  4. askback/adapters/base.py +189 -0
  5. askback/adapters/factory.py +176 -0
  6. askback/adapters/http.py +443 -0
  7. askback/adapters/in_process.py +135 -0
  8. askback/adapters/payload.py +207 -0
  9. askback/adapters/scripted.py +117 -0
  10. askback/adapters/socketio_adapter.py +885 -0
  11. askback/api/__init__.py +18 -0
  12. askback/api/corpus_store.py +85 -0
  13. askback/api/dashboard.py +444 -0
  14. askback/api/server.py +200 -0
  15. askback/api/state.py +82 -0
  16. askback/api/static/app.css +423 -0
  17. askback/api/static/app.js +98 -0
  18. askback/api/static/core.js +201 -0
  19. askback/api/static/index.html +175 -0
  20. askback/api/static/panels.js +651 -0
  21. askback/cli.py +618 -0
  22. askback/core/__init__.py +58 -0
  23. askback/core/config.py +224 -0
  24. askback/core/contracts.py +201 -0
  25. askback/core/errors.py +35 -0
  26. askback/core/logging.py +57 -0
  27. askback/core/registry.py +73 -0
  28. askback/core/text.py +151 -0
  29. askback/datasets/__init__.py +14 -0
  30. askback/datasets/loader.py +268 -0
  31. askback/datasets/schema.py +525 -0
  32. askback/demo/__init__.py +42 -0
  33. askback/demo/agent.py +195 -0
  34. askback/demo/suites/demo.yaml +100 -0
  35. askback/evaluation/__init__.py +70 -0
  36. askback/evaluation/base.py +214 -0
  37. askback/evaluation/calibration.py +405 -0
  38. askback/evaluation/citation_eval.py +580 -0
  39. askback/evaluation/clarification.py +477 -0
  40. askback/evaluation/engine.py +193 -0
  41. askback/evaluation/escalation_eval.py +109 -0
  42. askback/evaluation/gates.py +351 -0
  43. askback/evaluation/judge_cache.py +94 -0
  44. askback/evaluation/llm_judge.py +463 -0
  45. askback/evaluation/patterns.py +48 -0
  46. askback/evaluation/pii_eval.py +252 -0
  47. askback/evaluation/rag_eval.py +168 -0
  48. askback/evaluation/regression.py +438 -0
  49. askback/evaluation/results.py +588 -0
  50. askback/evaluation/rule_based.py +198 -0
  51. askback/evaluation/safety_eval.py +170 -0
  52. askback/providers/__init__.py +10 -0
  53. askback/providers/base.py +157 -0
  54. askback/providers/openai_compat.py +340 -0
  55. askback/py.typed +0 -0
  56. askback/reporting/__init__.py +77 -0
  57. askback/reporting/excel_report.py +117 -0
  58. askback/reporting/html_report.py +283 -0
  59. askback/reporting/json_report.py +27 -0
  60. askback/reporting/markdown_report.py +95 -0
  61. askback/reporting/redaction.py +185 -0
  62. askback/runner/__init__.py +5 -0
  63. askback/runner/responders.py +279 -0
  64. askback/runner/test_runner.py +424 -0
  65. askback-0.1.0.dist-info/METADATA +504 -0
  66. askback-0.1.0.dist-info/RECORD +71 -0
  67. askback-0.1.0.dist-info/WHEEL +5 -0
  68. askback-0.1.0.dist-info/entry_points.txt +2 -0
  69. askback-0.1.0.dist-info/licenses/LICENSE +176 -0
  70. askback-0.1.0.dist-info/licenses/NOTICE +15 -0
  71. askback-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,588 @@
1
+ """Run results: what one evaluated case produced, and what a whole run produced.
2
+
3
+ These structures are the wire format for JSON reports, so they round-trip
4
+ through `to_dict` / `from_dict`: reading a report back and writing it again
5
+ gives the same report. The JSON is the canonical form, and the other formats are
6
+ rendered from a report read back out of it, so anything `from_dict` dropped
7
+ would be missing from them too. A baseline is not one of these: it is a summary
8
+ of a run, and `regression.Baseline` owns its format.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from collections import defaultdict
14
+ from collections.abc import Mapping, Sequence
15
+ from dataclasses import dataclass, field
16
+ from datetime import UTC, datetime
17
+ from typing import Any
18
+
19
+ from ..core.contracts import ChatResponse
20
+ from ..core.errors import DatasetError
21
+ from .base import Check, EvalResult, JudgeUsage
22
+ from .clarification import Detection, PersonaUsage, Round
23
+
24
+
25
+ def _now() -> str:
26
+ return datetime.now(UTC).isoformat(timespec="seconds")
27
+
28
+
29
+ @dataclass(slots=True)
30
+ class TurnResult:
31
+ """One turn of a case's conversation: what was sent and what came back."""
32
+
33
+ #: 1-based, like the loader's "turn N" in a dataset error.
34
+ index: int
35
+ user: str
36
+ #: False for a setup turn, which is sent and never graded.
37
+ evaluated: bool = True
38
+ reply: str = ""
39
+ route: str = ""
40
+ status: str = ""
41
+ latency_ms: float = 0.0
42
+ #: None until the turn is graded. A setup turn, or one the adapter could not
43
+ #: deliver, has no verdict - which is not the same thing as a passing one.
44
+ score: float | None = None
45
+ passed: bool | None = None
46
+ results: list[EvalResult] = field(default_factory=list)
47
+ error: str | None = None
48
+ #: Every question the agent asked on the way to `reply`, and what we answered.
49
+ rounds: list[Round] = field(default_factory=list)
50
+ #: How the final reply was classified. Written only when it is a question, was ruled
51
+ #: out as one, or questions were answered on the way; a plain answer says nothing.
52
+ detection: Detection | None = None
53
+ #: Set when the agent asked and we had nothing to say: a gap in the suite, not a
54
+ #: failure of the agent. The turn is neither graded nor passed.
55
+ unmeasured: str | None = None
56
+
57
+ @property
58
+ def persona_usage(self) -> PersonaUsage | None:
59
+ usages = [r.usage for r in self.rounds if r.usage is not None]
60
+ return sum(usages[1:], usages[0]) if usages else None
61
+
62
+ def to_dict(self) -> dict[str, Any]:
63
+ data: dict[str, Any] = {
64
+ "index": self.index,
65
+ "user": self.user,
66
+ "evaluated": self.evaluated,
67
+ "reply": self.reply,
68
+ "route": self.route,
69
+ "status": self.status,
70
+ "latency_ms": self.latency_ms,
71
+ "score": self.score,
72
+ "passed": self.passed,
73
+ "error": self.error,
74
+ "results": [r.to_dict() for r in self.results],
75
+ }
76
+ # Each only when it has something to say, so a turn with no questions in it
77
+ # serialises exactly as it did before clarification existed.
78
+ if self.rounds:
79
+ data["rounds"] = [r.to_dict() for r in self.rounds]
80
+ if self.detection is not None and (
81
+ self.detection.is_question or self.detection.suppressed or self.rounds
82
+ ):
83
+ data["detection"] = self.detection.to_dict()
84
+ if self.unmeasured is not None:
85
+ data["unmeasured"] = self.unmeasured
86
+ return data
87
+
88
+ @classmethod
89
+ def from_dict(cls, data: Mapping[str, Any]) -> TurnResult:
90
+ score, passed = data.get("score"), data.get("passed")
91
+ return cls(
92
+ index=int(data["index"]),
93
+ user=str(data.get("user", "")),
94
+ evaluated=bool(data.get("evaluated", True)),
95
+ reply=str(data.get("reply", "")),
96
+ route=str(data.get("route", "")),
97
+ status=str(data.get("status", "")),
98
+ latency_ms=float(data.get("latency_ms", 0.0)),
99
+ # None stays None: an ungraded turn must not come back as a failed one.
100
+ score=None if score is None else float(score),
101
+ passed=None if passed is None else bool(passed),
102
+ results=[EvalResult.from_dict(r) for r in data.get("results", [])],
103
+ error=data.get("error"),
104
+ rounds=[Round.from_dict(r) for r in data.get("rounds", [])],
105
+ detection=Detection.from_dict(data["detection"]) if data.get("detection") else None,
106
+ unmeasured=data.get("unmeasured"),
107
+ )
108
+
109
+
110
+ @dataclass(slots=True)
111
+ class SampleResult:
112
+ """One run of a case's whole conversation, in a session of its own.
113
+
114
+ `message`, `reply`, `route`, `status`, `latency_ms` and `response` describe
115
+ the last graded turn that ran, as they do on `CaseResult`. A sample passes
116
+ only if every graded turn did; one the adapter could not deliver never does.
117
+ """
118
+
119
+ #: 1-based position among the case's samples.
120
+ index: int
121
+ session_id: str
122
+ message: str
123
+ score: float = 0.0
124
+ passed: bool = False
125
+ reply: str = ""
126
+ route: str = ""
127
+ status: str = ""
128
+ latency_ms: float = 0.0
129
+ response: dict[str, Any] = field(default_factory=dict)
130
+ results: list[EvalResult] = field(default_factory=list)
131
+ error: str | None = None
132
+ #: Every turn that was attempted. Turns after an error are absent, not blank.
133
+ turns: list[TurnResult] = field(default_factory=list)
134
+ #: Why the sample could not be measured (see `TurnResult.unmeasured`). Not an error.
135
+ unmeasured: str | None = None
136
+
137
+ def to_dict(self) -> dict[str, Any]:
138
+ data: dict[str, Any] = {
139
+ "index": self.index,
140
+ "session_id": self.session_id,
141
+ "score": round(self.score, 4),
142
+ "passed": self.passed,
143
+ "error": self.error,
144
+ "response": self.response,
145
+ "turns": [t.to_dict() for t in self.turns],
146
+ }
147
+ if self.unmeasured is not None:
148
+ data["unmeasured"] = self.unmeasured
149
+ return data
150
+
151
+ @classmethod
152
+ def from_dict(cls, data: Mapping[str, Any]) -> SampleResult:
153
+ return cls.from_turns(
154
+ [TurnResult.from_dict(t) for t in data.get("turns", [])],
155
+ index=int(data["index"]),
156
+ session_id=str(data.get("session_id", "")),
157
+ score=float(data.get("score", 0.0)),
158
+ passed=bool(data["passed"]),
159
+ response=dict(data.get("response") or {}),
160
+ error=data.get("error"),
161
+ unmeasured=data.get("unmeasured"),
162
+ )
163
+
164
+ @classmethod
165
+ def from_turns(
166
+ cls,
167
+ turns: list[TurnResult],
168
+ *,
169
+ index: int,
170
+ session_id: str,
171
+ score: float,
172
+ passed: bool,
173
+ response: dict[str, Any],
174
+ error: str | None,
175
+ unmeasured: str | None = None,
176
+ ) -> SampleResult:
177
+ """Rebuild a sample around its turns, re-deriving what `to_dict` leaves out.
178
+
179
+ The exchange a sample shows is its last graded turn that ran, and its
180
+ results are its graded turns' results; neither is written twice.
181
+ """
182
+ graded = [t for t in turns if t.evaluated and t.error is None]
183
+ shown = graded[-1] if graded else None
184
+ return cls(
185
+ index=index,
186
+ session_id=session_id,
187
+ message=shown.user if shown else (turns[0].user if turns else ""),
188
+ score=score,
189
+ passed=passed,
190
+ reply=shown.reply if shown else "",
191
+ route=shown.route if shown else "",
192
+ status=shown.status if shown else "",
193
+ latency_ms=shown.latency_ms if shown else 0.0,
194
+ response=response,
195
+ results=[r for t in turns if t.evaluated and t.error is None for r in t.results],
196
+ error=error,
197
+ turns=turns,
198
+ unmeasured=unmeasured,
199
+ )
200
+
201
+
202
+ @dataclass(slots=True)
203
+ class CaseResult:
204
+ """One test case, run and graded: one sample, or several when it declares them.
205
+
206
+ `message`, `reply`, `route`, `status`, `latency_ms`, `response` and `turns`
207
+ are the representative sample's - the first one that failed, else the last -
208
+ so a reader that knows nothing about samples or turns sees one coherent
209
+ exchange, and for a case run once that is simply the run. `results` is every
210
+ graded turn's results from every sample that ran: a critical failure in any
211
+ of them gates the run, however the case's K-of-N verdict came out.
212
+ """
213
+
214
+ case_id: str
215
+ suite: str
216
+ message: str
217
+ score: float = 0.0
218
+ passed: bool = False
219
+ description: str = ""
220
+ risk: str = "low"
221
+ tags: list[str] = field(default_factory=list)
222
+ reply: str = ""
223
+ route: str = ""
224
+ status: str = ""
225
+ latency_ms: float = 0.0
226
+ results: list[EvalResult] = field(default_factory=list)
227
+ #: Set when the case could not be run at all (an exception, not a low score).
228
+ error: str | None = None
229
+ response: dict[str, Any] = field(default_factory=dict)
230
+ #: Every sample that ran. Fewer than `samples_planned` when the gate ended early.
231
+ samples: list[SampleResult] = field(default_factory=list)
232
+ samples_planned: int = 1
233
+ #: The K of the K-of-N gate this case was judged against.
234
+ min_passing_samples: int = 1
235
+ #: The representative sample's `unmeasured`: our gap, so the case neither passed nor
236
+ #: counts as the agent failing it.
237
+ unmeasured: str | None = None
238
+
239
+ @property
240
+ def representative(self) -> SampleResult | None:
241
+ return next((s for s in self.samples if not s.passed), self.samples[-1] if self.samples else None)
242
+
243
+ @property
244
+ def turns(self) -> list[TurnResult]:
245
+ sample = self.representative
246
+ return sample.turns if sample else []
247
+
248
+ @property
249
+ def passing_samples(self) -> int:
250
+ return sum(1 for s in self.samples if s.passed)
251
+
252
+ @property
253
+ def sampled(self) -> bool:
254
+ """True when the case asked for more than one run, so non-determinism was measured."""
255
+ return self.samples_planned > 1
256
+
257
+ @property
258
+ def flaky(self) -> bool:
259
+ """Some samples passed and some did not: the same input gave different verdicts."""
260
+ return 0 < self.passing_samples < len(self.samples)
261
+
262
+ @property
263
+ def judge_usage(self) -> JudgeUsage | None:
264
+ """What the judge spent on every graded turn of every sample. None: no metered judge."""
265
+ usages = [r.judge for r in self.results if r.judge is not None]
266
+ return sum(usages[1:], usages[0]) if usages else None
267
+
268
+ @property
269
+ def persona_usage(self) -> PersonaUsage | None:
270
+ """What the simulated user cost across every turn of every sample. None: no persona ran."""
271
+ usages = [u for s in self.samples for t in s.turns if (u := t.persona_usage) is not None]
272
+ return sum(usages[1:], usages[0]) if usages else None
273
+
274
+ @property
275
+ def degraded_results(self) -> list[EvalResult]:
276
+ """Results graded by the fallback because the judge that was asked for failed."""
277
+ return [r for r in self.results if r.degraded is not None]
278
+
279
+ @property
280
+ def failed_checks(self) -> list[Check]:
281
+ return [c for r in self.results for c in r.failures]
282
+
283
+ @property
284
+ def critical_checks(self) -> list[Check]:
285
+ return [c for r in self.results for c in r.critical_failures]
286
+
287
+ def to_dict(self) -> dict[str, Any]:
288
+ data: dict[str, Any] = {
289
+ "case_id": self.case_id,
290
+ "suite": self.suite,
291
+ "message": self.message,
292
+ "description": self.description,
293
+ "risk": self.risk,
294
+ "tags": self.tags,
295
+ "score": round(self.score, 4),
296
+ "passed": self.passed,
297
+ "reply": self.reply,
298
+ "route": self.route,
299
+ "status": self.status,
300
+ "latency_ms": self.latency_ms,
301
+ "error": self.error,
302
+ "results": [r.to_dict() for r in self.results],
303
+ "response": self.response,
304
+ "turns": [t.to_dict() for t in self.turns],
305
+ }
306
+ if self.unmeasured is not None:
307
+ data["unmeasured"] = self.unmeasured
308
+ # Only a case that asked for sampling says anything about it, so a case
309
+ # run once serialises exactly as it did before samples existed.
310
+ if self.sampled:
311
+ data.update(
312
+ samples_planned=self.samples_planned,
313
+ min_passing_samples=self.min_passing_samples,
314
+ passing_samples=self.passing_samples,
315
+ flaky=self.flaky,
316
+ samples=[s.to_dict() for s in self.samples],
317
+ )
318
+ return data
319
+
320
+ @classmethod
321
+ def from_dict(cls, data: Mapping[str, Any]) -> CaseResult:
322
+ """Rebuild the whole case, samples and checks included.
323
+
324
+ A case run once wrote no `samples`, so its one run is rebuilt from the
325
+ case-level fields, and it serialises again exactly as it was read.
326
+ """
327
+ passed, error = bool(data["passed"]), data.get("error")
328
+ score = float(data.get("score", 0.0))
329
+ response = dict(data.get("response") or {})
330
+ if "samples" in data:
331
+ samples = [SampleResult.from_dict(s) for s in data["samples"]]
332
+ else:
333
+ samples = [
334
+ SampleResult.from_turns(
335
+ [TurnResult.from_dict(t) for t in data.get("turns", [])],
336
+ index=1,
337
+ session_id="",
338
+ score=score,
339
+ passed=passed,
340
+ response=response,
341
+ error=error,
342
+ unmeasured=data.get("unmeasured"),
343
+ )
344
+ ]
345
+ return cls(
346
+ case_id=str(data["case_id"]),
347
+ suite=str(data.get("suite", "")),
348
+ message=str(data.get("message", "")),
349
+ score=score,
350
+ passed=passed,
351
+ description=str(data.get("description", "")),
352
+ risk=str(data.get("risk", "low")),
353
+ tags=[str(t) for t in data.get("tags", [])],
354
+ reply=str(data.get("reply", "")),
355
+ route=str(data.get("route", "")),
356
+ status=str(data.get("status", "")),
357
+ latency_ms=float(data.get("latency_ms", 0.0)),
358
+ results=[EvalResult.from_dict(r) for r in data.get("results", [])],
359
+ error=error,
360
+ response=response,
361
+ samples=samples,
362
+ samples_planned=int(data.get("samples_planned", 1)),
363
+ min_passing_samples=int(data.get("min_passing_samples", 1)),
364
+ unmeasured=data.get("unmeasured"),
365
+ )
366
+
367
+
368
+ @dataclass(slots=True)
369
+ class RunReport:
370
+ """Every case in one execution, plus the context needed to reproduce it."""
371
+
372
+ run_id: str
373
+ started_at: str = field(default_factory=_now)
374
+ finished_at: str = ""
375
+ duration_s: float = 0.0
376
+ #: Provider, retriever, prompt version and thresholds. Two runs are only
377
+ #: comparable when this matches; the regression comparator says so if not.
378
+ context: dict[str, Any] = field(default_factory=dict)
379
+ cases: list[CaseResult] = field(default_factory=list)
380
+ #: Set when the run stopped before every case ran (judge budget exhausted).
381
+ #: Such a report has no verdict: its pass rate is about cases that happened to
382
+ #: run first, not about the suite.
383
+ incomplete: dict[str, Any] | None = None
384
+ #: How much of its text has been withheld: full | ci | metrics_only. A report
385
+ #: straight from the runner is `full`; `reporting.redaction` is what changes it.
386
+ redaction: str = "full"
387
+
388
+ # ------------------------------------------------------------- summaries #
389
+
390
+ @property
391
+ def judge_usage(self) -> JudgeUsage | None:
392
+ """The judge's total spend across the run; None when no metered judge ran."""
393
+ usages = [u for c in self.cases if (u := c.judge_usage) is not None]
394
+ return sum(usages[1:], usages[0]) if usages else None
395
+
396
+ @property
397
+ def persona_usage(self) -> PersonaUsage | None:
398
+ """The simulated user's total spend; None when no persona ran. Never part of the judge's."""
399
+ usages = [u for c in self.cases if (u := c.persona_usage) is not None]
400
+ return sum(usages[1:], usages[0]) if usages else None
401
+
402
+ @property
403
+ def degraded_results(self) -> list[tuple[CaseResult, EvalResult]]:
404
+ return [(c, r) for c in self.cases for r in c.degraded_results]
405
+
406
+ @property
407
+ def sampled(self) -> bool:
408
+ return any(c.sampled for c in self.cases)
409
+
410
+ @property
411
+ def flaky_cases(self) -> list[CaseResult]:
412
+ return [c for c in self.cases if c.flaky]
413
+
414
+ def summary(self) -> dict[str, Any]:
415
+ total = len(self.cases)
416
+ passed = sum(1 for c in self.cases if c.passed)
417
+ errors = sum(1 for c in self.cases if c.error)
418
+ summary: dict[str, Any] = {
419
+ "total": total,
420
+ "passed": passed,
421
+ "failed": total - passed,
422
+ "errors": errors,
423
+ "pass_rate": round(passed / total, 4) if total else 0.0,
424
+ "mean_score": round(sum(c.score for c in self.cases) / total, 4) if total else 0.0,
425
+ "critical_failures": sum(len(c.critical_checks) for c in self.cases),
426
+ "mean_latency_ms": round(
427
+ sum(c.latency_ms for c in self.cases) / total, 2
428
+ ) if total else 0.0,
429
+ }
430
+ # Absent, not zero: "0 unmeasured" would claim clarification was looked for.
431
+ if unmeasured := sum(1 for c in self.cases if c.unmeasured):
432
+ summary["unmeasured"] = unmeasured
433
+ if (persona := self.persona_usage) is not None:
434
+ summary.update(
435
+ persona_calls=persona.calls,
436
+ persona_prompt_tokens=persona.prompt_tokens,
437
+ persona_completion_tokens=persona.completion_tokens,
438
+ persona_cost_usd=round(persona.cost_usd, 6),
439
+ )
440
+ # Absent, not zero, when nothing was sampled: "0 flaky" is a finding
441
+ # about cases that were run more than once, and none were.
442
+ if self.sampled:
443
+ summary["flaky"] = len(self.flaky_cases)
444
+ # Likewise absent for a free judge: zero calls would claim a metered one ran.
445
+ if (usage := self.judge_usage) is not None:
446
+ summary.update(
447
+ judge_calls=usage.calls,
448
+ judge_cache_hits=usage.cache_hits,
449
+ judge_prompt_tokens=usage.prompt_tokens,
450
+ judge_completion_tokens=usage.completion_tokens,
451
+ judge_cost_usd=round(usage.cost_usd, 6),
452
+ judge_degraded=len(self.degraded_results),
453
+ )
454
+ return summary
455
+
456
+ def by_suite(self) -> dict[str, dict[str, Any]]:
457
+ buckets: dict[str, list[CaseResult]] = defaultdict(list)
458
+ for case in self.cases:
459
+ buckets[case.suite].append(case)
460
+ return {
461
+ suite: {
462
+ "total": len(items),
463
+ "passed": sum(1 for c in items if c.passed),
464
+ "pass_rate": round(sum(1 for c in items if c.passed) / len(items), 4),
465
+ "mean_score": round(sum(c.score for c in items) / len(items), 4),
466
+ }
467
+ for suite, items in sorted(buckets.items())
468
+ }
469
+
470
+ def by_evaluator(self) -> dict[str, dict[str, Any]]:
471
+ scores: dict[str, list[float]] = defaultdict(list)
472
+ failures: dict[str, int] = defaultdict(int)
473
+ for case in self.cases:
474
+ for result in case.results:
475
+ if result.skipped:
476
+ continue
477
+ scores[result.evaluator].append(result.score)
478
+ failures[result.evaluator] += 0 if result.passed else 1
479
+ return {
480
+ name: {
481
+ "cases": len(values),
482
+ "mean_score": round(sum(values) / len(values), 4),
483
+ "failed": failures[name],
484
+ }
485
+ for name, values in sorted(scores.items())
486
+ }
487
+
488
+ def failing_checks(self) -> dict[str, int]:
489
+ """Which named checks fail most often. The first thing to read in a report."""
490
+ counts: dict[str, int] = defaultdict(int)
491
+ for case in self.cases:
492
+ for check in case.failed_checks:
493
+ counts[check.name] += 1
494
+ return dict(sorted(counts.items(), key=lambda kv: kv[1], reverse=True))
495
+
496
+ def to_dict(self) -> dict[str, Any]:
497
+ data: dict[str, Any] = {
498
+ "run_id": self.run_id,
499
+ "started_at": self.started_at,
500
+ "finished_at": self.finished_at,
501
+ "duration_s": round(self.duration_s, 2),
502
+ "redaction": self.redaction,
503
+ "context": self.context,
504
+ "summary": self.summary(),
505
+ "by_suite": self.by_suite(),
506
+ "by_evaluator": self.by_evaluator(),
507
+ "failing_checks": self.failing_checks(),
508
+ "cases": [c.to_dict() for c in self.cases],
509
+ }
510
+ if self.incomplete is not None:
511
+ data["incomplete"] = self.incomplete
512
+ return data
513
+
514
+ @classmethod
515
+ def from_dict(cls, data: Mapping[str, Any]) -> RunReport:
516
+ """Read a report back whole. The summaries are recomputed from its cases.
517
+
518
+ A verdict, a severity or a case id that is not there is refused, not
519
+ defaulted: a baseline that read as passing because a key was missing
520
+ would hide every regression measured against it.
521
+ """
522
+ # A baseline has the same top-level keys and none of the checks. Read as
523
+ # a report it would look like a run with no critical failures.
524
+ if data.get("kind") == "baseline":
525
+ raise DatasetError(
526
+ "This file is a baseline summary, not a run report: it holds verdicts "
527
+ "only, and cannot be rendered or promoted again."
528
+ )
529
+ try:
530
+ return cls(
531
+ run_id=str(data.get("run_id", "")),
532
+ started_at=str(data.get("started_at", "")),
533
+ finished_at=str(data.get("finished_at", "")),
534
+ duration_s=float(data.get("duration_s", 0.0)),
535
+ context=dict(data.get("context", {})),
536
+ cases=[CaseResult.from_dict(c) for c in data.get("cases", [])],
537
+ incomplete=dict(data["incomplete"]) if data.get("incomplete") else None,
538
+ # Reports older than the redaction layer were written whole.
539
+ redaction=str(data.get("redaction", "full")),
540
+ )
541
+ except (KeyError, TypeError, ValueError, AttributeError) as exc:
542
+ raise DatasetError(
543
+ f"Not a readable run report ({type(exc).__name__}: {exc})."
544
+ ) from exc
545
+
546
+
547
+ def response_snapshot(response: ChatResponse, *, max_chunks: int = 5) -> dict[str, Any]:
548
+ """A compact, JSON-safe view of a response for the report.
549
+
550
+ Unredacted, like everything the runner produces: the reply is whole and the
551
+ top chunks carry their text, so a local `full` report can show what the agent
552
+ read. Whether any of it reaches a file is decided in one place, by
553
+ `reporting.redaction`, which withholds chunk text from every mode but `full`.
554
+ """
555
+ return {
556
+ "reply": response.reply,
557
+ "route": response.route,
558
+ "status": response.status.value,
559
+ "intent": response.intent.name,
560
+ "intent_confidence": round(response.intent.confidence, 4),
561
+ "citations": [c.doc_id for c in response.citations],
562
+ "retrieved": [
563
+ {"doc_id": c.doc_id, "score": c.score, "title": c.title, "text": c.text}
564
+ for c in response.retrieved[:max_chunks]
565
+ ],
566
+ "tool_calls": [
567
+ {"name": t.name, "ok": t.ok, "error": t.error} for t in response.tool_calls
568
+ ],
569
+ "safety": {
570
+ "blocked": response.safety.blocked,
571
+ "injection_detected": response.safety.injection_detected,
572
+ "risk_score": response.safety.risk_score,
573
+ "reasons": response.safety.reasons,
574
+ },
575
+ "latency_ms": response.latency_ms,
576
+ "trace_id": response.trace_id,
577
+ }
578
+
579
+
580
+ def merge_reports(reports: Sequence[RunReport], run_id: str) -> RunReport:
581
+ """Combine several runs into one report. Used when suites run separately."""
582
+ merged = RunReport(run_id=run_id)
583
+ for report in reports:
584
+ merged.cases.extend(report.cases)
585
+ merged.context.update(report.context)
586
+ merged.finished_at = _now()
587
+ merged.duration_s = sum(r.duration_s for r in reports)
588
+ return merged