benigntwin 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
benigntwin/__init__.py ADDED
@@ -0,0 +1,65 @@
1
+ """BenignTwin: benchmark prompt-injection detectors and LLM guardrails.
2
+
3
+ Every attack is evaluated next to a benign twin that shares its topic and
4
+ vocabulary, so a detector gets credit only when it catches the attack *and*
5
+ lets the legitimate request through. Results come with the false-positive
6
+ rate, per attack family and per language breakdowns, and confidence intervals.
7
+
8
+ Quick start::
9
+
10
+ from benigntwin import build_detector, evaluate, load_dataset, render
11
+
12
+ dataset = load_dataset("hf:3nesdeniz/agentic-prompt-injection-boundary-pairs")
13
+ result = evaluate(dataset, build_detector("heuristic"))
14
+ print(render(result.report, "markdown"))
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from ._version import __version__
20
+ from .compare import compare_reports, load_report, render_comparison
21
+ from .datasets import Dataset, DatasetError, FieldMap, load_dataset, validate_examples
22
+ from .detectors import (
23
+ Detector,
24
+ DetectorError,
25
+ DetectorOptions,
26
+ available_detectors,
27
+ build_detector,
28
+ get_detector,
29
+ register,
30
+ )
31
+ from .evaluate import Evaluation, EvaluationError, evaluate
32
+ from .gates import check_gates
33
+ from .models import BenignTwinError, Example, Prediction, Record
34
+ from .report import render, to_json, to_markdown, to_sarif
35
+
36
+ __all__ = [
37
+ "BenignTwinError",
38
+ "Dataset",
39
+ "DatasetError",
40
+ "Detector",
41
+ "DetectorError",
42
+ "DetectorOptions",
43
+ "Evaluation",
44
+ "EvaluationError",
45
+ "Example",
46
+ "FieldMap",
47
+ "Prediction",
48
+ "Record",
49
+ "__version__",
50
+ "available_detectors",
51
+ "build_detector",
52
+ "check_gates",
53
+ "compare_reports",
54
+ "evaluate",
55
+ "get_detector",
56
+ "load_dataset",
57
+ "load_report",
58
+ "register",
59
+ "render",
60
+ "render_comparison",
61
+ "to_json",
62
+ "to_markdown",
63
+ "to_sarif",
64
+ "validate_examples",
65
+ ]
benigntwin/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Allow ``python -m benigntwin``."""
2
+
3
+ from .cli import main
4
+
5
+ raise SystemExit(main())
benigntwin/_version.py ADDED
@@ -0,0 +1,3 @@
1
+ """Single source of the package version."""
2
+
3
+ __version__ = "1.0.0"
benigntwin/cli.py ADDED
@@ -0,0 +1,420 @@
1
+ """Command-line interface: ``benigntwin evaluate | validate | compare | detectors | export-promptfoo``.
2
+
3
+ Exit status of ``evaluate``:
4
+
5
+ * ``0`` success, and every requested gate passed
6
+ * ``1`` an error (bad arguments, unreadable data, failed predictions)
7
+ * ``2`` the ``--max-fpr`` gate failed
8
+ * ``3`` the ``--min-recall`` gate failed
9
+ * ``4`` the ``--min-pair-accuracy`` gate failed
10
+ * ``64`` the command line could not be understood
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import json
17
+ import os
18
+ import sys
19
+ from collections.abc import Sequence
20
+ from typing import Any, NoReturn
21
+
22
+ from ._version import __version__
23
+ from .compare import compare_reports, load_report, render_comparison
24
+ from .datasets import SPLITS, Dataset, FieldMap, load_dataset, validate_examples
25
+ from .detectors import DetectorOptions, available_detectors, build_detector
26
+ from .evaluate import DEFAULT_SLICES, evaluate
27
+ from .export import export_promptfoo
28
+ from .gates import EXIT_ERROR, EXIT_OK, EXIT_USAGE, check_gates, exit_code
29
+ from .models import BenignTwinError
30
+ from .report import FORMATS, predictions_jsonl, render, write_text
31
+
32
+ __all__ = ["build_parser", "main"]
33
+
34
+ _DETECTOR_HELP = (
35
+ "detector to evaluate: a registered name (see `benigntwin detectors`), "
36
+ "hf:owner/model[@revision], python:file.py:function, https://host/path "
37
+ "or predictions:file.jsonl"
38
+ )
39
+
40
+
41
+ def _add_dataset_arguments(parser: argparse.ArgumentParser) -> None:
42
+ group = parser.add_argument_group("dataset")
43
+ group.add_argument(
44
+ "--dataset",
45
+ required=True,
46
+ help="a .jsonl/.json/.parquet file, a directory, or hf:owner/name[@revision]",
47
+ )
48
+ group.add_argument(
49
+ "--split",
50
+ choices=(*SPLITS, "all"),
51
+ help="split to load (default: test for directories and Hub datasets, every row for a file)",
52
+ )
53
+ group.add_argument("--revision", help="branch, tag or commit of a Hub dataset")
54
+ group.add_argument("--hf-file", help="exact file inside a Hub dataset, for non-standard layouts")
55
+ group.add_argument("--refresh", action="store_true", help="download Hub files again")
56
+ group.add_argument("--text-field", help="column holding the input text")
57
+ group.add_argument("--label-field", help="column holding the label")
58
+ group.add_argument("--id-field", help="column holding the row id, or 'auto' to generate ids")
59
+ group.add_argument("--language-field", help="column holding the language tag")
60
+ group.add_argument("--pair-field", help="column linking an attack to its benign twin")
61
+ group.add_argument("--family-field", help="column holding the attack family")
62
+ group.add_argument("--default-language", help="language tag for rows that have none")
63
+ group.add_argument(
64
+ "--where",
65
+ action="append",
66
+ default=[],
67
+ metavar="COLUMN=VALUE",
68
+ help="keep only rows whose column equals the value; repeatable",
69
+ )
70
+ group.add_argument(
71
+ "--default-label",
72
+ help="label for rows that have none, e.g. 'benign' for a corpus of ordinary traffic",
73
+ )
74
+
75
+
76
+ def _load(args: argparse.Namespace) -> Dataset:
77
+ field_map = FieldMap.from_options(
78
+ text_field=args.text_field,
79
+ label_field=args.label_field,
80
+ id_field=args.id_field,
81
+ language_field=args.language_field,
82
+ pair_field=args.pair_field,
83
+ family_field=args.family_field,
84
+ default_language=args.default_language,
85
+ default_label=args.default_label,
86
+ )
87
+ return load_dataset(
88
+ args.dataset,
89
+ split=args.split,
90
+ revision=args.revision,
91
+ hf_file=args.hf_file,
92
+ field_map=field_map,
93
+ refresh=args.refresh,
94
+ where=_key_value(args.where, "filter"),
95
+ )
96
+
97
+
98
+ def _key_value(values: Sequence[str], what: str) -> dict[str, str]:
99
+ result: dict[str, str] = {}
100
+ for value in values:
101
+ key, separator, rest = value.partition("=")
102
+ if not separator or not key.strip():
103
+ raise BenignTwinError(f"invalid {what} {value!r}; expected KEY=VALUE")
104
+ result[key.strip()] = rest
105
+ return result
106
+
107
+
108
+ def _fraction(text: str) -> float:
109
+ try:
110
+ value = float(text)
111
+ except ValueError:
112
+ raise argparse.ArgumentTypeError(f"not a number: {text!r}") from None
113
+ if not 0.0 <= value <= 1.0:
114
+ raise argparse.ArgumentTypeError("must be between 0 and 1 (0.05 means 5%)")
115
+ return value
116
+
117
+
118
+ class _Parser(argparse.ArgumentParser):
119
+ """Usage errors exit with 64 so they cannot be mistaken for a failed gate."""
120
+
121
+ def error(self, message: str) -> NoReturn:
122
+ self.print_usage(sys.stderr)
123
+ self.exit(EXIT_USAGE, f"{self.prog}: error: {message}\n")
124
+
125
+
126
+ def build_parser() -> argparse.ArgumentParser:
127
+ """Build the argument parser."""
128
+
129
+ parser = _Parser(
130
+ prog="benigntwin",
131
+ description=(
132
+ "Benchmark prompt-injection detectors and LLM guardrails on matched benign/attack "
133
+ "pairs: attack recall, false-positive rate and pair accuracy, per attack family and "
134
+ "per language, with confidence intervals."
135
+ ),
136
+ epilog="Documentation: https://github.com/3nesdeniz/benigntwin",
137
+ )
138
+ parser.add_argument("--version", action="version", version=f"benigntwin {__version__}")
139
+ commands = parser.add_subparsers(dest="command", required=True, metavar="command")
140
+
141
+ evaluate_parser = commands.add_parser(
142
+ "evaluate",
143
+ help="run a detector on a dataset and report the results",
144
+ description=__doc__,
145
+ formatter_class=argparse.RawDescriptionHelpFormatter,
146
+ )
147
+ _add_dataset_arguments(evaluate_parser)
148
+ detector = evaluate_parser.add_argument_group("detector")
149
+ detector.add_argument("--detector", required=True, help=_DETECTOR_HELP)
150
+ detector.add_argument(
151
+ "--threshold", type=_fraction, default=0.5, help="flag when score >= threshold (default: 0.5)"
152
+ )
153
+ detector.add_argument(
154
+ "--decision",
155
+ choices=("auto", "label", "score"),
156
+ default="auto",
157
+ help="auto: the detector's own label if it gives one, else the threshold (default)",
158
+ )
159
+ detector.add_argument("--workers", type=int, default=1, help="parallel calls (python and HTTP detectors)")
160
+ detector.add_argument(
161
+ "--allow-errors",
162
+ action="store_true",
163
+ help="report on the rows that worked when the detector fails on some rows",
164
+ )
165
+ hf = evaluate_parser.add_argument_group("Hugging Face models (hf:...)")
166
+ hf.add_argument(
167
+ "--positive-label",
168
+ action="append",
169
+ default=[],
170
+ metavar="NAME",
171
+ help="output label that means attack; repeatable (default: recognized automatically)",
172
+ )
173
+ hf.add_argument("--device", default="cpu", help="cpu, cuda, mps or auto (default: cpu)")
174
+ hf.add_argument("--batch-size", type=int, default=16)
175
+ hf.add_argument("--max-length", type=int, help="token window (default: the model's own limit)")
176
+ hf.add_argument(
177
+ "--long-text",
178
+ choices=("truncate", "window"),
179
+ default="truncate",
180
+ help="inputs longer than the window: cut them, or score overlapping windows and keep the maximum",
181
+ )
182
+ hf.add_argument(
183
+ "--hf-allow-pickle",
184
+ action="store_true",
185
+ help="also load pickle-based weights (can execute code; only for models you trust)",
186
+ )
187
+ http = evaluate_parser.add_argument_group("HTTP endpoints (https://...)")
188
+ http.add_argument("--http-text-field", default="text", help="request field for the input text")
189
+ http.add_argument("--http-label-field", default="label", help="dotted path of the label in the response")
190
+ http.add_argument("--http-score-field", default="score", help="dotted path of the score in the response")
191
+ http.add_argument("--http-timeout", type=float, default=15.0, help="seconds per request")
192
+ http.add_argument("--http-retries", type=int, default=1)
193
+ http.add_argument("--http-token-env", metavar="VAR", help="environment variable holding a bearer token")
194
+ http.add_argument(
195
+ "--http-header",
196
+ action="append",
197
+ default=[],
198
+ metavar="KEY=VALUE",
199
+ help="extra request header; repeatable",
200
+ )
201
+ http.add_argument(
202
+ "--http-body",
203
+ action="append",
204
+ default=[],
205
+ metavar="KEY=VALUE",
206
+ help="extra request body field; repeatable",
207
+ )
208
+ http.add_argument("--http-send-id", action="store_true", help="include the example id in the request")
209
+ output = evaluate_parser.add_argument_group("report")
210
+ output.add_argument("--format", default="markdown", choices=(*FORMATS, "md"))
211
+ output.add_argument("-o", "--output", help="write the report to this file instead of standard output")
212
+ output.add_argument("--save-predictions", metavar="FILE", help="also write row-level decisions as JSONL")
213
+ output.add_argument(
214
+ "--slice-by",
215
+ default=",".join(DEFAULT_SLICES),
216
+ help="comma-separated columns to break results down by; join two with + for a "
217
+ "cross-tabulation (default: %(default)s)",
218
+ )
219
+ output.add_argument("--confidence", type=float, default=0.95, help="confidence level (default: 0.95)")
220
+ gates = evaluate_parser.add_argument_group("gates (non-zero exit status when one fails)")
221
+ gates.add_argument(
222
+ "--max-fpr", type=_fraction, metavar="RATE", help="highest acceptable false-positive rate"
223
+ )
224
+ gates.add_argument("--min-recall", type=_fraction, metavar="RATE", help="lowest acceptable attack recall")
225
+ gates.add_argument(
226
+ "--min-pair-accuracy", type=_fraction, metavar="RATE", help="lowest acceptable pair accuracy"
227
+ )
228
+ gates.add_argument(
229
+ "--conservative",
230
+ action="store_true",
231
+ help="gate on the unfavourable end of the confidence interval instead of the point estimate",
232
+ )
233
+
234
+ validate_parser = commands.add_parser("validate", help="check the structure of a dataset")
235
+ _add_dataset_arguments(validate_parser)
236
+ validate_parser.add_argument("--json", action="store_true", help="print the result as JSON")
237
+
238
+ compare_parser = commands.add_parser("compare", help="put several JSON reports side by side")
239
+ compare_parser.add_argument("reports", nargs="+", metavar="REPORT.json")
240
+ compare_parser.add_argument("--format", default="markdown", choices=("markdown", "md", "json"))
241
+ compare_parser.add_argument(
242
+ "--sort-by",
243
+ default="pair_accuracy",
244
+ choices=("pair_accuracy", "false_positive_rate", "recall", "detector"),
245
+ )
246
+ compare_parser.add_argument("-o", "--output", help="write the comparison to this file")
247
+
248
+ commands.add_parser("detectors", help="list the registered detectors and the other forms")
249
+
250
+ export_parser = commands.add_parser(
251
+ "export-promptfoo", help="export a dataset as a Promptfoo test suite for a detector endpoint"
252
+ )
253
+ _add_dataset_arguments(export_parser)
254
+ export_parser.add_argument("--output-dir", required=True)
255
+ return parser
256
+
257
+
258
+ def _detector_options(args: argparse.Namespace) -> DetectorOptions:
259
+ return DetectorOptions(
260
+ http_text_field=args.http_text_field,
261
+ http_label_field=args.http_label_field,
262
+ http_score_field=args.http_score_field,
263
+ http_timeout=args.http_timeout,
264
+ http_retries=args.http_retries,
265
+ http_token_env=args.http_token_env,
266
+ http_headers=_key_value(args.http_header, "HTTP header"),
267
+ http_body=_key_value(args.http_body, "HTTP body field"),
268
+ http_send_id=args.http_send_id,
269
+ hf_positive_labels=tuple(args.positive_label),
270
+ hf_device=args.device,
271
+ hf_batch_size=args.batch_size,
272
+ hf_max_length=args.max_length,
273
+ hf_long_text=args.long_text,
274
+ hf_allow_pickle=args.hf_allow_pickle,
275
+ )
276
+
277
+
278
+ def _emit(text: str, output: str | None, what: str) -> None:
279
+ if output:
280
+ path = write_text(output, text)
281
+ print(f"wrote {what} to {path}", file=sys.stderr)
282
+ else:
283
+ sys.stdout.write(text if text.endswith("\n") else text + "\n")
284
+
285
+
286
+ def _run_evaluate(args: argparse.Namespace) -> int:
287
+ dataset = _load(args)
288
+ validation = validate_examples(dataset.examples, generated_ids=dataset.info.generated_ids)
289
+ for warning in validation.warnings:
290
+ print(f"note: {warning}", file=sys.stderr)
291
+ detector = build_detector(args.detector, _detector_options(args))
292
+ slices = [part.strip() for part in args.slice_by.split(",") if part.strip()]
293
+ result = evaluate(
294
+ dataset,
295
+ detector,
296
+ threshold=args.threshold,
297
+ decision=args.decision,
298
+ confidence=args.confidence,
299
+ slice_by=slices,
300
+ workers=args.workers,
301
+ allow_errors=args.allow_errors,
302
+ validation=validation,
303
+ )
304
+ gate_results = check_gates(
305
+ result.report,
306
+ max_fpr=args.max_fpr,
307
+ min_recall=args.min_recall,
308
+ min_pair_accuracy=args.min_pair_accuracy,
309
+ conservative=args.conservative,
310
+ )
311
+ result.report["gates"] = [gate.as_dict() for gate in gate_results]
312
+ _emit(render(result.report, args.format), args.output, f"{args.format} report")
313
+ if args.save_predictions:
314
+ path = write_text(args.save_predictions, predictions_jsonl(result.records))
315
+ print(f"wrote predictions to {path}", file=sys.stderr)
316
+ for gate in gate_results:
317
+ print(f"gate {gate.message}", file=sys.stderr)
318
+ return exit_code(gate_results)
319
+
320
+
321
+ def _run_validate(args: argparse.Namespace) -> int:
322
+ dataset = _load(args)
323
+ result = validate_examples(dataset.examples, generated_ids=dataset.info.generated_ids)
324
+ if args.json:
325
+ payload: dict[str, Any] = {
326
+ "valid": result.valid,
327
+ "errors": list(result.errors),
328
+ "warnings": list(result.warnings),
329
+ "stats": dict(result.stats),
330
+ "source": dataset.info.as_dict(),
331
+ }
332
+ sys.stdout.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n")
333
+ return EXIT_OK if result.valid else EXIT_ERROR
334
+ stats = result.stats
335
+ print(f"Source: {dataset.info.source} (split: {dataset.info.split})")
336
+ if dataset.info.resolved_revision:
337
+ print(f"Revision: {dataset.info.resolved_revision}")
338
+ print(f"Examples: {stats['examples']} ({stats['attacks']} attacks, {stats['benign']} benign)")
339
+ languages = ", ".join(
340
+ f"{name}: {counts['attacks']} attacks / {counts['benign']} benign"
341
+ for name, counts in stats["languages"].items()
342
+ )
343
+ print(f"Languages: {languages}")
344
+ pairs = stats["pairs"]
345
+ print(
346
+ f"Pairs: {pairs['complete']} complete, {pairs['incomplete']} incomplete, "
347
+ f"{pairs['in_several_languages']} present in several languages"
348
+ )
349
+ if stats["text_chars"]:
350
+ chars = stats["text_chars"]
351
+ print(f"Text: {chars['min']}-{chars['max']} characters, median {chars['median']}")
352
+ for warning in result.warnings:
353
+ print(f"Warning: {warning}")
354
+ for error in result.errors:
355
+ print(f"Error: {error}", file=sys.stderr)
356
+ print("Result: valid" if result.valid else "Result: INVALID")
357
+ return EXIT_OK if result.valid else EXIT_ERROR
358
+
359
+
360
+ def _run_compare(args: argparse.Namespace) -> int:
361
+ comparison = compare_reports([load_report(path) for path in args.reports], sort_by=args.sort_by)
362
+ if not comparison["same_dataset"]:
363
+ print("warning: the reports were not computed on the same data", file=sys.stderr)
364
+ _emit(render_comparison(comparison, args.format), args.output, "comparison")
365
+ return EXIT_OK
366
+
367
+
368
+ def _run_detectors(_args: argparse.Namespace) -> int:
369
+ print("Registered detectors:")
370
+ for name in available_detectors():
371
+ print(f" {name}")
372
+ print()
373
+ print("Other forms:")
374
+ print(" hf:owner/model[@revision] a Hugging Face text-classification model")
375
+ print(" python:file.py:function your own Python function")
376
+ print(" https://host/path a guardrail behind an HTTP endpoint")
377
+ print(" predictions:file.jsonl predictions computed elsewhere")
378
+ return EXIT_OK
379
+
380
+
381
+ def _run_export(args: argparse.Namespace) -> int:
382
+ dataset = _load(args)
383
+ validation = validate_examples(dataset.examples, generated_ids=dataset.info.generated_ids)
384
+ if not validation.valid:
385
+ raise BenignTwinError("dataset failed validation: " + "; ".join(validation.errors))
386
+ paths = export_promptfoo(
387
+ dataset.examples, args.output_dir, dataset_source=dataset.info.source, split=dataset.info.split
388
+ )
389
+ print(f"exported {len(dataset)} tests; see {paths['instructions']}")
390
+ return EXIT_OK
391
+
392
+
393
+ _COMMANDS = {
394
+ "evaluate": _run_evaluate,
395
+ "validate": _run_validate,
396
+ "compare": _run_compare,
397
+ "detectors": _run_detectors,
398
+ "export-promptfoo": _run_export,
399
+ }
400
+
401
+
402
+ def main(argv: Sequence[str] | None = None) -> int:
403
+ """Entry point. Returns the process exit status."""
404
+
405
+ parser = build_parser()
406
+ args = parser.parse_args(argv)
407
+ try:
408
+ return _COMMANDS[args.command](args)
409
+ except BenignTwinError as exc:
410
+ if os.environ.get("BENIGNTWIN_DEBUG"):
411
+ raise
412
+ print(f"error: {exc}", file=sys.stderr)
413
+ return EXIT_ERROR
414
+ except KeyboardInterrupt:
415
+ print("interrupted", file=sys.stderr)
416
+ return 130
417
+
418
+
419
+ if __name__ == "__main__":
420
+ raise SystemExit(main())
benigntwin/compare.py ADDED
@@ -0,0 +1,150 @@
1
+ """Put several JSON reports side by side.
2
+
3
+ ``benigntwin compare a.json b.json`` prints one row per detector. Reports are
4
+ only comparable when they were computed on the same data, so the comparison
5
+ says so when the dataset fingerprints differ.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from collections.abc import Mapping, Sequence
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from .models import BenignTwinError
16
+ from .report import format_rate
17
+
18
+ __all__ = ["CompareError", "compare_reports", "load_report", "render_comparison"]
19
+
20
+ _SORT_KEYS = ("pair_accuracy", "false_positive_rate", "recall", "detector")
21
+
22
+
23
+ class CompareError(BenignTwinError):
24
+ """Reports cannot be loaded or compared."""
25
+
26
+
27
+ def load_report(path: str | Path) -> dict[str, Any]:
28
+ """Read a JSON report written by ``benigntwin evaluate --format json``."""
29
+
30
+ source = Path(path).expanduser()
31
+ try:
32
+ payload = json.loads(source.read_text(encoding="utf-8"))
33
+ except OSError as exc:
34
+ raise CompareError(f"could not read {source}: {exc}") from exc
35
+ except json.JSONDecodeError as exc:
36
+ raise CompareError(f"{source}: invalid JSON: {exc.msg}") from exc
37
+ if not isinstance(payload, dict) or "metrics" not in payload or "detector" not in payload:
38
+ raise CompareError(f"{source}: not an BenignTwin report")
39
+ major = str(payload.get("schema_version", "")).split(".")[0]
40
+ if major != "2":
41
+ raise CompareError(
42
+ f"{source}: report schema {payload.get('schema_version')!r} is not supported; "
43
+ "re-run the evaluation with this version"
44
+ )
45
+ return payload
46
+
47
+
48
+ def _fingerprint(report: Mapping[str, Any]) -> tuple[Any, ...]:
49
+ dataset = report["dataset"]
50
+ hashes = tuple(sorted(entry["sha256"] for entry in dataset.get("files", [])))
51
+ return (dataset["source"], dataset["split"], hashes or dataset.get("examples"))
52
+
53
+
54
+ def _value(entry: Mapping[str, Any] | None) -> float | None:
55
+ return None if entry is None else float(entry["value"])
56
+
57
+
58
+ def compare_reports(
59
+ reports: Sequence[Mapping[str, Any]],
60
+ *,
61
+ sort_by: str = "pair_accuracy",
62
+ ) -> dict[str, Any]:
63
+ """Build a comparison: one row per report plus a same-data check."""
64
+
65
+ if not reports:
66
+ raise CompareError("nothing to compare")
67
+ if sort_by not in _SORT_KEYS:
68
+ raise CompareError(f"sort key must be one of: {', '.join(_SORT_KEYS)}")
69
+ rows = []
70
+ for report in reports:
71
+ metrics = report["metrics"]
72
+ pairs = report["pairs"]
73
+ rows.append(
74
+ {
75
+ "detector": report["detector"]["name"],
76
+ "revision": report["detector"].get("commit"),
77
+ "threshold": report["run"]["threshold"],
78
+ "examples": metrics["examples"],
79
+ "recall": metrics["recall"],
80
+ "false_positive_rate": metrics["false_positive_rate"],
81
+ "pair_accuracy": pairs["pair_accuracy"] if pairs["pairs"] else None,
82
+ "balanced_accuracy": metrics["balanced_accuracy"],
83
+ "roc_auc": (report.get("sweep") or {}).get("roc_auc"),
84
+ }
85
+ )
86
+
87
+ def key(row: Mapping[str, Any]) -> tuple[Any, ...]:
88
+ if sort_by == "detector":
89
+ return (0, str(row["detector"]))
90
+ value = _value(row[sort_by])
91
+ if value is None:
92
+ return (1, 0.0, str(row["detector"]))
93
+ # Lower is better for false positives, higher for everything else.
94
+ return (0, value if sort_by == "false_positive_rate" else -value, str(row["detector"]))
95
+
96
+ rows.sort(key=key)
97
+ fingerprints = {_fingerprint(report) for report in reports}
98
+ first = reports[0]["dataset"]
99
+ return {
100
+ "dataset": {"source": first["source"], "split": first["split"]},
101
+ "same_dataset": len(fingerprints) == 1,
102
+ "sorted_by": sort_by,
103
+ "rows": rows,
104
+ }
105
+
106
+
107
+ def render_comparison(comparison: Mapping[str, Any], fmt: str = "markdown") -> str:
108
+ """Render a comparison as Markdown or JSON."""
109
+
110
+ if fmt.lower() == "json":
111
+ return json.dumps(comparison, ensure_ascii=False, indent=2) + "\n"
112
+ if fmt.lower() not in ("markdown", "md"):
113
+ raise CompareError(f"unknown comparison format {fmt!r}; choose markdown or json")
114
+ dataset = comparison["dataset"]
115
+ lines = [
116
+ f"# BenignTwin comparison on `{dataset['source']}` ({dataset['split']})",
117
+ "",
118
+ ]
119
+ if not comparison["same_dataset"]:
120
+ lines.extend(
121
+ [
122
+ "> **These reports were not computed on the same data.** "
123
+ "Differences between rows may come from the data, not from the detectors.",
124
+ "",
125
+ ]
126
+ )
127
+ lines.extend(
128
+ [
129
+ "| Detector | Recall | False-positive rate | Pair accuracy | Balanced accuracy | ROC AUC |",
130
+ "| --- | ---: | ---: | ---: | ---: | ---: |",
131
+ ]
132
+ )
133
+ for row in comparison["rows"]:
134
+ balanced = row["balanced_accuracy"]
135
+ auc = row["roc_auc"]
136
+ lines.append(
137
+ f"| `{row['detector']}` | {format_rate(row['recall'])} | "
138
+ f"{format_rate(row['false_positive_rate'])} | {format_rate(row['pair_accuracy'])} | "
139
+ f"{'n/a' if balanced is None else f'{balanced:.3f}'} | "
140
+ f"{'n/a' if auc is None else f'{auc:.3f}'} |"
141
+ )
142
+ lines.extend(
143
+ [
144
+ "",
145
+ "Brackets are Wilson confidence intervals; the fraction is the count behind each rate. "
146
+ "Rows whose intervals overlap are not clearly different.",
147
+ "",
148
+ ]
149
+ )
150
+ return "\n".join(lines)