openadapt-verify 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,742 @@
1
+ """Audit your automation's success check: how many wrong saves does it pass?
2
+
3
+ ``openadapt-verify audit`` starts an HTTP proxy in front of your app, runs your
4
+ command (a Playwright test, a browser-agent script, any program) against the
5
+ proxy several times, and on each run changes one write in a named way: the
6
+ save never reaches the app, half of it does, it lands twice, the reply is
7
+ lost. It records whether your command said it succeeded, then reads the store
8
+ through the contract's independent read path (never through the proxy) to
9
+ find out what actually happened.
10
+
11
+ Every fault is applied at the HTTP layer alone, so the audit needs no access
12
+ to the app's code or database. Kinds that need more than that are listed in
13
+ :data:`NOT_RUN` and reported as not run.
14
+
15
+ The command learns the proxy address and the values it should save from
16
+ ``OPENADAPT_AUDIT_URL``, ``OPENADAPT_AUDIT_VALUES`` (JSON) and
17
+ ``OPENADAPT_AUDIT_RUN``. It is never told which fault a run carries.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import http.client
23
+ import json
24
+ import os
25
+ import re
26
+ import socket
27
+ import subprocess
28
+ import sys
29
+ import tempfile
30
+ import threading
31
+ import time
32
+ from dataclasses import asdict, dataclass, field
33
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
34
+ from pathlib import Path
35
+ from typing import Any, Mapping, Optional, Sequence
36
+ from urllib.parse import parse_qsl, urlencode, urlsplit
37
+
38
+ FORMAT = "openadapt.verify.audit/v1"
39
+
40
+ #: Fault kinds, applied to the first write a run sends through the proxy.
41
+ FAULTS: dict[str, str] = {
42
+ "no_persist": "the write never reaches the app; the proxy answers with the "
43
+ "app's last real success reply",
44
+ "rejected": "the write never reaches the app; the proxy answers 409",
45
+ "partial": "one declared value is blanked before the write is forwarded",
46
+ "wrong_value": "one declared value is changed before the write is forwarded",
47
+ "duplicate": "the write is forwarded twice; the caller sees the first reply",
48
+ "extra_write": "the write is forwarded, then a copy with one value changed",
49
+ "lost_response": "the write is forwarded and lands, then the connection is "
50
+ "closed without a reply",
51
+ "wrong_record": "one field naming the target record is rewritten "
52
+ "(needs --wrong-record FIELD=VALUE)",
53
+ }
54
+
55
+ #: Runs by default: every kind that needs nothing beyond HTTP.
56
+ DEFAULT_FAULTS: tuple[str, ...] = tuple(k for k in FAULTS if k != "wrong_record")
57
+
58
+ #: Kinds from the fault set that HTTP alone can't produce.
59
+ NOT_RUN: dict[str, str] = {
60
+ "stale": "overwrites another user's concurrent change; needs the app's "
61
+ "concurrency, not one request",
62
+ "collateral_unaudited": "a change to a table the read path doesn't cover; "
63
+ "needs database access",
64
+ }
65
+
66
+ WRITE_METHODS = ("POST", "PUT", "PATCH", "DELETE")
67
+
68
+ _HOP_BY_HOP = {
69
+ "connection",
70
+ "keep-alive",
71
+ "proxy-authenticate",
72
+ "proxy-authorization",
73
+ "te",
74
+ "trailer",
75
+ "transfer-encoding",
76
+ "upgrade",
77
+ "content-length",
78
+ "host",
79
+ }
80
+
81
+
82
+ # -- body edits ----------------------------------------------------------------
83
+
84
+
85
+ def _parse(body: bytes, content_type: str) -> tuple[str, Any]:
86
+ if not body:
87
+ return "", None
88
+ kind = content_type.split(";")[0].strip().lower()
89
+ if kind == "application/x-www-form-urlencoded":
90
+ return "form", parse_qsl(body.decode("utf-8"), keep_blank_values=True)
91
+ try:
92
+ return "json", json.loads(body)
93
+ except ValueError:
94
+ return "", None
95
+
96
+
97
+ def _dump(kind: str, value: Any) -> bytes:
98
+ if kind == "form":
99
+ return urlencode(value).encode()
100
+ return json.dumps(value).encode()
101
+
102
+
103
+ def _changed(value: Any) -> Any:
104
+ if isinstance(value, bool):
105
+ return not value
106
+ if isinstance(value, (int, float)):
107
+ return value + 1
108
+ text = str(value)
109
+ return str(int(text) + 1) if text.isdigit() else text + "-changed"
110
+
111
+
112
+ def _edit(
113
+ kind: str, value: Any, declared: Sequence[str], edit: Any
114
+ ) -> tuple[Any, bool]:
115
+ """Apply ``edit`` to the first declared value found in the body."""
116
+
117
+ if kind == "form":
118
+ pairs = list(value)
119
+ for want in declared:
120
+ for i, (name, got) in enumerate(pairs):
121
+ if got == want:
122
+ pairs[i] = (name, edit(got))
123
+ return pairs, True
124
+ return pairs, False
125
+
126
+ def walk(node: Any, want: str) -> tuple[Any, bool]:
127
+ if isinstance(node, dict):
128
+ for key, child in node.items():
129
+ if not isinstance(child, (dict, list)) and str(child) == want:
130
+ return {**node, key: edit(child)}, True
131
+ new, done = walk(child, want)
132
+ if done:
133
+ return {**node, key: new}, True
134
+ elif isinstance(node, list):
135
+ for i, child in enumerate(node):
136
+ new, done = walk(child, want)
137
+ if done:
138
+ return node[:i] + [new] + node[i + 1 :], True
139
+ return node, False
140
+
141
+ for want in declared:
142
+ new, done = walk(value, want)
143
+ if done:
144
+ return new, True
145
+ return value, False
146
+
147
+
148
+ def _set_field(kind: str, value: Any, name: str, new: str) -> tuple[Any, bool]:
149
+ if kind == "form":
150
+ pairs = list(value)
151
+ hit = [i for i, (k, _) in enumerate(pairs) if k == name]
152
+ for i in hit:
153
+ pairs[i] = (name, new)
154
+ return pairs, bool(hit)
155
+ if isinstance(value, dict) and name in value:
156
+ old = value[name]
157
+ return {**value, name: type(old)(new) if isinstance(old, int) else new}, True
158
+ return value, False
159
+
160
+
161
+ # -- the proxy -----------------------------------------------------------------
162
+
163
+
164
+ @dataclass
165
+ class Injection:
166
+ """What the proxy did to one run's write."""
167
+
168
+ kind: str
169
+ applied: bool = False
170
+ request: str = ""
171
+ detail: str = ""
172
+ writes_seen: int = 0
173
+
174
+
175
+ @dataclass
176
+ class _Reply:
177
+ status: int
178
+ reason: str
179
+ headers: list[tuple[str, str]]
180
+ body: bytes
181
+
182
+
183
+ class _Server6(ThreadingHTTPServer):
184
+ address_family = socket.AF_INET6
185
+
186
+
187
+ class FaultProxy:
188
+ """A loopback HTTP proxy that changes one write per armed run."""
189
+
190
+ def __init__(
191
+ self,
192
+ upstream: str,
193
+ listen: tuple[str, int] = ("127.0.0.1", 0),
194
+ *,
195
+ write_methods: Sequence[str] = WRITE_METHODS,
196
+ write_path: Optional[str] = None,
197
+ wrong_record: Optional[tuple[str, str]] = None,
198
+ timeout_s: float = 60.0,
199
+ ) -> None:
200
+ split = urlsplit(upstream)
201
+ if split.scheme not in ("http", "https") or not split.hostname:
202
+ raise ValueError(f"--upstream must be an http(s) URL, got {upstream!r}")
203
+ self.upstream = split
204
+ self.host: str = split.hostname
205
+ self.prefix = split.path.rstrip("/")
206
+ self.write_methods = tuple(m.upper() for m in write_methods)
207
+ self.write_path = re.compile(write_path) if write_path else None
208
+ self.wrong_record = wrong_record
209
+ self.timeout_s = timeout_s
210
+ self.lock = threading.Lock()
211
+ self.injection: Optional[Injection] = None
212
+ self.declared: list[str] = []
213
+ self.last_success: dict[tuple[str, str], _Reply] = {}
214
+ server = _Server6 if ":" in listen[0] else ThreadingHTTPServer
215
+ self.httpd = server(listen, self._handler())
216
+ self.httpd.daemon_threads = True
217
+ host, port = self.httpd.server_address[:2]
218
+ host = f"[{host!s}]" if ":" in str(host) else str(host)
219
+ self.base = f"http://{host}:{port}"
220
+ self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
221
+ self.thread.start()
222
+
223
+ def close(self) -> None:
224
+ self.httpd.shutdown()
225
+ self.httpd.server_close()
226
+ self.thread.join()
227
+
228
+ def arm(self, kind: Optional[str], values: Mapping[str, Any]) -> None:
229
+ with self.lock:
230
+ self.injection = Injection(kind or "clean")
231
+ self.declared = [str(v) for v in values.values() if str(v)]
232
+
233
+ def disarm(self) -> Injection:
234
+ with self.lock:
235
+ done, self.injection = self.injection, None
236
+ return done or Injection("clean")
237
+
238
+ # -- forwarding
239
+
240
+ def _is_write(
241
+ self,
242
+ method: str,
243
+ path: str,
244
+ headers: list[tuple[str, str]],
245
+ body: bytes,
246
+ declared: Sequence[str],
247
+ ) -> bool:
248
+ """Whether this request is the save under audit.
249
+
250
+ With ``--write-path``, a matching request of a write method is. Without
251
+ it, the body must also carry a declared value, so a sign-in or other
252
+ write the command makes first is never mistaken for the save.
253
+ """
254
+
255
+ if method not in self.write_methods:
256
+ return False
257
+ if self.write_path is not None:
258
+ return bool(self.write_path.search(urlsplit(path).path))
259
+ content_type = dict((k.lower(), v) for k, v in headers).get("content-type", "")
260
+ kind, parsed = _parse(body, content_type)
261
+ return bool(kind) and _edit(kind, parsed, declared, lambda v: v)[1]
262
+
263
+ def _forward(
264
+ self, method: str, path: str, headers: list[tuple[str, str]], body: bytes
265
+ ) -> _Reply:
266
+ cls = (
267
+ http.client.HTTPSConnection
268
+ if self.upstream.scheme == "https"
269
+ else http.client.HTTPConnection
270
+ )
271
+ conn = cls(self.host, self.upstream.port, timeout=self.timeout_s)
272
+ try:
273
+ sent = [(k, v) for k, v in headers if k.lower() not in _HOP_BY_HOP]
274
+ sent.append(("Host", self.upstream.netloc))
275
+ if body or method in WRITE_METHODS:
276
+ sent.append(("Content-Length", str(len(body))))
277
+ conn.putrequest(
278
+ method, self.prefix + path, skip_host=True, skip_accept_encoding=True
279
+ )
280
+ for key, value in sent:
281
+ conn.putheader(key, value)
282
+ conn.endheaders(body or None)
283
+ resp = conn.getresponse()
284
+ return _Reply(resp.status, resp.reason, resp.getheaders(), resp.read())
285
+ finally:
286
+ conn.close()
287
+
288
+ def _handle(
289
+ self, method: str, path: str, headers: list[tuple[str, str]], body: bytes
290
+ ) -> Optional[_Reply]:
291
+ """The reply to send, or None to drop the connection."""
292
+
293
+ with self.lock:
294
+ injection = self.injection
295
+ declared = list(self.declared)
296
+ target = self._is_write(method, path, headers, body, declared)
297
+ with self.lock:
298
+ fire = False
299
+ if injection is not None and target:
300
+ injection.writes_seen += 1
301
+ # Only the run's first save: a retry always passes through.
302
+ fire = injection.kind != "clean" and injection.writes_seen == 1
303
+ if fire:
304
+ injection.applied = True
305
+ injection.request = f"{method} {urlsplit(path).path}"
306
+ key = (method, urlsplit(path).path)
307
+ if not fire:
308
+ reply = self._forward(method, path, headers, body)
309
+ if target and 200 <= reply.status < 300:
310
+ with self.lock:
311
+ self.last_success[key] = reply
312
+ return reply
313
+ assert injection is not None
314
+ kind = injection.kind
315
+ if kind == "no_persist":
316
+ with self.lock:
317
+ cached = self.last_success.get(key)
318
+ if cached is not None:
319
+ injection.detail = "answered with the last real success reply"
320
+ return cached
321
+ injection.detail = "no earlier success reply; answered 201 with the body"
322
+ return _Reply(
323
+ 201,
324
+ "Created",
325
+ [("Content-Type", "application/json")],
326
+ body if body.strip()[:1] in (b"{", b"[") else b"{}",
327
+ )
328
+ if kind == "rejected":
329
+ return _Reply(
330
+ 409,
331
+ "Conflict",
332
+ [("Content-Type", "application/json")],
333
+ b'{"error": "conflict"}',
334
+ )
335
+ if kind == "duplicate":
336
+ first = self._forward(method, path, headers, body)
337
+ self._forward(method, path, headers, body)
338
+ return first
339
+ if kind == "lost_response":
340
+ self._forward(method, path, headers, body)
341
+ return None
342
+ content_type = dict((k.lower(), v) for k, v in headers).get("content-type", "")
343
+ parsed_kind, parsed = _parse(body, content_type)
344
+ if not parsed_kind:
345
+ injection.applied = False
346
+ injection.detail = "the write's body is not JSON or a form; sent unchanged"
347
+ return self._forward(method, path, headers, body)
348
+ if kind == "wrong_record":
349
+ assert self.wrong_record is not None
350
+ edited, done = _set_field(parsed_kind, parsed, *self.wrong_record)
351
+ missing = f"the body has no field {self.wrong_record[0]!r}"
352
+ else:
353
+ edit = (lambda _v: "") if kind == "partial" else _changed
354
+ edited, done = _edit(parsed_kind, parsed, declared, edit)
355
+ missing = "no declared value appears in the write's body"
356
+ if not done:
357
+ injection.applied = False
358
+ injection.detail = missing + "; sent unchanged"
359
+ return self._forward(method, path, headers, body)
360
+ new_body = _dump(parsed_kind, edited)
361
+ if kind == "extra_write":
362
+ first = self._forward(method, path, headers, body)
363
+ self._forward(method, path, headers, new_body)
364
+ return first
365
+ return self._forward(method, path, headers, new_body)
366
+
367
+ def _handler(self) -> type[BaseHTTPRequestHandler]:
368
+ proxy = self
369
+
370
+ class Handler(BaseHTTPRequestHandler):
371
+ def log_message(self, *_args: Any) -> None:
372
+ pass
373
+
374
+ def _body(self) -> bytes:
375
+ if "chunked" in (self.headers.get("Transfer-Encoding") or "").lower():
376
+ chunks: list[bytes] = []
377
+ while True:
378
+ size = int(self.rfile.readline().split(b";")[0], 16)
379
+ if size == 0:
380
+ self.rfile.readline()
381
+ return b"".join(chunks)
382
+ chunks.append(self.rfile.read(size))
383
+ self.rfile.readline()
384
+ return self.rfile.read(int(self.headers.get("Content-Length") or 0))
385
+
386
+ def _serve(self) -> None:
387
+ body = self._body()
388
+ try:
389
+ reply = proxy._handle(
390
+ self.command, self.path, list(self.headers.items()), body
391
+ )
392
+ except OSError as exc:
393
+ reply = _Reply(502, "Bad Gateway", [], str(exc).encode())
394
+ if reply is None:
395
+ self.close_connection = True
396
+ return
397
+ self.send_response(reply.status, reply.reason)
398
+ origin = f"{proxy.upstream.scheme}://{proxy.upstream.netloc}"
399
+ for key, value in reply.headers:
400
+ if key.lower() in _HOP_BY_HOP:
401
+ continue
402
+ mount = origin + proxy.prefix
403
+ if key.lower() == "location" and (
404
+ value == mount or value.startswith(mount + "/")
405
+ ):
406
+ value = proxy.base + value[len(mount) :]
407
+ self.send_header(key, value)
408
+ self.send_header("Content-Length", str(len(reply.body)))
409
+ self.send_header("Connection", "close")
410
+ self.end_headers()
411
+ if self.command != "HEAD":
412
+ self.wfile.write(reply.body)
413
+ self.close_connection = True
414
+
415
+ do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = _serve # noqa: N815
416
+ do_HEAD = do_OPTIONS = _serve # noqa: N815
417
+
418
+ return Handler
419
+
420
+
421
+ # -- the audit -----------------------------------------------------------------
422
+
423
+
424
+ @dataclass
425
+ class Run:
426
+ index: int
427
+ fault: str
428
+ applied: bool
429
+ detail: str
430
+ request: str
431
+ command_returncode: Optional[int]
432
+ reported_success: bool
433
+ outcome: str
434
+ observed: str
435
+ reasons: list[str] = field(default_factory=list)
436
+ seconds: float = 0.0
437
+
438
+ @property
439
+ def measured(self) -> bool:
440
+ """The fault was applied (or the run is clean) and the store was read."""
441
+
442
+ if self.outcome == "NOT READ":
443
+ return False
444
+ if self.outcome == "UNCERTAIN" and self.observed == "unknown":
445
+ return False
446
+ return self.fault == "clean" or self.applied
447
+
448
+ @property
449
+ def bad(self) -> bool:
450
+ return self.outcome != "CONFIRMED"
451
+
452
+ @property
453
+ def missed(self) -> bool:
454
+ return self.measured and self.bad and self.reported_success
455
+
456
+ @property
457
+ def false_alarm(self) -> bool:
458
+ return self.measured and not self.bad and not self.reported_success
459
+
460
+
461
+ @dataclass
462
+ class Report:
463
+ command: list[str]
464
+ upstream: str
465
+ faults: list[str]
466
+ runs: list[Run]
467
+ not_run: dict[str, str]
468
+ success_rule: str
469
+
470
+ def summary(self) -> dict[str, dict[str, int]]:
471
+ rows: dict[str, dict[str, int]] = {}
472
+ for name in ["clean", *self.faults]:
473
+ runs = [r for r in self.runs if r.fault == name]
474
+ measured = [r for r in runs if r.measured]
475
+ rows[name] = {
476
+ "runs": len(runs),
477
+ "measured": len(measured),
478
+ "bad_saves": sum(r.bad for r in measured),
479
+ "missed": sum(r.missed for r in measured),
480
+ "caught": sum(r.bad and not r.reported_success for r in measured),
481
+ "false_alarms": sum(r.false_alarm for r in measured),
482
+ }
483
+ return rows
484
+
485
+ def totals(self) -> dict[str, int]:
486
+ rows = self.summary().values()
487
+ return {
488
+ key: sum(row[key] for row in rows)
489
+ for key in ("measured", "bad_saves", "missed", "caught", "false_alarms")
490
+ }
491
+
492
+ def to_dict(self) -> dict[str, Any]:
493
+ return {
494
+ "format": FORMAT,
495
+ "command": self.command,
496
+ "upstream": self.upstream,
497
+ "success_rule": self.success_rule,
498
+ "faults": self.faults,
499
+ "not_run": self.not_run,
500
+ "summary": self.summary(),
501
+ "totals": self.totals(),
502
+ "runs": [
503
+ {
504
+ **asdict(r),
505
+ "measured": r.measured,
506
+ "bad": r.bad,
507
+ "missed": r.missed,
508
+ "false_alarm": r.false_alarm,
509
+ }
510
+ for r in self.runs
511
+ ],
512
+ }
513
+
514
+
515
+ def schedule(faults: Sequence[str], runs: int) -> list[str]:
516
+ """Clean first (it gives ``no_persist`` a real reply to replay), then the
517
+ kinds in turn, a clean run starting each round."""
518
+
519
+ order = ["clean", *faults]
520
+ return [order[i % len(order)] for i in range(runs)]
521
+
522
+
523
+ def _values(params: Mapping[str, Any], index: int) -> dict[str, Any]:
524
+ return {
525
+ k: v.replace("{run}", str(index)) if isinstance(v, str) else v
526
+ for k, v in params.items()
527
+ }
528
+
529
+
530
+ def _proxied(contract: Mapping[str, Any], proxy: FaultProxy) -> bool:
531
+ base = str((contract.get("verifier") or {}).get("base_url") or "")
532
+ if not base:
533
+ return False
534
+ split = urlsplit(base)
535
+ host, port = proxy.httpd.server_address[:2]
536
+ same_host = split.hostname in (host, "localhost", "127.0.0.1", "::1", "0.0.0.0")
537
+ return same_host and split.port == port
538
+
539
+
540
+ def audit(
541
+ contract: Mapping[str, Any],
542
+ params: Mapping[str, Any],
543
+ command: Sequence[str],
544
+ *,
545
+ upstream: str,
546
+ listen: tuple[str, int] = ("127.0.0.1", 0),
547
+ runs: Optional[int] = None,
548
+ faults: Sequence[str] = DEFAULT_FAULTS,
549
+ success_regex: Optional[str] = None,
550
+ write_methods: Sequence[str] = WRITE_METHODS,
551
+ write_path: Optional[str] = None,
552
+ wrong_record: Optional[tuple[str, str]] = None,
553
+ command_timeout_s: float = 300.0,
554
+ progress: Any = None,
555
+ ) -> Report:
556
+ """Run ``command`` ``runs`` times through a fault proxy and judge each run."""
557
+
558
+ from openadapt_verify import (
559
+ Actor,
560
+ Contract,
561
+ WriteBlocked,
562
+ begin_write,
563
+ verify_write,
564
+ )
565
+ from openadapt_verify.quickstart import _STATE_ENV, isolated_state
566
+
567
+ faults = list(dict.fromkeys(faults))
568
+ for name in faults:
569
+ if name not in FAULTS:
570
+ raise ValueError(f"unknown fault {name!r}; choose from {', '.join(FAULTS)}")
571
+ if "wrong_record" in faults and wrong_record is None:
572
+ raise ValueError("wrong_record needs --wrong-record FIELD=VALUE")
573
+ if not command:
574
+ raise ValueError("audit needs a command after --")
575
+ checked = Contract.from_dict(dict(contract))
576
+ pattern = re.compile(success_regex) if success_regex else None
577
+ total = runs if runs is not None else 2 * (len(faults) + 1)
578
+ if total < 1:
579
+ raise ValueError("--runs must be at least 1")
580
+ proxy = FaultProxy(
581
+ upstream,
582
+ listen,
583
+ write_methods=write_methods,
584
+ write_path=write_path,
585
+ wrong_record=wrong_record,
586
+ )
587
+ if _proxied(contract, proxy):
588
+ proxy.close()
589
+ raise ValueError(
590
+ "the contract reads through the audit proxy; point its read path at "
591
+ "the app or its database directly"
592
+ )
593
+ child_env = {k: v for k, v in os.environ.items() if k not in _STATE_ENV}
594
+ child_env["OPENADAPT_AUDIT_URL"] = proxy.base
595
+ results: list[Run] = []
596
+ try:
597
+ for index, fault in enumerate(schedule(faults, total)):
598
+ values = _values(params, index)
599
+ started = time.monotonic()
600
+ with tempfile.TemporaryDirectory(prefix="openadapt-verify-audit-") as tmp:
601
+ with isolated_state(Path(tmp) / "state"):
602
+ try:
603
+ pending = begin_write(
604
+ checked,
605
+ values,
606
+ actor=Actor("agent", "audited-command"),
607
+ workdir=Path(tmp) / "writes",
608
+ )
609
+ except WriteBlocked as exc:
610
+ results.append(
611
+ Run(
612
+ index,
613
+ fault,
614
+ False,
615
+ "",
616
+ "",
617
+ None,
618
+ False,
619
+ "NOT READ",
620
+ "unknown",
621
+ [str(exc)],
622
+ )
623
+ )
624
+ continue
625
+ proxy.arm(None if fault == "clean" else fault, values)
626
+ env = {
627
+ **child_env,
628
+ "OPENADAPT_AUDIT_RUN": str(index),
629
+ "OPENADAPT_AUDIT_VALUES": json.dumps(values),
630
+ }
631
+ returncode: Optional[int]
632
+ try:
633
+ done = subprocess.run(
634
+ list(command),
635
+ env=env,
636
+ capture_output=True,
637
+ text=True,
638
+ timeout=command_timeout_s,
639
+ check=False,
640
+ )
641
+ returncode, output = done.returncode, done.stdout + done.stderr
642
+ except subprocess.TimeoutExpired as exc:
643
+ returncode, output = None, str(exc.output or "")
644
+ injection = proxy.disarm()
645
+ result = verify_write(pending, write_attempted=True, seal=False)
646
+ if pattern is not None:
647
+ success = returncode is not None and bool(pattern.search(output))
648
+ else:
649
+ success = returncode == 0
650
+ detail = injection.detail
651
+ if fault != "clean" and not injection.applied and not detail:
652
+ detail = "no write carrying a declared value went through the proxy"
653
+ run = Run(
654
+ index=index,
655
+ fault=fault,
656
+ applied=injection.applied,
657
+ detail=detail,
658
+ request=injection.request,
659
+ command_returncode=returncode,
660
+ reported_success=success,
661
+ outcome=result.outcome.value,
662
+ observed=result.observed,
663
+ reasons=list(result.reasons),
664
+ seconds=round(time.monotonic() - started, 3),
665
+ )
666
+ results.append(run)
667
+ if progress is not None:
668
+ progress(run)
669
+ finally:
670
+ proxy.close()
671
+ return Report(
672
+ command=list(command),
673
+ upstream=upstream,
674
+ faults=faults,
675
+ runs=results,
676
+ not_run={
677
+ **NOT_RUN,
678
+ **(
679
+ {}
680
+ if "wrong_record" in faults
681
+ else {
682
+ "wrong_record": "needs --wrong-record FIELD=VALUE naming another "
683
+ "record"
684
+ }
685
+ ),
686
+ },
687
+ success_rule=f"output matches {success_regex!r}"
688
+ if success_regex
689
+ else "exit code 0",
690
+ )
691
+
692
+
693
+ def render(report: Report) -> str:
694
+ lines = [
695
+ f"{'fault':<15}{'runs':>5}{'bad saves':>11}{'check said OK':>15}"
696
+ f"{'caught':>8}{'false alarms':>14}"
697
+ ]
698
+ for name, row in report.summary().items():
699
+ runs = str(row["measured"])
700
+ if row["measured"] != row["runs"]:
701
+ runs += f"/{row['runs']}"
702
+ lines.append(
703
+ f"{name:<15}{runs:>5}{row['bad_saves']:>11}{row['missed']:>15}"
704
+ f"{row['caught']:>8}{row['false_alarms']:>14}"
705
+ )
706
+ total = report.totals()
707
+ clean = report.summary()["clean"]
708
+ lines += [
709
+ "",
710
+ f"Your check ({report.success_rule}) reported success on "
711
+ f"{total['missed']} of {total['bad_saves']} bad saves.",
712
+ f"False alarms: {clean['false_alarms']} of {clean['measured']} clean runs, "
713
+ f"{total['false_alarms'] - clean['false_alarms']} on faulted runs whose "
714
+ "write still landed correctly.",
715
+ ]
716
+ skipped = [r for r in report.runs if not r.measured]
717
+ if skipped:
718
+ lines.append(
719
+ f"{len(skipped)} run(s) not measured (fault not applied or store "
720
+ "unreadable); see the JSON report."
721
+ )
722
+ lines.append("Not run: " + ", ".join(sorted(report.not_run)) + ".")
723
+ return "\n".join(lines)
724
+
725
+
726
+ def exit_code(report: Report) -> int:
727
+ """0 when every bad save was caught with no false alarms, else 1."""
728
+
729
+ total = report.totals()
730
+ if total["measured"] == 0:
731
+ return 2
732
+ return 0 if total["missed"] == 0 and total["false_alarms"] == 0 else 1
733
+
734
+
735
+ def print_progress(run: Run) -> None:
736
+ said = "success" if run.reported_success else "failure"
737
+ note = "" if run.measured else " (not measured)"
738
+ print(
739
+ f"run {run.index + 1}: {run.fault}: command said {said}, store says "
740
+ f"{run.outcome}{note}",
741
+ file=sys.stderr,
742
+ )