openadapt-verify 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openadapt_verify/__init__.py +65 -0
- openadapt_verify/__main__.py +3 -0
- openadapt_verify/_ledger_config.py +21 -0
- openadapt_verify/adapters/__init__.py +1 -0
- openadapt_verify/adapters/browser_use.py +39 -0
- openadapt_verify/adapters/claude_sdk.py +5 -0
- openadapt_verify/adapters/openai_agents.py +5 -0
- openadapt_verify/adapters/stagehand.py +32 -0
- openadapt_verify/audit.py +742 -0
- openadapt_verify/audit_demo.py +106 -0
- openadapt_verify/cli.py +463 -0
- openadapt_verify/guard.py +130 -0
- openadapt_verify/mcp.py +24 -0
- openadapt_verify/quickstart.py +567 -0
- openadapt_verify/shadow.py +5 -0
- openadapt_verify-0.2.0.dist-info/METADATA +382 -0
- openadapt_verify-0.2.0.dist-info/RECORD +20 -0
- openadapt_verify-0.2.0.dist-info/WHEEL +4 -0
- openadapt_verify-0.2.0.dist-info/entry_points.txt +2 -0
- openadapt_verify-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,742 @@
|
|
|
1
|
+
"""Audit your automation's success check: how many wrong saves does it pass?
|
|
2
|
+
|
|
3
|
+
``openadapt-verify audit`` starts an HTTP proxy in front of your app, runs your
|
|
4
|
+
command (a Playwright test, a browser-agent script, any program) against the
|
|
5
|
+
proxy several times, and on each run changes one write in a named way: the
|
|
6
|
+
save never reaches the app, half of it does, it lands twice, the reply is
|
|
7
|
+
lost. It records whether your command said it succeeded, then reads the store
|
|
8
|
+
through the contract's independent read path (never through the proxy) to
|
|
9
|
+
find out what actually happened.
|
|
10
|
+
|
|
11
|
+
Every fault is applied at the HTTP layer alone, so the audit needs no access
|
|
12
|
+
to the app's code or database. Kinds that need more than that are listed in
|
|
13
|
+
:data:`NOT_RUN` and reported as not run.
|
|
14
|
+
|
|
15
|
+
The command learns the proxy address and the values it should save from
|
|
16
|
+
``OPENADAPT_AUDIT_URL``, ``OPENADAPT_AUDIT_VALUES`` (JSON) and
|
|
17
|
+
``OPENADAPT_AUDIT_RUN``. It is never told which fault a run carries.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import http.client
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import re
|
|
26
|
+
import socket
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
import tempfile
|
|
30
|
+
import threading
|
|
31
|
+
import time
|
|
32
|
+
from dataclasses import asdict, dataclass, field
|
|
33
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Any, Mapping, Optional, Sequence
|
|
36
|
+
from urllib.parse import parse_qsl, urlencode, urlsplit
|
|
37
|
+
|
|
38
|
+
FORMAT = "openadapt.verify.audit/v1"
|
|
39
|
+
|
|
40
|
+
#: Fault kinds, applied to the first write a run sends through the proxy.
|
|
41
|
+
FAULTS: dict[str, str] = {
|
|
42
|
+
"no_persist": "the write never reaches the app; the proxy answers with the "
|
|
43
|
+
"app's last real success reply",
|
|
44
|
+
"rejected": "the write never reaches the app; the proxy answers 409",
|
|
45
|
+
"partial": "one declared value is blanked before the write is forwarded",
|
|
46
|
+
"wrong_value": "one declared value is changed before the write is forwarded",
|
|
47
|
+
"duplicate": "the write is forwarded twice; the caller sees the first reply",
|
|
48
|
+
"extra_write": "the write is forwarded, then a copy with one value changed",
|
|
49
|
+
"lost_response": "the write is forwarded and lands, then the connection is "
|
|
50
|
+
"closed without a reply",
|
|
51
|
+
"wrong_record": "one field naming the target record is rewritten "
|
|
52
|
+
"(needs --wrong-record FIELD=VALUE)",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
#: Runs by default: every kind that needs nothing beyond HTTP.
|
|
56
|
+
DEFAULT_FAULTS: tuple[str, ...] = tuple(k for k in FAULTS if k != "wrong_record")
|
|
57
|
+
|
|
58
|
+
#: Kinds from the fault set that HTTP alone can't produce.
|
|
59
|
+
NOT_RUN: dict[str, str] = {
|
|
60
|
+
"stale": "overwrites another user's concurrent change; needs the app's "
|
|
61
|
+
"concurrency, not one request",
|
|
62
|
+
"collateral_unaudited": "a change to a table the read path doesn't cover; "
|
|
63
|
+
"needs database access",
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
WRITE_METHODS = ("POST", "PUT", "PATCH", "DELETE")
|
|
67
|
+
|
|
68
|
+
_HOP_BY_HOP = {
|
|
69
|
+
"connection",
|
|
70
|
+
"keep-alive",
|
|
71
|
+
"proxy-authenticate",
|
|
72
|
+
"proxy-authorization",
|
|
73
|
+
"te",
|
|
74
|
+
"trailer",
|
|
75
|
+
"transfer-encoding",
|
|
76
|
+
"upgrade",
|
|
77
|
+
"content-length",
|
|
78
|
+
"host",
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# -- body edits ----------------------------------------------------------------
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _parse(body: bytes, content_type: str) -> tuple[str, Any]:
|
|
86
|
+
if not body:
|
|
87
|
+
return "", None
|
|
88
|
+
kind = content_type.split(";")[0].strip().lower()
|
|
89
|
+
if kind == "application/x-www-form-urlencoded":
|
|
90
|
+
return "form", parse_qsl(body.decode("utf-8"), keep_blank_values=True)
|
|
91
|
+
try:
|
|
92
|
+
return "json", json.loads(body)
|
|
93
|
+
except ValueError:
|
|
94
|
+
return "", None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _dump(kind: str, value: Any) -> bytes:
|
|
98
|
+
if kind == "form":
|
|
99
|
+
return urlencode(value).encode()
|
|
100
|
+
return json.dumps(value).encode()
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _changed(value: Any) -> Any:
|
|
104
|
+
if isinstance(value, bool):
|
|
105
|
+
return not value
|
|
106
|
+
if isinstance(value, (int, float)):
|
|
107
|
+
return value + 1
|
|
108
|
+
text = str(value)
|
|
109
|
+
return str(int(text) + 1) if text.isdigit() else text + "-changed"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _edit(
|
|
113
|
+
kind: str, value: Any, declared: Sequence[str], edit: Any
|
|
114
|
+
) -> tuple[Any, bool]:
|
|
115
|
+
"""Apply ``edit`` to the first declared value found in the body."""
|
|
116
|
+
|
|
117
|
+
if kind == "form":
|
|
118
|
+
pairs = list(value)
|
|
119
|
+
for want in declared:
|
|
120
|
+
for i, (name, got) in enumerate(pairs):
|
|
121
|
+
if got == want:
|
|
122
|
+
pairs[i] = (name, edit(got))
|
|
123
|
+
return pairs, True
|
|
124
|
+
return pairs, False
|
|
125
|
+
|
|
126
|
+
def walk(node: Any, want: str) -> tuple[Any, bool]:
|
|
127
|
+
if isinstance(node, dict):
|
|
128
|
+
for key, child in node.items():
|
|
129
|
+
if not isinstance(child, (dict, list)) and str(child) == want:
|
|
130
|
+
return {**node, key: edit(child)}, True
|
|
131
|
+
new, done = walk(child, want)
|
|
132
|
+
if done:
|
|
133
|
+
return {**node, key: new}, True
|
|
134
|
+
elif isinstance(node, list):
|
|
135
|
+
for i, child in enumerate(node):
|
|
136
|
+
new, done = walk(child, want)
|
|
137
|
+
if done:
|
|
138
|
+
return node[:i] + [new] + node[i + 1 :], True
|
|
139
|
+
return node, False
|
|
140
|
+
|
|
141
|
+
for want in declared:
|
|
142
|
+
new, done = walk(value, want)
|
|
143
|
+
if done:
|
|
144
|
+
return new, True
|
|
145
|
+
return value, False
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _set_field(kind: str, value: Any, name: str, new: str) -> tuple[Any, bool]:
|
|
149
|
+
if kind == "form":
|
|
150
|
+
pairs = list(value)
|
|
151
|
+
hit = [i for i, (k, _) in enumerate(pairs) if k == name]
|
|
152
|
+
for i in hit:
|
|
153
|
+
pairs[i] = (name, new)
|
|
154
|
+
return pairs, bool(hit)
|
|
155
|
+
if isinstance(value, dict) and name in value:
|
|
156
|
+
old = value[name]
|
|
157
|
+
return {**value, name: type(old)(new) if isinstance(old, int) else new}, True
|
|
158
|
+
return value, False
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# -- the proxy -----------------------------------------------------------------
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass
|
|
165
|
+
class Injection:
|
|
166
|
+
"""What the proxy did to one run's write."""
|
|
167
|
+
|
|
168
|
+
kind: str
|
|
169
|
+
applied: bool = False
|
|
170
|
+
request: str = ""
|
|
171
|
+
detail: str = ""
|
|
172
|
+
writes_seen: int = 0
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@dataclass
|
|
176
|
+
class _Reply:
|
|
177
|
+
status: int
|
|
178
|
+
reason: str
|
|
179
|
+
headers: list[tuple[str, str]]
|
|
180
|
+
body: bytes
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class _Server6(ThreadingHTTPServer):
|
|
184
|
+
address_family = socket.AF_INET6
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class FaultProxy:
|
|
188
|
+
"""A loopback HTTP proxy that changes one write per armed run."""
|
|
189
|
+
|
|
190
|
+
def __init__(
|
|
191
|
+
self,
|
|
192
|
+
upstream: str,
|
|
193
|
+
listen: tuple[str, int] = ("127.0.0.1", 0),
|
|
194
|
+
*,
|
|
195
|
+
write_methods: Sequence[str] = WRITE_METHODS,
|
|
196
|
+
write_path: Optional[str] = None,
|
|
197
|
+
wrong_record: Optional[tuple[str, str]] = None,
|
|
198
|
+
timeout_s: float = 60.0,
|
|
199
|
+
) -> None:
|
|
200
|
+
split = urlsplit(upstream)
|
|
201
|
+
if split.scheme not in ("http", "https") or not split.hostname:
|
|
202
|
+
raise ValueError(f"--upstream must be an http(s) URL, got {upstream!r}")
|
|
203
|
+
self.upstream = split
|
|
204
|
+
self.host: str = split.hostname
|
|
205
|
+
self.prefix = split.path.rstrip("/")
|
|
206
|
+
self.write_methods = tuple(m.upper() for m in write_methods)
|
|
207
|
+
self.write_path = re.compile(write_path) if write_path else None
|
|
208
|
+
self.wrong_record = wrong_record
|
|
209
|
+
self.timeout_s = timeout_s
|
|
210
|
+
self.lock = threading.Lock()
|
|
211
|
+
self.injection: Optional[Injection] = None
|
|
212
|
+
self.declared: list[str] = []
|
|
213
|
+
self.last_success: dict[tuple[str, str], _Reply] = {}
|
|
214
|
+
server = _Server6 if ":" in listen[0] else ThreadingHTTPServer
|
|
215
|
+
self.httpd = server(listen, self._handler())
|
|
216
|
+
self.httpd.daemon_threads = True
|
|
217
|
+
host, port = self.httpd.server_address[:2]
|
|
218
|
+
host = f"[{host!s}]" if ":" in str(host) else str(host)
|
|
219
|
+
self.base = f"http://{host}:{port}"
|
|
220
|
+
self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
|
|
221
|
+
self.thread.start()
|
|
222
|
+
|
|
223
|
+
def close(self) -> None:
|
|
224
|
+
self.httpd.shutdown()
|
|
225
|
+
self.httpd.server_close()
|
|
226
|
+
self.thread.join()
|
|
227
|
+
|
|
228
|
+
def arm(self, kind: Optional[str], values: Mapping[str, Any]) -> None:
|
|
229
|
+
with self.lock:
|
|
230
|
+
self.injection = Injection(kind or "clean")
|
|
231
|
+
self.declared = [str(v) for v in values.values() if str(v)]
|
|
232
|
+
|
|
233
|
+
def disarm(self) -> Injection:
|
|
234
|
+
with self.lock:
|
|
235
|
+
done, self.injection = self.injection, None
|
|
236
|
+
return done or Injection("clean")
|
|
237
|
+
|
|
238
|
+
# -- forwarding
|
|
239
|
+
|
|
240
|
+
def _is_write(
|
|
241
|
+
self,
|
|
242
|
+
method: str,
|
|
243
|
+
path: str,
|
|
244
|
+
headers: list[tuple[str, str]],
|
|
245
|
+
body: bytes,
|
|
246
|
+
declared: Sequence[str],
|
|
247
|
+
) -> bool:
|
|
248
|
+
"""Whether this request is the save under audit.
|
|
249
|
+
|
|
250
|
+
With ``--write-path``, a matching request of a write method is. Without
|
|
251
|
+
it, the body must also carry a declared value, so a sign-in or other
|
|
252
|
+
write the command makes first is never mistaken for the save.
|
|
253
|
+
"""
|
|
254
|
+
|
|
255
|
+
if method not in self.write_methods:
|
|
256
|
+
return False
|
|
257
|
+
if self.write_path is not None:
|
|
258
|
+
return bool(self.write_path.search(urlsplit(path).path))
|
|
259
|
+
content_type = dict((k.lower(), v) for k, v in headers).get("content-type", "")
|
|
260
|
+
kind, parsed = _parse(body, content_type)
|
|
261
|
+
return bool(kind) and _edit(kind, parsed, declared, lambda v: v)[1]
|
|
262
|
+
|
|
263
|
+
def _forward(
|
|
264
|
+
self, method: str, path: str, headers: list[tuple[str, str]], body: bytes
|
|
265
|
+
) -> _Reply:
|
|
266
|
+
cls = (
|
|
267
|
+
http.client.HTTPSConnection
|
|
268
|
+
if self.upstream.scheme == "https"
|
|
269
|
+
else http.client.HTTPConnection
|
|
270
|
+
)
|
|
271
|
+
conn = cls(self.host, self.upstream.port, timeout=self.timeout_s)
|
|
272
|
+
try:
|
|
273
|
+
sent = [(k, v) for k, v in headers if k.lower() not in _HOP_BY_HOP]
|
|
274
|
+
sent.append(("Host", self.upstream.netloc))
|
|
275
|
+
if body or method in WRITE_METHODS:
|
|
276
|
+
sent.append(("Content-Length", str(len(body))))
|
|
277
|
+
conn.putrequest(
|
|
278
|
+
method, self.prefix + path, skip_host=True, skip_accept_encoding=True
|
|
279
|
+
)
|
|
280
|
+
for key, value in sent:
|
|
281
|
+
conn.putheader(key, value)
|
|
282
|
+
conn.endheaders(body or None)
|
|
283
|
+
resp = conn.getresponse()
|
|
284
|
+
return _Reply(resp.status, resp.reason, resp.getheaders(), resp.read())
|
|
285
|
+
finally:
|
|
286
|
+
conn.close()
|
|
287
|
+
|
|
288
|
+
def _handle(
|
|
289
|
+
self, method: str, path: str, headers: list[tuple[str, str]], body: bytes
|
|
290
|
+
) -> Optional[_Reply]:
|
|
291
|
+
"""The reply to send, or None to drop the connection."""
|
|
292
|
+
|
|
293
|
+
with self.lock:
|
|
294
|
+
injection = self.injection
|
|
295
|
+
declared = list(self.declared)
|
|
296
|
+
target = self._is_write(method, path, headers, body, declared)
|
|
297
|
+
with self.lock:
|
|
298
|
+
fire = False
|
|
299
|
+
if injection is not None and target:
|
|
300
|
+
injection.writes_seen += 1
|
|
301
|
+
# Only the run's first save: a retry always passes through.
|
|
302
|
+
fire = injection.kind != "clean" and injection.writes_seen == 1
|
|
303
|
+
if fire:
|
|
304
|
+
injection.applied = True
|
|
305
|
+
injection.request = f"{method} {urlsplit(path).path}"
|
|
306
|
+
key = (method, urlsplit(path).path)
|
|
307
|
+
if not fire:
|
|
308
|
+
reply = self._forward(method, path, headers, body)
|
|
309
|
+
if target and 200 <= reply.status < 300:
|
|
310
|
+
with self.lock:
|
|
311
|
+
self.last_success[key] = reply
|
|
312
|
+
return reply
|
|
313
|
+
assert injection is not None
|
|
314
|
+
kind = injection.kind
|
|
315
|
+
if kind == "no_persist":
|
|
316
|
+
with self.lock:
|
|
317
|
+
cached = self.last_success.get(key)
|
|
318
|
+
if cached is not None:
|
|
319
|
+
injection.detail = "answered with the last real success reply"
|
|
320
|
+
return cached
|
|
321
|
+
injection.detail = "no earlier success reply; answered 201 with the body"
|
|
322
|
+
return _Reply(
|
|
323
|
+
201,
|
|
324
|
+
"Created",
|
|
325
|
+
[("Content-Type", "application/json")],
|
|
326
|
+
body if body.strip()[:1] in (b"{", b"[") else b"{}",
|
|
327
|
+
)
|
|
328
|
+
if kind == "rejected":
|
|
329
|
+
return _Reply(
|
|
330
|
+
409,
|
|
331
|
+
"Conflict",
|
|
332
|
+
[("Content-Type", "application/json")],
|
|
333
|
+
b'{"error": "conflict"}',
|
|
334
|
+
)
|
|
335
|
+
if kind == "duplicate":
|
|
336
|
+
first = self._forward(method, path, headers, body)
|
|
337
|
+
self._forward(method, path, headers, body)
|
|
338
|
+
return first
|
|
339
|
+
if kind == "lost_response":
|
|
340
|
+
self._forward(method, path, headers, body)
|
|
341
|
+
return None
|
|
342
|
+
content_type = dict((k.lower(), v) for k, v in headers).get("content-type", "")
|
|
343
|
+
parsed_kind, parsed = _parse(body, content_type)
|
|
344
|
+
if not parsed_kind:
|
|
345
|
+
injection.applied = False
|
|
346
|
+
injection.detail = "the write's body is not JSON or a form; sent unchanged"
|
|
347
|
+
return self._forward(method, path, headers, body)
|
|
348
|
+
if kind == "wrong_record":
|
|
349
|
+
assert self.wrong_record is not None
|
|
350
|
+
edited, done = _set_field(parsed_kind, parsed, *self.wrong_record)
|
|
351
|
+
missing = f"the body has no field {self.wrong_record[0]!r}"
|
|
352
|
+
else:
|
|
353
|
+
edit = (lambda _v: "") if kind == "partial" else _changed
|
|
354
|
+
edited, done = _edit(parsed_kind, parsed, declared, edit)
|
|
355
|
+
missing = "no declared value appears in the write's body"
|
|
356
|
+
if not done:
|
|
357
|
+
injection.applied = False
|
|
358
|
+
injection.detail = missing + "; sent unchanged"
|
|
359
|
+
return self._forward(method, path, headers, body)
|
|
360
|
+
new_body = _dump(parsed_kind, edited)
|
|
361
|
+
if kind == "extra_write":
|
|
362
|
+
first = self._forward(method, path, headers, body)
|
|
363
|
+
self._forward(method, path, headers, new_body)
|
|
364
|
+
return first
|
|
365
|
+
return self._forward(method, path, headers, new_body)
|
|
366
|
+
|
|
367
|
+
def _handler(self) -> type[BaseHTTPRequestHandler]:
|
|
368
|
+
proxy = self
|
|
369
|
+
|
|
370
|
+
class Handler(BaseHTTPRequestHandler):
|
|
371
|
+
def log_message(self, *_args: Any) -> None:
|
|
372
|
+
pass
|
|
373
|
+
|
|
374
|
+
def _body(self) -> bytes:
|
|
375
|
+
if "chunked" in (self.headers.get("Transfer-Encoding") or "").lower():
|
|
376
|
+
chunks: list[bytes] = []
|
|
377
|
+
while True:
|
|
378
|
+
size = int(self.rfile.readline().split(b";")[0], 16)
|
|
379
|
+
if size == 0:
|
|
380
|
+
self.rfile.readline()
|
|
381
|
+
return b"".join(chunks)
|
|
382
|
+
chunks.append(self.rfile.read(size))
|
|
383
|
+
self.rfile.readline()
|
|
384
|
+
return self.rfile.read(int(self.headers.get("Content-Length") or 0))
|
|
385
|
+
|
|
386
|
+
def _serve(self) -> None:
|
|
387
|
+
body = self._body()
|
|
388
|
+
try:
|
|
389
|
+
reply = proxy._handle(
|
|
390
|
+
self.command, self.path, list(self.headers.items()), body
|
|
391
|
+
)
|
|
392
|
+
except OSError as exc:
|
|
393
|
+
reply = _Reply(502, "Bad Gateway", [], str(exc).encode())
|
|
394
|
+
if reply is None:
|
|
395
|
+
self.close_connection = True
|
|
396
|
+
return
|
|
397
|
+
self.send_response(reply.status, reply.reason)
|
|
398
|
+
origin = f"{proxy.upstream.scheme}://{proxy.upstream.netloc}"
|
|
399
|
+
for key, value in reply.headers:
|
|
400
|
+
if key.lower() in _HOP_BY_HOP:
|
|
401
|
+
continue
|
|
402
|
+
mount = origin + proxy.prefix
|
|
403
|
+
if key.lower() == "location" and (
|
|
404
|
+
value == mount or value.startswith(mount + "/")
|
|
405
|
+
):
|
|
406
|
+
value = proxy.base + value[len(mount) :]
|
|
407
|
+
self.send_header(key, value)
|
|
408
|
+
self.send_header("Content-Length", str(len(reply.body)))
|
|
409
|
+
self.send_header("Connection", "close")
|
|
410
|
+
self.end_headers()
|
|
411
|
+
if self.command != "HEAD":
|
|
412
|
+
self.wfile.write(reply.body)
|
|
413
|
+
self.close_connection = True
|
|
414
|
+
|
|
415
|
+
do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = _serve # noqa: N815
|
|
416
|
+
do_HEAD = do_OPTIONS = _serve # noqa: N815
|
|
417
|
+
|
|
418
|
+
return Handler
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
# -- the audit -----------------------------------------------------------------
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
@dataclass
|
|
425
|
+
class Run:
|
|
426
|
+
index: int
|
|
427
|
+
fault: str
|
|
428
|
+
applied: bool
|
|
429
|
+
detail: str
|
|
430
|
+
request: str
|
|
431
|
+
command_returncode: Optional[int]
|
|
432
|
+
reported_success: bool
|
|
433
|
+
outcome: str
|
|
434
|
+
observed: str
|
|
435
|
+
reasons: list[str] = field(default_factory=list)
|
|
436
|
+
seconds: float = 0.0
|
|
437
|
+
|
|
438
|
+
@property
|
|
439
|
+
def measured(self) -> bool:
|
|
440
|
+
"""The fault was applied (or the run is clean) and the store was read."""
|
|
441
|
+
|
|
442
|
+
if self.outcome == "NOT READ":
|
|
443
|
+
return False
|
|
444
|
+
if self.outcome == "UNCERTAIN" and self.observed == "unknown":
|
|
445
|
+
return False
|
|
446
|
+
return self.fault == "clean" or self.applied
|
|
447
|
+
|
|
448
|
+
@property
|
|
449
|
+
def bad(self) -> bool:
|
|
450
|
+
return self.outcome != "CONFIRMED"
|
|
451
|
+
|
|
452
|
+
@property
|
|
453
|
+
def missed(self) -> bool:
|
|
454
|
+
return self.measured and self.bad and self.reported_success
|
|
455
|
+
|
|
456
|
+
@property
|
|
457
|
+
def false_alarm(self) -> bool:
|
|
458
|
+
return self.measured and not self.bad and not self.reported_success
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
@dataclass
|
|
462
|
+
class Report:
|
|
463
|
+
command: list[str]
|
|
464
|
+
upstream: str
|
|
465
|
+
faults: list[str]
|
|
466
|
+
runs: list[Run]
|
|
467
|
+
not_run: dict[str, str]
|
|
468
|
+
success_rule: str
|
|
469
|
+
|
|
470
|
+
def summary(self) -> dict[str, dict[str, int]]:
|
|
471
|
+
rows: dict[str, dict[str, int]] = {}
|
|
472
|
+
for name in ["clean", *self.faults]:
|
|
473
|
+
runs = [r for r in self.runs if r.fault == name]
|
|
474
|
+
measured = [r for r in runs if r.measured]
|
|
475
|
+
rows[name] = {
|
|
476
|
+
"runs": len(runs),
|
|
477
|
+
"measured": len(measured),
|
|
478
|
+
"bad_saves": sum(r.bad for r in measured),
|
|
479
|
+
"missed": sum(r.missed for r in measured),
|
|
480
|
+
"caught": sum(r.bad and not r.reported_success for r in measured),
|
|
481
|
+
"false_alarms": sum(r.false_alarm for r in measured),
|
|
482
|
+
}
|
|
483
|
+
return rows
|
|
484
|
+
|
|
485
|
+
def totals(self) -> dict[str, int]:
|
|
486
|
+
rows = self.summary().values()
|
|
487
|
+
return {
|
|
488
|
+
key: sum(row[key] for row in rows)
|
|
489
|
+
for key in ("measured", "bad_saves", "missed", "caught", "false_alarms")
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
def to_dict(self) -> dict[str, Any]:
|
|
493
|
+
return {
|
|
494
|
+
"format": FORMAT,
|
|
495
|
+
"command": self.command,
|
|
496
|
+
"upstream": self.upstream,
|
|
497
|
+
"success_rule": self.success_rule,
|
|
498
|
+
"faults": self.faults,
|
|
499
|
+
"not_run": self.not_run,
|
|
500
|
+
"summary": self.summary(),
|
|
501
|
+
"totals": self.totals(),
|
|
502
|
+
"runs": [
|
|
503
|
+
{
|
|
504
|
+
**asdict(r),
|
|
505
|
+
"measured": r.measured,
|
|
506
|
+
"bad": r.bad,
|
|
507
|
+
"missed": r.missed,
|
|
508
|
+
"false_alarm": r.false_alarm,
|
|
509
|
+
}
|
|
510
|
+
for r in self.runs
|
|
511
|
+
],
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def schedule(faults: Sequence[str], runs: int) -> list[str]:
|
|
516
|
+
"""Clean first (it gives ``no_persist`` a real reply to replay), then the
|
|
517
|
+
kinds in turn, a clean run starting each round."""
|
|
518
|
+
|
|
519
|
+
order = ["clean", *faults]
|
|
520
|
+
return [order[i % len(order)] for i in range(runs)]
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _values(params: Mapping[str, Any], index: int) -> dict[str, Any]:
|
|
524
|
+
return {
|
|
525
|
+
k: v.replace("{run}", str(index)) if isinstance(v, str) else v
|
|
526
|
+
for k, v in params.items()
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _proxied(contract: Mapping[str, Any], proxy: FaultProxy) -> bool:
|
|
531
|
+
base = str((contract.get("verifier") or {}).get("base_url") or "")
|
|
532
|
+
if not base:
|
|
533
|
+
return False
|
|
534
|
+
split = urlsplit(base)
|
|
535
|
+
host, port = proxy.httpd.server_address[:2]
|
|
536
|
+
same_host = split.hostname in (host, "localhost", "127.0.0.1", "::1", "0.0.0.0")
|
|
537
|
+
return same_host and split.port == port
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def audit(
|
|
541
|
+
contract: Mapping[str, Any],
|
|
542
|
+
params: Mapping[str, Any],
|
|
543
|
+
command: Sequence[str],
|
|
544
|
+
*,
|
|
545
|
+
upstream: str,
|
|
546
|
+
listen: tuple[str, int] = ("127.0.0.1", 0),
|
|
547
|
+
runs: Optional[int] = None,
|
|
548
|
+
faults: Sequence[str] = DEFAULT_FAULTS,
|
|
549
|
+
success_regex: Optional[str] = None,
|
|
550
|
+
write_methods: Sequence[str] = WRITE_METHODS,
|
|
551
|
+
write_path: Optional[str] = None,
|
|
552
|
+
wrong_record: Optional[tuple[str, str]] = None,
|
|
553
|
+
command_timeout_s: float = 300.0,
|
|
554
|
+
progress: Any = None,
|
|
555
|
+
) -> Report:
|
|
556
|
+
"""Run ``command`` ``runs`` times through a fault proxy and judge each run."""
|
|
557
|
+
|
|
558
|
+
from openadapt_verify import (
|
|
559
|
+
Actor,
|
|
560
|
+
Contract,
|
|
561
|
+
WriteBlocked,
|
|
562
|
+
begin_write,
|
|
563
|
+
verify_write,
|
|
564
|
+
)
|
|
565
|
+
from openadapt_verify.quickstart import _STATE_ENV, isolated_state
|
|
566
|
+
|
|
567
|
+
faults = list(dict.fromkeys(faults))
|
|
568
|
+
for name in faults:
|
|
569
|
+
if name not in FAULTS:
|
|
570
|
+
raise ValueError(f"unknown fault {name!r}; choose from {', '.join(FAULTS)}")
|
|
571
|
+
if "wrong_record" in faults and wrong_record is None:
|
|
572
|
+
raise ValueError("wrong_record needs --wrong-record FIELD=VALUE")
|
|
573
|
+
if not command:
|
|
574
|
+
raise ValueError("audit needs a command after --")
|
|
575
|
+
checked = Contract.from_dict(dict(contract))
|
|
576
|
+
pattern = re.compile(success_regex) if success_regex else None
|
|
577
|
+
total = runs if runs is not None else 2 * (len(faults) + 1)
|
|
578
|
+
if total < 1:
|
|
579
|
+
raise ValueError("--runs must be at least 1")
|
|
580
|
+
proxy = FaultProxy(
|
|
581
|
+
upstream,
|
|
582
|
+
listen,
|
|
583
|
+
write_methods=write_methods,
|
|
584
|
+
write_path=write_path,
|
|
585
|
+
wrong_record=wrong_record,
|
|
586
|
+
)
|
|
587
|
+
if _proxied(contract, proxy):
|
|
588
|
+
proxy.close()
|
|
589
|
+
raise ValueError(
|
|
590
|
+
"the contract reads through the audit proxy; point its read path at "
|
|
591
|
+
"the app or its database directly"
|
|
592
|
+
)
|
|
593
|
+
child_env = {k: v for k, v in os.environ.items() if k not in _STATE_ENV}
|
|
594
|
+
child_env["OPENADAPT_AUDIT_URL"] = proxy.base
|
|
595
|
+
results: list[Run] = []
|
|
596
|
+
try:
|
|
597
|
+
for index, fault in enumerate(schedule(faults, total)):
|
|
598
|
+
values = _values(params, index)
|
|
599
|
+
started = time.monotonic()
|
|
600
|
+
with tempfile.TemporaryDirectory(prefix="openadapt-verify-audit-") as tmp:
|
|
601
|
+
with isolated_state(Path(tmp) / "state"):
|
|
602
|
+
try:
|
|
603
|
+
pending = begin_write(
|
|
604
|
+
checked,
|
|
605
|
+
values,
|
|
606
|
+
actor=Actor("agent", "audited-command"),
|
|
607
|
+
workdir=Path(tmp) / "writes",
|
|
608
|
+
)
|
|
609
|
+
except WriteBlocked as exc:
|
|
610
|
+
results.append(
|
|
611
|
+
Run(
|
|
612
|
+
index,
|
|
613
|
+
fault,
|
|
614
|
+
False,
|
|
615
|
+
"",
|
|
616
|
+
"",
|
|
617
|
+
None,
|
|
618
|
+
False,
|
|
619
|
+
"NOT READ",
|
|
620
|
+
"unknown",
|
|
621
|
+
[str(exc)],
|
|
622
|
+
)
|
|
623
|
+
)
|
|
624
|
+
continue
|
|
625
|
+
proxy.arm(None if fault == "clean" else fault, values)
|
|
626
|
+
env = {
|
|
627
|
+
**child_env,
|
|
628
|
+
"OPENADAPT_AUDIT_RUN": str(index),
|
|
629
|
+
"OPENADAPT_AUDIT_VALUES": json.dumps(values),
|
|
630
|
+
}
|
|
631
|
+
returncode: Optional[int]
|
|
632
|
+
try:
|
|
633
|
+
done = subprocess.run(
|
|
634
|
+
list(command),
|
|
635
|
+
env=env,
|
|
636
|
+
capture_output=True,
|
|
637
|
+
text=True,
|
|
638
|
+
timeout=command_timeout_s,
|
|
639
|
+
check=False,
|
|
640
|
+
)
|
|
641
|
+
returncode, output = done.returncode, done.stdout + done.stderr
|
|
642
|
+
except subprocess.TimeoutExpired as exc:
|
|
643
|
+
returncode, output = None, str(exc.output or "")
|
|
644
|
+
injection = proxy.disarm()
|
|
645
|
+
result = verify_write(pending, write_attempted=True, seal=False)
|
|
646
|
+
if pattern is not None:
|
|
647
|
+
success = returncode is not None and bool(pattern.search(output))
|
|
648
|
+
else:
|
|
649
|
+
success = returncode == 0
|
|
650
|
+
detail = injection.detail
|
|
651
|
+
if fault != "clean" and not injection.applied and not detail:
|
|
652
|
+
detail = "no write carrying a declared value went through the proxy"
|
|
653
|
+
run = Run(
|
|
654
|
+
index=index,
|
|
655
|
+
fault=fault,
|
|
656
|
+
applied=injection.applied,
|
|
657
|
+
detail=detail,
|
|
658
|
+
request=injection.request,
|
|
659
|
+
command_returncode=returncode,
|
|
660
|
+
reported_success=success,
|
|
661
|
+
outcome=result.outcome.value,
|
|
662
|
+
observed=result.observed,
|
|
663
|
+
reasons=list(result.reasons),
|
|
664
|
+
seconds=round(time.monotonic() - started, 3),
|
|
665
|
+
)
|
|
666
|
+
results.append(run)
|
|
667
|
+
if progress is not None:
|
|
668
|
+
progress(run)
|
|
669
|
+
finally:
|
|
670
|
+
proxy.close()
|
|
671
|
+
return Report(
|
|
672
|
+
command=list(command),
|
|
673
|
+
upstream=upstream,
|
|
674
|
+
faults=faults,
|
|
675
|
+
runs=results,
|
|
676
|
+
not_run={
|
|
677
|
+
**NOT_RUN,
|
|
678
|
+
**(
|
|
679
|
+
{}
|
|
680
|
+
if "wrong_record" in faults
|
|
681
|
+
else {
|
|
682
|
+
"wrong_record": "needs --wrong-record FIELD=VALUE naming another "
|
|
683
|
+
"record"
|
|
684
|
+
}
|
|
685
|
+
),
|
|
686
|
+
},
|
|
687
|
+
success_rule=f"output matches {success_regex!r}"
|
|
688
|
+
if success_regex
|
|
689
|
+
else "exit code 0",
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def render(report: Report) -> str:
|
|
694
|
+
lines = [
|
|
695
|
+
f"{'fault':<15}{'runs':>5}{'bad saves':>11}{'check said OK':>15}"
|
|
696
|
+
f"{'caught':>8}{'false alarms':>14}"
|
|
697
|
+
]
|
|
698
|
+
for name, row in report.summary().items():
|
|
699
|
+
runs = str(row["measured"])
|
|
700
|
+
if row["measured"] != row["runs"]:
|
|
701
|
+
runs += f"/{row['runs']}"
|
|
702
|
+
lines.append(
|
|
703
|
+
f"{name:<15}{runs:>5}{row['bad_saves']:>11}{row['missed']:>15}"
|
|
704
|
+
f"{row['caught']:>8}{row['false_alarms']:>14}"
|
|
705
|
+
)
|
|
706
|
+
total = report.totals()
|
|
707
|
+
clean = report.summary()["clean"]
|
|
708
|
+
lines += [
|
|
709
|
+
"",
|
|
710
|
+
f"Your check ({report.success_rule}) reported success on "
|
|
711
|
+
f"{total['missed']} of {total['bad_saves']} bad saves.",
|
|
712
|
+
f"False alarms: {clean['false_alarms']} of {clean['measured']} clean runs, "
|
|
713
|
+
f"{total['false_alarms'] - clean['false_alarms']} on faulted runs whose "
|
|
714
|
+
"write still landed correctly.",
|
|
715
|
+
]
|
|
716
|
+
skipped = [r for r in report.runs if not r.measured]
|
|
717
|
+
if skipped:
|
|
718
|
+
lines.append(
|
|
719
|
+
f"{len(skipped)} run(s) not measured (fault not applied or store "
|
|
720
|
+
"unreadable); see the JSON report."
|
|
721
|
+
)
|
|
722
|
+
lines.append("Not run: " + ", ".join(sorted(report.not_run)) + ".")
|
|
723
|
+
return "\n".join(lines)
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def exit_code(report: Report) -> int:
|
|
727
|
+
"""0 when every bad save was caught with no false alarms, else 1."""
|
|
728
|
+
|
|
729
|
+
total = report.totals()
|
|
730
|
+
if total["measured"] == 0:
|
|
731
|
+
return 2
|
|
732
|
+
return 0 if total["missed"] == 0 and total["false_alarms"] == 0 else 1
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
def print_progress(run: Run) -> None:
|
|
736
|
+
said = "success" if run.reported_success else "failure"
|
|
737
|
+
note = "" if run.measured else " (not measured)"
|
|
738
|
+
print(
|
|
739
|
+
f"run {run.index + 1}: {run.fault}: command said {said}, store says "
|
|
740
|
+
f"{run.outcome}{note}",
|
|
741
|
+
file=sys.stderr,
|
|
742
|
+
)
|