agentcrash-core 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentcrash/__init__.py +45 -0
- agentcrash/__main__.py +6 -0
- agentcrash/_registry.py +68 -0
- agentcrash/cli.py +274 -0
- agentcrash/clock.py +37 -0
- agentcrash/db.py +411 -0
- agentcrash/evaluate.py +57 -0
- agentcrash/events.py +76 -0
- agentcrash/injector.py +237 -0
- agentcrash/invariants.py +73 -0
- agentcrash/lab/app.js +61 -0
- agentcrash/lab/index.html +20 -0
- agentcrash/lab/styles.css +7 -0
- agentcrash/ledger.py +92 -0
- agentcrash/policy.py +402 -0
- agentcrash/scan_pii.py +62 -0
- agentcrash/schemas/policy-patch.json +13 -0
- agentcrash/schemas/policy.json +42 -0
- agentcrash/sdk.py +352 -0
- agentcrash/server.py +495 -0
- agentcrash/tracer.py +191 -0
- agentcrash/verifiers.py +216 -0
- agentcrash_core-0.2.0.dist-info/METADATA +106 -0
- agentcrash_core-0.2.0.dist-info/RECORD +26 -0
- agentcrash_core-0.2.0.dist-info/WHEEL +4 -0
- agentcrash_core-0.2.0.dist-info/entry_points.txt +2 -0
agentcrash/db.py
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
"""SQLite store for runs, events, verdicts, patches, and ODRL documents.
|
|
2
|
+
|
|
3
|
+
The trace JSONL files stay the immutable artifact layer; this module is
|
|
4
|
+
the query layer the UI reads from. Standard library only.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import sqlite3
|
|
11
|
+
from datetime import UTC, datetime
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
SCHEMA_VERSION = 1
|
|
16
|
+
|
|
17
|
+
_SCHEMA = """
|
|
18
|
+
CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY);
|
|
19
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
20
|
+
run_key TEXT PRIMARY KEY,
|
|
21
|
+
run_id TEXT NOT NULL,
|
|
22
|
+
scenario_id TEXT NOT NULL,
|
|
23
|
+
faults_json TEXT NOT NULL,
|
|
24
|
+
policy_variant TEXT NOT NULL,
|
|
25
|
+
policy_file TEXT NOT NULL,
|
|
26
|
+
seed INTEGER NOT NULL,
|
|
27
|
+
max_steps INTEGER NOT NULL,
|
|
28
|
+
nl_text TEXT NOT NULL,
|
|
29
|
+
nl_label TEXT NOT NULL,
|
|
30
|
+
status TEXT NOT NULL,
|
|
31
|
+
verdict TEXT,
|
|
32
|
+
model_calls INTEGER NOT NULL DEFAULT 0,
|
|
33
|
+
prompt_tokens INTEGER NOT NULL DEFAULT 0,
|
|
34
|
+
completion_tokens INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
total_tokens INTEGER NOT NULL DEFAULT 0,
|
|
36
|
+
virtual_ms INTEGER NOT NULL DEFAULT 0,
|
|
37
|
+
wall_seconds REAL NOT NULL DEFAULT 0,
|
|
38
|
+
ledger_hash TEXT,
|
|
39
|
+
chain_ok INTEGER,
|
|
40
|
+
trace_path TEXT NOT NULL,
|
|
41
|
+
state_path TEXT NOT NULL,
|
|
42
|
+
error_code TEXT,
|
|
43
|
+
error_hint TEXT,
|
|
44
|
+
created_at TEXT NOT NULL,
|
|
45
|
+
completed_at TEXT
|
|
46
|
+
);
|
|
47
|
+
CREATE INDEX IF NOT EXISTS idx_runs_created ON runs(created_at DESC);
|
|
48
|
+
CREATE TABLE IF NOT EXISTS events (
|
|
49
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
50
|
+
run_key TEXT NOT NULL REFERENCES runs(run_key),
|
|
51
|
+
seq INTEGER NOT NULL,
|
|
52
|
+
event_id TEXT NOT NULL,
|
|
53
|
+
step INTEGER NOT NULL,
|
|
54
|
+
tool TEXT,
|
|
55
|
+
phase TEXT,
|
|
56
|
+
outcome TEXT NOT NULL,
|
|
57
|
+
latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
58
|
+
ts_virtual INTEGER NOT NULL DEFAULT 0,
|
|
59
|
+
model_usage_json TEXT,
|
|
60
|
+
injector_json TEXT,
|
|
61
|
+
policy_json TEXT,
|
|
62
|
+
detail_json TEXT NOT NULL
|
|
63
|
+
);
|
|
64
|
+
CREATE INDEX IF NOT EXISTS idx_events_run ON events(run_key, seq);
|
|
65
|
+
CREATE TABLE IF NOT EXISTS verdicts (
|
|
66
|
+
run_key TEXT NOT NULL REFERENCES runs(run_key),
|
|
67
|
+
invariant_id TEXT NOT NULL,
|
|
68
|
+
passed INTEGER NOT NULL,
|
|
69
|
+
evidence TEXT NOT NULL,
|
|
70
|
+
PRIMARY KEY (run_key, invariant_id)
|
|
71
|
+
);
|
|
72
|
+
CREATE TABLE IF NOT EXISTS patches (
|
|
73
|
+
run_key TEXT PRIMARY KEY REFERENCES runs(run_key),
|
|
74
|
+
patch_json TEXT NOT NULL
|
|
75
|
+
);
|
|
76
|
+
CREATE TABLE IF NOT EXISTS odrl_docs (
|
|
77
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
78
|
+
run_key TEXT NOT NULL REFERENCES runs(run_key),
|
|
79
|
+
step INTEGER NOT NULL,
|
|
80
|
+
outcome TEXT NOT NULL,
|
|
81
|
+
doc_json TEXT NOT NULL,
|
|
82
|
+
valid INTEGER NOT NULL,
|
|
83
|
+
errors_json TEXT NOT NULL,
|
|
84
|
+
attacker INTEGER NOT NULL
|
|
85
|
+
);
|
|
86
|
+
CREATE INDEX IF NOT EXISTS idx_odrl_run ON odrl_docs(run_key);
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def utcnow() -> str:
|
|
91
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def connect(path: Path) -> sqlite3.Connection:
|
|
95
|
+
conn = sqlite3.connect(str(path))
|
|
96
|
+
conn.row_factory = sqlite3.Row
|
|
97
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
98
|
+
return conn
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def migrate(conn: sqlite3.Connection) -> int:
|
|
102
|
+
"""Create tables idempotently; return the schema version."""
|
|
103
|
+
conn.executescript(_SCHEMA)
|
|
104
|
+
row = conn.execute("SELECT version FROM schema_version").fetchone()
|
|
105
|
+
if row is None:
|
|
106
|
+
conn.execute("INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,))
|
|
107
|
+
conn.commit()
|
|
108
|
+
return SCHEMA_VERSION
|
|
109
|
+
return int(row[0])
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def insert_run(conn: sqlite3.Connection, record: dict[str, Any]) -> None:
|
|
113
|
+
"""Insert or replace one finished run and all of its children."""
|
|
114
|
+
conn.execute(
|
|
115
|
+
"""
|
|
116
|
+
INSERT OR REPLACE INTO runs (
|
|
117
|
+
run_key, run_id, scenario_id, faults_json, policy_variant, policy_file,
|
|
118
|
+
seed, max_steps, nl_text, nl_label, status, verdict, model_calls,
|
|
119
|
+
prompt_tokens, completion_tokens, total_tokens, virtual_ms, wall_seconds,
|
|
120
|
+
ledger_hash, chain_ok, trace_path, state_path, error_code, error_hint,
|
|
121
|
+
created_at, completed_at
|
|
122
|
+
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
123
|
+
""",
|
|
124
|
+
(
|
|
125
|
+
record["run_key"],
|
|
126
|
+
record["run_id"],
|
|
127
|
+
record["scenario_id"],
|
|
128
|
+
json.dumps(record.get("faults", []), sort_keys=True),
|
|
129
|
+
record["policy_variant"],
|
|
130
|
+
record["policy_file"],
|
|
131
|
+
record["seed"],
|
|
132
|
+
record["max_steps"],
|
|
133
|
+
record["nl_text"],
|
|
134
|
+
record.get("nl_label", "user_supplied"),
|
|
135
|
+
record["status"],
|
|
136
|
+
record.get("verdict"),
|
|
137
|
+
record.get("model_calls", 0),
|
|
138
|
+
record.get("prompt_tokens", 0),
|
|
139
|
+
record.get("completion_tokens", 0),
|
|
140
|
+
record.get("total_tokens", 0),
|
|
141
|
+
record.get("virtual_ms", 0),
|
|
142
|
+
record.get("wall_seconds", 0.0),
|
|
143
|
+
record.get("ledger_hash"),
|
|
144
|
+
record.get("chain_ok"),
|
|
145
|
+
record["trace_path"],
|
|
146
|
+
record["state_path"],
|
|
147
|
+
record.get("error_code"),
|
|
148
|
+
record.get("error_hint"),
|
|
149
|
+
record.get("created_at", utcnow()),
|
|
150
|
+
record.get("completed_at"),
|
|
151
|
+
),
|
|
152
|
+
)
|
|
153
|
+
run_key = record["run_key"]
|
|
154
|
+
conn.execute("DELETE FROM events WHERE run_key = ?", (run_key,))
|
|
155
|
+
for seq, event in enumerate(record.get("events", [])):
|
|
156
|
+
detail = event.get("detail") or {}
|
|
157
|
+
conn.execute(
|
|
158
|
+
"""
|
|
159
|
+
INSERT INTO events (
|
|
160
|
+
run_key, seq, event_id, step, tool, phase, outcome, latency_ms,
|
|
161
|
+
ts_virtual, model_usage_json, injector_json, policy_json, detail_json
|
|
162
|
+
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
163
|
+
""",
|
|
164
|
+
(
|
|
165
|
+
run_key,
|
|
166
|
+
seq,
|
|
167
|
+
event.get("event_id", ""),
|
|
168
|
+
event.get("step", 0),
|
|
169
|
+
event.get("tool"),
|
|
170
|
+
detail.get("phase"),
|
|
171
|
+
event.get("outcome", "info"),
|
|
172
|
+
event.get("latency", 0),
|
|
173
|
+
event.get("ts_virtual", 0),
|
|
174
|
+
(
|
|
175
|
+
json.dumps(detail["model_usage"], sort_keys=True)
|
|
176
|
+
if isinstance(detail.get("model_usage"), dict)
|
|
177
|
+
else None
|
|
178
|
+
),
|
|
179
|
+
(
|
|
180
|
+
json.dumps(event["injector_event"], sort_keys=True)
|
|
181
|
+
if event.get("injector_event") is not None
|
|
182
|
+
else None
|
|
183
|
+
),
|
|
184
|
+
(
|
|
185
|
+
json.dumps(event["policy_decision"], sort_keys=True)
|
|
186
|
+
if event.get("policy_decision") is not None
|
|
187
|
+
else None
|
|
188
|
+
),
|
|
189
|
+
json.dumps(detail, sort_keys=True),
|
|
190
|
+
),
|
|
191
|
+
)
|
|
192
|
+
conn.execute("DELETE FROM verdicts WHERE run_key = ?", (run_key,))
|
|
193
|
+
for row in record.get("verdict_rows", []):
|
|
194
|
+
conn.execute(
|
|
195
|
+
"INSERT INTO verdicts (run_key, invariant_id, passed, evidence) VALUES (?,?,?,?)",
|
|
196
|
+
(run_key, row["id"], 1 if row["passed"] else 0, row.get("evidence", "")),
|
|
197
|
+
)
|
|
198
|
+
conn.execute("DELETE FROM patches WHERE run_key = ?", (run_key,))
|
|
199
|
+
if record.get("patch") is not None:
|
|
200
|
+
conn.execute(
|
|
201
|
+
"INSERT INTO patches (run_key, patch_json) VALUES (?,?)",
|
|
202
|
+
(run_key, json.dumps(record["patch"], sort_keys=True)),
|
|
203
|
+
)
|
|
204
|
+
conn.execute("DELETE FROM odrl_docs WHERE run_key = ?", (run_key,))
|
|
205
|
+
for doc in record.get("saves", []):
|
|
206
|
+
conn.execute(
|
|
207
|
+
"""
|
|
208
|
+
INSERT INTO odrl_docs (
|
|
209
|
+
run_key, step, outcome, doc_json, valid, errors_json, attacker
|
|
210
|
+
) VALUES (?,?,?,?,?,?,?)
|
|
211
|
+
""",
|
|
212
|
+
(
|
|
213
|
+
run_key,
|
|
214
|
+
doc.get("step", 0),
|
|
215
|
+
doc.get("outcome", ""),
|
|
216
|
+
json.dumps(doc.get("value"), sort_keys=True),
|
|
217
|
+
1 if doc.get("valid") else 0,
|
|
218
|
+
json.dumps(doc.get("errors", [])),
|
|
219
|
+
1 if doc.get("attacker") else 0,
|
|
220
|
+
),
|
|
221
|
+
)
|
|
222
|
+
conn.commit()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
_RUN_COLUMNS = (
|
|
226
|
+
"run_key, run_id, scenario_id, faults_json, policy_variant, policy_file,"
|
|
227
|
+
" seed, max_steps, nl_text, nl_label, status, verdict, model_calls,"
|
|
228
|
+
" prompt_tokens, completion_tokens, total_tokens, virtual_ms, wall_seconds,"
|
|
229
|
+
" ledger_hash, chain_ok, trace_path, state_path, error_code, error_hint,"
|
|
230
|
+
" created_at, completed_at"
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _run_row(row: sqlite3.Row) -> dict[str, Any]:
|
|
235
|
+
record = dict(row)
|
|
236
|
+
record["faults"] = json.loads(record.pop("faults_json") or "[]")
|
|
237
|
+
record["chain_ok"] = None if record["chain_ok"] is None else bool(record["chain_ok"])
|
|
238
|
+
return record
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def list_runs(
|
|
242
|
+
conn: sqlite3.Connection,
|
|
243
|
+
scenario: str | None = None,
|
|
244
|
+
verdict: str | None = None,
|
|
245
|
+
status: str | None = None,
|
|
246
|
+
limit: int = 100,
|
|
247
|
+
) -> list[dict[str, Any]]:
|
|
248
|
+
"""Newest-first run summaries for selectors and galleries."""
|
|
249
|
+
query = (
|
|
250
|
+
"SELECT run_key, run_id, scenario_id, faults_json, policy_variant, seed,"
|
|
251
|
+
" status, verdict, model_calls, total_tokens, wall_seconds, created_at,"
|
|
252
|
+
" substr(nl_text, 1, 160) AS nl_preview FROM runs"
|
|
253
|
+
)
|
|
254
|
+
clauses: list[str] = []
|
|
255
|
+
args: list[Any] = []
|
|
256
|
+
if scenario:
|
|
257
|
+
clauses.append("scenario_id = ?")
|
|
258
|
+
args.append(scenario)
|
|
259
|
+
if verdict:
|
|
260
|
+
clauses.append("verdict = ?")
|
|
261
|
+
args.append(verdict)
|
|
262
|
+
if status:
|
|
263
|
+
clauses.append("status = ?")
|
|
264
|
+
args.append(status)
|
|
265
|
+
if clauses:
|
|
266
|
+
query += " WHERE " + " AND ".join(clauses)
|
|
267
|
+
query += " ORDER BY created_at DESC, rowid DESC LIMIT ?"
|
|
268
|
+
args.append(limit)
|
|
269
|
+
rows = []
|
|
270
|
+
for row in conn.execute(query, args):
|
|
271
|
+
item = dict(row)
|
|
272
|
+
item["faults"] = json.loads(item.pop("faults_json") or "[]")
|
|
273
|
+
rows.append(item)
|
|
274
|
+
return rows
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def get_run(conn: sqlite3.Connection, run_key: str) -> dict[str, Any] | None:
|
|
278
|
+
"""Full run snapshot shaped like the live API response."""
|
|
279
|
+
row = conn.execute(f"SELECT {_RUN_COLUMNS} FROM runs WHERE run_key = ?", (run_key,)).fetchone()
|
|
280
|
+
if row is None:
|
|
281
|
+
return None
|
|
282
|
+
record = _run_row(row)
|
|
283
|
+
events = []
|
|
284
|
+
for event in conn.execute(
|
|
285
|
+
"SELECT event_id, step, tool, phase, outcome, latency_ms, ts_virtual,"
|
|
286
|
+
" model_usage_json, injector_json, policy_json, detail_json"
|
|
287
|
+
" FROM events WHERE run_key = ? ORDER BY seq",
|
|
288
|
+
(run_key,),
|
|
289
|
+
):
|
|
290
|
+
item = dict(event)
|
|
291
|
+
detail = json.loads(item.pop("detail_json") or "{}")
|
|
292
|
+
if item.pop("model_usage_json", None):
|
|
293
|
+
detail.setdefault("model_usage", json.loads(item.get("model_usage_json") or "{}"))
|
|
294
|
+
injector = item.pop("injector_json", None)
|
|
295
|
+
policy = item.pop("policy_json", None)
|
|
296
|
+
events.append(
|
|
297
|
+
{
|
|
298
|
+
"event_id": item["event_id"],
|
|
299
|
+
"step": item["step"],
|
|
300
|
+
"tool": item["tool"],
|
|
301
|
+
"outcome": item["outcome"],
|
|
302
|
+
"latency": item["latency_ms"],
|
|
303
|
+
"ts_virtual": item["ts_virtual"],
|
|
304
|
+
"injector_event": json.loads(injector) if injector else None,
|
|
305
|
+
"policy_decision": json.loads(policy) if policy else None,
|
|
306
|
+
"detail": detail,
|
|
307
|
+
"run_id": record["run_id"],
|
|
308
|
+
"scenario": record["scenario_id"],
|
|
309
|
+
"prev_hash": "",
|
|
310
|
+
}
|
|
311
|
+
)
|
|
312
|
+
verdict_rows = [
|
|
313
|
+
{"id": v["invariant_id"], "passed": bool(v["passed"]), "evidence": v["evidence"]}
|
|
314
|
+
for v in conn.execute(
|
|
315
|
+
"SELECT invariant_id, passed, evidence FROM verdicts WHERE run_key = ?",
|
|
316
|
+
(run_key,),
|
|
317
|
+
)
|
|
318
|
+
]
|
|
319
|
+
patch_row = conn.execute(
|
|
320
|
+
"SELECT patch_json FROM patches WHERE run_key = ?", (run_key,)
|
|
321
|
+
).fetchone()
|
|
322
|
+
saves = [
|
|
323
|
+
{
|
|
324
|
+
"step": doc["step"],
|
|
325
|
+
"outcome": doc["outcome"],
|
|
326
|
+
"value": json.loads(doc["doc_json"]),
|
|
327
|
+
"valid": bool(doc["valid"]),
|
|
328
|
+
"errors": json.loads(doc["errors_json"] or "[]"),
|
|
329
|
+
"attacker": bool(doc["attacker"]),
|
|
330
|
+
}
|
|
331
|
+
for doc in conn.execute(
|
|
332
|
+
"SELECT step, outcome, doc_json, valid, errors_json, attacker"
|
|
333
|
+
" FROM odrl_docs WHERE run_key = ? ORDER BY id",
|
|
334
|
+
(run_key,),
|
|
335
|
+
)
|
|
336
|
+
]
|
|
337
|
+
total = {
|
|
338
|
+
"prompt_tokens": record["prompt_tokens"],
|
|
339
|
+
"completion_tokens": record["completion_tokens"],
|
|
340
|
+
"total_tokens": record["total_tokens"],
|
|
341
|
+
}
|
|
342
|
+
return {
|
|
343
|
+
"run_key": record["run_key"],
|
|
344
|
+
"status": record["status"],
|
|
345
|
+
"spec": {
|
|
346
|
+
"scenario_id": record["scenario_id"],
|
|
347
|
+
"faults": record["faults"],
|
|
348
|
+
"policy_variant": record["policy_variant"],
|
|
349
|
+
"seed": record["seed"],
|
|
350
|
+
"run_id": record["run_id"],
|
|
351
|
+
},
|
|
352
|
+
"fault_summary": [f"{f.get('kind')}" for f in record["faults"] if isinstance(f, dict)],
|
|
353
|
+
"nl_text": record["nl_text"],
|
|
354
|
+
"nl_label": record["nl_label"],
|
|
355
|
+
"events": events,
|
|
356
|
+
"usage_total": total,
|
|
357
|
+
"model_calls": record["model_calls"],
|
|
358
|
+
"verdict": (
|
|
359
|
+
{"verdict": record["verdict"], "results": verdict_rows} if record["verdict"] else None
|
|
360
|
+
),
|
|
361
|
+
"patch": json.loads(patch_row["patch_json"]) if patch_row else None,
|
|
362
|
+
"saves": saves,
|
|
363
|
+
"chain_ok": record["chain_ok"],
|
|
364
|
+
"wall_seconds": record["wall_seconds"],
|
|
365
|
+
"ledger_hash": record["ledger_hash"],
|
|
366
|
+
"error": (
|
|
367
|
+
{"code": record["error_code"], "hint": record["error_hint"]}
|
|
368
|
+
if record["error_code"]
|
|
369
|
+
else None
|
|
370
|
+
),
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def list_odrl_docs(
|
|
375
|
+
conn: sqlite3.Connection,
|
|
376
|
+
scenario: str | None = None,
|
|
377
|
+
valid: bool | None = None,
|
|
378
|
+
attacker: bool | None = None,
|
|
379
|
+
limit: int = 100,
|
|
380
|
+
) -> list[dict[str, Any]]:
|
|
381
|
+
"""NL-to-ODRL pairs for Policy Studio, newest first."""
|
|
382
|
+
query = (
|
|
383
|
+
"SELECT o.run_key, o.step, o.outcome, o.doc_json, o.valid, o.errors_json,"
|
|
384
|
+
" o.attacker, r.scenario_id, r.policy_variant, r.verdict, r.nl_text,"
|
|
385
|
+
" r.nl_label, r.created_at FROM odrl_docs o"
|
|
386
|
+
" JOIN runs r ON r.run_key = o.run_key"
|
|
387
|
+
)
|
|
388
|
+
clauses: list[str] = []
|
|
389
|
+
args: list[Any] = []
|
|
390
|
+
if scenario:
|
|
391
|
+
clauses.append("r.scenario_id = ?")
|
|
392
|
+
args.append(scenario)
|
|
393
|
+
if valid is not None:
|
|
394
|
+
clauses.append("o.valid = ?")
|
|
395
|
+
args.append(1 if valid else 0)
|
|
396
|
+
if attacker is not None:
|
|
397
|
+
clauses.append("o.attacker = ?")
|
|
398
|
+
args.append(1 if attacker else 0)
|
|
399
|
+
if clauses:
|
|
400
|
+
query += " WHERE " + " AND ".join(clauses)
|
|
401
|
+
query += " ORDER BY r.created_at DESC, o.id DESC LIMIT ?"
|
|
402
|
+
args.append(limit)
|
|
403
|
+
pairs = []
|
|
404
|
+
for row in conn.execute(query, args):
|
|
405
|
+
item = dict(row)
|
|
406
|
+
item["doc"] = json.loads(item.pop("doc_json"))
|
|
407
|
+
item["errors"] = json.loads(item.pop("errors_json") or "[]")
|
|
408
|
+
item["valid"] = bool(item["valid"])
|
|
409
|
+
item["attacker"] = bool(item["attacker"])
|
|
410
|
+
pairs.append(item)
|
|
411
|
+
return pairs
|
agentcrash/evaluate.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Reliability evaluation over SDK traces.
|
|
2
|
+
|
|
3
|
+
Framework-neutral: callers pass explicit invariant objects with ``.id``
|
|
4
|
+
and ``.check(trace, state)`` (see :mod:`agentcrash.verifiers` for the
|
|
5
|
+
protocol). Produces machine-readable reports and baseline-vs-fault
|
|
6
|
+
comparisons for regression tracking.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any, Protocol
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Invariant(Protocol):
|
|
15
|
+
"""Structural contract for an invariant evaluator."""
|
|
16
|
+
|
|
17
|
+
id: str
|
|
18
|
+
|
|
19
|
+
def check(self, trace: list[dict[str, Any]], state: dict[str, Any]) -> Any: ...
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def evaluate_trace(
|
|
23
|
+
trace: list[dict[str, Any]],
|
|
24
|
+
invariants: list[Invariant],
|
|
25
|
+
state: dict[str, Any] | None = None,
|
|
26
|
+
run_id: str = "",
|
|
27
|
+
) -> dict[str, Any]:
|
|
28
|
+
"""Run invariants over one trace; returns a machine-readable report."""
|
|
29
|
+
state = state or {}
|
|
30
|
+
results = []
|
|
31
|
+
for invariant in invariants:
|
|
32
|
+
verdict = invariant.check(trace, state)
|
|
33
|
+
results.append(verdict.to_dict() if hasattr(verdict, "to_dict") else dict(verdict))
|
|
34
|
+
passed = sum(1 for r in results if r.get("passed"))
|
|
35
|
+
return {
|
|
36
|
+
"run_id": run_id,
|
|
37
|
+
"invariants": len(results),
|
|
38
|
+
"passed": passed,
|
|
39
|
+
"failed": len(results) - passed,
|
|
40
|
+
"verdict": "PASS" if passed == len(results) else "FAIL",
|
|
41
|
+
"results": results,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def compare_reports(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
|
|
46
|
+
"""Diff two evaluation reports; flags regressions and fixes."""
|
|
47
|
+
base = {r["id"]: r.get("passed") for r in baseline.get("results", [])}
|
|
48
|
+
cand = {r["id"]: r.get("passed") for r in candidate.get("results", [])}
|
|
49
|
+
regressions = sorted(i for i, ok in cand.items() if ok is False and base.get(i) is True)
|
|
50
|
+
fixes = sorted(i for i, ok in cand.items() if ok is True and base.get(i) is False)
|
|
51
|
+
return {
|
|
52
|
+
"baseline": baseline.get("verdict"),
|
|
53
|
+
"candidate": candidate.get("verdict"),
|
|
54
|
+
"regressions": regressions,
|
|
55
|
+
"fixes": fixes,
|
|
56
|
+
"regression": bool(regressions),
|
|
57
|
+
}
|
agentcrash/events.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Versioned SDK-to-server event schema (v1).
|
|
2
|
+
|
|
3
|
+
At-least-once delivery with idempotent ingestion: the server treats
|
|
4
|
+
``event_id`` as the idempotency key and dedupes. Ordering per run is by
|
|
5
|
+
``seq``; ``ts_virtual`` is a host-controlled virtual clock (monotonic per
|
|
6
|
+
run), never wall-clock.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
SCHEMA_VERSION = 1
|
|
14
|
+
|
|
15
|
+
REQUIRED_KEYS = (
|
|
16
|
+
"schema_version",
|
|
17
|
+
"event_id",
|
|
18
|
+
"run_id",
|
|
19
|
+
"project",
|
|
20
|
+
"workflow",
|
|
21
|
+
"step",
|
|
22
|
+
"kind",
|
|
23
|
+
"status",
|
|
24
|
+
"seq",
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def make_event(
|
|
29
|
+
*,
|
|
30
|
+
event_id: str,
|
|
31
|
+
run_id: str,
|
|
32
|
+
project: str,
|
|
33
|
+
workflow: str,
|
|
34
|
+
step: str | None = None,
|
|
35
|
+
kind: str,
|
|
36
|
+
status: str = "ok",
|
|
37
|
+
seq: int,
|
|
38
|
+
tool: str | None = None,
|
|
39
|
+
inputs: dict[str, Any] | None = None,
|
|
40
|
+
outputs: dict[str, Any] | None = None,
|
|
41
|
+
error: str | None = None,
|
|
42
|
+
fault: dict[str, Any] | None = None,
|
|
43
|
+
parent_step: str | None = None,
|
|
44
|
+
) -> dict[str, Any]:
|
|
45
|
+
"""Build a v1 event dict (JSON-serializable)."""
|
|
46
|
+
return {
|
|
47
|
+
"schema_version": SCHEMA_VERSION,
|
|
48
|
+
"event_id": event_id,
|
|
49
|
+
"run_id": run_id,
|
|
50
|
+
"project": project,
|
|
51
|
+
"workflow": workflow,
|
|
52
|
+
"step": step,
|
|
53
|
+
"parent_step": parent_step,
|
|
54
|
+
"kind": kind,
|
|
55
|
+
"status": status,
|
|
56
|
+
"seq": seq,
|
|
57
|
+
"tool": tool,
|
|
58
|
+
"inputs": inputs or {},
|
|
59
|
+
"outputs": outputs or {},
|
|
60
|
+
"error": error,
|
|
61
|
+
"fault": fault,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def validate_event(event: dict[str, Any]) -> list[str]:
|
|
66
|
+
"""Return a list of problems; empty means valid."""
|
|
67
|
+
problems: list[str] = []
|
|
68
|
+
for key in REQUIRED_KEYS:
|
|
69
|
+
if key not in event:
|
|
70
|
+
problems.append(f"missing key {key!r}")
|
|
71
|
+
if event.get("schema_version") != SCHEMA_VERSION:
|
|
72
|
+
problems.append(f"unsupported schema_version {event.get('schema_version')!r}")
|
|
73
|
+
seq = event.get("seq")
|
|
74
|
+
if not isinstance(seq, int) or seq < 1:
|
|
75
|
+
problems.append("seq must be a positive int")
|
|
76
|
+
return problems
|