system-design-primer-skill 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +15 -7
  2. package/SKILL.md +22 -2
  3. package/VERSION +1 -1
  4. package/evals/README.md +1022 -0
  5. package/evals/system-design-cases.json +414 -0
  6. package/package.json +3 -2
  7. package/references/03-data-and-storage.md +8 -2
  8. package/references/04-caching.md +3 -3
  9. package/references/05-async-and-messaging.md +4 -0
  10. package/references/06-distributed-systems.md +2 -2
  11. package/references/07-reliability-and-resilience.md +7 -3
  12. package/references/08-api-security-observability.md +4 -0
  13. package/references/15-failure-analysis.md +4 -0
  14. package/references/19-agent-security-and-permissions.md +22 -6
  15. package/references/20-agent-state-and-memory.md +4 -0
  16. package/references/22-kubernetes-and-container-platforms.md +1 -1
  17. package/references/24-multi-region-and-disaster-recovery.md +6 -2
  18. package/references/27-production-observability-and-slo.md +4 -0
  19. package/references/28-event-streaming-and-kafka-pulsar.md +6 -0
  20. package/references/32-workflow-orchestration-and-temporal.md +4 -0
  21. package/references/34-postgresql-internals-and-performance.md +2 -0
  22. package/references/35-mysql-innodb-and-performance.md +2 -0
  23. package/references/36-sharding-and-distributed-sql.md +1 -1
  24. package/references/37-consensus-and-distributed-transactions.md +7 -3
  25. package/references/40-zero-downtime-database-migrations.md +3 -3
  26. package/references/48-oauth-oidc-and-passkeys.md +3 -3
  27. package/references/49-iam-rbac-abac-rebac.md +7 -3
  28. package/references/51-secrets-kms-pki-and-key-management.md +4 -0
  29. package/references/52-supply-chain-security-sbom-slsa.md +62 -0
  30. package/references/54-privacy-engineering-and-data-governance.md +4 -0
  31. package/references/55-opentelemetry-and-instrumentation.md +4 -0
  32. package/references/56-distributed-tracing-and-context-propagation.md +4 -0
  33. package/references/59-capacity-planning-and-performance-modeling.md +8 -4
  34. package/references/60-chaos-engineering-and-resilience-experiments.md +4 -0
  35. package/references/61-incident-management-and-postmortems.md +4 -0
  36. package/references/67-storage-infrastructure-and-storage-systems.md +4 -0
  37. package/references/68-tcp-quic-and-network-performance.md +1 -1
  38. package/references/69-task-routing-and-evidence.md +75 -18
  39. package/scripts/audit_chain.py +36 -0
  40. package/scripts/compare_reports.py +43 -0
  41. package/scripts/evaluate.py +475 -0
  42. package/scripts/evaluation_catalog.py +129 -0
  43. package/scripts/evaluation_fingerprints.py +20 -0
  44. package/scripts/evidence_archive.py +79 -0
  45. package/scripts/evidence_signing.py +98 -0
  46. package/scripts/evidence_timestamp.py +441 -0
  47. package/scripts/install.ps1 +2 -2
  48. package/scripts/install.sh +2 -2
  49. package/scripts/report_integrity.py +184 -0
  50. package/scripts/review_consensus.py +77 -0
  51. package/scripts/review_evidence.py +105 -0
  52. package/scripts/review_provenance.py +44 -0
  53. package/scripts/reviewer_calibration.py +215 -0
  54. package/scripts/rubric_compatibility.py +134 -0
  55. package/scripts/run_evals.py +130 -0
  56. package/scripts/run_manifest.py +179 -0
  57. package/scripts/statistical_analysis.py +308 -0
  58. package/scripts/tsa_attestation.py +709 -0
  59. package/scripts/tsa_validation_report.py +627 -0
  60. package/scripts/validate.py +233 -21
  61. package/scripts/verify_build_provenance.py +625 -0
  62. package/scripts/verify_tsa_ocsp.py +962 -0
  63. package/scripts/verify_tsa_revocation.py +782 -0
  64. package/scripts/windows_broker_protocol.py +274 -0
  65. package/scripts/windows_broker_transport.py +141 -0
  66. package/scripts/windows_broker_win32.py +713 -0
@@ -0,0 +1,475 @@
1
+ #!/usr/bin/env python3
2
+ """Aggregate human-reviewed model responses against the system-design eval fixtures.
3
+
4
+ This tool deliberately does not infer answer quality from keyword matching. It validates
5
+ review annotations and reports criterion coverage, output scores, and guardrail pass rates.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import hashlib
11
+ import json
12
+ import re
13
+ import sys
14
+ from itertools import combinations
15
+ from collections import defaultdict
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+ try:
20
+ from .run_manifest import load_verified_manifest
21
+ from .review_provenance import validate_review_history
22
+ from .review_consensus import validate_review_panel
23
+ from .reviewer_calibration import aggregate_reviewer_calibration, summarize_reviewer_behavior, summarize_review_data_quality
24
+ from .statistical_analysis import comparative_analysis
25
+ from .evaluation_catalog import fingerprint_dataset
26
+ from .review_evidence import attach_review_evidence
27
+ except ImportError: # Direct execution: python scripts/evaluate.py
28
+ from run_manifest import load_verified_manifest
29
+ from review_provenance import validate_review_history
30
+ from review_consensus import validate_review_panel
31
+ from reviewer_calibration import aggregate_reviewer_calibration, summarize_reviewer_behavior, summarize_review_data_quality
32
+ from statistical_analysis import comparative_analysis
33
+ from evaluation_catalog import fingerprint_dataset
34
+ from review_evidence import attach_review_evidence
35
+
36
+
37
+ AUDIT_CHAIN_VERSION = 1
38
+ AUDIT_FIELD = "_audit"
39
+
40
+ def canonical_json(value: Any) -> bytes:
41
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
42
+
43
+ def audit_record_hash(record: dict[str, Any], previous_hash: str | None) -> str:
44
+ payload = {"chain_version": AUDIT_CHAIN_VERSION, "previous_hash": previous_hash,
45
+ "record": {key: value for key, value in record.items() if key != AUDIT_FIELD}}
46
+ return hashlib.sha256(canonical_json(payload)).hexdigest()
47
+
48
+ def seal_audit_chain(source: Path, destination: Path) -> int:
49
+ # Validate and construct the complete snapshot before creating the destination.
50
+ # This prevents malformed input from leaving an unusable partial sealed file.
51
+ records: list[dict[str, Any]] = []
52
+ for line_number, line in enumerate(source.read_text(encoding="utf-8").splitlines(), 1):
53
+ if not line.strip():
54
+ continue
55
+ try:
56
+ record = json.loads(line)
57
+ except json.JSONDecodeError as exc:
58
+ raise ValueError(f"line {line_number}: invalid JSON: {exc.msg}") from exc
59
+ if not isinstance(record, dict) or AUDIT_FIELD in record:
60
+ raise ValueError(f"line {line_number}: expected an unsealed JSON object")
61
+ records.append(record)
62
+ if not records:
63
+ raise ValueError("cannot seal an empty JSONL file")
64
+ sealed: list[str] = []
65
+ previous_hash: str | None = None
66
+ for record in records:
67
+ digest = audit_record_hash(record, previous_hash)
68
+ item = dict(record)
69
+ item[AUDIT_FIELD] = {"version": AUDIT_CHAIN_VERSION, "previous_hash": previous_hash, "record_hash": digest}
70
+ sealed.append(json.dumps(item, ensure_ascii=False, sort_keys=True))
71
+ previous_hash = digest
72
+ with destination.open("x", encoding="utf-8", newline="\n") as output:
73
+ output.write("\n".join(sealed) + "\n")
74
+ return len(sealed)
75
+
76
+ def verify_audit_chain(path: Path, expected_head_hash: str | None = None) -> dict[str, Any]:
77
+ if expected_head_hash is not None and not re.fullmatch(r"[0-9a-f]{64}", expected_head_hash):
78
+ raise ValueError("expected head hash must be a 64-character lowercase SHA-256 hex digest")
79
+ previous_hash: str | None = None
80
+ count = 0
81
+ for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
82
+ if not line.strip():
83
+ continue
84
+ try:
85
+ record = json.loads(line)
86
+ except json.JSONDecodeError as exc:
87
+ raise ValueError(f"line {line_number}: invalid JSON: {exc.msg}") from exc
88
+ if not isinstance(record, dict):
89
+ raise ValueError(f"line {line_number}: audit record must be an object")
90
+ audit = record.get(AUDIT_FIELD)
91
+ if (not isinstance(audit, dict) or set(audit) != {"version", "previous_hash", "record_hash"}
92
+ or audit.get("version") != AUDIT_CHAIN_VERSION):
93
+ raise ValueError(f"line {line_number}: missing or unsupported audit metadata")
94
+ if audit.get("previous_hash") != previous_hash:
95
+ raise ValueError(f"line {line_number}: audit chain link mismatch")
96
+ expected = audit_record_hash(record, previous_hash)
97
+ if not isinstance(audit.get("record_hash"), str) or audit["record_hash"] != expected:
98
+ raise ValueError(f"line {line_number}: audit record hash mismatch")
99
+ previous_hash = expected
100
+ count += 1
101
+ if count == 0:
102
+ raise ValueError("audit chain contains no records")
103
+ if expected_head_hash is not None and previous_hash != expected_head_hash:
104
+ raise ValueError("audit chain head hash does not match the external anchor")
105
+ return {"verified": True, "records": count, "chain_version": AUDIT_CHAIN_VERSION, "head_hash": previous_hash, "external_anchor_checked": expected_head_hash is not None}
106
+
107
+ def load_cases(path: Path) -> dict[str, dict[str, Any]]:
108
+ data = json.loads(path.read_text(encoding="utf-8"))
109
+ if not isinstance(data, dict) or not isinstance(data.get("cases"), list):
110
+ raise ValueError("case file must contain a 'cases' array")
111
+ cases: dict[str, dict[str, Any]] = {}
112
+ for item in data["cases"]:
113
+ if not isinstance(item, dict) or not isinstance(item.get("id"), str):
114
+ raise ValueError("each case must be an object with a string id")
115
+ case_id = item["id"]
116
+ if not case_id or case_id in cases:
117
+ raise ValueError(f"empty or duplicate case id: {case_id!r}")
118
+ for field in ("required_outputs", "guardrails"):
119
+ if not isinstance(item.get(field), list) or not item[field]:
120
+ raise ValueError(f"case {case_id!r} must have a non-empty {field} array")
121
+ weight = item.get("evaluation_weight", 1.0)
122
+ if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not 0 < weight < float("inf"):
123
+ raise ValueError(f"case {case_id!r} evaluation_weight must be a finite positive number")
124
+ stratum = item.get("evaluation_stratum", "default")
125
+ if not isinstance(stratum, str) or not stratum.strip():
126
+ raise ValueError(f"case {case_id!r} evaluation_stratum must be a non-empty string")
127
+ cases[case_id] = item
128
+ return cases
129
+
130
+
131
+ def validate_tool_trace(trace: Any, line_number: int = 0) -> None:
132
+ prefix = f"line {line_number}: " if line_number else ""
133
+ if not isinstance(trace, list):
134
+ raise ValueError(prefix + "tool_trace must be an array")
135
+ allowed = {"sequence", "event_id", "session_id", "turn_id", "turn_sequence", "parent_event_id", "tool", "operation", "authorization", "tenant_ref", "resource_ref", "approval_ref", "side_effect", "outcome"}
136
+ seen_events: dict[str, dict[str, Any]] = {}
137
+ previous_turn = 0
138
+ for index, event in enumerate(trace):
139
+ if not isinstance(event, dict) or set(event) - allowed:
140
+ raise ValueError(prefix + f"tool_trace event {index} must use the documented fields only")
141
+ if type(event.get("sequence")) is not int or event["sequence"] < 1:
142
+ raise ValueError(prefix + f"tool_trace event {index} requires a positive integer sequence")
143
+ for field in ("event_id", "session_id", "turn_id", "tool", "operation", "authorization", "outcome"):
144
+ if not isinstance(event.get(field), str) or not event[field].strip():
145
+ raise ValueError(prefix + f"tool_trace event {index} requires non-empty {field}")
146
+ if event.get("event_id") in seen_events:
147
+ raise ValueError(prefix + f"tool_trace event {index} has duplicate event_id")
148
+ if type(event.get("turn_sequence")) is not int or event["turn_sequence"] < 1:
149
+ raise ValueError(prefix + f"tool_trace event {index} requires a positive turn_sequence")
150
+ if event["turn_sequence"] < previous_turn:
151
+ raise ValueError(prefix + f"tool_trace event {index} moves backwards across turns")
152
+ previous_turn = event["turn_sequence"]
153
+ parent_id = event.get("parent_event_id")
154
+ if parent_id is not None and (parent_id not in seen_events or seen_events[parent_id]["session_id"] != event["session_id"]):
155
+ raise ValueError(prefix + f"tool_trace event {index} has an unknown or cross-session parent_event_id")
156
+ approval_id = event.get("approval_ref")
157
+ if approval_id is not None and (approval_id not in seen_events or seen_events[approval_id]["operation"] != "approval" or seen_events[approval_id]["outcome"] != "approved" or seen_events[approval_id]["session_id"] != event["session_id"]):
158
+ raise ValueError(prefix + f"tool_trace event {index} has no preceding approved approval_ref")
159
+ if event["authorization"] not in ("allowed", "denied", "not-applicable", "unknown"):
160
+ raise ValueError(prefix + f"tool_trace event {index} has invalid authorization")
161
+ if event.get("side_effect", "none") not in ("none", "read", "write", "external"):
162
+ raise ValueError(prefix + f"tool_trace event {index} has invalid side_effect")
163
+ for field in ("tenant_ref", "resource_ref", "approval_ref", "parent_event_id"):
164
+ if field in event and (not isinstance(event[field], str) or not event[field].strip()):
165
+ raise ValueError(prefix + f"tool_trace event {index} has invalid {field}")
166
+ seen_events[event["event_id"]] = event
167
+ sequences = [event["sequence"] for event in trace]
168
+ if sequences != sorted(set(sequences)):
169
+ raise ValueError(prefix + "tool_trace sequence values must be unique and increasing")
170
+
171
+
172
+ def read_responses(
173
+ path: Path,
174
+ cases: dict[str, dict[str, Any]],
175
+ expected_run_id: str | None = None,
176
+ expected_manifest_sha256: str | None = None,
177
+ ) -> list[dict[str, Any]]:
178
+ records: list[dict[str, Any]] = []
179
+ seen: set[tuple[str, str]] = set()
180
+ for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
181
+ if not line.strip():
182
+ continue
183
+ try:
184
+ item = json.loads(line)
185
+ except json.JSONDecodeError as exc:
186
+ raise ValueError(f"line {line_number}: invalid JSON: {exc.msg}") from exc
187
+ if not isinstance(item, dict):
188
+ raise ValueError(f"line {line_number}: response must be a JSON object")
189
+ case_id, model, output = item.get("case_id"), item.get("model"), item.get("output")
190
+ if expected_run_id is not None or expected_manifest_sha256 is not None:
191
+ if item.get("run_id") != expected_run_id:
192
+ raise ValueError(f"line {line_number}: response run_id does not match the supplied manifest")
193
+ if item.get("manifest_sha256") != expected_manifest_sha256:
194
+ raise ValueError(f"line {line_number}: response manifest_sha256 does not match the supplied manifest")
195
+ if case_id not in cases:
196
+ raise ValueError(f"line {line_number}: unknown case_id {case_id!r}")
197
+ if not isinstance(model, str) or not model.strip():
198
+ raise ValueError(f"line {line_number}: model must be a non-empty string")
199
+ if not isinstance(output, str) or not output.strip():
200
+ raise ValueError(f"line {line_number}: output must be a non-empty string")
201
+ key = (case_id, model)
202
+ if key in seen:
203
+ raise ValueError(f"line {line_number}: duplicate response for {case_id!r}/{model!r}")
204
+ seen.add(key)
205
+ review = item.get("review")
206
+ trace = item.get("tool_trace")
207
+ review_history = item.get("review_history")
208
+ review_provenance = None
209
+ review_panel = None
210
+ if "review_submissions" in item:
211
+ review_panel = validate_review_panel(item["review_submissions"], item.get("adjudication"), cases[case_id], validate_review, line_number)
212
+ if trace is not None:
213
+ validate_tool_trace(trace, line_number)
214
+ if review is not None:
215
+ validate_review(review, cases[case_id], line_number)
216
+ if review_history is not None:
217
+ review_provenance = validate_review_history(review_history, review, line_number)
218
+ findings = review.get("tool_trace_findings", [])
219
+ if not isinstance(findings, list):
220
+ raise ValueError(f"line {line_number}: review.tool_trace_findings must be an array")
221
+ known_event_ids = {event["event_id"] for event in (trace or [])}
222
+ for finding in findings:
223
+ if (not isinstance(finding, dict)
224
+ or set(finding) != {"event_id", "guardrail_index", "outcome"}
225
+ or finding.get("event_id") not in known_event_ids
226
+ or type(finding.get("guardrail_index")) is not int
227
+ or not 0 <= finding["guardrail_index"] < len(cases[case_id]["guardrails"])
228
+ or finding.get("outcome") not in ("violation", "compliant")):
229
+ raise ValueError(f"line {line_number}: invalid tool_trace_findings entry")
230
+ trace_scores = review.get("tool_trace_guardrails")
231
+ if trace_scores is not None:
232
+ if not isinstance(trace_scores, list) or len(trace_scores) != len(cases[case_id]["guardrails"]):
233
+ raise ValueError(f"line {line_number}: review.tool_trace_guardrails must assess every guardrail in order")
234
+ if any(type(value) is not bool for value in trace_scores):
235
+ raise ValueError(f"line {line_number}: tool trace guardrail assessments must be booleans")
236
+ records.append({"case_id": case_id, "model": model.strip(), "output": output, "review": review, "tool_trace": trace, "run_id": item.get("run_id"), "manifest_sha256": item.get("manifest_sha256"), "review_provenance": review_provenance, "review_panel": review_panel})
237
+ return records
238
+
239
+
240
+ def validate_review(review: Any, case: dict[str, Any], line_number: int = 0) -> None:
241
+ prefix = f"line {line_number}: " if line_number else ""
242
+ if not isinstance(review, dict):
243
+ raise ValueError(prefix + "review must be an object")
244
+ scores = review.get("required_outputs")
245
+ guards = review.get("guardrails")
246
+ if not isinstance(scores, list) or len(scores) != len(case["required_outputs"]):
247
+ raise ValueError(prefix + "review.required_outputs must score every required output in order")
248
+ if any(type(score) is not int or score not in (0, 1, 2) for score in scores):
249
+ raise ValueError(prefix + "required output scores must be integers: 0, 1, or 2")
250
+ if not isinstance(guards, list) or len(guards) != len(case["guardrails"]):
251
+ raise ValueError(prefix + "review.guardrails must assess every guardrail in order")
252
+ if any(type(passed) is not bool for passed in guards):
253
+ raise ValueError(prefix + "guardrail assessments must be booleans")
254
+
255
+
256
+ def summarize(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
257
+ grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
258
+ for record in records:
259
+ grouped[record["model"]].append(record)
260
+
261
+ models: dict[str, Any] = {}
262
+ for model, items in sorted(grouped.items()):
263
+ reviewed = [item for item in items if item["review"] is not None]
264
+ output_points = sum(sum(item["review"]["required_outputs"]) for item in reviewed)
265
+ output_max = 2 * sum(len(cases[item["case_id"]]["required_outputs"]) for item in reviewed)
266
+ guard_passes = sum(sum(item["review"]["guardrails"]) for item in reviewed)
267
+ guard_total = sum(len(cases[item["case_id"]]["guardrails"]) for item in reviewed)
268
+ present_case_ids = {item["case_id"] for item in items}
269
+ case_results: dict[str, Any] = {}
270
+ for item in items:
271
+ case = cases[item["case_id"]]
272
+ review = item["review"]
273
+ case_results[item["case_id"]] = {
274
+ "reviewed": review is not None,
275
+ "rubric_sha256": hashlib.sha256(canonical_json({"case_id": item["case_id"], "required_outputs": case["required_outputs"], "guardrails": case["guardrails"], "security_categories": case.get("security_categories", [])})).hexdigest(),
276
+ "review_provenance": item.get("review_provenance"),
277
+ "review_provenance_status": "verified" if item.get("review_provenance") else ("unattested" if review is not None else "unreviewed"),
278
+ "review_panel": item.get("review_panel"),
279
+ "run_id": item.get("run_id"),
280
+ "manifest_sha256": item.get("manifest_sha256"),
281
+ "tool_trace": item.get("tool_trace"),
282
+ "tool_trace_reviewed": review is not None and review.get("tool_trace_guardrails") is not None,
283
+ "tool_trace_findings": review.get("tool_trace_findings", []) if review is not None else [],
284
+ "tool_trace_guardrails": [
285
+ {"guardrail": guardrail, "passed": review["tool_trace_guardrails"][index]}
286
+ for index, guardrail in enumerate(case["guardrails"])
287
+ ] if review is not None and review.get("tool_trace_guardrails") is not None else [],
288
+ "required_outputs": [
289
+ {"criterion": criterion, "score": review["required_outputs"][index]}
290
+ for index, criterion in enumerate(case["required_outputs"])
291
+ ] if review is not None else [],
292
+ "guardrails": [
293
+ {"guardrail": guardrail, "passed": review["guardrails"][index]}
294
+ for index, guardrail in enumerate(case["guardrails"])
295
+ ] if review is not None else [],
296
+ }
297
+ models[model] = {
298
+ "responses": len(items),
299
+ "fixture_coverage": round(len(present_case_ids) / len(cases), 4) if cases else 0.0,
300
+ "missing_case_ids": sorted(set(cases) - present_case_ids),
301
+ "reviewed_responses": len(reviewed),
302
+ "review_coverage": round(len(reviewed) / len(items), 4) if items else 0.0,
303
+ "required_output_score": round(output_points / output_max, 4) if output_max else None,
304
+ "guardrail_pass_rate": round(guard_passes / guard_total, 4) if guard_total else None,
305
+ "unreviewed_case_ids": sorted(item["case_id"] for item in items if item["review"] is None),
306
+ "case_results": case_results,
307
+ }
308
+ calibration_panels = [{"submissions": item["review_panel"]["submissions"], "case": cases[item["case_id"]]} for item in records if item.get("review_panel")]
309
+ report_reviewer_metrics = aggregate_reviewer_calibration(calibration_panels)
310
+ reviewer_behavior = summarize_reviewer_behavior(records, cases)
311
+ review_data_quality = summarize_review_data_quality(records, cases)
312
+ low_pairs = {(item["rubric_version"], tuple(item["reviewer_ids"])) for item in report_reviewer_metrics["low_agreement_pairs"]}
313
+ re_review_queue = []
314
+ for record in records:
315
+ panel = record.get("review_panel")
316
+ if not panel:
317
+ continue
318
+ reviewers = panel.get("submissions", [])
319
+ pair_ids = {tuple(sorted((a["reviewer_id"], b["reviewer_id"]))) for a, b in combinations(reviewers, 2)}
320
+ panel_rubric = reviewers[0]["rubric_version"] if reviewers else None
321
+ flagged_pairs = sorted([list(pair) for pair in pair_ids if (panel_rubric, pair) in low_pairs])
322
+ if panel.get("status") == "disputed" or flagged_pairs:
323
+ re_review_queue.append({"case_id": record["case_id"], "model": record["model"],
324
+ "reason": "panel_disagreement" if panel.get("status") == "disputed" else "low_historical_reviewer_agreement",
325
+ "reviewer_pairs": flagged_pairs})
326
+ all_security_categories = sorted({
327
+ category
328
+ for case in cases.values()
329
+ for category in case.get("security_categories", [])
330
+ })
331
+ security_case_ids = sorted(
332
+ case_id for case_id, case in cases.items() if case.get("security_categories")
333
+ )
334
+ for model, metrics in models.items():
335
+ violations: list[str] = []
336
+ incomplete: list[str] = []
337
+ covered_categories: set[str] = set()
338
+ for case_id in security_case_ids:
339
+ result = metrics["case_results"].get(case_id)
340
+ requires_trace = cases[case_id].get("requires_tool_trace", False)
341
+ if (result is None or not result["reviewed"]
342
+ or (requires_trace and (result.get("tool_trace") is None or not result["tool_trace_reviewed"]))):
343
+ incomplete.append(case_id)
344
+ continue
345
+ covered_categories.update(cases[case_id].get("security_categories", []))
346
+ response_failed = any(not item["passed"] for item in result["guardrails"])
347
+ trace_failed = any(not item["passed"] for item in result["tool_trace_guardrails"])
348
+ trace_findings = [finding for finding in result["tool_trace_findings"] if finding["outcome"] == "violation"]
349
+ if response_failed or trace_failed or trace_findings:
350
+ violations.append(case_id)
351
+ for finding in trace_findings:
352
+ finding["case_id"] = case_id
353
+ finding["guardrail"] = cases[case_id]["guardrails"][finding["guardrail_index"]]
354
+ metrics.setdefault("security_violation_events", []).append(finding)
355
+ metrics["security_gate"] = {
356
+ "status": "fail" if violations else ("incomplete" if incomplete else "pass"),
357
+ "hard_fail": bool(violations),
358
+ "security_case_count": len(security_case_ids),
359
+ "reviewed_security_case_count": len(security_case_ids) - len(incomplete),
360
+ "security_case_coverage": round(
361
+ (len(security_case_ids) - len(incomplete)) / len(security_case_ids), 4
362
+ ) if security_case_ids else 0.0,
363
+ "covered_categories": sorted(covered_categories),
364
+ "uncovered_categories": sorted(set(all_security_categories) - covered_categories),
365
+ "guardrail_violation_case_ids": violations,
366
+ "incomplete_case_ids": incomplete,
367
+ "violation_event_count": len(metrics.get("security_violation_events", [])),
368
+ }
369
+ comparative = comparative_analysis(records, cases)
370
+ return {
371
+ "schema_version": 1,
372
+ "method": "human_review_aggregation",
373
+ "security_gate_policy": (
374
+ "Security guardrail failures are hard failures independent of required-output "
375
+ "scores. Missing or unreviewed security cases produce an incomplete gate."
376
+ ),
377
+ "security_attack_surface": {
378
+ "categories": all_security_categories,
379
+ "security_case_ids": security_case_ids,
380
+ },
381
+ "interpretation": (
382
+ "Scores summarize supplied human annotations only. They are not an automated "
383
+ "judgment of correctness and are not comparable unless prompts, model settings, "
384
+ "tool permissions, prompt/configuration fingerprints, review rubric, and reviewer "
385
+ "calibration are held constant."
386
+ ),
387
+ "reviewer_calibration": report_reviewer_metrics,
388
+ "reviewer_behavior": reviewer_behavior,
389
+ "review_data_quality": review_data_quality,
390
+ "comparative_analysis": comparative,
391
+ "re_review_queue": re_review_queue,
392
+ "models": models,
393
+ }
394
+
395
+
396
+ def main() -> int:
397
+ parser = argparse.ArgumentParser(description=__doc__)
398
+ parser.add_argument("--cases", type=Path, default=Path("evals/system-design-cases.json"))
399
+ parser.add_argument("--responses", type=Path, required=True, help="JSONL file with model outputs and optional reviews")
400
+ parser.add_argument("--output", type=Path, help="Optional path for the JSON report")
401
+ parser.add_argument("--integrity-sidecar", type=Path, help="Create a new report integrity sidecar; requires --output")
402
+ parser.add_argument("--manifest", type=Path, help="Sealed run manifest; requires every response to carry its run_id and manifest_sha256")
403
+ parser.add_argument("--verify-chain", action="store_true", help="Require and verify a versioned SHA-256 chain on the input JSONL")
404
+ parser.add_argument("--expected-chain-head-hash", help="Optional independently retained SHA-256 head hash for detecting tail truncation or full-chain replacement")
405
+ parser.add_argument("--system-prompt-sha256", help="SHA-256 fingerprint of the evaluated system prompt")
406
+ parser.add_argument("--tool-permission-profile", help="Stable identifier/version for the tool permission profile")
407
+ parser.add_argument("--run-config-id", help="Immutable identifier for model settings and evaluation configuration")
408
+ parser.add_argument("--enforce-security-gate", action="store_true",
409
+ help="Exit non-zero unless every model has a complete, passing security review")
410
+ args = parser.parse_args()
411
+ if args.integrity_sidecar and not args.output:
412
+ parser.error("--integrity-sidecar requires --output")
413
+ try:
414
+ cases = load_cases(args.cases)
415
+ dataset_document = json.loads(args.cases.read_text(encoding="utf-8"))
416
+ catalog_fingerprint = fingerprint_dataset(dataset_document, args.cases.read_bytes())
417
+ if args.expected_chain_head_hash and not args.verify_chain:
418
+ raise ValueError("--expected-chain-head-hash requires --verify-chain")
419
+ chain_status = verify_audit_chain(args.responses, args.expected_chain_head_hash) if args.verify_chain else None
420
+ manifest = load_verified_manifest(args.manifest) if args.manifest else None
421
+ expected_run_id = manifest["run"]["run_id"] if manifest else None
422
+ expected_manifest_hash = manifest["fingerprints"]["manifest_sha256"] if manifest else None
423
+ records = read_responses(args.responses, cases, expected_run_id, expected_manifest_hash)
424
+ report = summarize(records, cases)
425
+ report["evaluation_catalog"] = {key: catalog_fingerprint[key] for key in ("catalog_schema_version", "dataset_canonical_sha256", "dataset_source_sha256", "rubric_sha256", "case_count", "case_ids")}
426
+ report["run_manifest"] = ({
427
+ "verified": True, "run_id": expected_run_id,
428
+ "manifest_sha256": expected_manifest_hash,
429
+ "configuration_sha256": manifest["fingerprints"]["configuration_sha256"],
430
+ "response_binding_verified": True,
431
+ } if manifest else {"verified": False, "status": "not-supplied"})
432
+ report["audit_chain"] = chain_status or {"verified": False, "status": "not-checked"}
433
+ report["evaluation_provenance"] = {
434
+ "model_labels": sorted({record["model"] for record in records}),
435
+ "system_prompt_sha256": args.system_prompt_sha256,
436
+ "tool_permission_profile": args.tool_permission_profile,
437
+ "run_config_id": args.run_config_id,
438
+ "provenance_complete": all((
439
+ bool(args.system_prompt_sha256),
440
+ bool(args.tool_permission_profile),
441
+ bool(args.run_config_id),
442
+ )),
443
+ }
444
+ attach_review_evidence(report, records)
445
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
446
+ print(f"ERROR: {exc}", file=sys.stderr)
447
+ return 1
448
+ rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
449
+ if args.output:
450
+ args.output.write_text(rendered, encoding="utf-8")
451
+ if args.integrity_sidecar:
452
+ try:
453
+ from .report_integrity import seal_report
454
+ except ImportError:
455
+ from report_integrity import seal_report
456
+ try:
457
+ sidecar = seal_report(args.output, args.responses, chain_status, report["run_manifest"])
458
+ with args.integrity_sidecar.open("x", encoding="utf-8") as stream:
459
+ stream.write(json.dumps(sidecar, ensure_ascii=False, sort_keys=True, indent=2) + "\n")
460
+ except (OSError, ValueError) as exc:
461
+ print(f"ERROR: cannot create report integrity sidecar: {exc}", file=sys.stderr)
462
+ return 1
463
+ else:
464
+ sys.stdout.write(rendered)
465
+ if args.enforce_security_gate and any(
466
+ metrics["security_gate"]["status"] != "pass"
467
+ for metrics in report["models"].values()
468
+ ):
469
+ print("ERROR: security gate failed or is incomplete", file=sys.stderr)
470
+ return 2
471
+ return 0
472
+
473
+
474
+ if __name__ == "__main__":
475
+ raise SystemExit(main())
@@ -0,0 +1,129 @@
1
+ #!/usr/bin/env python3
2
+ """Fingerprint evaluation datasets and classify case/rubric changes."""
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import sys
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ try:
13
+ from .evaluation_fingerprints import dataset_fingerprints
14
+ except ImportError:
15
+ from evaluation_fingerprints import dataset_fingerprints
16
+
17
+ CASE_RUBRIC_FIELDS = ("required_outputs", "guardrails", "security_categories")
18
+ CASE_CONTEXT_FIELDS = ("task", "expected_mode", "references", "requires_tool_trace")
19
+ TOP_FIELDS = {"schema_version", "description", "cases"}
20
+
21
+
22
+ def canonical_json(value: Any) -> bytes:
23
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
24
+
25
+
26
+ def sha256(data: bytes) -> str:
27
+ return hashlib.sha256(data).hexdigest()
28
+
29
+
30
+ def load_dataset(path: Path) -> dict[str, Any]:
31
+ value = json.loads(path.read_text(encoding="utf-8"))
32
+ if not isinstance(value, dict) or set(value) != TOP_FIELDS or not isinstance(value.get("cases"), list):
33
+ raise ValueError("dataset must contain exactly schema_version, description, and cases")
34
+ if type(value["schema_version"]) is not int or value["schema_version"] < 1:
35
+ raise ValueError("dataset schema_version must be a positive integer")
36
+ seen: set[str] = set()
37
+ for index, case in enumerate(value["cases"]):
38
+ if not isinstance(case, dict) or not isinstance(case.get("id"), str) or not case["id"].strip():
39
+ raise ValueError(f"case at index {index} requires a non-empty string id")
40
+ if case["id"] in seen:
41
+ raise ValueError(f"duplicate case id: {case['id']}")
42
+ seen.add(case["id"])
43
+ for field in CASE_RUBRIC_FIELDS[:2]:
44
+ if not isinstance(case.get(field), list) or not case[field]:
45
+ raise ValueError(f"case {case['id']} requires non-empty {field}")
46
+ for field in CASE_RUBRIC_FIELDS[2:]:
47
+ if field in case and (not isinstance(case[field], list) or any(not isinstance(x, str) or not x for x in case[field])):
48
+ raise ValueError(f"case {case['id']} has invalid {field}")
49
+ return value
50
+
51
+
52
+ def fingerprint_dataset(dataset: dict[str, Any], source_bytes: bytes | None = None) -> dict[str, Any]:
53
+ return dataset_fingerprints(dataset, source_bytes)
54
+
55
+
56
+ def compare_datasets(left: dict[str, Any], right: dict[str, Any]) -> dict[str, Any]:
57
+ a = {case["id"]: case for case in left["cases"]}
58
+ b = {case["id"]: case for case in right["cases"]}
59
+ added, removed = sorted(set(b) - set(a)), sorted(set(a) - set(b))
60
+ changed = []
61
+ rubric_changes = []
62
+ context_changes = []
63
+ other_changes = []
64
+ for case_id in sorted(set(a) & set(b)):
65
+ fields = sorted(set(a[case_id]) | set(b[case_id]))
66
+ for field in fields:
67
+ av, bv = a[case_id].get(field), b[case_id].get(field)
68
+ if av == bv:
69
+ continue
70
+ item = {"case_id": case_id, "field": field, "left": av, "right": bv}
71
+ changed.append(item)
72
+ if field in CASE_RUBRIC_FIELDS:
73
+ rubric_changes.append(item)
74
+ elif field in CASE_CONTEXT_FIELDS:
75
+ context_changes.append(item)
76
+ else:
77
+ other_changes.append(item)
78
+ scope_changed = bool(added or removed)
79
+ strictly_comparable = not (scope_changed or rubric_changes or context_changes)
80
+ if strictly_comparable:
81
+ classification = "same_evaluation_content"
82
+ elif rubric_changes:
83
+ classification = "rubric_changed"
84
+ elif scope_changed:
85
+ classification = "case_scope_changed"
86
+ else:
87
+ classification = "prompt_or_context_changed"
88
+ return {
89
+ "strictly_comparable": strictly_comparable,
90
+ "classification": classification,
91
+ "left_case_count": len(a),
92
+ "right_case_count": len(b),
93
+ "added_case_ids": added,
94
+ "removed_case_ids": removed,
95
+ "rubric_changes": rubric_changes,
96
+ "prompt_context_changes": context_changes,
97
+ "other_case_metadata_changes": other_changes,
98
+ "note": "Any case-scope, rubric, task, mode, reference, or tool-trace requirement change prevents strict score comparability. Other metadata changes are disclosed but do not automatically block comparison.",
99
+ }
100
+
101
+
102
+ def read(path: Path) -> dict[str, Any]:
103
+ return load_dataset(path)
104
+
105
+
106
+ def main() -> int:
107
+ parser = argparse.ArgumentParser(description=__doc__)
108
+ sub = parser.add_subparsers(dest="command", required=True)
109
+ fp = sub.add_parser("fingerprint")
110
+ fp.add_argument("--dataset", type=Path, required=True)
111
+ compare = sub.add_parser("compare")
112
+ compare.add_argument("--left", type=Path, required=True)
113
+ compare.add_argument("--right", type=Path, required=True)
114
+ args = parser.parse_args()
115
+ try:
116
+ if args.command == "fingerprint":
117
+ raw = args.dataset.read_bytes()
118
+ result = fingerprint_dataset(load_dataset(args.dataset), raw)
119
+ else:
120
+ result = compare_datasets(read(args.left), read(args.right))
121
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
122
+ print(f"ERROR: {exc}", file=sys.stderr)
123
+ return 1
124
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True, indent=2))
125
+ return 0
126
+
127
+
128
+ if __name__ == "__main__":
129
+ raise SystemExit(main())
@@ -0,0 +1,20 @@
1
+ """Canonical fingerprints shared by evaluation report tooling."""
2
+ from __future__ import annotations
3
+ import hashlib, json
4
+ from typing import Any
5
+
6
+ def canonical_json(value: Any) -> bytes:
7
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
8
+
9
+ def dataset_fingerprints(dataset: dict[str, Any], source_bytes: bytes | None = None) -> dict[str, Any]:
10
+ cases = dataset["cases"]
11
+ rubric = [{"id": c["id"], "required_outputs": c.get("required_outputs", []),
12
+ "guardrails": c.get("guardrails", []), "security_categories": c.get("security_categories", [])}
13
+ for c in sorted(cases, key=lambda x: x["id"])]
14
+ return {
15
+ "catalog_schema_version": 1, "dataset_schema_version": dataset["schema_version"],
16
+ "case_count": len(cases), "case_ids": sorted(c["id"] for c in cases),
17
+ "dataset_canonical_sha256": hashlib.sha256(canonical_json(dataset)).hexdigest(),
18
+ "dataset_source_sha256": hashlib.sha256(source_bytes).hexdigest() if source_bytes is not None else None,
19
+ "rubric_sha256": hashlib.sha256(canonical_json(rubric)).hexdigest(),
20
+ }