system-design-primer-skill 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/SKILL.md +22 -2
- package/VERSION +1 -1
- package/evals/README.md +1022 -0
- package/evals/system-design-cases.json +414 -0
- package/package.json +3 -2
- package/references/03-data-and-storage.md +8 -2
- package/references/04-caching.md +3 -3
- package/references/05-async-and-messaging.md +4 -0
- package/references/06-distributed-systems.md +2 -2
- package/references/07-reliability-and-resilience.md +7 -3
- package/references/08-api-security-observability.md +4 -0
- package/references/15-failure-analysis.md +4 -0
- package/references/19-agent-security-and-permissions.md +22 -6
- package/references/20-agent-state-and-memory.md +4 -0
- package/references/22-kubernetes-and-container-platforms.md +1 -1
- package/references/24-multi-region-and-disaster-recovery.md +6 -2
- package/references/27-production-observability-and-slo.md +4 -0
- package/references/28-event-streaming-and-kafka-pulsar.md +6 -0
- package/references/32-workflow-orchestration-and-temporal.md +4 -0
- package/references/34-postgresql-internals-and-performance.md +2 -0
- package/references/35-mysql-innodb-and-performance.md +2 -0
- package/references/36-sharding-and-distributed-sql.md +1 -1
- package/references/37-consensus-and-distributed-transactions.md +7 -3
- package/references/40-zero-downtime-database-migrations.md +3 -3
- package/references/48-oauth-oidc-and-passkeys.md +3 -3
- package/references/49-iam-rbac-abac-rebac.md +7 -3
- package/references/51-secrets-kms-pki-and-key-management.md +4 -0
- package/references/52-supply-chain-security-sbom-slsa.md +62 -0
- package/references/54-privacy-engineering-and-data-governance.md +4 -0
- package/references/55-opentelemetry-and-instrumentation.md +4 -0
- package/references/56-distributed-tracing-and-context-propagation.md +4 -0
- package/references/59-capacity-planning-and-performance-modeling.md +8 -4
- package/references/60-chaos-engineering-and-resilience-experiments.md +4 -0
- package/references/61-incident-management-and-postmortems.md +4 -0
- package/references/67-storage-infrastructure-and-storage-systems.md +4 -0
- package/references/68-tcp-quic-and-network-performance.md +1 -1
- package/references/69-task-routing-and-evidence.md +75 -18
- package/scripts/audit_chain.py +36 -0
- package/scripts/compare_reports.py +43 -0
- package/scripts/evaluate.py +475 -0
- package/scripts/evaluation_catalog.py +129 -0
- package/scripts/evaluation_fingerprints.py +20 -0
- package/scripts/evidence_archive.py +79 -0
- package/scripts/evidence_signing.py +98 -0
- package/scripts/evidence_timestamp.py +441 -0
- package/scripts/install.ps1 +2 -2
- package/scripts/install.sh +2 -2
- package/scripts/report_integrity.py +184 -0
- package/scripts/review_consensus.py +77 -0
- package/scripts/review_evidence.py +105 -0
- package/scripts/review_provenance.py +44 -0
- package/scripts/reviewer_calibration.py +215 -0
- package/scripts/rubric_compatibility.py +134 -0
- package/scripts/run_evals.py +130 -0
- package/scripts/run_manifest.py +179 -0
- package/scripts/statistical_analysis.py +308 -0
- package/scripts/tsa_attestation.py +709 -0
- package/scripts/tsa_validation_report.py +627 -0
- package/scripts/validate.py +233 -21
- package/scripts/verify_build_provenance.py +625 -0
- package/scripts/verify_tsa_ocsp.py +962 -0
- package/scripts/verify_tsa_revocation.py +782 -0
- package/scripts/windows_broker_protocol.py +274 -0
- package/scripts/windows_broker_transport.py +141 -0
- package/scripts/windows_broker_win32.py +713 -0
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Aggregate human-reviewed model responses against the system-design eval fixtures.
|
|
3
|
+
|
|
4
|
+
This tool deliberately does not infer answer quality from keyword matching. It validates
|
|
5
|
+
review annotations and reports criterion coverage, output scores, and guardrail pass rates.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
from itertools import combinations
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
from .run_manifest import load_verified_manifest
|
|
21
|
+
from .review_provenance import validate_review_history
|
|
22
|
+
from .review_consensus import validate_review_panel
|
|
23
|
+
from .reviewer_calibration import aggregate_reviewer_calibration, summarize_reviewer_behavior, summarize_review_data_quality
|
|
24
|
+
from .statistical_analysis import comparative_analysis
|
|
25
|
+
from .evaluation_catalog import fingerprint_dataset
|
|
26
|
+
from .review_evidence import attach_review_evidence
|
|
27
|
+
except ImportError: # Direct execution: python scripts/evaluate.py
|
|
28
|
+
from run_manifest import load_verified_manifest
|
|
29
|
+
from review_provenance import validate_review_history
|
|
30
|
+
from review_consensus import validate_review_panel
|
|
31
|
+
from reviewer_calibration import aggregate_reviewer_calibration, summarize_reviewer_behavior, summarize_review_data_quality
|
|
32
|
+
from statistical_analysis import comparative_analysis
|
|
33
|
+
from evaluation_catalog import fingerprint_dataset
|
|
34
|
+
from review_evidence import attach_review_evidence
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
AUDIT_CHAIN_VERSION = 1
|
|
38
|
+
AUDIT_FIELD = "_audit"
|
|
39
|
+
|
|
40
|
+
def canonical_json(value: Any) -> bytes:
|
|
41
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
42
|
+
|
|
43
|
+
def audit_record_hash(record: dict[str, Any], previous_hash: str | None) -> str:
|
|
44
|
+
payload = {"chain_version": AUDIT_CHAIN_VERSION, "previous_hash": previous_hash,
|
|
45
|
+
"record": {key: value for key, value in record.items() if key != AUDIT_FIELD}}
|
|
46
|
+
return hashlib.sha256(canonical_json(payload)).hexdigest()
|
|
47
|
+
|
|
48
|
+
def seal_audit_chain(source: Path, destination: Path) -> int:
|
|
49
|
+
# Validate and construct the complete snapshot before creating the destination.
|
|
50
|
+
# This prevents malformed input from leaving an unusable partial sealed file.
|
|
51
|
+
records: list[dict[str, Any]] = []
|
|
52
|
+
for line_number, line in enumerate(source.read_text(encoding="utf-8").splitlines(), 1):
|
|
53
|
+
if not line.strip():
|
|
54
|
+
continue
|
|
55
|
+
try:
|
|
56
|
+
record = json.loads(line)
|
|
57
|
+
except json.JSONDecodeError as exc:
|
|
58
|
+
raise ValueError(f"line {line_number}: invalid JSON: {exc.msg}") from exc
|
|
59
|
+
if not isinstance(record, dict) or AUDIT_FIELD in record:
|
|
60
|
+
raise ValueError(f"line {line_number}: expected an unsealed JSON object")
|
|
61
|
+
records.append(record)
|
|
62
|
+
if not records:
|
|
63
|
+
raise ValueError("cannot seal an empty JSONL file")
|
|
64
|
+
sealed: list[str] = []
|
|
65
|
+
previous_hash: str | None = None
|
|
66
|
+
for record in records:
|
|
67
|
+
digest = audit_record_hash(record, previous_hash)
|
|
68
|
+
item = dict(record)
|
|
69
|
+
item[AUDIT_FIELD] = {"version": AUDIT_CHAIN_VERSION, "previous_hash": previous_hash, "record_hash": digest}
|
|
70
|
+
sealed.append(json.dumps(item, ensure_ascii=False, sort_keys=True))
|
|
71
|
+
previous_hash = digest
|
|
72
|
+
with destination.open("x", encoding="utf-8", newline="\n") as output:
|
|
73
|
+
output.write("\n".join(sealed) + "\n")
|
|
74
|
+
return len(sealed)
|
|
75
|
+
|
|
76
|
+
def verify_audit_chain(path: Path, expected_head_hash: str | None = None) -> dict[str, Any]:
|
|
77
|
+
if expected_head_hash is not None and not re.fullmatch(r"[0-9a-f]{64}", expected_head_hash):
|
|
78
|
+
raise ValueError("expected head hash must be a 64-character lowercase SHA-256 hex digest")
|
|
79
|
+
previous_hash: str | None = None
|
|
80
|
+
count = 0
|
|
81
|
+
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
|
82
|
+
if not line.strip():
|
|
83
|
+
continue
|
|
84
|
+
try:
|
|
85
|
+
record = json.loads(line)
|
|
86
|
+
except json.JSONDecodeError as exc:
|
|
87
|
+
raise ValueError(f"line {line_number}: invalid JSON: {exc.msg}") from exc
|
|
88
|
+
if not isinstance(record, dict):
|
|
89
|
+
raise ValueError(f"line {line_number}: audit record must be an object")
|
|
90
|
+
audit = record.get(AUDIT_FIELD)
|
|
91
|
+
if (not isinstance(audit, dict) or set(audit) != {"version", "previous_hash", "record_hash"}
|
|
92
|
+
or audit.get("version") != AUDIT_CHAIN_VERSION):
|
|
93
|
+
raise ValueError(f"line {line_number}: missing or unsupported audit metadata")
|
|
94
|
+
if audit.get("previous_hash") != previous_hash:
|
|
95
|
+
raise ValueError(f"line {line_number}: audit chain link mismatch")
|
|
96
|
+
expected = audit_record_hash(record, previous_hash)
|
|
97
|
+
if not isinstance(audit.get("record_hash"), str) or audit["record_hash"] != expected:
|
|
98
|
+
raise ValueError(f"line {line_number}: audit record hash mismatch")
|
|
99
|
+
previous_hash = expected
|
|
100
|
+
count += 1
|
|
101
|
+
if count == 0:
|
|
102
|
+
raise ValueError("audit chain contains no records")
|
|
103
|
+
if expected_head_hash is not None and previous_hash != expected_head_hash:
|
|
104
|
+
raise ValueError("audit chain head hash does not match the external anchor")
|
|
105
|
+
return {"verified": True, "records": count, "chain_version": AUDIT_CHAIN_VERSION, "head_hash": previous_hash, "external_anchor_checked": expected_head_hash is not None}
|
|
106
|
+
|
|
107
|
+
def load_cases(path: Path) -> dict[str, dict[str, Any]]:
|
|
108
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
109
|
+
if not isinstance(data, dict) or not isinstance(data.get("cases"), list):
|
|
110
|
+
raise ValueError("case file must contain a 'cases' array")
|
|
111
|
+
cases: dict[str, dict[str, Any]] = {}
|
|
112
|
+
for item in data["cases"]:
|
|
113
|
+
if not isinstance(item, dict) or not isinstance(item.get("id"), str):
|
|
114
|
+
raise ValueError("each case must be an object with a string id")
|
|
115
|
+
case_id = item["id"]
|
|
116
|
+
if not case_id or case_id in cases:
|
|
117
|
+
raise ValueError(f"empty or duplicate case id: {case_id!r}")
|
|
118
|
+
for field in ("required_outputs", "guardrails"):
|
|
119
|
+
if not isinstance(item.get(field), list) or not item[field]:
|
|
120
|
+
raise ValueError(f"case {case_id!r} must have a non-empty {field} array")
|
|
121
|
+
weight = item.get("evaluation_weight", 1.0)
|
|
122
|
+
if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not 0 < weight < float("inf"):
|
|
123
|
+
raise ValueError(f"case {case_id!r} evaluation_weight must be a finite positive number")
|
|
124
|
+
stratum = item.get("evaluation_stratum", "default")
|
|
125
|
+
if not isinstance(stratum, str) or not stratum.strip():
|
|
126
|
+
raise ValueError(f"case {case_id!r} evaluation_stratum must be a non-empty string")
|
|
127
|
+
cases[case_id] = item
|
|
128
|
+
return cases
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def validate_tool_trace(trace: Any, line_number: int = 0) -> None:
|
|
132
|
+
prefix = f"line {line_number}: " if line_number else ""
|
|
133
|
+
if not isinstance(trace, list):
|
|
134
|
+
raise ValueError(prefix + "tool_trace must be an array")
|
|
135
|
+
allowed = {"sequence", "event_id", "session_id", "turn_id", "turn_sequence", "parent_event_id", "tool", "operation", "authorization", "tenant_ref", "resource_ref", "approval_ref", "side_effect", "outcome"}
|
|
136
|
+
seen_events: dict[str, dict[str, Any]] = {}
|
|
137
|
+
previous_turn = 0
|
|
138
|
+
for index, event in enumerate(trace):
|
|
139
|
+
if not isinstance(event, dict) or set(event) - allowed:
|
|
140
|
+
raise ValueError(prefix + f"tool_trace event {index} must use the documented fields only")
|
|
141
|
+
if type(event.get("sequence")) is not int or event["sequence"] < 1:
|
|
142
|
+
raise ValueError(prefix + f"tool_trace event {index} requires a positive integer sequence")
|
|
143
|
+
for field in ("event_id", "session_id", "turn_id", "tool", "operation", "authorization", "outcome"):
|
|
144
|
+
if not isinstance(event.get(field), str) or not event[field].strip():
|
|
145
|
+
raise ValueError(prefix + f"tool_trace event {index} requires non-empty {field}")
|
|
146
|
+
if event.get("event_id") in seen_events:
|
|
147
|
+
raise ValueError(prefix + f"tool_trace event {index} has duplicate event_id")
|
|
148
|
+
if type(event.get("turn_sequence")) is not int or event["turn_sequence"] < 1:
|
|
149
|
+
raise ValueError(prefix + f"tool_trace event {index} requires a positive turn_sequence")
|
|
150
|
+
if event["turn_sequence"] < previous_turn:
|
|
151
|
+
raise ValueError(prefix + f"tool_trace event {index} moves backwards across turns")
|
|
152
|
+
previous_turn = event["turn_sequence"]
|
|
153
|
+
parent_id = event.get("parent_event_id")
|
|
154
|
+
if parent_id is not None and (parent_id not in seen_events or seen_events[parent_id]["session_id"] != event["session_id"]):
|
|
155
|
+
raise ValueError(prefix + f"tool_trace event {index} has an unknown or cross-session parent_event_id")
|
|
156
|
+
approval_id = event.get("approval_ref")
|
|
157
|
+
if approval_id is not None and (approval_id not in seen_events or seen_events[approval_id]["operation"] != "approval" or seen_events[approval_id]["outcome"] != "approved" or seen_events[approval_id]["session_id"] != event["session_id"]):
|
|
158
|
+
raise ValueError(prefix + f"tool_trace event {index} has no preceding approved approval_ref")
|
|
159
|
+
if event["authorization"] not in ("allowed", "denied", "not-applicable", "unknown"):
|
|
160
|
+
raise ValueError(prefix + f"tool_trace event {index} has invalid authorization")
|
|
161
|
+
if event.get("side_effect", "none") not in ("none", "read", "write", "external"):
|
|
162
|
+
raise ValueError(prefix + f"tool_trace event {index} has invalid side_effect")
|
|
163
|
+
for field in ("tenant_ref", "resource_ref", "approval_ref", "parent_event_id"):
|
|
164
|
+
if field in event and (not isinstance(event[field], str) or not event[field].strip()):
|
|
165
|
+
raise ValueError(prefix + f"tool_trace event {index} has invalid {field}")
|
|
166
|
+
seen_events[event["event_id"]] = event
|
|
167
|
+
sequences = [event["sequence"] for event in trace]
|
|
168
|
+
if sequences != sorted(set(sequences)):
|
|
169
|
+
raise ValueError(prefix + "tool_trace sequence values must be unique and increasing")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def read_responses(
|
|
173
|
+
path: Path,
|
|
174
|
+
cases: dict[str, dict[str, Any]],
|
|
175
|
+
expected_run_id: str | None = None,
|
|
176
|
+
expected_manifest_sha256: str | None = None,
|
|
177
|
+
) -> list[dict[str, Any]]:
|
|
178
|
+
records: list[dict[str, Any]] = []
|
|
179
|
+
seen: set[tuple[str, str]] = set()
|
|
180
|
+
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
|
181
|
+
if not line.strip():
|
|
182
|
+
continue
|
|
183
|
+
try:
|
|
184
|
+
item = json.loads(line)
|
|
185
|
+
except json.JSONDecodeError as exc:
|
|
186
|
+
raise ValueError(f"line {line_number}: invalid JSON: {exc.msg}") from exc
|
|
187
|
+
if not isinstance(item, dict):
|
|
188
|
+
raise ValueError(f"line {line_number}: response must be a JSON object")
|
|
189
|
+
case_id, model, output = item.get("case_id"), item.get("model"), item.get("output")
|
|
190
|
+
if expected_run_id is not None or expected_manifest_sha256 is not None:
|
|
191
|
+
if item.get("run_id") != expected_run_id:
|
|
192
|
+
raise ValueError(f"line {line_number}: response run_id does not match the supplied manifest")
|
|
193
|
+
if item.get("manifest_sha256") != expected_manifest_sha256:
|
|
194
|
+
raise ValueError(f"line {line_number}: response manifest_sha256 does not match the supplied manifest")
|
|
195
|
+
if case_id not in cases:
|
|
196
|
+
raise ValueError(f"line {line_number}: unknown case_id {case_id!r}")
|
|
197
|
+
if not isinstance(model, str) or not model.strip():
|
|
198
|
+
raise ValueError(f"line {line_number}: model must be a non-empty string")
|
|
199
|
+
if not isinstance(output, str) or not output.strip():
|
|
200
|
+
raise ValueError(f"line {line_number}: output must be a non-empty string")
|
|
201
|
+
key = (case_id, model)
|
|
202
|
+
if key in seen:
|
|
203
|
+
raise ValueError(f"line {line_number}: duplicate response for {case_id!r}/{model!r}")
|
|
204
|
+
seen.add(key)
|
|
205
|
+
review = item.get("review")
|
|
206
|
+
trace = item.get("tool_trace")
|
|
207
|
+
review_history = item.get("review_history")
|
|
208
|
+
review_provenance = None
|
|
209
|
+
review_panel = None
|
|
210
|
+
if "review_submissions" in item:
|
|
211
|
+
review_panel = validate_review_panel(item["review_submissions"], item.get("adjudication"), cases[case_id], validate_review, line_number)
|
|
212
|
+
if trace is not None:
|
|
213
|
+
validate_tool_trace(trace, line_number)
|
|
214
|
+
if review is not None:
|
|
215
|
+
validate_review(review, cases[case_id], line_number)
|
|
216
|
+
if review_history is not None:
|
|
217
|
+
review_provenance = validate_review_history(review_history, review, line_number)
|
|
218
|
+
findings = review.get("tool_trace_findings", [])
|
|
219
|
+
if not isinstance(findings, list):
|
|
220
|
+
raise ValueError(f"line {line_number}: review.tool_trace_findings must be an array")
|
|
221
|
+
known_event_ids = {event["event_id"] for event in (trace or [])}
|
|
222
|
+
for finding in findings:
|
|
223
|
+
if (not isinstance(finding, dict)
|
|
224
|
+
or set(finding) != {"event_id", "guardrail_index", "outcome"}
|
|
225
|
+
or finding.get("event_id") not in known_event_ids
|
|
226
|
+
or type(finding.get("guardrail_index")) is not int
|
|
227
|
+
or not 0 <= finding["guardrail_index"] < len(cases[case_id]["guardrails"])
|
|
228
|
+
or finding.get("outcome") not in ("violation", "compliant")):
|
|
229
|
+
raise ValueError(f"line {line_number}: invalid tool_trace_findings entry")
|
|
230
|
+
trace_scores = review.get("tool_trace_guardrails")
|
|
231
|
+
if trace_scores is not None:
|
|
232
|
+
if not isinstance(trace_scores, list) or len(trace_scores) != len(cases[case_id]["guardrails"]):
|
|
233
|
+
raise ValueError(f"line {line_number}: review.tool_trace_guardrails must assess every guardrail in order")
|
|
234
|
+
if any(type(value) is not bool for value in trace_scores):
|
|
235
|
+
raise ValueError(f"line {line_number}: tool trace guardrail assessments must be booleans")
|
|
236
|
+
records.append({"case_id": case_id, "model": model.strip(), "output": output, "review": review, "tool_trace": trace, "run_id": item.get("run_id"), "manifest_sha256": item.get("manifest_sha256"), "review_provenance": review_provenance, "review_panel": review_panel})
|
|
237
|
+
return records
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def validate_review(review: Any, case: dict[str, Any], line_number: int = 0) -> None:
|
|
241
|
+
prefix = f"line {line_number}: " if line_number else ""
|
|
242
|
+
if not isinstance(review, dict):
|
|
243
|
+
raise ValueError(prefix + "review must be an object")
|
|
244
|
+
scores = review.get("required_outputs")
|
|
245
|
+
guards = review.get("guardrails")
|
|
246
|
+
if not isinstance(scores, list) or len(scores) != len(case["required_outputs"]):
|
|
247
|
+
raise ValueError(prefix + "review.required_outputs must score every required output in order")
|
|
248
|
+
if any(type(score) is not int or score not in (0, 1, 2) for score in scores):
|
|
249
|
+
raise ValueError(prefix + "required output scores must be integers: 0, 1, or 2")
|
|
250
|
+
if not isinstance(guards, list) or len(guards) != len(case["guardrails"]):
|
|
251
|
+
raise ValueError(prefix + "review.guardrails must assess every guardrail in order")
|
|
252
|
+
if any(type(passed) is not bool for passed in guards):
|
|
253
|
+
raise ValueError(prefix + "guardrail assessments must be booleans")
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def summarize(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
|
257
|
+
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
258
|
+
for record in records:
|
|
259
|
+
grouped[record["model"]].append(record)
|
|
260
|
+
|
|
261
|
+
models: dict[str, Any] = {}
|
|
262
|
+
for model, items in sorted(grouped.items()):
|
|
263
|
+
reviewed = [item for item in items if item["review"] is not None]
|
|
264
|
+
output_points = sum(sum(item["review"]["required_outputs"]) for item in reviewed)
|
|
265
|
+
output_max = 2 * sum(len(cases[item["case_id"]]["required_outputs"]) for item in reviewed)
|
|
266
|
+
guard_passes = sum(sum(item["review"]["guardrails"]) for item in reviewed)
|
|
267
|
+
guard_total = sum(len(cases[item["case_id"]]["guardrails"]) for item in reviewed)
|
|
268
|
+
present_case_ids = {item["case_id"] for item in items}
|
|
269
|
+
case_results: dict[str, Any] = {}
|
|
270
|
+
for item in items:
|
|
271
|
+
case = cases[item["case_id"]]
|
|
272
|
+
review = item["review"]
|
|
273
|
+
case_results[item["case_id"]] = {
|
|
274
|
+
"reviewed": review is not None,
|
|
275
|
+
"rubric_sha256": hashlib.sha256(canonical_json({"case_id": item["case_id"], "required_outputs": case["required_outputs"], "guardrails": case["guardrails"], "security_categories": case.get("security_categories", [])})).hexdigest(),
|
|
276
|
+
"review_provenance": item.get("review_provenance"),
|
|
277
|
+
"review_provenance_status": "verified" if item.get("review_provenance") else ("unattested" if review is not None else "unreviewed"),
|
|
278
|
+
"review_panel": item.get("review_panel"),
|
|
279
|
+
"run_id": item.get("run_id"),
|
|
280
|
+
"manifest_sha256": item.get("manifest_sha256"),
|
|
281
|
+
"tool_trace": item.get("tool_trace"),
|
|
282
|
+
"tool_trace_reviewed": review is not None and review.get("tool_trace_guardrails") is not None,
|
|
283
|
+
"tool_trace_findings": review.get("tool_trace_findings", []) if review is not None else [],
|
|
284
|
+
"tool_trace_guardrails": [
|
|
285
|
+
{"guardrail": guardrail, "passed": review["tool_trace_guardrails"][index]}
|
|
286
|
+
for index, guardrail in enumerate(case["guardrails"])
|
|
287
|
+
] if review is not None and review.get("tool_trace_guardrails") is not None else [],
|
|
288
|
+
"required_outputs": [
|
|
289
|
+
{"criterion": criterion, "score": review["required_outputs"][index]}
|
|
290
|
+
for index, criterion in enumerate(case["required_outputs"])
|
|
291
|
+
] if review is not None else [],
|
|
292
|
+
"guardrails": [
|
|
293
|
+
{"guardrail": guardrail, "passed": review["guardrails"][index]}
|
|
294
|
+
for index, guardrail in enumerate(case["guardrails"])
|
|
295
|
+
] if review is not None else [],
|
|
296
|
+
}
|
|
297
|
+
models[model] = {
|
|
298
|
+
"responses": len(items),
|
|
299
|
+
"fixture_coverage": round(len(present_case_ids) / len(cases), 4) if cases else 0.0,
|
|
300
|
+
"missing_case_ids": sorted(set(cases) - present_case_ids),
|
|
301
|
+
"reviewed_responses": len(reviewed),
|
|
302
|
+
"review_coverage": round(len(reviewed) / len(items), 4) if items else 0.0,
|
|
303
|
+
"required_output_score": round(output_points / output_max, 4) if output_max else None,
|
|
304
|
+
"guardrail_pass_rate": round(guard_passes / guard_total, 4) if guard_total else None,
|
|
305
|
+
"unreviewed_case_ids": sorted(item["case_id"] for item in items if item["review"] is None),
|
|
306
|
+
"case_results": case_results,
|
|
307
|
+
}
|
|
308
|
+
calibration_panels = [{"submissions": item["review_panel"]["submissions"], "case": cases[item["case_id"]]} for item in records if item.get("review_panel")]
|
|
309
|
+
report_reviewer_metrics = aggregate_reviewer_calibration(calibration_panels)
|
|
310
|
+
reviewer_behavior = summarize_reviewer_behavior(records, cases)
|
|
311
|
+
review_data_quality = summarize_review_data_quality(records, cases)
|
|
312
|
+
low_pairs = {(item["rubric_version"], tuple(item["reviewer_ids"])) for item in report_reviewer_metrics["low_agreement_pairs"]}
|
|
313
|
+
re_review_queue = []
|
|
314
|
+
for record in records:
|
|
315
|
+
panel = record.get("review_panel")
|
|
316
|
+
if not panel:
|
|
317
|
+
continue
|
|
318
|
+
reviewers = panel.get("submissions", [])
|
|
319
|
+
pair_ids = {tuple(sorted((a["reviewer_id"], b["reviewer_id"]))) for a, b in combinations(reviewers, 2)}
|
|
320
|
+
panel_rubric = reviewers[0]["rubric_version"] if reviewers else None
|
|
321
|
+
flagged_pairs = sorted([list(pair) for pair in pair_ids if (panel_rubric, pair) in low_pairs])
|
|
322
|
+
if panel.get("status") == "disputed" or flagged_pairs:
|
|
323
|
+
re_review_queue.append({"case_id": record["case_id"], "model": record["model"],
|
|
324
|
+
"reason": "panel_disagreement" if panel.get("status") == "disputed" else "low_historical_reviewer_agreement",
|
|
325
|
+
"reviewer_pairs": flagged_pairs})
|
|
326
|
+
all_security_categories = sorted({
|
|
327
|
+
category
|
|
328
|
+
for case in cases.values()
|
|
329
|
+
for category in case.get("security_categories", [])
|
|
330
|
+
})
|
|
331
|
+
security_case_ids = sorted(
|
|
332
|
+
case_id for case_id, case in cases.items() if case.get("security_categories")
|
|
333
|
+
)
|
|
334
|
+
for model, metrics in models.items():
|
|
335
|
+
violations: list[str] = []
|
|
336
|
+
incomplete: list[str] = []
|
|
337
|
+
covered_categories: set[str] = set()
|
|
338
|
+
for case_id in security_case_ids:
|
|
339
|
+
result = metrics["case_results"].get(case_id)
|
|
340
|
+
requires_trace = cases[case_id].get("requires_tool_trace", False)
|
|
341
|
+
if (result is None or not result["reviewed"]
|
|
342
|
+
or (requires_trace and (result.get("tool_trace") is None or not result["tool_trace_reviewed"]))):
|
|
343
|
+
incomplete.append(case_id)
|
|
344
|
+
continue
|
|
345
|
+
covered_categories.update(cases[case_id].get("security_categories", []))
|
|
346
|
+
response_failed = any(not item["passed"] for item in result["guardrails"])
|
|
347
|
+
trace_failed = any(not item["passed"] for item in result["tool_trace_guardrails"])
|
|
348
|
+
trace_findings = [finding for finding in result["tool_trace_findings"] if finding["outcome"] == "violation"]
|
|
349
|
+
if response_failed or trace_failed or trace_findings:
|
|
350
|
+
violations.append(case_id)
|
|
351
|
+
for finding in trace_findings:
|
|
352
|
+
finding["case_id"] = case_id
|
|
353
|
+
finding["guardrail"] = cases[case_id]["guardrails"][finding["guardrail_index"]]
|
|
354
|
+
metrics.setdefault("security_violation_events", []).append(finding)
|
|
355
|
+
metrics["security_gate"] = {
|
|
356
|
+
"status": "fail" if violations else ("incomplete" if incomplete else "pass"),
|
|
357
|
+
"hard_fail": bool(violations),
|
|
358
|
+
"security_case_count": len(security_case_ids),
|
|
359
|
+
"reviewed_security_case_count": len(security_case_ids) - len(incomplete),
|
|
360
|
+
"security_case_coverage": round(
|
|
361
|
+
(len(security_case_ids) - len(incomplete)) / len(security_case_ids), 4
|
|
362
|
+
) if security_case_ids else 0.0,
|
|
363
|
+
"covered_categories": sorted(covered_categories),
|
|
364
|
+
"uncovered_categories": sorted(set(all_security_categories) - covered_categories),
|
|
365
|
+
"guardrail_violation_case_ids": violations,
|
|
366
|
+
"incomplete_case_ids": incomplete,
|
|
367
|
+
"violation_event_count": len(metrics.get("security_violation_events", [])),
|
|
368
|
+
}
|
|
369
|
+
comparative = comparative_analysis(records, cases)
|
|
370
|
+
return {
|
|
371
|
+
"schema_version": 1,
|
|
372
|
+
"method": "human_review_aggregation",
|
|
373
|
+
"security_gate_policy": (
|
|
374
|
+
"Security guardrail failures are hard failures independent of required-output "
|
|
375
|
+
"scores. Missing or unreviewed security cases produce an incomplete gate."
|
|
376
|
+
),
|
|
377
|
+
"security_attack_surface": {
|
|
378
|
+
"categories": all_security_categories,
|
|
379
|
+
"security_case_ids": security_case_ids,
|
|
380
|
+
},
|
|
381
|
+
"interpretation": (
|
|
382
|
+
"Scores summarize supplied human annotations only. They are not an automated "
|
|
383
|
+
"judgment of correctness and are not comparable unless prompts, model settings, "
|
|
384
|
+
"tool permissions, prompt/configuration fingerprints, review rubric, and reviewer "
|
|
385
|
+
"calibration are held constant."
|
|
386
|
+
),
|
|
387
|
+
"reviewer_calibration": report_reviewer_metrics,
|
|
388
|
+
"reviewer_behavior": reviewer_behavior,
|
|
389
|
+
"review_data_quality": review_data_quality,
|
|
390
|
+
"comparative_analysis": comparative,
|
|
391
|
+
"re_review_queue": re_review_queue,
|
|
392
|
+
"models": models,
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def main() -> int:
|
|
397
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
398
|
+
parser.add_argument("--cases", type=Path, default=Path("evals/system-design-cases.json"))
|
|
399
|
+
parser.add_argument("--responses", type=Path, required=True, help="JSONL file with model outputs and optional reviews")
|
|
400
|
+
parser.add_argument("--output", type=Path, help="Optional path for the JSON report")
|
|
401
|
+
parser.add_argument("--integrity-sidecar", type=Path, help="Create a new report integrity sidecar; requires --output")
|
|
402
|
+
parser.add_argument("--manifest", type=Path, help="Sealed run manifest; requires every response to carry its run_id and manifest_sha256")
|
|
403
|
+
parser.add_argument("--verify-chain", action="store_true", help="Require and verify a versioned SHA-256 chain on the input JSONL")
|
|
404
|
+
parser.add_argument("--expected-chain-head-hash", help="Optional independently retained SHA-256 head hash for detecting tail truncation or full-chain replacement")
|
|
405
|
+
parser.add_argument("--system-prompt-sha256", help="SHA-256 fingerprint of the evaluated system prompt")
|
|
406
|
+
parser.add_argument("--tool-permission-profile", help="Stable identifier/version for the tool permission profile")
|
|
407
|
+
parser.add_argument("--run-config-id", help="Immutable identifier for model settings and evaluation configuration")
|
|
408
|
+
parser.add_argument("--enforce-security-gate", action="store_true",
|
|
409
|
+
help="Exit non-zero unless every model has a complete, passing security review")
|
|
410
|
+
args = parser.parse_args()
|
|
411
|
+
if args.integrity_sidecar and not args.output:
|
|
412
|
+
parser.error("--integrity-sidecar requires --output")
|
|
413
|
+
try:
|
|
414
|
+
cases = load_cases(args.cases)
|
|
415
|
+
dataset_document = json.loads(args.cases.read_text(encoding="utf-8"))
|
|
416
|
+
catalog_fingerprint = fingerprint_dataset(dataset_document, args.cases.read_bytes())
|
|
417
|
+
if args.expected_chain_head_hash and not args.verify_chain:
|
|
418
|
+
raise ValueError("--expected-chain-head-hash requires --verify-chain")
|
|
419
|
+
chain_status = verify_audit_chain(args.responses, args.expected_chain_head_hash) if args.verify_chain else None
|
|
420
|
+
manifest = load_verified_manifest(args.manifest) if args.manifest else None
|
|
421
|
+
expected_run_id = manifest["run"]["run_id"] if manifest else None
|
|
422
|
+
expected_manifest_hash = manifest["fingerprints"]["manifest_sha256"] if manifest else None
|
|
423
|
+
records = read_responses(args.responses, cases, expected_run_id, expected_manifest_hash)
|
|
424
|
+
report = summarize(records, cases)
|
|
425
|
+
report["evaluation_catalog"] = {key: catalog_fingerprint[key] for key in ("catalog_schema_version", "dataset_canonical_sha256", "dataset_source_sha256", "rubric_sha256", "case_count", "case_ids")}
|
|
426
|
+
report["run_manifest"] = ({
|
|
427
|
+
"verified": True, "run_id": expected_run_id,
|
|
428
|
+
"manifest_sha256": expected_manifest_hash,
|
|
429
|
+
"configuration_sha256": manifest["fingerprints"]["configuration_sha256"],
|
|
430
|
+
"response_binding_verified": True,
|
|
431
|
+
} if manifest else {"verified": False, "status": "not-supplied"})
|
|
432
|
+
report["audit_chain"] = chain_status or {"verified": False, "status": "not-checked"}
|
|
433
|
+
report["evaluation_provenance"] = {
|
|
434
|
+
"model_labels": sorted({record["model"] for record in records}),
|
|
435
|
+
"system_prompt_sha256": args.system_prompt_sha256,
|
|
436
|
+
"tool_permission_profile": args.tool_permission_profile,
|
|
437
|
+
"run_config_id": args.run_config_id,
|
|
438
|
+
"provenance_complete": all((
|
|
439
|
+
bool(args.system_prompt_sha256),
|
|
440
|
+
bool(args.tool_permission_profile),
|
|
441
|
+
bool(args.run_config_id),
|
|
442
|
+
)),
|
|
443
|
+
}
|
|
444
|
+
attach_review_evidence(report, records)
|
|
445
|
+
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
|
446
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
447
|
+
return 1
|
|
448
|
+
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
449
|
+
if args.output:
|
|
450
|
+
args.output.write_text(rendered, encoding="utf-8")
|
|
451
|
+
if args.integrity_sidecar:
|
|
452
|
+
try:
|
|
453
|
+
from .report_integrity import seal_report
|
|
454
|
+
except ImportError:
|
|
455
|
+
from report_integrity import seal_report
|
|
456
|
+
try:
|
|
457
|
+
sidecar = seal_report(args.output, args.responses, chain_status, report["run_manifest"])
|
|
458
|
+
with args.integrity_sidecar.open("x", encoding="utf-8") as stream:
|
|
459
|
+
stream.write(json.dumps(sidecar, ensure_ascii=False, sort_keys=True, indent=2) + "\n")
|
|
460
|
+
except (OSError, ValueError) as exc:
|
|
461
|
+
print(f"ERROR: cannot create report integrity sidecar: {exc}", file=sys.stderr)
|
|
462
|
+
return 1
|
|
463
|
+
else:
|
|
464
|
+
sys.stdout.write(rendered)
|
|
465
|
+
if args.enforce_security_gate and any(
|
|
466
|
+
metrics["security_gate"]["status"] != "pass"
|
|
467
|
+
for metrics in report["models"].values()
|
|
468
|
+
):
|
|
469
|
+
print("ERROR: security gate failed or is incomplete", file=sys.stderr)
|
|
470
|
+
return 2
|
|
471
|
+
return 0
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
if __name__ == "__main__":
|
|
475
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fingerprint evaluation datasets and classify case/rubric changes."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
from .evaluation_fingerprints import dataset_fingerprints
|
|
14
|
+
except ImportError:
|
|
15
|
+
from evaluation_fingerprints import dataset_fingerprints
|
|
16
|
+
|
|
17
|
+
CASE_RUBRIC_FIELDS = ("required_outputs", "guardrails", "security_categories")
|
|
18
|
+
CASE_CONTEXT_FIELDS = ("task", "expected_mode", "references", "requires_tool_trace")
|
|
19
|
+
TOP_FIELDS = {"schema_version", "description", "cases"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def canonical_json(value: Any) -> bytes:
|
|
23
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def sha256(data: bytes) -> str:
|
|
27
|
+
return hashlib.sha256(data).hexdigest()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def load_dataset(path: Path) -> dict[str, Any]:
|
|
31
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
32
|
+
if not isinstance(value, dict) or set(value) != TOP_FIELDS or not isinstance(value.get("cases"), list):
|
|
33
|
+
raise ValueError("dataset must contain exactly schema_version, description, and cases")
|
|
34
|
+
if type(value["schema_version"]) is not int or value["schema_version"] < 1:
|
|
35
|
+
raise ValueError("dataset schema_version must be a positive integer")
|
|
36
|
+
seen: set[str] = set()
|
|
37
|
+
for index, case in enumerate(value["cases"]):
|
|
38
|
+
if not isinstance(case, dict) or not isinstance(case.get("id"), str) or not case["id"].strip():
|
|
39
|
+
raise ValueError(f"case at index {index} requires a non-empty string id")
|
|
40
|
+
if case["id"] in seen:
|
|
41
|
+
raise ValueError(f"duplicate case id: {case['id']}")
|
|
42
|
+
seen.add(case["id"])
|
|
43
|
+
for field in CASE_RUBRIC_FIELDS[:2]:
|
|
44
|
+
if not isinstance(case.get(field), list) or not case[field]:
|
|
45
|
+
raise ValueError(f"case {case['id']} requires non-empty {field}")
|
|
46
|
+
for field in CASE_RUBRIC_FIELDS[2:]:
|
|
47
|
+
if field in case and (not isinstance(case[field], list) or any(not isinstance(x, str) or not x for x in case[field])):
|
|
48
|
+
raise ValueError(f"case {case['id']} has invalid {field}")
|
|
49
|
+
return value
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def fingerprint_dataset(dataset: dict[str, Any], source_bytes: bytes | None = None) -> dict[str, Any]:
|
|
53
|
+
return dataset_fingerprints(dataset, source_bytes)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def compare_datasets(left: dict[str, Any], right: dict[str, Any]) -> dict[str, Any]:
|
|
57
|
+
a = {case["id"]: case for case in left["cases"]}
|
|
58
|
+
b = {case["id"]: case for case in right["cases"]}
|
|
59
|
+
added, removed = sorted(set(b) - set(a)), sorted(set(a) - set(b))
|
|
60
|
+
changed = []
|
|
61
|
+
rubric_changes = []
|
|
62
|
+
context_changes = []
|
|
63
|
+
other_changes = []
|
|
64
|
+
for case_id in sorted(set(a) & set(b)):
|
|
65
|
+
fields = sorted(set(a[case_id]) | set(b[case_id]))
|
|
66
|
+
for field in fields:
|
|
67
|
+
av, bv = a[case_id].get(field), b[case_id].get(field)
|
|
68
|
+
if av == bv:
|
|
69
|
+
continue
|
|
70
|
+
item = {"case_id": case_id, "field": field, "left": av, "right": bv}
|
|
71
|
+
changed.append(item)
|
|
72
|
+
if field in CASE_RUBRIC_FIELDS:
|
|
73
|
+
rubric_changes.append(item)
|
|
74
|
+
elif field in CASE_CONTEXT_FIELDS:
|
|
75
|
+
context_changes.append(item)
|
|
76
|
+
else:
|
|
77
|
+
other_changes.append(item)
|
|
78
|
+
scope_changed = bool(added or removed)
|
|
79
|
+
strictly_comparable = not (scope_changed or rubric_changes or context_changes)
|
|
80
|
+
if strictly_comparable:
|
|
81
|
+
classification = "same_evaluation_content"
|
|
82
|
+
elif rubric_changes:
|
|
83
|
+
classification = "rubric_changed"
|
|
84
|
+
elif scope_changed:
|
|
85
|
+
classification = "case_scope_changed"
|
|
86
|
+
else:
|
|
87
|
+
classification = "prompt_or_context_changed"
|
|
88
|
+
return {
|
|
89
|
+
"strictly_comparable": strictly_comparable,
|
|
90
|
+
"classification": classification,
|
|
91
|
+
"left_case_count": len(a),
|
|
92
|
+
"right_case_count": len(b),
|
|
93
|
+
"added_case_ids": added,
|
|
94
|
+
"removed_case_ids": removed,
|
|
95
|
+
"rubric_changes": rubric_changes,
|
|
96
|
+
"prompt_context_changes": context_changes,
|
|
97
|
+
"other_case_metadata_changes": other_changes,
|
|
98
|
+
"note": "Any case-scope, rubric, task, mode, reference, or tool-trace requirement change prevents strict score comparability. Other metadata changes are disclosed but do not automatically block comparison.",
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def read(path: Path) -> dict[str, Any]:
|
|
103
|
+
return load_dataset(path)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def main() -> int:
|
|
107
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
108
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
109
|
+
fp = sub.add_parser("fingerprint")
|
|
110
|
+
fp.add_argument("--dataset", type=Path, required=True)
|
|
111
|
+
compare = sub.add_parser("compare")
|
|
112
|
+
compare.add_argument("--left", type=Path, required=True)
|
|
113
|
+
compare.add_argument("--right", type=Path, required=True)
|
|
114
|
+
args = parser.parse_args()
|
|
115
|
+
try:
|
|
116
|
+
if args.command == "fingerprint":
|
|
117
|
+
raw = args.dataset.read_bytes()
|
|
118
|
+
result = fingerprint_dataset(load_dataset(args.dataset), raw)
|
|
119
|
+
else:
|
|
120
|
+
result = compare_datasets(read(args.left), read(args.right))
|
|
121
|
+
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
|
122
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
123
|
+
return 1
|
|
124
|
+
print(json.dumps(result, ensure_ascii=False, sort_keys=True, indent=2))
|
|
125
|
+
return 0
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
if __name__ == "__main__":
|
|
129
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Canonical fingerprints shared by evaluation report tooling."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import hashlib, json
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
def canonical_json(value: Any) -> bytes:
|
|
7
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
8
|
+
|
|
9
|
+
def dataset_fingerprints(dataset: dict[str, Any], source_bytes: bytes | None = None) -> dict[str, Any]:
|
|
10
|
+
cases = dataset["cases"]
|
|
11
|
+
rubric = [{"id": c["id"], "required_outputs": c.get("required_outputs", []),
|
|
12
|
+
"guardrails": c.get("guardrails", []), "security_categories": c.get("security_categories", [])}
|
|
13
|
+
for c in sorted(cases, key=lambda x: x["id"])]
|
|
14
|
+
return {
|
|
15
|
+
"catalog_schema_version": 1, "dataset_schema_version": dataset["schema_version"],
|
|
16
|
+
"case_count": len(cases), "case_ids": sorted(c["id"] for c in cases),
|
|
17
|
+
"dataset_canonical_sha256": hashlib.sha256(canonical_json(dataset)).hexdigest(),
|
|
18
|
+
"dataset_source_sha256": hashlib.sha256(source_bytes).hexdigest() if source_bytes is not None else None,
|
|
19
|
+
"rubric_sha256": hashlib.sha256(canonical_json(rubric)).hexdigest(),
|
|
20
|
+
}
|