system-design-primer-skill 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +15 -7
  2. package/SKILL.md +22 -2
  3. package/VERSION +1 -1
  4. package/evals/README.md +1022 -0
  5. package/evals/system-design-cases.json +414 -0
  6. package/package.json +3 -2
  7. package/references/03-data-and-storage.md +8 -2
  8. package/references/04-caching.md +3 -3
  9. package/references/05-async-and-messaging.md +4 -0
  10. package/references/06-distributed-systems.md +2 -2
  11. package/references/07-reliability-and-resilience.md +7 -3
  12. package/references/08-api-security-observability.md +4 -0
  13. package/references/15-failure-analysis.md +4 -0
  14. package/references/19-agent-security-and-permissions.md +22 -6
  15. package/references/20-agent-state-and-memory.md +4 -0
  16. package/references/22-kubernetes-and-container-platforms.md +1 -1
  17. package/references/24-multi-region-and-disaster-recovery.md +6 -2
  18. package/references/27-production-observability-and-slo.md +4 -0
  19. package/references/28-event-streaming-and-kafka-pulsar.md +6 -0
  20. package/references/32-workflow-orchestration-and-temporal.md +4 -0
  21. package/references/34-postgresql-internals-and-performance.md +2 -0
  22. package/references/35-mysql-innodb-and-performance.md +2 -0
  23. package/references/36-sharding-and-distributed-sql.md +1 -1
  24. package/references/37-consensus-and-distributed-transactions.md +7 -3
  25. package/references/40-zero-downtime-database-migrations.md +3 -3
  26. package/references/48-oauth-oidc-and-passkeys.md +3 -3
  27. package/references/49-iam-rbac-abac-rebac.md +7 -3
  28. package/references/51-secrets-kms-pki-and-key-management.md +4 -0
  29. package/references/52-supply-chain-security-sbom-slsa.md +62 -0
  30. package/references/54-privacy-engineering-and-data-governance.md +4 -0
  31. package/references/55-opentelemetry-and-instrumentation.md +4 -0
  32. package/references/56-distributed-tracing-and-context-propagation.md +4 -0
  33. package/references/59-capacity-planning-and-performance-modeling.md +8 -4
  34. package/references/60-chaos-engineering-and-resilience-experiments.md +4 -0
  35. package/references/61-incident-management-and-postmortems.md +4 -0
  36. package/references/67-storage-infrastructure-and-storage-systems.md +4 -0
  37. package/references/68-tcp-quic-and-network-performance.md +1 -1
  38. package/references/69-task-routing-and-evidence.md +75 -18
  39. package/scripts/audit_chain.py +36 -0
  40. package/scripts/compare_reports.py +43 -0
  41. package/scripts/evaluate.py +475 -0
  42. package/scripts/evaluation_catalog.py +129 -0
  43. package/scripts/evaluation_fingerprints.py +20 -0
  44. package/scripts/evidence_archive.py +79 -0
  45. package/scripts/evidence_signing.py +98 -0
  46. package/scripts/evidence_timestamp.py +441 -0
  47. package/scripts/install.ps1 +2 -2
  48. package/scripts/install.sh +2 -2
  49. package/scripts/report_integrity.py +184 -0
  50. package/scripts/review_consensus.py +77 -0
  51. package/scripts/review_evidence.py +105 -0
  52. package/scripts/review_provenance.py +44 -0
  53. package/scripts/reviewer_calibration.py +215 -0
  54. package/scripts/rubric_compatibility.py +134 -0
  55. package/scripts/run_evals.py +130 -0
  56. package/scripts/run_manifest.py +179 -0
  57. package/scripts/statistical_analysis.py +308 -0
  58. package/scripts/tsa_attestation.py +709 -0
  59. package/scripts/tsa_validation_report.py +627 -0
  60. package/scripts/validate.py +233 -21
  61. package/scripts/verify_build_provenance.py +625 -0
  62. package/scripts/verify_tsa_ocsp.py +962 -0
  63. package/scripts/verify_tsa_revocation.py +782 -0
  64. package/scripts/windows_broker_protocol.py +274 -0
  65. package/scripts/windows_broker_transport.py +141 -0
  66. package/scripts/windows_broker_win32.py +713 -0
@@ -0,0 +1,184 @@
1
+ #!/usr/bin/env python3
2
+ """Create and verify integrity sidecars for evaluation reports."""
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import re
9
+ import sys
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ SCHEMA_VERSION = 2
14
+ HASH_RE = re.compile(r"^[0-9a-f]{64}$")
15
+ SIDECAR_FIELDS = {
16
+ "schema_version", "report_sha256", "responses_sha256",
17
+ "audit_chain_head_hash", "manifest_sha256", "run_id",
18
+ }
19
+
20
+
21
+ def file_sha256(path: Path) -> str:
22
+ digest = hashlib.sha256()
23
+ with path.open("rb") as stream:
24
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
25
+ digest.update(chunk)
26
+ return digest.hexdigest()
27
+
28
+
29
+ def _read_report(path: Path) -> dict[str, Any]:
30
+ report = json.loads(path.read_text(encoding="utf-8"))
31
+ if not isinstance(report, dict):
32
+ raise ValueError("report must be a JSON object")
33
+ return report
34
+
35
+
36
+ def _validate_optional_binding(manifest_sha256: Any, run_id: Any) -> None:
37
+ if manifest_sha256 is None and run_id is None:
38
+ return
39
+ if not isinstance(manifest_sha256, str) or not HASH_RE.fullmatch(manifest_sha256):
40
+ raise ValueError("manifest_sha256 must be a lowercase SHA-256 digest or null")
41
+ if not isinstance(run_id, str) or not run_id.strip():
42
+ raise ValueError("run_id must be a non-empty string when a manifest is bound")
43
+
44
+
45
+ def seal_report(
46
+ report_path: Path,
47
+ responses_path: Path,
48
+ chain_status: dict[str, Any] | None,
49
+ run_manifest: dict[str, Any],
50
+ ) -> dict[str, Any]:
51
+ if not report_path.is_file() or not responses_path.is_file():
52
+ raise ValueError("report and response files must exist")
53
+ report = _read_report(report_path)
54
+ if chain_status is not None:
55
+ if not chain_status.get("verified"):
56
+ raise ValueError("cannot seal report with an unverified response audit chain")
57
+ head_hash = chain_status.get("head_hash")
58
+ if not isinstance(head_hash, str) or not HASH_RE.fullmatch(head_hash):
59
+ raise ValueError("verified audit chain must provide a valid head_hash")
60
+ else:
61
+ head_hash = None
62
+ manifest_verified = run_manifest.get("verified") is True
63
+ manifest_hash = run_manifest.get("manifest_sha256") if manifest_verified else None
64
+ run_id = run_manifest.get("run_id") if manifest_verified else None
65
+ _validate_optional_binding(manifest_hash, run_id)
66
+
67
+ report_chain = report.get("audit_chain", {})
68
+ if (report_chain.get("verified") is True) != (chain_status is not None):
69
+ raise ValueError("report audit_chain status does not match the supplied verification status")
70
+ if chain_status is not None and report_chain.get("head_hash") != head_hash:
71
+ raise ValueError("report audit-chain head does not match supplied chain status")
72
+ report_manifest = report.get("run_manifest", {})
73
+ if report_manifest.get("verified") is not manifest_verified:
74
+ raise ValueError("report run_manifest status does not match supplied manifest status")
75
+ if manifest_verified and (
76
+ report_manifest.get("manifest_sha256") != manifest_hash
77
+ or report_manifest.get("run_id") != run_id
78
+ ):
79
+ raise ValueError("report manifest binding does not match supplied manifest")
80
+
81
+ return {
82
+ "schema_version": SCHEMA_VERSION,
83
+ "report_sha256": file_sha256(report_path),
84
+ "responses_sha256": file_sha256(responses_path),
85
+ "audit_chain_head_hash": head_hash,
86
+ "manifest_sha256": manifest_hash,
87
+ "run_id": run_id,
88
+ }
89
+
90
+
91
+ def verify_report(
92
+ report_path: Path,
93
+ responses_path: Path,
94
+ sidecar_path: Path,
95
+ manifest_path: Path | None = None,
96
+ ) -> dict[str, Any]:
97
+ sidecar = json.loads(sidecar_path.read_text(encoding="utf-8"))
98
+ if not isinstance(sidecar, dict) or set(sidecar) != SIDECAR_FIELDS or sidecar.get("schema_version") != SCHEMA_VERSION:
99
+ raise ValueError("invalid report integrity sidecar schema")
100
+ for field in ("report_sha256", "responses_sha256"):
101
+ if not isinstance(sidecar[field], str) or not HASH_RE.fullmatch(sidecar[field]):
102
+ raise ValueError(f"invalid {field} in report integrity sidecar")
103
+ _validate_optional_binding(sidecar["manifest_sha256"], sidecar["run_id"])
104
+ chain_head = sidecar["audit_chain_head_hash"]
105
+ if chain_head is not None and (not isinstance(chain_head, str) or not HASH_RE.fullmatch(chain_head)):
106
+ raise ValueError("invalid audit_chain_head_hash in report integrity sidecar")
107
+
108
+ if file_sha256(report_path) != sidecar["report_sha256"]:
109
+ raise ValueError("report SHA-256 mismatch")
110
+ if file_sha256(responses_path) != sidecar["responses_sha256"]:
111
+ raise ValueError("response file SHA-256 mismatch")
112
+
113
+ report = _read_report(report_path)
114
+ report_chain = report.get("audit_chain", {})
115
+ if chain_head is None:
116
+ if report_chain.get("verified") is True:
117
+ raise ValueError("report claims a verified audit chain but sidecar has no chain head")
118
+ chain_checked = False
119
+ else:
120
+ if report_chain.get("verified") is not True or report_chain.get("head_hash") != chain_head:
121
+ raise ValueError("report audit-chain metadata does not match sidecar")
122
+ try:
123
+ from .evaluate import verify_audit_chain
124
+ except ImportError:
125
+ from evaluate import verify_audit_chain
126
+ verify_audit_chain(responses_path, expected_head_hash=chain_head)
127
+ chain_checked = True
128
+
129
+ report_manifest = report.get("run_manifest", {})
130
+ if sidecar["manifest_sha256"] is None:
131
+ if report_manifest.get("verified") is True:
132
+ raise ValueError("report claims a verified manifest but sidecar has no manifest binding")
133
+ if manifest_path is not None:
134
+ raise ValueError("manifest supplied but report sidecar is not manifest-bound")
135
+ manifest_checked = False
136
+ else:
137
+ if manifest_path is None:
138
+ raise ValueError("sidecar is manifest-bound; --manifest is required for verification")
139
+ try:
140
+ from .run_manifest import load_verified_manifest
141
+ except ImportError:
142
+ from run_manifest import load_verified_manifest
143
+ manifest = load_verified_manifest(manifest_path)
144
+ if manifest["fingerprints"]["manifest_sha256"] != sidecar["manifest_sha256"]:
145
+ raise ValueError("manifest fingerprint does not match report sidecar")
146
+ if manifest["run"]["run_id"] != sidecar["run_id"]:
147
+ raise ValueError("manifest run_id does not match report sidecar")
148
+ if (
149
+ report_manifest.get("verified") is not True
150
+ or report_manifest.get("manifest_sha256") != sidecar["manifest_sha256"]
151
+ or report_manifest.get("run_id") != sidecar["run_id"]
152
+ ):
153
+ raise ValueError("report manifest metadata does not match sidecar")
154
+ manifest_checked = True
155
+
156
+ return {
157
+ "verified": True,
158
+ "report_sha256": sidecar["report_sha256"],
159
+ "responses_sha256": sidecar["responses_sha256"],
160
+ "audit_chain_checked": chain_checked,
161
+ "manifest_checked": manifest_checked,
162
+ }
163
+
164
+
165
+ def main() -> int:
166
+ parser = argparse.ArgumentParser(description=__doc__)
167
+ sub = parser.add_subparsers(dest="command", required=True)
168
+ verify = sub.add_parser("verify", help="Verify report, response bytes, audit chain, and manifest binding")
169
+ verify.add_argument("--report", type=Path, required=True)
170
+ verify.add_argument("--responses", type=Path, required=True)
171
+ verify.add_argument("--sidecar", type=Path, required=True)
172
+ verify.add_argument("--manifest", type=Path)
173
+ args = parser.parse_args()
174
+ try:
175
+ result = verify_report(args.report, args.responses, args.sidecar, args.manifest)
176
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
177
+ print(f"ERROR: {exc}", file=sys.stderr)
178
+ return 1
179
+ print(json.dumps(result, sort_keys=True))
180
+ return 0
181
+
182
+
183
+ if __name__ == "__main__":
184
+ raise SystemExit(main())
@@ -0,0 +1,77 @@
1
+ from __future__ import annotations
2
+ from datetime import datetime
3
+ from typing import Any
4
+ try:
5
+ from .reviewer_calibration import pairwise_agreement
6
+ except ImportError: # Direct execution via evaluate.py
7
+ from reviewer_calibration import pairwise_agreement
8
+
9
+ def _aware(value: Any, field: str) -> None:
10
+ if not isinstance(value, str) or not value.strip():
11
+ raise ValueError(f"{field} must be a non-empty ISO-8601 timestamp")
12
+ try:
13
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
14
+ except ValueError as exc:
15
+ raise ValueError(f"{field} must be a valid ISO-8601 timestamp") from exc
16
+ if parsed.tzinfo is None or parsed.utcoffset() is None:
17
+ raise ValueError(f"{field} must include a timezone")
18
+
19
+ def validate_review_panel(submissions: Any, adjudication: Any, case: dict[str, Any],
20
+ validate_review: Any, line_number: int = 0) -> dict[str, Any]:
21
+ prefix = f"line {line_number}: " if line_number else ""
22
+ if not isinstance(submissions, list) or len(submissions) < 2:
23
+ raise ValueError(prefix + "review_submissions must contain at least two independent reviews")
24
+ ids: set[str] = set()
25
+ reviewers: set[str] = set()
26
+ normalized: list[dict[str, Any]] = []
27
+ for index, item in enumerate(submissions):
28
+ if not isinstance(item, dict) or set(item) != {"submission_id", "reviewer_id", "submitted_at", "rubric_version", "review"}:
29
+ raise ValueError(prefix + f"review_submissions entry {index} has invalid fields")
30
+ for field in ("submission_id", "reviewer_id", "rubric_version"):
31
+ if not isinstance(item[field], str) or not item[field].strip():
32
+ raise ValueError(prefix + f"review_submissions entry {index} requires {field}")
33
+ if item["submission_id"] in ids:
34
+ raise ValueError(prefix + "duplicate review submission_id")
35
+ if normalized and item["rubric_version"] != normalized[0]["rubric_version"]:
36
+ raise ValueError(prefix + "review panel submissions must use the same rubric_version")
37
+ if item["reviewer_id"] in reviewers:
38
+ raise ValueError(prefix + "independent reviews must have distinct reviewer_id values")
39
+ _aware(item["submitted_at"], f"review_submissions entry {index} submitted_at")
40
+ validate_review(item["review"], case, line_number)
41
+ ids.add(item["submission_id"])
42
+ reviewers.add(item["reviewer_id"])
43
+ normalized.append(item)
44
+ criteria = []
45
+ for kind, key in (("required_outputs", "required_outputs"), ("guardrails", "guardrails")):
46
+ for index, label in enumerate(case[key]):
47
+ values = [item["review"][key][index] for item in normalized]
48
+ distinct = sorted(set(values))
49
+ criteria.append({"type": kind, "index": index, "criterion": label,
50
+ "values": values, "disagreement": len(distinct) > 1,
51
+ "distinct_values": distinct})
52
+ disagreement_count = sum(entry["disagreement"] for entry in criteria)
53
+ result = {"reviewer_count": len(normalized), "submission_ids": sorted(ids),
54
+ "criteria": criteria, "disagreement_count": disagreement_count,
55
+ "agreement_metrics": pairwise_agreement(normalized, case),
56
+ "submissions": normalized,
57
+ "status": "disputed" if disagreement_count else "agreement"}
58
+ if adjudication is not None:
59
+ required = {"adjudicator_id", "decided_at", "rationale", "based_on_submission_ids", "review"}
60
+ if not isinstance(adjudication, dict) or set(adjudication) != required:
61
+ raise ValueError(prefix + "adjudication has invalid fields")
62
+ if not isinstance(adjudication["adjudicator_id"], str) or not adjudication["adjudicator_id"].strip():
63
+ raise ValueError(prefix + "adjudication requires adjudicator_id")
64
+ _aware(adjudication["decided_at"], "adjudication decided_at")
65
+ if not isinstance(adjudication["rationale"], str) or not adjudication["rationale"].strip():
66
+ raise ValueError(prefix + "adjudication requires a non-empty rationale")
67
+ refs = adjudication["based_on_submission_ids"]
68
+ if not isinstance(refs, list) or not refs or len(refs) != len(set(refs)) or not set(refs) <= ids:
69
+ raise ValueError(prefix + "adjudication must reference existing unique submission IDs")
70
+ validate_review(adjudication["review"], case, line_number)
71
+ result["adjudication"] = {"status": "recorded", "adjudicator_id": adjudication["adjudicator_id"],
72
+ "decided_at": adjudication["decided_at"], "rationale": adjudication["rationale"],
73
+ "based_on_submission_ids": refs, "review": adjudication["review"]}
74
+ result["status"] = "adjudicated"
75
+ elif disagreement_count:
76
+ result["status"] = "disputed"
77
+ return result
@@ -0,0 +1,105 @@
1
+ from __future__ import annotations
2
+ import argparse, hashlib, json, sys
3
+ from pathlib import Path
4
+ from typing import Any
5
+
6
+ def canonical_json(value: Any) -> bytes:
7
+ return json.dumps(value,ensure_ascii=False,sort_keys=True,separators=(",",":")).encode("utf-8")
8
+ def leaf_hash(value: dict[str,Any]) -> str:
9
+ return hashlib.sha256(bytes([0])+canonical_json(value)).hexdigest()
10
+ def node_hash(left: str,right: str) -> str:
11
+ return hashlib.sha256(bytes([1])+bytes.fromhex(left)+bytes.fromhex(right)).hexdigest()
12
+ def merkle_root(hashes: list[str]) -> str:
13
+ if not hashes: return hashlib.sha256(bytes([2])).hexdigest()
14
+ level=list(hashes)
15
+ while len(level)>1: level=[node_hash(level[i],level[i+1]) if i+1<len(level) else level[i] for i in range(0,len(level),2)]
16
+ return level[0]
17
+ def inclusion_proof(hashes: list[str],index: int) -> list[dict[str,str]]:
18
+ if not 0<=index<len(hashes): raise ValueError("proof index out of range")
19
+ proof=[]; level=list(hashes); pos=index
20
+ while len(level)>1:
21
+ sibling=pos^1
22
+ if sibling<len(level): proof.append({"side":"left" if sibling<pos else "right","hash":level[sibling]})
23
+ pos//=2
24
+ level=[node_hash(level[i],level[i+1]) if i+1<len(level) else level[i] for i in range(0,len(level),2)]
25
+ return proof
26
+ def verify_inclusion(value: dict[str,Any],proof: list[dict[str,str]],root: str) -> bool:
27
+ current=leaf_hash(value)
28
+ for step in proof:
29
+ if not isinstance(step,dict) or set(step)!={"side","hash"} or step["side"] not in ("left","right"): return False
30
+ sibling=step["hash"]
31
+ if not isinstance(sibling,str) or len(sibling)!=64: return False
32
+ try: bytes.fromhex(sibling)
33
+ except ValueError: return False
34
+ current=node_hash(sibling,current) if step["side"]=="left" else node_hash(current,sibling)
35
+ return current==root
36
+ def _evidence(record: dict[str,Any],result: dict[str,Any]) -> dict[str,Any]:
37
+ return {"case_id":record["case_id"],"model":record["model"],"rubric_sha256":result["rubric_sha256"],
38
+ "reviewed":result["reviewed"],"review":record.get("review"),"review_provenance":record.get("review_provenance"),
39
+ "review_panel":record.get("review_panel"),"response_output_sha256":hashlib.sha256(record["output"].encode("utf-8")).hexdigest(),
40
+ "tool_trace_sha256":hashlib.sha256(canonical_json(record.get("tool_trace"))).hexdigest(),
41
+ "run_id":record.get("run_id"),"manifest_sha256":record.get("manifest_sha256")}
42
+ def attach_review_evidence(report: dict[str,Any],records: list[dict[str,Any]]) -> None:
43
+ grouped={}
44
+ for record in records:
45
+ result=report["models"][record["model"]]["case_results"][record["case_id"]]
46
+ grouped.setdefault(record["model"],[]).append(_evidence(record,result))
47
+ roots={}
48
+ for model,entries in sorted(grouped.items()):
49
+ entries.sort(key=lambda x:x["case_id"]); hashes=[leaf_hash(x) for x in entries]; root=merkle_root(hashes); roots[model]=root
50
+ report["models"][model]["review_evidence"]={"algorithm":"sha256-merkle-v1","root":root,"leaf_count":len(entries),
51
+ "entries":[{"case_id":x["case_id"],"leaf":x,"leaf_sha256":hashes[i],"proof":inclusion_proof(hashes,i)} for i,x in enumerate(entries)]}
52
+ commitment={"algorithm":"sha256-merkle-v1","dataset_canonical_sha256":report.get("evaluation_catalog",{}).get("dataset_canonical_sha256"),
53
+ "rubric_sha256":report.get("evaluation_catalog",{}).get("rubric_sha256"),"evaluation_provenance":report.get("evaluation_provenance"),
54
+ "run_manifest":report.get("run_manifest"),"audit_chain":report.get("audit_chain"),"model_roots":roots}
55
+ report["review_evidence_commitment"]=commitment
56
+ report["review_evidence_root"]=hashlib.sha256(bytes([3])+canonical_json(commitment)).hexdigest()
57
+ def verify_report_evidence(report: dict[str,Any]) -> dict[str,Any]:
58
+ commitment=report.get("review_evidence_commitment")
59
+ if not isinstance(commitment,dict) or commitment.get("algorithm")!="sha256-merkle-v1": raise ValueError("unsupported or missing evidence commitment")
60
+ expected=hashlib.sha256(bytes([3])+canonical_json(commitment)).hexdigest()
61
+ if report.get("review_evidence_root")!=expected: raise ValueError("global review evidence root mismatch")
62
+ for key in ("dataset_canonical_sha256","rubric_sha256"):
63
+ if commitment.get(key)!=report.get("evaluation_catalog",{}).get(key): raise ValueError("global commitment does not match report catalog")
64
+ for key in ("evaluation_provenance","run_manifest","audit_chain"):
65
+ if commitment.get(key)!=report.get(key): raise ValueError("global commitment does not match report "+key)
66
+ roots={}; checked=0
67
+ for model,metrics in sorted(report.get("models",{}).items()):
68
+ evidence=metrics.get("review_evidence")
69
+ if not isinstance(evidence,dict) or evidence.get("algorithm")!="sha256-merkle-v1": raise ValueError("missing model evidence: "+model)
70
+ entries=evidence.get("entries")
71
+ if not isinstance(entries,list) or len(entries)!=evidence.get("leaf_count"): raise ValueError("evidence leaf count mismatch: "+model)
72
+ entries=sorted(entries,key=lambda x:x.get("case_id","")); hashes=[]
73
+ for entry in entries:
74
+ leaf=entry.get("leaf")
75
+ if not isinstance(leaf,dict) or leaf.get("model")!=model or leaf.get("case_id")!=entry.get("case_id"): raise ValueError("invalid leaf identity")
76
+ digest=leaf_hash(leaf)
77
+ if digest!=entry.get("leaf_sha256") or not verify_inclusion(leaf,entry.get("proof",[]),evidence.get("root","")): raise ValueError("case inclusion proof failed")
78
+ case_result=metrics.get("case_results",{}).get(leaf["case_id"])
79
+ if not isinstance(case_result,dict) or case_result.get("rubric_sha256")!=leaf.get("rubric_sha256") or case_result.get("reviewed")!=leaf.get("reviewed"):
80
+ raise ValueError("case leaf does not match report case result")
81
+ hashes.append(digest); checked+=1
82
+ root=merkle_root(hashes)
83
+ if root!=evidence.get("root"): raise ValueError("model Merkle root mismatch")
84
+ roots[model]=root
85
+ if commitment.get("model_roots")!=roots: raise ValueError("global model roots mismatch")
86
+ return {"verified":True,"models_checked":len(roots),"case_evidence_checked":checked,"review_evidence_root":expected}
87
+ def main() -> int:
88
+ p=argparse.ArgumentParser(description="Verify report review-evidence Merkle proofs"); sub=p.add_subparsers(dest="command",required=True)
89
+ v=sub.add_parser("verify"); v.add_argument("--report",type=Path,required=True)
90
+ q=sub.add_parser("verify-proof"); q.add_argument("--report",type=Path,required=True); q.add_argument("--model",required=True); q.add_argument("--case-id",required=True)
91
+ a=p.parse_args()
92
+ try:
93
+ report=json.loads(a.report.read_text(encoding="utf-8"))
94
+ if not isinstance(report,dict): raise ValueError("report must be a JSON object")
95
+ if a.command=="verify": result=verify_report_evidence(report)
96
+ else:
97
+ ev=report["models"][a.model]["review_evidence"]; entry=next(x for x in ev["entries"] if x["case_id"]==a.case_id)
98
+ commitment=report.get("review_evidence_commitment",{}); expected=hashlib.sha256(bytes([3])+canonical_json(commitment)).hexdigest()
99
+ if report.get("review_evidence_root")!=expected or commitment.get("model_roots",{}).get(a.model)!=ev.get("root"): raise ValueError("global report root does not commit to this model root")
100
+ if not verify_inclusion(entry["leaf"],entry["proof"],ev["root"]): raise ValueError("case inclusion proof failed")
101
+ result={"verified":True,"model":a.model,"case_id":a.case_id,"leaf_sha256":entry["leaf_sha256"],"model_merkle_root":ev["root"],"global_report_root":report.get("review_evidence_root")}
102
+ except (OSError,json.JSONDecodeError,KeyError,StopIteration,TypeError,ValueError) as exc:
103
+ print("ERROR: "+str(exc),file=sys.stderr); return 1
104
+ print(json.dumps(result,ensure_ascii=False,sort_keys=True,indent=2)); return 0
105
+ if __name__=="__main__": raise SystemExit(main())
@@ -0,0 +1,44 @@
1
+ from __future__ import annotations
2
+ import hashlib
3
+ from datetime import datetime
4
+ from typing import Any
5
+
6
+ def canonical_json(value: Any) -> bytes:
7
+ import json
8
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
9
+
10
+ def review_event_hash(event: dict[str, Any]) -> str:
11
+ payload = {key: value for key, value in event.items() if key != "event_sha256"}
12
+ return hashlib.sha256(canonical_json(payload)).hexdigest()
13
+
14
+ def validate_review_history(history: Any, current_review: Any, line_number: int = 0) -> dict[str, Any]:
15
+ prefix = f"line {line_number}: " if line_number else ""
16
+ if not isinstance(history, list) or not history:
17
+ raise ValueError(prefix + "review_history must be a non-empty array")
18
+ previous_hash = None
19
+ for index, event in enumerate(history):
20
+ required = {"reviewer_id", "reviewed_at", "rubric_version", "review", "previous_event_sha256", "event_sha256"}
21
+ if not isinstance(event, dict) or set(event) != required:
22
+ raise ValueError(prefix + f"review_history event {index} has invalid fields")
23
+ for field in ("reviewer_id", "reviewed_at", "rubric_version"):
24
+ if not isinstance(event[field], str) or not event[field].strip():
25
+ raise ValueError(prefix + f"review_history event {index} requires {field}")
26
+ try:
27
+ timestamp = datetime.fromisoformat(event["reviewed_at"].replace("Z", "+00:00"))
28
+ if timestamp.tzinfo is None or timestamp.utcoffset() is None:
29
+ raise ValueError("timezone required")
30
+ except ValueError as exc:
31
+ raise ValueError(prefix + f"review_history event {index} reviewed_at must be timezone-aware ISO-8601") from exc
32
+ if event["previous_event_sha256"] != previous_hash:
33
+ raise ValueError(prefix + f"review_history event {index} link mismatch")
34
+ if not isinstance(event["review"], dict):
35
+ raise ValueError(prefix + f"review_history event {index} review must be an object")
36
+ if event["event_sha256"] != review_event_hash(event):
37
+ raise ValueError(prefix + f"review_history event {index} hash mismatch")
38
+ previous_hash = event["event_sha256"]
39
+ if history[-1]["review"] != current_review:
40
+ raise ValueError(prefix + "current review does not match the latest review_history snapshot")
41
+ last = history[-1]
42
+ return {"verified": True, "events": len(history), "head_sha256": previous_hash,
43
+ "reviewer_id": last["reviewer_id"], "reviewed_at": last["reviewed_at"],
44
+ "rubric_version": last["rubric_version"]}
@@ -0,0 +1,215 @@
1
+ from __future__ import annotations
2
+ from itertools import combinations
3
+ import random
4
+ from typing import Any
5
+
6
+ def _cohen_kappa(left: list[int | bool], right: list[int | bool], categories: list[int | bool],
7
+ quadratic: bool = False) -> dict[str, Any]:
8
+ n = len(left)
9
+ if n == 0:
10
+ return {"observations": 0, "observed_agreement": None, "kappa": None}
11
+ observed = sum(a == b for a, b in zip(left, right)) / n
12
+ left_counts = {v: left.count(v) / n for v in categories}
13
+ right_counts = {v: right.count(v) / n for v in categories}
14
+ expected = sum(left_counts[v] * right_counts[v] for v in categories)
15
+ if expected >= 1.0:
16
+ kappa = None
17
+ else:
18
+ kappa = (observed - expected) / (1.0 - expected)
19
+ result: dict[str, Any] = {"observations": n, "observed_agreement": round(observed, 6),
20
+ "expected_agreement": round(expected, 6),
21
+ "kappa": round(kappa, 6) if kappa is not None else None}
22
+ if quadratic and len(categories) > 1:
23
+ low, high = min(categories), max(categories)
24
+ span = high - low
25
+ def weight(a: int | bool, b: int | bool) -> float:
26
+ return ((a - b) / span) ** 2
27
+ observed_disagreement = sum(weight(a, b) for a, b in zip(left, right)) / n
28
+ expected_disagreement = sum(left_counts[a] * right_counts[b] * weight(a, b)
29
+ for a in categories for b in categories)
30
+ weighted = (1 - observed_disagreement / expected_disagreement) if expected_disagreement else None
31
+ result["quadratic_weighted_kappa"] = round(weighted, 6) if weighted is not None else None
32
+ return result
33
+
34
+ def pairwise_agreement(submissions: list[dict[str, Any]], case: dict[str, Any]) -> dict[str, Any]:
35
+ pairs = []
36
+ for left, right in combinations(submissions, 2):
37
+ comparisons = []
38
+ for key in ("required_outputs", "guardrails"):
39
+ for index, criterion in enumerate(case[key]):
40
+ a = left["review"][key][index]
41
+ b = right["review"][key][index]
42
+ comparisons.append({"criterion_type": key, "criterion_index": index,
43
+ "criterion": criterion, "left_value": a,
44
+ "right_value": b, "agreement": a == b})
45
+ pairs.append({"reviewer_ids": [left["reviewer_id"], right["reviewer_id"]],
46
+ "criterion_comparisons": comparisons})
47
+ return {"method": "raw_pairwise_agreement", "pair_count": len(pairs), "pairs": pairs}
48
+
49
+ def _bootstrap_interval(values: list[tuple[int | bool, int | bool]], categories: list[int | bool],
50
+ quadratic: bool, seed: int = 1729, iterations: int = 1000) -> dict[str, Any] | None:
51
+ """Percentile bootstrap CI over paired case observations; omitted for small samples."""
52
+ if len(values) < 20:
53
+ return None
54
+ rng = random.Random(seed)
55
+ estimates: list[float] = []
56
+ for _ in range(iterations):
57
+ sample = [values[rng.randrange(len(values))] for _ in range(len(values))]
58
+ metric = _cohen_kappa([v[0] for v in sample], [v[1] for v in sample], categories, quadratic)
59
+ estimate = metric.get("quadratic_weighted_kappa") if quadratic else metric.get("kappa")
60
+ if estimate is not None:
61
+ estimates.append(float(estimate))
62
+ if len(estimates) < iterations * 0.8:
63
+ return {"status": "unstable", "iterations": iterations, "valid_resamples": len(estimates)}
64
+ estimates.sort()
65
+ low = estimates[int(0.025 * (len(estimates) - 1))]
66
+ high = estimates[int(0.975 * (len(estimates) - 1))]
67
+ return {"status": "estimated", "confidence_level": 0.95, "method": "paired_case_percentile_bootstrap",
68
+ "iterations": iterations, "valid_resamples": len(estimates),
69
+ "lower": round(low, 6), "upper": round(high, 6)}
70
+
71
+
72
+ def summarize_review_data_quality(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
73
+ """Describe panel coverage, reviewer concentration, and score distributions."""
74
+ reviewer_work: dict[tuple[str, str], set[str]] = {}
75
+ reviewer_counts: dict[tuple[str, str], int] = {}
76
+ case_panels: dict[str, int] = {}
77
+ distributions: dict[tuple[str, str, str], dict[str, int]] = {}
78
+ reviewed_records = 0
79
+ for record in records:
80
+ panel = record.get("review_panel")
81
+ if not panel:
82
+ continue
83
+ reviewed_records += 1
84
+ case_id = record["case_id"]
85
+ case = cases[case_id]
86
+ case_model = case_id + "|" + record["model"]
87
+ case_panels[case_model] = panel.get("reviewer_count", len(panel.get("submissions", [])))
88
+ for submission in panel.get("submissions", []):
89
+ rubric = submission["rubric_version"]
90
+ reviewer = submission["reviewer_id"]
91
+ reviewer_work.setdefault((rubric, reviewer), set()).add(case_model)
92
+ reviewer_counts[(rubric, reviewer)] = reviewer_counts.get((rubric, reviewer), 0) + 1
93
+ review = submission["review"]
94
+ for kind in ("required_outputs", "guardrails"):
95
+ for index, value in enumerate(review[kind]):
96
+ criterion = case[kind][index]
97
+ bucket = distributions.setdefault((rubric, kind, criterion), {})
98
+ label = str(value).lower() if isinstance(value, bool) else str(value)
99
+ bucket[label] = bucket.get(label, 0) + 1
100
+ total_records = len(records)
101
+ reviewer_rows = []
102
+ rubric_totals: dict[str, int] = {}
103
+ for (rubric, reviewer), count in reviewer_counts.items():
104
+ rubric_totals[rubric] = rubric_totals.get(rubric, 0) + count
105
+ for (rubric, reviewer), count in sorted(reviewer_counts.items()):
106
+ total = rubric_totals[rubric]
107
+ reviewer_rows.append({"rubric_version": rubric, "reviewer_id": reviewer,
108
+ "reviewed_case_model_count": len(reviewer_work[(rubric, reviewer)]),
109
+ "submission_count": count, "share_of_rubric_submissions": round(count / total, 6)})
110
+ concentration = []
111
+ for rubric, total in sorted(rubric_totals.items()):
112
+ shares = [count / total for (version, _), count in reviewer_counts.items() if version == rubric]
113
+ concentration.append({"rubric_version": rubric, "reviewer_count": len(shares),
114
+ "hhi": round(sum(share * share for share in shares), 6),
115
+ "largest_reviewer_share": round(max(shares), 6) if shares else None,
116
+ "interpretation": "high_concentration" if shares and max(shares) >= 0.5 else "distributed"})
117
+ case_rows = [{"case_id": key.split("|", 1)[0], "model": key.split("|", 1)[1],
118
+ "reviewer_count": count} for key, count in sorted(case_panels.items())]
119
+ distribution_rows = []
120
+ for (rubric, kind, criterion), counts in sorted(distributions.items()):
121
+ total = sum(counts.values())
122
+ distribution_rows.append({"rubric_version": rubric, "criterion_type": kind, "criterion": criterion,
123
+ "observation_count": total, "category_counts": counts,
124
+ "category_shares": {key: round(value / total, 6) for key, value in sorted(counts.items())}})
125
+ return {"status": "descriptive_only", "response_record_count": total_records,
126
+ "reviewed_record_count": reviewed_records,
127
+ "panel_coverage_rate": round(reviewed_records / total_records, 6) if total_records else None,
128
+ "unreviewed_record_count": total_records - reviewed_records,
129
+ "case_model_panel_coverage": case_rows, "reviewer_coverage": reviewer_rows,
130
+ "reviewer_concentration": concentration, "score_distributions": distribution_rows,
131
+ "limitations": ["Coverage is measured over supplied response records, not the full intended population unless all responses are supplied.",
132
+ "HHI and category distributions are descriptive and do not establish reviewer quality.",
133
+ "Repeated models or cases may not be statistically independent."]}
134
+
135
+ def aggregate_reviewer_calibration(panels: list[dict[str, Any]]) -> dict[str, Any]:
136
+ """Aggregate per-reviewer-pair categorical observations across independently reviewed cases."""
137
+ buckets: dict[tuple[str, str, str, str, str], list[tuple[int | bool, int | bool]]] = {}
138
+ for panel in panels:
139
+ submissions = panel["submissions"]
140
+ for left, right in combinations(submissions, 2):
141
+ if left.get("rubric_version") != right.get("rubric_version"):
142
+ continue
143
+ pair = tuple(sorted((left["reviewer_id"], right["reviewer_id"])))
144
+ rubric = left["rubric_version"]
145
+ for key in ("required_outputs", "guardrails"):
146
+ for index in range(len(left["review"][key])):
147
+ label = panel.get("case", {}).get(key, [])[index] if panel.get("case") else str(index)
148
+ bucket = (rubric, pair[0], pair[1], key, label)
149
+ buckets.setdefault(bucket, []).append((left["review"][key][index], right["review"][key][index]))
150
+ results = []
151
+ for (rubric, a_id, b_id, key, label), values in sorted(buckets.items()):
152
+ categories = [0, 1, 2] if key == "required_outputs" else [False, True]
153
+ metric = _cohen_kappa([v[0] for v in values], [v[1] for v in values],
154
+ categories, quadratic=(key == "required_outputs"))
155
+ confidence = _bootstrap_interval(values, categories, quadratic=(key == "required_outputs"))
156
+ results.append({"rubric_version": rubric, "reviewer_ids": [a_id, b_id], "criterion_type": key,
157
+ "criterion": label, **metric, "kappa_confidence_interval_95": confidence})
158
+ return {"method": "pairwise_cohen_kappa", "minimum_observations": 2,
159
+ "results": results,
160
+ "low_agreement_pairs": [item for item in results if item["observations"] >= 2
161
+ and item["kappa"] is not None and item["kappa"] < 0.6]}
162
+
163
+ def summarize_reviewer_behavior(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]],
164
+ min_cases: int = 3) -> dict[str, Any]:
165
+ """Describe reviewer scoring tendencies; never normalize or mutate model scores."""
166
+ observations: dict[tuple[str, str, str], list[tuple[str, str, float]]] = {}
167
+ reviewer_cases: dict[tuple[str, str], set[str]] = {}
168
+ panel_lookup: list[dict[str, Any]] = []
169
+ for record in records:
170
+ panel = record.get("review_panel")
171
+ if not panel:
172
+ continue
173
+ case_id = record["case_id"]
174
+ case = cases[case_id]
175
+ for submission in panel.get("submissions", []):
176
+ reviewer = submission["reviewer_id"]
177
+ rubric = submission["rubric_version"]
178
+ reviewer_cases.setdefault((rubric, reviewer), set()).add(case_id)
179
+ review = submission["review"]
180
+ for index, score in enumerate(review["required_outputs"]):
181
+ observations.setdefault((rubric, "required_outputs", case["required_outputs"][index]), []).append((reviewer, case_id + "|" + record["model"], float(score)))
182
+ for index, passed in enumerate(review["guardrails"]):
183
+ observations.setdefault((rubric, "guardrails", case["guardrails"][index]), []).append((reviewer, case_id + "|" + record["model"], 1.0 if passed else 0.0))
184
+ panel_lookup.append({"case_id": case_id, "rubric_version": sorted({s["rubric_version"] for s in panel.get("submissions", [])}),
185
+ "reviewer_ids": [s["reviewer_id"] for s in panel.get("submissions", [])],
186
+ "disagreement_count": panel.get("disagreement_count", 0),
187
+ "status": panel.get("status")})
188
+ reviewer_rows = []
189
+ for (rubric, reviewer), case_ids in sorted(reviewer_cases.items()):
190
+ row: dict[str, Any] = {"rubric_version": rubric, "reviewer_id": reviewer,
191
+ "reviewed_case_count": len(case_ids),
192
+ "sample_status": "sufficient" if len(case_ids) >= min_cases else "insufficient_sample",
193
+ "minimum_cases": min_cases, "criterion_tendencies": []}
194
+ for (r_version, kind, criterion), values in sorted(observations.items()):
195
+ if r_version != rubric:
196
+ continue
197
+ own = [value for who, _, value in values if who == reviewer]
198
+ own_case_keys = {case_key for who, case_key, _ in values if who == reviewer}
199
+ peers = [value for who, case_key, value in values if who != reviewer and case_key in own_case_keys]
200
+ if not own:
201
+ continue
202
+ mean = sum(own) / len(own)
203
+ peer_mean = sum(peers) / len(peers) if peers else None
204
+ row["criterion_tendencies"].append({"criterion_type": kind, "criterion": criterion,
205
+ "observation_count": len(own), "mean_score_or_pass_rate": round(mean, 4),
206
+ "peer_mean": round(peer_mean, 4) if peer_mean is not None else None,
207
+ "descriptive_delta": round(mean - peer_mean, 4) if peer_mean is not None else None,
208
+ "interpretation": "higher_than_peers" if peer_mean is not None and mean > peer_mean else ("lower_than_peers" if peer_mean is not None and mean < peer_mean else "not_comparable")})
209
+ reviewer_rows.append(row)
210
+ return {"status": "descriptive_only", "minimum_cases": min_cases,
211
+ "reviewers": reviewer_rows, "panels": panel_lookup,
212
+ "limitations": ["Reviewer IDs are caller-declared, not authenticated.",
213
+ "Deltas are descriptive and are not bias-corrected scores.",
214
+ "Small samples are flagged; confidence intervals are not estimated.",
215
+ "Comparisons are isolated by rubric version, criterion label, and shared case/model observations."]}