system-design-primer-skill 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/SKILL.md +22 -2
- package/VERSION +1 -1
- package/evals/README.md +1022 -0
- package/evals/system-design-cases.json +414 -0
- package/package.json +3 -2
- package/references/03-data-and-storage.md +8 -2
- package/references/04-caching.md +3 -3
- package/references/05-async-and-messaging.md +4 -0
- package/references/06-distributed-systems.md +2 -2
- package/references/07-reliability-and-resilience.md +7 -3
- package/references/08-api-security-observability.md +4 -0
- package/references/15-failure-analysis.md +4 -0
- package/references/19-agent-security-and-permissions.md +22 -6
- package/references/20-agent-state-and-memory.md +4 -0
- package/references/22-kubernetes-and-container-platforms.md +1 -1
- package/references/24-multi-region-and-disaster-recovery.md +6 -2
- package/references/27-production-observability-and-slo.md +4 -0
- package/references/28-event-streaming-and-kafka-pulsar.md +6 -0
- package/references/32-workflow-orchestration-and-temporal.md +4 -0
- package/references/34-postgresql-internals-and-performance.md +2 -0
- package/references/35-mysql-innodb-and-performance.md +2 -0
- package/references/36-sharding-and-distributed-sql.md +1 -1
- package/references/37-consensus-and-distributed-transactions.md +7 -3
- package/references/40-zero-downtime-database-migrations.md +3 -3
- package/references/48-oauth-oidc-and-passkeys.md +3 -3
- package/references/49-iam-rbac-abac-rebac.md +7 -3
- package/references/51-secrets-kms-pki-and-key-management.md +4 -0
- package/references/52-supply-chain-security-sbom-slsa.md +62 -0
- package/references/54-privacy-engineering-and-data-governance.md +4 -0
- package/references/55-opentelemetry-and-instrumentation.md +4 -0
- package/references/56-distributed-tracing-and-context-propagation.md +4 -0
- package/references/59-capacity-planning-and-performance-modeling.md +8 -4
- package/references/60-chaos-engineering-and-resilience-experiments.md +4 -0
- package/references/61-incident-management-and-postmortems.md +4 -0
- package/references/67-storage-infrastructure-and-storage-systems.md +4 -0
- package/references/68-tcp-quic-and-network-performance.md +1 -1
- package/references/69-task-routing-and-evidence.md +75 -18
- package/scripts/audit_chain.py +36 -0
- package/scripts/compare_reports.py +43 -0
- package/scripts/evaluate.py +475 -0
- package/scripts/evaluation_catalog.py +129 -0
- package/scripts/evaluation_fingerprints.py +20 -0
- package/scripts/evidence_archive.py +79 -0
- package/scripts/evidence_signing.py +98 -0
- package/scripts/evidence_timestamp.py +441 -0
- package/scripts/install.ps1 +2 -2
- package/scripts/install.sh +2 -2
- package/scripts/report_integrity.py +184 -0
- package/scripts/review_consensus.py +77 -0
- package/scripts/review_evidence.py +105 -0
- package/scripts/review_provenance.py +44 -0
- package/scripts/reviewer_calibration.py +215 -0
- package/scripts/rubric_compatibility.py +134 -0
- package/scripts/run_evals.py +130 -0
- package/scripts/run_manifest.py +179 -0
- package/scripts/statistical_analysis.py +308 -0
- package/scripts/tsa_attestation.py +709 -0
- package/scripts/tsa_validation_report.py +627 -0
- package/scripts/validate.py +233 -21
- package/scripts/verify_build_provenance.py +625 -0
- package/scripts/verify_tsa_ocsp.py +962 -0
- package/scripts/verify_tsa_revocation.py +782 -0
- package/scripts/windows_broker_protocol.py +274 -0
- package/scripts/windows_broker_transport.py +141 -0
- package/scripts/windows_broker_win32.py +713 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Create and verify integrity sidecars for evaluation reports."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
SCHEMA_VERSION = 2
|
|
14
|
+
HASH_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
15
|
+
SIDECAR_FIELDS = {
|
|
16
|
+
"schema_version", "report_sha256", "responses_sha256",
|
|
17
|
+
"audit_chain_head_hash", "manifest_sha256", "run_id",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def file_sha256(path: Path) -> str:
|
|
22
|
+
digest = hashlib.sha256()
|
|
23
|
+
with path.open("rb") as stream:
|
|
24
|
+
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
25
|
+
digest.update(chunk)
|
|
26
|
+
return digest.hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _read_report(path: Path) -> dict[str, Any]:
|
|
30
|
+
report = json.loads(path.read_text(encoding="utf-8"))
|
|
31
|
+
if not isinstance(report, dict):
|
|
32
|
+
raise ValueError("report must be a JSON object")
|
|
33
|
+
return report
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _validate_optional_binding(manifest_sha256: Any, run_id: Any) -> None:
|
|
37
|
+
if manifest_sha256 is None and run_id is None:
|
|
38
|
+
return
|
|
39
|
+
if not isinstance(manifest_sha256, str) or not HASH_RE.fullmatch(manifest_sha256):
|
|
40
|
+
raise ValueError("manifest_sha256 must be a lowercase SHA-256 digest or null")
|
|
41
|
+
if not isinstance(run_id, str) or not run_id.strip():
|
|
42
|
+
raise ValueError("run_id must be a non-empty string when a manifest is bound")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def seal_report(
|
|
46
|
+
report_path: Path,
|
|
47
|
+
responses_path: Path,
|
|
48
|
+
chain_status: dict[str, Any] | None,
|
|
49
|
+
run_manifest: dict[str, Any],
|
|
50
|
+
) -> dict[str, Any]:
|
|
51
|
+
if not report_path.is_file() or not responses_path.is_file():
|
|
52
|
+
raise ValueError("report and response files must exist")
|
|
53
|
+
report = _read_report(report_path)
|
|
54
|
+
if chain_status is not None:
|
|
55
|
+
if not chain_status.get("verified"):
|
|
56
|
+
raise ValueError("cannot seal report with an unverified response audit chain")
|
|
57
|
+
head_hash = chain_status.get("head_hash")
|
|
58
|
+
if not isinstance(head_hash, str) or not HASH_RE.fullmatch(head_hash):
|
|
59
|
+
raise ValueError("verified audit chain must provide a valid head_hash")
|
|
60
|
+
else:
|
|
61
|
+
head_hash = None
|
|
62
|
+
manifest_verified = run_manifest.get("verified") is True
|
|
63
|
+
manifest_hash = run_manifest.get("manifest_sha256") if manifest_verified else None
|
|
64
|
+
run_id = run_manifest.get("run_id") if manifest_verified else None
|
|
65
|
+
_validate_optional_binding(manifest_hash, run_id)
|
|
66
|
+
|
|
67
|
+
report_chain = report.get("audit_chain", {})
|
|
68
|
+
if (report_chain.get("verified") is True) != (chain_status is not None):
|
|
69
|
+
raise ValueError("report audit_chain status does not match the supplied verification status")
|
|
70
|
+
if chain_status is not None and report_chain.get("head_hash") != head_hash:
|
|
71
|
+
raise ValueError("report audit-chain head does not match supplied chain status")
|
|
72
|
+
report_manifest = report.get("run_manifest", {})
|
|
73
|
+
if report_manifest.get("verified") is not manifest_verified:
|
|
74
|
+
raise ValueError("report run_manifest status does not match supplied manifest status")
|
|
75
|
+
if manifest_verified and (
|
|
76
|
+
report_manifest.get("manifest_sha256") != manifest_hash
|
|
77
|
+
or report_manifest.get("run_id") != run_id
|
|
78
|
+
):
|
|
79
|
+
raise ValueError("report manifest binding does not match supplied manifest")
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
"schema_version": SCHEMA_VERSION,
|
|
83
|
+
"report_sha256": file_sha256(report_path),
|
|
84
|
+
"responses_sha256": file_sha256(responses_path),
|
|
85
|
+
"audit_chain_head_hash": head_hash,
|
|
86
|
+
"manifest_sha256": manifest_hash,
|
|
87
|
+
"run_id": run_id,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def verify_report(
|
|
92
|
+
report_path: Path,
|
|
93
|
+
responses_path: Path,
|
|
94
|
+
sidecar_path: Path,
|
|
95
|
+
manifest_path: Path | None = None,
|
|
96
|
+
) -> dict[str, Any]:
|
|
97
|
+
sidecar = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
|
98
|
+
if not isinstance(sidecar, dict) or set(sidecar) != SIDECAR_FIELDS or sidecar.get("schema_version") != SCHEMA_VERSION:
|
|
99
|
+
raise ValueError("invalid report integrity sidecar schema")
|
|
100
|
+
for field in ("report_sha256", "responses_sha256"):
|
|
101
|
+
if not isinstance(sidecar[field], str) or not HASH_RE.fullmatch(sidecar[field]):
|
|
102
|
+
raise ValueError(f"invalid {field} in report integrity sidecar")
|
|
103
|
+
_validate_optional_binding(sidecar["manifest_sha256"], sidecar["run_id"])
|
|
104
|
+
chain_head = sidecar["audit_chain_head_hash"]
|
|
105
|
+
if chain_head is not None and (not isinstance(chain_head, str) or not HASH_RE.fullmatch(chain_head)):
|
|
106
|
+
raise ValueError("invalid audit_chain_head_hash in report integrity sidecar")
|
|
107
|
+
|
|
108
|
+
if file_sha256(report_path) != sidecar["report_sha256"]:
|
|
109
|
+
raise ValueError("report SHA-256 mismatch")
|
|
110
|
+
if file_sha256(responses_path) != sidecar["responses_sha256"]:
|
|
111
|
+
raise ValueError("response file SHA-256 mismatch")
|
|
112
|
+
|
|
113
|
+
report = _read_report(report_path)
|
|
114
|
+
report_chain = report.get("audit_chain", {})
|
|
115
|
+
if chain_head is None:
|
|
116
|
+
if report_chain.get("verified") is True:
|
|
117
|
+
raise ValueError("report claims a verified audit chain but sidecar has no chain head")
|
|
118
|
+
chain_checked = False
|
|
119
|
+
else:
|
|
120
|
+
if report_chain.get("verified") is not True or report_chain.get("head_hash") != chain_head:
|
|
121
|
+
raise ValueError("report audit-chain metadata does not match sidecar")
|
|
122
|
+
try:
|
|
123
|
+
from .evaluate import verify_audit_chain
|
|
124
|
+
except ImportError:
|
|
125
|
+
from evaluate import verify_audit_chain
|
|
126
|
+
verify_audit_chain(responses_path, expected_head_hash=chain_head)
|
|
127
|
+
chain_checked = True
|
|
128
|
+
|
|
129
|
+
report_manifest = report.get("run_manifest", {})
|
|
130
|
+
if sidecar["manifest_sha256"] is None:
|
|
131
|
+
if report_manifest.get("verified") is True:
|
|
132
|
+
raise ValueError("report claims a verified manifest but sidecar has no manifest binding")
|
|
133
|
+
if manifest_path is not None:
|
|
134
|
+
raise ValueError("manifest supplied but report sidecar is not manifest-bound")
|
|
135
|
+
manifest_checked = False
|
|
136
|
+
else:
|
|
137
|
+
if manifest_path is None:
|
|
138
|
+
raise ValueError("sidecar is manifest-bound; --manifest is required for verification")
|
|
139
|
+
try:
|
|
140
|
+
from .run_manifest import load_verified_manifest
|
|
141
|
+
except ImportError:
|
|
142
|
+
from run_manifest import load_verified_manifest
|
|
143
|
+
manifest = load_verified_manifest(manifest_path)
|
|
144
|
+
if manifest["fingerprints"]["manifest_sha256"] != sidecar["manifest_sha256"]:
|
|
145
|
+
raise ValueError("manifest fingerprint does not match report sidecar")
|
|
146
|
+
if manifest["run"]["run_id"] != sidecar["run_id"]:
|
|
147
|
+
raise ValueError("manifest run_id does not match report sidecar")
|
|
148
|
+
if (
|
|
149
|
+
report_manifest.get("verified") is not True
|
|
150
|
+
or report_manifest.get("manifest_sha256") != sidecar["manifest_sha256"]
|
|
151
|
+
or report_manifest.get("run_id") != sidecar["run_id"]
|
|
152
|
+
):
|
|
153
|
+
raise ValueError("report manifest metadata does not match sidecar")
|
|
154
|
+
manifest_checked = True
|
|
155
|
+
|
|
156
|
+
return {
|
|
157
|
+
"verified": True,
|
|
158
|
+
"report_sha256": sidecar["report_sha256"],
|
|
159
|
+
"responses_sha256": sidecar["responses_sha256"],
|
|
160
|
+
"audit_chain_checked": chain_checked,
|
|
161
|
+
"manifest_checked": manifest_checked,
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def main() -> int:
|
|
166
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
167
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
168
|
+
verify = sub.add_parser("verify", help="Verify report, response bytes, audit chain, and manifest binding")
|
|
169
|
+
verify.add_argument("--report", type=Path, required=True)
|
|
170
|
+
verify.add_argument("--responses", type=Path, required=True)
|
|
171
|
+
verify.add_argument("--sidecar", type=Path, required=True)
|
|
172
|
+
verify.add_argument("--manifest", type=Path)
|
|
173
|
+
args = parser.parse_args()
|
|
174
|
+
try:
|
|
175
|
+
result = verify_report(args.report, args.responses, args.sidecar, args.manifest)
|
|
176
|
+
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
|
177
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
178
|
+
return 1
|
|
179
|
+
print(json.dumps(result, sort_keys=True))
|
|
180
|
+
return 0
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
if __name__ == "__main__":
|
|
184
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from typing import Any
|
|
4
|
+
try:
|
|
5
|
+
from .reviewer_calibration import pairwise_agreement
|
|
6
|
+
except ImportError: # Direct execution via evaluate.py
|
|
7
|
+
from reviewer_calibration import pairwise_agreement
|
|
8
|
+
|
|
9
|
+
def _aware(value: Any, field: str) -> None:
|
|
10
|
+
if not isinstance(value, str) or not value.strip():
|
|
11
|
+
raise ValueError(f"{field} must be a non-empty ISO-8601 timestamp")
|
|
12
|
+
try:
|
|
13
|
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
14
|
+
except ValueError as exc:
|
|
15
|
+
raise ValueError(f"{field} must be a valid ISO-8601 timestamp") from exc
|
|
16
|
+
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
|
17
|
+
raise ValueError(f"{field} must include a timezone")
|
|
18
|
+
|
|
19
|
+
def validate_review_panel(submissions: Any, adjudication: Any, case: dict[str, Any],
|
|
20
|
+
validate_review: Any, line_number: int = 0) -> dict[str, Any]:
|
|
21
|
+
prefix = f"line {line_number}: " if line_number else ""
|
|
22
|
+
if not isinstance(submissions, list) or len(submissions) < 2:
|
|
23
|
+
raise ValueError(prefix + "review_submissions must contain at least two independent reviews")
|
|
24
|
+
ids: set[str] = set()
|
|
25
|
+
reviewers: set[str] = set()
|
|
26
|
+
normalized: list[dict[str, Any]] = []
|
|
27
|
+
for index, item in enumerate(submissions):
|
|
28
|
+
if not isinstance(item, dict) or set(item) != {"submission_id", "reviewer_id", "submitted_at", "rubric_version", "review"}:
|
|
29
|
+
raise ValueError(prefix + f"review_submissions entry {index} has invalid fields")
|
|
30
|
+
for field in ("submission_id", "reviewer_id", "rubric_version"):
|
|
31
|
+
if not isinstance(item[field], str) or not item[field].strip():
|
|
32
|
+
raise ValueError(prefix + f"review_submissions entry {index} requires {field}")
|
|
33
|
+
if item["submission_id"] in ids:
|
|
34
|
+
raise ValueError(prefix + "duplicate review submission_id")
|
|
35
|
+
if normalized and item["rubric_version"] != normalized[0]["rubric_version"]:
|
|
36
|
+
raise ValueError(prefix + "review panel submissions must use the same rubric_version")
|
|
37
|
+
if item["reviewer_id"] in reviewers:
|
|
38
|
+
raise ValueError(prefix + "independent reviews must have distinct reviewer_id values")
|
|
39
|
+
_aware(item["submitted_at"], f"review_submissions entry {index} submitted_at")
|
|
40
|
+
validate_review(item["review"], case, line_number)
|
|
41
|
+
ids.add(item["submission_id"])
|
|
42
|
+
reviewers.add(item["reviewer_id"])
|
|
43
|
+
normalized.append(item)
|
|
44
|
+
criteria = []
|
|
45
|
+
for kind, key in (("required_outputs", "required_outputs"), ("guardrails", "guardrails")):
|
|
46
|
+
for index, label in enumerate(case[key]):
|
|
47
|
+
values = [item["review"][key][index] for item in normalized]
|
|
48
|
+
distinct = sorted(set(values))
|
|
49
|
+
criteria.append({"type": kind, "index": index, "criterion": label,
|
|
50
|
+
"values": values, "disagreement": len(distinct) > 1,
|
|
51
|
+
"distinct_values": distinct})
|
|
52
|
+
disagreement_count = sum(entry["disagreement"] for entry in criteria)
|
|
53
|
+
result = {"reviewer_count": len(normalized), "submission_ids": sorted(ids),
|
|
54
|
+
"criteria": criteria, "disagreement_count": disagreement_count,
|
|
55
|
+
"agreement_metrics": pairwise_agreement(normalized, case),
|
|
56
|
+
"submissions": normalized,
|
|
57
|
+
"status": "disputed" if disagreement_count else "agreement"}
|
|
58
|
+
if adjudication is not None:
|
|
59
|
+
required = {"adjudicator_id", "decided_at", "rationale", "based_on_submission_ids", "review"}
|
|
60
|
+
if not isinstance(adjudication, dict) or set(adjudication) != required:
|
|
61
|
+
raise ValueError(prefix + "adjudication has invalid fields")
|
|
62
|
+
if not isinstance(adjudication["adjudicator_id"], str) or not adjudication["adjudicator_id"].strip():
|
|
63
|
+
raise ValueError(prefix + "adjudication requires adjudicator_id")
|
|
64
|
+
_aware(adjudication["decided_at"], "adjudication decided_at")
|
|
65
|
+
if not isinstance(adjudication["rationale"], str) or not adjudication["rationale"].strip():
|
|
66
|
+
raise ValueError(prefix + "adjudication requires a non-empty rationale")
|
|
67
|
+
refs = adjudication["based_on_submission_ids"]
|
|
68
|
+
if not isinstance(refs, list) or not refs or len(refs) != len(set(refs)) or not set(refs) <= ids:
|
|
69
|
+
raise ValueError(prefix + "adjudication must reference existing unique submission IDs")
|
|
70
|
+
validate_review(adjudication["review"], case, line_number)
|
|
71
|
+
result["adjudication"] = {"status": "recorded", "adjudicator_id": adjudication["adjudicator_id"],
|
|
72
|
+
"decided_at": adjudication["decided_at"], "rationale": adjudication["rationale"],
|
|
73
|
+
"based_on_submission_ids": refs, "review": adjudication["review"]}
|
|
74
|
+
result["status"] = "adjudicated"
|
|
75
|
+
elif disagreement_count:
|
|
76
|
+
result["status"] = "disputed"
|
|
77
|
+
return result
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import argparse, hashlib, json, sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
def canonical_json(value: Any) -> bytes:
|
|
7
|
+
return json.dumps(value,ensure_ascii=False,sort_keys=True,separators=(",",":")).encode("utf-8")
|
|
8
|
+
def leaf_hash(value: dict[str,Any]) -> str:
|
|
9
|
+
return hashlib.sha256(bytes([0])+canonical_json(value)).hexdigest()
|
|
10
|
+
def node_hash(left: str,right: str) -> str:
|
|
11
|
+
return hashlib.sha256(bytes([1])+bytes.fromhex(left)+bytes.fromhex(right)).hexdigest()
|
|
12
|
+
def merkle_root(hashes: list[str]) -> str:
|
|
13
|
+
if not hashes: return hashlib.sha256(bytes([2])).hexdigest()
|
|
14
|
+
level=list(hashes)
|
|
15
|
+
while len(level)>1: level=[node_hash(level[i],level[i+1]) if i+1<len(level) else level[i] for i in range(0,len(level),2)]
|
|
16
|
+
return level[0]
|
|
17
|
+
def inclusion_proof(hashes: list[str],index: int) -> list[dict[str,str]]:
|
|
18
|
+
if not 0<=index<len(hashes): raise ValueError("proof index out of range")
|
|
19
|
+
proof=[]; level=list(hashes); pos=index
|
|
20
|
+
while len(level)>1:
|
|
21
|
+
sibling=pos^1
|
|
22
|
+
if sibling<len(level): proof.append({"side":"left" if sibling<pos else "right","hash":level[sibling]})
|
|
23
|
+
pos//=2
|
|
24
|
+
level=[node_hash(level[i],level[i+1]) if i+1<len(level) else level[i] for i in range(0,len(level),2)]
|
|
25
|
+
return proof
|
|
26
|
+
def verify_inclusion(value: dict[str,Any],proof: list[dict[str,str]],root: str) -> bool:
|
|
27
|
+
current=leaf_hash(value)
|
|
28
|
+
for step in proof:
|
|
29
|
+
if not isinstance(step,dict) or set(step)!={"side","hash"} or step["side"] not in ("left","right"): return False
|
|
30
|
+
sibling=step["hash"]
|
|
31
|
+
if not isinstance(sibling,str) or len(sibling)!=64: return False
|
|
32
|
+
try: bytes.fromhex(sibling)
|
|
33
|
+
except ValueError: return False
|
|
34
|
+
current=node_hash(sibling,current) if step["side"]=="left" else node_hash(current,sibling)
|
|
35
|
+
return current==root
|
|
36
|
+
def _evidence(record: dict[str,Any],result: dict[str,Any]) -> dict[str,Any]:
|
|
37
|
+
return {"case_id":record["case_id"],"model":record["model"],"rubric_sha256":result["rubric_sha256"],
|
|
38
|
+
"reviewed":result["reviewed"],"review":record.get("review"),"review_provenance":record.get("review_provenance"),
|
|
39
|
+
"review_panel":record.get("review_panel"),"response_output_sha256":hashlib.sha256(record["output"].encode("utf-8")).hexdigest(),
|
|
40
|
+
"tool_trace_sha256":hashlib.sha256(canonical_json(record.get("tool_trace"))).hexdigest(),
|
|
41
|
+
"run_id":record.get("run_id"),"manifest_sha256":record.get("manifest_sha256")}
|
|
42
|
+
def attach_review_evidence(report: dict[str,Any],records: list[dict[str,Any]]) -> None:
|
|
43
|
+
grouped={}
|
|
44
|
+
for record in records:
|
|
45
|
+
result=report["models"][record["model"]]["case_results"][record["case_id"]]
|
|
46
|
+
grouped.setdefault(record["model"],[]).append(_evidence(record,result))
|
|
47
|
+
roots={}
|
|
48
|
+
for model,entries in sorted(grouped.items()):
|
|
49
|
+
entries.sort(key=lambda x:x["case_id"]); hashes=[leaf_hash(x) for x in entries]; root=merkle_root(hashes); roots[model]=root
|
|
50
|
+
report["models"][model]["review_evidence"]={"algorithm":"sha256-merkle-v1","root":root,"leaf_count":len(entries),
|
|
51
|
+
"entries":[{"case_id":x["case_id"],"leaf":x,"leaf_sha256":hashes[i],"proof":inclusion_proof(hashes,i)} for i,x in enumerate(entries)]}
|
|
52
|
+
commitment={"algorithm":"sha256-merkle-v1","dataset_canonical_sha256":report.get("evaluation_catalog",{}).get("dataset_canonical_sha256"),
|
|
53
|
+
"rubric_sha256":report.get("evaluation_catalog",{}).get("rubric_sha256"),"evaluation_provenance":report.get("evaluation_provenance"),
|
|
54
|
+
"run_manifest":report.get("run_manifest"),"audit_chain":report.get("audit_chain"),"model_roots":roots}
|
|
55
|
+
report["review_evidence_commitment"]=commitment
|
|
56
|
+
report["review_evidence_root"]=hashlib.sha256(bytes([3])+canonical_json(commitment)).hexdigest()
|
|
57
|
+
def verify_report_evidence(report: dict[str,Any]) -> dict[str,Any]:
|
|
58
|
+
commitment=report.get("review_evidence_commitment")
|
|
59
|
+
if not isinstance(commitment,dict) or commitment.get("algorithm")!="sha256-merkle-v1": raise ValueError("unsupported or missing evidence commitment")
|
|
60
|
+
expected=hashlib.sha256(bytes([3])+canonical_json(commitment)).hexdigest()
|
|
61
|
+
if report.get("review_evidence_root")!=expected: raise ValueError("global review evidence root mismatch")
|
|
62
|
+
for key in ("dataset_canonical_sha256","rubric_sha256"):
|
|
63
|
+
if commitment.get(key)!=report.get("evaluation_catalog",{}).get(key): raise ValueError("global commitment does not match report catalog")
|
|
64
|
+
for key in ("evaluation_provenance","run_manifest","audit_chain"):
|
|
65
|
+
if commitment.get(key)!=report.get(key): raise ValueError("global commitment does not match report "+key)
|
|
66
|
+
roots={}; checked=0
|
|
67
|
+
for model,metrics in sorted(report.get("models",{}).items()):
|
|
68
|
+
evidence=metrics.get("review_evidence")
|
|
69
|
+
if not isinstance(evidence,dict) or evidence.get("algorithm")!="sha256-merkle-v1": raise ValueError("missing model evidence: "+model)
|
|
70
|
+
entries=evidence.get("entries")
|
|
71
|
+
if not isinstance(entries,list) or len(entries)!=evidence.get("leaf_count"): raise ValueError("evidence leaf count mismatch: "+model)
|
|
72
|
+
entries=sorted(entries,key=lambda x:x.get("case_id","")); hashes=[]
|
|
73
|
+
for entry in entries:
|
|
74
|
+
leaf=entry.get("leaf")
|
|
75
|
+
if not isinstance(leaf,dict) or leaf.get("model")!=model or leaf.get("case_id")!=entry.get("case_id"): raise ValueError("invalid leaf identity")
|
|
76
|
+
digest=leaf_hash(leaf)
|
|
77
|
+
if digest!=entry.get("leaf_sha256") or not verify_inclusion(leaf,entry.get("proof",[]),evidence.get("root","")): raise ValueError("case inclusion proof failed")
|
|
78
|
+
case_result=metrics.get("case_results",{}).get(leaf["case_id"])
|
|
79
|
+
if not isinstance(case_result,dict) or case_result.get("rubric_sha256")!=leaf.get("rubric_sha256") or case_result.get("reviewed")!=leaf.get("reviewed"):
|
|
80
|
+
raise ValueError("case leaf does not match report case result")
|
|
81
|
+
hashes.append(digest); checked+=1
|
|
82
|
+
root=merkle_root(hashes)
|
|
83
|
+
if root!=evidence.get("root"): raise ValueError("model Merkle root mismatch")
|
|
84
|
+
roots[model]=root
|
|
85
|
+
if commitment.get("model_roots")!=roots: raise ValueError("global model roots mismatch")
|
|
86
|
+
return {"verified":True,"models_checked":len(roots),"case_evidence_checked":checked,"review_evidence_root":expected}
|
|
87
|
+
def main() -> int:
|
|
88
|
+
p=argparse.ArgumentParser(description="Verify report review-evidence Merkle proofs"); sub=p.add_subparsers(dest="command",required=True)
|
|
89
|
+
v=sub.add_parser("verify"); v.add_argument("--report",type=Path,required=True)
|
|
90
|
+
q=sub.add_parser("verify-proof"); q.add_argument("--report",type=Path,required=True); q.add_argument("--model",required=True); q.add_argument("--case-id",required=True)
|
|
91
|
+
a=p.parse_args()
|
|
92
|
+
try:
|
|
93
|
+
report=json.loads(a.report.read_text(encoding="utf-8"))
|
|
94
|
+
if not isinstance(report,dict): raise ValueError("report must be a JSON object")
|
|
95
|
+
if a.command=="verify": result=verify_report_evidence(report)
|
|
96
|
+
else:
|
|
97
|
+
ev=report["models"][a.model]["review_evidence"]; entry=next(x for x in ev["entries"] if x["case_id"]==a.case_id)
|
|
98
|
+
commitment=report.get("review_evidence_commitment",{}); expected=hashlib.sha256(bytes([3])+canonical_json(commitment)).hexdigest()
|
|
99
|
+
if report.get("review_evidence_root")!=expected or commitment.get("model_roots",{}).get(a.model)!=ev.get("root"): raise ValueError("global report root does not commit to this model root")
|
|
100
|
+
if not verify_inclusion(entry["leaf"],entry["proof"],ev["root"]): raise ValueError("case inclusion proof failed")
|
|
101
|
+
result={"verified":True,"model":a.model,"case_id":a.case_id,"leaf_sha256":entry["leaf_sha256"],"model_merkle_root":ev["root"],"global_report_root":report.get("review_evidence_root")}
|
|
102
|
+
except (OSError,json.JSONDecodeError,KeyError,StopIteration,TypeError,ValueError) as exc:
|
|
103
|
+
print("ERROR: "+str(exc),file=sys.stderr); return 1
|
|
104
|
+
print(json.dumps(result,ensure_ascii=False,sort_keys=True,indent=2)); return 0
|
|
105
|
+
if __name__=="__main__": raise SystemExit(main())
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import hashlib
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
def canonical_json(value: Any) -> bytes:
|
|
7
|
+
import json
|
|
8
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
9
|
+
|
|
10
|
+
def review_event_hash(event: dict[str, Any]) -> str:
|
|
11
|
+
payload = {key: value for key, value in event.items() if key != "event_sha256"}
|
|
12
|
+
return hashlib.sha256(canonical_json(payload)).hexdigest()
|
|
13
|
+
|
|
14
|
+
def validate_review_history(history: Any, current_review: Any, line_number: int = 0) -> dict[str, Any]:
|
|
15
|
+
prefix = f"line {line_number}: " if line_number else ""
|
|
16
|
+
if not isinstance(history, list) or not history:
|
|
17
|
+
raise ValueError(prefix + "review_history must be a non-empty array")
|
|
18
|
+
previous_hash = None
|
|
19
|
+
for index, event in enumerate(history):
|
|
20
|
+
required = {"reviewer_id", "reviewed_at", "rubric_version", "review", "previous_event_sha256", "event_sha256"}
|
|
21
|
+
if not isinstance(event, dict) or set(event) != required:
|
|
22
|
+
raise ValueError(prefix + f"review_history event {index} has invalid fields")
|
|
23
|
+
for field in ("reviewer_id", "reviewed_at", "rubric_version"):
|
|
24
|
+
if not isinstance(event[field], str) or not event[field].strip():
|
|
25
|
+
raise ValueError(prefix + f"review_history event {index} requires {field}")
|
|
26
|
+
try:
|
|
27
|
+
timestamp = datetime.fromisoformat(event["reviewed_at"].replace("Z", "+00:00"))
|
|
28
|
+
if timestamp.tzinfo is None or timestamp.utcoffset() is None:
|
|
29
|
+
raise ValueError("timezone required")
|
|
30
|
+
except ValueError as exc:
|
|
31
|
+
raise ValueError(prefix + f"review_history event {index} reviewed_at must be timezone-aware ISO-8601") from exc
|
|
32
|
+
if event["previous_event_sha256"] != previous_hash:
|
|
33
|
+
raise ValueError(prefix + f"review_history event {index} link mismatch")
|
|
34
|
+
if not isinstance(event["review"], dict):
|
|
35
|
+
raise ValueError(prefix + f"review_history event {index} review must be an object")
|
|
36
|
+
if event["event_sha256"] != review_event_hash(event):
|
|
37
|
+
raise ValueError(prefix + f"review_history event {index} hash mismatch")
|
|
38
|
+
previous_hash = event["event_sha256"]
|
|
39
|
+
if history[-1]["review"] != current_review:
|
|
40
|
+
raise ValueError(prefix + "current review does not match the latest review_history snapshot")
|
|
41
|
+
last = history[-1]
|
|
42
|
+
return {"verified": True, "events": len(history), "head_sha256": previous_hash,
|
|
43
|
+
"reviewer_id": last["reviewer_id"], "reviewed_at": last["reviewed_at"],
|
|
44
|
+
"rubric_version": last["rubric_version"]}
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from itertools import combinations
|
|
3
|
+
import random
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
def _cohen_kappa(left: list[int | bool], right: list[int | bool], categories: list[int | bool],
|
|
7
|
+
quadratic: bool = False) -> dict[str, Any]:
|
|
8
|
+
n = len(left)
|
|
9
|
+
if n == 0:
|
|
10
|
+
return {"observations": 0, "observed_agreement": None, "kappa": None}
|
|
11
|
+
observed = sum(a == b for a, b in zip(left, right)) / n
|
|
12
|
+
left_counts = {v: left.count(v) / n for v in categories}
|
|
13
|
+
right_counts = {v: right.count(v) / n for v in categories}
|
|
14
|
+
expected = sum(left_counts[v] * right_counts[v] for v in categories)
|
|
15
|
+
if expected >= 1.0:
|
|
16
|
+
kappa = None
|
|
17
|
+
else:
|
|
18
|
+
kappa = (observed - expected) / (1.0 - expected)
|
|
19
|
+
result: dict[str, Any] = {"observations": n, "observed_agreement": round(observed, 6),
|
|
20
|
+
"expected_agreement": round(expected, 6),
|
|
21
|
+
"kappa": round(kappa, 6) if kappa is not None else None}
|
|
22
|
+
if quadratic and len(categories) > 1:
|
|
23
|
+
low, high = min(categories), max(categories)
|
|
24
|
+
span = high - low
|
|
25
|
+
def weight(a: int | bool, b: int | bool) -> float:
|
|
26
|
+
return ((a - b) / span) ** 2
|
|
27
|
+
observed_disagreement = sum(weight(a, b) for a, b in zip(left, right)) / n
|
|
28
|
+
expected_disagreement = sum(left_counts[a] * right_counts[b] * weight(a, b)
|
|
29
|
+
for a in categories for b in categories)
|
|
30
|
+
weighted = (1 - observed_disagreement / expected_disagreement) if expected_disagreement else None
|
|
31
|
+
result["quadratic_weighted_kappa"] = round(weighted, 6) if weighted is not None else None
|
|
32
|
+
return result
|
|
33
|
+
|
|
34
|
+
def pairwise_agreement(submissions: list[dict[str, Any]], case: dict[str, Any]) -> dict[str, Any]:
|
|
35
|
+
pairs = []
|
|
36
|
+
for left, right in combinations(submissions, 2):
|
|
37
|
+
comparisons = []
|
|
38
|
+
for key in ("required_outputs", "guardrails"):
|
|
39
|
+
for index, criterion in enumerate(case[key]):
|
|
40
|
+
a = left["review"][key][index]
|
|
41
|
+
b = right["review"][key][index]
|
|
42
|
+
comparisons.append({"criterion_type": key, "criterion_index": index,
|
|
43
|
+
"criterion": criterion, "left_value": a,
|
|
44
|
+
"right_value": b, "agreement": a == b})
|
|
45
|
+
pairs.append({"reviewer_ids": [left["reviewer_id"], right["reviewer_id"]],
|
|
46
|
+
"criterion_comparisons": comparisons})
|
|
47
|
+
return {"method": "raw_pairwise_agreement", "pair_count": len(pairs), "pairs": pairs}
|
|
48
|
+
|
|
49
|
+
def _bootstrap_interval(values: list[tuple[int | bool, int | bool]], categories: list[int | bool],
|
|
50
|
+
quadratic: bool, seed: int = 1729, iterations: int = 1000) -> dict[str, Any] | None:
|
|
51
|
+
"""Percentile bootstrap CI over paired case observations; omitted for small samples."""
|
|
52
|
+
if len(values) < 20:
|
|
53
|
+
return None
|
|
54
|
+
rng = random.Random(seed)
|
|
55
|
+
estimates: list[float] = []
|
|
56
|
+
for _ in range(iterations):
|
|
57
|
+
sample = [values[rng.randrange(len(values))] for _ in range(len(values))]
|
|
58
|
+
metric = _cohen_kappa([v[0] for v in sample], [v[1] for v in sample], categories, quadratic)
|
|
59
|
+
estimate = metric.get("quadratic_weighted_kappa") if quadratic else metric.get("kappa")
|
|
60
|
+
if estimate is not None:
|
|
61
|
+
estimates.append(float(estimate))
|
|
62
|
+
if len(estimates) < iterations * 0.8:
|
|
63
|
+
return {"status": "unstable", "iterations": iterations, "valid_resamples": len(estimates)}
|
|
64
|
+
estimates.sort()
|
|
65
|
+
low = estimates[int(0.025 * (len(estimates) - 1))]
|
|
66
|
+
high = estimates[int(0.975 * (len(estimates) - 1))]
|
|
67
|
+
return {"status": "estimated", "confidence_level": 0.95, "method": "paired_case_percentile_bootstrap",
|
|
68
|
+
"iterations": iterations, "valid_resamples": len(estimates),
|
|
69
|
+
"lower": round(low, 6), "upper": round(high, 6)}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def summarize_review_data_quality(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
|
73
|
+
"""Describe panel coverage, reviewer concentration, and score distributions."""
|
|
74
|
+
reviewer_work: dict[tuple[str, str], set[str]] = {}
|
|
75
|
+
reviewer_counts: dict[tuple[str, str], int] = {}
|
|
76
|
+
case_panels: dict[str, int] = {}
|
|
77
|
+
distributions: dict[tuple[str, str, str], dict[str, int]] = {}
|
|
78
|
+
reviewed_records = 0
|
|
79
|
+
for record in records:
|
|
80
|
+
panel = record.get("review_panel")
|
|
81
|
+
if not panel:
|
|
82
|
+
continue
|
|
83
|
+
reviewed_records += 1
|
|
84
|
+
case_id = record["case_id"]
|
|
85
|
+
case = cases[case_id]
|
|
86
|
+
case_model = case_id + "|" + record["model"]
|
|
87
|
+
case_panels[case_model] = panel.get("reviewer_count", len(panel.get("submissions", [])))
|
|
88
|
+
for submission in panel.get("submissions", []):
|
|
89
|
+
rubric = submission["rubric_version"]
|
|
90
|
+
reviewer = submission["reviewer_id"]
|
|
91
|
+
reviewer_work.setdefault((rubric, reviewer), set()).add(case_model)
|
|
92
|
+
reviewer_counts[(rubric, reviewer)] = reviewer_counts.get((rubric, reviewer), 0) + 1
|
|
93
|
+
review = submission["review"]
|
|
94
|
+
for kind in ("required_outputs", "guardrails"):
|
|
95
|
+
for index, value in enumerate(review[kind]):
|
|
96
|
+
criterion = case[kind][index]
|
|
97
|
+
bucket = distributions.setdefault((rubric, kind, criterion), {})
|
|
98
|
+
label = str(value).lower() if isinstance(value, bool) else str(value)
|
|
99
|
+
bucket[label] = bucket.get(label, 0) + 1
|
|
100
|
+
total_records = len(records)
|
|
101
|
+
reviewer_rows = []
|
|
102
|
+
rubric_totals: dict[str, int] = {}
|
|
103
|
+
for (rubric, reviewer), count in reviewer_counts.items():
|
|
104
|
+
rubric_totals[rubric] = rubric_totals.get(rubric, 0) + count
|
|
105
|
+
for (rubric, reviewer), count in sorted(reviewer_counts.items()):
|
|
106
|
+
total = rubric_totals[rubric]
|
|
107
|
+
reviewer_rows.append({"rubric_version": rubric, "reviewer_id": reviewer,
|
|
108
|
+
"reviewed_case_model_count": len(reviewer_work[(rubric, reviewer)]),
|
|
109
|
+
"submission_count": count, "share_of_rubric_submissions": round(count / total, 6)})
|
|
110
|
+
concentration = []
|
|
111
|
+
for rubric, total in sorted(rubric_totals.items()):
|
|
112
|
+
shares = [count / total for (version, _), count in reviewer_counts.items() if version == rubric]
|
|
113
|
+
concentration.append({"rubric_version": rubric, "reviewer_count": len(shares),
|
|
114
|
+
"hhi": round(sum(share * share for share in shares), 6),
|
|
115
|
+
"largest_reviewer_share": round(max(shares), 6) if shares else None,
|
|
116
|
+
"interpretation": "high_concentration" if shares and max(shares) >= 0.5 else "distributed"})
|
|
117
|
+
case_rows = [{"case_id": key.split("|", 1)[0], "model": key.split("|", 1)[1],
|
|
118
|
+
"reviewer_count": count} for key, count in sorted(case_panels.items())]
|
|
119
|
+
distribution_rows = []
|
|
120
|
+
for (rubric, kind, criterion), counts in sorted(distributions.items()):
|
|
121
|
+
total = sum(counts.values())
|
|
122
|
+
distribution_rows.append({"rubric_version": rubric, "criterion_type": kind, "criterion": criterion,
|
|
123
|
+
"observation_count": total, "category_counts": counts,
|
|
124
|
+
"category_shares": {key: round(value / total, 6) for key, value in sorted(counts.items())}})
|
|
125
|
+
return {"status": "descriptive_only", "response_record_count": total_records,
|
|
126
|
+
"reviewed_record_count": reviewed_records,
|
|
127
|
+
"panel_coverage_rate": round(reviewed_records / total_records, 6) if total_records else None,
|
|
128
|
+
"unreviewed_record_count": total_records - reviewed_records,
|
|
129
|
+
"case_model_panel_coverage": case_rows, "reviewer_coverage": reviewer_rows,
|
|
130
|
+
"reviewer_concentration": concentration, "score_distributions": distribution_rows,
|
|
131
|
+
"limitations": ["Coverage is measured over supplied response records, not the full intended population unless all responses are supplied.",
|
|
132
|
+
"HHI and category distributions are descriptive and do not establish reviewer quality.",
|
|
133
|
+
"Repeated models or cases may not be statistically independent."]}
|
|
134
|
+
|
|
135
|
+
def aggregate_reviewer_calibration(panels: list[dict[str, Any]]) -> dict[str, Any]:
|
|
136
|
+
"""Aggregate per-reviewer-pair categorical observations across independently reviewed cases."""
|
|
137
|
+
buckets: dict[tuple[str, str, str, str, str], list[tuple[int | bool, int | bool]]] = {}
|
|
138
|
+
for panel in panels:
|
|
139
|
+
submissions = panel["submissions"]
|
|
140
|
+
for left, right in combinations(submissions, 2):
|
|
141
|
+
if left.get("rubric_version") != right.get("rubric_version"):
|
|
142
|
+
continue
|
|
143
|
+
pair = tuple(sorted((left["reviewer_id"], right["reviewer_id"])))
|
|
144
|
+
rubric = left["rubric_version"]
|
|
145
|
+
for key in ("required_outputs", "guardrails"):
|
|
146
|
+
for index in range(len(left["review"][key])):
|
|
147
|
+
label = panel.get("case", {}).get(key, [])[index] if panel.get("case") else str(index)
|
|
148
|
+
bucket = (rubric, pair[0], pair[1], key, label)
|
|
149
|
+
buckets.setdefault(bucket, []).append((left["review"][key][index], right["review"][key][index]))
|
|
150
|
+
results = []
|
|
151
|
+
for (rubric, a_id, b_id, key, label), values in sorted(buckets.items()):
|
|
152
|
+
categories = [0, 1, 2] if key == "required_outputs" else [False, True]
|
|
153
|
+
metric = _cohen_kappa([v[0] for v in values], [v[1] for v in values],
|
|
154
|
+
categories, quadratic=(key == "required_outputs"))
|
|
155
|
+
confidence = _bootstrap_interval(values, categories, quadratic=(key == "required_outputs"))
|
|
156
|
+
results.append({"rubric_version": rubric, "reviewer_ids": [a_id, b_id], "criterion_type": key,
|
|
157
|
+
"criterion": label, **metric, "kappa_confidence_interval_95": confidence})
|
|
158
|
+
return {"method": "pairwise_cohen_kappa", "minimum_observations": 2,
|
|
159
|
+
"results": results,
|
|
160
|
+
"low_agreement_pairs": [item for item in results if item["observations"] >= 2
|
|
161
|
+
and item["kappa"] is not None and item["kappa"] < 0.6]}
|
|
162
|
+
|
|
163
|
+
def summarize_reviewer_behavior(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]],
|
|
164
|
+
min_cases: int = 3) -> dict[str, Any]:
|
|
165
|
+
"""Describe reviewer scoring tendencies; never normalize or mutate model scores."""
|
|
166
|
+
observations: dict[tuple[str, str, str], list[tuple[str, str, float]]] = {}
|
|
167
|
+
reviewer_cases: dict[tuple[str, str], set[str]] = {}
|
|
168
|
+
panel_lookup: list[dict[str, Any]] = []
|
|
169
|
+
for record in records:
|
|
170
|
+
panel = record.get("review_panel")
|
|
171
|
+
if not panel:
|
|
172
|
+
continue
|
|
173
|
+
case_id = record["case_id"]
|
|
174
|
+
case = cases[case_id]
|
|
175
|
+
for submission in panel.get("submissions", []):
|
|
176
|
+
reviewer = submission["reviewer_id"]
|
|
177
|
+
rubric = submission["rubric_version"]
|
|
178
|
+
reviewer_cases.setdefault((rubric, reviewer), set()).add(case_id)
|
|
179
|
+
review = submission["review"]
|
|
180
|
+
for index, score in enumerate(review["required_outputs"]):
|
|
181
|
+
observations.setdefault((rubric, "required_outputs", case["required_outputs"][index]), []).append((reviewer, case_id + "|" + record["model"], float(score)))
|
|
182
|
+
for index, passed in enumerate(review["guardrails"]):
|
|
183
|
+
observations.setdefault((rubric, "guardrails", case["guardrails"][index]), []).append((reviewer, case_id + "|" + record["model"], 1.0 if passed else 0.0))
|
|
184
|
+
panel_lookup.append({"case_id": case_id, "rubric_version": sorted({s["rubric_version"] for s in panel.get("submissions", [])}),
|
|
185
|
+
"reviewer_ids": [s["reviewer_id"] for s in panel.get("submissions", [])],
|
|
186
|
+
"disagreement_count": panel.get("disagreement_count", 0),
|
|
187
|
+
"status": panel.get("status")})
|
|
188
|
+
reviewer_rows = []
|
|
189
|
+
for (rubric, reviewer), case_ids in sorted(reviewer_cases.items()):
|
|
190
|
+
row: dict[str, Any] = {"rubric_version": rubric, "reviewer_id": reviewer,
|
|
191
|
+
"reviewed_case_count": len(case_ids),
|
|
192
|
+
"sample_status": "sufficient" if len(case_ids) >= min_cases else "insufficient_sample",
|
|
193
|
+
"minimum_cases": min_cases, "criterion_tendencies": []}
|
|
194
|
+
for (r_version, kind, criterion), values in sorted(observations.items()):
|
|
195
|
+
if r_version != rubric:
|
|
196
|
+
continue
|
|
197
|
+
own = [value for who, _, value in values if who == reviewer]
|
|
198
|
+
own_case_keys = {case_key for who, case_key, _ in values if who == reviewer}
|
|
199
|
+
peers = [value for who, case_key, value in values if who != reviewer and case_key in own_case_keys]
|
|
200
|
+
if not own:
|
|
201
|
+
continue
|
|
202
|
+
mean = sum(own) / len(own)
|
|
203
|
+
peer_mean = sum(peers) / len(peers) if peers else None
|
|
204
|
+
row["criterion_tendencies"].append({"criterion_type": kind, "criterion": criterion,
|
|
205
|
+
"observation_count": len(own), "mean_score_or_pass_rate": round(mean, 4),
|
|
206
|
+
"peer_mean": round(peer_mean, 4) if peer_mean is not None else None,
|
|
207
|
+
"descriptive_delta": round(mean - peer_mean, 4) if peer_mean is not None else None,
|
|
208
|
+
"interpretation": "higher_than_peers" if peer_mean is not None and mean > peer_mean else ("lower_than_peers" if peer_mean is not None and mean < peer_mean else "not_comparable")})
|
|
209
|
+
reviewer_rows.append(row)
|
|
210
|
+
return {"status": "descriptive_only", "minimum_cases": min_cases,
|
|
211
|
+
"reviewers": reviewer_rows, "panels": panel_lookup,
|
|
212
|
+
"limitations": ["Reviewer IDs are caller-declared, not authenticated.",
|
|
213
|
+
"Deltas are descriptive and are not bias-corrected scores.",
|
|
214
|
+
"Small samples are flagged; confidence intervals are not estimated.",
|
|
215
|
+
"Comparisons are isolated by rubric version, criterion label, and shared case/model observations."]}
|