system-design-primer-skill 1.2.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/README.md +114 -60
  2. package/SKILL.md +437 -93
  3. package/VERSION +1 -1
  4. package/evals/README.md +1022 -0
  5. package/evals/system-design-cases.json +414 -0
  6. package/package.json +46 -4
  7. package/references/03-data-and-storage.md +8 -2
  8. package/references/04-caching.md +3 -3
  9. package/references/05-async-and-messaging.md +4 -0
  10. package/references/06-distributed-systems.md +2 -2
  11. package/references/07-reliability-and-resilience.md +7 -3
  12. package/references/08-api-security-observability.md +4 -0
  13. package/references/10-workflows-and-checklists.md +1 -1
  14. package/references/13-architecture-intelligence.md +91 -0
  15. package/references/14-api-and-data-contracts.md +80 -0
  16. package/references/15-failure-analysis.md +88 -0
  17. package/references/16-multi-agent-architecture.md +76 -0
  18. package/references/17-agent-harness-and-runtime.md +87 -0
  19. package/references/18-mcp-architecture.md +67 -0
  20. package/references/19-agent-security-and-permissions.md +90 -0
  21. package/references/20-agent-state-and-memory.md +76 -0
  22. package/references/21-agent-evaluation-and-observability.md +90 -0
  23. package/references/22-kubernetes-and-container-platforms.md +29 -0
  24. package/references/23-service-mesh-and-traffic-management.md +21 -0
  25. package/references/24-multi-region-and-disaster-recovery.md +28 -0
  26. package/references/25-platform-engineering-and-developer-experience.md +26 -0
  27. package/references/26-finops-and-cloud-cost.md +29 -0
  28. package/references/27-production-observability-and-slo.md +33 -0
  29. package/references/28-event-streaming-and-kafka-pulsar.md +33 -0
  30. package/references/29-event-sourcing-and-cqrs.md +23 -0
  31. package/references/30-change-data-capture-and-debezium.md +27 -0
  32. package/references/31-data-lakehouse-and-stream-batch.md +24 -0
  33. package/references/32-workflow-orchestration-and-temporal.md +28 -0
  34. package/references/33-schema-registry-and-event-governance.md +22 -0
  35. package/references/34-postgresql-internals-and-performance.md +45 -0
  36. package/references/35-mysql-innodb-and-performance.md +29 -0
  37. package/references/36-sharding-and-distributed-sql.md +22 -0
  38. package/references/37-consensus-and-distributed-transactions.md +29 -0
  39. package/references/38-search-architecture-elasticsearch-opensearch.md +24 -0
  40. package/references/39-vector-databases-and-hybrid-retrieval.md +31 -0
  41. package/references/40-zero-downtime-database-migrations.md +27 -0
  42. package/references/41-api-gateway-and-edge-architecture.md +23 -0
  43. package/references/42-rest-api-design-and-lifecycle.md +23 -0
  44. package/references/43-graphql-architecture-and-governance.md +16 -0
  45. package/references/44-grpc-and-service-contracts.md +16 -0
  46. package/references/45-websocket-sse-and-realtime-systems.md +16 -0
  47. package/references/46-realtime-collaboration-ot-crdt.md +16 -0
  48. package/references/47-api-security-rate-limiting-and-abuse.md +16 -0
  49. package/references/48-oauth-oidc-and-passkeys.md +20 -0
  50. package/references/49-iam-rbac-abac-rebac.md +20 -0
  51. package/references/50-zero-trust-and-workload-identity.md +16 -0
  52. package/references/51-secrets-kms-pki-and-key-management.md +20 -0
  53. package/references/52-supply-chain-security-sbom-slsa.md +78 -0
  54. package/references/53-threat-modeling-stride-and-attack-trees.md +16 -0
  55. package/references/54-privacy-engineering-and-data-governance.md +20 -0
  56. package/references/55-opentelemetry-and-instrumentation.md +54 -0
  57. package/references/56-distributed-tracing-and-context-propagation.md +47 -0
  58. package/references/57-continuous-profiling-and-performance-diagnostics.md +45 -0
  59. package/references/58-load-stress-soak-testing-and-workload-modeling.md +44 -0
  60. package/references/59-capacity-planning-and-performance-modeling.md +38 -0
  61. package/references/60-chaos-engineering-and-resilience-experiments.md +45 -0
  62. package/references/61-incident-management-and-postmortems.md +38 -0
  63. package/references/62-dns-and-service-discovery.md +35 -0
  64. package/references/63-load-balancing-and-global-traffic-management.md +31 -0
  65. package/references/64-cdn-and-edge-caching.md +34 -0
  66. package/references/65-cloud-networking-and-connectivity.md +34 -0
  67. package/references/66-edge-computing-architecture.md +33 -0
  68. package/references/67-storage-infrastructure-and-storage-systems.md +34 -0
  69. package/references/68-tcp-quic-and-network-performance.md +27 -0
  70. package/references/69-task-routing-and-evidence.md +115 -0
  71. package/scripts/audit_chain.py +36 -0
  72. package/scripts/compare_reports.py +43 -0
  73. package/scripts/evaluate.py +475 -0
  74. package/scripts/evaluation_catalog.py +129 -0
  75. package/scripts/evaluation_fingerprints.py +20 -0
  76. package/scripts/evidence_archive.py +79 -0
  77. package/scripts/evidence_signing.py +98 -0
  78. package/scripts/evidence_timestamp.py +441 -0
  79. package/scripts/install.ps1 +12 -3
  80. package/scripts/install.sh +15 -8
  81. package/scripts/report_integrity.py +184 -0
  82. package/scripts/review_consensus.py +77 -0
  83. package/scripts/review_evidence.py +105 -0
  84. package/scripts/review_provenance.py +44 -0
  85. package/scripts/reviewer_calibration.py +215 -0
  86. package/scripts/rubric_compatibility.py +134 -0
  87. package/scripts/run_evals.py +130 -0
  88. package/scripts/run_manifest.py +179 -0
  89. package/scripts/statistical_analysis.py +308 -0
  90. package/scripts/tsa_attestation.py +709 -0
  91. package/scripts/tsa_validation_report.py +627 -0
  92. package/scripts/validate.py +356 -11
  93. package/scripts/verify_build_provenance.py +625 -0
  94. package/scripts/verify_tsa_ocsp.py +962 -0
  95. package/scripts/verify_tsa_revocation.py +782 -0
  96. package/scripts/windows_broker_protocol.py +274 -0
  97. package/scripts/windows_broker_transport.py +141 -0
  98. package/scripts/windows_broker_win32.py +713 -0
  99. package/templates/agent-evaluation-plan-template.md +57 -0
  100. package/templates/agent-security-review-template.md +46 -0
  101. package/templates/agent-system-design-template.md +74 -0
  102. package/templates/api-architecture-design-template.md +33 -0
  103. package/templates/api-security-and-rate-limit-review-template.md +29 -0
  104. package/templates/architecture-decision-matrix-template.md +48 -0
  105. package/templates/architecture-review-scorecard-template.md +36 -0
  106. package/templates/cdc-data-pipeline-template.md +41 -0
  107. package/templates/cdn-and-edge-strategy-template.md +36 -0
  108. package/templates/chaos-experiment-template.md +39 -0
  109. package/templates/cloud-cost-model-template.md +37 -0
  110. package/templates/database-architecture-review-template.md +33 -0
  111. package/templates/edge-computing-readiness-template.md +36 -0
  112. package/templates/event-sourcing-cqrs-template.md +41 -0
  113. package/templates/event-streaming-design-template.md +47 -0
  114. package/templates/failure-analysis-template.md +36 -0
  115. package/templates/identity-and-access-architecture-template.md +29 -0
  116. package/templates/incident-postmortem-template.md +39 -0
  117. package/templates/multi-region-disaster-recovery-template.md +43 -0
  118. package/templates/network-architecture-design-template.md +38 -0
  119. package/templates/observability-architecture-template.md +52 -0
  120. package/templates/performance-test-plan-template.md +52 -0
  121. package/templates/privacy-data-governance-template.md +24 -0
  122. package/templates/production-readiness-checklist-template.md +52 -0
  123. package/templates/realtime-system-design-template.md +28 -0
  124. package/templates/security-threat-model-template.md +26 -0
  125. package/templates/sharding-distributed-sql-design-template.md +34 -0
  126. package/templates/storage-architecture-design-template.md +35 -0
  127. package/templates/supply-chain-security-review-template.md +20 -0
  128. package/templates/zero-downtime-database-migration-template.md +32 -0
@@ -0,0 +1,215 @@
1
+ from __future__ import annotations
2
+ from itertools import combinations
3
+ import random
4
+ from typing import Any
5
+
6
+ def _cohen_kappa(left: list[int | bool], right: list[int | bool], categories: list[int | bool],
7
+ quadratic: bool = False) -> dict[str, Any]:
8
+ n = len(left)
9
+ if n == 0:
10
+ return {"observations": 0, "observed_agreement": None, "kappa": None}
11
+ observed = sum(a == b for a, b in zip(left, right)) / n
12
+ left_counts = {v: left.count(v) / n for v in categories}
13
+ right_counts = {v: right.count(v) / n for v in categories}
14
+ expected = sum(left_counts[v] * right_counts[v] for v in categories)
15
+ if expected >= 1.0:
16
+ kappa = None
17
+ else:
18
+ kappa = (observed - expected) / (1.0 - expected)
19
+ result: dict[str, Any] = {"observations": n, "observed_agreement": round(observed, 6),
20
+ "expected_agreement": round(expected, 6),
21
+ "kappa": round(kappa, 6) if kappa is not None else None}
22
+ if quadratic and len(categories) > 1:
23
+ low, high = min(categories), max(categories)
24
+ span = high - low
25
+ def weight(a: int | bool, b: int | bool) -> float:
26
+ return ((a - b) / span) ** 2
27
+ observed_disagreement = sum(weight(a, b) for a, b in zip(left, right)) / n
28
+ expected_disagreement = sum(left_counts[a] * right_counts[b] * weight(a, b)
29
+ for a in categories for b in categories)
30
+ weighted = (1 - observed_disagreement / expected_disagreement) if expected_disagreement else None
31
+ result["quadratic_weighted_kappa"] = round(weighted, 6) if weighted is not None else None
32
+ return result
33
+
34
+ def pairwise_agreement(submissions: list[dict[str, Any]], case: dict[str, Any]) -> dict[str, Any]:
35
+ pairs = []
36
+ for left, right in combinations(submissions, 2):
37
+ comparisons = []
38
+ for key in ("required_outputs", "guardrails"):
39
+ for index, criterion in enumerate(case[key]):
40
+ a = left["review"][key][index]
41
+ b = right["review"][key][index]
42
+ comparisons.append({"criterion_type": key, "criterion_index": index,
43
+ "criterion": criterion, "left_value": a,
44
+ "right_value": b, "agreement": a == b})
45
+ pairs.append({"reviewer_ids": [left["reviewer_id"], right["reviewer_id"]],
46
+ "criterion_comparisons": comparisons})
47
+ return {"method": "raw_pairwise_agreement", "pair_count": len(pairs), "pairs": pairs}
48
+
49
+ def _bootstrap_interval(values: list[tuple[int | bool, int | bool]], categories: list[int | bool],
50
+ quadratic: bool, seed: int = 1729, iterations: int = 1000) -> dict[str, Any] | None:
51
+ """Percentile bootstrap CI over paired case observations; omitted for small samples."""
52
+ if len(values) < 20:
53
+ return None
54
+ rng = random.Random(seed)
55
+ estimates: list[float] = []
56
+ for _ in range(iterations):
57
+ sample = [values[rng.randrange(len(values))] for _ in range(len(values))]
58
+ metric = _cohen_kappa([v[0] for v in sample], [v[1] for v in sample], categories, quadratic)
59
+ estimate = metric.get("quadratic_weighted_kappa") if quadratic else metric.get("kappa")
60
+ if estimate is not None:
61
+ estimates.append(float(estimate))
62
+ if len(estimates) < iterations * 0.8:
63
+ return {"status": "unstable", "iterations": iterations, "valid_resamples": len(estimates)}
64
+ estimates.sort()
65
+ low = estimates[int(0.025 * (len(estimates) - 1))]
66
+ high = estimates[int(0.975 * (len(estimates) - 1))]
67
+ return {"status": "estimated", "confidence_level": 0.95, "method": "paired_case_percentile_bootstrap",
68
+ "iterations": iterations, "valid_resamples": len(estimates),
69
+ "lower": round(low, 6), "upper": round(high, 6)}
70
+
71
+
72
+ def summarize_review_data_quality(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
73
+ """Describe panel coverage, reviewer concentration, and score distributions."""
74
+ reviewer_work: dict[tuple[str, str], set[str]] = {}
75
+ reviewer_counts: dict[tuple[str, str], int] = {}
76
+ case_panels: dict[str, int] = {}
77
+ distributions: dict[tuple[str, str, str], dict[str, int]] = {}
78
+ reviewed_records = 0
79
+ for record in records:
80
+ panel = record.get("review_panel")
81
+ if not panel:
82
+ continue
83
+ reviewed_records += 1
84
+ case_id = record["case_id"]
85
+ case = cases[case_id]
86
+ case_model = case_id + "|" + record["model"]
87
+ case_panels[case_model] = panel.get("reviewer_count", len(panel.get("submissions", [])))
88
+ for submission in panel.get("submissions", []):
89
+ rubric = submission["rubric_version"]
90
+ reviewer = submission["reviewer_id"]
91
+ reviewer_work.setdefault((rubric, reviewer), set()).add(case_model)
92
+ reviewer_counts[(rubric, reviewer)] = reviewer_counts.get((rubric, reviewer), 0) + 1
93
+ review = submission["review"]
94
+ for kind in ("required_outputs", "guardrails"):
95
+ for index, value in enumerate(review[kind]):
96
+ criterion = case[kind][index]
97
+ bucket = distributions.setdefault((rubric, kind, criterion), {})
98
+ label = str(value).lower() if isinstance(value, bool) else str(value)
99
+ bucket[label] = bucket.get(label, 0) + 1
100
+ total_records = len(records)
101
+ reviewer_rows = []
102
+ rubric_totals: dict[str, int] = {}
103
+ for (rubric, reviewer), count in reviewer_counts.items():
104
+ rubric_totals[rubric] = rubric_totals.get(rubric, 0) + count
105
+ for (rubric, reviewer), count in sorted(reviewer_counts.items()):
106
+ total = rubric_totals[rubric]
107
+ reviewer_rows.append({"rubric_version": rubric, "reviewer_id": reviewer,
108
+ "reviewed_case_model_count": len(reviewer_work[(rubric, reviewer)]),
109
+ "submission_count": count, "share_of_rubric_submissions": round(count / total, 6)})
110
+ concentration = []
111
+ for rubric, total in sorted(rubric_totals.items()):
112
+ shares = [count / total for (version, _), count in reviewer_counts.items() if version == rubric]
113
+ concentration.append({"rubric_version": rubric, "reviewer_count": len(shares),
114
+ "hhi": round(sum(share * share for share in shares), 6),
115
+ "largest_reviewer_share": round(max(shares), 6) if shares else None,
116
+ "interpretation": "high_concentration" if shares and max(shares) >= 0.5 else "distributed"})
117
+ case_rows = [{"case_id": key.split("|", 1)[0], "model": key.split("|", 1)[1],
118
+ "reviewer_count": count} for key, count in sorted(case_panels.items())]
119
+ distribution_rows = []
120
+ for (rubric, kind, criterion), counts in sorted(distributions.items()):
121
+ total = sum(counts.values())
122
+ distribution_rows.append({"rubric_version": rubric, "criterion_type": kind, "criterion": criterion,
123
+ "observation_count": total, "category_counts": counts,
124
+ "category_shares": {key: round(value / total, 6) for key, value in sorted(counts.items())}})
125
+ return {"status": "descriptive_only", "response_record_count": total_records,
126
+ "reviewed_record_count": reviewed_records,
127
+ "panel_coverage_rate": round(reviewed_records / total_records, 6) if total_records else None,
128
+ "unreviewed_record_count": total_records - reviewed_records,
129
+ "case_model_panel_coverage": case_rows, "reviewer_coverage": reviewer_rows,
130
+ "reviewer_concentration": concentration, "score_distributions": distribution_rows,
131
+ "limitations": ["Coverage is measured over supplied response records, not the full intended population unless all responses are supplied.",
132
+ "HHI and category distributions are descriptive and do not establish reviewer quality.",
133
+ "Repeated models or cases may not be statistically independent."]}
134
+
135
+ def aggregate_reviewer_calibration(panels: list[dict[str, Any]]) -> dict[str, Any]:
136
+ """Aggregate per-reviewer-pair categorical observations across independently reviewed cases."""
137
+ buckets: dict[tuple[str, str, str, str, str], list[tuple[int | bool, int | bool]]] = {}
138
+ for panel in panels:
139
+ submissions = panel["submissions"]
140
+ for left, right in combinations(submissions, 2):
141
+ if left.get("rubric_version") != right.get("rubric_version"):
142
+ continue
143
+ pair = tuple(sorted((left["reviewer_id"], right["reviewer_id"])))
144
+ rubric = left["rubric_version"]
145
+ for key in ("required_outputs", "guardrails"):
146
+ for index in range(len(left["review"][key])):
147
+ label = panel.get("case", {}).get(key, [])[index] if panel.get("case") else str(index)
148
+ bucket = (rubric, pair[0], pair[1], key, label)
149
+ buckets.setdefault(bucket, []).append((left["review"][key][index], right["review"][key][index]))
150
+ results = []
151
+ for (rubric, a_id, b_id, key, label), values in sorted(buckets.items()):
152
+ categories = [0, 1, 2] if key == "required_outputs" else [False, True]
153
+ metric = _cohen_kappa([v[0] for v in values], [v[1] for v in values],
154
+ categories, quadratic=(key == "required_outputs"))
155
+ confidence = _bootstrap_interval(values, categories, quadratic=(key == "required_outputs"))
156
+ results.append({"rubric_version": rubric, "reviewer_ids": [a_id, b_id], "criterion_type": key,
157
+ "criterion": label, **metric, "kappa_confidence_interval_95": confidence})
158
+ return {"method": "pairwise_cohen_kappa", "minimum_observations": 2,
159
+ "results": results,
160
+ "low_agreement_pairs": [item for item in results if item["observations"] >= 2
161
+ and item["kappa"] is not None and item["kappa"] < 0.6]}
162
+
163
+ def summarize_reviewer_behavior(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]],
164
+ min_cases: int = 3) -> dict[str, Any]:
165
+ """Describe reviewer scoring tendencies; never normalize or mutate model scores."""
166
+ observations: dict[tuple[str, str, str], list[tuple[str, str, float]]] = {}
167
+ reviewer_cases: dict[tuple[str, str], set[str]] = {}
168
+ panel_lookup: list[dict[str, Any]] = []
169
+ for record in records:
170
+ panel = record.get("review_panel")
171
+ if not panel:
172
+ continue
173
+ case_id = record["case_id"]
174
+ case = cases[case_id]
175
+ for submission in panel.get("submissions", []):
176
+ reviewer = submission["reviewer_id"]
177
+ rubric = submission["rubric_version"]
178
+ reviewer_cases.setdefault((rubric, reviewer), set()).add(case_id)
179
+ review = submission["review"]
180
+ for index, score in enumerate(review["required_outputs"]):
181
+ observations.setdefault((rubric, "required_outputs", case["required_outputs"][index]), []).append((reviewer, case_id + "|" + record["model"], float(score)))
182
+ for index, passed in enumerate(review["guardrails"]):
183
+ observations.setdefault((rubric, "guardrails", case["guardrails"][index]), []).append((reviewer, case_id + "|" + record["model"], 1.0 if passed else 0.0))
184
+ panel_lookup.append({"case_id": case_id, "rubric_version": sorted({s["rubric_version"] for s in panel.get("submissions", [])}),
185
+ "reviewer_ids": [s["reviewer_id"] for s in panel.get("submissions", [])],
186
+ "disagreement_count": panel.get("disagreement_count", 0),
187
+ "status": panel.get("status")})
188
+ reviewer_rows = []
189
+ for (rubric, reviewer), case_ids in sorted(reviewer_cases.items()):
190
+ row: dict[str, Any] = {"rubric_version": rubric, "reviewer_id": reviewer,
191
+ "reviewed_case_count": len(case_ids),
192
+ "sample_status": "sufficient" if len(case_ids) >= min_cases else "insufficient_sample",
193
+ "minimum_cases": min_cases, "criterion_tendencies": []}
194
+ for (r_version, kind, criterion), values in sorted(observations.items()):
195
+ if r_version != rubric:
196
+ continue
197
+ own = [value for who, _, value in values if who == reviewer]
198
+ own_case_keys = {case_key for who, case_key, _ in values if who == reviewer}
199
+ peers = [value for who, case_key, value in values if who != reviewer and case_key in own_case_keys]
200
+ if not own:
201
+ continue
202
+ mean = sum(own) / len(own)
203
+ peer_mean = sum(peers) / len(peers) if peers else None
204
+ row["criterion_tendencies"].append({"criterion_type": kind, "criterion": criterion,
205
+ "observation_count": len(own), "mean_score_or_pass_rate": round(mean, 4),
206
+ "peer_mean": round(peer_mean, 4) if peer_mean is not None else None,
207
+ "descriptive_delta": round(mean - peer_mean, 4) if peer_mean is not None else None,
208
+ "interpretation": "higher_than_peers" if peer_mean is not None and mean > peer_mean else ("lower_than_peers" if peer_mean is not None and mean < peer_mean else "not_comparable")})
209
+ reviewer_rows.append(row)
210
+ return {"status": "descriptive_only", "minimum_cases": min_cases,
211
+ "reviewers": reviewer_rows, "panels": panel_lookup,
212
+ "limitations": ["Reviewer IDs are caller-declared, not authenticated.",
213
+ "Deltas are descriptive and are not bias-corrected scores.",
214
+ "Small samples are flagged; confidence intervals are not estimated.",
215
+ "Comparisons are isolated by rubric version, criterion label, and shared case/model observations."]}
@@ -0,0 +1,134 @@
1
+ #!/usr/bin/env python3
2
+ """Conservatively assess whether two rubric revisions can be compared."""
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ TOP_FIELDS = {"rubric_version", "rubric_sha256", "score_scale", "criteria"}
12
+ CRITERIA_FIELDS = {"required_outputs", "guardrails", "security_categories"}
13
+ MIGRATION_FIELDS = {"from_version", "to_version", "decision", "rationale", "mapping"}
14
+
15
+
16
+ def load_definition(path: Path) -> dict[str, Any]:
17
+ value = json.loads(path.read_text(encoding="utf-8"))
18
+ if not isinstance(value, dict) or set(value) != TOP_FIELDS:
19
+ raise ValueError("rubric definition must contain exactly rubric_version, rubric_sha256, score_scale, and criteria")
20
+ if not isinstance(value["rubric_version"], str) or not value["rubric_version"].strip():
21
+ raise ValueError("rubric_version must be a non-empty string")
22
+ digest = value["rubric_sha256"]
23
+ if digest is not None and (not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest)):
24
+ raise ValueError("rubric_sha256 must be a lowercase SHA-256 digest or null")
25
+ scale = value["score_scale"]
26
+ if not isinstance(scale, dict) or set(scale) != {"minimum", "maximum", "labels"}:
27
+ raise ValueError("score_scale must contain minimum, maximum, and labels")
28
+ if type(scale["minimum"]) is not int or type(scale["maximum"]) is not int or scale["minimum"] >= scale["maximum"]:
29
+ raise ValueError("score_scale bounds must be integers with minimum < maximum")
30
+ if not isinstance(scale["labels"], dict):
31
+ raise ValueError("score_scale.labels must be an object")
32
+ criteria = value["criteria"]
33
+ if not isinstance(criteria, dict) or set(criteria) != CRITERIA_FIELDS:
34
+ raise ValueError("criteria must contain required_outputs, guardrails, and security_categories")
35
+ for field, entries in criteria.items():
36
+ if not isinstance(entries, list) or any(not isinstance(item, str) or not item.strip() for item in entries):
37
+ raise ValueError(f"criteria.{field} must be an array of non-empty strings")
38
+ if len(entries) != len(set(entries)):
39
+ raise ValueError(f"criteria.{field} must not contain duplicates")
40
+ return value
41
+
42
+
43
+ def _migration_result(left: dict[str, Any], right: dict[str, Any], migration: Any) -> dict[str, Any] | None:
44
+ if migration is None:
45
+ return None
46
+ if not isinstance(migration, dict) or set(migration) != MIGRATION_FIELDS:
47
+ raise ValueError("migration must contain exactly from_version, to_version, decision, rationale, and mapping")
48
+ if migration["from_version"] != left["rubric_version"] or migration["to_version"] != right["rubric_version"]:
49
+ raise ValueError("migration from_version/to_version must match the compared rubric versions")
50
+ if migration["decision"] not in ("equivalent", "mapped"):
51
+ raise ValueError("migration decision must be equivalent or mapped")
52
+ if not isinstance(migration["rationale"], str) or not migration["rationale"].strip():
53
+ raise ValueError("migration rationale must be non-empty")
54
+ if not isinstance(migration["mapping"], dict):
55
+ raise ValueError("migration mapping must be an object")
56
+ if migration["decision"] == "equivalent" and migration["mapping"]:
57
+ raise ValueError("equivalent migration must use an empty mapping")
58
+ if migration["decision"] == "mapped" and not migration["mapping"]:
59
+ raise ValueError("mapped migration must provide an explicit mapping")
60
+ return {"declared": True, "decision": migration["decision"], "rationale": migration["rationale"],
61
+ "mapping": migration["mapping"], "from_version": migration["from_version"],
62
+ "to_version": migration["to_version"]}
63
+
64
+
65
+ def compare_rubrics(left: dict[str, Any], right: dict[str, Any], migration: Any = None) -> dict[str, Any]:
66
+ left_missing = [key for key in TOP_FIELDS if key not in left]
67
+ right_missing = [key for key in TOP_FIELDS if key not in right]
68
+ if left_missing or right_missing:
69
+ return {"status": "unknown", "comparable": False, "reason": "rubric metadata is incomplete",
70
+ "left_missing_fields": sorted(left_missing), "right_missing_fields": sorted(right_missing)}
71
+ if left["rubric_version"] == right["rubric_version"] and left == right:
72
+ return {"status": "compatible", "comparable": True, "reason": "identical rubric definition", "changes": []}
73
+ changes, breaking, unknown = [], [], []
74
+ if left["score_scale"] != right["score_scale"]:
75
+ item = {"field": "score_scale", "left": left["score_scale"], "right": right["score_scale"], "impact": "breaking"}
76
+ changes.append(item)
77
+ breaking.append(item)
78
+ for field in sorted(CRITERIA_FIELDS):
79
+ a, b = left["criteria"][field], right["criteria"][field]
80
+ if a == b:
81
+ continue
82
+ if len(a) != len(b):
83
+ item = {"field": f"criteria.{field}", "added": sorted(set(b) - set(a)),
84
+ "removed": sorted(set(a) - set(b)), "impact": "breaking"}
85
+ changes.append(item)
86
+ breaking.append(item)
87
+ elif set(a) == set(b):
88
+ item = {"field": f"criteria.{field}", "impact": "breaking", "reason": "criterion order changed"}
89
+ changes.append(item)
90
+ breaking.append(item)
91
+ else:
92
+ item = {"field": f"criteria.{field}", "impact": "unknown",
93
+ "reason": "criterion text changed; semantic equivalence cannot be inferred"}
94
+ changes.append(item)
95
+ unknown.append(item)
96
+ if left["rubric_sha256"] is None or right["rubric_sha256"] is None:
97
+ unknown.append({"field": "rubric_sha256", "impact": "unknown", "reason": "fingerprint missing"})
98
+ elif left["rubric_sha256"] != right["rubric_sha256"] and not changes:
99
+ unknown.append({"field": "rubric_sha256", "impact": "unknown", "reason": "fingerprint differs without an explainable structural diff"})
100
+ if left["rubric_version"] != right["rubric_version"] and not changes:
101
+ unknown.append({"field": "rubric_version", "impact": "unknown", "reason": "version differs without a declared semantic change"})
102
+ declared = _migration_result(left, right, migration)
103
+ if declared:
104
+ return {"status": "compatible" if declared["decision"] == "equivalent" else "mapped",
105
+ "comparable": True, "reason": "explicit caller-supplied migration declaration",
106
+ "changes": changes, "breaking_changes": breaking, "unknown_changes": unknown,
107
+ "migration": declared,
108
+ "warning": "A migration declaration is an assertion to audit, not proof that historical scores were correctly transformed."}
109
+ status = "incompatible" if breaking else ("unknown" if unknown else "compatible")
110
+ return {"status": status, "comparable": status == "compatible",
111
+ "reason": "breaking scoring changes detected" if breaking else
112
+ ("insufficient evidence to establish equivalence" if unknown else "no scoring-semantic changes detected"),
113
+ "changes": changes, "breaking_changes": breaking, "unknown_changes": unknown, "migration": None}
114
+
115
+
116
+ def main() -> int:
117
+ parser = argparse.ArgumentParser(description=__doc__)
118
+ parser.add_argument("--left", type=Path, required=True)
119
+ parser.add_argument("--right", type=Path, required=True)
120
+ parser.add_argument("--migration", type=Path, help="Optional explicit migration declaration JSON")
121
+ args = parser.parse_args()
122
+ try:
123
+ left, right = load_definition(args.left), load_definition(args.right)
124
+ migration = json.loads(args.migration.read_text(encoding="utf-8")) if args.migration else None
125
+ result = compare_rubrics(left, right, migration)
126
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
127
+ print(f"ERROR: {exc}", file=sys.stderr)
128
+ return 1
129
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True, indent=2))
130
+ return 0 if result["comparable"] else 2
131
+
132
+
133
+ if __name__ == "__main__":
134
+ raise SystemExit(main())
@@ -0,0 +1,130 @@
1
+ #!/usr/bin/env python3
2
+ """Run system-design fixtures through a provider-neutral Python model adapter.
3
+
4
+ The adapter receives one JSON object on stdin and must emit one JSON object on stdout.
5
+ No provider SDK or credentials are managed by this runner.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import json
11
+ import subprocess
12
+ import sys
13
+ import time
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ try:
18
+ from .evaluate import load_cases, validate_tool_trace
19
+ from .run_manifest import load_verified_manifest
20
+ except ImportError: # Direct execution: python scripts/run_evals.py
21
+ from evaluate import load_cases, validate_tool_trace
22
+ from run_manifest import load_verified_manifest
23
+
24
+
25
+ def invoke_adapter(
26
+ adapter: Path,
27
+ adapter_args: list[str],
28
+ case: dict[str, Any],
29
+ model: str,
30
+ timeout: int,
31
+ ) -> tuple[str, int, list[dict[str, Any]] | None]:
32
+ request = {
33
+ "schema_version": 1,
34
+ "case_id": case["id"],
35
+ "model": model,
36
+ "task": case["task"],
37
+ "expected_mode": case.get("expected_mode"),
38
+ "references": case.get("references", []),
39
+ }
40
+ started = time.monotonic()
41
+ try:
42
+ completed = subprocess.run(
43
+ [sys.executable, str(adapter), *adapter_args],
44
+ input=json.dumps(request, ensure_ascii=False),
45
+ text=True,
46
+ encoding="utf-8",
47
+ capture_output=True,
48
+ timeout=timeout,
49
+ check=False,
50
+ )
51
+ except subprocess.TimeoutExpired as exc:
52
+ raise RuntimeError(f"adapter timed out after {timeout}s") from exc
53
+ elapsed_ms = round((time.monotonic() - started) * 1000)
54
+ if completed.returncode != 0:
55
+ raise RuntimeError(f"adapter exited with status {completed.returncode}")
56
+ try:
57
+ response = json.loads(completed.stdout)
58
+ except json.JSONDecodeError as exc:
59
+ raise RuntimeError("adapter stdout must contain one JSON object") from exc
60
+ if not isinstance(response, dict) or not isinstance(response.get("output"), str):
61
+ raise RuntimeError("adapter response must be an object with a string 'output' field")
62
+ trace = response.get("tool_trace")
63
+ if trace is not None:
64
+ try:
65
+ validate_tool_trace(trace)
66
+ except ValueError as exc:
67
+ raise RuntimeError(f"adapter returned invalid tool_trace: {exc}") from exc
68
+ if not response["output"].strip():
69
+ raise RuntimeError("adapter returned an empty output")
70
+ return response["output"], elapsed_ms, trace
71
+
72
+
73
+ def main() -> int:
74
+ parser = argparse.ArgumentParser(description=__doc__)
75
+ parser.add_argument("--cases", type=Path, default=Path("evals/system-design-cases.json"))
76
+ parser.add_argument("--adapter", type=Path, required=True, help="Python adapter script")
77
+ parser.add_argument("--adapter-arg", action="append", default=[], help="Argument passed to the adapter; may be repeated")
78
+ parser.add_argument("--model", required=True, help="Stable provider/model/version label for this run")
79
+ parser.add_argument("--output", type=Path, required=True, help="New JSONL file; existing files are never overwritten")
80
+ parser.add_argument("--manifest", type=Path, help="Sealed run manifest; embeds its run identity and fingerprint in every response")
81
+ parser.add_argument("--timeout", type=int, default=180, help="Per-case adapter timeout in seconds (default: 180)")
82
+ args = parser.parse_args()
83
+
84
+ if args.timeout < 1:
85
+ parser.error("--timeout must be at least 1 second")
86
+ if not args.adapter.is_file():
87
+ parser.error(f"adapter script does not exist: {args.adapter}")
88
+ try:
89
+ cases = load_cases(args.cases)
90
+ manifest = load_verified_manifest(args.manifest) if args.manifest else None
91
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
92
+ print(f"ERROR: cannot load cases or verify manifest: {exc}", file=sys.stderr)
93
+ return 1
94
+
95
+ try:
96
+ with args.output.open("x", encoding="utf-8", newline="\n") as stream:
97
+ for case in cases.values():
98
+ try:
99
+ output, elapsed_ms, tool_trace = invoke_adapter(
100
+ args.adapter, args.adapter_arg, case, args.model, args.timeout
101
+ )
102
+ except RuntimeError as exc:
103
+ print(f"ERROR: case {case['id']}: {exc}", file=sys.stderr)
104
+ return 1
105
+ record = {
106
+ "case_id": case["id"],
107
+ "model": args.model,
108
+ "task": case["task"],
109
+ "output": output,
110
+ "duration_ms": elapsed_ms,
111
+ }
112
+ if manifest is not None:
113
+ record["run_id"] = manifest["run"]["run_id"]
114
+ record["manifest_sha256"] = manifest["fingerprints"]["manifest_sha256"]
115
+ if tool_trace is not None:
116
+ record["tool_trace"] = tool_trace
117
+ stream.write(json.dumps(record, ensure_ascii=False) + "\n")
118
+ stream.flush()
119
+ print(f"Completed {case['id']} ({elapsed_ms} ms)", file=sys.stderr)
120
+ except FileExistsError:
121
+ print(f"ERROR: refusing to overwrite existing output: {args.output}", file=sys.stderr)
122
+ return 1
123
+ except OSError as exc:
124
+ print(f"ERROR: cannot write output: {exc}", file=sys.stderr)
125
+ return 1
126
+ return 0
127
+
128
+
129
+ if __name__ == "__main__":
130
+ raise SystemExit(main())
@@ -0,0 +1,179 @@
1
+ #!/usr/bin/env python3
2
+ """Create, verify, and compare versioned evaluation run manifests."""
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import re
9
+ import sys
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ SCHEMA_VERSION = 1
14
+ HASH_RE = re.compile(r"^[0-9a-f]{64}$")
15
+ TOP_LEVEL = {"schema_version", "run", "configuration", "fingerprints"}
16
+ RUN_FIELDS = {"run_id", "created_at", "code_revision"}
17
+ CONFIG_FIELDS = {
18
+ "model", "system_prompt_sha256", "dataset_sha256", "evaluator_version",
19
+ "tool_permission_profile", "rubric_version", "adapter_version",
20
+ "run_config", "runtime",
21
+ }
22
+
23
+
24
+ def canonical_json(value: Any) -> bytes:
25
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
26
+
27
+
28
+ def digest(value: Any) -> str:
29
+ return hashlib.sha256(canonical_json(value)).hexdigest()
30
+
31
+
32
+ def require_text(value: Any, label: str) -> None:
33
+ if not isinstance(value, str) or not value.strip():
34
+ raise ValueError(f"{label} must be a non-empty string")
35
+
36
+
37
+ def validate_manifest(manifest: dict[str, Any], require_fingerprints: bool = False) -> None:
38
+ if set(manifest) != (TOP_LEVEL if require_fingerprints else TOP_LEVEL - {"fingerprints"}):
39
+ raise ValueError("manifest has missing or unknown top-level fields")
40
+ if manifest.get("schema_version") != SCHEMA_VERSION:
41
+ raise ValueError("unsupported manifest schema_version")
42
+ run = manifest.get("run")
43
+ config = manifest.get("configuration")
44
+ if not isinstance(run, dict) or set(run) != RUN_FIELDS:
45
+ raise ValueError("run must contain exactly run_id, created_at, and code_revision")
46
+ if not isinstance(config, dict) or set(config) != CONFIG_FIELDS:
47
+ raise ValueError("configuration has missing or unknown fields")
48
+ require_text(run["run_id"], "run.run_id")
49
+ require_text(run["created_at"], "run.created_at")
50
+ if run["code_revision"] is not None:
51
+ require_text(run["code_revision"], "run.code_revision")
52
+ model = config["model"]
53
+ if not isinstance(model, dict) or set(model) != {"provider", "name", "revision"}:
54
+ raise ValueError("configuration.model must contain provider, name, and revision")
55
+ require_text(model["provider"], "model.provider")
56
+ require_text(model["name"], "model.name")
57
+ if model["revision"] is not None:
58
+ require_text(model["revision"], "model.revision")
59
+ for field in ("system_prompt_sha256", "dataset_sha256"):
60
+ value = config[field]
61
+ if value is not None and (not isinstance(value, str) or not HASH_RE.fullmatch(value)):
62
+ raise ValueError(f"{field} must be a lowercase SHA-256 hex digest or null")
63
+ for field in ("evaluator_version", "tool_permission_profile", "rubric_version", "adapter_version"):
64
+ require_text(config[field], f"configuration.{field}")
65
+ if not isinstance(config["run_config"], dict):
66
+ raise ValueError("configuration.run_config must be an object")
67
+ if not isinstance(config["runtime"], dict) or not config["runtime"]:
68
+ raise ValueError("configuration.runtime must be a non-empty object")
69
+ if require_fingerprints:
70
+ fingerprints = manifest.get("fingerprints")
71
+ if not isinstance(fingerprints, dict) or set(fingerprints) != {"manifest_sha256", "configuration_sha256"}:
72
+ raise ValueError("fingerprints must contain manifest_sha256 and configuration_sha256")
73
+
74
+
75
+ def seal_manifest(raw: dict[str, Any]) -> dict[str, Any]:
76
+ validate_manifest(raw)
77
+ configuration = raw["configuration"]
78
+ sealed = dict(raw)
79
+ sealed["fingerprints"] = {
80
+ "manifest_sha256": digest(raw),
81
+ "configuration_sha256": digest(configuration),
82
+ }
83
+ return sealed
84
+
85
+
86
+ def verify_manifest(manifest: dict[str, Any]) -> None:
87
+ validate_manifest(manifest, require_fingerprints=True)
88
+ expected = seal_manifest({key: manifest[key] for key in ("schema_version", "run", "configuration")})
89
+ if manifest["fingerprints"] != expected["fingerprints"]:
90
+ raise ValueError("manifest fingerprint mismatch")
91
+
92
+
93
+ def compare_manifests(left: dict[str, Any], right: dict[str, Any]) -> dict[str, Any]:
94
+ verify_manifest(left)
95
+ verify_manifest(right)
96
+ a, b = left["configuration"], right["configuration"]
97
+ differences: list[dict[str, Any]] = []
98
+ unknown: list[str] = []
99
+ comparable = [
100
+ "model.provider", "model.name", "model.revision", "system_prompt_sha256",
101
+ "dataset_sha256", "evaluator_version", "tool_permission_profile",
102
+ "rubric_version", "adapter_version", "run_config", "runtime",
103
+ ]
104
+ def get_path(obj: dict[str, Any], path: str) -> Any:
105
+ current: Any = obj
106
+ for part in path.split("."):
107
+ current = current[part]
108
+ return current
109
+ for path in comparable:
110
+ av, bv = get_path(a, path), get_path(b, path)
111
+ if av is None or bv is None:
112
+ unknown.append(path)
113
+ elif av != bv:
114
+ differences.append({"field": path, "left": av, "right": bv})
115
+ return {
116
+ "same_configuration": not differences and not unknown,
117
+ "comparable": not unknown,
118
+ "left_manifest_sha256": left["fingerprints"]["manifest_sha256"],
119
+ "right_manifest_sha256": right["fingerprints"]["manifest_sha256"],
120
+ "differences": differences,
121
+ "unknown_fields": unknown,
122
+ "note": "Matching configuration fingerprints establish metadata equality only; they do not guarantee deterministic model outputs.",
123
+ }
124
+
125
+
126
+ def load_verified_manifest(path: Path) -> dict[str, Any]:
127
+ """Load a sealed manifest and return it only after fingerprint verification."""
128
+ manifest = json.loads(path.read_text(encoding="utf-8"))
129
+ if not isinstance(manifest, dict):
130
+ raise ValueError(f"{path}: expected a JSON object")
131
+ verify_manifest(manifest)
132
+ return manifest
133
+
134
+
135
+ def read_json(path: Path) -> dict[str, Any]:
136
+ value = json.loads(path.read_text(encoding="utf-8"))
137
+ if not isinstance(value, dict):
138
+ raise ValueError(f"{path}: expected a JSON object")
139
+ return value
140
+
141
+
142
+ def write_new(path: Path, value: dict[str, Any]) -> None:
143
+ rendered = json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n"
144
+ with path.open("x", encoding="utf-8", newline="\n") as stream:
145
+ stream.write(rendered)
146
+
147
+
148
+ def main() -> int:
149
+ parser = argparse.ArgumentParser(description=__doc__)
150
+ sub = parser.add_subparsers(dest="command", required=True)
151
+ seal = sub.add_parser("seal", help="Validate an unsigned manifest and write its fingerprints")
152
+ seal.add_argument("--input", type=Path, required=True)
153
+ seal.add_argument("--output", type=Path, required=True)
154
+ verify = sub.add_parser("verify", help="Verify a sealed manifest")
155
+ verify.add_argument("--input", type=Path, required=True)
156
+ compare = sub.add_parser("compare", help="Compare two sealed manifests")
157
+ compare.add_argument("--left", type=Path, required=True)
158
+ compare.add_argument("--right", type=Path, required=True)
159
+ args = parser.parse_args()
160
+ try:
161
+ if args.command == "seal":
162
+ result = seal_manifest(read_json(args.input))
163
+ write_new(args.output, result)
164
+ output: dict[str, Any] = {"sealed": True, **result["fingerprints"], "output": str(args.output)}
165
+ elif args.command == "verify":
166
+ manifest = read_json(args.input)
167
+ verify_manifest(manifest)
168
+ output = {"verified": True, **manifest["fingerprints"]}
169
+ else:
170
+ output = compare_manifests(read_json(args.left), read_json(args.right))
171
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
172
+ print(f"ERROR: {exc}", file=sys.stderr)
173
+ return 1
174
+ print(json.dumps(output, ensure_ascii=False, sort_keys=True, indent=2))
175
+ return 0
176
+
177
+
178
+ if __name__ == "__main__":
179
+ raise SystemExit(main())