system-design-primer-skill 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/SKILL.md +22 -2
- package/VERSION +1 -1
- package/evals/README.md +1022 -0
- package/evals/system-design-cases.json +414 -0
- package/package.json +3 -2
- package/references/03-data-and-storage.md +8 -2
- package/references/04-caching.md +3 -3
- package/references/05-async-and-messaging.md +4 -0
- package/references/06-distributed-systems.md +2 -2
- package/references/07-reliability-and-resilience.md +7 -3
- package/references/08-api-security-observability.md +4 -0
- package/references/15-failure-analysis.md +4 -0
- package/references/19-agent-security-and-permissions.md +22 -6
- package/references/20-agent-state-and-memory.md +4 -0
- package/references/22-kubernetes-and-container-platforms.md +1 -1
- package/references/24-multi-region-and-disaster-recovery.md +6 -2
- package/references/27-production-observability-and-slo.md +4 -0
- package/references/28-event-streaming-and-kafka-pulsar.md +6 -0
- package/references/32-workflow-orchestration-and-temporal.md +4 -0
- package/references/34-postgresql-internals-and-performance.md +2 -0
- package/references/35-mysql-innodb-and-performance.md +2 -0
- package/references/36-sharding-and-distributed-sql.md +1 -1
- package/references/37-consensus-and-distributed-transactions.md +7 -3
- package/references/40-zero-downtime-database-migrations.md +3 -3
- package/references/48-oauth-oidc-and-passkeys.md +3 -3
- package/references/49-iam-rbac-abac-rebac.md +7 -3
- package/references/51-secrets-kms-pki-and-key-management.md +4 -0
- package/references/52-supply-chain-security-sbom-slsa.md +62 -0
- package/references/54-privacy-engineering-and-data-governance.md +4 -0
- package/references/55-opentelemetry-and-instrumentation.md +4 -0
- package/references/56-distributed-tracing-and-context-propagation.md +4 -0
- package/references/59-capacity-planning-and-performance-modeling.md +8 -4
- package/references/60-chaos-engineering-and-resilience-experiments.md +4 -0
- package/references/61-incident-management-and-postmortems.md +4 -0
- package/references/67-storage-infrastructure-and-storage-systems.md +4 -0
- package/references/68-tcp-quic-and-network-performance.md +1 -1
- package/references/69-task-routing-and-evidence.md +75 -18
- package/scripts/audit_chain.py +36 -0
- package/scripts/compare_reports.py +43 -0
- package/scripts/evaluate.py +475 -0
- package/scripts/evaluation_catalog.py +129 -0
- package/scripts/evaluation_fingerprints.py +20 -0
- package/scripts/evidence_archive.py +79 -0
- package/scripts/evidence_signing.py +98 -0
- package/scripts/evidence_timestamp.py +441 -0
- package/scripts/install.ps1 +2 -2
- package/scripts/install.sh +2 -2
- package/scripts/report_integrity.py +184 -0
- package/scripts/review_consensus.py +77 -0
- package/scripts/review_evidence.py +105 -0
- package/scripts/review_provenance.py +44 -0
- package/scripts/reviewer_calibration.py +215 -0
- package/scripts/rubric_compatibility.py +134 -0
- package/scripts/run_evals.py +130 -0
- package/scripts/run_manifest.py +179 -0
- package/scripts/statistical_analysis.py +308 -0
- package/scripts/tsa_attestation.py +709 -0
- package/scripts/tsa_validation_report.py +627 -0
- package/scripts/validate.py +233 -21
- package/scripts/verify_build_provenance.py +625 -0
- package/scripts/verify_tsa_ocsp.py +962 -0
- package/scripts/verify_tsa_revocation.py +782 -0
- package/scripts/windows_broker_protocol.py +274 -0
- package/scripts/windows_broker_transport.py +141 -0
- package/scripts/windows_broker_win32.py +713 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from itertools import combinations
|
|
4
|
+
import random
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
MIN_PAIRED_CASES = 5
|
|
8
|
+
PRACTICAL_EFFECT_THRESHOLD = 0.05
|
|
9
|
+
BOOTSTRAP_ITERATIONS = 2000
|
|
10
|
+
PERMUTATION_ITERATIONS = 10000
|
|
11
|
+
SEED = 271828
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _case_metrics(record: dict[str, Any], case: dict[str, Any]) -> dict[str, float] | None:
|
|
15
|
+
review = record.get("review")
|
|
16
|
+
if review is None:
|
|
17
|
+
return None
|
|
18
|
+
return {
|
|
19
|
+
"required_output_score": sum(review["required_outputs"]) / (2 * len(case["required_outputs"])),
|
|
20
|
+
"guardrail_pass_rate": sum(review["guardrails"]) / len(case["guardrails"]),
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _weighted_mean(values: list[float], weights: list[float]) -> float:
|
|
25
|
+
return sum(value * weight for value, weight in zip(values, weights)) / sum(weights)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _weighted_sd(values: list[float], weights: list[float]) -> float:
|
|
29
|
+
mean = _weighted_mean(values, weights)
|
|
30
|
+
return (sum(weight * (value - mean) ** 2 for value, weight in zip(values, weights)) / sum(weights)) ** 0.5
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _paired_permutation_pvalue(differences: list[float], weights: list[float], seed: int,
|
|
34
|
+
iterations: int = PERMUTATION_ITERATIONS) -> dict[str, Any]:
|
|
35
|
+
observed = abs(_weighted_mean(differences, weights))
|
|
36
|
+
n = len(differences)
|
|
37
|
+
if n <= 16:
|
|
38
|
+
total = 1 << n
|
|
39
|
+
extreme = 0
|
|
40
|
+
for mask in range(total):
|
|
41
|
+
signed = [value if (mask >> index) & 1 else -value for index, value in enumerate(differences)]
|
|
42
|
+
if abs(_weighted_mean(signed, weights)) >= observed - 1e-15:
|
|
43
|
+
extreme += 1
|
|
44
|
+
return {"p_value": extreme / total, "method": "exact_paired_sign_flip", "permutations": total}
|
|
45
|
+
rng = random.Random(seed)
|
|
46
|
+
extreme = 0
|
|
47
|
+
for _ in range(iterations):
|
|
48
|
+
signed = [value if rng.getrandbits(1) else -value for value in differences]
|
|
49
|
+
if abs(_weighted_mean(signed, weights)) >= observed - 1e-15:
|
|
50
|
+
extreme += 1
|
|
51
|
+
return {"p_value": (extreme + 1) / (iterations + 1),
|
|
52
|
+
"method": "monte_carlo_paired_sign_flip", "permutations": iterations}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _paired_bootstrap_ci(differences: list[float], weights: list[float], seed: int,
|
|
56
|
+
iterations: int = BOOTSTRAP_ITERATIONS) -> dict[str, Any]:
|
|
57
|
+
rng = random.Random(seed)
|
|
58
|
+
n = len(differences)
|
|
59
|
+
estimates = []
|
|
60
|
+
for _ in range(iterations):
|
|
61
|
+
indices = [rng.randrange(n) for _ in range(n)]
|
|
62
|
+
estimates.append(_weighted_mean([differences[i] for i in indices], [weights[i] for i in indices]))
|
|
63
|
+
estimates.sort()
|
|
64
|
+
return {"confidence_level": 0.95, "method": "paired_case_percentile_bootstrap",
|
|
65
|
+
"iterations": iterations,
|
|
66
|
+
"lower": estimates[int(0.025 * (iterations - 1))],
|
|
67
|
+
"upper": estimates[int(0.975 * (iterations - 1))]}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _holm_adjust(items: list[dict[str, Any]]) -> None:
|
|
71
|
+
ordered = sorted(items, key=lambda item: item["p_value"])
|
|
72
|
+
running = 0.0
|
|
73
|
+
for index, item in enumerate(ordered):
|
|
74
|
+
running = max(running, min(1.0, (len(ordered) - index) * item["p_value"]))
|
|
75
|
+
item["p_value_holm"] = round(running, 8)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def comparative_analysis(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
|
79
|
+
by_model: dict[str, dict[str, dict[str, Any]]] = {}
|
|
80
|
+
for record in records:
|
|
81
|
+
case = cases[record["case_id"]]
|
|
82
|
+
by_model.setdefault(record["model"], {})[record["case_id"]] = {
|
|
83
|
+
"metrics": _case_metrics(record, case),
|
|
84
|
+
"weight": float(case.get("evaluation_weight", 1.0)),
|
|
85
|
+
"stratum": case.get("evaluation_stratum", "default"),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
coverage = []
|
|
89
|
+
for model, observed in sorted(by_model.items()):
|
|
90
|
+
reviewed = [key for key, value in observed.items() if value["metrics"] is not None]
|
|
91
|
+
missing = sorted(set(cases) - set(observed))
|
|
92
|
+
unreviewed = sorted(key for key, value in observed.items() if value["metrics"] is None)
|
|
93
|
+
coverage.append({
|
|
94
|
+
"model": model, "response_case_count": len(observed), "reviewed_case_count": len(reviewed),
|
|
95
|
+
"reviewed_case_coverage": round(len(reviewed) / len(cases), 6) if cases else 0.0,
|
|
96
|
+
"missing_case_ids": missing, "unreviewed_case_ids": unreviewed,
|
|
97
|
+
"coverage_status": "complete" if not missing and not unreviewed else "incomplete",
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
comparisons = []
|
|
101
|
+
testable = []
|
|
102
|
+
for pair_index, (left_name, right_name) in enumerate(combinations(sorted(by_model), 2)):
|
|
103
|
+
left, right = by_model[left_name], by_model[right_name]
|
|
104
|
+
common = sorted(key for key in set(left) & set(right)
|
|
105
|
+
if left[key]["metrics"] is not None and right[key]["metrics"] is not None)
|
|
106
|
+
metric_rows = []
|
|
107
|
+
for metric_index, metric_name in enumerate(("required_output_score", "guardrail_pass_rate")):
|
|
108
|
+
differences = [left[key]["metrics"][metric_name] - right[key]["metrics"][metric_name] for key in common]
|
|
109
|
+
weights = [float(cases[key].get("evaluation_weight", 1.0)) for key in common]
|
|
110
|
+
strata: dict[str, int] = {}
|
|
111
|
+
for key in common:
|
|
112
|
+
label = str(cases[key].get("evaluation_stratum", "default"))
|
|
113
|
+
strata[label] = strata.get(label, 0) + 1
|
|
114
|
+
row: dict[str, Any] = {
|
|
115
|
+
"metric": metric_name, "paired_case_count": len(common), "paired_case_ids": common,
|
|
116
|
+
"stratum_counts": dict(sorted(strata.items())),
|
|
117
|
+
"weighting": "case_evaluation_weight" if any(weight != 1 for weight in weights) else "equal_case_weight",
|
|
118
|
+
"practical_effect_threshold_absolute": PRACTICAL_EFFECT_THRESHOLD,
|
|
119
|
+
}
|
|
120
|
+
if len(common) < MIN_PAIRED_CASES:
|
|
121
|
+
row.update({"status": "insufficient_paired_sample", "minimum_paired_cases": MIN_PAIRED_CASES,
|
|
122
|
+
"weighted_mean_difference": None, "confidence_interval_95": None,
|
|
123
|
+
"p_value": None, "effect_size_paired_dz": None})
|
|
124
|
+
else:
|
|
125
|
+
mean = _weighted_mean(differences, weights)
|
|
126
|
+
sd = _weighted_sd(differences, weights)
|
|
127
|
+
ci = _paired_bootstrap_ci(differences, weights, SEED + pair_index * 13 + metric_index)
|
|
128
|
+
permutation = _paired_permutation_pvalue(differences, weights, SEED + pair_index * 17 + metric_index)
|
|
129
|
+
row.update({
|
|
130
|
+
"status": "estimated", "weighted_mean_difference": round(mean, 8),
|
|
131
|
+
"confidence_interval_95": {key: round(value, 8) if isinstance(value, float) else value for key, value in ci.items()},
|
|
132
|
+
"p_value": round(permutation["p_value"], 8), "p_value_method": permutation["method"],
|
|
133
|
+
"permutations": permutation["permutations"],
|
|
134
|
+
"effect_size_paired_dz": round(mean / sd, 8) if sd else None,
|
|
135
|
+
"effect_size_interpretation": "practically_material" if abs(mean) >= PRACTICAL_EFFECT_THRESHOLD else "below_practical_threshold",
|
|
136
|
+
})
|
|
137
|
+
testable.append(row)
|
|
138
|
+
metric_rows.append(row)
|
|
139
|
+
comparisons.append({"models": [left_name, right_name], "metrics": metric_rows})
|
|
140
|
+
|
|
141
|
+
_holm_adjust(testable)
|
|
142
|
+
for comparison in comparisons:
|
|
143
|
+
for row in comparison["metrics"]:
|
|
144
|
+
if row in testable:
|
|
145
|
+
row["statistical_significance_0_05_after_holm"] = row["p_value_holm"] < 0.05
|
|
146
|
+
row["practical_significance"] = row["effect_size_interpretation"] == "practically_material"
|
|
147
|
+
row["interpretation"] = (
|
|
148
|
+
"statistically_and_practically_significant" if row["statistical_significance_0_05_after_holm"] and row["practical_significance"]
|
|
149
|
+
else "statistically_significant_only" if row["statistical_significance_0_05_after_holm"]
|
|
150
|
+
else "practically_material_but_not_statistically_significant" if row["practical_significance"]
|
|
151
|
+
else "no_demonstrated_material_difference"
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
return {
|
|
155
|
+
"status": "paired_descriptive_and_inferential_analysis",
|
|
156
|
+
"minimum_paired_cases": MIN_PAIRED_CASES,
|
|
157
|
+
"practical_effect_threshold_absolute": PRACTICAL_EFFECT_THRESHOLD,
|
|
158
|
+
"model_coverage": coverage, "comparisons": comparisons,
|
|
159
|
+
"robustness_analysis": robustness_analysis(records, cases),
|
|
160
|
+
"limitations": [
|
|
161
|
+
"Comparisons use only cases reviewed for both models; unequal coverage can create selection bias.",
|
|
162
|
+
"Case weights and strata are caller-supplied design metadata, not inferred corrections.",
|
|
163
|
+
"Strata are reported for audit; population post-stratification is not inferred.",
|
|
164
|
+
"Paired sign-flip tests assume exchangeable signs under the null; clustered cases can invalidate nominal p-values.",
|
|
165
|
+
"Bootstrap intervals quantify observed-sample uncertainty and do not repair non-representative sampling.",
|
|
166
|
+
"Statistical significance, practical effect threshold, and sample sufficiency are separate outputs.",
|
|
167
|
+
],
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
BOOTSTRAP_RANK_ITERATIONS = 1000
|
|
172
|
+
RANK_MIN_CASES = 5
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _average_ranks(scores: dict[str, float]) -> dict[str, float]:
|
|
176
|
+
ordered = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
|
|
177
|
+
ranks = {}
|
|
178
|
+
index = 0
|
|
179
|
+
while index < len(ordered):
|
|
180
|
+
end = index + 1
|
|
181
|
+
while end < len(ordered) and abs(ordered[end][1] - ordered[index][1]) <= 1e-12:
|
|
182
|
+
end += 1
|
|
183
|
+
rank = ((index + 1) + end) / 2
|
|
184
|
+
for model, _ in ordered[index:end]:
|
|
185
|
+
ranks[model] = rank
|
|
186
|
+
index = end
|
|
187
|
+
return ranks
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def robustness_analysis(records: list[dict[str, Any]], cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
|
191
|
+
"""Describe review missingness and test ranking sensitivity without imputing scores."""
|
|
192
|
+
by_model: dict[str, dict[str, dict[str, Any]]] = {}
|
|
193
|
+
for record in records:
|
|
194
|
+
case_id = record["case_id"]
|
|
195
|
+
by_model.setdefault(record["model"], {})[case_id] = {
|
|
196
|
+
"metrics": _case_metrics(record, cases[case_id]),
|
|
197
|
+
}
|
|
198
|
+
models = sorted(by_model)
|
|
199
|
+
strata = sorted({str(case.get("evaluation_stratum", "default")) for case in cases.values()})
|
|
200
|
+
coverage = []
|
|
201
|
+
for model in models:
|
|
202
|
+
observed = by_model[model]
|
|
203
|
+
for stratum in strata:
|
|
204
|
+
ids = sorted(k for k, c in cases.items()
|
|
205
|
+
if str(c.get("evaluation_stratum", "default")) == stratum)
|
|
206
|
+
reviewed = [k for k in ids if k in observed and observed[k]["metrics"] is not None]
|
|
207
|
+
coverage.append({
|
|
208
|
+
"model": model, "stratum": stratum, "case_count": len(ids),
|
|
209
|
+
"reviewed_count": len(reviewed),
|
|
210
|
+
"reviewed_coverage": round(len(reviewed) / len(ids), 6) if ids else None,
|
|
211
|
+
"reviewed_case_ids": reviewed,
|
|
212
|
+
"unreviewed_case_ids": [k for k in ids if k in observed and observed[k]["metrics"] is None],
|
|
213
|
+
"missing_case_ids": [k for k in ids if k not in observed],
|
|
214
|
+
})
|
|
215
|
+
gaps = {}
|
|
216
|
+
for model in models:
|
|
217
|
+
rates = [r["reviewed_coverage"] for r in coverage
|
|
218
|
+
if r["model"] == model and r["reviewed_coverage"] is not None]
|
|
219
|
+
gaps[model] = round(max(rates) - min(rates), 6) if rates else None
|
|
220
|
+
|
|
221
|
+
ids = sorted(cases)
|
|
222
|
+
base = {k: float(cases[k].get("evaluation_weight", 1.0)) for k in ids}
|
|
223
|
+
scenarios = {
|
|
224
|
+
"baseline": base,
|
|
225
|
+
"alternating_minus_plus_25_percent": {
|
|
226
|
+
k: base[k] * (0.75 if i % 2 == 0 else 1.25) for i, k in enumerate(ids)},
|
|
227
|
+
"alternating_plus_minus_25_percent": {
|
|
228
|
+
k: base[k] * (1.25 if i % 2 == 0 else 0.75) for i, k in enumerate(ids)},
|
|
229
|
+
}
|
|
230
|
+
sensitivity = {}
|
|
231
|
+
for metric in ("required_output_score", "guardrail_pass_rate"):
|
|
232
|
+
rows = {}
|
|
233
|
+
baseline_ranks = None
|
|
234
|
+
for name, weights in scenarios.items():
|
|
235
|
+
scores = {}
|
|
236
|
+
for model, observed in by_model.items():
|
|
237
|
+
pairs = [(item["metrics"][metric], weights[k])
|
|
238
|
+
for k, item in observed.items() if item["metrics"] is not None]
|
|
239
|
+
if pairs:
|
|
240
|
+
scores[model] = _weighted_mean([p[0] for p in pairs], [p[1] for p in pairs])
|
|
241
|
+
ranks = _average_ranks(scores)
|
|
242
|
+
if name == "baseline":
|
|
243
|
+
baseline_ranks = ranks
|
|
244
|
+
rows[name] = {"model_scores": {k: round(v, 8) for k, v in sorted(scores.items())},
|
|
245
|
+
"ranks": ranks}
|
|
246
|
+
orders = [tuple(k for k, _ in sorted(row["ranks"].items(), key=lambda x: (x[1], x[0])))
|
|
247
|
+
for row in rows.values()]
|
|
248
|
+
sensitivity[metric] = {
|
|
249
|
+
"scenarios": rows, "ranking_stable_across_scenarios": len(set(orders)) <= 1,
|
|
250
|
+
"weight_perturbation": "deterministic case-specific +/-25%; original caller weights unchanged",
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
common = sorted(set.intersection(*[
|
|
254
|
+
{k for k, item in observed.items() if item["metrics"] is not None}
|
|
255
|
+
for observed in by_model.values()
|
|
256
|
+
])) if models else []
|
|
257
|
+
rank_bootstrap = {}
|
|
258
|
+
for metric_index, metric in enumerate(("required_output_score", "guardrail_pass_rate")):
|
|
259
|
+
if len(models) < 2 or len(common) < RANK_MIN_CASES:
|
|
260
|
+
rank_bootstrap[metric] = {"status": "insufficient_paired_sample",
|
|
261
|
+
"paired_case_count": len(common),
|
|
262
|
+
"minimum_paired_cases": RANK_MIN_CASES,
|
|
263
|
+
"model_rank_distribution": {}}
|
|
264
|
+
continue
|
|
265
|
+
rng = random.Random(SEED + 700 + metric_index)
|
|
266
|
+
samples = {model: [] for model in models}
|
|
267
|
+
for _ in range(BOOTSTRAP_RANK_ITERATIONS):
|
|
268
|
+
picked = [common[rng.randrange(len(common))] for _ in common]
|
|
269
|
+
scores = {}
|
|
270
|
+
for model in models:
|
|
271
|
+
vals = [by_model[model][k]["metrics"][metric] for k in picked]
|
|
272
|
+
ws = [float(cases[k].get("evaluation_weight", 1.0)) for k in picked]
|
|
273
|
+
scores[model] = _weighted_mean(vals, ws)
|
|
274
|
+
for model, rank in _average_ranks(scores).items():
|
|
275
|
+
samples[model].append(rank)
|
|
276
|
+
distributions = {}
|
|
277
|
+
for model, values in samples.items():
|
|
278
|
+
ordered = sorted(values)
|
|
279
|
+
distributions[model] = {
|
|
280
|
+
"mean_rank": round(sum(values) / len(values), 6),
|
|
281
|
+
"rank_95_percentile_interval": [
|
|
282
|
+
ordered[int(0.025 * (len(ordered) - 1))],
|
|
283
|
+
ordered[int(0.975 * (len(ordered) - 1))]],
|
|
284
|
+
"top_rank_frequency": round(sum(v == 1.0 for v in values) / len(values), 6),
|
|
285
|
+
"rank_frequency": {str(r): round(values.count(r) / len(values), 6)
|
|
286
|
+
for r in sorted(set(values))},
|
|
287
|
+
}
|
|
288
|
+
rank_bootstrap[metric] = {
|
|
289
|
+
"status": "estimated", "paired_case_count": len(common),
|
|
290
|
+
"paired_case_ids": common, "iterations": BOOTSTRAP_RANK_ITERATIONS,
|
|
291
|
+
"method": "paired_case_bootstrap_shared_resamples",
|
|
292
|
+
"model_rank_distribution": distributions,
|
|
293
|
+
}
|
|
294
|
+
return {
|
|
295
|
+
"status": "descriptive_missingness_and_ranking_sensitivity",
|
|
296
|
+
"review_coverage_by_model_and_stratum": coverage,
|
|
297
|
+
"review_coverage_gap_by_model": gaps,
|
|
298
|
+
"missingness_policy": "missing and unreviewed are separate; neither is imputed as zero",
|
|
299
|
+
"weight_sensitivity": sensitivity,
|
|
300
|
+
"paired_bootstrap_rank_stability": rank_bootstrap,
|
|
301
|
+
"limitations": [
|
|
302
|
+
"Stratum coverage gaps are descriptive and do not identify a missingness mechanism.",
|
|
303
|
+
"Weight perturbations are stress scenarios, not corrected weights or confidence intervals.",
|
|
304
|
+
"Rank bootstrap uses shared case resamples over the complete reviewed intersection.",
|
|
305
|
+
"Bootstrap stability does not repair selection bias or non-representative sampling.",
|
|
306
|
+
"Tied scores receive average ranks; top-rank frequency counts only an unshared rank of one.",
|
|
307
|
+
],
|
|
308
|
+
}
|