rankcheck 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rankcheck/__init__.py ADDED
@@ -0,0 +1,2 @@
1
+ """rankcheck: how many real ranks and independent signals a leaderboard or a model panel carries."""
2
+ __version__ = "0.1.0"
rankcheck/__main__.py ADDED
@@ -0,0 +1,2 @@
1
+ from .cli import main
2
+ raise SystemExit(main())
@@ -0,0 +1,15 @@
1
+ {
2
+ "status": "SEALED",
3
+ "estimator_file": "rb5_v2.py",
4
+ "sealed_name": "harness/rb5_v2.py",
5
+ "rb5_sha256": "9de08ccb806cf6367226e32d6b8ab1101b8e41c96ca318fc0a75687939d3ccee",
6
+ "b3lib_sealed_name": "harness/b3lib_v1.py",
7
+ "b3lib_sha256": "75368618eecbf9ceae7305f7e5f9ef8666902842175f4cad41412cea029f696f",
8
+ "expected": {
9
+ "rb5_v2.py": "9de08ccb806cf6367226e32d6b8ab1101b8e41c96ca318fc0a75687939d3ccee",
10
+ "b3lib_v1.py": "75368618eecbf9ceae7305f7e5f9ef8666902842175f4cad41412cea029f696f"
11
+ },
12
+ "w5_leaf_check": "PASS: both files equal the w5 leaves (root 4c5166f996ba3402..., sealed 2026-11-15T23:16:24Z)",
13
+ "w5_root": "4c5166f996ba34024743efde42ea90b2b5998507dcb0ecfb3eb4821531266396",
14
+ "w5_sealed_utc": "2026-11-15T23:16:24Z"
15
+ }
@@ -0,0 +1,262 @@
1
+ """Shared loading and estimators for lane B3 (effective number of minds, lineage, validity).
2
+
3
+ Loading reuses the sealed scorer (out/sealed-trial/harness/score.py, sealed into wave w2's snapshot), so the
4
+ pre-registered parse rule and validity rule (forecast counts only if its wave's earliest verified RFC 3161 genTime
5
+ precedes the question start) are applied by the sealed code, not re-implemented here.
6
+
7
+ Labels used in every output:
8
+ PREREG fixed by out/sealed-trial/PREREG.md sections 4 and 5
9
+ EXPLORATORY anything PREREG.md does not list (its section 6: "Any analysis not listed here is labelled EXPLORATORY")
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import importlib.util
14
+ import json
15
+ import pathlib
16
+ import re
17
+
18
+ import numpy as np
19
+
20
+ HERE = pathlib.Path(__file__).resolve().parent
21
+ TRIAL = HERE.parent / "sealed-trial"
22
+ SEED = 20261115
23
+ B_PREREG = 10000
24
+
25
+
26
+ def load_sealed():
27
+ """forecasts[arm][model][qid] = p ; questions[qid] ; outcomes[qid] (RESOLVED only), via the sealed scorer."""
28
+ spec = importlib.util.spec_from_file_location("sealed_score", TRIAL / "harness" / "score.py")
29
+ mod = importlib.util.module_from_spec(spec)
30
+ spec.loader.exec_module(mod)
31
+ return mod.load()
32
+
33
+
34
+ # ---------------------------------------------------------------- families (vendor lineage) and event clusters
35
+
36
+ FAMILY_RULES = [ # (regex on model key, family); first match wins. Vendor = lineage owner of the base weights.
37
+ (r"claude", "anthropic"), (r"gemini|gemma", "google"), (r"gpt-oss|gpt-6|openai", "openai"),
38
+ (r"nova", "amazon"), (r"ernie|baidu", "baidu"), (r"deepseek", "deepseek"),
39
+ (r"llama|muse-spark|meta/", "meta"), (r"phi-", "microsoft"), (r"minimax", "minimax"),
40
+ (r"mistral|devstral", "mistral"), (r"kimi|moonshot", "moonshot"), (r"nemotron|nvidia", "nvidia"),
41
+ (r"beam|reflection", "reflection"), (r"inkling|thinkingmachines", "thinkingmachines"),
42
+ (r"solar|upstage", "upstage"), (r"grok|x-ai", "xai"), (r"mimo|xiaomi", "xiaomi"),
43
+ (r"glm|z-ai|zhipu", "zhipu"), (r"olmo|allenai", "ai2"), (r"qwen", "alibaba"), (r"command|cohere", "cohere"),
44
+ (r"granite|ibm", "ibm"), (r"seed|bytedance", "bytedance"), (r"hy4|tencent|hunyuan", "tencent"),
45
+ ]
46
+
47
+
48
+ def family(model_key: str) -> str:
49
+ k = model_key.lower()
50
+ for pat, fam in FAMILY_RULES:
51
+ if re.search(pat, k):
52
+ return fam
53
+ return "unknown:" + k
54
+
55
+
56
+ CRYPTO = re.compile(r"BTC|ETH|SOL|DOGE|XRP|ADA|LTC", re.I)
57
+
58
+
59
+ def cluster_of(qid: str, coarse: bool = False) -> str:
60
+ """Resolution cluster per the cycle-2 ruling R-B1: one game (winner and total together); one station and one
61
+ window (weather); one reference instant for all crypto assets; one Kalshi event (series plus close instant).
62
+ coarse=True is R-B1's sensitivity grouping: all crypto questions (Coinbase and Kalshi) one cluster, and all
63
+ Kalshi markets on one underlying (series) one cluster."""
64
+ p = qid.split("-")
65
+ if qid.startswith("S-"):
66
+ return "S-" + p[1]
67
+ if qid.startswith("W-"):
68
+ return "W-" + "-".join(p[1:3])
69
+ if qid.startswith("C-"):
70
+ return "crypto" if coarse else "C-" + p[2]
71
+ if qid.startswith("K-"):
72
+ if coarse:
73
+ return "crypto" if CRYPTO.search(p[1]) else "K-" + p[1]
74
+ return "K-" + "-".join(p[1:3])
75
+ return qid
76
+
77
+
78
+ # ---------------------------------------------------------------- matrices
79
+
80
+ def matrix(models: dict, qids: list[str], keys: list[str]) -> np.ndarray:
81
+ return np.array([[models[m][q] for q in qids] for m in keys], dtype=float)
82
+
83
+
84
+ def coverage_set(models: dict, pool: set, frac: float = 0.9):
85
+ """PREREG section 5 as the sealed scorer implements it: models answering >= 90% of the best-covered model's
86
+ count in the pool; questions = intersection of those models' answered questions."""
87
+ if not models:
88
+ return [], []
89
+ mx = max(len(set(f) & pool) for f in models.values())
90
+ cov = sorted(m for m, f in models.items() if len(set(f) & pool) >= frac * max(1, mx))
91
+ common = sorted(set.intersection(*[set(models[m]) & pool for m in cov])) if cov else []
92
+ return cov, common
93
+
94
+
95
+ # ---------------------------------------------------------------- estimators
96
+
97
+ def corr_from_cov(S: np.ndarray) -> np.ndarray:
98
+ d = np.sqrt(np.clip(np.diag(S), 1e-300, None))
99
+ return S / np.outer(d, d)
100
+
101
+
102
+ def pr_of(C: np.ndarray) -> float:
103
+ lam = np.clip(np.linalg.eigvalsh(C), 0, None)
104
+ return float(lam.sum() ** 2 / (lam ** 2).sum())
105
+
106
+
107
+ def exch_of(C: np.ndarray):
108
+ n = C.shape[0]
109
+ rho = (C.sum() - np.trace(C)) / (n * (n - 1))
110
+ return (float(n / (1 + (n - 1) * rho)) if rho > -1 / (n - 1) else None), float(rho)
111
+
112
+
113
+ def neff_corr(X: np.ndarray, c: float = 0.0) -> dict:
114
+ """PR and exchangeable N_eff from the across-question covariance of the rows of X.
115
+
116
+ c = 0 is PREREG (plain correlation of residuals or forecasts). c > 0 subtracts a common variance c from every
117
+ entry of the covariance (EXPLORATORY outcome-noise correction: a residual p - y carries the shared term pi - y,
118
+ whose variance E[pi(1-pi)] is common to every model, so plain residual correlation counts shared outcome noise
119
+ as shared error)."""
120
+ S = np.cov(X)
121
+ S = S - c
122
+ if np.any(np.diag(S) <= 0):
123
+ return {"pr": None, "exch": None, "rho_bar": None, "note": "c exceeds a model's residual variance"}
124
+ C = corr_from_cov(S)
125
+ ex, rho = exch_of(C)
126
+ return {"pr": pr_of(C), "exch": ex, "rho_bar": rho}
127
+
128
+
129
+ def neff_plain(X: np.ndarray) -> dict:
130
+ """Bit-for-bit the sealed scorer's neff(): np.corrcoef, NaN to 0."""
131
+ C = np.nan_to_num(np.corrcoef(X), nan=0.0)
132
+ lam = np.clip(np.linalg.eigvalsh(C), 0, None)
133
+ n = X.shape[0]
134
+ rho = (C.sum() - n) / (n * (n - 1))
135
+ return {"pr": float(lam.sum() ** 2 / (lam ** 2).sum()), "rho_bar": float(rho),
136
+ "exch": float(n / (1 + (n - 1) * rho)) if rho > -1 / (n - 1) else None}
137
+
138
+
139
+ def c_psd_max(R: np.ndarray) -> float:
140
+ """Largest common variance c with cov(R) - c*J still positive semidefinite: c <= 1 / (1' S^-1 1)."""
141
+ S = np.cov(R)
142
+ one = np.ones(S.shape[0])
143
+ try:
144
+ return float(1.0 / (one @ np.linalg.solve(S, one)))
145
+ except np.linalg.LinAlgError:
146
+ return float(1.0 / (one @ np.linalg.pinv(S) @ one))
147
+
148
+
149
+ def n_info(F: np.ndarray, target: np.ndarray, c: float = 0.0) -> dict:
150
+ """Third way (EXPLORATORY): error-reduction N. E_1 = mean single-model squared error against `target`
151
+ (outcome y, or a reference probability), E_N = squared error of the equal-weight linear pool of all N rows.
152
+ For k independent average-quality forecasters with shared irreducible term c, E_k = c + (E_1 - c)/k, so the
153
+ pool is worth N_info = (E_1 - c)/(E_N - c) independent forecasters."""
154
+ E1 = float(np.mean((F - target) ** 2))
155
+ EN = float(np.mean((F.mean(0) - target) ** 2))
156
+ if EN - c <= 0:
157
+ return {"E1": E1, "EN": EN, "n_info": None, "note": "E_N <= c: unbounded"}
158
+ return {"E1": E1, "EN": EN, "n_info": (E1 - c) / (EN - c)}
159
+
160
+
161
+ def subset_curve(F: np.ndarray, target: np.ndarray, rng, draws: int = 200) -> list[dict]:
162
+ """E_k for k = 1..N over random model subsets (the resampling curve behind N_info)."""
163
+ n = F.shape[0]
164
+ out = []
165
+ for k in range(1, n + 1):
166
+ vals = []
167
+ for _ in range(draws if k < n else 1):
168
+ idx = rng.choice(n, k, replace=False)
169
+ vals.append(float(np.mean((F[idx].mean(0) - target) ** 2)))
170
+ out.append({"k": k, "E_k": float(np.mean(vals)), "sd": float(np.std(vals))})
171
+ return out
172
+
173
+
174
+ def boot_index(qids: list[str], rng, cluster: bool):
175
+ """One bootstrap resample of column indices: over questions (PREREG) or over event clusters (EXPLORATORY)."""
176
+ n = len(qids)
177
+ if not cluster:
178
+ return rng.integers(0, n, n)
179
+ groups: dict[str, list[int]] = {}
180
+ for i, q in enumerate(qids):
181
+ groups.setdefault(cluster_of(q), []).append(i)
182
+ keys = list(groups)
183
+ pick = rng.integers(0, len(keys), len(keys))
184
+ return np.concatenate([groups[keys[j]] for j in pick])
185
+
186
+
187
+ def ci(vals) -> list | None:
188
+ v = [x for x in vals if x is not None and np.isfinite(x)]
189
+ if not v:
190
+ return None
191
+ return [float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5)), len(v)]
192
+
193
+
194
+ def wilson(k: int, n: int, z: float = 1.959964) -> list:
195
+ if n == 0:
196
+ return [None, None]
197
+ p = k / n
198
+ d = 1 + z * z / n
199
+ c = (p + z * z / (2 * n)) / d
200
+ h = z * np.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
201
+ return [float(c - h), float(c + h)]
202
+
203
+
204
+ def adjusted_rand(a: list, b: list) -> float:
205
+ from math import comb
206
+ n = len(a)
207
+ ct: dict = {}
208
+ for x, y in zip(a, b):
209
+ ct[(x, y)] = ct.get((x, y), 0) + 1
210
+ ra: dict = {}
211
+ rb: dict = {}
212
+ for (x, y), v in ct.items():
213
+ ra[x] = ra.get(x, 0) + v
214
+ rb[y] = rb.get(y, 0) + v
215
+ s = sum(comb(v, 2) for v in ct.values())
216
+ sa = sum(comb(v, 2) for v in ra.values())
217
+ sb = sum(comb(v, 2) for v in rb.values())
218
+ e = sa * sb / comb(n, 2) if n > 1 else 0
219
+ m = (sa + sb) / 2
220
+ return float((s - e) / (m - e)) if m != e else 1.0
221
+
222
+
223
+ def newick(Z, labels: list[str]) -> str:
224
+ from scipy.cluster.hierarchy import to_tree
225
+ root = to_tree(Z)
226
+
227
+ def rec(node, parent_h):
228
+ bl = (parent_h - node.dist) / 2
229
+ if node.is_leaf():
230
+ return f"{re.sub(r'[^A-Za-z0-9._-]', '_', labels[node.id])}:{bl:.4f}"
231
+ return f"({rec(node.left, node.dist)},{rec(node.right, node.dist)}):{bl:.4f}"
232
+
233
+ return rec(root, root.dist)[: -len(":0.0000")] + ";"
234
+
235
+
236
+ def dump(path: pathlib.Path, obj) -> None:
237
+ path.parent.mkdir(parents=True, exist_ok=True)
238
+ path.write_text(json.dumps(obj, indent=1, default=float) + "\n")
239
+
240
+
241
+ def logit(p):
242
+ p = np.clip(np.asarray(p, dtype=float), 1e-4, 1 - 1e-4)
243
+ return np.log(p / (1 - p))
244
+
245
+
246
+ def expit(x):
247
+ return 1 / (1 + np.exp(-np.asarray(x, dtype=float)))
248
+
249
+
250
+ def probit(p):
251
+ from scipy.stats import norm
252
+ return norm.ppf(np.clip(np.asarray(p, dtype=float), 1e-4, 1 - 1e-4))
253
+
254
+
255
+ def load_by_wave() -> dict:
256
+ """forecasts[record_wave][model][qid] = p, each wave's own forecast kept (#r2 keys included, for the self re-ask
257
+ control). Delegates to the sealed scorer, which rehashes every record and question file against its sealed leaf and
258
+ computes anchor times from the sealed bytes."""
259
+ spec = importlib.util.spec_from_file_location("sealed_score_bw", TRIAL / "harness" / "score.py")
260
+ sc = importlib.util.module_from_spec(spec)
261
+ spec.loader.exec_module(sc)
262
+ return sc.load_by_wave()
@@ -0,0 +1,232 @@
1
+ #!/usr/bin/env python3
2
+ """Addendum 6 estimator (a): outcome-free reference departure, per RULING-c4 section 1.2.
3
+
4
+ Estimand: "k forecasters whose departures from the reference forecast are independent" (reference = the market price,
5
+ sportsbook line or numerical forecast sealed with each item as baseline_p). On the market-only subset the wording
6
+ "beyond the market" may be used.
7
+
8
+ Definition (this file is the code that defines it; sealed with w4):
9
+ 1. lp[m,q] = logit p[m,q] winsorised to [logit 0.01, logit 0.99]; lb[q] = logit b[q] likewise.
10
+ 2. reference class of q = first letter of the qid (S sportsbook, K Kalshi, W ensemble weather, C crypto, ...);
11
+ a class with fewer than 15 panel items is pooled into class "_pooled". Per model and class, least squares
12
+ lp = a + s lb; u[m,q] = residual. The model's #r2 call is fitted the same way on its own cells.
13
+ 3. Missing cells stay missing (pairwise-complete covariances; never imputed).
14
+ 4. Split the R-B1 (fine) clusters into halves A and B: sorted cluster ids, numpy default_rng(20261115).permutation,
15
+ the first floor(C/2) ids are A.
16
+ 5. For (X, Y) in ((A, B), (B, A)): r_m = 0.5 * var(u[m] - u[m#r2]) over half X items where both calls answered
17
+ (ddof 1; 0 if fewer than 3 such items or no #r2); S = pairwise-complete covariance of u over half Y items;
18
+ G-O2 sequence: S[m,m] - r_m, floored at 0.1 * S[m,m]; standardise to a correlation; clip off-diagonals to
19
+ [-1, 1]; record the minimum eigenvalue; project to the nearest PSD unit-diagonal matrix (Higham 2002 alternating
20
+ projections, 100 iterations max); n_info_dir = M^2 / (1' R 1); k_dim_dir = (tr R)^2 / tr(R^2).
21
+ Observed n_info = mean of the two directions; k_dim likewise.
22
+ 6. Benchmark B(k), k = 1..M+1: k synthetic forecasters, lp = lb + e, e = e0 + e_r with e0 ~ N(0, (1 - h) s^2) and the
23
+ #r2 call carrying fresh e_r ~ N(0, h s^2); s = pooled sd of lp - lb over the calibration forecasts (w1-w3
24
+ price-withheld, reference-bearing items); h = the cross-fitted pooled re-ask share measured on the confirmatory
25
+ panel; the same items, clusters, split and missingness; through steps 2-5. B(1) = 1. B(k) = median of 1,000 draws,
26
+ seed 20261115.
27
+ 7. k_cont = inverse of B by linear interpolation between integers (n_info <= 1 gives 1; above B(M) gives M, flagged).
28
+ Printed point value: round(k_cont), half up, with the prefix "at least K_id" when the interval's upper end exceeds
29
+ K_id (see rb5_identifiability.py).
30
+ """
31
+ from __future__ import annotations
32
+
33
+ import pathlib
34
+ import sys
35
+
36
+ import numpy as np
37
+
38
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
39
+ import b3lib as L # noqa: E402
40
+
41
+ SEED = 20261115
42
+ WLO, WHI = np.log(0.01 / 0.99), np.log(0.99 / 0.01)
43
+ MIN_CLASS = 15
44
+ FLOOR = 0.1
45
+
46
+
47
+ def lg(p):
48
+ p = np.asarray(p, dtype=float)
49
+ with np.errstate(divide="ignore"):
50
+ x = np.log(np.clip(p, 1e-12, 1 - 1e-12) / (1 - np.clip(p, 1e-12, 1 - 1e-12)))
51
+ return np.where(np.isnan(p), np.nan, np.clip(x, WLO, WHI))
52
+
53
+
54
+ def ref_class(qid: str) -> str:
55
+ return qid.split("-")[0]
56
+
57
+
58
+ def classes_for(qids: list[str]) -> np.ndarray:
59
+ raw = [ref_class(q) for q in qids]
60
+ cnt = {c: raw.count(c) for c in set(raw)}
61
+ return np.array([c if cnt[c] >= MIN_CLASS else "_pooled" for c in raw])
62
+
63
+
64
+ def recal(LP: np.ndarray, lb: np.ndarray, cls: np.ndarray) -> np.ndarray:
65
+ """Per row and class: lp = a + s lb by least squares on the row's non-missing cells; residuals (NaN kept)."""
66
+ U = np.full_like(LP, np.nan)
67
+ for c in np.unique(cls):
68
+ cols = np.where(cls == c)[0]
69
+ A = np.vstack([np.ones(len(cols)), lb[cols]]).T
70
+ sub = LP[:, cols]
71
+ full = ~np.isnan(sub).any(axis=1)
72
+ if full.any():
73
+ coef, *_ = np.linalg.lstsq(A, sub[full].T, rcond=None)
74
+ U[np.ix_(np.where(full)[0], cols)] = sub[full] - (A @ coef).T
75
+ for i in np.where(~full)[0]:
76
+ ok = ~np.isnan(sub[i])
77
+ if ok.sum() >= 3:
78
+ cf, *_ = np.linalg.lstsq(A[ok], sub[i, ok], rcond=None)
79
+ U[i, cols[ok]] = sub[i, ok] - A[ok] @ cf
80
+ return U
81
+
82
+
83
+ def split_halves(clusters, seed: int = SEED):
84
+ ids = sorted(set(clusters))
85
+ perm = np.random.default_rng(seed).permutation(len(ids))
86
+ a = {ids[j] for j in perm[: len(ids) // 2]}
87
+ A = np.array([c in a for c in clusters])
88
+ return A, ~A
89
+
90
+
91
+ def pw_cov(X: np.ndarray) -> np.ndarray:
92
+ """Pairwise-complete covariance of rows (ddof 1); entries with fewer than 3 shared cells are 0. Vectorised form
93
+ from debate/C5-AMB-pwcov.py (matches the loop form to 6.7e-16)."""
94
+ if not np.isnan(X).any():
95
+ return np.cov(X)
96
+ O = (~np.isnan(X)).astype(float)
97
+ X0 = np.nan_to_num(X)
98
+ n = O @ O.T
99
+ sx = X0 @ O.T
100
+ sxy = X0 @ X0.T
101
+ with np.errstate(invalid="ignore", divide="ignore"):
102
+ S = (sxy - sx * sx.T / n) / (n - 1)
103
+ return np.where(n >= 3, S, 0.0)
104
+
105
+
106
+ def nearest_psd_corr(C: np.ndarray, iters: int = 100) -> np.ndarray:
107
+ """Higham (2002) alternating projections with Dykstra correction onto PSD and unit diagonal."""
108
+ Y = C.copy()
109
+ dS = np.zeros_like(C)
110
+ for _ in range(iters):
111
+ R = Y - dS
112
+ w, V = np.linalg.eigh(R)
113
+ X = (V * np.clip(w, 0, None)) @ V.T
114
+ dS = X - R
115
+ Y = X.copy()
116
+ np.fill_diagonal(Y, 1.0)
117
+ if np.linalg.eigvalsh(Y)[0] >= -1e-10:
118
+ break
119
+ return Y
120
+
121
+
122
+ def go2(S: np.ndarray, r: np.ndarray | None):
123
+ """G-O2 sequence. Returns (R, min eigenvalue before projection, number of floored diagonals)."""
124
+ S = S.copy()
125
+ d0 = np.diag(S).copy()
126
+ floored = 0
127
+ if r is not None:
128
+ nd = d0 - r
129
+ fl = FLOOR * d0
130
+ floored = int((nd < fl).sum())
131
+ np.fill_diagonal(S, np.maximum(nd, fl))
132
+ dd = np.sqrt(np.clip(np.diag(S), 1e-300, None))
133
+ C = np.clip(S / np.outer(dd, dd), -1, 1)
134
+ np.fill_diagonal(C, 1.0)
135
+ mine = float(np.linalg.eigvalsh(C)[0])
136
+ if mine < 0:
137
+ C = nearest_psd_corr(C)
138
+ return C, mine, floored
139
+
140
+
141
+ def n_info(R: np.ndarray) -> float:
142
+ M = R.shape[0]
143
+ return float(M * M / R.sum())
144
+
145
+
146
+ def k_dim(R: np.ndarray) -> float:
147
+ return float(np.trace(R) ** 2 / np.sum(R * R))
148
+
149
+
150
+ def reask_var(U: np.ndarray, U2: np.ndarray | None, cols: np.ndarray) -> np.ndarray | None:
151
+ if U2 is None:
152
+ return None
153
+ D = U[:, cols] - U2[:, cols]
154
+ if not np.isnan(D).any():
155
+ return 0.5 * np.var(D, axis=1, ddof=1) if D.shape[1] >= 3 else np.zeros(U.shape[0])
156
+ r = np.zeros(U.shape[0])
157
+ for m in range(U.shape[0]):
158
+ diff = U[m, cols] - U2[m, cols]
159
+ diff = diff[~np.isnan(diff)]
160
+ if len(diff) >= 3:
161
+ r[m] = 0.5 * float(np.var(diff, ddof=1))
162
+ return r
163
+
164
+
165
+ def estimate_from_U(U: np.ndarray, U2: np.ndarray | None, halves) -> dict:
166
+ A, B = halves
167
+ ni, kd, mins, fl = [], [], [], 0
168
+ for X, Y in ((A, B), (B, A)):
169
+ S = pw_cov(U[:, Y])
170
+ R, mine, f = go2(S, reask_var(U, U2, X))
171
+ ni.append(n_info(R))
172
+ kd.append(k_dim(R))
173
+ mins.append(mine)
174
+ fl += f
175
+ return {"n_info": float(np.mean(ni)), "k_dim": float(np.mean(kd)), "min_eig": float(min(mins)), "floored": fl}
176
+
177
+
178
+ def estimate(LP: np.ndarray, LP2: np.ndarray | None, lb: np.ndarray, cls: np.ndarray, halves) -> dict:
179
+ U = recal(LP, lb, cls)
180
+ U2 = recal(LP2, lb, cls) if LP2 is not None else None
181
+ return estimate_from_U(U, U2, halves)
182
+
183
+
184
+ def reask_share(LP: np.ndarray, LP2: np.ndarray, lb: np.ndarray, cls: np.ndarray, halves) -> float:
185
+ """Cross-fitted pooled re-ask share: mean over directions of sum_m r_m(X) / sum_m var(u_m on Y)."""
186
+ U, U2 = recal(LP, lb, cls), recal(LP2, lb, cls)
187
+ A, B = halves
188
+ out = []
189
+ for X, Y in ((A, B), (B, A)):
190
+ r = reask_var(U, U2, X)
191
+ v = np.nanvar(U[:, Y], axis=1, ddof=1)
192
+ out.append(float(np.sum(r) / np.sum(v)))
193
+ return float(np.mean(out))
194
+
195
+
196
+ def benchmark(lb, cls, halves, s, h, kmax, draws, rng, mask=None, mask2=None) -> list[float]:
197
+ """mask / mask2: optional M x Q booleans of observed first-call / #r2 cells; synthetic forecaster j takes row
198
+ j mod M's patterns (v2: realised masks)."""
199
+ Q = len(lb)
200
+ out = [1.0]
201
+ for k in range(2, kmax + 1):
202
+ v = np.empty(draws)
203
+ for t in range(draws):
204
+ e0 = s * np.sqrt(1 - h) * rng.standard_normal((k, Q))
205
+ LP = np.clip(lb[None, :] + e0 + s * np.sqrt(h) * rng.standard_normal((k, Q)), WLO, WHI)
206
+ LP2 = np.clip(lb[None, :] + e0 + s * np.sqrt(h) * rng.standard_normal((k, Q)), WLO, WHI) if h > 0 else None
207
+ if mask is not None:
208
+ mk = mask[np.arange(k) % mask.shape[0]]
209
+ LP = np.where(mk, LP, np.nan)
210
+ if LP2 is not None:
211
+ mk2 = mask2[np.arange(k) % mask2.shape[0]] if mask2 is not None else mk
212
+ LP2 = np.where(mk2, LP2, np.nan)
213
+ v[t] = estimate(LP, LP2, lb, cls, halves)["n_info"]
214
+ out.append(float(np.median(v)))
215
+ return out
216
+
217
+
218
+ def k_cont(B: list[float], obs: float) -> tuple[float, bool]:
219
+ if obs <= B[0]:
220
+ return 1.0, False
221
+ for i in range(len(B) - 1):
222
+ if B[i] < obs <= B[i + 1]:
223
+ return (i + 1) + (obs - B[i]) / (B[i + 1] - B[i]), False
224
+ return float(len(B)), True
225
+
226
+
227
+ def k_round(kc: float) -> int:
228
+ return int(np.floor(kc + 0.5))
229
+
230
+
231
+ def strictly_increasing(B) -> bool:
232
+ return all(B[i + 1] > B[i] for i in range(len(B) - 1))
rankcheck/cli.py ADDED
@@ -0,0 +1,150 @@
1
+ """rankcheck command line: tiers, ranks, indep, verify-seal."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+
8
+ from . import __version__
9
+
10
+
11
+ def _tiers(a: argparse.Namespace) -> int:
12
+ from . import tiers as T
13
+
14
+ if a.demo:
15
+ board = T.load_demo(a.demo, a.population)
16
+ elif a.board:
17
+ board = T.load_csv(a.board)
18
+ else:
19
+ print("rankcheck tiers: give a board CSV (columns model,item,score) or --demo swebench-verified", file=sys.stderr)
20
+ return 2
21
+ if len(board.scores) < 2:
22
+ print(f"rankcheck tiers: {board.name} has {len(board.scores)} system(s); need at least 2", file=sys.stderr)
23
+ return 2
24
+ adj = T.adjacent(board, a.alpha)
25
+ tr = T.tiers(board, a.alpha)
26
+ rec = T.records(board, a.alpha) if board.dates else None
27
+ if a.json:
28
+ print(json.dumps({"rankcheck": __version__, "board": board.name, "adjacent": adj, "tiers": tr,
29
+ "records": rec}, indent=1))
30
+ return 0
31
+ sys.stdout.write(T.render_text(board, adj, tr))
32
+ if rec and rec["n_records"]:
33
+ print(f"records (a new best by date) separable from the previous record: {rec['separable_holm']} of "
34
+ f"{rec['n_records']} (Holm)")
35
+ if a.pairs:
36
+ print("adjacent pairs (rank, upper, lower, upper-only items, lower-only items, p, p Holm):")
37
+ for r in adj["pairs"]:
38
+ print(f" {r['rank']:>3} {r['upper']} | {r['lower']} | {r['upper_only']:g} | {r['lower_only']:g} | "
39
+ f"{r['p']:.3g} | {r['p_holm']:.3g}")
40
+ print(f"rankcheck {__version__}; the test covers item sampling only (one run per system).")
41
+ return 0
42
+
43
+
44
+ def _ranks(a: argparse.Namespace) -> int:
45
+ from . import ranks as R
46
+ from . import tiers as T
47
+
48
+ if a.check_examples:
49
+ rows = R.check_examples(B=a.B, seed=a.seed)
50
+ if a.json:
51
+ print(json.dumps(rows, indent=1))
52
+ else:
53
+ for r in rows:
54
+ print(f"{r['example']}: {r['population']}: published {list(r['published'])}, ours {list(r['ours'])}"
55
+ f" {'MATCH' if r['match'] else 'DIFFERENT'}")
56
+ print(f"{sum(r['match'] for r in rows)} of {len(rows)} published simultaneous 95% sets reproduced "
57
+ f"(B {a.B}, seed {a.seed})")
58
+ return 0 if all(r["match"] for r in rows) else 1
59
+ if a.demo:
60
+ board = T.load_demo(a.demo, a.population)
61
+ elif a.board:
62
+ board = T.load_csv(a.board)
63
+ else:
64
+ print("rankcheck ranks: give a board CSV (columns model,item,score), --demo swebench-verified, or "
65
+ "--check-examples", file=sys.stderr)
66
+ return 2
67
+ if len(board.scores) < 2:
68
+ print(f"rankcheck ranks: {board.name} has {len(board.scores)} system(s); need at least 2", file=sys.stderr)
69
+ return 2
70
+ r = R.board_rank_ci(board, alpha=a.alpha, method=a.bootstrap, B=a.B, seed=a.seed, step=not a.single_step,
71
+ holm=not a.no_exact)
72
+ r.pop("_rej")
73
+ if a.json:
74
+ print(json.dumps({"rankcheck": __version__, "board": board.name, **r}, indent=1))
75
+ return 0
76
+ sys.stdout.write(R.render_text(board.name, r, None if a.all else 25))
77
+ print(f"rankcheck {__version__}; simultaneous 95% rank sets cover item sampling only (one run per system); "
78
+ "the bootstrap is asymptotic, so boards with few items are approximate.")
79
+ return 0
80
+
81
+
82
+ def _indep(a: argparse.Namespace) -> int:
83
+ from . import indep as N
84
+
85
+ return N.main_cli(a)
86
+
87
+
88
+ def _verify_seal(a: argparse.Namespace) -> int:
89
+ from . import seal as S
90
+
91
+ return S.main_cli(a)
92
+
93
+
94
+ def main(argv: list[str] | None = None) -> int:
95
+ ap = argparse.ArgumentParser(prog="rankcheck", description="How many real ranks and independent signals does a "
96
+ "leaderboard or a model panel carry?")
97
+ ap.add_argument("--version", action="version", version=f"rankcheck {__version__}")
98
+ sub = ap.add_subparsers(dest="cmd", required=True)
99
+
100
+ t = sub.add_parser("tiers", help="adjacent-rank separability and separable tiers of a per-item leaderboard")
101
+ t.add_argument("board", nargs="?", help="CSV with columns model,item,score (.csv or .csv.gz)")
102
+ t.add_argument("--demo", choices=["swebench-verified"], help="use a bundled per-item board")
103
+ t.add_argument("--population", default="results", choices=["results", "all", "bash-only"],
104
+ help="for --demo swebench-verified: which submissions (default: those with results.json)")
105
+ t.add_argument("--alpha", type=float, default=0.05)
106
+ t.add_argument("--pairs", action="store_true", help="print every adjacent pair")
107
+ t.add_argument("--json", action="store_true")
108
+ t.set_defaults(func=_tiers)
109
+
110
+ r = sub.add_parser("ranks", help="simultaneous confidence sets for every system's rank (Mogstad et al. 2024)")
111
+ r.add_argument("board", nargs="?", help="CSV with columns model,item,score (.csv or .csv.gz)")
112
+ r.add_argument("--demo", choices=["swebench-verified"], help="use a bundled per-item board")
113
+ r.add_argument("--population", default="results", choices=["results", "all", "bash-only"])
114
+ r.add_argument("--bootstrap", default="items", choices=["items", "gaussian"],
115
+ help="items: resample items (default); gaussian: N(0, Sigma) as in csranks")
116
+ r.add_argument("--B", type=int, default=4000, help="bootstrap draws (default 4000)")
117
+ r.add_argument("--seed", type=int, default=20261115)
118
+ r.add_argument("--alpha", type=float, default=0.05)
119
+ r.add_argument("--single-step", action="store_true", help="no stepwise improvement")
120
+ r.add_argument("--no-exact", action="store_true", help="skip the exact Holm construction on 0/1 boards")
121
+ r.add_argument("--check-examples", action="store_true",
122
+ help="recompute the published examples of Mogstad et al. (Tables 1 and 2) and compare")
123
+ r.add_argument("--all", action="store_true", help="print every system (default: the top 25)")
124
+ r.add_argument("--json", action="store_true")
125
+ r.set_defaults(func=_ranks)
126
+
127
+ n = sub.add_parser("indep", help="effective number of independent forecasters or models (n_indep)")
128
+ n.add_argument("forecasts", nargs="?", help="CSV with columns model,item,p and optionally baseline,cluster,"
129
+ "outcome; a re-ask of a model is a row set whose model ends in '#r2'")
130
+ n.add_argument("--demo", action="store_true", help="run on a bundled synthetic panel with a known answer")
131
+ n.add_argument("--min-coverage", type=float, default=0.9)
132
+ n.add_argument("--boot", type=int, default=2000, help="bootstrap resamples for the plain estimators")
133
+ n.add_argument("--draws", type=int, default=200, help="benchmark draws for the calibrated map (demo default 60)")
134
+ n.add_argument("--json", action="store_true")
135
+ n.set_defaults(func=_indep)
136
+
137
+ v = sub.add_parser("verify-seal", help="verify a sealed-trial wave bundle and its public registry entry")
138
+ v.add_argument("bundle", help="directory holding anchors/<wave>/ and harness/tsa_pins.txt")
139
+ v.add_argument("--wave", action="append", help="verify only this wave (repeatable)")
140
+ v.add_argument("--registry", help="local clone of the registry branch; default: fetch from GitHub")
141
+ v.add_argument("--offline", action="store_true", help="skip the public registry check (reported as NOT RUN)")
142
+ v.add_argument("--json", action="store_true")
143
+ v.set_defaults(func=_verify_seal)
144
+
145
+ a = ap.parse_args(argv)
146
+ return a.func(a)
147
+
148
+
149
+ if __name__ == "__main__":
150
+ sys.exit(main())