guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Score solver ladders of a QUALITY challenge: how good are the answers, at the same CPU budget? (Python standard library only; reuses score.py's hardened readers.)
|
|
3
|
+
|
|
4
|
+
python3 qscore.py score --pair BASELINE.jsonl SUBMISSION.jsonl [--pair ... --pair ...] the credit of one solver over the reference, in quality points
|
|
5
|
+
python3 qscore.py compare --a A1.jsonl A2.jsonl A3.jsonl --b B1.jsonl B2.jsonl B3.jsonl is solver A really better than solver B?
|
|
6
|
+
python3 qscore.py power [--noise X] [--min-gain G] what this challenge can and cannot certify (re-simulated if asked)
|
|
7
|
+
python3 qscore.py recheck RESULT.json [--dir D] recompute a --out result from its input ladders: same numbers or fail
|
|
8
|
+
|
|
9
|
+
A ladder is a JSON-lines file, one record per instance, id exactly "SIZE:INDEX" with INDEX running 0..n-1 for every size of the challenge (extra keys are ignored).
|
|
10
|
+
The first line should be {"meta": true, "commit": SHA256_OF_THE_SEED, "budget": CPU_SECONDS}: without a commitment the ladder cannot be tied to a published seed,
|
|
11
|
+
so `score` refuses it unless --lenient (and then the result is PROVISIONAL). `check.py`'s sibling `qcheck.py run` writes ladders in this format.
|
|
12
|
+
|
|
13
|
+
{"id": "60:0", "dim": 60, "status": "solved", "quality": 0.873} quality in [0, 1] for an answer the declaration's verify() accepted
|
|
14
|
+
{"id": "60:1", "dim": 60, "status": "timeout"} statuses: solved, timeout, crash, failed, wrong; anything but "solved" scores 0
|
|
15
|
+
|
|
16
|
+
The credit is the paired difference of qualities, solver minus reference, per instance: the mean over each size, then the mean over sizes (equal weight per size), with a
|
|
17
|
+
one-sided 95% lower bound from a bootstrap that resamples instances WITHIN each size. A seed's credit is max(0, lower bound); over several seeds it is the MINIMUM
|
|
18
|
+
(an unscorable seed counts 0). Units are quality points: 0.05 = five points of the 0..1 scale. What this does NOT know: how the answers were produced. A ladder
|
|
19
|
+
claims qualities; `qcheck.py qrerun` re-runs a sample of them. See README.md ("What a score does not show").
|
|
20
|
+
"""
|
|
21
|
+
import argparse
|
|
22
|
+
import io
|
|
23
|
+
import json
|
|
24
|
+
import math
|
|
25
|
+
import os
|
|
26
|
+
import random
|
|
27
|
+
import re
|
|
28
|
+
import statistics
|
|
29
|
+
import sys
|
|
30
|
+
|
|
31
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
32
|
+
ROOT = os.path.dirname(HERE)
|
|
33
|
+
sys.path.insert(0, HERE)
|
|
34
|
+
|
|
35
|
+
import quality_core as Q # noqa: E402
|
|
36
|
+
import score as S # noqa: E402 (its safe readers, argument types and size limits)
|
|
37
|
+
|
|
38
|
+
Refused = S.Refused
|
|
39
|
+
STATUSES = S.STATUSES
|
|
40
|
+
ID_RE = S.ID_RE
|
|
41
|
+
MAX_BOOT, MAX_TRIALS, MAX_SIM_WORK, MAX_RECORDS = S.MAX_BOOT, S.MAX_TRIALS, S.MAX_SIM_WORK, S.MAX_RECORDS
|
|
42
|
+
TOP_FRACTION = 0.5 # note when this share of the baseline's answers is already at the top of the scale
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def manifest():
|
|
46
|
+
ch = S.manifest()
|
|
47
|
+
if ch.get("metric") != "quality":
|
|
48
|
+
raise Refused("this bundle is a SPEED challenge: use score.py")
|
|
49
|
+
return ch
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def load_ladder(path, default_budget):
|
|
53
|
+
"""-> (meta, {size: {id: quality}}, budget, solved counts, wrong count). Anything but a verified answer is quality 0."""
|
|
54
|
+
meta, by_dim, solved, wrong = None, {}, {}, 0
|
|
55
|
+
with io.StringIO(S.read_text(path, S.LIMIT_LADDER)) as f:
|
|
56
|
+
for n, line in enumerate(f, 1):
|
|
57
|
+
if n > MAX_RECORDS:
|
|
58
|
+
raise Refused(f"{path}: more than {MAX_RECORDS} lines; this is not a ladder of this challenge and scoring it would not finish")
|
|
59
|
+
line = line.strip()
|
|
60
|
+
if not line:
|
|
61
|
+
continue
|
|
62
|
+
try:
|
|
63
|
+
r = json.loads(line)
|
|
64
|
+
except (ValueError, RecursionError):
|
|
65
|
+
raise Refused(f"{path}:{n}: not JSON")
|
|
66
|
+
if not isinstance(r, dict):
|
|
67
|
+
raise Refused(f"{path}:{n}: not a JSON object")
|
|
68
|
+
if r.get("meta"):
|
|
69
|
+
if meta is not None:
|
|
70
|
+
raise Refused(f"{path}:{n}: a second meta line")
|
|
71
|
+
meta = r
|
|
72
|
+
continue
|
|
73
|
+
for k in ("id", "dim", "status"):
|
|
74
|
+
if k not in r:
|
|
75
|
+
raise Refused(f"{path}:{n}: no {k!r}")
|
|
76
|
+
if not isinstance(r["status"], str) or r["status"] not in STATUSES:
|
|
77
|
+
raise Refused(f"{path}:{n}: status {str(r['status'])[:30]!r} is not one of {sorted(STATUSES)}")
|
|
78
|
+
dim = r["dim"]
|
|
79
|
+
if isinstance(dim, bool) or not isinstance(dim, int):
|
|
80
|
+
raise Refused(f"{path}:{n}: dim must be an integer")
|
|
81
|
+
if not (isinstance(r["id"], str) and ID_RE.match(r["id"]) and int(r["id"].split(":")[0]) == dim):
|
|
82
|
+
raise Refused(f"{path}:{n}: id {r['id']!r} must be 'SIZE:INDEX' with SIZE equal to dim ({dim})")
|
|
83
|
+
q = 0.0
|
|
84
|
+
if r["status"] == "solved":
|
|
85
|
+
q = S.finite_float(r.get("quality"))
|
|
86
|
+
if q is None or not 0.0 <= q <= 1.0:
|
|
87
|
+
raise Refused(f"{path}:{n}: a solved record needs a finite quality in [0, 1]")
|
|
88
|
+
rid = r["id"]
|
|
89
|
+
if rid in by_dim.setdefault(dim, {}):
|
|
90
|
+
raise Refused(f"{path}:{n}: instance {rid} appears twice")
|
|
91
|
+
by_dim[dim][rid] = q
|
|
92
|
+
solved[dim] = solved.get(dim, 0) + (r["status"] == "solved")
|
|
93
|
+
wrong += r["status"] == "wrong"
|
|
94
|
+
if not by_dim:
|
|
95
|
+
raise Refused(f"{path}: no records")
|
|
96
|
+
meta = meta or {}
|
|
97
|
+
commit = meta.get("commit")
|
|
98
|
+
if commit is not None and not (isinstance(commit, str) and 0 < len(commit) <= 128):
|
|
99
|
+
raise Refused(f"{path}: meta.commit must be a short string (the seed's sha256)")
|
|
100
|
+
budget = S.finite_float(meta.get("budget", default_budget))
|
|
101
|
+
if budget is None or budget <= 0:
|
|
102
|
+
raise Refused(f"{path}: budget must be a positive, finite number")
|
|
103
|
+
return meta, by_dim, float(budget), solved, wrong
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def ladder_problems(by_dim, ch):
|
|
107
|
+
want, n = set(ch["ladder"]["dims"]), ch["ladder"]["n"]
|
|
108
|
+
out = []
|
|
109
|
+
if set(by_dim) - want:
|
|
110
|
+
out.append(f"sizes {sorted(set(by_dim) - want)} are not on the challenge ladder")
|
|
111
|
+
if want - set(by_dim):
|
|
112
|
+
out.append(f"ladder sizes {sorted(want - set(by_dim))} are missing")
|
|
113
|
+
off = {d: len(set(recs) ^ {f"{d}:{i}" for i in range(n)}) for d, recs in by_dim.items() if d in want and set(recs) ^ {f"{d}:{i}" for i in range(n)}}
|
|
114
|
+
if off:
|
|
115
|
+
out.append(f"each size must hold exactly the instances SIZE:0..SIZE:{n - 1}; differing ids per size: {off}")
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def load_pair(base_path, sub_path, ch, strict):
|
|
120
|
+
cap = ch["ladder"]["budget_seconds"]
|
|
121
|
+
mb, base, bb, _, _ = load_ladder(base_path, cap)
|
|
122
|
+
ms, sub, bs, solved, wrong = load_ladder(sub_path, cap)
|
|
123
|
+
if bb != bs:
|
|
124
|
+
raise Refused(f"the two ladders ran with different CPU budgets ({bb} vs {bs}): the qualities would not be comparable")
|
|
125
|
+
notes = []
|
|
126
|
+
if bb != cap:
|
|
127
|
+
notes.append(f"budget {bb} s differs from the challenge's {cap} s")
|
|
128
|
+
cb, cs = mb.get("commit"), ms.get("commit")
|
|
129
|
+
if cb and cs and cb != cs:
|
|
130
|
+
raise Refused("the ladders were run on different seeds (commitments differ)")
|
|
131
|
+
commit = cb or cs
|
|
132
|
+
if not commit:
|
|
133
|
+
notes.append("no seed commitment in the ladders: cannot tie them to a published seed")
|
|
134
|
+
only_b = {i for d in base for i in base[d]} - {i for d in sub for i in sub[d]}
|
|
135
|
+
only_s = {i for d in sub for i in sub[d]} - {i for d in base for i in base[d]}
|
|
136
|
+
if only_b or only_s:
|
|
137
|
+
raise Refused(f"the ladders do not cover the same instances ({len(only_b)} only in the first, {len(only_s)} only in the second): "
|
|
138
|
+
"a truncated run would give a credit computed on a biased subset")
|
|
139
|
+
for label, by_dim in (("baseline", base), ("submission", sub)):
|
|
140
|
+
for p in ladder_problems(by_dim, ch):
|
|
141
|
+
notes.append(f"{label}: {p}")
|
|
142
|
+
if strict and notes:
|
|
143
|
+
raise Refused("; ".join(notes) + " (use --lenient to score it anyway; the result is then PROVISIONAL)")
|
|
144
|
+
return base, sub, bb, commit, notes, solved, wrong
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def pair_credit(base_path, sub_path, ch, strict, boot, headroom_note=True):
|
|
148
|
+
base, sub, budget, commit, notes, solved, wrong = load_pair(base_path, sub_path, ch, strict)
|
|
149
|
+
r = Q.seed_credit(base, sub, random.Random(1), boot)
|
|
150
|
+
head = Q.headroom(base)
|
|
151
|
+
top = statistics.fmean(v["at_top"] for v in head.values())
|
|
152
|
+
if headroom_note and top >= TOP_FRACTION: # not in `compare`, where the "baseline" is whichever solver is being measured against
|
|
153
|
+
notes = notes + [f"{top:.0%} of the reference's answers are already at the top of the scale: little room for any solver to show a gain"]
|
|
154
|
+
r.update({"baseline": os.path.basename(base_path), "submission": os.path.basename(sub_path), "commit": commit, "budget": budget,
|
|
155
|
+
"solved": {str(d): solved.get(d, 0) for d in sorted(solved)}, "wrong_answers": wrong, "notes": notes,
|
|
156
|
+
"baseline_mean_quality": statistics.fmean(v["mean"] for v in head.values()),
|
|
157
|
+
"submission_mean_quality": statistics.fmean(statistics.fmean(sub[d].values()) for d in sub)})
|
|
158
|
+
return r
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def combine(pairs, ch, strict):
|
|
162
|
+
seeds = ch["protocol"]["seeds_required"]
|
|
163
|
+
commits = [p["commit"] for p in pairs if p["commit"]]
|
|
164
|
+
dup = len(set(commits)) != len(commits)
|
|
165
|
+
if dup and strict:
|
|
166
|
+
raise Refused("two pairs share a seed commitment: one seed would be counted twice")
|
|
167
|
+
credits = [p["credit_quality"] for p in pairs]
|
|
168
|
+
notes = sorted({n for p in pairs for n in p["notes"]}) + (["two pairs share a seed commitment"] if dup else [])
|
|
169
|
+
complete = len(pairs) >= seeds and not notes
|
|
170
|
+
return {"credit_quality": Q.combine(credits), "per_seed": [0.0 if c is None else c for c in credits], "seeds": len(pairs), "seeds_required": seeds,
|
|
171
|
+
"status": "COMPLETE" if complete else "PROVISIONAL", "why_provisional": ([] if len(pairs) >= seeds else [f"{len(pairs)} seed(s); {seeds} required"]) + notes}
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def window_note(credit, ch):
|
|
175
|
+
pw = ch["power"]
|
|
176
|
+
lo, hi = pw["min_certifiable"], pw["max_certifiable"]
|
|
177
|
+
if lo is None:
|
|
178
|
+
return "this challenge's own power analysis found NO gain it can certify: treat every credit as weak"
|
|
179
|
+
if credit == 0:
|
|
180
|
+
return f"0 is 'nothing certified', not 'no gain': below about {lo:g} quality points this ladder usually cannot tell"
|
|
181
|
+
if credit >= hi:
|
|
182
|
+
return f"at or above the top of what this ladder can certify ({hi:g}): a larger true gain would read about the same"
|
|
183
|
+
if credit < lo:
|
|
184
|
+
return f"below the {lo:g} lower edge of the certifiable range: weak evidence"
|
|
185
|
+
return None
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def simulate_table(ch, noise, gap, trials, boot, effects=None):
|
|
189
|
+
sim = ch.get("simulation")
|
|
190
|
+
if not sim:
|
|
191
|
+
raise Refused("this bundle has no simulation block: re-export it")
|
|
192
|
+
return Q.power_table(sim, ch["ladder"]["n"], noise, gap, ch["protocol"]["seeds_required"], trials, boot, effects or Q.effects_for(ch["power"]["threshold"]))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def noise_note(pairs, ch, trials=20, boot=40):
|
|
196
|
+
"""The power table assumed a per-instance noise; if these ladders show clearly more, say so and re-simulate at the observed value."""
|
|
197
|
+
obs = [p["pair_noise"] for p in pairs if p.get("pair_noise") is not None]
|
|
198
|
+
assumed = ch["power"].get("noise")
|
|
199
|
+
if not obs or assumed is None or not ch.get("simulation"):
|
|
200
|
+
return None
|
|
201
|
+
o = statistics.median(obs)
|
|
202
|
+
if o <= 1.25 * assumed + 0.005:
|
|
203
|
+
return None
|
|
204
|
+
lo, hi = Q.certifiable_range(simulate_table(ch, o, ch["power"]["threshold"], trials, boot), ch["power"]["target_power"])
|
|
205
|
+
where = f"from about {lo:g} to about {hi:g} points" if lo is not None else "NO gain on the grid"
|
|
206
|
+
return (f"observed per-instance noise {o:.3f} is above the {assumed:g} the power table assumed: that table is optimistic here. Re-simulated at {o:.3f}, "
|
|
207
|
+
f"this ladder certifies {where} (all seeds, {int(100 * ch['power']['target_power'])}% power)")
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def print_score(res, pairs, ch):
|
|
211
|
+
for p in pairs:
|
|
212
|
+
if p["credit_quality"] is None:
|
|
213
|
+
print(f" {p['submission']} vs {p['baseline']}: NOT SCORABLE (no shared instances)")
|
|
214
|
+
continue
|
|
215
|
+
print(f" {p['submission']} vs {p['baseline']}: >= {p['credit_quality']:.4f} quality points (mean gain {p['estimate']:+.4f}; reference {p['baseline_mean_quality']:.3f} "
|
|
216
|
+
f"-> solver {p['submission_mean_quality']:.3f}; better on {p['improved']}, worse on {p['worse']} of {p['instances']})"
|
|
217
|
+
+ (f"; {p['wrong_answers']} WRONG answers" if p["wrong_answers"] else ""))
|
|
218
|
+
print(f"credit = min over {res['seeds']} seed(s): {res['credit_quality']:.4f} quality points [{res['status']}]")
|
|
219
|
+
for w in res["why_provisional"]:
|
|
220
|
+
print(" provisional because: " + w)
|
|
221
|
+
for note in (window_note(res["credit_quality"], ch), noise_note(pairs, ch)):
|
|
222
|
+
if note:
|
|
223
|
+
print(" " + note)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def run_score(pairs_paths, ch, strict, boot):
|
|
227
|
+
pairs = [pair_credit(b, s, ch, strict, boot) for b, s in pairs_paths]
|
|
228
|
+
return combine(pairs, ch, strict), pairs
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def run_compare(la, lb, ch, strict, boot, min_gap):
|
|
232
|
+
if len(la) != len(lb):
|
|
233
|
+
raise Refused("--a and --b need one ladder per seed, in the same seed order")
|
|
234
|
+
a_over_b = [pair_credit(b, x, ch, strict, boot, headroom_note=False) for x, b in zip(la, lb)]
|
|
235
|
+
b_over_a = [pair_credit(x, b, ch, strict, boot, headroom_note=False) for x, b in zip(la, lb)]
|
|
236
|
+
ra, rb = combine(a_over_b, ch, strict), combine(b_over_a, ch, strict)
|
|
237
|
+
ga, gb = ra["credit_quality"], rb["credit_quality"]
|
|
238
|
+
if ga >= min_gap and ga > gb:
|
|
239
|
+
verdict = f"A is better: >= {ga:.4f} quality points over B (lower bound, min over seeds)"
|
|
240
|
+
elif gb >= min_gap and gb > ga:
|
|
241
|
+
verdict = f"B is better: >= {gb:.4f} quality points over A (lower bound, min over seeds)"
|
|
242
|
+
else:
|
|
243
|
+
verdict = f"CANNOT SEPARATE: neither lower bound reaches {min_gap:g} points (A over B {ga:.4f}, B over A {gb:.4f}); the gap, if any, is inside the noise"
|
|
244
|
+
if ra["status"] != "COMPLETE" or rb["status"] != "COMPLETE":
|
|
245
|
+
verdict += " [PROVISIONAL]"
|
|
246
|
+
return {"verdict": verdict, "min_gap": min_gap, "a_over_b": ra, "b_over_a": rb}, a_over_b, b_over_a
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def tool_binding():
|
|
250
|
+
return {"challenge_json_sha256": S.sha256_file(os.path.join(ROOT, "challenge.json")), "quality_core_sha256": S.sha256_file(os.path.join(HERE, "quality_core.py")),
|
|
251
|
+
"score_py_sha256": S.sha256_file(os.path.abspath(__file__))}
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def cmd_score(a):
|
|
255
|
+
ch, strict = manifest(), not a.lenient
|
|
256
|
+
res, pairs = run_score(a.pair, ch, strict, a.boot)
|
|
257
|
+
print_score(res, pairs, ch)
|
|
258
|
+
paths = [p for pair in a.pair for p in pair]
|
|
259
|
+
S.write_out(a.out, {"challenge": ch["name"], "version": ch["version"], "command": {"name": "score", "pairs": a.pair, "boot": a.boot, "lenient": a.lenient},
|
|
260
|
+
"result": res, "pairs": pairs, "binding": {**tool_binding(), "inputs": S.input_binding(paths)}})
|
|
261
|
+
return 0
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def cmd_compare(a):
|
|
265
|
+
ch, strict = manifest(), not a.lenient
|
|
266
|
+
min_gap = ch["power"]["threshold"] if a.min_gap is None else a.min_gap
|
|
267
|
+
res, ab, ba = run_compare(a.a, a.b, ch, strict, a.boot, min_gap)
|
|
268
|
+
print(res["verdict"])
|
|
269
|
+
for w in res["a_over_b"]["why_provisional"]:
|
|
270
|
+
print(" provisional because: " + w)
|
|
271
|
+
obs = [p["pair_noise"] for p in ab + ba if p.get("pair_noise") is not None]
|
|
272
|
+
noise = statistics.median(obs) if obs else ch["power"]["noise"]
|
|
273
|
+
tab = simulate_table(ch, noise, min_gap, 20, 40)
|
|
274
|
+
lo, hi = Q.certifiable_range(tab, ch["power"]["target_power"])
|
|
275
|
+
where = f"a true gap of about {lo:g} to {hi:g} points" if lo is not None else "no gap on the grid"
|
|
276
|
+
print(f" power of this comparison (simulated, {'observed' if obs else 'assumed'} noise {noise:.3f}, declared at >= {min_gap:g}): {where} is declared with >= "
|
|
277
|
+
f"{int(100 * ch['power']['target_power'])}% probability; a true gap of 0 is declared {100 * tab[0]['all_seeds']:.0f}% of the time")
|
|
278
|
+
nn = noise_note(ab + ba, ch)
|
|
279
|
+
if nn:
|
|
280
|
+
print(" " + nn)
|
|
281
|
+
S.write_out(a.out, {"challenge": ch["name"], "version": ch["version"], "command": {"name": "compare", "a": a.a, "b": a.b, "boot": a.boot, "lenient": a.lenient, "min_gap": min_gap},
|
|
282
|
+
"result": res, "a_over_b_pairs": ab, "b_over_a_pairs": ba, "binding": {**tool_binding(), "inputs": S.input_binding(list(a.a) + list(a.b))}})
|
|
283
|
+
return 0
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def cmd_power(a):
|
|
287
|
+
ch = manifest()
|
|
288
|
+
pw = ch["power"]
|
|
289
|
+
print(f"{ch['name']}: {ch['ladder']['n']} instances per size, sizes {ch['ladder']['dims']}, budget {ch['ladder']['budget_seconds']} s, assumed pair noise {pw['noise']:g} quality points")
|
|
290
|
+
lo, hi = pw["min_certifiable"], pw["max_certifiable"]
|
|
291
|
+
print(f"certifiable (all {ch['protocol']['seeds_required']} seeds credited >= {pw['threshold']:g} with >= {int(100 * pw['target_power'])}% power): "
|
|
292
|
+
+ (f"{lo:g} to {hi:g} quality points" if lo is not None else "NOTHING on the grid"))
|
|
293
|
+
print("true gain | P(one seed credited) | P(all seeds credited)")
|
|
294
|
+
for e, v in pw["curve"].items():
|
|
295
|
+
print(f"{float(e):>9g} | {100 * v['single']:>19.0f}% | {100 * v['all_seeds']:>20.0f}%")
|
|
296
|
+
if a.noise is not None or a.min_gain is not None:
|
|
297
|
+
if a.trials * a.boot > MAX_SIM_WORK:
|
|
298
|
+
raise Refused(f"--trials {a.trials} x --boot {a.boot} = {a.trials * a.boot} exceeds the limit {MAX_SIM_WORK}: the simulation would run for hours")
|
|
299
|
+
noise = pw["noise"] if a.noise is None else a.noise
|
|
300
|
+
gap = pw["threshold"] if a.min_gain is None else a.min_gain
|
|
301
|
+
tab = simulate_table(ch, noise, gap, a.trials, a.boot, Q.effects_for(gap))
|
|
302
|
+
lo, hi = Q.certifiable_range(tab, pw["target_power"])
|
|
303
|
+
print(f"\nsimulated here: noise {noise:g}, a seed counts as credited at >= {gap:g}, {a.trials} trials per point "
|
|
304
|
+
f"(Monte-Carlo error about +-{100 * 0.5 / math.sqrt(a.trials):.0f} points); certifiable " + (f"{lo:g} to {hi:g}" if lo is not None else "NOTHING"))
|
|
305
|
+
print("true gain | P(one seed credited) | P(all seeds credited)")
|
|
306
|
+
for e, v in tab.items():
|
|
307
|
+
print(f"{e:>9g} | {100 * v['single']:>19.0f}% | {100 * v['all_seeds']:>20.0f}%")
|
|
308
|
+
return 0
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def validate_result(rec, name):
|
|
312
|
+
"""A result file is untrusted input: check its whole shape before anything is read from or computed with it. -> (command, binding)"""
|
|
313
|
+
bad = Refused(f"{name} is not a result written by this tool")
|
|
314
|
+
try:
|
|
315
|
+
cmd, bind = rec["command"], rec["binding"]
|
|
316
|
+
strs = lambda v: isinstance(v, list) and v and all(isinstance(p, str) for p in v) # noqa: E731
|
|
317
|
+
if not (isinstance(cmd, dict) and isinstance(bind, dict) and cmd["name"] in ("score", "compare") and isinstance(cmd["lenient"], bool)):
|
|
318
|
+
raise bad
|
|
319
|
+
boot = cmd["boot"]
|
|
320
|
+
if isinstance(boot, bool) or not isinstance(boot, int) or not 1 <= boot <= MAX_BOOT:
|
|
321
|
+
raise Refused(f"{name}: the bootstrap count must be an integer from 1 to {MAX_BOOT}")
|
|
322
|
+
if cmd["name"] == "score":
|
|
323
|
+
ok = isinstance(cmd["pairs"], list) and cmd["pairs"] and all(isinstance(p, list) and len(p) == 2 and all(isinstance(x, str) for x in p) for p in cmd["pairs"])
|
|
324
|
+
paths = [x for p in cmd["pairs"] for x in p] if ok else []
|
|
325
|
+
else:
|
|
326
|
+
ok = strs(cmd["a"]) and strs(cmd["b"]) and S.finite_float(cmd["min_gap"]) is not None
|
|
327
|
+
paths = list(cmd["a"]) + list(cmd["b"]) if ok else []
|
|
328
|
+
if not ok or not all(isinstance(bind.get(k), str) for k in ("challenge_json_sha256", "quality_core_sha256", "score_py_sha256")):
|
|
329
|
+
raise bad
|
|
330
|
+
if not (isinstance(bind["inputs"], dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in bind["inputs"].items())):
|
|
331
|
+
raise bad
|
|
332
|
+
if not set(paths) <= set(bind["inputs"]):
|
|
333
|
+
raise Refused(f"{name} does not bind all of its input ladders by hash, so it cannot be re-checked")
|
|
334
|
+
except (KeyError, TypeError, AttributeError):
|
|
335
|
+
raise bad
|
|
336
|
+
return cmd, bind
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def cmd_recheck(a):
|
|
340
|
+
"""Recompute a result file from its input ladders and demand the same numbers: tamper-evidence for a published score."""
|
|
341
|
+
try:
|
|
342
|
+
rec = json.loads(S.read_text(a.result, S.LIMIT_JSON))
|
|
343
|
+
except (ValueError, RecursionError) as e:
|
|
344
|
+
raise Refused(f"{a.result} is not a result written by this tool ({type(e).__name__})")
|
|
345
|
+
cmd, bind = validate_result(rec, a.result)
|
|
346
|
+
ch, problems = manifest(), []
|
|
347
|
+
for k, v in tool_binding().items():
|
|
348
|
+
if bind.get(k) != v:
|
|
349
|
+
problems.append(f"{k} differs: these numbers came from a different challenge or scorer than this one")
|
|
350
|
+
paths = list(bind["inputs"])
|
|
351
|
+
have = S.input_binding(paths, a.dir or "")
|
|
352
|
+
problems += [f"input changed: {p}" for p in paths if have[p] != bind["inputs"][p]]
|
|
353
|
+
if not problems:
|
|
354
|
+
base = a.dir or ""
|
|
355
|
+
fix = lambda p: os.path.join(base, p) if base and not os.path.isabs(p) else p # noqa: E731
|
|
356
|
+
strict = not cmd["lenient"]
|
|
357
|
+
if cmd["name"] == "score":
|
|
358
|
+
res, pairs = run_score([[fix(b), fix(s)] for b, s in cmd["pairs"]], ch, strict, cmd["boot"])
|
|
359
|
+
fresh = {"result": res, "pairs": pairs}
|
|
360
|
+
else:
|
|
361
|
+
res, ab, ba = run_compare([fix(p) for p in cmd["a"]], [fix(p) for p in cmd["b"]], ch, strict, cmd["boot"], cmd["min_gap"])
|
|
362
|
+
fresh = {"result": res, "a_over_b_pairs": ab, "b_over_a_pairs": ba}
|
|
363
|
+
got = json.loads(json.dumps(fresh))
|
|
364
|
+
for k, v in got.items():
|
|
365
|
+
if not S.same(v, rec.get(k)):
|
|
366
|
+
problems.append(f"{k}: recomputed numbers differ from the recorded ones")
|
|
367
|
+
print("RECHECK OK: recomputed from the input ladders, identical" if not problems else "RECHECK FAILED:\n " + "\n ".join(problems))
|
|
368
|
+
return 1 if problems else 0
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def main(argv=None):
|
|
372
|
+
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__)
|
|
373
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
374
|
+
p = sub.add_parser("score")
|
|
375
|
+
p.add_argument("--pair", nargs=2, action="append", required=True, metavar=("BASELINE", "SUBMISSION"))
|
|
376
|
+
p = sub.add_parser("compare")
|
|
377
|
+
p.add_argument("--a", nargs="+", required=True, metavar="LADDER")
|
|
378
|
+
p.add_argument("--b", nargs="+", required=True, metavar="LADDER")
|
|
379
|
+
p.add_argument("--min-gap", type=S.float_arg(0, 1), default=None, help="quality points a lower bound must reach to call one solver better (default: the challenge's threshold; a convention, not a pre-registered test)")
|
|
380
|
+
p = sub.add_parser("power", help="the compiled power table; with --noise / --min-gain it is re-simulated here")
|
|
381
|
+
p.add_argument("--noise", type=S.float_arg(0, 1), help="per-instance noise (quality points) to simulate at, e.g. the value `score` observed")
|
|
382
|
+
p.add_argument("--min-gain", type=S.float_arg(0, 1), help="credit threshold (quality points) to simulate at (default: the challenge's)")
|
|
383
|
+
p.add_argument("--trials", type=S.int_arg(1, MAX_TRIALS), default=30, help=f"Monte-Carlo trials per point (1..{MAX_TRIALS}; trials x boot may not exceed {MAX_SIM_WORK})")
|
|
384
|
+
p.add_argument("--boot", type=S.int_arg(1, MAX_BOOT), default=60)
|
|
385
|
+
p = sub.add_parser("recheck", help="recompute a --out result file from its input ladders; fails if anything differs")
|
|
386
|
+
p.add_argument("result")
|
|
387
|
+
p.add_argument("--dir", help="directory the recorded ladder paths are relative to (default: the current one)")
|
|
388
|
+
for name in ("score", "compare"):
|
|
389
|
+
q = sub.choices[name]
|
|
390
|
+
q.add_argument("--boot", type=S.int_arg(1, MAX_BOOT), default=400, help=f"bootstrap resamples (1..{MAX_BOOT}; default 400)")
|
|
391
|
+
q.add_argument("--lenient", action="store_true", help="score ladders that do not match the challenge's ladder or lack a commitment; the result is PROVISIONAL")
|
|
392
|
+
q.add_argument("--out", help="also write the full result as JSON (never overwrites)")
|
|
393
|
+
a = ap.parse_args(argv)
|
|
394
|
+
try:
|
|
395
|
+
return {"score": cmd_score, "compare": cmd_compare, "power": cmd_power, "recheck": cmd_recheck}[a.cmd](a)
|
|
396
|
+
except Refused as e:
|
|
397
|
+
print("REFUSED: " + str(e), file=sys.stderr)
|
|
398
|
+
return 1
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
if __name__ == "__main__":
|
|
402
|
+
sys.exit(main())
|