guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,649 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Score solver ladders with the compiler's own estimator (checker/estimator.py is a byte copy of the protocol's extrapolate.py).
|
|
3
|
+
|
|
4
|
+
python3 score.py score --pair BASELINE.jsonl SUBMISSION.jsonl [--pair ... --pair ...] credit of one solver vs the reference
|
|
5
|
+
python3 score.py compare --a A1.jsonl A2.jsonl A3.jsonl --b B1.jsonl B2.jsonl B3.jsonl is solver A really faster than solver B?
|
|
6
|
+
python3 score.py power [--pair-noise X] [--min-gap G] what this challenge can and cannot certify (re-simulated if asked)
|
|
7
|
+
python3 score.py recheck RESULT.json [--dir D] recompute a --out result from its input ladders: same numbers or fail
|
|
8
|
+
|
|
9
|
+
A ladder is a JSON-lines file, one record per instance, id exactly "SIZE:INDEX" with INDEX running 0..n-1 for every size of the challenge (extra keys are ignored).
|
|
10
|
+
The first line should be {"meta": true, "commit": SHA256_OF_THE_SEED, "budget": CAP}: without a commitment the ladder cannot be tied to a published seed, so
|
|
11
|
+
`score` refuses it unless --lenient (and then the result is PROVISIONAL).
|
|
12
|
+
|
|
13
|
+
{"id": "28:0", "dim": 28, "status": "solved", "cost": 3.41} cost > 0 is CPU seconds (or any unit that is "seconds" to the floor below)
|
|
14
|
+
{"id": "28:1", "dim": 28, "status": "timeout"} statuses: solved, timeout, crash, failed, wrong; only "solved" counts as solved
|
|
15
|
+
|
|
16
|
+
`cpu_net` is accepted in place of `cost` (the guldo harness writes it). An unsolved instance is charged the CPU cap, never excused.
|
|
17
|
+
|
|
18
|
+
The credit is the protocol's: instances are grouped by how hard the BASELINE found them (octaves of its cost above 0.5 s); the paired log2
|
|
19
|
+
speedup of the 3 hardest usable bins is pooled; a one-sided 95% joint-bootstrap lower bound is reported; the credit over several seeds is the
|
|
20
|
+
MINIMUM and an inconclusive seed counts as 0. Needs numpy. What this does NOT know: how the costs were measured. Garbage costs in, a
|
|
21
|
+
well-formed number out. See README.md ("What a score does not show").
|
|
22
|
+
"""
|
|
23
|
+
import argparse
|
|
24
|
+
import hashlib
|
|
25
|
+
import io
|
|
26
|
+
import json
|
|
27
|
+
import math
|
|
28
|
+
import os
|
|
29
|
+
import random
|
|
30
|
+
import re
|
|
31
|
+
import statistics
|
|
32
|
+
import sys
|
|
33
|
+
|
|
34
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
35
|
+
ROOT = os.path.dirname(HERE)
|
|
36
|
+
sys.path.insert(0, HERE)
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
import estimator as E # noqa: E402
|
|
40
|
+
except ImportError: # a QUALITY bundle ships no speed estimator: qscore.py is its scorer and reuses this file's readers and limits
|
|
41
|
+
E = None
|
|
42
|
+
|
|
43
|
+
STATUSES = {"solved", "timeout", "crash", "failed", "wrong"}
|
|
44
|
+
ID_RE = re.compile(r"^[0-9]{1,9}:[0-9]{1,9}$")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Refused(Exception):
|
|
48
|
+
"""The input cannot be turned into a trustworthy number: say why, produce no number."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def manifest():
|
|
52
|
+
with open(os.path.join(ROOT, "challenge.json"), encoding="utf-8") as f:
|
|
53
|
+
return json.load(f)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def int_arg(lo, hi):
|
|
57
|
+
"""argparse type: an integer in [lo, hi], else a one-line usage error (never a traceback, never an unbounded computation)."""
|
|
58
|
+
def parse(text):
|
|
59
|
+
try:
|
|
60
|
+
v = int(text)
|
|
61
|
+
except ValueError:
|
|
62
|
+
raise argparse.ArgumentTypeError(f"{text!r} is not an integer")
|
|
63
|
+
if not lo <= v <= hi:
|
|
64
|
+
raise argparse.ArgumentTypeError(f"{v} is outside the allowed range {lo}..{hi}")
|
|
65
|
+
return v
|
|
66
|
+
return parse
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def float_arg(lo, hi):
|
|
70
|
+
def parse(text):
|
|
71
|
+
try:
|
|
72
|
+
v = float(text)
|
|
73
|
+
except ValueError:
|
|
74
|
+
raise argparse.ArgumentTypeError(f"{text!r} is not a number")
|
|
75
|
+
if not math.isfinite(v) or not lo <= v <= hi:
|
|
76
|
+
raise argparse.ArgumentTypeError(f"{text} is not a finite number in {lo}..{hi}")
|
|
77
|
+
return v
|
|
78
|
+
return parse
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def finite_float(x):
|
|
82
|
+
"""x as a float if it is a finite real number (not a bool, not a string, not an int too big for a float), else None."""
|
|
83
|
+
if isinstance(x, bool) or not isinstance(x, (int, float)):
|
|
84
|
+
return None
|
|
85
|
+
try:
|
|
86
|
+
f = float(x)
|
|
87
|
+
except OverflowError:
|
|
88
|
+
return None
|
|
89
|
+
return f if math.isfinite(f) else None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
LIMIT_LADDER = 32 << 20 # 20,000 records of ~150 bytes is 3 MB
|
|
93
|
+
LIMIT_JSON = 16 << 20
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _open_regular(path, limit):
|
|
97
|
+
"""fd of a regular file no bigger than `limit`, opened WITHOUT blocking, else Refused. A FIFO would block every reader for ever and /dev/zero never ends (found by
|
|
98
|
+
probing: every tool hung on a FIFO, and a 300 MB single-line file took 626 MB)."""
|
|
99
|
+
import stat
|
|
100
|
+
try:
|
|
101
|
+
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)) # no O_NONBLOCK on Windows (no FIFOs to block on)
|
|
102
|
+
except OSError as e:
|
|
103
|
+
raise Refused(f"cannot read {path}: {e.strerror}")
|
|
104
|
+
st = os.fstat(fd)
|
|
105
|
+
if not stat.S_ISREG(st.st_mode):
|
|
106
|
+
os.close(fd)
|
|
107
|
+
raise Refused(f"{path} is not a regular file (a FIFO, device, socket or directory is refused: reading it could block or never end)")
|
|
108
|
+
if limit is not None and st.st_size > limit:
|
|
109
|
+
os.close(fd)
|
|
110
|
+
raise Refused(f"{path} is {st.st_size} bytes, over the {limit} byte limit for this kind of file")
|
|
111
|
+
return fd
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def read_text(path, limit):
|
|
115
|
+
fd = _open_regular(path, limit)
|
|
116
|
+
try:
|
|
117
|
+
chunks, total = [], 0
|
|
118
|
+
while True:
|
|
119
|
+
chunk = os.read(fd, 1 << 20)
|
|
120
|
+
if not chunk:
|
|
121
|
+
break
|
|
122
|
+
total += len(chunk)
|
|
123
|
+
if total > limit:
|
|
124
|
+
raise Refused(f"{path} grew past {limit} bytes while being read")
|
|
125
|
+
chunks.append(chunk)
|
|
126
|
+
finally:
|
|
127
|
+
os.close(fd)
|
|
128
|
+
try:
|
|
129
|
+
return b"".join(chunks).decode("utf-8")
|
|
130
|
+
except UnicodeDecodeError:
|
|
131
|
+
raise Refused(f"{path} is not UTF-8 text")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def load_ladder(path, default_budget):
|
|
135
|
+
meta, by_dim = None, {}
|
|
136
|
+
f = io.StringIO(read_text(path, LIMIT_LADDER))
|
|
137
|
+
with f:
|
|
138
|
+
for n, line in enumerate(f, 1):
|
|
139
|
+
if n > MAX_RECORDS: # the bootstrap is O(resamples x records): a million-line file ran >10 minutes (measured), and a real ladder is ~10^3 records
|
|
140
|
+
raise Refused(f"{path}: more than {MAX_RECORDS} lines; this is not a ladder of this challenge (sizes x instances per size is about {1000}) and scoring it would not finish")
|
|
141
|
+
line = line.strip()
|
|
142
|
+
if not line:
|
|
143
|
+
continue
|
|
144
|
+
try:
|
|
145
|
+
r = json.loads(line)
|
|
146
|
+
except ValueError:
|
|
147
|
+
raise Refused(f"{path}:{n}: not JSON")
|
|
148
|
+
if not isinstance(r, dict):
|
|
149
|
+
raise Refused(f"{path}:{n}: not a JSON object")
|
|
150
|
+
if r.get("meta"):
|
|
151
|
+
if meta is not None:
|
|
152
|
+
raise Refused(f"{path}:{n}: a second meta line")
|
|
153
|
+
meta = r
|
|
154
|
+
continue
|
|
155
|
+
for k in ("id", "dim", "status"):
|
|
156
|
+
if k not in r:
|
|
157
|
+
raise Refused(f"{path}:{n}: no {k!r}")
|
|
158
|
+
if not isinstance(r["status"], str) or r["status"] not in STATUSES:
|
|
159
|
+
raise Refused(f"{path}:{n}: status {str(r['status'])[:30]!r} is not one of {sorted(STATUSES)}")
|
|
160
|
+
dim = r["dim"]
|
|
161
|
+
if isinstance(dim, bool) or not isinstance(dim, int):
|
|
162
|
+
raise Refused(f"{path}:{n}: dim must be an integer")
|
|
163
|
+
if not (isinstance(r["id"], str) and ID_RE.match(r["id"]) and int(r["id"].split(":")[0]) == dim):
|
|
164
|
+
raise Refused(f"{path}:{n}: id {r['id']!r} must be 'SIZE:INDEX' with SIZE equal to dim ({dim})")
|
|
165
|
+
cost = r.get("cpu_net", r.get("cost"))
|
|
166
|
+
if r["status"] == "solved":
|
|
167
|
+
cost = finite_float(cost)
|
|
168
|
+
if cost is None or cost <= 0:
|
|
169
|
+
raise Refused(f"{path}:{n}: a solved record needs a finite cost > 0")
|
|
170
|
+
else:
|
|
171
|
+
cost = 0.0
|
|
172
|
+
rid = str(r["id"])
|
|
173
|
+
if rid in by_dim.setdefault(dim, {}):
|
|
174
|
+
raise Refused(f"{path}:{n}: instance {rid} appears twice")
|
|
175
|
+
by_dim[dim][rid] = {"id": rid, "dim": dim, "status": r["status"], "cpu_net": float(cost)}
|
|
176
|
+
if not by_dim:
|
|
177
|
+
raise Refused(f"{path}: no records")
|
|
178
|
+
meta = meta or {}
|
|
179
|
+
commit = meta.get("commit")
|
|
180
|
+
if commit is not None and not (isinstance(commit, str) and 0 < len(commit) <= 128):
|
|
181
|
+
raise Refused(f"{path}: meta.commit must be a short string (the seed's sha256)")
|
|
182
|
+
budget = finite_float(meta.get("budget", default_budget))
|
|
183
|
+
if budget is None or budget <= 0:
|
|
184
|
+
raise Refused(f"{path}: budget must be a positive, finite number")
|
|
185
|
+
for dim in by_dim.values():
|
|
186
|
+
for r in dim.values():
|
|
187
|
+
if r["status"] == "solved" and r["cpu_net"] > budget * 1.0000001:
|
|
188
|
+
raise Refused(f"{path}: instance {r['id']} is 'solved' at {r['cpu_net']} s, above the {budget} s cap (it is a timeout)")
|
|
189
|
+
return meta, by_dim, float(budget)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def ladder_problems(by_dim, ch):
|
|
193
|
+
"""How this ladder differs from the challenge's declared size ladder: a dropped hard size would hide failures."""
|
|
194
|
+
want, n = set(ch["ladder"]["dims"]), ch["ladder"]["n"]
|
|
195
|
+
out = []
|
|
196
|
+
if set(by_dim) - want:
|
|
197
|
+
out.append(f"sizes {sorted(set(by_dim) - want)} are not on the challenge ladder")
|
|
198
|
+
if want - set(by_dim):
|
|
199
|
+
out.append(f"ladder sizes {sorted(want - set(by_dim))} are missing")
|
|
200
|
+
off = {}
|
|
201
|
+
for d, recs in by_dim.items():
|
|
202
|
+
diff = set(recs) ^ {f"{d}:{i}" for i in range(n)}
|
|
203
|
+
if d in want and diff:
|
|
204
|
+
off[d] = len(diff)
|
|
205
|
+
if off:
|
|
206
|
+
out.append(f"each size must hold exactly the instances SIZE:0..SIZE:{n - 1}; differing ids per size: {off}")
|
|
207
|
+
return out
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def load_pair(base_path, sub_path, ch, strict):
|
|
211
|
+
mb, base, bb = load_ladder(base_path, ch["ladder"]["budget_seconds"])
|
|
212
|
+
ms, sub, bs = load_ladder(sub_path, ch["ladder"]["budget_seconds"])
|
|
213
|
+
if bb != bs:
|
|
214
|
+
raise Refused(f"the two ladders ran with different CPU caps ({bb} vs {bs}): censoring would not be comparable")
|
|
215
|
+
notes = []
|
|
216
|
+
if bb != ch["ladder"]["budget_seconds"]:
|
|
217
|
+
notes.append(f"cap {bb} s differs from the challenge's {ch['ladder']['budget_seconds']} s")
|
|
218
|
+
cb, cs = mb.get("commit"), ms.get("commit")
|
|
219
|
+
if cb and cs and cb != cs:
|
|
220
|
+
raise Refused("the ladders were run on different seeds (commitments differ)")
|
|
221
|
+
commit = cb or cs
|
|
222
|
+
if not commit:
|
|
223
|
+
notes.append("no seed commitment in the ladders: cannot tie them to a published seed")
|
|
224
|
+
only_b, only_s = E.coverage_mismatch(base, sub)
|
|
225
|
+
if only_b or only_s:
|
|
226
|
+
raise Refused(f"the ladders do not cover the same instances ({len(only_b)} only in the first, {len(only_s)} only in the second): "
|
|
227
|
+
"a truncated run would give a credit computed on a biased subset")
|
|
228
|
+
for label, by_dim in (("baseline", base), ("submission", sub)):
|
|
229
|
+
for p in ladder_problems(by_dim, ch):
|
|
230
|
+
notes.append(f"{label}: {p}")
|
|
231
|
+
if strict and notes:
|
|
232
|
+
raise Refused("; ".join(notes) + " (use --lenient to score it anyway; the result is then PROVISIONAL)")
|
|
233
|
+
return base, sub, bb, commit, notes
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def observed_pair_noise(tb, ts, bc, sc):
|
|
237
|
+
"""Pooled within-difficulty-bin sd of the paired log2 ratios (both solved, baseline above the floor): the per-instance noise the power table
|
|
238
|
+
ASSUMED (challenge.json power.pair_noise_bits), measured on this pair. None with fewer than 8 residuals from bins of >= 4."""
|
|
239
|
+
by_bin = {}
|
|
240
|
+
for d in tb:
|
|
241
|
+
for i, cb in tb[d].items():
|
|
242
|
+
if i in ts.get(d, {}) and not bc[d][i] and not sc[d][i] and cb >= E.FLOOR_ABS:
|
|
243
|
+
by_bin.setdefault(int(math.floor(math.log2(cb / E.FLOOR_ABS))), []).append(math.log2(cb / ts[d][i]))
|
|
244
|
+
res = [x - statistics.median(v) for v in by_bin.values() if len(v) >= 4 for x in v]
|
|
245
|
+
return statistics.pstdev(res) if len(res) >= 8 else None
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def pair_credit(base_path, sub_path, ch, strict, boot):
|
|
249
|
+
base, sub, budget, commit, notes = load_pair(base_path, sub_path, ch, strict)
|
|
250
|
+
(tb, bc), (ts, sc) = E.per_dim_times(base, budget), E.per_dim_times(sub, budget)
|
|
251
|
+
meas, top, info = E.measured_by_difficulty(tb, ts, bc, random.Random(1), boot=boot)
|
|
252
|
+
hp, hp_n, hp_sub = E.hard_pool_claim(tb, ts, bc, random.Random(1), boot=boot)
|
|
253
|
+
eligible = [(tb[d][i], ts[d][i]) for d in tb for i in tb[d] if i in ts.get(d, {}) and not bc[d][i] and tb[d][i] >= E.FLOOR_ABS]
|
|
254
|
+
below = sum(c < E.FLOOR_ABS for _, c in eligible)
|
|
255
|
+
return {
|
|
256
|
+
"eligible_instances": len(eligible),
|
|
257
|
+
"submission_below_floor_fraction": (below / len(eligible)) if eligible else None,
|
|
258
|
+
"raw_median_bits": statistics.median(math.log2(b / c) for b, c in eligible) if eligible else None,
|
|
259
|
+
"pair_noise_bits": observed_pair_noise(tb, ts, bc, sc),
|
|
260
|
+
"baseline": os.path.basename(base_path), "submission": os.path.basename(sub_path), "commit": commit, "budget": budget,
|
|
261
|
+
"credit_bits": None if meas is None else max(0.0, meas[1]),
|
|
262
|
+
"median_bits": None if meas is None else meas[0],
|
|
263
|
+
"bins": {str(k): v for k, v in sorted(info.items())},
|
|
264
|
+
"credited_bins": [[info[k]["lo"], info[k]["hi"]] for k in top] if meas is not None else [],
|
|
265
|
+
"hard_pool": None if hp is None else {"median": hp[0], "lower": hp[1], "n": hp_n},
|
|
266
|
+
"solved": {str(d): sum(r["status"] == "solved" for r in sub[d].values()) for d in sorted(sub)},
|
|
267
|
+
"wrong_answers": sum(r["status"] == "wrong" for d in sub for r in sub[d].values()),
|
|
268
|
+
"notes": notes,
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def combine(pairs, ch, strict):
|
|
273
|
+
seeds = ch["protocol"]["seeds_required"]
|
|
274
|
+
commits = [p["commit"] for p in pairs if p["commit"]]
|
|
275
|
+
if len(set(commits)) != len(commits):
|
|
276
|
+
msg = "two pairs share a seed commitment: one seed would be counted twice"
|
|
277
|
+
if strict:
|
|
278
|
+
raise Refused(msg)
|
|
279
|
+
credits = [0.0 if p["credit_bits"] is None else p["credit_bits"] for p in pairs]
|
|
280
|
+
inconclusive = sum(p["credit_bits"] is None for p in pairs)
|
|
281
|
+
notes = sorted({n for p in pairs for n in p["notes"]})
|
|
282
|
+
if len(set(commits)) != len(commits):
|
|
283
|
+
notes.append("two pairs share a seed commitment")
|
|
284
|
+
complete = len(pairs) >= seeds and not notes
|
|
285
|
+
return {"credit_bits": min(credits), "per_seed": credits, "inconclusive_seeds": inconclusive, "seeds": len(pairs),
|
|
286
|
+
"seeds_required": seeds, "status": "COMPLETE" if complete else "PROVISIONAL",
|
|
287
|
+
"why_provisional": ([] if len(pairs) >= seeds else [f"{len(pairs)} seed(s); {seeds} required"]) + notes}
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def window_note(credit, ch, pairs=()):
|
|
291
|
+
pw = ch["power"]
|
|
292
|
+
lo, hi = pw["min_certifiable_bits"], pw["max_certifiable_bits"]
|
|
293
|
+
if lo is None:
|
|
294
|
+
return "this challenge's own power analysis found NO speedup it can certify: treat every credit as weak"
|
|
295
|
+
if credit == 0 and any(floor_limited(p) for p in pairs):
|
|
296
|
+
return "0 here means TOO FAST TO MEASURE on this ladder, not slow: the submission falls under the cost floor (see the raw medians above)"
|
|
297
|
+
if credit == 0:
|
|
298
|
+
return f"0 bits is 'nothing certified', not 'no speedup': below about {lo} bits this ladder usually cannot tell"
|
|
299
|
+
if credit >= hi:
|
|
300
|
+
return f"at or above the top of what this ladder can certify ({hi} bits): a larger true speedup would read about the same"
|
|
301
|
+
if credit < lo:
|
|
302
|
+
return f"below the {lo}-bit lower edge of the certifiable range: weak evidence"
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def floor_limited(p):
|
|
307
|
+
"""The faster side costs under the 0.5 s floor on most instances the slower side finds hard: the estimator discards exactly the evidence."""
|
|
308
|
+
f = p.get("submission_below_floor_fraction")
|
|
309
|
+
return f is not None and f >= 0.5
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def resolve_hint(p):
|
|
313
|
+
"""What it would take to resolve a gap this big: the slower solver must cost about FLOOR * 2^gap on the hard instances, which the cap may forbid."""
|
|
314
|
+
raw = p.get("raw_median_bits")
|
|
315
|
+
if raw is None or raw <= 0:
|
|
316
|
+
return ""
|
|
317
|
+
need, cap = E.FLOOR_ABS * 2 ** raw, p["budget"]
|
|
318
|
+
return (f" Resolving a gap of about {raw:.1f} bits needs the slower solver to cost about {need:,.0f} s on the hard instances; this challenge's cap is {cap:g} s, "
|
|
319
|
+
+ ("so no ladder under this cap can resolve it: it needs a higher cap or a stronger reference (recompile the declaration)" if need > cap
|
|
320
|
+
else "so a ladder with larger sizes can: recompile with a harder size range"))
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def why_inconclusive(p):
|
|
324
|
+
if floor_limited(p):
|
|
325
|
+
return (f"the submission costs < {E.FLOOR_ABS:g} s on {p['submission_below_floor_fraction']:.0%} of the {p['eligible_instances']} instances the baseline finds hard: "
|
|
326
|
+
f"faster than this ladder can resolve (raw median {p['raw_median_bits']:.1f} bits, descriptive only, NOT a bound)." + resolve_hint(p))
|
|
327
|
+
return "fewer than 2 difficulty bins hold >= %d instances where the submission also costs >= %g s" % (E.MIN_BIN, E.FLOOR_ABS)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
GRID = (0, 0.5, 1, 1.5, 2, 3, 4, 5, 6)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def simulate(ch, effect, noise, gap, trials, boot, rng):
|
|
334
|
+
"""P(one seed credited >= gap) and P(all seeds credited >= gap) for a true speedup of `effect` bits at per-instance noise `noise`, through the
|
|
335
|
+
estimator in this bundle, on ladders drawn from the compiled profile (challenge.json "simulation"). Mirrors the compiler's own power analysis."""
|
|
336
|
+
sim = ch.get("simulation")
|
|
337
|
+
if not sim:
|
|
338
|
+
raise Refused("this bundle has no simulation block (it was exported by an older tool): re-export it")
|
|
339
|
+
n, cap, sig, k = ch["ladder"]["n"], sim["cap"], sim["sigma"], ch["protocol"]["seeds_required"]
|
|
340
|
+
credits = []
|
|
341
|
+
for _ in range(k * trials):
|
|
342
|
+
tb, ts, bc = {}, {}, {}
|
|
343
|
+
for s in sim["sizes"]:
|
|
344
|
+
m = sim["median_log2"][str(s)]
|
|
345
|
+
tb[s], ts[s], bc[s] = {}, {}, {}
|
|
346
|
+
for i in range(n):
|
|
347
|
+
c = 2 ** (m + sig * rng.gauss(0, 1))
|
|
348
|
+
cens = c > cap
|
|
349
|
+
cost = cap if cens else max(c, 1e-3)
|
|
350
|
+
tb[s][i], bc[s][i] = cost, cens
|
|
351
|
+
ts[s][i] = min(max(cost * 2 ** -(effect + rng.gauss(0, noise)), 1e-3), cap)
|
|
352
|
+
meas, _, _ = E.measured_by_difficulty(tb, ts, bc, rng, boot=boot)
|
|
353
|
+
credits.append(None if meas is None else max(0.0, meas[1]))
|
|
354
|
+
trip = [min(0.0 if c is None else c for c in credits[i:i + k]) for i in range(0, len(credits) - k + 1, k)]
|
|
355
|
+
return {"single": sum(c is not None and c >= gap for c in credits) / len(credits), "all_seeds": sum(c >= gap for c in trip) / len(trip)}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def simulate_table(ch, noise, gap, trials, boot, rng):
|
|
359
|
+
return {e: simulate(ch, e, noise, gap, trials, boot, rng) for e in GRID}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def certifiable_range(table, target):
|
|
363
|
+
ok = [e for e, v in table.items() if v["all_seeds"] >= target]
|
|
364
|
+
return (min(ok), max(ok)) if ok else (None, None)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def noise_note(pairs, ch, trials=20, boot=40):
|
|
368
|
+
"""The power table assumed a per-instance noise; if these ladders show clearly more, say so and re-simulate at the observed value."""
|
|
369
|
+
obs = [p["pair_noise_bits"] for p in pairs if p.get("pair_noise_bits") is not None]
|
|
370
|
+
assumed = ch["power"].get("pair_noise_bits")
|
|
371
|
+
if not obs or assumed is None or not ch.get("simulation"):
|
|
372
|
+
return None
|
|
373
|
+
o = statistics.median(obs)
|
|
374
|
+
if o <= assumed + 0.15:
|
|
375
|
+
return None
|
|
376
|
+
lo, hi = certifiable_range(simulate_table(ch, o, ch["power"]["threshold_bits"], trials, boot, random.Random(0)), ch["power"]["target_power"])
|
|
377
|
+
where = f"from about {lo} to about {hi} bits" if lo is not None else "NO speedup on the grid"
|
|
378
|
+
return (f"observed per-instance noise {o:.2f} bits is above the {assumed} the power table assumed: that table is optimistic here. Re-simulated at {o:.2f}, "
|
|
379
|
+
f"this ladder certifies {where} (all seeds, {int(100 * ch['power']['target_power'])}% power)")
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def print_score(res, pairs, ch):
|
|
383
|
+
for p in pairs:
|
|
384
|
+
c = p["credit_bits"]
|
|
385
|
+
print(f" {p['submission']} vs {p['baseline']}: " + (f"INCONCLUSIVE: {why_inconclusive(p)}" if c is None else
|
|
386
|
+
f">= {c:.2f} bits (median {p['median_bits']:.2f}) over bins {p['credited_bins']}") + (f"; {p['wrong_answers']} WRONG answers" if p["wrong_answers"] else ""))
|
|
387
|
+
print(f"credit = min over {res['seeds']} seed(s), inconclusive counted as 0: {res['credit_bits']:.2f} bits [{res['status']}]")
|
|
388
|
+
for w in res["why_provisional"]:
|
|
389
|
+
print(" provisional because: " + w)
|
|
390
|
+
note = window_note(res["credit_bits"], ch, pairs)
|
|
391
|
+
if note:
|
|
392
|
+
print(" " + note)
|
|
393
|
+
nn = noise_note(pairs, ch)
|
|
394
|
+
if nn:
|
|
395
|
+
print(" " + nn)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def write_out(path, obj):
|
|
399
|
+
if path is None:
|
|
400
|
+
return
|
|
401
|
+
try:
|
|
402
|
+
f = open(path, "x", encoding="utf-8") # exclusive create: no window between 'does it exist' and 'write it'
|
|
403
|
+
except FileExistsError:
|
|
404
|
+
raise Refused(f"{path} exists: this tool never overwrites")
|
|
405
|
+
except OSError as e:
|
|
406
|
+
raise Refused(f"cannot write {path}: {e.strerror}")
|
|
407
|
+
with f:
|
|
408
|
+
json.dump(obj, f, indent=1, sort_keys=True)
|
|
409
|
+
print("wrote", path)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def sha256_file(path):
|
|
413
|
+
fd = _open_regular(path, None)
|
|
414
|
+
try:
|
|
415
|
+
h = hashlib.sha256()
|
|
416
|
+
while True:
|
|
417
|
+
chunk = os.read(fd, 1 << 20)
|
|
418
|
+
if not chunk:
|
|
419
|
+
return h.hexdigest()
|
|
420
|
+
h.update(chunk)
|
|
421
|
+
finally:
|
|
422
|
+
os.close(fd)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def tool_binding():
|
|
426
|
+
return {"challenge_json_sha256": sha256_file(os.path.join(ROOT, "challenge.json")), "estimator_sha256": sha256_file(os.path.join(HERE, "estimator.py")),
|
|
427
|
+
"score_py_sha256": sha256_file(os.path.abspath(__file__))}
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def input_binding(paths, base=""):
|
|
431
|
+
try:
|
|
432
|
+
return {p: sha256_file(os.path.join(base, p) if base and not os.path.isabs(p) else p) for p in sorted(set(paths))}
|
|
433
|
+
except OSError as e:
|
|
434
|
+
raise Refused(f"cannot read {e.filename}: {e.strerror}")
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def run_score(pairs_paths, ch, strict, boot):
|
|
438
|
+
pairs = [pair_credit(b, s, ch, strict, boot) for b, s in pairs_paths]
|
|
439
|
+
return combine(pairs, ch, strict), pairs
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def run_compare(la, lb, ch, strict, boot, min_gap):
|
|
443
|
+
if len(la) != len(lb):
|
|
444
|
+
raise Refused("--a and --b need one ladder per seed, in the same seed order")
|
|
445
|
+
a_over_b = [pair_credit(b, x, ch, strict, boot) for x, b in zip(la, lb)] # baseline = the other solver, so bins follow ITS difficulty
|
|
446
|
+
b_over_a = [pair_credit(x, b, ch, strict, boot) for x, b in zip(la, lb)]
|
|
447
|
+
ra, rb = combine(a_over_b, ch, strict), combine(b_over_a, ch, strict)
|
|
448
|
+
ga, gb = ra["credit_bits"], rb["credit_bits"]
|
|
449
|
+
if ga >= min_gap and ga > gb:
|
|
450
|
+
verdict = f"A is faster: >= {ga:.2f} bits over B (lower bound, min over seeds)"
|
|
451
|
+
elif gb >= min_gap and gb > ga:
|
|
452
|
+
verdict = f"B is faster: >= {gb:.2f} bits over A (lower bound, min over seeds)"
|
|
453
|
+
elif any(floor_limited(p) for p in a_over_b + b_over_a):
|
|
454
|
+
who, ps = ("A", a_over_b) if any(floor_limited(p) for p in a_over_b) else ("B", b_over_a)
|
|
455
|
+
raw = [p["raw_median_bits"] for p in ps if p["raw_median_bits"] is not None]
|
|
456
|
+
verdict = (f"UNRESOLVABLE BY THIS LADDER: {who} costs < {E.FLOOR_ABS:g} s on most instances the other finds hard, so the estimator discards the evidence "
|
|
457
|
+
f"(raw median gap {statistics.median(raw):.1f} bits, descriptive only, NOT a bound)." + resolve_hint(ps[0] | {"raw_median_bits": statistics.median(raw)}))
|
|
458
|
+
else:
|
|
459
|
+
verdict = f"CANNOT SEPARATE: neither lower bound reaches {min_gap} bits (A over B {ga:.2f}, B over A {gb:.2f}); the gap, if any, is inside the noise"
|
|
460
|
+
if ra["status"] != "COMPLETE" or rb["status"] != "COMPLETE":
|
|
461
|
+
verdict += " [PROVISIONAL]"
|
|
462
|
+
return {"verdict": verdict, "min_gap": min_gap, "a_over_b": ra, "b_over_a": rb}, a_over_b, b_over_a
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def cmd_score(a):
|
|
466
|
+
ch, strict = manifest(), not a.lenient
|
|
467
|
+
res, pairs = run_score(a.pair, ch, strict, a.boot)
|
|
468
|
+
print_score(res, pairs, ch)
|
|
469
|
+
paths = [p for pair in a.pair for p in pair]
|
|
470
|
+
write_out(a.out, {"challenge": ch["name"], "version": ch["version"], "command": {"name": "score", "pairs": a.pair, "boot": a.boot, "lenient": a.lenient},
|
|
471
|
+
"result": res, "pairs": pairs, "binding": {**tool_binding(), "inputs": input_binding(paths)}})
|
|
472
|
+
return 0
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def same(x, y, tol=1e-9):
|
|
476
|
+
if isinstance(x, dict) and isinstance(y, dict):
|
|
477
|
+
return x.keys() == y.keys() and all(same(x[k], y[k], tol) for k in x)
|
|
478
|
+
if isinstance(x, list) and isinstance(y, list):
|
|
479
|
+
return len(x) == len(y) and all(same(p, q, tol) for p, q in zip(x, y))
|
|
480
|
+
if isinstance(x, float) or isinstance(y, float):
|
|
481
|
+
if not (isinstance(x, (int, float)) and isinstance(y, (int, float))):
|
|
482
|
+
return False
|
|
483
|
+
try:
|
|
484
|
+
return abs(x - y) <= tol * max(1.0, abs(x), abs(y))
|
|
485
|
+
except OverflowError: # an int too big for a float: equal only if exactly equal
|
|
486
|
+
return x == y
|
|
487
|
+
return x == y
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
MAX_BOOT = 5000 # measured: scoring is linear in records x resamples; 50,000 records at 400 resamples took 48 s,
|
|
491
|
+
MAX_TRIALS = 2000
|
|
492
|
+
MAX_SIM_WORK = 200000 # trials x resamples for `power`: the default (30 x 60, 9 effects x 3 seeds) is ~2000; this allows 100x that (minutes), not hours
|
|
493
|
+
MAX_RECORDS = 20000 # so these two caps bound the worst accepted case near two minutes (a real ladder: ~10^3 records, 400 resamples)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def validate_result(rec, name):
|
|
497
|
+
"""A result file is untrusted input: check its whole shape before anything is read from or computed with it. -> (command, binding)"""
|
|
498
|
+
bad = Refused(f"{name} is not a result written by this tool")
|
|
499
|
+
try:
|
|
500
|
+
cmd, bind = rec["command"], rec["binding"]
|
|
501
|
+
strs = lambda v: isinstance(v, list) and v and all(isinstance(p, str) for p in v) # noqa: E731
|
|
502
|
+
if not (isinstance(cmd, dict) and isinstance(bind, dict) and cmd["name"] in ("score", "compare") and isinstance(cmd["lenient"], bool)):
|
|
503
|
+
raise bad
|
|
504
|
+
boot = cmd["boot"]
|
|
505
|
+
if isinstance(boot, bool) or not isinstance(boot, int) or not 1 <= boot <= MAX_BOOT:
|
|
506
|
+
raise Refused(f"{name}: the bootstrap count must be an integer from 1 to {MAX_BOOT}")
|
|
507
|
+
if cmd["name"] == "score":
|
|
508
|
+
ok = isinstance(cmd["pairs"], list) and cmd["pairs"] and all(isinstance(p, list) and len(p) == 2 and all(isinstance(x, str) for x in p) for p in cmd["pairs"])
|
|
509
|
+
paths = [x for p in cmd["pairs"] for x in p] if ok else []
|
|
510
|
+
else:
|
|
511
|
+
ok = strs(cmd["a"]) and strs(cmd["b"]) and finite_float(cmd["min_gap"]) is not None
|
|
512
|
+
paths = list(cmd["a"]) + list(cmd["b"]) if ok else []
|
|
513
|
+
if not ok or not all(isinstance(bind.get(k), str) for k in ("challenge_json_sha256", "estimator_sha256", "score_py_sha256")):
|
|
514
|
+
raise bad
|
|
515
|
+
if not (isinstance(bind["inputs"], dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in bind["inputs"].items())):
|
|
516
|
+
raise bad
|
|
517
|
+
if not set(paths) <= set(bind["inputs"]):
|
|
518
|
+
raise Refused(f"{name} does not bind all of its input ladders by hash, so it cannot be re-checked")
|
|
519
|
+
except (KeyError, TypeError, AttributeError):
|
|
520
|
+
raise bad
|
|
521
|
+
return cmd, bind
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def cmd_recheck(a):
|
|
525
|
+
"""Recompute a result file from its input ladders and demand the same numbers: tamper-evidence for a published score."""
|
|
526
|
+
try:
|
|
527
|
+
rec = json.loads(read_text(a.result, LIMIT_JSON))
|
|
528
|
+
except (OSError, ValueError, RecursionError) as e:
|
|
529
|
+
raise Refused(f"{a.result} is not a result written by this tool ({type(e).__name__})")
|
|
530
|
+
cmd, bind = validate_result(rec, a.result)
|
|
531
|
+
ch, problems = manifest(), []
|
|
532
|
+
now = tool_binding()
|
|
533
|
+
for k, v in now.items():
|
|
534
|
+
if bind.get(k) != v:
|
|
535
|
+
problems.append(f"{k} differs: these numbers came from a different challenge, estimator or scorer than this one")
|
|
536
|
+
paths = list(bind["inputs"])
|
|
537
|
+
have = input_binding(paths, a.dir or "")
|
|
538
|
+
problems += [f"input changed: {p}" for p in paths if have[p] != bind["inputs"][p]]
|
|
539
|
+
if not problems:
|
|
540
|
+
base = a.dir or ""
|
|
541
|
+
fix = lambda p: os.path.join(base, p) if base and not os.path.isabs(p) else p # noqa: E731
|
|
542
|
+
strict = not cmd["lenient"]
|
|
543
|
+
if cmd["name"] == "score":
|
|
544
|
+
res, pairs = run_score([[fix(b), fix(s)] for b, s in cmd["pairs"]], ch, strict, cmd["boot"])
|
|
545
|
+
fresh = {"result": res, "pairs": pairs}
|
|
546
|
+
else:
|
|
547
|
+
res, ab, ba = run_compare([fix(p) for p in cmd["a"]], [fix(p) for p in cmd["b"]], ch, strict, cmd["boot"], cmd["min_gap"])
|
|
548
|
+
fresh = {"result": res, "a_over_b_pairs": ab, "b_over_a_pairs": ba}
|
|
549
|
+
got = json.loads(json.dumps(fresh))
|
|
550
|
+
for k, v in got.items():
|
|
551
|
+
if not same(v, rec.get(k)):
|
|
552
|
+
problems.append(f"{k}: recomputed numbers differ from the recorded ones")
|
|
553
|
+
print("RECHECK OK: recomputed from the input ladders, identical" if not problems else "RECHECK FAILED:\n " + "\n ".join(problems))
|
|
554
|
+
return 1 if problems else 0
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def compare_power_line(res, pairs, ch, min_gap, trials=20, boot=40):
|
|
558
|
+
"""The power of THIS comparison, simulated at the noise actually observed (the assumed one if the ladders are too thin to measure it)."""
|
|
559
|
+
if not ch.get("simulation"):
|
|
560
|
+
return None
|
|
561
|
+
obs = [p["pair_noise_bits"] for p in pairs if p.get("pair_noise_bits") is not None]
|
|
562
|
+
noise = statistics.median(obs) if obs else ch["power"].get("pair_noise_bits")
|
|
563
|
+
if noise is None:
|
|
564
|
+
return None
|
|
565
|
+
tab = simulate_table(ch, noise, min_gap, trials, boot, random.Random(0))
|
|
566
|
+
lo, hi = certifiable_range(tab, ch["power"]["target_power"])
|
|
567
|
+
where = f"a true gap of about {lo} to {hi} bits" if lo is not None else "no gap on the grid"
|
|
568
|
+
return (f"power of this comparison (simulated, {'observed' if obs else 'assumed'} noise {noise:.2f} bit, declared at >= {min_gap} bit): {where} is declared with >= "
|
|
569
|
+
f"{int(100 * ch['power']['target_power'])}% probability; a true gap of 0 is declared {100 * tab[0]['all_seeds']:.0f}% of the time")
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def cmd_compare(a):
|
|
573
|
+
ch, strict = manifest(), not a.lenient
|
|
574
|
+
res, a_over_b, b_over_a = run_compare(a.a, a.b, ch, strict, a.boot, a.min_gap)
|
|
575
|
+
print(res["verdict"])
|
|
576
|
+
for w in res["a_over_b"]["why_provisional"]:
|
|
577
|
+
print(" provisional because: " + w)
|
|
578
|
+
print(" resolves only gaps on instances where the slower solver costs >= 0.5 s and the faster one does too; a tie below that says nothing.")
|
|
579
|
+
line = None if res["verdict"].startswith("UNRESOLVABLE") else compare_power_line(res, a_over_b + b_over_a, ch, a.min_gap) # a power range says nothing about a gap beyond the ladder
|
|
580
|
+
if line:
|
|
581
|
+
print(" " + line)
|
|
582
|
+
nn = noise_note(a_over_b + b_over_a, ch)
|
|
583
|
+
if nn:
|
|
584
|
+
print(" " + nn)
|
|
585
|
+
write_out(a.out, {"challenge": ch["name"], "version": ch["version"], "command": {"name": "compare", "a": a.a, "b": a.b, "boot": a.boot, "lenient": a.lenient, "min_gap": a.min_gap},
|
|
586
|
+
"result": res, "a_over_b_pairs": a_over_b, "b_over_a_pairs": b_over_a,
|
|
587
|
+
"binding": {**tool_binding(), "inputs": input_binding(list(a.a) + list(a.b))}})
|
|
588
|
+
return 0
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def cmd_power(a):
|
|
592
|
+
ch = manifest()
|
|
593
|
+
pw = ch["power"]
|
|
594
|
+
print(f"{ch['name']}: {ch['ladder']['n']} instances per size, sizes {ch['ladder']['dims']}, cap {ch['ladder']['budget_seconds']} s, assumed pair noise {pw['pair_noise_bits']} bit")
|
|
595
|
+
print(f"certifiable (all {ch['protocol']['seeds_required']} seeds credited >= {pw['threshold_bits']} bit with >= {int(100 * pw['target_power'])}% power): "
|
|
596
|
+
f"{pw['min_certifiable_bits']} to {pw['max_certifiable_bits']} bits")
|
|
597
|
+
print("true speedup (bits) | P(one seed credited) | P(all seeds credited)")
|
|
598
|
+
for e, v in pw["curve"].items():
|
|
599
|
+
print(f"{e:>19} | {100 * v['single']:>19.0f}% | {100 * v['all_seeds']:>20.0f}%")
|
|
600
|
+
if a.pair_noise is not None or a.min_gap is not None:
|
|
601
|
+
if a.trials * a.boot > MAX_SIM_WORK:
|
|
602
|
+
raise Refused(f"--trials {a.trials} x --boot {a.boot} = {a.trials * a.boot} exceeds the limit {MAX_SIM_WORK} (the default is {30 * 60}): the simulation would run for hours")
|
|
603
|
+
noise = pw["pair_noise_bits"] if a.pair_noise is None else a.pair_noise
|
|
604
|
+
gap = pw["threshold_bits"] if a.min_gap is None else a.min_gap
|
|
605
|
+
tab = simulate_table(ch, noise, gap, a.trials, a.boot, random.Random(0))
|
|
606
|
+
lo, hi = certifiable_range(tab, pw["target_power"])
|
|
607
|
+
print(f"\nsimulated here: pair noise {noise} bit, a seed counts as credited at >= {gap} bit, {a.trials} trials per point "
|
|
608
|
+
f"(Monte-Carlo error about +-{100 * 0.5 / math.sqrt(a.trials):.0f} points); certifiable {lo} to {hi} bits")
|
|
609
|
+
print("true effect (bits) | P(one seed credited) | P(all seeds credited)")
|
|
610
|
+
for e, v in tab.items():
|
|
611
|
+
print(f"{e:>18} | {100 * v['single']:>19.0f}% | {100 * v['all_seeds']:>20.0f}%")
|
|
612
|
+
return 0
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def main(argv=None):
|
|
616
|
+
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__)
|
|
617
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
618
|
+
p = sub.add_parser("score")
|
|
619
|
+
p.add_argument("--pair", nargs=2, action="append", required=True, metavar=("BASELINE", "SUBMISSION"))
|
|
620
|
+
p = sub.add_parser("compare")
|
|
621
|
+
p.add_argument("--a", nargs="+", required=True, metavar="LADDER")
|
|
622
|
+
p.add_argument("--b", nargs="+", required=True, metavar="LADDER")
|
|
623
|
+
p.add_argument("--min-gap", type=float_arg(0, 100), default=0.5, help="bits a lower bound must reach to call one solver faster (default 0.5; the power printed with the verdict is simulated, "
|
|
624
|
+
"but the threshold itself is a convention, not a pre-registered test)")
|
|
625
|
+
p = sub.add_parser("power", help="the compiled power table; with --pair-noise / --min-gap it is re-simulated here")
|
|
626
|
+
p.add_argument("--pair-noise", type=float_arg(0, 50), help="per-instance noise (bits) to simulate at, e.g. the value `score` observed")
|
|
627
|
+
p.add_argument("--min-gap", type=float_arg(0, 100), help="credit threshold (bits) to simulate at (default: the challenge's)")
|
|
628
|
+
p.add_argument("--trials", type=int_arg(1, MAX_TRIALS), default=30, help=f"Monte-Carlo trials per point (1..{MAX_TRIALS}; trials x boot may not exceed {MAX_SIM_WORK})")
|
|
629
|
+
p.add_argument("--boot", type=int_arg(1, MAX_BOOT), default=60)
|
|
630
|
+
p = sub.add_parser("recheck", help="recompute a --out result file from its input ladders; fails if anything differs")
|
|
631
|
+
p.add_argument("result")
|
|
632
|
+
p.add_argument("--dir", help="directory the recorded ladder paths are relative to (default: the current one)")
|
|
633
|
+
for name in ("score", "compare"):
|
|
634
|
+
q = sub.choices[name]
|
|
635
|
+
q.add_argument("--boot", type=int_arg(1, MAX_BOOT), default=400, help=f"bootstrap resamples (1..{MAX_BOOT}; default 400)")
|
|
636
|
+
q.add_argument("--lenient", action="store_true", help="score ladders that do not match the challenge's ladder or lack a commitment; the result is PROVISIONAL")
|
|
637
|
+
q.add_argument("--out", help="also write the full result as JSON (never overwrites)")
|
|
638
|
+
a = ap.parse_args(argv)
|
|
639
|
+
try:
|
|
640
|
+
if manifest().get("metric") == "quality":
|
|
641
|
+
raise Refused("this bundle is a QUALITY challenge (answers are scored, not the clock): use qscore.py")
|
|
642
|
+
return {"score": cmd_score, "compare": cmd_compare, "power": cmd_power, "recheck": cmd_recheck}[a.cmd](a)
|
|
643
|
+
except Refused as e:
|
|
644
|
+
print("REFUSED: " + str(e), file=sys.stderr)
|
|
645
|
+
return 1
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
if __name__ == "__main__":
|
|
649
|
+
sys.exit(main())
|