guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
"""Static HTML report of speedup-verification runs: leaderboard, per-run charts, tables, limits.
|
|
2
|
+
|
|
3
|
+
python3 report_html.py --out results/report.html [--pair BASELINE.jsonl SUBMISSION.jsonl ...]
|
|
4
|
+
|
|
5
|
+
With no --pair it finds every (baseline, submission) ladder pair under results/ (judged_<family>_<tag>_...). Everything on
|
|
6
|
+
the page is computed from the saved ladders by the same functions the judge uses (extrapolate.py), so each number can be
|
|
7
|
+
traced to data. The page is one self-contained file: inline CSS/SVG, a few lines of JS for tooltips, Google Fonts only.
|
|
8
|
+
"""
|
|
9
|
+
import argparse
|
|
10
|
+
import html
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import os
|
|
14
|
+
import random
|
|
15
|
+
import re
|
|
16
|
+
import statistics
|
|
17
|
+
import time
|
|
18
|
+
|
|
19
|
+
import extrapolate as E
|
|
20
|
+
import judge
|
|
21
|
+
|
|
22
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
23
|
+
PAIR_RE = re.compile(r"^judged_([a-z]+)_([0-9a-f]{8})_(baseline|submission)_(.+)\.py\.jsonl$") # any registered family
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def esc(x):
|
|
27
|
+
return html.escape(str(x), quote=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# ---------------------------------------------------------------- data
|
|
31
|
+
def load_pair(base_path, sub_path):
|
|
32
|
+
mb, base = E.load(base_path)
|
|
33
|
+
ms, sub = E.load(sub_path)
|
|
34
|
+
only_b, only_s = E.coverage_mismatch(base, sub)
|
|
35
|
+
if only_b or only_s: # shown as "Could not read": a page must never chart a truncated run as if it were complete
|
|
36
|
+
raise ValueError(f"the ladders do not cover the same instances ({len(only_b)} only in the baseline, "
|
|
37
|
+
f"{len(only_s)} only in the submission)")
|
|
38
|
+
budget = mb["budget"]
|
|
39
|
+
(tb, bc), (ts, sc) = E.per_dim_times(base, budget), E.per_dim_times(sub, budget)
|
|
40
|
+
pts = [] # (baseline cost, submission cost, bits, id)
|
|
41
|
+
for d in tb:
|
|
42
|
+
for i, cb in tb[d].items():
|
|
43
|
+
if i in ts.get(d, {}) and not bc[d][i] and cb >= E.FLOOR_ABS:
|
|
44
|
+
pts.append((cb, ts[d][i], math.log2(cb / ts[d][i]), f"{d}:{i}"))
|
|
45
|
+
bins = E._group([(cb, cs) for cb, cs, _, _ in pts])
|
|
46
|
+
info, top = E._usable_top(bins)
|
|
47
|
+
meas, mtop, _ = E.measured_by_difficulty(tb, ts, bc, random.Random(1))
|
|
48
|
+
hp, hp_n, hp_sub = E.hard_pool_claim(tb, ts, bc, random.Random(1))
|
|
49
|
+
n_total = sum(len(v) for v in tb.values())
|
|
50
|
+
n_censored = sum(1 for d in bc for i in bc[d] if bc[d][i])
|
|
51
|
+
return {"meta_b": mb, "meta_s": ms, "budget": budget, "pts": pts, "info": info, "top": mtop, "meas": meas, "hp": hp,
|
|
52
|
+
"hp_n": hp_n, "hp_sub": hp_sub, "n_total": n_total, "n_censored": n_censored, "curve": dim_curve(tb, ts, bc, sc)}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def find_pairs(results_dir):
|
|
56
|
+
files = {}
|
|
57
|
+
for fn in sorted(os.listdir(results_dir)):
|
|
58
|
+
m = PAIR_RE.match(fn)
|
|
59
|
+
if m:
|
|
60
|
+
files.setdefault((m.group(1), m.group(2)), {}).setdefault(m.group(3), []).append((m.group(4), os.path.join(results_dir, fn)))
|
|
61
|
+
out = []
|
|
62
|
+
for (fam, tag), roles in sorted(files.items()):
|
|
63
|
+
for base in roles.get("baseline", [])[:1]:
|
|
64
|
+
for name, path in roles.get("submission", []):
|
|
65
|
+
out.append({"family": fam, "tag": tag, "baseline": base[1], "submission": path, "name": name})
|
|
66
|
+
return out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# ---------------------------------------------------------------- charts (hand-placed SVG, one scale per axis)
|
|
70
|
+
W, H = 640, 310
|
|
71
|
+
ML, MR, MT, MB = 46, 84, 16, 58 # right margin holds the credit label outside the plot; bottom holds tick, n and axis title
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def fmt_tick(t):
|
|
75
|
+
return "0" if t == 0 else f"{t:+g}"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _nice_ticks(lo, hi):
|
|
79
|
+
step = 1 if hi - lo <= 8 else 2
|
|
80
|
+
t = math.floor(lo / step) * step
|
|
81
|
+
out = [t]
|
|
82
|
+
while out[-1] < hi - 1e-9: # always end on a tick at or above the data maximum, or the top point leaves the plot
|
|
83
|
+
out.append(out[-1] + step)
|
|
84
|
+
return out
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def bin_chart(info, top, credit, budget):
|
|
88
|
+
keys = sorted(info)
|
|
89
|
+
if not keys:
|
|
90
|
+
return '<p class="muted">No instance reached the 0.5 s floor on the baseline, so there are no bins to draw.</p>'
|
|
91
|
+
vals = [info[k]["median_bits"] for k in keys]
|
|
92
|
+
lo, hi = min(-1.0, min(vals) - 0.5, (credit or 0) - 0.5), max(1.0, max(vals) + 0.5)
|
|
93
|
+
ticks = _nice_ticks(lo, hi)
|
|
94
|
+
lo, hi = ticks[0], ticks[-1]
|
|
95
|
+
|
|
96
|
+
def y(v):
|
|
97
|
+
return MT + (H - MT - MB) * (hi - v) / (hi - lo)
|
|
98
|
+
|
|
99
|
+
pw = (W - ML - MR) / len(keys)
|
|
100
|
+
bw = min(46, pw * 0.62)
|
|
101
|
+
g = []
|
|
102
|
+
for t in ticks:
|
|
103
|
+
g.append(f'<line class="grid" x1="{ML}" x2="{W - MR}" y1="{y(t):.1f}" y2="{y(t):.1f}"/>'
|
|
104
|
+
f'<text class="tick" x="{ML - 8}" y="{y(t) + 4:.1f}" text-anchor="end">{fmt_tick(t)}</text>')
|
|
105
|
+
g.append(f'<line class="zero" x1="{ML}" x2="{W - MR}" y1="{y(0):.1f}" y2="{y(0):.1f}"/>')
|
|
106
|
+
for j, k in enumerate(keys):
|
|
107
|
+
x = ML + pw * j + pw / 2
|
|
108
|
+
v = info[k]["median_bits"]
|
|
109
|
+
usable, used = info[k]["usable"], k in top
|
|
110
|
+
cls = "bar used" if used else ("bar usable" if usable else "bar unusable")
|
|
111
|
+
top_y, h = (y(v), y(0) - y(v)) if v >= 0 else (y(0), y(v) - y(0))
|
|
112
|
+
tip = (f"baseline cost {info[k]['lo']:g} to {info[k]['hi']:g} s: {info[k]['n']} instances, median speedup {v:+.2f} bits, "
|
|
113
|
+
f"submission median cost {info[k]['median_sub_cost']} s, "
|
|
114
|
+
+ ("in the credit" if used else ("usable, not among the 3 hardest" if usable else "not usable (too few instances or below the cost floor)")))
|
|
115
|
+
g.append(f'<rect class="{cls}" x="{x - bw / 2:.1f}" y="{top_y:.1f}" width="{bw:.1f}" height="{max(h, 1.5):.1f}" rx="3" '
|
|
116
|
+
f'data-tip="{esc(tip)}" tabindex="0"/>')
|
|
117
|
+
g.append(f'<text class="tick" x="{x:.1f}" y="{H - MB + 16}" text-anchor="middle">{info[k]["lo"]:g}–{info[k]["hi"]:g}</text>')
|
|
118
|
+
g.append(f'<text class="n" x="{x:.1f}" y="{H - MB + 31}" text-anchor="middle">n={info[k]["n"]}</text>')
|
|
119
|
+
if credit is not None:
|
|
120
|
+
g.append(f'<line class="credit" x1="{ML}" x2="{W - MR + 4}" y1="{y(credit):.1f}" y2="{y(credit):.1f}"/>'
|
|
121
|
+
f'<text class="creditlbl" x="{W - MR + 8}" y="{y(credit) + 4:.1f}">credit {credit:.1f}</text>')
|
|
122
|
+
g.append(f'<text class="axis" x="{(ML + W - MR) / 2}" y="{H - 6}" text-anchor="middle">baseline cost of the instance (s, octaves)</text>')
|
|
123
|
+
g.append(f'<text class="axis" transform="translate(12 {(MT + H - MB) / 2}) rotate(-90)" text-anchor="middle">median speedup (bits)</text>')
|
|
124
|
+
return f'<svg class="chart" viewBox="0 0 {W} {H}" role="img" aria-label="Median speedup per baseline-cost bin">{"".join(g)}</svg>'
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def scatter_chart(pts, info, top, budget):
|
|
128
|
+
if not pts:
|
|
129
|
+
return ""
|
|
130
|
+
lo_x, hi_x = math.log2(E.FLOOR_ABS), math.log2(max(budget, max(p[0] for p in pts)) * 1.05)
|
|
131
|
+
bits = [p[2] for p in pts]
|
|
132
|
+
ticks = _nice_ticks(min(-1.0, min(bits) - 0.5), max(1.0, max(bits) + 0.5))
|
|
133
|
+
lo, hi = ticks[0], ticks[-1]
|
|
134
|
+
|
|
135
|
+
def x(c):
|
|
136
|
+
return ML + (W - ML - MR) * (math.log2(c) - lo_x) / (hi_x - lo_x)
|
|
137
|
+
|
|
138
|
+
def y(v):
|
|
139
|
+
return MT + (H - MT - MB) * (hi - v) / (hi - lo)
|
|
140
|
+
|
|
141
|
+
g = []
|
|
142
|
+
for t in ticks:
|
|
143
|
+
g.append(f'<line class="grid" x1="{ML}" x2="{W - MR}" y1="{y(t):.1f}" y2="{y(t):.1f}"/>'
|
|
144
|
+
f'<text class="tick" x="{ML - 8}" y="{y(t) + 4:.1f}" text-anchor="end">{fmt_tick(t)}</text>')
|
|
145
|
+
g.append(f'<line class="zero" x1="{ML}" x2="{W - MR}" y1="{y(0):.1f}" y2="{y(0):.1f}"/>')
|
|
146
|
+
c = E.FLOOR_ABS
|
|
147
|
+
while c <= max(budget, 1):
|
|
148
|
+
g.append(f'<text class="tick" x="{x(c):.1f}" y="{H - MB + 16}" text-anchor="middle">{c:g}</text>')
|
|
149
|
+
c *= 4
|
|
150
|
+
for cb, cs, b, iid in pts:
|
|
151
|
+
k = int(math.floor(math.log2(cb / E.FLOOR_ABS)))
|
|
152
|
+
used = k in top
|
|
153
|
+
tip = f"instance {iid}: baseline {cb:.2f} s, submission {cs:.2f} s, {b:+.2f} bits" + (" (in the credit)" if used else "")
|
|
154
|
+
g.append(f'<circle class="pt {"used" if used else "other"}" cx="{x(cb):.1f}" cy="{y(b):.1f}" r="4" data-tip="{esc(tip)}" tabindex="0"/>')
|
|
155
|
+
g.append(f'<text class="axis" x="{(ML + W - MR) / 2}" y="{H - 6}" text-anchor="middle">baseline cost of the instance (s, log scale)</text>')
|
|
156
|
+
g.append(f'<text class="axis" transform="translate(12 {(MT + H - MB) / 2}) rotate(-90)" text-anchor="middle">speedup (bits)</text>')
|
|
157
|
+
return f'<svg class="chart" viewBox="0 0 {W} {H}" role="img" aria-label="Per-instance speedup against baseline cost">{"".join(g)}</svg>'
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
MIN_DIM_N = 3 # fewer paired instances than this at a dimension and no median is reported for it
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def dim_curve(tb, ts, bc, sc, boot=400):
|
|
164
|
+
"""Speedup BY DIMENSION, computed the way the estimator computes it (extrapolate.speedup_table, usable_dims) so the chart cannot disagree
|
|
165
|
+
with the credit. tb/ts: {dim: {instance: cost}} for baseline and submission (a failure is charged the cap); bc/sc: {dim: {instance: failed}}.
|
|
166
|
+
* the median is over ALL paired instances. Where the baseline hit the cap its cost is taken as the cap, so that instance contributes a
|
|
167
|
+
LOWER bound: this is what keeps the hardest instances (where the speedup is largest) from being silently dropped at large dimensions
|
|
168
|
+
(an earlier version averaged only the instances the baseline finished and showed a non-monotone, understated curve)
|
|
169
|
+
* `usable` / `why` are the estimator's own verdict (at most half the baseline runs capped, costs at least the 0.5 s floor)
|
|
170
|
+
* a 90% bootstrap interval of the median, seeded by the dimension so the same data always gives the same interval
|
|
171
|
+
A dimension with fewer than MIN_DIM_N paired instances has no median at all."""
|
|
172
|
+
usable, dropped = E.usable_dims(bc, sc, tb, ts)
|
|
173
|
+
out = {}
|
|
174
|
+
for d in sorted(tb):
|
|
175
|
+
ids = sorted(set(tb[d]) & set(ts.get(d, {})))
|
|
176
|
+
bits = [math.log2(tb[d][i] / ts[d][i]) for i in ids]
|
|
177
|
+
rec = {"n": len(ids), "censored": sum(1 for i in ids if bc[d].get(i)), "sub_failed": sum(1 for i in ids if sc.get(d, {}).get(i)),
|
|
178
|
+
"usable": d in usable, "why": dropped.get(d, ""), "median": None, "lo": None, "hi": None}
|
|
179
|
+
if len(bits) >= MIN_DIM_N:
|
|
180
|
+
rng = random.Random(d)
|
|
181
|
+
rec["median"] = statistics.median(bits)
|
|
182
|
+
meds = sorted(statistics.median([bits[rng.randrange(len(bits))] for _ in bits]) for _ in range(boot))
|
|
183
|
+
rec["lo"], rec["hi"] = meds[int(0.05 * boot)], meds[int(0.95 * boot) - 1]
|
|
184
|
+
out[d] = rec
|
|
185
|
+
return out
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def dim_chart(curve, budget):
|
|
189
|
+
"""A filled point with its 90% interval at each dimension the estimator uses; a HOLLOW point (no interval) where it does not, with the
|
|
190
|
+
reason in its tooltip. No line is drawn between points: a trend across a gap would be a claim the data does not make."""
|
|
191
|
+
if not curve:
|
|
192
|
+
return '<p class="muted">No dimensions to chart.</p>'
|
|
193
|
+
dims = sorted(curve)
|
|
194
|
+
vals = [v for r in curve.values() if r["usable"] for v in (r["median"], r["lo"], r["hi"]) if v is not None]
|
|
195
|
+
vals += [r["median"] for r in curve.values() if not r["usable"] and r["median"] is not None]
|
|
196
|
+
ticks = _nice_ticks(min(-1.0, min(vals, default=0) - 0.5), max(1.0, max(vals, default=0) + 0.5))
|
|
197
|
+
lo, hi = ticks[0], ticks[-1]
|
|
198
|
+
|
|
199
|
+
def y(v):
|
|
200
|
+
return MT + (H - MT - MB) * (hi - v) / (hi - lo)
|
|
201
|
+
pw = (W - ML - MR) / len(dims)
|
|
202
|
+
g = []
|
|
203
|
+
for t in ticks:
|
|
204
|
+
g.append(f'<line class="grid" x1="{ML}" x2="{W - MR}" y1="{y(t):.1f}" y2="{y(t):.1f}"/>'
|
|
205
|
+
f'<text class="tick" x="{ML - 8}" y="{y(t) + 4:.1f}" text-anchor="end">{fmt_tick(t)}</text>')
|
|
206
|
+
g.append(f'<line class="zero" x1="{ML}" x2="{W - MR}" y1="{y(0):.1f}" y2="{y(0):.1f}"/>')
|
|
207
|
+
for j, d in enumerate(dims):
|
|
208
|
+
r, x = curve[d], ML + pw * j + pw / 2
|
|
209
|
+
g.append(f'<text class="tick" x="{x:.1f}" y="{H - MB + 16}" text-anchor="middle">{d}</text>')
|
|
210
|
+
capped = f"{r['censored']} of {r['n']} baseline runs hit the cap and count as lower bounds" if r["censored"] else "no baseline run hit the cap"
|
|
211
|
+
if r["median"] is None:
|
|
212
|
+
why = f"dimension {d}: only {r['n']} paired instance(s); {MIN_DIM_N} are needed for a median, so nothing is shown"
|
|
213
|
+
g.append(f'<text class="n" x="{x:.1f}" y="{H - MB + 31}" text-anchor="middle">n={r["n"]}</text>'
|
|
214
|
+
f'<circle class="pt other" cx="{x:.1f}" cy="{y(0):.1f}" r="5" data-tip="{esc(why)}" tabindex="0"/>')
|
|
215
|
+
elif r["usable"]:
|
|
216
|
+
g.append(f'<line class="whisk" x1="{x:.1f}" x2="{x:.1f}" y1="{y(r["lo"]):.1f}" y2="{y(r["hi"]):.1f}"/>'
|
|
217
|
+
f'<line class="whisk" x1="{x - 5:.1f}" x2="{x + 5:.1f}" y1="{y(r["lo"]):.1f}" y2="{y(r["lo"]):.1f}"/>'
|
|
218
|
+
f'<line class="whisk" x1="{x - 5:.1f}" x2="{x + 5:.1f}" y1="{y(r["hi"]):.1f}" y2="{y(r["hi"]):.1f}"/>')
|
|
219
|
+
tip = (f"dimension {d}: median speedup {r['median']:+.2f} bits (90% interval {r['lo']:+.2f} to {r['hi']:+.2f}) over {r['n']} paired instances; {capped}")
|
|
220
|
+
g.append(f'<circle class="pt used" cx="{x:.1f}" cy="{y(r["median"]):.1f}" r="5" data-tip="{esc(tip)}" tabindex="0"/>')
|
|
221
|
+
g.append(f'<text class="n" x="{x:.1f}" y="{H - MB + 31}" text-anchor="middle">'
|
|
222
|
+
f'{(str(r["censored"]) + " capped") if r["censored"] else "n=" + str(r["n"])}</text>')
|
|
223
|
+
else:
|
|
224
|
+
if r["why"].startswith("baseline timeouts"):
|
|
225
|
+
why = (f"dimension {d}: the baseline hit the cap on {r['censored']} of {r['n']} instances (more than half), so the estimator does not use "
|
|
226
|
+
f"this dimension. The hollow point is the median with capped baselines taken as the cap: a lower bound, {r['median']:+.2f} bits, not a measurement")
|
|
227
|
+
short = "capped"
|
|
228
|
+
else:
|
|
229
|
+
why = (f"dimension {d}: {r['why']}, so the estimator does not use this dimension (a fixed setup cost, not the algorithm, would dominate). "
|
|
230
|
+
f"Median {r['median']:+.2f} bits shown for reference only")
|
|
231
|
+
short = "< floor"
|
|
232
|
+
g.append(f'<circle class="pt other" cx="{x:.1f}" cy="{y(r["median"]):.1f}" r="5" data-tip="{esc(why)}" tabindex="0"/>')
|
|
233
|
+
g.append(f'<text class="n" x="{x:.1f}" y="{H - MB + 31}" text-anchor="middle">{short}</text>')
|
|
234
|
+
g.append(f'<text class="axis" x="{(ML + W - MR) / 2}" y="{H - 6}" text-anchor="middle">dimension</text>')
|
|
235
|
+
g.append(f'<text class="axis" transform="translate(12 {(MT + H - MB) / 2}) rotate(-90)" text-anchor="middle">median speedup (bits)</text>')
|
|
236
|
+
return f'<svg class="chart" viewBox="0 0 {W} {H}" role="img" aria-label="Median speedup per dimension with 90% intervals">{"".join(g)}</svg>'
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# ---------------------------------------------------------------- page parts
|
|
240
|
+
ICON = {
|
|
241
|
+
"OFFICIAL": '<svg viewBox="0 0 16 16" width="14" height="14" aria-hidden="true"><circle cx="8" cy="8" r="7" fill="none" stroke="currentColor" stroke-width="1.6"/><path d="M4.6 8.4l2.2 2.2 4.6-5" fill="none" stroke="currentColor" stroke-width="1.8"/></svg>',
|
|
242
|
+
"PROVISIONAL": '<svg viewBox="0 0 16 16" width="14" height="14" aria-hidden="true"><circle cx="8" cy="8" r="7" fill="none" stroke="currentColor" stroke-width="1.6"/><path d="M8 1.5a6.5 6.5 0 0 1 0 13z" fill="currentColor"/></svg>',
|
|
243
|
+
"INCONCLUSIVE": '<svg viewBox="0 0 16 16" width="14" height="14" aria-hidden="true"><circle cx="8" cy="8" r="7" fill="none" stroke="currentColor" stroke-width="1.6"/><path d="M4.5 8h7" stroke="currentColor" stroke-width="1.8"/></svg>',
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def pill(kind, text):
|
|
248
|
+
return f'<span class="pill {kind.lower()}">{ICON[kind]}<span>{esc(text)}</span></span>'
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def status_pill(status):
|
|
252
|
+
return pill("OFFICIAL", "Official") if status == "OFFICIAL" else pill("PROVISIONAL", "Provisional")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def leaderboard(board):
|
|
256
|
+
rows = judge.leaderboard_rows(board)
|
|
257
|
+
if not rows:
|
|
258
|
+
return '<p class="muted">No scored runs yet.</p>'
|
|
259
|
+
out = ['<div class="scroll"><table class="lb"><thead><tr><th>Submission</th><th>Family</th><th class="num">Credit (bits)</th>'
|
|
260
|
+
'<th>Status</th><th>Per seed</th></tr></thead><tbody>']
|
|
261
|
+
for r in rows:
|
|
262
|
+
seeds = ", ".join("inconclusive" if v == "inconclusive" else f"{v:g}" for v in r["per_seed"])
|
|
263
|
+
why = "".join(f'<li>{esc(x)}</li>' for x in r["reasons"])
|
|
264
|
+
out.append(f'<tr><td class="strong">{esc(r["name"])}</td><td>{esc(r["family"])}</td><td class="num mono">{r["credit"]:.1f}</td>'
|
|
265
|
+
f'<td>{status_pill(r["status"])}<div class="small muted">{esc(r["status"])}</div></td><td class="mono">{esc(seeds)}</td></tr>')
|
|
266
|
+
if why:
|
|
267
|
+
out.append(f'<tr class="why"><td colspan="5"><span class="small muted">Not official because</span><ul>{why}</ul></td></tr>')
|
|
268
|
+
out.append("</tbody></table></div>")
|
|
269
|
+
return "".join(out)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def run_section(spec, d):
|
|
273
|
+
meas = d["meas"]
|
|
274
|
+
credit = None if meas is None else max(0.0, meas[1])
|
|
275
|
+
ms = d["meta_s"]
|
|
276
|
+
head = (f'<h3>{esc(spec["name"])} <span class="muted">on {esc(spec["family"])}, seed {esc(spec["tag"])}</span></h3>')
|
|
277
|
+
if credit is None:
|
|
278
|
+
verdict = pill("INCONCLUSIVE", "Inconclusive") + ('<p>Fewer than two difficulty bins had at least 5 instances and a submission '
|
|
279
|
+
'cost of at least 0.5 s, so no credit can be certified from this run.</p>')
|
|
280
|
+
else:
|
|
281
|
+
verdict = (f'<p class="big"><span class="mono">{credit:.1f}</span> bits certified '
|
|
282
|
+
f'<span class="muted">(median speedup {meas[0]:.1f}; the credit is the one-sided 95% lower bound)</span></p>')
|
|
283
|
+
hp = d["hp"]
|
|
284
|
+
hp_txt = ("inconclusive (needs 12 instances at 4 s or more and a submission median cost of at least 0.5 s)" if hp is None else
|
|
285
|
+
f"at least {max(0.0, hp[1]):.1f} bits (median {hp[0]:.1f}, {d['hp_n']} instances)")
|
|
286
|
+
facts = [("Instances", f"{d['n_total']} generated, {d['n_censored']} where the baseline hit the {d['budget']:g} s cap (excluded)"),
|
|
287
|
+
("Run", f"{esc(ms.get('sandbox') or 'bare local process')}, {esc(ms.get('jobs', 1))} concurrent job(s), {esc(ms.get('cpu', 'unknown CPU'))}"),
|
|
288
|
+
("Hard-pool claim (exploratory)", hp_txt)]
|
|
289
|
+
dl = "".join(f"<div><dt>{a}</dt><dd>{b}</dd></div>" for a, b in facts)
|
|
290
|
+
rows = "".join(
|
|
291
|
+
f'<tr><td class="mono">{x["lo"]:g}–{x["hi"]:g}</td><td class="num mono">{x["n"]}</td><td class="num mono">{x["median_bits"]:+.2f}</td>'
|
|
292
|
+
f'<td class="num mono">{x["median_sub_cost"]}</td><td>{"in the credit" if k in d["top"] else ("usable" if x["usable"] else "not usable")}</td></tr>'
|
|
293
|
+
for k, x in sorted(d["info"].items()))
|
|
294
|
+
table = ('<details class="tv"><summary>Table view</summary><div class="scroll"><table class="lb"><thead><tr><th>Baseline cost (s)</th>'
|
|
295
|
+
'<th class="num">Instances</th><th class="num">Median bits</th><th class="num">Submission median (s)</th><th>Role</th></tr></thead>'
|
|
296
|
+
f'<tbody>{rows}</tbody></table></div></details>')
|
|
297
|
+
curve = d.get("curve") # absent in dicts built by older callers: then the section simply has no dimension chart
|
|
298
|
+
dimfig = ""
|
|
299
|
+
if curve:
|
|
300
|
+
dimfig = (f'<figure><figcaption>Speedup by dimension</figcaption>{dim_chart(curve, d["budget"])}'
|
|
301
|
+
'<p class="small muted">Median over all paired instances, with a 90% bootstrap interval; where the baseline hit its cap its cost is '
|
|
302
|
+
'taken as the cap, so those instances can only understate the speedup. Hollow points are dimensions the estimator does not use '
|
|
303
|
+
'("capped": more than half the baseline runs hit the cap; "< floor": costs under 0.5 s): they are <strong>not measured</strong>, '
|
|
304
|
+
'shown only as lower bounds or for reference. Intervals of neighbouring dimensions often overlap, so a rising median is a tendency, '
|
|
305
|
+
'not a measured rate. Growth beyond the last measured dimension is not measured either: no line is drawn across a gap, and '
|
|
306
|
+
'nothing here is extrapolated.</p></figure>')
|
|
307
|
+
return (f'<section class="run">{head}{verdict}<dl class="facts">{dl}</dl>'
|
|
308
|
+
f'<figure><figcaption>Speedup by how hard the baseline found each instance</figcaption>{bin_chart(d["info"], d["top"], credit, d["budget"])}'
|
|
309
|
+
f'<p class="legend"><span class="sw used"></span>in the credit <span class="sw usable"></span>usable <span class="sw unusable"></span>not usable'
|
|
310
|
+
f'<span class="sw line"></span>credit (lower bound)</p></figure>{dimfig}'
|
|
311
|
+
f'<figure><figcaption>Every instance the baseline finished</figcaption>{scatter_chart(d["pts"], d["info"], d["top"], d["budget"])}</figure>{table}</section>')
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
CSS = """
|
|
315
|
+
:root{--bg:#EFF1F3;--card:#FCFCFB;--ink:#12161A;--ink2:#464E55;--muted:#66707A;--line:#D8DDE1;--accent:#2A78D6;--accent-soft:#BBD5F3;
|
|
316
|
+
--ok:#1D6B3B;--ok-bg:#DCEFE2;--warn:#8A4B00;--warn-bg:#F9E8CC;--idle:#4F5961;--idle-bg:#E3E6E9;
|
|
317
|
+
--display:"Bricolage Grotesque","Avenir Next","Segoe UI",system-ui,sans-serif;--body:"Instrument Sans","Segoe UI",system-ui,sans-serif;
|
|
318
|
+
--mono:"JetBrains Mono",ui-monospace,SFMono-Regular,Menlo,monospace}
|
|
319
|
+
@media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#101214;--card:#1A1A19;--ink:#ECEFF1;--ink2:#BAC2C8;--muted:#8E98A0;--line:#2C3237;
|
|
320
|
+
--accent:#3987E5;--accent-soft:#1E3F66;--ok:#79D09A;--ok-bg:#16301F;--warn:#F0B565;--warn-bg:#3A2A10;--idle:#B6BEC5;--idle-bg:#262B2F;color-scheme:dark}}
|
|
321
|
+
:root[data-theme="dark"]{--bg:#101214;--card:#1A1A19;--ink:#ECEFF1;--ink2:#BAC2C8;--muted:#8E98A0;--line:#2C3237;
|
|
322
|
+
--accent:#3987E5;--accent-soft:#1E3F66;--ok:#79D09A;--ok-bg:#16301F;--warn:#F0B565;--warn-bg:#3A2A10;--idle:#B6BEC5;--idle-bg:#262B2F;color-scheme:dark}
|
|
323
|
+
*{box-sizing:border-box}
|
|
324
|
+
body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.55 var(--body);padding-inline:max(16px,4vw);padding-block:28px 56px}
|
|
325
|
+
.wrap{max-width:980px;margin:0 auto;display:grid;gap:22px}
|
|
326
|
+
h1{font:600 clamp(26px,5vw,40px)/1.08 var(--display);margin:0 0 6px;letter-spacing:-.02em;text-wrap:balance}
|
|
327
|
+
h2{font:600 19px/1.2 var(--display);margin:0 0 12px;letter-spacing:-.01em}
|
|
328
|
+
h3{font:600 16px/1.3 var(--display);margin:0 0 8px}
|
|
329
|
+
p{margin:0 0 10px;max-width:68ch}
|
|
330
|
+
.lede{color:var(--ink2);font-size:16px;max-width:62ch}
|
|
331
|
+
.muted{color:var(--muted)} .small{font-size:12.5px} .strong{font-weight:600} .mono{font-family:var(--mono);font-variant-numeric:tabular-nums}
|
|
332
|
+
.card{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:20px;min-width:0}
|
|
333
|
+
.banner{display:flex;gap:12px;align-items:flex-start;background:var(--warn-bg);color:var(--warn);border-radius:10px;padding:14px 16px}
|
|
334
|
+
.banner p{margin:0;color:var(--ink)} .banner svg{flex:none;margin-top:3px}
|
|
335
|
+
.scroll{overflow-x:auto}
|
|
336
|
+
table.lb{border-collapse:collapse;width:100%;font-size:14px}
|
|
337
|
+
.lb th{font:600 11.5px/1.2 var(--body);letter-spacing:.06em;text-transform:uppercase;color:var(--muted);text-align:left;padding:8px 12px 8px 0;border-bottom:1px solid var(--line)}
|
|
338
|
+
.lb td{padding:10px 12px 10px 0;border-bottom:1px solid var(--line);vertical-align:top}
|
|
339
|
+
.lb .num{text-align:right;font-variant-numeric:tabular-nums} .lb tr.why td{padding-top:0;border-bottom:1px solid var(--line)}
|
|
340
|
+
.lb ul{margin:4px 0 0;padding-left:18px;color:var(--ink2);font-size:13px}
|
|
341
|
+
.pill{display:inline-flex;gap:6px;align-items:center;border-radius:999px;padding:3px 10px 3px 8px;font-size:12.5px;font-weight:600}
|
|
342
|
+
.pill.official{background:var(--ok-bg);color:var(--ok)} .pill.provisional{background:var(--warn-bg);color:var(--warn)} .pill.inconclusive{background:var(--idle-bg);color:var(--idle)}
|
|
343
|
+
.run{display:grid;gap:12px} .big{font-size:20px;font-family:var(--display);font-weight:600;margin:0} .big .muted{font:400 13px var(--body)}
|
|
344
|
+
dl.facts{display:grid;gap:6px;margin:0;font-size:13.5px} dl.facts div{display:grid;grid-template-columns:minmax(120px,190px) 1fr;gap:10px} dt{color:var(--muted)} dd{margin:0;min-width:0}
|
|
345
|
+
figure{margin:0;display:grid;gap:6px} figcaption{font-weight:600;font-size:13.5px}
|
|
346
|
+
svg.chart{width:100%;height:auto;display:block;overflow:visible}
|
|
347
|
+
.grid{stroke:var(--line);stroke-width:1} .zero{stroke:var(--muted);stroke-width:1.2}
|
|
348
|
+
.tick,.n,.axis{fill:var(--ink2);font:11.5px var(--body)} .n{fill:var(--muted);font:11px var(--mono)} .axis{fill:var(--muted)}
|
|
349
|
+
.bar.used{fill:var(--accent)} .bar.usable{fill:var(--accent-soft);stroke:var(--accent);stroke-width:1.2}
|
|
350
|
+
.bar.unusable{fill:none;stroke:var(--muted);stroke-width:1.2;stroke-dasharray:4 3}
|
|
351
|
+
.credit{stroke:var(--ink);stroke-width:1.4;stroke-dasharray:6 4} .creditlbl{fill:var(--ink);font:600 12px var(--body)}
|
|
352
|
+
.pt.used{fill:var(--accent);stroke:var(--card);stroke-width:1.5} .pt.other{fill:none;stroke:var(--muted);stroke-width:1.5}
|
|
353
|
+
.whisk{stroke:var(--accent);stroke-width:1.6}
|
|
354
|
+
.bar:focus-visible,.pt:focus-visible{outline:2px solid var(--ink);outline-offset:2px}
|
|
355
|
+
.legend{display:flex;flex-wrap:wrap;gap:6px 16px;align-items:center;font-size:12.5px;color:var(--ink2);margin:0}
|
|
356
|
+
.sw{display:inline-block;width:12px;height:12px;border-radius:3px;margin-right:6px;vertical-align:-1px}
|
|
357
|
+
.sw.used{background:var(--accent)} .sw.usable{background:var(--accent-soft);border:1.2px solid var(--accent)} .sw.unusable{border:1.2px dashed var(--muted)}
|
|
358
|
+
.sw.line{width:18px;height:0;border-top:2px dashed var(--ink);border-radius:0;margin-left:10px}
|
|
359
|
+
details.tv summary{cursor:pointer;font-weight:600;font-size:13.5px;padding:6px 0} details.tv summary:focus-visible{outline:2px solid var(--ink);outline-offset:2px}
|
|
360
|
+
.limits ul{margin:0;padding-left:20px;display:grid;gap:7px;color:var(--ink2);max-width:74ch}
|
|
361
|
+
#tip{position:fixed;z-index:9;pointer-events:none;background:var(--ink);color:var(--card);font-size:12.5px;line-height:1.4;padding:7px 10px;border-radius:6px;max-width:300px;opacity:0;transition:opacity .08s}
|
|
362
|
+
@media (max-width:560px){dl.facts div{grid-template-columns:1fr;gap:0}.card{padding:16px}}
|
|
363
|
+
@media (prefers-reduced-motion:reduce){#tip{transition:none}}
|
|
364
|
+
"""
|
|
365
|
+
|
|
366
|
+
JS = """
|
|
367
|
+
(function(){var tip=document.getElementById('tip');function show(e){var t=e.target.getAttribute&&e.target.getAttribute('data-tip');
|
|
368
|
+
if(!t){tip.style.opacity=0;return}tip.textContent=t;tip.style.opacity=1;var x=(e.clientX||e.target.getBoundingClientRect().left)+14,y=(e.clientY||e.target.getBoundingClientRect().top)+14;
|
|
369
|
+
tip.style.left=Math.min(x,window.innerWidth-tip.offsetWidth-8)+'px';tip.style.top=Math.min(y,window.innerHeight-tip.offsetHeight-8)+'px'}
|
|
370
|
+
document.addEventListener('mousemove',show);document.addEventListener('focusin',show);document.addEventListener('focusout',function(){tip.style.opacity=0});
|
|
371
|
+
document.addEventListener('mouseleave',function(){tip.style.opacity=0})})();
|
|
372
|
+
"""
|
|
373
|
+
|
|
374
|
+
LIMITS = [
|
|
375
|
+
"The Docker sandbox that makes it safe to run other people's code has passed its tests on real Linux machines, but not on a dedicated evaluation machine separate from the web service, and no result on this page was measured in it. Every run on this page used bare local processes and my own solvers, so none of it can be an official result.",
|
|
376
|
+
"Official credit needs three independent seeds, each with a published commitment, the sandbox, and the family's full instance count and CPU cap. No seed on this page meets all of those.",
|
|
377
|
+
"The credit rule was designed while looking at seed A, so seed A is not an independent test. The out-of-sample test is a pre-registered run on a fresh seed, with its pass criterion written down before the data existed.",
|
|
378
|
+
"The certifiable range is limited. At the official ladder (24 instances per size, 120 s cap), a simulation with real per-instance noise certifies a constant speedup of 1.5 to 3 bits with roughly 65 to 100% probability (each figure is good to about 10 points), 1 bit rarely (6.5%) and 4 bits almost never (1%), because too few hard instances remain for the bound to stay stable. A round can ask for more instances per size (48: 68% at 4 bits; 96: 100% at 4 and 98% at 5 bits), but about 5 bits is the ceiling at a 120 s cap whatever the count, since a very fast submission falls under the 0.5 s cost floor wherever the baseline finishes. Results outside the range are reported as inconclusive, never credited, and speedups of tens of bits cannot be certified by this design at all.",
|
|
379
|
+
"The paper's own code (the \"egg\") cannot yet be used as a baseline: its sieve does not reach the target past dimension 30.",
|
|
380
|
+
]
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def prereg_verdict(primary_credit, hard_pool):
|
|
384
|
+
"""Apply the UNCONDITIONAL criteria written in a pre-registration file, mechanically.
|
|
385
|
+
primary_credit: None if the primary claim was inconclusive, else the credit in bits.
|
|
386
|
+
hard_pool: None if inconclusive, else (median, lower bound). Returns (primary verdict, exploratory verdict)."""
|
|
387
|
+
if primary_credit is None:
|
|
388
|
+
primary = ("(c) inconclusive", "FAILS for this configuration (insufficient power)")
|
|
389
|
+
elif primary_credit >= 1.0:
|
|
390
|
+
primary = ("(a) credit >= 1.0 bit", "SUPPORTS the claim that this ladder size can certify a pruning-sized speedup")
|
|
391
|
+
else:
|
|
392
|
+
primary = ("(b) conclusive but below 1.0 bit", "FAILS")
|
|
393
|
+
if hard_pool is None:
|
|
394
|
+
hp = "NOT promoted (inconclusive)"
|
|
395
|
+
elif hard_pool[1] >= 1.0:
|
|
396
|
+
hp = "first out-of-sample support for promoting it (lower bound >= 1.0 bit)"
|
|
397
|
+
else:
|
|
398
|
+
hp = "NOT promoted (lower bound below 1.0 bit)"
|
|
399
|
+
return primary, hp
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def prereg_card(path, data, tag, remeasure_note):
|
|
403
|
+
run = next(((s, d) for s, d in data if s["tag"] == tag and "error" not in d), None)
|
|
404
|
+
if run is None:
|
|
405
|
+
return ""
|
|
406
|
+
with open(path, encoding="utf-8") as f:
|
|
407
|
+
text = f.read()
|
|
408
|
+
d = run[1]
|
|
409
|
+
credit = None if d["meas"] is None else max(0.0, d["meas"][1])
|
|
410
|
+
primary, hp = prereg_verdict(credit, d["hp"])
|
|
411
|
+
# two decimals here: the criterion is a hard threshold at 1.0, so 0.96 must not be displayed as "1.0"
|
|
412
|
+
shown = "inconclusive" if credit is None else f"{credit:.2f} bits"
|
|
413
|
+
hp_shown = "inconclusive" if d["hp"] is None else f"lower bound {max(0.0, d['hp'][1]):.2f} bits"
|
|
414
|
+
return (f'<section class="card"><h2>Pre-registered test, seed {esc(tag)}</h2>'
|
|
415
|
+
f'<p>The pass criteria were written before any data from this seed existed and are applied here without interpretation.</p>'
|
|
416
|
+
f'<dl class="facts"><div><dt>Primary claim</dt><dd>{esc(shown)}: outcome {esc(primary[0])}, which {esc(primary[1])}.</dd></div>'
|
|
417
|
+
f'<div><dt>Exploratory hard-pool claim</dt><dd>{esc(hp_shown)}: {esc(hp)}.</dd></div>'
|
|
418
|
+
f'<div><dt>Concurrency check</dt><dd>{esc(remeasure_note or "not recorded on this page; see the remeasure output")}</dd></div></dl>'
|
|
419
|
+
f'<details class="tv"><summary>The registration, verbatim</summary><pre class="small mono" style="white-space:pre-wrap;overflow-x:auto">{esc(text)}</pre></details></section>')
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def page(specs, board, sandbox_verified=False, prereg=None, remeasure_note=""):
|
|
423
|
+
data = []
|
|
424
|
+
for s in specs:
|
|
425
|
+
try:
|
|
426
|
+
data.append((s, load_pair(s["baseline"], s["submission"])))
|
|
427
|
+
except (OSError, KeyError, ValueError) as e: # a corrupt or incompatible pair is reported, not hidden
|
|
428
|
+
data.append((s, {"error": str(e)}))
|
|
429
|
+
runs = "".join(f'<div class="card">{run_section(s, d)}</div>' if "error" not in d else
|
|
430
|
+
f'<div class="card"><p>Could not read {esc(s["name"])} ({esc(s["tag"])}): {esc(d["error"])}</p></div>' for s, d in data)
|
|
431
|
+
official = sum(1 for r in judge.leaderboard_rows(board) if r["status"] == "OFFICIAL")
|
|
432
|
+
local = any("error" not in d and not d["meta_s"].get("sandbox") for _, d in data)
|
|
433
|
+
reasons = []
|
|
434
|
+
if not sandbox_verified:
|
|
435
|
+
reasons.append("the sandbox has not passed its tests")
|
|
436
|
+
if local:
|
|
437
|
+
reasons.append("some runs were bare local processes")
|
|
438
|
+
banner = f"{official} official result(s) below." if official else "No result on this page is official."
|
|
439
|
+
banner_rest = (f"Why: {'; '.join(reasons)}. These are measurements of the method, not certified results. Limits are listed at the bottom."
|
|
440
|
+
if reasons and not official else "Limits are listed at the bottom.")
|
|
441
|
+
limits = list(LIMITS)
|
|
442
|
+
if sandbox_verified:
|
|
443
|
+
limits[0] = ("The sandbox passed its tests on this machine, and the verification is tied to a fingerprint of the sandbox files, "
|
|
444
|
+
"so it lapses if they change. Runs labelled \"bare local process\" above still cannot count as official.")
|
|
445
|
+
stamp = time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())
|
|
446
|
+
prereg_html = prereg_card(prereg[0], data, prereg[1], remeasure_note) if prereg else ""
|
|
447
|
+
return f"""<title>Lattice Speedup Ledger</title>
|
|
448
|
+
<style>{CSS}</style>
|
|
449
|
+
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:wght@600&family=Instrument+Sans:wght@400;600&family=JetBrains+Mono:wght@400;600&display=swap">
|
|
450
|
+
<main class="wrap">
|
|
451
|
+
<header><h1>Lattice Speedup Ledger</h1>
|
|
452
|
+
<p class="lede">How many bits faster than a reference solver does a lattice algorithm find short vectors, measured on instances nobody could prepare for. Generated {esc(stamp)} from saved ladders.</p></header>
|
|
453
|
+
<div class="banner" role="note">{ICON["PROVISIONAL"]}<p><strong>{esc(banner)}</strong> {esc(banner_rest)}</p></div>
|
|
454
|
+
<section class="card"><h2>Leaderboard</h2>{leaderboard(board)}</section>
|
|
455
|
+
{prereg_html}
|
|
456
|
+
<section><h2>Runs</h2><div style="display:grid;gap:16px">{runs or '<p class="muted">No ladder pairs found.</p>'}</div></section>
|
|
457
|
+
<section class="card limits"><h2>What this page does not show</h2><ul>{"".join(f"<li>{esc(x)}</li>" for x in limits)}</ul></section>
|
|
458
|
+
</main><div id="tip" role="tooltip"></div>
|
|
459
|
+
<script>{JS}</script>"""
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def main():
|
|
463
|
+
ap = argparse.ArgumentParser()
|
|
464
|
+
ap.add_argument("--out", default=os.path.join(HERE, "results", "report.html"))
|
|
465
|
+
ap.add_argument("--pair", nargs=2, action="append", metavar=("BASELINE", "SUBMISSION"))
|
|
466
|
+
ap.add_argument("--no-rescore", action="store_true", help="use the board file as it is (it may predate the current estimator)")
|
|
467
|
+
ap.add_argument("--prereg", nargs=2, metavar=("FILE", "SEED_TAG"), help="add a card applying a pre-registration's criteria to that seed's run")
|
|
468
|
+
ap.add_argument("--remeasure-note", default="", help="one line summarising the remeasure.py concurrency check, shown on the card")
|
|
469
|
+
a = ap.parse_args()
|
|
470
|
+
if not a.no_rescore:
|
|
471
|
+
# SIDE EFFECT: this rewrites results/leaderboard_v2.json (and each ladder's .claim.json) so the leaderboard comes from the
|
|
472
|
+
# same estimator as the charts; without it the page can contradict itself. Do not run it while a ladder is still being written.
|
|
473
|
+
import subprocess
|
|
474
|
+
import sys
|
|
475
|
+
subprocess.run([sys.executable, os.path.join(HERE, "judge.py"), "rescore"], cwd=HERE, capture_output=True)
|
|
476
|
+
if a.pair:
|
|
477
|
+
specs = [{"family": "gm", "tag": "custom", "baseline": b, "submission": s, "name": os.path.basename(s)} for b, s in a.pair]
|
|
478
|
+
else:
|
|
479
|
+
specs = find_pairs(os.path.join(HERE, "results"))
|
|
480
|
+
board = []
|
|
481
|
+
if os.path.exists(judge.BOARD):
|
|
482
|
+
with open(judge.BOARD, encoding="utf-8") as f:
|
|
483
|
+
board = json.load(f)
|
|
484
|
+
import portal_core # standard-library-only home of the sandbox gate (portal imports this module, so importing portal here would be circular)
|
|
485
|
+
verified = portal_core.sandbox_verified(os.path.join(HERE, "portal_data")) # read-only: data_dir() would create the directory
|
|
486
|
+
with open(a.out, "w", encoding="utf-8") as f:
|
|
487
|
+
f.write(page(specs, board, sandbox_verified=verified, prereg=tuple(a.prereg) if a.prereg else None,
|
|
488
|
+
remeasure_note=a.remeasure_note))
|
|
489
|
+
print(f"wrote {a.out} ({len(specs)} run(s))")
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
if __name__ == "__main__":
|
|
493
|
+
main()
|
guldo/_src/results/.keep
ADDED
|
File without changes
|