guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/build_site.py
ADDED
|
@@ -0,0 +1,958 @@
|
|
|
1
|
+
"""Build the Guldo website (static, multi-page) from the saved ladders and pre-registration files.
|
|
2
|
+
|
|
3
|
+
python3 build_site.py --out site [--reveal 755a57fd=/path/seedD.seed ...]
|
|
4
|
+
|
|
5
|
+
Every number comes from the saved ladders through extrapolate.py (the judge's own estimator); nothing is typed in by hand
|
|
6
|
+
except the prose. This script READS the results: it does not rescore (run `python3 judge.py rescore` first if the board
|
|
7
|
+
may predate the current estimator) and it never executes a solver. A revealed seed is checked against its published
|
|
8
|
+
commitment before it is shipped, and one that does not hash to the commitment is refused.
|
|
9
|
+
"""
|
|
10
|
+
import argparse
|
|
11
|
+
import ast
|
|
12
|
+
import hashlib
|
|
13
|
+
import html
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import shutil
|
|
18
|
+
import statistics
|
|
19
|
+
import tempfile
|
|
20
|
+
import time
|
|
21
|
+
from urllib.parse import quote
|
|
22
|
+
|
|
23
|
+
import families
|
|
24
|
+
import judge
|
|
25
|
+
import report_html as R
|
|
26
|
+
import site_assets
|
|
27
|
+
|
|
28
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
29
|
+
esc = R.esc
|
|
30
|
+
MARKER = ".speedup-ledger-site"
|
|
31
|
+
|
|
32
|
+
PAGES = [("index.html", "Overview"), ("install.html", "Install"), ("domains.html", "Domains"), ("guide.html", "Guide"), ("method.html", "Method"), ("results.html", "Results"),
|
|
33
|
+
("preregistrations.html", "Pre-registrations"), ("compete.html", "Compete"), ("certify.html", "Certify"), ("claims.html", "Other claims"), ("limits.html", "Limits")]
|
|
34
|
+
|
|
35
|
+
# what each registration became; the registration FILES are the record, this is only a label
|
|
36
|
+
PREREG_STATUS = {
|
|
37
|
+
"2315b668": ("void", "Void: the run was killed and the harness changed before it finished. Superseded by seed C."),
|
|
38
|
+
"14a1b0db": ("void", "Void: the run was killed and the harness changed before it finished. Superseded by seed D."),
|
|
39
|
+
"755a57fd": ("done", "Completed. Criterion (a) met; see Results."),
|
|
40
|
+
"88ecb627": ("done", "Positive control on a second domain (integer factoring). The written criteria are applied to the saved ladders on the Results page."),
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
EXTRA_CSS = """
|
|
44
|
+
.big{font-size:20px;margin:0 0 10px;max-width:none} .run{display:grid;gap:14px} .tv{margin-top:4px}
|
|
45
|
+
.crumbs{display:flex;flex-wrap:wrap;gap:8px 18px}
|
|
46
|
+
.two{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(300px,100%),1fr));gap:22px;align-items:start}
|
|
47
|
+
.fine{font-size:14px;color:var(--ink2)}
|
|
48
|
+
.gloss{display:grid;grid-template-columns:minmax(120px,max-content) 1fr;gap:8px 18px;margin:0;font-size:15px} .gloss dt{font-weight:600} .gloss dd{margin:0;color:var(--ink2)}
|
|
49
|
+
@media (max-width:560px){.gloss{grid-template-columns:1fr;gap:2px} .gloss dd{margin-bottom:10px}}
|
|
50
|
+
details.more{margin-top:12px} details.more summary{cursor:pointer;font-weight:600;color:var(--accent-ink)} details.more[open] summary{margin-bottom:8px}
|
|
51
|
+
.guide{min-width:0} .guide h2{margin-top:34px} .guide h3,.guide h4{margin-top:22px} .guide p,.guide li{max-width:72ch}
|
|
52
|
+
pre.codeblock{background:var(--chip);border:1px solid var(--line);border-radius:10px;padding:12px 14px;overflow-x:auto;font-family:var(--mono);font-size:13px;line-height:1.5}
|
|
53
|
+
.guide code{font-family:var(--mono);font-size:.92em;background:var(--chip);border-radius:5px;padding:1px 5px} .guide pre code{background:none;padding:0}
|
|
54
|
+
.toc{align-self:start;position:sticky;top:16px} .toc ul{margin:8px 0 0;padding-left:18px}
|
|
55
|
+
.bar{height:10px;min-width:90px;background:var(--line);border-radius:5px;overflow:hidden} .bar span{display:block;height:100%;background:var(--accent)}
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
LOGO = ('<svg viewBox="0 0 28 28" width="26" height="26" aria-hidden="true"><rect x="1.5" y="1.5" width="25" height="25" rx="6" fill="none" '
|
|
59
|
+
'stroke="var(--ink)" stroke-width="2"/><path d="M6 20h4v-4h4v-4h4V8h4" fill="none" stroke="var(--accent)" stroke-width="2.4" '
|
|
60
|
+
'stroke-linecap="round" stroke-linejoin="round"/></svg>')
|
|
61
|
+
FAVICON = ("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 28 28'%3E%3Crect width='28' height='28' rx='6' fill='%232A78D6'/%3E"
|
|
62
|
+
"%3Cpath d='M6 20h4v-4h4v-4h4V8h4' fill='none' stroke='white' stroke-width='2.6' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E")
|
|
63
|
+
FONTS = ('<link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>'
|
|
64
|
+
'<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:wght@600&family=Instrument+Sans:wght@400;600'
|
|
65
|
+
'&family=JetBrains+Mono:wght@400;600&display=swap">')
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# ---------------------------------------------------------------- shared layout
|
|
69
|
+
def layout(fname, title, description, body, runs_manifest=None):
|
|
70
|
+
nav = "".join(f'<a href="{f}"' + (' aria-current="page"' if f == fname else "") + f'>{esc(t)}</a>' for f, t in PAGES)
|
|
71
|
+
manifest = ""
|
|
72
|
+
if runs_manifest is not None:
|
|
73
|
+
manifest = '<script type="application/json" id="runs-manifest">' + json.dumps(runs_manifest).replace("</", "<\\/") + "</script>"
|
|
74
|
+
stamp = time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())
|
|
75
|
+
return f"""<!doctype html>
|
|
76
|
+
<html lang="en" data-theme="dark"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
77
|
+
<title>{esc(title)}</title><meta name="description" content="{esc(description)}">
|
|
78
|
+
<meta name="color-scheme" content="dark light"><link rel="icon" href="{FAVICON}">
|
|
79
|
+
<script>try{{if(localStorage.getItem('theme')==='light')document.documentElement.setAttribute('data-theme','light')}}catch(e){{}}</script>
|
|
80
|
+
{FONTS}<link rel="stylesheet" href="assets/site.css"></head>
|
|
81
|
+
<body><a class="skip" href="#main">Skip to content</a>
|
|
82
|
+
<div class="wrap"><header class="top"><a class="brand" href="index.html">{LOGO}Guldo</a>
|
|
83
|
+
<nav class="main" aria-label="Main">{nav}<button class="theme" id="theme-toggle" type="button" aria-pressed="true">Light mode</button></nav></header>
|
|
84
|
+
<main id="main">{body}</main>
|
|
85
|
+
<footer class="foot"><p>Generated {esc(stamp)} from the saved measurement files in <span class="mono">results/</span>; every figure on these pages is recomputed from that data by the judge's own scoring code.</p>
|
|
86
|
+
<p>No result on this site is official: they are all practice runs. <a href="limits.html">Why, and what would make one official</a>.</p></footer></div>
|
|
87
|
+
<div id="tip" role="tooltip"></div>{manifest}<script src="assets/site.js"></script></body></html>"""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def banner(text):
|
|
91
|
+
return f'<div class="banner" role="note">{R.ICON["PROVISIONAL"]}<p>{text}</p></div>'
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def cmd(text):
|
|
95
|
+
return f'<div class="cmd"><code>{esc(text)}</code><button class="copy" type="button" data-copy="{esc(text)}">Copy</button></div>'
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# ---------------------------------------------------------------- data
|
|
99
|
+
SELFTEST_CONTROLS = {"control_zero", "control_none"} # the compiler's own self-test submissions (compile_domain.selftest): controls, not results
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def collect(results_dir):
|
|
103
|
+
specs = [s for s in R.find_pairs(results_dir) if s["name"] not in SELFTEST_CONTROLS]
|
|
104
|
+
data = []
|
|
105
|
+
for s in specs:
|
|
106
|
+
try:
|
|
107
|
+
data.append((s, R.load_pair(s["baseline"], s["submission"])))
|
|
108
|
+
except (OSError, KeyError, ValueError, TypeError, AttributeError, IndexError) as e: # an incomplete, empty or corrupt pair is listed as such, never charted, and never stops the build
|
|
109
|
+
data.append((s, {"error": str(e)}))
|
|
110
|
+
return data
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def credit_of(d):
|
|
114
|
+
return None if d.get("meas") is None else max(0.0, d["meas"][1])
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def read_prereg(results_dir):
|
|
118
|
+
out = []
|
|
119
|
+
for fn in sorted(os.listdir(results_dir)):
|
|
120
|
+
if not (fn.startswith("PREREGISTERED_") and fn.endswith(".txt")):
|
|
121
|
+
continue
|
|
122
|
+
with open(os.path.join(results_dir, fn), encoding="utf-8") as f:
|
|
123
|
+
text = f.read()
|
|
124
|
+
m = re.search(r"commitment \(sha256\): ([0-9a-f]{64})", text)
|
|
125
|
+
d = re.search(r"^date: (\S+)", text, re.M)
|
|
126
|
+
if not m:
|
|
127
|
+
continue
|
|
128
|
+
out.append({"file": fn, "text": text, "commit": m.group(1), "tag": m.group(1)[:8], "date": d.group(1) if d else "",
|
|
129
|
+
"title": text.splitlines()[0][:150]})
|
|
130
|
+
return out
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def load_reveals(pairs, prereg):
|
|
134
|
+
commits = {p["commit"] for p in prereg}
|
|
135
|
+
seeds = {}
|
|
136
|
+
for item in pairs or []:
|
|
137
|
+
tag, _, path = item.partition("=")
|
|
138
|
+
raw = open(os.path.expanduser(path), "rb").read().strip()
|
|
139
|
+
h = hashlib.sha256(raw).hexdigest()
|
|
140
|
+
if h not in commits or not h.startswith(tag):
|
|
141
|
+
raise SystemExit(f"REFUSED: {path} hashes to {h[:16]}..., which is not the registered commitment {tag}")
|
|
142
|
+
seeds[h] = raw.decode()
|
|
143
|
+
return seeds
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
# ---------------------------------------------------------------- pages
|
|
147
|
+
def compiler_sample():
|
|
148
|
+
"""A card of the compiler's REAL verdict for the first compiled domain installed (the text `compile_domain.py show NAME` prints), or
|
|
149
|
+
nothing when none is installed. Never typed by hand, so it cannot show a result the compiler did not produce."""
|
|
150
|
+
import compile_domain as CD
|
|
151
|
+
for name, info in sorted(families.DECLARED.items()):
|
|
152
|
+
try:
|
|
153
|
+
_, lines = CD.verdict(info["compiled"])
|
|
154
|
+
except (KeyError, TypeError, ValueError):
|
|
155
|
+
continue
|
|
156
|
+
# the summary only: the notes and the next-step commands are long, and the full card has them
|
|
157
|
+
head = [ln for ln in lines if not ln.startswith((" note", " next:", " "))]
|
|
158
|
+
n_notes = sum(ln.startswith(" note") for ln in lines)
|
|
159
|
+
if n_notes:
|
|
160
|
+
head.append(f" notes : {n_notes} (listed on the full card)")
|
|
161
|
+
return (f'<div><p class="kicker">Real compiler output, for the {esc(name)} example</p><div class="card">'
|
|
162
|
+
f'<pre class="codeblock"><code>{esc(chr(10).join(head))}</code></pre>'
|
|
163
|
+
f'<p class="fine">A shortened version of what <span class="mono">python3 compile_domain.py show {esc(name)}</span> prints. '
|
|
164
|
+
f'<a href="domains.html#{esc(name)}">See the full card</a>.</p></div></div>')
|
|
165
|
+
return ""
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def page_index(data, board, prereg, notes=None):
|
|
169
|
+
sample = compiler_sample()
|
|
170
|
+
hero_side = ('<div><p class="kicker">An illustration, not a result</p><div class="card"><h3>Two solvers, the same questions</h3>'
|
|
171
|
+
'<p class="fine">Say two people each write a program that factors big numbers. Both are given the same 100 numbers, made from a secret number that nobody could see in advance. '
|
|
172
|
+
'The first program takes 40 seconds on the hard ones. The second takes 5. That is 8 times faster, or 3 bits. '
|
|
173
|
+
'This site is a way for a stranger to confirm a claim like that without taking anyone\'s word for it.</p></div>'
|
|
174
|
+
+ (f'<details class="more"><summary>Show what the compiler really prints</summary>{sample}'
|
|
175
|
+
'<p class="fine">In that printout, the <em>size ladder</em> is the list of question sizes and the <em>CPU cap</em> is the time limit per question.</p></details>' if sample else "")
|
|
176
|
+
+ '</div>')
|
|
177
|
+
body = f"""
|
|
178
|
+
<section class="hero"><div><p class="kicker">Verified speedups, without trusting the claimant</p>
|
|
179
|
+
<h1>Declare a problem. Get a challenge <em style="font-style:normal;color:var(--accent-ink)">anyone can verify</em>.</h1>
|
|
180
|
+
<p class="lede">Someone says their algorithm is much faster than the standard one. How would you check? Describe the problem once. The compiler builds a fair test, runs it, and produces a report that anyone can re-check.</p>
|
|
181
|
+
<p class="fine">How we count speed: <strong>3 bits faster means 8 times faster</strong>, and 10 bits means about 1,000 times. Each extra bit doubles it.</p>
|
|
182
|
+
<div class="btnrow"><a class="btn primary" href="domains.html">See the domains</a><a class="btn" href="domains.html#add">Compile your own</a><a class="btn" href="method.html">How it works</a><a class="btn" href="install.html">Install the tool</a><a class="btn" href="preregistrations.html">Check a commitment yourself</a></div></div>
|
|
183
|
+
{hero_side}</section>
|
|
184
|
+
<section class="block"><p class="kicker">Start here</p><div class="grid4">
|
|
185
|
+
<div class="card"><h3>I have a problem</h3><p class="fine">You can say exactly when an answer is right, and you have a plain solver to compare against. The compiler turns that into a fair test.</p><p class="fine"><a href="domains.html#add">Compile your own</a></p></div>
|
|
186
|
+
<div class="card"><h3>I have a faster solver</h3><p class="fine">Write one Python file with a function called <span class="mono">solve</span>, check it locally, and see how it would be measured.</p><p class="fine"><a href="compete.html">Go to Compete</a> · <a href="install.html">Install the tool</a></p></div>
|
|
187
|
+
<div class="card"><h3>I want a verified measurement</h3><p class="fine">Not entering a competition. You have a solver, an attack, or a benchmark and you want a certificate anyone can check independently.</p><p class="fine"><a href="certify.html">Go to Certify</a> · <a href="install.html">Install the tool</a></p></div>
|
|
188
|
+
<div class="card"><h3>I want to check a claim</h3><p class="fine">You do not have to trust anyone. Recompute a result from its file, check a commitment yourself, or read what a result does not prove.</p><p class="fine"><a href="guide.html">Read the guide</a> · <a href="preregistrations.html">Check a commitment</a> · <a href="limits.html">What it does not show</a></p></div></div></section>
|
|
189
|
+
{banner("<strong>Everything here is a practice result.</strong> A certified result has to be measured inside a strict test environment: a locked box that runs other people's programs so they can't cheat. That box has passed its own tests on real Linux machines, but no official result has been measured in it yet. So every number on this site was measured on the author's own computer, with the author's own programs. The numbers are real, but nothing is certified.")}
|
|
190
|
+
<section class="block"><p class="kicker">From a declaration to a certificate</p><div class="grid3">
|
|
191
|
+
<div class="card step"><span class="n">1</span><h3>Describe the problem</h3><p class="fine">Write three small pieces: how to make a test question from a secret number (with the answer hidden inside), how to check an answer exactly, and a plain baseline solver to compare against.</p>{cmd("python3 compile_domain.py init mything")}</div>
|
|
192
|
+
<div class="card step"><span class="n">2</span><h3>Compile it into a test</h3><p class="fine">The compiler checks that your problem can't be gamed, times your baseline, picks the test sizes and time limit, and works out what size of speedup it can actually prove. It ends with READY or NOT READY.</p>{cmd("python3 compile_domain.py compile mything")}</div>
|
|
193
|
+
<div class="card step"><span class="n">3</span><h3>Get a certificate</h3><p class="fine">After a round, one file holds the claim, the rules, the secret numbers and every raw measurement. Anyone can recompute the result from it, and checking it never runs the file's code unless you ask. It cannot show the CPU costs were honestly measured: that needs the verified sandbox, or re-running instances yourself.</p>{cmd("python3 certificate.py verify x.cert.json")}</div></div>
|
|
194
|
+
<p class="fine" style="margin-top:12px">The compiler and certificates are built and tested, but no round has run on a verified sandbox yet, so no certificate here is official. <a href="domains.html">Domains</a> shows what each test can and can't prove; <a href="guide.html">the guide</a> explains how to write a problem and check a certificate. Measuring AI speedups (model inference, operation counts, GPU time) is covered under <a href="domains.html#ai">AI workloads</a>, with what each kind of evidence does and does not show.</p></section>
|
|
195
|
+
<section class="block"><p class="kicker">How a test stays fair, in four steps</p><div class="grid4">
|
|
196
|
+
<div class="card step"><span class="n">1</span><h3>Commit</h3><p class="fine">Before anyone submits, the judge publishes a fingerprint (a hash) of a secret number. Every test question is generated from that number, so nobody can prepare answers in advance, and the judge can't swap the questions afterwards.</p></div>
|
|
197
|
+
<div class="card step"><span class="n">2</span><h3>Solve</h3><p class="fine">The submitted solver and the baseline get the same questions, one process each, with a time limit. Every question has a known answer hidden inside it, so every one can be solved.</p></div>
|
|
198
|
+
<div class="card step"><span class="n">3</span><h3>Check</h3><p class="fine">Every answer is checked exactly. A wrong or missing answer is charged the full time limit, so giving up on the hard questions can't look like being fast.</p></div>
|
|
199
|
+
<div class="card step"><span class="n">4</span><h3>Score</h3><p class="fine">Questions are grouped by how hard the baseline found them. The score is a cautious estimate of the speedup on the hardest groups: a number we are 95% sure the true speedup is at least as large as. A lucky run can't inflate it.</p></div></div></section>
|
|
200
|
+
<section class="block"><p class="kicker">Words used on this site</p><div class="card"><dl class="gloss">
|
|
201
|
+
<dt>Bits</dt><dd>How we count speed. Each bit doubles it: 3 bits is 8 times faster.</dd>
|
|
202
|
+
<dt>Baseline (reference solver)</dt><dd>The plain, standard solver every other solver is compared against.</dd>
|
|
203
|
+
<dt>Instance</dt><dd>One test question. A solver gets it and must return an answer.</dd>
|
|
204
|
+
<dt>Size ladder</dt><dd>The list of question sizes used, from easier to harder.</dd>
|
|
205
|
+
<dt>Time limit (CPU cap)</dt><dd>The most CPU time one question may take. Failing or timing out is charged the full limit.</dd>
|
|
206
|
+
<dt>Seed and commitment</dt><dd>A secret number that every question is generated from, and its published fingerprint. The fingerprint proves the questions were fixed before anyone submitted.</dd>
|
|
207
|
+
<dt>Certificate</dt><dd>One file with the claim, the rules and every raw measurement, so anyone can recompute the result.</dd>
|
|
208
|
+
<dt>Official (certified)</dt><dd>A result measured inside the strict, locked-box test environment, on three independent secret numbers. Everything on this site is a practice result instead.</dd>
|
|
209
|
+
<dt>Locked box (sandbox)</dt><dd>A sealed environment that runs other people's programs so they cannot read the answers, hide work or tamper with the timer.</dd>
|
|
210
|
+
<dt>Round</dt><dd>One contest: the secret numbers are fingerprinted first, solvers are submitted until a deadline, then everything is run and the secrets are revealed.</dd>
|
|
211
|
+
<dt>Lattice, factoring, dlog</dt><dd>The puzzle types used so far: finding a short vector in a grid of points; splitting a big number into smaller ones; finding the exponent in a modular power (the discrete logarithm).</dd>
|
|
212
|
+
<dt>Exact count (step counting)</dt><dd>Counting the Python instructions a program runs instead of timing it. Anyone on the same Python version gets the same number, so nobody has to trust a clock or a computer.</dd></dl></div></section>"""
|
|
213
|
+
return layout("index.html", "Guldo: verifiable speedup claims", "A challenge compiler and a protocol for crediting claimed algorithmic speedups: declare a problem, get a calibrated challenge and a certificate anyone can recompute.", body)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def page_method():
|
|
217
|
+
body = f"""
|
|
218
|
+
<section class="block"><p class="kicker">Method</p><h1>How a result is checked</h1>
|
|
219
|
+
<p class="lede">The test assumes whoever submits a solver may be mistaken, careless or cheating, and asks what can still be concluded. Every rule below closes a specific way a result could be gamed or misread.</p></section>
|
|
220
|
+
<section class="block"><h2>The rules</h2><div class="grid3">
|
|
221
|
+
<div class="card"><h3>Seed first, reveal later</h3><p class="fine">Before the deadline we publish a fingerprint of a secret number, the seed. After the deadline we reveal the seed. Every test question is computed from it, so nobody could have tuned a solver to the questions, and anyone can recompute them. You can run that check yourself on the <a href="preregistrations.html">pre-registrations page</a>.</p></div>
|
|
222
|
+
<div class="card"><h3>Every question has a known answer</h3><p class="fine">Each question is built around an answer we hid in it: a short point in a lattice, or two known primes multiplied together. So no question is impossible, and a table of old answers cannot cover new questions.</p></div>
|
|
223
|
+
<div class="card"><h3>Answers are checked exactly</h3><p class="fine">Whole-number arithmetic only. For lattices we check that the point is on the lattice and short enough; for factoring, that the number divides evenly. A solver can claim anything. The checker decides.</p></div>
|
|
224
|
+
<div class="card"><h3>Same question, both solvers</h3><p class="fine">The speedup on a question is <span class="mono">log2(baseline cost / your cost)</span>, measured on that same question. Runs under 0.5 s are ignored: a fixed startup cost would otherwise look like a speedup or a slowdown.</p></div>
|
|
225
|
+
<div class="card"><h3>Failing costs the maximum</h3><p class="fine">If the baseline runs out of time on a question, that question is dropped. If your solver fails, it is charged the full time limit. Dropping questions only when the <em>baseline</em> fails closes the trick of failing the hard ones on purpose.</p></div>
|
|
226
|
+
<div class="card"><h3>Grouped by difficulty</h3><p class="fine">Questions are grouped by how hard the baseline found them, each group costing about twice the one before. A group counts only if it has at least 5 questions and your solver's median cost there is at least 0.5 s. The credit comes from the three hardest groups that count.</p>
|
|
227
|
+
<p class="fine"><strong>For example:</strong> if the baseline takes about 1, 2, 4 and 8 seconds on four groups of questions, a group is used only when it holds at least 5 questions and your solver needs half a second or more on them. A solver that is so fast it finishes the easy groups in a blink simply has no say about those groups.</p></div></div></section>
|
|
228
|
+
<section class="block"><div class="two"><div><h2>The credit</h2>
|
|
229
|
+
<p>The credit is deliberately cautious: a number we are 95% sure the true speedup reaches or beats. It is found by re-running the whole analysis on many reshuffled copies of the data, including the step that picks the groups, so the choice of groups is part of the uncertainty. What the credit still cannot show is on the <a href="limits.html">Limits page</a>.</p>
|
|
230
|
+
<p>A second, exploratory claim covers only the hardest questions (baseline cost of 4 s or more). It picks nothing after the fact and is reported separately. It is promoted only if a pre-registered test on a fresh seed supports it.</p></div>
|
|
231
|
+
<div class="card"><h3>"Official" means all of this</h3><ul class="fine"><li>The locked-box environment, with a meter that bills every process a solver starts</li><li>At least three independent seeds, each with its fingerprint published first</li><li>The problem's full question count and time limit</li><li>The same computer, the same solver file and the same baseline file for every seed</li><li>The credit is the <strong>lowest</strong> over the seeds, and a seed with no conclusion counts as zero</li></ul></div></div></section>
|
|
232
|
+
<section class="block"><h2>Pre-registration</h2><p>Before the test on a fresh seed, we wrote the setup, the fingerprints of the code and the pass criteria into a file, and kept the file. The criteria are fixed in advance: whichever outcome occurs is the one the file says it is. A registration that is abandoned is marked void and kept, never edited; every one is listed on the <a href="preregistrations.html">pre-registrations page</a>.</p></section>
|
|
233
|
+
<section class="block"><h2>What a certificate adds</h2><p>A certificate is one file that carries everything needed to check a result on your own machine: the questions' seeds, every measured time, both solvers and the log showing the fingerprints came first. <a href="guide.html">The guide</a> says exactly what checking it proves, and what it cannot.</p></section>
|
|
234
|
+
<section class="block"><h2>Beyond speedups</h2><p>The same file can back other claims, such as a benchmark score on questions fixed in advance, or a result that nobody in a contest managed to beat a target. They are on the <a href="claims.html">Other claims</a> page, each with what it does not show.</p></section>"""
|
|
235
|
+
return layout("method.html", "Method - Guldo", "The rules of the speedup-verification protocol and why each one exists.", body)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def control_verdict(credit, median):
|
|
239
|
+
"""The seed-E criteria, exactly as written in results/PREREGISTERED_seedE_factor.txt, applied mechanically.
|
|
240
|
+
credit: None if inconclusive. Returns (outcome label, meaning)."""
|
|
241
|
+
if credit is None:
|
|
242
|
+
return "(d) inconclusive", "FAILS: a known large effect was not certifiable at this ladder"
|
|
243
|
+
if credit < 2.5:
|
|
244
|
+
return "(c) conclusive credit below 2.5 bits", "FAILS: the pipeline under-certifies a known effect"
|
|
245
|
+
if 3.5 <= median <= 5.5:
|
|
246
|
+
return "(a) credit at least 2.5 and median within [3.5, 5.5]", "the pipeline RECOVERS a known large speedup"
|
|
247
|
+
return "(b) credit at least 2.5 but median outside [3.5, 5.5]", "FAILS as a recovery: the number disagrees with the calibration"
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def prereg_cards(data, prereg, notes):
|
|
251
|
+
cards = []
|
|
252
|
+
for p in sorted(prereg, key=lambda p: p["date"]):
|
|
253
|
+
note = notes.get(p["tag"], "")
|
|
254
|
+
if p["tag"] == "755a57fd":
|
|
255
|
+
c = R.prereg_card(os.path.join(HERE, "results", p["file"]), data, p["tag"], note)
|
|
256
|
+
if c:
|
|
257
|
+
cards.append(c)
|
|
258
|
+
elif p["tag"] == "88ecb627":
|
|
259
|
+
run = next(((s, d) for s, d in data if s["family"] == "factor" and s["tag"] == p["tag"][:8] and "error" not in d), None)
|
|
260
|
+
if run:
|
|
261
|
+
d = run[1]
|
|
262
|
+
cr = credit_of(d)
|
|
263
|
+
label, meaning = control_verdict(cr, d["meas"][0] if d["meas"] else None)
|
|
264
|
+
shown = "inconclusive" if cr is None else f"{cr:.2f} bits (median {d['meas'][0]:.2f})"
|
|
265
|
+
failed = note.strip().upper().startswith("FAILED")
|
|
266
|
+
caveat = ('<div class="banner" role="note">' + R.ICON["PROVISIONAL"] + '<p><strong>The registered concurrency criterion failed.</strong> '
|
|
267
|
+
'The credit outcome above is real, but the timings it rests on were biased by running several jobs at once, so it must not be read '
|
|
268
|
+
'as a clean recovery of the known speedup. Treat it as: a speedup at least this large was certified under biased timings.</p></div>'
|
|
269
|
+
if failed else "")
|
|
270
|
+
cards.append(f'<section class="card"><h2>Positive control, seed {esc(p["tag"])}</h2><p>The criteria were written before any data existed '
|
|
271
|
+
f'and are applied here without interpretation. The true effect was calibrated in-process at about 4.4 bits.</p>'
|
|
272
|
+
f'<dl class="facts"><div><dt>Primary claim</dt><dd>{esc(shown)}: outcome {esc(label)}, which means {esc(meaning)}.</dd></div>'
|
|
273
|
+
f'<div><dt>Concurrency check</dt><dd>{esc(note or "not recorded on this page yet")}</dd></div></dl>{caveat}</section>')
|
|
274
|
+
return "".join(cards)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def page_results(data, board, prereg=None, notes=None, results_dir=None, certs_dir=None):
|
|
278
|
+
import certs_site
|
|
279
|
+
import dlog_experiment
|
|
280
|
+
runs = []
|
|
281
|
+
for s, d in data:
|
|
282
|
+
if "error" in d:
|
|
283
|
+
runs.append(f'<div class="card"><h3>{esc(s["name"])} <span class="muted">on {esc(s["family"])}, seed {esc(s["tag"])}</span></h3>'
|
|
284
|
+
f'<span class="pill idle">{R.ICON["INCONCLUSIVE"]}<span>Not shown</span></span>'
|
|
285
|
+
f'<p class="fine">The saved ladders cannot be charted yet: {esc(d["error"])}. A run still being written is expected to look like this; '
|
|
286
|
+
f'a page never charts a truncated ladder as if it were complete.</p></div>')
|
|
287
|
+
else:
|
|
288
|
+
runs.append(f'<div class="card">{R.run_section(s, d)}</div>')
|
|
289
|
+
body = f"""
|
|
290
|
+
<section class="block"><p class="kicker">Results</p><h1>What has been measured</h1>
|
|
291
|
+
<p class="lede">Every number here comes from saved timings you can download and recompute. Each run below shows the data behind its number: the difficulty groups, every question, and a table of the same figures.</p></section>
|
|
292
|
+
{banner("<strong>None of these is an official result.</strong> They were run on one computer, with solvers I wrote, without the locked-box environment. “Provisional” means measured and reproducible from the saved data, not certified.")}
|
|
293
|
+
<section class="card"><h2>Leaderboard</h2>{R.leaderboard(board)}
|
|
294
|
+
<p class="fine" style="margin-top:14px">The credit on the board is the <strong>lowest over a submission's seeds</strong>, so one weak or small run pulls it down. A row that reads lower than the pre-registered result below it usually includes an early design run (12 questions per size, 60 s limit, the data the credit rule was built from). That is why each row lists its per-seed values and the reasons it is not official.</p></section>
|
|
295
|
+
{prereg_cards(data, prereg or [], notes or {})}
|
|
296
|
+
{dlog_experiment.card_html(results_dir) if results_dir else ""}
|
|
297
|
+
{certs_site.section_html(certs_dir)}
|
|
298
|
+
<section class="block"><h2>Runs</h2><div style="display:grid;gap:18px">{"".join(runs) or '<p class="muted">No ladders found.</p>'}</div></section>
|
|
299
|
+
<section class="card"><h2>Reading these charts</h2><ul class="fine"><li>Each bar is the typical (median) speedup in bits for questions grouped by how hard the <em>baseline</em> found them. Each bit is a factor of two.</li>
|
|
300
|
+
<li>Solid bars are the three hardest groups that count, and they make up the credit. Light bars count but are not among the hardest. Dashed bars have too few questions, or sit below the 0.5 s floor, so they do not count.</li>
|
|
301
|
+
<li>The dashed line is the credit. It sits below the bars on purpose: it is the cautious number, not the typical one.</li>
|
|
302
|
+
<li>The "speedup by problem size" chart asks a different question: does the speedup grow as the problem gets harder? It shows the typical speedup at each size with a 90% range. Where the baseline ran out of time, its cost is taken as the time limit, so those questions can only understate the speedup. Hollow points are sizes the credit does not use (too many baseline time-outs, or costs under the 0.5 s floor). They are lower bounds or reference points, not measurements, and nothing is guessed beyond them or joined by a line.</li>
|
|
303
|
+
<li>A group can show a negative speedup: a tuned solver with a fixed setup cost is slower on trivial questions. That is why the floor exists, and it is shown, not hidden.</li></ul></section>"""
|
|
304
|
+
return layout("results.html", "Results - Guldo", "Measured speedups with the data behind each number.", body)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def page_prereg(prereg, seeds):
|
|
308
|
+
manifest = [{"label": f"seed {p['tag']}: {p['title'][:70]}", "commit": p["commit"], "seed": seeds.get(p["commit"], "")} for p in prereg]
|
|
309
|
+
items = []
|
|
310
|
+
for p in sorted(prereg, key=lambda p: p["date"]):
|
|
311
|
+
kind, why = PREREG_STATUS.get(p["tag"], ("run", ""))
|
|
312
|
+
cls = "void" if kind == "void" else ""
|
|
313
|
+
pill = {"void": '<span class="pill idle">' + R.ICON["INCONCLUSIVE"] + '<span>Void</span></span>',
|
|
314
|
+
"done": '<span class="pill ok">' + R.ICON["OFFICIAL"] + '<span>Completed</span></span>',
|
|
315
|
+
"run": '<span class="pill warn">' + R.ICON["PROVISIONAL"] + '<span>Registered</span></span>'}[kind]
|
|
316
|
+
revealed = (f'<p class="small muted">Seed revealed: <span class="mono">{esc(seeds[p["commit"]][:12])}...</span> (shipped with the checker below)</p>'
|
|
317
|
+
if p["commit"] in seeds else '<p class="small muted">Seed not revealed yet.</p>')
|
|
318
|
+
items.append(f'<div class="item {cls}"><div class="small muted mono">{esc(p["date"])}</div><h3>{esc(p["title"])}</h3>{pill}'
|
|
319
|
+
f'<p class="fine">{esc(why)}</p><p class="small mono" style="word-break:break-all">commitment {esc(p["commit"])}</p>{revealed}'
|
|
320
|
+
f'<details><summary>The registration, verbatim</summary><pre>{esc(p["text"])}</pre>'
|
|
321
|
+
f'<p class="small"><a href="prereg/{esc(p["file"])}">Download {esc(p["file"])}</a></p></details></div>')
|
|
322
|
+
body = f"""
|
|
323
|
+
<section class="block"><p class="kicker">Pre-registrations</p><h1>Written before the data existed</h1>
|
|
324
|
+
<p class="lede">A claim that the method works is only worth something if the pass criteria were fixed before looking. Each registration here carries its date, the seed commitment, the criteria and the hashes of the code it will run.</p></section>
|
|
325
|
+
<section class="block"><div class="tl">{"".join(items)}</div></section>
|
|
326
|
+
<section class="card"><h2>Check a commitment yourself</h2>
|
|
327
|
+
<p class="fine">Pick a run, paste the revealed seed, and your own browser computes its SHA-256 and compares it with the commitment above. Nothing is sent anywhere.</p>
|
|
328
|
+
<form class="verify" id="verify" onsubmit="return false"><label for="v-run" class="small muted">Run</label><select id="v-run"></select>
|
|
329
|
+
<label for="v-seed" class="small muted">Seed</label><textarea id="v-seed" spellcheck="false" placeholder="paste the revealed seed (hex text)"></textarea>
|
|
330
|
+
<div class="btnrow"><button class="btn primary" id="v-check" type="button">Check</button><button class="btn" id="v-fill" type="button">Fill in the revealed seed</button></div>
|
|
331
|
+
<div class="vres" id="v-result" role="status" aria-live="polite"></div></form></section>"""
|
|
332
|
+
return layout("preregistrations.html", "Pre-registrations - Guldo", "Dated registrations with seed commitments and unconditional criteria.", body, manifest)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def live_rounds(index, portal_url, sandbox_ok):
|
|
336
|
+
"""The 'Live rounds' block for the Compete page, from rounds.export_public's index (never hand-typed)."""
|
|
337
|
+
if not index:
|
|
338
|
+
return ""
|
|
339
|
+
rows = []
|
|
340
|
+
for r in index:
|
|
341
|
+
title = (f'<a href="{esc(portal_url.rstrip("/"))}/round/{esc(r["id"])}">{esc(r["title"])}</a>' if portal_url else esc(r["title"]))
|
|
342
|
+
when = time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(r["closes_at"]))
|
|
343
|
+
log = (f'<a href="ledger/log_{esc(r["id"])}.jsonl">log</a>, head <span class="mono small">{esc((r["log_head"] or "")[:12])}...</span>'
|
|
344
|
+
if r["log_ok"] else '<span class="pill warn"><span>log does not verify</span></span>')
|
|
345
|
+
rows.append(f'<tr><td class="strong">{title}</td><td>{esc(r["family"])}</td><td><span class="pill info"><span>{esc(r["state"])}</span></span></td>'
|
|
346
|
+
f'<td class="mono small">{esc(when)}</td><td class="num mono">{r["submissions"]}</td><td>{log}</td></tr>')
|
|
347
|
+
gate = ("The operator reports the sandbox as verified for this deployment." if sandbox_ok else
|
|
348
|
+
"The timing sandbox is <strong>not</strong> verified for this deployment, so timed rounds cannot be evaluated or produce an official result yet. Counted rounds (exact step counts, not seconds) can be evaluated, but they run on the operator's own machine without operating-system isolation.")
|
|
349
|
+
return ('<section class="card"><h2>Live rounds</h2><div class="scroll"><table class="lb"><thead><tr><th>Round</th><th>Family</th><th>State</th><th>Closes</th>'
|
|
350
|
+
'<th class="num">Submissions</th><th>Transparency log</th></tr></thead><tbody>' + "".join(rows) + '</tbody></table></div>'
|
|
351
|
+
f'<p class="fine" style="margin-top:12px">{gate} Each round publishes its seed commitments before it opens, every accepted file\'s hash goes into a hash-chained log you can download and verify, '
|
|
352
|
+
'and the seeds are revealed only after evaluation.</p></section>')
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def declared_rows():
|
|
356
|
+
"""(name, signature, what to return) for every compiled declared domain (COMPILER.md), from its own declaration's SIGNATURE and DOC."""
|
|
357
|
+
import declared
|
|
358
|
+
out = []
|
|
359
|
+
for name, info in sorted(families.DECLARED.items()):
|
|
360
|
+
try:
|
|
361
|
+
mod = declared.load_declaration(info["dir"])
|
|
362
|
+
except declared.DeclarationError:
|
|
363
|
+
continue
|
|
364
|
+
sig = getattr(mod, "SIGNATURE", None)
|
|
365
|
+
if not sig: # default: the reference solver's own parameter list
|
|
366
|
+
import ast
|
|
367
|
+
try:
|
|
368
|
+
with open(declared.reference_path(info["dir"], mod), "rb") as f:
|
|
369
|
+
fn = next(n for n in ast.parse(f.read()).body if isinstance(n, ast.FunctionDef) and n.name == "solve")
|
|
370
|
+
sig = "solve(" + ", ".join(a.arg for a in fn.args.args) + ")"
|
|
371
|
+
except (StopIteration, SyntaxError, OSError, declared.DeclarationError):
|
|
372
|
+
sig = "solve(*args)"
|
|
373
|
+
doc = getattr(mod, "DOC", "See the challenge page for this family.")
|
|
374
|
+
m = re.match(r"^solve\([^)]*\):?\s*", doc) # a DOC that opens with the signature would repeat the column beside it
|
|
375
|
+
if m and m.end() < len(doc):
|
|
376
|
+
doc = doc[m.end():]
|
|
377
|
+
doc = doc[:1].upper() + doc[1:]
|
|
378
|
+
out.append((name, sig, doc))
|
|
379
|
+
return tuple(out)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
# one plain sentence per problem, shown beside its name: the names themselves (gm, mlwe, dlog) mean nothing to a newcomer
|
|
383
|
+
PROBLEM_GLOSS = {
|
|
384
|
+
"gm": "A lattice puzzle: find a short vector in a huge grid of points. Cryptography relies on this being hard.",
|
|
385
|
+
"mlwe": "The same kind of lattice puzzle, in the shape that modern encryption schemes use.",
|
|
386
|
+
"factor": "Factoring: split a large number into smaller whole numbers that multiply back to it.",
|
|
387
|
+
"dlog": "Discrete logarithm: find the exponent x so that g to the power x leaves the remainder h, modulo a prime p.",
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
_LI = re.compile(r"^(\s*)([*-]|\d+\.) (.*)$")
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _inline(s):
|
|
395
|
+
"""Inline Markdown on ESCAPED text: `code`, **bold**, *italic*. Escaping first means nothing in the source can inject markup."""
|
|
396
|
+
out = []
|
|
397
|
+
for part in re.split(r"(`[^`]+`)", esc(s)):
|
|
398
|
+
if len(part) > 1 and part.startswith("`") and part.endswith("`"):
|
|
399
|
+
out.append(f"<code>{part[1:-1]}</code>")
|
|
400
|
+
continue
|
|
401
|
+
part = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", part)
|
|
402
|
+
out.append(re.sub(r"(?<![\w*])\*(?!\s)(.+?)(?<!\s)\*(?![\w*])", r"<em>\1</em>", part))
|
|
403
|
+
return "".join(out)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _md_list(lines, i):
|
|
407
|
+
"""Parse consecutive list items (nested by indentation) starting at lines[i]. -> (html, next index)"""
|
|
408
|
+
items = []
|
|
409
|
+
while i < len(lines):
|
|
410
|
+
m = _LI.match(lines[i])
|
|
411
|
+
if m:
|
|
412
|
+
items.append([len(m.group(1)), m.group(2)[0].isdigit(), m.group(3)])
|
|
413
|
+
elif items and lines[i].strip() and lines[i].startswith(" ") and not lines[i].startswith(" "):
|
|
414
|
+
items[-1][2] += " " + lines[i].strip() # a wrapped continuation of the previous item
|
|
415
|
+
else:
|
|
416
|
+
break
|
|
417
|
+
i += 1
|
|
418
|
+
html, stack = [], []
|
|
419
|
+
for indent, ordered, text in items:
|
|
420
|
+
while stack and indent < stack[-1][0]:
|
|
421
|
+
html.append(f"</li></{stack.pop()[1]}>")
|
|
422
|
+
if stack and indent == stack[-1][0]:
|
|
423
|
+
html.append("</li>")
|
|
424
|
+
else:
|
|
425
|
+
html.append("<ol>" if ordered else "<ul>")
|
|
426
|
+
stack.append((indent, "ol" if ordered else "ul"))
|
|
427
|
+
html.append(f"<li>{_inline(text)}")
|
|
428
|
+
while stack:
|
|
429
|
+
html.append(f"</li></{stack.pop()[1]}>")
|
|
430
|
+
return "".join(html), i
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def md_to_html(md):
|
|
434
|
+
"""A deliberately small Markdown subset (headings, paragraphs, nested lists, tables, indented code, inline code/bold/italic): what COMPILER.md
|
|
435
|
+
uses and nothing else. -> (html, title, [(id, heading)]). All text is escaped; there is no raw-HTML passthrough."""
|
|
436
|
+
lines, i, out, toc, title, used = md.splitlines(), 0, [], [], "", set()
|
|
437
|
+
is_block = lambda ln: (ln.startswith(("#", "|", " ")) or _LI.match(ln) is not None)
|
|
438
|
+
while i < len(lines):
|
|
439
|
+
ln = lines[i]
|
|
440
|
+
if not ln.strip():
|
|
441
|
+
i += 1
|
|
442
|
+
elif ln.startswith("# "):
|
|
443
|
+
title, i = ln[2:].strip(), i + 1
|
|
444
|
+
elif re.match(r"^#{2,4} ", ln):
|
|
445
|
+
level = len(ln) - len(ln.lstrip("#"))
|
|
446
|
+
text = ln[level:].strip()
|
|
447
|
+
slug = base = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") or "section"
|
|
448
|
+
n = 2
|
|
449
|
+
while slug in used:
|
|
450
|
+
slug, n = f"{base}-{n}", n + 1
|
|
451
|
+
used.add(slug)
|
|
452
|
+
if level == 2:
|
|
453
|
+
toc.append((slug, text))
|
|
454
|
+
out.append(f'<h{level} id="{slug}">{_inline(text)}</h{level}>')
|
|
455
|
+
i += 1
|
|
456
|
+
elif ln.startswith(" "):
|
|
457
|
+
block = []
|
|
458
|
+
while i < len(lines) and (lines[i].startswith(" ") or (not lines[i].strip() and i + 1 < len(lines) and lines[i + 1].startswith(" "))):
|
|
459
|
+
block.append(lines[i][4:])
|
|
460
|
+
i += 1
|
|
461
|
+
out.append('<pre class="codeblock"><code>' + esc("\n".join(block)) + "</code></pre>")
|
|
462
|
+
elif ln.startswith("|"):
|
|
463
|
+
rows = []
|
|
464
|
+
while i < len(lines) and lines[i].startswith("|"):
|
|
465
|
+
rows.append([c.strip() for c in lines[i].strip().strip("|").split("|")])
|
|
466
|
+
i += 1
|
|
467
|
+
body_rows = [r for r in rows[2:]] if len(rows) > 1 and set("".join(rows[1])) <= set("-: ") else rows[1:]
|
|
468
|
+
head = "".join(f"<th>{_inline(c)}</th>" for c in rows[0])
|
|
469
|
+
out.append('<div class="scroll"><table class="lb"><thead><tr>' + head + "</tr></thead><tbody>"
|
|
470
|
+
+ "".join("<tr>" + "".join(f"<td>{_inline(c)}</td>" for c in r) + "</tr>" for r in body_rows) + "</tbody></table></div>")
|
|
471
|
+
elif _LI.match(ln):
|
|
472
|
+
html, i = _md_list(lines, i)
|
|
473
|
+
out.append(html)
|
|
474
|
+
else:
|
|
475
|
+
para = []
|
|
476
|
+
while i < len(lines) and lines[i].strip() and not is_block(lines[i]):
|
|
477
|
+
para.append(lines[i].strip())
|
|
478
|
+
i += 1
|
|
479
|
+
out.append(f"<p>{_inline(' '.join(para))}</p>")
|
|
480
|
+
return "\n".join(out), title, toc
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def page_guide():
|
|
484
|
+
"""The compiler guide, rendered from COMPILER.md at build time so the page cannot drift from the document in the repository."""
|
|
485
|
+
try:
|
|
486
|
+
with open(os.path.join(HERE, "COMPILER.md"), encoding="utf-8") as f:
|
|
487
|
+
html, title, toc = md_to_html(f.read())
|
|
488
|
+
except OSError:
|
|
489
|
+
html, title, toc = '<p class="fine">COMPILER.md is not present in this checkout, so this guide could not be generated.</p>', "The challenge compiler", []
|
|
490
|
+
toc_html = ('<nav class="card toc" aria-label="On this page"><strong>On this page</strong><ul class="fine">'
|
|
491
|
+
+ "".join(f'<li><a href="#{esc(s)}">{_inline(t)}</a></li>' for s, t in toc) + "</ul></nav>") if toc else ""
|
|
492
|
+
body = f"""
|
|
493
|
+
<section class="block"><p class="kicker">Guide</p><h1>{esc(title or "The challenge compiler")}</h1>
|
|
494
|
+
<p class="lede">How to describe a problem, turn it into a fair test, and check a certificate, including what each step can't do. This page is generated from <span class="mono">COMPILER.md</span> in the repository, so it matches the code. It is the technical reference: the later sections (counted rounds, the restricted subset, derived claims) assume you know the basics, and the plain-words version is on <a href="claims.html">Other claims</a>.</p></section>
|
|
495
|
+
<section class="block"><div class="card"><h2 style="margin-top:0">If you read only this</h2><ol class="fine">
|
|
496
|
+
<li>You write three small pieces: make a test question from a secret number, check an answer exactly, and a plain baseline solver.</li>
|
|
497
|
+
<li><span class="mono">compile_domain.py check NAME</span> takes seconds and tells you what to fix. <span class="mono">compile NAME --quick</span> gives a draft in minutes. <span class="mono">compile NAME</span> is the real one and can take hours.</li>
|
|
498
|
+
<li>A certificate lets anyone recompute a result, but only shows the costs were honestly measured if the verified sandbox or a re-run backs it. Checking one never runs its code unless you pass <span class="mono">--execute</span>.</li>
|
|
499
|
+
<li>Past speedups: a cost can be an exact instruction count that anyone reproduces, and a certificate can back a benchmark score or a verified negative result. These are newer and have never been run: see <a href="claims.html">Other claims</a>.</li></ol></div></section>
|
|
500
|
+
{banner("<strong>Practice stage.</strong> A certificate can show the arithmetic adds up, but not that the CPU costs were honestly measured. Nothing is certified as official until a full round has been evaluated in the strict test environment (the locked box) on a dedicated machine.")}
|
|
501
|
+
<section class="block"><div class="two guide-wrap">{toc_html}<div class="guide">{html}</div></div></section>"""
|
|
502
|
+
return layout("guide.html", "Guide - Guldo", "How to declare a problem, compile it into a challenge, and verify a certificate.", body)
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
BUILTIN_FAMILIES = (("gm", "Lattice: a short vector", "Given a lattice (a regular grid of points in many dimensions), find a short non-zero point on it. Each test lattice has a short point hidden in it, so a good answer is known to exist."),
|
|
506
|
+
("mlwe", "Lattice: encryption-style", "The same task on lattices of the kind used in some modern encryption schemes."),
|
|
507
|
+
("factor", "Integer factoring", "Given a large number that is the product of two primes, find a factor of it."))
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _power_rows(compiled):
|
|
511
|
+
rows = []
|
|
512
|
+
for e, v in compiled["power"]["curve"].items():
|
|
513
|
+
pct = round(100 * v["all_three"])
|
|
514
|
+
rows.append(f'<tr><td class="num mono">{esc(e)}</td><td><div class="bar" role="img" aria-label="{pct}% of simulated three-seed rounds certify this speedup">'
|
|
515
|
+
f'<span style="width:{pct}%"></span></div></td><td class="num mono">{pct}%</td></tr>')
|
|
516
|
+
return ('<div class="scroll"><table class="lb"><thead><tr><th class="num">Real speedup (bits)</th><th>Proven in all three rounds</th><th class="num">%</th></tr></thead>'
|
|
517
|
+
'<tbody>' + "".join(rows) + '</tbody></table></div>')
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def domain_card(name, info):
|
|
521
|
+
"""One compiled domain, entirely from its compiled.json and declaration: nothing on this card is typed by hand."""
|
|
522
|
+
import compile_domain as CD
|
|
523
|
+
import declared
|
|
524
|
+
c = info["compiled"]
|
|
525
|
+
level, _ = CD.verdict(c)
|
|
526
|
+
cls = "ok" if level == "READY" else "idle" if level.startswith("NOT READY") else "warn"
|
|
527
|
+
icon = R.ICON["OFFICIAL"] if cls == "ok" else R.ICON["INCONCLUSIVE"] if cls == "idle" else R.ICON["PROVISIONAL"]
|
|
528
|
+
label = ("Ready" if level == "READY" else "Not ready" if level.startswith("NOT READY") else "Ready, with notes" if level.startswith("READY") else "Draft")
|
|
529
|
+
meaning = {"Ready": "", "Ready, with notes": "Usable, but read the compiler's notes below: they say how far to trust the test.",
|
|
530
|
+
"Draft": "Practice only until it passes the compiler's self-test.", "Not ready": "The compiler could not build a test that proves anything useful."}[label]
|
|
531
|
+
try:
|
|
532
|
+
doc = getattr(declared.load_declaration(info["dir"]), "DOC", "")
|
|
533
|
+
except declared.DeclarationError:
|
|
534
|
+
doc = ""
|
|
535
|
+
pw, st = c["power"], (c.get("selftest") or {}).get("status", "pending")
|
|
536
|
+
mce, top = pw["min_certifiable_effect"], pw.get("max_certifiable_effect")
|
|
537
|
+
certifies = (f"speedups from about {mce:g} to {top:g} bits ({2 ** mce:.1f} to {2 ** top:.1f} times faster) than the baseline"
|
|
538
|
+
if mce is not None else "nothing yet: the test is too weak to prove a speedup")
|
|
539
|
+
selftest = {"pass": "passed", "pending": "not finished yet", "skipped": "not run (draft)", "fail": "FAILED"}.get(st, st)
|
|
540
|
+
notes = list(c["plan"]["notes"]) + list(pw["notes"])
|
|
541
|
+
note_html = (f'<details class="more"><summary>Notes from the compiler ({len(notes)})</summary><ul class="fine">'
|
|
542
|
+
+ "".join(f"<li>{esc(x)}</li>" for x in notes) + "</ul></details>") if notes else ""
|
|
543
|
+
return (f'<article class="card" id="{esc(name)}"><span class="pill {cls}">{icon}<span>{esc(label)}</span></span>'
|
|
544
|
+
f'<h3 style="margin-top:10px">{esc(name)}</h3>'
|
|
545
|
+
+ (f'<p class="fine">{esc(doc)}</p>' if doc else "")
|
|
546
|
+
+ f'<p><strong>What it can prove:</strong> {esc(certifies)}.</p>'
|
|
547
|
+
+ (f'<p class="fine">{esc(meaning)}</p>' if meaning else "")
|
|
548
|
+
+ f'<ul class="fine"><li><strong>The test:</strong> problem sizes {esc(c["dims"])}, {esc(c["official"]["n"])} questions per size, time limit {esc(c["official"]["budget"])} s per question; '
|
|
549
|
+
f'about {esc(c["cpu_hours"]["expected"])} CPU-hours to score one solver.</li>'
|
|
550
|
+
f'<li><strong>The baseline:</strong> {esc(c.get("reference_note") or "the author has not said what it is or what is known to beat it, so a speedup over it may be easy.")}</li>'
|
|
551
|
+
f'<li><strong>Compiler self-test:</strong> {esc(selftest)}. It checks that the baseline against itself scores about 0, that a solver which never answers scores 0, and that a known 1-bit speedup is recovered.</li></ul>'
|
|
552
|
+
+ '<h4>How often a real speedup of each size would be proven</h4>' + _power_rows(c)
|
|
553
|
+
+ f'<p class="fine">Assumes a solver\'s speed varies about {esc(pw["pair_noise"])} bit from question to question. The real figure is unknown until someone submits.</p>'
|
|
554
|
+
+ note_html
|
|
555
|
+
+ f'<p class="fine">Version <span class="mono">{esc(c.get("version"))}</span>, fingerprint <span class="mono">{esc(c["decl_sha256"][:12])}</span>.</p>'
|
|
556
|
+
+ '<p class="fine">Check the compiler\'s arithmetic yourself (it recomputes the whole table from the saved record and says if anything differs):</p>'
|
|
557
|
+
+ cmd(f"python3 compile_domain.py recheck {name}")
|
|
558
|
+
+ cmd(f"python3 submit_check.py my_solver.py --family {name}") + '</article>')
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
# ---------------------------------------------------------------- AI workloads (counted-operations ladders and the GPU meter)
|
|
562
|
+
COUNTED_LADDER = re.compile(r"^tflops_([a-z]+)_([0-9a-f]{8})_(baseline|submission)_(.+)\.py\.jsonl$")
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _blurb(solver_name):
|
|
566
|
+
"""The first line of a saved solver's own docstring (what the file says it is), or ''. Read as text; nothing is executed."""
|
|
567
|
+
try:
|
|
568
|
+
with open(os.path.join(HERE, "solvers", solver_name + ".py"), encoding="utf-8") as f:
|
|
569
|
+
doc = ast.get_docstring(ast.parse(f.read())) or ""
|
|
570
|
+
except (OSError, SyntaxError, ValueError):
|
|
571
|
+
return ""
|
|
572
|
+
return doc.strip().splitlines()[0][:170] if doc.strip() else ""
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _ratio_text(base_recs, sub_recs):
|
|
576
|
+
"""'21x at 40, 29x at 56': baseline median cost over submission median cost, per size, from the saved records (solved instances only)."""
|
|
577
|
+
by = {}
|
|
578
|
+
for role, recs in (("b", base_recs), ("s", sub_recs)):
|
|
579
|
+
for r in recs:
|
|
580
|
+
if r.get("status") == "solved" and isinstance(r.get("cpu_net"), (int, float)) and r["cpu_net"] > 0:
|
|
581
|
+
by.setdefault(r.get("dim"), {}).setdefault(role, []).append(r["cpu_net"])
|
|
582
|
+
out = []
|
|
583
|
+
for dim in sorted(d for d in by if isinstance(d, int)):
|
|
584
|
+
if "b" in by[dim] and "s" in by[dim]:
|
|
585
|
+
q = statistics.median(by[dim]["b"]) / statistics.median(by[dim]["s"])
|
|
586
|
+
out.append((f"{q:.0f}x" if q >= 10 else f"{q:.1f}x") + f" at {dim}")
|
|
587
|
+
return ", ".join(out) or "n/a"
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def _counted_row(bname, bpath, sname, spath):
|
|
591
|
+
import tflops
|
|
592
|
+
bmeta, brecs = tflops.read_ladder(bpath)
|
|
593
|
+
smeta, srecs = tflops.read_ladder(spath)
|
|
594
|
+
if not bmeta or not smeta or bmeta.get("cost_model") != "tflops" or smeta.get("cost_model") != "tflops" or not srecs:
|
|
595
|
+
return None
|
|
596
|
+
credit = "not scored"
|
|
597
|
+
try:
|
|
598
|
+
with open(spath + ".claim.json", encoding="utf-8") as f:
|
|
599
|
+
m = json.load(f).get("measured")
|
|
600
|
+
credit = "inconclusive" if m is None else f"{float(m):.1f} bits"
|
|
601
|
+
except (OSError, ValueError, TypeError):
|
|
602
|
+
pass
|
|
603
|
+
return {"solver": sname, "baseline": bname, "same": sname == bname, "solved": sum(r.get("status") == "solved" for r in srecs), "total": len(srecs),
|
|
604
|
+
"wrong": sum(r.get("status") == "wrong" for r in srecs), "ratio": _ratio_text(brecs, srecs), "credit": credit,
|
|
605
|
+
"consistent": bool(tflops.verify_ladder(spath)[0]) and bool(tflops.verify_ladder(bpath)[0]),
|
|
606
|
+
"files": [os.path.basename(bpath), os.path.basename(spath)]}
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def counted_rows(results_dir):
|
|
610
|
+
"""One row per saved counted-operations pair (a baseline ladder and a submission ladder from the same seed). A pair that cannot be read is left out, never guessed."""
|
|
611
|
+
if not results_dir or not os.path.isdir(results_dir):
|
|
612
|
+
return []
|
|
613
|
+
found = {}
|
|
614
|
+
for fn in sorted(os.listdir(results_dir)):
|
|
615
|
+
m = COUNTED_LADDER.match(fn)
|
|
616
|
+
if m:
|
|
617
|
+
found.setdefault((m.group(1), m.group(2)), {}).setdefault(m.group(3), []).append((m.group(4), os.path.join(results_dir, fn)))
|
|
618
|
+
rows = []
|
|
619
|
+
for (fam, tag), roles in sorted(found.items()):
|
|
620
|
+
for bname, bpath in roles.get("baseline", [])[:1]:
|
|
621
|
+
for sname, spath in roles.get("submission", []):
|
|
622
|
+
try:
|
|
623
|
+
row = _counted_row(bname, bpath, sname, spath)
|
|
624
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
625
|
+
row = None
|
|
626
|
+
if row:
|
|
627
|
+
row["family"] = fam
|
|
628
|
+
rows.append(row)
|
|
629
|
+
return rows
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def ai_section(results_dir=None):
|
|
633
|
+
"""The AI-workloads section of the Domains page. The prose says what each way of measuring is and is NOT; every number and every status is read from the saved files."""
|
|
634
|
+
rows = counted_rows(results_dir)
|
|
635
|
+
gpu = sorted(fn for fn in os.listdir(results_dir) if fn.startswith("gpu_") and fn.endswith(".jsonl")) if results_dir and os.path.isdir(results_dir) else []
|
|
636
|
+
if rows:
|
|
637
|
+
body = "".join(
|
|
638
|
+
f'<tr><td><span class="mono">{esc(r["solver"])}</span>'
|
|
639
|
+
+ ('<br><span class="fine">The reference run against itself: it should earn nothing.</span>' if r["same"] else (f'<br><span class="fine">{esc(_blurb(r["solver"]))}</span>' if _blurb(r["solver"]) else ""))
|
|
640
|
+
+ f'</td><td class="mono">{esc(r["family"])}</td><td class="num mono">{r["solved"]}/{r["total"]}</td><td class="num mono">{r["wrong"]}</td>'
|
|
641
|
+
f'<td class="mono">{esc(r["ratio"])}</td><td class="num mono"><strong>{esc(r["credit"])}</strong></td>'
|
|
642
|
+
f'<td class="fine">{"cost arithmetic re-checks" if r["consistent"] else "DOES NOT VERIFY"}</td>'
|
|
643
|
+
f'<td class="fine"><a href="data/{quote(r["files"][0])}">reference ladder</a> · <a href="data/{quote(r["files"][1])}">solver ladder</a></td></tr>'
|
|
644
|
+
for r in rows)
|
|
645
|
+
table = ('<div class="scroll"><table class="lb"><thead><tr><th>Solver (against the reference)</th><th>Domain</th><th class="num">Solved</th><th class="num">Wrong answers</th>'
|
|
646
|
+
'<th>Operations saved (reference / solver)</th><th class="num">Credit</th><th>Ledger</th><th>Files</th></tr></thead><tbody>' + body + '</tbody></table></div>'
|
|
647
|
+
'<p class="fine">Read from the saved ladders: the credit is the estimator\'s lower bound on the operations saved, in bits (each bit is a factor of two), the ratios are medians over the saved '
|
|
648
|
+
'instances, and the last column is re-checked when this page is built. A solver that answers wrongly earns nothing, and the reference against itself earns nothing. '
|
|
649
|
+
'The sizes in these runs were chosen by whoever ran them, not committed in advance.</p>')
|
|
650
|
+
else:
|
|
651
|
+
table = '<p class="fine">No counted-operations run is saved in this build.</p>'
|
|
652
|
+
gpu_text = (f"{len(gpu)} GPU ladder file(s) are saved. They are not summarised here: a GPU time is one operator's measurement on one device model and says nothing about other hardware." if gpu
|
|
653
|
+
else "No GPU measurement has been made. The meter exists and its decision rules have been tested against a simulated device, but it has never been run on a real GPU.")
|
|
654
|
+
return f"""<section class="block" id="ai"><h2>AI workloads</h2>
|
|
655
|
+
<p class="fine">There are three ways to measure an AI speedup here. They give different kinds of evidence, and each says what it does not show.</p>
|
|
656
|
+
<div class="two">
|
|
657
|
+
<article class="card"><h3 style="margin-top:0">Timed on the CPU</h3><p class="fine">The domains above, such as <span class="mono">aiinfer</span> (classify a committed batch with a small neural network, at equivalent accuracy). This is the full protocol: seeds committed first, a public log, a certificate anyone can recompute. The timings still rest on the locked-box sandbox, which has not been run on a real machine. Saved timed runs appear on the <a href="results.html">Results page</a>.</p></article>
|
|
658
|
+
<article class="card"><h3 style="margin-top:0">Counted operations</h3><p class="fine">Every tensor operation is charged a cost computed from its shapes, and the solver's own Python adds one unit per instruction. Anyone can re-run any record and must get exactly the same counts, on any machine with the same Python version, trusting no clock. It can show that a solver needs N times fewer operations (a cache, pruning, an early exit). It cannot show that a solver is faster at the same arithmetic (a better kernel, a lower precision, a different memory layout).</p></article>
|
|
659
|
+
<article class="card"><h3 style="margin-top:0">GPU time</h3><p class="fine">The operator's own clock times a request to a solver running on one pinned device model, with warm-up on public instances and a different instance every run. It can be re-measured on the same model but never re-executed exactly, so it is the weakest evidence here. {esc(gpu_text)}</p></article>
|
|
660
|
+
</div>
|
|
661
|
+
<h3>Counted-operations results</h3>{table}
|
|
662
|
+
{cmd("python3 tflops.py verify LADDER --rerun all")}
|
|
663
|
+
<p class="fine">That command re-executes every record of a ladder (the files are linked in the table; the solvers and the tool ship in the installable package) and demands identical counts. The build only re-checks the cost arithmetic; it never runs a solver.</p>
|
|
664
|
+
</section>"""
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def page_domains(results_dir=None):
|
|
668
|
+
cards = [domain_card(n, i) for n, i in sorted(families.DECLARED.items())]
|
|
669
|
+
# a reason can quote an exception or a path from the operator's machine: this page is public, so show the sentence, not the filesystem
|
|
670
|
+
scrub = lambda s: re.sub(r"(/[\w.@+\-]+){2,}", "<path>", str(s).replace(HERE, "."))[:300]
|
|
671
|
+
errors = [(k, scrub(v)) for k, v in sorted(families.DECLARED_ERRORS.items())]
|
|
672
|
+
builtin = "".join(
|
|
673
|
+
f'<article class="card" id="{n}"><span class="pill info"><span>Built in</span></span><h3 style="margin-top:10px">{esc(n)}: {esc(t)}</h3><p class="fine">{esc(d)}</p>'
|
|
674
|
+
f'<ul class="fine"><li><strong>The test:</strong> problem sizes {esc(families.DEFAULT_DIMS[n])}, {esc(families.OFFICIAL[n]["n"])} questions per size, '
|
|
675
|
+
f'time limit {esc(families.OFFICIAL[n]["budget"])} s per question</li><li>Set up by hand, not by the compiler. What has been measured is on the <a href="results.html">Results page</a> and its limits on the <a href="limits.html">Limits page</a>.</li></ul></article>'
|
|
676
|
+
for n, t, d in BUILTIN_FAMILIES)
|
|
677
|
+
err_html = ("<section class=\"block\"><h2>Declared but not registered</h2><p class=\"fine\">A declaration is registered only while its compiled record matches "
|
|
678
|
+
"its files. These are not, and why:</p><ul class=\"fine\">" + "".join(f"<li><span class=\"mono\">{esc(k)}</span>: {esc(v)}</li>" for k, v in errors) + "</ul></section>") if errors else ""
|
|
679
|
+
body = f"""
|
|
680
|
+
<section class="block"><p class="kicker">Domains</p><h1>Problems you can test speedups on</h1>
|
|
681
|
+
<p class="lede">Each card is a problem we can measure speedups on, and says what size of speedup its test can prove. The built-in ones were set up by hand. The others were made by the compiler from a short description, and every number on their cards is read from the compiler's own record.</p></section>
|
|
682
|
+
{banner("<strong>These are practice results too.</strong> None of these problems can produce a certified result until the locked-box test environment has been tried on a real machine.")}
|
|
683
|
+
<section class="block"><h2>Compiled domains</h2>{('<div class="two">' + "".join(cards) + '</div>') if cards else '<p class="fine">No compiled domain is installed in this build.</p>'}</section>
|
|
684
|
+
{ai_section(results_dir)}
|
|
685
|
+
{err_html}
|
|
686
|
+
<section class="block"><h2>Built-in families</h2><div class="two">{builtin}</div></section>
|
|
687
|
+
<section class="block" id="add"><h2>Add your own</h2><div class="two"><div>
|
|
688
|
+
<p>You write three small pieces: how to make a test question from a secret number with the answer hidden inside (<span class="mono">derive</span>), how to check an answer exactly (<span class="mono">verify</span>), and a plain baseline solver. The first command below creates a working example to edit.</p>
|
|
689
|
+
{cmd("python3 compile_domain.py init mything")}{cmd("python3 compile_domain.py check mything")}{cmd("python3 compile_domain.py compile mything --quick")}{cmd("python3 compile_domain.py compile mything")}{cmd("python3 compile_domain.py show mything")}
|
|
690
|
+
<p class="fine">New to this? <a href="guide.html">The guide</a> walks through it, including what the compiler checks, how to verify a certificate, and the limits.</p>
|
|
691
|
+
</div><div class="card"><h3>What the compiler turns down</h3><ul class="fine"><li>Questions that change depending on the clock, chance or the machine</li><li>A checker that accepts junk such as nothing, 0 or 1, or an answer read straight off the question</li><li>An answer hidden in the question text, or one answer that fits every question</li><li>A baseline that gets answers wrong, or that depends on the judge's own code</li></ul>
|
|
692
|
+
<h3 style="margin-top:14px">What it can't do</h3><ul class="fine"><li>Judge whether your baseline is a good algorithm. A certificate only says "faster than this baseline"</li><li>Prove a problem has no shortcut</li><li>Prove speedups above about 5 bits (about 32 times faster) with a 120 s time limit</li></ul></div></div></section>"""
|
|
693
|
+
return layout("domains.html", "Domains - Guldo", "Problems the protocol can measure, built in and compiled from a declaration.", body)
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def page_compete(rounds_index=None, portal_url="", sandbox_ok=False):
|
|
697
|
+
rows ="".join(f'<tr><td class="strong">{esc(k)}' + (f'<br><span class="fine">{esc(PROBLEM_GLOSS[k])}</span>' if k in PROBLEM_GLOSS else "") + f'</td><td class="mono">{esc(a)}</td><td>{esc(b)}</td></tr>'
|
|
698
|
+
for k, a, b in (("gm", "solve(basis, bound2)", "Return any nonzero lattice vector, as a list of ints, with squared norm at most bound2 (1.05 x the Gaussian heuristic)."),
|
|
699
|
+
("mlwe", "solve(basis, bound2)", "The same call on module-LWE-shaped lattices; the target is 1.5 x the planted short vector's norm."),
|
|
700
|
+
("factor", "solve(n)", "Return any nontrivial divisor of n, as an int.")) + declared_rows())
|
|
701
|
+
body = f"""
|
|
702
|
+
<section class="block"><p class="kicker">Compete</p><h1>Submit a solver</h1>
|
|
703
|
+
<p class="lede">Write one Python file with a function called <span class="mono">solve</span>. The judge announces which problem is being tested, and the problem decides what <span class="mono">solve</span> receives.</p></section>
|
|
704
|
+
{banner("<strong>Submissions are taken in rounds.</strong> Your file is read, never run, while a round is open; it is run only after the round closes, on instances fixed by seeds committed before the round opened: in a container for a timed round, and inside the step counter's restricted subset of Python for a counted round. Nothing is official until the sandbox has been verified on the operator's host; a counted round is certified by exact, re-executable step counts instead." if rounds_index else "<strong>No public submission service exists yet.</strong> Running other people's code safely needs the sandbox verified on a real host. Today you can check a solver locally against the same verifiers the judge uses; the judge cannot yet evaluate it officially.")}
|
|
705
|
+
{live_rounds(rounds_index, portal_url, sandbox_ok)}
|
|
706
|
+
<section class="card"><h2>What you write</h2><div class="scroll"><table class="lb"><thead><tr><th>Problem</th><th>Your function</th><th>What it returns</th></tr></thead><tbody>{rows}</tbody></table></div>
|
|
707
|
+
<ul class="fine" style="margin-top:14px"><li>Return a whole number, a list of whole numbers, or <span class="mono">None</span>. Anything else counts as a crash, and a failure is charged the full time limit.</li>
|
|
708
|
+
<li>Give the same output every time, use one process, and do not read files or use the network. Only <span class="mono">fpylll</span>, <span class="mono">numpy</span> and the standard library exist in the locked box.</li>
|
|
709
|
+
<li><span class="mono">print()</span> is safe: it goes to the error stream and cannot corrupt your answer.</li>
|
|
710
|
+
<li>Some rounds count Python instructions instead of seconds (the round's page says so). Then your solver must be plain computation: loops, arithmetic, lists and dictionaries, and only these imports: <span class="mono">math</span>, <span class="mono">itertools</span>, <span class="mono">heapq</span>, <span class="mono">bisect</span>, <span class="mono">functools</span>, <span class="mono">collections</span>, <span class="mono">fractions</span>. No files, no <span class="mono">sys</span>, no <span class="mono">getattr</span>. Only the factoring problem can be counted so far. Try <span class="mono">python3 counted.py check my_solver.py --family factor --dim 38</span> to see whether yours runs.</li></ul></section>
|
|
711
|
+
<section class="block"><div class="two"><div><h2>Check it first</h2><p class="fine">The pre-flight check looks for anything the locked box blocks, runs your file on small public practice questions, checks every answer, and tests that it gives the same output each time. Passing means well-formed, not fast.</p>
|
|
712
|
+
{cmd("python3 submit_check.py solvers/my_solver.py --family gm")}{cmd("python3 submit_check.py solvers/my_solver.py --family factor")}</div>
|
|
713
|
+
<div><h2>See how you'd be measured</h2><p class="fine">Work out the cost of an official run before using machine time, and read the baseline solver for each problem.</p>
|
|
714
|
+
{cmd("python3 official.py plan --family gm")}{cmd("ls solvers/baseline_bkz.py solvers/factor_trial_py.py")}</div></div></section>
|
|
715
|
+
<section class="card"><h2>From the command line</h2><p class="fine">All of this is also one command, <span class="mono">guldo</span>. It checks your file, uploads it, and confirms your receipt in the round's public log using its own copy of the chain. It follows the round, and once the seeds are revealed it downloads your certificate and checks it on your own computer. No browser is needed. The <a href="install.html">Install page</a> has the download and the three steps; the full guide is <span class="mono">SUBMIT.md</span> in the repository.</p>
|
|
716
|
+
{cmd("guldo practice my_solver.py --family dlog")}{cmd("guldo submit my_solver.py --round ROUND --handle NAME --accept-rules")}{cmd("guldo verify --round ROUND --id SUBMISSION_ID")}
|
|
717
|
+
<p class="fine">The first command runs your solver on three seeds of your own and certifies the run, with no server involved. It is provisional by construction. Checking a certificate proves the arithmetic and the fairness, and it says whether the timings came from the locked box or from a plain local run.</p></section>
|
|
718
|
+
<section class="card"><h2>What wins</h2><p>Bits of speedup, certified: the cautious number described on the <a href="method.html">Method page</a>, taken as the lowest over at least three seeds. A fast solver that fails the hard questions earns nothing, and so does one that only wins on trivial questions. A result that cannot be told apart from noise is reported as inconclusive.</p>
|
|
719
|
+
<p class="fine"><strong>Your algorithm can stay private.</strong> If you choose to publish your source, your certificate carries it and anyone can re-run it. If you don't, the public certificate carries only your solver's fingerprint: everyone can still check the log, the questions and the arithmetic, but only the operator can run your solver, so your timings rest on the operator's sandbox. If you publish the file later, anyone can check it against that fingerprint and re-run it.</p></section>"""
|
|
720
|
+
return layout("compete.html", "Compete - Guldo", "How to write, check and submit a solver.", body)
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def page_certify():
|
|
724
|
+
"""Standalone certification: measure and verify a result without entering a competition."""
|
|
725
|
+
body = f"""
|
|
726
|
+
<section class="block"><p class="kicker">Certify</p><h1>Measure a result. Prove it to anyone.</h1>
|
|
727
|
+
<p class="lede">Not here to compete. You have a solver, an attack, or a benchmark and you want a verifiable measurement that anyone can check independently. No round, no opponent, no deadline.</p></section>
|
|
728
|
+
<section class="card"><h2>One command</h2>
|
|
729
|
+
<p class="fine">Run your solver on three secret seeds of your own. The tool measures the cost, checks every answer, and writes a certificate: one file that carries the claim, the rules, the seeds and every raw measurement.</p>
|
|
730
|
+
{cmd("guldo practice my_solver.py --family gm --yes")}
|
|
731
|
+
<p class="fine">For exact instruction counts instead of seconds (reproducible on any machine with the same Python version):</p>
|
|
732
|
+
{cmd("guldo practice my_solver.py --family factor --count --dims 38 40 42 --yes")}
|
|
733
|
+
<p class="fine">The certificate is a JSON file. It is self-contained: nothing about it depends on trust in the operator, the server, or this site.</p></section>
|
|
734
|
+
<section class="block"><div class="two"><div class="card"><h2>Anyone can check it</h2>
|
|
735
|
+
<p class="fine">Hand the certificate to a reviewer, a standards body, or a journal. They verify it on their own machine with one command. No account, no server, no trust required.</p>
|
|
736
|
+
{cmd("guldo verify their.cert.json")}
|
|
737
|
+
<p class="fine">Checking never runs any code from the certificate. It verifies the log chain, checks that the seed commitments came first, confirms the certificate holds exactly the instances those seeds produce, and re-derives the credit from the raw timing records with the verifier's own copy of the protocol code.</p></div>
|
|
738
|
+
<div class="card"><h2>Others can attest</h2>
|
|
739
|
+
<p class="fine">An independent party can go further: re-run a sample of instances on their own hardware and sign the outcome. Their attestation is a separate file anyone can check.</p>
|
|
740
|
+
{cmd("guldo attest sign x.cert.json --key my.key --rerun 5 --execute")}
|
|
741
|
+
{cmd("guldo attest check x.cert.json a1.att.json a2.att.json")}
|
|
742
|
+
<p class="fine">An attestation is worth exactly what the signer's identity is worth. The tool separates attestations from keys you chose to trust from ones you did not.</p></div></div></section>
|
|
743
|
+
<section class="card"><h2>Use cases</h2>
|
|
744
|
+
<ul class="fine">
|
|
745
|
+
<li><strong>Lattice security.</strong> Measure a lattice attack's real cost across dimensions. The certificate gives the cryptographic community verifiable data points for setting post-quantum security parameters, not self-reported estimates.</li>
|
|
746
|
+
<li><strong>Algorithm benchmarks.</strong> Prove your solver is faster than the baseline on instances neither side could have prepared for. The seeds are yours, but the protocol is the same one the competitions use.</li>
|
|
747
|
+
<li><strong>Reproducible research.</strong> Attach a certificate to a paper. A reviewer can check the claim without re-running anything, or re-run a sample if they want to.</li>
|
|
748
|
+
<li><strong>AI model evaluation.</strong> Measure inference quality or operation counts on committed instances. See <a href="domains.html#ai">AI workloads</a> for what each kind of evidence shows.</li>
|
|
749
|
+
</ul></section>
|
|
750
|
+
<section class="card"><h2>What it proves and what it does not</h2>
|
|
751
|
+
<p class="fine">A practice certificate uses your own seeds and your own machine. That means:</p>
|
|
752
|
+
<ul class="fine">
|
|
753
|
+
<li><strong>Proven:</strong> the instances are the ones the seeds produce; the solver answered them; the credit follows from the recorded costs; the protocol code is identified by hash.</li>
|
|
754
|
+
<li><strong>Not proven:</strong> that the timings are honest (you ran it). Exact instruction counts remove this gap entirely for pure-Python solvers. For timed runs, independent attestations from people who re-run instances on their own hardware close it partially.</li>
|
|
755
|
+
<li><strong>Not proven:</strong> that the seeds were committed before you saw them (they are yours). In a competition round, the seeds are committed publicly before the deadline. A practice certificate is weaker on this axis, and says so.</li>
|
|
756
|
+
</ul>
|
|
757
|
+
<p class="fine">The certificate itself states whether it was produced in a round or standalone, whether costs are seconds or counts, and whether the solver source is included. The verifier reports all of this. <a href="limits.html">Full limits</a>.</p></section>
|
|
758
|
+
<section class="block"><p class="fine">To enter a competition instead, see the <a href="compete.html">Compete</a> page. To define your own problem domain, see the <a href="guide.html">Guide</a>.</p></section>"""
|
|
759
|
+
return layout("certify.html", "Certify - Guldo", "Measure a solver, attack, or benchmark and produce a certificate anyone can verify independently. No competition required.", body)
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def package_zip(out):
|
|
763
|
+
"""Stage the installable package (build_package.py) and zip it into out/downloads/. -> (file name, sha256, size in bytes). The page shows the hash, so a visitor can
|
|
764
|
+
check what they downloaded; the zip is built from the same checkout as this site, so its protocol files are the ones every certificate here was made with."""
|
|
765
|
+
import build_package
|
|
766
|
+
name = f"{build_package.NAME}-{build_package.DEFAULT_VERSION}"
|
|
767
|
+
os.makedirs(os.path.join(out, "downloads"), exist_ok=True)
|
|
768
|
+
with tempfile.TemporaryDirectory() as td:
|
|
769
|
+
build_package.stage(td, build_package.DEFAULT_VERSION)
|
|
770
|
+
shutil.make_archive(os.path.join(out, "downloads", name), "zip", root_dir=td, base_dir=build_package.NAME)
|
|
771
|
+
path = os.path.join(out, "downloads", name + ".zip")
|
|
772
|
+
with open(path, "rb") as f:
|
|
773
|
+
data = f.read()
|
|
774
|
+
return name + ".zip", hashlib.sha256(data).hexdigest(), len(data)
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def page_install(zip_name, zip_sha, zip_size):
|
|
778
|
+
"""How to install the command-line tool, in plain words, with the honest status of the package."""
|
|
779
|
+
z = esc(zip_name)
|
|
780
|
+
body = f"""
|
|
781
|
+
<section class="block"><p class="kicker">Install</p><h1>Install the command-line tool</h1>
|
|
782
|
+
<p class="lede">One command, <span class="mono">guldo</span>, lets you try a solver on your own computer, submit it to a round, and check any certificate yourself. No account, no browser.</p></section>
|
|
783
|
+
{banner("<strong>Honest status.</strong> This package was built and tried on the author's own computer: it was installed once (with <span class='mono'>pip</span>; <span class='mono'>pipx</span> itself has not been tried) and used to check one real certificate. It is not on the public Python package index, and it has not been installed on a clean computer by anyone else. Exact step counting and the extra claims pass their own tests. A counted test round has run end to end on the project's server and a first real round is open, but no outside participant has used any of it yet.")}
|
|
784
|
+
<section class="block"><p class="kicker">Three steps</p><div class="grid3">
|
|
785
|
+
<div class="card step"><span class="n">1</span><h3>Download</h3><p class="fine"><a href="downloads/{z}">{z}</a> ({zip_size // 1024} KB). To check what you got, compare its SHA-256 with this one:</p><p class="mono fine" style="word-break:break-all">{esc(zip_sha)}</p>{cmd("shasum -a 256 " + zip_name)}<p class="fine">On Linux the command is <span class="mono">sha256sum</span>.</p></div>
|
|
786
|
+
<div class="card step"><span class="n">2</span><h3>Install</h3><p class="fine">The easy way is <span class="mono">pipx</span>, which keeps the tool in its own private folder:</p>{cmd("pipx install ./" + zip_name)}<p class="fine">Or use a virtual environment:</p>{cmd("python3 -m venv ~/.guldo-venv && ~/.guldo-venv/bin/pip install ./" + zip_name)}</div>
|
|
787
|
+
<div class="card step"><span class="n">3</span><h3>Check it works</h3><p class="fine">You should see a version number and a list of the other tools:</p>{cmd("guldo --version")}{cmd("guldo tools")}<p class="fine">With a virtual environment, run <span class="mono">~/.guldo-venv/bin/guldo</span> instead.</p></div></div></section>
|
|
788
|
+
<section class="block"><div class="two"><div class="card"><h2>What you need</h2><ul class="fine"><li>A Mac or a Linux computer. Windows is not supported.</li><li>Python 3.9 or newer. Exact step counting needs Python 3.12 or newer.</li><li>The numerical library <span class="mono">numpy</span>, which is installed for you.</li><li>Optional: <span class="mono">pip install "./{z}[attest]"</span> in a virtual environment adds signed attestations; <span class="mono">[lattice]</span> adds the library the lattice problems use.</li></ul></div>
|
|
789
|
+
<div class="card"><h2>Try it</h2><ul class="fine"><li>Check a solver before you submit: <span class="mono">guldo lint my_solver.py --family factor</span></li><li>See what a practice run on three secret numbers of your own would cost: <span class="mono">guldo practice my_solver.py --family factor --count --dims 38 40 42</span>. It only prints the plan; add <span class="mono">--yes</span> to run it, which can take hours.</li><li>Check a certificate someone gave you: <span class="mono">guldo verify their.cert.json</span></li><li>Work out a benchmark score from it: <span class="mono">guldo claims correctness their.cert.json</span></li><li>See a round and submit: <span class="mono">guldo rounds --server URL</span></li></ul></div></div></section>
|
|
790
|
+
<section class="card"><h2>Good to know</h2><ul class="fine"><li><strong>Checking is safe, re-running is not.</strong> <span class="mono">guldo verify FILE</span> never runs any code from the certificate. Adding <span class="mono">--rerun all --execute</span> re-runs the solvers inside it on your machine, with no protection: do that only on a throwaway computer or virtual machine.</li>
|
|
791
|
+
<li>Your receipts and tokens are kept in <span class="mono">~/.svp</span>, readable only by you. A submission token is shown once; keep it.</li>
|
|
792
|
+
<li>Some commands write under the tool's own folder, which is why <span class="mono">pipx</span> or a virtual environment is the right way to install it. A problem you compile yourself is saved there too, and an upgrade would replace it: keep your own problem declarations in a copy of the repository.</li>
|
|
793
|
+
<li>To update, install the newer file over the old one: <span class="mono">pipx install --force ./new-file.zip</span>. To remove it: <span class="mono">pipx uninstall guldo</span>.</li>
|
|
794
|
+
<li>Prefer to build it yourself? In a checkout of the repository, <span class="mono">make package</span> creates the same folder, and the hash of this download changes with every build of this site, so always compare against the hash shown above.</li></ul>
|
|
795
|
+
<p class="fine">What each result does and does not prove is on the <a href="limits.html">Limits</a> page, and the plain-words list of extra claims is on <a href="claims.html">Other claims</a>.</p></section>"""
|
|
796
|
+
return layout("install.html", "Install - Guldo", "Install the guldo command-line tool: download, check the hash, install with pipx, and try it.", body)
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
def page_claims():
|
|
800
|
+
"""What else a verified certificate can back (claims.py, counted.py). Plain words; every card says what the claim does NOT show."""
|
|
801
|
+
items = [
|
|
802
|
+
("Exact cost, not seconds", "Instead of timing a program, count the Python instructions it runs. For a program that behaves the same way every time, anyone with the same Python version gets exactly the same number, so a re-run proves the recorded cost without trusting anyone's computer, clock or sealed box.",
|
|
803
|
+
"It is not a time. Work done inside one built-in function (sorting, big-number arithmetic) counts as a single instruction, so it only means something for plain Python on both sides. The solver must be written in a restricted, pure-computation subset of Python.",
|
|
804
|
+
"guldo practice my_solver.py --family factor --count --dims 38 40 42"),
|
|
805
|
+
("Rounds that count", "A contest can announce, before anyone submits, that every solver will be counted on the same question sizes. The sizes, the cap, the Python version and the counting program are written into the round's first log entry. Every source is public, so anyone can re-run any count.",
|
|
806
|
+
"It is called COUNTED, not official: official means timed inside the verified locked box. Today it is for the factoring problem only, and the operator's machine still runs the submitted code, so it needs a throwaway user or machine.",
|
|
807
|
+
"guldo ops round-new --count --count-dims 38 40 42 ..."),
|
|
808
|
+
("A benchmark score", "How many of the questions fixed in advance did a program answer correctly, with an uncertainty range, for both the submitted solver and the baseline. The questions come from secret numbers fingerprinted before the program was seen, so nobody could have trained on them.",
|
|
809
|
+
"It is a rate on this question set, under the time limit, for this kind of puzzle. It says nothing about other questions. With seconds as the cost, 'within the limit' depends on the measuring computer; an exact-count certificate removes that.",
|
|
810
|
+
"guldo claims correctness x.cert.json"),
|
|
811
|
+
("A verified negative result", "'Nobody in this contest reached a speedup of T bits.' The claim holds only if there is a certificate for every submission the round's log accepted and the log shows the round was closed, so nobody can hide the attempt that worked.",
|
|
812
|
+
"It covers the programs actually submitted, not every possible approach. The credit is deliberately cautious, so 'did not reach T' means 'T was not shown', not 'T is impossible'. A submission whose evaluation failed makes the claim incomplete.",
|
|
813
|
+
"guldo claims negative a.cert.json b.cert.json --threshold 2"),
|
|
814
|
+
("The worst case seen", "For an exact-count certificate: the largest number of instructions seen at each question size, and the chance that one more question from the same source would need more (at most 1 in n+1).",
|
|
815
|
+
"It is the largest value seen, not a guarantee for every possible input of that size. A size where any question timed out or failed has no bound at all.",
|
|
816
|
+
"guldo claims worst-case x.cert.json"),
|
|
817
|
+
]
|
|
818
|
+
cards = "".join(f'<div class="card"><h3>{esc(t)}</h3><p>{esc(a)}</p><p class="fine"><strong>What it does not show:</strong> {esc(b)}</p>{cmd(c)}</div>' for t, a, b, c in items)
|
|
819
|
+
body = f"""
|
|
820
|
+
<section class="block"><p class="kicker">Other claims</p><h1>What else a certificate can prove</h1>
|
|
821
|
+
<p class="lede">A certificate records one plain fact: this program, on questions fixed before anyone saw it, gave these answers at this cost. A speedup is just one thing you can read off it. These are others.</p></section>
|
|
822
|
+
{banner("<strong>Tested, not yet used.</strong> Everything on this page passes its own tests on the author's computer, and the claims have been worked out from one real certificate. Nobody else has used any of it, and no real round has finished yet (a counted test round has run end to end on the project's server, and a first real round is open), so expect rough edges before anyone relies on it.")}
|
|
823
|
+
<section class="block"><div class="run">{cards}</div></section>
|
|
824
|
+
<section class="card"><h2>How these stay honest</h2><ul class="fine"><li>Each claim is recomputed from a certificate that has first been checked: nothing is read off a file that does not verify.</li>
|
|
825
|
+
<li>A claim file is not trusted either. <span class="mono">guldo claims check CLAIM.json CERT.json</span> recomputes it and demands the same answer.</li>
|
|
826
|
+
<li>Every claim prints what it does not show, every time.</li></ul>
|
|
827
|
+
<p class="fine">The guide on this site explains how to check a certificate; the <a href="limits.html">Limits</a> page lists what checking cannot prove.</p></section>"""
|
|
828
|
+
return layout("claims.html", "Other claims - Guldo", "Exact costs, benchmark scores, verified negative results and worst-case figures: what else a certificate can back.", body)
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
# (limit, what would close it, status). The status is "Open" until the evidence exists (limits_plan.py finds it; RESOLVE.md has the steps) and "Stays" for a property of the method.
|
|
832
|
+
LIMIT_PLAN = [
|
|
833
|
+
("The sandbox that runs other people's code has passed its tests on real Linux machines, but not yet on a dedicated machine that is separate from the web service", "A separate, dedicated-CPU evaluation machine where the sandbox's own tests and self-test pass, and a full timed round evaluated there", "Open"),
|
|
834
|
+
("No result is official", "An official round: three independent secret numbers, run inside the verified sandbox at the problem's full size", "Open"),
|
|
835
|
+
("Runs used several jobs at once", "Official runs evaluate one question at a time inside the sandbox", "Open"),
|
|
836
|
+
("One seed is one sample", "Three or more fresh-seed runs, each registered before its data and judged against its written criteria", "Open"),
|
|
837
|
+
("The factoring control is not a clean recovery of a known number", "A calibrated pair at the ladder's own sizes, run one job at a time in the sandbox", "Open"),
|
|
838
|
+
("Differences between computers are not measured", "Re-measurements from two or more different computers", "Open"),
|
|
839
|
+
("The paper's own code cannot be used as a baseline", "A stronger solver for the baseline (research, not engineering)", "Open"),
|
|
840
|
+
("The certifiable range is about 1.5 to 3 bits, with a ceiling near 5", "Nothing removes it: it follows from the 0.5 s cost floor and the time limit. Bigger rounds stretch it", "Stays on this page"),
|
|
841
|
+
("A private solver's timings rest on the operator's sandbox", "Would need trusted hardware or proofs of execution, which are not feasible for solvers that run for minutes", "Stays on this page"),
|
|
842
|
+
("The baseline decides what counts", "It is a judgement: authors state what beats their baseline and independent baselines help", "Stays on this page"),
|
|
843
|
+
]
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def page_limits():
|
|
847
|
+
limits = R.LIMITS
|
|
848
|
+
body = f"""
|
|
849
|
+
<section class="block"><p class="kicker">Limits</p><h1>What this does not show</h1>
|
|
850
|
+
<p class="lede">The most useful part of a verification site is a list of what it has not verified. This page is kept as plain as the results.</p></section>
|
|
851
|
+
<section class="card"><h2>What a checked result still does not prove</h2><p class="fine">A certificate can be checked by anyone, and checking it proves the arithmetic and the fairness. It does not, by itself, prove the timings were honest.</p>
|
|
852
|
+
<ul class="fine"><li><strong>Proven by checking:</strong> the seeds' fingerprints were public first, the questions are exactly the ones the seeds produce, the rules are the ones named, and the credit follows from the recorded timings.</li>
|
|
853
|
+
<li><strong>Not proven by checking:</strong> that the recorded timings are what a real run took. Someone who runs the test could write plausible numbers. What helps: the locked-box environment (its tests have passed on real Linux machines, but no official result has been measured in it yet), and other people re-measuring on their own computers. A re-measurement only counts as independent if you chose to trust whoever made it: a signature shows who signed, not that they are unconnected to the operator.</li>
|
|
854
|
+
<li><strong>One way to prove a cost exactly:</strong> instead of seconds, count the Python instructions a solver executes. That number is the same on every computer, so anyone can re-run a few questions and must get identical counts, trusting nobody's machine or clock. It has limits: it is not a time, it cannot see work done inside a built-in C function (which counts as one instruction however long it runs), it needs a recent Python and pure-Python solvers, and, unless a counted round declared the sizes in its first log entry before any submission, the question sizes are chosen by whoever ran it, so it is not an official round result.</li>
|
|
855
|
+
<li><strong>AI workloads carry weaker evidence than the lattice and factoring ones:</strong> a counted-operations certificate proves how much arithmetic a solver asked for, not how fast it ran, and cannot see memory traffic, precision or kernel quality. A GPU time is one operator's clock on one device model and cannot be re-executed exactly by anyone else. See <a href="domains.html#ai">AI workloads</a> on the Domains page.</li>
|
|
856
|
+
<li><strong>The baseline decides what counts:</strong> a speedup is only as meaningful as the solver it beats. A weak baseline makes big speedups cheap. Each problem's author is asked to say what the baseline is and what is known to beat it, and the problem's card shows when they have not.</li>
|
|
857
|
+
<li><strong>Computers differ:</strong> a speedup is a ratio, which mostly cancels differences in machine speed, but the spread across different machines has not been measured yet, so the tolerance for re-measurement is deliberately loose.</li>
|
|
858
|
+
<li><strong>Private solvers are hidden from the public, not from the operator:</strong> a submitter may keep their solver private, and then the public certificate shows only its fingerprint. Everyone can still verify the log, the questions and the arithmetic, but nobody except the operator can inspect or re-run it, so its timings rest on the operator's sandbox alone. If the owner later publishes the file, anyone can check it against the fingerprint. Hiding a solver from the operator as well would need special hardware or a zero-knowledge proof, and neither is feasible for solvers that run for minutes.</li></ul></section>
|
|
859
|
+
<section class="card"><h2>Open limits</h2><ul>{"".join(f"<li>{esc(x)}</li>" for x in limits)}</ul>
|
|
860
|
+
<ul><li>Several jobs at once: runs use several processes on one computer at the same time. Re-measuring one job at a time shifted the baseline by -0.050 bits and the tuned solver by -0.084, so the speedup was slightly underestimated, not inflated. The pass line was a shift under 0.10.</li>
|
|
861
|
+
<li>That effect depends on the solver. On the factoring control, re-measured one job at a time, the baseline (a plain Python loop) moved by -0.033 bits, but the numpy solver ran 1.16 bits faster than it had with six jobs at once, against a pre-registered limit of 0.10. So that check failed. The bias happened to favour the baseline, which makes the credit cautious, but for another pair of solvers it could point the other way. Runs with several jobs at once are therefore valid only after re-measuring both sides one job at a time, and official runs use one question at a time in the locked box.</li>
|
|
862
|
+
<li>The factoring control's known effect (about 4.4 bits) was calibrated on 44 to 52-bit numbers. The 4.2 bits measured at 58 to 64 bits agrees with it partly because the several-jobs bias cancels a real increase with size. It is not a clean recovery of a known number.</li>
|
|
863
|
+
<li>One seed is one sample. The pre-registered result is a single test on a fresh seed, not a replication.</li></ul></section>
|
|
864
|
+
<section class="card"><h2>What would close each limit</h2><p class="fine">Some limits can be closed with more work, and this table says what that work is. Others are properties of the method and will stay on this page.</p>
|
|
865
|
+
<div class="scroll"><table class="lb"><thead><tr><th>Limit</th><th>What would close it</th><th>Status</th></tr></thead><tbody>{"".join(f'<tr><td>{esc(a)}</td><td class="fine">{esc(b)}</td><td><strong>{esc(c)}</strong></td></tr>' for a, b, c in LIMIT_PLAN)}</tbody></table></div></section>
|
|
866
|
+
<section class="block"><h2>Related work</h2><p class="fine">Public challenges already exist in this area. The <a href="https://www.latticechallenge.org/svp-challenge/index.php" rel="noopener">Darmstadt SVP challenge</a> and its LWE counterpart publish instances and record solutions. <a href="https://www.bankless.com/read/news/eigen-labs-launches-yukon-for-open-ai-research" rel="noopener">Yukon</a>, from Eigen Labs, is an open platform where people and AI agents compete on posted technical challenges. This project is about checking claims: seeds committed before the questions exist, a public log, and certificates anyone can recompute.</p></section>"""
|
|
867
|
+
return layout("limits.html", "Limits - Guldo", "What this does not show: open limits of the protocol and its results.", body)
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
# ---------------------------------------------------------------- build
|
|
871
|
+
def build(out, results_dir, reveals, notes=None, ledger_data=None, portal_url="", certs_dir=None, cname=None):
|
|
872
|
+
data = collect(results_dir)
|
|
873
|
+
board = []
|
|
874
|
+
if os.path.exists(judge.BOARD):
|
|
875
|
+
with open(judge.BOARD, encoding="utf-8") as f:
|
|
876
|
+
board = json.load(f)
|
|
877
|
+
prereg = read_prereg(results_dir)
|
|
878
|
+
seeds = load_reveals(reveals, prereg)
|
|
879
|
+
for what, src in (("--certs", certs_dir), ("the results directory", results_dir), ("--ledger-data", ledger_data)):
|
|
880
|
+
if src: # the build deletes and recreates `out`: an input that lives inside it would be destroyed first
|
|
881
|
+
s, o = os.path.realpath(src), os.path.realpath(out)
|
|
882
|
+
if s == o or s.startswith(o + os.sep):
|
|
883
|
+
raise SystemExit(f"REFUSED: {what} ({src}) is inside the output directory {out}, which this build deletes and recreates. Keep inputs outside it.")
|
|
884
|
+
if os.path.exists(out):
|
|
885
|
+
# never delete a directory this script did not create: --out ~ or --out . must not be able to wipe anything
|
|
886
|
+
if not (os.path.isdir(out) and os.path.exists(os.path.join(out, MARKER))):
|
|
887
|
+
if os.path.isdir(out) and not os.listdir(out):
|
|
888
|
+
pass # an empty directory is safe to reuse
|
|
889
|
+
else:
|
|
890
|
+
raise SystemExit(f"REFUSED: {out} exists and is not a site this script built (no {MARKER} marker); "
|
|
891
|
+
"choose a new directory or remove it yourself")
|
|
892
|
+
else:
|
|
893
|
+
shutil.rmtree(out)
|
|
894
|
+
os.makedirs(os.path.join(out, "assets"), exist_ok=True)
|
|
895
|
+
os.makedirs(os.path.join(out, "prereg"), exist_ok=True)
|
|
896
|
+
os.makedirs(os.path.join(out, "data"), exist_ok=True)
|
|
897
|
+
with open(os.path.join(out, MARKER), "w", encoding="utf-8") as f:
|
|
898
|
+
f.write("built by build_site.py; this directory may be deleted and rebuilt by it\n")
|
|
899
|
+
if cname: # for a static host that serves this folder under a custom domain (GitHub Pages reads CNAME; .nojekyll stops it processing the files)
|
|
900
|
+
if not re.fullmatch(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+", cname):
|
|
901
|
+
raise SystemExit(f"REFUSED: --cname {cname!r} is not a plain domain name like example.org")
|
|
902
|
+
with open(os.path.join(out, "CNAME"), "w", encoding="utf-8") as f:
|
|
903
|
+
f.write(cname + "\n")
|
|
904
|
+
with open(os.path.join(out, ".nojekyll"), "w", encoding="utf-8") as f:
|
|
905
|
+
f.write("")
|
|
906
|
+
rounds_index, sandbox_ok = None, False
|
|
907
|
+
if ledger_data: # the public competition's rounds, exported (tokens and unrevealed sources are never in the export)
|
|
908
|
+
import portal_core
|
|
909
|
+
import rounds
|
|
910
|
+
rounds_index = rounds.export_public(ledger_data, os.path.join(out, "ledger"))
|
|
911
|
+
sandbox_ok = portal_core.sandbox_verified(os.path.abspath(ledger_data))
|
|
912
|
+
with open(os.path.join(out, "assets", "site.css"), "w", encoding="utf-8") as f:
|
|
913
|
+
f.write(site_assets.CSS + EXTRA_CSS)
|
|
914
|
+
with open(os.path.join(out, "assets", "site.js"), "w", encoding="utf-8") as f:
|
|
915
|
+
f.write(site_assets.JS)
|
|
916
|
+
for p in prereg:
|
|
917
|
+
with open(os.path.join(out, "prereg", p["file"]), "w", encoding="utf-8") as f:
|
|
918
|
+
f.write(p["text"])
|
|
919
|
+
import dlog_experiment
|
|
920
|
+
import certs_site
|
|
921
|
+
certs_site.copy_files(certs_dir, out)
|
|
922
|
+
exp_reg = os.path.join(results_dir, dlog_experiment.PREREG)
|
|
923
|
+
if os.path.exists(exp_reg): # not in `prereg` (its commitments are on separate lines), but the card links it
|
|
924
|
+
shutil.copy(exp_reg, os.path.join(out, "prereg", dlog_experiment.PREREG))
|
|
925
|
+
for s, d in data:
|
|
926
|
+
if "error" not in d:
|
|
927
|
+
for path in (s["baseline"], s["submission"]):
|
|
928
|
+
shutil.copy(path, os.path.join(out, "data", os.path.basename(path)))
|
|
929
|
+
for row in counted_rows(results_dir): # the counted-operations ladders the AI section links to: anyone can re-run them
|
|
930
|
+
for fn in row["files"]:
|
|
931
|
+
shutil.copy(os.path.join(results_dir, fn), os.path.join(out, "data", fn))
|
|
932
|
+
zip_name, zip_sha, zip_size = package_zip(out) # the installable tool, built from this same checkout and shipped with the site
|
|
933
|
+
pages = {"index.html": page_index(data, board, prereg, notes), "install.html": page_install(zip_name, zip_sha, zip_size), "domains.html": page_domains(results_dir), "guide.html": page_guide(), "method.html": page_method(), "results.html": page_results(data, board, prereg, notes, results_dir, certs_dir),
|
|
934
|
+
"preregistrations.html": page_prereg(prereg, seeds), "compete.html": page_compete(rounds_index, portal_url, sandbox_ok), "certify.html": page_certify(), "claims.html": page_claims(), "limits.html": page_limits()}
|
|
935
|
+
for fn, text in pages.items():
|
|
936
|
+
with open(os.path.join(out, fn), "w", encoding="utf-8") as f:
|
|
937
|
+
f.write(text)
|
|
938
|
+
return sorted(pages), len(data), sum(1 for _, d in data if "error" in d)
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def main():
|
|
942
|
+
ap = argparse.ArgumentParser()
|
|
943
|
+
ap.add_argument("--out", default=os.path.join(HERE, "site"))
|
|
944
|
+
ap.add_argument("--reveal", action="append", metavar="TAG=SEEDFILE", help="ship a revealed seed (checked against its commitment)")
|
|
945
|
+
ap.add_argument("--note", action="append", metavar="TAG=TEXT",
|
|
946
|
+
help="one-line concurrency re-measurement summary shown on that registration's card (TAG = first 8 hex of its commitment)")
|
|
947
|
+
ap.add_argument("--ledger-data", help="the public competition's data directory (ops.py --data): adds its rounds to the Compete page")
|
|
948
|
+
ap.add_argument("--portal-url", default="", help="public URL of the submission service (public_portal.py), linked from the Compete page")
|
|
949
|
+
ap.add_argument("--certs", help="a directory of NAME.cert.json, NAME.*.att.json and trusted.txt: each certificate is re-verified and shown on the Results page")
|
|
950
|
+
ap.add_argument("--cname", help="a custom domain for a static host (writes CNAME and .nojekyll into the site), e.g. guldo.org")
|
|
951
|
+
a = ap.parse_args()
|
|
952
|
+
notes = dict(n.partition("=")[::2] for n in a.note or [])
|
|
953
|
+
pages, n, bad = build(a.out, os.path.join(HERE, "results"), a.reveal, notes, a.ledger_data, a.portal_url, a.certs, a.cname)
|
|
954
|
+
print(f"wrote {len(pages)} pages to {a.out} ({n} ladder pair(s), {bad} not chartable)")
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
if __name__ == "__main__":
|
|
958
|
+
main()
|