guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
"""PUBLIC submission service: the web tier. Run it behind a TLS-terminating reverse proxy (deploy/Caddyfile).
|
|
2
|
+
|
|
3
|
+
python3 public_portal.py serve --data /srv/ledger/data --host speedup.example.org [--bind 127.0.0.1] [--port 8080] [--trust-proxy]
|
|
4
|
+
|
|
5
|
+
What this process does: lists rounds, shows each round's commitments and public log, accepts an upload while a round is open,
|
|
6
|
+
parses it with `ast` (never imports or runs it), appends it to the round's hash-chained log, and shows its status. What it
|
|
7
|
+
never does: execute an upload, read a seed (seeds live in a directory this user cannot read), evaluate anything, or store
|
|
8
|
+
contact details or IP addresses. Evaluation is a separate process on a separate user or host (ops.py worker, sandbox-gated).
|
|
9
|
+
|
|
10
|
+
Threat model, each item with a test (test_public_portal.py):
|
|
11
|
+
* hostile uploads -> never executed; size limit; UTF-8 only; lint; identical files rejected; rejections store nothing
|
|
12
|
+
* flooding -> per-IP and global token buckets (429), a hard cap on concurrent connections (503), 10 s socket timeout
|
|
13
|
+
* Host / DNS rebinding, cross-site POST -> Host must be on the allowlist; a POST's Origin, when sent, must be one of our own
|
|
14
|
+
* handle squatting -> a handle is bound to a secret token issued once; its hash is all that is stored
|
|
15
|
+
* operator cheating -> every accepted file's hash is in a public hash-chained log (rounds.py) that anyone can verify
|
|
16
|
+
* emergency -> `touch <data>/KILL_SWITCH` pauses submissions at once, reads keep working
|
|
17
|
+
* XSS / injection -> every dynamic string is escaped, handles are validated, strict CSP (no inline script or style)
|
|
18
|
+
"""
|
|
19
|
+
import argparse
|
|
20
|
+
import hashlib
|
|
21
|
+
import html
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import signal
|
|
26
|
+
import threading
|
|
27
|
+
import time
|
|
28
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
29
|
+
|
|
30
|
+
import families
|
|
31
|
+
import portal_core
|
|
32
|
+
import rounds
|
|
33
|
+
import site_assets
|
|
34
|
+
|
|
35
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
36
|
+
RULES_VERSION = "draft-1"
|
|
37
|
+
LOOPBACK = ("127.0.0.1", "localhost")
|
|
38
|
+
# the CSP forbids inline style attributes, so everything the pages need beyond the site stylesheet lives here
|
|
39
|
+
EXTRA_CSS = """
|
|
40
|
+
.brk{word-break:break-all}
|
|
41
|
+
.verify input[type=text],.verify input[type=file]{font:13px var(--mono);padding:10px;border:1px solid var(--line);border-radius:8px;background:var(--card);color:var(--ink);width:100%;min-width:0}
|
|
42
|
+
.verify label.small{display:flex;gap:8px;align-items:flex-start}
|
|
43
|
+
button.btn{cursor:pointer;font:600 15px var(--body)}
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class RateLimiter:
|
|
48
|
+
"""Token buckets keyed by a hash of the client address (the address itself is never kept). Memory is bounded."""
|
|
49
|
+
|
|
50
|
+
def __init__(self, per_second, burst, max_keys=20000, clock=time.monotonic):
|
|
51
|
+
self.rate, self.burst, self.max_keys, self.clock = per_second, burst, max_keys, clock
|
|
52
|
+
self.buckets = {}
|
|
53
|
+
self.lock = threading.Lock()
|
|
54
|
+
|
|
55
|
+
def allow(self, key):
|
|
56
|
+
now = self.clock()
|
|
57
|
+
with self.lock:
|
|
58
|
+
tokens, last = self.buckets.get(key, (self.burst, now))
|
|
59
|
+
tokens = min(self.burst, tokens + (now - last) * self.rate)
|
|
60
|
+
if tokens < 1:
|
|
61
|
+
self.buckets[key] = (tokens, now)
|
|
62
|
+
return False
|
|
63
|
+
if len(self.buckets) >= self.max_keys and key not in self.buckets:
|
|
64
|
+
oldest = sorted(self.buckets, key=lambda k: self.buckets[k][1])[: self.max_keys // 10]
|
|
65
|
+
for k in oldest: # evict the stalest tenth: an attacker spraying addresses cannot grow memory without bound
|
|
66
|
+
del self.buckets[k]
|
|
67
|
+
self.buckets[key] = (tokens - 1, now)
|
|
68
|
+
return True
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Config:
|
|
72
|
+
def __init__(self, base, allowed_hosts=(), trust_proxy=False, max_conns=64, site_url="", clock=time.time):
|
|
73
|
+
self.base = os.path.abspath(base)
|
|
74
|
+
self.allowed_hosts = {h.lower() for h in allowed_hosts}
|
|
75
|
+
self.trust_proxy = trust_proxy
|
|
76
|
+
self.site_url = site_url.rstrip("/")
|
|
77
|
+
self.clock = clock
|
|
78
|
+
self.ip_salt = os.urandom(16) # per-process: address hashes cannot be linked across restarts
|
|
79
|
+
self.get_limiter = RateLimiter(2.0, 40) # 120 requests / minute, burst 40
|
|
80
|
+
self.submit_limiter = RateLimiter(6 / 3600, 3) # 6 submissions / hour / address, burst 3
|
|
81
|
+
self.global_submit_limiter = RateLimiter(300 / 3600, 20)
|
|
82
|
+
self.max_conns = max_conns
|
|
83
|
+
os.makedirs(os.path.join(self.base, "rounds"), exist_ok=True)
|
|
84
|
+
|
|
85
|
+
def killed(self):
|
|
86
|
+
return os.path.exists(os.path.join(self.base, "KILL_SWITCH"))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ------------------------------------------------------------------ pages
|
|
90
|
+
def e(x):
|
|
91
|
+
return html.escape(str(x), quote=True)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def utc(ts):
|
|
95
|
+
return time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(ts))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
FAVICON = ("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 28 28'%3E%3Crect width='28' height='28' rx='6' fill='%232A78D6'/%3E"
|
|
99
|
+
"%3Cpath d='M6 20h4v-4h4v-4h4V8h4' fill='none' stroke='white' stroke-width='2.6' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def layout(cfg, title, body):
|
|
103
|
+
back = f'<a href="{e(cfg.site_url)}/index.html">Main site</a>' if cfg.site_url else ""
|
|
104
|
+
return f"""<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
105
|
+
<title>{e(title)} - Guldo</title><meta name="color-scheme" content="light dark"><meta name="robots" content="noindex">
|
|
106
|
+
<link rel="icon" href="{FAVICON}"><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
107
|
+
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:wght@600&family=Instrument+Sans:wght@400;600&family=JetBrains+Mono:wght@400;600&display=swap">
|
|
108
|
+
<link rel="stylesheet" href="/portal-assets/site.css"></head><body><a class="skip" href="#main">Skip to content</a>
|
|
109
|
+
<div class="wrap"><header class="top"><a class="brand" href="/">Guldo <span class="muted small">submissions</span></a>
|
|
110
|
+
<nav class="main" aria-label="Main"><a href="/">Rounds</a>{back}<button class="theme" id="theme-toggle" type="button" aria-pressed="false">Dark mode</button></nav></header>
|
|
111
|
+
<main id="main">{body}</main>
|
|
112
|
+
<footer class="foot"><p>Nothing here is run until a round has closed, and then only inside a verified sandbox. Every accepted file's hash is in a public log you can check.</p></footer></div>
|
|
113
|
+
<div id="tip" role="tooltip"></div><script src="/portal-assets/site.js"></script></body></html>"""
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def state_pill(state):
|
|
117
|
+
kind = {"open": "ok", "closed": "warn", "evaluating": "warn", "evaluated": "info", "revealed": "info"}[state]
|
|
118
|
+
return f'<span class="pill {kind}"><span>{e(state)}</span></span>'
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def page_rounds(cfg):
|
|
122
|
+
rows = []
|
|
123
|
+
for r in rounds.list_rounds(cfg.base):
|
|
124
|
+
n = len(rounds.list_submissions(cfg.base, r["id"]))
|
|
125
|
+
rows.append(f'<tr><td class="strong"><a href="/round/{e(r["id"])}">{e(r["title"])}</a></td><td>{e(r["family"])}</td><td>{state_pill(r["state"])}</td>'
|
|
126
|
+
f'<td class="mono small">{e(utc(r["closes_at"]))}</td><td class="num mono">{n}</td></tr>')
|
|
127
|
+
table = ('<div class="scroll"><table class="lb"><thead><tr><th>Round</th><th>Family</th><th>State</th><th>Closes</th><th class="num">Submissions</th></tr></thead>'
|
|
128
|
+
f'<tbody>{"".join(rows)}</tbody></table></div>') if rows else '<p class="muted">No rounds yet.</p>'
|
|
129
|
+
paused = '<div class="banner" role="note"><p><strong>Submissions are paused.</strong> The operator has switched them off for now.</p></div>' if cfg.killed() else ""
|
|
130
|
+
body = f"""<section class="block"><p class="kicker">Submissions</p><h1>Rounds</h1>
|
|
131
|
+
<p class="lede">Each round publishes its seed commitments before it opens. You submit one Python file; after the deadline it is run in a sandbox on the committed instances, and the seeds are revealed.</p></section>
|
|
132
|
+
{paused}<section class="card"><h2>All rounds</h2>{table}</section>"""
|
|
133
|
+
return layout(cfg, "Rounds", body)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def page_round(cfg, rid, flash=""):
|
|
137
|
+
pr = rounds.public_round(cfg.base, rid)
|
|
138
|
+
ok = pr["log_ok"]
|
|
139
|
+
commits = "".join(f'<li class="mono small brk">{e(c)}</li>' for c in pr["commitments"])
|
|
140
|
+
subs = "".join(f'<tr><td class="strong">{e(s["handle"])}</td><td class="mono small">{e(s["sha256"][:16])}...</td><td class="mono small">{e(utc(s["submitted_at"]))}</td>'
|
|
141
|
+
f'<td class="num mono">{s["seq"]}</td><td>{e(s["state"])}</td>'
|
|
142
|
+
f'<td>{result_cell(s)}</td><td><a href="/s/{e(rid)}/{e(s["id"])}">status</a></td></tr>' for s in pr["submissions"])
|
|
143
|
+
table = ('<div class="scroll"><table class="lb"><thead><tr><th>Handle</th><th>File hash</th><th>Received</th><th class="num">Log #</th><th>State</th><th>Credit</th><th></th></tr></thead>'
|
|
144
|
+
f'<tbody>{subs}</tbody></table></div>') if subs else '<p class="muted">No submissions yet.</p>'
|
|
145
|
+
form = ""
|
|
146
|
+
must_publish = not pr.get("allow_private", True)
|
|
147
|
+
pub_required = " required" if must_publish else ""
|
|
148
|
+
pub_suffix = " (this round requires it)" if must_publish else ""
|
|
149
|
+
pub_hint = "" if must_publish else ('<p class="small muted">Leave it unticked and your solver stays private: after the reveal the public certificate carries only its '
|
|
150
|
+
'fingerprint, nobody but the operator can run it, and others must trust the operator\'s sandbox for your timings.</p>')
|
|
151
|
+
if pr["state"] == "open" and not cfg.killed():
|
|
152
|
+
form = f"""<section class="card"><h2>Submit a solver</h2>
|
|
153
|
+
<form class="verify" method="post" action="/round/{e(rid)}/submit" enctype="multipart/form-data">
|
|
154
|
+
<label for="handle" class="small muted">Handle (public)</label><input id="handle" name="handle" type="text" maxlength="32" required autocomplete="off" placeholder="my_solver">
|
|
155
|
+
<label for="token" class="small muted">Token (only if you have already submitted under this handle this round)</label><input id="token" name="token" type="text" autocomplete="off">
|
|
156
|
+
<label for="file" class="small muted">Python file</label><input id="file" name="file" type="file" accept=".py" required>
|
|
157
|
+
<label class="small"><input type="checkbox" name="publish_source"{pub_required}> Publish my source after the seeds are revealed{pub_suffix}</label>{pub_hint}
|
|
158
|
+
<label class="small"><input type="checkbox" name="agree" required> I accept the rules ({e(RULES_VERSION)}, a draft: see RULES.md)</label>
|
|
159
|
+
<div class="btnrow"><button class="btn primary" type="submit">Check and submit</button></div></form>
|
|
160
|
+
<p class="small muted">Your file is read, never run, until the round closes. It should expose <span class="mono">solve</span> for this family; see the Compete page of the main site.</p></section>"""
|
|
161
|
+
elif pr["state"] == "open":
|
|
162
|
+
form = '<div class="banner" role="note"><p><strong>Submissions are paused.</strong></p></div>'
|
|
163
|
+
seeds = ""
|
|
164
|
+
if pr["state"] == "revealed":
|
|
165
|
+
seeds = ('<section class="card"><h2>Revealed seeds</h2><p class="fine">Each seed hashes to one of the commitments above; check it on the main site\'s Pre-registrations page.</p><ul>'
|
|
166
|
+
+ "".join(f'<li class="mono small brk">{e(s)}</li>' for s in pr["seeds"].values()) + "</ul></section>")
|
|
167
|
+
flash_html = f'<div class="banner" role="status"><p>{e(flash)}</p></div>' if flash else ""
|
|
168
|
+
logline = ("The log verifies: every entry chains to the one before it." if ok else f"LOG PROBLEM: {e(pr['log_error'])}")
|
|
169
|
+
if pr["consistency_problems"]:
|
|
170
|
+
logline += " CONSISTENCY PROBLEM: " + e("; ".join(pr["consistency_problems"][:3])) + ("..." if len(pr["consistency_problems"]) > 3 else "")
|
|
171
|
+
body = f"""<section class="block"><p class="kicker">{e(pr["family"])} round</p><h1>{e(pr["title"])}</h1>
|
|
172
|
+
<p class="lede">Opens {e(utc(pr["opens_at"]))}. Closes {e(utc(pr["closes_at"]))}. Accepts up to {pr["max_submissions"]} submissions ({len(pr["submissions"])} so far). {pr["n"]} instances per size. State: {state_pill(pr["state"])}</p></section>{flash_html}{form}
|
|
173
|
+
<section class="card"><h2>Seed commitments</h2><p class="fine">SHA-256 of each secret seed, published when the round was created and recorded as entry 0 of the log. Instances are derived from the seeds, so they could not be chosen after seeing submissions.</p><ul>{commits}</ul></section>
|
|
174
|
+
<section class="card"><h2>Submissions</h2>{table}
|
|
175
|
+
<p class="fine">{logline} {pr["log_entries"]} entries, head <span class="mono small">{e((pr["log_head"] or "")[:16])}...</span>. <a href="/api/log/{e(rid)}.jsonl">Download the log</a> and <a href="/api/round/{e(rid)}.json">the round as JSON</a>.</p></section>{seeds}"""
|
|
176
|
+
return layout(cfg, pr["title"], body)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def result_cell(s):
|
|
180
|
+
res = s.get("result")
|
|
181
|
+
if not res:
|
|
182
|
+
return '<span class="muted small">pending</span>' if "result" not in s else '<span class="muted small">none</span>'
|
|
183
|
+
return f'<span class="mono">{e(res["credit"])}</span> <span class="small muted">{e(res["status"])}</span>'
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def page_submission(cfg, rid, sid):
|
|
187
|
+
r = rounds.get_round(cfg.base, rid)
|
|
188
|
+
m = rounds.public_submission(cfg.base, rid, rounds.get_submission(cfg.base, rid, sid), r)
|
|
189
|
+
warns = "".join(f"<li>{e(w)}</li>" for w in m["warnings"])
|
|
190
|
+
res = ""
|
|
191
|
+
if "result" in m:
|
|
192
|
+
res = (f'<dl class="facts"><div><dt>Credit</dt><dd class="mono">{e(m["result"]["credit"] if m["result"] else "none")}</dd></div>'
|
|
193
|
+
f'<div><dt>Status</dt><dd>{e(m["result"]["status"] if m["result"] else m.get("reason", ""))}</dd></div></dl>')
|
|
194
|
+
src = f'<details><summary>Source</summary><pre>{e(m["source"])}</pre></details>' if "source" in m else ""
|
|
195
|
+
body = f"""<section class="block"><p class="kicker">Submission</p><h1>{e(m["handle"])}</h1></section>
|
|
196
|
+
<section class="card"><dl class="facts"><div><dt>Round</dt><dd><a href="/round/{e(rid)}">{e(r["title"])}</a></dd></div>
|
|
197
|
+
<div><dt>State</dt><dd>{e(m["state"])}</dd></div><div><dt>Received</dt><dd>{e(utc(m["submitted_at"]))}</dd></div>
|
|
198
|
+
<div><dt>File sha256</dt><dd class="mono small brk">{e(m["sha256"])}</dd></div>
|
|
199
|
+
<div><dt>Log entry</dt><dd class="mono small">#{m["seq"]}, hash {e(m["log_hash"])}</dd></div></dl>
|
|
200
|
+
{('<h3>Static-check warnings</h3><ul>' + warns + '</ul>') if warns else ''}{res}{src}
|
|
201
|
+
<p class="fine">To check this receipt, download the round's log and confirm entry #{m["seq"]} carries your file's hash and the chain verifies.</p></section>"""
|
|
202
|
+
return layout(cfg, f"Submission {sid}", body)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def page_receipt(cfg, rid, out, handle):
|
|
206
|
+
token = (f'<div class="banner" role="note"><p><strong>Keep this token: {e(out["token"])}</strong> It is shown once and proves the handle is yours; '
|
|
207
|
+
'it is needed to submit again under this handle in this round. We store only its hash.</p></div>') if "token" in out else ""
|
|
208
|
+
warns = "".join(f"<li>{e(w)}</li>" for w in out["warnings"])
|
|
209
|
+
body = f"""<section class="block"><p class="kicker">Received</p><h1>Submission accepted</h1></section>{token}
|
|
210
|
+
<section class="card"><dl class="facts"><div><dt>Handle</dt><dd>{e(handle)}</dd></div><div><dt>File sha256</dt><dd class="mono small brk">{e(out["sha256"])}</dd></div>
|
|
211
|
+
<div><dt>Log entry</dt><dd class="mono small">#{out["seq"]}, hash {e(out["hash"])}</dd></div></dl>
|
|
212
|
+
{('<h3>Warnings</h3><ul>' + warns + '</ul><p class="fine">Warnings are not rejections; they list things the sandbox will block or treat as evasion.</p>') if warns else ''}
|
|
213
|
+
<p><a class="btn" href="/s/{e(rid)}/{e(out["id"])}">Status page</a> <a class="btn" href="/round/{e(rid)}">Back to the round</a></p></section>"""
|
|
214
|
+
return layout(cfg, "Submission accepted", body)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# ------------------------------------------------------------------ http
|
|
218
|
+
class Server(ThreadingHTTPServer):
|
|
219
|
+
# Request threads are daemon threads and the server counts its OWN in-flight requests. server_close() waits for that counter to reach
|
|
220
|
+
# zero (bounded by drain_seconds, itself bounded by the 10 s socket timeout), so a restart during a submission cannot cut it between the
|
|
221
|
+
# log append and the metadata write. Not using the standard library's non-daemon thread list is deliberate: how it tracks and prunes
|
|
222
|
+
# finished threads has changed between Python versions, and a list that kept every finished thread would grow without bound.
|
|
223
|
+
daemon_threads = True
|
|
224
|
+
block_on_close = False
|
|
225
|
+
request_queue_size = 64
|
|
226
|
+
drain_seconds = 15
|
|
227
|
+
|
|
228
|
+
def __init__(self, addr, handler, max_conns):
|
|
229
|
+
# Set BEFORE super().__init__: if binding fails, TCPServer.__init__ calls server_close() from inside it, and server_close() uses
|
|
230
|
+
# these. (Setting them afterwards masked the real "address already in use" with an AttributeError.)
|
|
231
|
+
self._gate = threading.BoundedSemaphore(max_conns)
|
|
232
|
+
self._idle = threading.Condition()
|
|
233
|
+
self._active = 0
|
|
234
|
+
super().__init__(addr, handler)
|
|
235
|
+
|
|
236
|
+
def _finish(self):
|
|
237
|
+
self._gate.release()
|
|
238
|
+
with self._idle:
|
|
239
|
+
self._active -= 1
|
|
240
|
+
self._idle.notify_all()
|
|
241
|
+
|
|
242
|
+
def wait_idle(self, timeout):
|
|
243
|
+
"""True once no request is being handled, False if `timeout` seconds pass first."""
|
|
244
|
+
with self._idle:
|
|
245
|
+
return self._idle.wait_for(lambda: self._active == 0, timeout)
|
|
246
|
+
|
|
247
|
+
def process_request(self, request, client_address):
|
|
248
|
+
if not self._gate.acquire(blocking=False): # over the connection cap: refuse at once rather than queue threads (not counted)
|
|
249
|
+
try:
|
|
250
|
+
request.sendall(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nRetry-After: 5\r\nConnection: close\r\n\r\n")
|
|
251
|
+
except OSError:
|
|
252
|
+
pass
|
|
253
|
+
self.shutdown_request(request)
|
|
254
|
+
return
|
|
255
|
+
with self._idle:
|
|
256
|
+
self._active += 1
|
|
257
|
+
try:
|
|
258
|
+
super().process_request(request, client_address)
|
|
259
|
+
except BaseException:
|
|
260
|
+
self._finish() # the thread never started, so nothing else will release the slot
|
|
261
|
+
raise
|
|
262
|
+
|
|
263
|
+
def process_request_thread(self, request, client_address):
|
|
264
|
+
try:
|
|
265
|
+
super().process_request_thread(request, client_address)
|
|
266
|
+
finally:
|
|
267
|
+
self._finish()
|
|
268
|
+
|
|
269
|
+
def server_close(self):
|
|
270
|
+
super().server_close() # stop accepting first
|
|
271
|
+
self.wait_idle(self.drain_seconds) # then let the requests already in flight finish
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class Handler(BaseHTTPRequestHandler):
|
|
275
|
+
cfg = None
|
|
276
|
+
timeout = 10 # seconds a client may stall on any read
|
|
277
|
+
server_version = "ledger"
|
|
278
|
+
sys_version = ""
|
|
279
|
+
|
|
280
|
+
def log_message(self, *a):
|
|
281
|
+
pass # no access log: addresses are not recorded
|
|
282
|
+
|
|
283
|
+
# -- guards
|
|
284
|
+
def client_key(self):
|
|
285
|
+
ip = self.client_address[0]
|
|
286
|
+
if self.cfg.trust_proxy and ip in LOOPBACK:
|
|
287
|
+
xff = (self.headers.get("X-Forwarded-For") or "").split(",")
|
|
288
|
+
if xff and xff[-1].strip(): # the LAST hop is the one our own proxy appended; earlier entries are client-supplied
|
|
289
|
+
ip = xff[-1].strip()
|
|
290
|
+
return hashlib.sha256(self.cfg.ip_salt + ip.encode()).hexdigest()[:20]
|
|
291
|
+
|
|
292
|
+
def _host_ok(self):
|
|
293
|
+
host = (self.headers.get("Host") or "").lower()
|
|
294
|
+
port = self.server.server_address[1]
|
|
295
|
+
return host in self.cfg.allowed_hosts or host in (f"127.0.0.1:{port}", f"localhost:{port}")
|
|
296
|
+
|
|
297
|
+
def _origin_ok(self):
|
|
298
|
+
origin = (self.headers.get("Origin") or "").lower()
|
|
299
|
+
if not origin:
|
|
300
|
+
return self.headers.get("Sec-Fetch-Site") != "cross-site" # no ambient credentials exist, so non-browser clients are fine
|
|
301
|
+
port = self.server.server_address[1]
|
|
302
|
+
allowed = {f"https://{h}" for h in self.cfg.allowed_hosts} | {f"http://{h}" for h in self.cfg.allowed_hosts}
|
|
303
|
+
allowed |= {f"http://127.0.0.1:{port}", f"http://localhost:{port}"}
|
|
304
|
+
return origin in allowed
|
|
305
|
+
|
|
306
|
+
def _send(self, code, payload, ctype="text/html; charset=utf-8", cache="no-store", extra=()):
|
|
307
|
+
if isinstance(payload, str):
|
|
308
|
+
payload = payload.encode("utf-8")
|
|
309
|
+
self.send_response(code)
|
|
310
|
+
self.send_header("Content-Type", ctype)
|
|
311
|
+
self.send_header("Content-Length", str(len(payload)))
|
|
312
|
+
self.send_header("Cache-Control", cache)
|
|
313
|
+
self.send_header("X-Content-Type-Options", "nosniff")
|
|
314
|
+
self.send_header("X-Frame-Options", "DENY")
|
|
315
|
+
self.send_header("Referrer-Policy", "no-referrer")
|
|
316
|
+
self.send_header("Content-Security-Policy", "default-src 'none'; style-src 'self' https://fonts.googleapis.com; font-src https://fonts.gstatic.com; "
|
|
317
|
+
"script-src 'self'; img-src data:; form-action 'self'; base-uri 'none'; frame-ancestors 'none'")
|
|
318
|
+
if self.cfg.allowed_hosts: # a public deployment is served over HTTPS by the proxy; browsers ignore this header on plain HTTP
|
|
319
|
+
self.send_header("Strict-Transport-Security", "max-age=31536000")
|
|
320
|
+
for k, v in extra:
|
|
321
|
+
self.send_header(k, v)
|
|
322
|
+
self.end_headers()
|
|
323
|
+
if self.command != "HEAD":
|
|
324
|
+
self.wfile.write(payload)
|
|
325
|
+
|
|
326
|
+
def _text(self, code, msg, extra=()):
|
|
327
|
+
self._send(code, msg, "text/plain; charset=utf-8", extra=extra)
|
|
328
|
+
|
|
329
|
+
def send_error(self, code, message=None, explain=None):
|
|
330
|
+
"""Every protocol-level error (unsupported method, overlong URI, bad request line, bad version, too many headers) is answered here
|
|
331
|
+
by the stdlib's own send_error, which would bypass our security headers and return an HTML page that reflects the request.
|
|
332
|
+
Answer them with the same hardened plain-text response as everything else, and never reflect anything the client sent."""
|
|
333
|
+
if self.request_version == "HTTP/0.9": # a request line that did not parse: the stdlib would send a bare body with no headers
|
|
334
|
+
self.request_version = "HTTP/1.0"
|
|
335
|
+
self.close_connection = True
|
|
336
|
+
try:
|
|
337
|
+
extra = (("Allow", "GET, HEAD, POST"),) if code in (405, 501) else ()
|
|
338
|
+
self._text(code, f"{code} {self.responses.get(code, ('error',))[0]}", extra=extra)
|
|
339
|
+
except OSError:
|
|
340
|
+
pass
|
|
341
|
+
|
|
342
|
+
def _guard(self, limiter):
|
|
343
|
+
if not self._host_ok():
|
|
344
|
+
self._text(403, "forbidden: unexpected Host header")
|
|
345
|
+
return False
|
|
346
|
+
if not limiter.allow(self.client_key()):
|
|
347
|
+
self._text(429, "too many requests", extra=(("Retry-After", "60"),))
|
|
348
|
+
return False
|
|
349
|
+
return True
|
|
350
|
+
|
|
351
|
+
# -- routes
|
|
352
|
+
def do_HEAD(self):
|
|
353
|
+
self.do_GET()
|
|
354
|
+
|
|
355
|
+
def do_GET(self):
|
|
356
|
+
path = self.path.split("?")[0]
|
|
357
|
+
if path == "/healthz":
|
|
358
|
+
return self._text(200, "ok")
|
|
359
|
+
if not self._guard(self.cfg.get_limiter):
|
|
360
|
+
return
|
|
361
|
+
try:
|
|
362
|
+
rounds.close_due(self.cfg.base, self.cfg.clock())
|
|
363
|
+
if path == "/":
|
|
364
|
+
return self._send(200, page_rounds(self.cfg))
|
|
365
|
+
if path == "/portal-assets/site.css": # NOT /assets/: that path belongs to the static site on a single-domain deployment
|
|
366
|
+
return self._send(200, site_assets.CSS + EXTRA_CSS, "text/css; charset=utf-8", cache="public, max-age=300")
|
|
367
|
+
if path == "/portal-assets/site.js":
|
|
368
|
+
return self._send(200, site_assets.JS, "text/javascript; charset=utf-8", cache="public, max-age=300")
|
|
369
|
+
if path == "/api/rounds.json":
|
|
370
|
+
return self._send(200, json.dumps([rounds.public_round(self.cfg.base, r["id"]) for r in rounds.list_rounds(self.cfg.base)]), "application/json")
|
|
371
|
+
m = re.fullmatch(r"/round/([a-z0-9-]+)", path)
|
|
372
|
+
if m:
|
|
373
|
+
return self._send(200, page_round(self.cfg, m.group(1)))
|
|
374
|
+
m = re.fullmatch(r"/s/([a-z0-9-]+)/([0-9a-f]{12})", path)
|
|
375
|
+
if m:
|
|
376
|
+
return self._send(200, page_submission(self.cfg, m.group(1), m.group(2)))
|
|
377
|
+
m = re.fullmatch(r"/api/round/([a-z0-9-]+)\.json", path)
|
|
378
|
+
if m:
|
|
379
|
+
return self._send(200, json.dumps(rounds.public_round(self.cfg.base, m.group(1))), "application/json")
|
|
380
|
+
m = re.fullmatch(r"/api/log/([a-z0-9-]+)\.jsonl", path)
|
|
381
|
+
if m:
|
|
382
|
+
with open(os.path.join(rounds.rdir(self.cfg.base, m.group(1)), "log.jsonl"), "rb") as f:
|
|
383
|
+
return self._send(200, f.read(), "application/x-ndjson")
|
|
384
|
+
m = re.fullmatch(r"/api/cert(-full)?/([a-z0-9-]+)/([0-9a-f]{12})\.json", path)
|
|
385
|
+
if m:
|
|
386
|
+
full = bool(m.group(1))
|
|
387
|
+
token = None
|
|
388
|
+
if full: # only the FULL certificate (it carries the solver's source) needs the owner's token; the public one never reads the header
|
|
389
|
+
auth = self.headers.get("Authorization") or ""
|
|
390
|
+
token = auth[7:].strip() if auth.lower().startswith("bearer ") else None # a header, never the URL: URLs end up in logs and referrers
|
|
391
|
+
try:
|
|
392
|
+
cert = rounds.cert_for_download(self.cfg.base, m.group(2), m.group(3), token, full)
|
|
393
|
+
except rounds.Rejected as ex:
|
|
394
|
+
return self._send(403, json.dumps({"ok": False, "error": str(ex)}), "application/json")
|
|
395
|
+
with open(cert, "rb") as f:
|
|
396
|
+
return self._send(200, f.read(), "application/json")
|
|
397
|
+
except (KeyError, OSError):
|
|
398
|
+
pass
|
|
399
|
+
except Exception: # a bug must be a clean 500 (nothing echoed), not a dropped connection; the service keeps running
|
|
400
|
+
return self._text(500, "internal error")
|
|
401
|
+
self._text(404, "not found")
|
|
402
|
+
|
|
403
|
+
def do_POST(self):
|
|
404
|
+
m = re.fullmatch(r"/round/([a-z0-9-]+)/submit", self.path.split("?")[0])
|
|
405
|
+
if not self._host_ok():
|
|
406
|
+
return self._text(403, "forbidden: unexpected Host header")
|
|
407
|
+
if not self._origin_ok():
|
|
408
|
+
return self._text(403, "forbidden: cross-site request")
|
|
409
|
+
if not m:
|
|
410
|
+
return self._text(404, "not found")
|
|
411
|
+
rid = m.group(1)
|
|
412
|
+
if self.cfg.killed():
|
|
413
|
+
return self._text(503, "submissions are paused", extra=(("Retry-After", "3600"),))
|
|
414
|
+
if not (self.cfg.submit_limiter.allow(self.client_key()) and self.cfg.global_submit_limiter.allow("global")):
|
|
415
|
+
return self._text(429, "too many submissions; try again later", extra=(("Retry-After", "3600"),))
|
|
416
|
+
raw_len = self.headers.get("Content-Length") or ""
|
|
417
|
+
if not re.fullmatch(r"[0-9]{1,9}", raw_len):
|
|
418
|
+
return self._text(400, "invalid Content-Length")
|
|
419
|
+
n = int(raw_len)
|
|
420
|
+
if n > rounds.MAX_UPLOAD + 20_000:
|
|
421
|
+
return self._text(413, "that upload is too large")
|
|
422
|
+
try:
|
|
423
|
+
body = self.rfile.read(n)
|
|
424
|
+
except OSError: # includes socket.timeout: a stalled client is dropped, not waited for
|
|
425
|
+
self.close_connection = True
|
|
426
|
+
return
|
|
427
|
+
if len(body) < n:
|
|
428
|
+
return self._text(400, "the upload ended early")
|
|
429
|
+
try:
|
|
430
|
+
form = portal_core.parse_multipart(self.headers.get("Content-Type", ""), body)
|
|
431
|
+
except Exception:
|
|
432
|
+
return self._text(400, "could not read the form")
|
|
433
|
+
if "file" not in form or "handle" not in form:
|
|
434
|
+
return self._text(400, "missing handle or file")
|
|
435
|
+
as_json = "application/json" in (self.headers.get("Accept") or "") # command-line clients (svp.py) ask for JSON; browsers get the pages
|
|
436
|
+
if "agree" not in form:
|
|
437
|
+
if as_json:
|
|
438
|
+
return self._send(422, json.dumps({"ok": False, "error": "you need to accept the rules to submit"}), "application/json")
|
|
439
|
+
return self._send(422, page_round(self.cfg, rid, "You need to accept the rules to submit."))
|
|
440
|
+
handle = form["handle"][1].decode("utf-8", "replace").strip()
|
|
441
|
+
token = form.get("token", (None, b""))[1].decode("utf-8", "replace").strip()
|
|
442
|
+
try:
|
|
443
|
+
out = rounds.submit(self.cfg.base, rid, handle, token, form["file"][1], publish_source="publish_source" in form, now=self.cfg.clock())
|
|
444
|
+
except rounds.Rejected as ex:
|
|
445
|
+
if as_json:
|
|
446
|
+
return self._send(422, json.dumps({"ok": False, "error": str(ex)}), "application/json")
|
|
447
|
+
return self._send(422, page_round(self.cfg, rid, f"Rejected: {ex}"))
|
|
448
|
+
except KeyError:
|
|
449
|
+
return self._text(404, "no such round")
|
|
450
|
+
except Exception: # disk full, memory, a bug: a clean 500 with nothing echoed; rounds.submit has already removed any partial file
|
|
451
|
+
return self._text(500, "internal error")
|
|
452
|
+
if as_json:
|
|
453
|
+
return self._send(200, json.dumps({"ok": True, "handle": handle, **out}), "application/json")
|
|
454
|
+
self._send(200, page_receipt(self.cfg, rid, out, handle))
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def make_server(cfg, bind="127.0.0.1", port=0):
|
|
458
|
+
handler = type("H", (Handler,), {"cfg": cfg})
|
|
459
|
+
return Server((bind, port), handler, cfg.max_conns)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def main():
|
|
463
|
+
rounds.require_python()
|
|
464
|
+
ap = argparse.ArgumentParser()
|
|
465
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
466
|
+
p = sub.add_parser("serve")
|
|
467
|
+
p.add_argument("--data", required=True)
|
|
468
|
+
p.add_argument("--host", action="append", default=[], help="public host name(s) this service answers to (repeat); loopback always works")
|
|
469
|
+
p.add_argument("--bind", default="127.0.0.1", help="leave as 127.0.0.1 and put a TLS reverse proxy in front")
|
|
470
|
+
p.add_argument("--port", type=int, default=8080)
|
|
471
|
+
p.add_argument("--trust-proxy", action="store_true", help="read the client address from X-Forwarded-For (only when bound to loopback behind your proxy)")
|
|
472
|
+
p.add_argument("--site-url", default="", help="public URL of the static site, for the 'Main site' link")
|
|
473
|
+
p.add_argument("--max-conns", type=int, default=64)
|
|
474
|
+
a = ap.parse_args()
|
|
475
|
+
if a.trust_proxy and a.bind not in LOOPBACK:
|
|
476
|
+
raise SystemExit("REFUSED: --trust-proxy lets any client choose its address; use it only when bound to loopback behind your own proxy")
|
|
477
|
+
os.umask(0o007) # the operator (same group) must be able to write what this process creates, and others must not read it
|
|
478
|
+
cfg = Config(a.data, a.host, a.trust_proxy, a.max_conns, a.site_url)
|
|
479
|
+
try:
|
|
480
|
+
srv = make_server(cfg, a.bind, a.port)
|
|
481
|
+
except OSError as e: # port taken, no permission, bad address: a sentence, not a traceback
|
|
482
|
+
raise SystemExit(f"REFUSED: cannot listen on {a.bind}:{a.port} ({e.strerror or e}). Is another copy already running?")
|
|
483
|
+
print(f"public portal on {a.bind}:{srv.server_address[1]} (data {cfg.base}; hosts {sorted(cfg.allowed_hosts) or 'loopback only'}; "
|
|
484
|
+
f"evaluation happens elsewhere: ops.py worker)", flush=True)
|
|
485
|
+
|
|
486
|
+
def stop(signum, frame):
|
|
487
|
+
# shutdown() waits for serve_forever() to return, and serve_forever() runs in THIS thread (which is executing this handler),
|
|
488
|
+
# so it must be called from another thread or it would deadlock.
|
|
489
|
+
threading.Thread(target=srv.shutdown, daemon=True).start()
|
|
490
|
+
signal.signal(signal.SIGTERM, stop) # systemctl stop/restart
|
|
491
|
+
signal.signal(signal.SIGINT, stop)
|
|
492
|
+
try:
|
|
493
|
+
srv.serve_forever()
|
|
494
|
+
finally:
|
|
495
|
+
srv.server_close() # stops accepting, then WAITS for the handler threads still running: an in-flight submission is completed, not cut
|
|
496
|
+
print("public portal stopped cleanly", flush=True)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
if __name__ == "__main__":
|
|
500
|
+
main()
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Quality-metric statistics for a challenge (standard library only). The compiler imports this file; an exported bundle carries a byte copy as checker/quality_core.py.
|
|
2
|
+
|
|
3
|
+
A QUALITY challenge scores the ANSWER, not the clock. Every instance is solved under the same CPU cap; the declaration's quality(inst, ans) says how good a VALID answer is,
|
|
4
|
+
from 0 (worst) to 1 (best); an instance the solver does not answer validly counts as 0. The credit of a solver over the reference is the paired difference of qualities:
|
|
5
|
+
|
|
6
|
+
per size, the mean over instances of q_solver - q_reference; the overall estimate is the mean over sizes (equal weight per size);
|
|
7
|
+
one-sided 95% lower bound by a stratified bootstrap (instances resampled with replacement WITHIN each size);
|
|
8
|
+
credit of one seed = max(0, lower bound); credit over several seeds = the MINIMUM (the speed protocol's convention).
|
|
9
|
+
|
|
10
|
+
Units are quality points (0.05 = five points of the 0..1 scale). A credit above the challenge's min_gain is "certified"; below it is "nothing certified", not "no gain".
|
|
11
|
+
What this does NOT know: how the answers were produced or whether quality() measures what you care about. It scores the numbers it is given.
|
|
12
|
+
"""
|
|
13
|
+
import math
|
|
14
|
+
import random
|
|
15
|
+
|
|
16
|
+
ALPHA = 0.05 # one-sided: the lower bound is the 5th percentile of the bootstrap distribution
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def clip01(x):
|
|
20
|
+
return 0.0 if x < 0.0 else 1.0 if x > 1.0 else x
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def strata_diffs(base, sub):
|
|
24
|
+
"""{size: [q_sub - q_base, ...]} (ids in sorted order) over the ids both ladders hold. `base`, `sub`: {size: {id: quality}}."""
|
|
25
|
+
out = {}
|
|
26
|
+
for d in sorted(base):
|
|
27
|
+
ids = sorted(set(base[d]) & set(sub.get(d, {})))
|
|
28
|
+
if ids:
|
|
29
|
+
out[d] = [sub[d][i] - base[d][i] for i in ids]
|
|
30
|
+
return out
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def estimate(strata):
|
|
34
|
+
return sum(sum(v) / len(v) for v in strata.values()) / len(strata)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def lower_bound(strata, rng, boot, alpha=ALPHA):
|
|
38
|
+
"""One-sided (1 - alpha) lower confidence bound of estimate(): stratified bootstrap, boot resamples."""
|
|
39
|
+
dims = sorted(strata)
|
|
40
|
+
ests = []
|
|
41
|
+
for _ in range(boot):
|
|
42
|
+
tot = 0.0
|
|
43
|
+
for d in dims:
|
|
44
|
+
v = strata[d]
|
|
45
|
+
tot += sum(rng.choices(v, k=len(v))) / len(v)
|
|
46
|
+
ests.append(tot / len(dims))
|
|
47
|
+
ests.sort()
|
|
48
|
+
return ests[min(len(ests) - 1, int(alpha * len(ests)))]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def pair_noise(strata):
|
|
52
|
+
"""Pooled within-size sd of the paired differences: what the power table calls the per-instance noise. None if fewer than 8 residuals."""
|
|
53
|
+
res = []
|
|
54
|
+
for v in strata.values():
|
|
55
|
+
if len(v) >= 2:
|
|
56
|
+
m = sum(v) / len(v)
|
|
57
|
+
res += [x - m for x in v]
|
|
58
|
+
dof = len(res) - len(strata)
|
|
59
|
+
if len(res) < 8 or dof < 1:
|
|
60
|
+
return None
|
|
61
|
+
return math.sqrt(sum(x * x for x in res) / dof)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def seed_credit(base, sub, rng, boot):
|
|
65
|
+
"""-> dict for one seed's pair of ladders. credit_quality = max(0, lower bound); None estimates when no instance is shared."""
|
|
66
|
+
strata = strata_diffs(base, sub)
|
|
67
|
+
if not strata:
|
|
68
|
+
return {"estimate": None, "lower": None, "credit_quality": None, "instances": 0, "pair_noise": None, "improved": None, "worse": None}
|
|
69
|
+
flat = [x for v in strata.values() for x in v]
|
|
70
|
+
lo = lower_bound(strata, rng, boot)
|
|
71
|
+
return {"estimate": estimate(strata), "lower": lo, "credit_quality": max(0.0, lo), "instances": len(flat), "pair_noise": pair_noise(strata),
|
|
72
|
+
"improved": sum(x > 1e-12 for x in flat), "worse": sum(x < -1e-12 for x in flat)}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def combine(credits):
|
|
76
|
+
"""Credit over seeds: the minimum, an unscorable seed counting as 0."""
|
|
77
|
+
return min(0.0 if c is None else c for c in credits)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def headroom(base):
|
|
81
|
+
"""Per size: the baseline's mean quality and the fraction of instances already at (>= 0.999) the top. A size with no headroom cannot show a gain."""
|
|
82
|
+
out = {}
|
|
83
|
+
for d in sorted(base):
|
|
84
|
+
qs = list(base[d].values())
|
|
85
|
+
out[d] = {"mean": sum(qs) / len(qs), "at_top": sum(q >= 0.999 for q in qs) / len(qs)}
|
|
86
|
+
return out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ------------------------------------------------------------------------------------------------------------------------------ power
|
|
90
|
+
def synth_ladders(profile, n, effect, noise, rng):
|
|
91
|
+
"""A synthetic (baseline, solver) pair of ladders. profile = {"sizes": [..], "mean": {size: mu}, "sd": {size: s}} describes the reference's quality;
|
|
92
|
+
the solver is `effect` better per instance plus N(0, noise), clipped to [0, 1] like real qualities (so a gain near the top is truncated: headroom is modelled)."""
|
|
93
|
+
base, sub = {}, {}
|
|
94
|
+
for s in profile["sizes"]:
|
|
95
|
+
mu, sd = profile["mean"][str(s)], profile["sd"][str(s)]
|
|
96
|
+
base[s], sub[s] = {}, {}
|
|
97
|
+
for i in range(n):
|
|
98
|
+
b = clip01(rng.gauss(mu, sd))
|
|
99
|
+
base[s][i] = b
|
|
100
|
+
sub[s][i] = clip01(b + effect + rng.gauss(0, noise))
|
|
101
|
+
return base, sub
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def power_at(profile, n, effect, noise, gap, seeds, trials, boot, rng):
|
|
105
|
+
"""P(one seed credited >= gap) and P(all `seeds` seeds credited >= gap) for a true per-instance gain of `effect`."""
|
|
106
|
+
credits = []
|
|
107
|
+
for _ in range(seeds * trials):
|
|
108
|
+
b, s = synth_ladders(profile, n, effect, noise, rng)
|
|
109
|
+
credits.append(seed_credit(b, s, rng, boot)["credit_quality"])
|
|
110
|
+
trip = [combine(credits[i:i + seeds]) for i in range(0, len(credits) - seeds + 1, seeds)]
|
|
111
|
+
return {"single": sum(c >= gap for c in credits) / len(credits), "all_seeds": sum(c >= gap for c in trip) / len(trip)}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def power_table(profile, n, noise, gap, seeds, trials, boot, effects, seed=0):
|
|
115
|
+
rng = random.Random(seed)
|
|
116
|
+
return {e: power_at(profile, n, e, noise, gap, seeds, trials, boot, rng) for e in effects}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def certifiable_range(table, target):
|
|
120
|
+
ok = [e for e, v in table.items() if v["all_seeds"] >= target]
|
|
121
|
+
return (min(ok), max(ok)) if ok else (None, None)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def effects_for(gap):
|
|
125
|
+
"""The grid of true gains a power table covers, as multiples of the certification threshold."""
|
|
126
|
+
return tuple(round(gap * m, 4) for m in (0, 0.5, 1, 1.5, 2, 3, 5, 10))
|