guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/counted.py
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Deterministic step counting: a cost that EVERYONE re-running gets exactly, instead of seconds that depend on a machine and a sandbox.
|
|
2
|
+
|
|
3
|
+
counted.py evaluate --family F --seed-file S --submission X.py --dims 38 40 42 [--n 12] [--cap 20] [--baseline B.py] [--results-dir R]
|
|
4
|
+
counted.py check SOLVER.py --family F --dim 16 run one instance twice: are the two counts identical on this interpreter?
|
|
5
|
+
|
|
6
|
+
The cost of an instance is the number of Python bytecode instructions the solver executes (count_runner.py: sys.monitoring, Python 3.12+), divided by
|
|
7
|
+
STEPS_PER_SECOND to give "counted seconds", so the existing estimator, certificates and site read these ladders unchanged. Two programs that execute
|
|
8
|
+
the same instructions on the same input get the same number on any machine, so a verifier does not have to trust the operator's hardware, sandbox or
|
|
9
|
+
clock: it re-executes and compares, and equality is the proof (certificate.py verify --rerun K --execute).
|
|
10
|
+
|
|
11
|
+
WHAT THIS PROVES, AND WHAT IT DOES NOT. It proves the recorded cost of each re-run instance is exactly what that program executes, for this Python
|
|
12
|
+
version. It does NOT measure time, and it is blind to work done inside a C function: `pow(a, b, m)` on huge integers, `sorted`, `sum`, numpy and
|
|
13
|
+
fpylll each count as ONE instruction however long they run. A solver can therefore move its work into a C call and look free, so a counted result is
|
|
14
|
+
meaningful only when the work is in Python code on both sides, and it is not a comparison of an algorithm that leans on C builtins against one that
|
|
15
|
+
does not. The ladder (the sizes, how many instances, the cap) is chosen by whoever ran it: a counted certificate is not an official round result.
|
|
16
|
+
Counting is also slow (a callback per instruction, roughly 100x), so counted ladders use small sizes and a small cap. The solver runs in a restricted subset of Python
|
|
17
|
+
that count_runner.py enforces (pure computation; see its docstring and COMPILER.md), which protects the counter's integrity but is an in-process sandbox, not operating-system
|
|
18
|
+
isolation: for code you did not write, still use a throwaway user or VM.
|
|
19
|
+
"""
|
|
20
|
+
import argparse
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
import tempfile
|
|
28
|
+
import time
|
|
29
|
+
|
|
30
|
+
import count_runner # (its check_source is the restricted-subset rule; importing it defines functions only, main() runs only as a script)
|
|
31
|
+
import families
|
|
32
|
+
import instances
|
|
33
|
+
|
|
34
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
35
|
+
RUNNER = os.path.join(HERE, "count_runner.py")
|
|
36
|
+
STEPS_PER_SECOND = 1_000_000 # 1 "counted second" = one million bytecode instructions. It is only a UNIT, not a duration: it is chosen so that the estimator's
|
|
37
|
+
# 0.5 s cost floor (a size counts only if its typical cost reaches it) is 500,000 instructions, which counting can reach. With
|
|
38
|
+
# 5e7 the floor would be 25 million instructions per solver and no counted ladder could ever be conclusive.
|
|
39
|
+
SLOW_STEPS_PER_SECOND = 300_000 # a PESSIMISTIC counting throughput: used only for the wall-clock backstop and for cost estimates
|
|
40
|
+
MIN_PYTHON = (3, 12)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def supported():
|
|
44
|
+
return sys.version_info >= MIN_PYTHON
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def python_version():
|
|
48
|
+
"""Identifies the interpreter whose counts a certificate records: instruction counts are only comparable on the same one."""
|
|
49
|
+
v = sys.version_info
|
|
50
|
+
return f"{sys.implementation.name} {v.major}.{v.minor}.{v.micro}"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def count_instance(solver, inst, cap_seconds):
|
|
54
|
+
"""Run `solver` on `inst` under exact counting. -> a ladder record: id, dim, status (solved / failed / wrong / crash / timeout), steps, cpu, cpu_net, wall.
|
|
55
|
+
A timeout is a solver that used more than cap_seconds * STEPS_PER_SECOND instructions (deterministic), or, rarely, hit the wall-clock backstop."""
|
|
56
|
+
limit = int(cap_seconds * STEPS_PER_SECOND)
|
|
57
|
+
rec = {"id": inst["id"], "dim": inst["dim"]}
|
|
58
|
+
payload = json.dumps({"args": inst["args"], "limit": limit})
|
|
59
|
+
env = {"PYTHONHASHSEED": "0", "PYTHONDONTWRITEBYTECODE": "1", "PATH": os.environ.get("PATH", "")} # no environment variable of this machine reaches the solver (the count still depends on the Python version and on the solver being deterministic)
|
|
60
|
+
t0 = time.time()
|
|
61
|
+
with tempfile.TemporaryDirectory() as empty:
|
|
62
|
+
try:
|
|
63
|
+
r = subprocess.run([sys.executable, "-B", RUNNER, os.path.abspath(solver)], input=payload.encode(), capture_output=True, env=env, cwd=empty,
|
|
64
|
+
timeout=limit / SLOW_STEPS_PER_SECOND + 60)
|
|
65
|
+
except subprocess.TimeoutExpired:
|
|
66
|
+
rec.update({"status": "timeout", "steps": None, "wall_timeout": True, "cpu": cap_seconds, "cpu_net": cap_seconds, "wall": time.time() - t0})
|
|
67
|
+
return rec
|
|
68
|
+
rec["wall"] = time.time() - t0
|
|
69
|
+
lines = [ln for ln in r.stdout.decode(errors="replace").splitlines() if ln.strip()]
|
|
70
|
+
try:
|
|
71
|
+
out = json.loads(lines[-1])
|
|
72
|
+
steps = int(out["steps"])
|
|
73
|
+
status = out["status"]
|
|
74
|
+
except (IndexError, ValueError, KeyError, TypeError):
|
|
75
|
+
rec.update({"status": "crash", "steps": None, "cpu": cap_seconds, "cpu_net": cap_seconds,
|
|
76
|
+
"err": ("the counter produced no result: " + r.stderr.decode(errors="replace")[-300:])})
|
|
77
|
+
return rec
|
|
78
|
+
rec["steps"] = steps
|
|
79
|
+
if status == "limit":
|
|
80
|
+
rec.update({"status": "timeout", "cpu": cap_seconds, "cpu_net": cap_seconds})
|
|
81
|
+
elif status == "error":
|
|
82
|
+
rec.update({"status": "crash", "cpu": cap_seconds, "cpu_net": cap_seconds, "err": str(out.get("err", ""))[-1500:]})
|
|
83
|
+
else:
|
|
84
|
+
ans = out.get("ans")
|
|
85
|
+
try:
|
|
86
|
+
ok = families.verify(inst, ans)
|
|
87
|
+
except Exception:
|
|
88
|
+
ok = False
|
|
89
|
+
rec["status"] = "solved" if ok else ("failed" if ans is None else "wrong")
|
|
90
|
+
rec["cpu"] = steps / STEPS_PER_SECOND
|
|
91
|
+
rec["cpu_net"] = max(rec["cpu"], 1e-3) # the same floor the harness applies; a unit conversion, not a measurement
|
|
92
|
+
return rec
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def counter_sha256():
|
|
96
|
+
"""Hash of the counting program. It is not one of the round protocol's hashed files, so every counted ladder and certificate records it: counts made by a
|
|
97
|
+
different (or doctored) counter are then named as such, and a later fix to the counter cannot silently change what an old certificate means."""
|
|
98
|
+
with open(RUNNER, "rb") as f:
|
|
99
|
+
return hashlib.sha256(f.read()).hexdigest()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def ladder_header(solver, family, master, cap):
|
|
103
|
+
with open(solver, "rb") as f:
|
|
104
|
+
sha = hashlib.sha256(f.read()).hexdigest()
|
|
105
|
+
return {"meta": True, "solver": os.path.basename(solver), "commit": instances.commit(master), "overhead": 0.0, "budget": cap, "family": family,
|
|
106
|
+
"sandbox": "count", "instance_version": families.VERSIONS[family], "image": None, "host": "deterministic step counting (no machine timing)",
|
|
107
|
+
"cpu": "none: instruction counts", "ncpu": 1, "jobs": 1, "solver_sha256": sha,
|
|
108
|
+
"cost_model": "count", "steps_per_second": STEPS_PER_SECOND, "python": python_version(), "counter_sha256": counter_sha256()}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def evaluate_ladder(solver, family, master, dims, n, cap, out_path, progress=None):
|
|
112
|
+
"""One ladder file: a header, then the counted record of every (dim, idx). Never overwrites an existing file."""
|
|
113
|
+
if os.path.exists(out_path):
|
|
114
|
+
raise FileExistsError(out_path)
|
|
115
|
+
recs = []
|
|
116
|
+
fd = os.open(out_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
117
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
118
|
+
f.write(json.dumps(ladder_header(solver, family, master, cap)) + "\n")
|
|
119
|
+
for dim in dims:
|
|
120
|
+
for idx in range(n):
|
|
121
|
+
rec = count_instance(solver, families.derive(family, master, dim, idx), cap)
|
|
122
|
+
f.write(json.dumps(rec) + "\n")
|
|
123
|
+
f.flush()
|
|
124
|
+
recs.append(rec)
|
|
125
|
+
if progress:
|
|
126
|
+
progress(os.path.basename(solver), rec)
|
|
127
|
+
return recs
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def c_extension_imports(path):
|
|
131
|
+
"""Libraries whose work happens inside C and so counts as ONE instruction however long it runs: a counted run that leans on them says almost nothing."""
|
|
132
|
+
try:
|
|
133
|
+
with open(path, encoding="utf-8", errors="replace") as f:
|
|
134
|
+
text = f.read()
|
|
135
|
+
except OSError:
|
|
136
|
+
return []
|
|
137
|
+
return sorted(set(re.findall(r"^\s*(?:import|from)\s+(numpy|scipy|fpylll|gmpy2|sympy|cython|cysignals)\b", text, re.M)))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def source_problems(text):
|
|
141
|
+
"""-> [problems]: why this source cannot be run by the counter, or [] if it is inside the restricted subset a counted solver is written in (count_runner.check_source:
|
|
142
|
+
an AST whitelist, no dunder/private/frame attributes, no getattr/type/open/eval/exec, imports only from a few pure modules). The counter enforces the same rules when it
|
|
143
|
+
loads the solver, so this is the SAME check, made early: a counted round refuses such a source at submission, and `practice --count` before it spends any time. Because the
|
|
144
|
+
rules live in count_runner.py they are pinned by the counter hash."""
|
|
145
|
+
return count_runner.check_source(text)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def family_problem(family):
|
|
149
|
+
"""None if `family` can be counted, otherwise why not. Its baseline must be pure Python: a credit is a ratio against the baseline, and work hidden in C is free."""
|
|
150
|
+
import judge # (lazily: judge imports the harness)
|
|
151
|
+
if family not in families.FAMILIES:
|
|
152
|
+
return f"unknown family {family!r}"
|
|
153
|
+
base = judge.default_baseline(family)
|
|
154
|
+
ext = c_extension_imports(base)
|
|
155
|
+
if ext:
|
|
156
|
+
return (f"the default baseline for {family} imports {', '.join(ext)}, so its work happens inside C and counts as almost nothing: a counted credit against it would be "
|
|
157
|
+
"meaningless (step counting needs a pure-Python baseline)")
|
|
158
|
+
try:
|
|
159
|
+
with open(base, encoding="utf-8") as f:
|
|
160
|
+
problems = source_problems(f.read())
|
|
161
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
162
|
+
return f"the default baseline for {family} cannot be read ({type(e).__name__})"
|
|
163
|
+
if problems: # the counter runs the baseline in the same restricted subset as every submission
|
|
164
|
+
return f"the default baseline for {family} is outside the restricted subset the counter runs solvers in: " + "; ".join(problems[:3])
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def baseline_reusable(path, solver, family, master, dims, n, cap):
|
|
169
|
+
"""(ok, why). A baseline ladder already on disk may be reused only if it is exactly what this evaluation would write: the same header (seed commitment, baseline
|
|
170
|
+
file hash, family, cap, instance version, unit, interpreter and counter) and a record for every (size, index). Counting is deterministic, so a complete
|
|
171
|
+
identical ladder is the same measurement."""
|
|
172
|
+
try:
|
|
173
|
+
with open(path, encoding="utf-8") as f:
|
|
174
|
+
lines = [ln for ln in f.read().splitlines() if ln.strip()]
|
|
175
|
+
meta = json.loads(lines[0])
|
|
176
|
+
recs = [json.loads(ln) for ln in lines[1:]]
|
|
177
|
+
except (OSError, ValueError, IndexError):
|
|
178
|
+
return False, "no readable ladder"
|
|
179
|
+
if meta != ladder_header(solver, family, master, cap):
|
|
180
|
+
return False, "its header differs (another seed, baseline file, cap, unit, Python or counter)"
|
|
181
|
+
want = {f"{d}:{i}" for d in dims for i in range(n)}
|
|
182
|
+
have = [r.get("id") for r in recs]
|
|
183
|
+
if len(have) != len(set(have)) or set(have) != want:
|
|
184
|
+
return False, "incomplete or with unexpected instances"
|
|
185
|
+
return True, ""
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def evaluate_seed(family, seed_file, submission, baseline, dims, n, cap, results_dir, progress=None, reuse_baseline=False):
|
|
189
|
+
"""The baseline ladder and the submission's ladder for ONE seed, written where judge.py writes its own (so certificate.py finds them).
|
|
190
|
+
reuse_baseline: a round counts many submissions against ONE baseline, so an identical complete baseline ladder already on disk is kept (anything else that
|
|
191
|
+
is already there is refused, never overwritten). -> (seed tag, baseline ladder path, submission ladder path)."""
|
|
192
|
+
with open(seed_file, "rb") as f:
|
|
193
|
+
master = f.read().strip()
|
|
194
|
+
tag = hashlib.sha256(master).hexdigest()[:8]
|
|
195
|
+
paths = []
|
|
196
|
+
for role, solver in (("baseline", baseline), ("submission", submission)):
|
|
197
|
+
p = os.path.join(results_dir, f"judged_{family}_{tag}_{role}_{os.path.basename(solver)}.jsonl")
|
|
198
|
+
if role == "baseline" and reuse_baseline and os.path.exists(p):
|
|
199
|
+
ok, why = baseline_reusable(p, solver, family, master, dims, n, cap)
|
|
200
|
+
if not ok:
|
|
201
|
+
raise FileExistsError(f"{p} exists but is not the complete, identical baseline ladder ({why}): move it aside and run again")
|
|
202
|
+
paths.append(p)
|
|
203
|
+
continue
|
|
204
|
+
evaluate_ladder(solver, family, master, dims, n, cap, p, progress)
|
|
205
|
+
paths.append(p)
|
|
206
|
+
return tag, paths[0], paths[1]
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def selfcheck(solver, family, dim, cap, master=instances.PRACTICE_SEED, idx=0):
|
|
210
|
+
"""Run one instance twice. -> (identical?, first record, second record). A solver or interpreter that is not deterministic cannot be certified."""
|
|
211
|
+
inst = families.derive(family, master, dim, idx)
|
|
212
|
+
a, b = count_instance(solver, inst, cap), count_instance(solver, inst, cap)
|
|
213
|
+
return (a.get("steps") == b.get("steps") and a.get("status") == b.get("status")), a, b
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def selfcheck_problem(label, same, a, b):
|
|
217
|
+
"""None when the two runs agree AND solved the question; otherwise the reason this solver cannot be counted. Two identical crashes or timeouts are
|
|
218
|
+
'deterministic' but useless: a whole run of them would be charged the cap on every question and certify nothing."""
|
|
219
|
+
if not same:
|
|
220
|
+
return f"{label} is not deterministic under step counting ({a.get('steps')} vs {b.get('steps')} steps on the same input): it cannot be certified this way"
|
|
221
|
+
if a.get("status") != "solved":
|
|
222
|
+
why = {"timeout": "it used more than the whole budget (raise --cap or choose a smaller size)", "crash": "it crashed: " + str(a.get("err", ""))[-200:],
|
|
223
|
+
"failed": "it returned no answer", "wrong": "it returned a wrong answer"}.get(a.get("status"), a.get("status"))
|
|
224
|
+
return f"{label} did not solve the smallest question: {why}"
|
|
225
|
+
return None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def worst_case_seconds(dims, n, cap, seeds=3):
|
|
229
|
+
"""A pessimistic wall-clock bound for counting a whole practice run (every instance spending its whole cap at the slow throughput)."""
|
|
230
|
+
return len(dims) * n * 2 * seeds * cap * STEPS_PER_SECOND / SLOW_STEPS_PER_SECOND
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def main(argv=None):
|
|
234
|
+
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
235
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
236
|
+
p = sub.add_parser("evaluate")
|
|
237
|
+
p.add_argument("--family", required=True)
|
|
238
|
+
p.add_argument("--seed-file", required=True)
|
|
239
|
+
p.add_argument("--submission", required=True)
|
|
240
|
+
p.add_argument("--baseline")
|
|
241
|
+
p.add_argument("--dims", type=int, nargs="+", required=True)
|
|
242
|
+
p.add_argument("--n", type=int, default=12)
|
|
243
|
+
p.add_argument("--cap", type=float, default=20.0, help="the most counted seconds (x 1e6 instructions) one instance may use; to be usable at all a size must "
|
|
244
|
+
"cost the solver at least 0.5 (the estimator's floor), so the baseline needs a few counted seconds")
|
|
245
|
+
p.add_argument("--results-dir", default=os.path.join(HERE, "results"))
|
|
246
|
+
p = sub.add_parser("check")
|
|
247
|
+
p.add_argument("solver")
|
|
248
|
+
p.add_argument("--family", required=True)
|
|
249
|
+
p.add_argument("--dim", type=int, required=True)
|
|
250
|
+
p.add_argument("--cap", type=float, default=20.0)
|
|
251
|
+
a = ap.parse_args(argv)
|
|
252
|
+
if not supported():
|
|
253
|
+
raise SystemExit(f"REFUSED: step counting needs Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]} or newer (this is {python_version()})")
|
|
254
|
+
if a.family not in families.FAMILIES:
|
|
255
|
+
raise SystemExit(f"REFUSED: unknown family {a.family!r}")
|
|
256
|
+
if a.cmd == "check":
|
|
257
|
+
same, x, y = selfcheck(a.solver, a.family, a.dim, a.cap)
|
|
258
|
+
problem = selfcheck_problem("this solver", same, x, y)
|
|
259
|
+
print(f"run 1: {x['status']} {x.get('steps')} steps\nrun 2: {y['status']} {y.get('steps')} steps\n"
|
|
260
|
+
+ ("DETERMINISTIC, and it solved the question" if problem is None else "NOT USABLE: " + problem))
|
|
261
|
+
return 0 if problem is None else 1
|
|
262
|
+
import certificate # (lazily: certificate imports this module)
|
|
263
|
+
if a.n < 1 or not (0 < a.cap <= certificate.MAX_COUNTED_CAP) or not a.dims or min(a.dims) < 1:
|
|
264
|
+
raise SystemExit(f"REFUSED: --n and --dims must be positive and --cap a number between 0 and {certificate.MAX_COUNTED_CAP:g}")
|
|
265
|
+
import judge
|
|
266
|
+
baseline = a.baseline or judge.default_baseline(a.family)
|
|
267
|
+
tag, bp, sp = evaluate_seed(a.family, a.seed_file, a.submission, baseline, a.dims, a.n, a.cap, a.results_dir,
|
|
268
|
+
lambda name, rec: print(f"{name:22s} {rec['id']:>7s} {rec['status']:8s} {rec.get('steps')}", flush=True))
|
|
269
|
+
print(f"wrote {bp}\nwrote {sp}")
|
|
270
|
+
return 0
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
if __name__ == "__main__":
|
|
274
|
+
sys.exit(main())
|
guldo/_src/declared.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Declared domains: how a compiled problem declaration becomes a registered family (see COMPILER.md).
|
|
2
|
+
|
|
3
|
+
A declaration is a directory domains/<name>/ holding
|
|
4
|
+
decl.py the author's file: NAME, VERSION, SIZE_RANGE, REFERENCE_SOLVER, derive(master, size, idx), verify(inst, ans)
|
|
5
|
+
reference.py (any file name; REFERENCE_SOLVER) the baseline every submission is measured against
|
|
6
|
+
compiled.json written ONLY by compile_domain.py: the calibrated ladder, cap, official n, power analysis and self-test result
|
|
7
|
+
|
|
8
|
+
families.py calls load_all() when it is imported, so every process (harness, judge, portal, worker) sees the same registry. A domain is
|
|
9
|
+
registered only if compiled.json is FRESH: its recorded hashes of decl.py and the reference solver must equal the files on disk. Editing
|
|
10
|
+
either file un-registers the domain until it is compiled again, because a stale ladder or cap was calibrated to different instances.
|
|
11
|
+
|
|
12
|
+
decl.py is OPERATOR code and runs in the judge's process, exactly like instances.py does. Submissions never touch it: they run in the
|
|
13
|
+
sandbox with only runner.py and their own file.
|
|
14
|
+
"""
|
|
15
|
+
import hashlib
|
|
16
|
+
import importlib.util
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
|
|
21
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
22
|
+
NAME_RE = re.compile(r"^[a-z]{2,16}$") # letters only: judge.py rescore and report_html recognise ladder files by this pattern
|
|
23
|
+
FORMAT = "svp-compiled-v1"
|
|
24
|
+
COMPILED = "compiled.json"
|
|
25
|
+
DECL = "decl.py"
|
|
26
|
+
REQUIRED = ("NAME", "VERSION", "SIZE_RANGE", "REFERENCE_SOLVER", "derive", "verify")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DeclarationError(Exception):
|
|
30
|
+
"""A declaration or its compiled record is unusable; the message is safe to show to its author."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def domains_dir():
|
|
34
|
+
return os.environ.get("SVP_DOMAINS_DIR") or os.path.join(HERE, "domains")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def sha256_file(path):
|
|
38
|
+
with open(path, "rb") as f:
|
|
39
|
+
return hashlib.sha256(f.read()).hexdigest()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def load_declaration(dirpath):
|
|
43
|
+
"""Import decl.py from `dirpath` and check its shape (not its behaviour: that is compile_domain's validate pass)."""
|
|
44
|
+
dirpath = os.path.abspath(dirpath)
|
|
45
|
+
path = os.path.join(dirpath, DECL)
|
|
46
|
+
if not os.path.isfile(path):
|
|
47
|
+
raise DeclarationError(f"{dirpath} has no {DECL}")
|
|
48
|
+
spec = importlib.util.spec_from_file_location(f"svp_decl_{sha256_file(path)[:16]}", path)
|
|
49
|
+
mod = importlib.util.module_from_spec(spec)
|
|
50
|
+
try:
|
|
51
|
+
spec.loader.exec_module(mod)
|
|
52
|
+
except Exception as e: # the author's own code: report it, never crash the registry
|
|
53
|
+
raise DeclarationError(f"{DECL} failed to import: {type(e).__name__}: {e}")
|
|
54
|
+
missing = [k for k in REQUIRED if not hasattr(mod, k)]
|
|
55
|
+
if missing:
|
|
56
|
+
raise DeclarationError(f"{DECL} does not define {', '.join(missing)}")
|
|
57
|
+
if not isinstance(mod.NAME, str) or not NAME_RE.match(mod.NAME):
|
|
58
|
+
raise DeclarationError("NAME must be 2-16 lowercase letters a-z (no digits or hyphens)")
|
|
59
|
+
if mod.NAME != os.path.basename(dirpath):
|
|
60
|
+
raise DeclarationError(f"NAME {mod.NAME!r} must equal the directory name {os.path.basename(dirpath)!r}")
|
|
61
|
+
if not isinstance(mod.VERSION, str) or not mod.VERSION.strip():
|
|
62
|
+
raise DeclarationError("VERSION must be a non-empty string; bump it whenever derive() changes")
|
|
63
|
+
sr = mod.SIZE_RANGE
|
|
64
|
+
if not (isinstance(sr, (tuple, list)) and len(sr) == 2 and all(isinstance(x, int) and not isinstance(x, bool) for x in sr)
|
|
65
|
+
and 1 <= sr[0] < sr[1]):
|
|
66
|
+
raise DeclarationError("SIZE_RANGE must be (lo, hi): two integers with 1 <= lo < hi")
|
|
67
|
+
step = getattr(mod, "SIZE_STEP", 1)
|
|
68
|
+
if not (isinstance(step, int) and not isinstance(step, bool) and step >= 1):
|
|
69
|
+
raise DeclarationError("SIZE_STEP, if given, must be a positive integer")
|
|
70
|
+
if not callable(mod.derive) or not callable(mod.verify):
|
|
71
|
+
raise DeclarationError("derive and verify must be functions")
|
|
72
|
+
return mod
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def reference_path(dirpath, mod):
|
|
76
|
+
"""Absolute path of the declared reference solver. It must be a file INSIDE the declaration directory."""
|
|
77
|
+
dirpath = os.path.realpath(dirpath)
|
|
78
|
+
rel = mod.REFERENCE_SOLVER
|
|
79
|
+
if not isinstance(rel, str) or not rel or os.path.isabs(rel):
|
|
80
|
+
raise DeclarationError("REFERENCE_SOLVER must be a relative path inside the declaration directory")
|
|
81
|
+
p = os.path.realpath(os.path.join(dirpath, rel))
|
|
82
|
+
if not p.startswith(dirpath + os.sep):
|
|
83
|
+
raise DeclarationError("REFERENCE_SOLVER must stay inside the declaration directory")
|
|
84
|
+
if not os.path.isfile(p):
|
|
85
|
+
raise DeclarationError(f"REFERENCE_SOLVER {rel!r} does not exist")
|
|
86
|
+
return p
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def read_compiled(dirpath):
|
|
90
|
+
try:
|
|
91
|
+
with open(os.path.join(dirpath, COMPILED), encoding="utf-8") as f:
|
|
92
|
+
c = json.load(f)
|
|
93
|
+
return c if isinstance(c, dict) else None
|
|
94
|
+
except (OSError, ValueError):
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def freshness_problems(dirpath, mod, compiled):
|
|
99
|
+
"""Why `compiled` cannot be trusted for the declaration on disk (empty list = fresh)."""
|
|
100
|
+
if compiled is None:
|
|
101
|
+
return ["not compiled (no readable compiled.json)"]
|
|
102
|
+
out = []
|
|
103
|
+
if compiled.get("format") != FORMAT:
|
|
104
|
+
out.append(f"compiled.json has format {compiled.get('format')!r}, expected {FORMAT!r}")
|
|
105
|
+
if compiled.get("name") != mod.NAME:
|
|
106
|
+
out.append("compiled.json is for a different NAME")
|
|
107
|
+
if compiled.get("version") != mod.VERSION:
|
|
108
|
+
out.append(f"VERSION changed ({compiled.get('version')!r} -> {mod.VERSION!r})")
|
|
109
|
+
if compiled.get("decl_sha256") != sha256_file(os.path.join(dirpath, DECL)):
|
|
110
|
+
out.append("decl.py changed since it was compiled")
|
|
111
|
+
try:
|
|
112
|
+
if compiled.get("reference_sha256") != sha256_file(reference_path(dirpath, mod)):
|
|
113
|
+
out.append("the reference solver changed since it was compiled")
|
|
114
|
+
except DeclarationError as e:
|
|
115
|
+
out.append(str(e))
|
|
116
|
+
for k in ("dims", "practice_dims", "official"):
|
|
117
|
+
if k not in compiled:
|
|
118
|
+
out.append(f"compiled.json lacks {k!r}")
|
|
119
|
+
# the SHAPE of what register() will index: a hand-edited or damaged record must be refused here, not crash every process that imports families
|
|
120
|
+
isint = lambda v: isinstance(v, int) and not isinstance(v, bool)
|
|
121
|
+
off = compiled.get("official")
|
|
122
|
+
if "official" in compiled and not (isinstance(off, dict) and isint(off.get("n")) and off["n"] >= 1
|
|
123
|
+
and isinstance(off.get("budget"), (int, float)) and not isinstance(off.get("budget"), bool) and off["budget"] > 0):
|
|
124
|
+
out.append("compiled.json: 'official' must be {n: positive int, budget: positive number}")
|
|
125
|
+
for k in ("dims", "practice_dims"):
|
|
126
|
+
v = compiled.get(k)
|
|
127
|
+
if k in compiled and not (isinstance(v, list) and v and all(isint(x) for x in v)):
|
|
128
|
+
out.append(f"compiled.json: {k!r} must be a non-empty list of integers")
|
|
129
|
+
return out
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def make_spec(mod):
|
|
133
|
+
"""(derive, verify) in the form families.FAMILIES expects. derive adds id/dim/idx so a declaration need not."""
|
|
134
|
+
def derive(master, size, idx):
|
|
135
|
+
inst = dict(mod.derive(master, size, idx))
|
|
136
|
+
inst.setdefault("id", f"{size}:{idx}")
|
|
137
|
+
inst.setdefault("dim", size)
|
|
138
|
+
inst.setdefault("idx", idx)
|
|
139
|
+
return inst
|
|
140
|
+
return derive, mod.verify
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def register(fam, dirpath, mod, compiled):
|
|
144
|
+
"""Fill the six tables of the `families` module for this domain. Raises DeclarationError on a name clash."""
|
|
145
|
+
name = mod.NAME
|
|
146
|
+
if name in fam.FAMILIES and name not in fam.DECLARED:
|
|
147
|
+
raise DeclarationError(f"{name!r} is a built-in family; choose another NAME")
|
|
148
|
+
real_here = os.path.realpath(fam.HERE)
|
|
149
|
+
ref = os.path.relpath(reference_path(dirpath, mod), real_here) if os.path.realpath(dirpath).startswith(real_here + os.sep) \
|
|
150
|
+
else reference_path(dirpath, mod) # outside the project (tests): judge.py joins with HERE, and join(HERE, absolute) is the absolute path
|
|
151
|
+
# compute everything first, then assign: a bad record must leave the registry untouched, never half a domain
|
|
152
|
+
spec, official = make_spec(mod), {"n": int(compiled["official"]["n"]), "budget": compiled["official"]["budget"]}
|
|
153
|
+
dims, practice = [int(d) for d in compiled["dims"]], [int(d) for d in compiled["practice_dims"]]
|
|
154
|
+
info = {"dir": os.path.abspath(dirpath), "compiled": compiled, "decl_sha256": compiled["decl_sha256"]}
|
|
155
|
+
fam.FAMILIES[name] = spec
|
|
156
|
+
fam.VERSIONS[name] = mod.VERSION
|
|
157
|
+
fam.OFFICIAL[name] = official
|
|
158
|
+
fam.DEFAULT_DIMS[name] = dims
|
|
159
|
+
fam.DEFAULT_BASELINE[name] = ref
|
|
160
|
+
fam.PRACTICE_DIMS[name] = practice
|
|
161
|
+
fam.DECLARED[name] = info
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
BUILTIN = ("gm", "mlwe", "factor")
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def unregister(fam, name):
|
|
168
|
+
if name in BUILTIN: # never remove a built-in family, whatever calls this
|
|
169
|
+
return
|
|
170
|
+
for table in (fam.FAMILIES, fam.VERSIONS, fam.OFFICIAL, fam.DEFAULT_DIMS, fam.DEFAULT_BASELINE, fam.PRACTICE_DIMS, fam.DECLARED):
|
|
171
|
+
table.pop(name, None)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def load_all(fam, root=None):
|
|
175
|
+
"""Register every fresh compiled declaration under `root` (default domains/). Anything unusable is recorded in fam.DECLARED_ERRORS,
|
|
176
|
+
never raised: one broken declaration must not stop the judge, the portal or the other domains from starting."""
|
|
177
|
+
root = os.path.abspath(root or domains_dir())
|
|
178
|
+
loaded = []
|
|
179
|
+
try:
|
|
180
|
+
names = sorted(os.listdir(root))
|
|
181
|
+
except OSError:
|
|
182
|
+
return loaded
|
|
183
|
+
for entry in names:
|
|
184
|
+
d = os.path.join(root, entry)
|
|
185
|
+
if not os.path.isfile(os.path.join(d, DECL)):
|
|
186
|
+
continue
|
|
187
|
+
try:
|
|
188
|
+
mod = load_declaration(d)
|
|
189
|
+
problems = freshness_problems(d, mod, read_compiled(d))
|
|
190
|
+
if problems:
|
|
191
|
+
raise DeclarationError("; ".join(problems) + " (run compile_domain.py compile again)")
|
|
192
|
+
if (read_compiled(d) or {}).get("kind") == "quality":
|
|
193
|
+
# the judge, harness and portal score by SPEED: registering a quality challenge would rank answers by the clock. Its product is the bundle.
|
|
194
|
+
raise DeclarationError("quality challenge: not a speed family, so no judge/portal rounds; use `compile_domain.py export` for its bundle")
|
|
195
|
+
register(fam, d, mod, read_compiled(d))
|
|
196
|
+
fam.DECLARED_ERRORS.pop(entry, None)
|
|
197
|
+
loaded.append(mod.NAME)
|
|
198
|
+
except DeclarationError as e:
|
|
199
|
+
fam.DECLARED_ERRORS[entry] = str(e)
|
|
200
|
+
except Exception as e: # a damaged compiled.json or odd declaration must never stop `import families` (the judge, portal and worker all import it)
|
|
201
|
+
fam.DECLARED_ERRORS[entry] = f"unusable: {type(e).__name__}: {e}"
|
|
202
|
+
return loaded
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def domain_files(fam, name):
|
|
206
|
+
"""{repo-relative or absolute path: sha256} of the files a declared domain's rules rest on. rounds.code_fingerprint records them in
|
|
207
|
+
entry 0 of a round's log, so a declaration or its compiled ladder cannot change under a running round unnoticed."""
|
|
208
|
+
info = fam.DECLARED.get(name)
|
|
209
|
+
if not info:
|
|
210
|
+
return {}
|
|
211
|
+
d = info["dir"]
|
|
212
|
+
mod = load_declaration(d)
|
|
213
|
+
files = [os.path.join(d, DECL), os.path.join(d, COMPILED), reference_path(d, mod)]
|
|
214
|
+
real_here = os.path.realpath(fam.HERE)
|
|
215
|
+
return {(os.path.relpath(p, real_here) if os.path.realpath(p).startswith(real_here + os.sep) else p): sha256_file(p) for p in files}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# TLS-terminating reverse proxy for the PUBLIC submission service. Replace ledger.example.org.
|
|
2
|
+
# Caddy obtains and renews the certificate itself. The Python service binds to 127.0.0.1:8080 only.
|
|
3
|
+
#
|
|
4
|
+
# Two settings matter for safety:
|
|
5
|
+
# * X-Forwarded-For is SET to the real client address by header_up (it overwrites whatever the client sent). public_portal.py
|
|
6
|
+
# (run with --trust-proxy) rate-limits on that value, so a client must not be able to choose it.
|
|
7
|
+
# * the request body is capped well above MAX_UPLOAD (200 KB) so oversized uploads are refused at the proxy.
|
|
8
|
+
|
|
9
|
+
ledger.example.org {
|
|
10
|
+
encode gzip
|
|
11
|
+
|
|
12
|
+
request_body {
|
|
13
|
+
max_size 300KB
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
# the static site (python3 build_site.py --out /srv/ledger/site ...) is served from disk, INCLUDING its own /assets/site.css;
|
|
17
|
+
# only /round, /s, /api and /portal-assets (the service's own stylesheet, a different path on purpose) go to the Python service
|
|
18
|
+
@service path /round/* /s/* /api/* /portal-assets/* /healthz
|
|
19
|
+
handle @service {
|
|
20
|
+
reverse_proxy 127.0.0.1:8080 {
|
|
21
|
+
header_up X-Forwarded-For {remote_host}
|
|
22
|
+
transport http {
|
|
23
|
+
dial_timeout 3s
|
|
24
|
+
response_header_timeout 30s
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
handle {
|
|
30
|
+
root * /srv/ledger/site
|
|
31
|
+
# the static pages carry one inline theme script and a few inline style attributes, hence 'unsafe-inline'; nothing here handles user input
|
|
32
|
+
header X-Frame-Options "DENY"
|
|
33
|
+
header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src https://fonts.gstatic.com; script-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'"
|
|
34
|
+
file_server
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
header {
|
|
38
|
+
Strict-Transport-Security "max-age=31536000"
|
|
39
|
+
X-Content-Type-Options "nosniff"
|
|
40
|
+
Referrer-Policy "no-referrer"
|
|
41
|
+
-Server
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
# Any other Host name pointed at this machine (DNS rebinding, a stray domain) is refused; without this block Caddy answers an empty 200.
|
|
46
|
+
:443 {
|
|
47
|
+
respond "forbidden" 403
|
|
48
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Back up and verify the competition state (logs, submissions, certificates and the secret seeds). Pair with ledger-backup.timer. Runs as root: the archive holds the seeds.
|
|
2
|
+
[Unit]
|
|
3
|
+
Description=Back up and verify the Guldo ledger (data and seeds)
|
|
4
|
+
|
|
5
|
+
[Service]
|
|
6
|
+
Type=oneshot
|
|
7
|
+
ExecStart=/bin/bash /opt/speedup-verification/deploy/ledger-backup.sh
|
|
8
|
+
Nice=10
|
|
9
|
+
IOSchedulingClass=idle
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Back up the competition's state (the data directory with every round's log, submissions and certificates, AND the secret seeds) and VERIFY the archive:
|
|
3
|
+
# every round's hash chain must verify and every seed must hash to a commitment published in its round's log. A seed that is lost or wrong cannot be fixed later:
|
|
4
|
+
# the round could then never be evaluated or revealed. Run as root; the archive is readable by root only because it contains the seeds.
|
|
5
|
+
#
|
|
6
|
+
# bash deploy/ledger-backup.sh (the systemd timer runs it daily; see deploy/ledger-backup.timer)
|
|
7
|
+
# Environment: LEDGER_ROOT (default /srv/ledger), BACKUP_DIR (default /root/backups), KEEP (archives to keep, default 14), CODE_DIR (default /opt/speedup-verification).
|
|
8
|
+
# Exit status is non-zero if the archive could not be made or does not verify, so a failing run shows up in `systemctl --failed`.
|
|
9
|
+
# This is a copy on the SAME machine: it survives a mistake, not the loss of the disk. Copy BACKUP_DIR off the machine too (EVAL_BOX.md, "Backups").
|
|
10
|
+
set -euo pipefail
|
|
11
|
+
|
|
12
|
+
ROOT="${LEDGER_ROOT:-/srv/ledger}"
|
|
13
|
+
OUT="${BACKUP_DIR:-/root/backups}"
|
|
14
|
+
KEEP="${KEEP:-14}"
|
|
15
|
+
CODE="${CODE_DIR:-/opt/speedup-verification}"
|
|
16
|
+
[ -d "$ROOT/data" ] || { echo "no $ROOT/data: nothing to back up" >&2; exit 1; }
|
|
17
|
+
|
|
18
|
+
umask 077
|
|
19
|
+
mkdir -p "$OUT"
|
|
20
|
+
name="$(basename "$ROOT")"
|
|
21
|
+
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
22
|
+
file="$OUT/ledger-$stamp.tgz"
|
|
23
|
+
|
|
24
|
+
# tar exits 1 for the harmless "file changed as we read it" (a submission arrived while it ran); the archive is verified below either way, so only a real failure (2+) stops here
|
|
25
|
+
rc=0
|
|
26
|
+
tar czf "$file" -C "$(dirname "$ROOT")" --exclude="$name/site" --exclude="$name/site.new" --exclude="$name/site.old" --exclude="$name/rehearsal" "$name" || rc=$?
|
|
27
|
+
[ "$rc" -le 1 ] || { echo "tar failed with status $rc" >&2; rm -f "$file"; exit 1; }
|
|
28
|
+
if command -v sha256sum >/dev/null; then SUM="sha256sum"; else SUM="shasum -a 256"; fi
|
|
29
|
+
( cd "$OUT" && $SUM "$(basename "$file")" > "$(basename "$file").sha256" )
|
|
30
|
+
|
|
31
|
+
tmp="$(mktemp -d)"
|
|
32
|
+
trap 'rm -rf "$tmp"' EXIT
|
|
33
|
+
tar xzf "$file" -C "$tmp"
|
|
34
|
+
status=0
|
|
35
|
+
CODE_DIR="$CODE" python3 - "$tmp/$name" <<'PY' || status=$?
|
|
36
|
+
import hashlib, json, os, sys
|
|
37
|
+
sys.path.insert(0, os.environ["CODE_DIR"])
|
|
38
|
+
import rounds
|
|
39
|
+
|
|
40
|
+
root = sys.argv[1]
|
|
41
|
+
base, seeds = os.path.join(root, "data"), os.path.join(root, "seeds")
|
|
42
|
+
bad = 0
|
|
43
|
+
rs = rounds.list_rounds(base)
|
|
44
|
+
for r in rs:
|
|
45
|
+
ok, n, head, err = rounds.verify_log(rounds.rdir(base, r["id"]), r["commitments"])
|
|
46
|
+
print(f"round {r['id']}: log {'OK' if ok else 'FAILED: ' + str(err)} ({n} entries)")
|
|
47
|
+
bad += 0 if ok else 1
|
|
48
|
+
sd = os.path.join(seeds, r["id"])
|
|
49
|
+
if os.path.isdir(sd):
|
|
50
|
+
found = 0
|
|
51
|
+
for fn in sorted(os.listdir(sd)):
|
|
52
|
+
if fn.endswith(".seed"):
|
|
53
|
+
with open(os.path.join(sd, fn), "rb") as f:
|
|
54
|
+
good = hashlib.sha256(f.read()).hexdigest() in r["commitments"]
|
|
55
|
+
found += 1
|
|
56
|
+
if not good:
|
|
57
|
+
print(f" seed {fn}: does NOT match any published commitment")
|
|
58
|
+
bad += 1
|
|
59
|
+
print(f" {found} seed file(s), each checked against the commitments")
|
|
60
|
+
if found != len(r["commitments"]) and r["state"] != "revealed":
|
|
61
|
+
print(f" WARNING: {found} seeds for {len(r['commitments'])} commitments")
|
|
62
|
+
bad += 1
|
|
63
|
+
sys.exit(1 if bad else 0)
|
|
64
|
+
PY
|
|
65
|
+
|
|
66
|
+
# keep the newest KEEP archives (and their checksums)
|
|
67
|
+
ls -1t "$OUT"/ledger-*.tgz 2>/dev/null | tail -n +"$((KEEP + 1))" | while read -r old; do rm -f "$old" "$old.sha256"; done
|
|
68
|
+
|
|
69
|
+
if [ "$status" -ne 0 ]; then
|
|
70
|
+
echo "BACKUP WRITTEN BUT DOES NOT VERIFY: $file" >&2
|
|
71
|
+
exit 1
|
|
72
|
+
fi
|
|
73
|
+
echo "backup verified: $file ($(du -h "$file" | cut -f1))"
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Rebuild the static site (and export the rounds' public data) from the saved results. Run as the OPERATOR user.
|
|
2
|
+
# Pair with ledger-export.timer. Never run while a ladder is being written: build_site.py only READS results/, but a
|
|
3
|
+
# half-written ladder would be shown as "not chartable".
|
|
4
|
+
[Unit]
|
|
5
|
+
Description=Rebuild the Speedup Ledger static site
|
|
6
|
+
|
|
7
|
+
[Service]
|
|
8
|
+
Type=oneshot
|
|
9
|
+
User=ledger-ops
|
|
10
|
+
WorkingDirectory=/opt/speedup-verification
|
|
11
|
+
ExecStart=/usr/bin/python3 build_site.py --out /srv/ledger/site.new --ledger-data /srv/ledger/data --portal-url https://ledger.example.org
|
|
12
|
+
ExecStartPost=/bin/sh -c 'rm -rf /srv/ledger/site.old && { [ -d /srv/ledger/site ] && mv /srv/ledger/site /srv/ledger/site.old; }; mv /srv/ledger/site.new /srv/ledger/site'
|