guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,828 @@
|
|
|
1
|
+
"""Speedup certificates: one JSON file that proves a measured speedup to someone who does not trust the operator (see COMPILER.md).
|
|
2
|
+
|
|
3
|
+
certificate.py build --family F --submission solvers/x.py --seed-dir S [--reveal-seeds] [--round-data D --round-id R] --out x.cert.json
|
|
4
|
+
certificate.py verify x.cert.json [--rerun K]
|
|
5
|
+
|
|
6
|
+
A certificate carries: the claim; the rules fingerprint (hashes of the estimator, harness, instance code and, for a declared domain, its
|
|
7
|
+
declaration, compiled ladder and reference solver); the seed commitments and (only with --reveal-seeds, i.e. after the reveal) the seeds;
|
|
8
|
+
both solvers' source; every raw per-instance record of every ladder; and the round's hash-chained log. `verify` needs nothing from the
|
|
9
|
+
operator. It checks, and reports each as PASS / FAIL / SKIP:
|
|
10
|
+
|
|
11
|
+
rules this checkout's protocol code hashes to the certificate's (a different estimator would score the same data differently)
|
|
12
|
+
domain a declared domain's files hash to what the certificate names, and load
|
|
13
|
+
log the hash chain verifies, entry 0 holds the commitments and the rules fingerprint, the submission's hash was logged
|
|
14
|
+
seeds each revealed seed hashes to a committed value
|
|
15
|
+
sources the solver sources are the ones the ladders were measured with (sha256 in every ladder header)
|
|
16
|
+
coverage re-derives every instance from the seeds; each ladder holds exactly those instances, for the same size set
|
|
17
|
+
credit recomputes every seed's credit from the raw records with this checkout's estimator and compares it with the claim
|
|
18
|
+
rerun (--rerun K, needs --execute) re-measures K instances on THIS machine: the solver must still solve them, and the speedup should reproduce
|
|
19
|
+
|
|
20
|
+
A certificate is untrusted input, so by default `verify` EXECUTES NOTHING from it: a declared domain's decl.py is hash-checked and read as data
|
|
21
|
+
(instance coverage of a declared domain is then SKIPPED). `--execute` imports that decl.py and, with --rerun, runs the solver sources, on this
|
|
22
|
+
machine, unsandboxed: use it only in a throwaway VM or container. Built-in families derive instances with this checkout's own code, no flag needed.
|
|
23
|
+
|
|
24
|
+
A certificate is as strong as its weakest link: without a log it can never be OFFICIAL (the commitment's publication cannot be shown);
|
|
25
|
+
a ladder measured in local mode is PROVISIONAL (only the Docker sandbox meters untrusted code); the verifier says which.
|
|
26
|
+
|
|
27
|
+
PRIVATE-SOLVER certificates. `make_private` (or `build --private`) removes the solver's source and every failure message from its ladders (a
|
|
28
|
+
traceback quotes the code); everything else is unchanged, including the solver's SHA-256, which the round's log committed to before the seeds were
|
|
29
|
+
revealed. A verifier can still check the log, the fingerprints, the questions, the rules and the credit arithmetic. What it cannot do is re-run the
|
|
30
|
+
solver or inspect it: the timings rest on the operator's sandbox, and the certificate says so. If the owner later publishes the file,
|
|
31
|
+
`verify --source FILE` checks that it hashes to the committed value and, with --rerun --execute, re-measures it: the certificate is then as strong
|
|
32
|
+
as a full one. A private-solver certificate hides the algorithm from the PUBLIC, not from the operator, who ran it. (Not to be confused with a
|
|
33
|
+
"sealed" certificate: one built before the reveal, without the seeds.)
|
|
34
|
+
"""
|
|
35
|
+
import argparse
|
|
36
|
+
import contextlib
|
|
37
|
+
import hashlib
|
|
38
|
+
import json
|
|
39
|
+
import math
|
|
40
|
+
import os
|
|
41
|
+
import random
|
|
42
|
+
import re
|
|
43
|
+
import secrets
|
|
44
|
+
import sys
|
|
45
|
+
import tempfile
|
|
46
|
+
import time
|
|
47
|
+
|
|
48
|
+
import declared
|
|
49
|
+
import extrapolate as E
|
|
50
|
+
import families
|
|
51
|
+
import harness
|
|
52
|
+
import judge
|
|
53
|
+
import official
|
|
54
|
+
import rounds
|
|
55
|
+
|
|
56
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
57
|
+
FORMAT = "svp-certificate-v1"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class CertificateError(Exception):
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def sha256(data):
|
|
65
|
+
return hashlib.sha256(data if isinstance(data, bytes) else data.encode()).hexdigest()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def canon(obj):
|
|
69
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _read(path):
|
|
73
|
+
# newline="": NO translation. The harness hashes the file's raw bytes (solver_sha256 in every ladder header), so a certificate that stores the text must
|
|
74
|
+
# give back the same bytes, or a solver saved with Windows line endings would fail its own `sources` check.
|
|
75
|
+
with open(path, encoding="utf-8", newline="") as f:
|
|
76
|
+
return f.read()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def rules_of(family):
|
|
80
|
+
"""The protocol-code part of the fingerprint (declared-domain files are carried separately, by role)."""
|
|
81
|
+
fp = rounds.code_fingerprint(family)
|
|
82
|
+
return {"dims": fp["dims"], "instance_version": fp["instance_version"],
|
|
83
|
+
"code": {rel: fp["code"][rel] for rel in rounds.CODE_FILES}}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def commit_published(log_entries, commitments, round_id_hint=None):
|
|
87
|
+
"""(ok, why): the log's hash chain verifies and entry 0 contains every commitment. Verified in a temporary directory named for the round."""
|
|
88
|
+
if not log_entries:
|
|
89
|
+
return False, "no round log included"
|
|
90
|
+
rid = (log_entries[0].get("data") or {}).get("id")
|
|
91
|
+
if not isinstance(rid, str) or not rounds.ROUND_ID_RE.match(rid):
|
|
92
|
+
return False, "the first log entry has no valid round id"
|
|
93
|
+
with tempfile.TemporaryDirectory() as td:
|
|
94
|
+
d = os.path.join(td, rid)
|
|
95
|
+
os.makedirs(d)
|
|
96
|
+
with open(os.path.join(d, "log.jsonl"), "w", encoding="utf-8") as f:
|
|
97
|
+
for e in log_entries:
|
|
98
|
+
f.write(rounds.canonical(e) + "\n")
|
|
99
|
+
ok, n, head, err = rounds.verify_log(d)
|
|
100
|
+
if not ok:
|
|
101
|
+
return False, f"the log does not verify: {err}"
|
|
102
|
+
if not set(commitments) <= set(log_entries[0]["data"].get("commitments", [])):
|
|
103
|
+
return False, "a seed commitment is not in entry 0 of the log"
|
|
104
|
+
return True, f"{n} entries, head {head[:16]}"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def score_seed(family, name, tag, published, base_text, sub_text, work):
|
|
108
|
+
"""The board entry for one seed, computed exactly as judge.py computes it: extrapolate.py on the two ladder files, then make_entry."""
|
|
109
|
+
import subprocess
|
|
110
|
+
bp, sp = os.path.join(work, f"{tag}_baseline.jsonl"), os.path.join(work, f"{tag}_submission.jsonl")
|
|
111
|
+
with open(bp, "w", encoding="utf-8") as f:
|
|
112
|
+
f.write(base_text)
|
|
113
|
+
with open(sp, "w", encoding="utf-8") as f:
|
|
114
|
+
f.write(sub_text)
|
|
115
|
+
r = subprocess.run([sys.executable, os.path.join(HERE, "extrapolate.py"), "--baseline", bp, "--submission", sp], cwd=HERE, capture_output=True, text=True)
|
|
116
|
+
if r.returncode:
|
|
117
|
+
raise CertificateError(f"seed {tag}: the estimator refused these ladders: {(r.stdout.strip() or r.stderr.strip())[-300:]}")
|
|
118
|
+
with open(sp + ".claim.json", encoding="utf-8") as f:
|
|
119
|
+
claim = json.load(f)
|
|
120
|
+
return judge.make_entry(name, family, tag, sp, claim, 512, published, base_file=bp)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def claim_of(entries):
|
|
124
|
+
rows = judge.leaderboard_rows(sorted(entries, key=lambda e: e["seed_tag"])) # per_seed lists follow entry order: fix it, or the same data compares unequal
|
|
125
|
+
if len(rows) != 1:
|
|
126
|
+
raise CertificateError("the entries do not describe exactly one submission")
|
|
127
|
+
r = rows[0]
|
|
128
|
+
status, reasons = r["status"], r["reasons"]
|
|
129
|
+
if all(e.get("sandbox") == "count" for e in entries):
|
|
130
|
+
status, reasons = _counted_status(entries) # judge.py's OFFICIAL means "Docker sandbox, timed": a counted ladder is held to its own rules
|
|
131
|
+
return {"credit_bits": r["credit"], "status": status, "reasons": reasons, "per_seed": r["per_seed"]}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _counted_status(entries):
|
|
135
|
+
"""-> (status, reasons) for entries made by step counting. COUNTED = at least judge.MIN_SEEDS committed seeds whose commitments were shown to be public
|
|
136
|
+
first, one solver file and one baseline file throughout, hashes and instance version recorded. Anything less is COUNTED, PROVISIONAL, with the reasons."""
|
|
137
|
+
why = []
|
|
138
|
+
if len(entries) < judge.MIN_SEEDS:
|
|
139
|
+
why.append(f"{len(entries)} seed(s), fewer than {judge.MIN_SEEDS}")
|
|
140
|
+
if not all(e.get("commit_checked") for e in entries):
|
|
141
|
+
why.append("seed commitment not checked against a published value")
|
|
142
|
+
for key, label in (("solver_sha256", "solver file hash"), ("baseline_sha256", "baseline file hash"), ("instance_version", "instance version")):
|
|
143
|
+
if not all(e.get(key) for e in entries):
|
|
144
|
+
why.append(f"no {label} recorded")
|
|
145
|
+
if len({e.get("solver_sha256") for e in entries}) > 1:
|
|
146
|
+
why.append("the seeds used different versions of the submission file")
|
|
147
|
+
if len({e.get("baseline_sha256") for e in entries}) > 1:
|
|
148
|
+
why.append("the seeds used different baseline files")
|
|
149
|
+
return ("COUNTED" if not why else "COUNTED, PROVISIONAL"), why
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# ------------------------------------------------------------------------------------------------------------- build
|
|
153
|
+
def _counted_info(ladders):
|
|
154
|
+
"""None for ordinary (timed) ladders. For ladders made by counted.py: {steps_per_second, python, dims, n, cap}. A mix of counted and timed ladders, or
|
|
155
|
+
counted ladders that disagree about the unit, the interpreter, the cap or the sizes, is refused: they would not be comparable."""
|
|
156
|
+
heads, shapes = [], set()
|
|
157
|
+
for tag, lad in sorted(ladders.items()):
|
|
158
|
+
for role in ("baseline", "submission"):
|
|
159
|
+
lines = [ln for ln in lad[role].splitlines() if ln.strip()]
|
|
160
|
+
heads.append(json.loads(lines[0]))
|
|
161
|
+
recs = [json.loads(ln) for ln in lines[1:]]
|
|
162
|
+
per = {}
|
|
163
|
+
for r in recs:
|
|
164
|
+
per[r["dim"]] = per.get(r["dim"], 0) + 1
|
|
165
|
+
shapes.add(tuple(sorted(per.items())))
|
|
166
|
+
flags = {h.get("cost_model") == "count" for h in heads}
|
|
167
|
+
if flags == {False}:
|
|
168
|
+
return None
|
|
169
|
+
if flags != {True}:
|
|
170
|
+
raise CertificateError("the ladders mix counted and timed cost models: they are not comparable")
|
|
171
|
+
for key in ("steps_per_second", "python", "budget", "counter_sha256"):
|
|
172
|
+
if len({json.dumps(h.get(key)) for h in heads}) != 1:
|
|
173
|
+
raise CertificateError(f"the counted ladders disagree about {key}")
|
|
174
|
+
if len(shapes) != 1:
|
|
175
|
+
raise CertificateError("the counted ladders cover different sizes or numbers of instances")
|
|
176
|
+
shape = dict(next(iter(shapes)))
|
|
177
|
+
return {"steps_per_second": heads[0]["steps_per_second"], "python": heads[0]["python"], "dims": sorted(shape), "n": min(shape.values()), "cap": heads[0]["budget"],
|
|
178
|
+
"counter_sha256": heads[0].get("counter_sha256")}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def make_private(cert):
|
|
182
|
+
"""A PRIVATE-SOLVER copy of a certificate: no solver source, and no failure text in the submission's ladders (a crash message is a traceback, and a
|
|
183
|
+
traceback quotes the solver's own lines). Credit, claim, log, rules and hashes are untouched, so everything but re-running still verifies.
|
|
184
|
+
The input is not modified."""
|
|
185
|
+
import copy
|
|
186
|
+
out = copy.deepcopy(cert)
|
|
187
|
+
if not isinstance(out.get("submission"), dict):
|
|
188
|
+
raise CertificateError("not a certificate: no submission")
|
|
189
|
+
out["submission"]["source"] = None
|
|
190
|
+
out["private"] = True
|
|
191
|
+
for tag, lad in (out.get("ladders") or {}).items():
|
|
192
|
+
lines = []
|
|
193
|
+
for line in lad["submission"].splitlines():
|
|
194
|
+
if not line.strip():
|
|
195
|
+
continue
|
|
196
|
+
rec = json.loads(line)
|
|
197
|
+
if "err" in rec: # only per-instance records carry failure text
|
|
198
|
+
rec.pop("err")
|
|
199
|
+
rec["err_withheld"] = True
|
|
200
|
+
lines.append(json.dumps(rec))
|
|
201
|
+
lad["submission"] = "\n".join(lines) + "\n"
|
|
202
|
+
return out
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def build(family, submission, seed_dir, reveal_seeds=False, baseline=None, results_dir=None, round_data=None, round_id=None, now=None, private=False):
|
|
206
|
+
if family not in families.FAMILIES:
|
|
207
|
+
raise CertificateError(f"unknown family {family!r}")
|
|
208
|
+
results_dir = results_dir or os.path.join(HERE, "results")
|
|
209
|
+
baseline = os.path.abspath(baseline or judge.default_baseline(family))
|
|
210
|
+
submission = os.path.abspath(submission)
|
|
211
|
+
name = os.path.splitext(os.path.basename(submission))[0]
|
|
212
|
+
seeds = official.verified_seeds(seed_dir)
|
|
213
|
+
base_text_src, sub_text_src = _read(baseline), _read(submission)
|
|
214
|
+
log = None
|
|
215
|
+
if round_id:
|
|
216
|
+
if not round_data:
|
|
217
|
+
raise CertificateError("--round-id needs --round-data")
|
|
218
|
+
r = rounds.get_round(round_data, round_id)
|
|
219
|
+
if r["family"] != family:
|
|
220
|
+
raise CertificateError(f"round {round_id} is a {r['family']} round, not {family}")
|
|
221
|
+
if reveal_seeds and r["state"] != "revealed":
|
|
222
|
+
# putting the seeds in a file IS the reveal: before the round is evaluated they would let anyone precompute every instance
|
|
223
|
+
raise CertificateError(f"round {round_id} is {r['state']!r}, not 'revealed': a certificate with seeds may only be built after the reveal "
|
|
224
|
+
"(build a sealed one, without --reveal-seeds, until then)")
|
|
225
|
+
log = rounds.read_log(rounds.rdir(round_data, round_id))
|
|
226
|
+
commits = sorted(h for _, h in seeds)
|
|
227
|
+
published, why = commit_published(log, commits)
|
|
228
|
+
ladders, entries, seed_blob = {}, [], {}
|
|
229
|
+
with tempfile.TemporaryDirectory() as work:
|
|
230
|
+
for path, h in seeds:
|
|
231
|
+
tag = h[:8]
|
|
232
|
+
bfile = os.path.join(results_dir, f"judged_{family}_{tag}_baseline_{os.path.basename(baseline)}.jsonl")
|
|
233
|
+
sfile = os.path.join(results_dir, f"judged_{family}_{tag}_submission_{os.path.basename(submission)}.jsonl")
|
|
234
|
+
for f in (bfile, sfile):
|
|
235
|
+
if not os.path.exists(f):
|
|
236
|
+
raise CertificateError(f"missing ladder {f}: evaluate this submission on seed {tag} first")
|
|
237
|
+
ladders[tag] = {"baseline": _read(bfile), "submission": _read(sfile)}
|
|
238
|
+
entries.append(score_seed(family, name, tag, published, ladders[tag]["baseline"], ladders[tag]["submission"], work))
|
|
239
|
+
if reveal_seeds:
|
|
240
|
+
with open(path, "rb") as f:
|
|
241
|
+
seed_blob[h] = f.read().strip().decode()
|
|
242
|
+
cert = {"format": FORMAT, "created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now or time.time())), "family": family,
|
|
243
|
+
"claim": claim_of(entries), "entries": entries, "rules": rules_of(family), "commitments": commits,
|
|
244
|
+
"seeds": seed_blob or None, "seeds_revealed": bool(reveal_seeds),
|
|
245
|
+
"submission": {"name": name, "sha256": sha256(sub_text_src), "source": sub_text_src},
|
|
246
|
+
"baseline": {"name": os.path.basename(baseline), "sha256": sha256(base_text_src), "source": base_text_src},
|
|
247
|
+
"ladders": ladders, "log": log, "log_note": why}
|
|
248
|
+
if family in families.DECLARED:
|
|
249
|
+
d = families.DECLARED[family]["dir"]
|
|
250
|
+
mod = declared.load_declaration(d)
|
|
251
|
+
files = {"decl.py": os.path.join(d, declared.DECL), "compiled.json": os.path.join(d, declared.COMPILED),
|
|
252
|
+
"reference": declared.reference_path(d, mod)}
|
|
253
|
+
cert["domain"] = {"name": family, "reference_name": mod.REFERENCE_SOLVER,
|
|
254
|
+
"files": {role: {"sha256": sha256(_read(p).encode()), "text": _read(p)} for role, p in files.items()}}
|
|
255
|
+
cert["private"] = False
|
|
256
|
+
counted = _counted_info(ladders)
|
|
257
|
+
if counted:
|
|
258
|
+
# counted ladders carry THEIR OWN size ladder (small: counting is slow), chosen by whoever ran them: the verifier then holds the ladder check to
|
|
259
|
+
# these sizes, not to the family's official ones, and the verdict says that a counted certificate is not an official round result
|
|
260
|
+
cert["count"] = counted
|
|
261
|
+
cert["rules"]["dims"] = counted["dims"]
|
|
262
|
+
cert["rules"]["cost_model"] = "count"
|
|
263
|
+
return make_private(cert) if private else cert
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# ------------------------------------------------------------------------------------------------------------- verify
|
|
267
|
+
def _chk(out, check, status, detail=""):
|
|
268
|
+
out.append({"check": check, "status": status, "detail": detail})
|
|
269
|
+
return status == "pass"
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _domain_derive(cert, family, work, out, metas, stack, execute=False):
|
|
273
|
+
"""(derive, verify) for the certificate's family: this checkout's registry for a built-in, the certificate's own files for a declared domain.
|
|
274
|
+
A declared domain's decl.py is CODE FROM THE CERTIFICATE: it is hash-checked and read as data, and imported (executed) only with execute=True."""
|
|
275
|
+
dom = cert.get("domain")
|
|
276
|
+
if not dom:
|
|
277
|
+
if family in families.DECLARED:
|
|
278
|
+
_chk(out, "domain", "fail", f"{family!r} is a declared domain here but the certificate carries no declaration files")
|
|
279
|
+
return None
|
|
280
|
+
if family not in families.FAMILIES:
|
|
281
|
+
_chk(out, "domain", "fail", f"{family!r} is not a family this checkout knows, and the certificate carries no declaration")
|
|
282
|
+
return None
|
|
283
|
+
_chk(out, "domain", "pass", f"built-in family {family}")
|
|
284
|
+
return families.FAMILIES[family]
|
|
285
|
+
if dom.get("name") != family or not declared.NAME_RE.match(str(dom.get("name"))):
|
|
286
|
+
_chk(out, "domain", "fail", "the declaration's name is not the certificate's family")
|
|
287
|
+
return None
|
|
288
|
+
d = os.path.join(work, "domains", dom["name"])
|
|
289
|
+
os.makedirs(d)
|
|
290
|
+
ref_rel = dom["reference_name"]
|
|
291
|
+
if os.path.isabs(ref_rel) or ".." in ref_rel.split("/"):
|
|
292
|
+
_chk(out, "domain", "fail", "the reference path escapes the declaration directory")
|
|
293
|
+
return None
|
|
294
|
+
targets = {"decl.py": os.path.join(d, declared.DECL), "compiled.json": os.path.join(d, declared.COMPILED), "reference": os.path.join(d, ref_rel)}
|
|
295
|
+
for role, f in dom["files"].items():
|
|
296
|
+
if role not in targets:
|
|
297
|
+
_chk(out, "domain", "fail", f"unexpected file role {role!r}")
|
|
298
|
+
return None
|
|
299
|
+
if sha256(f["text"].encode()) != f["sha256"]:
|
|
300
|
+
_chk(out, "domain", "fail", f"{role} does not match its recorded hash")
|
|
301
|
+
return None
|
|
302
|
+
os.makedirs(os.path.dirname(targets[role]), exist_ok=True)
|
|
303
|
+
with open(targets[role], "w", encoding="utf-8") as g:
|
|
304
|
+
g.write(f["text"])
|
|
305
|
+
compiled = declared.read_compiled(d)
|
|
306
|
+
texts = {role: f["text"].encode() for role, f in dom["files"].items()}
|
|
307
|
+
if (compiled is None or compiled.get("format") != declared.FORMAT or compiled.get("name") != family
|
|
308
|
+
or compiled.get("decl_sha256") != sha256(texts.get("decl.py", b"")) or compiled.get("reference_sha256") != sha256(texts.get("reference", b""))
|
|
309
|
+
or not all(k in compiled for k in ("dims", "practice_dims", "official", "version"))):
|
|
310
|
+
_chk(out, "domain", "fail", "the compiled record does not match the declaration and reference files it sits beside")
|
|
311
|
+
return None
|
|
312
|
+
if execute:
|
|
313
|
+
try:
|
|
314
|
+
mod = declared.load_declaration(d) # EXECUTES the certificate's decl.py
|
|
315
|
+
problems = declared.freshness_problems(d, mod, compiled)
|
|
316
|
+
except declared.DeclarationError as e:
|
|
317
|
+
_chk(out, "domain", "fail", str(e))
|
|
318
|
+
return None
|
|
319
|
+
if problems:
|
|
320
|
+
_chk(out, "domain", "fail", "the compiled record does not match the declaration: " + "; ".join(problems))
|
|
321
|
+
return None
|
|
322
|
+
if family in families.DECLARED:
|
|
323
|
+
# this checkout already has a domain of that name: it must be THE certificate's declaration, or the registry (verify, OFFICIAL, rerun)
|
|
324
|
+
# would silently judge the certificate by someone else's rules
|
|
325
|
+
if families.DECLARED[family]["decl_sha256"] != compiled["decl_sha256"]:
|
|
326
|
+
_chk(out, "domain", "fail", f"this checkout's own {family!r} declaration differs from the certificate's (decl.py hashes differ)")
|
|
327
|
+
return None
|
|
328
|
+
elif family in families.FAMILIES:
|
|
329
|
+
_chk(out, "domain", "fail", f"{family!r} is a built-in family here, but the certificate carries a declaration for it")
|
|
330
|
+
return None
|
|
331
|
+
else:
|
|
332
|
+
# judge.leaderboard_rows looks the family's thresholds up in the registry: register them (data from compiled.json; with execute, the whole
|
|
333
|
+
# declaration) for the duration of this verification, then remove them
|
|
334
|
+
if execute:
|
|
335
|
+
declared.register(families, d, mod, compiled)
|
|
336
|
+
else:
|
|
337
|
+
families.VERSIONS[family] = compiled["version"]
|
|
338
|
+
families.OFFICIAL[family] = compiled["official"]
|
|
339
|
+
families.DEFAULT_DIMS[family] = compiled["dims"]
|
|
340
|
+
stack.callback(declared.unregister, families, family)
|
|
341
|
+
ver_ok = all(m.get("instance_version") == compiled["version"] for m in metas)
|
|
342
|
+
_chk(out, "domain", "pass" if ver_ok else "fail",
|
|
343
|
+
(f"declaration {compiled['version']}, decl.py {compiled['decl_sha256'][:12]}..., compiled ladder {compiled['dims']}, cap {compiled['official']['budget']}; "
|
|
344
|
+
+ ("declaration executed (--execute)" if execute else "files hash-checked and read as data; the declaration was NOT executed"))
|
|
345
|
+
if ver_ok else "a ladder was measured with a different instance version than this declaration")
|
|
346
|
+
return declared.make_spec(mod) if execute else None
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
ALL_INSTANCES = 10 ** 9 # what `--rerun all` means: more than any certificate holds, so every instance is re-run (the sample is then the whole pool)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def rerun_count(text):
|
|
353
|
+
"""argparse type for --rerun: a non-negative integer, or `all` (re-run EVERY instance: the only way to leave nothing covered merely statistically)."""
|
|
354
|
+
if str(text).strip().lower() == "all":
|
|
355
|
+
return ALL_INSTANCES
|
|
356
|
+
try:
|
|
357
|
+
k = int(text)
|
|
358
|
+
except ValueError:
|
|
359
|
+
raise argparse.ArgumentTypeError("--rerun takes a number or `all`")
|
|
360
|
+
if k < 0:
|
|
361
|
+
raise argparse.ArgumentTypeError("--rerun cannot be negative")
|
|
362
|
+
return k
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
MAX_COUNTED_CAP = 10_000 # counted seconds (x steps_per_second instructions): a larger per-instance budget in an untrusted certificate would make --rerun --execute run for hours
|
|
366
|
+
RERUN_TOLERANCE = 2.0 # bits: LOOSE on purpose until cross-machine spread has been measured (compile_domain.py recheck --measure); tighten with --tolerance
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def verify(cert, rerun=0, execute=False, log=print, tolerance=RERUN_TOLERANCE, sample_seed=0, source=None):
|
|
370
|
+
"""-> {"ok": bool, "checks": [...], "claim": ..., "private": bool, "source_revealed": bool}. Never trusts the certificate's own claim: everything is recomputed.
|
|
371
|
+
A certificate is untrusted input. By default nothing in it is executed: a declared domain's decl.py is hash-checked and read as data, so
|
|
372
|
+
instance coverage of a declared domain is SKIPPED, and --rerun is refused. execute=True imports the certificate's decl.py and (with
|
|
373
|
+
rerun) runs its solver sources on this machine, unsandboxed: do that only in a throwaway VM or container.
|
|
374
|
+
`source` (text) is a solver the owner has revealed for a PRIVATE-SOLVER certificate: it must hash to the certificate's committed solver hash, and
|
|
375
|
+
then --rerun can re-measure it."""
|
|
376
|
+
try:
|
|
377
|
+
return _verify(cert, rerun, execute, tolerance, sample_seed, source)
|
|
378
|
+
except (KeyError, TypeError, ValueError, AttributeError, IndexError, StopIteration, OverflowError, ZeroDivisionError) as e:
|
|
379
|
+
return {"ok": False, "claim": None, "private": False, "source_revealed": False, "counted": False,
|
|
380
|
+
"checks": [{"check": "format", "status": "fail", "detail": f"malformed certificate ({type(e).__name__}: {e})"}]}
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _verify(cert, rerun, execute, tolerance=RERUN_TOLERANCE, sample_seed=0, source=None):
|
|
384
|
+
out = []
|
|
385
|
+
if not isinstance(cert, dict) or cert.get("format") != FORMAT:
|
|
386
|
+
return {"ok": False, "checks": [{"check": "format", "status": "fail", "detail": f"not a {FORMAT} certificate"}], "claim": None}
|
|
387
|
+
family = cert["family"]
|
|
388
|
+
_chk(out, "format", "pass", f"{FORMAT}, family {family}, created {cert.get('created')}")
|
|
389
|
+
# rules: this checkout's protocol code against the certificate's
|
|
390
|
+
mine_code = {rel: (sha256(open(os.path.join(HERE, rel), "rb").read()) if os.path.exists(os.path.join(HERE, rel)) else None) for rel in rounds.CODE_FILES}
|
|
391
|
+
diff = sorted(rel for rel in rounds.CODE_FILES if mine_code[rel] != cert["rules"]["code"].get(rel))
|
|
392
|
+
_chk(out, "rules", "fail" if diff else "pass",
|
|
393
|
+
("this checkout differs from the certificate's rules in: " + ", ".join(diff) + ". Check out the version it names; a different estimator or "
|
|
394
|
+
"harness would score the same data differently") if diff else f"{len(mine_code)} protocol files hash exactly as the certificate names")
|
|
395
|
+
# sources vs ladders
|
|
396
|
+
metas, ladder_ok = [], True
|
|
397
|
+
for tag, lad in sorted(cert["ladders"].items()):
|
|
398
|
+
for role in ("baseline", "submission"):
|
|
399
|
+
try:
|
|
400
|
+
meta = json.loads(lad[role].splitlines()[0])
|
|
401
|
+
except (ValueError, IndexError):
|
|
402
|
+
ladder_ok = False
|
|
403
|
+
_chk(out, "sources", "fail", f"seed {tag}: the {role} ladder has no readable header")
|
|
404
|
+
continue
|
|
405
|
+
metas.append(meta)
|
|
406
|
+
want = cert[role]["sha256"]
|
|
407
|
+
if meta.get("solver_sha256") != want:
|
|
408
|
+
ladder_ok = False
|
|
409
|
+
_chk(out, "sources", "fail", f"seed {tag}: the {role} ladder was measured with a solver whose hash is {str(meta.get('solver_sha256'))[:12]}, "
|
|
410
|
+
f"not the included source ({want[:12]})")
|
|
411
|
+
private = cert.get("private") is True
|
|
412
|
+
sub = cert["submission"]
|
|
413
|
+
revealed = None # the solver's source, once the owner has revealed it and the caller supplied it
|
|
414
|
+
if private:
|
|
415
|
+
if sub.get("source") is not None:
|
|
416
|
+
ladder_ok = False
|
|
417
|
+
_chk(out, "sources", "fail", "a certificate marked private-solver carries the solver's source: it is malformed or was made private badly")
|
|
418
|
+
elif sha256(sub["source"]) != sub["sha256"]:
|
|
419
|
+
ladder_ok = False
|
|
420
|
+
_chk(out, "sources", "fail", "the included solver source does not match its own recorded hash")
|
|
421
|
+
if sha256(cert["baseline"]["source"]) != cert["baseline"]["sha256"]:
|
|
422
|
+
ladder_ok = False
|
|
423
|
+
_chk(out, "sources", "fail", "the included baseline source does not match its own recorded hash")
|
|
424
|
+
in_log = " (and the round's log)" if cert.get("log") else "" # a certificate built outside a round has no log: do not claim one committed to anything
|
|
425
|
+
if private and source is not None:
|
|
426
|
+
if sha256(source) != sub["sha256"]:
|
|
427
|
+
ladder_ok = False
|
|
428
|
+
_chk(out, "solver disclosure", "fail", f"the file supplied does not hash to the solver hash this certificate{in_log} committed to: "
|
|
429
|
+
f"it hashes to {sha256(source)[:12]}..., the certificate says {sub['sha256'][:12]}...")
|
|
430
|
+
else:
|
|
431
|
+
revealed = source
|
|
432
|
+
_chk(out, "solver disclosure", "pass", f"the file supplied is exactly the solver this certificate measured (sha256 {sub['sha256'][:12]}...)")
|
|
433
|
+
elif private:
|
|
434
|
+
_chk(out, "solver disclosure", "skip", "PRIVATE SOLVER: its source is not in this certificate, so it cannot be inspected or re-run here. Its hash is the one every "
|
|
435
|
+
f"ladder header names{' and the round' + chr(39) + 's log committed to' if cert.get('log') else ''}; if the owner publishes the file, check it with --source FILE")
|
|
436
|
+
if ladder_ok:
|
|
437
|
+
_chk(out, "sources", "pass", f"submission {sub['sha256'][:12]}...{' (private solver: source withheld)' if private else ''} and baseline {cert['baseline']['sha256'][:12]}... "
|
|
438
|
+
"are what every ladder header names")
|
|
439
|
+
metas = [m for m in metas if m.get("meta")]
|
|
440
|
+
with tempfile.TemporaryDirectory() as work, contextlib.ExitStack() as stack:
|
|
441
|
+
spec = _domain_derive(cert, family, work, out, metas, stack, execute)
|
|
442
|
+
# log
|
|
443
|
+
commits = cert["commitments"]
|
|
444
|
+
published, why = commit_published(cert.get("log"), commits)
|
|
445
|
+
round_declared = False # True when the log's entry 0 declared THIS counted ladder before any submission (see rounds.create_round(count=...))
|
|
446
|
+
try:
|
|
447
|
+
log_count = _counted_info(cert["ladders"]) # a malformed mix is reported by the cost-model check below
|
|
448
|
+
except CertificateError:
|
|
449
|
+
log_count = None
|
|
450
|
+
if cert.get("log"):
|
|
451
|
+
if published:
|
|
452
|
+
data0 = cert["log"][0]["data"]
|
|
453
|
+
fp = data0.get("fingerprint") or {}
|
|
454
|
+
same = fp.get("code") is not None and all(fp["code"].get(rel) == cert["rules"]["code"].get(rel) for rel in rounds.CODE_FILES)
|
|
455
|
+
subs = {e["data"].get("sha256") for e in cert["log"] if e.get("kind") == "submission"}
|
|
456
|
+
changed = [e for e in cert["log"] if e.get("kind") == "code_changed"]
|
|
457
|
+
bad = []
|
|
458
|
+
if not same or fp.get("dims") != cert["rules"]["dims"] or fp.get("instance_version") != cert["rules"]["instance_version"]:
|
|
459
|
+
bad.append("the rules fingerprint in log entry 0 (code, size ladder, instance version) differs from the certificate's rules")
|
|
460
|
+
if cert["submission"]["sha256"] not in subs:
|
|
461
|
+
bad.append("the submission's hash was never logged as an accepted submission")
|
|
462
|
+
if cert.get("domain"): # the declaration, its compiled ladder and its reference were committed to in entry 0 too
|
|
463
|
+
anchored = set((fp.get("code") or {}).values())
|
|
464
|
+
off = sorted(role for role, f in cert["domain"]["files"].items() if f["sha256"] not in anchored)
|
|
465
|
+
if off:
|
|
466
|
+
bad.append("the domain file(s) " + ", ".join(off) + " are not the ones the round committed to in log entry 0")
|
|
467
|
+
declared_count = data0.get("count") if data0.get("cost_model") == "count" and isinstance(data0.get("count"), dict) else None
|
|
468
|
+
if declared_count is not None or log_count is not None:
|
|
469
|
+
if log_count is None:
|
|
470
|
+
bad.append("the round declared step counting in log entry 0 but this certificate's ladders are not step-counted")
|
|
471
|
+
elif declared_count is None:
|
|
472
|
+
bad.append("this certificate's ladders are step-counted but the round did not declare step counting in log entry 0: their sizes and cap would be the prover's")
|
|
473
|
+
elif any(declared_count.get(k) != log_count.get(k) for k in ("dims", "n", "cap", "steps_per_second", "python", "counter_sha256")):
|
|
474
|
+
bad.append("the counted ladder (sizes, instances per size, cap, unit, Python version or counter) is not the one the round declared in log entry 0")
|
|
475
|
+
elif (fp.get("code") or {}).get("count_runner.py") != log_count["counter_sha256"]:
|
|
476
|
+
bad.append("the counting program is not the one the round's rules fingerprint names")
|
|
477
|
+
elif private:
|
|
478
|
+
bad.append("a counted round requires every solver's source to be published (the proof is re-execution), but this certificate withholds it")
|
|
479
|
+
else:
|
|
480
|
+
round_declared = True
|
|
481
|
+
import counted
|
|
482
|
+
src = cert["submission"].get("source")
|
|
483
|
+
problems = counted.source_problems(src) if isinstance(src, str) else []
|
|
484
|
+
if problems: # the round should have refused it at submission; a verifier must not rely on the operator having done so (the counter refuses to run it anyway)
|
|
485
|
+
bad.append("the solver's source is outside the restricted subset a counted solver is written in: " + "; ".join(problems[:3]))
|
|
486
|
+
round_declared = False
|
|
487
|
+
_chk(out, "log", "fail" if bad else "pass", ("; ".join(bad)) if bad else
|
|
488
|
+
f"{why}; commitments and rules were public before submissions; submission logged" + (f"; NOTE {len(changed)} code_changed entry(ies)" if changed else ""))
|
|
489
|
+
else:
|
|
490
|
+
_chk(out, "log", "fail", why)
|
|
491
|
+
else:
|
|
492
|
+
_chk(out, "log", "skip", "no log included: the publication time of the commitments cannot be shown, so the credit is at most PROVISIONAL")
|
|
493
|
+
# seeds
|
|
494
|
+
seeds = cert.get("seeds") or {}
|
|
495
|
+
if cert.get("seeds_revealed"):
|
|
496
|
+
bad = [h for h, s in seeds.items() if sha256(s.strip().encode()) != h]
|
|
497
|
+
missing = [h for h in commits if h not in seeds]
|
|
498
|
+
_chk(out, "seeds", "fail" if bad or missing else "pass",
|
|
499
|
+
f"{len(bad)} seed(s) do not hash to their commitment, {len(missing)} missing" if bad or missing else f"{len(seeds)} revealed seed(s) hash to their commitments")
|
|
500
|
+
else:
|
|
501
|
+
_chk(out, "seeds", "skip", "seeds not revealed in this certificate: instance coverage and re-measurement cannot be checked")
|
|
502
|
+
# ladders: one baseline/submission pair per committed seed (a dropped seed would raise the minimum), on the rules' own size ladder
|
|
503
|
+
pair_bad = []
|
|
504
|
+
if sorted(cert["ladders"]) != sorted(c[:8] for c in commits):
|
|
505
|
+
pair_bad.append("the ladders are not exactly one pair per committed seed")
|
|
506
|
+
for tag, lad in cert["ladders"].items():
|
|
507
|
+
for role in ("baseline", "submission"):
|
|
508
|
+
recs = [json.loads(line) for line in lad[role].splitlines()[1:] if line.strip()]
|
|
509
|
+
if sorted({r["dim"] for r in recs}) != sorted(cert["rules"]["dims"]):
|
|
510
|
+
pair_bad.append(f"seed {tag} {role}: sizes {sorted({r['dim'] for r in recs})} are not the rules' ladder {sorted(cert['rules']['dims'])}")
|
|
511
|
+
per = {}
|
|
512
|
+
for r in recs:
|
|
513
|
+
per[r["dim"]] = per.get(r["dim"], 0) + 1
|
|
514
|
+
if len(set(per.values())) > 1:
|
|
515
|
+
pair_bad.append(f"seed {tag} {role}: the number of instances differs between sizes")
|
|
516
|
+
_chk(out, "ladders", "fail" if pair_bad else "pass",
|
|
517
|
+
"; ".join(pair_bad) if pair_bad else f"{len(cert['ladders'])} seed(s), each with a baseline and a submission ladder on the rules' sizes {cert['rules']['dims']}")
|
|
518
|
+
# cost model: deterministic step counting (counted.py) is a different KIND of evidence from a timed ladder, and the verdict must say so
|
|
519
|
+
count = cert.get("count") if isinstance(cert.get("count"), dict) else None
|
|
520
|
+
try: # the ladders themselves say which cost model they use: a certificate may not hide that by dropping its own `count` field (instruction counts passing as seconds)
|
|
521
|
+
actual = _counted_info(cert["ladders"])
|
|
522
|
+
except CertificateError as e:
|
|
523
|
+
_chk(out, "cost model", "fail", f"the ladders are inconsistent: {e}")
|
|
524
|
+
actual, count = None, None
|
|
525
|
+
else:
|
|
526
|
+
if actual is None and any(m.get("sandbox") == "count" for m in metas):
|
|
527
|
+
# claim_of grants the COUNTED status to entries whose sandbox is "count": a timed ladder must not be able to take it by labelling its header so
|
|
528
|
+
_chk(out, "cost model", "fail", "a ladder header says sandbox \"count\" but the ladders are not step-counted: the label is only for counted ladders")
|
|
529
|
+
count = None
|
|
530
|
+
elif (actual is not None) != (count is not None):
|
|
531
|
+
_chk(out, "cost model", "fail", "the ladders are " + ("step-COUNTED" if actual is not None else "timed") + " but the certificate says "
|
|
532
|
+
+ ("they are counted" if count is not None else "nothing about step counting") + ": a counted cost must never be presented as seconds")
|
|
533
|
+
count = None # already reported: skip the detailed counted checks below
|
|
534
|
+
if count is not None:
|
|
535
|
+
bad = _check_counted(cert, count)
|
|
536
|
+
_chk(out, "cost model", "fail" if bad else "pass", "; ".join(bad[:3]) + (f" (and {len(bad) - 3} more)" if len(bad) > 3 else "") if bad else
|
|
537
|
+
f"deterministic STEP COUNTING: every cost is a bytecode-instruction count / {count['steps_per_second']:g}, recorded on {count['python']}. It is exactly "
|
|
538
|
+
"reproducible (--rerun K --execute must give identical counts) but it is NOT a time, it is blind to work inside C functions, and the ladder "
|
|
539
|
+
f"(sizes {count['dims']}, {count['n']} per size, cap {count['cap']:g}) "
|
|
540
|
+
+ ("was declared by the round in log entry 0 before any submission, the same for every participant (a ROUND-DECLARED ladder)" if round_declared else
|
|
541
|
+
"was chosen by whoever ran it: not an official round result"))
|
|
542
|
+
# coverage
|
|
543
|
+
if spec is not None and cert.get("seeds_revealed") and ladder_ok:
|
|
544
|
+
derive = spec[0]
|
|
545
|
+
cov_bad = []
|
|
546
|
+
for tag, lad in sorted(cert["ladders"].items()):
|
|
547
|
+
h = next((c for c in commits if c.startswith(tag)), None)
|
|
548
|
+
if h is None or h not in seeds:
|
|
549
|
+
cov_bad.append(f"seed {tag}: no matching revealed seed")
|
|
550
|
+
continue
|
|
551
|
+
master = seeds[h].strip().encode()
|
|
552
|
+
for role in ("baseline", "submission"):
|
|
553
|
+
recs = [json.loads(line) for line in lad[role].splitlines()[1:] if line.strip()]
|
|
554
|
+
dims = sorted({r["dim"] for r in recs})
|
|
555
|
+
n = len(recs) // max(len(dims), 1)
|
|
556
|
+
want = set()
|
|
557
|
+
for dm in dims:
|
|
558
|
+
for i in range(n):
|
|
559
|
+
want.add(derive(master, dm, i)["id"])
|
|
560
|
+
have = [r["id"] for r in recs]
|
|
561
|
+
if len(have) != len(set(have)) or set(have) != want:
|
|
562
|
+
cov_bad.append(f"seed {tag} {role}: records are not exactly the {len(want)} instances the seed derives")
|
|
563
|
+
_chk(out, "coverage", "fail" if cov_bad else "pass",
|
|
564
|
+
"; ".join(cov_bad) if cov_bad else "every ladder holds exactly the instances its seed derives, no more, no fewer, none repeated")
|
|
565
|
+
else:
|
|
566
|
+
_chk(out, "coverage", "skip", "needs the revealed seeds" + ("" if spec is not None or not cert.get("domain") else
|
|
567
|
+
" and the declaration executed (--execute: runs the certificate's decl.py on this machine)"))
|
|
568
|
+
# credit
|
|
569
|
+
try:
|
|
570
|
+
entries = [score_seed(family, cert["submission"]["name"], tag, published, lad["baseline"], lad["submission"], work)
|
|
571
|
+
for tag, lad in sorted(cert["ladders"].items())]
|
|
572
|
+
mine, theirs = canon(sorted(entries, key=lambda e: e["seed_tag"])), canon(sorted(cert["entries"], key=lambda e: e["seed_tag"]))
|
|
573
|
+
claim = claim_of(entries)
|
|
574
|
+
if mine != theirs or canon(claim) != canon(cert["claim"]):
|
|
575
|
+
_chk(out, "credit", "fail", f"recomputed credit {claim['credit_bits']} bits ({claim['status']}) differs from the claimed "
|
|
576
|
+
f"{cert['claim']['credit_bits']} bits ({cert['claim']['status']})")
|
|
577
|
+
else:
|
|
578
|
+
_chk(out, "credit", "pass", f"recomputed {claim['credit_bits']} bits, {claim['status']}, per seed {claim['per_seed']}; identical to the claim")
|
|
579
|
+
except CertificateError as e:
|
|
580
|
+
claim = None
|
|
581
|
+
_chk(out, "credit", "fail", str(e))
|
|
582
|
+
# rerun
|
|
583
|
+
if rerun and private and revealed is None:
|
|
584
|
+
_chk(out, "rerun", "skip", "PRIVATE solver: its source is not in the certificate, so there is nothing to re-run. If the owner publishes it, "
|
|
585
|
+
"pass it with --source FILE")
|
|
586
|
+
elif rerun and execute and spec is not None and cert.get("seeds_revealed") and ladder_ok:
|
|
587
|
+
_rerun(cert, spec, seeds, commits, rerun, work, out, tolerance, sample_seed, revealed)
|
|
588
|
+
elif rerun:
|
|
589
|
+
_chk(out, "rerun", "skip", "needs --execute (it runs the certificate's solver sources on this machine, unsandboxed), the revealed seeds and a loadable domain")
|
|
590
|
+
ok = all(c["status"] != "fail" for c in out)
|
|
591
|
+
return {"ok": ok, "checks": out, "claim": cert.get("claim") if ok else None, "private": private, "source_revealed": revealed is not None,
|
|
592
|
+
"counted": count is not None}
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _write_sources(cert, work, solver_source=None):
|
|
596
|
+
"""The two solver files, written where they can be run. A revealed solver is written byte-for-byte: it is the file whose hash the log committed to."""
|
|
597
|
+
paths = {}
|
|
598
|
+
for role in ("baseline", "submission"):
|
|
599
|
+
paths[role] = os.path.join(work, f"{role}.py")
|
|
600
|
+
src = solver_source if role == "submission" and solver_source is not None else cert[role]["source"]
|
|
601
|
+
if isinstance(src, bytes):
|
|
602
|
+
with open(paths[role], "wb") as f:
|
|
603
|
+
f.write(src)
|
|
604
|
+
else:
|
|
605
|
+
with open(paths[role], "w", encoding="utf-8") as f:
|
|
606
|
+
f.write(src)
|
|
607
|
+
return paths
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _check_counted(cert, count):
|
|
611
|
+
"""-> [problems]. Every record of a counted ladder must be internally consistent: a solved instance's cost is exactly its step count over the unit
|
|
612
|
+
(floored like the harness), an unsolved one is charged the cap, and every header names the same unit and interpreter as the certificate."""
|
|
613
|
+
bad = []
|
|
614
|
+
sps = count.get("steps_per_second")
|
|
615
|
+
if not (isinstance(sps, (int, float)) and not isinstance(sps, bool) and sps > 0) or not isinstance(count.get("python"), str):
|
|
616
|
+
return ["the certificate's counted-cost record is malformed"]
|
|
617
|
+
import counted
|
|
618
|
+
csha = count.get("counter_sha256")
|
|
619
|
+
if not (isinstance(csha, str) and re.fullmatch(r"[0-9a-f]{64}", csha)):
|
|
620
|
+
return ["the certificate does not name the counting program that produced its counts (counter_sha256)"]
|
|
621
|
+
if csha != counted.counter_sha256():
|
|
622
|
+
return [f"these counts were made by a different counting program (count_runner.py hash {csha[:12]}, this checkout's is {counted.counter_sha256()[:12]}): "
|
|
623
|
+
"check out the version it names; another counter would count the same program differently"]
|
|
624
|
+
if sps != counted.STEPS_PER_SECOND:
|
|
625
|
+
# the unit is part of what a re-run reproduces (the budget in instructions is cap x unit); the counter hash pins count_runner.py, not this constant
|
|
626
|
+
return [f"the certificate's unit is {sps:g} instructions per counted second, this checkout's is {counted.STEPS_PER_SECOND:g}: check out the version it names"]
|
|
627
|
+
ccap = count.get("cap")
|
|
628
|
+
if not (isinstance(ccap, (int, float)) and not isinstance(ccap, bool) and 0 < ccap <= MAX_COUNTED_CAP): # also rejects NaN and Infinity (JSON allows both)
|
|
629
|
+
return [f"the certificate's counted cap is not a number between 0 and {MAX_COUNTED_CAP:g} counted seconds"]
|
|
630
|
+
for tag, lad in sorted(cert["ladders"].items()):
|
|
631
|
+
for role in ("baseline", "submission"):
|
|
632
|
+
lines = [ln for ln in lad[role].splitlines() if ln.strip()]
|
|
633
|
+
meta = json.loads(lines[0])
|
|
634
|
+
if meta.get("budget") != ccap:
|
|
635
|
+
bad.append(f"seed {tag} {role}: the ladder's budget is not the certificate's counted cap")
|
|
636
|
+
continue
|
|
637
|
+
if (meta.get("cost_model") != "count" or meta.get("sandbox") != "count" or meta.get("steps_per_second") != sps or meta.get("python") != count["python"]
|
|
638
|
+
or meta.get("counter_sha256") != csha): # (sandbox "count": a counted ladder cannot borrow judge.py's OFFICIAL by labelling itself "docker")
|
|
639
|
+
bad.append(f"seed {tag} {role}: the ladder header disagrees with the certificate's counted-cost record")
|
|
640
|
+
continue
|
|
641
|
+
cap = meta["budget"]
|
|
642
|
+
limit = int(cap * sps)
|
|
643
|
+
for ln in lines[1:]:
|
|
644
|
+
r = json.loads(ln)
|
|
645
|
+
s = r.get("steps")
|
|
646
|
+
if isinstance(s, bool) or not isinstance(s, int) or s < 0:
|
|
647
|
+
# a wall-clock timeout or a counter failure has no count: it can never be re-executed to the same number, and "charged the cap" without a count would let a
|
|
648
|
+
# prover turn any baseline instance into a maximal cost that no re-run will ever check
|
|
649
|
+
bad.append(f"seed {tag} {role} {r['id']}: no step count, so it cannot be reproduced and is not provable")
|
|
650
|
+
elif r["status"] == "solved":
|
|
651
|
+
if s > limit:
|
|
652
|
+
bad.append(f"seed {tag} {role} {r['id']}: a solved instance cannot have used more than the budget")
|
|
653
|
+
elif abs(r["cpu_net"] - max(s / sps, 1e-3)) > 1e-9 * max(1.0, r["cpu_net"]):
|
|
654
|
+
bad.append(f"seed {tag} {role} {r['id']}: the cost does not follow from the step count")
|
|
655
|
+
elif r["cpu_net"] != cap:
|
|
656
|
+
bad.append(f"seed {tag} {role} {r['id']}: an unsolved instance must be charged the cap")
|
|
657
|
+
elif r["status"] == "timeout":
|
|
658
|
+
if s != limit + 1:
|
|
659
|
+
# the counter stops a solver at exactly limit+1 instructions, so a timeout with any other count (a baseline "timing out" in 5 steps) is not one
|
|
660
|
+
bad.append(f"seed {tag} {role} {r['id']}: a counted timeout must have used exactly the budget plus one instruction")
|
|
661
|
+
elif s > limit:
|
|
662
|
+
bad.append(f"seed {tag} {role} {r['id']}: a {r['status']} instance cannot have used more than the budget")
|
|
663
|
+
return bad
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _rerun_counted(cert, spec, paths, seeds, commits, k, out, sample_seed):
|
|
667
|
+
"""Re-EXECUTE k instances of BOTH solvers here under step counting and require IDENTICAL counts and statuses: no tolerance, no trust in anyone's
|
|
668
|
+
machine. That equality is the proof. Instruction counts are only comparable on the same interpreter, so a different Python version is a SKIP."""
|
|
669
|
+
import counted
|
|
670
|
+
derive, check = spec
|
|
671
|
+
count = cert["count"]
|
|
672
|
+
if not counted.supported() or counted.python_version() != count["python"]:
|
|
673
|
+
_chk(out, "rerun", "skip", f"these counts were recorded on {count['python']} and this machine runs {counted.python_version()}: instruction counts are only "
|
|
674
|
+
"comparable on the same interpreter version, so there is nothing exact to compare (use that version)")
|
|
675
|
+
return
|
|
676
|
+
rng = random.Random(sample_seed)
|
|
677
|
+
pool = []
|
|
678
|
+
for tag, lad in cert["ladders"].items():
|
|
679
|
+
recs = {role: {r["id"]: r for r in map(json.loads, (ln for ln in lad[role].splitlines()[1:] if ln.strip()))} for role in ("baseline", "submission")}
|
|
680
|
+
for i, rb in recs["baseline"].items():
|
|
681
|
+
rs = recs["submission"].get(i)
|
|
682
|
+
if rs and rb.get("steps") is not None and rs.get("steps") is not None:
|
|
683
|
+
pool.append((tag, i, rb, rs))
|
|
684
|
+
pool.sort(key=lambda t: (t[0], t[1]))
|
|
685
|
+
picks = rng.sample(pool, min(k, len(pool)))
|
|
686
|
+
if not picks:
|
|
687
|
+
_chk(out, "rerun", "skip", "no instance has a recorded step count for both solvers")
|
|
688
|
+
return
|
|
689
|
+
mismatches = []
|
|
690
|
+
for tag, iid, rb, rs in picks:
|
|
691
|
+
h = next(c for c in commits if c.startswith(tag))
|
|
692
|
+
master = seeds[h].strip().encode()
|
|
693
|
+
dim, idx = rb["dim"], int(iid.rsplit(":", 1)[1])
|
|
694
|
+
inst = derive(master, dim, idx)
|
|
695
|
+
inst["family"] = cert["family"]
|
|
696
|
+
if "args" not in inst: # lattice families: the solver receives (basis, bound2), as families.derive arranges
|
|
697
|
+
inst["args"] = [inst["basis"], inst["bound2"]]
|
|
698
|
+
for role, rec in (("baseline", rb), ("submission", rs)):
|
|
699
|
+
got = counted.count_instance(paths[role], inst, count["cap"])
|
|
700
|
+
if got.get("steps") != rec.get("steps") or got.get("status") != rec.get("status"):
|
|
701
|
+
mismatches.append(f"{iid} (seed {tag}) {role}: recorded {rec.get('status')} in {rec.get('steps')} steps, re-executed {got.get('status')} in {got.get('steps')}")
|
|
702
|
+
_chk(out, "rerun", "fail" if mismatches else "pass",
|
|
703
|
+
("did not reproduce: " + "; ".join(mismatches[:3]) + (f" (and {len(mismatches) - 3} more)" if len(mismatches) > 3 else "")) if mismatches else
|
|
704
|
+
f"re-executed {len(picks)} of {len(pool)} instance(s) of BOTH solvers here: every step count and status is IDENTICAL to the record (exact equality, not a tolerance)"
|
|
705
|
+
+ ("" if len(picks) == len(pool) else f". The other {len(pool) - len(picks)} are covered only statistically (a record falsified in a fraction f of the instances escapes "
|
|
706
|
+
f"{len(picks)} random checks with probability (1-f)^{len(picks)}): re-run them all for certainty"))
|
|
707
|
+
out[-1]["data"] = {"k": len(picks), "of": len(pool), "exact": True, "mismatches": len(mismatches), "sample_seed": sample_seed}
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _rerun(cert, spec, seeds, commits, k, work, out, tolerance=RERUN_TOLERANCE, sample_seed=0, solver_source=None):
|
|
711
|
+
derive, check = spec
|
|
712
|
+
paths = _write_sources(cert, work, solver_source)
|
|
713
|
+
if isinstance(cert.get("count"), dict):
|
|
714
|
+
return _rerun_counted(cert, spec, paths, seeds, commits, k, out, sample_seed)
|
|
715
|
+
rng = random.Random(sample_seed)
|
|
716
|
+
pool = []
|
|
717
|
+
for tag, lad in cert["ladders"].items():
|
|
718
|
+
recs = {role: {r["id"]: r for r in map(json.loads, lad[role].splitlines()[1:]) if r} for role in ("baseline", "submission")}
|
|
719
|
+
meta = json.loads(lad["baseline"].splitlines()[0])
|
|
720
|
+
for i, rb in recs["baseline"].items():
|
|
721
|
+
rs = recs["submission"].get(i)
|
|
722
|
+
if rs and rb["status"] == "solved" and rs["status"] == "solved" and rb["cpu_net"] >= E.FLOOR_ABS:
|
|
723
|
+
pool.append((tag, i, rb, rs, meta["budget"]))
|
|
724
|
+
pool.sort(key=lambda t: (t[0], t[1]))
|
|
725
|
+
picks = rng.sample(pool, min(k, len(pool)))
|
|
726
|
+
if not picks:
|
|
727
|
+
_chk(out, "rerun", "skip", "no instance solved by both solvers at a cost above the 0.5 s floor")
|
|
728
|
+
return
|
|
729
|
+
dev, bad = [], []
|
|
730
|
+
ov = harness.calibrate()
|
|
731
|
+
for tag, iid, rb, rs, budget in picks:
|
|
732
|
+
h = next(c for c in commits if c.startswith(tag))
|
|
733
|
+
master = seeds[h].strip().encode()
|
|
734
|
+
dim, idx = rb["dim"], int(iid.rsplit(":", 1)[1])
|
|
735
|
+
inst = derive(master, dim, idx)
|
|
736
|
+
inst["family"] = cert["family"]
|
|
737
|
+
if "args" not in inst: # lattice families: the solver receives (basis, bound2), as families.derive arranges
|
|
738
|
+
inst["args"] = [inst["basis"], inst["bound2"]]
|
|
739
|
+
a, b = (harness.run_one(paths[role], inst, budget) for role in ("baseline", "submission"))
|
|
740
|
+
if a["status"] != "solved" or b["status"] != "solved":
|
|
741
|
+
bad.append(f"{iid} (seed {tag}): baseline {a['status']}, submission {b['status']} here")
|
|
742
|
+
continue
|
|
743
|
+
ratio_here = (max(a["cpu"] - ov, 1e-3)) / max(b["cpu"] - ov, 1e-3)
|
|
744
|
+
dev.append(abs(math.log2(ratio_here) - math.log2(rb["cpu_net"] / rs["cpu_net"])))
|
|
745
|
+
if bad:
|
|
746
|
+
_chk(out, "rerun", "fail", "did not reproduce: " + "; ".join(bad))
|
|
747
|
+
else:
|
|
748
|
+
med = sorted(dev)[len(dev) // 2]
|
|
749
|
+
_chk(out, "rerun", "pass" if med <= tolerance else "fail",
|
|
750
|
+
f"re-measured {len(dev)} instance(s) here: all solved; median |difference| in speedup {med:.2f} bits (machine and load differ; > {tolerance:g} bits fails)")
|
|
751
|
+
out[-1]["data"] = {"k": len(dev), "median_gap_bits": round(med, 3), "gaps_bits": [round(d, 3) for d in dev], "tolerance_bits": tolerance, "sample_seed": sample_seed}
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def main(argv=None):
|
|
755
|
+
ap = argparse.ArgumentParser()
|
|
756
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
757
|
+
p = sub.add_parser("build")
|
|
758
|
+
p.add_argument("--family", required=True)
|
|
759
|
+
p.add_argument("--submission", required=True)
|
|
760
|
+
p.add_argument("--seed-dir", required=True)
|
|
761
|
+
p.add_argument("--reveal-seeds", action="store_true", help="include the seeds: this IS the reveal. Only after the round is evaluated")
|
|
762
|
+
p.add_argument("--baseline")
|
|
763
|
+
p.add_argument("--results-dir")
|
|
764
|
+
p.add_argument("--round-data", help="the operator's data directory (to include the round's log)")
|
|
765
|
+
p.add_argument("--round-id")
|
|
766
|
+
p.add_argument("--private", action="store_true", help="a PRIVATE-SOLVER certificate: no solver source, no failure text (see the module docstring)")
|
|
767
|
+
p.add_argument("--out", required=True)
|
|
768
|
+
p = sub.add_parser("make-private", help="write a private-solver copy of a certificate (the original is untouched)")
|
|
769
|
+
p.add_argument("cert")
|
|
770
|
+
p.add_argument("--out", required=True)
|
|
771
|
+
p = sub.add_parser("verify")
|
|
772
|
+
p.add_argument("cert")
|
|
773
|
+
p.add_argument("--source", metavar="FILE", help="for a private-solver certificate: the solver file its owner has published; it must hash to the committed value, "
|
|
774
|
+
"and then --rerun can re-measure it")
|
|
775
|
+
p.add_argument("--rerun", type=rerun_count, default=0, metavar="K|all", help="re-measure K instances here, or `all` of them (needs --execute)")
|
|
776
|
+
p.add_argument("--execute", action="store_true", help="RUN code from the certificate on this machine (a declared domain's decl.py, and with --rerun "
|
|
777
|
+
"its solver sources, unsandboxed): needed for instance coverage of a declared domain and for --rerun. Use a throwaway VM or container.")
|
|
778
|
+
p.add_argument("--tolerance", type=float, default=RERUN_TOLERANCE, help="bits of disagreement --rerun accepts (default %(default)g, loose until cross-machine spread is measured)")
|
|
779
|
+
p.add_argument("--sample-seed", type=int, default=None, help="which instances --rerun re-measures (default: a fresh random draw, yours alone, printed in the verdict)")
|
|
780
|
+
a = ap.parse_args(argv)
|
|
781
|
+
if a.cmd == "build":
|
|
782
|
+
if os.path.exists(a.out):
|
|
783
|
+
raise SystemExit(f"REFUSED: {a.out} exists; a certificate is never overwritten")
|
|
784
|
+
try:
|
|
785
|
+
cert = build(a.family, a.submission, a.seed_dir, a.reveal_seeds, a.baseline, a.results_dir, a.round_data, a.round_id, private=a.private)
|
|
786
|
+
except (CertificateError, SystemExit, KeyError, OSError) as e:
|
|
787
|
+
raise SystemExit(f"REFUSED: {e}")
|
|
788
|
+
fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
789
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
790
|
+
json.dump(cert, f, indent=1, sort_keys=True)
|
|
791
|
+
c = cert["claim"]
|
|
792
|
+
print(f"wrote {a.out}: {c['credit_bits']} bits, {c['status']}" + ("" if cert["seeds_revealed"] else " (seeds NOT included: sealed certificate)")
|
|
793
|
+
+ (" (PRIVATE SOLVER: source withheld)" if cert.get("private") else ""))
|
|
794
|
+
elif a.cmd == "make-private":
|
|
795
|
+
if os.path.exists(a.out):
|
|
796
|
+
raise SystemExit(f"REFUSED: {a.out} exists; a certificate is never overwritten")
|
|
797
|
+
with open(a.cert, encoding="utf-8") as f:
|
|
798
|
+
cert = json.load(f)
|
|
799
|
+
try:
|
|
800
|
+
priv = make_private(cert)
|
|
801
|
+
except (CertificateError, KeyError, ValueError, TypeError, AttributeError) as e:
|
|
802
|
+
raise SystemExit(f"REFUSED: {e}")
|
|
803
|
+
fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
804
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
805
|
+
json.dump(priv, f, indent=1, sort_keys=True)
|
|
806
|
+
print(f"wrote {a.out}: the same certificate without the solver's source (verify it: certificate.py verify {a.out})")
|
|
807
|
+
else:
|
|
808
|
+
with open(a.cert, encoding="utf-8") as f:
|
|
809
|
+
cert = json.load(f)
|
|
810
|
+
source = None
|
|
811
|
+
if a.source:
|
|
812
|
+
with open(a.source, "rb") as f: # BYTES: the committed hash is of the file's exact bytes
|
|
813
|
+
source = f.read()
|
|
814
|
+
if a.sample_seed is None: # the prover must not be able to predict which instances are re-run: the default is the verifier's own random draw
|
|
815
|
+
a.sample_seed = secrets.randbelow(2 ** 31)
|
|
816
|
+
res = verify(cert, rerun=a.rerun, execute=a.execute, tolerance=a.tolerance, sample_seed=a.sample_seed, source=source)
|
|
817
|
+
for c in res["checks"]:
|
|
818
|
+
print(f" {c['status'].upper():4s} {c['check']}: {c['detail']}")
|
|
819
|
+
if res["ok"]:
|
|
820
|
+
print(f"VERIFIED: {res['claim']['credit_bits']} bits ({res['claim']['status']})"
|
|
821
|
+
+ (" [PRIVATE SOLVER: not inspectable or re-runnable here]" if res.get("private") and not res.get("source_revealed") else ""))
|
|
822
|
+
else:
|
|
823
|
+
print("NOT VERIFIED")
|
|
824
|
+
sys.exit(0 if res["ok"] else 1)
|
|
825
|
+
|
|
826
|
+
|
|
827
|
+
if __name__ == "__main__":
|
|
828
|
+
main()
|