guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1382 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Standalone checker for one challenge bundle (made by `compile_domain.py export`). Python standard library only, plus whatever the
|
|
3
|
+
declaration itself imports (challenge.json, "requires").
|
|
4
|
+
|
|
5
|
+
python3 check.py info what this challenge is
|
|
6
|
+
python3 check.py selftest does this bundle work, here, as shipped? (exit 1 if not)
|
|
7
|
+
python3 check.py commit --seed-file F sha256 of a seed file: the value to publish BEFORE the deadline
|
|
8
|
+
python3 check.py derive --seed-file F --size N --idx I the instance a solver receives (never the planted answer)
|
|
9
|
+
python3 check.py derive --practice --size N --idx I the same on the public practice seed
|
|
10
|
+
python3 check.py verify --seed-file F --size N --idx I --answer JSON exit 0 if the answer is valid, 1 if not
|
|
11
|
+
python3 check.py rerun --seed-file F --solver S.py --claimed SUB.jsonl --baseline REF.jsonl [-k 5] [--sign KEY --attest-out FILE]
|
|
12
|
+
re-measure a random sample of a CLAIMED result on this machine, sandboxed: do the claims reproduce?
|
|
13
|
+
python3 check.py keygen --out KEY a signing key for attestations; publish the public key it prints
|
|
14
|
+
python3 check.py attest-check A1.json A2.json --trusted KEYS --min 2 do independent verifiers (keys YOU pinned) attest the claim reproduces?
|
|
15
|
+
python3 check.py log append LOG --file F [--kind K] append-only hash-chained log; log verify LOG [--head H] [--checkpoint C]
|
|
16
|
+
python3 check.py log checkpoint LOG --key K --out C a SIGNED (head, count): a log that no longer contains it was truncated or forked
|
|
17
|
+
|
|
18
|
+
An answer is JSON: `7`, `[3, 5]`, `null`. `--answer @file` reads a file, `--answer -` reads stdin.
|
|
19
|
+
`derive` and `verify` need the seed, so only the operator runs them with a real seed; competitors use --practice.
|
|
20
|
+
|
|
21
|
+
WARNING: every command except `info` and `commit` IMPORTS AND RUNS checker/decl.py (and `selftest` runs checker/reference.py), unsandboxed, as you.
|
|
22
|
+
That is the author's code. Read it, or run this in a throwaway VM or container, before running a bundle you did not make. The pinned-file hash
|
|
23
|
+
check in `selftest` proves the files are the ones the manifest names, not that they are safe. `rerun` additionally EXECUTES the competitor's solver
|
|
24
|
+
file (and the reference) in child processes with a CPU limit, inside an OS sandbox by default (macOS sandbox-exec; Linux bubblewrap if present): no network, no
|
|
25
|
+
writes outside a scratch directory, the seed and your keys unreadable. That stops the practical cheats and the obvious damage; it is not a proof of isolation.
|
|
26
|
+
"""
|
|
27
|
+
import argparse
|
|
28
|
+
import hashlib
|
|
29
|
+
import importlib.util
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
import random
|
|
33
|
+
import sys
|
|
34
|
+
|
|
35
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
36
|
+
ROOT = os.path.dirname(HERE)
|
|
37
|
+
sys.path.insert(0, HERE)
|
|
38
|
+
sys.path.append(os.path.join(HERE, "vendor")) # after the standard library: a vendored file can never shadow it
|
|
39
|
+
|
|
40
|
+
PRACTICE_SEED = b"practice-seed-public"
|
|
41
|
+
MAX_ARGS_BYTES = 1 << 20
|
|
42
|
+
JUNK = [None, 0, 1, -1, True, False, "", "0", "x", [], [0], [1], [None], {}, 1.5, -0.0, 10 ** 40, [[]], ["0"]]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def int_arg(lo, hi):
|
|
46
|
+
"""argparse type: an integer in [lo, hi], else a one-line usage error (never a traceback, never an unbounded computation)."""
|
|
47
|
+
def parse(text):
|
|
48
|
+
try:
|
|
49
|
+
v = int(text)
|
|
50
|
+
except ValueError:
|
|
51
|
+
raise argparse.ArgumentTypeError(f"{text!r} is not an integer")
|
|
52
|
+
if not lo <= v <= hi:
|
|
53
|
+
raise argparse.ArgumentTypeError(f"{v} is outside the allowed range {lo}..{hi}")
|
|
54
|
+
return v
|
|
55
|
+
return parse
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def float_arg(lo, hi):
|
|
59
|
+
def parse(text):
|
|
60
|
+
import math
|
|
61
|
+
try:
|
|
62
|
+
v = float(text)
|
|
63
|
+
except ValueError:
|
|
64
|
+
raise argparse.ArgumentTypeError(f"{text!r} is not a number")
|
|
65
|
+
if not math.isfinite(v) or not lo <= v <= hi:
|
|
66
|
+
raise argparse.ArgumentTypeError(f"{text} is not a finite number in {lo}..{hi}")
|
|
67
|
+
return v
|
|
68
|
+
return parse
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def hex_arg(text):
|
|
72
|
+
try:
|
|
73
|
+
raw = bytes.fromhex(text)
|
|
74
|
+
except ValueError:
|
|
75
|
+
raise argparse.ArgumentTypeError(f"{text!r} is not hexadecimal")
|
|
76
|
+
if not 1 <= len(raw) <= 64:
|
|
77
|
+
raise argparse.ArgumentTypeError("the seed must be 1 to 64 bytes (2 to 128 hex digits)")
|
|
78
|
+
return text
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def size_idx_problem(ch, size, idx):
|
|
82
|
+
"""A size outside the challenge's declared range (or a huge index) is handed to the author's derive(), which may hang, allocate gigabytes or crash."""
|
|
83
|
+
lad = ch["ladder"]
|
|
84
|
+
sizes = list(lad["dims"]) + list(lad["practice_dims"])
|
|
85
|
+
rng = lad.get("size_range") or [min(sizes), max(sizes)]
|
|
86
|
+
if not rng[0] <= size <= rng[1]:
|
|
87
|
+
return f"size {size} is outside this challenge's declared range {rng[0]}..{rng[1]}"
|
|
88
|
+
if not 0 <= idx < 1_000_000:
|
|
89
|
+
return f"index {idx} is outside 0..999999"
|
|
90
|
+
return None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def manifest():
|
|
94
|
+
with open(os.path.join(ROOT, "challenge.json"), encoding="utf-8") as f:
|
|
95
|
+
return json.load(f)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class FileRefused(ValueError):
|
|
99
|
+
"""A path that is not a regular file within the size limit. Opening or reading a FIFO blocks for ever and /dev/zero never ends (found by probing: every reader
|
|
100
|
+
hung on a FIFO, and a 300 MB single-line file took 626 MB); a symlink in an uploaded tree can point at either."""
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
LIMIT_SMALL = 1 << 20 # seeds, key lists, keys
|
|
104
|
+
LIMIT_JSON = 16 << 20 # results, attestations, checkpoints, measurements
|
|
105
|
+
LIMIT_LADDER = 32 << 20 # 20,000 records of ~150 bytes is 3 MB
|
|
106
|
+
LIMIT_LOG = 64 << 20
|
|
107
|
+
LIMIT_ANSWER = 64 << 20
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _flock(fd, exclusive, required):
|
|
111
|
+
"""flock. Appending MUST hold the exclusive lock across read, verify and write (eight concurrent appends without it wrote 4 entries, broke the chain and lost 4
|
|
112
|
+
submissions); reading takes a shared lock so it never sees a half-written line. Needs POSIX; where there is none a read goes unlocked, an append refuses."""
|
|
113
|
+
try:
|
|
114
|
+
import fcntl
|
|
115
|
+
except ImportError:
|
|
116
|
+
if required:
|
|
117
|
+
raise SystemExit("appending to a log needs POSIX file locking (fcntl), which this system does not have")
|
|
118
|
+
return
|
|
119
|
+
fcntl.flock(fd, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _open_regular(path, limit):
|
|
123
|
+
"""-> (fd, size) of a regular file no bigger than `limit`, opened WITHOUT blocking (a FIFO opens at once and is then refused), else FileRefused."""
|
|
124
|
+
import stat
|
|
125
|
+
try:
|
|
126
|
+
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)) # no O_NONBLOCK on Windows (there are no FIFOs to block on); O_BINARY avoids text-mode newline mangling there
|
|
127
|
+
except OSError as e:
|
|
128
|
+
raise FileRefused(f"cannot read {path}: {e.strerror}")
|
|
129
|
+
st = os.fstat(fd)
|
|
130
|
+
if not stat.S_ISREG(st.st_mode):
|
|
131
|
+
os.close(fd)
|
|
132
|
+
raise FileRefused(f"{path} is not a regular file (a FIFO, device, socket or directory is refused: reading it could block or never end)")
|
|
133
|
+
if limit is not None and st.st_size > limit:
|
|
134
|
+
os.close(fd)
|
|
135
|
+
raise FileRefused(f"{path} is {st.st_size} bytes, over the {limit} byte limit for this kind of file")
|
|
136
|
+
return fd
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def read_bytes(path, limit, shared_lock=False):
|
|
140
|
+
fd = _open_regular(path, limit)
|
|
141
|
+
try:
|
|
142
|
+
if shared_lock:
|
|
143
|
+
_flock(fd, False, False)
|
|
144
|
+
chunks, total = [], 0
|
|
145
|
+
while True:
|
|
146
|
+
chunk = os.read(fd, 1 << 20)
|
|
147
|
+
if not chunk:
|
|
148
|
+
break
|
|
149
|
+
total += len(chunk)
|
|
150
|
+
if total > limit: # it grew while we read it
|
|
151
|
+
raise FileRefused(f"{path} grew past {limit} bytes while being read")
|
|
152
|
+
chunks.append(chunk)
|
|
153
|
+
return b"".join(chunks)
|
|
154
|
+
finally:
|
|
155
|
+
os.close(fd)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def read_text(path, limit, shared_lock=False):
|
|
159
|
+
data = read_bytes(path, limit, shared_lock)
|
|
160
|
+
try:
|
|
161
|
+
return data.decode("utf-8")
|
|
162
|
+
except UnicodeDecodeError:
|
|
163
|
+
raise FileRefused(f"{path} is not UTF-8 text")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def sha256_file(path):
|
|
167
|
+
"""Streamed, and only for a regular file (hashing /dev/zero would never finish)."""
|
|
168
|
+
fd = _open_regular(path, None)
|
|
169
|
+
try:
|
|
170
|
+
h = hashlib.sha256()
|
|
171
|
+
while True:
|
|
172
|
+
chunk = os.read(fd, 1 << 20)
|
|
173
|
+
if not chunk:
|
|
174
|
+
return h.hexdigest()
|
|
175
|
+
h.update(chunk)
|
|
176
|
+
finally:
|
|
177
|
+
os.close(fd)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def file_problems(ch):
|
|
181
|
+
"""Every file the challenge pins must hash as recorded. [] = the bundle is intact."""
|
|
182
|
+
out = []
|
|
183
|
+
for rel, want in sorted(ch["files"].items()):
|
|
184
|
+
p = os.path.join(ROOT, rel)
|
|
185
|
+
if not os.path.isfile(p):
|
|
186
|
+
out.append(f"missing: {rel}")
|
|
187
|
+
elif sha256_file(p) != want:
|
|
188
|
+
out.append(f"changed: {rel}")
|
|
189
|
+
return out
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def load_decl():
|
|
193
|
+
import decl
|
|
194
|
+
return decl
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def load_reference(ch):
|
|
198
|
+
path = os.path.join(ROOT, ch["reference"]["file"])
|
|
199
|
+
spec = importlib.util.spec_from_file_location("bundle_reference", path)
|
|
200
|
+
mod = importlib.util.module_from_spec(spec)
|
|
201
|
+
spec.loader.exec_module(mod)
|
|
202
|
+
return mod
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def master_from(a):
|
|
206
|
+
if a.practice:
|
|
207
|
+
return PRACTICE_SEED
|
|
208
|
+
if a.seed_file:
|
|
209
|
+
return read_bytes(a.seed_file, LIMIT_SMALL).strip()
|
|
210
|
+
raise SystemExit("give --seed-file F (the operator's seed) or --practice")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def instance(decl, master, size, idx):
|
|
214
|
+
inst = dict(decl.derive(master, size, idx))
|
|
215
|
+
inst.setdefault("id", f"{size}:{idx}")
|
|
216
|
+
inst.setdefault("dim", size)
|
|
217
|
+
inst.setdefault("idx", idx)
|
|
218
|
+
return inst
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def args_problem(inst):
|
|
222
|
+
try:
|
|
223
|
+
text = json.dumps(inst["args"])
|
|
224
|
+
except (TypeError, ValueError) as e:
|
|
225
|
+
return f"args are not JSON-serializable ({e})"
|
|
226
|
+
if not isinstance(inst["args"], list):
|
|
227
|
+
return "args is not a list"
|
|
228
|
+
if len(text) > MAX_ARGS_BYTES:
|
|
229
|
+
return f"args are {len(text)} bytes (limit {MAX_ARGS_BYTES})"
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def read_answer(text):
|
|
234
|
+
if text == "-":
|
|
235
|
+
text = sys.stdin.read()
|
|
236
|
+
elif text.startswith("@"):
|
|
237
|
+
text = read_text(text[1:], LIMIT_ANSWER)
|
|
238
|
+
try:
|
|
239
|
+
return json.loads(text)
|
|
240
|
+
except (ValueError, RecursionError) as e:
|
|
241
|
+
raise SystemExit(f"--answer is not usable JSON: {type(e).__name__}: {str(e)[:100]}")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def safe_verify(decl, inst, ans, report=False):
|
|
245
|
+
"""A verifier that raises on junk rejects it: a crash must never count as a valid answer. `report` says so on stderr (a real answer that
|
|
246
|
+
crashes the verifier is a bug in the declaration the operator should see, not a quiet 'invalid')."""
|
|
247
|
+
try:
|
|
248
|
+
return bool(decl.verify(inst, ans))
|
|
249
|
+
except Exception as e:
|
|
250
|
+
if report:
|
|
251
|
+
print(f"note: verify() raised {type(e).__name__}: {e}; counted as invalid", file=sys.stderr)
|
|
252
|
+
return False
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def cmd_info(a):
|
|
256
|
+
ch = manifest()
|
|
257
|
+
print(f"{ch['name']} ({ch['version']}), bundle status {ch['status']}")
|
|
258
|
+
print(ch["problem"]["doc"])
|
|
259
|
+
if ch.get("metric") == "quality": # answers are scored, not the clock: the fields below are the speed ones
|
|
260
|
+
pw = ch["power"]
|
|
261
|
+
print(f"solver contract: {ch['solver_contract']}")
|
|
262
|
+
print(f"QUALITY challenge: sizes {ch['ladder']['dims']}, {ch['ladder']['n']} instances per size, {ch['ladder']['budget_seconds']} CPU s per instance for everyone, "
|
|
263
|
+
f"{ch['protocol']['seeds_required']} committed seeds; tools: qcheck.py, qscore.py")
|
|
264
|
+
print(f"certifiable mean gain: {pw['min_certifiable']} to {pw['max_certifiable']} quality points (None = this ladder cannot certify)")
|
|
265
|
+
print(f"reference: {ch['reference']['note'] or 'NOT STATED'}")
|
|
266
|
+
return 0
|
|
267
|
+
print(f"solver contract: {ch['solver_contract']}")
|
|
268
|
+
print(f"sizes {ch['ladder']['dims']}, {ch['ladder']['n']} instances per size, CPU cap {ch['ladder']['budget_seconds']} s, "
|
|
269
|
+
f"{ch['protocol']['seeds_required']} committed seeds")
|
|
270
|
+
pw = ch["power"]
|
|
271
|
+
print(f"certifiable speedup range: {pw['min_certifiable_bits']} to {pw['max_certifiable_bits']} bits (None = this ladder cannot certify)")
|
|
272
|
+
print(f"reference: {ch['reference']['note'] or 'NOT STATED'}")
|
|
273
|
+
return 0
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def cmd_commit(a):
|
|
277
|
+
print(hashlib.sha256(master_from(a)).hexdigest())
|
|
278
|
+
return 0
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def cmd_derive(a):
|
|
282
|
+
bad = size_idx_problem(manifest(), a.size, a.idx)
|
|
283
|
+
if bad:
|
|
284
|
+
raise SystemExit(bad)
|
|
285
|
+
decl = load_decl()
|
|
286
|
+
master = master_from(a)
|
|
287
|
+
inst = instance(decl, master, a.size, a.idx)
|
|
288
|
+
bad = args_problem(inst)
|
|
289
|
+
if bad:
|
|
290
|
+
raise SystemExit(bad)
|
|
291
|
+
print(json.dumps({"commit": hashlib.sha256(master).hexdigest(), "id": inst["id"], "size": a.size, "idx": a.idx, "args": inst["args"]}))
|
|
292
|
+
return 0
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def cmd_verify(a):
|
|
296
|
+
bad = size_idx_problem(manifest(), a.size, a.idx)
|
|
297
|
+
if bad:
|
|
298
|
+
raise SystemExit(bad)
|
|
299
|
+
decl = load_decl()
|
|
300
|
+
inst = instance(decl, master_from(a), a.size, a.idx)
|
|
301
|
+
ok = safe_verify(decl, inst, read_answer(a.answer), report=True)
|
|
302
|
+
print(json.dumps({"valid": ok}))
|
|
303
|
+
return 0 if ok else 1
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
RUNNER = r'''
|
|
307
|
+
import importlib.util, json, sys
|
|
308
|
+
path = sys.argv[1]
|
|
309
|
+
args = json.loads(sys.stdin.read())
|
|
310
|
+
real_out, sys.stdout = sys.stdout, sys.stderr
|
|
311
|
+
spec = importlib.util.spec_from_file_location("rerun_solver", path)
|
|
312
|
+
mod = importlib.util.module_from_spec(spec)
|
|
313
|
+
spec.loader.exec_module(mod)
|
|
314
|
+
ans = mod.solve(*args)
|
|
315
|
+
real_out.write("\n" + json.dumps(ans) + "\n")
|
|
316
|
+
'''
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def sandbox_prefix(mode, deny_read, scratch):
|
|
320
|
+
"""argv prefix that runs a solver with no network, no writes outside `scratch`, and no reads of `deny_read` (the seed, signing keys) or of ~/.ssh, ~/.aws, ~/.gnupg,
|
|
321
|
+
the keychain. macOS: sandbox-exec (deprecated by Apple but present; TESTED here). Linux: bubblewrap if installed (WRITTEN, NOT RUN by its author on a real Linux
|
|
322
|
+
kernel; run `make test` there to find out: the sandbox tests run wherever a sandbox actually works). None = nothing available.
|
|
323
|
+
It stops the practical cheats (read the seed, phone home, write a lookup table). It is not a proof of isolation: CPU time, memory and the kernel are shared."""
|
|
324
|
+
if mode == "off":
|
|
325
|
+
return []
|
|
326
|
+
import shutil
|
|
327
|
+
real = lambda p: os.path.realpath(p) # noqa: E731 (/tmp and /var are symlinks on macOS; the sandbox matches real paths)
|
|
328
|
+
exe, base = real(sys.executable), real(sys.base_prefix) # macOS framework Python re-executes itself via Python.app: allow its whole installation, nothing else
|
|
329
|
+
paths = [real(p) for p in deny_read if p] + [real(scratch)]
|
|
330
|
+
if any(c in p for p in paths + [exe, base] for c in '"\\\n'):
|
|
331
|
+
raise SystemExit("a path contains a quote, backslash or newline: refusing to build a sandbox profile from it")
|
|
332
|
+
home = real(os.path.expanduser("~"))
|
|
333
|
+
sb = shutil.which("sandbox-exec") if sys.platform == "darwin" else None
|
|
334
|
+
if sb:
|
|
335
|
+
denies = " ".join([f'(literal "{p}")' for p in paths[:-1]] + [f'(subpath "{home}/{d}")' for d in CREDENTIAL_DIRS])
|
|
336
|
+
# No exec outside the interpreter: with it, a solver could run `open` (hand a URL to LaunchServices, i.e. network egress through ANOTHER process) or `sh`.
|
|
337
|
+
# Found by trying: /bin/sh and open(1) both ran under the first version of this profile. Solvers are pure Python; they have no use for exec.
|
|
338
|
+
profile = (f'(version 1)(allow default)(deny network*)(deny file-write*)(allow file-write* (subpath "{paths[-1]}") (literal "/dev/null"))'
|
|
339
|
+
f'(deny file-read* {denies})(deny process-exec*)(allow process-exec (literal "{exe}") (subpath "{base}"))'
|
|
340
|
+
# No Mach service lookups: a solver using ctypes could otherwise talk to LaunchServices, the pasteboard or preferences daemons directly (measured: all
|
|
341
|
+
# reachable under the earlier profile, all denied under this one). Only user-NAME lookups stay (pwd.getpwuid / getpass.getuser fail without it).
|
|
342
|
+
'(deny mach-lookup)(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo"))')
|
|
343
|
+
return [sb, "-p", profile]
|
|
344
|
+
bwrap = shutil.which("bwrap") if sys.platform.startswith("linux") else None
|
|
345
|
+
if bwrap:
|
|
346
|
+
# Everything read-only, scratch writable, no network, own pid/ipc namespaces, its own session (without --new-session a sandboxed process can push keystrokes
|
|
347
|
+
# into your terminal with TIOCSTI). NOT `--tmpfs /tmp`: an earlier draft had it, which hides every solver and bundle that lives under /tmp from the sandbox
|
|
348
|
+
# (all of the tests', and any CI checkout in a temp dir). Credential directories are masked with an empty tmpfs; the seed and keys with /dev/null.
|
|
349
|
+
# --as-pid-1 is not optional: with --unshare-pid alone bubblewrap puts a reaper between us and the program, and the kernel's CPU accounting for the program
|
|
350
|
+
# VANISHES (measured on a real Linux 7.0 kernel: a 2 s burn read as 0.00 s, so every `rerun` "measured" 0 and could pass or fail by accident). Running the program
|
|
351
|
+
# itself as the namespace's init keeps the accounting (2.03 s) AND the PID namespace (a solver cannot see or signal your other processes).
|
|
352
|
+
argv = [bwrap, "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc", "--bind", paths[-1], paths[-1], "--unshare-net", "--unshare-pid", "--as-pid-1",
|
|
353
|
+
"--unshare-ipc", "--new-session", "--die-with-parent"]
|
|
354
|
+
for d in CREDENTIAL_DIRS:
|
|
355
|
+
if os.path.isdir(os.path.join(home, d)):
|
|
356
|
+
argv += ["--tmpfs", os.path.join(home, d)]
|
|
357
|
+
for p in paths[:-1]:
|
|
358
|
+
argv += ["--ro-bind", "/dev/null", p]
|
|
359
|
+
return argv
|
|
360
|
+
return None
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def working_sandbox(mode, deny_read, scratch):
|
|
364
|
+
"""-> (prefix, why_not). prefix is [] for mode "off", None when no sandbox can be used, else an argv prefix that was VERIFIED by running Python under it once.
|
|
365
|
+
A sandbox tool that is installed but cannot run (Ubuntu 24.04 restricts unprivileged user namespaces, so bubblewrap fails there; sandbox-exec cannot nest inside
|
|
366
|
+
another sandbox) must read as 'unavailable, and why', not as a mysterious failure of every command."""
|
|
367
|
+
prefix = sandbox_prefix(mode, deny_read, scratch)
|
|
368
|
+
if not prefix:
|
|
369
|
+
return prefix, None
|
|
370
|
+
import subprocess
|
|
371
|
+
try:
|
|
372
|
+
r = subprocess.run(prefix + [sys.executable, "-I", "-B", "-c", "pass"], capture_output=True, text=True, timeout=30)
|
|
373
|
+
except (OSError, subprocess.TimeoutExpired) as e:
|
|
374
|
+
return None, f"{type(e).__name__}: {e}"
|
|
375
|
+
if r.returncode != 0:
|
|
376
|
+
lines = [ln for ln in (r.stderr or r.stdout).splitlines() if ln.strip()]
|
|
377
|
+
return None, (lines[-1][:200] if lines else f"exit {r.returncode}")
|
|
378
|
+
return prefix, None
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
CREDENTIAL_DIRS = (".ssh", ".aws", ".gnupg", ".config/gcloud", "Library/Keychains")
|
|
382
|
+
DENIAL_WORDS = ("Permission", "Operation not permitted", "Read-only file system")
|
|
383
|
+
TOOL_ERRORS = ("bwrap:", "sandbox-exec:") # a sandbox tool that could not start prints its own error first; the command it wraps never ran
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
class RunResult(tuple):
|
|
387
|
+
"""(status, answer, cpu seconds, stderr tail), still unpackable as 4 values, that also carries `.wall`: the wall-clock seconds of the run."""
|
|
388
|
+
wall = None
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
ABSOLUTE_SHARE = 0.3 # with no honest run to compare against: the least CPU/wall share a CPU-bound solver can have
|
|
392
|
+
RELATIVE_SHARE = 0.6 # against the reference, which ran under the same load a moment earlier: the least fraction of ITS share the solver may have
|
|
393
|
+
REFERENCE_SHARE = 0.15 # the reference is the author's own code; only an absurdly low share suggests its meter is blind
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def hides_cpu(run, cpu, ref=None, limit=None):
|
|
397
|
+
"""`ref` = (the reference's run, its CPU), measured on the same instance under the same load. When it ran long enough to have a meaningful share (over 0.5 s), the solver
|
|
398
|
+
is flagged against RELATIVE_SHARE of that share instead of the fixed ABSOLUTE_SHARE: a loaded machine starves both alike (no false failure), and a solver that pads its CPU
|
|
399
|
+
to just over 30% to hide the rest (up to 1.7 bits of fake speedup) is caught once it falls under ~60% of what the reference got (under ~0.8 bit). `limit` overrides all of it.
|
|
400
|
+
The rule: """
|
|
401
|
+
"""True when a run took over half a second of wall time (the estimator ignores costs below FLOOR_ABS = 0.5 s, so shorter hidden work cannot move a score; an empty solver takes ~0.03 s, a sandboxed one ~0.1 s) but its CPU is under 30% of that. A CPU-bound solver cannot do that honestly; one that WAITS (sleeps, polls) or whose
|
|
402
|
+
work runs in a process the parent never reaps can. Linux credits a child's CPU to its parent only when the parent waits for it, so a solver that forks the work and returns
|
|
403
|
+
the answer through a pipe without waiting escapes `RUSAGE_CHILDREN` entirely (macOS credits it at the child's exit, so the attack fails there: the accounting differs by OS,
|
|
404
|
+
the wall clock does not)."""
|
|
405
|
+
wall = getattr(run, "wall", None)
|
|
406
|
+
if wall is None or wall <= 0.5:
|
|
407
|
+
return False
|
|
408
|
+
if limit is None:
|
|
409
|
+
limit = ABSOLUTE_SHARE
|
|
410
|
+
ref_wall = getattr(ref[0], "wall", None) if ref else None
|
|
411
|
+
if ref_wall is not None and ref_wall > 0.5:
|
|
412
|
+
limit = RELATIVE_SHARE * min(1.0, ref[1] / ref_wall)
|
|
413
|
+
return cpu < limit * wall
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def run_solver(path, args, cap, prefix=(), cwd=None):
|
|
417
|
+
"""One solve() in a fresh process: -> (status, answer, cpu seconds of the process tree). The CPU cap is enforced with RLIMIT_CPU and a wall backstop.
|
|
418
|
+
`prefix` is a sandbox_prefix(); without one the solver runs as you and can do anything you can."""
|
|
419
|
+
import resource
|
|
420
|
+
import subprocess
|
|
421
|
+
|
|
422
|
+
def limit():
|
|
423
|
+
resource.setrlimit(resource.RLIMIT_CPU, (int(cap) + 1, int(cap) + 1))
|
|
424
|
+
env = {"PATH": os.environ.get("PATH", ""), "HOME": os.environ.get("HOME", ""), "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "MKL_NUM_THREADS": "1",
|
|
425
|
+
"VECLIB_MAXIMUM_THREADS": "1", "PYTHONHASHSEED": "0"}
|
|
426
|
+
import time
|
|
427
|
+
t_wall = time.monotonic()
|
|
428
|
+
before = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
429
|
+
status, answer, tail = "crash", None, ""
|
|
430
|
+
try:
|
|
431
|
+
r = subprocess.run(list(prefix) + [sys.executable, "-I", "-B", "-c", RUNNER, os.path.abspath(path)], input=json.dumps(args), capture_output=True, text=True,
|
|
432
|
+
timeout=cap * 3 + 10, env=env, preexec_fn=limit, cwd=cwd)
|
|
433
|
+
err_lines = [ln for ln in r.stderr.splitlines() if ln.strip()]
|
|
434
|
+
tail = err_lines[-1][:200] if err_lines else ""
|
|
435
|
+
lines = [ln for ln in r.stdout.splitlines() if ln.strip()]
|
|
436
|
+
if r.returncode == 0 and lines:
|
|
437
|
+
answer, status = json.loads(lines[-1]), "ran"
|
|
438
|
+
elif r.returncode in (-24, -9, 128 + 24, 128 + 9): # SIGXCPU / SIGKILL, directly or reported by a wrapper (bubblewrap exits 137 when its program is killed)
|
|
439
|
+
status = "timeout"
|
|
440
|
+
except subprocess.TimeoutExpired:
|
|
441
|
+
status = "timeout"
|
|
442
|
+
except ValueError:
|
|
443
|
+
status = "crash"
|
|
444
|
+
after = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
445
|
+
result = RunResult((status, answer, (after.ru_utime - before.ru_utime) + (after.ru_stime - before.ru_stime), tail))
|
|
446
|
+
result.wall = time.monotonic() - t_wall
|
|
447
|
+
return result
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def solver_overhead(tmp_path, prefix=(), cwd=None):
|
|
451
|
+
with open(tmp_path, "w") as f:
|
|
452
|
+
f.write("def solve(*a):\n return None\n")
|
|
453
|
+
runs = [run_solver(tmp_path, [], 30, prefix, cwd) for _ in range(5)]
|
|
454
|
+
if any(r[0] != "ran" for r in runs):
|
|
455
|
+
raise SystemExit("the sandbox or the interpreter cannot run even an empty solver here: " + (runs[0][3] or runs[0][0]))
|
|
456
|
+
overhead = sorted(r[2] for r in runs)[2]
|
|
457
|
+
if overhead < 0.002: # starting Python costs ~10-50 ms of CPU anywhere; ~0 means the meter is not seeing the process (found on Linux under bwrap --unshare-pid)
|
|
458
|
+
raise SystemExit(f"CPU accounting does not work here: an empty Python start measured {overhead * 1000:.2f} ms of CPU (it costs 10 ms or more). Every re-measurement would read ~0 "
|
|
459
|
+
"and 'pass' or 'fail' by accident, so rerun refuses. This sandbox configuration hides the program's CPU time from the parent.")
|
|
460
|
+
return overhead
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def load_costs(path, master):
|
|
464
|
+
"""{id: cost} of the SOLVED records of a ladder; refuses a ladder whose commitment is not this seed's."""
|
|
465
|
+
costs, commit = {}, None
|
|
466
|
+
for n, line in enumerate(read_text(path, LIMIT_LADDER).splitlines(), 1):
|
|
467
|
+
if not line.strip():
|
|
468
|
+
continue
|
|
469
|
+
try:
|
|
470
|
+
r = json.loads(line)
|
|
471
|
+
if r.get("meta"):
|
|
472
|
+
commit = r.get("commit")
|
|
473
|
+
elif r.get("status") == "solved":
|
|
474
|
+
costs[str(r["id"])] = float(r.get("cpu_net", r.get("cost")))
|
|
475
|
+
except (ValueError, TypeError, AttributeError, OverflowError, RecursionError):
|
|
476
|
+
raise SystemExit(f"{path}:{n}: not a ladder record")
|
|
477
|
+
if commit is not None and commit != hashlib.sha256(master).hexdigest():
|
|
478
|
+
raise SystemExit(f"{path} was run on a different seed (its commitment is not this seed's sha256)")
|
|
479
|
+
return costs
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
# ---- Ed25519 (RFC 8032 section 6), pure Python, so signing needs no package. Slow and not constant-time: fine for signing one short statement on your own
|
|
483
|
+
# machine, and for checking signatures; do not use it for anything else.
|
|
484
|
+
_P = 2 ** 255 - 19
|
|
485
|
+
_Q = 2 ** 252 + 27742317777372353535851937790883648493
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _inv(x):
|
|
489
|
+
return pow(x, _P - 2, _P)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
_D = -121665 * _inv(121666) % _P
|
|
493
|
+
_SQRT_M1 = pow(2, (_P - 1) // 4, _P)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _padd(p, q):
|
|
497
|
+
a, b = (p[1] - p[0]) * (q[1] - q[0]) % _P, (p[1] + p[0]) * (q[1] + q[0]) % _P
|
|
498
|
+
c, d = 2 * p[3] * q[3] * _D % _P, 2 * p[2] * q[2] % _P
|
|
499
|
+
e, f, g, h = b - a, d - c, d + c, b + a
|
|
500
|
+
return (e * f % _P, g * h % _P, f * g % _P, e * h % _P)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _pmul(s, p):
|
|
504
|
+
q = (0, 1, 1, 0)
|
|
505
|
+
while s > 0:
|
|
506
|
+
if s & 1:
|
|
507
|
+
q = _padd(q, p)
|
|
508
|
+
p = _padd(p, p)
|
|
509
|
+
s >>= 1
|
|
510
|
+
return q
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _peq(p, q):
|
|
514
|
+
return (p[0] * q[2] - q[0] * p[2]) % _P == 0 and (p[1] * q[2] - q[1] * p[2]) % _P == 0
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _recover_x(y, sign):
|
|
518
|
+
if y >= _P:
|
|
519
|
+
return None
|
|
520
|
+
x2 = (y * y - 1) * _inv(_D * y * y + 1) % _P
|
|
521
|
+
if x2 == 0:
|
|
522
|
+
return None if sign else 0
|
|
523
|
+
x = pow(x2, (_P + 3) // 8, _P)
|
|
524
|
+
if (x * x - x2) % _P != 0:
|
|
525
|
+
x = x * _SQRT_M1 % _P
|
|
526
|
+
if (x * x - x2) % _P != 0:
|
|
527
|
+
return None
|
|
528
|
+
return _P - x if (x & 1) != sign else x
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
_GY = 4 * _inv(5) % _P
|
|
532
|
+
_GX = _recover_x(_GY, 0)
|
|
533
|
+
_G = (_GX, _GY, 1, _GX * _GY % _P)
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _compress(p):
|
|
537
|
+
zi = _inv(p[2])
|
|
538
|
+
x, y = p[0] * zi % _P, p[1] * zi % _P
|
|
539
|
+
return int.to_bytes(y | ((x & 1) << 255), 32, "little")
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _decompress(s):
|
|
543
|
+
if len(s) != 32:
|
|
544
|
+
return None
|
|
545
|
+
y = int.from_bytes(s, "little")
|
|
546
|
+
sign, y = y >> 255, y & ((1 << 255) - 1)
|
|
547
|
+
x = _recover_x(y, sign)
|
|
548
|
+
return None if x is None else (x, y, 1, x * y % _P)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _h512(m):
|
|
552
|
+
return hashlib.sha512(m).digest()
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _expand(secret):
|
|
556
|
+
h = _h512(secret)
|
|
557
|
+
a = int.from_bytes(h[:32], "little")
|
|
558
|
+
a &= (1 << 254) - 8
|
|
559
|
+
a |= 1 << 254
|
|
560
|
+
return a, h[32:]
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def ed25519_public(secret):
|
|
564
|
+
return _compress(_pmul(_expand(secret)[0], _G))
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def ed25519_sign(secret, msg):
|
|
568
|
+
a, prefix = _expand(secret)
|
|
569
|
+
pub = _compress(_pmul(a, _G))
|
|
570
|
+
r = int.from_bytes(_h512(prefix + msg), "little") % _Q
|
|
571
|
+
rs = _compress(_pmul(r, _G))
|
|
572
|
+
h = int.from_bytes(_h512(rs + pub + msg), "little") % _Q
|
|
573
|
+
return rs + int.to_bytes((r + h * a) % _Q, 32, "little")
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def ed25519_verify(public, msg, sig):
|
|
577
|
+
if len(public) != 32 or len(sig) != 64:
|
|
578
|
+
return False
|
|
579
|
+
a_pt, r_pt = _decompress(public), _decompress(sig[:32])
|
|
580
|
+
s = int.from_bytes(sig[32:], "little")
|
|
581
|
+
if a_pt is None or r_pt is None or s >= _Q:
|
|
582
|
+
return False
|
|
583
|
+
h = int.from_bytes(_h512(sig[:32] + public + msg), "little") % _Q
|
|
584
|
+
return _peq(_pmul(s, _G), _padd(r_pt, _pmul(h, a_pt)))
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def canonical(obj):
|
|
588
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode()
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
ATTEST_FORMAT = "guldo-bundle-attestation-v1"
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def cmd_keygen(a):
|
|
595
|
+
if os.path.exists(a.out):
|
|
596
|
+
raise SystemExit(f"{a.out} exists: this tool never overwrites a key")
|
|
597
|
+
secret = os.urandom(32)
|
|
598
|
+
fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
599
|
+
with os.fdopen(fd, "w") as f:
|
|
600
|
+
f.write(secret.hex() + "\n")
|
|
601
|
+
print(f"wrote {a.out} (keep it private; mode 600)")
|
|
602
|
+
print("public key (publish it, out of band, wherever people look for who you are):")
|
|
603
|
+
print(ed25519_public(secret).hex())
|
|
604
|
+
return 0
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def read_key(path):
|
|
608
|
+
try:
|
|
609
|
+
secret = bytes.fromhex(read_text(path, LIMIT_SMALL).strip())
|
|
610
|
+
except (OSError, ValueError) as e:
|
|
611
|
+
raise SystemExit(f"cannot read a key from {path}: {e}")
|
|
612
|
+
if len(secret) != 32:
|
|
613
|
+
raise SystemExit(f"{path} does not hold a 32-byte hex key")
|
|
614
|
+
return secret
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def write_attestation(path, statement, secret):
|
|
618
|
+
sig = ed25519_sign(secret, canonical(statement))
|
|
619
|
+
try:
|
|
620
|
+
f = open(path, "x", encoding="utf-8") # exclusive create: never overwrites, with no race
|
|
621
|
+
except FileExistsError:
|
|
622
|
+
raise SystemExit(f"{path} exists: this tool never overwrites")
|
|
623
|
+
except OSError as e:
|
|
624
|
+
raise SystemExit(f"cannot write {path}: {e.strerror}")
|
|
625
|
+
with f:
|
|
626
|
+
json.dump({"statement": statement, "public_key": ed25519_public(secret).hex(), "signature": sig.hex()}, f, indent=1, sort_keys=True)
|
|
627
|
+
print(f"wrote {path}; signed by {ed25519_public(secret).hex()[:16]}...")
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def cmd_attest_check(a):
|
|
631
|
+
"""Verify signed rerun attestations. A signature proves WHO signed, not that they are independent of the operator: only keys you pinned in --trusted
|
|
632
|
+
(obtained out of band) count as vouched. A valid dissent (outcome FAIL) from ANY key makes this fail."""
|
|
633
|
+
ch = manifest()
|
|
634
|
+
chsha = sha256_file(os.path.join(ROOT, "challenge.json"))
|
|
635
|
+
trusted = read_trusted(a.trusted)
|
|
636
|
+
seen, passes, dissent, problems = set(), set(), [], []
|
|
637
|
+
for path in a.files:
|
|
638
|
+
try:
|
|
639
|
+
rec = json.loads(read_text(path, LIMIT_JSON))
|
|
640
|
+
st, pub, sig = rec["statement"], rec["public_key"].lower(), bytes.fromhex(rec["signature"])
|
|
641
|
+
assert isinstance(st, dict) and st.get("format") == ATTEST_FORMAT and st.get("outcome") in ("PASS", "FAIL")
|
|
642
|
+
except (OSError, ValueError, KeyError, AssertionError, TypeError, AttributeError, RecursionError, OverflowError):
|
|
643
|
+
problems.append(f"{path}: not an attestation this tool writes (ignored)")
|
|
644
|
+
continue
|
|
645
|
+
try:
|
|
646
|
+
good = ed25519_verify(bytes.fromhex(pub), canonical(st), sig)
|
|
647
|
+
except ValueError:
|
|
648
|
+
good = False
|
|
649
|
+
if not good:
|
|
650
|
+
problems.append(f"{path}: the signature does not verify (ignored)")
|
|
651
|
+
continue
|
|
652
|
+
about = st.get("challenge")
|
|
653
|
+
if not isinstance(about, dict) or about.get("challenge_json_sha256") != chsha:
|
|
654
|
+
problems.append(f"{path}: signed for a different challenge bundle than this one (ignored)")
|
|
655
|
+
continue
|
|
656
|
+
bound = []
|
|
657
|
+
for key, given in (("claimed_sha256", a.claimed), ("baseline_sha256", a.baseline)):
|
|
658
|
+
if given:
|
|
659
|
+
try:
|
|
660
|
+
want = sha256_file(given)
|
|
661
|
+
except OSError as e:
|
|
662
|
+
raise SystemExit(f"cannot read {given}: {e.strerror}")
|
|
663
|
+
if st.get(key) != want:
|
|
664
|
+
bound.append(key.split("_")[0])
|
|
665
|
+
if bound:
|
|
666
|
+
problems.append(f"{path}: refers to different {'/'.join(bound)} ladder(s) than the ones you gave (ignored)")
|
|
667
|
+
continue
|
|
668
|
+
vouched = pub in trusted
|
|
669
|
+
machine = st.get("machine") if isinstance(st.get("machine"), dict) else {}
|
|
670
|
+
show = lambda v: str(v)[:60].replace("\n", " ") # noqa: E731 (a signer controls these fields: bound what is printed)
|
|
671
|
+
print(f"{path}: valid signature by {pub[:16]}... ({'PINNED' if vouched else 'unvouched'}), outcome {st['outcome']}, sample {show(st.get('sample_seed', '?'))}, "
|
|
672
|
+
+ (f"median gap {show(st.get('median_gap_quality', '?'))} quality points" if "median_gap_quality" in st else f"median gap {show(st.get('median_gap_bits', '?'))} bits")
|
|
673
|
+
+ f" on {show(machine.get('platform', '?'))}")
|
|
674
|
+
if st["outcome"] == "FAIL":
|
|
675
|
+
dissent.append(path)
|
|
676
|
+
elif vouched and pub not in seen:
|
|
677
|
+
passes.add(pub)
|
|
678
|
+
seen.add(pub)
|
|
679
|
+
for p in problems:
|
|
680
|
+
print("note: " + p)
|
|
681
|
+
ok = not dissent and len(passes) >= a.min
|
|
682
|
+
print(f"{len(passes)} distinct PINNED key(s) attest the claim reproduces (need {a.min}); {len(dissent)} valid dissent(s)")
|
|
683
|
+
if dissent:
|
|
684
|
+
print("FAIL: a valid signed attestation says the claim does NOT reproduce: " + ", ".join(dissent))
|
|
685
|
+
print("ATTESTATION " + ("OK" if ok else "NOT ESTABLISHED"))
|
|
686
|
+
return 0 if ok else 1
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
# ---- an append-only, hash-chained log: tamper-evidence for a list of submissions, results or attestations. Without a head hash published somewhere the
|
|
690
|
+
# operator cannot edit, a truncated copy still verifies; the time in an entry is the logger's clock, not evidence.
|
|
691
|
+
def entry_hash(entry):
|
|
692
|
+
return hashlib.sha256(canonical({k: v for k, v in entry.items() if k != "hash"})).hexdigest()
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def lock_file(f, exclusive, required):
|
|
696
|
+
_flock(f.fileno(), exclusive, required)
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def parse_log(text, path):
|
|
700
|
+
entries = []
|
|
701
|
+
for n, line in enumerate(text.splitlines(), 1):
|
|
702
|
+
if not line.strip():
|
|
703
|
+
continue
|
|
704
|
+
try:
|
|
705
|
+
entries.append(json.loads(line))
|
|
706
|
+
except (ValueError, RecursionError):
|
|
707
|
+
raise SystemExit(f"{path}:{n}: not JSON")
|
|
708
|
+
return entries
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def read_log(path):
|
|
712
|
+
return parse_log(read_text(path, LIMIT_LOG, shared_lock=True), path)
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def log_problems(entries):
|
|
716
|
+
prev, out = "0" * 64, []
|
|
717
|
+
for i, e in enumerate(entries):
|
|
718
|
+
if not isinstance(e, dict) or e.get("seq") != i:
|
|
719
|
+
out.append(f"entry {i}: out of order, missing or reordered")
|
|
720
|
+
break
|
|
721
|
+
if e.get("prev") != prev:
|
|
722
|
+
out.append(f"entry {i}: does not follow entry {i - 1} (an entry was removed, reordered or altered)")
|
|
723
|
+
break
|
|
724
|
+
if e.get("hash") != entry_hash(e):
|
|
725
|
+
out.append(f"entry {i}: its content does not match its hash (altered)")
|
|
726
|
+
break
|
|
727
|
+
prev = e["hash"]
|
|
728
|
+
return out
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
CHECKPOINT_FORMAT = "guldo-log-checkpoint-v1"
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def read_trusted(path):
|
|
735
|
+
"""Public keys (hex, one per line, # comments) the reader obtained out of band."""
|
|
736
|
+
if not path:
|
|
737
|
+
return set()
|
|
738
|
+
return {ln.split("#")[0].strip().lower() for ln in read_text(path, LIMIT_SMALL).splitlines() if ln.split("#")[0].strip()}
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def load_signed(path):
|
|
742
|
+
"""A file {statement, public_key, signature} with a VALID signature -> (statement, public key hex). Raises ValueError otherwise."""
|
|
743
|
+
try:
|
|
744
|
+
rec = json.loads(read_text(path, LIMIT_JSON))
|
|
745
|
+
st, pub, sig = rec["statement"], rec["public_key"].lower(), bytes.fromhex(rec["signature"])
|
|
746
|
+
if not isinstance(st, dict) or not ed25519_verify(bytes.fromhex(pub), canonical(st), sig):
|
|
747
|
+
raise ValueError("signature does not verify")
|
|
748
|
+
except (OSError, KeyError, TypeError, AttributeError, RecursionError) as e:
|
|
749
|
+
raise ValueError(type(e).__name__)
|
|
750
|
+
return st, pub
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def cmd_log(a):
|
|
754
|
+
import time
|
|
755
|
+
if a.action == "verify":
|
|
756
|
+
entries = read_log(a.log)
|
|
757
|
+
bad = log_problems(entries)
|
|
758
|
+
head = entries[-1]["hash"] if entries and not bad else None
|
|
759
|
+
print(f"{len(entries)} entries; head {head}")
|
|
760
|
+
if a.head and head != a.head.lower():
|
|
761
|
+
bad.append(f"the head is not the published one ({a.head}): the log was truncated, extended or replaced since")
|
|
762
|
+
trusted = read_trusted(a.trusted)
|
|
763
|
+
for cp in a.checkpoint or []:
|
|
764
|
+
try:
|
|
765
|
+
st, pub = load_signed(cp)
|
|
766
|
+
n = st["count"]
|
|
767
|
+
if st.get("format") != CHECKPOINT_FORMAT or isinstance(n, bool) or not isinstance(n, int) or n < 1 or not isinstance(st.get("head"), str):
|
|
768
|
+
raise ValueError("not a checkpoint")
|
|
769
|
+
except (ValueError, KeyError):
|
|
770
|
+
bad.append(f"{cp}: not a validly signed checkpoint (ignored)")
|
|
771
|
+
continue
|
|
772
|
+
if not entries or bad and head is None:
|
|
773
|
+
continue
|
|
774
|
+
if n > len(entries) or entries[n - 1].get("hash") != st["head"]:
|
|
775
|
+
bad.append(f"{cp}: this log does NOT contain the checkpoint signed by {pub[:16]}... (its entry {n - 1} is not {st['head'][:16]}...): it was truncated, forked or replaced")
|
|
776
|
+
else:
|
|
777
|
+
print(f"checkpoint {cp}: consistent, signed by {pub[:16]}... ({'PINNED' if pub in trusted else 'unvouched'}) at {n} entries")
|
|
778
|
+
print("LOG OK" if not bad else "LOG PROBLEMS:\n " + "\n ".join(bad))
|
|
779
|
+
return 1 if bad else 0
|
|
780
|
+
if a.action == "checkpoint":
|
|
781
|
+
if not (a.key and a.out):
|
|
782
|
+
raise SystemExit("log checkpoint needs --key KEYFILE and --out FILE")
|
|
783
|
+
entries = read_log(a.log)
|
|
784
|
+
bad = log_problems(entries)
|
|
785
|
+
if bad or not entries:
|
|
786
|
+
raise SystemExit("refusing to checkpoint a log that is empty or does not verify: " + "; ".join(bad))
|
|
787
|
+
write_attestation(a.out, {"format": CHECKPOINT_FORMAT, "head": entries[-1]["hash"], "count": len(entries), "time_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())},
|
|
788
|
+
read_key(a.key))
|
|
789
|
+
print(f"checkpoint at {len(entries)} entries, head {entries[-1]['hash']}")
|
|
790
|
+
return 0
|
|
791
|
+
try:
|
|
792
|
+
file_sha = sha256_file(a.file)
|
|
793
|
+
except OSError as e:
|
|
794
|
+
raise SystemExit(f"cannot read {a.file}: {e.strerror}")
|
|
795
|
+
import stat
|
|
796
|
+
try:
|
|
797
|
+
st = os.stat(a.log)
|
|
798
|
+
except FileNotFoundError:
|
|
799
|
+
st = None
|
|
800
|
+
except OSError as e:
|
|
801
|
+
raise SystemExit(f"cannot open {a.log}: {e.strerror}")
|
|
802
|
+
if st is not None and not stat.S_ISREG(st.st_mode):
|
|
803
|
+
raise FileRefused(f"{a.log} is not a regular file (a FIFO, device or directory cannot be a log)")
|
|
804
|
+
try:
|
|
805
|
+
f = open(a.log, "a+", encoding="utf-8")
|
|
806
|
+
except OSError as e:
|
|
807
|
+
raise SystemExit(f"cannot open {a.log}: {e.strerror}")
|
|
808
|
+
with f:
|
|
809
|
+
lock_file(f, True, True) # held until the entry is on disk: read, verify and write are one step
|
|
810
|
+
f.seek(0)
|
|
811
|
+
text = f.read(LIMIT_LOG + 1)
|
|
812
|
+
if len(text) > LIMIT_LOG:
|
|
813
|
+
raise FileRefused(f"{a.log} is over the {LIMIT_LOG} byte limit for a log")
|
|
814
|
+
entries = parse_log(text, a.log)
|
|
815
|
+
bad = log_problems(entries)
|
|
816
|
+
if bad:
|
|
817
|
+
raise SystemExit("refusing to extend a log that does not verify: " + "; ".join(bad))
|
|
818
|
+
entry = {"seq": len(entries), "prev": entries[-1]["hash"] if entries else "0" * 64, "kind": a.kind, "file_name": os.path.basename(a.file),
|
|
819
|
+
"file_sha256": file_sha, "note": a.note or "", "time_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
|
820
|
+
entry["hash"] = entry_hash(entry)
|
|
821
|
+
f.seek(0, 2)
|
|
822
|
+
f.write(json.dumps(entry, sort_keys=True) + "\n")
|
|
823
|
+
f.flush()
|
|
824
|
+
os.fsync(f.fileno())
|
|
825
|
+
print(f"appended entry {entry['seq']}; head {entry['hash']} (publish it somewhere the operator cannot edit)")
|
|
826
|
+
return 0
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
MEASURE_FORMAT = "guldo-bundle-measurement-v1"
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def cmd_spread(a):
|
|
833
|
+
"""Merge `rerun --save` files made on DIFFERENT machines for the SAME instances and recommend a tolerance from the measured machine-to-machine spread of the
|
|
834
|
+
re-measured speedup, instead of a loose guess. Needs >= 2 distinct machines and >= 3 instances measured on at least two of them (use the same --sample-seed
|
|
835
|
+
everywhere). The recommendation is an estimate from the machines and instances you give it, not a bound."""
|
|
836
|
+
import math
|
|
837
|
+
by_machine, problems, chash = {}, [], None
|
|
838
|
+
for path in a.files:
|
|
839
|
+
try:
|
|
840
|
+
rec = json.loads(read_text(path, LIMIT_JSON))
|
|
841
|
+
assert rec["format"] == MEASURE_FORMAT and isinstance(rec["machine_id"], str) and isinstance(rec["instances"], list)
|
|
842
|
+
rows = {str(r["id"]): r["remeasured_bits"] for r in rec["instances"] if isinstance(r, dict) and isinstance(r.get("remeasured_bits"), (int, float))
|
|
843
|
+
and not isinstance(r["remeasured_bits"], bool) and math.isfinite(r["remeasured_bits"])}
|
|
844
|
+
ch_sha = rec["challenge_json_sha256"]
|
|
845
|
+
assert isinstance(ch_sha, str)
|
|
846
|
+
except (OSError, ValueError, KeyError, TypeError, AssertionError, AttributeError, RecursionError, OverflowError):
|
|
847
|
+
problems.append(f"{path}: not a measurement file this tool writes (ignored)")
|
|
848
|
+
continue
|
|
849
|
+
if chash is not None and ch_sha != chash:
|
|
850
|
+
problems.append(f"{path}: measured a different challenge bundle (ignored)")
|
|
851
|
+
continue
|
|
852
|
+
chash = ch_sha
|
|
853
|
+
by_machine.setdefault(rec["machine_id"], {}).update(rows)
|
|
854
|
+
for p in problems:
|
|
855
|
+
print("note: " + p)
|
|
856
|
+
if len(by_machine) < 2:
|
|
857
|
+
print(f"{len(by_machine)} distinct machine(s): the spread of a machine against itself is not machine-to-machine spread. Run `rerun --sample-seed S --save F` "
|
|
858
|
+
"with the SAME sample seed on at least two different machines and merge the files. The default tolerance stays as it is.")
|
|
859
|
+
return 1
|
|
860
|
+
shared = {}
|
|
861
|
+
for rows in by_machine.values():
|
|
862
|
+
for iid, v in rows.items():
|
|
863
|
+
shared.setdefault(iid, []).append(v)
|
|
864
|
+
shared = {i: v for i, v in shared.items() if len(v) >= 2}
|
|
865
|
+
if len(shared) < 3:
|
|
866
|
+
print(f"only {len(shared)} instance(s) measured on two or more machines (need 3): use the same --sample-seed and a larger -k")
|
|
867
|
+
return 1
|
|
868
|
+
ranges = sorted(max(v) - min(v) for v in shared.values())
|
|
869
|
+
worst = ranges[-1]
|
|
870
|
+
rec_tol = max(1.0, math.ceil(worst * 4) / 4)
|
|
871
|
+
print(f"{len(by_machine)} machines, {len(shared)} shared instances: the re-measured speedup differs between machines by a median {ranges[len(ranges) // 2]:.2f} bit, "
|
|
872
|
+
f"at worst {worst:.2f} bit")
|
|
873
|
+
print(f"recommended --tolerance: {rec_tol} bits (the worst observed range rounded up to a quarter bit, never below 1: single-instance timing noise is not in this number). "
|
|
874
|
+
"An estimate from these machines and instances, not a bound.")
|
|
875
|
+
return 0
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def cmd_rerun(a):
|
|
879
|
+
"""Re-measure a sample of a claimed result on THIS machine with THIS verifier's own choice of instances. Costs typed into a ladder, or taken from
|
|
880
|
+
another machine's different hardware class, show up as a spread between claimed and re-measured speedup."""
|
|
881
|
+
import math
|
|
882
|
+
import random
|
|
883
|
+
import statistics
|
|
884
|
+
import tempfile
|
|
885
|
+
try:
|
|
886
|
+
import resource # noqa: F401
|
|
887
|
+
except ImportError:
|
|
888
|
+
raise SystemExit("rerun needs a POSIX system (resource limits)")
|
|
889
|
+
ch = manifest()
|
|
890
|
+
decl = load_decl()
|
|
891
|
+
master = master_from(a)
|
|
892
|
+
cap = ch["ladder"]["budget_seconds"]
|
|
893
|
+
base, claimed = load_costs(a.baseline, master), load_costs(a.claimed, master)
|
|
894
|
+
ids = sorted(set(base) & set(claimed))
|
|
895
|
+
if not ids:
|
|
896
|
+
raise SystemExit("no instance is solved in both ladders: nothing to re-run")
|
|
897
|
+
sample_seed = a.sample_seed or os.urandom(8).hex()
|
|
898
|
+
chosen = random.Random(sample_seed).sample(ids, min(a.k, len(ids)))
|
|
899
|
+
print(f"sample seed {sample_seed} (an independent verifier picks their own; the operator must not know it in advance): {len(chosen)} of {len(ids)} instances")
|
|
900
|
+
ref_path = os.path.join(ROOT, ch["reference"]["file"])
|
|
901
|
+
secret = read_key(a.sign) if a.sign else None
|
|
902
|
+
if a.sign and not a.attest_out:
|
|
903
|
+
raise SystemExit("--sign needs --attest-out FILE")
|
|
904
|
+
for out in (a.attest_out, a.save): # fail BEFORE minutes of measurement, not after
|
|
905
|
+
if out and os.path.exists(out):
|
|
906
|
+
raise SystemExit(f"{out} exists: this tool never overwrites")
|
|
907
|
+
if a.repeat < 1:
|
|
908
|
+
raise SystemExit("--repeat must be at least 1")
|
|
909
|
+
solver_sha = sha256_file(a.solver)
|
|
910
|
+
scratch = os.path.realpath(tempfile.mkdtemp(prefix="rerun-"))
|
|
911
|
+
import atexit
|
|
912
|
+
import shutil
|
|
913
|
+
atexit.register(shutil.rmtree, scratch, True)
|
|
914
|
+
prefix, why_not = working_sandbox(a.sandbox, [a.seed_file, a.sign], scratch)
|
|
915
|
+
if prefix is None:
|
|
916
|
+
raise SystemExit("no sandbox is available here (needs macOS sandbox-exec or Linux bwrap" + (f"; the tool exists but cannot run: {why_not}" if why_not else "")
|
|
917
|
+
+ "). To run the solver anyway, unsandboxed, as you, pass --sandbox off (only for a solver you have read).")
|
|
918
|
+
print(f"solver {os.path.basename(a.solver)} sha256 {solver_sha[:16]}...; " + ("sandboxed: no network, no writes outside a scratch directory, seed and keys unreadable"
|
|
919
|
+
if prefix else "UNSANDBOXED: the solver runs as you"))
|
|
920
|
+
overhead = solver_overhead(os.path.join(scratch, "noop.py"), prefix, scratch)
|
|
921
|
+
rows, invalid, repeat_sd, meter_bad, hidden = [], [], [], [], []
|
|
922
|
+
for iid in chosen:
|
|
923
|
+
size, idx = (int(x) for x in iid.split(":"))
|
|
924
|
+
inst = instance(decl, master, size, idx)
|
|
925
|
+
ref_run = run_solver(ref_path, inst["args"], cap, prefix, scratch)
|
|
926
|
+
sol_run = run_solver(a.solver, inst["args"], cap, prefix, scratch)
|
|
927
|
+
rs, ra, rc, rerr = ref_run
|
|
928
|
+
ss, sa, sc, serr = sol_run
|
|
929
|
+
ref_ok = rs == "ran" and safe_verify(decl, inst, ra)
|
|
930
|
+
sol_ok = ss == "ran" and safe_verify(decl, inst, sa)
|
|
931
|
+
if not sol_ok:
|
|
932
|
+
invalid.append(iid)
|
|
933
|
+
if ref_ok and rc < 0.005 and base[iid] >= 0.5: # the claim says this instance costs seconds; a meter reading ~0 for the reference means it is not seeing the process
|
|
934
|
+
meter_bad.append(iid)
|
|
935
|
+
if ref_ok and hides_cpu(ref_run, rc, limit=REFERENCE_SHARE):
|
|
936
|
+
meter_bad.append(iid)
|
|
937
|
+
if sol_ok and hides_cpu(sol_run, sc, ref=(ref_run, rc) if ref_ok else None):
|
|
938
|
+
hidden.append(iid)
|
|
939
|
+
rc, sc = max(rc - overhead, 1e-3), max(sc - overhead, 1e-3)
|
|
940
|
+
now = math.log2(rc / sc) if ref_ok and sol_ok else None
|
|
941
|
+
was = math.log2(base[iid] / claimed[iid])
|
|
942
|
+
rows.append((iid, was, now, math.log2(rc / base[iid]) if ref_ok else None, rc, sc, ss))
|
|
943
|
+
print(f" {iid:>8}: claimed {was:+.2f} bits, re-measured {('%+.2f' % now) if now is not None else 'n/a'} "
|
|
944
|
+
f"(reference now {rc:.2f} s, solver now {sc:.2f} s, solver {'valid' if sol_ok else 'INVALID/' + ss})")
|
|
945
|
+
if a.repeat > 1 and ref_ok and sol_ok: # the same instance again, to measure THIS machine's timing noise instead of guessing it
|
|
946
|
+
reps_r, reps_s = [rc], [sc]
|
|
947
|
+
for _ in range(a.repeat - 1):
|
|
948
|
+
reps_r.append(max(run_solver(ref_path, inst["args"], cap, prefix, scratch)[2] - overhead, 1e-3))
|
|
949
|
+
reps_s.append(max(run_solver(a.solver, inst["args"], cap, prefix, scratch)[2] - overhead, 1e-3))
|
|
950
|
+
repeat_sd.append((statistics.pstdev(math.log2(x) for x in reps_r), statistics.pstdev(math.log2(x) for x in reps_s)))
|
|
951
|
+
if ss != "ran" and serr:
|
|
952
|
+
print(f" solver stderr: {serr}" + (" <- the sandbox probably blocked something the solver tried to do" if any(w in serr for w in DENIAL_WORDS) else ""))
|
|
953
|
+
if rs != "ran" and rerr:
|
|
954
|
+
print(f" reference stderr: {rerr}")
|
|
955
|
+
gaps = [abs(was - now) for _, was, now, _, _, _, _ in rows if now is not None]
|
|
956
|
+
machine = [m for _, _, _, m, _, _, _ in rows if m is not None]
|
|
957
|
+
ok = not invalid and not meter_bad and not hidden and bool(gaps) and statistics.median(gaps) <= a.tolerance
|
|
958
|
+
if hidden:
|
|
959
|
+
try:
|
|
960
|
+
load = os.getloadavg()[0]
|
|
961
|
+
busy = f" This machine's load average is {load:.1f} on {os.cpu_count()} cores: a heavily loaded machine also starves an honest solver of CPU, so if it is busy, re-run when it is idle."
|
|
962
|
+
except (OSError, AttributeError):
|
|
963
|
+
busy = ""
|
|
964
|
+
print(f"FAIL: on {sorted(set(hidden))} the solver ran for over half a second of wall time but its CPU time is far below it (under 60% of the share the reference got on the same instance, or under 30% with no reference to compare): it waits, or its work runs where the CPU meter cannot see it "
|
|
965
|
+
"(on Linux a child the solver forks and never waits for escapes the accounting entirely). The measured cost is not trustworthy, so the speedup is not confirmed." + busy)
|
|
966
|
+
if meter_bad:
|
|
967
|
+
print(f"FAIL: the CPU meter read ~0 for the REFERENCE on {meter_bad}, whose published cost is seconds: the measurement is not valid on this machine/sandbox, so no "
|
|
968
|
+
"claim is confirmed or refuted by it")
|
|
969
|
+
print(f"median |claimed - re-measured| = {statistics.median(gaps):.2f} bits (tolerance {a.tolerance})" if gaps else "no instance could be compared")
|
|
970
|
+
if machine:
|
|
971
|
+
spread = max(machine) - min(machine)
|
|
972
|
+
print(f"this machine vs the baseline ladder's: median {statistics.median(machine):+.2f} bits (hardware), spread {spread:.2f} bits"
|
|
973
|
+
+ (" <- a large spread means the baseline costs are not from one machine's measurements" if spread > a.tolerance else ""))
|
|
974
|
+
short = [r[0] for r in rows if r[4] < 0.05 or r[5] < 0.05]
|
|
975
|
+
if short:
|
|
976
|
+
print(f"note: {len(short)} instance(s) cost under 50 ms here; timer noise dominates those ratios")
|
|
977
|
+
rep = None
|
|
978
|
+
if repeat_sd:
|
|
979
|
+
rep = {"reference_sd_bits": round(statistics.median(x[0] for x in repeat_sd), 4), "solver_sd_bits": round(statistics.median(x[1] for x in repeat_sd), 4),
|
|
980
|
+
"repeats": a.repeat, "instances": len(repeat_sd)}
|
|
981
|
+
noise = (rep["reference_sd_bits"] ** 2 + rep["solver_sd_bits"] ** 2) ** 0.5
|
|
982
|
+
print(f"timing repeatability on THIS machine over {a.repeat} runs (median sd of log2 cost): reference {rep['reference_sd_bits']} bit, solver {rep['solver_sd_bits']} bit; "
|
|
983
|
+
f"a re-measured speedup then has noise of about {noise:.2f} bit on its own (this is within-machine only: machine-to-machine spread needs `spread`)")
|
|
984
|
+
if invalid:
|
|
985
|
+
print(f"FAIL: the solver's answer is wrong or missing on {invalid}")
|
|
986
|
+
print("RERUN " + ("PASS: the sampled claims reproduce" if ok else "FAIL"))
|
|
987
|
+
if a.save:
|
|
988
|
+
import platform
|
|
989
|
+
mid = hashlib.sha256(f"{platform.platform()}|{platform.processor()}|{platform.node()}".encode()).hexdigest()[:16] # a hash, never the host name
|
|
990
|
+
saved = {"format": MEASURE_FORMAT, "challenge_json_sha256": sha256_file(os.path.join(ROOT, "challenge.json")), "claimed_sha256": sha256_file(a.claimed),
|
|
991
|
+
"baseline_sha256": sha256_file(a.baseline), "machine_id": mid, "platform": platform.platform(), "sample_seed": sample_seed, "repeatability": rep,
|
|
992
|
+
"instances": [{"id": r[0], "claimed_bits": round(r[1], 4), "remeasured_bits": None if r[2] is None else round(r[2], 4),
|
|
993
|
+
"machine_bits": None if r[3] is None else round(r[3], 4)} for r in rows]}
|
|
994
|
+
with open(a.save, "x", encoding="utf-8") as f:
|
|
995
|
+
json.dump(saved, f, indent=1, sort_keys=True)
|
|
996
|
+
print(f"saved the measurements to {a.save} (merge several machines' files with `spread`)")
|
|
997
|
+
if secret:
|
|
998
|
+
import platform
|
|
999
|
+
import time
|
|
1000
|
+
statement = {
|
|
1001
|
+
"format": ATTEST_FORMAT, "kind": "rerun",
|
|
1002
|
+
"challenge": {"name": ch["name"], "version": ch["version"], "challenge_json_sha256": sha256_file(os.path.join(ROOT, "challenge.json"))},
|
|
1003
|
+
"solver_sha256": solver_sha, "claimed_sha256": sha256_file(a.claimed), "baseline_sha256": sha256_file(a.baseline),
|
|
1004
|
+
"seed_commit": hashlib.sha256(master).hexdigest(), "sample_seed": sample_seed, "k": len(chosen), "tolerance_bits": a.tolerance,
|
|
1005
|
+
"sandboxed": bool(prefix), "outcome": "PASS" if ok else "FAIL",
|
|
1006
|
+
"median_gap_bits": round(statistics.median(gaps), 3) if gaps else None, "repeatability": rep,
|
|
1007
|
+
"instances": [{"id": r[0], "claimed_bits": round(r[1], 3), "remeasured_bits": None if r[2] is None else round(r[2], 3), "solver_status": r[6]} for r in rows],
|
|
1008
|
+
"machine": {"platform": platform.platform(), "python": platform.python_version(), "processor": platform.processor() or platform.machine()},
|
|
1009
|
+
"time_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
|
1010
|
+
write_attestation(a.attest_out, statement, secret)
|
|
1011
|
+
return 0 if ok else 1
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def reexec_sandboxed(argv, script=None):
|
|
1015
|
+
"""Run this same command again inside the OS sandbox (no network, no writes outside a scratch directory, ~/.ssh and friends unreadable). The declaration and
|
|
1016
|
+
the reference are the bundle AUTHOR's code and `selftest`/`derive`/`verify` import and run them. -> the child's exit code, or None to run here."""
|
|
1017
|
+
if os.environ.get("GULDO_BUNDLE_SANDBOXED") == "1":
|
|
1018
|
+
return None
|
|
1019
|
+
import shutil
|
|
1020
|
+
import subprocess
|
|
1021
|
+
import tempfile
|
|
1022
|
+
scratch = os.path.realpath(tempfile.mkdtemp(prefix="check-"))
|
|
1023
|
+
try:
|
|
1024
|
+
prefix, reason = sandbox_prefix("auto", [], scratch), ""
|
|
1025
|
+
if prefix is not None:
|
|
1026
|
+
# No separate probe on the hot path (a verification spawn added 50 ms to EVERY call, measured): run the real command, and only if the SANDBOX TOOL itself
|
|
1027
|
+
# failed to start (its own error line: `bwrap: ...`, `sandbox-exec: ...`) diagnose it as unavailable. stdout streams; stderr is held to inspect and re-emitted.
|
|
1028
|
+
env = dict(os.environ, GULDO_BUNDLE_SANDBOXED="1", TMPDIR=scratch)
|
|
1029
|
+
r = subprocess.run(prefix + [sys.executable, "-I", "-B", script or os.path.abspath(__file__)] + list(argv), env=env, stderr=subprocess.PIPE, text=True)
|
|
1030
|
+
err = r.stderr or ""
|
|
1031
|
+
first = next((ln for ln in err.splitlines() if ln.strip()), "")
|
|
1032
|
+
if r.returncode != 0 and (first.startswith(TOOL_ERRORS) or "sandbox_apply" in err):
|
|
1033
|
+
prefix, reason = None, f" (the tool exists but cannot run: {first[:200]})"
|
|
1034
|
+
else:
|
|
1035
|
+
if err:
|
|
1036
|
+
sys.stderr.write(err)
|
|
1037
|
+
return r.returncode
|
|
1038
|
+
if os.environ.get("GULDO_REQUIRE_SANDBOX") == "1":
|
|
1039
|
+
raise SystemExit("no OS sandbox is available here (macOS sandbox-exec or Linux bwrap)" + reason + " and one was required")
|
|
1040
|
+
print("warning: no OS sandbox is available here" + reason + ": the declaration and reference run unsandboxed, as you (pass --sandbox off to silence this)", file=sys.stderr)
|
|
1041
|
+
return None
|
|
1042
|
+
finally:
|
|
1043
|
+
shutil.rmtree(scratch, ignore_errors=True)
|
|
1044
|
+
|
|
1045
|
+
|
|
1046
|
+
def cmd_selftest(a):
|
|
1047
|
+
ch = manifest()
|
|
1048
|
+
results = []
|
|
1049
|
+
|
|
1050
|
+
def check(name, ok, detail=""):
|
|
1051
|
+
results.append((name, ok, detail))
|
|
1052
|
+
print(("PASS " if ok else "FAIL ") + name + (f": {detail}" if detail else ""))
|
|
1053
|
+
|
|
1054
|
+
problems = file_problems(ch)
|
|
1055
|
+
check("pinned files intact", not problems, "; ".join(problems))
|
|
1056
|
+
if problems:
|
|
1057
|
+
return 1
|
|
1058
|
+
try:
|
|
1059
|
+
decl = load_decl()
|
|
1060
|
+
except Exception as e: # the author's code, run on import: whatever it does, say so instead of a traceback
|
|
1061
|
+
denied = isinstance(e, OSError) and e.errno in (1, 13, 30) # EPERM, EACCES, EROFS: macOS sandbox-exec says EPERM, bubblewrap's read-only root says EROFS
|
|
1062
|
+
check("the declaration imports", False, f"{type(e).__name__}: {str(e)[:150]}" + (" <- blocked by the sandbox?" if denied else ""))
|
|
1063
|
+
return 1
|
|
1064
|
+
check("declaration matches the manifest", decl.NAME == ch["name"] and decl.VERSION == ch["version"], f"{decl.NAME} {decl.VERSION}")
|
|
1065
|
+
try:
|
|
1066
|
+
ref = load_reference(ch)
|
|
1067
|
+
except Exception as e:
|
|
1068
|
+
check("the reference imports", False, f"{type(e).__name__}: {str(e)[:150]}")
|
|
1069
|
+
return 1
|
|
1070
|
+
check("reference has solve()", callable(getattr(ref, "solve", None)))
|
|
1071
|
+
sizes = ch["ladder"]["practice_dims"]
|
|
1072
|
+
fresh = bytes.fromhex(a.fresh_seed) if a.fresh_seed else os.urandom(16)
|
|
1073
|
+
print(f"fresh seed for this run (reproduce with --fresh-seed): {fresh.hex()}")
|
|
1074
|
+
for label, master in (("practice", PRACTICE_SEED), ("fresh", fresh)):
|
|
1075
|
+
for size in sizes:
|
|
1076
|
+
instance_checks(decl, ref, master, label, size, check)
|
|
1077
|
+
if ch.get("metric") == "quality":
|
|
1078
|
+
quality_checks(decl, ref, master, label, size, check)
|
|
1079
|
+
else:
|
|
1080
|
+
answer_space_checks(decl, master, label, size, check)
|
|
1081
|
+
cross_seed_checks(decl, fresh, sizes, check)
|
|
1082
|
+
other = instance(decl, b"another-seed", sizes[0], 0)
|
|
1083
|
+
base = instance(decl, PRACTICE_SEED, sizes[0], 0)
|
|
1084
|
+
check("a different seed gives a different instance", json.dumps(other["args"]) != json.dumps(base["args"]))
|
|
1085
|
+
failed = [n for n, ok, _ in results if not ok]
|
|
1086
|
+
print(f"{len(results) - len(failed)} passed, {len(failed)} failed")
|
|
1087
|
+
return 1 if failed else 0
|
|
1088
|
+
|
|
1089
|
+
|
|
1090
|
+
def witness_of(decl, inst):
|
|
1091
|
+
return decl.witness(inst) if hasattr(decl, "witness") else inst["planted"]
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def cross_seed_checks(decl, fresh, sizes, check):
|
|
1095
|
+
"""Instances must DEPEND ON THE SEED at every size and index, or the commitment commits to nothing and a table of old answers works for ever. Compared at
|
|
1096
|
+
the SAME index: that is where a seed-ignoring derive() shows (6 of 6 identical), whereas a witness-acceptance rate over all pairs diluted it to 6 of 36
|
|
1097
|
+
(found by mutation testing). Different-index pairs carry no extra information: statistically they are the within-seed reuse check."""
|
|
1098
|
+
for size in sizes:
|
|
1099
|
+
same = sum(json.dumps(instance(decl, PRACTICE_SEED, size, i)["args"], sort_keys=True) == json.dumps(instance(decl, fresh, size, i)["args"], sort_keys=True)
|
|
1100
|
+
for i in range(6))
|
|
1101
|
+
check(f"size {size}: instances differ between seeds", same <= 2, f"{same} of 6 identical across two seeds")
|
|
1102
|
+
if same:
|
|
1103
|
+
print(f"WARN size {size}: {same} of 6 instances are identical across two different seeds (a small instance space at this size; fine if few)")
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
def instance_checks(decl, ref, master, label, size, check):
|
|
1107
|
+
for idx in (0, 1):
|
|
1108
|
+
tag = f"{label} size {size} idx {idx}"
|
|
1109
|
+
inst = instance(decl, master, size, idx)
|
|
1110
|
+
again = instance(decl, master, size, idx)
|
|
1111
|
+
check(f"{tag}: deterministic", json.dumps(inst["args"], sort_keys=True) == json.dumps(again["args"], sort_keys=True))
|
|
1112
|
+
bad = args_problem(inst)
|
|
1113
|
+
check(f"{tag}: args are a JSON list under 1 MiB", bad is None, bad or "")
|
|
1114
|
+
if bad:
|
|
1115
|
+
continue
|
|
1116
|
+
check(f"{tag}: the planted witness verifies", safe_verify(decl, inst, witness_of(decl, inst)))
|
|
1117
|
+
accepted = [repr(j)[:20] for j in JUNK if safe_verify(decl, inst, j)]
|
|
1118
|
+
check(f"{tag}: junk answers rejected", not accepted, ("accepted: " + ", ".join(accepted)) if accepted else "")
|
|
1119
|
+
try:
|
|
1120
|
+
got = ref.solve(*json.loads(json.dumps(inst["args"])))
|
|
1121
|
+
except Exception as e:
|
|
1122
|
+
check(f"{tag}: reference solves", False, f"{type(e).__name__}: {e}")
|
|
1123
|
+
continue
|
|
1124
|
+
check(f"{tag}: reference solves", safe_verify(decl, inst, got))
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
def leaves(x, depth=0):
|
|
1128
|
+
if depth > 6:
|
|
1129
|
+
return
|
|
1130
|
+
if isinstance(x, dict):
|
|
1131
|
+
for v in x.values():
|
|
1132
|
+
yield from leaves(v, depth + 1)
|
|
1133
|
+
elif isinstance(x, (list, tuple)):
|
|
1134
|
+
for v in x:
|
|
1135
|
+
yield from leaves(v, depth + 1)
|
|
1136
|
+
else:
|
|
1137
|
+
yield x
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def trivial_candidates(inst):
|
|
1141
|
+
"""Answers a solver could write down without searching: the arguments, their reverse, every integer in them and its neighbours."""
|
|
1142
|
+
ints = [v for v in leaves(inst["args"]) if isinstance(v, int) and not isinstance(v, bool)][:64]
|
|
1143
|
+
return [inst["args"], list(inst["args"])[::-1]] + ints + [v + d for v in ints[:8] for d in (-1, 1)]
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def mutants(w):
|
|
1147
|
+
"""Near-misses of a witness. Not all are wrong (a factor's cofactor may be valid), so these are evidence about the answer space, not a verdict each."""
|
|
1148
|
+
out = []
|
|
1149
|
+
if isinstance(w, bool):
|
|
1150
|
+
return out
|
|
1151
|
+
if isinstance(w, int):
|
|
1152
|
+
out = [w + 1, w - 1, w + 2, w - 2, -w, w * 2, w // 2, w ^ 1, 0, 1]
|
|
1153
|
+
elif isinstance(w, str):
|
|
1154
|
+
out = [w[:-1], w + "0", w[::-1]]
|
|
1155
|
+
elif isinstance(w, (list, tuple)):
|
|
1156
|
+
items = list(w)
|
|
1157
|
+
out = [items[:-1], items[1:], items + [0], items[::-1], [0] * len(items)]
|
|
1158
|
+
for k in range(min(len(items), 16)):
|
|
1159
|
+
out += [items[:k] + [m] + items[k + 1:] for m in mutants(items[k])[:4]]
|
|
1160
|
+
seen, uniq = set(), []
|
|
1161
|
+
for m in out:
|
|
1162
|
+
key = json.dumps(m, sort_keys=True)
|
|
1163
|
+
if m != w and key not in seen:
|
|
1164
|
+
seen.add(key)
|
|
1165
|
+
uniq.append(m)
|
|
1166
|
+
return uniq
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def map_ints(w, f):
|
|
1170
|
+
if isinstance(w, bool):
|
|
1171
|
+
return w
|
|
1172
|
+
if isinstance(w, int):
|
|
1173
|
+
return f(w)
|
|
1174
|
+
if isinstance(w, (list, tuple)):
|
|
1175
|
+
return [map_ints(x, f) for x in w]
|
|
1176
|
+
return w
|
|
1177
|
+
|
|
1178
|
+
|
|
1179
|
+
def heavy_mutants(w):
|
|
1180
|
+
"""The witness with EVERY integer in it damaged at once. A verifier with a tolerance may accept one wrong part; it must not accept all parts wrong: if
|
|
1181
|
+
it does, anything of the right shape verifies."""
|
|
1182
|
+
if isinstance(w, bool) or not isinstance(w, (int, list, tuple)):
|
|
1183
|
+
return []
|
|
1184
|
+
out = [map_ints(w, f) for f in (lambda x: x + 1, lambda x: x - 1, lambda x: -x - 1, lambda x: x ^ 1, lambda x: 0, lambda x: x + 7)]
|
|
1185
|
+
seen, uniq = set(), []
|
|
1186
|
+
for m in out:
|
|
1187
|
+
key = json.dumps(m, sort_keys=True)
|
|
1188
|
+
if m != w and key not in seen:
|
|
1189
|
+
seen.add(key)
|
|
1190
|
+
uniq.append(m)
|
|
1191
|
+
return uniq
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def random_like(w, rng):
|
|
1195
|
+
"""A random answer with the witness's type, length and magnitude. Whatever the verifier accepts of these, it accepts without anyone solving anything."""
|
|
1196
|
+
if isinstance(w, bool):
|
|
1197
|
+
return rng.random() < 0.5
|
|
1198
|
+
if isinstance(w, int):
|
|
1199
|
+
b = max(1, abs(w).bit_length())
|
|
1200
|
+
return rng.randrange(-(1 << b), (1 << b) + 1)
|
|
1201
|
+
if isinstance(w, float):
|
|
1202
|
+
return rng.uniform(-abs(w) - 1, abs(w) + 1)
|
|
1203
|
+
if isinstance(w, (list, tuple)):
|
|
1204
|
+
return [random_like(x, rng) for x in w]
|
|
1205
|
+
if isinstance(w, str):
|
|
1206
|
+
return "".join(chr(rng.randrange(32, 127)) for _ in w)
|
|
1207
|
+
return w
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
def answer_space_checks(decl, master, label, size, check):
|
|
1211
|
+
"""What the fixed junk list cannot show: answers DERIVED FROM THE INSTANCE ITSELF. If the arguments, another instance's witness (a lookup table
|
|
1212
|
+
of past answers), or a slightly damaged witness verifies, the answer space is small or leaks, whatever the junk list says. Run on a seed the
|
|
1213
|
+
declaration's author has never seen (the fresh one), so a verifier cannot be tuned to the public practice seed."""
|
|
1214
|
+
insts = [instance(decl, master, size, i) for i in range(6)] # 30 ordered pairs for the reuse rate: few enough pairs and a tiny answer space makes chance hits common
|
|
1215
|
+
tag = f"{label} size {size}"
|
|
1216
|
+
hit = next((str(c)[:50] for x in insts[:3] for c in trivial_candidates(x) if safe_verify(decl, x, c)), None)
|
|
1217
|
+
check(f"{tag}: no answer can be read off the arguments", hit is None, f"verify() accepts {hit!r} without any search" if hit else "")
|
|
1218
|
+
ws = [witness_of(decl, x) for x in insts]
|
|
1219
|
+
pairs = [(j, i) for i in range(len(insts)) for j in range(len(insts)) if i != j]
|
|
1220
|
+
reused = sum(bool(safe_verify(decl, insts[i], ws[j])) for j, i in pairs)
|
|
1221
|
+
# FAIL above 50%, warn above 25%: measured on a sound toy domain at size 20 (10-bit primes, so instances often share a factor and hits CLUSTER), 2 seeds in
|
|
1222
|
+
# 3000 reach 8 of 30 pairs, so a 25% hard limit failed ~0.07% of random seeds. A real lookup table accepts nearly everything.
|
|
1223
|
+
check(f"{tag}: another instance's witness is not accepted (no lookup table)", reused / len(pairs) <= 0.5, f"{reused} of {len(pairs)} accepted")
|
|
1224
|
+
if reused / len(pairs) > 0.25:
|
|
1225
|
+
print(f"WARN {tag}: {reused} of {len(pairs)} other instances' witnesses accepted: a small answer space at this size")
|
|
1226
|
+
htot = hacc = tot = acc = 0
|
|
1227
|
+
for x, w in zip(insts, ws):
|
|
1228
|
+
for m in heavy_mutants(w):
|
|
1229
|
+
htot += 1
|
|
1230
|
+
hacc += bool(safe_verify(decl, x, m))
|
|
1231
|
+
for m in mutants(w):
|
|
1232
|
+
tot += 1
|
|
1233
|
+
acc += bool(safe_verify(decl, x, m))
|
|
1234
|
+
check(f"{tag}: a witness with EVERY part damaged is mostly rejected", htot == 0 or hacc / htot <= 0.5, f"{hacc} of {htot} accepted")
|
|
1235
|
+
rng = random.Random(int.from_bytes(hashlib.sha256(master + b":fuzz:%d" % size).digest()[:8], "big"))
|
|
1236
|
+
ftot = facc = 0
|
|
1237
|
+
for x, w in zip(insts, ws):
|
|
1238
|
+
for _ in range(100):
|
|
1239
|
+
ftot += 1
|
|
1240
|
+
facc += bool(safe_verify(decl, x, random_like(w, rng)))
|
|
1241
|
+
check(f"{tag}: random answers of the witness's shape are mostly rejected", facc / ftot <= 0.5, f"{facc} of {ftot} accepted")
|
|
1242
|
+
if facc:
|
|
1243
|
+
print(f"WARN {tag}: {facc} of {ftot} random answers of the witness's shape verified: the answer space is dense, so little work separates a solver from a guess")
|
|
1244
|
+
if reused or acc:
|
|
1245
|
+
print(f"WARN {tag}: {reused} reused witnesses and {acc} of {tot} single-part damaged witnesses accepted; expected when the verifier has a tolerance "
|
|
1246
|
+
"or the problem has many valid answers, a smell if it should not")
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
QUALITY_TRIVIAL = 0.9 # QUALITY challenges: an answer that needs no search (derived from the arguments, another instance's witness, a random guess) must not score above this
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
def quality_of(decl, inst, ans):
|
|
1253
|
+
"""-> (valid, quality): quality is None unless verify() accepted the answer. A quality() that raises, or returns anything but a finite number in [0, 1], on a VERIFIED
|
|
1254
|
+
answer is the declaration's bug: ValueError, never a score."""
|
|
1255
|
+
import math
|
|
1256
|
+
if not safe_verify(decl, inst, ans):
|
|
1257
|
+
return False, None
|
|
1258
|
+
try:
|
|
1259
|
+
q = decl.quality(inst, ans)
|
|
1260
|
+
except Exception as e:
|
|
1261
|
+
raise ValueError(f"quality() raised {type(e).__name__}: {str(e)[:100]} on an answer verify() accepted")
|
|
1262
|
+
if isinstance(q, bool) or not isinstance(q, (int, float)) or not math.isfinite(q) or not 0.0 <= q <= 1.0:
|
|
1263
|
+
raise ValueError(f"quality() returned {str(q)[:30]!r}, not a finite number in [0, 1]")
|
|
1264
|
+
return True, float(q)
|
|
1265
|
+
|
|
1266
|
+
|
|
1267
|
+
def quality_checks(decl, ref, master, label, size, check):
|
|
1268
|
+
"""The quality challenge's counterpart of answer_space_checks. The valid answer space is LARGE here (any feasible solution is valid), so what must hold is not
|
|
1269
|
+
'junk is rejected' but 'quality() is well-defined, and no answer that needs no search scores high'. Answers derived from the planted witness (a damaged
|
|
1270
|
+
witness, its complement) are not no-search answers: a solver is never handed the witness, so those only have to score in range."""
|
|
1271
|
+
insts = [instance(decl, master, size, i) for i in range(6)]
|
|
1272
|
+
tag = f"{label} size {size}"
|
|
1273
|
+
problems = []
|
|
1274
|
+
|
|
1275
|
+
def q(x, ans, what):
|
|
1276
|
+
try:
|
|
1277
|
+
return quality_of(decl, x, ans)[1]
|
|
1278
|
+
except ValueError as e:
|
|
1279
|
+
problems.append(f"{what}: {e}")
|
|
1280
|
+
return None
|
|
1281
|
+
ws = [witness_of(decl, x) for x in insts]
|
|
1282
|
+
first = [q(x, w, "planted witness") for x, w in zip(insts, ws)]
|
|
1283
|
+
check(f"{tag}: the planted witness scores a number in [0, 1]", not problems and all(v is not None for v in first), "; ".join(problems[:2]))
|
|
1284
|
+
again = [q(x, w, "planted witness") for x, w in zip(insts, ws)]
|
|
1285
|
+
check(f"{tag}: quality() is deterministic", first == again, f"{first[:3]} then {again[:3]}")
|
|
1286
|
+
got = []
|
|
1287
|
+
for x in insts[:3]:
|
|
1288
|
+
try:
|
|
1289
|
+
got.append(q(x, ref.solve(*json.loads(json.dumps(x["args"]))), "reference answer"))
|
|
1290
|
+
except Exception as e:
|
|
1291
|
+
problems.append(f"reference raised {type(e).__name__}: {e}")
|
|
1292
|
+
check(f"{tag}: the reference's answers score a number in [0, 1]", len(got) == 3 and all(v is not None for v in got), "; ".join(problems[:2]))
|
|
1293
|
+
nosearch = [(x, c) for x in insts[:3] for c in trivial_candidates(x)] + [(insts[i], ws[j]) for i in range(len(insts)) for j in range(len(insts)) if i != j]
|
|
1294
|
+
high = [str(c)[:40] for x, c in nosearch if (q(x, c, "no-search answer") or 0.0) > QUALITY_TRIVIAL]
|
|
1295
|
+
check(f"{tag}: no answer that needs no search scores above {QUALITY_TRIVIAL}", not high, f"{len(high)} did, e.g. {high[0]!r}" if high else "")
|
|
1296
|
+
rng = random.Random(int.from_bytes(hashlib.sha256(master + b":qfuzz:%d" % size).digest()[:8], "big"))
|
|
1297
|
+
tot = top = 0
|
|
1298
|
+
for x, w in zip(insts, ws):
|
|
1299
|
+
for _ in range(100):
|
|
1300
|
+
tot += 1
|
|
1301
|
+
top += (q(x, random_like(w, rng), "random answer") or 0.0) > QUALITY_TRIVIAL
|
|
1302
|
+
check(f"{tag}: random answers of the witness's shape rarely score above {QUALITY_TRIVIAL}", top / tot <= 0.05, f"{top} of {tot}")
|
|
1303
|
+
for x, w in zip(insts, ws):
|
|
1304
|
+
for m in mutants(w) + heavy_mutants(w):
|
|
1305
|
+
q(x, m, "damaged witness")
|
|
1306
|
+
check(f"{tag}: quality() is defined on every accepted damaged witness", not problems, "; ".join(problems[:2]))
|
|
1307
|
+
|
|
1308
|
+
|
|
1309
|
+
def main(argv=None):
|
|
1310
|
+
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__)
|
|
1311
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
1312
|
+
sub.add_parser("info")
|
|
1313
|
+
sandbox_help = ("auto (default): run inside an OS sandbox (no network, no writes outside a scratch directory) because this imports and runs the bundle's decl.py and "
|
|
1314
|
+
"reference, which are its author's code; off: run as you")
|
|
1315
|
+
p = sub.add_parser("selftest")
|
|
1316
|
+
p.add_argument("--fresh-seed", metavar="HEX", type=hex_arg, help="hex of the fresh seed to use (default: random each run, printed so a failure can be reproduced)")
|
|
1317
|
+
p.add_argument("--sandbox", choices=["auto", "off"], default="auto", help=sandbox_help)
|
|
1318
|
+
p = sub.add_parser("rerun")
|
|
1319
|
+
p.add_argument("--seed-file")
|
|
1320
|
+
p.add_argument("--practice", action="store_true")
|
|
1321
|
+
p.add_argument("--solver", required=True, help="the solver file whose claimed ladder is being checked (it is EXECUTED here, in an OS sandbox unless --sandbox off)")
|
|
1322
|
+
p.add_argument("--claimed", required=True, help="the claimed ladder of that solver")
|
|
1323
|
+
p.add_argument("--baseline", required=True, help="the claimed ladder of the reference on the same seed")
|
|
1324
|
+
p.add_argument("-k", type=int_arg(1, 1000), default=5, help="instances to re-run (1..1000, default 5)")
|
|
1325
|
+
p.add_argument("--sample-seed", help="fix the sample (default: random, printed)")
|
|
1326
|
+
p.add_argument("--tolerance", type=float_arg(0, 1000), default=2.0, help="bits the median claimed-vs-re-measured gap may reach (default 2; machine-to-machine spread is not measured)")
|
|
1327
|
+
p.add_argument("--sandbox", choices=["auto", "off"], default="auto", help="auto (default): run solver and reference without network, without writes outside a scratch "
|
|
1328
|
+
"directory and without reading the seed or keys, or refuse if the system has no sandbox; off: run them as you")
|
|
1329
|
+
p.add_argument("--sign", metavar="KEYFILE", help="sign the outcome as an attestation (see keygen); needs --attest-out")
|
|
1330
|
+
p.add_argument("--attest-out", metavar="FILE")
|
|
1331
|
+
p.add_argument("--repeat", type=int_arg(1, 50), default=1, help="run each sampled instance N times (1..50) and report this machine's own timing repeatability (default 1: not measured)")
|
|
1332
|
+
p.add_argument("--save", metavar="FILE", help="write the per-instance measurements (machine id is a hash) so several machines' runs can be merged with `spread`")
|
|
1333
|
+
p = sub.add_parser("spread", help="merge `rerun --save` files from different machines and recommend a tolerance from their measured spread")
|
|
1334
|
+
p.add_argument("files", nargs="+")
|
|
1335
|
+
p = sub.add_parser("keygen", help="make an Ed25519 signing key for attestations (never overwrites)")
|
|
1336
|
+
p.add_argument("--out", required=True)
|
|
1337
|
+
p = sub.add_parser("attest-check", help="verify signed rerun attestations; only keys listed in --trusted count as vouched")
|
|
1338
|
+
p.add_argument("files", nargs="+")
|
|
1339
|
+
p.add_argument("--trusted", help="file of public keys (hex, one per line) you obtained out of band")
|
|
1340
|
+
p.add_argument("--min", type=int, default=1, help="distinct pinned keys that must attest PASS (default 1)")
|
|
1341
|
+
p.add_argument("--claimed", help="also require the attestations to name THIS claimed ladder")
|
|
1342
|
+
p.add_argument("--baseline", help="also require the attestations to name THIS baseline ladder")
|
|
1343
|
+
p = sub.add_parser("log", help="an append-only, hash-chained log of files (submissions, results, attestations); signed checkpoints catch truncation")
|
|
1344
|
+
p.add_argument("action", choices=["append", "verify", "checkpoint"])
|
|
1345
|
+
p.add_argument("log")
|
|
1346
|
+
p.add_argument("--file", help="append: the file to record")
|
|
1347
|
+
p.add_argument("--kind", default="submission", help="append: what it is")
|
|
1348
|
+
p.add_argument("--note")
|
|
1349
|
+
p.add_argument("--head", help="verify: the head hash you were told, so a truncated log is caught")
|
|
1350
|
+
p.add_argument("--key", help="checkpoint: your signing key (see keygen)")
|
|
1351
|
+
p.add_argument("--out", help="checkpoint: where to write the signed (head, count) statement")
|
|
1352
|
+
p.add_argument("--checkpoint", action="append", help="verify: a signed checkpoint the log must contain (repeatable)")
|
|
1353
|
+
p.add_argument("--trusted", help="verify: file of public keys you pinned; says which checkpoint signers are vouched")
|
|
1354
|
+
for name in ("commit", "derive", "verify"):
|
|
1355
|
+
p = sub.add_parser(name)
|
|
1356
|
+
p.add_argument("--seed-file")
|
|
1357
|
+
p.add_argument("--practice", action="store_true")
|
|
1358
|
+
if name != "commit":
|
|
1359
|
+
p.add_argument("--size", type=int_arg(-10 ** 12, 10 ** 12), required=True) # parsed here, checked against the challenge's declared range in the command
|
|
1360
|
+
p.add_argument("--idx", type=int_arg(-10 ** 12, 10 ** 12), required=True)
|
|
1361
|
+
p.add_argument("--sandbox", choices=["auto", "off"], default="auto", help=sandbox_help)
|
|
1362
|
+
if name == "verify":
|
|
1363
|
+
p.add_argument("--answer", required=True)
|
|
1364
|
+
a = ap.parse_args(argv)
|
|
1365
|
+
if a.cmd in ("selftest", "derive", "verify") and a.sandbox == "auto":
|
|
1366
|
+
code = reexec_sandboxed(argv if argv is not None else sys.argv[1:])
|
|
1367
|
+
if code is not None:
|
|
1368
|
+
return code
|
|
1369
|
+
if a.cmd == "log" and a.action == "append" and not a.file:
|
|
1370
|
+
raise SystemExit("log append needs --file")
|
|
1371
|
+
if a.cmd in ("rerun", "spread") and manifest().get("metric") == "quality":
|
|
1372
|
+
raise SystemExit(f"this bundle is a QUALITY challenge: `{a.cmd}` re-measures speed. Use qcheck.py qrerun")
|
|
1373
|
+
try:
|
|
1374
|
+
return {"info": cmd_info, "selftest": cmd_selftest, "commit": cmd_commit, "derive": cmd_derive, "verify": cmd_verify, "rerun": cmd_rerun,
|
|
1375
|
+
"keygen": cmd_keygen, "attest-check": cmd_attest_check, "log": cmd_log, "spread": cmd_spread}[a.cmd](a)
|
|
1376
|
+
except FileRefused as e: # a path that is not a regular file of sane size: one line, exit 1
|
|
1377
|
+
print(f"REFUSED: {e}", file=sys.stderr)
|
|
1378
|
+
return 1
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
if __name__ == "__main__":
|
|
1382
|
+
sys.exit(main())
|