guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,828 @@
1
+ """Speedup certificates: one JSON file that proves a measured speedup to someone who does not trust the operator (see COMPILER.md).
2
+
3
+ certificate.py build --family F --submission solvers/x.py --seed-dir S [--reveal-seeds] [--round-data D --round-id R] --out x.cert.json
4
+ certificate.py verify x.cert.json [--rerun K]
5
+
6
+ A certificate carries: the claim; the rules fingerprint (hashes of the estimator, harness, instance code and, for a declared domain, its
7
+ declaration, compiled ladder and reference solver); the seed commitments and (only with --reveal-seeds, i.e. after the reveal) the seeds;
8
+ both solvers' source; every raw per-instance record of every ladder; and the round's hash-chained log. `verify` needs nothing from the
9
+ operator. It checks, and reports each as PASS / FAIL / SKIP:
10
+
11
+ rules this checkout's protocol code hashes to the certificate's (a different estimator would score the same data differently)
12
+ domain a declared domain's files hash to what the certificate names, and load
13
+ log the hash chain verifies, entry 0 holds the commitments and the rules fingerprint, the submission's hash was logged
14
+ seeds each revealed seed hashes to a committed value
15
+ sources the solver sources are the ones the ladders were measured with (sha256 in every ladder header)
16
+ coverage re-derives every instance from the seeds; each ladder holds exactly those instances, for the same size set
17
+ credit recomputes every seed's credit from the raw records with this checkout's estimator and compares it with the claim
18
+ rerun (--rerun K, needs --execute) re-measures K instances on THIS machine: the solver must still solve them, and the speedup should reproduce
19
+
20
+ A certificate is untrusted input, so by default `verify` EXECUTES NOTHING from it: a declared domain's decl.py is hash-checked and read as data
21
+ (instance coverage of a declared domain is then SKIPPED). `--execute` imports that decl.py and, with --rerun, runs the solver sources, on this
22
+ machine, unsandboxed: use it only in a throwaway VM or container. Built-in families derive instances with this checkout's own code, no flag needed.
23
+
24
+ A certificate is as strong as its weakest link: without a log it can never be OFFICIAL (the commitment's publication cannot be shown);
25
+ a ladder measured in local mode is PROVISIONAL (only the Docker sandbox meters untrusted code); the verifier says which.
26
+
27
+ PRIVATE-SOLVER certificates. `make_private` (or `build --private`) removes the solver's source and every failure message from its ladders (a
28
+ traceback quotes the code); everything else is unchanged, including the solver's SHA-256, which the round's log committed to before the seeds were
29
+ revealed. A verifier can still check the log, the fingerprints, the questions, the rules and the credit arithmetic. What it cannot do is re-run the
30
+ solver or inspect it: the timings rest on the operator's sandbox, and the certificate says so. If the owner later publishes the file,
31
+ `verify --source FILE` checks that it hashes to the committed value and, with --rerun --execute, re-measures it: the certificate is then as strong
32
+ as a full one. A private-solver certificate hides the algorithm from the PUBLIC, not from the operator, who ran it. (Not to be confused with a
33
+ "sealed" certificate: one built before the reveal, without the seeds.)
34
+ """
35
+ import argparse
36
+ import contextlib
37
+ import hashlib
38
+ import json
39
+ import math
40
+ import os
41
+ import random
42
+ import re
43
+ import secrets
44
+ import sys
45
+ import tempfile
46
+ import time
47
+
48
+ import declared
49
+ import extrapolate as E
50
+ import families
51
+ import harness
52
+ import judge
53
+ import official
54
+ import rounds
55
+
56
+ HERE = os.path.dirname(os.path.abspath(__file__))
57
+ FORMAT = "svp-certificate-v1"
58
+
59
+
60
+ class CertificateError(Exception):
61
+ pass
62
+
63
+
64
+ def sha256(data):
65
+ return hashlib.sha256(data if isinstance(data, bytes) else data.encode()).hexdigest()
66
+
67
+
68
+ def canon(obj):
69
+ return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
70
+
71
+
72
+ def _read(path):
73
+ # newline="": NO translation. The harness hashes the file's raw bytes (solver_sha256 in every ladder header), so a certificate that stores the text must
74
+ # give back the same bytes, or a solver saved with Windows line endings would fail its own `sources` check.
75
+ with open(path, encoding="utf-8", newline="") as f:
76
+ return f.read()
77
+
78
+
79
+ def rules_of(family):
80
+ """The protocol-code part of the fingerprint (declared-domain files are carried separately, by role)."""
81
+ fp = rounds.code_fingerprint(family)
82
+ return {"dims": fp["dims"], "instance_version": fp["instance_version"],
83
+ "code": {rel: fp["code"][rel] for rel in rounds.CODE_FILES}}
84
+
85
+
86
+ def commit_published(log_entries, commitments, round_id_hint=None):
87
+ """(ok, why): the log's hash chain verifies and entry 0 contains every commitment. Verified in a temporary directory named for the round."""
88
+ if not log_entries:
89
+ return False, "no round log included"
90
+ rid = (log_entries[0].get("data") or {}).get("id")
91
+ if not isinstance(rid, str) or not rounds.ROUND_ID_RE.match(rid):
92
+ return False, "the first log entry has no valid round id"
93
+ with tempfile.TemporaryDirectory() as td:
94
+ d = os.path.join(td, rid)
95
+ os.makedirs(d)
96
+ with open(os.path.join(d, "log.jsonl"), "w", encoding="utf-8") as f:
97
+ for e in log_entries:
98
+ f.write(rounds.canonical(e) + "\n")
99
+ ok, n, head, err = rounds.verify_log(d)
100
+ if not ok:
101
+ return False, f"the log does not verify: {err}"
102
+ if not set(commitments) <= set(log_entries[0]["data"].get("commitments", [])):
103
+ return False, "a seed commitment is not in entry 0 of the log"
104
+ return True, f"{n} entries, head {head[:16]}"
105
+
106
+
107
+ def score_seed(family, name, tag, published, base_text, sub_text, work):
108
+ """The board entry for one seed, computed exactly as judge.py computes it: extrapolate.py on the two ladder files, then make_entry."""
109
+ import subprocess
110
+ bp, sp = os.path.join(work, f"{tag}_baseline.jsonl"), os.path.join(work, f"{tag}_submission.jsonl")
111
+ with open(bp, "w", encoding="utf-8") as f:
112
+ f.write(base_text)
113
+ with open(sp, "w", encoding="utf-8") as f:
114
+ f.write(sub_text)
115
+ r = subprocess.run([sys.executable, os.path.join(HERE, "extrapolate.py"), "--baseline", bp, "--submission", sp], cwd=HERE, capture_output=True, text=True)
116
+ if r.returncode:
117
+ raise CertificateError(f"seed {tag}: the estimator refused these ladders: {(r.stdout.strip() or r.stderr.strip())[-300:]}")
118
+ with open(sp + ".claim.json", encoding="utf-8") as f:
119
+ claim = json.load(f)
120
+ return judge.make_entry(name, family, tag, sp, claim, 512, published, base_file=bp)
121
+
122
+
123
+ def claim_of(entries):
124
+ rows = judge.leaderboard_rows(sorted(entries, key=lambda e: e["seed_tag"])) # per_seed lists follow entry order: fix it, or the same data compares unequal
125
+ if len(rows) != 1:
126
+ raise CertificateError("the entries do not describe exactly one submission")
127
+ r = rows[0]
128
+ status, reasons = r["status"], r["reasons"]
129
+ if all(e.get("sandbox") == "count" for e in entries):
130
+ status, reasons = _counted_status(entries) # judge.py's OFFICIAL means "Docker sandbox, timed": a counted ladder is held to its own rules
131
+ return {"credit_bits": r["credit"], "status": status, "reasons": reasons, "per_seed": r["per_seed"]}
132
+
133
+
134
+ def _counted_status(entries):
135
+ """-> (status, reasons) for entries made by step counting. COUNTED = at least judge.MIN_SEEDS committed seeds whose commitments were shown to be public
136
+ first, one solver file and one baseline file throughout, hashes and instance version recorded. Anything less is COUNTED, PROVISIONAL, with the reasons."""
137
+ why = []
138
+ if len(entries) < judge.MIN_SEEDS:
139
+ why.append(f"{len(entries)} seed(s), fewer than {judge.MIN_SEEDS}")
140
+ if not all(e.get("commit_checked") for e in entries):
141
+ why.append("seed commitment not checked against a published value")
142
+ for key, label in (("solver_sha256", "solver file hash"), ("baseline_sha256", "baseline file hash"), ("instance_version", "instance version")):
143
+ if not all(e.get(key) for e in entries):
144
+ why.append(f"no {label} recorded")
145
+ if len({e.get("solver_sha256") for e in entries}) > 1:
146
+ why.append("the seeds used different versions of the submission file")
147
+ if len({e.get("baseline_sha256") for e in entries}) > 1:
148
+ why.append("the seeds used different baseline files")
149
+ return ("COUNTED" if not why else "COUNTED, PROVISIONAL"), why
150
+
151
+
152
+ # ------------------------------------------------------------------------------------------------------------- build
153
+ def _counted_info(ladders):
154
+ """None for ordinary (timed) ladders. For ladders made by counted.py: {steps_per_second, python, dims, n, cap}. A mix of counted and timed ladders, or
155
+ counted ladders that disagree about the unit, the interpreter, the cap or the sizes, is refused: they would not be comparable."""
156
+ heads, shapes = [], set()
157
+ for tag, lad in sorted(ladders.items()):
158
+ for role in ("baseline", "submission"):
159
+ lines = [ln for ln in lad[role].splitlines() if ln.strip()]
160
+ heads.append(json.loads(lines[0]))
161
+ recs = [json.loads(ln) for ln in lines[1:]]
162
+ per = {}
163
+ for r in recs:
164
+ per[r["dim"]] = per.get(r["dim"], 0) + 1
165
+ shapes.add(tuple(sorted(per.items())))
166
+ flags = {h.get("cost_model") == "count" for h in heads}
167
+ if flags == {False}:
168
+ return None
169
+ if flags != {True}:
170
+ raise CertificateError("the ladders mix counted and timed cost models: they are not comparable")
171
+ for key in ("steps_per_second", "python", "budget", "counter_sha256"):
172
+ if len({json.dumps(h.get(key)) for h in heads}) != 1:
173
+ raise CertificateError(f"the counted ladders disagree about {key}")
174
+ if len(shapes) != 1:
175
+ raise CertificateError("the counted ladders cover different sizes or numbers of instances")
176
+ shape = dict(next(iter(shapes)))
177
+ return {"steps_per_second": heads[0]["steps_per_second"], "python": heads[0]["python"], "dims": sorted(shape), "n": min(shape.values()), "cap": heads[0]["budget"],
178
+ "counter_sha256": heads[0].get("counter_sha256")}
179
+
180
+
181
+ def make_private(cert):
182
+ """A PRIVATE-SOLVER copy of a certificate: no solver source, and no failure text in the submission's ladders (a crash message is a traceback, and a
183
+ traceback quotes the solver's own lines). Credit, claim, log, rules and hashes are untouched, so everything but re-running still verifies.
184
+ The input is not modified."""
185
+ import copy
186
+ out = copy.deepcopy(cert)
187
+ if not isinstance(out.get("submission"), dict):
188
+ raise CertificateError("not a certificate: no submission")
189
+ out["submission"]["source"] = None
190
+ out["private"] = True
191
+ for tag, lad in (out.get("ladders") or {}).items():
192
+ lines = []
193
+ for line in lad["submission"].splitlines():
194
+ if not line.strip():
195
+ continue
196
+ rec = json.loads(line)
197
+ if "err" in rec: # only per-instance records carry failure text
198
+ rec.pop("err")
199
+ rec["err_withheld"] = True
200
+ lines.append(json.dumps(rec))
201
+ lad["submission"] = "\n".join(lines) + "\n"
202
+ return out
203
+
204
+
205
+ def build(family, submission, seed_dir, reveal_seeds=False, baseline=None, results_dir=None, round_data=None, round_id=None, now=None, private=False):
206
+ if family not in families.FAMILIES:
207
+ raise CertificateError(f"unknown family {family!r}")
208
+ results_dir = results_dir or os.path.join(HERE, "results")
209
+ baseline = os.path.abspath(baseline or judge.default_baseline(family))
210
+ submission = os.path.abspath(submission)
211
+ name = os.path.splitext(os.path.basename(submission))[0]
212
+ seeds = official.verified_seeds(seed_dir)
213
+ base_text_src, sub_text_src = _read(baseline), _read(submission)
214
+ log = None
215
+ if round_id:
216
+ if not round_data:
217
+ raise CertificateError("--round-id needs --round-data")
218
+ r = rounds.get_round(round_data, round_id)
219
+ if r["family"] != family:
220
+ raise CertificateError(f"round {round_id} is a {r['family']} round, not {family}")
221
+ if reveal_seeds and r["state"] != "revealed":
222
+ # putting the seeds in a file IS the reveal: before the round is evaluated they would let anyone precompute every instance
223
+ raise CertificateError(f"round {round_id} is {r['state']!r}, not 'revealed': a certificate with seeds may only be built after the reveal "
224
+ "(build a sealed one, without --reveal-seeds, until then)")
225
+ log = rounds.read_log(rounds.rdir(round_data, round_id))
226
+ commits = sorted(h for _, h in seeds)
227
+ published, why = commit_published(log, commits)
228
+ ladders, entries, seed_blob = {}, [], {}
229
+ with tempfile.TemporaryDirectory() as work:
230
+ for path, h in seeds:
231
+ tag = h[:8]
232
+ bfile = os.path.join(results_dir, f"judged_{family}_{tag}_baseline_{os.path.basename(baseline)}.jsonl")
233
+ sfile = os.path.join(results_dir, f"judged_{family}_{tag}_submission_{os.path.basename(submission)}.jsonl")
234
+ for f in (bfile, sfile):
235
+ if not os.path.exists(f):
236
+ raise CertificateError(f"missing ladder {f}: evaluate this submission on seed {tag} first")
237
+ ladders[tag] = {"baseline": _read(bfile), "submission": _read(sfile)}
238
+ entries.append(score_seed(family, name, tag, published, ladders[tag]["baseline"], ladders[tag]["submission"], work))
239
+ if reveal_seeds:
240
+ with open(path, "rb") as f:
241
+ seed_blob[h] = f.read().strip().decode()
242
+ cert = {"format": FORMAT, "created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now or time.time())), "family": family,
243
+ "claim": claim_of(entries), "entries": entries, "rules": rules_of(family), "commitments": commits,
244
+ "seeds": seed_blob or None, "seeds_revealed": bool(reveal_seeds),
245
+ "submission": {"name": name, "sha256": sha256(sub_text_src), "source": sub_text_src},
246
+ "baseline": {"name": os.path.basename(baseline), "sha256": sha256(base_text_src), "source": base_text_src},
247
+ "ladders": ladders, "log": log, "log_note": why}
248
+ if family in families.DECLARED:
249
+ d = families.DECLARED[family]["dir"]
250
+ mod = declared.load_declaration(d)
251
+ files = {"decl.py": os.path.join(d, declared.DECL), "compiled.json": os.path.join(d, declared.COMPILED),
252
+ "reference": declared.reference_path(d, mod)}
253
+ cert["domain"] = {"name": family, "reference_name": mod.REFERENCE_SOLVER,
254
+ "files": {role: {"sha256": sha256(_read(p).encode()), "text": _read(p)} for role, p in files.items()}}
255
+ cert["private"] = False
256
+ counted = _counted_info(ladders)
257
+ if counted:
258
+ # counted ladders carry THEIR OWN size ladder (small: counting is slow), chosen by whoever ran them: the verifier then holds the ladder check to
259
+ # these sizes, not to the family's official ones, and the verdict says that a counted certificate is not an official round result
260
+ cert["count"] = counted
261
+ cert["rules"]["dims"] = counted["dims"]
262
+ cert["rules"]["cost_model"] = "count"
263
+ return make_private(cert) if private else cert
264
+
265
+
266
+ # ------------------------------------------------------------------------------------------------------------- verify
267
+ def _chk(out, check, status, detail=""):
268
+ out.append({"check": check, "status": status, "detail": detail})
269
+ return status == "pass"
270
+
271
+
272
+ def _domain_derive(cert, family, work, out, metas, stack, execute=False):
273
+ """(derive, verify) for the certificate's family: this checkout's registry for a built-in, the certificate's own files for a declared domain.
274
+ A declared domain's decl.py is CODE FROM THE CERTIFICATE: it is hash-checked and read as data, and imported (executed) only with execute=True."""
275
+ dom = cert.get("domain")
276
+ if not dom:
277
+ if family in families.DECLARED:
278
+ _chk(out, "domain", "fail", f"{family!r} is a declared domain here but the certificate carries no declaration files")
279
+ return None
280
+ if family not in families.FAMILIES:
281
+ _chk(out, "domain", "fail", f"{family!r} is not a family this checkout knows, and the certificate carries no declaration")
282
+ return None
283
+ _chk(out, "domain", "pass", f"built-in family {family}")
284
+ return families.FAMILIES[family]
285
+ if dom.get("name") != family or not declared.NAME_RE.match(str(dom.get("name"))):
286
+ _chk(out, "domain", "fail", "the declaration's name is not the certificate's family")
287
+ return None
288
+ d = os.path.join(work, "domains", dom["name"])
289
+ os.makedirs(d)
290
+ ref_rel = dom["reference_name"]
291
+ if os.path.isabs(ref_rel) or ".." in ref_rel.split("/"):
292
+ _chk(out, "domain", "fail", "the reference path escapes the declaration directory")
293
+ return None
294
+ targets = {"decl.py": os.path.join(d, declared.DECL), "compiled.json": os.path.join(d, declared.COMPILED), "reference": os.path.join(d, ref_rel)}
295
+ for role, f in dom["files"].items():
296
+ if role not in targets:
297
+ _chk(out, "domain", "fail", f"unexpected file role {role!r}")
298
+ return None
299
+ if sha256(f["text"].encode()) != f["sha256"]:
300
+ _chk(out, "domain", "fail", f"{role} does not match its recorded hash")
301
+ return None
302
+ os.makedirs(os.path.dirname(targets[role]), exist_ok=True)
303
+ with open(targets[role], "w", encoding="utf-8") as g:
304
+ g.write(f["text"])
305
+ compiled = declared.read_compiled(d)
306
+ texts = {role: f["text"].encode() for role, f in dom["files"].items()}
307
+ if (compiled is None or compiled.get("format") != declared.FORMAT or compiled.get("name") != family
308
+ or compiled.get("decl_sha256") != sha256(texts.get("decl.py", b"")) or compiled.get("reference_sha256") != sha256(texts.get("reference", b""))
309
+ or not all(k in compiled for k in ("dims", "practice_dims", "official", "version"))):
310
+ _chk(out, "domain", "fail", "the compiled record does not match the declaration and reference files it sits beside")
311
+ return None
312
+ if execute:
313
+ try:
314
+ mod = declared.load_declaration(d) # EXECUTES the certificate's decl.py
315
+ problems = declared.freshness_problems(d, mod, compiled)
316
+ except declared.DeclarationError as e:
317
+ _chk(out, "domain", "fail", str(e))
318
+ return None
319
+ if problems:
320
+ _chk(out, "domain", "fail", "the compiled record does not match the declaration: " + "; ".join(problems))
321
+ return None
322
+ if family in families.DECLARED:
323
+ # this checkout already has a domain of that name: it must be THE certificate's declaration, or the registry (verify, OFFICIAL, rerun)
324
+ # would silently judge the certificate by someone else's rules
325
+ if families.DECLARED[family]["decl_sha256"] != compiled["decl_sha256"]:
326
+ _chk(out, "domain", "fail", f"this checkout's own {family!r} declaration differs from the certificate's (decl.py hashes differ)")
327
+ return None
328
+ elif family in families.FAMILIES:
329
+ _chk(out, "domain", "fail", f"{family!r} is a built-in family here, but the certificate carries a declaration for it")
330
+ return None
331
+ else:
332
+ # judge.leaderboard_rows looks the family's thresholds up in the registry: register them (data from compiled.json; with execute, the whole
333
+ # declaration) for the duration of this verification, then remove them
334
+ if execute:
335
+ declared.register(families, d, mod, compiled)
336
+ else:
337
+ families.VERSIONS[family] = compiled["version"]
338
+ families.OFFICIAL[family] = compiled["official"]
339
+ families.DEFAULT_DIMS[family] = compiled["dims"]
340
+ stack.callback(declared.unregister, families, family)
341
+ ver_ok = all(m.get("instance_version") == compiled["version"] for m in metas)
342
+ _chk(out, "domain", "pass" if ver_ok else "fail",
343
+ (f"declaration {compiled['version']}, decl.py {compiled['decl_sha256'][:12]}..., compiled ladder {compiled['dims']}, cap {compiled['official']['budget']}; "
344
+ + ("declaration executed (--execute)" if execute else "files hash-checked and read as data; the declaration was NOT executed"))
345
+ if ver_ok else "a ladder was measured with a different instance version than this declaration")
346
+ return declared.make_spec(mod) if execute else None
347
+
348
+
349
+ ALL_INSTANCES = 10 ** 9 # what `--rerun all` means: more than any certificate holds, so every instance is re-run (the sample is then the whole pool)
350
+
351
+
352
+ def rerun_count(text):
353
+ """argparse type for --rerun: a non-negative integer, or `all` (re-run EVERY instance: the only way to leave nothing covered merely statistically)."""
354
+ if str(text).strip().lower() == "all":
355
+ return ALL_INSTANCES
356
+ try:
357
+ k = int(text)
358
+ except ValueError:
359
+ raise argparse.ArgumentTypeError("--rerun takes a number or `all`")
360
+ if k < 0:
361
+ raise argparse.ArgumentTypeError("--rerun cannot be negative")
362
+ return k
363
+
364
+
365
+ MAX_COUNTED_CAP = 10_000 # counted seconds (x steps_per_second instructions): a larger per-instance budget in an untrusted certificate would make --rerun --execute run for hours
366
+ RERUN_TOLERANCE = 2.0 # bits: LOOSE on purpose until cross-machine spread has been measured (compile_domain.py recheck --measure); tighten with --tolerance
367
+
368
+
369
+ def verify(cert, rerun=0, execute=False, log=print, tolerance=RERUN_TOLERANCE, sample_seed=0, source=None):
370
+ """-> {"ok": bool, "checks": [...], "claim": ..., "private": bool, "source_revealed": bool}. Never trusts the certificate's own claim: everything is recomputed.
371
+ A certificate is untrusted input. By default nothing in it is executed: a declared domain's decl.py is hash-checked and read as data, so
372
+ instance coverage of a declared domain is SKIPPED, and --rerun is refused. execute=True imports the certificate's decl.py and (with
373
+ rerun) runs its solver sources on this machine, unsandboxed: do that only in a throwaway VM or container.
374
+ `source` (text) is a solver the owner has revealed for a PRIVATE-SOLVER certificate: it must hash to the certificate's committed solver hash, and
375
+ then --rerun can re-measure it."""
376
+ try:
377
+ return _verify(cert, rerun, execute, tolerance, sample_seed, source)
378
+ except (KeyError, TypeError, ValueError, AttributeError, IndexError, StopIteration, OverflowError, ZeroDivisionError) as e:
379
+ return {"ok": False, "claim": None, "private": False, "source_revealed": False, "counted": False,
380
+ "checks": [{"check": "format", "status": "fail", "detail": f"malformed certificate ({type(e).__name__}: {e})"}]}
381
+
382
+
383
+ def _verify(cert, rerun, execute, tolerance=RERUN_TOLERANCE, sample_seed=0, source=None):
384
+ out = []
385
+ if not isinstance(cert, dict) or cert.get("format") != FORMAT:
386
+ return {"ok": False, "checks": [{"check": "format", "status": "fail", "detail": f"not a {FORMAT} certificate"}], "claim": None}
387
+ family = cert["family"]
388
+ _chk(out, "format", "pass", f"{FORMAT}, family {family}, created {cert.get('created')}")
389
+ # rules: this checkout's protocol code against the certificate's
390
+ mine_code = {rel: (sha256(open(os.path.join(HERE, rel), "rb").read()) if os.path.exists(os.path.join(HERE, rel)) else None) for rel in rounds.CODE_FILES}
391
+ diff = sorted(rel for rel in rounds.CODE_FILES if mine_code[rel] != cert["rules"]["code"].get(rel))
392
+ _chk(out, "rules", "fail" if diff else "pass",
393
+ ("this checkout differs from the certificate's rules in: " + ", ".join(diff) + ". Check out the version it names; a different estimator or "
394
+ "harness would score the same data differently") if diff else f"{len(mine_code)} protocol files hash exactly as the certificate names")
395
+ # sources vs ladders
396
+ metas, ladder_ok = [], True
397
+ for tag, lad in sorted(cert["ladders"].items()):
398
+ for role in ("baseline", "submission"):
399
+ try:
400
+ meta = json.loads(lad[role].splitlines()[0])
401
+ except (ValueError, IndexError):
402
+ ladder_ok = False
403
+ _chk(out, "sources", "fail", f"seed {tag}: the {role} ladder has no readable header")
404
+ continue
405
+ metas.append(meta)
406
+ want = cert[role]["sha256"]
407
+ if meta.get("solver_sha256") != want:
408
+ ladder_ok = False
409
+ _chk(out, "sources", "fail", f"seed {tag}: the {role} ladder was measured with a solver whose hash is {str(meta.get('solver_sha256'))[:12]}, "
410
+ f"not the included source ({want[:12]})")
411
+ private = cert.get("private") is True
412
+ sub = cert["submission"]
413
+ revealed = None # the solver's source, once the owner has revealed it and the caller supplied it
414
+ if private:
415
+ if sub.get("source") is not None:
416
+ ladder_ok = False
417
+ _chk(out, "sources", "fail", "a certificate marked private-solver carries the solver's source: it is malformed or was made private badly")
418
+ elif sha256(sub["source"]) != sub["sha256"]:
419
+ ladder_ok = False
420
+ _chk(out, "sources", "fail", "the included solver source does not match its own recorded hash")
421
+ if sha256(cert["baseline"]["source"]) != cert["baseline"]["sha256"]:
422
+ ladder_ok = False
423
+ _chk(out, "sources", "fail", "the included baseline source does not match its own recorded hash")
424
+ in_log = " (and the round's log)" if cert.get("log") else "" # a certificate built outside a round has no log: do not claim one committed to anything
425
+ if private and source is not None:
426
+ if sha256(source) != sub["sha256"]:
427
+ ladder_ok = False
428
+ _chk(out, "solver disclosure", "fail", f"the file supplied does not hash to the solver hash this certificate{in_log} committed to: "
429
+ f"it hashes to {sha256(source)[:12]}..., the certificate says {sub['sha256'][:12]}...")
430
+ else:
431
+ revealed = source
432
+ _chk(out, "solver disclosure", "pass", f"the file supplied is exactly the solver this certificate measured (sha256 {sub['sha256'][:12]}...)")
433
+ elif private:
434
+ _chk(out, "solver disclosure", "skip", "PRIVATE SOLVER: its source is not in this certificate, so it cannot be inspected or re-run here. Its hash is the one every "
435
+ f"ladder header names{' and the round' + chr(39) + 's log committed to' if cert.get('log') else ''}; if the owner publishes the file, check it with --source FILE")
436
+ if ladder_ok:
437
+ _chk(out, "sources", "pass", f"submission {sub['sha256'][:12]}...{' (private solver: source withheld)' if private else ''} and baseline {cert['baseline']['sha256'][:12]}... "
438
+ "are what every ladder header names")
439
+ metas = [m for m in metas if m.get("meta")]
440
+ with tempfile.TemporaryDirectory() as work, contextlib.ExitStack() as stack:
441
+ spec = _domain_derive(cert, family, work, out, metas, stack, execute)
442
+ # log
443
+ commits = cert["commitments"]
444
+ published, why = commit_published(cert.get("log"), commits)
445
+ round_declared = False # True when the log's entry 0 declared THIS counted ladder before any submission (see rounds.create_round(count=...))
446
+ try:
447
+ log_count = _counted_info(cert["ladders"]) # a malformed mix is reported by the cost-model check below
448
+ except CertificateError:
449
+ log_count = None
450
+ if cert.get("log"):
451
+ if published:
452
+ data0 = cert["log"][0]["data"]
453
+ fp = data0.get("fingerprint") or {}
454
+ same = fp.get("code") is not None and all(fp["code"].get(rel) == cert["rules"]["code"].get(rel) for rel in rounds.CODE_FILES)
455
+ subs = {e["data"].get("sha256") for e in cert["log"] if e.get("kind") == "submission"}
456
+ changed = [e for e in cert["log"] if e.get("kind") == "code_changed"]
457
+ bad = []
458
+ if not same or fp.get("dims") != cert["rules"]["dims"] or fp.get("instance_version") != cert["rules"]["instance_version"]:
459
+ bad.append("the rules fingerprint in log entry 0 (code, size ladder, instance version) differs from the certificate's rules")
460
+ if cert["submission"]["sha256"] not in subs:
461
+ bad.append("the submission's hash was never logged as an accepted submission")
462
+ if cert.get("domain"): # the declaration, its compiled ladder and its reference were committed to in entry 0 too
463
+ anchored = set((fp.get("code") or {}).values())
464
+ off = sorted(role for role, f in cert["domain"]["files"].items() if f["sha256"] not in anchored)
465
+ if off:
466
+ bad.append("the domain file(s) " + ", ".join(off) + " are not the ones the round committed to in log entry 0")
467
+ declared_count = data0.get("count") if data0.get("cost_model") == "count" and isinstance(data0.get("count"), dict) else None
468
+ if declared_count is not None or log_count is not None:
469
+ if log_count is None:
470
+ bad.append("the round declared step counting in log entry 0 but this certificate's ladders are not step-counted")
471
+ elif declared_count is None:
472
+ bad.append("this certificate's ladders are step-counted but the round did not declare step counting in log entry 0: their sizes and cap would be the prover's")
473
+ elif any(declared_count.get(k) != log_count.get(k) for k in ("dims", "n", "cap", "steps_per_second", "python", "counter_sha256")):
474
+ bad.append("the counted ladder (sizes, instances per size, cap, unit, Python version or counter) is not the one the round declared in log entry 0")
475
+ elif (fp.get("code") or {}).get("count_runner.py") != log_count["counter_sha256"]:
476
+ bad.append("the counting program is not the one the round's rules fingerprint names")
477
+ elif private:
478
+ bad.append("a counted round requires every solver's source to be published (the proof is re-execution), but this certificate withholds it")
479
+ else:
480
+ round_declared = True
481
+ import counted
482
+ src = cert["submission"].get("source")
483
+ problems = counted.source_problems(src) if isinstance(src, str) else []
484
+ if problems: # the round should have refused it at submission; a verifier must not rely on the operator having done so (the counter refuses to run it anyway)
485
+ bad.append("the solver's source is outside the restricted subset a counted solver is written in: " + "; ".join(problems[:3]))
486
+ round_declared = False
487
+ _chk(out, "log", "fail" if bad else "pass", ("; ".join(bad)) if bad else
488
+ f"{why}; commitments and rules were public before submissions; submission logged" + (f"; NOTE {len(changed)} code_changed entry(ies)" if changed else ""))
489
+ else:
490
+ _chk(out, "log", "fail", why)
491
+ else:
492
+ _chk(out, "log", "skip", "no log included: the publication time of the commitments cannot be shown, so the credit is at most PROVISIONAL")
493
+ # seeds
494
+ seeds = cert.get("seeds") or {}
495
+ if cert.get("seeds_revealed"):
496
+ bad = [h for h, s in seeds.items() if sha256(s.strip().encode()) != h]
497
+ missing = [h for h in commits if h not in seeds]
498
+ _chk(out, "seeds", "fail" if bad or missing else "pass",
499
+ f"{len(bad)} seed(s) do not hash to their commitment, {len(missing)} missing" if bad or missing else f"{len(seeds)} revealed seed(s) hash to their commitments")
500
+ else:
501
+ _chk(out, "seeds", "skip", "seeds not revealed in this certificate: instance coverage and re-measurement cannot be checked")
502
+ # ladders: one baseline/submission pair per committed seed (a dropped seed would raise the minimum), on the rules' own size ladder
503
+ pair_bad = []
504
+ if sorted(cert["ladders"]) != sorted(c[:8] for c in commits):
505
+ pair_bad.append("the ladders are not exactly one pair per committed seed")
506
+ for tag, lad in cert["ladders"].items():
507
+ for role in ("baseline", "submission"):
508
+ recs = [json.loads(line) for line in lad[role].splitlines()[1:] if line.strip()]
509
+ if sorted({r["dim"] for r in recs}) != sorted(cert["rules"]["dims"]):
510
+ pair_bad.append(f"seed {tag} {role}: sizes {sorted({r['dim'] for r in recs})} are not the rules' ladder {sorted(cert['rules']['dims'])}")
511
+ per = {}
512
+ for r in recs:
513
+ per[r["dim"]] = per.get(r["dim"], 0) + 1
514
+ if len(set(per.values())) > 1:
515
+ pair_bad.append(f"seed {tag} {role}: the number of instances differs between sizes")
516
+ _chk(out, "ladders", "fail" if pair_bad else "pass",
517
+ "; ".join(pair_bad) if pair_bad else f"{len(cert['ladders'])} seed(s), each with a baseline and a submission ladder on the rules' sizes {cert['rules']['dims']}")
518
+ # cost model: deterministic step counting (counted.py) is a different KIND of evidence from a timed ladder, and the verdict must say so
519
+ count = cert.get("count") if isinstance(cert.get("count"), dict) else None
520
+ try: # the ladders themselves say which cost model they use: a certificate may not hide that by dropping its own `count` field (instruction counts passing as seconds)
521
+ actual = _counted_info(cert["ladders"])
522
+ except CertificateError as e:
523
+ _chk(out, "cost model", "fail", f"the ladders are inconsistent: {e}")
524
+ actual, count = None, None
525
+ else:
526
+ if actual is None and any(m.get("sandbox") == "count" for m in metas):
527
+ # claim_of grants the COUNTED status to entries whose sandbox is "count": a timed ladder must not be able to take it by labelling its header so
528
+ _chk(out, "cost model", "fail", "a ladder header says sandbox \"count\" but the ladders are not step-counted: the label is only for counted ladders")
529
+ count = None
530
+ elif (actual is not None) != (count is not None):
531
+ _chk(out, "cost model", "fail", "the ladders are " + ("step-COUNTED" if actual is not None else "timed") + " but the certificate says "
532
+ + ("they are counted" if count is not None else "nothing about step counting") + ": a counted cost must never be presented as seconds")
533
+ count = None # already reported: skip the detailed counted checks below
534
+ if count is not None:
535
+ bad = _check_counted(cert, count)
536
+ _chk(out, "cost model", "fail" if bad else "pass", "; ".join(bad[:3]) + (f" (and {len(bad) - 3} more)" if len(bad) > 3 else "") if bad else
537
+ f"deterministic STEP COUNTING: every cost is a bytecode-instruction count / {count['steps_per_second']:g}, recorded on {count['python']}. It is exactly "
538
+ "reproducible (--rerun K --execute must give identical counts) but it is NOT a time, it is blind to work inside C functions, and the ladder "
539
+ f"(sizes {count['dims']}, {count['n']} per size, cap {count['cap']:g}) "
540
+ + ("was declared by the round in log entry 0 before any submission, the same for every participant (a ROUND-DECLARED ladder)" if round_declared else
541
+ "was chosen by whoever ran it: not an official round result"))
542
+ # coverage
543
+ if spec is not None and cert.get("seeds_revealed") and ladder_ok:
544
+ derive = spec[0]
545
+ cov_bad = []
546
+ for tag, lad in sorted(cert["ladders"].items()):
547
+ h = next((c for c in commits if c.startswith(tag)), None)
548
+ if h is None or h not in seeds:
549
+ cov_bad.append(f"seed {tag}: no matching revealed seed")
550
+ continue
551
+ master = seeds[h].strip().encode()
552
+ for role in ("baseline", "submission"):
553
+ recs = [json.loads(line) for line in lad[role].splitlines()[1:] if line.strip()]
554
+ dims = sorted({r["dim"] for r in recs})
555
+ n = len(recs) // max(len(dims), 1)
556
+ want = set()
557
+ for dm in dims:
558
+ for i in range(n):
559
+ want.add(derive(master, dm, i)["id"])
560
+ have = [r["id"] for r in recs]
561
+ if len(have) != len(set(have)) or set(have) != want:
562
+ cov_bad.append(f"seed {tag} {role}: records are not exactly the {len(want)} instances the seed derives")
563
+ _chk(out, "coverage", "fail" if cov_bad else "pass",
564
+ "; ".join(cov_bad) if cov_bad else "every ladder holds exactly the instances its seed derives, no more, no fewer, none repeated")
565
+ else:
566
+ _chk(out, "coverage", "skip", "needs the revealed seeds" + ("" if spec is not None or not cert.get("domain") else
567
+ " and the declaration executed (--execute: runs the certificate's decl.py on this machine)"))
568
+ # credit
569
+ try:
570
+ entries = [score_seed(family, cert["submission"]["name"], tag, published, lad["baseline"], lad["submission"], work)
571
+ for tag, lad in sorted(cert["ladders"].items())]
572
+ mine, theirs = canon(sorted(entries, key=lambda e: e["seed_tag"])), canon(sorted(cert["entries"], key=lambda e: e["seed_tag"]))
573
+ claim = claim_of(entries)
574
+ if mine != theirs or canon(claim) != canon(cert["claim"]):
575
+ _chk(out, "credit", "fail", f"recomputed credit {claim['credit_bits']} bits ({claim['status']}) differs from the claimed "
576
+ f"{cert['claim']['credit_bits']} bits ({cert['claim']['status']})")
577
+ else:
578
+ _chk(out, "credit", "pass", f"recomputed {claim['credit_bits']} bits, {claim['status']}, per seed {claim['per_seed']}; identical to the claim")
579
+ except CertificateError as e:
580
+ claim = None
581
+ _chk(out, "credit", "fail", str(e))
582
+ # rerun
583
+ if rerun and private and revealed is None:
584
+ _chk(out, "rerun", "skip", "PRIVATE solver: its source is not in the certificate, so there is nothing to re-run. If the owner publishes it, "
585
+ "pass it with --source FILE")
586
+ elif rerun and execute and spec is not None and cert.get("seeds_revealed") and ladder_ok:
587
+ _rerun(cert, spec, seeds, commits, rerun, work, out, tolerance, sample_seed, revealed)
588
+ elif rerun:
589
+ _chk(out, "rerun", "skip", "needs --execute (it runs the certificate's solver sources on this machine, unsandboxed), the revealed seeds and a loadable domain")
590
+ ok = all(c["status"] != "fail" for c in out)
591
+ return {"ok": ok, "checks": out, "claim": cert.get("claim") if ok else None, "private": private, "source_revealed": revealed is not None,
592
+ "counted": count is not None}
593
+
594
+
595
+ def _write_sources(cert, work, solver_source=None):
596
+ """The two solver files, written where they can be run. A revealed solver is written byte-for-byte: it is the file whose hash the log committed to."""
597
+ paths = {}
598
+ for role in ("baseline", "submission"):
599
+ paths[role] = os.path.join(work, f"{role}.py")
600
+ src = solver_source if role == "submission" and solver_source is not None else cert[role]["source"]
601
+ if isinstance(src, bytes):
602
+ with open(paths[role], "wb") as f:
603
+ f.write(src)
604
+ else:
605
+ with open(paths[role], "w", encoding="utf-8") as f:
606
+ f.write(src)
607
+ return paths
608
+
609
+
610
+ def _check_counted(cert, count):
611
+ """-> [problems]. Every record of a counted ladder must be internally consistent: a solved instance's cost is exactly its step count over the unit
612
+ (floored like the harness), an unsolved one is charged the cap, and every header names the same unit and interpreter as the certificate."""
613
+ bad = []
614
+ sps = count.get("steps_per_second")
615
+ if not (isinstance(sps, (int, float)) and not isinstance(sps, bool) and sps > 0) or not isinstance(count.get("python"), str):
616
+ return ["the certificate's counted-cost record is malformed"]
617
+ import counted
618
+ csha = count.get("counter_sha256")
619
+ if not (isinstance(csha, str) and re.fullmatch(r"[0-9a-f]{64}", csha)):
620
+ return ["the certificate does not name the counting program that produced its counts (counter_sha256)"]
621
+ if csha != counted.counter_sha256():
622
+ return [f"these counts were made by a different counting program (count_runner.py hash {csha[:12]}, this checkout's is {counted.counter_sha256()[:12]}): "
623
+ "check out the version it names; another counter would count the same program differently"]
624
+ if sps != counted.STEPS_PER_SECOND:
625
+ # the unit is part of what a re-run reproduces (the budget in instructions is cap x unit); the counter hash pins count_runner.py, not this constant
626
+ return [f"the certificate's unit is {sps:g} instructions per counted second, this checkout's is {counted.STEPS_PER_SECOND:g}: check out the version it names"]
627
+ ccap = count.get("cap")
628
+ if not (isinstance(ccap, (int, float)) and not isinstance(ccap, bool) and 0 < ccap <= MAX_COUNTED_CAP): # also rejects NaN and Infinity (JSON allows both)
629
+ return [f"the certificate's counted cap is not a number between 0 and {MAX_COUNTED_CAP:g} counted seconds"]
630
+ for tag, lad in sorted(cert["ladders"].items()):
631
+ for role in ("baseline", "submission"):
632
+ lines = [ln for ln in lad[role].splitlines() if ln.strip()]
633
+ meta = json.loads(lines[0])
634
+ if meta.get("budget") != ccap:
635
+ bad.append(f"seed {tag} {role}: the ladder's budget is not the certificate's counted cap")
636
+ continue
637
+ if (meta.get("cost_model") != "count" or meta.get("sandbox") != "count" or meta.get("steps_per_second") != sps or meta.get("python") != count["python"]
638
+ or meta.get("counter_sha256") != csha): # (sandbox "count": a counted ladder cannot borrow judge.py's OFFICIAL by labelling itself "docker")
639
+ bad.append(f"seed {tag} {role}: the ladder header disagrees with the certificate's counted-cost record")
640
+ continue
641
+ cap = meta["budget"]
642
+ limit = int(cap * sps)
643
+ for ln in lines[1:]:
644
+ r = json.loads(ln)
645
+ s = r.get("steps")
646
+ if isinstance(s, bool) or not isinstance(s, int) or s < 0:
647
+ # a wall-clock timeout or a counter failure has no count: it can never be re-executed to the same number, and "charged the cap" without a count would let a
648
+ # prover turn any baseline instance into a maximal cost that no re-run will ever check
649
+ bad.append(f"seed {tag} {role} {r['id']}: no step count, so it cannot be reproduced and is not provable")
650
+ elif r["status"] == "solved":
651
+ if s > limit:
652
+ bad.append(f"seed {tag} {role} {r['id']}: a solved instance cannot have used more than the budget")
653
+ elif abs(r["cpu_net"] - max(s / sps, 1e-3)) > 1e-9 * max(1.0, r["cpu_net"]):
654
+ bad.append(f"seed {tag} {role} {r['id']}: the cost does not follow from the step count")
655
+ elif r["cpu_net"] != cap:
656
+ bad.append(f"seed {tag} {role} {r['id']}: an unsolved instance must be charged the cap")
657
+ elif r["status"] == "timeout":
658
+ if s != limit + 1:
659
+ # the counter stops a solver at exactly limit+1 instructions, so a timeout with any other count (a baseline "timing out" in 5 steps) is not one
660
+ bad.append(f"seed {tag} {role} {r['id']}: a counted timeout must have used exactly the budget plus one instruction")
661
+ elif s > limit:
662
+ bad.append(f"seed {tag} {role} {r['id']}: a {r['status']} instance cannot have used more than the budget")
663
+ return bad
664
+
665
+
666
+ def _rerun_counted(cert, spec, paths, seeds, commits, k, out, sample_seed):
667
+ """Re-EXECUTE k instances of BOTH solvers here under step counting and require IDENTICAL counts and statuses: no tolerance, no trust in anyone's
668
+ machine. That equality is the proof. Instruction counts are only comparable on the same interpreter, so a different Python version is a SKIP."""
669
+ import counted
670
+ derive, check = spec
671
+ count = cert["count"]
672
+ if not counted.supported() or counted.python_version() != count["python"]:
673
+ _chk(out, "rerun", "skip", f"these counts were recorded on {count['python']} and this machine runs {counted.python_version()}: instruction counts are only "
674
+ "comparable on the same interpreter version, so there is nothing exact to compare (use that version)")
675
+ return
676
+ rng = random.Random(sample_seed)
677
+ pool = []
678
+ for tag, lad in cert["ladders"].items():
679
+ recs = {role: {r["id"]: r for r in map(json.loads, (ln for ln in lad[role].splitlines()[1:] if ln.strip()))} for role in ("baseline", "submission")}
680
+ for i, rb in recs["baseline"].items():
681
+ rs = recs["submission"].get(i)
682
+ if rs and rb.get("steps") is not None and rs.get("steps") is not None:
683
+ pool.append((tag, i, rb, rs))
684
+ pool.sort(key=lambda t: (t[0], t[1]))
685
+ picks = rng.sample(pool, min(k, len(pool)))
686
+ if not picks:
687
+ _chk(out, "rerun", "skip", "no instance has a recorded step count for both solvers")
688
+ return
689
+ mismatches = []
690
+ for tag, iid, rb, rs in picks:
691
+ h = next(c for c in commits if c.startswith(tag))
692
+ master = seeds[h].strip().encode()
693
+ dim, idx = rb["dim"], int(iid.rsplit(":", 1)[1])
694
+ inst = derive(master, dim, idx)
695
+ inst["family"] = cert["family"]
696
+ if "args" not in inst: # lattice families: the solver receives (basis, bound2), as families.derive arranges
697
+ inst["args"] = [inst["basis"], inst["bound2"]]
698
+ for role, rec in (("baseline", rb), ("submission", rs)):
699
+ got = counted.count_instance(paths[role], inst, count["cap"])
700
+ if got.get("steps") != rec.get("steps") or got.get("status") != rec.get("status"):
701
+ mismatches.append(f"{iid} (seed {tag}) {role}: recorded {rec.get('status')} in {rec.get('steps')} steps, re-executed {got.get('status')} in {got.get('steps')}")
702
+ _chk(out, "rerun", "fail" if mismatches else "pass",
703
+ ("did not reproduce: " + "; ".join(mismatches[:3]) + (f" (and {len(mismatches) - 3} more)" if len(mismatches) > 3 else "")) if mismatches else
704
+ f"re-executed {len(picks)} of {len(pool)} instance(s) of BOTH solvers here: every step count and status is IDENTICAL to the record (exact equality, not a tolerance)"
705
+ + ("" if len(picks) == len(pool) else f". The other {len(pool) - len(picks)} are covered only statistically (a record falsified in a fraction f of the instances escapes "
706
+ f"{len(picks)} random checks with probability (1-f)^{len(picks)}): re-run them all for certainty"))
707
+ out[-1]["data"] = {"k": len(picks), "of": len(pool), "exact": True, "mismatches": len(mismatches), "sample_seed": sample_seed}
708
+
709
+
710
+ def _rerun(cert, spec, seeds, commits, k, work, out, tolerance=RERUN_TOLERANCE, sample_seed=0, solver_source=None):
711
+ derive, check = spec
712
+ paths = _write_sources(cert, work, solver_source)
713
+ if isinstance(cert.get("count"), dict):
714
+ return _rerun_counted(cert, spec, paths, seeds, commits, k, out, sample_seed)
715
+ rng = random.Random(sample_seed)
716
+ pool = []
717
+ for tag, lad in cert["ladders"].items():
718
+ recs = {role: {r["id"]: r for r in map(json.loads, lad[role].splitlines()[1:]) if r} for role in ("baseline", "submission")}
719
+ meta = json.loads(lad["baseline"].splitlines()[0])
720
+ for i, rb in recs["baseline"].items():
721
+ rs = recs["submission"].get(i)
722
+ if rs and rb["status"] == "solved" and rs["status"] == "solved" and rb["cpu_net"] >= E.FLOOR_ABS:
723
+ pool.append((tag, i, rb, rs, meta["budget"]))
724
+ pool.sort(key=lambda t: (t[0], t[1]))
725
+ picks = rng.sample(pool, min(k, len(pool)))
726
+ if not picks:
727
+ _chk(out, "rerun", "skip", "no instance solved by both solvers at a cost above the 0.5 s floor")
728
+ return
729
+ dev, bad = [], []
730
+ ov = harness.calibrate()
731
+ for tag, iid, rb, rs, budget in picks:
732
+ h = next(c for c in commits if c.startswith(tag))
733
+ master = seeds[h].strip().encode()
734
+ dim, idx = rb["dim"], int(iid.rsplit(":", 1)[1])
735
+ inst = derive(master, dim, idx)
736
+ inst["family"] = cert["family"]
737
+ if "args" not in inst: # lattice families: the solver receives (basis, bound2), as families.derive arranges
738
+ inst["args"] = [inst["basis"], inst["bound2"]]
739
+ a, b = (harness.run_one(paths[role], inst, budget) for role in ("baseline", "submission"))
740
+ if a["status"] != "solved" or b["status"] != "solved":
741
+ bad.append(f"{iid} (seed {tag}): baseline {a['status']}, submission {b['status']} here")
742
+ continue
743
+ ratio_here = (max(a["cpu"] - ov, 1e-3)) / max(b["cpu"] - ov, 1e-3)
744
+ dev.append(abs(math.log2(ratio_here) - math.log2(rb["cpu_net"] / rs["cpu_net"])))
745
+ if bad:
746
+ _chk(out, "rerun", "fail", "did not reproduce: " + "; ".join(bad))
747
+ else:
748
+ med = sorted(dev)[len(dev) // 2]
749
+ _chk(out, "rerun", "pass" if med <= tolerance else "fail",
750
+ f"re-measured {len(dev)} instance(s) here: all solved; median |difference| in speedup {med:.2f} bits (machine and load differ; > {tolerance:g} bits fails)")
751
+ out[-1]["data"] = {"k": len(dev), "median_gap_bits": round(med, 3), "gaps_bits": [round(d, 3) for d in dev], "tolerance_bits": tolerance, "sample_seed": sample_seed}
752
+
753
+
754
+ def main(argv=None):
755
+ ap = argparse.ArgumentParser()
756
+ sub = ap.add_subparsers(dest="cmd", required=True)
757
+ p = sub.add_parser("build")
758
+ p.add_argument("--family", required=True)
759
+ p.add_argument("--submission", required=True)
760
+ p.add_argument("--seed-dir", required=True)
761
+ p.add_argument("--reveal-seeds", action="store_true", help="include the seeds: this IS the reveal. Only after the round is evaluated")
762
+ p.add_argument("--baseline")
763
+ p.add_argument("--results-dir")
764
+ p.add_argument("--round-data", help="the operator's data directory (to include the round's log)")
765
+ p.add_argument("--round-id")
766
+ p.add_argument("--private", action="store_true", help="a PRIVATE-SOLVER certificate: no solver source, no failure text (see the module docstring)")
767
+ p.add_argument("--out", required=True)
768
+ p = sub.add_parser("make-private", help="write a private-solver copy of a certificate (the original is untouched)")
769
+ p.add_argument("cert")
770
+ p.add_argument("--out", required=True)
771
+ p = sub.add_parser("verify")
772
+ p.add_argument("cert")
773
+ p.add_argument("--source", metavar="FILE", help="for a private-solver certificate: the solver file its owner has published; it must hash to the committed value, "
774
+ "and then --rerun can re-measure it")
775
+ p.add_argument("--rerun", type=rerun_count, default=0, metavar="K|all", help="re-measure K instances here, or `all` of them (needs --execute)")
776
+ p.add_argument("--execute", action="store_true", help="RUN code from the certificate on this machine (a declared domain's decl.py, and with --rerun "
777
+ "its solver sources, unsandboxed): needed for instance coverage of a declared domain and for --rerun. Use a throwaway VM or container.")
778
+ p.add_argument("--tolerance", type=float, default=RERUN_TOLERANCE, help="bits of disagreement --rerun accepts (default %(default)g, loose until cross-machine spread is measured)")
779
+ p.add_argument("--sample-seed", type=int, default=None, help="which instances --rerun re-measures (default: a fresh random draw, yours alone, printed in the verdict)")
780
+ a = ap.parse_args(argv)
781
+ if a.cmd == "build":
782
+ if os.path.exists(a.out):
783
+ raise SystemExit(f"REFUSED: {a.out} exists; a certificate is never overwritten")
784
+ try:
785
+ cert = build(a.family, a.submission, a.seed_dir, a.reveal_seeds, a.baseline, a.results_dir, a.round_data, a.round_id, private=a.private)
786
+ except (CertificateError, SystemExit, KeyError, OSError) as e:
787
+ raise SystemExit(f"REFUSED: {e}")
788
+ fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
789
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
790
+ json.dump(cert, f, indent=1, sort_keys=True)
791
+ c = cert["claim"]
792
+ print(f"wrote {a.out}: {c['credit_bits']} bits, {c['status']}" + ("" if cert["seeds_revealed"] else " (seeds NOT included: sealed certificate)")
793
+ + (" (PRIVATE SOLVER: source withheld)" if cert.get("private") else ""))
794
+ elif a.cmd == "make-private":
795
+ if os.path.exists(a.out):
796
+ raise SystemExit(f"REFUSED: {a.out} exists; a certificate is never overwritten")
797
+ with open(a.cert, encoding="utf-8") as f:
798
+ cert = json.load(f)
799
+ try:
800
+ priv = make_private(cert)
801
+ except (CertificateError, KeyError, ValueError, TypeError, AttributeError) as e:
802
+ raise SystemExit(f"REFUSED: {e}")
803
+ fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
804
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
805
+ json.dump(priv, f, indent=1, sort_keys=True)
806
+ print(f"wrote {a.out}: the same certificate without the solver's source (verify it: certificate.py verify {a.out})")
807
+ else:
808
+ with open(a.cert, encoding="utf-8") as f:
809
+ cert = json.load(f)
810
+ source = None
811
+ if a.source:
812
+ with open(a.source, "rb") as f: # BYTES: the committed hash is of the file's exact bytes
813
+ source = f.read()
814
+ if a.sample_seed is None: # the prover must not be able to predict which instances are re-run: the default is the verifier's own random draw
815
+ a.sample_seed = secrets.randbelow(2 ** 31)
816
+ res = verify(cert, rerun=a.rerun, execute=a.execute, tolerance=a.tolerance, sample_seed=a.sample_seed, source=source)
817
+ for c in res["checks"]:
818
+ print(f" {c['status'].upper():4s} {c['check']}: {c['detail']}")
819
+ if res["ok"]:
820
+ print(f"VERIFIED: {res['claim']['credit_bits']} bits ({res['claim']['status']})"
821
+ + (" [PRIVATE SOLVER: not inspectable or re-runnable here]" if res.get("private") and not res.get("source_revealed") else ""))
822
+ else:
823
+ print("NOT VERIFIED")
824
+ sys.exit(0 if res["ok"] else 1)
825
+
826
+
827
+ if __name__ == "__main__":
828
+ main()