guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1312 @@
1
+ """The challenge compiler: a problem declaration in, a verified, calibrated challenge system out (see COMPILER.md).
2
+
3
+ compile_domain.py init NAME scaffold domains/NAME/{decl.py,reference.py} (new files only; never overwrites)
4
+ compile_domain.py compile domains/NAME run every pass; writes compiled.json and out/ (the four emit targets)
5
+ compile_domain.py check domains/NAME parse + validate only (fast; no timing): CI-friendly, exit 1 on any FAIL
6
+ compile_domain.py selftest NAME re-run the end-to-end controls through judge.py (zero effect, failure, known effect)
7
+ compile_domain.py emit NAME [--target T] regenerate out/ from compiled.json
8
+ compile_domain.py status every declaration under domains/: fresh, compiled, self-tested, or why not
9
+
10
+ Passes: 1 parse -> 2 validate -> 3 profile -> 4 optimize -> 5 power -> 6 emit -> 7 self-test
11
+ parse the declaration has the right shape; the reference solver is lint-clean and imports nothing of the judge's
12
+ validate derive() is deterministic and process-pure; the planted witness verifies and is not in the arguments; verify() rejects
13
+ junk, the arguments themselves and another instance's witness; the reference solves small instances correctly
14
+ profile measure the reference over the size range -> a CALIBRATION PROFILE (cost vs size, per-instance spread, timeout rate)
15
+ optimize choose the size ladder, the CPU cap and the instances per size from the profile
16
+ power the real estimator on synthetic ladders drawn from the profile: the smallest speedup this challenge can certify
17
+ emit local kit, server plan, CI script, certificate instructions
18
+ self-test the compiled system, run through the real pipeline: reference-vs-reference must earn ~0, a solver that never answers must
19
+ earn 0, a known 1-bit speedup (the same ladder, costs scaled) must be recovered
20
+
21
+ The profile is public-seed data about the REFERENCE only: it contains no competition instance. Nothing here weakens the commit-reveal
22
+ protocol; a compiled domain runs under exactly the rules of the built-in families.
23
+ """
24
+ import argparse
25
+ import ast
26
+ import contextlib
27
+ import hashlib
28
+ import json
29
+ import math
30
+ import os
31
+ import random
32
+ import secrets
33
+ import shutil
34
+ import statistics
35
+ import subprocess
36
+ import sys
37
+ import tempfile
38
+ import time
39
+
40
+ import declared
41
+ import families
42
+ import harness
43
+ import submit_check
44
+
45
+ HERE = os.path.dirname(os.path.abspath(__file__))
46
+ BUILTIN = {"gm", "mlwe", "factor"}
47
+ PROFILE_SEED = b"svp-compiler-profile-seed-v1" # public on purpose: profiling measures the reference, never a competition instance
48
+ CAPS = (30, 60, 120, 240) # CPU-second caps the optimizer may choose from (the built-in families use 60 and 120)
49
+ NS = (12, 24, 48, 96, 192) # instances per size it may choose from (rounds.MAX_N is 200)
50
+ MAX_SIZES, MIN_SIZES = 6, 3 # ladder length bounds
51
+ LOW_COST = 2.0 # s: smallest predicted median cost of a ladder size (4x the estimator's 0.5 s floor)
52
+ Z_TAIL = 1.28 # the top size's median sits Z_TAIL sigma below the cap: about 10% of its instances time out
53
+ THRESHOLD = 1.0 # bits: "certified" in the power analysis means a credit of at least this
54
+ SELFTEST_EFFECT = 1.0 # bits of the known-effect control
55
+ SEEDS = 3 # judge.MIN_SEEDS
56
+ MAX_JUNK_RATE = 0.0
57
+
58
+ JUNK = [None, True, False, 0, 1, -1, 2, "", "x", "0", [], [0], [1], [None], {}, 1.5, 10 ** 30, -10 ** 30, [[]]]
59
+
60
+
61
+ def _log(*a):
62
+ print(*a, flush=True) # a compile runs for minutes to hours: progress must reach a redirected log as it happens
63
+
64
+
65
+ class CompileError(Exception):
66
+ """A pass cannot continue; the message tells the declaration's author what to change."""
67
+
68
+
69
+ def R(check, status, detail=""):
70
+ return {"check": check, "status": status, "detail": detail}
71
+
72
+
73
+ def worst(results):
74
+ order = {"pass": 0, "warn": 1, "fail": 2}
75
+ return max((r["status"] for r in results), key=order.get, default="pass")
76
+
77
+
78
+ def canon(obj):
79
+ return json.dumps(obj, sort_keys=True, separators=(",", ":"), allow_nan=False)
80
+
81
+
82
+ def _sha(path):
83
+ return declared.sha256_file(path)
84
+
85
+
86
+ # ------------------------------------------------------------------------------------------------------------- 1. parse
87
+ def parse(dirpath):
88
+ """-> (module, reference path, results). module is None when the declaration cannot be loaded."""
89
+ try:
90
+ mod = declared.load_declaration(dirpath)
91
+ except declared.DeclarationError as e:
92
+ return None, None, [R("declaration", "fail", str(e))]
93
+ out = [R("declaration", "pass", f"{mod.NAME} {mod.VERSION}, sizes {mod.SIZE_RANGE[0]}..{mod.SIZE_RANGE[1]} step {getattr(mod, 'SIZE_STEP', 1)}")]
94
+ if mod.NAME in BUILTIN:
95
+ out.append(R("name", "fail", f"{mod.NAME!r} is a built-in family"))
96
+ try:
97
+ ref = declared.reference_path(dirpath, mod)
98
+ except declared.DeclarationError as e:
99
+ return mod, None, out + [R("reference", "fail", str(e))]
100
+ warnings, fatal = submit_check.lint(ref)
101
+ for f in fatal:
102
+ out.append(R("reference lint", "fail", f))
103
+ for w in warnings:
104
+ # the baseline is measured inside the sandbox image: an import the image lacks is not a style point, it is a crash
105
+ out.append(R("reference lint", "fail" if "not in the sandbox image" in w else "warn", w))
106
+ with open(ref, "rb") as f:
107
+ try:
108
+ tree = ast.parse(f.read())
109
+ except (SyntaxError, ValueError):
110
+ tree = None
111
+ if tree is not None:
112
+ judge_side = {"families", "instances", "instances_mlwe", "instances_factor", "harness", "judge", "declared", "decl", "rounds",
113
+ "extrapolate", "compile_domain", mod.NAME}
114
+ for node in ast.walk(tree):
115
+ mods = [a.name for a in node.names] if isinstance(node, ast.Import) else \
116
+ [node.module] if isinstance(node, ast.ImportFrom) and node.module else []
117
+ for m in mods:
118
+ if m.split(".")[0] in judge_side:
119
+ out.append(R("reference isolation", "fail", f"imports `{m}`: the reference must be self-contained (it runs in the sandbox, which has none of the judge's modules)"))
120
+ if not any(r["check"].startswith("reference") for r in out):
121
+ out.append(R("reference", "pass", os.path.basename(ref)))
122
+ note = getattr(mod, "REFERENCE_NOTE", None)
123
+ if isinstance(note, str) and note.strip():
124
+ out.append(R("baseline statement", "pass", "REFERENCE_NOTE says what the reference is and what is known to beat it"))
125
+ else:
126
+ out.append(R("baseline statement", "warn", "no REFERENCE_NOTE: say in one or two sentences what algorithm the reference is and what is known to beat it. "
127
+ "A weak reference makes big speedups cheap and nothing else in the compile can detect that"))
128
+ return mod, ref, out
129
+
130
+
131
+ # ------------------------------------------------------------------------------------------------------------- 2. validate
132
+ def _leaves(x, depth=0):
133
+ if depth > 6:
134
+ return
135
+ if isinstance(x, dict):
136
+ for v in x.values():
137
+ yield from _leaves(v, depth + 1)
138
+ elif isinstance(x, (list, tuple)):
139
+ for v in x:
140
+ yield from _leaves(v, depth + 1)
141
+ else:
142
+ yield x
143
+
144
+
145
+ def _subtrees(x, depth=0):
146
+ """Every node of a JSON-like value (containers and leaves)."""
147
+ yield x
148
+ if depth > 6:
149
+ return
150
+ if isinstance(x, dict):
151
+ for v in x.values():
152
+ yield from _subtrees(v, depth + 1)
153
+ elif isinstance(x, (list, tuple)):
154
+ for v in x:
155
+ yield from _subtrees(v, depth + 1)
156
+
157
+
158
+ def _witness(mod, inst):
159
+ return mod.witness(inst) if hasattr(mod, "witness") else inst.get("planted")
160
+
161
+
162
+ @contextlib.contextmanager
163
+ def provisional(mod):
164
+ """harness.run_one verifies answers through families.FAMILIES, and a domain is not registered until it is compiled. While the compiler
165
+ measures its reference, register the declaration's (derive, verify) for this process only, then restore the registry exactly."""
166
+ had = mod.NAME in families.FAMILIES
167
+ if not had:
168
+ families.FAMILIES[mod.NAME] = declared.make_spec(mod)
169
+ try:
170
+ yield
171
+ finally:
172
+ if not had:
173
+ families.FAMILIES.pop(mod.NAME, None)
174
+
175
+
176
+ def _pure_subprocess(decl_path, picks, hashseed):
177
+ code = ("import sys, json, importlib.util\nsys.path.insert(0, %r)\n"
178
+ "spec = importlib.util.spec_from_file_location('d', %r)\nm = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)\n"
179
+ "print(json.dumps([m.derive(bytes.fromhex(a), b, c) for a, b, c in json.loads(sys.argv[1])], sort_keys=True, separators=(',', ':')))\n"
180
+ % (HERE, decl_path))
181
+ r = subprocess.run([sys.executable, "-c", code, json.dumps(picks)], capture_output=True, text=True, timeout=300, cwd=os.path.dirname(decl_path),
182
+ env={"PATH": os.environ.get("PATH", ""), "PYTHONHASHSEED": str(hashseed)})
183
+ if r.returncode:
184
+ raise CompileError("derive() failed in a fresh interpreter: " + r.stderr[-300:])
185
+ return json.loads(r.stdout)
186
+
187
+
188
+ QUALITY_TRIVIAL = 0.9 # a quality challenge: an answer derived WITHOUT search that verify() accepts must still score at most this (valid-but-poor answers are expected)
189
+
190
+
191
+ def validate(mod, dirpath, ref, measure=None, n_ref=3, run_reference=True, quality=None):
192
+ """The anti-cheat and sanity checks. `measure(size, idx)` runs the reference (default: the real harness) for the reference checks.
193
+ `quality` (quality challenges only: the declaration's quality(inst, ans)) relaxes the two checks that assume a SMALL answer space: verify() may accept a no-search
194
+ answer or another instance's witness as long as it scores at most QUALITY_TRIVIAL, because in an optimisation problem many answers are valid and the score tells them apart."""
195
+
196
+ def counts(x, ans):
197
+ """verify() accepted `ans`: does it count as a hit against the anti-cheat checks? Always, unless a quality score says it is a poor answer."""
198
+ if quality is None:
199
+ return True
200
+ try:
201
+ q = float(quality(x, ans))
202
+ except Exception:
203
+ return True # a quality() that raises on a verified answer is a defect of its own (compile_quality reports it); do not excuse the answer
204
+ return q > QUALITY_TRIVIAL or q != q
205
+ derive, verify = declared.make_spec(mod)
206
+ lo, hi = mod.SIZE_RANGE
207
+ step = getattr(mod, "SIZE_STEP", 1)
208
+ mid = lo + ((hi - lo) // (2 * step)) * step
209
+ master = b"svp-compiler-validate-A"
210
+ master2 = b"svp-compiler-validate-B"
211
+ out = []
212
+
213
+ def guarded(name, fn):
214
+ try:
215
+ fn()
216
+ except CompileError as e:
217
+ out.append(R(name, "fail", str(e)))
218
+ except Exception as e: # the author's code can raise anything
219
+ out.append(R(name, "fail", f"{type(e).__name__}: {e}"))
220
+
221
+ sizes = sorted({lo, mid, hi})
222
+ cache = {}
223
+
224
+ def inst(master_, size, idx):
225
+ k = (master_, size, idx)
226
+ if k not in cache:
227
+ t0 = time.time()
228
+ cache[k] = derive(master_, size, idx)
229
+ cache[k]["_t"] = time.time() - t0
230
+ return cache[k]
231
+
232
+ def shape():
233
+ for s in sizes:
234
+ for i in range(2):
235
+ x = inst(master, s, i)
236
+ if not isinstance(x.get("args"), (list, tuple)):
237
+ raise CompileError(f"derive({s}, {i}) did not return an 'args' list: the solver is called as solve(*args)")
238
+ payload = json.dumps({"args": x["args"]}, allow_nan=False) # raises on sets, bytes, NaN: they cannot cross the process boundary
239
+ if len(payload) > (1 << 20):
240
+ raise CompileError(f"the arguments at size {s} are {len(payload)} bytes; keep an instance under 1 MiB")
241
+ slow = max(x["_t"] for x in cache.values())
242
+ out.append(R("shape", "warn" if slow > 5 else "pass",
243
+ f"args are JSON-serializable lists at sizes {sizes}" + (f"; derive() took {slow:.1f} s" if slow > 5 else "")))
244
+ guarded("shape", shape)
245
+ if any(r["status"] == "fail" for r in out):
246
+ return out
247
+
248
+ def determinism():
249
+ for s in sizes:
250
+ a = {k: v for k, v in derive(master, s, 0).items()}
251
+ b = {k: v for k, v in derive(master, s, 0).items()}
252
+ if canon(a) != canon(b):
253
+ raise CompileError(f"derive({s}, 0) returned different instances on two calls with the same inputs")
254
+ distinct = {canon(inst(master, mid, i)["args"]) for i in range(8)}
255
+ if len(distinct) < 7:
256
+ raise CompileError(f"only {len(distinct)} distinct instances from 8 indices at size {mid}: instances must depend on the index")
257
+ other = sum(canon(inst(master, mid, i)["args"]) != canon(inst(master2, mid, i)["args"]) for i in range(4))
258
+ if other < 3:
259
+ raise CompileError("instances do not depend on the master seed: the commit-reveal protocol would commit to nothing")
260
+ out.append(R("determinism", "pass", f"pure in (master, size, index); {len(distinct)}/8 distinct indices; seed-dependent"))
261
+ guarded("determinism", determinism)
262
+
263
+ def purity():
264
+ picks = [[master.hex(), s, i] for s in sizes for i in range(2)]
265
+ mine = [{k: v for k, v in derive(master, s, i).items()} for _, s, i in picks]
266
+ decl_path = os.path.join(os.path.abspath(dirpath), declared.DECL)
267
+ for seed in (1, 2):
268
+ theirs = _pure_subprocess(decl_path, picks, seed)
269
+ for a, b, p in zip(mine, theirs, picks):
270
+ # the declaration's own derive(), not the wrapper: compare only what the author produced
271
+ if canon({k: a[k] for k in b}) != canon(b):
272
+ raise CompileError(f"derive({p[1]}, {p[2]}) differs between interpreters (PYTHONHASHSEED={seed}): it depends on hash "
273
+ "order, the clock, global random state or the environment, so another machine would derive different instances")
274
+ out.append(R("process purity", "pass", "identical instances in fresh interpreters with two hash seeds"))
275
+ guarded("process purity", purity)
276
+
277
+ def witness():
278
+ bad = 0
279
+ for s in sizes:
280
+ for i in range(4):
281
+ x = inst(master, s, i)
282
+ if "planted" not in x and not hasattr(mod, "witness"):
283
+ raise CompileError("derive() must return the witness under 'planted' (or the declaration must define witness(inst)): "
284
+ "without a known answer the compiler cannot prove the verifier accepts a correct one")
285
+ if not verify(x, _witness(mod, x)):
286
+ bad += 1
287
+ if bad:
288
+ raise CompileError(f"verify() rejected the planted witness on {bad} of {len(sizes) * 4} instances")
289
+ for s in sizes:
290
+ x = inst(master, s, 0)
291
+ w = _witness(mod, x)
292
+ if len(canon(w)) >= 6 and canon(w) in {canon(t) for t in _subtrees(x["args"])}:
293
+ raise CompileError(f"the witness appears verbatim in the arguments at size {s}: the solver is handed the answer")
294
+ big = {v for v in _leaves(w) if (isinstance(v, int) and not isinstance(v, bool) and abs(v) >= 1 << 16) or (isinstance(v, str) and len(v) >= 8)}
295
+ if big & set(_leaves(x["args"])):
296
+ raise CompileError(f"a component of the witness is one of the arguments at size {s}: the solver is handed part of the answer")
297
+ out.append(R("witness", "pass", f"verify() accepts the planted witness on {len(sizes) * 4} instances; it is not in the arguments"))
298
+ guarded("witness", witness)
299
+
300
+ def junk():
301
+ raised = set()
302
+ for s in (lo, mid):
303
+ for i in range(3):
304
+ x = inst(master, s, i)
305
+ for j in JUNK:
306
+ try:
307
+ ok = verify(x, j)
308
+ except Exception as e:
309
+ raised.add(type(e).__name__)
310
+ continue
311
+ if ok:
312
+ raise CompileError(f"verify() ACCEPTED {j!r} at size {s}. An answer space this small is guessable; the answer must be a witness a "
313
+ "solver has to search for")
314
+ out.append(R("junk rejected", "warn" if raised else "pass",
315
+ f"verify() raised {sorted(raised)} on some junk answers (the harness scores that as wrong, but return False instead)" if raised
316
+ else f"verify() rejected all {len(JUNK)} junk answers on 6 instances"))
317
+ guarded("junk rejected", junk)
318
+
319
+ def trivial():
320
+ for s in sizes:
321
+ for i in range(3):
322
+ x = inst(master, s, i)
323
+ cands = [x["args"], list(x["args"])[::-1]] + [v for v in _leaves(x["args"]) if isinstance(v, int) and not isinstance(v, bool)][:64]
324
+ cands += [v + d for v in cands[2:10] if isinstance(v, int) for d in (-1, 1)]
325
+ for c in cands:
326
+ try:
327
+ hit = verify(x, c)
328
+ except Exception:
329
+ continue
330
+ if hit and counts(x, c):
331
+ raise CompileError(f"verify() accepts {str(c)[:60]!r}, which is derived from the arguments without any search (size {s})")
332
+ out.append(R("no trivial answer", "pass", "no argument, its neighbour or the argument list verifies"))
333
+ guarded("no trivial answer", trivial)
334
+
335
+ def cross():
336
+ pairs = hit = 0
337
+ for i in range(6):
338
+ w = _witness(mod, inst(master, mid, i))
339
+ for j in range(6):
340
+ if i != j:
341
+ pairs += 1
342
+ try:
343
+ xj = inst(master, mid, j)
344
+ hit += bool(verify(xj, w)) and counts(xj, w)
345
+ except Exception:
346
+ pass
347
+ rate = hit / pairs
348
+ if rate > 0.25:
349
+ raise CompileError(f"another instance's witness is accepted {100 * rate:.0f}% of the time at size {mid}: a lookup table of past answers would solve it")
350
+ out.append(R("witness reuse", "warn" if hit else "pass", f"{hit} of {pairs} cross-instance witnesses accepted" + (" (small answer space)" if hit else "")))
351
+ guarded("witness reuse", cross)
352
+
353
+ if any(r["status"] == "fail" for r in out):
354
+ return out
355
+ if not run_reference:
356
+ return out + [R("reference solves", "warn", "skipped (--no-reference): the reference was not run")]
357
+
358
+ def reference():
359
+ meas = measure or default_measure(mod, ref, PROFILE_SEED, 10, harness.calibrate() if measure is None else 0.0)
360
+ sub = [lo + k * step for k in range(3) if lo + k * step <= hi]
361
+ notes = []
362
+ for s in sub:
363
+ recs = [meas(s, i) for i in range(n_ref)]
364
+ for r in recs:
365
+ if r["status"] == "wrong":
366
+ raise CompileError(f"the reference returned an answer verify() rejects at size {s}: either the reference or verify() is wrong")
367
+ if r["status"] == "crash":
368
+ raise CompileError(f"the reference crashed at size {s}: {str(r.get('err', ''))[-200:]}")
369
+ if r["status"] == "failed":
370
+ raise CompileError(f"the reference returned no answer at size {s}")
371
+ if r["status"] == "timeout":
372
+ raise CompileError(f"the reference exceeded 10 s at the smallest size {s}: lower SIZE_RANGE[0]")
373
+ notes.append(f"{s}: {statistics.median(r['cpu_net'] for r in recs):.3f}s")
374
+ r1, r2 = meas(sub[0], 0), meas(sub[0], 0)
375
+ if r1["status"] != r2["status"]:
376
+ raise CompileError("the reference is not deterministic: two runs on one instance ended differently")
377
+ out.append(R("reference solves", "pass", f"solved {n_ref} instances at each of sizes {sub}; median cost " + ", ".join(notes)))
378
+ with provisional(mod):
379
+ guarded("reference solves", reference)
380
+ return out
381
+
382
+
383
+ # ------------------------------------------------------------------------------------------------------------- 3. profile
384
+ def default_measure(mod, ref, master, budget, overhead):
385
+ derive, _ = declared.make_spec(mod)
386
+ fam = mod.NAME
387
+
388
+ def measure(size, idx, budget_=None):
389
+ b = budget_ or budget
390
+ x = derive(master, size, idx)
391
+ x["family"] = fam
392
+ rec = harness.run_one(ref, x, b)
393
+ rec["cpu_net"] = max(rec["cpu"] - overhead, 1e-3)
394
+ rec["budget"] = b
395
+ rec["idx"] = idx # which instance this is: a filtered or re-measured list must not be read by position
396
+ return rec
397
+ return measure
398
+
399
+
400
+ def _cost(r, default_cap):
401
+ return r["cpu_net"] if r["status"] == "solved" else r.get("budget", default_cap)
402
+
403
+
404
+ class Model:
405
+ """Cost-vs-size, piecewise linear in log2(median cost). Built from every record measured so far (timeouts count at their cap)."""
406
+
407
+ def __init__(self, pts, cap):
408
+ self.pts, self.cap = pts, cap
409
+ self.xs = sorted(s for s, rs in pts.items() if statistics.median(_cost(r, cap) for r in rs) >= 0.02) # below ~20 ms: overhead, not work
410
+ # the trend over ALL usable sizes: a median of a few heavy-tailed instances can sit below its neighbour's, and a slope read off two
411
+ # such points can be ~0 (it once predicted a flat cost for sizes that actually double per step)
412
+ self.gslope = 0.0
413
+ if len(self.xs) >= 2:
414
+ ys = [self.y(x) for x in self.xs]
415
+ mx, my = statistics.fmean(self.xs), statistics.fmean(ys)
416
+ self.gslope = sum((x - mx) * (y - my) for x, y in zip(self.xs, ys)) / sum((x - mx) ** 2 for x in self.xs)
417
+
418
+ def y(self, s):
419
+ return math.log2(max(statistics.median(_cost(r, self.cap) for r in self.pts[s]), 1e-3))
420
+
421
+ def _local_line(self, s):
422
+ """Least squares of log2(cost) on size over the INDIVIDUAL instances of the (up to) four usable sizes nearest `s`. The median of 8
423
+ heavy-tailed instances is noisy (on dlog, sizes 28/29/30 came out 3.7 s, 2.05 s, 43 s where the cost doubles per size); a line through
424
+ all the instances is far steadier. -> (mean size, mean log2 cost, slope) or None."""
425
+ near = sorted(self.xs, key=lambda x: (abs(x - s), x))[:4]
426
+ px = [x for x in near for _ in self.pts[x]]
427
+ py = [math.log2(max(_cost(r, self.cap), 1e-3)) for x in near for r in self.pts[x]]
428
+ if len(set(px)) < 2:
429
+ return None
430
+ mx, my = statistics.fmean(px), statistics.fmean(py)
431
+ return mx, my, sum((x - mx) * (y - my) for x, y in zip(px, py)) / sum((x - mx) ** 2 for x in px)
432
+
433
+ def predict(self, s):
434
+ if len(self.xs) < 2:
435
+ raise CompileError("fewer than two sizes with a measurable cost: the reference is too cheap across SIZE_RANGE (raise its upper end)")
436
+ line = self._local_line(s)
437
+ if line is not None:
438
+ mx, my, b = line
439
+ return my + max(b, 0.5 * self.gslope, 1e-3) * (s - mx) # never flatter than half the overall trend; cost does not fall with size
440
+ if s in self.pts and s in self.xs:
441
+ return self.y(s)
442
+ if s <= self.xs[0]:
443
+ a, b = self.xs[0], self.xs[1]
444
+ elif s >= self.xs[-1]:
445
+ a, b = self.xs[-2], self.xs[-1]
446
+ else:
447
+ b = next(x for x in self.xs if x > s)
448
+ a = self.xs[self.xs.index(b) - 1]
449
+ # cost never "falls" with size in a challenge worth compiling, and a local slope may not fall below half the overall trend
450
+ slope = max((self.y(b) - self.y(a)) / (b - a), 0.5 * self.gslope, 1e-3)
451
+ return self.y(a) + slope * (s - a)
452
+
453
+
454
+ def pooled_sigma(pts, sizes, cap, model=None):
455
+ """Per-instance spread of the baseline (sd of log2 cost). With a model: the sd of the residuals of every solved instance on the ladder
456
+ around the model's line (>= 8 of them), which pools all sizes. Without: the median over sizes of the per-size sd (>= 4 solved each),
457
+ which on 8 instances per size is badly noisy (0.64, 2.26 and 0.64 for a true 1.44)."""
458
+ if model is not None:
459
+ res = [math.log2(r["cpu_net"]) - model.predict(s) for s in sizes for r in pts.get(s, []) if r["status"] == "solved" and r["cpu_net"] >= 0.05]
460
+ if len(res) >= 8:
461
+ return min(max(math.sqrt(sum(x * x for x in res) / (len(res) - 2)), 0.15), 3.0), True # two fitted parameters
462
+ sds = []
463
+ for s in sizes:
464
+ v = [math.log2(r["cpu_net"]) for r in pts.get(s, []) if r["status"] == "solved" and r["cpu_net"] >= 0.05]
465
+ if len(v) >= 4:
466
+ sds.append(statistics.pstdev(v))
467
+ if not sds:
468
+ return 1.0, False
469
+ return min(max(statistics.median(sds), 0.15), 3.0), True
470
+
471
+
472
+ def probe(mod, measure, k, probe_budget, log=_log):
473
+ """Coarse gallop up the size range until the reference costs more than probe_budget/3 (or a timeout). -> {size: [records]}"""
474
+ lo, hi = mod.SIZE_RANGE
475
+ step = getattr(mod, "SIZE_STEP", 1)
476
+
477
+ def snap(s):
478
+ return max(lo, min(hi, lo + int(round((s - lo) / step)) * step))
479
+ pts, hist = {}, []
480
+ gap0 = max(step, snap(lo + (hi - lo) // 6) - lo)
481
+ s = lo
482
+ while True:
483
+ recs = [measure(s, i) for i in range(k)]
484
+ pts[s] = recs
485
+ for r in recs:
486
+ if r["status"] in ("wrong", "crash", "failed"):
487
+ raise CompileError(f"the reference {r['status']} at size {s}: {str(r.get('err', ''))[-200:] or 'see validate'}")
488
+ med = statistics.median(_cost(r, probe_budget) for r in recs)
489
+ log(f" probe size {s}: median {med:.3f}s" + (f", {sum(r['status'] == 'timeout' for r in recs)} timeout(s)" if any(r['status'] == 'timeout' for r in recs) else ""))
490
+ if any(r["status"] == "timeout" for r in recs) or med > probe_budget / 3 or s >= hi:
491
+ break
492
+ if med >= 0.02:
493
+ hist.append((s, math.log2(med)))
494
+ if len(hist) >= 2 and hist[-1][0] != hist[-2][0]:
495
+ slope = (hist[-1][1] - hist[-2][1]) / (hist[-1][0] - hist[-2][0])
496
+ if slope > 1e-3:
497
+ want = s + (math.log2(probe_budget / 4) - hist[-1][1]) / slope
498
+ delta = min(max(want - s, step), 3 * max(s - hist[-2][0], step))
499
+ else:
500
+ delta = 2 * gap0
501
+ else:
502
+ delta = 2 * gap0 if med < 0.02 else gap0
503
+ nxt = snap(s + delta)
504
+ s = nxt if nxt > s else min(hi, s + step)
505
+ return pts
506
+
507
+
508
+ def choose_ladder(model, sigma, mod, caps=CAPS):
509
+ lo, hi = mod.SIZE_RANGE
510
+ step = getattr(mod, "SIZE_STEP", 1)
511
+ grid = list(range(lo, hi + 1, step))
512
+ best, notes = None, []
513
+ for cap in caps:
514
+ y_hi, y_lo = math.log2(cap) - Z_TAIL * sigma, math.log2(LOW_COST)
515
+ S = [s for s in grid if y_lo <= model.predict(s) <= y_hi]
516
+ if not S:
517
+ continue
518
+ if len(S) > MAX_SIZES:
519
+ targets = [model.predict(S[-1]) - j * (model.predict(S[-1]) - model.predict(S[0])) / (MAX_SIZES - 1) for j in range(MAX_SIZES)]
520
+ S = sorted({min(S, key=lambda s: abs(model.predict(s) - t)) for t in targets})
521
+ span = model.predict(S[-1]) - model.predict(S[0])
522
+ cand = {"cap": cap, "sizes": S, "predicted_median": [round(2 ** model.predict(s), 3) for s in S], "span_octaves": round(span, 2)}
523
+ if best is None or (cand["span_octaves"], len(S)) > (best["span_octaves"], len(best["sizes"])):
524
+ best = cand
525
+ if len(S) >= MIN_SIZES + 1 and span >= 3.0:
526
+ best = cand
527
+ break
528
+ if best is None:
529
+ raise CompileError(f"no size in SIZE_RANGE {mod.SIZE_RANGE} has a predicted median cost between {LOW_COST:g} s and the cap (largest cap tried "
530
+ f"{caps[-1]} s); the reference is too cheap or too expensive over this range: move SIZE_RANGE")
531
+ if best["sizes"][-1] == hi and model.predict(hi) < math.log2(best["cap"]) - Z_TAIL * sigma - 1:
532
+ notes.append(f"the ladder is limited by SIZE_RANGE[1]={hi}: the reference is predicted to cost only {2 ** model.predict(hi):.1f} s there; raise it for a harder challenge")
533
+ if len(best["sizes"]) < MIN_SIZES or best["span_octaves"] < 3.0:
534
+ notes.append(f"narrow ladder: only {len(best['sizes'])} size(s), whose costs span {best['span_octaves']} doublings (about {2 ** best['span_octaves']:.0f}x); "
535
+ "at least 3 doublings are needed for the scoring groups to fill up: widen SIZE_RANGE or let the reference's cost grow faster with size")
536
+ best["notes"] = notes
537
+ return best
538
+
539
+
540
+ def missing_indices(have, n_prof):
541
+ """The instance indices still to measure so that a size has exactly indices 0..n_prof-1. Records carry their own index: counting from len(have)
542
+ re-measured an index already held whenever a lower-cap timeout had been dropped (one instance counted twice, the dropped one never re-measured).
543
+ Records without an index (a caller's own measure) keep the positional rule."""
544
+ have_idx = {r["idx"] for r in have if "idx" in r}
545
+ return [i for i in range(n_prof) if i not in have_idx] if have_idx else list(range(len(have), n_prof))
546
+
547
+
548
+ def profile_and_plan(mod, measure, k=3, n_prof=8, probe_budget=20, caps=CAPS, log=_log, max_refine=2):
549
+ """Passes 3 and 4. Returns (points, plan) where plan carries the ladder, the cap and sigma, validated by measuring the chosen sizes."""
550
+ log("profile: coarse probe")
551
+ pts = probe(mod, lambda s, i: measure(s, i, probe_budget) if _accepts_budget(measure) else measure(s, i), k, probe_budget, log)
552
+ plan, prev_plan, off, first_pass_ok = None, None, [], True
553
+ first = pts[min(pts)]
554
+ if all(r["status"] == "timeout" for r in first):
555
+ raise CompileError(f"the reference exceeds the {probe_budget:g} s probe cap even at the smallest size {min(pts)}: lower SIZE_RANGE[0] "
556
+ "(or raise --probe-budget)")
557
+ for it in range(max_refine + 1):
558
+ cap0 = caps[-1]
559
+ model = Model(pts, cap0)
560
+ sigma, measured_sigma = pooled_sigma(pts, model.xs, cap0, model)
561
+ plan = choose_ladder(model, sigma, mod, caps)
562
+ plan["sigma"], plan["sigma_measured"] = round(sigma, 3), measured_sigma
563
+ if it > 0 and (plan["cap"], plan["sizes"]) == prev_plan: # a re-plan that changes nothing has nothing new to measure: stop (it re-ran an identical pass before)
564
+ log(f"optimize (pass {it + 1}): the same ladder as the previous pass; stopping")
565
+ break
566
+ prev_plan = (plan["cap"], list(plan["sizes"]))
567
+ log(f"optimize (pass {it + 1}): cap {plan['cap']} s, sizes {plan['sizes']}, predicted medians {plan['predicted_median']}, sigma {sigma:.2f}")
568
+ off = []
569
+ for s in plan["sizes"]:
570
+ # keep what is valid at THIS cap (a solved record, or a timeout at a cap at least this large); a timeout at a lower cap is re-measured
571
+ have = pts[s] = [r for r in pts.get(s, []) if r.get("budget", plan["cap"]) >= plan["cap"] or r["status"] == "solved"]
572
+ for i in missing_indices(have, n_prof):
573
+ pts.setdefault(s, []).append(measure(s, i, plan["cap"]) if _accepts_budget(measure) else measure(s, i))
574
+ meas_med = statistics.median(_cost(r, plan["cap"]) for r in pts[s])
575
+ pred = 2 ** model.predict(s)
576
+ log(f" confirm size {s}: measured median {meas_med:.2f}s, predicted {pred:.2f}s")
577
+ miss = abs(math.log2(max(meas_med, 1e-3) / max(pred, 1e-3)))
578
+ if miss > 1.0:
579
+ off.append(s)
580
+ if miss > 2.0: # a wildly wrong model: stop measuring the rest of this ladder (hours at the cap) and re-plan from what was learned
581
+ log(f" prediction off by {miss:.1f} octaves: re-planning without measuring the remaining sizes")
582
+ break
583
+ if it == 0:
584
+ first_pass_ok = not off
585
+ if not off:
586
+ break
587
+ sizes = plan["sizes"]
588
+ final = Model(pts, plan["cap"])
589
+ sigma, measured_sigma = pooled_sigma(pts, sizes, plan["cap"], final)
590
+ plan["sigma"], plan["sigma_measured"] = round(sigma, 3), measured_sigma
591
+ # "confirmed" means the FIRST plan's predictions held when measured. After a re-plan the model contains the very data it would be
592
+ # confirmed by, so agreement proves nothing (the earlier version reported True for exactly that).
593
+ plan["confirmed"] = first_pass_ok
594
+ if not first_pass_ok:
595
+ plan["notes"].append("the first plan's cost predictions were off by more than a factor of 2, so the ladder was re-planned from the confirming data: "
596
+ "it is the best available but NOT independently confirmed")
597
+ if off:
598
+ plan["notes"].append(f"the cost model was still off by more than a factor of 2 at size(s) {off} after {max_refine} refinements")
599
+ # the simulated truth for the power analysis: the model's smoothed line, not 8-instance medians that can sit an octave off their neighbours
600
+ plan["median_log2"] = {s: round(final.predict(s), 4) for s in sizes}
601
+ raw = {s: statistics.median(math.log2(max(_cost(r, plan["cap"]), 1e-3)) for r in pts[s]) for s in sizes}
602
+ worst_gap = max(abs(raw[s] - plan["median_log2"][s]) for s in sizes)
603
+ if worst_gap > 1.0:
604
+ plan["notes"].append(f"the measured typical cost at one size is a factor of {2 ** worst_gap:.1f} away from the smoothed trend (chance variation with only "
605
+ f"{min(len(pts[s]) for s in sizes)} instances per size): the power analysis uses the smoothed trend; raise --n-prof for a steadier profile")
606
+ return pts, plan
607
+
608
+
609
+ def _accepts_budget(fn):
610
+ try:
611
+ import inspect
612
+ return len(inspect.signature(fn).parameters) >= 3
613
+ except (TypeError, ValueError):
614
+ return False
615
+
616
+
617
+ def profile_json(pts, plan, overhead):
618
+ out = {}
619
+ for s, rs in sorted(pts.items()):
620
+ solved = [r["cpu_net"] for r in rs if r["status"] == "solved"]
621
+ out[str(s)] = {"n": len(rs), "solved": len(solved), "timeouts": sum(r["status"] == "timeout" for r in rs),
622
+ "costs": [[r.get("idx", pos), round(_cost(r, plan["cap"]), 4), r["status"] == "solved"] for pos, r in enumerate(rs)], # [instance index, cost, solved]: lets `recheck --measure` compare the SAME instances
623
+ "median_cost": round(statistics.median(_cost(r, plan["cap"]) for r in rs), 4),
624
+ "sd_log2": round(statistics.pstdev([math.log2(c) for c in solved]), 3) if len(solved) >= 2 else None}
625
+ return {"seed": PROFILE_SEED.decode(), "overhead": round(overhead, 4), "sizes": out}
626
+
627
+
628
+ # ------------------------------------------------------------------------------------------------------------- 5. power
629
+ def synth_base(plan, n, rng):
630
+ """One synthetic baseline ladder drawn from the profile: per size, costs 2^(median + sigma z); above the cap = a timeout (censored)."""
631
+ cap, sig = plan["cap"], plan["sigma"]
632
+ base = {}
633
+ for s in plan["sizes"]:
634
+ m = plan["median_log2"][s] if s in plan["median_log2"] else plan["median_log2"][str(s)]
635
+ base[s] = {}
636
+ for i in range(n):
637
+ c = 2 ** (m + sig * rng.gauss(0, 1))
638
+ base[s][i] = {"status": "timeout", "cpu_net": cap} if c > cap else {"status": "solved", "cpu_net": max(c, 1e-3)}
639
+ return base
640
+
641
+
642
+ def power_at(plan, n, effect, triples, boot, pair_noise, rng):
643
+ """P(one seed's credit >= THRESHOLD), P(all three seeds >= THRESHOLD) for a true speedup of `effect` bits, through the REAL estimator."""
644
+ import power
645
+ credits = []
646
+ for _ in range(3 * triples):
647
+ base = synth_base(plan, n, rng)
648
+ pool = {"__all__": [rng.gauss(0, pair_noise) for _ in range(400)]}
649
+ c, _ = power.synth_trial(base, plan["cap"], pool, effect, rng, boot)
650
+ credits.append(c)
651
+ single = sum(c is not None and c >= THRESHOLD for c in credits) / len(credits)
652
+ trip = [min(0.0 if c is None else c for c in credits[i:i + 3]) for i in range(0, len(credits) - 2, 3)]
653
+ return {"single": round(single, 3), "all_three": round(sum(c >= THRESHOLD for c in trip) / len(trip), 3),
654
+ "inconclusive": round(sum(c is None for c in credits) / len(credits), 3)}
655
+
656
+
657
+ def power_plan(plan, target_effect=2.0, target_power=0.8, pair_noise=0.6, triples=30, boot=60, effects=(0, 0.5, 1, 1.5, 2, 3, 4, 5, 6), ns=NS, log=_log):
658
+ """Pass 5. Chooses the smallest n whose all-three-seeds power at `target_effect` reaches `target_power`, then tabulates power by effect."""
659
+ rng = random.Random(0)
660
+ chosen, tried = None, {}
661
+ for n in ns:
662
+ p = power_at(plan, n, target_effect, triples, boot, pair_noise, rng)
663
+ tried[n] = p["all_three"]
664
+ log(f"power: n={n}: P(all 3 seeds >= {THRESHOLD:g} bit | true effect {target_effect:g}) = {p['all_three']:.2f}")
665
+ chosen = n
666
+ if p["all_three"] >= target_power:
667
+ break
668
+ n = chosen
669
+ curve = {}
670
+ for e in effects:
671
+ curve[str(e)] = power_at(plan, n, e, triples, boot, pair_noise, rng)
672
+ mce = next((e for e in effects if curve[str(e)]["all_three"] >= target_power), None)
673
+ top = max((e for e in effects if curve[str(e)]["all_three"] >= target_power), default=None) # the ceiling: a submission cost below the 0.5 s floor is unusable
674
+ notes = []
675
+ if tried[n] < target_power:
676
+ notes.append(f"even n={n} (the largest allowed) certifies a {target_effect:g}-bit speedup only {100 * tried[n]:.0f}% of the time")
677
+ if curve["0"]["single"] > 0.10:
678
+ notes.append(f"FALSE-CREDIT RATE {100 * curve['0']['single']:.0f}% at zero effect (> 10%): the lower bound is not holding on this ladder")
679
+ return {"n": n, "target_effect": target_effect, "target_power": target_power, "pair_noise": pair_noise, "triples": triples, "boot": boot,
680
+ "threshold": THRESHOLD, "tried": {str(k): v for k, v in tried.items()}, "curve": curve, "min_certifiable_effect": mce, "max_certifiable_effect": top, "notes": notes}
681
+
682
+
683
+ def cpu_hours(plan, n, sigma=None):
684
+ """Worst case (every instance charged the cap) CPU-hours of ONE submission's evaluation: 3 seeds x (baseline + submission) ladders."""
685
+ per = len(plan["sizes"]) * n * plan["cap"] * 2 * SEEDS / 3600.0
686
+ exp = 0.0
687
+ for s in plan["sizes"]:
688
+ m = plan["median_log2"][s] if s in plan["median_log2"] else plan["median_log2"][str(s)]
689
+ sig = plan["sigma"] if sigma is None else sigma
690
+ exp += min(2 ** m * math.exp((sig * math.log(2)) ** 2 / 2), plan["cap"]) # mean of a lognormal whose log2 has sd sig, capped (an upper-ish figure)
691
+ return {"worst_case": round(per, 2), "expected": round(exp * n * 2 * SEEDS / 3600.0, 2)}
692
+
693
+
694
+ # ------------------------------------------------------------------------------------------------------------- 6. emit
695
+ def practice_sizes(mod, pts, cap):
696
+ lo, hi = mod.SIZE_RANGE
697
+ ok = [s for s in sorted(pts) if statistics.median(_cost(r, cap) for r in pts[s]) <= 1.0 and all(r["status"] == "solved" for r in pts[s])]
698
+ return ok[-2:] if ok else [lo]
699
+
700
+
701
+ def build_compiled(mod, dirpath, ref, plan, pw, pts, overhead, validation, selftest):
702
+ cost = cpu_hours(plan, pw["n"])
703
+ return {"format": declared.FORMAT, "name": mod.NAME, "version": mod.VERSION, "compiled_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
704
+ "decl_sha256": _sha(os.path.join(dirpath, declared.DECL)), "reference_sha256": _sha(ref),
705
+ "compiler_sha256": _sha(os.path.abspath(__file__)), "reference_note": (getattr(mod, "REFERENCE_NOTE", "") or "").strip(),
706
+ "size_range": list(mod.SIZE_RANGE), "size_step": getattr(mod, "SIZE_STEP", 1),
707
+ "official": {"n": pw["n"], "budget": plan["cap"]}, "dims": plan["sizes"], "practice_dims": practice_sizes(mod, pts, plan["cap"]),
708
+ "plan": {k: plan[k] for k in ("cap", "sizes", "predicted_median", "span_octaves", "sigma", "sigma_measured", "confirmed", "notes", "median_log2")},
709
+ "profile": profile_json(pts, plan, overhead), "power": pw, "cpu_hours": cost,
710
+ "validation": validation, "selftest": selftest}
711
+
712
+
713
+ def write_compiled(dirpath, compiled):
714
+ path = os.path.join(dirpath, declared.COMPILED)
715
+ if os.path.exists(path):
716
+ shutil.copyfile(path, os.path.join(dirpath, "compiled.prev.json")) # the previous compile is kept, never discarded
717
+ tmp = path + ".tmp"
718
+ with open(tmp, "w", encoding="utf-8") as f:
719
+ json.dump(compiled, f, indent=1, sort_keys=True)
720
+ os.replace(tmp, path)
721
+
722
+
723
+ def _write(path, text, mode=None):
724
+ os.makedirs(os.path.dirname(path), exist_ok=True)
725
+ with open(path, "w", encoding="utf-8") as f:
726
+ f.write(text)
727
+ if mode:
728
+ os.chmod(path, mode)
729
+
730
+
731
+ def power_table(compiled):
732
+ rows = ["| true speedup (bits) | P(one seed credited >= %g bit) | P(all three seeds) |" % compiled["power"]["threshold"], "|---|---|---|"]
733
+ for e, v in compiled["power"]["curve"].items():
734
+ rows.append(f"| {e} | {100 * v['single']:.0f}% | {100 * v['all_three']:.0f}% |")
735
+ return "\n".join(rows)
736
+
737
+
738
+ def emit(dirpath, mod, compiled, targets=("local", "server", "ci", "certificate")):
739
+ out = os.path.join(dirpath, "out")
740
+ name, cap, n, dims = mod.NAME, compiled["official"]["budget"], compiled["official"]["n"], compiled["dims"]
741
+ mce = compiled["power"]["min_certifiable_effect"]
742
+ st = (compiled.get("selftest") or {}).get("status", "pending")
743
+ doc = getattr(mod, "DOC", "(the declaration gives no DOC string)")
744
+ written = []
745
+ if "local" in targets:
746
+ p = os.path.join(out, "CHALLENGE.md")
747
+ _write(p, f"""# {name}: challenge
748
+
749
+ {doc}
750
+
751
+ * sizes (ladder): {dims}; CPU cap {cap} s per instance; {n} instances per size; credit = MIN over 3 committed seeds of a one-sided 95% lower bound
752
+ * the reference solver is `{mod.REFERENCE_SOLVER}`; a credit is a speedup over it, in bits (log2 of the cost ratio) on the hardest instances it solved
753
+ * what the reference is, and what is known to beat it (the author's statement; a speedup is only as meaningful as its baseline): {compiled.get('reference_note') or 'NOT STATED by the declaration author'}
754
+ * speedups this ladder can certify (all three seeds, {int(100 * compiled['power']['target_power'])}% power): {('from about %g to about %g bits' % (mce, compiled['power'].get('max_certifiable_effect'))) + ('' if compiled['power'].get('max_certifiable_effect') == max(float(e) for e in compiled['power']['curve']) else ". Above the top of that range a fast solver's own cost falls under the estimator's 0.5 s floor, so a bigger speedup cannot be certified by this ladder (a harder ladder or a model baseline is needed)") if mce is not None else 'NONE on the grid: this challenge is too weak to certify'}
755
+ * compiler self-test: {st}
756
+
757
+ ## Try a solver (public practice sizes, well-formedness only)
758
+
759
+ python3 submit_check.py my_solver.py --family {name}
760
+
761
+ ## What this challenge can certify (simulated through the real estimator; pair noise {compiled['power']['pair_noise']} bit)
762
+
763
+ {power_table(compiled)}
764
+ """)
765
+ written.append(p)
766
+ if "server" in targets:
767
+ cost = compiled["cpu_hours"]
768
+ p = os.path.join(out, "server_plan.json")
769
+ _write(p, json.dumps({"family": name, "n": n, "budget": cap, "dims": dims, "seeds": SEEDS, "cpu_hours_per_submission": cost,
770
+ "selftest": st, "commands": [
771
+ f"python3 ops.py round-new --data DATA --seed-dir SEEDS --id {name}-1 --title \"{name} round 1\" --family {name} --closes <UTC deadline> --n {n}",
772
+ "python3 ops.py close --data DATA --id " + name + "-1", "python3 ops.py worker --data DATA --id " + name + "-1 --seed-dir SEEDS/" + name + "-1",
773
+ "python3 ops.py reveal --data DATA --id " + name + "-1 --seed-dir SEEDS/" + name + "-1"],
774
+ "note": "the domain directory is part of the round's committed rules (its hashes go into log entry 0); deploy domains/" + name + " with the code"},
775
+ indent=1))
776
+ written.append(p)
777
+ if "ci" in targets:
778
+ p = os.path.join(out, "ci_check.sh")
779
+ _write(p, f"""#!/bin/sh
780
+ # Pre-merge check for solver submissions to the {name} challenge. Usage: sh ci_check.sh path/to/solver.py
781
+ # Fast (no timing study): the declaration still validates, the compile is fresh, and the solver is well-formed on the public practice sizes.
782
+ set -e
783
+ SOLVER="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" # resolve BEFORE changing directory: $1 is relative to where the caller ran this
784
+ cd "$(dirname "$0")/../../.."
785
+ python3 compile_domain.py check domains/{name}
786
+ python3 submit_check.py "$SOLVER" --family {name}
787
+ """, 0o755)
788
+ _write(os.path.join(out, "ci_workflow.yml.template"), f"""# TEMPLATE for a CI system that supports GitHub-Actions-style files. Not installed anywhere: copy it where your CI expects it.
789
+ name: {name}-submission-check
790
+ on: [pull_request]
791
+ jobs:
792
+ check:
793
+ runs-on: ubuntu-latest
794
+ steps:
795
+ - uses: actions/checkout@v4
796
+ - uses: actions/setup-python@v5
797
+ with: {{python-version: "3.12"}}
798
+ - run: pip install numpy
799
+ - run: sh domains/{name}/out/ci_check.sh solvers/submission.py # path to the solver under review
800
+ """)
801
+ written.append(p)
802
+ if "certificate" in targets:
803
+ p = os.path.join(out, "CERTIFICATE.md")
804
+ _write(p, f"""# Certificates for {name}
805
+
806
+ After a round is evaluated and its seeds revealed, one command bundles everything a third party needs:
807
+
808
+ python3 certificate.py build --family {name} --submission solvers/x.py --seed-dir SEEDS/{name}-1 --reveal-seeds --out x.{name}.cert.json
809
+
810
+ Anyone can check it, with no access to the operator's machine:
811
+
812
+ python3 certificate.py verify x.{name}.cert.json
813
+
814
+ `verify` re-derives every instance from the revealed seeds with this declaration, checks the hash chain and commitments, checks that the
815
+ rules (estimator, harness, declaration, compiled ladder) are the ones the certificate names, and recomputes the credit from the raw ladders.
816
+ Add `--rerun 5` to re-measure five random instances on your own machine.
817
+
818
+ Rules fingerprint of this domain: decl {compiled['decl_sha256'][:16]}..., reference {compiled['reference_sha256'][:16]}...
819
+ """)
820
+ written.append(p)
821
+ return written
822
+
823
+
824
+ # ------------------------------------------------------------------------------------------------------------- 7. self-test
825
+ def evaluate_controls(zero, none, positive, effect=SELFTEST_EFFECT):
826
+ """Pure decision rule over the three control credits (None = inconclusive). -> (status, [results])"""
827
+ res = []
828
+ if zero is None:
829
+ res.append(R("zero-effect control", "fail", "inconclusive: the ladder yields fewer than 2 usable difficulty bins, so the challenge could not certify anything"))
830
+ elif zero >= THRESHOLD:
831
+ res.append(R("zero-effect control", "fail", f"the reference measured against ITSELF earned {zero:.2f} bits (>= {THRESHOLD:g}): the estimator credits nothing as something"))
832
+ else:
833
+ res.append(R("zero-effect control", "pass", f"reference vs reference earned {zero:.2f} bits"))
834
+ if none is None or none == 0:
835
+ res.append(R("failure control", "pass", "a solver that never answers earned 0" if none == 0 else "a solver that never answers was inconclusive (earns nothing)"))
836
+ else:
837
+ res.append(R("failure control", "fail", f"a solver that never answers earned {none:.2f} bits"))
838
+ if positive is None:
839
+ res.append(R("known-effect control", "fail", "inconclusive on a known speedup: too few usable difficulty bins"))
840
+ elif 0.4 <= positive <= effect + 0.3:
841
+ res.append(R("known-effect control", "pass", f"a known {effect:g}-bit speedup earned {positive:.2f} bits"))
842
+ else:
843
+ res.append(R("known-effect control", "fail", f"a known {effect:g}-bit speedup earned {positive:.2f} bits (expected between 0.4 and {effect + 0.3:g})"))
844
+ return ("pass" if all(r["status"] == "pass" for r in res) else "fail"), res
845
+
846
+
847
+ def _claim_bits(sub_path):
848
+ with open(sub_path + ".claim.json", encoding="utf-8") as f:
849
+ return json.load(f)["measured"]
850
+
851
+
852
+ def selftest(dirpath, mod, n, runner=subprocess.run, effect=SELFTEST_EFFECT, timeout=24 * 3600, log=_log):
853
+ """Run the COMPILED domain through the real pipeline (judge.py evaluate -> harness -> extrapolate), local mode, throwaway seed."""
854
+ name = mod.NAME
855
+ ref = declared.reference_path(dirpath, mod)
856
+ work = tempfile.mkdtemp(prefix="svp-selftest-") # outside the project: the seed must not sit in the tree
857
+ seed = os.path.join(work, "selftest.seed")
858
+ fd = os.open(seed, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
859
+ os.write(fd, secrets.token_hex(32).encode())
860
+ os.close(fd)
861
+ with open(seed, "rb") as f:
862
+ tag = hashlib.sha256(f.read().strip()).hexdigest()[:8]
863
+ shutil.copyfile(ref, os.path.join(work, "control_zero.py"))
864
+ _write(os.path.join(work, "control_none.py"), "def solve(*args):\n return None\n")
865
+ env = dict(os.environ, SVP_BOARD=os.path.join(work, "board.json"), SVP_DOMAINS_DIR=os.path.dirname(os.path.abspath(dirpath)))
866
+ results = {}
867
+ for ctrl in ("control_zero", "control_none"):
868
+ cmd = [sys.executable, "judge.py", "evaluate", "--seed", seed, "--submission", os.path.join(work, ctrl + ".py"), "--family", name,
869
+ "--n", str(n), "--insecure-local", "--reuse-baseline"]
870
+ log(f"selftest: {ctrl} ({n} instances per size)")
871
+ r = runner(cmd, cwd=HERE, env=env, capture_output=True, text=True, timeout=timeout)
872
+ if r.returncode:
873
+ return {"status": "fail", "n": n, "controls": [R(ctrl, "fail", (r.stdout + r.stderr)[-400:])], "work": work}
874
+ results[ctrl] = os.path.join(HERE, "results", f"judged_{name}_{tag}_submission_{ctrl}.py.jsonl")
875
+ base = os.path.join(HERE, "results", f"judged_{name}_{tag}_baseline_{os.path.basename(ref)}.jsonl")
876
+ zero, none = _claim_bits(results["control_zero"]), _claim_bits(results["control_none"])
877
+ scaled = os.path.join(work, "scaled_effect.jsonl")
878
+ with open(results["control_zero"], encoding="utf-8") as f, open(scaled, "w", encoding="utf-8") as g:
879
+ for line in f:
880
+ rec = json.loads(line)
881
+ if not rec.get("meta") and rec["status"] == "solved":
882
+ rec["cpu_net"] = max(rec["cpu_net"] * 2 ** -effect, 1e-3)
883
+ g.write(json.dumps(rec) + "\n")
884
+ r = runner([sys.executable, "extrapolate.py", "--baseline", base, "--submission", scaled], cwd=HERE, env=env, capture_output=True, text=True, timeout=timeout)
885
+ positive = _claim_bits(scaled) if r.returncode == 0 else None
886
+ status, res = evaluate_controls(zero, none, positive, effect)
887
+ return {"status": status, "n": n, "effect": effect, "controls": res, "credits": {"zero": zero, "none": none, "known_effect": positive},
888
+ "ladders": {k: os.path.relpath(v, HERE) for k, v in results.items()} | {"baseline": os.path.relpath(base, HERE)},
889
+ "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
890
+
891
+
892
+ # ------------------------------------------------------------------------------------------------------------- driver
893
+ def _print(results):
894
+ for r in results:
895
+ print(f" {r['status'].upper():4s} {r['check']}: {r['detail']}")
896
+
897
+
898
+ def compile_dir(dirpath, *, targets=("local", "server", "ci", "certificate"), skip_selftest=False, measure=None, k=3, n_prof=8, probe_budget=20,
899
+ caps=CAPS, target_effect=2.0, target_power=0.8, pair_noise=0.6, triples=30, boot=60, selftest_n=None, runner=subprocess.run, log=_log):
900
+ dirpath = os.path.abspath(dirpath)
901
+ mod, ref, res = parse(dirpath)
902
+ log("parse:")
903
+ _print(res)
904
+ if worst(res) == "fail":
905
+ raise CompileError("the declaration did not parse")
906
+ overhead = 0.0 if measure is not None else harness.calibrate()
907
+ meas = measure or default_measure(mod, ref, PROFILE_SEED, probe_budget, overhead)
908
+ log("validate:")
909
+ val = validate(mod, dirpath, ref, measure=(lambda s, i: meas(s, i)) if measure is not None else None)
910
+ _print(val)
911
+ if worst(val) == "fail":
912
+ raise CompileError("validation failed; nothing was compiled. Fix the FAIL items above.")
913
+ with provisional(mod):
914
+ pts, plan = profile_and_plan(mod, meas, k=k, n_prof=n_prof, probe_budget=probe_budget, caps=caps, log=log)
915
+ pw = power_plan(plan, target_effect, target_power, pair_noise, triples, boot, log=log)
916
+ compiled = build_compiled(mod, dirpath, ref, plan, pw, pts, overhead, res + val, {"status": "pending" if not skip_selftest else "skipped"})
917
+ write_compiled(dirpath, compiled)
918
+ emit(dirpath, mod, compiled, targets)
919
+ log(f"compiled: cap {plan['cap']} s, sizes {plan['sizes']}, n={pw['n']}, min certifiable effect {pw['min_certifiable_effect']} bits, "
920
+ f"worst-case {compiled['cpu_hours']['worst_case']} CPU-h per submission")
921
+ for note in plan["notes"] + pw["notes"]:
922
+ log(" NOTE " + note)
923
+ if not skip_selftest:
924
+ # default: the chosen official n. A smaller n has less power than the compile just certified, so a known-effect control that fails at it
925
+ # would say nothing about the compiled system (the power analysis already said n=12 certified a 2-bit effect only 23% of the time on dlog)
926
+ st = selftest(dirpath, mod, pw["n"] if selftest_n is None else min(selftest_n, pw["n"]), runner=runner, log=log)
927
+ compiled["selftest"] = st
928
+ write_compiled(dirpath, compiled)
929
+ emit(dirpath, mod, compiled, targets)
930
+ log("selftest:")
931
+ _print(st["controls"])
932
+ return compiled
933
+
934
+
935
+ TEMPLATE_DECL = '''"""Declaration of the @@NAME@@ domain. This is a WORKING starting point (find x with g^x = h mod p); it already passes `compile_domain.py check`.
936
+ Change the parts marked EDIT to your own problem, keep the shape. Full reference: COMPILER.md. A larger worked example: domains/dlog."""
937
+ import hashlib
938
+ import hmac
939
+ import random
940
+
941
+ import instances # EDIT (optional): the project's Miller-Rabin primality test; remove if you do not need primes
942
+
943
+ NAME = "@@NAME@@" # fixed: must equal this directory's name
944
+ VERSION = "@@NAME@@-v1" # EDIT: bump whenever derive() changes (old results then refuse to mix with new ones)
945
+ SIZE_RANGE = (16, 40) # EDIT: the sizes the compiler may use. It finds the ladder inside this range, so be generous
946
+ SIZE_STEP = 1 # EDIT: 2 if only every other size is meaningful (e.g. even bit lengths)
947
+ REFERENCE_SOLVER = "reference.py"
948
+ DOC = "solve(p, g, h): return an integer x with pow(g, x, p) == h, 0 <= x < p - 1. Return None for no answer." # EDIT: shown to solver authors
949
+ REFERENCE_NOTE = "A plain linear scan over exponents. Baby-step/giant-step is known to beat it by about size/2 bits." # EDIT: what the reference is and what is known to beat it
950
+
951
+
952
+ def derive(master: bytes, size: int, idx: int) -> dict:
953
+ """EDIT: build instance number `idx` of size `size` from the secret `master`, and NOTHING else (no clock, no global random, no files).
954
+ Return {"args": [...JSON values the solver receives...], "planted": <a valid answer; it never reaches the solver>}."""
955
+ rng = random.Random(hmac.new(master, f"@@NAME@@-v1:{size}:{idx}".encode(), hashlib.sha256).digest())
956
+ while True:
957
+ p = rng.getrandbits(size + 2) | (1 << (size + 1)) | 1
958
+ if instances._is_prime(p):
959
+ break
960
+ g = rng.randrange(2, p - 1)
961
+ x = rng.randrange(1, min(1 << size, p - 1))
962
+ return {"args": [p, g, pow(g, x, p)], "planted": x}
963
+
964
+
965
+ def verify(inst: dict, ans) -> bool:
966
+ """EDIT: an exact check of an answer. It must reject None, 0, 1, strings and lists of the wrong kind: an answer a solver can guess is a bug."""
967
+ p, g, h = inst["args"]
968
+ return isinstance(ans, int) and not isinstance(ans, bool) and 0 <= ans < p - 1 and pow(g, ans, p) == h
969
+ '''
970
+
971
+ TEMPLATE_REF = '''"""EDIT: the baseline every submission is measured against: the plain, honest algorithm. Self-contained (it runs in a sandbox that has
972
+ only the standard library, numpy and fpylll). Called as solve(*args); return an int, a list of ints, or None."""
973
+
974
+
975
+ def solve(p, g, h):
976
+ cur = 1
977
+ for x in range(p - 1):
978
+ if cur == h:
979
+ return x
980
+ cur = cur * g % p
981
+ return None
982
+ '''
983
+
984
+
985
+ def cmd_init(a):
986
+ """Scaffold a WORKING declaration (so there is always something that passes `check` to edit from), and validate it at once."""
987
+ if a.blank:
988
+ return cmd_init_blank(a)
989
+ d = os.path.join(declared.domains_dir(), a.name)
990
+ if not declared.NAME_RE.match(a.name):
991
+ raise SystemExit("REFUSED: NAME must be 2-16 lowercase letters a-z (no digits, hyphens or capitals)")
992
+ if a.name in BUILTIN or os.path.exists(d):
993
+ raise SystemExit(f"REFUSED: {d} exists or {a.name!r} is a built-in family; nothing was written")
994
+ _write(os.path.join(d, "decl.py"), TEMPLATE_DECL.replace("@@NAME@@", a.name))
995
+ _write(os.path.join(d, "reference.py"), TEMPLATE_REF)
996
+ mod, ref, res = parse(d)
997
+ res += validate(mod, d, ref, run_reference=False) if mod else []
998
+ print(f"created {d}/decl.py and reference.py: a working discrete-log example. It already validates:" if worst(res) != "fail"
999
+ else f"created {d}, but the template did not validate (this is a bug in the scaffold):")
1000
+ _print(res)
1001
+ print(f"""
1002
+ next:
1003
+ 1. edit the lines marked EDIT in decl.py (derive, verify, DOC, SIZE_RANGE) and reference.py (the baseline solver)
1004
+ 2. python3 compile_domain.py check {a.name} # seconds; tells you exactly what to fix
1005
+ 3. python3 compile_domain.py compile {a.name} --quick # a 1-2 minute draft to see the ladder (practice only)
1006
+ 4. python3 compile_domain.py compile {a.name} # the real thing: tens of minutes to hours, official-eligible
1007
+ 5. python3 compile_domain.py show {a.name} # plain-English summary and the exact next commands""")
1008
+
1009
+
1010
+ def cmd_init_blank(a):
1011
+ d = os.path.join(declared.domains_dir(), a.name)
1012
+ if not declared.NAME_RE.match(a.name):
1013
+ raise SystemExit("REFUSED: NAME must be 2-16 lowercase letters a-z")
1014
+ if a.name in BUILTIN or os.path.exists(d):
1015
+ raise SystemExit(f"REFUSED: {d} exists or {a.name!r} is a built-in family; nothing was written")
1016
+ _write(os.path.join(d, "decl.py"), f'''"""Declaration of the {a.name} domain. Fill in derive/verify; see COMPILER.md and domains/dlog for a worked example."""
1017
+ import hashlib
1018
+ import hmac
1019
+ import random
1020
+
1021
+ NAME = "{a.name}" # must equal this directory's name: 2-16 lowercase letters
1022
+ VERSION = "{a.name}-v1" # bump whenever derive() changes
1023
+ SIZE_RANGE = (16, 40) # sizes the compiler may use; it finds the ladder inside this range
1024
+ SIZE_STEP = 1
1025
+ REFERENCE_SOLVER = "reference.py"
1026
+ DOC = "solve(...): describe what the solver receives and must return (an integer, a list of integers, or None)."
1027
+ REFERENCE_NOTE = "What the reference algorithm is, and what is known to beat it." # a weak reference makes big speedups cheap
1028
+
1029
+
1030
+ def derive(master: bytes, size: int, idx: int) -> dict:
1031
+ rng = random.Random(hmac.new(master, f"{a.name}-v1:{{size}}:{{idx}}".encode(), hashlib.sha256).digest()) # everything from (master, size, idx)
1032
+ raise NotImplementedError("return {{'args': [...JSON values the solver receives...], 'planted': <a valid answer>}}")
1033
+
1034
+
1035
+ def verify(inst: dict, ans) -> bool:
1036
+ raise NotImplementedError("exact check of an answer; must reject None, 0, 1, strings, lists of the wrong kind")
1037
+ ''')
1038
+ _write(os.path.join(d, "reference.py"), '"""The baseline every submission is measured against. Self-contained: no imports of the judge\'s modules."""\n\n\ndef solve(*args):\n raise NotImplementedError\n')
1039
+ print(f"wrote {d}/decl.py and reference.py; next: edit them, then python3 compile_domain.py compile {d}")
1040
+
1041
+
1042
+ def verdict(compiled):
1043
+ """(level, lines): the plain-English reading of a compiled record, with the exact commands to run next."""
1044
+ name, pw = compiled["name"], compiled["power"]
1045
+ st = (compiled.get("selftest") or {}).get("status", "pending")
1046
+ notes = list(compiled["plan"]["notes"]) + list(pw["notes"])
1047
+ mce, top = pw["min_certifiable_effect"], pw.get("max_certifiable_effect")
1048
+ cap, n, dims, cost = compiled["official"]["budget"], compiled["official"]["n"], compiled["dims"], compiled["cpu_hours"]
1049
+ if st == "fail" or mce is None:
1050
+ level = "NOT READY"
1051
+ elif st in ("skipped", "pending"):
1052
+ level = "DRAFT (practice only: no official round until it passes the self-test)"
1053
+ elif notes:
1054
+ level = "READY, WITH NOTES (read them: they say how far to trust the ladder)"
1055
+ else:
1056
+ level = "READY"
1057
+ lines = [f"{name}: {level}",
1058
+ f" the challenge : sizes {dims}, CPU cap {cap} s, {n} instances per size; one submission costs about {cost['expected']} CPU-hours (worst case {cost['worst_case']})",
1059
+ " it can certify: " + (f"speedups from about {mce:g} to about {top:g} bits over your reference" if mce is not None else
1060
+ "NOTHING on the grid tried: the ladder is too weak (see the notes)"),
1061
+ f" self-test : {st}"]
1062
+ lines += [f" note : {x}" for x in notes]
1063
+ if level == "NOT READY":
1064
+ why = ("a control failed: python3 compile_domain.py selftest " + name + " (prints which one and why)" if st == "fail" else
1065
+ "widen SIZE_RANGE, or let the reference's cost grow faster with size, then compile again")
1066
+ nxt = [why]
1067
+ else:
1068
+ nxt = [f"python3 submit_check.py my_solver.py --family {name} # try a solver on public practice sizes"]
1069
+ if st in ("skipped", "pending"):
1070
+ nxt.append(f"python3 compile_domain.py compile {name} # the full compile with the self-test (official-eligible)")
1071
+ else:
1072
+ nxt.append(f"python3 ops.py round-new --data DATA --seed-dir SEEDS --id {name}-1 --title \"{name} round 1\" --family {name} --closes <UTC> --n {n}")
1073
+ nxt.append("python3 certificate.py build ... / verify ... # after the round: see domains/" + name + "/out/CERTIFICATE.md")
1074
+ lines += [" next:"] + [f" {x}" for x in nxt]
1075
+ return level, lines
1076
+
1077
+
1078
+ def cmd_show(a):
1079
+ d = _resolve(a.name)
1080
+ mod = _load(d, a.name)
1081
+ compiled = declared.read_compiled(d)
1082
+ problems = declared.freshness_problems(d, mod, compiled)
1083
+ if problems:
1084
+ raise SystemExit("not compiled, or stale: " + "; ".join(problems) + f"\n run: python3 compile_domain.py compile {a.name}")
1085
+ if compiled.get("kind") == "quality":
1086
+ import compile_quality
1087
+ print("\n".join(compile_quality.verdict(compiled)[1]))
1088
+ return
1089
+ print("\n".join(verdict(compiled)[1]))
1090
+ print(f"\n the full text: {os.path.join(d, 'out', 'CHALLENGE.md')}")
1091
+
1092
+
1093
+ QUICK = dict(k=3, n_prof=4, probe_budget=5.0, caps=[30], trials=8, boot=30, skip_selftest=True)
1094
+
1095
+
1096
+ def cmd_compile(a):
1097
+ if a.quick: # a draft in minutes, for iterating on a declaration: small samples, one cap, no self-test (so never official-eligible)
1098
+ for k, v in QUICK.items():
1099
+ setattr(a, k, v)
1100
+ print("--quick: a DRAFT with small samples and no self-test; good for iterating, not for a real round. Run without --quick when the declaration is final.")
1101
+ else:
1102
+ print("this runs your reference solver for real (tens of minutes to hours: the confirming step alone is n-prof instances at up to the cap, per size).\n"
1103
+ " to iterate faster use --quick (1-2 minutes). Progress prints as it goes.")
1104
+ a.dir = _resolve(a.dir)
1105
+ qmod = _load(a.dir, a.dir)
1106
+ if callable(getattr(qmod, "quality", None)): # a quality challenge has its own compile (compile_quality.py): the speed passes below do not apply
1107
+ import compile_quality
1108
+ try:
1109
+ c = compile_quality.compile_dir(a.dir, quick=a.quick, target_power=a.power, skip_selftest=a.skip_selftest)
1110
+ except CompileError as e:
1111
+ raise SystemExit(f"REFUSED: {e}\n (nothing was written; `python3 compile_domain.py check {a.dir}` re-runs just the validation in seconds)")
1112
+ print("\n" + "\n".join(compile_quality.verdict(c)[1]))
1113
+ if c["selftest"].get("status") == "fail":
1114
+ raise SystemExit("SELF-TEST FAILED: see the controls above")
1115
+ return
1116
+ try:
1117
+ c = compile_dir(a.dir, targets=("local", "server", "ci", "certificate") if a.target == "all" else (a.target,), skip_selftest=a.skip_selftest,
1118
+ k=a.k, n_prof=a.n_prof, probe_budget=a.probe_budget, caps=tuple(a.caps), target_effect=a.target_effect, target_power=a.power,
1119
+ pair_noise=a.pair_noise, triples=a.trials, boot=a.boot, selftest_n=a.selftest_n)
1120
+ except CompileError as e:
1121
+ raise SystemExit(f"REFUSED: {e}\n (nothing was written; `python3 compile_domain.py check {a.dir}` re-runs just the validation in seconds)")
1122
+ print("\n" + "\n".join(verdict(c)[1]))
1123
+ if c["selftest"].get("status") == "fail": # the record is written either way (it says why); the exit code must not claim success
1124
+ raise SystemExit("SELF-TEST FAILED: the domain is compiled but no official round can be created for it (see the controls above)")
1125
+
1126
+
1127
+ def _resolve(arg):
1128
+ """A NAME means domains/NAME (even if a same-named directory sits in the current one); anything with a path separator is a path."""
1129
+ if os.sep not in arg and not arg.startswith("."):
1130
+ named = os.path.join(declared.domains_dir(), arg)
1131
+ if os.path.isdir(named) or not os.path.isdir(arg):
1132
+ return named
1133
+ return arg
1134
+
1135
+
1136
+ def _load(d, name):
1137
+ """declared.load_declaration, but a missing or broken declaration exits with a sentence instead of a traceback."""
1138
+ try:
1139
+ return declared.load_declaration(d)
1140
+ except declared.DeclarationError as e:
1141
+ raise SystemExit(f"REFUSED: {e}\n (a declaration lives in domains/NAME/decl.py; `python3 compile_domain.py init {name}` creates one, "
1142
+ "`python3 compile_domain.py status` lists them)")
1143
+
1144
+
1145
+ def cmd_check(a):
1146
+ d = _resolve(a.dir)
1147
+ mod, ref, res = parse(d)
1148
+ if worst(res) != "fail":
1149
+ isq = callable(getattr(mod, "quality", None))
1150
+ res += validate(mod, d, ref, run_reference=not a.no_reference, quality=mod.quality if isq else None)
1151
+ if isq:
1152
+ import compile_quality
1153
+ res += compile_quality.quality_validation(mod, d)
1154
+ problems = declared.freshness_problems(d, mod, declared.read_compiled(d))
1155
+ res.append(R("compiled record", "fail" if problems else "pass", "; ".join(problems) if problems else "compiled.json matches decl.py and the reference"))
1156
+ _print(res)
1157
+ failed = [r for r in res if r["status"] == "fail"]
1158
+ nm = os.path.basename(os.path.abspath(d))
1159
+ if not failed:
1160
+ print(f"\nOK: the declaration is valid and its compiled record is current. python3 compile_domain.py show {nm}")
1161
+ elif all(r["check"] == "compiled record" for r in failed):
1162
+ print(f"\nThe declaration is VALID; it is just not compiled (or was edited since). python3 compile_domain.py compile {nm} --quick (draft, minutes)"
1163
+ f" or python3 compile_domain.py compile {nm} (the real one)\n(exit status 1 on purpose: CI should not accept an uncompiled domain)")
1164
+ else:
1165
+ print(f"\n{len(failed)} problem(s) above: each FAIL line says what is wrong; fix decl.py or reference.py and run this again.")
1166
+ sys.exit(1 if failed else 0)
1167
+
1168
+
1169
+ def cmd_recheck(a):
1170
+ import recheck
1171
+ d = _resolve(a.name)
1172
+ _load(d, a.name)
1173
+ if (declared.read_compiled(d) or {}).get("kind") == "quality":
1174
+ raise SystemExit("REFUSED: recheck re-derives a SPEED ladder's claims; for a quality challenge use `compile_domain.py selftest NAME` and the bundle's own checks")
1175
+ if a.save and not a.measure:
1176
+ raise SystemExit("REFUSED: --save stores the --measure results; add --measure K")
1177
+ if a.save and os.path.exists(a.save):
1178
+ raise SystemExit(f"REFUSED: {a.save} exists; measurement files are never overwritten")
1179
+ res = recheck.recheck(d, measure_k=a.measure)
1180
+ _print(res)
1181
+ if a.save:
1182
+ recheck.save_measurements(res, declared.read_compiled(d), a.save)
1183
+ print(f"\nsaved {a.save}: send it to whoever merges them with `compile_domain.py spread FILE FILE ...` (it contains a hash of this host's name, not the name)")
1184
+ failed = [r for r in res if r["status"] == "fail"]
1185
+ print("\nrecheck: every recomputable claim reproduces." if not failed else f"\nrecheck: {len(failed)} claim(s) did NOT reproduce (FAIL lines above).")
1186
+ sys.exit(1 if failed else 0)
1187
+
1188
+
1189
+ def cmd_spread(a):
1190
+ import recheck
1191
+ res, rec = recheck.spread(a.files)
1192
+ _print(res)
1193
+ sys.exit(1 if any(r["status"] == "fail" for r in res) else 0)
1194
+
1195
+
1196
+ def cmd_selftest(a):
1197
+ d = _resolve(a.name)
1198
+ mod = _load(d, a.name)
1199
+ compiled = declared.read_compiled(d)
1200
+ problems = declared.freshness_problems(d, mod, compiled)
1201
+ if problems:
1202
+ raise SystemExit("REFUSED: " + "; ".join(problems))
1203
+ if compiled.get("kind") == "quality":
1204
+ import compile_quality
1205
+ qc = compiled["quality"]
1206
+ st = compile_quality.selftest(qc["simulation"], compiled["official"]["n"] if a.n is None else min(a.n, compiled["official"]["n"]), qc["min_gain"], qc["power"]["noise"])
1207
+ compiled["selftest"] = st
1208
+ write_compiled(d, compiled)
1209
+ _print(st["controls"])
1210
+ sys.exit(0 if st["status"] == "pass" else 1)
1211
+ st = selftest(d, mod, compiled["official"]["n"] if a.n is None else min(a.n, compiled["official"]["n"]))
1212
+ compiled["selftest"] = st
1213
+ write_compiled(d, compiled)
1214
+ emit(d, mod, compiled)
1215
+ _print(st["controls"])
1216
+ sys.exit(0 if st["status"] == "pass" else 1)
1217
+
1218
+
1219
+ def cmd_emit(a):
1220
+ d = _resolve(a.name)
1221
+ mod = _load(d, a.name)
1222
+ compiled = declared.read_compiled(d)
1223
+ problems = declared.freshness_problems(d, mod, compiled)
1224
+ if problems:
1225
+ raise SystemExit("REFUSED: " + "; ".join(problems))
1226
+ if compiled.get("kind") == "quality":
1227
+ raise SystemExit("REFUSED: a quality challenge has no speed-round kit to emit; `compile_domain.py export NAME` writes its bundle")
1228
+ for p in emit(d, mod, compiled, ("local", "server", "ci", "certificate") if a.target == "all" else (a.target,)):
1229
+ print("wrote", p)
1230
+
1231
+
1232
+ def cmd_export(a):
1233
+ import export_bundle
1234
+ argv = ["export", a.name, "--field", a.field] + (["--out", a.out] if a.out else []) + (["--no-selftest"] if a.no_selftest else [])
1235
+ sys.exit(export_bundle.main(argv))
1236
+
1237
+
1238
+ def cmd_status(a):
1239
+ root = declared.domains_dir()
1240
+ for entry in sorted(os.listdir(root)) if os.path.isdir(root) else []:
1241
+ d = os.path.join(root, entry)
1242
+ if not os.path.isfile(os.path.join(d, declared.DECL)):
1243
+ continue
1244
+ reg = entry in families.DECLARED
1245
+ c = declared.read_compiled(d)
1246
+ st = ((c or {}).get("selftest") or {}).get("status", "-")
1247
+ old = bool(c) and c.get("compiler_sha256") != _sha(os.path.abspath(__file__)) # advisory: the registry only checks decl.py and the reference
1248
+ print(f"{entry:14s} {'REGISTERED' if reg else 'not registered':15s} selftest={st:8s}" + ("" if reg else f" ({families.DECLARED_ERRORS.get(entry, 'unknown')})")
1249
+ + (" [compiled by an older compile_domain.py: consider compiling again]" if old else ""))
1250
+
1251
+
1252
+ def main(argv=None):
1253
+ ap = argparse.ArgumentParser(description=__doc__.split("\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter, epilog="""\
1254
+ the whole workflow (a NAME is 2-16 lowercase letters; every command also accepts a path like domains/NAME):
1255
+ python3 compile_domain.py init NAME a working example to edit (the lines marked EDIT)
1256
+ python3 compile_domain.py check NAME seconds: what is wrong with the declaration, and how to fix it
1257
+ python3 compile_domain.py compile NAME --quick minutes: a draft ladder, for iterating
1258
+ python3 compile_domain.py compile NAME the real compile + self-test (long); prints a READY / NOT READY verdict
1259
+ python3 compile_domain.py show NAME that verdict again, with the exact next commands
1260
+ python3 compile_domain.py status every declaration: registered? self-tested? stale?
1261
+ more: COMPILER.md""")
1262
+ sub = ap.add_subparsers(dest="cmd", required=True, metavar="{init,check,compile,show,selftest,emit,export,status,recheck,spread}")
1263
+ p = sub.add_parser("init", help="scaffold a WORKING declaration to edit")
1264
+ p.add_argument("name")
1265
+ p.add_argument("--blank", action="store_true", help="a skeleton with NotImplementedError instead of the working example")
1266
+ p = sub.add_parser("show", help="plain-English summary of a compiled domain and the next commands")
1267
+ p.add_argument("name")
1268
+ p = sub.add_parser("compile", help="validate, profile, choose the ladder, simulate power, emit, self-test")
1269
+ p.add_argument("dir", metavar="NAME_or_dir")
1270
+ p.add_argument("--quick", action="store_true", help="a 1-2 minute DRAFT (small samples, no self-test): practice only, for iterating")
1271
+ p.add_argument("--target", choices=["local", "server", "ci", "certificate", "all"], default="all")
1272
+ p.add_argument("--skip-selftest", action="store_true", help="the domain can then be practised but no OFFICIAL round can be created for it")
1273
+ p.add_argument("--k", type=int, default=5, help="instances per size in the coarse probe")
1274
+ p.add_argument("--n-prof", type=int, default=8, help="instances per ladder size in the confirming measurement")
1275
+ p.add_argument("--probe-budget", type=float, default=20, help="CPU-second cap during the coarse probe")
1276
+ p.add_argument("--caps", type=int, nargs="+", default=list(CAPS), help="CPU caps the optimizer may choose from")
1277
+ p.add_argument("--target-effect", type=float, default=2.0, help="speedup (bits) the chosen n must certify with --power probability")
1278
+ p.add_argument("--power", type=float, default=0.8)
1279
+ p.add_argument("--pair-noise", type=float, default=0.6, help="ASSUMED sd (bits) of a submission's per-instance log2 speedup; measured values in this project: see PROTOCOL.md")
1280
+ p.add_argument("--trials", type=int, default=30, help="independent seed triples per power estimate")
1281
+ p.add_argument("--boot", type=int, default=60)
1282
+ p.add_argument("--selftest-n", type=int, default=None, help="instances per size in the self-test (default: the chosen official n; fewer saves machine time but then "
1283
+ "the known-effect control may fail for lack of power, not because anything is broken)")
1284
+ p = sub.add_parser("check")
1285
+ p.add_argument("dir")
1286
+ p.add_argument("--no-reference", action="store_true", help="skip running the reference solver (CI without the solver's dependencies)")
1287
+ p = sub.add_parser("selftest")
1288
+ p.add_argument("name")
1289
+ p.add_argument("--n", type=int, default=None, help="instances per size (default: the compiled official n)")
1290
+ p = sub.add_parser("emit")
1291
+ p.add_argument("name")
1292
+ p.add_argument("--target", choices=["local", "server", "ci", "certificate", "all"], default="all")
1293
+ p = sub.add_parser("export", help="write a platform-neutral challenge bundle (checker, scorer, power analysis, manifest) for a compiled, fresh domain; see COMPILER.md")
1294
+ p.add_argument("name")
1295
+ p.add_argument("--out", help="bundle directory (default domains/NAME/out/bundle-VERSION-HASH); never overwritten")
1296
+ p.add_argument("--field", default="Computer Science", help="the Field entry PROPOSAL.md pre-fills")
1297
+ p.add_argument("--no-selftest", action="store_true", help="skip the clean-interpreter self-test of the bundle")
1298
+ p = sub.add_parser("recheck", help="recompute the compiled record's checkable claims (and optionally re-measure on this machine). "
1299
+ "Executes the declaration and reference unsandboxed: only for declarations you have read")
1300
+ p.add_argument("name")
1301
+ p.add_argument("--measure", type=int, default=0, metavar="K", help="also re-measure each ladder size with K instances here and report the gap to the recorded profile")
1302
+ p.add_argument("--save", metavar="FILE", help="write the --measure gaps to FILE (never overwritten) so measurements from several machines can be merged with `spread`")
1303
+ p = sub.add_parser("spread", help="merge --save files from different machines and recommend a re-measurement tolerance")
1304
+ p.add_argument("files", nargs="+")
1305
+ sub.add_parser("status")
1306
+ a = ap.parse_args(argv)
1307
+ {"init": cmd_init, "compile": cmd_compile, "check": cmd_check, "selftest": cmd_selftest, "emit": cmd_emit, "status": cmd_status,
1308
+ "show": cmd_show, "recheck": cmd_recheck, "spread": cmd_spread, "export": cmd_export}[a.cmd](a)
1309
+
1310
+
1311
+ if __name__ == "__main__":
1312
+ main()