guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,750 @@
1
+ """Export a compiled domain as a platform-neutral CHALLENGE BUNDLE: everything another leaderboard needs to host the challenge and to say how
2
+ much of a measured speedup is real, with no copy of this repository.
3
+
4
+ python3 export_bundle.py export NAME [--out DIR] [--field FIELD] [--no-selftest] (also: compile_domain.py export NAME)
5
+ python3 export_bundle.py verify DIR [--selftest] re-check a bundle's hashes (and run its self-test)
6
+
7
+ A bundle holds the declaration (derive/verify), the reference solver, the compiled ladder and power analysis, a standalone checker
8
+ (`checker/check.py`: derive instances, verify answers, self-test), a standalone scorer (`checker/score.py`: the protocol's difficulty-stratified
9
+ credit, and a head-to-head `compare` of two solvers), a byte copy of the estimator, a manifest of every file's sha256, a README, and the
10
+ intake text a platform like Yukon asks a proposer for (PROPOSAL.md). Export refuses a stale compile, and refuses to emit a bundle that fails
11
+ its own self-test in a clean interpreter (`python3 -I`, empty directory): a bundle that only works inside this checkout is not a bundle.
12
+
13
+ Export executes the declaration and the reference solver (the self-test does), unsandboxed, like `compile`: export only declarations you
14
+ have read. It never overwrites an existing directory.
15
+ """
16
+ import argparse
17
+ import ast
18
+ import hashlib
19
+ import json
20
+ import os
21
+ import re
22
+ import shutil
23
+ import subprocess
24
+ import sys
25
+ import tempfile
26
+
27
+ import declared
28
+
29
+ HERE = os.path.dirname(os.path.abspath(__file__))
30
+ TEMPLATES = os.path.join(HERE, "bundle_templates")
31
+ FORMAT = "guldo-challenge-bundle-v1"
32
+ RESERVED = {"decl", "reference", "estimator", "check", "score", "challenge", "quality_core", "qcheck", "qscore"} # module names the bundle itself uses
33
+ MAX_VENDORED = 12
34
+ SELFTEST_TIMEOUT = 900
35
+ STD = set(sys.stdlib_module_names)
36
+
37
+
38
+ class ExportError(Exception):
39
+ """Nothing was written; the message says what to fix."""
40
+
41
+
42
+ def sha256_file(path):
43
+ return declared.sha256_file(path)
44
+
45
+
46
+ def imports_of(path):
47
+ with open(path, encoding="utf-8") as f:
48
+ tree = ast.parse(f.read(), path)
49
+ names = set()
50
+ for node in ast.walk(tree):
51
+ if isinstance(node, ast.Import):
52
+ names.update(a.name.split(".")[0] for a in node.names)
53
+ elif isinstance(node, ast.ImportFrom):
54
+ if node.level:
55
+ raise ExportError(f"{os.path.basename(path)} uses a relative import; a bundle cannot resolve it")
56
+ if node.module:
57
+ names.add(node.module.split(".")[0])
58
+ return names
59
+
60
+
61
+ def resolve_imports(entry, search_dirs, seen=None, vendored=None, third=None):
62
+ """Walk the imports of the declaration. A module that lives beside it or in the project is VENDORED (copied byte for byte); anything
63
+ else that is not standard library is a third-party requirement the bundle names but cannot ship. Imports made through importlib with a
64
+ string are invisible here: the clean-interpreter self-test is what catches those."""
65
+ seen = set() if seen is None else seen
66
+ vendored = {} if vendored is None else vendored
67
+ third = set() if third is None else third
68
+ for name in sorted(imports_of(entry)):
69
+ if name in STD or name in seen:
70
+ continue
71
+ seen.add(name)
72
+ src = next((os.path.join(d, name + ".py") for d in search_dirs if os.path.isfile(os.path.join(d, name + ".py"))), None)
73
+ if src is None:
74
+ third.add(name)
75
+ continue
76
+ if name in RESERVED:
77
+ raise ExportError(f"the declaration imports a module called {name!r}, which the bundle uses for itself; rename it")
78
+ vendored[name] = src
79
+ if len(vendored) > MAX_VENDORED:
80
+ raise ExportError(f"the declaration pulls in more than {MAX_VENDORED} project modules ({sorted(vendored)}...): it is not portable as written")
81
+ resolve_imports(src, search_dirs, seen, vendored, third)
82
+ return vendored, third
83
+
84
+
85
+ PROBE = r'''
86
+ import importlib.util, json, os, sys
87
+ d, here, sizes = sys.argv[1], sys.argv[2], [int(x) for x in sys.argv[3:]]
88
+ opened = set()
89
+ def hook(event, args):
90
+ if event == "open" and args and isinstance(args[0], (str, bytes, os.PathLike)):
91
+ opened.add(os.fsdecode(args[0]))
92
+ sys.addaudithook(hook)
93
+ sys.path[:0] = [d, here]
94
+ spec = importlib.util.spec_from_file_location("probe_decl", os.path.join(d, "decl.py"))
95
+ decl = importlib.util.module_from_spec(spec)
96
+ spec.loader.exec_module(decl)
97
+ for size in sizes:
98
+ for idx in range(3):
99
+ inst = dict(decl.derive(b"practice-seed-public", size, idx))
100
+ for ans in [decl.witness(inst) if hasattr(decl, "witness") else inst["planted"], None, 0, [], "x"]:
101
+ try:
102
+ decl.verify(inst, ans)
103
+ except Exception:
104
+ pass
105
+ rd, rh = os.path.realpath(d), os.path.realpath(here)
106
+ found, data, foreign = {}, set(), set()
107
+ for name, mod in list(sys.modules.items()):
108
+ f = getattr(mod, "__file__", None)
109
+ if f and "." not in name and name != "probe_decl":
110
+ rp = os.path.realpath(f)
111
+ if rp == os.path.join(rd, name + ".py") or rp == os.path.join(rh, name + ".py"):
112
+ found[name] = rp
113
+ for p in opened:
114
+ if not os.path.isabs(p):
115
+ continue
116
+ rp = os.path.realpath(p)
117
+ if not os.path.isfile(rp) or rp.endswith((".py", ".pyc")) or "__pycache__" in rp:
118
+ continue
119
+ if rp.startswith(rd + os.sep):
120
+ data.add(os.path.relpath(rp, rd))
121
+ elif rp.startswith(rh + os.sep):
122
+ foreign.add(rp)
123
+ print(json.dumps({"modules": found, "data": sorted(data), "foreign": sorted(foreign)}))
124
+ '''
125
+
126
+
127
+ def probe_imports(domain_dir, sizes):
128
+ """Run the declaration (import, then derive and verify several practice instances at every practice size, the witness and a few junk answers) and
129
+ report what it ACTUALLY touched: project or sibling modules it loaded (importlib.import_module("x"), __import__, imports inside functions that
130
+ run), and data files it opened (an audit hook sees every open()). The static import walk cannot see either. What this still misses: a code path
131
+ none of those calls takes. The clean-interpreter self-test is the backstop."""
132
+ with tempfile.TemporaryDirectory() as empty:
133
+ try:
134
+ r = subprocess.run([sys.executable, "-B", "-c", PROBE, domain_dir, HERE] + [str(s) for s in sizes], cwd=empty, capture_output=True, text=True,
135
+ timeout=SELFTEST_TIMEOUT)
136
+ except subprocess.TimeoutExpired:
137
+ raise ExportError("the declaration did not finish deriving its practice instances while probing it")
138
+ if r.returncode != 0:
139
+ tail = "\n".join(r.stderr.strip().splitlines()[-8:])
140
+ hint = ("\n(a data file must be opened by a path relative to the declaration itself, e.g. os.path.join(os.path.dirname(__file__), 'x.json'), "
141
+ "not relative to the working directory)" if "FileNotFoundError" in tail else "")
142
+ raise ExportError("the declaration failed while probing it:\n" + tail + hint)
143
+ return json.loads(r.stdout.strip().splitlines()[-1])
144
+
145
+
146
+ def simulation_block(compiled):
147
+ """What `score.py` needs to re-run the power simulation at a different noise or threshold: the compiled profile."""
148
+ plan = compiled.get("plan") or {}
149
+ try:
150
+ return {"cap": plan["cap"], "sizes": [int(s) for s in plan["sizes"]], "median_log2": {str(k): v for k, v in plan["median_log2"].items()}, "sigma": plan["sigma"]}
151
+ except (KeyError, TypeError, ValueError, AttributeError):
152
+ return None
153
+
154
+
155
+ def read_template(name):
156
+ with open(os.path.join(TEMPLATES, name), "rb") as f:
157
+ return f.read()
158
+
159
+
160
+ def write_bytes(path, data, mode=0o644):
161
+ os.makedirs(os.path.dirname(path), exist_ok=True)
162
+ with open(path, "wb") as f:
163
+ f.write(data)
164
+ os.chmod(path, mode)
165
+
166
+
167
+ def copy_exact(src, dst):
168
+ with open(src, "rb") as f:
169
+ write_bytes(dst, f.read())
170
+
171
+
172
+ def power_block(compiled):
173
+ cp = compiled.get("power") or {}
174
+ return {
175
+ "threshold_bits": cp.get("threshold", 1.0),
176
+ "target_power": cp.get("target_power", 0.8),
177
+ "target_effect_bits": cp.get("target_effect"),
178
+ "pair_noise_bits": cp.get("pair_noise"),
179
+ "min_certifiable_bits": cp.get("min_certifiable_effect"),
180
+ "max_certifiable_bits": cp.get("max_certifiable_effect"),
181
+ "curve": {str(e): {"single": v["single"], "all_seeds": v["all_three"], "inconclusive": v.get("inconclusive")}
182
+ for e, v in (cp.get("curve") or {}).items()},
183
+ "notes": list(cp.get("notes") or []),
184
+ }
185
+
186
+
187
+ def status_of(compiled, power):
188
+ reasons = []
189
+ st = (compiled.get("selftest") or {}).get("status", "pending")
190
+ if st != "pass":
191
+ reasons.append(f"the compiler's own self-test is {st!r}, not 'pass'")
192
+ if power["min_certifiable_bits"] is None:
193
+ reasons.append("the power analysis found no speedup this ladder can certify")
194
+ if not compiled.get("reference_note"):
195
+ reasons.append("the declaration states no REFERENCE_NOTE: nobody can tell how strong the baseline is")
196
+ if not compiled.get("plan", {}).get("confirmed", True):
197
+ reasons.append("the ladder's predicted costs were not confirmed by measurement")
198
+ return ("READY" if not reasons else "DRAFT"), reasons
199
+
200
+
201
+ def table(power):
202
+ rows = ["| true speedup (bits) | P(one seed credited) | P(all seeds credited) |", "|---|---|---|"]
203
+ rows += [f"| {e} | {100 * v['single']:.0f}% | {100 * v['all_seeds']:.0f}% |" for e, v in power["curve"].items()]
204
+ return "\n".join(rows)
205
+
206
+
207
+ def readme(ch):
208
+ pw, lad, name = ch["power"], ch["ladder"], ch["name"]
209
+ rng = (f"from about {pw['min_certifiable_bits']} to about {pw['max_certifiable_bits']} bits" if pw["min_certifiable_bits"] is not None
210
+ else "NONE: this challenge is too weak to certify anything")
211
+ reasons = "".join(f"\n* {r}" for r in ch["status_reasons"])
212
+ return f"""# {name} ({ch['version']}): challenge bundle, status {ch['status']}
213
+ {('The bundle is a DRAFT because:' + reasons) if reasons else ''}
214
+
215
+ {ch['problem']['doc']}
216
+
217
+ * solver: {ch['solver_contract']}
218
+ * sizes {lad['dims']}; {lad['n']} instances per size; CPU cap {lad['budget_seconds']} s per instance; credit needs {ch['protocol']['seeds_required']} committed seeds
219
+ * reference: {ch['reference']['note'] or 'NOT STATED by the declaration author'}
220
+ * this ladder can certify speedups {rng} (simulated through the real estimator; assumed per-instance pair noise {pw['pair_noise_bits']} bit)
221
+
222
+ {table(pw)}
223
+
224
+ ## What is in here
225
+
226
+ | path | what |
227
+ |---|---|
228
+ | `challenge.json` | the manifest: problem, ladder, protocol, power, provenance, requirements, and the sha256 of every pinned file |
229
+ | `checker/decl.py`, `checker/reference.py` | the declaration (instances are a pure function of seed, size, index; `verify` is exact) and the baseline, byte-for-byte what was compiled |
230
+ | `checker/check.py` | `info`, `selftest`, `commit`, `derive`, `verify`, `rerun` (sandboxed re-measurement of a claimed result; `--repeat`, `--save`), `spread` (merge several machines' saved measurements into a tolerance), `keygen`, `attest-check`, `log append|verify|checkpoint`: the operator's instance generator and answer checker, a verifier's tools, and signed attestations. `selftest`, `derive` and `verify` run sandboxed by default. Standard library only. `selftest` tests the verifier on a fresh random seed (printed), including answers derived from the instance itself and random answers of the witness's shape |
231
+ | `checker/score.py` | `score` (one solver against the reference), `compare` (is solver A really faster than solver B?), `power` (re-simulable), `recheck`. Needs numpy |
232
+ | `checker/estimator.py` | byte copy of the protocol's estimator (`extrapolate.py`); its hash is in the manifest |
233
+ | `checker/vendor/` | project modules the declaration imports, copied byte-for-byte (empty if none) |
234
+ | `compiled.json` | the compiler's full record, for audit |
235
+ | `MANIFEST.sha256` | `sha256sum -c MANIFEST.sha256` (macOS: `shasum -a 256 -c MANIFEST.sha256`), run in this directory |
236
+ | `PROPOSAL.md` | the text a platform's proposal form asks for, pre-filled from this record |
237
+
238
+ ## Before you run anything here
239
+
240
+ `checker/check.py derive|verify|selftest` import and run `checker/decl.py`, and `selftest` runs `checker/reference.py`. That is the author's code, and the manifest
241
+ hashes prove only that the files are the ones listed, not that they are safe. All three therefore re-run themselves INSIDE an OS sandbox by default (macOS
242
+ `sandbox-exec`, tested; Linux `bwrap`, tested on one real machine (Ubuntu 26.04, kernel 7.0, bubblewrap 0.11.1, as root: an attack probe and 63 unit tests passed). On Linux it
243
+ gives no network for any process, a read-only root, a writable scratch directory and masked credential directories, but does NOT restrict which programs may run. Ubuntu
244
+ restricts unprivileged user namespaces (AppArmor), so bwrap may be unusable for a normal user unless installed from the distribution package, which carries the exemption;
245
+ the tools say so when that happens): no network, no writes outside a scratch directory, no running any program except the Python interpreter,
246
+ `~/.ssh`, `~/.aws`, `~/.gnupg` and the keychain unreadable. Where there is no sandbox they warn and run as you; `export_bundle.py verify --selftest` instead refuses unless you pass `--unsandboxed`. A sandbox stops
247
+ the practical damage, not a determined attacker with a kernel exploit, and your own `--sandbox off` runs are as you. `check.py rerun` sandboxes a COMPETITOR's solver the same
248
+ way. (`score.py` runs nothing from the ladders it reads.) Export runs its own self-test sandboxed too, so a declaration that secretly needs to write a file or reach the
249
+ network fails at export, not for the next person. Cost: the sandbox adds about 75 ms to each call (measured on `verify`: 57 ms unsandboxed, 130 ms sandboxed), which is
250
+ nothing for a few hundred verifications per submission; for bulk verification of a bundle you have read, pass `--sandbox off` or run the whole batch inside your own VM.
251
+
252
+ ## Hosting it on a board
253
+
254
+ A board needs three things from its own side; the bundle supplies the rest.
255
+
256
+ 1. **Seeds.** Pick at least {ch['protocol']['seeds_required']} secret seeds, publish `python3 checker/check.py commit --seed-file S` for each BEFORE the deadline, reveal after. Instances are
257
+ `python3 checker/check.py derive --seed-file S --size N --idx I`; nothing can be precomputed.
258
+ 2. **A meter.** Run each submission once per instance, one at a time on one core, charge failures the cap, and write a ladder file: a first line
259
+ `{{"meta": true, "commit": "<the seed's sha256>", "budget": {lad['budget_seconds']}}}`, then one line per instance, ids `SIZE:0` .. `SIZE:{lad['n'] - 1}` for every size:
260
+ `{{"id": "28:0", "dim": 28, "status": "solved", "cost": 3.41}}` (cost > 0; statuses solved, timeout, crash, failed, wrong). Do the same for the reference
261
+ (`checker/reference.py`). Answers are checked with `python3 checker/check.py verify --seed-file S --size N --idx I --answer @answer.json`.
262
+ 3. **A scorer run.** `python3 checker/score.py score --pair REF_SEED1.jsonl SUB_SEED1.jsonl --pair ... --pair ... --out result.json` prints the credit and writes JSON.
263
+ `python3 checker/score.py compare --a A1 A2 A3 --b B1 B2 B3` answers "is #1 really ahead of #2, or is that noise?". Anyone can then run
264
+ `python3 checker/score.py recheck result.json`, and a verifier with the revealed seed can run
265
+ `python3 checker/check.py rerun --seed-file S --solver sub.py --claimed SUB_SEED1.jsonl --baseline REF_SEED1.jsonl` on their own machine.
266
+
267
+ Competitors can try the contract with `python3 checker/check.py derive --practice --size {lad['practice_dims'][0]} --idx 0`.
268
+
269
+ ## What a score does not show (read before publishing one)
270
+
271
+ * **`score.py` does not know how the costs were measured**, but `check.py rerun` can test them. `rerun` takes the seed, the solver and the two claimed ladders,
272
+ picks a random sample of instances (the verifier's own choice: the operator must not know it in advance), re-runs the reference and the solver on THIS machine,
273
+ checks the answers, and compares the re-measured speedup with the claimed one instance by instance. Costs typed into a file, or a solver slower than claimed, show as a
274
+ gap; a baseline ladder that was not measured on one machine shows as a spread. What it does not show: a meter whose bias is the same on every instance; ladders
275
+ from a different hardware class (it reports the machine factor, which cancels in a ratio only if it is the same at every size); instances you did not sample. The
276
+ default tolerance (2 bits) is loose on purpose, because machine-to-machine spread has been measured only once: `rerun --repeat N` measures THIS machine's own timing noise
277
+ (real dlog claims: 0.01 bit), and `rerun --sample-seed S --save F` on two or more machines followed by `check.py spread F1 F2 ...` turns their disagreement into a recommended
278
+ tolerance (needs 2+ distinct machines and 3+ shared instances; one machine is refused, not faked; never below 1 bit). The one real measurement: an Apple M4 Pro under macOS
279
+ and a Hetzner x86 server under Linux (about 0.5 bit apart in raw speed) agreed on the re-measured speedup to a median 0.01 bit, worst 0.03 bit, over 4 instances of one
280
+ pure-Python domain: encouraging, not a bound (one domain, one solver family; BLAS-heavy work may differ). Under bubblewrap, `rerun` also checks that the CPU meter works at
281
+ all: it refuses if an empty Python start measures ~0, and fails if the reference reads ~0 where its published cost is seconds. It also fails a claim when a run took over
282
+ half a second of wall time (the scorer ignores costs below 0.5 s anyway) but its CPU share is under 60% of the share the reference got on the same instance (30% absolute when the reference ran under 0.5 s): a solver that waits, or that forks its work and returns without waiting (on Linux that child's CPU escapes the kernel's
283
+ accounting; measured: 2.0 s of work read as 0.02 s without the sandbox, and a solver padding its CPU to 34% still faked 1.0 bit against a fixed 30% rule). Judged against the reference, a
284
+ loaded machine does not fail an honest solver, and the most a solver can still hide is about 0.7 bit. `--sandbox off` is for solvers you trust. It runs the solver and the reference in the OS sandbox described above (and additionally hides the seed and
285
+ your signing key from them) and refuses to run at all where there is none unless you pass `--sandbox off`. It is not a proof of isolation: CPU, memory and the kernel are
286
+ shared, and `sandbox-exec` is deprecated by Apple. (On macOS the profile also denies every Mach service lookup except user-name lookups, so a solver using `ctypes` cannot
287
+ talk to LaunchServices, the pasteboard or the preferences daemon: measured, all three were reachable before.)
288
+ Run the meter you trust, on one machine, one instance at a time, the same CPU for the reference and the submission.
289
+ * **The credit is the speedup over THIS reference**, in bits, on the instances the reference finds hard (cost above 0.5 s). A weak reference makes
290
+ big speedups cheap. The declaration author's statement of what the reference is: {ch['reference']['note'] or 'not stated'}
291
+ * **It measures speed, not quality.** The cost is one number per instance (seconds, or any unit that plays the role of seconds against the 0.5 s floor, for
292
+ example counted bytecode steps scaled to seconds). A challenge whose score is accuracy, size or a trade-off is not expressible here; the verifier is
293
+ all-or-nothing per instance.
294
+ * **A speedup beyond the certifiable range cannot be certified**: a very fast solver costs less than the 0.5 s floor on the instances that matter, so
295
+ they stop counting. Above the range the result is inconclusive or flat, not a measure of how fast. `score.py` says so in those words, gives the raw
296
+ median gap as description (never as a bound), and says what it would take: the slower solver must cost about 0.5 s x 2^gap on the hard instances, and if that is
297
+ above the cap, no ladder under this cap can resolve it (raise the cap or use a stronger reference, and recompile).
298
+ * **The power table assumed a per-instance noise; `score.py` measures it.** Each pair's within-bin noise is reported; if it is clearly above the assumed value the
299
+ scorer says the table is optimistic and re-simulates the certifiable range at the observed value. `python3 checker/score.py power --pair-noise X --min-gap G` re-simulates
300
+ on demand from the compiled profile (Monte-Carlo error is printed). It is a simulation of ladders drawn from the compile-time profile, not of your solver.
301
+ * **`compare` resolves only gaps on instances where the slower solver costs at least 0.5 s** (and the faster one does too). It prints its own simulated power at the
302
+ observed noise, including how often a true gap of 0 would be declared. The `--min-gap` threshold itself (default 0.5 bit) is a convention, not a pre-registered test.
303
+ * **A result can be re-checked, not trusted.** `score.py --out` records the sha256 of every input ladder, of this challenge, of the estimator and of the scorer;
304
+ `score.py recheck RESULT.json` recomputes from the ladders and fails on any difference: tamper-evidence for a published number.
305
+ * **Attestations and a log, with their honest limits.** `check.py keygen` makes an Ed25519 key (pure Python, checked against the RFC 8032 vectors); `rerun --sign KEY
306
+ --attest-out F` signs the outcome (sample, per-instance gaps, solver and ladder hashes, machine, sandboxed or not); `attest-check` verifies signatures and counts only
307
+ keys YOU pinned (`--trusted`, obtained out of band; an operator can make five keys) as vouched, while a valid signed dissent from ANY key fails the check.
308
+ `check.py log append|verify` is an append-only hash chain of submissions, results or attestations: it detects an edited, removed or reordered entry. Appending holds a file
309
+ lock from reading the log to writing the entry (POSIX; concurrent appends queue up instead of corrupting the chain, tested with 16 at once; elsewhere it refuses). A truncated copy
310
+ still verifies ALONE, so anyone (the operator, a competitor, a watcher) can `log checkpoint LOG --key K --out C`, a signed (head, count); `log verify LOG --checkpoint C`
311
+ then fails for a log that no longer contains it, whether truncated or forked into a different but internally valid history. Checkpoints only help if someone other
312
+ than the operator keeps them (and, for it to mean independence, you pinned their key). The time in an entry is the logger's clock, not evidence. None of this is a
313
+ trusted clock or a trusted-hardware proof; for claims that must hold against an adversarial operator, the full protocol (guldo certificates, rounds) is still stronger.
314
+ * **Malformed or hostile input is refused with a message, not crashed on**: ladders, result files, attestations, measurements, checkpoints, logs, answers and every numeric
315
+ command-line option (fuzz-tested in the guldo project). Options have bounds (`--boot` 1..5000, `-k` 1..1000, `--repeat` 1..50, `derive`/`verify` sizes inside the
316
+ challenge's declared range, ...), so no option can request an unbounded computation or hand the author's `derive()` a size it was never meant for. Files named on the
317
+ command line must be REGULAR files under a size cap (1 MB for seeds and key lists, 16 MB for JSON, 32 MB for ladders, 64 MB for logs and answers): a FIFO, a device such as
318
+ `/dev/zero` (also behind a symlink), a socket or a directory is refused in one line, because opening a FIFO blocks for ever and a device never ends. A result file that asks
319
+ for an unreasonable bootstrap count or leaves an input ladder unbound is refused up front.
320
+ * **The verifier is tested, not proved.** `check.py selftest` tries a fixed list of junk answers AND answers derived from each instance (its arguments and their
321
+ neighbours, other instances' witnesses, that instances really differ between seeds, witnesses with every part damaged; a witness with one part changed only warns, because a verifier with a
322
+ tolerance accepts it by design, and 100 random answers of the witness's type, length and magnitude per instance) on the public practice seed and on a fresh random seed.
323
+ Reuse checks are rates over many pairs: more than 50% fails and more than 25% warns, because at small sizes instances can share factors and chance hits cluster. So a
324
+ mildly small answer space is a warning; a lookup-table verifier (accepts nearly every other instance's answer) fails. A verifier that accepts only something outside
325
+ all of that still passes.
326
+ * **Not tied to any particular platform.** `PROPOSAL.md` is pre-filled intake text; no platform has reviewed or accepted this bundle.
327
+
328
+ Provenance: compiled {ch['provenance']['compiled_at']}, compiler self-test {ch['provenance']['selftest']}, decl {ch['files']['checker/decl.py'][:16]}..., reference {ch['files']['checker/reference.py'][:16]}...
329
+ """
330
+
331
+
332
+ def proposal(ch, field):
333
+ pw, lad = ch["power"], ch["ladder"]
334
+ rng = (f"about {pw['min_certifiable_bits']} to {pw['max_certifiable_bits']} bits" if pw["min_certifiable_bits"] is not None else "none yet")
335
+ return f"""# Proposal text for {ch['name']} (pre-filled; edit before sending)
336
+
337
+ Written from the bundle's own record. Nothing here has been reviewed by any platform. Bundle status: {ch['status']}.
338
+ {('Fix first: ' + '; '.join(ch['status_reasons'])) if ch['status_reasons'] else ''}
339
+
340
+ **Name:** [you]
341
+ **Email:** [you]
342
+ **Institution / affiliation:** [you]
343
+ **Field:** {field}
344
+
345
+ ## The Problem
346
+ {ch['problem']['doc']}
347
+
348
+ Goal metric: speedup over a stated reference solver, in bits (log2 of the cost ratio), measured on instances the reference finds hard
349
+ (cost above 0.5 s), reported as a one-sided 95% lower bound and, across {ch['protocol']['seeds_required']} committed seeds, the minimum. The reference is: {ch['reference']['note'] or '[state what the reference is and what is known to beat it]'}
350
+ Why it matters: [one or two sentences about why a faster solver for this problem is worth having].
351
+
352
+ ## The Verifier
353
+ * Instances are a pure function of (seed, size, index), derived by the declaration's `derive`; the seed's sha256 is published before the deadline and the seed after, so nothing can be precomputed.
354
+ * `verify(instance, answer)` is exact and rejects junk (checked: None, 0, 1, -1, booleans, empty and nested lists, strings, floats, huge ints).
355
+ * Cost is CPU seconds of the whole process tree, one instance at a time on one core, with a cap of {lad['budget_seconds']} s per instance; a failure is charged the cap.
356
+ * Sizes {lad['dims']}, {lad['n']} instances per size. By simulation through the real estimator this ladder can certify speedups of {rng}; the full power table is in the bundle.
357
+ * A standalone checker, a scorer and the estimator are in the bundle, with a sha256 manifest. `python3 checker/check.py selftest` runs in seconds, on a fresh random seed.
358
+ * Claimed results can be re-measured by a verifier (`check.py rerun`, random sample, own machine) and re-computed by anyone (`score.py recheck`).
359
+
360
+ ## Link
361
+ [URL of the published bundle]
362
+
363
+ ## Honest limits to state up front
364
+ * The bundle measures speed only, and the credit is only as meaningful as the reference.
365
+ * The scorer takes measured costs as input; it does not itself provide a sandbox or meter (the re-measurement tool samples; it does not replace a meter).
366
+ * Speedups beyond the certifiable range read as inconclusive, with a statement of what cap would be needed to resolve them.
367
+ """
368
+
369
+
370
+ def build(arg, out=None, field="Computer Science", selftest=True):
371
+ import compile_domain # lazy: it imports the whole project, and `verify` must not need that
372
+ d = compile_domain._resolve(arg)
373
+ mod = compile_domain._load(d, arg)
374
+ compiled = declared.read_compiled(d)
375
+ problems = declared.freshness_problems(d, mod, compiled)
376
+ if problems:
377
+ raise ExportError("REFUSED: " + "; ".join(problems) + " (compile it again)")
378
+ is_q = compiled.get("kind") == "quality" # answers are scored, not the clock: a different scorer, power analysis and README (see COMPILER.md, "Quality challenges")
379
+ d = os.path.abspath(d)
380
+ ref_path = declared.reference_path(d, mod)
381
+ safe_version = re.sub(r"[^A-Za-z0-9._-]", "_", mod.VERSION) # the author's string becomes a directory name
382
+ out = os.path.abspath(out or os.path.join(d, "out", f"bundle-{safe_version}-{compiled['decl_sha256'][:8]}"))
383
+ if os.path.exists(out):
384
+ raise ExportError(f"{out} exists; this tool never overwrites (choose another --out, or remove it yourself)")
385
+ os.makedirs(os.path.dirname(out), exist_ok=True)
386
+
387
+ vendored, third = resolve_imports(os.path.join(d, declared.DECL), [d, HERE])
388
+ dynamic = {}
389
+ probed = probe_imports(d, compiled["practice_dims"][:3]) # what the walk could not see: imports made at run time, and data files opened
390
+ if probed["foreign"]:
391
+ raise ExportError("the declaration reads project files that cannot travel with a bundle: " + ", ".join(os.path.relpath(p, HERE) for p in probed["foreign"])
392
+ + ". Put the data beside decl.py (it is then copied), or remove the dependency.")
393
+ data_files = sorted(probed["data"])
394
+ for name, path in probed["modules"].items():
395
+ if name not in vendored:
396
+ if name in RESERVED:
397
+ raise ExportError(f"the declaration loads a module called {name!r} at run time, which the bundle uses for itself; rename it")
398
+ dynamic[name] = path
399
+ for name, path in sorted(dynamic.items()):
400
+ vendored[name] = path
401
+ for path in list(dynamic.values()):
402
+ resolve_imports(path, [d, HERE], set(vendored) | STD, vendored, third)
403
+ ref_vendored, ref_third = resolve_imports(ref_path, [d, HERE])
404
+ if ref_vendored:
405
+ raise ExportError(f"the reference solver imports project modules {sorted(ref_vendored)}; it runs in a sandbox that has only the standard library, "
406
+ "numpy and fpylll, so it must not")
407
+ third |= ref_third
408
+
409
+ stage = tempfile.mkdtemp(prefix=".bundle-stage-", dir=os.path.dirname(out))
410
+ try:
411
+ files = {}
412
+
413
+ def put(rel, src):
414
+ copy_exact(src, os.path.join(stage, rel))
415
+ files[rel] = sha256_file(os.path.join(stage, rel)) # the hash of what was written, not of a source that could change meanwhile
416
+
417
+ put("checker/decl.py", os.path.join(d, declared.DECL))
418
+ put("checker/reference.py", ref_path)
419
+ put("checker/quality_core.py" if is_q else "checker/estimator.py", os.path.join(HERE, "quality_core.py" if is_q else "extrapolate.py"))
420
+ put("compiled.json", os.path.join(d, declared.COMPILED))
421
+ for name, src in sorted(vendored.items()):
422
+ put(f"checker/vendor/{name}.py", src)
423
+ for rel in data_files: # beside decl.py in the domain, beside checker/decl.py in the bundle: a path relative to __file__ keeps working
424
+ if rel in ("compiled.json",) or rel.split(os.sep)[0] in ("out",):
425
+ continue
426
+ put("checker/" + rel.replace(os.sep, "/"), os.path.join(d, rel))
427
+ write_bytes(os.path.join(stage, "checker", "check.py"), read_template("check.py"), 0o755)
428
+ write_bytes(os.path.join(stage, "checker", "score.py"), read_template("score.py"), 0o755)
429
+ if is_q:
430
+ write_bytes(os.path.join(stage, "checker", "qscore.py"), read_template("qscore.py"), 0o755)
431
+ write_bytes(os.path.join(stage, "checker", "qcheck.py"), read_template("qcheck.py"), 0o755)
432
+
433
+ import extrapolate as E
434
+ power = power_block(compiled)
435
+ status, reasons = status_of(compiled, power)
436
+ ch = {
437
+ "format": FORMAT, "name": mod.NAME, "version": mod.VERSION, "status": status, "status_reasons": reasons,
438
+ "problem": {"doc": getattr(mod, "DOC", "(the declaration gives no DOC string)"), "signature": getattr(mod, "SIGNATURE", None)},
439
+ "solver_contract": "one Python file defining a top-level solve(*args) returning an int, a list of ints, or None; deterministic, one process, no files, no network",
440
+ "reference": {"file": "checker/reference.py", "note": compiled.get("reference_note")},
441
+ "ladder": {"dims": compiled["dims"], "practice_dims": compiled["practice_dims"], "size_range": compiled.get("size_range"),
442
+ "size_step": compiled.get("size_step"), "n": compiled["official"]["n"], "budget_seconds": compiled["official"]["budget"]},
443
+ "protocol": {"seeds_required": compiled_seeds(), "commitment": "sha256 of the seed file's bytes (surrounding whitespace stripped)",
444
+ "instances": "a pure function of (seed, size, idx): checker/check.py derive",
445
+ "cost_floor_seconds": E.FLOOR_ABS, "min_instances_per_bin": E.MIN_BIN, "top_bins_pooled": E.TOP_K,
446
+ "credit": "one-sided 95% joint-bootstrap lower bound of the pooled median log2 speedup over the hardest usable difficulty bins; min over seeds; inconclusive = 0"},
447
+ "power": power,
448
+ "provenance": {"compiled_at": compiled.get("compiled_at"), "compiler_sha256": compiled.get("compiler_sha256"),
449
+ "selftest": (compiled.get("selftest") or {}).get("status", "pending"), "cpu_hours": compiled.get("cpu_hours"),
450
+ "exported_with": f"python {sys.version.split()[0]}"},
451
+ "requires": {"third_party": sorted(third), "vendored": sorted(vendored), "vendored_found_only_at_run_time": sorted(dynamic), "data_files": [x for x in files if x.startswith("checker/") and not x.startswith(("checker/vendor/", "checker/decl.py", "checker/reference.py", "checker/estimator.py", "checker/quality_core.py"))],
452
+ "checker": "standard library", "scorer": "numpy"},
453
+ "simulation": simulation_block(compiled),
454
+ "files": dict(sorted(files.items())),
455
+ }
456
+ if is_q:
457
+ ch = quality_challenge(ch, compiled)
458
+ status, reasons = ch["status"], ch["status_reasons"] # what build() returns and prints: the speed record's reasons above do not apply
459
+ write_bytes(os.path.join(stage, "challenge.json"), (json.dumps(ch, indent=1, sort_keys=True) + "\n").encode())
460
+ write_bytes(os.path.join(stage, "README.md"), (readme_quality(ch) if is_q else readme(ch)).encode())
461
+ write_bytes(os.path.join(stage, "PROPOSAL.md"), (proposal_quality(ch, field) if is_q else proposal(ch, field)).encode())
462
+ write_manifest(stage)
463
+ if selftest: # a fixed fresh seed (from the declaration's hash) keeps an export reproducible; `check.py selftest` run later draws a new one each time
464
+ run_selftest(stage, hashlib.sha256((compiled["decl_sha256"] + ":fresh").encode()).hexdigest()[:32])
465
+ os.chmod(stage, 0o755) # mkdtemp makes it 0700; a bundle is meant to be copied and shared
466
+ os.rename(stage, out)
467
+ except BaseException:
468
+ shutil.rmtree(stage, ignore_errors=True)
469
+ raise
470
+ return out, status, reasons
471
+
472
+
473
+ def quality_challenge(ch, compiled):
474
+ """The manifest of a QUALITY bundle: `ch` is the speed-shaped record built above; everything that describes speed is replaced."""
475
+ import compile_quality
476
+ qc = compiled["quality"]
477
+ pw = qc["power"]
478
+ reasons = []
479
+ st = (compiled.get("selftest") or {}).get("status", "pending")
480
+ if st != "pass":
481
+ reasons.append(f"the compiler's own self-test is {st!r}, not 'pass'")
482
+ if pw["min_certifiable"] is None:
483
+ reasons.append("the power analysis found no gain this ladder can certify")
484
+ if not compiled.get("reference_note"):
485
+ reasons.append("the declaration states no REFERENCE_NOTE: nobody can tell how strong the baseline is")
486
+ out = dict(ch)
487
+ out.update({
488
+ "metric": "quality", "status": "READY" if not reasons else "DRAFT", "status_reasons": reasons,
489
+ "solver_contract": "one Python file defining a top-level solve(*args) returning a JSON value the declaration's verify() accepts; deterministic or not, one process, no files, no network; "
490
+ f"it gets {qc['budget']:g} CPU seconds per instance",
491
+ "protocol": {"seeds_required": compiled_seeds(), "commitment": "sha256 of the seed file's bytes (surrounding whitespace stripped)",
492
+ "instances": "a pure function of (seed, size, idx): checker/check.py derive",
493
+ "credit": "one-sided 95% lower bound of the mean paired quality difference (solver minus reference), bootstrapped within each size, averaged over sizes; "
494
+ "max(0, .) per seed; min over seeds. An instance without a verified answer scores 0."},
495
+ "power": {"threshold": pw["threshold"], "target_power": pw["target_power"], "target_effect": pw["target_effect"], "noise": pw["noise"], "noise_assumed": pw["noise_assumed"],
496
+ "min_certifiable": pw["min_certifiable"], "max_certifiable": pw["max_certifiable"], "curve": pw["curve"], "notes": list(pw["notes"]) + list(qc.get("notes", []))},
497
+ "quality": {"budget_seconds": qc["budget"], "min_gain": qc["min_gain"], "headroom_mean_cap": qc["headroom_mean_cap"],
498
+ "reference_profile": {s: {"mean": v["mean"], "sd": v["sd"], "timeouts": v["timeouts"]} for s, v in qc["profile"]["sizes"].items()}},
499
+ "simulation": qc["simulation"],
500
+ })
501
+ out["provenance"] = dict(out["provenance"], cpu_hours=qc["cpu_hours"], quality_compiler_sha256=compiled.get("quality_compiler_sha256"))
502
+ out["requires"] = dict(out["requires"], scorer="standard library", checker="standard library")
503
+ return out
504
+
505
+
506
+ def readme_quality(ch):
507
+ pw, lad, name = ch["power"], ch["ladder"], ch["name"]
508
+ reasons = "".join(f"\n* {r}" for r in ch["status_reasons"])
509
+ rng = (f"from about {pw['min_certifiable']:g} to about {pw['max_certifiable']:g}" if pw["min_certifiable"] is not None else "NONE: this challenge is too weak to certify anything")
510
+ rows = ["| true mean gain (quality points) | P(one seed credited) | P(all seeds credited) |", "|---|---|---|"]
511
+ rows += [f"| {float(e):g} | {100 * v['single']:.0f}% | {100 * v['all_seeds']:.0f}% |" for e, v in pw["curve"].items()]
512
+ prof = "\n".join(f" * size {s}: reference mean quality {v['mean']:.3f} (sd {v['sd']:.3f}, {v['timeouts']} timeouts in the profile)" for s, v in ch["quality"]["reference_profile"].items()
513
+ if int(s) in set(lad["dims"]))
514
+ return f"""# {name} ({ch['version']}): QUALITY challenge bundle, status {ch['status']}
515
+ {('The bundle is a DRAFT because:' + reasons) if reasons else ''}
516
+
517
+ {ch['problem']['doc']}
518
+
519
+ * **What is scored:** the QUALITY of each answer (0 to 1, from the declaration's `quality(inst, ans)`), at the same CPU budget for everyone: {lad['budget_seconds']} s per instance. A solver that is
520
+ faster but no better gets nothing; one that finds better answers inside the budget gets the difference. An instance with no verified answer scores 0.
521
+ * solver: {ch['solver_contract']}
522
+ * sizes {lad['dims']}; {lad['n']} instances per size; credit needs {ch['protocol']['seeds_required']} committed seeds
523
+ * reference: {ch['reference']['note'] or 'NOT STATED by the declaration author'}
524
+ {prof}
525
+ * this ladder can certify mean gains {rng} quality points over the reference (threshold {pw['threshold']:g}, simulated; assumed per-instance noise {pw['noise']:g}, an ASSUMPTION that `qscore.py` replaces by the value it measures)
526
+
527
+ {chr(10).join(rows)}
528
+
529
+ ## What is in here
530
+
531
+ | path | what |
532
+ |---|---|
533
+ | `challenge.json` | the manifest (`"metric": "quality"`): problem, ladder, protocol, power, provenance, requirements, the sha256 of every pinned file |
534
+ | `checker/decl.py`, `checker/reference.py` | the declaration (`derive`, `verify`, `quality`) and the baseline, byte-for-byte what was compiled |
535
+ | `checker/check.py` | `info`, `selftest`, `commit`, `derive`, `keygen`, `attest-check`, `log`: the speed bundle's tools, which work here. `selftest` tests `verify` AND `quality()` (range, determinism, that no answer needing no search scores above 0.9). `rerun` and `spread` are speed tools and refuse |
536
+ | `checker/qcheck.py` | `run` (run a solver on every instance, sandboxed, and write the ladder), `verify` (valid? and its quality), `qrerun` (re-run a random sample of a CLAIMED result; `--sign`) |
537
+ | `checker/qscore.py` | `score` (one solver against the reference), `compare` (is A really better than B?), `power` (re-simulable), `recheck`. Standard library only |
538
+ | `checker/quality_core.py` | the statistics (paired difference, stratified bootstrap bound, power simulation); its hash is in the manifest |
539
+ | `checker/score.py` | the speed scorer's readers, reused by `qscore.py`; its speed commands refuse on this bundle |
540
+ | `checker/vendor/` | project modules the declaration imports, copied byte-for-byte (empty if none) |
541
+ | `compiled.json` | the compiler's full record, for audit |
542
+ | `MANIFEST.sha256` | `sha256sum -c MANIFEST.sha256` (macOS: `shasum -a 256 -c MANIFEST.sha256`), run in this directory |
543
+ | `PROPOSAL.md` | intake text pre-filled from this record |
544
+
545
+ ## Before you run anything here
546
+
547
+ `check.py derive|selftest` and `qcheck.py verify` import and run `checker/decl.py` (and `selftest` the reference): that is the author's code, and the manifest hashes prove only that the files are the
548
+ ones listed, not that they are safe. They re-run themselves inside an OS sandbox by default (macOS `sandbox-exec`; Linux `bwrap`: no network, no writes outside a scratch directory, credentials
549
+ unreadable) and warn where there is none. `qcheck.py run` and `qrerun` execute a COMPETITOR's solver the same way and REFUSE to run without a sandbox unless you pass `--sandbox off`. They also
550
+ call `decl.verify()` and `decl.quality()` in the parent process, unsandboxed: read `decl.py` first. A sandbox stops the practical damage, not a determined attacker.
551
+
552
+ ## Hosting it on a board
553
+
554
+ 1. **Seeds.** Pick at least {ch['protocol']['seeds_required']} secret seeds, publish `python3 checker/check.py commit --seed-file S` for each BEFORE the deadline, reveal after.
555
+ 2. **A meter.** `python3 checker/qcheck.py run --seed-file S --solver sub.py --out SUB_S.jsonl` runs the solver on every instance (one at a time, CPU-limited to the budget, sandboxed), writes the
556
+ ladder, and prints each size's mean quality. Do the same with `--solver checker/reference.py` for the baseline ladder. A ladder is JSON lines: a first line
557
+ `{{"meta": true, "commit": "<the seed's sha256>", "budget": {lad['budget_seconds']}}}`, then `{{"id": "SIZE:IDX", "dim": SIZE, "status": "solved", "quality": 0.873}}` for every instance
558
+ (statuses solved, timeout, crash, failed, wrong; anything but solved scores 0).
559
+ 3. **A scorer run.** `python3 checker/qscore.py score --pair REF_S1.jsonl SUB_S1.jsonl --pair ... --pair ... --out result.json` prints the credit in quality points and writes JSON;
560
+ `python3 checker/qscore.py compare --a A1 A2 A3 --b B1 B2 B3` answers "is #1 really ahead of #2?"; anyone can `qscore.py recheck result.json`; a verifier with the revealed seed can
561
+ `python3 checker/qcheck.py qrerun --seed-file S --solver sub.py --claimed SUB_S1.jsonl --baseline REF_S1.jsonl --sign KEY --attest-out A.json` on their own machine, and
562
+ `check.py attest-check` counts attestations from keys you pinned.
563
+
564
+ Competitors can try the contract with `python3 checker/qcheck.py run --practice --solver my.py` (public practice seed, sizes {lad['practice_dims']}).
565
+
566
+ ## What a score does not show (read before publishing one)
567
+
568
+ * **The credit is only as meaningful as `quality()`.** The scorer averages whatever the declaration returns. If quality() rewards the wrong thing, the score does too. It is checked
569
+ to be a finite number in [0, 1], deterministic, and not high for answers that need no search; whether it measures what you care about is not checkable by a tool.
570
+ * **The credit is the gain over THIS reference, in quality points, at THIS budget.** A weak reference makes gains cheap; a saturated one (the reference already near 1) makes every solver tie:
571
+ the compiler drops sizes where the reference averages above {ch['quality']['headroom_mean_cap']}, and `qscore.py` warns when half the reference's answers are at the top. Declaration author's statement of
572
+ the reference: {ch['reference']['note'] or 'not stated'}
573
+ * **Equal weight per size, equal budget for everyone.** The mean is over sizes, so a gain on the largest size counts as much as one on the smallest. A solver that does better by
574
+ using more CPU than the budget is not credited: `run` scores an answer that came after the cap as 0 and `qrerun` fails a solver whose CPU exceeds it. Wall-clock, memory and
575
+ parallelism are NOT limited beyond the sandbox: the budget is CPU seconds of the process tree, and a solver that hides its CPU (waits, or forks work it never waits for) is detected only
576
+ when more than 70% is hidden.
577
+ * **The power table assumed a per-instance noise; `qscore.py` measures it** (pooled within-size sd of the paired differences) and, if the ladders show clearly more, re-simulates the
578
+ certifiable range at the observed value. The assumption itself is a share of the reference's own spread, not measured on any competitor.
579
+ * **Randomised solvers do not reproduce exactly.** `qrerun` compares claimed and re-measured qualities within a tolerance (default 0.05); a solver whose own run-to-run spread is larger
580
+ will fail it, which is honest: its claimed number was not reproducible.
581
+ * **Machine speed matters here in a way it does not for a speed bundle's ratios:** an anytime solver does more work per CPU second on a faster machine, so the same budget gives a better
582
+ answer. Run the reference and every submission on the same machine (`run` records no machine), or re-run with `qrerun` on yours. Cross-machine spread of quality has NOT been measured.
583
+ * **A result can be re-checked, not trusted.** `qscore.py --out` records the sha256 of every input ladder, of this challenge and of the scorer; `recheck` recomputes and fails on any difference.
584
+ * **Hostile input is refused, not crashed on** (ladders, result files, answers, options), with the speed bundle's size caps and bounds. The quality code paths were added later than the
585
+ speed ones and have had less testing.
586
+ * **Not tied to any particular platform.** `PROPOSAL.md` is intake text; no platform has reviewed or accepted this bundle.
587
+
588
+ Provenance: compiled {ch['provenance']['compiled_at']}, compiler self-test {ch['provenance']['selftest']}, decl {ch['files']['checker/decl.py'][:16]}..., reference {ch['files']['checker/reference.py'][:16]}...
589
+ """
590
+
591
+
592
+ def proposal_quality(ch, field):
593
+ pw, lad = ch["power"], ch["ladder"]
594
+ rng = (f"about {pw['min_certifiable']:g} to {pw['max_certifiable']:g} quality points" if pw["min_certifiable"] is not None else "none yet")
595
+ return f"""# Proposal text for {ch['name']} (pre-filled; edit before sending)
596
+
597
+ Written from the bundle's own record. Nothing here has been reviewed by any platform. Bundle status: {ch['status']}.
598
+ {('Fix first: ' + '; '.join(ch['status_reasons'])) if ch['status_reasons'] else ''}
599
+
600
+ **Name:** [you]
601
+ **Email:** [you]
602
+ **Institution / affiliation:** [you]
603
+ **Field:** {field}
604
+
605
+ ## The Problem
606
+ {ch['problem']['doc']}
607
+
608
+ Goal metric: the QUALITY of the answer (0 to 1) at a fixed CPU budget of {lad['budget_seconds']} s per instance, as a gain over a stated reference solver. The score is the one-sided 95% lower bound
609
+ of the mean paired quality difference, bootstrapped within each size, and across {ch['protocol']['seeds_required']} committed seeds the minimum. The reference is: {ch['reference']['note'] or '[state what the reference is and what is known to beat it]'}
610
+ Why it matters: [one or two sentences about why better answers to this problem are worth having].
611
+
612
+ ## The Verifier
613
+ * Instances are a pure function of (seed, size, index); the seed's sha256 is published before the deadline and the seed after, so nothing can be precomputed.
614
+ * `verify(instance, answer)` decides validity; `quality(instance, answer)` scores a valid answer in [0, 1] (checked: finite, in range, deterministic, and no answer that needs no search scores above 0.9). An instance without a verified answer scores 0.
615
+ * CPU seconds of the whole process tree, one instance at a time, a cap of {lad['budget_seconds']} s per instance, the same for the reference and every submission.
616
+ * Sizes {lad['dims']}, {lad['n']} instances per size. By simulation this ladder can certify mean gains of {rng}; the full power table is in the bundle.
617
+ * A standalone checker (`check.py`, `qcheck.py`), a scorer (`qscore.py`) and the statistics module are in the bundle, with a sha256 manifest. `python3 checker/check.py selftest` runs in seconds on a fresh random seed.
618
+ * Claimed results can be re-run by a verifier on their own machine (`qcheck.py qrerun`, random sample, signed attestation) and re-computed by anyone (`qscore.py recheck`).
619
+
620
+ ## Link
621
+ [URL of the published bundle]
622
+
623
+ ## Honest limits to state up front
624
+ * The score is only as meaningful as `quality()`; the tools check its range and determinism, not its meaning.
625
+ * The scorer takes measured qualities as input; `qcheck.py run` is a sandboxed local meter, not a hosted, isolated one.
626
+ * Cross-machine reproducibility of quality (an anytime solver does more per CPU second on a faster machine) has not been measured.
627
+ """
628
+
629
+
630
+ def compiled_seeds():
631
+ import compile_domain
632
+ return compile_domain.SEEDS
633
+
634
+
635
+ def write_manifest(root):
636
+ lines = []
637
+ for dp, dns, fns in os.walk(root):
638
+ dns[:] = sorted(x for x in dns if x != "__pycache__")
639
+ for fn in sorted(fns):
640
+ if fn == "MANIFEST.sha256" or fn.endswith(".pyc"):
641
+ continue
642
+ p = os.path.join(dp, fn)
643
+ lines.append(f"{sha256_file(p)} {os.path.relpath(p, root)}")
644
+ write_bytes(os.path.join(root, "MANIFEST.sha256"), ("\n".join(sorted(lines, key=lambda s: s.split(" ", 1)[1])) + "\n").encode())
645
+
646
+
647
+ def manifest_problems(root):
648
+ path = os.path.join(root, "MANIFEST.sha256")
649
+ if not os.path.isfile(path):
650
+ return ["MANIFEST.sha256 is missing"]
651
+ listed, out = {}, []
652
+ with open(path, encoding="utf-8") as f:
653
+ for line in f:
654
+ if line.strip():
655
+ h, _, rel = line.rstrip("\n").partition(" ")
656
+ listed[rel] = h
657
+ for rel, h in sorted(listed.items()):
658
+ p = os.path.join(root, rel)
659
+ if not os.path.isfile(p):
660
+ out.append(f"missing: {rel}")
661
+ elif sha256_file(p) != h:
662
+ out.append(f"changed: {rel}")
663
+ for dp, dns, fns in os.walk(root):
664
+ dns[:] = [x for x in dns if x != "__pycache__"]
665
+ for fn in fns:
666
+ rel = os.path.relpath(os.path.join(dp, fn), root)
667
+ if rel != "MANIFEST.sha256" and not fn.endswith(".pyc") and rel not in listed:
668
+ out.append(f"not in the manifest: {rel}")
669
+ return out
670
+
671
+
672
+ def run_selftest(root, fresh_seed=None, sandbox="auto", require=False):
673
+ """check.py selftest in a clean interpreter (-I: no PYTHONPATH, no user site, no script directory; -B: leave no bytecode in the bundle) from an empty working
674
+ directory, inside the OS sandbox unless sandbox="off" (so a declaration that secretly needs to write files or reach the network fails HERE, at export, not for
675
+ the next person). require=True refuses to run at all where no sandbox exists."""
676
+ with tempfile.TemporaryDirectory() as empty:
677
+ env = {"PATH": os.environ.get("PATH", ""), "HOME": empty, "PYTHONHASHSEED": "0", "PYTHONDONTWRITEBYTECODE": "1"}
678
+ if require:
679
+ env["GULDO_REQUIRE_SANDBOX"] = "1"
680
+ try:
681
+ cmd = ([sys.executable, "-I", "-B", os.path.join(root, "checker", "check.py"), "selftest", "--sandbox", sandbox]
682
+ + (["--fresh-seed", fresh_seed] if fresh_seed else []))
683
+ r = subprocess.run(cmd, cwd=empty, env=env,
684
+ capture_output=True, text=True, timeout=SELFTEST_TIMEOUT)
685
+ except subprocess.TimeoutExpired:
686
+ raise ExportError(f"the bundle's self-test did not finish in {SELFTEST_TIMEOUT} s")
687
+ if r.returncode != 0:
688
+ tail = "\n".join((r.stdout + r.stderr).strip().splitlines()[-15:])
689
+ raise ExportError("the bundle FAILS its own self-test in a clean interpreter, so it is not portable as written:\n" + tail)
690
+ return r.stdout
691
+
692
+
693
+ def verify_bundle(root, selftest=False, unsandboxed=False):
694
+ """Hashes and pins, and with selftest=True the bundle's own self-test. That runs the bundle's decl.py and reference, which are its AUTHOR's code, so it runs
695
+ inside the OS sandbox and refuses to run at all where there is none, unless unsandboxed=True."""
696
+ problems = manifest_problems(root)
697
+ if not problems:
698
+ try:
699
+ with open(os.path.join(root, "challenge.json"), encoding="utf-8") as f:
700
+ ch = json.load(f)
701
+ if ch.get("format") != FORMAT:
702
+ problems.append(f"challenge.json has format {ch.get('format')!r}, expected {FORMAT!r}")
703
+ for rel, h in ch.get("files", {}).items():
704
+ p = os.path.join(root, rel)
705
+ if not os.path.isfile(p) or sha256_file(p) != h:
706
+ problems.append(f"challenge.json pins {rel} but it differs")
707
+ except (OSError, ValueError) as e:
708
+ problems.append(f"challenge.json unreadable: {e}")
709
+ if selftest and not problems:
710
+ try:
711
+ run_selftest(root, None, "off" if unsandboxed else "auto", require=not unsandboxed)
712
+ except ExportError as e:
713
+ problems.append(str(e))
714
+ return problems
715
+
716
+
717
+ def main(argv=None):
718
+ argv = list(sys.argv[1:] if argv is None else argv)
719
+ if argv and argv[0] not in ("export", "verify") and not argv[0].startswith("-"):
720
+ argv = ["export"] + argv # `guldo export NAME` (the form the docs show) means `export NAME`; `guldo export export NAME` also still works
721
+ ap = argparse.ArgumentParser(description=__doc__.split("\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__)
722
+ sub = ap.add_subparsers(dest="cmd", required=True)
723
+ p = sub.add_parser("export", help="write a bundle for a compiled, fresh domain")
724
+ p.add_argument("name", metavar="NAME_or_dir")
725
+ p.add_argument("--out", help="bundle directory (default domains/NAME/out/bundle-VERSION-DECLHASH); never overwritten")
726
+ p.add_argument("--field", default="Computer Science", help="the Field entry PROPOSAL.md pre-fills (default: Computer Science)")
727
+ p.add_argument("--no-selftest", action="store_true", help="skip the clean-interpreter self-test (the bundle is then not known to be portable)")
728
+ p = sub.add_parser("verify", help="re-check a bundle's hashes (and optionally run its self-test)")
729
+ p.add_argument("dir")
730
+ p.add_argument("--selftest", action="store_true", help="ALSO run the bundle's self-test, which executes its decl.py and reference (its author's code): inside the OS sandbox, "
731
+ "and refusing to run where there is none")
732
+ p.add_argument("--unsandboxed", action="store_true", help="with --selftest: run it as you, without the sandbox (only for a bundle you have read)")
733
+ a = ap.parse_args(argv)
734
+ try:
735
+ if a.cmd == "export":
736
+ out, status, reasons = build(a.name, a.out, a.field, not a.no_selftest)
737
+ print(f"wrote {out}")
738
+ print(f"status: {status}" + "".join(f"\n - {r}" for r in reasons))
739
+ print("next: python3 " + os.path.join(out, "checker", "check.py") + " selftest (and read " + os.path.join(out, "README.md") + ")")
740
+ return 0
741
+ bad = verify_bundle(a.dir, a.selftest, a.unsandboxed)
742
+ print("bundle OK" if not bad else "bundle PROBLEMS:\n " + "\n ".join(bad))
743
+ return 1 if bad else 0
744
+ except ExportError as e:
745
+ print(str(e), file=sys.stderr)
746
+ return 1
747
+
748
+
749
+ if __name__ == "__main__":
750
+ sys.exit(main())