guldo-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo_cli-0.1.0/MANIFEST.in +4 -0
- guldo_cli-0.1.0/PKG-INFO +30 -0
- guldo_cli-0.1.0/README.md +18 -0
- guldo_cli-0.1.0/guldo/__init__.py +0 -0
- guldo_cli-0.1.0/guldo/_src/COMPILER.md +329 -0
- guldo_cli-0.1.0/guldo/_src/DEPLOY.md +200 -0
- guldo_cli-0.1.0/guldo/_src/LAUNCH.md +85 -0
- guldo_cli-0.1.0/guldo/_src/PROTOCOL.md +294 -0
- guldo_cli-0.1.0/guldo/_src/RULES.md +25 -0
- guldo_cli-0.1.0/guldo/_src/SUBMIT.md +170 -0
- guldo_cli-0.1.0/guldo/_src/__init__.py +0 -0
- guldo_cli-0.1.0/guldo/_src/attest.py +257 -0
- guldo_cli-0.1.0/guldo/_src/build_site.py +958 -0
- guldo_cli-0.1.0/guldo/_src/bundle_templates/check.py +1382 -0
- guldo_cli-0.1.0/guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo_cli-0.1.0/guldo/_src/bundle_templates/qscore.py +402 -0
- guldo_cli-0.1.0/guldo/_src/bundle_templates/score.py +649 -0
- guldo_cli-0.1.0/guldo/_src/certificate.py +828 -0
- guldo_cli-0.1.0/guldo/_src/certs_site.py +108 -0
- guldo_cli-0.1.0/guldo/_src/claims.py +343 -0
- guldo_cli-0.1.0/guldo/_src/compile_domain.py +1312 -0
- guldo_cli-0.1.0/guldo/_src/compile_quality.py +286 -0
- guldo_cli-0.1.0/guldo/_src/count_runner.py +248 -0
- guldo_cli-0.1.0/guldo/_src/counted.py +274 -0
- guldo_cli-0.1.0/guldo/_src/declared.py +215 -0
- guldo_cli-0.1.0/guldo/_src/deploy/Caddyfile +48 -0
- guldo_cli-0.1.0/guldo/_src/deploy/ledger-backup.service +9 -0
- guldo_cli-0.1.0/guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo_cli-0.1.0/guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo_cli-0.1.0/guldo/_src/deploy/ledger-export.service +12 -0
- guldo_cli-0.1.0/guldo/_src/deploy/ledger-export.timer +9 -0
- guldo_cli-0.1.0/guldo/_src/deploy/ledger-web.service +38 -0
- guldo_cli-0.1.0/guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo_cli-0.1.0/guldo/_src/deploy/provision_eval.sh +128 -0
- guldo_cli-0.1.0/guldo/_src/deploy_smoke.py +148 -0
- guldo_cli-0.1.0/guldo/_src/dlog_experiment.py +165 -0
- guldo_cli-0.1.0/guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo_cli-0.1.0/guldo/_src/domains/aidecode/decl.py +134 -0
- guldo_cli-0.1.0/guldo/_src/domains/aidecode/reference.py +73 -0
- guldo_cli-0.1.0/guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo_cli-0.1.0/guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo_cli-0.1.0/guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo_cli-0.1.0/guldo/_src/domains/dlog/compiled.json +248 -0
- guldo_cli-0.1.0/guldo/_src/domains/dlog/decl.py +41 -0
- guldo_cli-0.1.0/guldo/_src/domains/dlog/reference.py +10 -0
- guldo_cli-0.1.0/guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo_cli-0.1.0/guldo/_src/domains/maxcut/decl.py +46 -0
- guldo_cli-0.1.0/guldo/_src/domains/maxcut/reference.py +16 -0
- guldo_cli-0.1.0/guldo/_src/drift_check.py +89 -0
- guldo_cli-0.1.0/guldo/_src/export_bundle.py +750 -0
- guldo_cli-0.1.0/guldo/_src/extrapolate.py +326 -0
- guldo_cli-0.1.0/guldo/_src/families.py +53 -0
- guldo_cli-0.1.0/guldo/_src/gen_cheat_table.py +36 -0
- guldo_cli-0.1.0/guldo/_src/gpu_meter.py +634 -0
- guldo_cli-0.1.0/guldo/_src/gpu_runner.py +63 -0
- guldo_cli-0.1.0/guldo/_src/harness.py +273 -0
- guldo_cli-0.1.0/guldo/_src/instances.py +86 -0
- guldo_cli-0.1.0/guldo/_src/instances_factor.py +40 -0
- guldo_cli-0.1.0/guldo/_src/instances_mlwe.py +69 -0
- guldo_cli-0.1.0/guldo/_src/judge.py +278 -0
- guldo_cli-0.1.0/guldo/_src/limits_plan.py +158 -0
- guldo_cli-0.1.0/guldo/_src/official.py +189 -0
- guldo_cli-0.1.0/guldo/_src/ops.py +281 -0
- guldo_cli-0.1.0/guldo/_src/portal.py +313 -0
- guldo_cli-0.1.0/guldo/_src/portal_core.py +57 -0
- guldo_cli-0.1.0/guldo/_src/power.py +174 -0
- guldo_cli-0.1.0/guldo/_src/prereg.py +173 -0
- guldo_cli-0.1.0/guldo/_src/public_portal.py +500 -0
- guldo_cli-0.1.0/guldo/_src/quality_core.py +126 -0
- guldo_cli-0.1.0/guldo/_src/recheck.py +236 -0
- guldo_cli-0.1.0/guldo/_src/remeasure.py +88 -0
- guldo_cli-0.1.0/guldo/_src/report_html.py +493 -0
- guldo_cli-0.1.0/guldo/_src/results/.keep +0 -0
- guldo_cli-0.1.0/guldo/_src/rounds.py +846 -0
- guldo_cli-0.1.0/guldo/_src/runner.py +52 -0
- guldo_cli-0.1.0/guldo/_src/sandbox/Dockerfile +11 -0
- guldo_cli-0.1.0/guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo_cli-0.1.0/guldo/_src/sandbox/README.md +40 -0
- guldo_cli-0.1.0/guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo_cli-0.1.0/guldo/_src/sandbox/meter.py +89 -0
- guldo_cli-0.1.0/guldo/_src/sandbox/meter_core.py +28 -0
- guldo_cli-0.1.0/guldo/_src/site_assets.py +176 -0
- guldo_cli-0.1.0/guldo/_src/site_live_check.py +79 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo_cli-0.1.0/guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo_cli-0.1.0/guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo_cli-0.1.0/guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo_cli-0.1.0/guldo/_src/solvers/cheat_table.json +1 -0
- guldo_cli-0.1.0/guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo_cli-0.1.0/guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo_cli-0.1.0/guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo_cli-0.1.0/guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo_cli-0.1.0/guldo/_src/solvers/egg_base.py +11 -0
- guldo_cli-0.1.0/guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo_cli-0.1.0/guldo/_src/solvers/egg_template.py +11 -0
- guldo_cli-0.1.0/guldo/_src/solvers/factor_brent.py +34 -0
- guldo_cli-0.1.0/guldo/_src/solvers/factor_floyd.py +19 -0
- guldo_cli-0.1.0/guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo_cli-0.1.0/guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo_cli-0.1.0/guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo_cli-0.1.0/guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo_cli-0.1.0/guldo/_src/solvers/fork_bomb.py +18 -0
- guldo_cli-0.1.0/guldo/_src/solvers/gpu_noop.py +6 -0
- guldo_cli-0.1.0/guldo/_src/solvers/hostile_probe.py +36 -0
- guldo_cli-0.1.0/guldo/_src/solvers/kill_meter.py +27 -0
- guldo_cli-0.1.0/guldo/_src/solvers/memory_bomb.py +10 -0
- guldo_cli-0.1.0/guldo/_src/solvers/noop.py +5 -0
- guldo_cli-0.1.0/guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo_cli-0.1.0/guldo/_src/solvers/planted_peek.py +23 -0
- guldo_cli-0.1.0/guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo_cli-0.1.0/guldo/_src/solvers/small_n_trick.py +22 -0
- guldo_cli-0.1.0/guldo/_src/solvers/starter_template.py +21 -0
- guldo_cli-0.1.0/guldo/_src/solvers/stdout_flood.py +9 -0
- guldo_cli-0.1.0/guldo/_src/solvers/tmp_fill.py +10 -0
- guldo_cli-0.1.0/guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo_cli-0.1.0/guldo/_src/static_check.py +214 -0
- guldo_cli-0.1.0/guldo/_src/submit_check.py +166 -0
- guldo_cli-0.1.0/guldo/_src/svp.py +889 -0
- guldo_cli-0.1.0/guldo/_src/tensorops.py +304 -0
- guldo_cli-0.1.0/guldo/_src/tflops.py +453 -0
- guldo_cli-0.1.0/guldo/_src/tflops_runner.py +134 -0
- guldo_cli-0.1.0/guldo/cli.py +60 -0
- guldo_cli-0.1.0/guldo_cli.egg-info/PKG-INFO +30 -0
- guldo_cli-0.1.0/guldo_cli.egg-info/SOURCES.txt +138 -0
- guldo_cli-0.1.0/guldo_cli.egg-info/dependency_links.txt +1 -0
- guldo_cli-0.1.0/guldo_cli.egg-info/entry_points.txt +2 -0
- guldo_cli-0.1.0/guldo_cli.egg-info/requires.txt +7 -0
- guldo_cli-0.1.0/guldo_cli.egg-info/top_level.txt +1 -0
- guldo_cli-0.1.0/pyproject.toml +24 -0
- guldo_cli-0.1.0/setup.cfg +4 -0
guldo_cli-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: guldo-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Command-line client and tools for verifiable speedup claims: commit-reveal rounds, certificates anyone can recompute, exact step counting.
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: numpy
|
|
8
|
+
Provides-Extra: attest
|
|
9
|
+
Requires-Dist: cryptography; extra == "attest"
|
|
10
|
+
Provides-Extra: lattice
|
|
11
|
+
Requires-Dist: fpylll; extra == "lattice"
|
|
12
|
+
|
|
13
|
+
# guldo
|
|
14
|
+
|
|
15
|
+
Command-line client and tools for verifiable speedup claims. One command, `guldo`.
|
|
16
|
+
|
|
17
|
+
pipx install ./guldo # or, in a virtual environment: pip install ./guldo
|
|
18
|
+
guldo --help # the client: practice, submit, status, verify, certificate, rounds, receipts
|
|
19
|
+
guldo tools # the other tools: claims, counted, attest, cert, compile, ops, lint, judge
|
|
20
|
+
|
|
21
|
+
guldo lint my_solver.py --family factor
|
|
22
|
+
guldo practice my_solver.py --family factor --count --dims 38 40 42 # prints the plan; add --yes to run it (exact step counting, Python 3.12+)
|
|
23
|
+
guldo verify my.cert.json --rerun all --execute # re-execute everything: runs the certificate's solvers, use a throwaway VM
|
|
24
|
+
guldo claims correctness my.cert.json
|
|
25
|
+
|
|
26
|
+
Needs Python 3.9+ (step counting needs 3.12+) and numpy. `pip install "guldo[attest]"` adds signatures; `[lattice]` adds fpylll for the lattice problems.
|
|
27
|
+
Install with pipx or in a virtual environment: some commands write under the package directory.
|
|
28
|
+
|
|
29
|
+
Nothing in this package is official, and parts of it (step counting, counted rounds, claims) pass their own tests but have not been used in a real round: read
|
|
30
|
+
`COMPILER.md` and `SUBMIT.md` in the package (`guldo/_src/`) for what each result does and does not show.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# guldo
|
|
2
|
+
|
|
3
|
+
Command-line client and tools for verifiable speedup claims. One command, `guldo`.
|
|
4
|
+
|
|
5
|
+
pipx install ./guldo # or, in a virtual environment: pip install ./guldo
|
|
6
|
+
guldo --help # the client: practice, submit, status, verify, certificate, rounds, receipts
|
|
7
|
+
guldo tools # the other tools: claims, counted, attest, cert, compile, ops, lint, judge
|
|
8
|
+
|
|
9
|
+
guldo lint my_solver.py --family factor
|
|
10
|
+
guldo practice my_solver.py --family factor --count --dims 38 40 42 # prints the plan; add --yes to run it (exact step counting, Python 3.12+)
|
|
11
|
+
guldo verify my.cert.json --rerun all --execute # re-execute everything: runs the certificate's solvers, use a throwaway VM
|
|
12
|
+
guldo claims correctness my.cert.json
|
|
13
|
+
|
|
14
|
+
Needs Python 3.9+ (step counting needs 3.12+) and numpy. `pip install "guldo[attest]"` adds signatures; `[lattice]` adds fpylll for the lattice problems.
|
|
15
|
+
Install with pipx or in a virtual environment: some commands write under the package directory.
|
|
16
|
+
|
|
17
|
+
Nothing in this package is official, and parts of it (step counting, counted rounds, claims) pass their own tests but have not been used in a real round: read
|
|
18
|
+
`COMPILER.md` and `SUBMIT.md` in the package (`guldo/_src/`) for what each result does and does not show.
|
|
File without changes
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
# The challenge compiler
|
|
2
|
+
|
|
3
|
+
You declare a problem. The compiler turns the declaration into a verified, calibrated challenge: an instance family with a size ladder, a CPU
|
|
4
|
+
cap and an instance count chosen from measurements, a power analysis saying which speedups it can and cannot certify, a self-tested pipeline,
|
|
5
|
+
a server plan, and, afterwards, a certificate anyone can check.
|
|
6
|
+
|
|
7
|
+
Everything downstream of the declaration is the protocol that already exists (commit-reveal seeds, the difficulty-stratified lower bound,
|
|
8
|
+
the hash-chained log, the sandbox). The compiler writes no new trust rules; it automates the part that used to take a week of hand
|
|
9
|
+
calibration per domain (PROTOCOL.md, "Power at the official configuration"), and it refuses to emit a challenge it cannot validate.
|
|
10
|
+
|
|
11
|
+
## Quick start
|
|
12
|
+
|
|
13
|
+
python3 compile_domain.py init mything # a WORKING example in domains/mything/ (it already passes check); never overwrites
|
|
14
|
+
# ... edit the lines marked EDIT: derive(), verify(), DOC, SIZE_RANGE, and the reference solver ...
|
|
15
|
+
python3 compile_domain.py check mything # seconds: what is wrong, and how to fix it (shape, determinism, anti-cheat, reference)
|
|
16
|
+
python3 compile_domain.py compile mything --quick # 1-2 minute DRAFT ladder, for iterating (practice only)
|
|
17
|
+
python3 compile_domain.py compile mything # the real compile + self-test: tens of minutes to hours; prints READY / NOT READY
|
|
18
|
+
python3 compile_domain.py show mything # that verdict again, with the exact next commands
|
|
19
|
+
python3 compile_domain.py recheck mything # seconds: recompute every checkable claim in the compiled record and compare (see "Checking a compile")
|
|
20
|
+
python3 compile_domain.py status # every declaration: registered, self-tested, or stale (and why)
|
|
21
|
+
python3 submit_check.py my_solver.py --family mything
|
|
22
|
+
|
|
23
|
+
Every command takes the NAME or a path (`domains/mything`). `compile_domain.py --help` prints this workflow. `init --blank` gives an empty skeleton instead of the working example.
|
|
24
|
+
|
|
25
|
+
A compiled domain is a family like `gm`, `mlwe` and `factor`: `judge.py`, `official.py`, `ops.py`, the portal and the site all accept
|
|
26
|
+
`--family mything`. `domains/dlog/` is a worked example the compiler was not written for (discrete logarithm by exhaustive search).
|
|
27
|
+
|
|
28
|
+
## The declaration (`domains/<name>/decl.py`)
|
|
29
|
+
|
|
30
|
+
| name | meaning |
|
|
31
|
+
|---|---|
|
|
32
|
+
| `NAME` | 2-16 lowercase letters, equal to the directory name (ladder files are recognised by that pattern) |
|
|
33
|
+
| `VERSION` | any string; bump it whenever `derive()` changes (it is recorded in every results file and refuses to compare across versions) |
|
|
34
|
+
| `SIZE_RANGE`, `SIZE_STEP` | the sizes the compiler may use; it finds the ladder inside this range |
|
|
35
|
+
| `REFERENCE_SOLVER` | a file inside the directory with a top-level `solve`; every submission is measured against it |
|
|
36
|
+
| `derive(master, size, idx)` | a pure function of its arguments returning `{"args": [...JSON...], "planted": <a valid answer>}` |
|
|
37
|
+
| `verify(inst, ans)` | exact check; must reject everything that is not a real answer |
|
|
38
|
+
| `witness(inst)` | optional: the answer to feed `verify` when `planted` is not itself an answer |
|
|
39
|
+
| `DOC`, `SIGNATURE` | optional text for the Compete page and the challenge document |
|
|
40
|
+
| `REFERENCE_NOTE` | one or two sentences: what the reference algorithm is and what is known to beat it. Strongly advised: `check` warns without it, and the challenge document, the domain card and the compiled record say "not stated" |
|
|
41
|
+
|
|
42
|
+
The solver is called as `solve(*args)` and returns an int, a list of ints, or `None` (the runner's contract, unchanged).
|
|
43
|
+
|
|
44
|
+
`decl.py` is **operator code**. It runs in the judge's process exactly like `instances.py` does. Submissions never see it: they run in the
|
|
45
|
+
sandbox with only `runner.py` and their own file. Do not compile a declaration you did not read.
|
|
46
|
+
|
|
47
|
+
## The passes
|
|
48
|
+
|
|
49
|
+
1. **parse.** The declaration has the required names; `NAME` is not a built-in; the reference lints clean (an import the sandbox image lacks is a FAIL, because the baseline is measured there) and imports none of the judge's modules.
|
|
50
|
+
2. **validate.** Each is a PASS / WARN / FAIL with the reason, and any FAIL stops the compile and writes nothing:
|
|
51
|
+
* *shape*: `args` is a JSON-serializable list and under 1 MiB (it crosses a process boundary);
|
|
52
|
+
* *determinism*: the same `(master, size, idx)` gives the same instance; different indices and different seeds give different ones (an instance that ignores the seed would make the commitment commit to nothing);
|
|
53
|
+
* *process purity*: the same instances come out of fresh interpreters under two different `PYTHONHASHSEED`s (catches hash-order, clock and global-`random` dependence, which would make another machine derive different instances);
|
|
54
|
+
* *witness*: `verify` accepts the planted witness at three sizes, and the witness (or a large part of it) is not in the arguments;
|
|
55
|
+
* *junk rejected*: `verify` rejects 19 junk answers (`None`, `0`, `1`, `-1`, strings, empty lists, ...). A verifier that accepts `1` has an answer space a solver can guess;
|
|
56
|
+
* *no trivial answer*: no argument, neighbour of an argument, or the argument list verifies, so the answer is not read off the question;
|
|
57
|
+
* *witness reuse*: one instance's witness is not accepted by other instances (otherwise a lookup table of past answers solves the challenge, the "egg" problem in EGG.md);
|
|
58
|
+
* *reference solves*: the reference solves the smallest instances with an answer `verify` accepts, deterministically.
|
|
59
|
+
3. **profile.** The reference is run, locally, on a **public** profiling seed (never a competition seed) at growing sizes until its cost passes a third of the probe budget: a calibration profile of median cost, per-instance spread (sd of log2 cost) and timeout rate.
|
|
60
|
+
4. **optimize.** From the profile: the CPU cap (30, 60, 120 or 240 s, the smallest that gives a usable ladder), and 3-6 sizes whose predicted median costs run from 2 s up to a point where about 10% of instances time out, spanning at least three octaves (the estimator's difficulty bins are octaves of baseline cost above 0.5 s). The chosen sizes are then **measured** (8 instances each) and the ladder is re-chosen if a prediction was off by more than an octave.
|
|
61
|
+
5. **power.** The real estimator (`extrapolate.measured_by_difficulty`, through `power.synth_trial`) on synthetic ladders drawn from the profile, with a synthetic submission that is exactly `e` bits faster. The compiler picks the smallest instances-per-size `n` (12, 24, 48, 96 or 192) for which all three seeds are credited at >= 1 bit with probability >= 80% at a 2-bit true speedup, then tabulates power by speedup. Output: the range of speedups the challenge can certify and the zero-effect false-credit rate.
|
|
62
|
+
6. **emit.** `compiled.json`, plus `out/CHALLENGE.md` (local kit), `out/server_plan.json` (the `ops.py` commands and CPU-hours), `out/ci_check.sh` and a CI workflow template, `out/CERTIFICATE.md`.
|
|
63
|
+
7. **self-test.** The compiled domain is run through the real pipeline (`judge.py evaluate` -> harness -> `extrapolate.py`) on a throwaway seed in local mode, with three controls:
|
|
64
|
+
* *zero effect*: the reference submitted against itself must earn < 1 bit;
|
|
65
|
+
* *failure*: a solver that never answers must earn 0;
|
|
66
|
+
* *known effect*: the same ladder with every solved cost scaled by 2^-1 must be recovered (a credit between 0.4 and 1.3 bits).
|
|
67
|
+
If any control fails the domain's record says so, and `rounds.create_round` refuses an official round for it. The control ladders are kept in `results/` (named `control_zero` and `control_none`, so do not give a real submission those names: the site's Results page leaves them out on purpose, as they are controls, not results).
|
|
68
|
+
|
|
69
|
+
### What registers a domain, and what un-registers it
|
|
70
|
+
|
|
71
|
+
`families.py` calls `declared.load_all()` at import, so every process sees the same registry. A domain registers only if `compiled.json` is
|
|
72
|
+
fresh: it records the SHA-256 of `decl.py` and of the reference solver, and a mismatch (any edit to either) un-registers the domain until it is
|
|
73
|
+
compiled again. `compile_domain.py status` lists the reason. A round's rules fingerprint (log entry 0) includes the hashes of `decl.py`,
|
|
74
|
+
`compiled.json` and the reference, so the declaration cannot change under a running round without the evaluator refusing, and a certificate
|
|
75
|
+
can show which declaration its numbers came from.
|
|
76
|
+
|
|
77
|
+
## Submitting and verifying from the command line
|
|
78
|
+
|
|
79
|
+
`svp.py` (guide: SUBMIT.md) is the user-facing path: `submit` a solver to a round, `status`, and `verify --round R --id I`, which downloads the
|
|
80
|
+
certificate described below and checks it against the server's own log and your own receipt. `svp.py practice SOLVER --family F` certifies a
|
|
81
|
+
local run on your own seeds, no server needed. `ops.py certs` builds the certificates after the reveal.
|
|
82
|
+
|
|
83
|
+
## Provable costs (`counted.py`, `count_runner.py`)
|
|
84
|
+
|
|
85
|
+
The one thing a certificate cannot prove by checking is that recorded timings are what a real run took. Step counting replaces the clock with a number
|
|
86
|
+
anyone can reproduce exactly: the count of Python bytecode instructions a solver executes (`sys.monitoring`, Python 3.12+; imports, the garbage collector and
|
|
87
|
+
the hash seed are controlled, so for a deterministic solver nothing about the machine enters the count; it still depends on the Python version and build, and a solver that
|
|
88
|
+
depends on memory addresses, the clock or randomness counts differently between runs, which the determinism self-check refuses to certify). `counted.py evaluate` writes ladders in the ordinary format (cost = count / 1e6, a unit
|
|
89
|
+
called a "counted second", plus the exact `steps` of every record), so the estimator, `certificate.py build` and the site read them unchanged. A counted
|
|
90
|
+
certificate carries `count` (unit, interpreter, its own size ladder), `verify` adds a **cost model** check (every cost follows from its step count; every
|
|
91
|
+
unsolved instance is charged the cap; every header agrees), and `--rerun K --execute` re-executes K instances of both solvers and demands **identical**
|
|
92
|
+
counts and statuses: exact equality, no tolerance, no trust in anyone's machine, sandbox or clock. On a different Python version the re-run is a SKIP. The counting program is pinned too: every counted ladder and certificate records the hash of `count_runner.py`
|
|
93
|
+
(it is not one of the protocol's hashed files), and the cost-model check fails if this checkout's differs, so counts from another counter are named as such and
|
|
94
|
+
a later fix to the counter cannot silently change what an old certificate means (it invalidates it: re-measure with the new one).
|
|
95
|
+
Limits, stated in the verdict: it is not a time; it cannot see work inside a C function (one instruction however long); the ladder is chosen by whoever ran
|
|
96
|
+
it, so it is not an official round result (unless a counted round declared it, below); and it needs Python 3.12+ and pure-Python solvers. It is not wired
|
|
97
|
+
into the compiler's calibrated challenges (those are in seconds). `svp.py practice --count` is the way to produce one of your own; see SUBMIT.md.
|
|
98
|
+
|
|
99
|
+
### Counted rounds
|
|
100
|
+
|
|
101
|
+
`ops.py round-new ... --count --count-dims 38 40 42 [--count-n 8 --count-cap 20]` creates a **counted round**: the sizes, the instances per size, the cap, the unit, the
|
|
102
|
+
Python version and the hash of the counting program are committed in log entry 0 (with `count_runner.py` and `counted.py` in the rules fingerprint) before any
|
|
103
|
+
submission, so the ladder is no longer the prover's: it is the round's, the same for every participant. Source publication is forced (the proof is re-execution),
|
|
104
|
+
and only pure-Python families are allowed (`counted.family_problem`: not `gm` or `mlwe`, whose baseline is fpylll). `ops.py worker --unsandboxed` evaluates it:
|
|
105
|
+
no Docker, the solvers run as the operator's user under step counting, so use a throwaway user or VM; the worker refuses another Python version or counter than the
|
|
106
|
+
round committed to (even with `--accept-code-change`). A solver that cheats by reading the operator's files behaves differently when anyone re-runs it, so the first
|
|
107
|
+
re-execution exposes it. The status of a counted result is `COUNTED` (3 committed seeds, commitments public first, one solver and one baseline file throughout) or
|
|
108
|
+
`COUNTED, PROVISIONAL` with the reasons; it is not judge.py's `OFFICIAL`, which means "timed in the Docker sandbox". The certificate's log check ties the counted
|
|
109
|
+
ladder to entry 0 (a counted ladder in a round that did not declare it, or the reverse, fails), and the cost-model line then says "ROUND-DECLARED". Every record
|
|
110
|
+
must carry a step count a re-run can reproduce (a wall-clock timeout cannot, so a certificate containing one does not verify), and the unit must equal this
|
|
111
|
+
checkout's.
|
|
112
|
+
|
|
113
|
+
**The restricted subset (what a counted solver may be).** The solver runs in the SAME process as the counter, so unrestricted it could switch its own counting off
|
|
114
|
+
(`sys.monitoring.set_events(3, 0)`), walk up to the runner's stack frame that holds the counter (`sys._getframe`, `__traceback__`, `gi_frame`), forge a result line on the real
|
|
115
|
+
stdout and `_exit`, or read files: and a re-run would reproduce all of it identically, so the "proof" would pass. `count_runner.py` therefore enforces a restricted subset
|
|
116
|
+
itself, before and while the solver runs (`check_source`, the same check `rounds.submit`, `practice --count` and `svp.py submit` make early):
|
|
117
|
+
1. an AST whitelist: no dunder or underscore names or attributes, no frame/generator/traceback/code attributes (`f_`, `gi_`, `cr_`, `tb_`, `co_`), no `str.format`
|
|
118
|
+
(`"{0.__class__}"`), no `eval/exec/compile/open/getattr/setattr/vars/globals/type/object/super/id`, no async code, no relative imports;
|
|
119
|
+
2. a minimal `__builtins__` (arithmetic, containers, `print`, the common exceptions) and a guarded `__import__`;
|
|
120
|
+
3. imports only from `math`, `itertools`, `heapq`, `bisect`, `functools`, `collections`, `fractions`, and each import yields a PROXY exposing only named functions
|
|
121
|
+
(a real module would leak others: `fractions.sys`, `collections.abc.sys`);
|
|
122
|
+
4. `sys.monitoring` is removed before the solver loads, a final check fails a run whose counting was reconfigured, and, on Linux only (on macOS the limit would trip over the process's own address space), the process has a 2 GiB address-space limit;
|
|
123
|
+
5. the module body is counted as well as `solve()` (otherwise precomputation at import time, such as a lookup table, would be free, and a solver could spin there for ever).
|
|
124
|
+
The rules live in `count_runner.py`, so the counter hash pins them: a different rule set is a different counter. The baseline must be inside the subset too
|
|
125
|
+
(`counted.family_problem`). A pure-arithmetic solver needs nothing outside it (`solvers/factor_trial_py.py` and `factor_trial_fast_py.py` are inside).
|
|
126
|
+
What it is NOT: a defence against bugs in CPython itself, against memory or time spent inside ONE C call (`10**10**10`: the instruction limit cannot interrupt it, the wall-clock
|
|
127
|
+
backstop does, and that submission then fails to verify), or a replacement for running the worker as an unprivileged throwaway user (it refuses root). Source stays mandatory
|
|
128
|
+
for counted rounds. Counting a round costs about 100x a timed run per instance: size the ladder small (the worker prints a worst case).
|
|
129
|
+
|
|
130
|
+
## Derived claims (`claims.py`)
|
|
131
|
+
|
|
132
|
+
The machinery proves one atomic statement: program P, on inputs committed before P was seen, produced these outputs at this cost. A speedup is one reading of it.
|
|
133
|
+
`claims.py` reads three others off a certificate that **verifies**, recomputing each from the raw per-instance records (a claim file is not trusted either:
|
|
134
|
+
`claims.py check CLAIM CERT...` redoes it and demands identical output):
|
|
135
|
+
|
|
136
|
+
claims.py correctness CERT [--out F] a benchmark score: solved/total on the committed questions (per size, per seed, Wilson 95% lower bound), for the solver AND the baseline
|
|
137
|
+
claims.py worst-case CERT [--out F] COUNTED certificates only: the largest step count seen per size, and the chance a fresh instance exceeds it (<= 1/(n+1))
|
|
138
|
+
claims.py negative CERT... --threshold BITS [--out F] "no submission in this round reached BITS": needs a certificate for EVERY submission the round's log accepted
|
|
139
|
+
|
|
140
|
+
What each does NOT show, printed with every claim: a correctness score is a rate on THIS test set within the cap (with timed costs "within the cap" depends on the
|
|
141
|
+
measuring machine; a counted certificate makes it reproducible by anyone); the worst case is the largest value SEEN, not a bound over every input; a negative result is
|
|
142
|
+
about the submissions made (participants chose to submit) and about what could be certified (credit is a conservative lower bound: the minimum over seeds, an
|
|
143
|
+
inconclusive seed counts as 0), so it says "no speedup of that size was shown", not "none exists". A submission whose evaluation failed has no certificate: it is listed as
|
|
144
|
+
missing and the negative claim is then incomplete and does not hold. The log must also contain the round's `round_closed` entry with no submission after it (a PREFIX of a
|
|
145
|
+
hash-chained log still verifies, so certificates built from a truncated copy would otherwise hide later submissions), and the claim records `log_head`: compare it with a
|
|
146
|
+
head hash published somewhere the operator cannot edit. The test set is the family's generator under committed seeds, so this is a benchmark of those
|
|
147
|
+
problem families (factoring, dlog, ...), not of arbitrary data; a new benchmark is a declared domain (`compile_domain.py`). Tests: `test_claims.py` (unrun).
|
|
148
|
+
|
|
149
|
+
## Certificates (`certificate.py`)
|
|
150
|
+
|
|
151
|
+
python3 certificate.py build --family F --submission solvers/x.py --seed-dir S --reveal-seeds \
|
|
152
|
+
--round-data D --round-id R --out x.cert.json
|
|
153
|
+
python3 certificate.py verify x.cert.json [--rerun 5]
|
|
154
|
+
|
|
155
|
+
A certificate carries the claim; the rules fingerprint; the commitments and (with `--reveal-seeds`, which is the reveal) the seeds; both
|
|
156
|
+
solvers' source; every raw per-instance record of every ladder; for a declared domain its three files; and the round's log. `verify`, on any
|
|
157
|
+
machine with a checkout of this protocol, checks and reports each as PASS / FAIL / SKIP:
|
|
158
|
+
|
|
159
|
+
| check | what it proves |
|
|
160
|
+
|---|---|
|
|
161
|
+
| rules | this checkout's estimator, harness and instance code hash exactly as the certificate names |
|
|
162
|
+
| domain | the declaration, compiled ladder and reference are the hashed files, load, and are fresh |
|
|
163
|
+
| log | the hash chain verifies; entry 0 holds the commitments and the same rules fingerprint (including the domain files); the submission's hash was logged |
|
|
164
|
+
| seeds | each revealed seed hashes to a commitment |
|
|
165
|
+
| sources | the solver sources are what every ladder header's `solver_sha256` names |
|
|
166
|
+
| ladders | one baseline/submission pair per committed seed (a dropped seed would raise the minimum), each on exactly the rules' size ladder with equal instance counts per size (a dropped hard size would otherwise hide failures) |
|
|
167
|
+
| coverage | re-deriving every instance from the seeds gives exactly the ids in each ladder: none missing, none repeated, none foreign |
|
|
168
|
+
| credit | the credit recomputed from the raw records, with this checkout's estimator, equals the claim exactly |
|
|
169
|
+
| rerun | (optional) K instances re-measured on the verifier's own machine: both solvers still solve them and the speedup reproduces within 2 bits |
|
|
170
|
+
|
|
171
|
+
**A certificate is untrusted input, so `verify` executes nothing from it by default.** A declared domain's `decl.py` is hash-checked and read as
|
|
172
|
+
data (its coverage check is then SKIPPED), and `--rerun` is refused. `--execute` imports the certificate's `decl.py` and, with `--rerun`, runs its
|
|
173
|
+
solver sources on your machine, unsandboxed: use it only in a throwaway VM or container. Built-in families derive instances with your own checkout's
|
|
174
|
+
code and need no flag for coverage.
|
|
175
|
+
|
|
176
|
+
`verify --rerun K` picks the same K instances every time unless you give `--sample-seed`; an independent verifier should choose its own seed so that
|
|
177
|
+
the operator cannot know which instances will be re-measured. `--tolerance BITS` sets how far the re-measured speedup may differ (default 2 bits:
|
|
178
|
+
loose on purpose, because the real machine-to-machine spread has not been measured; see "Checking a compile" for how to measure it).
|
|
179
|
+
|
|
180
|
+
Without a log a certificate can still verify but is at most PROVISIONAL (the publication time of the commitments cannot be shown); a ladder
|
|
181
|
+
measured in local mode is PROVISIONAL regardless (only the Docker sandbox meters untrusted code). `verify` says which.
|
|
182
|
+
|
|
183
|
+
## Checking a compile (`recheck`)
|
|
184
|
+
|
|
185
|
+
python3 compile_domain.py recheck dlog # about 5 seconds
|
|
186
|
+
python3 compile_domain.py recheck dlog --measure 3 # also re-measure each ladder size with 3 instances on this machine
|
|
187
|
+
|
|
188
|
+
`recheck` recomputes what is arithmetic and compares it with the record: the declaration and reference hash as recorded; the validation checks
|
|
189
|
+
give the recorded verdicts; the ladder, `n`, CPU cap and CPU-hours follow from the recorded plan; and the **whole power table** (seeded, so it
|
|
190
|
+
must come out identical) is recomputed through the real estimator. Any difference is a FAIL. The arithmetic needs no trust beyond the protocol code you checked out. **But `recheck` is not safe on a stranger's domain:** like `compile`, it executes the declaration's `decl.py` (validation derives instances with it) and the reference solver, unsandboxed, and `--measure` runs the reference many more times. Read a declaration before rechecking it, or do it in a throwaway VM.
|
|
191
|
+
|
|
192
|
+
What it does not recompute: the profile's timings and the self-test, which are measurements. `--measure K` re-measures the ladder sizes here on
|
|
193
|
+
the same public profiling instances (instance `i` is a function of the public profiling seed) and prints the gap in bits per size. A compile
|
|
194
|
+
now records every instance's cost, so the gap compares the SAME K instances and is the machine, not which instances were drawn. A record made
|
|
195
|
+
before that (dlog's, today) keeps only a median over all its instances, so the comparison includes sampling noise: with a per-instance spread
|
|
196
|
+
of about 1.7 bits and K=3, noise alone gives a typical gap of 0.6 bit and one above 1.8 bits one time in ten. The output then says so and
|
|
197
|
+
widens its warning threshold, and `spread` flags the recommendation as inflated. That does not prove the recorded timings; it tells you how far this
|
|
198
|
+
machine is from the one that made them. A gap that is the same at every size is a hardware speed difference and cancels in a speedup ratio; a
|
|
199
|
+
gap that changes with size is not. Two or more machines' `--measure` output is the data needed to set `--tolerance` honestly. None has been
|
|
200
|
+
collected yet. To collect it: on each machine run `compile_domain.py recheck NAME --measure 3 --save NAME.<machine>.json` (never overwrites; it
|
|
201
|
+
stores a hash of the host name, not the name), then merge them anywhere with `compile_domain.py spread *.json`. `spread` recommends a tolerance
|
|
202
|
+
(the largest within-machine range of gaps across sizes, rounded up to a quarter bit, never below 1 bit because the re-measurement compares single instances whose own noise is about 0.6 bit and `spread` does not measure it) only when at least two distinct machines measured the same
|
|
203
|
+
declaration; with fewer it says so and the default stays loose. The recommendation is an estimate from a few sizes, not a bound.
|
|
204
|
+
|
|
205
|
+
## Attestations (`attest.py`)
|
|
206
|
+
|
|
207
|
+
python3 attest.py keygen --out my.key # a key you keep; publish the public key it prints
|
|
208
|
+
python3 attest.py sign x.cert.json --key my.key --out x.me.att.json --rerun 5 --execute # verify on YOUR machine, sign the outcome
|
|
209
|
+
python3 attest.py check x.cert.json a1.json a2.json --trusted keys.txt
|
|
210
|
+
|
|
211
|
+
A verifier runs `certificate.py verify` (and optionally `--rerun K`, on instances drawn from the verifier's own random seed) and signs the
|
|
212
|
+
result with an Ed25519 key: the outcome of every check, the re-measured gap, a hash of the certificate, and a self-reported machine description.
|
|
213
|
+
`check` verifies the signatures and counts distinct keys.
|
|
214
|
+
|
|
215
|
+
**What an attestation is worth is what its signer's identity is worth.** A key proves the same party signed; it does not prove independence
|
|
216
|
+
from the operator, because an operator can create five keys. So `check` reports a key as independent only if it is listed in the `--trusted`
|
|
217
|
+
file you built yourself from keys you obtained out of band, and every other key is reported as "unvouched", never counted. A signed statement
|
|
218
|
+
that the certificate does NOT verify (a dissent) makes `check` fail, from any key. A valid signature proves only who signed, so the statement's
|
|
219
|
+
contents are still untrusted input: `check` accepts exactly the shape `sign` writes (a real boolean `verified`, short plain-text check names, a
|
|
220
|
+
finite gap) and ignores anything else with a warning, so a hostile attestation can neither crash a caller nor count. The same goes for the
|
|
221
|
+
measurement files `spread` merges, and for certificates on the site: whatever they make the verifier raise costs only that file. One key signing twice is one party. Needs the `cryptography`
|
|
222
|
+
package, which the project's own venv does NOT have (install it only where you sign or check attestations). Without it, `sign` and `keygen` refuse with a message, `check` and the site report the attestation files as "NOT checked" and count none of them (a dissent among them would not be seen), and the signature tests in `make test` are skipped, not passed. Attestations do not touch the certificate, so adding one never changes its hash.
|
|
223
|
+
|
|
224
|
+
To show them on the site, put `NAME.cert.json`, `NAME.<who>.att.json` and (optionally) `trusted.txt` in one directory and build with
|
|
225
|
+
`python3 build_site.py --certs DIR` (keep DIR outside the site's output directory: the build deletes and recreates that directory, and refuses to run if an input lives inside it). Each certificate is re-verified at build time (nothing in it executed), shown with its checks, and its
|
|
226
|
+
attestations are counted as pinned or unvouched exactly as `attest.py check` does. `make attest CERT=... KEY=... OUT=...` is the one-command
|
|
227
|
+
version for an outside verifier.
|
|
228
|
+
|
|
229
|
+
## Exporting a challenge bundle (`export_bundle.py`)
|
|
230
|
+
|
|
231
|
+
python3 compile_domain.py export dlog [--out DIR] [--field "Computer Science"] (or `python3 export_bundle.py export dlog`, `guldo export dlog`)
|
|
232
|
+
python3 export_bundle.py verify DIR --selftest
|
|
233
|
+
|
|
234
|
+
The compiler's output so far runs inside this project. `export` writes the part that ports: a directory any leaderboard can host without a copy of this
|
|
235
|
+
repository. It refuses a stale compile (same freshness rule as `emit`), never overwrites, and refuses to emit a bundle that fails its OWN self-test run
|
|
236
|
+
in a clean interpreter (`python3 -I`, empty working directory), so "works only inside this checkout" cannot ship.
|
|
237
|
+
|
|
238
|
+
| path | what |
|
|
239
|
+
|---|---|
|
|
240
|
+
| `challenge.json` | manifest: problem, solver contract, ladder, protocol, power analysis, provenance, requirements, sha256 of every pinned file. `status` is `READY` or `DRAFT` with the reasons (compiler self-test not passed, no certifiable range, no `REFERENCE_NOTE`, ladder unconfirmed) |
|
|
241
|
+
| `checker/decl.py`, `reference.py`, `vendor/` | the declaration and baseline byte-for-byte, plus any project module the declaration imports (`instances.py` for dlog), found by walking its imports |
|
|
242
|
+
| `checker/check.py` | standard library only: `selftest` (pinned files intact, deterministic, witness verifies, 19 junk answers rejected, reference solves), `commit`, `derive`, `verify` |
|
|
243
|
+
| `checker/score.py` | needs numpy: `score` (credit vs the reference, min over seeds, inconclusive = 0), `compare` (is solver A really faster than B?), `power` |
|
|
244
|
+
| `checker/estimator.py` | byte copy of `extrapolate.py`: the scorer's numbers ARE the protocol's (a test pins them equal on the same files; on the saved dlog stride-4 ladders it reproduces the earlier claims 1.67 / 1.61 / 1.54 bits) |
|
|
245
|
+
| `README.md`, `PROPOSAL.md` | how a board hosts it (seeds, meter, scorer run) and its limits; and the text a proposal form like Yukon's asks for (problem, verifier, link), pre-filled from the record |
|
|
246
|
+
| `MANIFEST.sha256` | `sha256sum -c`-compatible; `export_bundle.py verify` also checks the pins in `challenge.json` |
|
|
247
|
+
|
|
248
|
+
What the scorer adds beyond the estimator: it takes ladders in a plain JSON-lines shape (`{"id", "dim", "status", "cost"}`, so a board's own meter can write them), and
|
|
249
|
+
applies the certificate's input rules to them: the two ladders must cover exactly the same instances (a truncated run is refused, not scored on the easy half), share a seed
|
|
250
|
+
commitment (a `{"meta": true, "commit": ..., "budget": ...}` first line; without it the ladder is refused unless `--lenient`), share a CPU cap, sit on exactly the challenge's size ladder with ids
|
|
251
|
+
exactly `SIZE:0`..`SIZE:n-1` per size (a relabelled or duplicated-easy ladder with the right count is refused), and not reuse a seed. `--lenient` scores anyway and labels the result
|
|
252
|
+
PROVISIONAL. Malformed records are refused with the line number: a solved cost must be finite and > 0 and not above the cap; a status outside solved/timeout/crash/failed/wrong
|
|
253
|
+
is refused rather than silently charged.
|
|
254
|
+
|
|
255
|
+
The commands, in short: `check.py` = `info`, `selftest`, `commit`, `derive`, `verify`, `rerun`, `spread`, `keygen`, `attest-check`, `log append|verify|checkpoint`; `score.py` = `score`, `compare`, `power`, `recheck`. `selftest`, `derive` and `verify` run the bundle's own code, so they run inside the OS sandbox by default (`--sandbox off` to opt out).
|
|
256
|
+
|
|
257
|
+
`compare` answers the question a leaderboard cannot answer from ranks alone. It runs the estimator in both directions on the two solvers' own ladders (the slower one plays the
|
|
258
|
+
baseline, so difficulty bins follow ITS cost) and says "A is faster" only when the minimum-over-seeds lower bound reaches `--min-gap` (0.5 bit, a convention, not a
|
|
259
|
+
pre-registered test, but the power at that threshold is simulated and printed with every verdict). It distinguishes two reasons for no verdict. `CANNOT SEPARATE` means the gap is inside the noise. `UNRESOLVABLE BY THIS LADDER` means one solver costs under the 0.5 s
|
|
260
|
+
floor on most instances the other finds hard, so the estimator discards exactly the evidence (the ceiling in "What this does not do"); it prints the raw median gap as description, never as a
|
|
261
|
+
bound. On the real dlog ladders, baby-step/giant-step against stride-4 is the second kind (raw median about 11.5 bits, no certifiable bound), and stride-4 against the reference
|
|
262
|
+
is "A is faster, >= 1.54 bits".
|
|
263
|
+
|
|
264
|
+
Limits, in addition to the ones above and in the bundle's own README:
|
|
265
|
+
* **Speed only.** One cost number per instance and an all-or-nothing verifier. A challenge scored by accuracy, size, or a trade-off cannot be declared; the compiler's cost is CPU seconds (any unit that plays the role of seconds against the 0.5 s floor works, for example scaled step counts).
|
|
266
|
+
* **`score.py` does not know how costs were measured; `check.py rerun` tests them.** `rerun --seed-file S --solver X.py --claimed X.jsonl --baseline REF.jsonl -k 5` re-runs a random sample (the verifier's own choice, printed) of the claimed instances on the verifier's machine, checks the answers, and compares re-measured with claimed speedup per instance; it also reports the machine factor and its spread, so a baseline ladder that was not measured on one machine shows up. On the real dlog stride-4 claims (seed `d00100a7`, 4 random instances, this Mac) the median claimed-vs-re-measured gap was 0.04 bit, machine factor +0.03 bit, spread 0.08. It samples (instances not drawn are not tested), cannot see a bias identical on every instance, is only as good as the hardware match, and its 2-bit default tolerance is loose because machine-to-machine spread has been measured only once (4 instances, one domain: median 0.01 bit, worst 0.03 bit between a Mac and a Linux server; see the Linux run below). `score.py --out` also records the sha256 of every input ladder and of the challenge, estimator and scorer, and `score.py recheck RESULT.json` recomputes and fails on any difference: tamper-evidence for a published number. A second sample on a different random seed (`audit2`, 3 instances, sandboxed, signed) gave 0.08 bit.
|
|
267
|
+
* **`rerun` runs the solver in an OS sandbox by default.** No network, no writes outside a scratch directory, and the seed file, signing key, `~/.ssh`, `~/.aws`, `~/.gnupg` and the keychain unreadable; the reference runs under the same wrapper so the overhead is comparable. macOS `sandbox-exec` is tested here against solvers that return a correct answer ONLY if they can read the seed, write outside scratch, or connect to a local socket (each is blocked; each succeeds with `--sandbox off`, so the tests are not vacuous). Linux `bwrap` was written blind, then run for real (see "The Linux run" below, which found a fifth bug, the vanishing CPU accounting). Reading it again, before it ever ran, had found four real bugs: `--tmpfs /tmp` hid every solver and bundle that lives under `/tmp` from the sandbox, the credential directories were not masked, there was no `--new-session` (the classic TIOCSTI terminal injection), and a `bwrap` that is installed but unusable (Ubuntu 24.04 restricts unprivileged user namespaces) was not detected. All four are fixed, and each is protected by a test that fails when it is reverted; the tests use a FAKE `bwrap` (a simulation of argv assembly, the verification probe and failure reporting, not of isolation). Every sandbox test used to be gated on macOS `sandbox-exec`, so on Linux they were skipped, never failed; they are now gated on "a sandbox actually works here" (Python is run under it once), so `make test` on a Linux box where `bwrap` works runs them for real, and the cheating solvers check file CONTENT (under bubblewrap a hidden file reads as empty rather than raising). On Linux the exec restriction is absent by design (network is unshared for every process instead). guldo's own Linux CI (GitHub Actions, per project notes) runs the suite and the Docker sandbox, which is a different mechanism; whether it has run this bundle code, and whether `bwrap` works on its runners, is not known to the author of this section (note that Ubuntu runners restrict unprivileged user namespaces, so install the distribution `bubblewrap` package and see the template). `BUNDLE_LINUX_CI.yml.template` is a ready job for it. Where no sandbox works, `rerun` refuses unless `--sandbox off`, and says why when the tool exists but cannot run. The profile also denies running any program except the interpreter's own installation (macOS framework Python re-executes itself through `Python.app`, so the whole base prefix is allowed): found by trying, because under the first profile a solver could still run `/bin/sh` and `open(1)`, and `open` hands a URL to LaunchServices, which is network egress through another process. Tried and blocked now: a hard link, a symlink, a `../` path and a copy of the seed, `cat` of it through a child, `curl`, `sh`, `open`, and a unix socket to `mDNSResponder`; numpy still runs. Mach lookups are denied too (except Apple's user-name lookup service): this was the one documented hole, "a solver that calls macOS services through `ctypes` is not stopped", and it was real: a harmless `bootstrap_look_up` from inside the sandbox reached LaunchServices, the pasteboard, the preferences daemon and the directory service. Denying all of them broke `pwd.getpwuid`/`getpass.getuser` (they go through the directory service), so exactly that one service is allowed back; numpy's BLAS/LAPACK, threads and the numpy-heavy `aiinfer` domain (33/33 self-test checks sandboxed) are unaffected, and a real dlog `rerun` still reproduces to 0.01 bit. Tests: three services are reachable unsandboxed and denied sandboxed, user-name lookups and linear algebra still work, and removing either profile line is caught. A kernel exploit, or any service not reached through a Mach lookup, is still outside what this stops. It is not a proof of isolation: CPU time, memory and the kernel are shared, `sandbox-exec` is deprecated by Apple, and a solver can still burn its CPU cap. Found by running the real dlog solver: a solver given as a relative path stopped resolving inside the sandboxed scratch directory (every test had used an absolute one); fixed, pinned by a test, and a crash now prints the solver's stderr.
|
|
268
|
+
* **Signed attestations and a hash-chained log.** `check.py keygen`, `rerun --sign KEY --attest-out F`, `attest-check F... --trusted KEYS --min N [--claimed L --baseline L]`: Ed25519 in pure Python (checked against the RFC 8032 vectors and against the `cryptography` library). A signature proves who signed, not independence from the operator, so only keys you pinned count as vouched (an operator can make five keys), one key twice is one party, an edited statement or a dissent turned into a pass no longer verifies, a valid signed dissent from ANY key fails the check, and an attestation for a different bundle or ladder is ignored. `check.py log append|verify|checkpoint` is an append-only hash chain (edit, removal, reordering caught). A truncated copy still verifies ALONE; `log checkpoint LOG --key K --out C` is a signed (head, count) that anyone can make and keep, and `log verify LOG --checkpoint C` fails for a log that no longer contains it, whether truncated or forked into a different but internally valid history (both tested). It only helps if someone other than the operator holds checkpoints and you pinned their key; the time is the logger's clock. Not a trusted clock, not trusted hardware, and guldo's own rounds and certificates remain the stronger protocol for adversarial settings.
|
|
269
|
+
* **Machine-to-machine spread now has a tool, and the within-machine part is measured.** `rerun --repeat N` re-runs each sampled instance and reports this machine's timing repeatability: on the real dlog claims (4 instances x 3 runs) the median sd of log2 cost was 0.009 bit for the reference and 0.005 for the solver, and the claimed-vs-re-measured gap on that sample was 0.02 bit, so the 2-bit default tolerance is two orders of magnitude above same-machine noise. What is still NOT measured is cross-machine spread: `rerun --sample-seed S --save F` on each machine (same seed everywhere; the machine is stored as a hash, never the host name) and `check.py spread F1 F2 ...` give a recommended tolerance (the worst observed range rounded up to a quarter bit, never below 1; an estimate, not a bound), refusing with fewer than 2 distinct machines or 3 shared instances rather than inventing one. A real two-machine measurement now exists (Mac vs a Linux server, see the Linux run bullet), but it is one domain and four instances: it supports, and does not set, a tighter tolerance.
|
|
270
|
+
* **Checked on other interpreters and from the installed package.** `check.py` (standard library only) passes its full self-test on Python 3.9, 3.12, 3.13 and 3.14, sandboxed. A scoring result made on 3.14 / numpy 2.3 rechecks identically on 3.9 / numpy 2.0 and on 3.12 / numpy 2.4 (the estimator's bootstrap is seeded, and `recheck` compares to 1e-9). The package was staged, installed into a clean virtual environment and used from there; that found that the documented `guldo export NAME` did not work (the packaged tool routes straight to `export_bundle.main`, which wanted `export NAME`). Fixed, and pinned by a test that runs the documented short form. Not run: an installed package on Linux, or Windows (`rerun` needs POSIX `resource`).
|
|
271
|
+
* **The 95% lower bound was measured, not assumed.** Through the real estimator (400 bootstrap resamples) on ladders drawn from dlog's compiled profile, 1500 trials per cell, the credit exceeded the TRUE speedup in 0.9% to 3.3% of trials across true effects 0 to 3 bits and per-instance noise 0.3, 0.6 and 1.5 bits (nominal limit 5%), and exceeded it by more than a quarter bit in at most 0.4%. The median credit sits 0.1 to 0.7 bit below the truth: the bound is valid and conservative, under the model that generated it (log-normal baseline costs, Gaussian pair noise). It says nothing about heavy-tailed real noise beyond what the real dlog ladders show.
|
|
272
|
+
* **The test suite was mutation-tested.** 43 deliberate bugs (an Ed25519 verifier that always accepts, a log that ignores broken links, each sandbox rule removed in turn, `max` instead of `min` over seeds, a scorer that accepts truncated ladders, an exporter that skips its own self-test, ...) were applied one at a time to a copy of the project and run against the tests meant to catch them. First run: 7 survived, i.e. real test gaps: nothing tested that the credit is the MINIMUM over seeds (every test used identical seeds), that a log edit with a recomputed hash is caught by the next entry's link, that a solver wrong on only SOME instances fails, that a bundle with an extra unlisted file is flagged, that export really runs its own self-test, that the every-part-damaged check is the one that fires, and the cross-seed check. The last one turned out to be a design flaw: it compared witnesses over all instance pairs, which diluted a seed-ignoring `derive()` to 6 of 36 pairs (below the 25% warning), so it could never fire where it mattered; it now compares instances at the SAME index. All 7 are fixed with tests that name what they pin, and all 43 mutations are killed. Mutation testing finds gaps in tests, not bugs in untested paths: the 43 are the paths I thought of.
|
|
273
|
+
* **The Linux run (11-21, one Hetzner server: Ubuntu 26.04.1, kernel 7.0.0, 2 vCPU, Python 3.14.4, bubblewrap 0.11.1 extracted from the distribution package into a temp directory, nothing installed system-wide; run only after the owner explicitly authorized it, because an earlier attempt was refused by the safety guard).** A kit (`~/Downloads/guldo-linux-check/`) did four things. (1) *The sandbox holds.* An attack probe run sandboxed AND unsandboxed (so a vacuous probe shows): the seed, `~/.ssh`, an outside write, a loopback socket and a hard link to the seed were all blocked, an in-scratch write worked, and running another program was allowed (by design on Linux). The bundle self-test passed 62/62 through bubblewrap; 63 unit tests (sandbox, meter, FIFOs, `flock`, attestations, checkpoints, data files) passed with bubblewrap in use, 3 skipped (macOS-only). (2) *The first run found a real bug that no amount of reading would have:* under `bwrap --unshare-pid` the kernel's CPU accounting for the program VANISHES (a 2 s burn read as 0.00 s with `--unshare-pid`, 2.03 s without it). Every `rerun` on Linux "measured" ~0 and still printed `RERUN PASS` (the claims were small enough to sit inside the 2-bit tolerance), and the "1.5 bit" spread it produced was garbage, which I discarded. Fix: `--unshare-pid --as-pid-1` (the program runs as the namespace's init, no reaper in between) keeps both the accounting (2.03 s) and the PID namespace; the CPU cap (kills at cap+1 s), the wall backstop and the orphan cleanup (a forked CPU-burner dies with the namespace) were verified on the real kernel. Added so this can never pass silently again: `rerun` refuses when an empty Python start measures under 2 ms, FAILS when the reference reads ~0 where its published cost is seconds, and a kill reported as exit 137 is a timeout, not a crash; each is protected by a test that fails when reverted. (3) *Unprivileged use needs the distribution's bubblewrap:* as the unprivileged user, the copy running from `/tmp` failed with `loopback: Failed RTM_NEWADDR: Operation not permitted` (Ubuntu's AppArmor user-namespace policy ships a `bwrap-userns-restrict` profile for `/usr/bin/bwrap`; mine was not covered). The tools diagnosed it exactly as designed (the reason in a warning, or a refusal when a sandbox is required). I did not install the package. (4) *The first real cross-machine measurement:* four published dlog claims re-measured on an Apple M4 Pro (macOS) and on the server (about 0.5 bit slower in raw speed) agree on the re-measured speedup to a median 0.01 bit, worst 0.03 bit (Mac +1.34/+1.35/+1.36/+1.35, Linux +1.37/+1.34/+1.36/+1.35; `spread` recommends its floor of 1.0 bit). Limits of that: 4 instances, one pure-Python domain, one solver family; BLAS-heavy work may differ, so the default tolerance stays at 2 bits. Side effects to know about: a timed rehearsal round (a `docker run` of `baseline_bkz` from `/srv/ledger/rehearsal`) was running on that server at the time; my probes used about 30 CPU-seconds at lowest priority alongside it, so judge whether that rehearsal's timings need repeating. Everything I placed on the server was deleted and its services were untouched.
|
|
274
|
+
* **CPU hiding against `rerun` (found by attacking the fix, not by reading it).** The meter guards from the Linux run only catch a meter that reads ~0. A solver can also HIDE its CPU: fork the real work into a child and return the answer through a pipe WITHOUT waiting for it. Linux credits a child's CPU to its parent only when the parent waits, so on the real server (no sandbox) 2.0 s of work measured as 0.02 s: the attack works there, and a fabricated speedup would reproduce. macOS credits CPU at the child's exit, so the same solver was measured honestly there (real dlog ladders, a fabricated 2048x claim: the hider's CPU was counted), which is why this had never shown up. Under `bwrap --as-pid-1` it is already closed (the kernel reaps everything in the namespace when its init exits, rolling the CPU up: measured 2.03 s), so the exposure is `--sandbox off` on Linux. The defense does not depend on the OS: `rerun` now compares each run's CPU with its wall clock and FAILS the claim when a run took over half a second of wall time with under 30% of it as CPU (first version said one second; the next audit measured 0.6 s and 0.9 s of hidden work passing it, and lowered the minimum to the estimator's own 0.5 s floor, below which costs cannot move a score) (a CPU-bound solver cannot do that honestly; one that waits, or whose work runs where the meter cannot see, can). Tests: a stand-in that waits is failed with a 50-bit tolerance (so only this check can fail it), an honest CPU-bound solver is not flagged, the one-second minimum and the 30% boundary are pinned, and each part of the logic is caught when disabled. SUPERSEDED BY THE RELATIVE RULE (next audit, "figure out what's not closed"): the fixed 30% share is now only the fallback. Each instance's solver run is judged against the reference's share on the SAME instance (flag below 60% of it, the share capped at 100% so a threaded reference does not raise the bar; used only when the reference ran over 0.5 s; the reference itself is flagged only under 15%). Verified on the real Linux server with the padding attack: reference 100% share; padders at 29%/34%/42% share faked 1.3/1.0/0.5 bit, and the fixed rule missed the last two, the relative rule flagged all; on a loaded machine both runs starve alike so an honest solver passes (a wired-in `rerun` test with a 28%-share reference and solver: PASS). What a solver can still hide: it must stay above 60% of the reference's share, so at most 1/0.6 = 1.67x = 0.74 bit. 7 mutations of the rule, all killed. The two limits below (false positives under load, the 1.7-bit padding bound) described the FIXED rule and are now closed down to that 0.74 bit, except that a load that hits the solver's run but not the reference's (a burst between the two runs) can still false-flag, which fails safe. Original re-audit, measured: (a) FALSE POSITIVES: an honest 1.5 s burner kept 97% CPU share idle, 78% with as many busy competitors as cores, but only 28% (flagged) with 3x oversubscription, so a heavily loaded machine fails honest solvers; the verdict is the safe side for a verifier and the failure message now prints the load average and says to re-run when idle. (b) BOUND on the evasion: a solver can pad its parent's CPU to just over 30% of wall and hide the rest, which fakes at most about 1/0.3 = 3.3x, i.e. 1.7 bits, inside the 2-bit default tolerance; so this check closes the large fakes (the 2048x one) but not a 1.7-bit one. Tighten with `--tolerance` below 1.7 if that matters, or keep the sandbox on (bwrap `--as-pid-1` counts everything). Limits: a solver that hides only PART of its CPU (say half) stays above 30% and is not caught; `--sandbox off` on Linux is for solvers you trust; a solver that waits for a legitimate reason (I/O) would be wrongly failed, which is the conservative side for a verifier.
|
|
275
|
+
* **Re-audit of the Linux changes.** (1) My own fix regressed performance: the "does the sandbox actually work" probe I added to every `selftest`/`derive`/`verify` re-exec raised the sandboxed call from 135 ms to 180 ms (the documented 75 ms overhead had become 120 ms). The probe is now only used where it runs once (`rerun`, and the tests' gating); the hot path runs the real command and diagnoses a sandbox TOOL that could not start by its own error line (`bwrap: ...`, `sandbox-exec: ...`), not by any non-zero exit, so a wrapped command's own failure is passed through and never re-run unsandboxed. Back to 58 ms unsandboxed, 135 ms sandboxed; both the "never diagnose" and the "diagnose every failure" mutations are caught. (2) The CI template I wrote would have shown green on failing tests: `... | tee log` reports `tee`'s exit status unless the shell is bash with pipefail (a step needs `shell: bash` explicitly). Fixed in `BUNDLE_LINUX_CI.yml.template`. (3) Seed and solver paths with spaces, accents and parentheses were never tested against a profile built from path strings; they work (a cheater is blocked and, unsandboxed, leaks, so the test is not vacuous), and a test pins it. My first attempt at that check was vacuous in my own script (a factoring cheater against the dlog domain crashed even unsandboxed): I caught it because the comparison arm did not leak. (4) The non-blocking open no longer assumes `os.O_NONBLOCK` exists.
|
|
276
|
+
* **Concurrency and hostile paths (the next round of auditing the audit).** (1) Eight simultaneous `log append`s wrote 4 entries, broke the hash chain and refused the other 4 submissions: read, verify and write were three steps with no lock. Appending now holds an exclusive `flock` from read to `fsync` (reads take a shared lock so they never see a half-written line); 24 concurrent appends all land with a valid chain, a 16-way test pins it, and removing the lock in a copy is caught. Needs POSIX; elsewhere an append refuses. (2) Every tool that read a file named on its command line hung for ever on a FIFO (score, recheck, derive, verify, log, attest-check, spread, log append, rerun) and read a file with no newlines whole (300 MB became 626 MB, so a symlink to `/dev/zero` would exhaust memory). All reads now go through one path: regular files only, opened non-blocking so a FIFO is refused rather than waited on, size-capped (1 MB seeds and key lists, 16 MB JSON, 32 MB ladders, 64 MB logs and answers) with the cap re-checked while reading in case the file grows, hashing streamed and regular-file-only. All 19 probe cases now refuse in about 0.1 s; a test covers a FIFO, a symlink to `/dev/zero`, a sparse 70 MB file and a directory across 13 invocations, and removing the regular-file check from either tool is caught (the size pre-check is deliberately redundant with the bounded read loop, so removing only it is not). Hashing a big regular file (`log append --file`) is intended and not capped. Also closed on the way: deeply nested JSON in an attestation file raised `RecursionError` (the file fuzz had never nested deeply there). My own slip while probing: I first ran the probe with `timeout`, which does not exist on macOS, so every case "exited 127" and looked clean; my notes already warned about exactly this. I caught it because 0 tracebacks across 9 hanging tools was implausible.
|
|
277
|
+
* **Numeric command-line options are bounded and fuzzed (found by auditing the audit).** The file fuzz corrupted files but never the options. Fuzzing 146 odd values (0, negative, NaN, inf, 1e400, 20-digit integers, text, empty) over every numeric option of both tools found 29 failures: `--boot`, `--trials` and `-k` at 0 or below crashed with `IndexError`/`ZeroDivisionError`/`ValueError`; `derive --size -1` hung and `--size 0` crashed inside the author's code; an invalid `--fresh-seed` crashed; huge `--boot`/`--trials` ran until killed; and `score --boot 6000` wrote a result that `recheck` then refused (the two caps disagreed after I tightened one). All options now have ranges (`--boot` 1..5000, `--trials` 1..2000 with trials x boot <= 200,000, `-k` 1..1000, `--repeat` 1..50, `--tolerance` 0..1000, `--min-gap` 0..100, `derive`/`verify` sizes inside the challenge's declared range), violations are one-line usage errors, and the whole 146-case sweep now takes 13 s instead of timing out. A test runs it and another pins that the largest accepted `--boot` still rechecks; reverting each of five fixes in a copy is caught. Measured cost of the default sandbox: `verify` 57 ms unsandboxed, 130 ms sandboxed.
|
|
278
|
+
* **Re-audit of those fixes.** The redesigned cross-seed check was measured for false failures over 6,000 (seed, size) cells on the toy domain (worst case 1 of 6 instances identical, limit is 3), 3,000 on dlog and 25 on aiinfer: none. The full export and fuzz suites passed six times concurrently under CPU load (no timing flakes). The docs' sandbox promises were each given a test that can fail: the five credential directories (`~/.ssh`, `~/.aws`, `~/.gnupg`, `~/.config/gcloud`, `Library/Keychains`; removing any one from the deny list is caught), `verify --selftest` refusing where there is no sandbox, and export refusing a declaration that writes a cache file next to itself (it compiles and probes fine because the author's directory is writable; the sandboxed self-test is what stops it). A mistake in my own write-up was caught too: I had documented 40 mutations; there are 43.
|
|
279
|
+
* **Limits that bound the work a file can demand, and reproducibility.** Scoring is linear in records x bootstrap resamples (measured: 50,000 records at 400 resamples took 48 s per pair; a one-million-line ladder ran for more than ten minutes), so a scored ladder may not exceed 20,000 lines and a result file may not request more than 5,000 resamples, which bounds the worst accepted case near two minutes (real ladders have about a thousand records and use 400 resamples). The first version of this cap, 50,000 records with 20,000 resamples, still allowed about 40 minutes: found by timing it. Exporting the same declaration twice produces byte-identical bundles (tested), so anyone can re-export and compare `MANIFEST.sha256` with a published bundle's. Paths with spaces and non-ASCII characters work through `score` and `recheck`.
|
|
280
|
+
* **Every input is fuzzed.** `test_fuzz_bundle.py` corrupts, at random, every input the tools read (ladders, result files, signed and unsigned attestations, checkpoints, measurement files, logs, answers, ladder records) and requires that nothing raises. The first run found 31 distinct crash classes (the worst: a validly signed attestation missing one field crashed `attest-check`, a denial of service a hostile signer could cause); further seeds found `OverflowError`s in `spread` and `recheck`; all fixed. Twelve further seeds at 400 cases per surface ran clean. A result file may not request more than 20000 bootstrap resamples, must bind every ladder it names by hash, and files are written with exclusive-create so "never overwrites" has no race.
|
|
281
|
+
* **Statistical checks in the self-test have measured false-failure rates.** The reuse check (another instance's witness accepted) failed ~5% of random-seed runs as single samples, then ~0.07% as a 25% rate over 30 pairs, because at tiny sizes instances share factors and hits cluster (distribution measured over 3000 seeds); they now fail above 50% and warn above 25%. A real lookup-table verifier (accepts any odd number) still fails; a test pins both directions, and a 400-seed test pins the false-failure rate at zero.
|
|
282
|
+
* **The power table's noise is measured, not just assumed.** `score.py` reports each pair's observed within-bin noise (real dlog stride-4 vs reference: 0.19 bit, against the 0.6 assumed), says so when it is clearly higher than assumed, and re-simulates the certifiable range at the observed value; `score.py power --pair-noise X --min-gap G` re-simulates on demand from the compiled profile, and `compare` prints its own simulated power and its false-declaration rate at a true gap of 0. The simulation draws ladders from the compile-time profile (median per size, spread, cap), so it answers "what could this ladder resolve", not "what is this solver's real noise" (that is the measured number).
|
|
283
|
+
* **A solver beyond the ladder is told what it would take.** When the faster solver costs under the 0.5 s floor on most instances the slower one finds hard, `score` and `compare` print the raw median gap as description (never a bound) and the cost the slower solver would need (about 0.5 s x 2^gap), and say plainly when that exceeds the cap so no ladder under this cap can resolve it (dlog's bsgs vs stride-4: about 11.5 bits needs about 1,463 s, cap 120 s).
|
|
284
|
+
* **No platform has reviewed this.** Yukon's checker format is not public (its proposal form asks for a problem and a verifier description, and the checker is designed with their team afterwards), so the bundle targets nothing in particular. `PROPOSAL.md` is intake text, not an integration.
|
|
285
|
+
* **A bundle is code, so the tools that run it sandbox themselves.** `check.py derive|verify|selftest` import and run the bundle's `decl.py` (and `selftest` its reference): by default they re-execute inside the OS sandbox (no network, no writes outside a scratch directory, `~/.ssh` and friends unreadable). Tested here with a bundle whose `decl.py` writes a file outside the scratch directory at import, with every hash re-pinned so only the sandbox can stop it: blocked by default (reported as `FAIL the declaration imports ... blocked by the sandbox?`), written with `--sandbox off`. `export_bundle.py verify --selftest` also sandboxes and REFUSES to run where there is no sandbox unless `--unsandboxed`; plain `check.py` warns and runs as you there. Export runs its own self-test sandboxed, so a declaration that secretly needs to write files fails at export. Same limits as for `rerun`: not a proof of isolation, Linux `bwrap` simulation-tested only. The manifest hashes prove the files are the listed ones, not that they are safe. `score.py` runs nothing from the ladders it reads.
|
|
286
|
+
* **The verifier is tested, not proved.** The self-test no longer rests on a fixed junk list: on the public practice seed AND a fresh random seed (printed; fixed from the declaration's hash during export so exports are reproducible) it also tries answers derived from each instance (every integer in the arguments, their neighbours, the argument list and its reverse), other instances' witnesses (a lookup table of past answers) and whether instances at the same index differ between the two seeds (a seed-ignoring `derive()` would make the commitment commit to nothing; an all-pairs witness-acceptance version of this diluted that to 6 of 36 and never fired, found by mutation testing), and damaged witnesses. Two kinds of damage are kept apart because one is legitimate: a witness with EVERY part damaged (all integers shifted, negated, zeroed) must be mostly rejected (more than half accepted fails), while a witness with ONE part changed only warns when accepted, since a verifier with a tolerance (aiinfer accepts up to 1% wrong labels) accepts those by design. The first version failed single-part damage and wrongly rejected aiinfer's export: a real false positive found by exporting a second domain, now pinned by a tolerance-style toy domain whose loosened twin (accepts any list of the right length, which no junk list catches) must fail. Tests also show it catches a verifier loosened to accept 1 and one loosened to accept `n` even when the file's hash was re-pinned. A verifier that accepts only something outside all of that still passes.
|
|
287
|
+
* **Dynamic imports and data files are found by running the declaration.** Export runs it (import, then derive and verify three instances at every practice size, with the witness and a few junk answers) in a child process with an audit hook. It vendors every project or sibling module the declaration actually loaded, including ones the import walk cannot see (`importlib.import_module("x")`; listed under `vendored_found_only_at_run_time`), copies every data file it opened beside `decl.py` (listed under `data_files`, hash-pinned, relative to `__file__` so it keeps working), and refuses with the file's name if it reads a project file that cannot travel. Still missed: a module or file used only on a code path none of those calls takes; the clean-interpreter self-test is the backstop. Data it reads from the network or a database is out of reach by construction. The reference may not import project modules (it runs in a sandbox that has only the standard library, numpy and fpylll).
|
|
288
|
+
* **`DRAFT` is common today.** `dlog` exports as DRAFT because its declaration has no `REFERENCE_NOTE` and its compile record says the ladder was not confirmed; the bundle says so in its README and its proposal text.
|
|
289
|
+
|
|
290
|
+
Tests: `test_export.py` (compiles a toy domain, exports it, then drives the shipped files as a stranger would: moved to another directory, `-I`, tampering caught three ways, scorer equal to the project estimator on identical files, every refusal above, both `compare` verdicts).
|
|
291
|
+
|
|
292
|
+
## The pre-registered real-solver test (`dlog_experiment.py`)
|
|
293
|
+
|
|
294
|
+
`results/PREREGISTERED_dlog_realsolver.txt` was written before any data existed: two real dlog solvers (a 4-entry baby-step scan, true speedup
|
|
295
|
+
about 1.3-1.6 bits, and full baby-step/giant-step, far above the ceiling) on three seeds. `python3 dlog_experiment.py` applies its criteria
|
|
296
|
+
mechanically from the saved ladders and claim files (and re-hashes every registered code file, failing if any changed), and the Results page shows
|
|
297
|
+
the same table. Criteria that cannot be evaluated yet print PENDING, never PASS.
|
|
298
|
+
|
|
299
|
+
## What this does not do (read before relying on it)
|
|
300
|
+
|
|
301
|
+
* **A certificate proves consistency, not honest measurement.** It shows the commitments were public first, the instances are the seeds' own, the rules are the named ones, and the credit follows from the records. It cannot show the CPU costs in those records are what a sandbox measured: an operator could write plausible ones. Only the Docker meter (unverified), `verify --rerun` (re-measuring instances yourself, needs `--execute`) and independent attestations from keys you chose to trust address that, and the first of those has never been run on a real host. `build --reveal-seeds` is refused until the round is revealed.
|
|
302
|
+
* **A private solver is hidden from the public, not from the operator.** `certificate.py make-private` (or `build --private`; in a round, any submission whose owner did not choose "publish source") removes the solver's source and every failure message from its ladders (a traceback quotes code). The log, the fingerprints, the questions, the rules and the credit arithmetic still verify, and the solver's hash is the one the round's log committed to before the seeds were revealed; but nobody except the operator can inspect or re-run it, so its timings rest on the operator's sandbox alone (the verdict says so). If the owner publishes the file, `verify --source FILE` checks that it hashes to the committed value and, with `--rerun --execute`, re-measures it: the certificate is then as strong as a full one. Hiding a solver from the OPERATOR too would need trusted hardware (not available) or a zero-knowledge proof of execution, which is infeasible here: these solvers run for seconds to minutes (billions of cycles), a zkVM would count abstract cycles not CPU seconds, and it cannot run numpy or fpylll.
|
|
303
|
+
* **The reference defines the baseline.** A certificate says "faster than this reference by this many bits on its hardest instances". It does not say the reference is a good algorithm. A weak reference makes big speedups cheap; the compiler cannot judge that and neither can the estimator. What it does is make the author say it: a missing `REFERENCE_NOTE` makes `check` warn (it does not fail), and the note is copied into the compiled record, the challenge document and the domain card, and reads "not stated" when missing. When someone beats the reference by a large margin, replace the reference, bump `VERSION` and recompile; old results then refuse to mix with new ones.
|
|
304
|
+
* **The power analysis rests on an assumed pair noise.** Real per-instance noise of a submission against the baseline cannot be known before a submission exists. The default is 0.6 bit (`--pair-noise`); compare it with the measured values in PROTOCOL.md and re-run with yours. The compile also fixes everything from 8-instance medians per size, so the ladder is a good choice, not an optimal one.
|
|
305
|
+
* **Certifiable range has a ceiling** (about 5 bits at a 120 s cap, as in PROTOCOL.md): a solver that is 2^5 times faster than the reference costs less than the estimator's 0.5 s floor on the instances the baseline finds hard. The compiled `CHALLENGE.md` reports the actual range; beyond it a model baseline is needed (not built).
|
|
306
|
+
* **Validation finds the defects it knows.** The anti-cheat checks catch a verifier that is guessable, derivable from the arguments, or reusable across instances. They cannot prove a domain has no shortcut: a domain whose instances are structured so that a solver can skip the work is the author's to rule out (EGG.md is the worked example of a family that could not be tested).
|
|
307
|
+
* **The sandbox is still unverified.** Everything here inherits `sandbox/README.md`: the Docker meter has never passed `test_sandbox` on a real host, so no compiled domain can have an OFFICIAL result yet, exactly like `gm`, `mlwe` and `factor`.
|
|
308
|
+
* **The self-test is local mode.** It runs at the chosen official `n` (`--selftest-n` lowers it to save machine time, but a known-effect control run below the n the power analysis chose can fail for lack of power alone), without Docker. It tests that the compiled pipeline behaves; it is not an official run.
|
|
309
|
+
* **What the compiler's notes caught on `dlog`.** The reference's cost does not double smoothly per size: size 30 measured a 39-43 s median against 3.4-3.8 s at sizes 28-29 (twice, with 8 and with 16 instances), a cliff that is probably CPython's big-integer arithmetic leaving its single-digit fast path once the modulus passes 2^30. The compile re-planned to a 3-size ladder (28-30, 2.6 octaves), and its output says so: "narrow ladder", "NOT independently confirmed", "still off by more than one octave at size 30". It did not hide the cliff or claim a confirmation. The honest reading is that this reference is a poor baseline for a ladder (a perfectly valid, deliberately plain demo), and that a declaration author should read those notes, not just the exit code.
|
|
310
|
+
* **The dlog challenge's size-29 profile is slightly optimistic.** Its compiled record dropped one slow instance and counted another twice, so the recorded size-29 median (3.75 s), spread and the ladder chosen from them lean toward cheaper instances. How far is not measured (that needs all 16 instances at the 120 s cap). Recompiling the domain settles it; that is held until the pre-registered experiment ends, because the compiled record is part of what that registration hashed.
|
|
311
|
+
* **The profile is noisy.** The ladder and the power simulation rest on 8-instance medians per size (`--n-prof`). On the first real run (`dlog`) neighbouring sizes came out as 3.7 s, 2.05 s and 43 s where the true cost doubles per size, so the compile re-plans, and the resulting ladder can be short (3 sizes there) and irregular. Raise `--n-prof` for a steadier profile.
|
|
312
|
+
* **Python 3.14 here.** Tested on the interpreter at hand; 3.10-3.13 and Linux are untested like the rest of the project (PROTOCOL.md).
|
|
313
|
+
|
|
314
|
+
## Files
|
|
315
|
+
|
|
316
|
+
| file | role |
|
|
317
|
+
|---|---|
|
|
318
|
+
| `declared.py` | loads and checks declarations, registers fresh compiled domains into `families.py`'s six tables |
|
|
319
|
+
| `compile_domain.py` | the passes and the CLI (`init`, `compile`, `check`, `selftest`, `emit`, `status`) |
|
|
320
|
+
| `certificate.py` | `build` and `verify` for speedup certificates |
|
|
321
|
+
| `domains/<name>/` | `decl.py`, the reference, `compiled.json` (+ `compiled.prev.json`), `out/` |
|
|
322
|
+
| `recheck.py` | `compile_domain.py recheck`: recompute a compiled record's checkable claims; `--measure` compares this machine with the record |
|
|
323
|
+
| `attest.py` | `keygen`, `sign`, `check` for independent signed attestations of a certificate |
|
|
324
|
+
| `counted.py`, `count_runner.py` | deterministic step counting: exactly reproducible costs (Python 3.12+), the counted ladder writer and the determinism self-check |
|
|
325
|
+
| `export_bundle.py`, `bundle_templates/` | `compile_domain.py export`: the platform-neutral challenge bundle (checker, scorer, estimator copy, manifest); `check.py` and `score.py` are shipped byte-for-byte from `bundle_templates/` |
|
|
326
|
+
| `test_export.py` | the bundle's tests |
|
|
327
|
+
| `certs_site.py` | the site's Certificates section (`build_site.py --certs DIR`) |
|
|
328
|
+
| `dlog_experiment.py` | mechanical judge of the pre-registered dlog real-solver experiment |
|
|
329
|
+
| `test_compiler.py`, `test_certificate.py`, `test_recheck.py`, `test_attest.py` | the tests (`make test` runs them) |
|