guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/DEPLOY.md ADDED
@@ -0,0 +1,200 @@
1
+ # Running the public competition
2
+
3
+ This is the runbook for putting the submission path on the internet. It assumes you have read PROTOCOL.md. Everything here is
4
+ written and unit-tested, and it has now run on one real server (a Hetzner VPS, November 2026): the portal behind Caddy with TLS, a counted test
5
+ round taken through submit, close, evaluate, reveal and a certificate verified from another computer, and a real counted round open for submissions. The
6
+ sandbox passed its full test suite (including resource-exhaustion and meter-tampering attacks) on GitHub's Linux runners and on that server.
7
+ **Not yet done:** a full timed round end to end on an evaluation machine that is separate from the web service (EVAL_BOX.md), a public timed round, and a
8
+ timed evaluation at the official size (one is running as a private rehearsal). Read the sections below with that in mind: the parts marked untested
9
+ are still untested.
10
+
11
+ Two files go with this one: EVAL_BOX.md (the second machine, backups, CPU pinning, running the sandbox on the web machine for a trusted rehearsal) and
12
+ LATTICE_ROUND.md (what a lattice round would certify, and what it costs).
13
+
14
+ ## 1. The three trust zones
15
+
16
+ | zone | process | user | can | cannot |
17
+ |---|---|---|---|---|
18
+ | web tier | `public_portal.py serve` | `ledger-web` | list rounds, accept an upload, `ast`-parse it, append to the round's log | execute an upload, read a seed, reach Docker |
19
+ | operator | `ops.py` | `ledger-ops` | create rounds (makes the seeds), close, evaluate, reveal, export | (trusted) |
20
+ | sandbox | docker container per solver run | uid 10001 inside it | run one solver on one instance with a CPU cap | network, filesystem outside `/work`, seeds |
21
+
22
+ Why this split: the web tier handles hostile input all day, so it must not hold anything worth stealing. The seeds are the only
23
+ secret the protocol has, and they live under `ledger-ops` only (`/srv/ledger/seeds`, mode 0700). If the web tier is fully
24
+ compromised, an attacker can read submissions (public after the reveal anyway), flood the queue (bounded, see below), or take
25
+ the site down. They cannot learn a seed, so they cannot pre-solve a round.
26
+
27
+ ## 2. What keeps the operator honest
28
+
29
+ Each round has an append-only **hash-chained log**. Entry 0 holds the seed commitments **and the rules**: the ladder sizes, the
30
+ instance version, the baseline solver's hash and the hashes of the code that generates, runs, meters and scores instances. It is
31
+ written when the round is created, so all of it is public before any submission exists. The worker refuses to evaluate if any of
32
+ that has changed since (so a bug fix mid-round means a new round); `ops.py worker --accept-code-change` overrides, and writes a
33
+ public `code_changed` entry with the old and new hashes, so a competitor can see exactly what changed and when. Every accepted file's SHA-256 is an entry. Anyone can
34
+ download `/api/log/<round>.jsonl` and check the chain (`ops.py verify-log`, or `rounds.verify_log`): a dropped, reordered,
35
+ altered or back-dated entry breaks it. **Publish the head hash somewhere you cannot edit it** after each round closes (a signed
36
+ post, a git tag in another repository): that anchors the log, so even you cannot rewrite history. The software refuses to
37
+ append to a log that does not verify.
38
+
39
+ ## 3. Setup, in order
40
+
41
+ 1. **Host.** A small Linux VPS for the web tier and site. For stronger isolation, evaluate on a *separate* VM (step 6).
42
+ 2. **Users and directories.**
43
+ ```
44
+ useradd --system ledger-web; useradd -m ledger-ops; groupadd ledger
45
+ usermod -aG ledger ledger-web; usermod -aG ledger ledger-ops
46
+ install -d -o ledger-ops -g ledger-ops -m 0755 /srv/ledger # the operator builds site.new / site.old inside it
47
+ install -d -o ledger-ops -g ledger -m 2770 /srv/ledger/data
48
+ install -d -o ledger-ops -g ledger-ops -m 0700 /srv/ledger/seeds # web tier has no access
49
+ install -d -o ledger-ops -m 0755 /srv/ledger/site
50
+ ```
51
+ Both the web tier (systemd `UMask=0007`) and `ops.py` (it sets umask 007 itself) create group-writable files, so the two users
52
+ can share the data directory. If you create anything under `data/` by hand, make it group-writable too.
53
+ 3. **Code.** Copy this directory to `/opt/speedup-verification`, owned by root and read-only to `ledger-web`.
54
+ 4. **Edit** `deploy/Caddyfile`, `deploy/ledger-web.service` and `deploy/ledger-export.service`: replace `ledger.example.org`.
55
+ 5. **Web tier.** `cp deploy/ledger-web.service /etc/systemd/system/ && systemctl enable --now ledger-web`; install Caddy and
56
+ use `deploy/Caddyfile` (it terminates TLS and overwrites `X-Forwarded-For`). Test: `curl -s https://<host>/healthz`.
57
+ 6. **Sandbox (the gate).** On the machine that will evaluate: install Docker, build the image
58
+ (`sandbox/README.md`), then `python3 portal.py verify-sandbox --data /srv/ledger/data`. It writes a marker **only if the
59
+ sandbox tests all run and pass with none skipped**, and the marker is tied to a fingerprint of the sandbox files, so it lapses
60
+ if they change. Also run `sandbox/linux_selftest.py` as root on a Linux box. **Do not open a round for evaluation before this
61
+ passes.** The worker refuses to run without the marker, and there is no flag to override that.
62
+ 7. **Create a round** (as `ledger-ops`):
63
+ ```
64
+ python3 ops.py round-new --data /srv/ledger/data --seed-dir /srv/ledger/seeds --id winter-1 \
65
+ --title "Winter round: lattice SVP" --family gm --closes 2026-12-31T23:59:00Z
66
+ ```
67
+ It creates the seeds (mode 0600), writes the commitments into the log, and opens the round. Announce the commitments.
68
+ **Decide about private solvers now**, because it becomes a public rule in the log's first entry. By default a submitter may keep their solver private:
69
+ the public certificate then carries only its hash, nobody but you can run it, and its timings rest on your sandbox alone. Add `--require-source` to
70
+ require every submission to publish its source after the reveal (the server refuses a submission without it, `svp.py` refuses before uploading, and the
71
+ web form marks the box required). Which to choose is your call: private solvers invite more entrants, published ones make every result re-checkable.
72
+ **Then check the host and the running service, before you announce anything:**
73
+ ```
74
+ python3 ops.py preflight --data /srv/ledger/data --seed-dir /srv/ledger/seeds # on the host, as ledger-ops: permissions, no seed in data/, sandbox marker, logs
75
+ python3 deploy_smoke.py https://<host> # from anywhere: headers, Host/Origin refusals, every round's log, no early seed
76
+ ```
77
+ `preflight` is read-only and fails if the seed directory can be entered by the group (the web tier's group), if any seed file sits under the data
78
+ directory, or if a round's log does not verify; it warns when no sandbox is verified. `deploy_smoke.py` tests the **running** service over
79
+ the network (a stripped header, a proxy that forwards a foreign Host, a log that does not verify, a seed shown early are all findings). Both
80
+ are tested against deliberately broken setups. `deploy_smoke.py URL --submit FILE --round ID --handle NAME` adds one real, permanent submission.
81
+ 8. **Build the site** with the round listed: `python3 build_site.py --out /srv/ledger/site.new --ledger-data /srv/ledger/data --portal-url https://<host>`
82
+ (or enable `ledger-export.timer`, which does this every 5 minutes).
83
+
84
+ ## 4. Round lifecycle
85
+
86
+ `open` (submissions accepted; files only read) -> `closed` (deadline passed, nothing more accepted) -> `evaluating` -> `evaluated`
87
+ -> `revealed` (seeds published, each checked against its commitment).
88
+
89
+ ```
90
+ python3 ops.py close --data D --id winter-1 # automatic at the deadline on the next request; --force closes early (logged)
91
+ python3 portal.py verify-sandbox --data D # must pass on the evaluating host
92
+ python3 ops.py worker --data D --id winter-1 --seed-dir /srv/ledger/seeds/winter-1
93
+ python3 ops.py reveal --data D --id winter-1 --seed-dir /srv/ledger/seeds/winter-1
94
+ python3 ops.py certs --data D --id winter-1 --seed-dir /srv/ledger/seeds/winter-1 # one verified certificate per evaluated submission
95
+ python3 ops.py verify-log --data D --id winter-1 # then publish the head hash
96
+ ```
97
+ `ops.py certs` runs after the reveal (a certificate carries the seeds) and stores a certificate only if it verifies. Submitters then fetch it
98
+ with `svp.py` (SUBMIT.md). `GET /api/cert/<round>/<id>.json` is open to everyone: it carries the solver's source only when the owner chose "publish
99
+ source", and is otherwise a private-solver copy (hash, no source, no failure text); `GET /api/cert-full/<round>/<id>.json` is the full certificate and needs the
100
+ handle's token in an `Authorization: Bearer` header, because the certificate contains the solver. `ops.py export` copies the public ones next to
101
+ the round's log. **Fails closed:** on every release the portal re-checks that the public certificate of a submission whose owner did NOT publish their
102
+ source really is a private-solver certificate (marked so, no source). A file that is not (a stale one from an older version, a mistake) is refused with
103
+ 403, not advertised in the round's description and not exported; `ops.py certs` rebuilds it in place.
104
+ Evaluation of one submission is 3 seeds x 2 ladders at the family's official size, up to several CPU-hours (`official.py plan`).
105
+ Run it deliberately, not from a timer. Results appear on the round page only after the round is `evaluated`.
106
+
107
+ **A counted round** (`round-new --count --count-dims ...`, COMPILER.md "Counted rounds") is evaluated without Docker:
108
+ `python3 ops.py worker --data D --id r --seed-dir S/r --unsandboxed [--results-dir R]`. Submitted code runs in the counter's restricted subset of Python (an in-process sandbox that protects the counter's integrity: no `sys`, files, frames, `getattr`/`eval`;
109
+ COMPILER.md "The restricted subset"), but there is no operating-system isolation, hence the flag: it still runs as the operator's user, so do it as a
110
+ throwaway user or VM that holds nothing of value (and, ideally, no seeds you still need secret: the seeds are revealed right after anyway). It needs the same
111
+ Python the round committed to, to the patch level (check `svp.py round ID` for the version). On a machine with automatic updates, an interpreter update in the middle of
112
+ a round would make the evaluation refuse to run, so hold the interpreter for the life of the round: `apt-mark hold python3.14 python3.14-minimal` plus the
113
+ `libpython3.14*` packages (`dpkg -l | grep libpython`), and release it with `apt-mark unhold` once the round is evaluated and revealed. `reveal` and `certs` are unchanged.
114
+ This path has run end to end on a real server for a test round; see the top of this file for what has and has not been tried.
115
+
116
+ **Evaluating on a separate VM.** After `close` the web tier no longer writes the round, so you can `rsync -a D/rounds/winter-1/`
117
+ to the evaluation VM, run the worker there against its own copy, and rsync the directory back before `reveal`. Keep the seeds on
118
+ the evaluation VM only.
119
+
120
+ ## 4b. What evaluation costs (read before you set a limit)
121
+
122
+ A solver that works is cheap. A solver that FAILS is not: every failed, wrong, crashed or timed-out instance is charged the whole CPU
123
+ cap, and the judge really spends that time (the sandbox runs it until the cap or the wall deadline). For `gm` that is 120 instances x
124
+ 3 seeds x 120 s = **12 CPU-hours per submission that fails everything**, plus the baseline ladders once (about the same again, shared
125
+ by all). A round that accepts 200 junk submissions could take months of machine time. Consequences, and what to do:
126
+ * `ops.py round-new --max-submissions N` (default 50, logged in the round's first entry so it is a public rule) bounds the count, and
127
+ prints this worst case when you create the round. Pick N from the machine time you actually have.
128
+ * `ops.py round-new --n N` sets the instances per size for the round (default: the family's official 24 for `gm`; at most 200; also logged
129
+ in the round's first entry). More instances is how a round certifies larger speedups: measured with `power.py --copies K --resample`, a
130
+ 4-bit constant speedup is certified 1% of the time at n=24, 68% at n=48 and 100% at n=96, and 5 bits 50% at n=48 and 98% at n=96
131
+ (PROTOCOL.md, "Power at the OFFICIAL configuration"): for 4 to 5 bits use about 96. The price is linear: n=48 doubles the worst-case
132
+ evaluation time per submission (24 CPU-hours for `gm` at 3 seeds) and n=96 quadruples it (48), so lower `--max-submissions` accordingly. No n lifts the ceiling of about 5 bits at a 120 s cap: at 6 bits the result is
133
+ inconclusive whatever the count.
134
+ * Evaluation is one worker at a time. Several machines means several data directories, not one shared one (the board file is global).
135
+ * A hostile solver that sleeps or hides orphan processes is charged the cap too, and the meter holds each such instance until its wall
136
+ deadline (3x the cap + 10 s), so it costs the judge wall time without costing its owner anything. The cap on submissions is the defence.
137
+ * Not implemented, a design choice for you: a cheap screen that runs each submission on a few small public instances in the sandbox and
138
+ skips the official run if it cannot solve even those. It would save hours per junk file, but it excludes solvers that only work at
139
+ large sizes, so it would have to be a published rule in the round's first entry. The code to add it is small; the rule is the decision.
140
+
141
+ ## 5. Abuse and failure controls (each has a test)
142
+
143
+ * **Kill switch:** `ops.py kill-switch --data D on` pauses submissions instantly (reads keep working); `off` resumes.
144
+ * **Graceful stop:** on SIGTERM/SIGINT (`systemctl stop` or `restart`) the web tier stops accepting connections, finishes the requests
145
+ already in flight (at most about 10 s, the socket timeout), then exits 0. A restart during a submission therefore cannot cut it between
146
+ the log append and the metadata write (tested with a real subprocess and a half-sent upload). A hard kill (SIGKILL, power loss) still can:
147
+ that leaves a log entry with nothing stored, which `ops.py verify-log` reports and the section below explains. The worker is different:
148
+ stopping it mid-evaluation leaves that submission `evaluating`, and the next run retries it.
149
+ * **Limits:** 200 KB per file; 3 submissions per handle per round; `--max-submissions` per round (default 50, at most 200); 6 per hour per address (burst 3) and 300 per hour overall;
150
+ 120 requests/min per address; 64 concurrent connections (the rest get a 503); a 10 s stall timeout. Rejections store nothing.
151
+ * **No accounts.** A handle is bound to a secret token issued with its first submission; only the token's hash is stored.
152
+ * **Privacy:** the service stores the handle, the file and its hash. No IP address, no contact detail. Rate limiting uses a
153
+ per-process salted hash of the address held in memory only.
154
+ * **Logs:** no access log. Add one at the proxy if you need it and decide your retention.
155
+
156
+ ## 6. Things only you can decide (RULES.md is a draft)
157
+
158
+ * the rules text, licence terms for submitted code, and any prize or eligibility limits (get legal review before publishing);
159
+ * the domain, the host and who is on call; whether to publish sources by default (it is opt-in per submission here);
160
+ * what happens to a handle someone claims is theirs (the answer today: nothing, the token is the only proof);
161
+ * retention: log entries are permanent by design; sources can be deleted on request after the reveal, but their hash stays in the log.
162
+
163
+ ## 7. Known limits
164
+
165
+ * The sandbox is unverified (above). Official credit also needs 3 seeds per round, which `ops.py round-new` enforces.
166
+ * The round page shows a submission's result only once the round is `evaluated`, but the main site's Results leaderboard reads the
167
+ global board file as the judge writes it, so a credit can appear there a little earlier (never a seed; submissions are already
168
+ closed by then, so it cannot be used to adapt). If you need the two to agree, rebuild the site only after `ops.py worker` finishes
169
+ (disable `ledger-export.timer` during an evaluation).
170
+ * A failed evaluation's raw output is written to `rounds/<id>/submissions/<sid>/eval_error.log` (group-readable only, never
171
+ exported); the public page says only "evaluation failed; the operator has the details", because tracebacks can contain seed paths.
172
+ * One worker at a time for the whole data directory (every evaluation writes the one board file).
173
+ * **The baseline is measured once per seed and reused** by every later submission in the round (`--reuse-baseline`, accepted only when the
174
+ seed commitment, baseline file hash, family, budget, instance version, sandbox, image, CPU and job count all match and the ladder is
175
+ complete; otherwise it is measured again). That keeps a round's cost near one baseline ladder per seed, and every submission is
176
+ compared with the same measurement. The trade-off: a submission evaluated hours after the baseline is compared with a baseline timed
177
+ earlier, so keep the evaluation machine otherwise idle and thermally steady, and spot-check with `remeasure.py`.
178
+ * **The worker runs a file only if it still matches the hash in the log**, and `ops.py verify-log` also cross-checks every stored file
179
+ against the log (a swapped file, a stored submission with no log entry, or a log entry with nothing stored). The last one is what a
180
+ crash between the log append and the metadata write looks like: that submission was never acknowledged to its sender, so do NOT edit
181
+ the log; leave it, and say in the announcement that entry #N is an unacknowledged crash artifact.
182
+ * **Python 3.9 or newer** (both entry points refuse older ones with a clear message). The web tier (`public_portal.py`) and the operator
183
+ tool (`ops.py`) need only the standard library: `test_python_compat.py` blocks numpy and fpylll outright and has them create a round,
184
+ accept and lint a submission and render a page. numpy IS needed by `build_site.py` and by evaluation (the estimator computes the
185
+ credits), which run on the operator/evaluating machine; fpylll only inside the sandbox image. Verified on Python 3.9.6 and 3.14 (macOS),
186
+ where `test_python_compat.py` also re-checks every module on the oldest interpreter it can find. NOT run on 3.10 to 3.13 or on Linux.
187
+ (Found by audit: the service could not even be imported before 3.12 because of an f-string, and the web tier needlessly imported
188
+ numpy through the local portal; both fixed.)
189
+ * Linux/macOS only (`fcntl` file locks).
190
+ * **The server clock is the authority for deadlines.** Keep NTP running. A clock that steps BACKWARD is tolerated (log timestamps are
191
+ clamped so they never go backwards). A clock that jumps FORWARD past a deadline closes the round, and closing is permanent: there
192
+ is no reopen, by design, because the log records it. If that happens, create a new round and say why in the announcement.
193
+ * Serve the static site and the service on one domain only with the Caddyfile's split: the static site owns `/assets/*`, the service
194
+ owns `/round`, `/s`, `/api` and `/portal-assets` (its stylesheet has a different path on purpose).
195
+ * A hostile solver that escapes the sandbox is the one threat the whole design leans on the container for: keep the evaluating
196
+ host disposable, with no credentials on it, and the seeds only for the duration of a round.
197
+ * The static site is a snapshot; it shows what was exported at build time.
198
+ * There is no spam protection beyond rate limits and the token: an attacker with many addresses can fill a round's slots
199
+ (`--max-submissions`, 50 by default) with junk, which also costs you evaluation time (section 4b). The response is the kill switch,
200
+ and creating the next round with a lower limit; a filled round cannot be reopened or have entries removed, by design.
guldo/_src/LAUNCH.md ADDED
@@ -0,0 +1,85 @@
1
+ # Launch checklist: everything together
2
+
3
+ Honest state before you start: the whole Docker-free test suite (746 tests) passes on the author's Mac under Python 3.14, and that includes the real step
4
+ counter, the restricted-subset sandbox, counted rounds (with a stand-in counter), derived claims, the package and the site. What has **not** happened: a real counted round
5
+ with real submissions, anything on Linux, and the Docker sandbox on a real host (never run anywhere). This checklist is the order
6
+ that finds out, privately, before anything is public. `DEPLOY.md` has the long-form detail for each server step; this is the spine.
7
+
8
+ ## What launching means here
9
+
10
+ | piece | needs | label it can honestly carry |
11
+ |---|---|---|
12
+ | the static site | any web host | "no result is official" (already on every page) |
13
+ | counted rounds (`ops.py round-new --count ...`) | a Linux host, Python 3.12, an unprivileged throwaway user; **no Docker** | COUNTED, never OFFICIAL |
14
+ | official timed rounds | the verified Docker sandbox on a Linux host, 3 independent seeds | OFFICIAL, only after Gate 2 passes |
15
+
16
+ If Gate 2 fails, launch with counted rounds only and say so on the Compete page.
17
+
18
+ ## You do (I cannot)
19
+
20
+ 1. A Linux VM: Ubuntu 24.04 LTS, x86-64, 2-4 vCPU, 4-8 GB RAM (Hetzner, DigitalOcean, Linode, EC2). Use a dedicated-CPU plan if you want clean timings. Add your SSH key.
21
+ 2. Install on it: `apt install -y docker.io python3-pip rsync` and `pip install numpy cryptography` (Ubuntu 24.04 already has Python 3.12).
22
+ 3. A domain and a DNS A record to the web VM (Caddy then gets TLS by itself, `deploy/Caddyfile`).
23
+ 4. Decide: who is `ledger-ops`; whether rounds `--require-source` (counted rounds force it); where you will publish each round's log head hash
24
+ (somewhere you cannot edit: a signed post, a git tag in another repo).
25
+ 5. Treat the evaluating VM as disposable: the self-test runs hostile test programs. Put no keys on it.
26
+
27
+ ## Gates (run in order; stop at the first failure)
28
+
29
+ From your Mac: `./launch_push.sh user@host` copies the project. On the box, in the project directory:
30
+
31
+ ```
32
+ python3 launch_gates.py # Gate 0-1: environment; counter + counted rounds + claims (FIRST REAL RUN); whole suite; Linux meter self-test
33
+ python3 launch_gates.py --with-docker --data /srv/ledger/data # Gate 2: sandbox image, test_sandbox, verify-sandbox (writes the marker the worker needs)
34
+ python3 launch_gates.py --with-rehearsal # Gate 3: a real counted practice run, full re-execution, the three claims (slow: hours)
35
+ ```
36
+
37
+ | gate | passes when | unlocks | if it fails |
38
+ |---|---|---|---|
39
+ | 0 env + counted + suite | `launch_gates.py` shows PASS for env, counted, suite | everything else; counted rounds are *possible* | fix the failure it prints, one at a time. Expect real findings in `count_runner.py` (does removing `sys.monitoring` break an import? does the proxy import work?), the estimator on counted ladders, and the log checks. Do not launch anything. |
40
+ | 1 selftest | `sandbox/linux_selftest.py` passes as root | the meter is trustworthy on Linux | read the failure: it names orphan/daemon/hang/privilege cases |
41
+ | 2 docker | `test_sandbox` passes and `portal.py verify-sandbox` writes the marker | official timed rounds | counted-only launch |
42
+ | 3 rehearsal | practice, `verify --rerun all --execute` and `claims.py check` all succeed | publish its certificate as the site's real example | the failure says which step; the work directory is kept |
43
+
44
+ The rehearsal is also where the defaults get measured: `--dims 38 40 42 --cap 40` are estimates from reading the loops. If the baseline costs under
45
+ about 0.5 counted seconds at the smallest size, or over the cap at the largest, change `--dims` and `--cap` and run again before creating a round.
46
+
47
+ ## Server setup (after Gate 0-1; full detail in DEPLOY.md section 3)
48
+
49
+ 1. Users, directories, code under `/opt/speedup-verification`, edit the three `deploy/` files for your domain (steps 2-4).
50
+ 2. Web tier: systemd unit + Caddy (step 5). `curl -s https://<host>/healthz`.
51
+ 3. Create the rounds as `ledger-ops`. Counted: `python3 ops.py round-new --data D --seed-dir S --id counted-1 --title "..." --family factor --closes ... --count --count-dims 38 40 42 --count-n 8 --count-cap 40`.
52
+ Timed (only if Gate 2 passed): as in DEPLOY.md step 7. Announce the commitments.
53
+ 4. Before announcing anything: `python3 ops.py preflight --data D --seed-dir S` (as ledger-ops) and `python3 deploy_smoke.py https://<host>` (from elsewhere).
54
+
55
+ ## Private rehearsal round (before public)
56
+
57
+ 1. Open a small counted round and invite two or three friendly submitters (one honest, one deliberately outside the subset, one that is slow).
58
+ 2. `ops.py close`, then `ops.py worker --data D --id R --seed-dir S/R --unsandboxed` **as an unprivileged user** (it refuses root), `ops.py reveal`, `ops.py certs`.
59
+ 3. From a different machine: `svp.py verify --round R --id X --rerun all --execute` for each certificate (a throwaway VM; it runs their code), then
60
+ `claims.py correctness|worst-case CERT` and `claims.py negative CERT... --threshold 2`.
61
+ 4. Check that the refused submission was refused *before* upload (`svp.py submit` says what is outside the subset) and that the slow one fails to verify, not the others.
62
+
63
+ ## Launch day
64
+
65
+ 1. Rebuild the site with the real certificates: `python3 build_site.py --out /srv/ledger/site.new --ledger-data /srv/ledger/data --portal-url https://<host> --certs certs --reveal ... --note ...`
66
+ (the earlier build's `--reveal` and `--note` values are in the memory note; seed A's file does not match its commitment, so do not reveal it). Look at it in a browser first.
67
+ 2. Update the places that say "Tested, not yet used" (the home page status row, the Other claims banner, the Install page's status banner) to what is now true after the rehearsal. Do not remove them before Gate 0-3 pass.
68
+ 3. Open the public round. Publish the commitments and, after each close, the log head hash.
69
+ Give submitters the command-line package: `make package` stages `dist/guldo/` (zip it as a download, or publish it to a package index), and they
70
+ `pipx install` it and use `svp`. Build it from the same checkout the round committed to: the files a round hashes are copied byte for byte, so a mismatch
71
+ would show up as a failed "rules" check. It has been installed and used on one real certificate, never on a clean machine.
72
+ 4. Keep the kill switch in reach: `ops.py kill-switch --data D on` pauses submissions.
73
+
74
+ ## Closing the open limits
75
+
76
+ `RESOLVE.md` is the runbook: each closable limit, the exact steps, the evidence that shows it closed, and the website text to change. `python3 limits_plan.py --data D`
77
+ prints where each stands (read-only). The Gate 2 and rehearsal runs above feed three of them directly (the sandbox, the first official round, one job at a time), and two
78
+ pre-registered experiments (`prereg_templates/`) should be written while the box is being set up, because they only count if registered before their data.
79
+ Three limits stay on the website for good; the Limits page says which.
80
+
81
+ ## After launch
82
+
83
+ - After each round closes: publish the head hash; after the reveal, `ops.py certs`, then `ops.py export` and rebuild the site.
84
+ - Negative results need a certificate for every logged submission: if one failed evaluation, say so rather than hiding it (`claims.py negative` reports it as missing).
85
+ - Anything that fails a gate later (a Python upgrade on the box changes counts): counted certificates are only comparable on the same Python version; start a new round.
guldo/_src/PROTOCOL.md ADDED
@@ -0,0 +1,294 @@
1
+ # Adversarial speedup verification: lattice SVP track
2
+
3
+ Goal: credit a claim of the form "my algorithm is X bits faster than the baseline" without trusting the claimant.
4
+ A solution is NP-checkable; a speedup is not. This protocol converts the second into the first plus a conservative
5
+ statistical bound.
6
+
7
+ ## Interface
8
+ A submission is one Python file exposing `solve(basis: list[list[int]], bound2: int) -> list[int] | None`.
9
+ It must return a nonzero lattice vector with squared norm <= `bound2` (1.05 x Gaussian heuristic, squared). It receives
10
+ nothing else: no seed, no instance index, no network, no files.
11
+
12
+ ## Instances (instances.py, instances_mlwe.py, families.py)
13
+ Three families (two lattice, one integer factoring: `factor`, semiprimes of a given bit size, accepting any nontrivial divisor),
14
+ each a pure function of (master_seed, dim, idx) via HMAC-SHA256, each with a judge-side planted witness so every instance is
15
+ provably solvable (the solver never sees it). The two lattice families:
16
+ - `gm`: Goldstein-Mayer SVP-challenge lattices (accept any vector <= 1.05 x Gaussian heuristic). A typical-length vector is
17
+ planted. Without planting, some small-dimension instances had NO vector within the bound (two seen at dims 40 and 45,
18
+ confirmed by exhaustive enumeration; 2 of ~160 small-dimension baseline runs recorded).
19
+ - `mlwe`: Kannan-embedding lattice of a module-LWE-shaped instance (Z_3329[x]/(x^4+1), rank k, dim 8k+1, Gaussian error
20
+ sigma=8; ladder 97,105,...,137). The judge accepts any short lattice vector, checked by modular arithmetic, not only the
21
+ planted secret. Ring dimension 4 (not 8) so the ladder steps by 8: with step 16 only one dimension fell between the
22
+ 0.5 s floor and the timeout ceiling and the family could never produce a credit. Calibration (baseline BKZ, 3 instances,
23
+ 60 s cap): 97: 0.3 s, 105: 0.5 s, 113: 0.7 s, 121: ~1.5 s (1 of 3 timed out), 129: 2-5 s (1 of 3), 137: 24 s (2 of 3).
24
+ Measured with BOTH solvers (practice seed, 6 instances/dim, tuned_bkz vs baseline_bkz, medians in seconds):
25
+ 105: 0.52 / 0.55, 113: 0.58 / 0.74, 121: 1.77 / 2.04 (baseline 1 of 6 timed out), 129: 32.4 / 2.00 (baseline 3 of 6
26
+ timed out). All four clear the 0.5 s floor, but 105 does so by a hair and 129 sits exactly at the 50% baseline-timeout
27
+ drop threshold, so with other seeds or n the usable set can shrink to two or three dimensions. Per-dimension speedup
28
+ of tuned over baseline: -0.25, -0.38, +0.45, +2.69 bits; Tier-1 credit 0 (pooled over 113/121/129: median 0.4, lower
29
+ bound -0.1). So the ladder can produce a measurement, but this solver pair has no certifiable speedup on it. (An
30
+ earlier version of this note, on the old N=8 ladder with 12 instances, found only dim 113 usable.)
31
+ 1. Judge draws a 256-bit master seed and publishes sha256(seed) **before** the submission deadline (`judge.py new-seed`).
32
+ 2. Submissions are frozen at the deadline.
33
+ 3. Judge reveals the seed; anyone can check the commitment (`judge.py check`) and regenerate every instance.
34
+ Consequence: a submission cannot contain answers for, or be tuned to, the judged instances. The public practice seed is
35
+ for development only; `solvers/cheat_lookup.py` demonstrates the attack this blocks.
36
+
37
+ ## Cost
38
+ CPU seconds (user+sys) of the solver process TREE, minus the cost of a no-op solver, strictly sequential.
39
+ In the Docker sandbox (the only mode `judge.py evaluate` accepts by default) `sandbox/meter.py` runs as root, drops the
40
+ solver to uid 10001, and after the solver exits waits for and reaps every descendant (orphans, double-forked daemons)
41
+ so their CPU is billed; stragglers are killed at the wall deadline and the instance counts as a timeout.
42
+ Failures (timeout, crash, no answer, invalid answer) are censored at the budget, so they count against the solver.
43
+ Bare local mode exists only for your own solvers (`--insecure-local`): a solver there can read the seed and the judge's
44
+ code and can orphan processes to hide CPU. Both exploits were reproduced against the bare harness
45
+ (solvers/planted_peek.py, solvers/orphan_cpu_hider.py).
46
+ STATUS OF THE SANDBOX (be precise): the first meter was verified against a hostile probe (non-root uid, no forging the
47
+ meter's output, no setuid, read-only filesystem, no network, pid cap). The CURRENT meter (descendant reaping, subreaper,
48
+ kill-all-at-deadline) and its tests in test_sandbox.py have NOT been run against a real Docker daemon: Docker became
49
+ unavailable during testing. Treat orphan-CPU billing and the hang protection as unverified until
50
+ `python3 -m unittest test_sandbox` passes. The harness's Docker control flow (flags, parsing, timeout cleanup) is
51
+ tested against a fake `docker` (test_docker_shim.py), which says nothing about the sandbox itself.
52
+ A results file records `instance_version`, `budget`, `sandbox` and the image ID; `extrapolate.py` refuses to compare
53
+ ladders that differ in any of them. Seeds must live outside the project tree (`judge.py new-seed` enforces it).
54
+
55
+ ## Claim (extrapolate.py)
56
+ s(n) = median over instances of log2(baseline cost / submission cost) on the same instance, per dimension n on the
57
+ ladder (`families.DEFAULT_DIMS`, tuned to the default baseline; `--dims` overrides it for another baseline; cap per
58
+ `families.OFFICIAL`). A dimension is used only if both solvers' median
59
+ cost is at least 0.5 s (fixed-cost guard: setup costs fade with n and would fake a speedup that grows with n; the
60
+ floor is absolute so genuinely fast solvers are not penalized). Consequence: a speedup can only be certified in a
61
+ regime where the faster solver still costs >= 0.5 s, so the ladder must reach dimensions that are expensive enough.
62
+ Censoring only ever hurts the claimant: a baseline timeout understates its cost and a submission timeout overstates
63
+ the submission's, and both shrink s(n). A ladder where the baseline mostly times out is a bad ladder, not a big
64
+ speedup; pick dimensions the baseline can finish.
65
+ Two tiers, reported separately:
66
+
67
+ **Tier 1, MEASURED (this is the credit): difficulty-stratified.** Group instances (all dimensions together) into octaves
68
+ of the BASELINE's cost, starting at 0.5 s: [0.5,1), [1,2), [2,4), ... A bin is usable when it holds >= 5 instances and
69
+ the submission's median cost in it is >= 0.5 s (fixed-cost guard). Take the 3 hardest usable bins, pool their paired
70
+ per-instance log2(baseline cost / submission cost), and report the one-sided 95% JOINT-bootstrap lower bound of the pooled
71
+ median: every resample redraws instances from the whole eligible pool and RE-SELECTS the usable and hardest bins, so the
72
+ uncertainty of that selection is inside the bound (a resample with fewer than 2 usable bins counts as 0 bits). An earlier
73
+ version resampled within fixed bins; by simulation its bound exceeded the truth 5-9% of the time instead of <= 5%, and on
74
+ real seed-A data it overstated the credit (1.3 bits vs 0.1 corrected). Instances the baseline timed out on are excluded (unknown cost:
75
+ conservative); submission failures stay in at the budget cost. No model, no extrapolation: "on instances the baseline
76
+ found hard, this solver is at least X bits faster". Needs >= 2 usable bins, else inconclusive. Leaderboard credit is
77
+ the MIN over >= 3 independent committed seeds; an inconclusive seed counts as 0.
78
+ Why not by dimension: on real data the baseline's cost is heavy-tailed and non-monotone in dimension (median 1.2 s at
79
+ dim 66, 60 s at 68, 4.6 s at 70, 60 s at 72 on one seed), so choosing "the largest dimensions" was erratic and dropped
80
+ the dimensions where the effect was large. Selecting by the baseline's own cost depends only on the reference solver,
81
+ so it cannot favour the submission. The rule (top-3 hardest bins, bin size, floor) was fixed BEFORE computing any
82
+ credit with it, but the author had looked at the seed-A bin medians while designing it; independent data was checked
83
+ afterwards (see Real-data checks). The old dimension-pooled claim is still printed as an informational line.
84
+
85
+ **Tier 2, EXTRAPOLATED (information, never credit).** Four growth models (constant, linear, power-law, log-decay) are
86
+ fit to the per-dimension medians, each tested out-of-sample (fit without the largest dimension, predict it, reject if
87
+ off by > 1 bit); the extrapolated bits at the target dimension are min(min over surviving models, bootstrap 5th
88
+ percentile, 2 x the largest measured speedup). No surviving model => inconclusive. It is model-dependent and
89
+ statistically weak at realistic noise (see Statistical power), which is why it is not the credit.
90
+
91
+ **What makes a leaderboard credit official** (`judge.py leaderboard`): >= 3 distinct seeds, each run in the Docker
92
+ sandbox, with the family's official instances per dimension and CPU cap (`families.OFFICIAL`: gm 24 instances / 120 s,
93
+ mlwe 12 / 60 s), an instance version, and a seed verified against a
94
+ commitment published beforehand (`judge.py evaluate --published-commit HEX`). Credit = MIN over those seeds, an
95
+ inconclusive seed counts as 0. Anything else is shown as PROVISIONAL with the reasons; board entries record n, budget,
96
+ commitment, sandbox, instance version and image ID. `judge.py rescore` rebuilds entries from saved ladder files after
97
+ a rule change.
98
+
99
+ Both tiers use paired speedups (same instance, both solvers). A dimension is dropped when more than half of the
100
+ BASELINE's instances time out (its cost there is unmeasured) or when either solver costs under 0.5 s. A dimension is
101
+ NOT dropped because the submission failed there: failures count at the budget cost and pull the pooled speedup down.
102
+ (An earlier rule dropped dimensions on submission timeouts too; a solver that failed every instance at dims >= 66
103
+ then earned the same credit as one that solved them all. Fixed, with regression tests.) `solvers/fixed_overhead_ref.py` (reference with 1 CPU-second of fixed setup cost) shows a
104
+ fading advantage earning < 1 bit at the top dimensions (test_extrapolate.py).
105
+
106
+ ## What this does NOT guarantee
107
+ - The credit is a measured lower bound at the dimensions the ladder reaches (about 60-72 for GM). It says nothing
108
+ certain about NIST-scale dimensions; the extrapolated tier is a model, not a proof.
109
+ - CPU seconds depend on hardware; judges must run baseline and submission on the same pinned machine.
110
+ - The Docker sandbox is a container, not a VM, and its current meter is unverified (see Cost). Run it on a dedicated
111
+ judge machine with a pinned core.
112
+ - "dimension" means different things in the two families (GM dimension vs 16k+1), so compare within a family only.
113
+ - A speedup is certifiable only where the faster solver still costs >= 0.5 s (fixed-cost floor) and the baseline mostly
114
+ finishes within the cap; a speedup of a few tenths of a bit is below what 1.5-2.8 bits of per-instance noise allows.
115
+
116
+ ## Accelerator (GPU) workloads: a weaker, device-pinned tier (gpu_meter.py)
117
+ Every cost in the CPU path is CPU time of a process tree the meter can bill (`getrusage` / the root meter), or, for counted certificates, a Python bytecode count.
118
+ Neither sees a GPU: kernels run asynchronously on a device whose time is not in the process's CPU time, the Docker sandbox image has no device passthrough, and no
119
+ deterministic instruction counter exists for GPU kernels. So a GPU claim cannot be proved, only MEASURED by a trusted operator, and `gpu_meter.py` does that and nothing
120
+ more: the wall-clock time to the answer on a PINNED device model + driver, with ladders that `extrapolate.py` refuses to compare across devices. The PARENT does the timing:
121
+ it writes a request line to a child that has loaded the solver and been warmed up on public instances, reads the reply line, and takes the elapsed time on its own clock, so
122
+ the child supplies no timing and cannot under-report; each timed run is a different instance, so nothing can be cached across repetitions; one-time setup (imports, loading
123
+ a model) is untimed and per-instance work must happen in the solve call. A measurement is rejected, not kept, if another process shares the measured GPU (checked before and
124
+ after the run) or the GPU reports a hardware or thermal slowdown while it runs. The ladder can be fixed in advance (`plan`: family, sizes, instances per size, budget, seed
125
+ commitment, device, with a hash to publish first) and a ladder verifies against the plan only if it is exactly the planned one.
126
+ Limits, stated in every verdict: it is an operator's clock on an operator's box (nothing proves which device ran it, that clocks were not locked or raised, or that the box
127
+ was not tuned for one solver), nobody can re-execute a time exactly (`verify --rerun` re-measures single runs on the same device model and agrees within a factor of 1.3), the
128
+ solver runs unsandboxed, and a result says nothing about any other device. The parent-side rules are tested against a fake nvidia-smi and a fake transport with a fake clock,
129
+ and the process transport against tiny real children; the child (`gpu_runner.py`) and the torch solvers were written without a GPU and have never run. The CPU domains
130
+ (`aiinfer`, `aidecode`) remain the evidence that carries full weight.
131
+
132
+ ## Statistical power of the earlier DIMENSION-POOLED estimator (superseded as the credit; kept as an informational line)
133
+ NOTE: everything in this section describes pooling by dimension. The credit is now the difficulty-stratified claim; its
134
+ real-data checks and real-noise power are in "Real-data checks" above.
135
+ (simulated; Gaussian per-instance noise sd 2.2 bits, the measured range is 1.5-2.8)
136
+ 12 instances/dimension, dims 56..72 step 2, effect rising 0.2 -> 5.5 bits (the tuned-vs-baseline trend measured):
137
+ | claim | zero effect credited >= 1 bit | tuned-like effect credited >= 1 bit | constant 0.5-bit effect |
138
+ |---|---|---|---|
139
+ | Tier 1 measured | 0/150 (CI 0-2%) | 139/150 = 93% (CI 87-96%) | 0/150 |
140
+ | Tier 2 extrapolated | 0/120 | 2/120 = 2% (n=12), 50% only at 64/dim | n/a |
141
+ The Tier 2 claim was nearly unusable at realistic noise, which is why Tier 1 is the credit. These are simulations of
142
+ the estimator under an assumed noise model, not measurements of the real pipeline; real noise is heavier-tailed.
143
+
144
+ **Caveat: the Tier-1 power above does NOT transfer to the real ladder as configured.** The simulation assumed the top
145
+ three usable dimensions are uncensored and reach 72, where the tuned-like effect is ~4.7 bits. In real data the
146
+ baseline times out on most instances at the top dimensions (cap 60 s), those dimensions are dropped, and the usable
147
+ ones sit where the effect is small. Real-data check (GM v1, practice seed, tuned vs baseline, 12 instances/dim; HISTORICAL: these ladders predate the `instance_version` field and
148
+ the current estimator refuses them, so these numbers cannot be re-scored today and are kept for the reasoning, not as current results): usable
149
+ dims {60, 65}; pooled median 0.23 bits, one-sided lower bound -0.49, i.e. credit 0, although the same solver is 4.6 bits
150
+ faster at dim 70 (censored, dropped). Certifying a real speedup therefore needs a CPU budget large enough that the
151
+ baseline mostly finishes at the dimensions where the effect is large (budget vs dimension is the binding constraint),
152
+ or a baseline that scales more gently. Treat "93%" as the estimator's power given such data, not as the pipeline's
153
+ power. A second real-data run (GM, fresh seed, 12 instances/dim, 60 s cap, fixed rules) agrees: usable dims {64, 66, 70}
154
+ (68 and 72 dropped for 58% baseline timeouts), per-dimension speedups 0.7 / 0.17 / 1.31 bits, credit 0.0.
155
+ **Minimum detectable effect (simulated power curve).** A constant true effect at every dimension, per-instance noise
156
+ sd 2.2 bits (the measured range is 1.5-2.8), costs high enough that the 0.5 s floor does not bind, 120 trials per cell;
157
+ cell = probability that the Tier-1 credit is >= 1 bit:
158
+ | true effect | 1.0 bit | 1.5 bits | 2.0 bits | 2.5 bits | 3.0 bits |
159
+ |---|---|---|---|---|---|
160
+ | n = 12 per dimension | 10% | 35% | 75% | 93% | 100% |
161
+ | n = 24 per dimension | 9% | 58% | 91% | 100% | 100% |
162
+ A back-of-envelope formula (credit = median - 1.645 x 1.25 x sigma / sqrt(3n)) puts the break-even at about 1.8 bits
163
+ (n=12) and 1.5 bits (n=24); that is the ~50% point, NOT a power guarantee. For 80% power plan on about 2.1 bits at n=12
164
+ and about 1.8 bits at n=24 (hence gm official n=24). A real speedup of 0.5-1 bit is inconclusive by design, and the
165
+ honest tuned-vs-unpruned pair (about 0.2-1.3 bits at the dimensions where the baseline finishes) sits below that.
166
+ With floor and censoring effects the real thresholds are higher than these clean-case figures.
167
+ Two independent runs of the same baseline differ by only +0.03 bits median (worst dimension +0.08), so timing
168
+ drift is not the limiting factor; per-instance difficulty noise (1.5-2.8 bits) and censoring are.
169
+
170
+ ## Parallel local runs (`harness.py --jobs K`, local mode only)
171
+ The official GM config is hours of CPU, so local runs can use K concurrent instances. CPU time is each child's own
172
+ `wait4` rusage, so concurrent runs do not corrupt each other's accounting. Concurrency does slow each process a little,
173
+ and that varies between runs (absolute time +0.01 to +0.10 bits at 4-10 jobs on this machine). What matters for the credit
174
+ is the SPEEDUP estimate, a ratio in which a uniform slowdown cancels: `drift_check.py` measured it shifting by only
175
+ -0.02 bits (sd 0.04) at 4, 6 and 8 jobs, baseline vs tuned, same instances, against a sequential noise floor of 0.01.
176
+ Both ladders of a comparison must use the same `--jobs` (`extrapolate.py` refuses otherwise); the sandbox path is
177
+ sequential only. Re-run `drift_check.py --solver ... --solver2 ...` on a new machine before using --jobs there.
178
+
179
+ ## Real-data checks of the difficulty-stratified claim
180
+ (`extrapolate.py`; `power.py` for the noise/power numbers; baseline = baseline_bkz, submission = tuned_bkz unless stated)
181
+ | data | result |
182
+ |---|---|
183
+ | GM v2, fresh seed A, 12 inst/dim, dims 56-72, cap 60 s (in-sample: used to design the rule) | credit 0.1 bits (median 2.2) with the corrected joint bootstrap (1.3 with the withdrawn fixed-bin bootstrap); the credit pools the three hardest usable bins, 1-2 s (+0.54 bits, 5 instances), 2-4 s (+2.2, 8) and 16-32 s (+5.0, 7), each holding only 5-8 instances; the 0.5-1 s bin (+0.17, 15 instances) is usable but not among the three hardest, and 4-8 s (+1.9, 4 instances), 8-16 s and 32-64 s (1 each) are not usable; exploratory hard-pool claim >= 2.3 bits (n=13), also in-sample (its 4 s threshold was chosen after seeing this table) |
184
+ | GM v1, practice seed, independent older ladder (HISTORICAL: the files predate `instance_version`, so the current estimator refuses them and this row cannot be re-scored; kept as the record of what was found then) | credit 0.00 (median 0.6); effects +4 to +5.5 bits exist but sit in bins with 1-3 instances (< 5) |
185
+ | **GM v2, seed D, PRE-REGISTERED, 24 inst/dim, dims 64-72, cap 120 s, 6 jobs (OUT-OF-SAMPLE)** | **primary credit 2.6 bits (median 4.1)**: the credit pools the three hardest USABLE bins: 4-8 s (+2.56 bits, 11 instances), 16-32 s (+4.91, 13) and 32-64 s (+5.04, 5); the 2-4 s bin (+2.28, 10) is usable but not among the three hardest, and 64-128 s has 1 instance so it is not usable; criterion (a) >= 1.0 MET. Exploratory hard-pool claim 2.9 bits (30 instances, submission median cost 0.96 s): first out-of-sample support for promoting it. Cheapest bin (0.5-1 s) is -0.86 bits: the tuned solver's fixed cost makes it slower there, which is why the cost floor exists |
186
+ | seed D concurrency check (pre-registered, baseline only) | 25 baseline instances re-measured sequentially: median -0.050 bits, per-octave medians -0.041 to -0.074, none solved differently; criteria (median within +-0.10, no octave beyond +-0.20) MET |
187
+ | **factoring positive control, seed E, PRE-REGISTERED, sizes 58-64 bits, 24/size, cap 120 s, 6 jobs** | credit 4.1 bits (median 4.2) vs the calibrated ~4.4: credit criterion (a) met. **Concurrency criterion FAILED** (pre-registered, both ladders): baseline re-measured sequentially -0.033 bits (passes), numpy submission **-1.160 bits** (n=16, sd 0.08; limit 0.10). The sequential speedup is therefore probably ~5.2-5.6 bits (inference, not measured), and the match with 4.4 is partly a coincidence of two offsetting effects. Solver-dependent concurrency bias is a measurement hazard of `--jobs > 1` |
188
+ | seed D concurrency, tuned ladder (POST-HOC, not pre-registered) | all 113 tuned instances at >= 0.5 s re-measured: median -0.084 bits. The baseline is slowed by concurrency 0.050 bits and the tuned solver 0.084, so the speedup is UNDERestimated by about 0.034 bits: concurrency makes this credit slightly conservative |
189
+ | two independent runs of the SAME baseline (true effect 0) | inconclusive; every bin -0.03 to -0.05 bits (timing drift) |
190
+ | module-LWE v2, n=6 | inconclusive (1 usable bin) |
191
+ | fixed-overhead fallacy (reference = baseline + 1 s) | no credit: its +3.2-bit easy bin is below the floor |
192
+ | cheat on the PUBLIC seed (lookup table) | no credit: its cost is milliseconds, below the floor (commit-reveal is the real defence) |
193
+ Real per-instance noise inside difficulty bins is 0.5-1.5 bits (vs ~2.2 pooling by dimension). `power.py` with that real
194
+ noise on the seed-A ladder size (12/dim, 60 s cap), CORRECTED estimator, RE-RUN 2026-11-19 with the default settings (200 trials per
195
+ effect, seed 0; output saved as results/power_seedA_default.txt): false credit 0%, and P(credit >= 1) = 0.5% at a true effect of 1 bit,
196
+ 1.0% at 1.5 and 0% from 2 to 4 bits (inconclusive 28% at 2.5, 58% at 3, 66% at 4). An earlier version of this paragraph quoted "0% at every
197
+ effect" and inconclusive 61% / 79% at 3 and 4 bits; the re-run does not reproduce those exactly (the 4-bit figure is outside sampling noise).
198
+ The conclusion stands: that ladder is too small to certify anything with the corrected bound. (The figures 12% / 46% / 41% / 31% / 6% / 27% quoted in earlier versions came from the withdrawn
199
+ fixed-bin bootstrap.) Larger true effects get HARDER to certify here because
200
+ the submission's cost falls under the 0.5 s floor in the bins where the baseline is cheap, leaving too few hard
201
+ instances. This is inherent to a cost floor (subtracting each solver's measured fixed cost instead was rejected: a solver
202
+ could inflate that measurement on trivial instances and make every real instance look free). More instances per
203
+ dimension and a larger cap fill the hard bins; the gm official config (24/dim, 120 s) exists for that reason and has been run once
204
+ (seed D, above).
205
+
206
+ **Power at the OFFICIAL configuration (measured; re-run 2026-11-19).** `power.py` on the seed-D pair (24/dim, 120 s cap, REAL per-instance
207
+ noise, 200 trials per effect, default settings; output saved as results/power_seedD_default.txt). P(credit >= 1 bit) for a CONSTANT true
208
+ speedup: 0% at 0 bits (no false credit), 6.5% at 1, 70.5% at 1.5, 98% at 2, 100% at 2.5, 73% at 3 and **1% at 4 bits**. Power FALLS for
209
+ large effects at n=24. Mechanism (checked, not assumed): at 4 bits the point estimate is still exactly right (4.00 bits) but the 95% lower bound
210
+ is 0.00 in every trial, because only 2-3 hard bins remain usable (this ladder has 13, 5 and 1 instances above 16 s and none at 8-16 s), some
211
+ of them marginal (5 instances), and the joint bootstrap re-selects bins in every resample, so the bound collapses when the usable set is
212
+ unstable. That is a property of how many HARD baseline instances the ladder has, not of the estimator being blind.
213
+
214
+ **More instances help, up to a hard ceiling.** `power.py --copies K --resample` builds each synthetic ladder with K x the instances per size, drawn at
215
+ random WITH REPLACEMENT from the real seed-D baseline records (so the number of hard instances varies from ladder to ladder, as it would for a genuinely
216
+ larger ladder of fresh instances); outputs saved as results/power_seedD_copies2_resample.txt and _copies4_resample.txt, 100 trials per effect.
217
+ P(credit >= 1):
218
+
219
+ | true speedup | n=24 (official) | n=48 | n=96 |
220
+ |---|---|---|---|
221
+ | 3 bits | 73% | 99% | 100% |
222
+ | 4 bits | 1% | 68% | 100% |
223
+ | 5 bits | not run | 50% | 98% |
224
+ | 6 bits | not run | 0% (inconclusive 93%) | 0% (inconclusive 66%) |
225
+
226
+ (An earlier version of this section used `--copies` WITHOUT `--resample`, which repeats the observed instances exactly and so guarantees K times
227
+ seed D's hard-instance count: that gave 99% at 4 bits and 73% at 5 bits for n=48, and 100% for both at n=96, results/power_seedD_copies2.txt and
228
+ _copies4.txt. Those are the optimistic case where a larger ladder happens to have exactly the observed composition; the resampled numbers above are
229
+ the honest ones, and they differ materially at n=48.)
230
+
231
+ So a round that expects a 4 to 5-bit speedup should set `n` to about 96 (`ops.py round-new --n 96`; logged in the round's first entry, accepted by the
232
+ board because the rule is n >= the minimum; the price is proportionally more machine time): n=48 is borderline (68% at 4 bits, 50% at 5). But NO n
233
+ removes the ceiling: at 6 bits the submission
234
+ costs under 0.5 s on every instance the baseline finished in under 32 s, only the 32-128 s bins could be usable, and the credit needs at least two
235
+ usable bins with 5 instances each, so the result is inconclusive. **The practical ceiling of the solver-versus-solver design at a 120 s cap is
236
+ about 5 bits.** The formula that stood here before, "maximum certifiable effect is about log2(hardest-bin baseline cost / 0.5 s), at most ~7 bits
237
+ at a 60 s cap" is therefore not attainable and is WITHDRAWN as a statement of what can be certified. Caveats: the simulation applies one constant
238
+ effect to every instance, while a real solver's effect varies (tuned BKZ's grows with dimension and it WAS certified at 2.6 bits at n=24), and
239
+ `--copies` assumes a larger ladder would have the same cost distribution as seed D's (a different seed or solver pair could have more or fewer hard
240
+ instances), and every cell carries roughly +-10 points of simulation noise: `power.py` draws 150 bootstrap resamples per trial while the real estimator
241
+ draws 300, and a re-run at 300 (results/power_seedD_boot300_n24.txt and _n48_resample.txt, 100 trials) gave 64% at 3 bits for n=24 (73% at 150), and
242
+ 61% and 54% at 4 and 5 bits for n=48 (68% and 50% at 150), with the same regime in every cell and 0% false credit. So these show the shape of the limits, not a prediction for any particular
243
+ solver. Claims above about 5 bits need a different design (a baseline that is a model rather than a solver: EGG.md, "Ways forward" 2); not built.
244
+
245
+ **What to expect from the official run of the real pair (bootstrap of the real instances, n=24 only).** `power.py --real-pairs` resamples seed D's REAL
246
+ paired instances (the solver's real, instance-dependent speedup and its real failures) into 300 fresh ladders of the same size (boot 300; output saved as
247
+ results/power_seedD_realpairs_n24.txt). Credit per ladder: median 2.56 (the run itself gave 2.58), 5th percentile 1.15, 95th 2.94. P(one seed's credit >= 1
248
+ bit) = 96.7% and P(ALL THREE seeds >= 1 bit) = 91.0% (100 independent triples); the board credit is the minimum over seeds, so the likely official figure
249
+ sits near the low end of those. Caveats: this bootstraps ONE seed's instances, so it captures instance-sampling variation but not seed-to-seed or machine
250
+ variation; seed D ran with 6 concurrent jobs, which slowed the tuned solver more than the baseline (about 0.03 bits post hoc), so a sequential sandbox run
251
+ is, if anything, slightly better for this pair; the 1-bit threshold is the one registered for seed D, and the official run's own criteria are written when
252
+ it is registered. The same bootstrap scaled to 2x and 4x the instances is REFUSED by `power.py`: duplicating observed submission costs duplicates their
253
+ noise, so the bootstrap believes it has more information and reports an inflated credit (median 2.56 -> 2.94 -> 3.67); the invalid attempt is kept as
254
+ results/INVALID_power_seedD_realpairs_n48_duplicated_noise.txt and _n96_. Larger-ladder power comes from the synthetic mode above, where the noise is
255
+ redrawn for every instance.
256
+
257
+ **Exploratory hard-pool claim.** Because the bin-selection step is what makes small samples fragile, `extrapolate.py` also
258
+ prints a selection-free claim: pool every instance the baseline solved at cost >= 4 s (threshold fixed in advance as 8 x the
259
+ floor), require >= 12 of them and a submission median cost >= 0.5 s, and bootstrap the median normally. It is NOT the
260
+ credit; it is reported beside it so a validation run can say whether it should become the credit. Its 4 s threshold was
261
+ fixed after the author had seen seed A's bin table, so only runs pre-registered afterwards count as validation.
262
+
263
+ **Harness fixes found by audit:** the CPU cap used to be applied with `subprocess` `preexec_fn`, documented as unsafe when
264
+ spawning from several threads (possible deadlock); it is now set inside `runner.py` (soft = hard, set before the solver is
265
+ imported, so the solver cannot raise it). The hand-managed pipes also leaked 3 descriptors per run (macOS default limit
266
+ 256); they are closed explicitly. Both have tests (test_harness_local.py), including 72 spawns at 12 jobs and an
267
+ fd-leak check. `remeasure.py` re-runs a stratified sample of a finished ladder sequentially to check, per cost octave,
268
+ that concurrent timings of HARD instances match sequential ones (end-to-end check on the sequential seed-A baseline ladder:
269
+ 12 instances up to 38 s reproduced within +-0.06 bits each, median -0.012).
270
+
271
+ A second audit found the first fix introduced a hang: with an orphan holding the output pipe, the reader thread stayed
272
+ blocked in `read()` and my explicit `close()` then blocked on the stream's lock, so the harness never returned (reproduced:
273
+ no return after 90 s on a 2 s budget). Readers are now non-blocking and stoppable (select with a stop flag), the child runs
274
+ in its own session, and after a 2 s grace period everything left in that session is killed. Regression tests cover a
275
+ sleeping orphan and a CPU-burning orphan. KNOWN LIMIT of bare local mode: a solver that calls `setsid()` escapes that
276
+ cleanup and its daemon keeps running (and burning CPU) after the harness returns, unbilled; only the Linux/Docker meter
277
+ (unverified) can bill and kill such descendants. Local mode is for your own solvers.
278
+ Also found: on macOS a solver that ignores SIGXCPU is not killed at the CPU limit (Linux kills at the hard limit); it ran to
279
+ 18.95 s CPU on a 3 s budget before the wall timer stopped it. It is still billed and marked a timeout, so there is no credit
280
+ advantage, but it stalls a run. The local wall backstop is now 1.5 x budget + 10 s (a solver killed by it, e.g. one that only
281
+ sleeps, is labelled `timeout`, not `crash`). `--jobs` above the safe maximum is refused. The maximum counts PERFORMANCE
282
+ cores only, minus 2 (8 on this 10-P-core machine, the range the drift check validated): jobs beyond that would run on
283
+ efficiency cores or oversubscribe, distorting timings and making wall time exceed CPU time, which the backstop assumes.
284
+
285
+ ## Related work (checked by web search, not from memory)
286
+ The TU Darmstadt SVP Challenge and LWE Challenge are long-running fixed-instance lattice challenges with checkable
287
+ solutions and public records (SVP up to dimension ~180); they rank by capability (dimension solved), not by paired
288
+ bits-of-speedup over a reference under a committed seed. A 2024 benchmarking paper (arXiv 2408.00882) calls the Darmstadt
289
+ challenges the only concrete LWE benchmark and notes they omit standardized-parameter (small-secret, ring/module) cases.
290
+ This protocol is a complement to those, not the first lattice competition.
291
+
292
+ ## Reproducing the baseline
293
+ `python3 judge.py evaluate --seed <seed> --submission solvers/baseline_bkz.py` must credit ~0 bits. If it doesn't, the
294
+ verifier is biased and results should not be trusted.