guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Turn two ladders (baseline, submission) into conservative, falsifiable speedup claims. Prints, in order of authority:
|
|
2
|
+
|
|
3
|
+
1. MEASURED claim (THE CREDIT, difficulty-stratified): instances are grouped by how hard the BASELINE found them (octaves of
|
|
4
|
+
its cost from 0.5 s); pool the paired log2 speedup of the 3 hardest usable bins; report a one-sided 95% joint-bootstrap lower
|
|
5
|
+
bound. No model, no extrapolation. See measured_by_difficulty().
|
|
6
|
+
2. hard-pool claim (exploratory): every instance the baseline solved at >= 4 s, plain bootstrap.
|
|
7
|
+
3. dimension-pooled claim (informational, the older credit) and
|
|
8
|
+
4. EXTRAPOLATED claim (information only): four growth models (constant, linear, power-law, log-decay) fit to per-dimension
|
|
9
|
+
medians, each tested out-of-sample on the largest dimension; the minimum over surviving models, capped at 2x the largest
|
|
10
|
+
measured speedup. Statistically weak at realistic noise; never the credit.
|
|
11
|
+
|
|
12
|
+
Refuses (exit 1) to compare ladders that differ in seed commitment, instance version/family, budget, sandbox, image, --jobs,
|
|
13
|
+
CPU, or that do not cover exactly the same instances (a truncated run would otherwise give a credit on a biased subset).
|
|
14
|
+
"""
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import math
|
|
18
|
+
import random
|
|
19
|
+
import statistics
|
|
20
|
+
from collections import defaultdict
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
TOL = 1.0 # bits of out-of-sample prediction error tolerated
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load(path):
|
|
28
|
+
meta, by_dim = None, defaultdict(dict)
|
|
29
|
+
with open(path) as f:
|
|
30
|
+
for line in f:
|
|
31
|
+
r = json.loads(line)
|
|
32
|
+
if r.get("meta"):
|
|
33
|
+
meta = r
|
|
34
|
+
else:
|
|
35
|
+
by_dim[r["dim"]][r["id"]] = r
|
|
36
|
+
return meta, by_dim
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
CENSOR_MAX = 0.5 # drop a dimension if more than this fraction of BASELINE instances timed out
|
|
40
|
+
CAP = 2.0 # credited bits never exceed CAP x the largest speedup actually measured
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def coverage_mismatch(base, sub):
|
|
44
|
+
"""(ids only in the baseline, ids only in the submission) for two {dim: {id: record}} ladders. Both empty = the ladders
|
|
45
|
+
cover exactly the same instances. A truncated or mismatched pair must never be turned into a credit."""
|
|
46
|
+
ids_b = {(d, i) for d in base for i in base[d]}
|
|
47
|
+
ids_s = {(d, i) for d in sub for i in sub[d]}
|
|
48
|
+
return ids_b - ids_s, ids_s - ids_b
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def per_dim_times(by_dim, budget):
|
|
52
|
+
"""CPU time per instance; failures are censored at the budget (they count against the solver)."""
|
|
53
|
+
times, cens = {}, {}
|
|
54
|
+
for d, recs in by_dim.items():
|
|
55
|
+
times[d] = {i: (r["cpu_net"] if r["status"] == "solved" else budget) for i, r in recs.items()}
|
|
56
|
+
cens[d] = {i: r["status"] != "solved" for i, r in recs.items()}
|
|
57
|
+
return times, cens
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
FLOOR_ABS = 0.5 # seconds. A dimension counts only if BOTH solvers' median cost is at least this
|
|
61
|
+
|
|
62
|
+
def usable_dims(bc, sc, tb, ts):
|
|
63
|
+
"""Drop dimensions where (a) the BASELINE timed out on too many instances (its cost there is unmeasured), or (b)
|
|
64
|
+
either solver costs less than FLOOR_ABS.
|
|
65
|
+
(a) is deliberately one-sided. Dropping a dimension because the SUBMISSION failed there would remove its failures
|
|
66
|
+
from the claim: a solver that fails every large instance would earn the same credit as one that solves them (this was
|
|
67
|
+
a real defect). Submission failures stay in, costing the budget, which pulls the pooled speedup down.
|
|
68
|
+
(b) is the fixed-cost guard: interpreter/import/setup costs (harness ~0.1 s, a solver loading its tables ~0.1 s)
|
|
69
|
+
fade as work grows and would otherwise masquerade as a speedup that grows with n. The floor is absolute, not
|
|
70
|
+
relative to the solver's cheapest point, so a genuinely very fast solver is not penalized for being fast."""
|
|
71
|
+
common = sorted(d for d in set(bc) & set(sc) if set(bc[d]) & set(sc[d]))
|
|
72
|
+
ok, dropped = [], {}
|
|
73
|
+
for d in common:
|
|
74
|
+
ids = sorted(set(bc[d]) & set(sc[d]))
|
|
75
|
+
frac = sum(bc[d][i] for i in ids) / len(ids)
|
|
76
|
+
cheapest = min(statistics.median(tb[d].values()), statistics.median(ts[d].values()))
|
|
77
|
+
if frac > CENSOR_MAX:
|
|
78
|
+
dropped[d] = f"baseline timeouts {frac:.2f}"
|
|
79
|
+
elif cheapest < FLOOR_ABS:
|
|
80
|
+
dropped[d] = f"cost {cheapest:.3f}s < {FLOOR_ABS:g}s floor"
|
|
81
|
+
else:
|
|
82
|
+
ok.append(d)
|
|
83
|
+
return ok, dropped
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def speedup_table(base, sub, dims, rng=None):
|
|
87
|
+
"""Paired: median over instances of log2(baseline cost / submission cost) on the SAME instance."""
|
|
88
|
+
s = {}
|
|
89
|
+
for d in dims:
|
|
90
|
+
ids = sorted(set(base[d]) & set(sub[d]))
|
|
91
|
+
if not ids:
|
|
92
|
+
continue
|
|
93
|
+
if rng:
|
|
94
|
+
ids = [rng.choice(ids) for _ in ids]
|
|
95
|
+
s[d] = statistics.median(math.log2(base[d][i] / sub[d][i]) for i in ids)
|
|
96
|
+
return s
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def fit(model, ns, ss):
|
|
100
|
+
ns, ss = np.array(ns, float), np.array(ss, float)
|
|
101
|
+
if model == "constant":
|
|
102
|
+
a = ss.mean()
|
|
103
|
+
return lambda n: a
|
|
104
|
+
if model == "linear":
|
|
105
|
+
b, a = np.polyfit(ns, ss, 1)
|
|
106
|
+
return lambda n: a + b * n
|
|
107
|
+
if model == "logdecay":
|
|
108
|
+
b, a = np.polyfit(np.log(ns), ss, 1)
|
|
109
|
+
return lambda n: a + b * math.log(n)
|
|
110
|
+
if model == "power":
|
|
111
|
+
if (ss <= 0).any():
|
|
112
|
+
return None
|
|
113
|
+
b, la = np.polyfit(np.log(ns), np.log(ss), 1)
|
|
114
|
+
return lambda n: math.exp(la) * n ** b
|
|
115
|
+
raise ValueError(model)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
MODELS = ["constant", "linear", "power", "logdecay"]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def credited(s, target):
|
|
122
|
+
ds = sorted(s)
|
|
123
|
+
verdict = {}
|
|
124
|
+
for m in MODELS:
|
|
125
|
+
if len(ds) < 4:
|
|
126
|
+
verdict[m] = {"status": "too few dimensions"}
|
|
127
|
+
continue
|
|
128
|
+
f_in = fit(m, ds[:-1], [s[d] for d in ds[:-1]])
|
|
129
|
+
if f_in is None:
|
|
130
|
+
verdict[m] = {"status": "unfittable"}
|
|
131
|
+
continue
|
|
132
|
+
err = f_in(ds[-1]) - s[ds[-1]]
|
|
133
|
+
f_all = fit(m, ds, [s[d] for d in ds])
|
|
134
|
+
verdict[m] = {"holdout_err_bits": round(err, 2), "status": "ok" if abs(err) <= TOL else "rejected",
|
|
135
|
+
"at_target": round(f_all(target), 2) if f_all else None}
|
|
136
|
+
ok = [v["at_target"] for v in verdict.values() if v["status"] == "ok" and v["at_target"] is not None]
|
|
137
|
+
if not ok:
|
|
138
|
+
return verdict, None
|
|
139
|
+
return verdict, min(min(ok), CAP * max(s.values()))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
TOP_K = 3 # the measured claim pools the K largest usable dimensions
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def measured_lower(base, sub, dims, rng, boot=400, pct=5):
|
|
146
|
+
"""Tier 1, no extrapolation: pooled paired speedup over the largest usable dimensions, with a one-sided bootstrap
|
|
147
|
+
lower bound (resampling instances within each dimension). Needs >= 2 usable dimensions."""
|
|
148
|
+
top = sorted(dims)[-TOP_K:]
|
|
149
|
+
if len(top) < 2:
|
|
150
|
+
return None, top
|
|
151
|
+
|
|
152
|
+
def stat(resample):
|
|
153
|
+
vals = []
|
|
154
|
+
for d in top:
|
|
155
|
+
ids = sorted(set(base[d]) & set(sub[d]))
|
|
156
|
+
if resample:
|
|
157
|
+
ids = [rng.choice(ids) for _ in ids]
|
|
158
|
+
vals += [math.log2(base[d][i] / sub[d][i]) for i in ids]
|
|
159
|
+
return statistics.median(vals)
|
|
160
|
+
|
|
161
|
+
return (stat(False), float(np.percentile([stat(True) for _ in range(boot)], pct))), top
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
MIN_BIN = 5 # instances needed in a difficulty bin for it to count
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def difficulty_bins(tb, ts, bc):
|
|
168
|
+
"""Group instances by how hard the BASELINE found them: octaves of baseline cost starting at the 0.5 s floor.
|
|
169
|
+
Selection depends on the reference solver only, so it cannot bias the submission's side, and it is immune to the
|
|
170
|
+
non-monotone, heavy-tailed cost-vs-dimension behaviour that made dimension-based selection erratic. Instances where
|
|
171
|
+
the baseline timed out are excluded (their true cost is unknown: conservative); submission failures stay in at the
|
|
172
|
+
budget cost. Returns {octave index: [(baseline cost, submission cost), ...]}."""
|
|
173
|
+
bins = {}
|
|
174
|
+
for d in tb:
|
|
175
|
+
for i, cb in tb[d].items():
|
|
176
|
+
if i in ts.get(d, {}) and not bc[d][i] and cb >= FLOOR_ABS:
|
|
177
|
+
bins.setdefault(int(math.floor(math.log2(cb / FLOOR_ABS))), []).append((cb, ts[d][i]))
|
|
178
|
+
return bins
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _group(points):
|
|
182
|
+
"""[(baseline cost, submission cost)] -> {octave index: [points]} (octaves of baseline cost above the floor)."""
|
|
183
|
+
bins = {}
|
|
184
|
+
for cb, cs in points:
|
|
185
|
+
bins.setdefault(int(math.floor(math.log2(cb / FLOOR_ABS))), []).append((cb, cs))
|
|
186
|
+
return bins
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _usable_top(bins):
|
|
190
|
+
info = {k: {"lo": FLOOR_ABS * 2 ** k, "hi": FLOOR_ABS * 2 ** (k + 1), "n": len(v),
|
|
191
|
+
"median_sub_cost": round(statistics.median(c for _, c in v), 3),
|
|
192
|
+
"median_bits": round(statistics.median(math.log2(b / c) for b, c in v), 2),
|
|
193
|
+
"usable": len(v) >= MIN_BIN and statistics.median(c for _, c in v) >= FLOOR_ABS}
|
|
194
|
+
for k, v in bins.items()}
|
|
195
|
+
return info, sorted(k for k, x in info.items() if x["usable"])[-TOP_K:]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _pooled_median(bins, top):
|
|
199
|
+
return statistics.median(math.log2(b / c) for k in top for b, c in bins[k])
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def measured_by_difficulty(tb, ts, bc, rng, boot=400, pct=5):
|
|
203
|
+
"""Tier 1 (THE credit). Pooled median of the paired log2 speedup over the TOP_K hardest usable bins, with a one-sided
|
|
204
|
+
JOINT bootstrap lower bound: each resample redraws instances from the whole eligible pool and RE-SELECTS which bins are
|
|
205
|
+
usable and hardest, so the uncertainty of that selection is inside the bound (a resample with < 2 usable bins counts as 0
|
|
206
|
+
bits). A bin is usable when it holds >= MIN_BIN instances and the submission's median cost there is >= FLOOR_ABS
|
|
207
|
+
(fixed-cost guard). Needs >= 2 usable bins in the real data. (The earlier version resampled within fixed bins; by
|
|
208
|
+
simulation its bound exceeded the truth 5-9% of the time instead of <= 5%.)"""
|
|
209
|
+
pts = [p for k in difficulty_bins(tb, ts, bc).values() for p in k]
|
|
210
|
+
bins = _group(pts)
|
|
211
|
+
info, top = _usable_top(bins)
|
|
212
|
+
if len(top) < 2:
|
|
213
|
+
return None, top, info
|
|
214
|
+
|
|
215
|
+
def resampled():
|
|
216
|
+
b2 = _group([rng.choice(pts) for _ in pts])
|
|
217
|
+
_, t2 = _usable_top(b2)
|
|
218
|
+
return _pooled_median(b2, t2) if len(t2) >= 2 else 0.0
|
|
219
|
+
|
|
220
|
+
return (_pooled_median(bins, top), float(np.percentile([resampled() for _ in range(boot)], pct))), top, info
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
HARD_POOL_MIN_COST = 4.0 # seconds of baseline cost; fixed in advance (8 x the 0.5 s floor), never tuned to data
|
|
224
|
+
HARD_POOL_MIN_N = 12
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def hard_pool_claim(tb, ts, bc, rng, boot=400, pct=5):
|
|
228
|
+
"""EXPLORATORY second claim with no selection step at all: pool every instance the baseline solved at a cost >=
|
|
229
|
+
HARD_POOL_MIN_COST and report the one-sided bootstrap lower bound of the median paired log2 speedup. Inconclusive if
|
|
230
|
+
there are < HARD_POOL_MIN_N such instances or the submission's median cost on them is under the 0.5 s floor.
|
|
231
|
+
Returns ((median, lower bound) or None, n, median submission cost)."""
|
|
232
|
+
pts = [(cb, ts[d][i]) for d in tb for i, cb in tb[d].items()
|
|
233
|
+
if i in ts.get(d, {}) and not bc[d][i] and cb >= HARD_POOL_MIN_COST]
|
|
234
|
+
if len(pts) < HARD_POOL_MIN_N:
|
|
235
|
+
return None, len(pts), None
|
|
236
|
+
med_sub = statistics.median(c for _, c in pts)
|
|
237
|
+
if med_sub < FLOOR_ABS:
|
|
238
|
+
return None, len(pts), med_sub
|
|
239
|
+
vals = [math.log2(b / c) for b, c in pts]
|
|
240
|
+
boots = [statistics.median(rng.choice(vals) for _ in vals) for _ in range(boot)]
|
|
241
|
+
return (statistics.median(vals), float(np.percentile(boots, pct))), len(pts), med_sub
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def main():
|
|
245
|
+
ap = argparse.ArgumentParser()
|
|
246
|
+
ap.add_argument("--baseline", required=True)
|
|
247
|
+
ap.add_argument("--submission", required=True)
|
|
248
|
+
ap.add_argument("--target-dim", type=int, default=512)
|
|
249
|
+
ap.add_argument("--boot", type=int, default=300)
|
|
250
|
+
a = ap.parse_args()
|
|
251
|
+
mb, base = load(a.baseline)
|
|
252
|
+
ms, sub = load(a.submission)
|
|
253
|
+
if mb["commit"] != ms["commit"]:
|
|
254
|
+
raise SystemExit("REFUSED: ladders were run on different instance sets (commitments differ)")
|
|
255
|
+
if mb.get("family", "gm") != ms.get("family", "gm") or mb.get("sandbox") != ms.get("sandbox"):
|
|
256
|
+
raise SystemExit("REFUSED: ladders differ in instance family or sandbox mode")
|
|
257
|
+
if mb.get("instance_version") != ms.get("instance_version") or mb.get("instance_version") is None:
|
|
258
|
+
raise SystemExit("REFUSED: ladders have different (or missing) instance_version: same seed, different "
|
|
259
|
+
"instances. Re-run both ladders with the current harness.")
|
|
260
|
+
if mb["budget"] != ms["budget"]:
|
|
261
|
+
raise SystemExit("REFUSED: ladders ran with different CPU budgets (censoring would not be comparable)")
|
|
262
|
+
if mb.get("image") != ms.get("image"):
|
|
263
|
+
raise SystemExit("REFUSED: ladders ran in different sandbox images")
|
|
264
|
+
if mb.get("jobs", 1) != ms.get("jobs", 1): # a uniform concurrency slowdown cancels in the ratio only if shared
|
|
265
|
+
raise SystemExit("REFUSED: ladders ran with different --jobs (concurrency changes absolute timings)")
|
|
266
|
+
if mb.get("cpu") != ms.get("cpu"): # both absent (older files) compare equal; one absent is a mismatch
|
|
267
|
+
raise SystemExit(f"REFUSED: ladders ran on different CPUs ({mb.get('cpu')!r} vs {ms.get('cpu')!r}); cost "
|
|
268
|
+
"ratios across machines are meaningless")
|
|
269
|
+
only_b, only_s = coverage_mismatch(base, sub)
|
|
270
|
+
if only_b or only_s:
|
|
271
|
+
raise SystemExit(f"REFUSED: the ladders do not cover the same instances ({len(only_b)} only in the baseline, "
|
|
272
|
+
f"{len(only_s)} only in the submission): a truncated or mismatched run would give a credit computed "
|
|
273
|
+
"on a biased subset (typically the early, easier instances)")
|
|
274
|
+
budget = mb["budget"]
|
|
275
|
+
(tb, bc), (ts, sc) = per_dim_times(base, budget), per_dim_times(sub, budget)
|
|
276
|
+
dims, dropped = usable_dims(bc, sc, tb, ts)
|
|
277
|
+
if dropped:
|
|
278
|
+
print(f"dimensions dropped from fitting: {dropped}")
|
|
279
|
+
s = speedup_table(tb, ts, dims)
|
|
280
|
+
verdict, point = credited(s, a.target_dim)
|
|
281
|
+
boots = []
|
|
282
|
+
rng = random.Random(0)
|
|
283
|
+
for _ in range(a.boot):
|
|
284
|
+
_, p = credited(speedup_table(tb, ts, dims, rng), a.target_dim)
|
|
285
|
+
boots.append(p)
|
|
286
|
+
lo = float(np.percentile([b if b is not None else 0.0 for b in boots], 5)) # a resample with no surviving model earns 0
|
|
287
|
+
solved = {d: sum(r["status"] == "solved" for r in sub[d].values()) for d in sorted(sub)}
|
|
288
|
+
wrong = sum(r["status"] == "wrong" for d in sub for r in sub[d].values())
|
|
289
|
+
print(f"submission={ms['solver']} baseline={mb['solver']} commit={ms['commit'][:12]}")
|
|
290
|
+
print("measured speedup (bits) per dim:", {d: round(v, 2) for d, v in s.items()})
|
|
291
|
+
print("submission solved per dim:", solved, f" wrong answers: {wrong}")
|
|
292
|
+
for m, v in verdict.items():
|
|
293
|
+
print(f" {m:9s} " + str({k: (float(x) if hasattr(x, 'item') else x) for k, x in v.items()}))
|
|
294
|
+
dmeas, dtop = measured_lower(tb, ts, dims, random.Random(1))
|
|
295
|
+
dim_bits = None if dmeas is None else max(0.0, dmeas[1])
|
|
296
|
+
print("dimension-pooled claim (informational): " + ("INCONCLUSIVE" if dim_bits is None else
|
|
297
|
+
f">= {dim_bits:.1f} bits over dims {dtop} (median {dmeas[0]:.1f})"))
|
|
298
|
+
meas, top, info = measured_by_difficulty(tb, ts, bc, random.Random(1))
|
|
299
|
+
print("difficulty bins (baseline cost octave): " + "; ".join(
|
|
300
|
+
f"[{x['lo']:g},{x['hi']:g})s n={x['n']} sub-median {x['median_sub_cost']}s {x['median_bits']:+} bits"
|
|
301
|
+
f"{'' if x['usable'] else ' (unusable)'}" for _, x in sorted(info.items())))
|
|
302
|
+
if meas is None:
|
|
303
|
+
print(f"MEASURED claim (credit): INCONCLUSIVE (needs >= 2 usable difficulty bins; usable: {top})")
|
|
304
|
+
measured_bits = None
|
|
305
|
+
else:
|
|
306
|
+
measured_bits = max(0.0, meas[1])
|
|
307
|
+
spans = [f"[{info[k]['lo']:g},{info[k]['hi']:g})s" for k in top]
|
|
308
|
+
print(f"MEASURED claim (credit): >= {measured_bits:.1f} bits pooled over the hardest usable baseline-cost bins "
|
|
309
|
+
f"{', '.join(spans)} (median {meas[0]:.1f}, one-sided 95% lower bound {meas[1]:.1f}); no extrapolation")
|
|
310
|
+
hp, hp_n, hp_sub = hard_pool_claim(tb, ts, bc, random.Random(1))
|
|
311
|
+
print(f"hard-pool claim (EXPLORATORY, baseline cost >= {HARD_POOL_MIN_COST:g}s, n={hp_n}): " + (
|
|
312
|
+
f"INCONCLUSIVE (needs >= {HARD_POOL_MIN_N} instances and submission median cost >= {FLOOR_ABS:g}s; got median "
|
|
313
|
+
f"{hp_sub if hp_sub is None else round(hp_sub, 2)})" if hp is None else
|
|
314
|
+
f">= {max(0.0, hp[1]):.1f} bits (median {hp[0]:.1f}, submission median cost {hp_sub:.2f}s)"))
|
|
315
|
+
if point is None or lo is None:
|
|
316
|
+
print(f"EXTRAPOLATED @ n={a.target_dim}: INCONCLUSIVE (no model survived out-of-sample + bootstrap)")
|
|
317
|
+
else:
|
|
318
|
+
print(f"EXTRAPOLATED @ n={a.target_dim} (model-dependent, not credit): >= {max(0.0, min(point, lo)):.1f} bits "
|
|
319
|
+
f"(min over surviving models {point:.1f}; bootstrap p5 {lo:.1f})")
|
|
320
|
+
json.dump({"measured": measured_bits, "measured_bins": [[info[k]["lo"], info[k]["hi"]] for k in top],
|
|
321
|
+
"hard_pool": None if hp is None else {"median": hp[0], "lower": hp[1], "n": hp_n},
|
|
322
|
+
"measured_by_dim": dim_bits, "measured_dims": dtop, "s": s, "dropped": dropped, "verdict": {m: {k: (float(x) if hasattr(x, "item") else x) for k, x in v.items()} for m, v in verdict.items()}, "point": point, "p5": lo}, open(a.submission + ".claim.json", "w"), indent=1)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
if __name__ == "__main__":
|
|
326
|
+
main()
|
guldo/_src/families.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Domain registry. A domain (family) is (derive(master, size, idx) -> inst, verify(inst, answer) -> bool) plus per-family
|
|
2
|
+
configuration below. An instance dict must carry "args": the JSON-serializable list the solver receives as solve(*args).
|
|
3
|
+
Lattice families supply (basis, bound2) as their args; factoring supplies (N,). Everything downstream of this file (harness,
|
|
4
|
+
estimator, judge, report, portal) is domain-neutral. To add a domain: write instances_<x>.py, then fill in every table here
|
|
5
|
+
(test_domains.py checks the configuration is complete). Or declare the domain and let compile_domain.py fill the tables (COMPILER.md)."""
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
import instances
|
|
10
|
+
import instances_factor
|
|
11
|
+
import instances_mlwe
|
|
12
|
+
|
|
13
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
14
|
+
DECLARED = {} # name -> {"dir", "compiled", "decl_sha256"} for domains registered from domains/<name>/ by declared.load_all (see COMPILER.md)
|
|
15
|
+
DECLARED_ERRORS = {} # directory name -> why a declaration under domains/ was NOT registered (stale compile, bad declaration, name clash)
|
|
16
|
+
|
|
17
|
+
FAMILIES = {
|
|
18
|
+
"gm": (instances.derive, instances.verify),
|
|
19
|
+
"mlwe": (instances_mlwe.derive, instances_mlwe.verify),
|
|
20
|
+
"factor": (instances_factor.derive, instances_factor.verify),
|
|
21
|
+
}
|
|
22
|
+
VERSIONS = {"gm": "svp-gm-v2", "mlwe": "mlwe-v2", "factor": "factor-v1"} # bump when a derive() changes; recorded in every results file
|
|
23
|
+
# Official-run thresholds per family. GM needs more instances and a larger cap: at (12 inst, 60 s) the baseline times
|
|
24
|
+
# out at dims >= 68 and an honest ~1-bit pruning speedup is not certifiable (measured twice on real data).
|
|
25
|
+
OFFICIAL = {"gm": {"n": 24, "budget": 120}, "mlwe": {"n": 12, "budget": 60}, "factor": {"n": 24, "budget": 120}}
|
|
26
|
+
# Ladders are tuned to the DEFAULT baseline (baseline_bkz): dims below its 0.5 s cost floor can never be used. A different
|
|
27
|
+
# baseline needs its own ladder (judge.py evaluate --dims ...).
|
|
28
|
+
DEFAULT_DIMS = {
|
|
29
|
+
"gm": [64, 66, 68, 70, 72],
|
|
30
|
+
"mlwe": [97, 105, 113, 121, 129, 137],
|
|
31
|
+
"factor": [58, 60, 62, 64], # bits of N; trial division costs ~1 s at 52 bits and doubles every 2 bits
|
|
32
|
+
}
|
|
33
|
+
# The reference solver instances are compared against, per family (paths relative to the project directory).
|
|
34
|
+
DEFAULT_BASELINE = {"gm": "solvers/baseline_bkz.py", "mlwe": "solvers/baseline_bkz.py", "factor": "solvers/factor_trial_py.py"}
|
|
35
|
+
# Small public sizes for `submit_check.py`: quick well-formedness checks, not performance.
|
|
36
|
+
PRACTICE_DIMS = {"gm": [24, 30, 36], "mlwe": [33, 49], "factor": [36, 40]}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def derive(family, master, dim, idx):
|
|
40
|
+
inst = FAMILIES[family][0](master, dim, idx)
|
|
41
|
+
inst["family"] = family
|
|
42
|
+
if "args" not in inst: # lattice families: the solver receives (basis, bound2)
|
|
43
|
+
inst["args"] = [inst["basis"], inst["bound2"]]
|
|
44
|
+
return inst
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def verify(inst, v):
|
|
48
|
+
return FAMILIES[inst.get("family", "gm")][1](inst, v)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
import declared # noqa: E402 (needs the tables above to exist)
|
|
52
|
+
|
|
53
|
+
declared.load_all(sys.modules[__name__]) # compiled declarations under domains/ become families like the built-in three
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Build the red-team lookup table by solving the public practice instances once (run only when no timing ladder is running)."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "solvers"))
|
|
8
|
+
import families
|
|
9
|
+
import instances
|
|
10
|
+
import tuned_bkz
|
|
11
|
+
|
|
12
|
+
def main():
|
|
13
|
+
import argparse
|
|
14
|
+
ap = argparse.ArgumentParser()
|
|
15
|
+
ap.add_argument("--n", type=int, default=families.OFFICIAL["gm"]["n"], help="instances per dimension (default: the official n)")
|
|
16
|
+
ap.add_argument("--force", action="store_true", help="overwrite an existing table")
|
|
17
|
+
a = ap.parse_args()
|
|
18
|
+
out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "solvers", "cheat_table.json")
|
|
19
|
+
if os.path.exists(out) and not a.force:
|
|
20
|
+
raise SystemExit(f"REFUSED: {out} exists (pass --force to overwrite)")
|
|
21
|
+
table = {}
|
|
22
|
+
for dim in families.DEFAULT_DIMS["gm"]:
|
|
23
|
+
for idx in range(a.n):
|
|
24
|
+
inst = families.derive("gm", instances.PRACTICE_SEED, dim, idx)
|
|
25
|
+
v = tuned_bkz.solve(inst["basis"], inst["bound2"])
|
|
26
|
+
if v is None: # the cheater is allowed to know the judge-side planted answer for PUBLIC practice instances
|
|
27
|
+
v = inst["planted"]
|
|
28
|
+
assert instances.verify(inst, v), (dim, idx)
|
|
29
|
+
table[hashlib.sha256(json.dumps(inst["basis"]).encode()).hexdigest()] = v
|
|
30
|
+
print(dim, idx, flush=True)
|
|
31
|
+
with open(out, "w") as f:
|
|
32
|
+
json.dump(table, f)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
if __name__ == "__main__":
|
|
36
|
+
main()
|