guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/recheck.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Recompute a compiled domain's checkable claims from its own record, and (optionally) re-measure its profile on this machine.
|
|
2
|
+
|
|
3
|
+
python3 compile_domain.py recheck NAME seconds to a minute: everything that is arithmetic, recomputed and compared
|
|
4
|
+
python3 compile_domain.py recheck NAME --measure 3 also re-measure each ladder size with 3 instances on THIS machine and report the gap
|
|
5
|
+
|
|
6
|
+
What it proves: the declaration and reference are the hashed files; the validation checks still give the recorded verdicts; the ladder, n and
|
|
7
|
+
CPU-hour figures follow from the recorded plan; and the whole power table (seeded) comes out identical. What it cannot prove: that the
|
|
8
|
+
recorded profile timings, or the recorded self-test, were honestly measured. `--measure` does not prove them either: it shows how far this
|
|
9
|
+
machine's timings sit from the recorded ones, which is the evidence a reader needs about machine-to-machine spread. The self-test is not
|
|
10
|
+
recomputed (it takes hours); `compile_domain.py selftest NAME` reruns it.
|
|
11
|
+
|
|
12
|
+
SAFETY: this EXECUTES the declaration's decl.py and the reference solver on this machine, unsandboxed (validation derives instances with the
|
|
13
|
+
declaration; --measure runs the reference). Do not recheck a declaration you did not read, except in a throwaway VM.
|
|
14
|
+
"""
|
|
15
|
+
import math
|
|
16
|
+
import os
|
|
17
|
+
import statistics
|
|
18
|
+
|
|
19
|
+
import compile_domain as C
|
|
20
|
+
import declared
|
|
21
|
+
|
|
22
|
+
MEASURE_WARN_BITS = 1.0 # this machine's median differs from the recorded one by more than this many bits: say so
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _same_curve(a, b):
|
|
26
|
+
return C.canon(a) == C.canon(b)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def recheck(dirpath, measure_k=0, log=C._log):
|
|
30
|
+
out = []
|
|
31
|
+
mod, ref, res = C.parse(dirpath)
|
|
32
|
+
out += res
|
|
33
|
+
if C.worst(res) == "fail":
|
|
34
|
+
return out
|
|
35
|
+
compiled = declared.read_compiled(dirpath)
|
|
36
|
+
problems = declared.freshness_problems(dirpath, mod, compiled)
|
|
37
|
+
out.append(C.R("record is current", "fail" if problems else "pass", "; ".join(problems) if problems else "decl.py and the reference hash as recorded"))
|
|
38
|
+
if problems:
|
|
39
|
+
return out
|
|
40
|
+
|
|
41
|
+
plan, pw, official = compiled["plan"], compiled["power"], compiled["official"]
|
|
42
|
+
bad = []
|
|
43
|
+
if list(compiled["dims"]) != list(plan["sizes"]):
|
|
44
|
+
bad.append(f"dims {compiled['dims']} != plan sizes {plan['sizes']}")
|
|
45
|
+
if official["n"] != pw["n"]:
|
|
46
|
+
bad.append(f"official n {official['n']} != power n {pw['n']}")
|
|
47
|
+
if official["budget"] != plan["cap"]:
|
|
48
|
+
bad.append(f"official budget {official['budget']} != plan cap {plan['cap']}")
|
|
49
|
+
cost = C.cpu_hours(plan, pw["n"])
|
|
50
|
+
if cost != compiled["cpu_hours"]:
|
|
51
|
+
bad.append(f"CPU-hours recompute to {cost}, recorded {compiled['cpu_hours']}")
|
|
52
|
+
out.append(C.R("plan arithmetic", "fail" if bad else "pass",
|
|
53
|
+
"; ".join(bad) if bad else f"ladder, n={pw['n']}, cap {plan['cap']} s and CPU-hours ({cost['expected']} expected, {cost['worst_case']} worst) follow from the record"))
|
|
54
|
+
|
|
55
|
+
log("recheck: re-running the validation checks ...")
|
|
56
|
+
fresh = C.validate(mod, dirpath, ref)
|
|
57
|
+
old = {r["check"]: r["status"] for r in compiled["validation"]}
|
|
58
|
+
new = {r["check"]: r["status"] for r in fresh}
|
|
59
|
+
parsed = {r["check"] for r in out} # the parse pass's own checks are recorded beside validation's, but validate() does not regenerate them
|
|
60
|
+
diff = [f"{k}: recorded {old.get(k, 'absent')}, now {new.get(k, 'absent')}" for k in sorted(set(old) | set(new))
|
|
61
|
+
if k not in parsed and old.get(k) != new.get(k)]
|
|
62
|
+
out.append(C.R("validation reproduces", "fail" if diff else "pass",
|
|
63
|
+
"; ".join(diff) if diff else f"{len(new)} checks give the recorded verdicts (the recorded ones were run at compile time with the same seeds)"))
|
|
64
|
+
|
|
65
|
+
log("recheck: recomputing the power table (seeded) ...")
|
|
66
|
+
again = C.power_plan(plan, target_effect=pw["target_effect"], target_power=pw["target_power"], pair_noise=pw["pair_noise"],
|
|
67
|
+
triples=pw["triples"], boot=pw["boot"], log=lambda *a: None)
|
|
68
|
+
if _same_curve(again, pw):
|
|
69
|
+
out.append(C.R("power table reproduces", "pass", f"identical: n={again['n']}, certifies {again['min_certifiable_effect']} to {again['max_certifiable_effect']} bits, "
|
|
70
|
+
f"zero-effect false credit {100 * again['curve']['0']['single']:.1f}%"))
|
|
71
|
+
else:
|
|
72
|
+
keys = [k for k in again if C.canon(again[k]) != C.canon(pw.get(k))]
|
|
73
|
+
out.append(C.R("power table reproduces", "fail", f"differs in {keys}: the record was not produced by this estimator and plan, or the Python/random version differs"))
|
|
74
|
+
|
|
75
|
+
st = compiled.get("selftest") or {}
|
|
76
|
+
out.append(C.R("self-test", "pass" if st.get("status") == "pass" else "warn",
|
|
77
|
+
f"recorded status {st.get('status', 'absent')} at {st.get('at', '?')}; NOT recomputed (hours): `python3 compile_domain.py selftest {os.path.basename(os.path.abspath(dirpath))}`"))
|
|
78
|
+
|
|
79
|
+
if measure_k > 0:
|
|
80
|
+
out += _measure(mod, ref, compiled, measure_k, log)
|
|
81
|
+
else:
|
|
82
|
+
out.append(C.R("profile timings", "warn", "recorded, not re-measured: add --measure K to compare this machine's costs with the recorded ones"))
|
|
83
|
+
return out
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _positive(x):
|
|
87
|
+
return isinstance(x, (int, float)) and not isinstance(x, bool) and math.isfinite(x) and x > 0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _usable_costs(raw):
|
|
91
|
+
"""{instance index: (cost, solved)} from a record's `costs`, keeping only well-formed entries. A compiled record may come from someone else, so
|
|
92
|
+
anything malformed (wrong shape, non-positive or non-finite cost) is dropped, never raised on and never divided by."""
|
|
93
|
+
out = {}
|
|
94
|
+
if not isinstance(raw, list):
|
|
95
|
+
return out
|
|
96
|
+
for e in raw:
|
|
97
|
+
if isinstance(e, (list, tuple)) and len(e) == 3:
|
|
98
|
+
i, c, ok = e
|
|
99
|
+
if isinstance(i, int) and not isinstance(i, bool) and _positive(c) and isinstance(ok, bool):
|
|
100
|
+
out[i] = (c, ok)
|
|
101
|
+
return out
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _measure(mod, ref, compiled, k, log):
|
|
105
|
+
prof = compiled["profile"]
|
|
106
|
+
measure = C.default_measure(mod, ref, C.PROFILE_SEED, compiled["plan"]["cap"], prof["overhead"])
|
|
107
|
+
res, gaps = [], []
|
|
108
|
+
for s in compiled["dims"]:
|
|
109
|
+
log(f"recheck: measuring size {s} ({k} instances) ...")
|
|
110
|
+
recs = [measure(s, i) for i in range(k)]
|
|
111
|
+
mine = statistics.median(C._cost(r, compiled["plan"]["cap"]) for r in recs)
|
|
112
|
+
entry = prof["sizes"].get(str(s), {})
|
|
113
|
+
recorded = _usable_costs(entry.get("costs"))
|
|
114
|
+
# the SAME instances (instance i is a function of the public profiling seed), and only those SOLVED both when compiled and now: a timeout is a
|
|
115
|
+
# censored cost (the cap it hit may differ between then and now), not a measurement
|
|
116
|
+
both = [i for i in range(k) if i in recorded and recorded[i][1] and recs[i]["status"] == "solved"]
|
|
117
|
+
used = k
|
|
118
|
+
if recorded and both:
|
|
119
|
+
mine = statistics.median(C._cost(recs[i], compiled["plan"]["cap"]) for i in both)
|
|
120
|
+
rec, basis, warn_at, used = statistics.median(recorded[i][0] for i in both), "same", MEASURE_WARN_BITS, len(both)
|
|
121
|
+
note = f" [on {len(both)} of {k} instances solved both then and now; the rest timed out or are not in the record]" if len(both) < k else ""
|
|
122
|
+
elif _positive(entry.get("median_cost")):
|
|
123
|
+
# an older record keeps only the median over all its instances: comparing K of them with it includes sampling noise, which is stated, not hidden
|
|
124
|
+
sd = entry.get("sd_log2") or 1.5
|
|
125
|
+
noise = sd / math.sqrt(k)
|
|
126
|
+
rec, basis, warn_at = entry["median_cost"], "median", max(MEASURE_WARN_BITS, 2 * noise)
|
|
127
|
+
note = (f" [recorded median over {entry.get('n', '?')} instances, compared with the first {k}: sampling noise alone is about ±{noise:.1f} bits (per-instance sd "
|
|
128
|
+
f"{sd:.1f}); recompile to record per-instance costs for an exact comparison]")
|
|
129
|
+
else:
|
|
130
|
+
res.append(C.R(f"size {s} timing", "warn", f"this machine {mine:.2f} s; no recorded median at this size"))
|
|
131
|
+
continue
|
|
132
|
+
gap = math.log2(mine / rec)
|
|
133
|
+
gaps.append(gap)
|
|
134
|
+
res.append(C.R(f"size {s} timing", "warn" if abs(gap) > warn_at else "pass",
|
|
135
|
+
f"this machine median {mine:.2f} s ({k} instances), recorded {rec:.2f} s: {gap:+.2f} bits{note}"))
|
|
136
|
+
res[-1]["data"] = {"size": s, "mine": round(mine, 4), "recorded": round(rec, 4), "gap_bits": round(gap, 4), "k": k, "used": used, "basis": basis}
|
|
137
|
+
if gaps:
|
|
138
|
+
res.append(C.R("machine spread", "pass", f"this machine vs the recorded one: mean {statistics.mean(gaps):+.2f} bits, worst {max(gaps, key=abs):+.2f} bits over {len(gaps)} sizes "
|
|
139
|
+
"(a gap that is the same at every size is a hardware speed difference and cancels in a speedup ratio; one that changes with size is not)"))
|
|
140
|
+
return res
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# ------------------------------------------------------------------------------------------------------------- machine spread
|
|
144
|
+
SPREAD_FORMAT = "svp-measurements-v1"
|
|
145
|
+
MIN_TOLERANCE = 1.0 # bits. certificate.py's re-measurement compares single instances (pair noise ~0.6 bit); a tolerance below this fails honest certificates
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def machine_id():
|
|
149
|
+
"""A description of this machine, and a short hash of its host name (so two files from one machine are recognised without printing the name;
|
|
150
|
+
a guessable host name can still be confirmed from the hash, so do not treat it as anonymous)."""
|
|
151
|
+
import hashlib
|
|
152
|
+
import platform
|
|
153
|
+
return {"platform": platform.platform(), "machine": platform.machine(), "python": platform.python_version(),
|
|
154
|
+
"host_hash": hashlib.sha256(platform.node().encode()).hexdigest()[:12]}
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def save_measurements(res, compiled, path):
|
|
158
|
+
"""Write the per-size gaps from a `recheck --measure` run. Never overwrites."""
|
|
159
|
+
import json
|
|
160
|
+
import time
|
|
161
|
+
rows = [r["data"] for r in res if r.get("data") and "gap_bits" in r["data"]]
|
|
162
|
+
if not rows:
|
|
163
|
+
raise SystemExit("nothing to save: run with --measure K first")
|
|
164
|
+
doc = {"format": SPREAD_FORMAT, "domain": compiled["name"], "version": compiled["version"], "decl_sha256": compiled["decl_sha256"],
|
|
165
|
+
"machine": machine_id(), "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "sizes": rows}
|
|
166
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
167
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
168
|
+
json.dump(doc, f, indent=1, sort_keys=True)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _read_measurements(path):
|
|
172
|
+
"""A measurement file comes from another machine, so it is untrusted: accept exactly the shape save_measurements writes, and make every string
|
|
173
|
+
that will be printed plain ASCII (a hostile platform string must not be able to drive the terminal)."""
|
|
174
|
+
import json
|
|
175
|
+
import re
|
|
176
|
+
try:
|
|
177
|
+
with open(path, encoding="utf-8") as f:
|
|
178
|
+
d = json.load(f)
|
|
179
|
+
if not isinstance(d, dict) or d.get("format") != SPREAD_FORMAT:
|
|
180
|
+
raise ValueError(f"not a {SPREAD_FORMAT} file")
|
|
181
|
+
m = d["machine"]
|
|
182
|
+
if not (isinstance(d["domain"], str) and isinstance(d["decl_sha256"], str) and re.fullmatch(r"[0-9a-f]{1,64}", d["decl_sha256"])
|
|
183
|
+
and isinstance(m, dict) and isinstance(m["host_hash"], str) and re.fullmatch(r"[0-9a-f]{1,32}", m["host_hash"]) and isinstance(m["platform"], str)):
|
|
184
|
+
raise ValueError("header fields are malformed")
|
|
185
|
+
if not isinstance(d["sizes"], list) or not d["sizes"] or len(d["sizes"]) > 100:
|
|
186
|
+
raise ValueError("sizes is not a short, non-empty list")
|
|
187
|
+
for r in d["sizes"]:
|
|
188
|
+
if not (isinstance(r, dict) and isinstance(r["size"], int) and not isinstance(r["size"], bool)
|
|
189
|
+
and isinstance(r["gap_bits"], (int, float)) and not isinstance(r["gap_bits"], bool) and math.isfinite(r["gap_bits"]) and abs(r["gap_bits"]) <= 64):
|
|
190
|
+
raise ValueError("a size entry is malformed")
|
|
191
|
+
except (OSError, ValueError, KeyError, TypeError) as e:
|
|
192
|
+
raise SystemExit(f"REFUSED: {path} is not a valid measurement file ({type(e).__name__}: {str(e)[:80]})")
|
|
193
|
+
d["machine"]["platform"] = re.sub(r"[^\x20-\x7e]", "?", m["platform"])[:80]
|
|
194
|
+
d["domain"] = re.sub(r"[^\x20-\x7e]", "?", d["domain"])[:40]
|
|
195
|
+
return d
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def spread(files, min_machines=2):
|
|
199
|
+
"""Merge measurement files from different machines. -> (results, recommended tolerance in bits or None).
|
|
200
|
+
|
|
201
|
+
A speedup is a ratio, so a uniform hardware difference cancels. What does not cancel is how a machine's gap CHANGES from size to size (the
|
|
202
|
+
baseline and a solver sit at different costs, so a machine that is relatively slower at large sizes distorts the ratio). Each machine's
|
|
203
|
+
within-machine range of gaps (max - min over sizes) bounds that; the recommendation is the largest range, rounded up to a quarter bit,
|
|
204
|
+
never below 0.25. It is an estimate from a handful of sizes and machines, and is None until at least `min_machines` DISTINCT machines
|
|
205
|
+
measured the SAME declaration."""
|
|
206
|
+
import json
|
|
207
|
+
docs = [_read_measurements(p) for p in files]
|
|
208
|
+
res = []
|
|
209
|
+
if len({(d["domain"], d["decl_sha256"]) for d in docs}) > 1:
|
|
210
|
+
return [C.R("same declaration", "fail", "the files measured different declarations (name or decl hash differ): they cannot be merged")], None
|
|
211
|
+
res.append(C.R("same declaration", "pass", f"{docs[0]['domain']} {docs[0]['decl_sha256'][:12]}"))
|
|
212
|
+
machines = {}
|
|
213
|
+
for d in docs:
|
|
214
|
+
machines.setdefault(d["machine"]["host_hash"], []).append(d)
|
|
215
|
+
ranges = {}
|
|
216
|
+
for h, ds in machines.items():
|
|
217
|
+
by_size = {}
|
|
218
|
+
for d in ds:
|
|
219
|
+
for r in d["sizes"]:
|
|
220
|
+
by_size.setdefault(r["size"], []).append(r["gap_bits"])
|
|
221
|
+
gaps = {s: statistics.mean(v) for s, v in by_size.items()}
|
|
222
|
+
rng = max(gaps.values()) - min(gaps.values()) if len(gaps) > 1 else 0.0
|
|
223
|
+
ranges[h] = rng
|
|
224
|
+
res.append(C.R(f"machine {h}", "pass", f"{ds[0]['machine']['platform']}: mean gap {statistics.mean(gaps.values()):+.2f} bits, "
|
|
225
|
+
f"range across sizes {rng:.2f} bits ({len(gaps)} sizes)"))
|
|
226
|
+
if len(machines) < min_machines:
|
|
227
|
+
res.append(C.R("tolerance", "warn", f"{len(machines)} distinct machine(s): need at least {min_machines} to recommend a tolerance. The default stays loose"))
|
|
228
|
+
return res, None
|
|
229
|
+
if any(r.get("basis") != "same" for d in docs for r in d["sizes"]):
|
|
230
|
+
res.append(C.R("comparison basis", "warn", "some gaps were measured against a recorded MEDIAN over different instances, so they include sampling noise "
|
|
231
|
+
"(often a bit or more): the recommendation below is inflated by it. Recompile the domain (records per-instance costs) and measure again for a clean figure"))
|
|
232
|
+
rec = max(MIN_TOLERANCE, math.ceil(max(ranges.values()) * 4) / 4)
|
|
233
|
+
res.append(C.R("tolerance", "pass", f"recommended --tolerance {rec:g} bits (largest within-machine range across sizes, {len(machines)} machines, never below {MIN_TOLERANCE:g}: "
|
|
234
|
+
"the re-measurement compares single instances, whose own noise is about 0.6 bit and is NOT measured here). "
|
|
235
|
+
"An estimate from a few sizes: use it for `certificate.py verify --tolerance`, do not treat it as a bound"))
|
|
236
|
+
return res, rec
|
guldo/_src/remeasure.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Re-measure a stratified sample of a finished ladder SEQUENTIALLY and compare with the original (possibly concurrent)
|
|
2
|
+
timings, by baseline-cost bin. This is the direct check that --jobs K did not bias the timings of the HARD instances,
|
|
3
|
+
where long runs, sustained load and thermal throttling matter most (drift_check.py only covers short instances).
|
|
4
|
+
|
|
5
|
+
python3 remeasure.py --ladder results/judged_gm_<tag>_baseline_baseline_bkz.py.jsonl --seed-file ~/judge-seeds/a.seed \
|
|
6
|
+
--solver solvers/baseline_bkz.py [--per-bin 4]
|
|
7
|
+
|
|
8
|
+
Reports, per octave of the ORIGINAL cost, the median of log2(new cost / original cost). Near 0 = concurrency did not bias
|
|
9
|
+
the ladder. A systematic positive value means the concurrent run measured faster than sequential (or the reverse).
|
|
10
|
+
"""
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import os
|
|
15
|
+
import statistics
|
|
16
|
+
|
|
17
|
+
import families
|
|
18
|
+
import harness
|
|
19
|
+
|
|
20
|
+
FLOOR = 0.5
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def octave(cost):
|
|
24
|
+
return int(math.floor(math.log2(cost / FLOOR)))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def select_sample(records, per_bin):
|
|
28
|
+
"""Deterministic stratified sample: from each octave (original cost >= FLOOR, solved), `per_bin` records spread evenly
|
|
29
|
+
over the records sorted by id."""
|
|
30
|
+
bins = {}
|
|
31
|
+
for r in records:
|
|
32
|
+
if r["status"] == "solved" and r["cpu_net"] >= FLOOR:
|
|
33
|
+
bins.setdefault(octave(r["cpu_net"]), []).append(r)
|
|
34
|
+
out = []
|
|
35
|
+
for k in sorted(bins):
|
|
36
|
+
rs = sorted(bins[k], key=lambda r: r["id"])
|
|
37
|
+
if len(rs) <= per_bin:
|
|
38
|
+
out += rs
|
|
39
|
+
else:
|
|
40
|
+
step = len(rs) / per_bin
|
|
41
|
+
out += [rs[int(i * step)] for i in range(per_bin)]
|
|
42
|
+
return out
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def compare(pairs):
|
|
46
|
+
"""pairs: [(original cpu_net, new cpu_net)] -> {octave: (n, median log2 ratio, sd)} plus an 'all' entry."""
|
|
47
|
+
by = {}
|
|
48
|
+
for old, new in pairs:
|
|
49
|
+
by.setdefault(octave(old), []).append(math.log2(new / old))
|
|
50
|
+
out = {k: (len(v), statistics.median(v), statistics.pstdev(v)) for k, v in sorted(by.items())}
|
|
51
|
+
allv = [x for v in by.values() for x in v]
|
|
52
|
+
if allv:
|
|
53
|
+
out["all"] = (len(allv), statistics.median(allv), statistics.pstdev(allv))
|
|
54
|
+
return out
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def main():
|
|
58
|
+
ap = argparse.ArgumentParser()
|
|
59
|
+
ap.add_argument("--ladder", required=True)
|
|
60
|
+
ap.add_argument("--seed-file", required=True)
|
|
61
|
+
ap.add_argument("--solver", required=True)
|
|
62
|
+
ap.add_argument("--per-bin", type=int, default=4)
|
|
63
|
+
a = ap.parse_args()
|
|
64
|
+
with open(a.ladder) as f:
|
|
65
|
+
meta = json.loads(f.readline())
|
|
66
|
+
records = [json.loads(line) for line in f]
|
|
67
|
+
with open(a.seed_file, "rb") as f:
|
|
68
|
+
master = f.read().strip()
|
|
69
|
+
sample = select_sample(records, a.per_bin)
|
|
70
|
+
print(f"original ladder jobs={meta.get('jobs', 1)}; re-measuring {len(sample)} instances sequentially", flush=True)
|
|
71
|
+
overhead = meta["overhead"] # the SAME no-op overhead the original ladder subtracted, so the comparison is like-for-like
|
|
72
|
+
pairs = []
|
|
73
|
+
for r in sample:
|
|
74
|
+
dim, idx = map(int, r["id"].split(":"))
|
|
75
|
+
new = harness.run_one(os.path.abspath(a.solver), families.derive(meta["family"], master, dim, idx), meta["budget"])
|
|
76
|
+
if new["status"] != "solved":
|
|
77
|
+
print(f" {r['id']}: not solved on re-measurement ({new['status']}); skipped")
|
|
78
|
+
continue
|
|
79
|
+
net = max(new["cpu"] - overhead, 1e-3)
|
|
80
|
+
pairs.append((r["cpu_net"], net))
|
|
81
|
+
print(f" {r['id']:>7s} original {r['cpu_net']:8.2f}s sequential {net:8.2f}s {math.log2(net / r['cpu_net']):+.3f} bits", flush=True)
|
|
82
|
+
print("\nlog2(sequential / original) by original-cost octave [0.5*2^k s]:")
|
|
83
|
+
for k, (n, med, sd) in compare(pairs).items():
|
|
84
|
+
print(f" {str(k):>4s}: n={n:2d} median {med:+.3f} bits sd {sd:.3f}")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
if __name__ == "__main__":
|
|
88
|
+
main()
|