guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""The few helpers shared by the local portal (portal.py), the PUBLIC web tier (public_portal.py), the operator tool (ops.py) and the
|
|
2
|
+
site builder, kept in a module that imports only the standard library.
|
|
3
|
+
|
|
4
|
+
Why separate: portal.py imports the report generator, which imports the estimator, which imports numpy. A process that parses strangers'
|
|
5
|
+
uploads has no business loading a numeric library, and a server without numpy must be able to run it (test_python_compat.py checks
|
|
6
|
+
that public_portal, ops, rounds and submit_check import no third-party library).
|
|
7
|
+
|
|
8
|
+
sandbox_fingerprint / write_marker / sandbox_verified the "sandbox verified" gate (a marker file tied to the sandbox files' hashes)
|
|
9
|
+
parse_multipart a minimal multipart/form-data parser using the stdlib email package
|
|
10
|
+
"""
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import time
|
|
15
|
+
from email.parser import BytesParser
|
|
16
|
+
from email.policy import default as email_policy
|
|
17
|
+
|
|
18
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
19
|
+
MARKER = "SANDBOX_VERIFIED"
|
|
20
|
+
SANDBOX_FILES = ["sandbox/meter.py", "sandbox/meter_core.py", "sandbox/Dockerfile.v2", "runner.py", "harness.py"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def sandbox_fingerprint():
|
|
24
|
+
"""sha256 of every file that defines the sandbox (meter, image recipe, runner, and harness.py which builds the docker
|
|
25
|
+
command). The verification marker is only valid while these are unchanged."""
|
|
26
|
+
out = {}
|
|
27
|
+
for rel in SANDBOX_FILES:
|
|
28
|
+
with open(os.path.join(HERE, rel), "rb") as f:
|
|
29
|
+
out[rel] = hashlib.sha256(f.read()).hexdigest()
|
|
30
|
+
return out
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def write_marker(base):
|
|
34
|
+
with open(os.path.join(base, MARKER), "w", encoding="utf-8") as f:
|
|
35
|
+
json.dump({"verified_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "files": sandbox_fingerprint()}, f, indent=1)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def sandbox_verified(base):
|
|
39
|
+
"""True only if the marker exists AND the sandbox files are byte-identical to what was verified."""
|
|
40
|
+
try:
|
|
41
|
+
with open(os.path.join(base, MARKER), encoding="utf-8") as f:
|
|
42
|
+
return json.load(f).get("files") == sandbox_fingerprint()
|
|
43
|
+
except (OSError, ValueError):
|
|
44
|
+
return False
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def parse_multipart(content_type, body):
|
|
48
|
+
"""Minimal multipart/form-data parser using the stdlib email package. Returns {field: (filename or None, bytes)}."""
|
|
49
|
+
msg = BytesParser(policy=email_policy).parsebytes(b"Content-Type: " + content_type.encode() + b"\r\n\r\n" + body)
|
|
50
|
+
out = {}
|
|
51
|
+
if not msg.is_multipart():
|
|
52
|
+
return out
|
|
53
|
+
for part in msg.iter_parts():
|
|
54
|
+
name = part.get_param("name", header="content-disposition")
|
|
55
|
+
if name:
|
|
56
|
+
out[name] = (part.get_filename(), part.get_payload(decode=True) or b"")
|
|
57
|
+
return out
|
guldo/_src/power.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""Empirical power of the Tier-1 claim using REAL per-instance noise instead of an assumed Gaussian.
|
|
2
|
+
|
|
3
|
+
Given a baseline ladder and one or more "reference pair" ladders (a solver measured against that same baseline on the
|
|
4
|
+
same instances), this takes the real paired log2-ratios, removes each dimension's median (leaving the real noise
|
|
5
|
+
residuals, heavy tails included), then synthesises a submission whose true speedup is exactly `e` bits:
|
|
6
|
+
|
|
7
|
+
sub_cost(i) = baseline_cost(i) * 2 ** -(e + residual drawn from that dimension's real residuals)
|
|
8
|
+
|
|
9
|
+
and runs the actual usable_dims + measured_lower pipeline on it. Output: P(credit >= threshold) per effect size, plus
|
|
10
|
+
the zero-effect false-credit rate. Baseline timeouts stay censored exactly as in the real data, so the effect of the
|
|
11
|
+
timeout cap on power is included (the Gaussian simulation in PROTOCOL.md ignores it).
|
|
12
|
+
|
|
13
|
+
python3 power.py --baseline B.jsonl --pair S.jsonl [--pair S2.jsonl ...] [--effects 0 1 2 3 4] [--trials 200]
|
|
14
|
+
"""
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import math
|
|
18
|
+
import random
|
|
19
|
+
import statistics
|
|
20
|
+
from collections import defaultdict
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
|
|
24
|
+
import extrapolate as E
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load(path):
|
|
28
|
+
meta, by_dim = None, defaultdict(dict)
|
|
29
|
+
with open(path) as f:
|
|
30
|
+
for line in f:
|
|
31
|
+
r = json.loads(line)
|
|
32
|
+
if r.get("meta"):
|
|
33
|
+
meta = r
|
|
34
|
+
else:
|
|
35
|
+
by_dim[r["dim"]][r["id"]] = r
|
|
36
|
+
return meta, by_dim
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def residuals(base, sub, budget):
|
|
40
|
+
"""Real noise: paired log2 ratios minus the median of their baseline-difficulty octave (both solved, so no
|
|
41
|
+
censoring artefact). Returns {octave: [residuals]} for octaves with >= 4 instances."""
|
|
42
|
+
by_bin = defaultdict(list)
|
|
43
|
+
for d in base:
|
|
44
|
+
for i in base[d]:
|
|
45
|
+
if i in sub.get(d, {}) and base[d][i]["status"] == "solved" and sub[d][i]["status"] == "solved":
|
|
46
|
+
cb = base[d][i]["cpu_net"]
|
|
47
|
+
if cb >= E.FLOOR_ABS:
|
|
48
|
+
by_bin[int(math.floor(math.log2(cb / E.FLOOR_ABS)))].append(
|
|
49
|
+
math.log2(cb / sub[d][i]["cpu_net"]))
|
|
50
|
+
return {k: [x - statistics.median(v) for x in v] for k, v in by_bin.items() if len(v) >= 4}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def synth_trial(base, budget, resid_pool, effect, rng, boot, copies=1, resample=False):
|
|
54
|
+
"""Synthetic submission with a TRUE constant speedup `effect` over the real baseline costs. Baseline-censored
|
|
55
|
+
instances stay censored (and are excluded by the estimator, exactly as in a real run).
|
|
56
|
+
copies > 1 replicates the real baseline ladder that many times (same cost distribution, copies x the instances per dimension, each copy
|
|
57
|
+
with its own noise draw): the answer to "what would a ladder with more instances per dimension certify?" without running one.
|
|
58
|
+
The copies are IDENTICAL baseline costs, so a larger ladder always has exactly `copies` x the observed hard instances. resample=True draws
|
|
59
|
+
each dimension's instances (copies x as many) at random WITH REPLACEMENT from the real ones instead, so the number of hard instances varies
|
|
60
|
+
from ladder to ladder, as it would for a genuinely larger ladder of fresh instances: the honest version of the question."""
|
|
61
|
+
tb, ts, bc = {}, {}, {}
|
|
62
|
+
for d, recs in base.items():
|
|
63
|
+
tb[d], ts[d], bc[d] = {}, {}, {}
|
|
64
|
+
if resample:
|
|
65
|
+
ids = list(recs)
|
|
66
|
+
chosen = [(f"{j}", recs[rng.choice(ids)]) for j in range(len(ids) * copies)]
|
|
67
|
+
else:
|
|
68
|
+
chosen = [((i if copies == 1 else f"{i}#{c}"), r) for i, r in recs.items() for c in range(copies)]
|
|
69
|
+
for key, r in chosen:
|
|
70
|
+
censored = r["status"] != "solved"
|
|
71
|
+
cost = budget if censored else r["cpu_net"]
|
|
72
|
+
k = int(math.floor(math.log2(max(cost, E.FLOOR_ABS) / E.FLOOR_ABS)))
|
|
73
|
+
pool = resid_pool.get(k) or resid_pool["__all__"]
|
|
74
|
+
tb[d][key], bc[d][key] = cost, censored
|
|
75
|
+
ts[d][key] = min(max(cost * 2 ** -(effect + rng.choice(pool)), 1e-3), budget)
|
|
76
|
+
m, top, _ = E.measured_by_difficulty(tb, ts, bc, rng, boot=boot)
|
|
77
|
+
return None if m is None else max(0.0, m[1]), top
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def real_pair_credits(base, sub, budget, rng, trials, boot, copies=1):
|
|
81
|
+
"""Credits of `trials` ladders built by resampling the REAL paired instances (baseline record + the submission's record on the SAME
|
|
82
|
+
instance) with replacement, copies x as many per dimension. No synthetic effect: the solver's real, instance-dependent speedup and its
|
|
83
|
+
real failures (charged the cap, exactly as in a run) are what is resampled. None = inconclusive."""
|
|
84
|
+
tb0, bc0 = E.per_dim_times(base, budget)
|
|
85
|
+
ts0, _ = E.per_dim_times(sub, budget)
|
|
86
|
+
out = []
|
|
87
|
+
for _ in range(trials):
|
|
88
|
+
tb, ts, bc = {}, {}, {}
|
|
89
|
+
for d in tb0:
|
|
90
|
+
ids = sorted(set(tb0[d]) & set(ts0.get(d, {})))
|
|
91
|
+
tb[d], ts[d], bc[d] = {}, {}, {}
|
|
92
|
+
for j in range(len(ids) * copies):
|
|
93
|
+
i = rng.choice(ids)
|
|
94
|
+
tb[d][j], ts[d][j], bc[d][j] = tb0[d][i], ts0[d][i], bc0[d][i]
|
|
95
|
+
m, _, _ = E.measured_by_difficulty(tb, ts, bc, rng, boot=boot)
|
|
96
|
+
out.append(None if m is None else max(0.0, m[1]))
|
|
97
|
+
return out
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def main():
|
|
101
|
+
ap = argparse.ArgumentParser()
|
|
102
|
+
ap.add_argument("--baseline", required=True)
|
|
103
|
+
ap.add_argument("--pair", action="append", required=True, help="a submission ladder measured against --baseline")
|
|
104
|
+
ap.add_argument("--effects", type=float, nargs="+", default=[0, 1, 1.5, 2, 2.5, 3, 4])
|
|
105
|
+
ap.add_argument("--threshold", type=float, default=1.0)
|
|
106
|
+
ap.add_argument("--trials", type=int, default=200)
|
|
107
|
+
ap.add_argument("--boot", type=int, default=150)
|
|
108
|
+
ap.add_argument("--seed", type=int, default=0)
|
|
109
|
+
ap.add_argument("--copies", type=int, default=1, help="replicate the real baseline ladder this many times: the power of a ladder with "
|
|
110
|
+
"copies x the instances per dimension (same cost distribution)")
|
|
111
|
+
ap.add_argument("--real-pairs", action="store_true", help="no synthetic effect: bootstrap the REAL paired instances of each --pair (its real, "
|
|
112
|
+
"instance-dependent speedup and real failures) and report P(credit >= threshold) for one seed and for ALL THREE seeds (the board credit "
|
|
113
|
+
"is the minimum over seeds); --copies scales the ladder")
|
|
114
|
+
ap.add_argument("--resample", action="store_true", help="with --copies: draw each synthetic ladder's instances at random from the real ones "
|
|
115
|
+
"(the number of hard instances then varies from ladder to ladder, as for a genuinely larger ladder) instead of repeating them exactly")
|
|
116
|
+
a = ap.parse_args()
|
|
117
|
+
if a.real_pairs and a.copies > 1:
|
|
118
|
+
raise SystemExit("REFUSED: --real-pairs with --copies > 1 would DUPLICATE the real submission costs (and their noise), so the bootstrap would "
|
|
119
|
+
"believe it has copies x the information and report an artificially tight bound and an inflated credit. Larger-ladder power needs "
|
|
120
|
+
"a noise model: use the synthetic mode with --copies and --resample.")
|
|
121
|
+
mb, base = load(a.baseline)
|
|
122
|
+
budget = mb["budget"]
|
|
123
|
+
pool = defaultdict(list)
|
|
124
|
+
for p in a.pair:
|
|
125
|
+
ms, sub = load(p)
|
|
126
|
+
if ms["commit"] != mb["commit"]:
|
|
127
|
+
raise SystemExit(f"{p}: different instance set than the baseline")
|
|
128
|
+
only_b, only_s = E.coverage_mismatch(base, sub)
|
|
129
|
+
if only_b or only_s:
|
|
130
|
+
raise SystemExit(f"{p}: does not cover the same instances as the baseline ({len(only_b)} only in the baseline, "
|
|
131
|
+
f"{len(only_s)} only in the pair); residuals from a truncated run would be biased")
|
|
132
|
+
if a.real_pairs:
|
|
133
|
+
rng = random.Random(a.seed)
|
|
134
|
+
credits = real_pair_credits(base, sub, budget, rng, 3 * a.trials, a.boot, a.copies)
|
|
135
|
+
ok = sorted(c for c in credits if c is not None)
|
|
136
|
+
single = sum(c is not None and c >= a.threshold for c in credits) / len(credits)
|
|
137
|
+
triples = [min(0.0 if c is None else c for c in credits[i:i + 3]) for i in range(0, len(credits) - 2, 3)]
|
|
138
|
+
all3 = sum(c >= a.threshold for c in triples) / len(triples)
|
|
139
|
+
pct = lambda q: ok[min(len(ok) - 1, int(q * len(ok)))] if ok else float("nan")
|
|
140
|
+
n_real = max(len(v) for v in base.values())
|
|
141
|
+
print(f"REAL pair {p.rsplit('/', 1)[-1]} bootstrapped: {n_real * a.copies} instances per size (x{a.copies}), {len(credits)} ladders, "
|
|
142
|
+
f"boot {a.boot}, threshold {a.threshold} bit(s)")
|
|
143
|
+
print(f" credit per ladder: median {pct(0.5):.2f}, 5th percentile {pct(0.05):.2f}, 95th {pct(0.95):.2f}, inconclusive "
|
|
144
|
+
f"{100 * (len(credits) - len(ok)) / len(credits):.0f}%")
|
|
145
|
+
print(f" P(one seed's credit >= {a.threshold}) = {100 * single:.1f}%")
|
|
146
|
+
print(f" P(ALL THREE seeds >= {a.threshold}) = {100 * all3:.1f}% ({len(triples)} independent triples; the board credit is the minimum over seeds)")
|
|
147
|
+
if a.real_pairs:
|
|
148
|
+
return
|
|
149
|
+
for k, v in residuals(base, sub, budget).items():
|
|
150
|
+
pool[k] += v
|
|
151
|
+
pool["__all__"] += v
|
|
152
|
+
if not pool["__all__"]:
|
|
153
|
+
raise SystemExit("no difficulty bin has >= 4 instances solved by both solvers; cannot estimate real noise")
|
|
154
|
+
sd = {k: statistics.pstdev(v) for k, v in pool.items() if k != "__all__"}
|
|
155
|
+
print("real per-instance noise (sd of paired log2 ratio, bits) by baseline-cost octave "
|
|
156
|
+
f"[0.5*2^k s]: { {k: round(s, 2) for k, s in sorted(sd.items())} }")
|
|
157
|
+
print(f"pooled residuals: {len(pool['__all__'])}; baseline timeouts kept censored; budget {budget}s")
|
|
158
|
+
if a.copies > 1:
|
|
159
|
+
n_real = max(len(v) for v in base.values())
|
|
160
|
+
print(f"ladder replicated {a.copies}x: {n_real} -> {n_real * a.copies} instances per dimension, same cost distribution"
|
|
161
|
+
+ ("; instances RESAMPLED at random with replacement (hard-instance count varies per ladder)" if a.resample else ""))
|
|
162
|
+
rng = random.Random(a.seed)
|
|
163
|
+
print(f"\nP(Tier-1 credit >= {a.threshold} bit) by true effect ({a.trials} trials each):")
|
|
164
|
+
for e in a.effects:
|
|
165
|
+
credits = [synth_trial(base, budget, pool, e, rng, a.boot, a.copies, a.resample)[0] for _ in range(a.trials)]
|
|
166
|
+
hit = sum(c is not None and c >= a.threshold for c in credits)
|
|
167
|
+
incon = sum(c is None for c in credits)
|
|
168
|
+
ok = [c for c in credits if c is not None]
|
|
169
|
+
print(f" true effect {e:4.1f} bits: {100 * hit / a.trials:5.1f}% "
|
|
170
|
+
f"(inconclusive {100 * incon / a.trials:3.0f}%, mean credit {np.mean(ok) if ok else 0:.2f})")
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
if __name__ == "__main__":
|
|
174
|
+
main()
|
guldo/_src/prereg.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Pre-registration files: write one, and check later that the code it pins has not changed.
|
|
2
|
+
|
|
3
|
+
A pre-registration is only worth something if the pass criteria and the exact code were fixed BEFORE any data existed. Two were written
|
|
4
|
+
by hand with shell heredocs and checked with an ad-hoc loop; this makes both steps mechanical.
|
|
5
|
+
|
|
6
|
+
prereg.py new --out results/PREREGISTERED_x.txt --title "..." --family gm \\
|
|
7
|
+
--baseline solvers/baseline_bkz.py --submission solvers/tuned_bkz.py \\
|
|
8
|
+
--commitment HEX [--commitment HEX ...] --criteria "credit >= 1.0 bit ..." [--criteria-file F] \\
|
|
9
|
+
[--sandbox docker|local] [--jobs N] [--pin EXTRA_FILE ...] [--root DIR]
|
|
10
|
+
prereg.py check FILE [--root DIR] exit 0 if every pinned file still hashes as registered; 1 if any changed or is missing
|
|
11
|
+
|
|
12
|
+
What it records: the date (UTC), every seed commitment (public from this moment), the family and its ladder, the official n and CPU cap,
|
|
13
|
+
the sandbox mode, the two solvers, your criteria (required, verbatim), and the sha256 of every file that generates, runs, meters and
|
|
14
|
+
scores instances (rounds.CODE_FILES) plus both solvers. It refuses to overwrite an existing registration. It cannot judge whether your
|
|
15
|
+
criteria are sensible or unconditional; that is yours to write. Note: build_site.py lists only the FIRST commitment of a multi-seed file.
|
|
16
|
+
"""
|
|
17
|
+
import argparse
|
|
18
|
+
import hashlib
|
|
19
|
+
import os
|
|
20
|
+
import re
|
|
21
|
+
import sys
|
|
22
|
+
import time
|
|
23
|
+
|
|
24
|
+
import families
|
|
25
|
+
import rounds
|
|
26
|
+
|
|
27
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
28
|
+
HASH_LINE = re.compile(r"^([0-9a-f]{64}) (\S.*)$")
|
|
29
|
+
COMMIT_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def sha256_file(path):
|
|
33
|
+
with open(path, "rb") as f:
|
|
34
|
+
return hashlib.sha256(f.read()).hexdigest()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def inside_root(rel, root):
|
|
38
|
+
"""True only for a relative path that, after resolving symlinks, stays inside `root`. An absolute path, a `..` that climbs out, or a
|
|
39
|
+
symlink pointing elsewhere would let a registration pin (and `check` hash) a file that is not part of the project."""
|
|
40
|
+
if os.path.isabs(rel):
|
|
41
|
+
return False
|
|
42
|
+
real_root = os.path.realpath(root)
|
|
43
|
+
real = os.path.realpath(os.path.join(root, rel))
|
|
44
|
+
return real == real_root or real.startswith(real_root + os.sep)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def pinned_files(baseline, submission, extra=()):
|
|
48
|
+
"""The files a registration pins, relative to the root, in a stable order without duplicates."""
|
|
49
|
+
out = []
|
|
50
|
+
for rel in list(rounds.CODE_FILES) + [baseline, submission] + list(extra):
|
|
51
|
+
if rel not in out:
|
|
52
|
+
out.append(rel)
|
|
53
|
+
return out
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def render(title, family, baseline, submission, commitments, criteria, sandbox, jobs, files, root, now=None):
|
|
57
|
+
cfg = families.OFFICIAL[family]
|
|
58
|
+
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now if now is not None else time.time()))
|
|
59
|
+
title = " ".join(title.split()) # one line: a newline in the title would break the header and the date line after it
|
|
60
|
+
lines = [f"PRE-REGISTRATION, {title}. Written BEFORE any data from these seeds exists.", f"date: {stamp}"]
|
|
61
|
+
lines += [f"seed commitment (sha256): {c}" for c in commitments]
|
|
62
|
+
lines += [f"config: family {family}, sizes {families.DEFAULT_DIMS[family]}, n={cfg['n']} per size, cap {cfg['budget']} s, "
|
|
63
|
+
f"sandbox {sandbox}, jobs {jobs}.",
|
|
64
|
+
f"baseline: {baseline}. submission: {submission}.", "",
|
|
65
|
+
"UNCONDITIONAL criteria (written before any data; applied without reinterpretation):", criteria.strip(), "",
|
|
66
|
+
"code hashes (sha256):"]
|
|
67
|
+
lines += [f"{sha256_file(os.path.join(root, rel))} {rel}" for rel in files]
|
|
68
|
+
return "\n".join(lines) + "\n"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def parse(text):
|
|
72
|
+
"""{relative path: sha256} for every pinned file in a registration. Only the block after the LAST `code hashes` header counts: the
|
|
73
|
+
criteria above it are free text, and a criteria line must not be able to open the hash block early and plant pins of its own."""
|
|
74
|
+
lines = text.splitlines()
|
|
75
|
+
starts = [i for i, line in enumerate(lines) if line.startswith("code hashes")]
|
|
76
|
+
pins = {}
|
|
77
|
+
for line in lines[starts[-1] + 1:] if starts else []:
|
|
78
|
+
m = HASH_LINE.match(line)
|
|
79
|
+
if m:
|
|
80
|
+
pins[m.group(2)] = m.group(1)
|
|
81
|
+
return pins
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def check(path, root=HERE):
|
|
85
|
+
"""(ok, report lines). Every pinned file must exist and hash as registered."""
|
|
86
|
+
with open(path, encoding="utf-8") as f:
|
|
87
|
+
pins = parse(f.read())
|
|
88
|
+
if not pins:
|
|
89
|
+
return False, ["no code hashes found in this file: not a registration made by prereg.py"]
|
|
90
|
+
report, ok = [], True
|
|
91
|
+
for rel, want in pins.items():
|
|
92
|
+
full = os.path.join(root, rel)
|
|
93
|
+
if not inside_root(rel, root): # never hash a file the registration points at outside the project
|
|
94
|
+
ok = False
|
|
95
|
+
report.append(f"REFUSED path outside the root: {rel}")
|
|
96
|
+
elif not os.path.exists(full):
|
|
97
|
+
ok = False
|
|
98
|
+
report.append(f"MISSING {rel}")
|
|
99
|
+
elif sha256_file(full) != want:
|
|
100
|
+
ok = False
|
|
101
|
+
report.append(f"CHANGED {rel}")
|
|
102
|
+
else:
|
|
103
|
+
report.append(f"same {rel}")
|
|
104
|
+
return ok, report
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main(argv=None):
|
|
108
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
109
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
110
|
+
p = sub.add_parser("new")
|
|
111
|
+
p.add_argument("--out", required=True)
|
|
112
|
+
p.add_argument("--title", required=True)
|
|
113
|
+
p.add_argument("--family", required=True, choices=sorted(families.FAMILIES))
|
|
114
|
+
p.add_argument("--baseline", required=True)
|
|
115
|
+
p.add_argument("--submission", required=True)
|
|
116
|
+
p.add_argument("--commitment", action="append", required=True, help="a seed commitment (64 hex); repeat for several seeds")
|
|
117
|
+
p.add_argument("--criteria", help="the unconditional pass criteria, in your words")
|
|
118
|
+
p.add_argument("--criteria-file", help="read the criteria from a file instead")
|
|
119
|
+
p.add_argument("--sandbox", choices=["docker", "local"], default="docker")
|
|
120
|
+
p.add_argument("--jobs", type=int, default=1)
|
|
121
|
+
p.add_argument("--pin", action="append", default=[], help="an extra file to pin (repeatable)")
|
|
122
|
+
p.add_argument("--root", default=HERE, help="directory the pinned paths are relative to (default: the project)")
|
|
123
|
+
p = sub.add_parser("check")
|
|
124
|
+
p.add_argument("file")
|
|
125
|
+
p.add_argument("--root", default=HERE)
|
|
126
|
+
a = ap.parse_args(argv)
|
|
127
|
+
if a.cmd == "check":
|
|
128
|
+
try:
|
|
129
|
+
ok, report = check(a.file, a.root)
|
|
130
|
+
except OSError as e:
|
|
131
|
+
print(f"cannot read {a.file}: {e}", file=sys.stderr)
|
|
132
|
+
return 1
|
|
133
|
+
print("\n".join(report))
|
|
134
|
+
print("OK: every pinned file is unchanged" if ok else "FAILED: the code differs from what was registered")
|
|
135
|
+
return 0 if ok else 1
|
|
136
|
+
criteria = a.criteria
|
|
137
|
+
if a.criteria_file:
|
|
138
|
+
with open(a.criteria_file, encoding="utf-8") as f:
|
|
139
|
+
criteria = f.read()
|
|
140
|
+
if not (criteria or "").strip():
|
|
141
|
+
print("REFUSED: criteria are required (--criteria or --criteria-file): a registration without them registers nothing", file=sys.stderr)
|
|
142
|
+
return 1
|
|
143
|
+
bad = [c for c in a.commitment if not COMMIT_RE.match(c)]
|
|
144
|
+
if bad or len(set(a.commitment)) != len(a.commitment):
|
|
145
|
+
print("REFUSED: commitments must be distinct 64-character lowercase hex sha256 values", file=sys.stderr)
|
|
146
|
+
return 1
|
|
147
|
+
if os.path.exists(a.out):
|
|
148
|
+
print(f"REFUSED: {a.out} already exists; a registration is never overwritten (write a new one and mark the old one void)", file=sys.stderr)
|
|
149
|
+
return 1
|
|
150
|
+
files = pinned_files(a.baseline, a.submission, a.pin)
|
|
151
|
+
escaping = [rel for rel in files if not inside_root(rel, a.root)]
|
|
152
|
+
if escaping:
|
|
153
|
+
print(f"REFUSED: pinned paths must be relative and stay inside the project root; outside the root: {escaping}", file=sys.stderr)
|
|
154
|
+
return 1
|
|
155
|
+
missing = [rel for rel in files if not os.path.exists(os.path.join(a.root, rel))]
|
|
156
|
+
if missing:
|
|
157
|
+
print(f"REFUSED: cannot pin files that do not exist: {missing}", file=sys.stderr)
|
|
158
|
+
return 1
|
|
159
|
+
not_files = [rel for rel in files if not os.path.isfile(os.path.join(a.root, rel))]
|
|
160
|
+
if not_files:
|
|
161
|
+
print(f"REFUSED: not a regular file: {not_files}", file=sys.stderr)
|
|
162
|
+
return 1
|
|
163
|
+
text = render(a.title, a.family, a.baseline, a.submission, a.commitment, criteria, a.sandbox, a.jobs, files, a.root)
|
|
164
|
+
fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) # exclusive: never overwrite, even in a race
|
|
165
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
166
|
+
f.write(text)
|
|
167
|
+
print(f"wrote {a.out}: {len(a.commitment)} commitment(s), {len(files)} pinned file(s). Publish the commitments now; run nothing that "
|
|
168
|
+
"changes a pinned file until the run is finished (`prereg.py check` verifies).")
|
|
169
|
+
return 0
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
if __name__ == "__main__":
|
|
173
|
+
sys.exit(main())
|