guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/judge.py ADDED
@@ -0,0 +1,278 @@
1
+ """Judge workflow: new-seed -> (publish commitment) -> evaluate submissions -> reveal -> leaderboard.
2
+
3
+ judge.py new-seed --out judging.seed writes a 256-bit seed (mode 0600), prints the commitment to publish
4
+ judge.py check --seed judging.seed --commit HEX anyone can run this after the reveal
5
+ judge.py evaluate --seed judging.seed --submission solvers/x.py --baseline solvers/baseline_bkz.py
6
+ judge.py leaderboard
7
+ """
8
+ import argparse
9
+ import hashlib
10
+ import json
11
+ import os
12
+ import secrets
13
+ import subprocess
14
+ import sys
15
+
16
+ import families
17
+ import harness
18
+
19
+ HERE = os.path.dirname(os.path.abspath(__file__))
20
+ BOARD = os.environ.get("SVP_BOARD", os.path.join(HERE, "results", "leaderboard_v2.json"))
21
+ MIN_SEEDS = 3 # a leaderboard credit needs at least this many independent committed seeds
22
+ SCHEMA = 2
23
+
24
+
25
+ def default_baseline(family):
26
+ """The reference solver instances of this family are compared against (families.DEFAULT_BASELINE)."""
27
+ return os.path.join(HERE, families.DEFAULT_BASELINE[family])
28
+
29
+
30
+ def sh(*args):
31
+ r = subprocess.run([sys.executable, *args], cwd=HERE, capture_output=True, text=True)
32
+ if r.returncode:
33
+ raise SystemExit(r.stdout + r.stderr)
34
+ return r.stdout
35
+
36
+
37
+ def ladder_meta(path):
38
+ with open(path) as f:
39
+ return json.loads(f.readline())
40
+
41
+
42
+ def ladder_n(path):
43
+ """Smallest number of instances recorded at any dimension of a ladder file."""
44
+ counts = {}
45
+ with open(path) as f:
46
+ f.readline()
47
+ for line in f:
48
+ r = json.loads(line)
49
+ counts[r["dim"]] = counts.get(r["dim"], 0) + 1
50
+ return min(counts.values()) if counts else 0
51
+
52
+
53
+ def baseline_reusable(path, expect):
54
+ """(ok, reason). True only if the baseline ladder at `path` is PROVABLY the measurement this evaluation would make: the same seed
55
+ commitment, baseline file hash, family, budget, instance version, sandbox, image, CPU and job count, and every instance of the
56
+ ladder present. Anything else, including a file that cannot be read, means "measure it again"."""
57
+ try:
58
+ with open(path) as f:
59
+ meta = json.loads(f.readline())
60
+ recs = [json.loads(line) for line in f if line.strip()]
61
+ except (OSError, ValueError):
62
+ return False, "no readable baseline ladder"
63
+ if not isinstance(meta, dict) or not meta.get("meta"):
64
+ return False, "no readable baseline ladder"
65
+ for key, label in (("commit", "seed commitment"), ("solver_sha256", "baseline file"), ("family", "family"), ("budget", "budget"),
66
+ ("instance_version", "instance version"), ("sandbox", "sandbox"), ("image", "image"), ("cpu", "cpu"), ("jobs", "jobs")):
67
+ if meta.get(key) != expect[key]:
68
+ return False, f"the {label} differs ({meta.get(key)!r} vs {expect[key]!r})"
69
+ want = {f"{d}:{i}" for d in expect["dims"] for i in range(expect["n"])}
70
+ have = {r.get("id") for r in recs}
71
+ if have != want:
72
+ return False, f"incomplete: {len(want - have)} instance(s) missing, {len(have - want)} unexpected"
73
+ return True, ""
74
+
75
+
76
+ def make_entry(name, family, tag, sub_file, claim, target_dim, commit_checked, base_file=None):
77
+ m = ladder_meta(sub_file)
78
+ mb = ladder_meta(base_file) if base_file else {}
79
+ return {"schema": SCHEMA, "submission": name, "family": family, "seed_tag": tag, "commit": m["commit"],
80
+ "commit_checked": bool(commit_checked), "n": ladder_n(sub_file), "budget": m["budget"],
81
+ "sandbox": m.get("sandbox"), "instance_version": m.get("instance_version"), "image": m.get("image"),
82
+ "cpu": m.get("cpu"), "solver_sha256": m.get("solver_sha256"), "baseline": mb.get("solver"),
83
+ "baseline_sha256": mb.get("solver_sha256"),
84
+ "target_dim": int(target_dim),
85
+ "bits": None if claim["measured"] is None else round(claim["measured"], 1),
86
+ "measured_bins": claim.get("measured_bins"), "measured_dims": claim.get("measured_dims"),
87
+ "dim_pooled": claim.get("measured_by_dim"),
88
+ "extrapolated": None if claim["point"] is None or claim["p5"] is None
89
+ else round(max(0.0, min(claim["point"], claim["p5"])), 1),
90
+ "per_dim": {str(d): round(v, 2) for d, v in claim["s"].items()}}
91
+
92
+
93
+ def official_reasons(e):
94
+ """Why a board entry does NOT count toward an official credit (empty list = it counts)."""
95
+ if e.get("schema") != SCHEMA:
96
+ return ["legacy entry (no run metadata); rescore it"]
97
+ why = []
98
+ need = families.OFFICIAL[e["family"]]
99
+ if e["sandbox"] != "docker":
100
+ why.append("not run in the sandbox")
101
+ if e["n"] < need["n"]:
102
+ why.append(f"n={e['n']} < {need['n']} instances/dim")
103
+ if e["budget"] < need["budget"]:
104
+ why.append(f"budget {e['budget']} < {need['budget']}s")
105
+ if not e["commit_checked"]:
106
+ why.append("seed commitment not checked against a published value")
107
+ if not e.get("instance_version"):
108
+ why.append("no instance version")
109
+ if not e.get("cpu"):
110
+ why.append("no CPU model recorded")
111
+ if not e.get("solver_sha256"):
112
+ why.append("solver file hash not recorded")
113
+ if not e.get("baseline_sha256"):
114
+ why.append("baseline file hash not recorded")
115
+ return why
116
+
117
+
118
+ def leaderboard_rows(board):
119
+ groups = {}
120
+ for e in board:
121
+ groups.setdefault((e["submission"], e["family"]), []).append(e)
122
+ rows = []
123
+ for (name, fam), es in groups.items():
124
+ official = {}
125
+ for e in es:
126
+ if not official_reasons(e):
127
+ official[e["seed_tag"]] = e
128
+ # official seeds must be one machine type, one submission file and one baseline file: a minimum over seeds scored
129
+ # with different code (an edited solver, a different reference) is not a credit for anything
130
+ combine = []
131
+ if len({e["cpu"] for e in official.values()}) > 1:
132
+ combine.append("official seeds ran on different CPUs")
133
+ if len({e["solver_sha256"] for e in official.values()}) > 1:
134
+ combine.append("official seeds used different versions of the submission file")
135
+ if len({e["baseline_sha256"] for e in official.values()}) > 1:
136
+ combine.append("official seeds used different baseline files")
137
+ if combine:
138
+ official = {}
139
+ counted = list(official.values())
140
+ pool = counted if len(counted) >= MIN_SEEDS else es
141
+ vals = [e["bits"] for e in pool]
142
+ credit = 0.0 if any(v is None for v in vals) else min(vals)
143
+ status = "OFFICIAL" if len(counted) >= MIN_SEEDS else f"PROVISIONAL ({len(counted)}/{MIN_SEEDS} official seeds)"
144
+ reasons = (sorted({r for e in es for r in official_reasons(e)}) + combine) if len(counted) < MIN_SEEDS else []
145
+ rows.append({"credit": credit, "name": name, "family": fam, "status": status, "reasons": reasons,
146
+ "per_seed": ["inconclusive" if v is None else v for v in vals]})
147
+ rows.sort(key=lambda r: -r["credit"])
148
+ return rows
149
+
150
+
151
+ def main():
152
+ ap = argparse.ArgumentParser()
153
+ sub = ap.add_subparsers(dest="cmd", required=True)
154
+ p = sub.add_parser("new-seed")
155
+ p.add_argument("--out", required=True)
156
+ p.add_argument("--allow-in-tree", action="store_true", help="test seeds only; never for a real competition")
157
+ p = sub.add_parser("check")
158
+ p.add_argument("--seed", required=True)
159
+ p.add_argument("--commit", required=True)
160
+ p = sub.add_parser("evaluate")
161
+ p.add_argument("--seed", required=True)
162
+ p.add_argument("--submission", required=True)
163
+ p.add_argument("--baseline", help="reference solver (default: the family's own, families.DEFAULT_BASELINE)")
164
+ p.add_argument("--n", help="instances per dimension (default: the family's official value)")
165
+ p.add_argument("--budget", help="CPU-second cap per instance (default: the family's official value)")
166
+ p.add_argument("--dims", type=int, nargs="+", help="override the family's default ladder (e.g. for a non-default baseline)")
167
+ p.add_argument("--target-dim", default="512")
168
+ p.add_argument("--jobs", default="1", help="concurrent instances per ladder (local mode only; see drift_check.py)")
169
+ p.add_argument("--family", default="gm", choices=sorted(families.FAMILIES))
170
+ p.add_argument("--sandbox", choices=["docker"])
171
+ p.add_argument("--insecure-local", action="store_true",
172
+ help="run solvers as bare local processes (only for solvers you wrote; they can read the seed and cheat)")
173
+ p.add_argument("--published-commit", help="the sha256 you published BEFORE the deadline; the run is refused if the "
174
+ "seed does not hash to it, and without it the run can never be official")
175
+ p.add_argument("--reuse-baseline", action="store_true",
176
+ help="reuse this seed's baseline ladder if it is provably the same measurement (same seed, baseline file, budget, sandbox, "
177
+ "image, CPU, jobs, all instances present); otherwise measure it again. A multi-submission round measures it once.")
178
+ sub.add_parser("leaderboard")
179
+ sub.add_parser("rescore", help="rebuild board entries from saved ladder files (after a rule change)")
180
+ a = ap.parse_args()
181
+
182
+ if a.cmd == "new-seed":
183
+ if os.path.abspath(a.out).startswith(HERE + os.sep) and not a.allow_in_tree:
184
+ raise SystemExit("REFUSED: a seed inside the project tree is readable by bare-local-mode solvers and other "
185
+ "tooling. Write it elsewhere (e.g. ~/judge-seeds/), or pass --allow-in-tree for a throwaway test seed.")
186
+ try:
187
+ fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
188
+ except FileExistsError: # a seed is never overwritten: that would silently change a commitment that may already be published
189
+ raise SystemExit(f"REFUSED: {a.out} already exists. A seed is never overwritten (its commitment may already be published); "
190
+ "choose another path or remove it yourself.")
191
+ seed = secrets.token_hex(32).encode()
192
+ os.write(fd, seed)
193
+ os.close(fd)
194
+ print("PUBLISH THIS COMMITMENT NOW, keep the seed file secret until the deadline:")
195
+ print(hashlib.sha256(seed).hexdigest())
196
+ elif a.cmd == "check":
197
+ got = hashlib.sha256(open(a.seed, "rb").read().strip()).hexdigest()
198
+ print("OK: seed matches commitment" if got == a.commit else f"MISMATCH: seed hashes to {got}")
199
+ sys.exit(got != a.commit)
200
+ elif a.cmd == "evaluate":
201
+ if not a.sandbox and not a.insecure_local:
202
+ raise SystemExit("REFUSED: pass --sandbox docker (untrusted submissions), or --insecure-local for your own "
203
+ "solvers. Local mode lets a solver read the seed/judge code and orphan processes to hide CPU.")
204
+ name = os.path.splitext(os.path.basename(a.submission))[0]
205
+ full = hashlib.sha256(open(a.seed, "rb").read().strip()).hexdigest()
206
+ if a.published_commit and a.published_commit != full:
207
+ raise SystemExit(f"REFUSED: seed hashes to {full}, not the published commitment {a.published_commit}")
208
+ tag = full[:8]
209
+ out = {}
210
+ dims = [str(d) for d in (a.dims or families.DEFAULT_DIMS[a.family])]
211
+ n = a.n or str(families.OFFICIAL[a.family]["n"])
212
+ budget = a.budget or str(families.OFFICIAL[a.family]["budget"])
213
+ for role, path in (("baseline", a.baseline or default_baseline(a.family)), ("submission", a.submission)):
214
+ out[role] = os.path.join(HERE, "results", f"judged_{a.family}_{tag}_{role}_{os.path.basename(path)}.jsonl")
215
+ if role == "baseline" and a.reuse_baseline and os.path.exists(out[role]):
216
+ with open(path, "rb") as f:
217
+ base_sha = hashlib.sha256(f.read()).hexdigest()
218
+ ok, why = baseline_reusable(out[role], {
219
+ "commit": full, "solver_sha256": base_sha, "family": a.family, "budget": float(budget),
220
+ "instance_version": families.VERSIONS[a.family], "sandbox": a.sandbox,
221
+ "image": harness.image_id() if a.sandbox else None, "cpu": harness.cpu_brand(), "jobs": int(a.jobs),
222
+ "n": int(n), "dims": [int(d) for d in dims]})
223
+ print(f"baseline ladder: {'REUSED (identical measurement, already complete)' if ok else 'not reusable (' + why + '); measuring it again'}",
224
+ flush=True)
225
+ if ok:
226
+ continue
227
+ sh("harness.py", "--solver", path, "--family", a.family, "--dims", *dims, "--n", n, "--budget", budget,
228
+ "--seed-file", a.seed, "--out", out[role], "--jobs", a.jobs,
229
+ *(["--sandbox", a.sandbox] if a.sandbox else []))
230
+ report = sh("extrapolate.py", "--baseline", out["baseline"], "--submission", out["submission"],
231
+ "--target-dim", a.target_dim)
232
+ print(report)
233
+ claim = json.load(open(out["submission"] + ".claim.json"))
234
+ board = json.load(open(BOARD)) if os.path.exists(BOARD) else []
235
+ board = [r for r in board if not (r["submission"] == name and r["family"] == a.family and r["seed_tag"] == tag)]
236
+ board.append(make_entry(name, a.family, tag, out["submission"], claim, a.target_dim, bool(a.published_commit),
237
+ base_file=out["baseline"]))
238
+ with open(BOARD, "w") as f:
239
+ json.dump(board, f, indent=1)
240
+ elif a.cmd == "rescore":
241
+ import re
242
+ pat = re.compile(r"^judged_([a-z]+)_([0-9a-f]{8})_(baseline|submission)_(.+)\.py\.jsonl$") # any registered family
243
+ files = {}
244
+ for fn in sorted(os.listdir(os.path.join(HERE, "results"))):
245
+ m = pat.match(fn)
246
+ if m:
247
+ files.setdefault((m.group(1), m.group(2)), {}).setdefault(m.group(3), []).append((m.group(4), fn))
248
+ board = json.load(open(BOARD)) if os.path.exists(BOARD) else []
249
+ old = {(e["submission"], e["family"], e["seed_tag"]): e for e in board}
250
+ for (fam, tag), roles in sorted(files.items()):
251
+ if "baseline" not in roles:
252
+ continue
253
+ base_fn = os.path.join(HERE, "results", roles["baseline"][0][1])
254
+ for name, fn in roles.get("submission", []):
255
+ sub_fn = os.path.join(HERE, "results", fn)
256
+ r = subprocess.run([sys.executable, "extrapolate.py", "--baseline", base_fn, "--submission", sub_fn],
257
+ cwd=HERE, capture_output=True, text=True)
258
+ if r.returncode:
259
+ print(f"skip {fn}: {r.stdout.strip().splitlines()[-1] if r.stdout.strip() else r.stderr.strip()[-200:]}")
260
+ continue
261
+ claim = json.load(open(sub_fn + ".claim.json"))
262
+ keep = old.get((name, fam, tag), {}).get("commit_checked", False)
263
+ old[(name, fam, tag)] = make_entry(name, fam, tag, sub_fn, claim, 512, keep, base_file=base_fn)
264
+ print(f"rescored {name} {fam} seed {tag}: measured credit {old[(name, fam, tag)]['bits']}")
265
+ with open(BOARD, "w") as f:
266
+ json.dump(list(old.values()), f, indent=1)
267
+ elif a.cmd == "leaderboard":
268
+ board = json.load(open(BOARD)) if os.path.exists(BOARD) else []
269
+ print(f"credit = MIN over >= {MIN_SEEDS} OFFICIAL seeds (docker sandbox, per-family n and budget {families.OFFICIAL}, "
270
+ f"commitment checked); an inconclusive seed counts as 0. Otherwise PROVISIONAL.")
271
+ for i, r in enumerate(leaderboard_rows(board), 1):
272
+ print(f"{i:2d}. {r['name']:22s} {r['family']:5s} {r['credit']:5.1f} bits {r['status']:36s} per-seed {r['per_seed']}")
273
+ for why in r["reasons"]:
274
+ print(f" not official: {why}")
275
+
276
+
277
+ if __name__ == "__main__":
278
+ main()
@@ -0,0 +1,158 @@
1
+ """Which open limits can be closed, and what evidence of closing exists on disk right now. Read-only: it runs nothing, evaluates nothing, writes nothing.
2
+
3
+ python3 limits_plan.py [--data D] [--spread FILE FILE ...]
4
+
5
+ --data D the operator's data directory (ops.py --data): checks the sandbox marker there
6
+ --spread F F ... `compile_domain.py recheck NAME --measure K --save FILE` files from DIFFERENT machines: checks that a tolerance can be recommended
7
+
8
+ Each limit is one of
9
+ CLOSABLE engineering or more data closes it: the line says what, and RESOLVE.md has the commands
10
+ NOT CLOSABLE a property of the method, stated on the website for good (the site keeps these)
11
+ Evidence is only what is findable: a human still judges each pre-registered run against the criteria it wrote down. "evidence found" means the necessary
12
+ artefacts exist, not that the limit may be removed from the website; RESOLVE.md says which text to change when it is.
13
+ """
14
+ import argparse
15
+ import glob
16
+ import json
17
+ import os
18
+ import re
19
+ import sys
20
+
21
+ HERE = os.path.dirname(os.path.abspath(__file__))
22
+ RESULTS = os.path.join(HERE, "results")
23
+
24
+
25
+ def _headers(pattern):
26
+ """The first (header) line of every ladder file matching `pattern` in results/, as dicts."""
27
+ out = []
28
+ for p in sorted(glob.glob(os.path.join(RESULTS, pattern))):
29
+ try:
30
+ with open(p, encoding="utf-8") as f:
31
+ h = json.loads(f.readline())
32
+ if isinstance(h, dict) and h.get("meta"):
33
+ out.append(h)
34
+ except (OSError, ValueError):
35
+ continue
36
+ return out
37
+
38
+
39
+ def _preregs():
40
+ """[(file name, text)] of every registration in results/."""
41
+ out = []
42
+ for p in sorted(glob.glob(os.path.join(RESULTS, "PREREGISTERED_*.txt"))):
43
+ try:
44
+ with open(p, encoding="utf-8") as f:
45
+ out.append((os.path.basename(p), f.read()))
46
+ except OSError:
47
+ continue
48
+ return out
49
+
50
+
51
+ def _completed(text):
52
+ """A registration counts as run when ladders exist for its first seed commitment (the tag is the first 8 hex)."""
53
+ m = re.search(r"commitment[^:\n]*:\s*([0-9a-f]{64})", text)
54
+ return bool(m) and bool(glob.glob(os.path.join(RESULTS, f"judged_*_{m.group(1)[:8]}_*.jsonl")))
55
+
56
+
57
+ def check_sandbox(a):
58
+ if not a.data:
59
+ return None, "pass --data D (the operator's data directory) to look for the marker `portal.py verify-sandbox` writes"
60
+ try:
61
+ import portal_core
62
+ ok = portal_core.sandbox_verified(os.path.abspath(a.data))
63
+ except Exception as e: # (any import or read problem: report, do not guess)
64
+ return None, f"could not check: {type(e).__name__}"
65
+ return ok, ("the marker is present and matches the sandbox files" if ok else "no valid marker: run `python3 launch_gates.py --with-docker --data D` on the Linux box")
66
+
67
+
68
+ def check_official(a):
69
+ try:
70
+ import judge
71
+ with open(judge.BOARD, encoding="utf-8") as f:
72
+ board = json.load(f)
73
+ rows = judge.leaderboard_rows(board)
74
+ except (OSError, ValueError, ImportError):
75
+ return False, "no leaderboard on disk"
76
+ official = [r for r in rows if r["status"] == "OFFICIAL"]
77
+ return bool(official), (f"{len(official)} OFFICIAL row(s): " + ", ".join(f"{r['name']} ({r['family']}) {r['credit']} bits" for r in official)) if official else \
78
+ f"no OFFICIAL row among {len(rows)} (an official row needs 3 seeds, the docker sandbox, the family's full n and cap)"
79
+
80
+
81
+ def check_one_job(a):
82
+ n = [h for h in _headers("judged_*_submission_*.jsonl") if h.get("sandbox") == "docker" and h.get("jobs") == 1]
83
+ return bool(n), (f"{len(n)} submission ladder(s) measured in the docker sandbox with one job at a time" if n else
84
+ "no ladder was measured in the docker sandbox with --jobs 1 (every ladder on disk is local-mode or several jobs at once)")
85
+
86
+
87
+ def check_fresh_seed(a):
88
+ regs = _preregs()
89
+ done = []
90
+ for n, t in regs:
91
+ fam = re.search(r"family\s+(\w+)", t)
92
+ void = re.search(r"(?:this|the)\s+(?:pre-?)?registration\s+is\s+void", t, re.I) # (a file that merely mentions voided predecessors is not void itself)
93
+ if fam and fam.group(1) in ("gm", "mlwe") and not void and _completed(t):
94
+ done.append(n)
95
+ return len(done) >= 3, (f"{len(regs)} registration(s) on disk; {len(done)} completed, non-void lattice run(s): {', '.join(done) or 'none'}. Closing needs at least 3 FRESH-seed lattice runs "
96
+ "(the first fresh-seed run counts), each registered before its data and judged against its written criteria (a person judges: see Results)")
97
+
98
+
99
+ def check_clean_control(a):
100
+ hits = []
101
+ for n, t in _preregs():
102
+ fam = re.search(r"family\s+(\w+)", t)
103
+ if fam and fam.group(1) == "factor" and re.search(r"--jobs\s+1\b|jobs\s*[:=]\s*1\b", t) and re.search(r"docker", t) and _completed(t):
104
+ hits.append(n)
105
+ return bool(hits), (f"a factoring control run one job at a time in docker: {', '.join(hits)}" if hits else
106
+ "no registered factoring control was run in docker with one job at a time (the seed E control ran six jobs at once, locally)")
107
+
108
+
109
+ def check_spread(a):
110
+ if not a.spread:
111
+ return None, "pass --spread FILE FILE ... (measurement files from at least two different machines) to check whether a tolerance can be recommended"
112
+ try:
113
+ import recheck
114
+ res, rec = recheck.spread(a.spread)
115
+ except Exception as e:
116
+ return None, f"could not read them: {type(e).__name__}: {str(e)[:100]}"
117
+ return rec is not None, (f"recommended tolerance {rec:g} bits from {len(a.spread)} file(s)" if rec is not None else "; ".join(f"{r['check']}: {r['detail']}" for r in res if r["status"] != "pass")[:300])
118
+
119
+
120
+ # (id, the limit in a few words, closable?, what closes it, evidence function or None, RESOLVE.md section, website text to change when it is closed)
121
+ LIMITS = [
122
+ ("sandbox", "The Docker sandbox has never been verified on a real host", True, "a Linux host: `launch_gates.py --with-docker --data D` (test_sandbox + verify-sandbox)", check_sandbox, "1",
123
+ "the first item of report_html.LIMITS, the 'locked box has not been tried' banner on the home page, 'never yet run on a real machine' on Limits"),
124
+ ("official", "No result is official (3 seeds, sandbox, full n and cap)", True, "an official round: `ops.py round-new` with 3 seeds, worker, reveal, certs", check_official, "2",
125
+ "the second item of report_html.LIMITS and the footer 'No result on this site is official'"),
126
+ ("one-job", "Runs used several jobs at once (a solver-dependent bias)", True, "the official worker evaluates one question at a time in the sandbox: the first docker ladder with jobs=1", check_one_job, "3",
127
+ "the two 'Several jobs at once' bullets on Limits"),
128
+ ("fresh-seed", "One seed is one sample; the credit rule was designed on seed A", True, "3+ pre-registered fresh-seed lattice runs (prereg_templates/fresh_seed_replication.criteria.txt)", check_fresh_seed, "4",
129
+ "'One seed is one sample' and the seed A item of report_html.LIMITS"),
130
+ ("clean-control", "The factoring control is not a clean recovery of a known number", True, "a calibrated pair at the ladder's own sizes, docker, one job (prereg_templates/clean_factoring_control.criteria.txt)", check_clean_control, "5",
131
+ "the factoring-control bullet on Limits"),
132
+ ("spread", "Cross-machine spread is unmeasured, so the re-measurement tolerance is loose", True, "`compile_domain.py recheck NAME --measure K --save F` on 2+ machines, then `spread`", check_spread, "6",
133
+ "'Computers differ' on Limits and certificate.RERUN_TOLERANCE"),
134
+ ("egg", "The paper's own code cannot yet be used as a baseline", True, "research, not engineering: a sieve that reaches the target past dimension 30", None, "7",
135
+ "the last item of report_html.LIMITS"),
136
+ ("range", "The certifiable range is about 1.5 to 3 bits (ceiling near 5)", False, "set by the 0.5 s cost floor and the cap; a larger n per round extends it but nothing removes the ceiling", None, "-", "stays on the website"),
137
+ ("private", "A private solver's timings rest on the operator's sandbox", False, "would need trusted hardware or zero-knowledge proofs of execution: not feasible for solvers that run for minutes", None, "-", "stays on the website"),
138
+ ("baseline", "The baseline decides what counts", False, "authors state what beats their baseline and independent baselines help, but it stays a judgement", None, "-", "stays on the website"),
139
+ ]
140
+
141
+
142
+ def main(argv=None):
143
+ ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
144
+ ap.add_argument("--data")
145
+ ap.add_argument("--spread", nargs="+")
146
+ a = ap.parse_args(argv)
147
+ for key, title, closable, how, fn, section, site in LIMITS:
148
+ if not closable:
149
+ print(f"NOT CLOSABLE {title}\n {how}. {site}.\n")
150
+ continue
151
+ ok, detail = (fn(a) if fn else (False, "needs research; there is nothing to check on disk"))
152
+ tag = "evidence found" if ok else ("unknown" if ok is None else "open")
153
+ print(f"CLOSABLE ({tag}) {title}\n closes with: {how}\n evidence: {detail}\n RESOLVE.md section {section}; when closed, change: {site}\n")
154
+ return 0
155
+
156
+
157
+ if __name__ == "__main__":
158
+ sys.exit(main())
guldo/_src/official.py ADDED
@@ -0,0 +1,189 @@
1
+ """Orchestration of an OFFICIAL multi-seed run.
2
+
3
+ official.py plan --family gm [--seeds 3] [--calib results/<baseline ladder>.jsonl]
4
+ How many instances, and the worst-case / expected CPU time, before you commit hours of machine time.
5
+ official.py commit --seed-dir ~/judge-seeds --seeds 3
6
+ Create the seeds (mode 0600, outside the project tree) and commitments.json. PUBLISH the commitments now.
7
+ official.py evaluate --family gm --submission solvers/x.py --seed-dir ~/judge-seeds --sandbox docker
8
+ Run judge.py evaluate for every seed, each checked against its published commitment.
9
+ official.py report --family gm --submission x Markdown report of the leaderboard row and its seeds.
10
+ """
11
+ import argparse
12
+ import hashlib
13
+ import json
14
+ import os
15
+ import secrets
16
+ import subprocess
17
+ import sys
18
+ import time
19
+
20
+ import families
21
+ import judge
22
+
23
+ HERE = os.path.dirname(os.path.abspath(__file__))
24
+ COMMITMENTS = "commitments.json"
25
+
26
+
27
+ def plan_cost(family, seeds=3, n=None, budget=None, dims=None, calib_costs=None):
28
+ """Worst-case and (if a baseline calibration is given) expected CPU seconds for an official run.
29
+ calib_costs: {dim: mean baseline cost in s} from an earlier baseline ladder (timeouts counted at the cap)."""
30
+ cfg = families.OFFICIAL[family]
31
+ n = n or cfg["n"]
32
+ budget = budget or cfg["budget"]
33
+ dims = dims or families.DEFAULT_DIMS[family]
34
+ per_ladder = len(dims) * n
35
+ out = {"family": family, "seeds": seeds, "n": n, "budget": budget, "dims": dims,
36
+ "instances_per_ladder": per_ladder, "ladders": 2 * seeds,
37
+ "worst_case_cpu_hours": per_ladder * 2 * seeds * budget / 3600.0, "expected_cpu_hours": None}
38
+ if calib_costs and all(d in calib_costs for d in dims):
39
+ base_h = sum(calib_costs[d] * n for d in dims) * seeds / 3600.0
40
+ out["expected_cpu_hours"] = base_h * 2 # submission assumed no slower than the baseline: an upper-ish guess
41
+ out["expected_baseline_cpu_hours"] = base_h
42
+ return out
43
+
44
+
45
+ def calib_from_ladder(path):
46
+ costs, count = {}, {}
47
+ with open(path) as f:
48
+ meta = json.loads(f.readline())
49
+ for line in f:
50
+ r = json.loads(line)
51
+ costs[r["dim"]] = costs.get(r["dim"], 0.0) + (r["cpu_net"] if r["status"] == "solved" else meta["budget"])
52
+ count[r["dim"]] = count.get(r["dim"], 0) + 1
53
+ return {d: costs[d] / count[d] for d in costs}
54
+
55
+
56
+ def make_commitments(seed_dir, k):
57
+ seed_dir = os.path.abspath(os.path.expanduser(seed_dir))
58
+ if seed_dir.startswith(HERE + os.sep) or seed_dir == HERE:
59
+ raise SystemExit("REFUSED: the seed directory must be outside the project tree")
60
+ os.makedirs(seed_dir, exist_ok=True)
61
+ cfile = os.path.join(seed_dir, COMMITMENTS)
62
+ if os.path.exists(cfile):
63
+ raise SystemExit(f"REFUSED: {cfile} already exists; commitments are created once and never replaced")
64
+ commits = []
65
+ for i in range(k):
66
+ seed = secrets.token_hex(32).encode()
67
+ fd = os.open(os.path.join(seed_dir, f"seed{i + 1}.seed"), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
68
+ os.write(fd, seed)
69
+ os.close(fd)
70
+ commits.append(hashlib.sha256(seed).hexdigest())
71
+ with open(cfile, "w") as f:
72
+ json.dump({"created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commitments": commits}, f, indent=1)
73
+ return commits
74
+
75
+
76
+ def verified_seeds(seed_dir):
77
+ """[(seed file, full sha256)] for every seed whose hash appears in commitments.json; raises on any mismatch."""
78
+ seed_dir = os.path.abspath(os.path.expanduser(seed_dir))
79
+ with open(os.path.join(seed_dir, COMMITMENTS)) as f:
80
+ commits = json.load(f)["commitments"]
81
+ seeds = sorted(fn for fn in os.listdir(seed_dir) if fn.endswith(".seed"))
82
+ out = []
83
+ for fn in seeds:
84
+ with open(os.path.join(seed_dir, fn), "rb") as f:
85
+ h = hashlib.sha256(f.read().strip()).hexdigest()
86
+ if h not in commits:
87
+ raise SystemExit(f"REFUSED: {fn} hashes to {h}, which is not in the published commitments")
88
+ out.append((os.path.join(seed_dir, fn), h))
89
+ if len(out) < len(commits):
90
+ raise SystemExit(f"REFUSED: {len(commits)} commitments published but only {len(out)} seed files present")
91
+ return out
92
+
93
+
94
+ def render_report(board, family, submission):
95
+ rows = [r for r in judge.leaderboard_rows(board) if r["name"] == submission and r["family"] == family]
96
+ lines = [f"# Result: {submission} on {family}", ""]
97
+ if not rows:
98
+ return "\n".join(lines + ["No board entries for this submission."])
99
+ r = rows[0]
100
+ lines += [f"**Credit: {r['credit']:.1f} bits ({r['status']})**", "",
101
+ f"Credit is the MIN over seeds of the difficulty-stratified lower bound; an inconclusive seed counts as 0.", "",
102
+ "| seed | credit (bits) | n/dim | cap | sandbox | commitment checked | bins used |", "|---|---|---|---|---|---|---|"]
103
+ for e in sorted((e for e in board if e["submission"] == submission and e["family"] == family),
104
+ key=lambda e: e["seed_tag"]):
105
+ bits = "inconclusive" if e["bits"] is None else f"{e['bits']:.1f}"
106
+ bins = ", ".join(f"[{lo:g},{hi:g})s" for lo, hi in (e.get("measured_bins") or [])) or "-"
107
+ lines.append(f"| {e['seed_tag']} | {bits} | {e.get('n')} | {e.get('budget')} | {e.get('sandbox')} | "
108
+ f"{e.get('commit_checked')} | {bins} |")
109
+ if r["reasons"]:
110
+ lines += ["", "Not official because:"] + [f"- {x}" for x in r["reasons"]]
111
+ return "\n".join(lines)
112
+
113
+
114
+ def load_board():
115
+ return json.load(open(judge.BOARD)) if os.path.exists(judge.BOARD) else []
116
+
117
+
118
+ def main():
119
+ ap = argparse.ArgumentParser()
120
+ sub = ap.add_subparsers(dest="cmd", required=True)
121
+ p = sub.add_parser("plan")
122
+ p.add_argument("--family", default="gm", choices=sorted(families.FAMILIES))
123
+ p.add_argument("--seeds", type=int, default=judge.MIN_SEEDS)
124
+ p.add_argument("--calib")
125
+ p = sub.add_parser("commit")
126
+ p.add_argument("--seed-dir", required=True)
127
+ p.add_argument("--seeds", type=int, default=judge.MIN_SEEDS)
128
+ p = sub.add_parser("evaluate")
129
+ p.add_argument("--family", default="gm", choices=sorted(families.FAMILIES))
130
+ p.add_argument("--submission", required=True)
131
+ p.add_argument("--seed-dir", required=True)
132
+ p.add_argument("--baseline", help="reference solver (default: the family's own, families.DEFAULT_BASELINE)")
133
+ p.add_argument("--sandbox", choices=["docker"])
134
+ p.add_argument("--insecure-local", action="store_true")
135
+ p.add_argument("--n", help="instances per size (default: the family's official value). More than the official minimum is allowed and "
136
+ "can still be official; FEWER is a smoke test and is never official")
137
+ p.add_argument("--budget", help="override the CPU cap (smoke tests only)")
138
+ p.add_argument("--dims", type=int, nargs="+", help="override the ladder (smoke tests, or a non-default baseline)")
139
+ p.add_argument("--jobs", default="1", help="concurrent instances per ladder, local mode only (see drift_check.py)")
140
+ p.add_argument("--reuse-baseline", action="store_true",
141
+ help="reuse each seed's baseline ladder when it is provably the same measurement (see judge.py); a round measures it once")
142
+ p = sub.add_parser("report")
143
+ p.add_argument("--family", default="gm", choices=sorted(families.FAMILIES))
144
+ p.add_argument("--submission", required=True)
145
+ a = ap.parse_args()
146
+
147
+ if a.cmd == "plan":
148
+ pl = plan_cost(a.family, a.seeds, calib_costs=calib_from_ladder(a.calib) if a.calib else None)
149
+ print(json.dumps(pl, indent=1))
150
+ if a.calib:
151
+ with open(a.calib) as f:
152
+ cal_budget = json.loads(f.readline())["budget"]
153
+ if cal_budget < pl["budget"]:
154
+ print(f"\nCAUTION: the calibration ladder used a {cal_budget:g} s cap but this run uses {pl['budget']} s: "
155
+ "instances that timed out there will take longer here, so the expected figure is an underestimate.")
156
+ print(f"\n{pl['instances_per_ladder']} instances per ladder x {pl['ladders']} ladders. Worst case "
157
+ f"{pl['worst_case_cpu_hours']:.1f} CPU-hours" + (f"; expected about {pl['expected_cpu_hours']:.1f} h."
158
+ if pl["expected_cpu_hours"] else "; no calibration given."))
159
+ elif a.cmd == "commit":
160
+ commits = make_commitments(a.seed_dir, a.seeds)
161
+ print("PUBLISH THESE COMMITMENTS NOW (before any submission is evaluated); keep the seed files secret:")
162
+ print("\n".join(commits))
163
+ elif a.cmd == "evaluate":
164
+ if not a.sandbox and not a.insecure_local:
165
+ raise SystemExit("REFUSED: pass --sandbox docker, or --insecure-local for your own solvers")
166
+ for seed_file, h in verified_seeds(a.seed_dir):
167
+ cmd = [sys.executable, "judge.py", "evaluate", "--seed", seed_file, "--submission", a.submission,
168
+ "--family", a.family, "--published-commit", h]
169
+ if a.baseline: # otherwise judge.py uses the family's own default baseline
170
+ cmd += ["--baseline", a.baseline]
171
+ cmd += ["--sandbox", a.sandbox] if a.sandbox else ["--insecure-local", "--jobs", a.jobs]
172
+ if a.reuse_baseline:
173
+ cmd.append("--reuse-baseline")
174
+ for flag, val in (("--n", a.n), ("--budget", a.budget)):
175
+ if val:
176
+ cmd += [flag, val]
177
+ if a.dims:
178
+ cmd += ["--dims", *map(str, a.dims)]
179
+ print(f"== seed {h[:8]}", flush=True)
180
+ r = subprocess.run(cmd, cwd=HERE)
181
+ if r.returncode:
182
+ raise SystemExit(f"judge.py evaluate failed for seed {h[:8]}")
183
+ print(render_report(load_board(), a.family, os.path.splitext(os.path.basename(a.submission))[0]))
184
+ elif a.cmd == "report":
185
+ print(render_report(load_board(), a.family, a.submission))
186
+
187
+
188
+ if __name__ == "__main__":
189
+ main()