guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,108 @@
1
+ """The site's Certificates section: every certificate in a directory, re-verified at build time, with the attestations that name it.
2
+
3
+ A directory holds NAME.cert.json, NAME.<anything>.att.json (attest.py sign output) and optionally trusted.txt (public keys, one hex per
4
+ line, `# name` allowed) that the site operator pinned from keys obtained out of band. Nothing here is trusted from the files: the certificate is
5
+ verified by certificate.verify with this checkout (nothing in it is executed), and each attestation is checked by attest.check.
6
+ The section says plainly what an unpinned key is worth: nothing about independence.
7
+ """
8
+ import glob
9
+ import html
10
+ import json
11
+ import os
12
+ import shutil
13
+
14
+ import attest
15
+ import certificate
16
+
17
+
18
+ def _read(path):
19
+ with open(path, encoding="utf-8") as f:
20
+ return json.load(f)
21
+
22
+
23
+ def collect(certs_dir):
24
+ """-> [{"name", "file", "cert", "result", "atts": [(file, attestation)], "attest": check result}] (a malformed file is shown as a failure, never skipped)."""
25
+ trusted = []
26
+ tpath = os.path.join(certs_dir, "trusted.txt")
27
+ if os.path.exists(tpath):
28
+ trusted = list(attest.read_trusted(tpath))
29
+ out = []
30
+ for cpath in sorted(glob.glob(os.path.join(certs_dir, "*.cert.json"))):
31
+ name = os.path.basename(cpath)[: -len(".cert.json")]
32
+ try:
33
+ cert = _read(cpath)
34
+ result = certificate.verify(cert)
35
+ except Exception as e: # a certificate is untrusted input: whatever it makes the verifier raise is a failed certificate, never a failed site build
36
+ out.append({"name": name, "file": os.path.basename(cpath), "cert": None,
37
+ "result": {"ok": False, "checks": [{"check": "format", "status": "fail", "detail": f"unreadable or hostile: {type(e).__name__}"}], "claim": None},
38
+ "atts": [], "attest": None})
39
+ continue
40
+ atts = []
41
+ for apath in sorted(glob.glob(os.path.join(glob.escape(certs_dir), glob.escape(name) + ".*.att.json"))):
42
+ try:
43
+ atts.append((os.path.basename(apath), _read(apath)))
44
+ except (OSError, ValueError):
45
+ atts.append((os.path.basename(apath), {}))
46
+ try:
47
+ checked = attest.check(cert, [a for _, a in atts], trusted) if atts else None
48
+ except Exception as e: # attest.check is hardened, but one hostile attestation must never cost the whole page
49
+ checked = {"ok": False, "pinned": 0, "unvouched": 0, "checks": [{"check": "attestations", "status": "fail", "detail": f"could not be checked: {type(e).__name__}"}]}
50
+ out.append({"name": name, "file": os.path.basename(cpath), "cert": cert, "result": result, "atts": atts, "attest": checked})
51
+ return out
52
+
53
+
54
+ def section_html(certs_dir):
55
+ """The Results-page section for `certs_dir`, or "" when it holds no certificate."""
56
+ items = collect(certs_dir) if certs_dir and os.path.isdir(certs_dir) else []
57
+ if not items:
58
+ return ""
59
+ label = {"pass": "Checked", "fail": "Failed", "skip": "Not checked", "warn": "Note"}
60
+ cards = []
61
+ for it in items:
62
+ res = it["result"]
63
+ claim = res["claim"]
64
+ private = isinstance(it["cert"], dict) and it["cert"].get("private") is True
65
+ head = (f'{html.escape(str(claim["credit_bits"]))} bits, {html.escape(str(claim["status"]))}' if claim else "does not verify")
66
+ if private and claim:
67
+ head += " (private solver)"
68
+ counted = isinstance(it["cert"], dict) and isinstance(it["cert"].get("count"), dict)
69
+ if counted and claim:
70
+ head += " (step counting)"
71
+ rlog = it["cert"].get("log") if isinstance(it["cert"], dict) else None
72
+ data0 = rlog[0].get("data") if isinstance(rlog, list) and rlog and isinstance(rlog[0], dict) else None
73
+ round_declared = bool(claim) and isinstance(data0, dict) and data0.get("cost_model") == "count" # a verified certificate whose round declared the counted ladder in entry 0
74
+ counted_note = ('<p class="fine"><strong>Step counting.</strong> The costs in this certificate are counts of Python instructions, not seconds. Anyone on the same Python '
75
+ "version can re-execute instances and must get identical counts, which proves those costs without trusting any machine. It cannot see work done inside "
76
+ "built-in C functions, "
77
+ + ("and the question sizes, the number of questions and the cap were declared by the round in its first log entry, before any submission, the same for every "
78
+ "participant.</p>" if round_declared else
79
+ "and the question sizes were chosen by whoever ran it, so it is not an official round result.</p>")) if counted else ""
80
+ private_note = ('<p class="fine"><strong>Private solver.</strong> Its source is not in this certificate, so it can be neither inspected nor re-run here: '
81
+ "the timings rest on the operator's sandbox alone. Its hash is recorded in the certificate (for a round, it was also in the round's log before "
82
+ "the seeds were revealed), so if its owner publishes the file anyone can check it.</p>") if private else ""
83
+ rows = "".join(f'<tr><td>{html.escape(c["check"])}</td><td><strong>{label.get(c["status"], c["status"])}</strong></td><td class="fine">{html.escape(c["detail"])}</td></tr>'
84
+ for c in res["checks"])
85
+ att_html = '<p class="fine">No one has attested to this certificate yet.</p>'
86
+ if it["attest"]:
87
+ a = it["attest"]
88
+ att_html = ("<h4>Independent re-checks</h4>"
89
+ f'<p class="fine"><strong>{a["pinned"]}</strong> from keys the operator pinned out of band; <strong>{a["unvouched"]}</strong> from keys nobody has vouched for. '
90
+ "A signature shows who signed, not that they are unconnected to the operator, so only the first number is counted as independent.</p>"
91
+ '<ul class="fine">' + "".join(f'<li>{html.escape(c["check"])}: {html.escape(c["detail"])}</li>' for c in a["checks"]) + "</ul>")
92
+ links = " ".join(f'<a href="certs/{html.escape(f)}">{html.escape(f)}</a>' for f in [it["file"]] + [f for f, _ in it["atts"]])
93
+ cards.append(f'<article class="card"><h3>{html.escape(it["name"])}: {head}</h3>'
94
+ '<p class="fine">Re-verified when this page was built, with nothing from the file executed. Checking proves the arithmetic and the fairness; '
95
+ f'it does not prove the recorded timings were honest (see the Limits page).</p>{private_note}{counted_note}'
96
+ f'<div class="scroll"><table class="lb"><thead><tr><th>Check</th><th>Outcome</th><th>What it found</th></tr></thead><tbody>{rows}</tbody></table></div>'
97
+ f'{att_html}<p class="fine">Files: {links}</p></article>')
98
+ return '<section class="block"><h2>Certificates</h2><div style="display:grid;gap:18px">' + "".join(cards) + "</div></section>"
99
+
100
+
101
+ def copy_files(certs_dir, out_dir):
102
+ """Copy the certificates, attestations and the pinned-key list (all public) into out_dir/certs."""
103
+ if not certs_dir or not os.path.isdir(certs_dir):
104
+ return
105
+ os.makedirs(os.path.join(out_dir, "certs"), exist_ok=True)
106
+ wanted = glob.glob(os.path.join(certs_dir, "*.cert.json")) + glob.glob(os.path.join(certs_dir, "*.att.json")) + glob.glob(os.path.join(certs_dir, "trusted.txt"))
107
+ for p in sorted(wanted): # exactly these: any other file in the directory (a key, a note, someone's JSON) is never published
108
+ shutil.copy(p, os.path.join(out_dir, "certs", os.path.basename(p)))
guldo/_src/claims.py ADDED
@@ -0,0 +1,343 @@
1
+ """Derived claims: other things a VERIFIED certificate can prove, from the same ladders, with no new measurement.
2
+
3
+ claims.py correctness CERT [--out F] [--rerun K|all --execute] "this program solved X% of committed, blind questions" (a benchmark score)
4
+ claims.py worst-case CERT [--out F] [--rerun K|all --execute] the largest step count observed per size (a COUNTED certificate only)
5
+ claims.py negative CERT [CERT ...] --threshold BITS [--out F] "none of the N submissions in this round reached BITS": a verified negative result
6
+ claims.py check CLAIM CERT [CERT ...] recompute a claim file from the certificates: it must be identical
7
+
8
+ Every claim starts by running certificate.verify on each certificate and refuses if it does not verify; it then RECOMPUTES the claim from the raw per-instance
9
+ records. A claim file is therefore not trusted either: `check` redoes it. The claims inherit the certificate's strength and its limits (a timed ladder rests on the
10
+ operator's sandbox, a counted one is reproducible by anyone on the same Python), and each says what it does NOT show.
11
+
12
+ What these are and are not. They are statements about the committed instances and the submissions actually made. A correctness score is a rate on THIS test
13
+ set (with a Wilson interval, which is about sampling the generator, not about other test sets); the worst-case figure is the largest value SEEN, not a proof about every
14
+ input of that size (the chance a fresh instance from the same generator exceeds the maximum of n is at most 1/(n+1), by exchangeability); a negative result is about the
15
+ submissions made (participants chose to submit) and about what could be CERTIFIED, not about every possible approach: the credit is a conservative lower bound (the minimum
16
+ over seeds, with an inconclusive seed counting as 0), so "did not reach T bits" means "no speedup of T bits was shown", not "none exists".
17
+ """
18
+ import argparse
19
+ import json
20
+ import math
21
+ import os
22
+ import secrets
23
+ import sys
24
+
25
+ import certificate as K
26
+
27
+ FORMAT = "svp-claim-v1"
28
+ STATUSES = ("solved", "wrong", "failed", "crash", "timeout")
29
+
30
+
31
+ class ClaimError(Exception):
32
+ pass
33
+
34
+
35
+ def _digest(cert):
36
+ return K.sha256(K.canon(cert))
37
+
38
+
39
+ def _verified(cert, rerun=0, execute=False, sample_seed=None):
40
+ """The certificate's verification result, or ClaimError: nothing is derived from a certificate that does not verify."""
41
+ if sample_seed is None:
42
+ sample_seed = secrets.randbelow(2 ** 31) # the verifier's own draw (see certificate.py)
43
+ res = K.verify(cert, rerun=rerun, execute=execute, sample_seed=sample_seed)
44
+ if not res["ok"]:
45
+ fails = [f"{c['check']}: {str(c['detail'])[:150]}" for c in res["checks"] if c["status"] == "fail"]
46
+ raise ClaimError("the certificate does not verify, so nothing can be derived from it (" + "; ".join(fails[:3]) + ")")
47
+ return res
48
+
49
+
50
+ def _check_status(res, name):
51
+ return next((c["status"] for c in res["checks"] if c["check"] == name), None)
52
+
53
+
54
+ def _records(cert):
55
+ """{role: {seed tag: [record, ...]}} from the certificate's raw ladders."""
56
+ out = {"baseline": {}, "submission": {}}
57
+ for tag, lad in cert["ladders"].items():
58
+ for role in out:
59
+ out[role][tag] = [json.loads(ln) for ln in lad[role].splitlines()[1:] if ln.strip()]
60
+ return out
61
+
62
+
63
+ def wilson_lower(k, n, z=1.96):
64
+ """Lower end of the Wilson score interval for k successes in n trials (z = 1.96: 95%). 0 when n is 0."""
65
+ if n <= 0:
66
+ return 0.0
67
+ p = k / n
68
+ denom = 1 + z * z / n
69
+ centre = p + z * z / (2 * n)
70
+ adj = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
71
+ return max(0.0, (centre - adj) / denom)
72
+
73
+
74
+ def _tally(recs):
75
+ t = {s: 0 for s in STATUSES}
76
+ t["other"] = 0
77
+ for r in recs:
78
+ t[r.get("status") if r.get("status") in STATUSES else "other"] += 1
79
+ t["total"] = len(recs)
80
+ return t
81
+
82
+
83
+ def _summ(t):
84
+ acc = t["solved"] / t["total"] if t["total"] else 0.0
85
+ return dict(t, accuracy=round(acc, 6), wilson95_lower=round(wilson_lower(t["solved"], t["total"]), 6))
86
+
87
+
88
+ def _reexecuted(res):
89
+ """{"k", "of"} when this derivation also re-executed instances under step counting (certificate.verify --rerun --execute), else None. It describes the run that
90
+ produced a claim, not the claim, so `check` leaves it out of the comparison."""
91
+ rr = next((c.get("data") for c in res["checks"] if c["check"] == "rerun" and c.get("data")), None)
92
+ return {"k": rr["k"], "of": rr["of"]} if isinstance(rr, dict) and rr.get("exact") is True else None
93
+
94
+
95
+ def _cap_of(cert):
96
+ count = cert.get("count") if isinstance(cert.get("count"), dict) else None
97
+ if count:
98
+ return {"cost_model": "count", "cap": count["cap"], "cap_unit": f"counted seconds ({count['steps_per_second']:g} bytecode instructions each), on {count['python']}",
99
+ "reproducible": True}
100
+ head = json.loads(next(iter(cert["ladders"].values()))["submission"].splitlines()[0])
101
+ return {"cost_model": "time", "cap": head.get("budget"), "cap_unit": "CPU seconds on the machine that measured it (see the certificate's sandbox)", "reproducible": False}
102
+
103
+
104
+ # ------------------------------------------------------------------------------------------------------------- correctness
105
+ def correctness(cert, res):
106
+ """A benchmark score: for each solver in the certificate, the fraction of the committed questions its answer passed the family's own verifier within the cap."""
107
+ recs = _records(cert)
108
+ roles = {}
109
+ for role, by_tag in recs.items():
110
+ allr = [r for rs in by_tag.values() for r in rs]
111
+ per_seed = {tag: _summ(_tally(rs))["accuracy"] for tag, rs in sorted(by_tag.items())}
112
+ roles[role] = dict(_summ(_tally(allr)), per_seed=per_seed, worst_seed_accuracy=min(per_seed.values()) if per_seed else 0.0,
113
+ per_size={str(d): _summ(_tally([r for r in allr if r["dim"] == d])) for d in sorted({r["dim"] for r in allr})})
114
+ cap = _cap_of(cert)
115
+ return {"format": FORMAT, "kind": "correctness", "params": {}, "family": cert["family"], "solvers": {"submission": cert["submission"]["sha256"], "baseline": cert["baseline"]["sha256"]},
116
+ "committed_before_submission": _check_status(res, "log") == "pass", "seeds": len(cert["ladders"]), "seeds_revealed": bool(cert.get("seeds_revealed")),
117
+ "scoring": dict(cap, rule="solved = the answer passed the family's own verifier within the cap; wrong, no answer, a crash and a timeout all count as not solved"),
118
+ "results": roles, "reexecuted": _reexecuted(res),
119
+ "limits": ["a rate on THIS committed test set; the Wilson interval is for sampling variation of the generator, not a claim about other test sets",
120
+ "the cap is part of the question: an answer that needs longer than the cap counts as not solved"
121
+ + ("" if cap["reproducible"] else "; with timed costs 'within the cap' depends on the measuring machine, and only a counted certificate makes it reproducible by anyone"),
122
+ *([] if not cap["reproducible"] else ["a counted record is only PROVEN for the instances someone re-executes (`claims.py correctness CERT --rerun all --execute`, on the "
123
+ "same Python, in a throwaway VM): a status that was edited in an instance nobody re-ran would not be noticed by the arithmetic checks"])],
124
+ "from": [_digest(cert)]}
125
+
126
+
127
+ # ------------------------------------------------------------------------------------------------------------- worst case (counted only)
128
+ def worst_case(cert, res):
129
+ """The largest step count SEEN at each size, for a counted certificate (the counts are exact and re-executable). A size with any unsolved instance has no bound."""
130
+ if not isinstance(cert.get("count"), dict):
131
+ raise ClaimError("a worst-case claim needs a COUNTED certificate: timed costs are machine-dependent, so a maximum of them proves nothing portable "
132
+ "(make one with `svp.py practice --count` or from a counted round)")
133
+ recs = _records(cert)
134
+ roles = {}
135
+ for role, by_tag in recs.items():
136
+ allr = [r for rs in by_tag.values() for r in rs]
137
+ per_size = {}
138
+ for d in sorted({r["dim"] for r in allr}):
139
+ at = [r for r in allr if r["dim"] == d]
140
+ solved = [r for r in at if r.get("status") == "solved"]
141
+ unsolved = len(at) - len(solved)
142
+ bounded = unsolved == 0 and bool(at)
143
+ per_size[str(d)] = {"instances": len(at), "unsolved": unsolved, "bounded": bounded,
144
+ "max_steps": max(r["steps"] for r in solved) if bounded else None,
145
+ "max_steps_among_solved": max((r["steps"] for r in solved), default=None),
146
+ "exceed_probability_bound": round(1 / (len(at) + 1), 6) if bounded else None}
147
+ roles[role] = {"per_size": per_size, "all_bounded": all(v["bounded"] for v in per_size.values())}
148
+ return {"format": FORMAT, "kind": "worst-case", "params": {}, "family": cert["family"], "solvers": {"submission": cert["submission"]["sha256"], "baseline": cert["baseline"]["sha256"]},
149
+ "committed_before_submission": _check_status(res, "log") == "pass", "seeds": len(cert["ladders"]),
150
+ "unit": f"bytecode instructions on {cert['count']['python']} (exactly reproducible: `--rerun K --execute` demands identical counts)",
151
+ "cap_instructions": int(cert["count"]["cap"] * cert["count"]["steps_per_second"]), "results": roles, "reexecuted": _reexecuted(res),
152
+ "limits": ["the largest value SEEN on the committed instances, not a bound over every input of that size: the chance that one more instance from the same "
153
+ "generator exceeds the maximum of n is at most 1/(n+1) (exchangeability), which is what exceed_probability_bound says",
154
+ "instructions are not seconds, and work inside C functions counts as one instruction (see COMPILER.md)",
155
+ "a size where any instance timed out or failed has no bound: those instances used more than the cap or did not finish"],
156
+ "from": [_digest(cert)]}
157
+
158
+
159
+ # ------------------------------------------------------------------------------------------------------------- verified negative results
160
+ def submission_coverage(log, certified):
161
+ """(logged, missing): {sha256: log data} of every submission the round's log accepted, and those among them with no certificate in `certified` (a set of sha256)."""
162
+ logged = {}
163
+ for e in log:
164
+ data = e.get("data") if isinstance(e, dict) else None
165
+ if e.get("kind") == "submission" and isinstance(data, dict) and isinstance(data.get("sha256"), str):
166
+ logged[data["sha256"]] = data
167
+ return logged, {h: d for h, d in logged.items() if h not in certified}
168
+
169
+
170
+ def negative(certs, results, threshold):
171
+ """'No submission in this round reached `threshold` bits.' Needs a certificate for EVERY submission the round's log accepted, or it says the claim is incomplete.
172
+ All certificates must verify, be of one round (same log), family, baseline and seeds."""
173
+ if not certs:
174
+ raise ClaimError("no certificate given")
175
+ if not (isinstance(threshold, (int, float)) and not isinstance(threshold, bool) and math.isfinite(threshold) and threshold > 0):
176
+ raise ClaimError("--threshold must be a positive number of bits")
177
+ for c in certs:
178
+ if not c.get("log"):
179
+ raise ClaimError("a certificate without the round's log cannot support a negative result: without it nobody can know who else submitted")
180
+ base = max((c["log"] for c in certs), key=len)
181
+ for c in certs:
182
+ if any(e.get("hash") != base[i].get("hash") for i, e in enumerate(c["log"])):
183
+ raise ClaimError("these certificates are not from one round: their logs diverge")
184
+ for key in ("family", "commitments"):
185
+ if len({json.dumps(c[key], sort_keys=True) for c in certs}) != 1:
186
+ raise ClaimError(f"the certificates disagree about {key}: they are not from one round")
187
+ if len({c["baseline"]["sha256"] for c in certs}) != 1:
188
+ raise ClaimError("the certificates were scored against different baselines")
189
+ shas = [c["submission"]["sha256"] for c in certs]
190
+ if len(set(shas)) != len(shas):
191
+ raise ClaimError("two certificates are for the same submission")
192
+ kinds = [e.get("kind") for e in base]
193
+ if "round_closed" not in kinds:
194
+ # a log is hash-chained, but a PREFIX of it verifies too: certificates built from a copy cut before the close would hide every later submission. The close entry
195
+ # is the end of the submission list (nothing is accepted after it), so without it nobody can say these are all the submissions
196
+ raise ClaimError("the round's log has no round_closed entry: submissions could still have been accepted after it ends, so these cannot be shown to be all of them")
197
+ if "submission" in kinds[kinds.index("round_closed") + 1:]:
198
+ raise ClaimError("the log records a submission after the round closed: the round did not follow its own rules")
199
+ logged, missing = submission_coverage(base, set(shas))
200
+ stray = [h for h in shas if h not in logged]
201
+ if stray:
202
+ raise ClaimError("a certificate is for a solver the round's log never accepted")
203
+ rows = []
204
+ for c in sorted(certs, key=lambda c: c["submission"]["sha256"]):
205
+ d = logged[c["submission"]["sha256"]]
206
+ rows.append({"id": d.get("id"), "handle": d.get("handle"), "sha256": c["submission"]["sha256"], "credit_bits": c["claim"]["credit_bits"], "status": c["claim"]["status"],
207
+ "reached_threshold": c["claim"]["credit_bits"] >= threshold, "private_solver": c.get("private") is True,
208
+ "cost_model": "count" if isinstance(c.get("count"), dict) else "time"})
209
+ complete = not missing
210
+ none_reached = not any(r["reached_threshold"] for r in rows)
211
+ weak = [r["handle"] for r in rows if r["status"] not in ("OFFICIAL", "COUNTED")]
212
+ return {"format": FORMAT, "kind": "negative", "params": {"threshold_bits": threshold}, "round": base[0]["data"].get("id"), "family": certs[0]["family"],
213
+ "log_head": base[-1].get("hash"), "log_entries": len(base), # publish/compare the head: it anchors "these are all the submissions"
214
+ "baseline": certs[0]["baseline"]["sha256"], "logged_submissions": len(logged), "certified": len(rows), "complete": complete,
215
+ "missing": [{"id": d.get("id"), "handle": d.get("handle"), "sha256": h} for h, d in sorted(missing.items())],
216
+ "max_credit_bits": max(r["credit_bits"] for r in rows), "none_reached_threshold": none_reached,
217
+ "claim_holds": complete and none_reached, "submissions": rows,
218
+ "limits": ["about the submissions actually made (participants chose to submit), not about every possible approach",
219
+ "a credit is a conservative lower bound (minimum over seeds; an inconclusive seed counts as 0): 'did not reach the threshold' means no speedup of that size "
220
+ "was SHOWN, not that none exists",
221
+ "the list of submissions is the round's own log up to its close entry; its head hash (log_head) should be compared with one published somewhere the operator cannot edit",
222
+ *([] if not weak else ["PROVISIONAL credits (" + ", ".join(str(w) for w in weak) + ") were not made by the sandbox or by counting: weaker evidence that they did not reach the threshold"]),
223
+ "a submission whose evaluation failed has no certificate: it is listed under `missing`, and the claim is then NOT complete"
224
+ + ("" if all(r["cost_model"] == "count" for r in rows) else "; timed certificates rest on the operator's sandbox, only counted ones are reproducible by anyone")],
225
+ "from": sorted(_digest(c) for c in certs)}
226
+
227
+
228
+ # ------------------------------------------------------------------------------------------------------------- driver
229
+ def derive(kind, certs, threshold=None, rerun=0, execute=False, sample_seed=None):
230
+ if kind == "negative":
231
+ results = [_verified(c) for c in certs]
232
+ return negative(certs, results, threshold)
233
+ if len(certs) != 1:
234
+ raise ClaimError(f"a {kind} claim is about one certificate")
235
+ res = _verified(certs[0], rerun, execute, sample_seed)
236
+ return correctness(certs[0], res) if kind == "correctness" else worst_case(certs[0], res)
237
+
238
+
239
+ def render(c):
240
+ """Plain-language lines for a claim."""
241
+ out = []
242
+ if c["kind"] == "correctness":
243
+ out.append(f"CORRECTNESS on family {c['family']}, {c['seeds']} committed seed(s)" + ("" if c["committed_before_submission"] else " (NO round log: the seeds were not shown to be public first)"))
244
+ out.append(f" scoring: {c['scoring']['rule']}; cap {c['scoring']['cap']:g} {c['scoring']['cap_unit']}")
245
+ for role, r in c["results"].items():
246
+ out.append(f" {role}: {r['solved']}/{r['total']} solved = {r['accuracy']:.1%} (95% Wilson lower bound {r['wilson95_lower']:.1%}); worst seed {r['worst_seed_accuracy']:.1%}; "
247
+ f"wrong {r['wrong']}, no answer {r['failed']}, crashed {r['crash']}, timed out {r['timeout']}")
248
+ elif c["kind"] == "worst-case":
249
+ out.append(f"WORST CASE SEEN on family {c['family']}, in {c['unit']}; cap {c['cap_instructions']:,} instructions")
250
+ for role, r in c["results"].items():
251
+ for d, v in r["per_size"].items():
252
+ out.append(f" {role} size {d}: " + (f"at most {v['max_steps']:,} over {v['instances']} instances (chance a fresh one exceeds it <= {v['exceed_probability_bound']:g})" if v["bounded"] else
253
+ f"NO BOUND: {v['unsolved']} of {v['instances']} not solved within the cap"))
254
+ else:
255
+ out.append(f"NEGATIVE RESULT, round {c['round']}, family {c['family']}: threshold {c['params']['threshold_bits']:g} bits")
256
+ out.append(f" {c['certified']} of {c['logged_submissions']} logged submission(s) certified; highest credit {c['max_credit_bits']} bits")
257
+ for r in c["submissions"]:
258
+ out.append(f" {str(r['handle']):16s} {r['credit_bits']:5} bits {r['status']}" + (" REACHED" if r["reached_threshold"] else ""))
259
+ if not c["complete"]:
260
+ out.append(f" INCOMPLETE: {len(c['missing'])} logged submission(s) have no certificate ({', '.join(str(m['handle']) for m in c['missing'])}), so the claim does not hold")
261
+ out.append(" CLAIM HOLDS: no logged submission reached the threshold" if c["claim_holds"] else " claim does not hold" + ("" if c["complete"] else " (incomplete)"))
262
+ if c["kind"] != "negative":
263
+ rx = c.get("reexecuted")
264
+ out.append(f" re-executed here under step counting: {rx['k']} of {rx['of']} instances, all identical" if rx else
265
+ " re-executed here: none (the records were checked arithmetically, not re-run)")
266
+ out.append(" does not show: " + "; ".join(c["limits"]))
267
+ return out
268
+
269
+
270
+ def _load(path):
271
+ with open(path, encoding="utf-8") as f:
272
+ return json.load(f)
273
+
274
+
275
+ def check_claim(claim, certs):
276
+ """Recompute `claim` from the certificates: (ok, why). Equality is exact; the claim file is not trusted."""
277
+ if not isinstance(claim, dict) or claim.get("format") != FORMAT or claim.get("kind") not in ("correctness", "worst-case", "negative"):
278
+ return False, "not a claim file"
279
+ if not (isinstance(claim.get("from"), list) and all(isinstance(x, str) for x in claim["from"])):
280
+ return False, "not a claim file (no list of certificate digests)"
281
+ try:
282
+ if sorted(claim["from"]) != sorted(_digest(c) for c in certs):
283
+ return False, "the certificates given are not the ones this claim was derived from"
284
+ params = claim.get("params") if isinstance(claim.get("params"), dict) else {}
285
+ again = derive(claim["kind"], certs, threshold=params.get("threshold_bits"))
286
+ same = _comparable(again) == _comparable(claim)
287
+ except ClaimError as e:
288
+ return False, str(e)
289
+ except (KeyError, TypeError, ValueError, AttributeError, IndexError) as e: # a malformed claim or certificate: a refusal, never a traceback
290
+ return False, f"malformed input ({type(e).__name__})"
291
+ return same, ("recomputed identically" if same else "the recomputed claim differs from the file")
292
+
293
+
294
+ def _comparable(claim):
295
+ return K.canon({k: v for k, v in claim.items() if k != "reexecuted"})
296
+
297
+
298
+ def main(argv=None):
299
+ ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
300
+ sub = ap.add_subparsers(dest="cmd", required=True)
301
+ for name in ("correctness", "worst-case"):
302
+ p = sub.add_parser(name)
303
+ p.add_argument("cert")
304
+ p.add_argument("--out")
305
+ p.add_argument("--rerun", type=K.rerun_count, default=0, metavar="K|all")
306
+ p.add_argument("--execute", action="store_true", help="re-run instances here (RUNS the certificate's solvers, unsandboxed: a throwaway VM)")
307
+ p = sub.add_parser("negative")
308
+ p.add_argument("certs", nargs="+")
309
+ p.add_argument("--threshold", type=float, required=True, help="the speedup in bits that no submission should have reached")
310
+ p.add_argument("--out")
311
+ p = sub.add_parser("check")
312
+ p.add_argument("claim")
313
+ p.add_argument("certs", nargs="+")
314
+ a = ap.parse_args(argv)
315
+ try:
316
+ if a.cmd == "check":
317
+ ok, why = check_claim(_load(a.claim), [_load(c) for c in a.certs])
318
+ print(("CONFIRMED: " if ok else "NOT CONFIRMED: ") + why)
319
+ return 0 if ok else 1
320
+ if a.cmd == "negative":
321
+ claim = derive("negative", [_load(c) for c in a.certs], threshold=a.threshold)
322
+ else:
323
+ claim = derive(a.cmd, [_load(a.cert)], rerun=a.rerun, execute=a.execute)
324
+ except ClaimError as e:
325
+ print(f"REFUSED: {e}", file=sys.stderr)
326
+ return 1
327
+ except (OSError, ValueError, KeyError, TypeError) as e:
328
+ print(f"REFUSED: could not read the input ({type(e).__name__}: {str(e)[:120]})", file=sys.stderr)
329
+ return 1
330
+ print("\n".join(render(claim)))
331
+ if a.out:
332
+ if os.path.exists(a.out):
333
+ print(f"REFUSED: {a.out} exists; a claim file is never overwritten", file=sys.stderr)
334
+ return 1
335
+ fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
336
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
337
+ json.dump(claim, f, indent=1, sort_keys=True)
338
+ print(f"wrote {a.out} (anyone can confirm it: claims.py check {a.out} CERT...)")
339
+ return 0
340
+
341
+
342
+ if __name__ == "__main__":
343
+ sys.exit(main())