guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/tflops.py ADDED
@@ -0,0 +1,453 @@
1
+ """Counted-OPERATIONS costs for AI workloads: the one cost for tensor code that anyone can reproduce exactly, on any machine.
2
+
3
+ python3 tflops.py practice SOLVER --baseline REF --family aidecode --dims 64 96 128 --n 8 --cap 2000 [--seed SEEDFILE]
4
+ python3 tflops.py claim BASELINE_LADDER SUBMISSION_LADDER the estimator's credit on the two ladders (extrapolate.py, unchanged)
5
+ python3 tflops.py verify LADDER [LADDER ...] [--rerun K] [--seed SEEDFILE]
6
+ python3 tflops.py selfcheck SOLVER --family aidecode --dim 64 does the same instance count the same twice?
7
+
8
+ A solver is written in the restricted subset PLUS `import tensorops as tp` (tensorops.py): every tensor operation charges an analytic cost computed from shapes, and
9
+ its own Python is charged one unit per bytecode instruction (tflops_runner.py). cost = flops + steps; one "counted second" = 1e7 such operations, so the estimator's
10
+ 0.5 s floor is 5e6 operations. Ladders are written in the ordinary format, so extrapolate.py reads them unchanged; `verify` redoes every cost from its record and, with
11
+ --rerun K, re-executes K records per ladder, drawn at random by the verifier (the draw's seed is printed), and demands IDENTICAL status, flops and steps: no tolerance, and no
12
+ trust in anyone's clock or hardware. A ladder with a fraction f of fabricated records passes a sample of K only with probability (1-f)^K, which is printed; the records
13
+ not sampled are checked for internal consistency only. `--rerun all` re-executes every record: a full audit.
14
+
15
+ SAFETY: the solver runs in the operator's process tree with NO container. What contains it is the restricted subset (no files, frames, getattr, eval, imports beyond math,
16
+ itertools and tensorops), the size and live-memory caps on tensors (tensorops.py), and a PARENT-side watchdog that samples the child's resident memory every 50 ms and
17
+ kills it past MAX_RSS_MB (so a Python-level allocation such as `[0.0] * 10**9` is stopped on macOS too, overshooting by at most what one 50 ms interval can touch). It is not
18
+ an operating-system sandbox: there is no file, network or process isolation beyond the restricted subset. Run untrusted solvers as an unprivileged throwaway user on a
19
+ machine you can lose; this tool refuses to run as root.
20
+
21
+ What a verified ladder shows, printed with every verdict: how many OPERATIONS the solver asked for versus the baseline. It is not a time. It cannot see memory traffic,
22
+ cache, vectorisation, dtype or kernel quality, so it certifies "N times fewer operations" (a KV cache, pruning, early exit, a smaller model at equal output) and cannot
23
+ certify "faster at the same arithmetic"; use a timed domain for that. Work inside one C call other than a tensor op is one instruction (as in counted.py). It needs
24
+ Python 3.12+. It is NOT wired into rounds, the portal or certificate.py: `verify` here is the checker. The counting program (tflops_runner.py, tensorops.py and the
25
+ unchanged count_runner.py it reuses) is pinned by hash in every ladder; a different counter is named as such.
26
+ """
27
+ import argparse
28
+ import hashlib
29
+ import json
30
+ import os
31
+ import random
32
+ import secrets
33
+ import subprocess
34
+ import sys
35
+ import tempfile
36
+ import time
37
+
38
+ import declared
39
+ import instances
40
+
41
+ HERE = os.path.dirname(os.path.abspath(__file__))
42
+ RUNNER = os.path.join(HERE, "tflops_runner.py")
43
+ UNIT_OPS = 10_000_000 # 1 "counted second" = 1e7 operations (a unit, not a duration). Chosen so the estimator's 0.5 s floor is 5e6 operations: a KV-cached
44
+ # decoder costs about 0.8e6 operations per token, so with 1e9 it would stay under the floor until T ~ 600 (where the reference costs ~150
45
+ # units) and nothing smaller could ever be certified; with 1e7 it clears the floor at small T and the reference is still affordable to count
46
+ SLOW_OPS_PER_SECOND = 20_000_000 # pessimistic wall throughput: used only for the wall-clock backstop
47
+ MAX_RSS_MB = 4096 # the most resident memory a counted solver may reach (the timed sandbox allows 4 GB too); enforced by the parent on every platform
48
+ MIN_PYTHON = (3, 12)
49
+ COUNTER_FILES = ("tflops_runner.py", "tensorops.py", "count_runner.py")
50
+ CLAIM_LIMITS = ("A count is not a time. It measures arithmetic asked for (tensor operations by shape, plus the solver's own Python instructions), not speed: memory traffic, "
51
+ "cache, vectorisation, dtype and kernel quality are invisible to it, so it can show 'N times fewer operations' and cannot show 'faster at the same arithmetic'. "
52
+ "The ladder is the prover's unless it was fixed in advance.")
53
+
54
+
55
+ def python_version():
56
+ v = sys.version_info
57
+ return f"{sys.implementation.name} {v.major}.{v.minor}.{v.micro}"
58
+
59
+
60
+ def counter_sha256():
61
+ h = hashlib.sha256()
62
+ for name in COUNTER_FILES:
63
+ with open(os.path.join(HERE, name), "rb") as f:
64
+ h.update(name.encode() + b"\0" + hashlib.sha256(f.read()).digest())
65
+ return h.hexdigest()
66
+
67
+
68
+ def sha256_file(path):
69
+ with open(path, "rb") as f:
70
+ return hashlib.sha256(f.read()).hexdigest()
71
+
72
+
73
+ def load_family(name):
74
+ d = os.path.join(declared.domains_dir(), name)
75
+ mod = declared.load_declaration(d)
76
+ derive, verify = declared.make_spec(mod)
77
+ return mod, derive, verify, d
78
+
79
+
80
+ def _rss_mb(pid):
81
+ """Resident memory of a process in MB, read by the PARENT with `ps` (works on macOS and Linux). 0 when it cannot be read (the process is gone)."""
82
+ try:
83
+ out = subprocess.run(["ps", "-o", "rss=", "-p", str(pid)], capture_output=True, text=True, timeout=5).stdout.strip()
84
+ return int(out) / 1024.0 if out else 0.0
85
+ except (OSError, ValueError, subprocess.SubprocessError):
86
+ return 0.0
87
+
88
+
89
+ def _run_watched(argv, payload, env, cwd, wall_timeout, max_rss_mb, poll=0.05):
90
+ """Run a child with a PARENT-side memory and wall-clock watchdog. -> (stdout, stderr, killed) with killed in (None, "memory", "wall").
91
+ Why the parent: a solver can allocate gigabytes in ONE C call (`[0.0] * 10**9`), which holds the GIL, so a watchdog thread inside the child would never get to run, and
92
+ macOS has no working address-space limit. A separate process sampling the child's resident size every `poll` seconds does not depend on either."""
93
+ p = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=cwd)
94
+ deadline, first = time.time() + wall_timeout, True
95
+ while True:
96
+ try:
97
+ out, err = p.communicate(input=payload if first else None, timeout=poll)
98
+ return out, err, None
99
+ except subprocess.TimeoutExpired:
100
+ first = False
101
+ killed = "memory" if _rss_mb(p.pid) > max_rss_mb else "wall" if time.time() > deadline else None
102
+ if killed:
103
+ p.kill()
104
+ out, err = p.communicate()
105
+ return out, err, killed
106
+
107
+
108
+ def count_instance(solver, inst, verify, cap, max_rss_mb=MAX_RSS_MB):
109
+ """Run `solver` on `inst` under operation counting. -> a ladder record: id, dim, status (solved/failed/wrong/crash/timeout), steps, flops, cpu, cpu_net, wall.
110
+ A child whose resident memory passes `max_rss_mb` is killed and recorded as a crash charged the cap."""
111
+ limit = int(cap * UNIT_OPS)
112
+ rec = {"id": inst["id"], "dim": inst["dim"]}
113
+ payload = json.dumps({"args": inst["args"], "limit": limit}).encode()
114
+ env = {"PYTHONHASHSEED": "0", "PYTHONDONTWRITEBYTECODE": "1", "OMP_NUM_THREADS": "1", "PATH": os.environ.get("PATH", "")}
115
+ t0 = time.time()
116
+ with tempfile.TemporaryDirectory() as empty:
117
+ stdout, stderr, killed = _run_watched([sys.executable, "-B", RUNNER, os.path.abspath(solver)], payload, env, empty, limit / SLOW_OPS_PER_SECOND + 120, max_rss_mb)
118
+ rec["wall"] = time.time() - t0
119
+ if killed == "wall":
120
+ rec.update({"status": "timeout", "steps": None, "flops": None, "wall_timeout": True, "cpu": cap, "cpu_net": cap})
121
+ return rec
122
+ if killed == "memory":
123
+ rec.update({"status": "crash", "steps": None, "flops": None, "cpu": cap, "cpu_net": cap, "err": f"killed: resident memory passed {max_rss_mb} MB"})
124
+ return rec
125
+ lines = [ln for ln in stdout.decode(errors="replace").splitlines() if ln.strip()]
126
+ try:
127
+ out = json.loads(lines[-1])
128
+ steps, flops, status = int(out["steps"]), int(out["flops"]), out["status"]
129
+ except (IndexError, ValueError, KeyError, TypeError):
130
+ rec.update({"status": "crash", "steps": None, "flops": None, "cpu": cap, "cpu_net": cap,
131
+ "err": "the counter produced no result: " + stderr.decode(errors="replace")[-300:]})
132
+ return rec
133
+ rec["steps"], rec["flops"] = steps, flops
134
+ if status == "limit":
135
+ rec.update({"status": "timeout", "cpu": cap, "cpu_net": cap})
136
+ elif status == "error":
137
+ rec.update({"status": "crash", "cpu": cap, "cpu_net": cap, "err": str(out.get("err", ""))[-1500:]})
138
+ else:
139
+ ans = out.get("ans")
140
+ try:
141
+ ok = bool(verify(inst, ans))
142
+ except Exception as e: # a verifier that RAISES is the operator's bug, not a wrong answer: say so in the record instead of passing for `wrong`
143
+ ok = False
144
+ rec["verify_error"] = f"{type(e).__name__}: {str(e)[:200]}"
145
+ rec["status"] = "solved" if ok else ("failed" if ans is None else "wrong")
146
+ rec["cpu"] = (steps + flops) / UNIT_OPS
147
+ rec["cpu_net"] = max(rec["cpu"], 1e-3)
148
+ if rec["status"] != "solved":
149
+ rec["cpu"] = rec["cpu_net"] = cap
150
+ return rec
151
+
152
+
153
+ def ladder_header(solver, family, mod, decl_dir, master, cap):
154
+ return {"meta": True, "solver": os.path.basename(solver), "commit": instances.commit(master), "overhead": 0.0, "budget": cap, "family": family,
155
+ "sandbox": "tflops", "instance_version": mod.VERSION, "image": None, "host": "analytic operation counting (no machine timing)",
156
+ "cpu": "none: operation counts", "ncpu": 1, "jobs": 1, "solver_sha256": sha256_file(solver),
157
+ "cost_model": "tflops", "unit_ops": UNIT_OPS, "python": python_version(), "counter_sha256": counter_sha256(),
158
+ "decl_sha256": sha256_file(os.path.join(decl_dir, "decl.py"))}
159
+
160
+
161
+ def evaluate_ladder(solver, family, master, dims, n, cap, out_path, progress=None):
162
+ if os.path.exists(out_path):
163
+ raise FileExistsError(out_path)
164
+ mod, derive, verify, d = load_family(family)
165
+ fd = os.open(out_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
166
+ recs = []
167
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
168
+ f.write(json.dumps(ladder_header(solver, family, mod, d, master, cap)) + "\n")
169
+ for dim in dims:
170
+ for idx in range(n):
171
+ rec = count_instance(solver, derive(master, dim, idx), verify, cap)
172
+ f.write(json.dumps(rec) + "\n")
173
+ f.flush()
174
+ recs.append(rec)
175
+ if progress:
176
+ progress(os.path.basename(solver), rec)
177
+ return recs
178
+
179
+
180
+ def read_ladder(path):
181
+ with open(path, encoding="utf-8") as f:
182
+ rows = [json.loads(ln) for ln in f if ln.strip()]
183
+ meta = rows[0] if rows and rows[0].get("meta") else None
184
+ return meta, [r for r in rows if not r.get("meta")]
185
+
186
+
187
+ def _master(args, meta=None):
188
+ if args.seed:
189
+ with open(args.seed, "rb") as f:
190
+ return f.read().strip()
191
+ return instances.PRACTICE_SEED
192
+
193
+
194
+ def verify_ladder(path, rerun=0, master=None, solver_path=None, sample_seed=None):
195
+ """-> (ok, [lines]). The cost-model check on every record; with rerun > 0 also the exact re-execution of that many instances, or of ALL of them with rerun="all".
196
+ The sample is drawn by the VERIFIER, at random (the draw's seed is printed, so a run can be reproduced): a prover cannot know which records will be re-executed, so one
197
+ who fabricated a fraction f of the records escapes a sample of k only with probability (1-f)^k, which is also printed."""
198
+ lines, ok = [], True
199
+ meta, recs = read_ladder(path)
200
+ if meta is None or meta.get("cost_model") != "tflops":
201
+ return False, [f"FAIL {os.path.basename(path)}: not a tflops ladder"]
202
+ cap, unit = meta.get("budget"), meta.get("unit_ops")
203
+ if isinstance(cap, bool) or not isinstance(cap, (int, float)) or cap <= 0 or "solver_sha256" not in meta or "commit" not in meta:
204
+ return False, [f"FAIL {os.path.basename(path)}: the header lacks a positive budget, a seed commitment or a solver hash"]
205
+ if not recs:
206
+ return False, [f"FAIL {os.path.basename(path)}: the ladder holds no records (an empty ladder certifies nothing, and a full audit of nothing would pass vacuously)"]
207
+ if unit != UNIT_OPS:
208
+ ok = False
209
+ lines.append(f"FAIL unit: the ladder's unit is {unit}, this checkout's is {UNIT_OPS}")
210
+ if meta.get("counter_sha256") != counter_sha256():
211
+ ok = False
212
+ lines.append("FAIL counter: the ladder was counted by a different counting program (hash differs from this checkout's): re-measure with this one")
213
+ bad = 0
214
+ isint = lambda v: isinstance(v, int) and not isinstance(v, bool) and v >= 0
215
+ for r in recs:
216
+ try:
217
+ if r["status"] == "solved":
218
+ if not (isint(r["steps"]) and isint(r["flops"])):
219
+ bad += 1 # a solved record must carry the exact counts the cost is made of
220
+ continue
221
+ exp = (r["steps"] + r["flops"]) / UNIT_OPS
222
+ if abs(r["cpu"] - exp) > 1e-12 or abs(r["cpu_net"] - max(exp, 1e-3)) > 1e-12:
223
+ bad += 1
224
+ elif r["cpu_net"] != cap:
225
+ bad += 1
226
+ except (KeyError, TypeError): # a malformed or hand-edited record is a failed check, never a crash
227
+ bad += 1
228
+ lines.append(f"{'PASS' if not bad else 'FAIL'} cost model: every solved cost = (flops + steps) / {UNIT_OPS:g} and every unsolved instance is charged the cap" + (f" ({bad} records disagree)" if bad else ""))
229
+ ok = ok and not bad
230
+ if rerun:
231
+ if meta.get("python") != python_version():
232
+ lines.append(f"SKIP re-run: ladder counted on {meta.get('python')}, this interpreter is {python_version()} (counts are only comparable on the same one)")
233
+ ok = False # a re-run that was asked for and did not happen is not a pass
234
+ else:
235
+ solver = solver_path or os.path.join(HERE, "solvers", meta["solver"])
236
+ if not os.path.exists(solver) or sha256_file(solver) != meta["solver_sha256"]:
237
+ lines.append(f"FAIL re-run: the solver file {meta['solver']} is missing here or its hash differs from the ladder's")
238
+ return False, lines
239
+ mod, derive, verify, d = load_family(meta["family"])
240
+ if sha256_file(os.path.join(d, "decl.py")) != meta.get("decl_sha256"):
241
+ lines.append("FAIL re-run: this checkout's decl.py differs from the one the ladder was made with")
242
+ return False, lines
243
+ if instances.commit(master) != meta["commit"]:
244
+ lines.append("SKIP re-run: the seed given does not match the ladder's commitment (pass the revealed seed with --seed)")
245
+ return False, lines # asked for, not done: not a pass
246
+ k = len(recs) if rerun == "all" else min(int(rerun), len(recs))
247
+ if k >= len(recs):
248
+ pick = list(recs)
249
+ lines.append(f"FULL AUDIT: all {len(recs)} records are re-executed")
250
+ else:
251
+ seed = secrets.randbits(64) if sample_seed is None else sample_seed
252
+ pick = random.Random(seed).sample(recs, k)
253
+ lines.append(f"sample: {k} of {len(recs)} records drawn at random (sample seed {seed}); a ladder with a fraction f of fabricated records passes only with "
254
+ f"probability (1-f)^{k}, i.e. under 5% once f exceeds {100 * (1 - 0.05 ** (1.0 / k)):.0f}%")
255
+ same = 0
256
+ for r in pick:
257
+ dim, idx = r["id"].split(":")
258
+ again = count_instance(solver, derive(master, int(dim), int(idx)), verify, cap)
259
+ if (again.get("status"), again.get("steps"), again.get("flops")) == (r["status"], r.get("steps"), r.get("flops")):
260
+ same += 1
261
+ else:
262
+ lines.append(f" differs at {r['id']}: recorded {r['status']} {r.get('steps')}+{r.get('flops')}, re-run {again.get('status')} {again.get('steps')}+{again.get('flops')}")
263
+ good = same == len(pick)
264
+ lines.append(f"{'PASS' if good else 'FAIL'} re-run: {same}/{len(pick)} instances re-executed with identical status, flops and steps")
265
+ ok = ok and good
266
+ return ok, lines
267
+
268
+
269
+ def baseline_reusable(path, solver, family, master, dims, n, cap):
270
+ """-> (ok, why). An existing baseline ladder may be reused only if its header is identical to the one that would be written now (same seed, solver bytes, family and
271
+ declaration, budget, interpreter and counter) and it holds exactly the expected instances, each once: a ladder left half-written by an interrupted run must not pass."""
272
+ try:
273
+ meta, recs = read_ladder(path)
274
+ except (OSError, ValueError) as e:
275
+ return False, f"unreadable ({type(e).__name__})"
276
+ mod, derive, verify, d = load_family(family)
277
+ if meta != ladder_header(solver, family, mod, d, master, cap):
278
+ return False, "its header differs from the one this run would write (different seed, solver, declaration, budget, interpreter or counter)"
279
+ want = {f"{dim}:{i}" for dim in dims for i in range(n)}
280
+ got = [r.get("id") for r in recs]
281
+ if len(got) != len(set(got)) or set(got) != want:
282
+ return False, f"it holds {len(set(got) & want)} of the {len(want)} expected instances (an interrupted run, or a different ladder)"
283
+ return True, ""
284
+
285
+
286
+ PLAN_FORMAT = "svp-ladder-plan-v1"
287
+
288
+
289
+ def plan_hash(plan):
290
+ """The value to PUBLISH before any submission: sha256 of the plan's canonical JSON."""
291
+ return hashlib.sha256(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
292
+
293
+
294
+ def plan_new(cost_model, family, dims, n, cap, master, extra=None):
295
+ """A ladder plan: everything the prover would otherwise choose (the family, the sizes, how many instances per size, the budget, the seed commitment, the declaration
296
+ and, for counted ladders, the counter) fixed in advance. Publish plan_hash(plan) somewhere the operator cannot edit BEFORE any solver is submitted; a ladder then
297
+ verifies against the plan only if it matches it exactly. `extra` carries mode-specific pins (counter hash and unit for tflops, the device for the GPU meter)."""
298
+ mod, derive, verify, d = load_family(family)
299
+ plan = {"format": PLAN_FORMAT, "cost_model": cost_model, "family": family, "dims": sorted({int(x) for x in dims}), "n": int(n), "cap": cap,
300
+ "commit": instances.commit(master), "instance_version": mod.VERSION, "decl_sha256": sha256_file(os.path.join(d, "decl.py")),
301
+ "created_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
302
+ plan.update(extra or {})
303
+ return plan
304
+
305
+
306
+ def check_plan(plan, meta, recs, expect_hash=None):
307
+ """-> [problems] (empty: the ladder is exactly the planned one). Header fields, the pins in `extra`, and the exact instance set, each instance once."""
308
+ problems = []
309
+ if not isinstance(plan, dict) or plan.get("format") != PLAN_FORMAT:
310
+ return [f"not a {PLAN_FORMAT} file"]
311
+ if expect_hash is not None and plan_hash(plan) != expect_hash:
312
+ problems.append("the plan file does not hash to the published value: it is not the plan that was committed to")
313
+ for plan_key, meta_key in (("cost_model", "cost_model"), ("family", "family"), ("commit", "commit"), ("cap", "budget"), ("instance_version", "instance_version"),
314
+ ("decl_sha256", "decl_sha256"), ("counter_sha256", "counter_sha256"), ("unit_ops", "unit_ops"), ("cpu", "cpu")):
315
+ if plan_key in plan and plan[plan_key] != meta.get(meta_key):
316
+ problems.append(f"{meta_key}: the ladder has {meta.get(meta_key)!r}, the plan says {plan[plan_key]!r}")
317
+ want = {f"{d}:{i}" for d in plan.get("dims", []) for i in range(int(plan.get("n", 0)))}
318
+ got = [r.get("id") for r in recs]
319
+ if len(got) != len(set(got)):
320
+ problems.append("the ladder holds some instance more than once")
321
+ if set(got) != want:
322
+ problems.append(f"the ladder's instances are not the planned ones ({len(set(got) & want)} of {len(want)} planned present, {len(set(got) - want)} unplanned)")
323
+ return problems
324
+
325
+
326
+ def load_plan(path):
327
+ with open(path, encoding="utf-8") as f:
328
+ return json.load(f)
329
+
330
+
331
+ def cmd_plan(a):
332
+ master = _master(a)
333
+ plan = plan_new("tflops", a.family, a.dims, a.n, a.cap, master, {"counter_sha256": counter_sha256(), "unit_ops": UNIT_OPS})
334
+ fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) # never overwrites a plan
335
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
336
+ json.dump(plan, f, indent=1, sort_keys=True)
337
+ print(f"plan written to {a.out}\nplan hash: {plan_hash(plan)}\nPublish this hash (somewhere the operator cannot edit) BEFORE any solver is submitted.")
338
+
339
+
340
+ def cmd_practice(a):
341
+ master = _master(a)
342
+ tag = instances.commit(master)[:8]
343
+ if a.plan:
344
+ plan = load_plan(a.plan)
345
+ if (plan.get("family"), sorted(set(a.dims)), a.n, a.cap, plan.get("commit")) != (a.family, plan.get("dims"), plan.get("n"), plan.get("cap"), instances.commit(master)):
346
+ sys.exit("refused: the dims, n, budget, family or seed on the command line are not the plan's. A planned ladder is run exactly as planned.")
347
+ os.makedirs(os.path.join(HERE, "results"), exist_ok=True)
348
+ paths = []
349
+ for role, solver in (("baseline", a.baseline), ("submission", a.solver)):
350
+ p = os.path.join(HERE, "results", f"tflops_{a.family}_{tag}_{role}_{os.path.basename(solver)}.jsonl")
351
+ if os.path.exists(p) and role == "baseline" and a.reuse_baseline:
352
+ ok, why = baseline_reusable(p, solver, a.family, master, a.dims, a.n, a.cap)
353
+ if not ok:
354
+ sys.exit(f"{p} exists but cannot be reused: {why}. Move it aside and run again.")
355
+ paths.append(p)
356
+ continue
357
+ evaluate_ladder(solver, a.family, master, a.dims, a.n, a.cap, p, progress=lambda name, rec: print(f" {name} {rec['id']}: {rec['status']} {rec.get('steps')}+{rec.get('flops')} ops", flush=True))
358
+ paths.append(p)
359
+ print("ladders:\n " + "\n ".join(paths))
360
+ print("next: python3 tflops.py claim " + " ".join(paths))
361
+
362
+
363
+ def cmd_claim(a):
364
+ r = subprocess.run([sys.executable, os.path.join(HERE, "extrapolate.py"), "--baseline", a.baseline, "--submission", a.submission], capture_output=True, text=True, cwd=HERE)
365
+ print(r.stdout + r.stderr)
366
+ if r.returncode:
367
+ sys.exit(r.returncode)
368
+ with open(a.submission + ".claim.json", encoding="utf-8") as f:
369
+ c = json.load(f)
370
+ print(f"credit (difficulty-stratified lower bound): {c.get('measured')} bits\n\nWhat this does NOT show: {CLAIM_LIMITS}")
371
+
372
+
373
+ def cmd_verify(a):
374
+ master = _master(a)
375
+ allok = True
376
+ plan = load_plan(a.plan) if a.plan else None
377
+ if a.plan_hash and not plan:
378
+ sys.exit("--plan-hash needs --plan")
379
+ for p in a.ladders:
380
+ ok, lines = verify_ladder(p, a.rerun, master, sample_seed=a.sample_seed)
381
+ if plan is not None:
382
+ meta, recs = read_ladder(p)
383
+ problems = check_plan(plan, meta or {}, recs, a.plan_hash)
384
+ lines.append(("FAIL plan: " + "; ".join(problems)) if problems else "PASS plan: the ladder is exactly the committed plan" + ("" if a.plan_hash else " (no --plan-hash given: the plan file itself was not checked against a published hash)"))
385
+ ok = ok and not problems
386
+ print(p)
387
+ for ln in lines:
388
+ print(" " + ln)
389
+ allok = allok and ok
390
+ print(("\nVERIFIED (costs consistent and re-executed as requested)" if a.rerun else "\nCONSISTENT ONLY: the cost arithmetic checks out, but nothing was re-executed (use --rerun K or --rerun all)") if allok else "\nNOT VERIFIED")
391
+ print("What this does NOT show: " + CLAIM_LIMITS)
392
+ sys.exit(0 if allok else 1)
393
+
394
+
395
+ def cmd_selfcheck(a):
396
+ mod, derive, verify, d = load_family(a.family)
397
+ inst = derive(instances.PRACTICE_SEED, a.dim, 0)
398
+ x, y = count_instance(a.solver, inst, verify, a.cap), count_instance(a.solver, inst, verify, a.cap)
399
+ same = (x["status"], x.get("steps"), x.get("flops")) == (y["status"], y.get("steps"), y.get("flops"))
400
+ print(f"run 1: {x['status']} steps {x.get('steps')} flops {x.get('flops')}\nrun 2: {y['status']} steps {y.get('steps')} flops {y.get('flops')}\n" + ("IDENTICAL" if same else "DIFFERENT: not countable"))
401
+ sys.exit(0 if same and x["status"] == "solved" else 1)
402
+
403
+
404
+ def main(argv=None):
405
+ if sys.version_info < MIN_PYTHON:
406
+ sys.exit("tflops needs Python 3.12 or newer")
407
+ if hasattr(os, "geteuid") and os.geteuid() == 0:
408
+ sys.exit("tflops refuses to run as root: the solver is contained by a restricted subset, not a sandbox. Use an unprivileged throwaway user.")
409
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
410
+ sub = ap.add_subparsers(dest="cmd", required=True)
411
+ p = sub.add_parser("practice")
412
+ p.add_argument("solver")
413
+ p.add_argument("--baseline", required=True)
414
+ p.add_argument("--family", required=True)
415
+ p.add_argument("--dims", type=int, nargs="+", required=True)
416
+ p.add_argument("--n", type=int, default=8)
417
+ p.add_argument("--cap", type=float, default=2000, help="budget in counted seconds (1 = 1e7 operations)")
418
+ p.add_argument("--seed")
419
+ p.add_argument("--reuse-baseline", action="store_true")
420
+ p.add_argument("--plan", help="run exactly this committed plan (refuses anything else)")
421
+ p.set_defaults(fn=cmd_practice)
422
+ p = sub.add_parser("plan", help="fix the ladder in advance and print the hash to publish")
423
+ p.add_argument("--family", required=True)
424
+ p.add_argument("--dims", type=int, nargs="+", required=True)
425
+ p.add_argument("--n", type=int, required=True)
426
+ p.add_argument("--cap", type=float, required=True)
427
+ p.add_argument("--seed")
428
+ p.add_argument("--out", required=True)
429
+ p.set_defaults(fn=cmd_plan)
430
+ p = sub.add_parser("claim")
431
+ p.add_argument("baseline")
432
+ p.add_argument("submission")
433
+ p.set_defaults(fn=cmd_claim)
434
+ p = sub.add_parser("verify")
435
+ p.add_argument("ladders", nargs="+")
436
+ p.add_argument("--rerun", type=lambda s: s if s == "all" else int(s), default=0, help="K random records to re-execute, or `all` for a full audit")
437
+ p.add_argument("--sample-seed", type=int, help="fix the random draw (default: fresh; the seed used is printed)")
438
+ p.add_argument("--plan", help="also require the ladder to be exactly this committed plan")
439
+ p.add_argument("--plan-hash", help="the hash published for the plan: the plan file must hash to it")
440
+ p.add_argument("--seed")
441
+ p.set_defaults(fn=cmd_verify)
442
+ p = sub.add_parser("selfcheck")
443
+ p.add_argument("solver")
444
+ p.add_argument("--family", required=True)
445
+ p.add_argument("--dim", type=int, required=True)
446
+ p.add_argument("--cap", type=float, default=2000, help="budget in counted seconds (1 = 1e7 operations)")
447
+ p.set_defaults(fn=cmd_selfcheck)
448
+ a = ap.parse_args(argv)
449
+ a.fn(a)
450
+
451
+
452
+ if __name__ == "__main__":
453
+ main()
@@ -0,0 +1,134 @@
1
+ """Run `solve(*args)` of one solver file under EXACT operation counting and report the count. Child process of tflops.py; not meant to be run by hand. Needs Python 3.12+.
2
+
3
+ argv[1] the solver file (a top-level `solve`), written in the restricted subset PLUS `import tensorops`
4
+ stdin {"args": [...], "limit": N} N = the most operations the solver may spend (tensor ops + Python bytecode instructions)
5
+ stdout the LAST line is {"status": "ok" | "limit" | "error", "steps": int, "flops": int, "ans": ..., "err": "..."}
6
+
7
+ Cost = flops + steps.
8
+ flops the analytic tensor-operation charges of tensorops.py (shapes only: see its docstring for the table)
9
+ steps Python bytecode instructions executed in the SOLVER'S OWN CODE (its module body and everything it defines), exactly as count_runner.py counts them.
10
+ Library code (tensorops, numpy, the interpreter's own Python) is never counted: its instruction locations are disabled the first time they fire. Without the steps
11
+ term a solver could do real arithmetic in plain-float Python loops where no tensor op charges it; with it, such work costs at least one unit per instruction.
12
+
13
+ The restricted subset is count_runner.py's (no frames, getattr, open, dunders, str.format, eval/exec, ...), imported and reused unchanged, with one change made at run
14
+ time in THIS process only: the module allowlist is math, itertools and tensorops (no fractions/collections/heapq/bisect/functools: parts of those are pure Python
15
+ whose instructions would not be counted under the solver-code rule). count_runner.py itself is not modified, so existing counted certificates are unaffected.
16
+
17
+ Limits, stated in every verdict: it is not a time; work inside ONE C call other than the tensor ops (a giant big-int power, say) is one instruction; and it is blind to
18
+ memory traffic, cache and kernel quality (see tensorops.py).
19
+ """
20
+ import gc
21
+ import json
22
+ import os
23
+ import resource
24
+ import sys
25
+
26
+ import count_runner as cr
27
+ import tensorops
28
+
29
+ TOOL_ID = 4
30
+
31
+ cr.ALLOWED_MODULES = {"math": None, "itertools": None, "tensorops": tuple(tensorops.EXPORTS)} # this process only
32
+
33
+
34
+ class _StepLimit(BaseException):
35
+ pass
36
+
37
+
38
+ def _emit(obj):
39
+ sys.__stdout__.write("\n" + json.dumps(obj) + "\n")
40
+ sys.__stdout__.flush()
41
+
42
+
43
+ def main():
44
+ if len(sys.argv) != 2:
45
+ _emit({"status": "error", "steps": 0, "flops": 0, "err": "usage: tflops_runner.py SOLVER"})
46
+ return
47
+ try:
48
+ payload = json.loads(sys.stdin.read())
49
+ args, limit = payload["args"], int(payload["limit"])
50
+ except (ValueError, KeyError, TypeError) as e:
51
+ _emit({"status": "error", "steps": 0, "flops": 0, "err": f"bad payload: {type(e).__name__}"})
52
+ return
53
+ if sys.version_info < (3, 12):
54
+ _emit({"status": "error", "steps": 0, "flops": 0, "err": "counting needs Python 3.12 or newer (sys.monitoring)"})
55
+ return
56
+ here = os.path.dirname(os.path.abspath(__file__))
57
+ sys.path[:] = [p for p in sys.path if p and os.path.abspath(p) != here] # everything needed is imported already; a solver must not reach the judge's modules
58
+ mon = sys.monitoring
59
+ try:
60
+ delattr(sys, "monitoring")
61
+ except (AttributeError, TypeError):
62
+ pass
63
+ sys.modules.pop("sys.monitoring", None)
64
+ if sys.platform.startswith("linux"):
65
+ try:
66
+ resource.setrlimit(resource.RLIMIT_AS, (4 << 30, 4 << 30))
67
+ except (ValueError, OSError):
68
+ pass
69
+ solver_name = sys.argv[1]
70
+ try:
71
+ code, namespace = cr._prepare(solver_name)
72
+ except BaseException as e:
73
+ _emit({"status": "error", "steps": 0, "flops": 0, "err": f"could not load the solver: {type(e).__name__}: {str(e)[:200]}"})
74
+ return
75
+ count = [0]
76
+ tripped = [False]
77
+ tensorops._FLOPS[0] = 0
78
+ tensorops._LIMIT[0] = limit
79
+
80
+ def on_instruction(c, offset):
81
+ if c.co_filename != solver_name:
82
+ return mon.DISABLE # library code: never counted, and not asked about again at this location
83
+ if tripped[0]:
84
+ return None
85
+ n = count[0] + 1
86
+ count[0] = n
87
+ if n + tensorops._FLOPS[0] > limit:
88
+ tripped[0] = True
89
+ raise _StepLimit()
90
+ return None
91
+
92
+ status, ans, err = "ok", None, ""
93
+ real_stdout = sys.stdout
94
+ sys.stdout = sys.stderr
95
+ gc.collect()
96
+ gc.disable()
97
+ mon.use_tool_id(TOOL_ID, "svp-tflops")
98
+ mon.register_callback(TOOL_ID, mon.events.INSTRUCTION, on_instruction)
99
+ try:
100
+ mon.set_events(TOOL_ID, mon.events.INSTRUCTION)
101
+ exec(code, namespace)
102
+ ans = namespace["solve"](*args)
103
+ except (_StepLimit, tensorops.BudgetExceeded):
104
+ status = "limit"
105
+ except BaseException as e:
106
+ status, err = "error", f"{type(e).__name__}: {str(e)[:200]}"
107
+ finally:
108
+ try:
109
+ intact = mon.get_events(TOOL_ID) == mon.events.INSTRUCTION
110
+ except ValueError:
111
+ intact = False
112
+ try:
113
+ mon.set_events(TOOL_ID, mon.events.NO_EVENTS)
114
+ mon.register_callback(TOOL_ID, mon.events.INSTRUCTION, None)
115
+ mon.free_tool_id(TOOL_ID)
116
+ except ValueError:
117
+ intact = False
118
+ gc.enable()
119
+ sys.stdout = real_stdout
120
+ steps, flops = count[0], tensorops._FLOPS[0]
121
+ if tripped[0] or flops > limit:
122
+ status, ans, err = "limit", None, ""
123
+ elif status == "ok" and not intact:
124
+ status, ans, err = "error", None, "the solver reconfigured the counter: its count means nothing"
125
+ if status == "ok":
126
+ try:
127
+ json.dumps(ans)
128
+ except (TypeError, ValueError):
129
+ status, err, ans = "error", "the answer is not JSON-serialisable (an int, a list of ints or None is expected)", None
130
+ _emit({"status": status, "steps": steps, "flops": flops, "ans": ans if status == "ok" else None, "err": err})
131
+
132
+
133
+ if __name__ == "__main__":
134
+ main()
guldo/cli.py ADDED
@@ -0,0 +1,60 @@
1
+ """The `guldo` command: the client, and `guldo <tool>` for the other tools. See `guldo tools`."""
2
+ import importlib
3
+ import inspect
4
+ import os
5
+ import sys
6
+
7
+ HERE = os.path.dirname(os.path.abspath(__file__))
8
+ SRC = os.path.join(HERE, "_src")
9
+ TOOLS = {
10
+ "claims": ("claims", "benchmark score, worst case, verified negative result from a verified certificate; `claims check` recomputes a claim file"),
11
+ "counted": ("counted", "step counting: `counted check SOLVER --family F --dim D` (is it deterministic and inside the restricted subset?), `counted evaluate`"),
12
+ "attest": ("attest", "sign and check independent attestations of a certificate"),
13
+ "cert": ("certificate", "build and verify certificates (the standalone verifier). Not `guldo certificate`: that is the client's own command, which downloads one"),
14
+ "compile": ("compile_domain", "declare a new problem and compile it into a challenge"),
15
+ "export": ("export_bundle", "export a compiled challenge as a platform-neutral bundle (checker, scorer, power analysis); `export verify DIR` re-checks one"),
16
+ "ops": ("ops", "operator tools: create, close, evaluate, reveal, export rounds"),
17
+ "lint": ("submit_check", "pre-flight check of a solver file"),
18
+ "judge": ("judge", "the judge (seeds, evaluate, leaderboard)"),
19
+ }
20
+
21
+
22
+ def _version():
23
+ try:
24
+ from importlib.metadata import version
25
+ return version("guldo")
26
+ except Exception:
27
+ return "unknown"
28
+
29
+
30
+ def _call(module, rest, prog):
31
+ fn = importlib.import_module(module).main
32
+ sys.argv = [prog] + rest
33
+ try:
34
+ takes_argv = bool(inspect.signature(fn).parameters)
35
+ except (TypeError, ValueError):
36
+ takes_argv = False
37
+ return fn(rest) if takes_argv else fn()
38
+
39
+
40
+ def main(argv=None):
41
+ argv = list(sys.argv[1:] if argv is None else argv)
42
+ if SRC not in sys.path:
43
+ sys.path.insert(0, SRC) # our modules first: a flat name like `judge` or `harness` must never resolve to something else installed
44
+ if argv[:1] in (["--version"], ["-V"]):
45
+ print(f"guldo {_version()}")
46
+ return 0
47
+ if argv[:1] == ["tools"]:
48
+ print("guldo <tool> ... (guldo --help shows the client's own commands)\n")
49
+ for name, (_, what) in sorted(TOOLS.items()):
50
+ print(f" guldo {name:12s} {what}")
51
+ return 0
52
+ if argv and argv[0] in TOOLS:
53
+ return _call(TOOLS[argv[0]][0], argv[1:], f"guldo {argv[0]}")
54
+ import svp # the client module keeps its file name; only the command a person types is `guldo`
55
+ svp.PROG = "guldo"
56
+ return svp.main(argv)
57
+
58
+
59
+ if __name__ == "__main__":
60
+ sys.exit(main())