guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,634 @@
1
+ """GPU timing for the speedup protocol: a DEVICE-PINNED, PARENT-TIMED wall-clock meter. Written without a GPU (this machine has none): everything on the parent side is tested
2
+ against a fake device and a fake transport (test_gpu_meter.py); the child (gpu_runner.py) and every torch solver are UNRUN until someone runs them on an NVIDIA box.
3
+
4
+ python3 gpu_meter.py info what device, driver, memory; is it idle; is it throttling
5
+ python3 gpu_meter.py plan --family aiinfer --dims 2048 3072 --n 6 --cap 60 --out PLAN.json fix the ladder in advance; prints the hash to publish
6
+ python3 gpu_meter.py practice SOLVER --baseline REF --family aiinfer --dims 2048 3072 --n 6 --cap 60 [--plan PLAN.json] [--seed SEEDFILE]
7
+ python3 gpu_meter.py claim BASELINE_LADDER SUBMISSION_LADDER the estimator's credit (extrapolate.py, unchanged)
8
+ python3 gpu_meter.py verify LADDER [--rerun K|all] [--plan PLAN.json --plan-hash HEX] consistency; --rerun RE-MEASURES records on this very device model
9
+
10
+ HOW IT MEASURES. One child process per size loads the solver and warms up on PUBLIC instances (never ladder instances). Then, for each ladder instance, the PARENT writes a
11
+ request line to the child and reads the reply line, and the time is the elapsed time on the PARENT's own clock between the two (the child brackets the solve with a device
12
+ synchronize). The child supplies no timing at all, so it cannot under-report; every timed run is a different instance, so nothing can be cached across repetitions; the answer
13
+ must come back, correct, inside the measured interval. One-time setup (imports, module-level work, loading a model) happens before the child says `ready` and is not measured,
14
+ exactly as loading weights is not part of an inference time; per-instance work must happen in solve(). Ladders are the ordinary format with `cpu` = the device model + driver, so
15
+ extrapolate.py REFUSES to compare ladders from different devices. One run per instance: noise is the estimator's job, as on the CPU path.
16
+
17
+ WHAT MAKES A MEASUREMENT INVALID (a crash charged the cap, flagged `invalid`, to be re-measured, never silently kept): another process is using the measured GPU; the GPU reported
18
+ a hardware or thermal slowdown while the solver ran. Hardware or driver trouble the parent cannot see is not covered.
19
+
20
+ TRUST: still the weakest evidence in the protocol, and every verdict says so. The parent's clock replaces the child's claim, but the solver runs unsandboxed in a process the
21
+ operator started: nothing here proves WHICH device it ran on beyond nvidia-smi, that clocks were not locked or raised, or that nobody tuned the box for one solver. Nobody else can
22
+ reproduce a time without the same device model: `verify --rerun` re-measures and agrees only within RERUN_TOLERANCE. A GPU speedup is a claim about one device model, certified
23
+ by an operator you trust, not a proof.
24
+ """
25
+ import argparse
26
+ import json
27
+ import os
28
+ import queue
29
+ import random
30
+ import secrets
31
+ import shutil
32
+ import statistics
33
+ import subprocess
34
+ import sys
35
+ import tempfile
36
+ import threading
37
+ import time
38
+
39
+ import instances
40
+ import tflops
41
+
42
+ HERE = os.path.dirname(os.path.abspath(__file__))
43
+ RUNNER = os.path.join(HERE, "gpu_runner.py")
44
+ NOOP = os.path.join(HERE, "solvers", "gpu_noop.py")
45
+ GPU_INDEX = os.environ.get("SVP_GPU_INDEX", "0") # the one GPU measured: nvidia-smi is asked about it alone (-i), and the child only sees it (CUDA_VISIBLE_DEVICES)
46
+ THROTTLE_BAD = 0x8 | 0x20 | 0x40 | 0x80 # hardware slowdown, software thermal, hardware thermal, hardware power brake: the measurement is not the device's normal speed
47
+ THROTTLE_NOTE = 0x4 # software power cap: routine under load on many data-centre parts, recorded but not fatal
48
+ WARM_INSTANCES = 2 # public warm-up instances run (untimed) per size before the ladder instances
49
+ CALIBRATION_CALLS = 5
50
+ RERUN_TOLERANCE = 1.3 # `verify --rerun` accepts a single re-measured run within this factor of the recorded one
51
+ POLL_S = 1.0 # background throttle sampling while a run is in progress: nvidia-smi queries inside a timed interval are interference, so only 1 Hz (plus a poll just before and just after every run)
52
+ GPU_MAX_RSS_MB = 16384 # host memory a GPU solver may reach: a torch + CUDA process alone holds a few GB resident, so the CPU path's 4 GB would kill honest ones
53
+ WARMUP_SEED = b"gpu-warmup-public-seed" # PUBLIC: warm-up instances come from this seed and are never ladder instances
54
+ CLAIM_LIMITS = ("A GPU time is a wall-clock measurement on ONE device model, taken by an operator's process on the operator's clock: nobody else can re-execute it exactly, "
55
+ "the solver runs unsandboxed, clocks, thermal state and other tenants move it, and nothing proves which device ran it beyond nvidia-smi. It supports 'faster on this "
56
+ "device, as measured by this operator', not a proof, and says nothing about other devices.")
57
+
58
+
59
+ class GpuUnavailable(Exception):
60
+ pass
61
+
62
+
63
+ class SessionError(Exception):
64
+ pass
65
+
66
+
67
+ # ----------------------------------------------------------------------------------------------------------------- nvidia-smi and its parsers (pure, tested)
68
+ def smi(kind, query):
69
+ """Run nvidia-smi for the measured GPU and return its stdout, or None when it is absent or fails. kind: "gpu" or "compute-apps"."""
70
+ try:
71
+ r = subprocess.run(["nvidia-smi", "-i", GPU_INDEX, f"--query-{kind}={query}", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=20)
72
+ except (OSError, subprocess.SubprocessError):
73
+ return None
74
+ return r.stdout if r.returncode == 0 else None
75
+
76
+
77
+ def device_info(smi_fn=smi):
78
+ text = smi_fn("gpu", "name,driver_version,memory.total")
79
+ if not text or not text.strip():
80
+ raise GpuUnavailable("nvidia-smi reported no GPU (or is not installed)")
81
+ first = text.strip().splitlines()[0]
82
+ parts = [p.strip() for p in first.split(",")]
83
+ if len(parts) != 3 or not parts[2].replace(".", "").isdigit():
84
+ raise GpuUnavailable(f"could not read nvidia-smi output: {first!r}")
85
+ return {"name": parts[0], "driver": parts[1], "memory_mib": int(float(parts[2]))}
86
+
87
+
88
+ def device_key(info):
89
+ return f"{info['name']} | driver {info['driver']} | {info['memory_mib']} MiB"
90
+
91
+
92
+ def parse_mask(text):
93
+ """The throttle-reason bitmask nvidia-smi prints (hex like 0x0000000000000004, or decimal). None when unreadable."""
94
+ try:
95
+ t = text.strip().splitlines()[0].strip()
96
+ return int(t, 16) if t.lower().startswith("0x") else int(t)
97
+ except (ValueError, IndexError, AttributeError):
98
+ return None
99
+
100
+
101
+ def other_gpu_processes(text):
102
+ """How many compute processes are on the GPU (csv `pid, used_memory` lines; empty or 'No running ...' means none)."""
103
+ if not text:
104
+ return 0
105
+ return sum(1 for ln in text.strip().splitlines() if ln.strip() and "no running" not in ln.lower())
106
+
107
+
108
+ def throttle_mask(smi_fn=smi):
109
+ for field in ("clocks_throttle_reasons.active", "clocks_event_reasons.active"): # the field was renamed in newer drivers
110
+ m = parse_mask(smi_fn("gpu", field) or "")
111
+ if m is not None:
112
+ return m
113
+ return None
114
+
115
+
116
+ class Sampler(threading.Thread):
117
+ """Polls the throttle mask in the background (the parent is a separate process from the solver, so the solver's GIL cannot starve it). take() polls once more and returns
118
+ the union of the bits seen since the previous take, then starts a fresh window: one window per measured instance."""
119
+ def __init__(self, smi_fn, poll=POLL_S):
120
+ super().__init__(daemon=True)
121
+ self.smi_fn, self.poll, self._mask, self._lock, self._stop_evt = smi_fn, poll, 0, threading.Lock(), threading.Event()
122
+
123
+ def _add(self, m):
124
+ if m is not None:
125
+ with self._lock:
126
+ self._mask |= m
127
+
128
+ def run(self):
129
+ while not self._stop_evt.is_set():
130
+ self._add(throttle_mask(self.smi_fn))
131
+ self._stop_evt.wait(self.poll)
132
+
133
+ def take(self):
134
+ self._add(throttle_mask(self.smi_fn))
135
+ with self._lock:
136
+ out, self._mask = self._mask, 0
137
+ return out
138
+
139
+ def stop(self):
140
+ self._stop_evt.set()
141
+ self.join(timeout=5)
142
+
143
+
144
+ def invalid_reason(mask, busy):
145
+ """None when the measurement stands; otherwise why it must be re-measured."""
146
+ if busy:
147
+ return f"another process was using the GPU ({busy} compute process(es) present)"
148
+ if mask is not None and mask & THROTTLE_BAD:
149
+ return f"the GPU reported a slowdown while the solver ran (throttle mask {mask:#x})"
150
+ return None
151
+
152
+
153
+ # ----------------------------------------------------------------------------------------------------------------- the child session (parent side)
154
+ class ProcTransport:
155
+ """The real transport: a child process spoken to line by line. A reader thread turns its stdout into a queue so the parent can wait with a timeout and keep watching memory."""
156
+ def __init__(self, argv, env, cwd, cleanup_dir=None):
157
+ self._err = tempfile.TemporaryFile()
158
+ self._cleanup_dir = cleanup_dir
159
+ self.p = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self._err, env=env, cwd=cwd)
160
+ self.q = queue.Queue()
161
+ threading.Thread(target=self._pump, daemon=True).start()
162
+
163
+ def _pump(self):
164
+ for line in self.p.stdout:
165
+ line = line.strip()
166
+ if line:
167
+ self.q.put(line)
168
+ self.q.put(b"") # end of output
169
+
170
+ def send(self, data):
171
+ try:
172
+ self.p.stdin.write(data + b"\n")
173
+ self.p.stdin.flush()
174
+ except (BrokenPipeError, OSError, ValueError):
175
+ raise SessionError("the child is gone (its input pipe is closed)")
176
+
177
+ def recv(self, timeout):
178
+ """A reply line (bytes), b"" at the end of the child's output, or None if nothing arrived within `timeout`."""
179
+ try:
180
+ return self.q.get(timeout=timeout)
181
+ except queue.Empty:
182
+ return None
183
+
184
+ def rss_mb(self):
185
+ return tflops._rss_mb(self.p.pid)
186
+
187
+ def stderr_tail(self):
188
+ try:
189
+ self._err.seek(0)
190
+ return self._err.read()[-400:].decode(errors="replace")
191
+ except (OSError, ValueError):
192
+ return ""
193
+
194
+ def _close_pipes(self):
195
+ for pipe in (self.p.stdin, self.p.stdout):
196
+ try:
197
+ pipe.close()
198
+ except (OSError, ValueError):
199
+ pass # a pipe whose peer is already gone (BrokenPipeError on flush) is closed regardless
200
+
201
+ def kill(self):
202
+ try:
203
+ self.p.kill()
204
+ self.p.wait(timeout=10)
205
+ except (OSError, subprocess.SubprocessError):
206
+ pass
207
+ self._close_pipes()
208
+
209
+ def close(self):
210
+ try:
211
+ self.p.stdin.write(b'{"op": "quit"}\n')
212
+ self.p.stdin.flush()
213
+ self.p.wait(timeout=5)
214
+ except (OSError, ValueError, subprocess.SubprocessError):
215
+ self.kill()
216
+ finally:
217
+ self._close_pipes()
218
+ if self._cleanup_dir:
219
+ shutil.rmtree(self._cleanup_dir, ignore_errors=True)
220
+ self._err.close()
221
+
222
+
223
+ class Session:
224
+ """One child, one solver, many requests. run() returns the answer AND the time the PARENT saw pass between writing the request and reading the reply."""
225
+ RSS_EVERY_S = 0.5 # resident memory is sampled this often, not at every poll: spawning `ps` inside a timed interval would put host-CPU noise into the measurement
226
+
227
+ def __init__(self, transport, clock=time.perf_counter, max_rss_mb=GPU_MAX_RSS_MB, poll=0.05, ready_timeout=900):
228
+ self.t, self.clock, self.max_rss_mb, self.poll, self.ready_timeout = transport, clock, max_rss_mb, poll, ready_timeout
229
+ self._last_rss = None
230
+
231
+ def _guard(self, started, limit):
232
+ """None while all is well; "memory" or "timeout" (the child is killed first) when a limit is passed."""
233
+ now = self.clock()
234
+ if self._last_rss is None or now - self._last_rss >= self.RSS_EVERY_S:
235
+ self._last_rss = now
236
+ if self.t.rss_mb() > self.max_rss_mb:
237
+ self.t.kill()
238
+ return "memory"
239
+ if self.clock() - started > limit:
240
+ self.t.kill()
241
+ return "timeout"
242
+ return None
243
+
244
+ def wait_ready(self):
245
+ started = self.clock()
246
+ while True:
247
+ line = self.t.recv(self.poll)
248
+ if line is None:
249
+ why = self._guard(started, self.ready_timeout)
250
+ if why:
251
+ raise SessionError(f"the child was killed before it was ready ({why})")
252
+ continue
253
+ if line == b"":
254
+ raise SessionError("the child exited before it was ready: " + self.t.stderr_tail())
255
+ try:
256
+ msg = json.loads(line)
257
+ except ValueError:
258
+ raise SessionError(f"the child's first line was not JSON: {line[:100]!r}")
259
+ if msg.get("status") != "ready":
260
+ raise SessionError(str(msg.get("err", "the child did not become ready"))[-300:])
261
+ return msg
262
+
263
+ def run(self, args, timeout):
264
+ """-> {"status": "ok"|"error"|"dead"|"memory"|"timeout", "elapsed": seconds on the parent's clock, "answer": ..., "err": ...}"""
265
+ t0 = self.clock()
266
+ try:
267
+ self.t.send(json.dumps({"op": "run", "args": args}).encode())
268
+ except SessionError as e:
269
+ return {"status": "dead", "elapsed": 0.0, "err": str(e)}
270
+ while True:
271
+ line = self.t.recv(self.poll)
272
+ if line is not None:
273
+ elapsed = self.clock() - t0 # taken at the moment the reply arrives, before anything is parsed
274
+ if line == b"":
275
+ return {"status": "dead", "elapsed": elapsed, "err": "the child exited: " + self.t.stderr_tail()}
276
+ try:
277
+ msg = json.loads(line)
278
+ except ValueError:
279
+ return {"status": "error", "elapsed": elapsed, "err": f"unparseable reply {line[:100]!r}"}
280
+ if msg.get("status") == "ok":
281
+ return {"status": "ok", "elapsed": elapsed, "answer": msg.get("answer")}
282
+ return {"status": "error", "elapsed": elapsed, "err": str(msg.get("err", ""))[-1500:]}
283
+ why = self._guard(t0, timeout)
284
+ if why:
285
+ return {"status": why, "elapsed": self.clock() - t0, "err": f"killed: {why} limit"}
286
+
287
+ def close(self):
288
+ self.t.close()
289
+
290
+
291
+ def default_session_factory(solver, max_rss_mb=GPU_MAX_RSS_MB):
292
+ empty = tempfile.mkdtemp(prefix="svp-gpu-") # an empty working directory: the solver cannot find the judge's files by a relative path
293
+ env = {"PATH": os.environ.get("PATH", ""), "PYTHONHASHSEED": "0", "CUDA_VISIBLE_DEVICES": GPU_INDEX, "OMP_NUM_THREADS": "1"}
294
+ return Session(ProcTransport([sys.executable, "-B", RUNNER, os.path.abspath(solver)], env, empty, cleanup_dir=empty), max_rss_mb=max_rss_mb)
295
+
296
+
297
+ def _open_warm(session_factory, solver, derive, dim, warm, cap):
298
+ """A ready child that has run `warm` public instances of this size. Raises SessionError (and closes the child) if anything fails."""
299
+ s = session_factory(solver)
300
+ try:
301
+ s.wait_ready()
302
+ for i in range(warm):
303
+ r = s.run(derive(WARMUP_SEED, dim, i)["args"], cap)
304
+ if r["status"] != "ok":
305
+ raise SessionError(f"warm-up failed ({r['status']}): {str(r.get('err', ''))[-200:]}")
306
+ except BaseException:
307
+ s.close()
308
+ raise
309
+ return s
310
+
311
+
312
+ def calibrate_call_overhead(session_factory, calls=CALIBRATION_CALLS):
313
+ """The fixed cost of one request through the parent/child pipe plus two device synchronizes, measured on the empty solver. Recorded in the ladder header for transparency; NOT
314
+ subtracted (that keeps every time a conservative one)."""
315
+ s = session_factory(NOOP)
316
+ try:
317
+ s.wait_ready()
318
+ times = []
319
+ for _ in range(calls):
320
+ r = s.run([], 60)
321
+ if r["status"] != "ok":
322
+ raise SessionError(f"calibration failed ({r['status']}): {str(r.get('err', ''))[-200:]}")
323
+ times.append(r["elapsed"])
324
+ return statistics.median(times)
325
+ finally:
326
+ s.close()
327
+
328
+
329
+ # ----------------------------------------------------------------------------------------------------------------- measuring instances
330
+ def finish_record(rec, res, busy, mask, verify, inst, cap):
331
+ """Turn one session result into a ladder record. Kept separate from the I/O so every branch is testable."""
332
+ st = res["status"]
333
+ rec["elapsed"] = res.get("elapsed")
334
+ rec["throttle_mask"] = mask
335
+ if st == "timeout":
336
+ rec.update({"status": "timeout", "cpu": cap, "cpu_net": cap, "err": res.get("err", "")})
337
+ return rec
338
+ if st != "ok":
339
+ rec.update({"status": "crash", "cpu": cap, "cpu_net": cap, "err": f"{st}: {res.get('err', '')}"[-1500:]})
340
+ return rec
341
+ why = invalid_reason(mask, busy)
342
+ if why:
343
+ rec.update({"status": "crash", "invalid": True, "cpu": cap, "cpu_net": cap, "err": "measurement invalid: " + why})
344
+ return rec
345
+ elapsed, ans = res["elapsed"], res.get("answer")
346
+ if elapsed > cap:
347
+ rec.update({"status": "timeout", "cpu": cap, "cpu_net": cap})
348
+ return rec
349
+ try:
350
+ ok = bool(verify(inst, ans))
351
+ except Exception as e: # a verifier that RAISES is the operator's bug, not a wrong answer: say so
352
+ ok = False
353
+ rec["verify_error"] = f"{type(e).__name__}: {str(e)[:200]}"
354
+ if not ok:
355
+ rec.update({"status": "failed" if ans is None else "wrong", "cpu": cap, "cpu_net": cap})
356
+ else:
357
+ rec.update({"status": "solved", "cpu": elapsed, "cpu_net": max(elapsed, 1e-3)})
358
+ return rec
359
+
360
+
361
+ def measure_one(session, inst, verify, cap, info, smi_fn, sampler):
362
+ rec = {"id": inst["id"], "dim": inst["dim"], "device": device_key(info)}
363
+ busy_before = other_gpu_processes(smi_fn("compute-apps", "pid,used_memory"))
364
+ sampler.take() # a clean throttle window for exactly this run
365
+ res = session.run(inst["args"], cap)
366
+ mask = sampler.take()
367
+ busy = max(busy_before, other_gpu_processes(smi_fn("compute-apps", "pid,used_memory"))) # a process that arrived DURING the run counts too
368
+ return finish_record(rec, res, busy, mask, verify, inst, cap)
369
+
370
+
371
+ def ladder_header(solver, family, mod, decl_dir, master, cap, info, warm, call_overhead):
372
+ return {"meta": True, "solver": os.path.basename(solver), "commit": instances.commit(master), "overhead": 0.0, "budget": cap, "family": family,
373
+ "sandbox": "gpu-local", "instance_version": mod.VERSION, "image": None, "host": "NVIDIA GPU, parent clock (operator-trusted; see gpu_meter.py)",
374
+ "cpu": device_key(info), "ncpu": 1, "jobs": 1, "solver_sha256": tflops.sha256_file(solver), "cost_model": "gpu_time", "warm": warm,
375
+ "timing": "parent clock around a request/reply; one run per instance; warm-up on public instances", "call_overhead_s": call_overhead,
376
+ "decl_sha256": tflops.sha256_file(os.path.join(decl_dir, "decl.py"))}
377
+
378
+
379
+ def evaluate_ladder(solver, family, master, dims, n, cap, out_path, warm=WARM_INSTANCES, progress=None, smi_fn=smi, session_factory=None):
380
+ session_factory = session_factory or default_session_factory
381
+ if os.path.exists(out_path):
382
+ raise FileExistsError(out_path)
383
+ mod, derive, verify, d = tflops.load_family(family)
384
+ info = device_info(smi_fn)
385
+ call_overhead = calibrate_call_overhead(session_factory)
386
+ fd = os.open(out_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
387
+ recs = []
388
+ sampler = Sampler(smi_fn)
389
+ sampler.start()
390
+ try:
391
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
392
+ f.write(json.dumps(ladder_header(solver, family, mod, d, master, cap, info, warm, call_overhead)) + "\n")
393
+ for dim in dims:
394
+ session, start_error = None, None
395
+ try:
396
+ for idx in range(n):
397
+ inst = derive(master, dim, idx)
398
+ if session is None:
399
+ if start_error is None:
400
+ try:
401
+ session = _open_warm(session_factory, solver, derive, dim, warm, cap)
402
+ except SessionError as e:
403
+ start_error = f"could not start: {e}"[-1500:] # one failed start per size: the rest of the size is charged without waiting out more start-up timeouts
404
+ if session is None:
405
+ rec = {"id": inst["id"], "dim": inst["dim"], "device": device_key(info), "status": "crash", "cpu": cap, "cpu_net": cap, "err": start_error}
406
+ f.write(json.dumps(rec) + "\n")
407
+ f.flush()
408
+ recs.append(rec)
409
+ if progress:
410
+ progress(os.path.basename(solver), rec)
411
+ continue
412
+ rec = measure_one(session, inst, verify, cap, info, smi_fn, sampler)
413
+ f.write(json.dumps(rec) + "\n")
414
+ f.flush()
415
+ recs.append(rec)
416
+ if progress:
417
+ progress(os.path.basename(solver), rec)
418
+ if rec["status"] in ("timeout", "crash") and not rec.get("invalid"):
419
+ session.close() # killed or dead: the next instance gets a fresh, re-warmed child
420
+ session = None
421
+ finally:
422
+ if session is not None:
423
+ session.close()
424
+ finally:
425
+ sampler.stop()
426
+ return recs
427
+
428
+
429
+ def verify_ladder(path, rerun=0, master=None, solver_path=None, sample_seed=None, smi_fn=smi, session_factory=None):
430
+ """-> (ok, [lines]). Consistency of every record; no record flagged invalid; with rerun, K random solved records re-MEASURED once each on this device (same model only), each
431
+ within RERUN_TOLERANCE of the recorded time."""
432
+ session_factory = session_factory or default_session_factory
433
+ lines, ok = [], True
434
+ meta, recs = tflops.read_ladder(path)
435
+ if meta is None or meta.get("cost_model") != "gpu_time":
436
+ return False, [f"FAIL {os.path.basename(path)}: not a gpu_time ladder"]
437
+ cap = meta.get("budget")
438
+ if isinstance(cap, bool) or not isinstance(cap, (int, float)) or cap <= 0 or not isinstance(meta.get("cpu"), str) or "solver_sha256" not in meta:
439
+ return False, [f"FAIL {os.path.basename(path)}: the header lacks a positive budget, a device or a solver hash"]
440
+ if not recs:
441
+ return False, [f"FAIL {os.path.basename(path)}: the ladder holds no records (an empty ladder certifies nothing)"]
442
+ bad = 0
443
+ for r in recs:
444
+ try:
445
+ if r.get("device", meta["cpu"]) != meta["cpu"]:
446
+ bad += 1 # a record measured on a different device than the ladder claims
447
+ elif r["status"] == "solved":
448
+ if not (isinstance(r["cpu"], (int, float)) and abs(r["cpu_net"] - max(r["cpu"], 1e-3)) < 1e-12 and 0 <= r["cpu"] <= cap):
449
+ bad += 1
450
+ elif r["cpu_net"] != cap:
451
+ bad += 1
452
+ except (KeyError, TypeError):
453
+ bad += 1
454
+ lines.append(f"{'PASS' if not bad else 'FAIL'} cost model: solved cost = the parent-measured time (floored at 1e-3 s), every other record charged the cap" + (f" ({bad} disagree)" if bad else ""))
455
+ ok = ok and not bad
456
+ invalid = [r.get("id") for r in recs if r.get("invalid")]
457
+ lines.append(f"{'FAIL' if invalid else 'PASS'} no invalid measurements" + (f": {len(invalid)} record(s) flagged invalid ({', '.join(map(str, invalid[:5]))}): re-measure them" if invalid else ""))
458
+ ok = ok and not invalid
459
+ if rerun:
460
+ try:
461
+ now = device_key(device_info(smi_fn))
462
+ except GpuUnavailable as e:
463
+ lines.append(f"SKIP re-measure: {e}")
464
+ return False, lines # asked for, not done: not a pass
465
+ if now != meta.get("cpu"):
466
+ lines.append(f"SKIP re-measure: this machine's device is {now!r}, the ladder's is {meta.get('cpu')!r} (a time means nothing on another device)")
467
+ return False, lines
468
+ solver = solver_path or os.path.join(HERE, "solvers", meta["solver"])
469
+ if not os.path.exists(solver) or tflops.sha256_file(solver) != meta["solver_sha256"]:
470
+ lines.append(f"FAIL re-measure: the solver file {meta['solver']} is missing here or differs from the ladder's")
471
+ return False, lines
472
+ mod, derive, verify, d = tflops.load_family(meta["family"])
473
+ if tflops.sha256_file(os.path.join(d, "decl.py")) != meta.get("decl_sha256"):
474
+ lines.append("FAIL re-measure: this checkout's decl.py differs from the ladder's")
475
+ return False, lines
476
+ if instances.commit(master) != meta["commit"]:
477
+ lines.append("SKIP re-measure: the seed given does not match the ladder's commitment")
478
+ return False, lines
479
+ solved = [r for r in recs if r["status"] == "solved"]
480
+ if not solved:
481
+ lines.append("FAIL re-measure: the ladder has no solved record to re-measure")
482
+ return False, lines
483
+ k = len(solved) if rerun == "all" else min(int(rerun), len(solved))
484
+ seed = secrets.randbits(64) if sample_seed is None else sample_seed
485
+ pick = random.Random(seed).sample(solved, k) if k < len(solved) else solved
486
+ lines.append(f"re-measure: {len(pick)} of {len(solved)} solved records, once each (sample seed {seed}); agreement means within a factor of {RERUN_TOLERANCE}, not identical")
487
+ info = device_info(smi_fn)
488
+ warm = meta.get("warm", WARM_INSTANCES)
489
+ agree = 0
490
+ sampler = Sampler(smi_fn)
491
+ sampler.start()
492
+ try:
493
+ for r in pick:
494
+ dim, idx = r["id"].split(":")
495
+ inst = derive(master, int(dim), int(idx))
496
+ try:
497
+ session = _open_warm(session_factory, solver, derive, int(dim), warm, cap)
498
+ except SessionError as e:
499
+ lines.append(f" could not re-measure {r['id']}: {e}")
500
+ continue
501
+ try:
502
+ again = measure_one(session, inst, verify, cap, info, smi_fn, sampler)
503
+ finally:
504
+ session.close()
505
+ if again["status"] == "solved" and 1 / RERUN_TOLERANCE <= again["cpu_net"] / r["cpu_net"] <= RERUN_TOLERANCE: # cpu_net: floored at 1e-3 s, never zero
506
+ agree += 1
507
+ else:
508
+ lines.append(f" differs at {r['id']}: recorded {r['cpu']:.4f} s, re-measured {again['status']} {again.get('cpu')}")
509
+ finally:
510
+ sampler.stop()
511
+ good = agree == len(pick)
512
+ lines.append(f"{'PASS' if good else 'FAIL'} re-measure: {agree}/{len(pick)} agree")
513
+ ok = ok and good
514
+ return ok, lines
515
+
516
+
517
+ # ----------------------------------------------------------------------------------------------------------------- command line
518
+ def cmd_info(a):
519
+ try:
520
+ info = device_info()
521
+ except GpuUnavailable as e:
522
+ sys.exit(f"no usable GPU: {e}")
523
+ busy = other_gpu_processes(smi("compute-apps", "pid,used_memory"))
524
+ mask = throttle_mask()
525
+ print(f"device : {device_key(info)} (GPU index {GPU_INDEX})\nidle : {'yes' if not busy else f'NO ({busy} compute process(es))'}\nthrottle mask now: {mask if mask is None else hex(mask)} "
526
+ f"({'slowdown active' if mask and mask & THROTTLE_BAD else 'none that invalidates a measurement'})")
527
+
528
+
529
+ def cmd_plan(a):
530
+ master = tflops._master(a)
531
+ try:
532
+ key = device_key(device_info())
533
+ except GpuUnavailable as e:
534
+ sys.exit(f"cannot plan: the plan pins this machine's GPU, and {e}")
535
+ plan = tflops.plan_new("gpu_time", a.family, a.dims, a.n, a.cap, master, {"cpu": key})
536
+ fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) # never overwrites a plan
537
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
538
+ json.dump(plan, f, indent=1, sort_keys=True)
539
+ print(f"plan written to {a.out}\nplan hash: {tflops.plan_hash(plan)}\nPublish this hash (somewhere the operator cannot edit) BEFORE any solver is submitted.")
540
+
541
+
542
+ def cmd_practice(a):
543
+ master = tflops._master(a)
544
+ tag = instances.commit(master)[:8]
545
+ if a.plan:
546
+ plan = tflops.load_plan(a.plan)
547
+ if (plan.get("family"), sorted(set(a.dims)), a.n, a.cap, plan.get("commit")) != (a.family, plan.get("dims"), plan.get("n"), plan.get("cap"), instances.commit(master)):
548
+ sys.exit("refused: the dims, n, budget, family or seed on the command line are not the plan's. A planned ladder is run exactly as planned.")
549
+ os.makedirs(os.path.join(HERE, "results"), exist_ok=True)
550
+ paths = []
551
+ for role, solver in (("baseline", a.baseline), ("submission", a.solver)):
552
+ p = os.path.join(HERE, "results", f"gpu_{a.family}_{tag}_{role}_{os.path.basename(solver)}.jsonl")
553
+ try:
554
+ evaluate_ladder(solver, a.family, master, a.dims, a.n, a.cap, p, a.warm,
555
+ progress=lambda name, rec: print(f" {name} {rec['id']}: {rec['status']} {rec.get('cpu_net', 0):.4f}s" + (f" INVALID: {rec['err']}" if rec.get("invalid") else ""), flush=True))
556
+ except (GpuUnavailable, SessionError) as e:
557
+ sys.exit(f"cannot measure: {e}")
558
+ paths.append(p)
559
+ print("ladders:\n " + "\n ".join(paths) + "\nnext: python3 gpu_meter.py claim " + " ".join(paths))
560
+
561
+
562
+ def cmd_claim(a):
563
+ r = subprocess.run([sys.executable, os.path.join(HERE, "extrapolate.py"), "--baseline", a.baseline, "--submission", a.submission], capture_output=True, text=True, cwd=HERE)
564
+ print(r.stdout + r.stderr)
565
+ if r.returncode:
566
+ sys.exit(r.returncode)
567
+ with open(a.submission + ".claim.json", encoding="utf-8") as f:
568
+ c = json.load(f)
569
+ print(f"credit (difficulty-stratified lower bound): {c.get('measured')} bits\n\nWhat this does NOT show: {CLAIM_LIMITS}")
570
+
571
+
572
+ def cmd_verify(a):
573
+ master = tflops._master(a)
574
+ plan = tflops.load_plan(a.plan) if a.plan else None
575
+ if a.plan_hash and not plan:
576
+ sys.exit("--plan-hash needs --plan")
577
+ allok = True
578
+ for p in a.ladders:
579
+ ok, lines = verify_ladder(p, a.rerun, master, sample_seed=a.sample_seed)
580
+ if plan is not None:
581
+ meta, recs = tflops.read_ladder(p)
582
+ problems = tflops.check_plan(plan, meta or {}, recs, a.plan_hash)
583
+ lines.append(("FAIL plan: " + "; ".join(problems)) if problems else "PASS plan: the ladder is exactly the committed plan" + ("" if a.plan_hash else " (no --plan-hash given: the plan file itself was not checked against a published hash)"))
584
+ ok = ok and not problems
585
+ print(p)
586
+ for ln in lines:
587
+ print(" " + ln)
588
+ allok = allok and ok
589
+ print(("\nVERIFIED (consistent and re-measured on this device as requested)" if a.rerun else "\nCONSISTENT ONLY: nothing was re-measured (use --rerun K or --rerun all on the same device model)") if allok else "\nNOT VERIFIED")
590
+ print("What this does NOT show: " + CLAIM_LIMITS)
591
+ sys.exit(0 if allok else 1)
592
+
593
+
594
+ def main(argv=None):
595
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
596
+ sub = ap.add_subparsers(dest="cmd", required=True)
597
+ sub.add_parser("info").set_defaults(fn=cmd_info)
598
+ p = sub.add_parser("plan")
599
+ p.add_argument("--family", required=True)
600
+ p.add_argument("--dims", type=int, nargs="+", required=True)
601
+ p.add_argument("--n", type=int, required=True)
602
+ p.add_argument("--cap", type=float, required=True)
603
+ p.add_argument("--seed")
604
+ p.add_argument("--out", required=True)
605
+ p.set_defaults(fn=cmd_plan)
606
+ p = sub.add_parser("practice")
607
+ p.add_argument("solver")
608
+ p.add_argument("--baseline", required=True)
609
+ p.add_argument("--family", required=True)
610
+ p.add_argument("--dims", type=int, nargs="+", required=True)
611
+ p.add_argument("--n", type=int, default=6)
612
+ p.add_argument("--cap", type=float, default=60)
613
+ p.add_argument("--warm", type=int, default=WARM_INSTANCES)
614
+ p.add_argument("--plan")
615
+ p.add_argument("--seed")
616
+ p.set_defaults(fn=cmd_practice)
617
+ p = sub.add_parser("claim")
618
+ p.add_argument("baseline")
619
+ p.add_argument("submission")
620
+ p.set_defaults(fn=cmd_claim)
621
+ p = sub.add_parser("verify")
622
+ p.add_argument("ladders", nargs="+")
623
+ p.add_argument("--rerun", type=lambda s: s if s == "all" else int(s), default=0)
624
+ p.add_argument("--sample-seed", type=int)
625
+ p.add_argument("--plan")
626
+ p.add_argument("--plan-hash")
627
+ p.add_argument("--seed")
628
+ p.set_defaults(fn=cmd_verify)
629
+ a = ap.parse_args(argv)
630
+ a.fn(a)
631
+
632
+
633
+ if __name__ == "__main__":
634
+ main()