guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/gpu_meter.py
ADDED
|
@@ -0,0 +1,634 @@
|
|
|
1
|
+
"""GPU timing for the speedup protocol: a DEVICE-PINNED, PARENT-TIMED wall-clock meter. Written without a GPU (this machine has none): everything on the parent side is tested
|
|
2
|
+
against a fake device and a fake transport (test_gpu_meter.py); the child (gpu_runner.py) and every torch solver are UNRUN until someone runs them on an NVIDIA box.
|
|
3
|
+
|
|
4
|
+
python3 gpu_meter.py info what device, driver, memory; is it idle; is it throttling
|
|
5
|
+
python3 gpu_meter.py plan --family aiinfer --dims 2048 3072 --n 6 --cap 60 --out PLAN.json fix the ladder in advance; prints the hash to publish
|
|
6
|
+
python3 gpu_meter.py practice SOLVER --baseline REF --family aiinfer --dims 2048 3072 --n 6 --cap 60 [--plan PLAN.json] [--seed SEEDFILE]
|
|
7
|
+
python3 gpu_meter.py claim BASELINE_LADDER SUBMISSION_LADDER the estimator's credit (extrapolate.py, unchanged)
|
|
8
|
+
python3 gpu_meter.py verify LADDER [--rerun K|all] [--plan PLAN.json --plan-hash HEX] consistency; --rerun RE-MEASURES records on this very device model
|
|
9
|
+
|
|
10
|
+
HOW IT MEASURES. One child process per size loads the solver and warms up on PUBLIC instances (never ladder instances). Then, for each ladder instance, the PARENT writes a
|
|
11
|
+
request line to the child and reads the reply line, and the time is the elapsed time on the PARENT's own clock between the two (the child brackets the solve with a device
|
|
12
|
+
synchronize). The child supplies no timing at all, so it cannot under-report; every timed run is a different instance, so nothing can be cached across repetitions; the answer
|
|
13
|
+
must come back, correct, inside the measured interval. One-time setup (imports, module-level work, loading a model) happens before the child says `ready` and is not measured,
|
|
14
|
+
exactly as loading weights is not part of an inference time; per-instance work must happen in solve(). Ladders are the ordinary format with `cpu` = the device model + driver, so
|
|
15
|
+
extrapolate.py REFUSES to compare ladders from different devices. One run per instance: noise is the estimator's job, as on the CPU path.
|
|
16
|
+
|
|
17
|
+
WHAT MAKES A MEASUREMENT INVALID (a crash charged the cap, flagged `invalid`, to be re-measured, never silently kept): another process is using the measured GPU; the GPU reported
|
|
18
|
+
a hardware or thermal slowdown while the solver ran. Hardware or driver trouble the parent cannot see is not covered.
|
|
19
|
+
|
|
20
|
+
TRUST: still the weakest evidence in the protocol, and every verdict says so. The parent's clock replaces the child's claim, but the solver runs unsandboxed in a process the
|
|
21
|
+
operator started: nothing here proves WHICH device it ran on beyond nvidia-smi, that clocks were not locked or raised, or that nobody tuned the box for one solver. Nobody else can
|
|
22
|
+
reproduce a time without the same device model: `verify --rerun` re-measures and agrees only within RERUN_TOLERANCE. A GPU speedup is a claim about one device model, certified
|
|
23
|
+
by an operator you trust, not a proof.
|
|
24
|
+
"""
|
|
25
|
+
import argparse
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import queue
|
|
29
|
+
import random
|
|
30
|
+
import secrets
|
|
31
|
+
import shutil
|
|
32
|
+
import statistics
|
|
33
|
+
import subprocess
|
|
34
|
+
import sys
|
|
35
|
+
import tempfile
|
|
36
|
+
import threading
|
|
37
|
+
import time
|
|
38
|
+
|
|
39
|
+
import instances
|
|
40
|
+
import tflops
|
|
41
|
+
|
|
42
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
43
|
+
RUNNER = os.path.join(HERE, "gpu_runner.py")
|
|
44
|
+
NOOP = os.path.join(HERE, "solvers", "gpu_noop.py")
|
|
45
|
+
GPU_INDEX = os.environ.get("SVP_GPU_INDEX", "0") # the one GPU measured: nvidia-smi is asked about it alone (-i), and the child only sees it (CUDA_VISIBLE_DEVICES)
|
|
46
|
+
THROTTLE_BAD = 0x8 | 0x20 | 0x40 | 0x80 # hardware slowdown, software thermal, hardware thermal, hardware power brake: the measurement is not the device's normal speed
|
|
47
|
+
THROTTLE_NOTE = 0x4 # software power cap: routine under load on many data-centre parts, recorded but not fatal
|
|
48
|
+
WARM_INSTANCES = 2 # public warm-up instances run (untimed) per size before the ladder instances
|
|
49
|
+
CALIBRATION_CALLS = 5
|
|
50
|
+
RERUN_TOLERANCE = 1.3 # `verify --rerun` accepts a single re-measured run within this factor of the recorded one
|
|
51
|
+
POLL_S = 1.0 # background throttle sampling while a run is in progress: nvidia-smi queries inside a timed interval are interference, so only 1 Hz (plus a poll just before and just after every run)
|
|
52
|
+
GPU_MAX_RSS_MB = 16384 # host memory a GPU solver may reach: a torch + CUDA process alone holds a few GB resident, so the CPU path's 4 GB would kill honest ones
|
|
53
|
+
WARMUP_SEED = b"gpu-warmup-public-seed" # PUBLIC: warm-up instances come from this seed and are never ladder instances
|
|
54
|
+
CLAIM_LIMITS = ("A GPU time is a wall-clock measurement on ONE device model, taken by an operator's process on the operator's clock: nobody else can re-execute it exactly, "
|
|
55
|
+
"the solver runs unsandboxed, clocks, thermal state and other tenants move it, and nothing proves which device ran it beyond nvidia-smi. It supports 'faster on this "
|
|
56
|
+
"device, as measured by this operator', not a proof, and says nothing about other devices.")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class GpuUnavailable(Exception):
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class SessionError(Exception):
|
|
64
|
+
pass
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ----------------------------------------------------------------------------------------------------------------- nvidia-smi and its parsers (pure, tested)
|
|
68
|
+
def smi(kind, query):
|
|
69
|
+
"""Run nvidia-smi for the measured GPU and return its stdout, or None when it is absent or fails. kind: "gpu" or "compute-apps"."""
|
|
70
|
+
try:
|
|
71
|
+
r = subprocess.run(["nvidia-smi", "-i", GPU_INDEX, f"--query-{kind}={query}", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=20)
|
|
72
|
+
except (OSError, subprocess.SubprocessError):
|
|
73
|
+
return None
|
|
74
|
+
return r.stdout if r.returncode == 0 else None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def device_info(smi_fn=smi):
|
|
78
|
+
text = smi_fn("gpu", "name,driver_version,memory.total")
|
|
79
|
+
if not text or not text.strip():
|
|
80
|
+
raise GpuUnavailable("nvidia-smi reported no GPU (or is not installed)")
|
|
81
|
+
first = text.strip().splitlines()[0]
|
|
82
|
+
parts = [p.strip() for p in first.split(",")]
|
|
83
|
+
if len(parts) != 3 or not parts[2].replace(".", "").isdigit():
|
|
84
|
+
raise GpuUnavailable(f"could not read nvidia-smi output: {first!r}")
|
|
85
|
+
return {"name": parts[0], "driver": parts[1], "memory_mib": int(float(parts[2]))}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def device_key(info):
|
|
89
|
+
return f"{info['name']} | driver {info['driver']} | {info['memory_mib']} MiB"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def parse_mask(text):
|
|
93
|
+
"""The throttle-reason bitmask nvidia-smi prints (hex like 0x0000000000000004, or decimal). None when unreadable."""
|
|
94
|
+
try:
|
|
95
|
+
t = text.strip().splitlines()[0].strip()
|
|
96
|
+
return int(t, 16) if t.lower().startswith("0x") else int(t)
|
|
97
|
+
except (ValueError, IndexError, AttributeError):
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def other_gpu_processes(text):
|
|
102
|
+
"""How many compute processes are on the GPU (csv `pid, used_memory` lines; empty or 'No running ...' means none)."""
|
|
103
|
+
if not text:
|
|
104
|
+
return 0
|
|
105
|
+
return sum(1 for ln in text.strip().splitlines() if ln.strip() and "no running" not in ln.lower())
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def throttle_mask(smi_fn=smi):
|
|
109
|
+
for field in ("clocks_throttle_reasons.active", "clocks_event_reasons.active"): # the field was renamed in newer drivers
|
|
110
|
+
m = parse_mask(smi_fn("gpu", field) or "")
|
|
111
|
+
if m is not None:
|
|
112
|
+
return m
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class Sampler(threading.Thread):
|
|
117
|
+
"""Polls the throttle mask in the background (the parent is a separate process from the solver, so the solver's GIL cannot starve it). take() polls once more and returns
|
|
118
|
+
the union of the bits seen since the previous take, then starts a fresh window: one window per measured instance."""
|
|
119
|
+
def __init__(self, smi_fn, poll=POLL_S):
|
|
120
|
+
super().__init__(daemon=True)
|
|
121
|
+
self.smi_fn, self.poll, self._mask, self._lock, self._stop_evt = smi_fn, poll, 0, threading.Lock(), threading.Event()
|
|
122
|
+
|
|
123
|
+
def _add(self, m):
|
|
124
|
+
if m is not None:
|
|
125
|
+
with self._lock:
|
|
126
|
+
self._mask |= m
|
|
127
|
+
|
|
128
|
+
def run(self):
|
|
129
|
+
while not self._stop_evt.is_set():
|
|
130
|
+
self._add(throttle_mask(self.smi_fn))
|
|
131
|
+
self._stop_evt.wait(self.poll)
|
|
132
|
+
|
|
133
|
+
def take(self):
|
|
134
|
+
self._add(throttle_mask(self.smi_fn))
|
|
135
|
+
with self._lock:
|
|
136
|
+
out, self._mask = self._mask, 0
|
|
137
|
+
return out
|
|
138
|
+
|
|
139
|
+
def stop(self):
|
|
140
|
+
self._stop_evt.set()
|
|
141
|
+
self.join(timeout=5)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def invalid_reason(mask, busy):
|
|
145
|
+
"""None when the measurement stands; otherwise why it must be re-measured."""
|
|
146
|
+
if busy:
|
|
147
|
+
return f"another process was using the GPU ({busy} compute process(es) present)"
|
|
148
|
+
if mask is not None and mask & THROTTLE_BAD:
|
|
149
|
+
return f"the GPU reported a slowdown while the solver ran (throttle mask {mask:#x})"
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# ----------------------------------------------------------------------------------------------------------------- the child session (parent side)
|
|
154
|
+
class ProcTransport:
|
|
155
|
+
"""The real transport: a child process spoken to line by line. A reader thread turns its stdout into a queue so the parent can wait with a timeout and keep watching memory."""
|
|
156
|
+
def __init__(self, argv, env, cwd, cleanup_dir=None):
|
|
157
|
+
self._err = tempfile.TemporaryFile()
|
|
158
|
+
self._cleanup_dir = cleanup_dir
|
|
159
|
+
self.p = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self._err, env=env, cwd=cwd)
|
|
160
|
+
self.q = queue.Queue()
|
|
161
|
+
threading.Thread(target=self._pump, daemon=True).start()
|
|
162
|
+
|
|
163
|
+
def _pump(self):
|
|
164
|
+
for line in self.p.stdout:
|
|
165
|
+
line = line.strip()
|
|
166
|
+
if line:
|
|
167
|
+
self.q.put(line)
|
|
168
|
+
self.q.put(b"") # end of output
|
|
169
|
+
|
|
170
|
+
def send(self, data):
|
|
171
|
+
try:
|
|
172
|
+
self.p.stdin.write(data + b"\n")
|
|
173
|
+
self.p.stdin.flush()
|
|
174
|
+
except (BrokenPipeError, OSError, ValueError):
|
|
175
|
+
raise SessionError("the child is gone (its input pipe is closed)")
|
|
176
|
+
|
|
177
|
+
def recv(self, timeout):
|
|
178
|
+
"""A reply line (bytes), b"" at the end of the child's output, or None if nothing arrived within `timeout`."""
|
|
179
|
+
try:
|
|
180
|
+
return self.q.get(timeout=timeout)
|
|
181
|
+
except queue.Empty:
|
|
182
|
+
return None
|
|
183
|
+
|
|
184
|
+
def rss_mb(self):
|
|
185
|
+
return tflops._rss_mb(self.p.pid)
|
|
186
|
+
|
|
187
|
+
def stderr_tail(self):
|
|
188
|
+
try:
|
|
189
|
+
self._err.seek(0)
|
|
190
|
+
return self._err.read()[-400:].decode(errors="replace")
|
|
191
|
+
except (OSError, ValueError):
|
|
192
|
+
return ""
|
|
193
|
+
|
|
194
|
+
def _close_pipes(self):
|
|
195
|
+
for pipe in (self.p.stdin, self.p.stdout):
|
|
196
|
+
try:
|
|
197
|
+
pipe.close()
|
|
198
|
+
except (OSError, ValueError):
|
|
199
|
+
pass # a pipe whose peer is already gone (BrokenPipeError on flush) is closed regardless
|
|
200
|
+
|
|
201
|
+
def kill(self):
|
|
202
|
+
try:
|
|
203
|
+
self.p.kill()
|
|
204
|
+
self.p.wait(timeout=10)
|
|
205
|
+
except (OSError, subprocess.SubprocessError):
|
|
206
|
+
pass
|
|
207
|
+
self._close_pipes()
|
|
208
|
+
|
|
209
|
+
def close(self):
|
|
210
|
+
try:
|
|
211
|
+
self.p.stdin.write(b'{"op": "quit"}\n')
|
|
212
|
+
self.p.stdin.flush()
|
|
213
|
+
self.p.wait(timeout=5)
|
|
214
|
+
except (OSError, ValueError, subprocess.SubprocessError):
|
|
215
|
+
self.kill()
|
|
216
|
+
finally:
|
|
217
|
+
self._close_pipes()
|
|
218
|
+
if self._cleanup_dir:
|
|
219
|
+
shutil.rmtree(self._cleanup_dir, ignore_errors=True)
|
|
220
|
+
self._err.close()
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class Session:
|
|
224
|
+
"""One child, one solver, many requests. run() returns the answer AND the time the PARENT saw pass between writing the request and reading the reply."""
|
|
225
|
+
RSS_EVERY_S = 0.5 # resident memory is sampled this often, not at every poll: spawning `ps` inside a timed interval would put host-CPU noise into the measurement
|
|
226
|
+
|
|
227
|
+
def __init__(self, transport, clock=time.perf_counter, max_rss_mb=GPU_MAX_RSS_MB, poll=0.05, ready_timeout=900):
|
|
228
|
+
self.t, self.clock, self.max_rss_mb, self.poll, self.ready_timeout = transport, clock, max_rss_mb, poll, ready_timeout
|
|
229
|
+
self._last_rss = None
|
|
230
|
+
|
|
231
|
+
def _guard(self, started, limit):
|
|
232
|
+
"""None while all is well; "memory" or "timeout" (the child is killed first) when a limit is passed."""
|
|
233
|
+
now = self.clock()
|
|
234
|
+
if self._last_rss is None or now - self._last_rss >= self.RSS_EVERY_S:
|
|
235
|
+
self._last_rss = now
|
|
236
|
+
if self.t.rss_mb() > self.max_rss_mb:
|
|
237
|
+
self.t.kill()
|
|
238
|
+
return "memory"
|
|
239
|
+
if self.clock() - started > limit:
|
|
240
|
+
self.t.kill()
|
|
241
|
+
return "timeout"
|
|
242
|
+
return None
|
|
243
|
+
|
|
244
|
+
def wait_ready(self):
|
|
245
|
+
started = self.clock()
|
|
246
|
+
while True:
|
|
247
|
+
line = self.t.recv(self.poll)
|
|
248
|
+
if line is None:
|
|
249
|
+
why = self._guard(started, self.ready_timeout)
|
|
250
|
+
if why:
|
|
251
|
+
raise SessionError(f"the child was killed before it was ready ({why})")
|
|
252
|
+
continue
|
|
253
|
+
if line == b"":
|
|
254
|
+
raise SessionError("the child exited before it was ready: " + self.t.stderr_tail())
|
|
255
|
+
try:
|
|
256
|
+
msg = json.loads(line)
|
|
257
|
+
except ValueError:
|
|
258
|
+
raise SessionError(f"the child's first line was not JSON: {line[:100]!r}")
|
|
259
|
+
if msg.get("status") != "ready":
|
|
260
|
+
raise SessionError(str(msg.get("err", "the child did not become ready"))[-300:])
|
|
261
|
+
return msg
|
|
262
|
+
|
|
263
|
+
def run(self, args, timeout):
|
|
264
|
+
"""-> {"status": "ok"|"error"|"dead"|"memory"|"timeout", "elapsed": seconds on the parent's clock, "answer": ..., "err": ...}"""
|
|
265
|
+
t0 = self.clock()
|
|
266
|
+
try:
|
|
267
|
+
self.t.send(json.dumps({"op": "run", "args": args}).encode())
|
|
268
|
+
except SessionError as e:
|
|
269
|
+
return {"status": "dead", "elapsed": 0.0, "err": str(e)}
|
|
270
|
+
while True:
|
|
271
|
+
line = self.t.recv(self.poll)
|
|
272
|
+
if line is not None:
|
|
273
|
+
elapsed = self.clock() - t0 # taken at the moment the reply arrives, before anything is parsed
|
|
274
|
+
if line == b"":
|
|
275
|
+
return {"status": "dead", "elapsed": elapsed, "err": "the child exited: " + self.t.stderr_tail()}
|
|
276
|
+
try:
|
|
277
|
+
msg = json.loads(line)
|
|
278
|
+
except ValueError:
|
|
279
|
+
return {"status": "error", "elapsed": elapsed, "err": f"unparseable reply {line[:100]!r}"}
|
|
280
|
+
if msg.get("status") == "ok":
|
|
281
|
+
return {"status": "ok", "elapsed": elapsed, "answer": msg.get("answer")}
|
|
282
|
+
return {"status": "error", "elapsed": elapsed, "err": str(msg.get("err", ""))[-1500:]}
|
|
283
|
+
why = self._guard(t0, timeout)
|
|
284
|
+
if why:
|
|
285
|
+
return {"status": why, "elapsed": self.clock() - t0, "err": f"killed: {why} limit"}
|
|
286
|
+
|
|
287
|
+
def close(self):
|
|
288
|
+
self.t.close()
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def default_session_factory(solver, max_rss_mb=GPU_MAX_RSS_MB):
|
|
292
|
+
empty = tempfile.mkdtemp(prefix="svp-gpu-") # an empty working directory: the solver cannot find the judge's files by a relative path
|
|
293
|
+
env = {"PATH": os.environ.get("PATH", ""), "PYTHONHASHSEED": "0", "CUDA_VISIBLE_DEVICES": GPU_INDEX, "OMP_NUM_THREADS": "1"}
|
|
294
|
+
return Session(ProcTransport([sys.executable, "-B", RUNNER, os.path.abspath(solver)], env, empty, cleanup_dir=empty), max_rss_mb=max_rss_mb)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _open_warm(session_factory, solver, derive, dim, warm, cap):
|
|
298
|
+
"""A ready child that has run `warm` public instances of this size. Raises SessionError (and closes the child) if anything fails."""
|
|
299
|
+
s = session_factory(solver)
|
|
300
|
+
try:
|
|
301
|
+
s.wait_ready()
|
|
302
|
+
for i in range(warm):
|
|
303
|
+
r = s.run(derive(WARMUP_SEED, dim, i)["args"], cap)
|
|
304
|
+
if r["status"] != "ok":
|
|
305
|
+
raise SessionError(f"warm-up failed ({r['status']}): {str(r.get('err', ''))[-200:]}")
|
|
306
|
+
except BaseException:
|
|
307
|
+
s.close()
|
|
308
|
+
raise
|
|
309
|
+
return s
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def calibrate_call_overhead(session_factory, calls=CALIBRATION_CALLS):
|
|
313
|
+
"""The fixed cost of one request through the parent/child pipe plus two device synchronizes, measured on the empty solver. Recorded in the ladder header for transparency; NOT
|
|
314
|
+
subtracted (that keeps every time a conservative one)."""
|
|
315
|
+
s = session_factory(NOOP)
|
|
316
|
+
try:
|
|
317
|
+
s.wait_ready()
|
|
318
|
+
times = []
|
|
319
|
+
for _ in range(calls):
|
|
320
|
+
r = s.run([], 60)
|
|
321
|
+
if r["status"] != "ok":
|
|
322
|
+
raise SessionError(f"calibration failed ({r['status']}): {str(r.get('err', ''))[-200:]}")
|
|
323
|
+
times.append(r["elapsed"])
|
|
324
|
+
return statistics.median(times)
|
|
325
|
+
finally:
|
|
326
|
+
s.close()
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
# ----------------------------------------------------------------------------------------------------------------- measuring instances
|
|
330
|
+
def finish_record(rec, res, busy, mask, verify, inst, cap):
|
|
331
|
+
"""Turn one session result into a ladder record. Kept separate from the I/O so every branch is testable."""
|
|
332
|
+
st = res["status"]
|
|
333
|
+
rec["elapsed"] = res.get("elapsed")
|
|
334
|
+
rec["throttle_mask"] = mask
|
|
335
|
+
if st == "timeout":
|
|
336
|
+
rec.update({"status": "timeout", "cpu": cap, "cpu_net": cap, "err": res.get("err", "")})
|
|
337
|
+
return rec
|
|
338
|
+
if st != "ok":
|
|
339
|
+
rec.update({"status": "crash", "cpu": cap, "cpu_net": cap, "err": f"{st}: {res.get('err', '')}"[-1500:]})
|
|
340
|
+
return rec
|
|
341
|
+
why = invalid_reason(mask, busy)
|
|
342
|
+
if why:
|
|
343
|
+
rec.update({"status": "crash", "invalid": True, "cpu": cap, "cpu_net": cap, "err": "measurement invalid: " + why})
|
|
344
|
+
return rec
|
|
345
|
+
elapsed, ans = res["elapsed"], res.get("answer")
|
|
346
|
+
if elapsed > cap:
|
|
347
|
+
rec.update({"status": "timeout", "cpu": cap, "cpu_net": cap})
|
|
348
|
+
return rec
|
|
349
|
+
try:
|
|
350
|
+
ok = bool(verify(inst, ans))
|
|
351
|
+
except Exception as e: # a verifier that RAISES is the operator's bug, not a wrong answer: say so
|
|
352
|
+
ok = False
|
|
353
|
+
rec["verify_error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
|
354
|
+
if not ok:
|
|
355
|
+
rec.update({"status": "failed" if ans is None else "wrong", "cpu": cap, "cpu_net": cap})
|
|
356
|
+
else:
|
|
357
|
+
rec.update({"status": "solved", "cpu": elapsed, "cpu_net": max(elapsed, 1e-3)})
|
|
358
|
+
return rec
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def measure_one(session, inst, verify, cap, info, smi_fn, sampler):
|
|
362
|
+
rec = {"id": inst["id"], "dim": inst["dim"], "device": device_key(info)}
|
|
363
|
+
busy_before = other_gpu_processes(smi_fn("compute-apps", "pid,used_memory"))
|
|
364
|
+
sampler.take() # a clean throttle window for exactly this run
|
|
365
|
+
res = session.run(inst["args"], cap)
|
|
366
|
+
mask = sampler.take()
|
|
367
|
+
busy = max(busy_before, other_gpu_processes(smi_fn("compute-apps", "pid,used_memory"))) # a process that arrived DURING the run counts too
|
|
368
|
+
return finish_record(rec, res, busy, mask, verify, inst, cap)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def ladder_header(solver, family, mod, decl_dir, master, cap, info, warm, call_overhead):
|
|
372
|
+
return {"meta": True, "solver": os.path.basename(solver), "commit": instances.commit(master), "overhead": 0.0, "budget": cap, "family": family,
|
|
373
|
+
"sandbox": "gpu-local", "instance_version": mod.VERSION, "image": None, "host": "NVIDIA GPU, parent clock (operator-trusted; see gpu_meter.py)",
|
|
374
|
+
"cpu": device_key(info), "ncpu": 1, "jobs": 1, "solver_sha256": tflops.sha256_file(solver), "cost_model": "gpu_time", "warm": warm,
|
|
375
|
+
"timing": "parent clock around a request/reply; one run per instance; warm-up on public instances", "call_overhead_s": call_overhead,
|
|
376
|
+
"decl_sha256": tflops.sha256_file(os.path.join(decl_dir, "decl.py"))}
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def evaluate_ladder(solver, family, master, dims, n, cap, out_path, warm=WARM_INSTANCES, progress=None, smi_fn=smi, session_factory=None):
|
|
380
|
+
session_factory = session_factory or default_session_factory
|
|
381
|
+
if os.path.exists(out_path):
|
|
382
|
+
raise FileExistsError(out_path)
|
|
383
|
+
mod, derive, verify, d = tflops.load_family(family)
|
|
384
|
+
info = device_info(smi_fn)
|
|
385
|
+
call_overhead = calibrate_call_overhead(session_factory)
|
|
386
|
+
fd = os.open(out_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
387
|
+
recs = []
|
|
388
|
+
sampler = Sampler(smi_fn)
|
|
389
|
+
sampler.start()
|
|
390
|
+
try:
|
|
391
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
392
|
+
f.write(json.dumps(ladder_header(solver, family, mod, d, master, cap, info, warm, call_overhead)) + "\n")
|
|
393
|
+
for dim in dims:
|
|
394
|
+
session, start_error = None, None
|
|
395
|
+
try:
|
|
396
|
+
for idx in range(n):
|
|
397
|
+
inst = derive(master, dim, idx)
|
|
398
|
+
if session is None:
|
|
399
|
+
if start_error is None:
|
|
400
|
+
try:
|
|
401
|
+
session = _open_warm(session_factory, solver, derive, dim, warm, cap)
|
|
402
|
+
except SessionError as e:
|
|
403
|
+
start_error = f"could not start: {e}"[-1500:] # one failed start per size: the rest of the size is charged without waiting out more start-up timeouts
|
|
404
|
+
if session is None:
|
|
405
|
+
rec = {"id": inst["id"], "dim": inst["dim"], "device": device_key(info), "status": "crash", "cpu": cap, "cpu_net": cap, "err": start_error}
|
|
406
|
+
f.write(json.dumps(rec) + "\n")
|
|
407
|
+
f.flush()
|
|
408
|
+
recs.append(rec)
|
|
409
|
+
if progress:
|
|
410
|
+
progress(os.path.basename(solver), rec)
|
|
411
|
+
continue
|
|
412
|
+
rec = measure_one(session, inst, verify, cap, info, smi_fn, sampler)
|
|
413
|
+
f.write(json.dumps(rec) + "\n")
|
|
414
|
+
f.flush()
|
|
415
|
+
recs.append(rec)
|
|
416
|
+
if progress:
|
|
417
|
+
progress(os.path.basename(solver), rec)
|
|
418
|
+
if rec["status"] in ("timeout", "crash") and not rec.get("invalid"):
|
|
419
|
+
session.close() # killed or dead: the next instance gets a fresh, re-warmed child
|
|
420
|
+
session = None
|
|
421
|
+
finally:
|
|
422
|
+
if session is not None:
|
|
423
|
+
session.close()
|
|
424
|
+
finally:
|
|
425
|
+
sampler.stop()
|
|
426
|
+
return recs
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def verify_ladder(path, rerun=0, master=None, solver_path=None, sample_seed=None, smi_fn=smi, session_factory=None):
|
|
430
|
+
"""-> (ok, [lines]). Consistency of every record; no record flagged invalid; with rerun, K random solved records re-MEASURED once each on this device (same model only), each
|
|
431
|
+
within RERUN_TOLERANCE of the recorded time."""
|
|
432
|
+
session_factory = session_factory or default_session_factory
|
|
433
|
+
lines, ok = [], True
|
|
434
|
+
meta, recs = tflops.read_ladder(path)
|
|
435
|
+
if meta is None or meta.get("cost_model") != "gpu_time":
|
|
436
|
+
return False, [f"FAIL {os.path.basename(path)}: not a gpu_time ladder"]
|
|
437
|
+
cap = meta.get("budget")
|
|
438
|
+
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or cap <= 0 or not isinstance(meta.get("cpu"), str) or "solver_sha256" not in meta:
|
|
439
|
+
return False, [f"FAIL {os.path.basename(path)}: the header lacks a positive budget, a device or a solver hash"]
|
|
440
|
+
if not recs:
|
|
441
|
+
return False, [f"FAIL {os.path.basename(path)}: the ladder holds no records (an empty ladder certifies nothing)"]
|
|
442
|
+
bad = 0
|
|
443
|
+
for r in recs:
|
|
444
|
+
try:
|
|
445
|
+
if r.get("device", meta["cpu"]) != meta["cpu"]:
|
|
446
|
+
bad += 1 # a record measured on a different device than the ladder claims
|
|
447
|
+
elif r["status"] == "solved":
|
|
448
|
+
if not (isinstance(r["cpu"], (int, float)) and abs(r["cpu_net"] - max(r["cpu"], 1e-3)) < 1e-12 and 0 <= r["cpu"] <= cap):
|
|
449
|
+
bad += 1
|
|
450
|
+
elif r["cpu_net"] != cap:
|
|
451
|
+
bad += 1
|
|
452
|
+
except (KeyError, TypeError):
|
|
453
|
+
bad += 1
|
|
454
|
+
lines.append(f"{'PASS' if not bad else 'FAIL'} cost model: solved cost = the parent-measured time (floored at 1e-3 s), every other record charged the cap" + (f" ({bad} disagree)" if bad else ""))
|
|
455
|
+
ok = ok and not bad
|
|
456
|
+
invalid = [r.get("id") for r in recs if r.get("invalid")]
|
|
457
|
+
lines.append(f"{'FAIL' if invalid else 'PASS'} no invalid measurements" + (f": {len(invalid)} record(s) flagged invalid ({', '.join(map(str, invalid[:5]))}): re-measure them" if invalid else ""))
|
|
458
|
+
ok = ok and not invalid
|
|
459
|
+
if rerun:
|
|
460
|
+
try:
|
|
461
|
+
now = device_key(device_info(smi_fn))
|
|
462
|
+
except GpuUnavailable as e:
|
|
463
|
+
lines.append(f"SKIP re-measure: {e}")
|
|
464
|
+
return False, lines # asked for, not done: not a pass
|
|
465
|
+
if now != meta.get("cpu"):
|
|
466
|
+
lines.append(f"SKIP re-measure: this machine's device is {now!r}, the ladder's is {meta.get('cpu')!r} (a time means nothing on another device)")
|
|
467
|
+
return False, lines
|
|
468
|
+
solver = solver_path or os.path.join(HERE, "solvers", meta["solver"])
|
|
469
|
+
if not os.path.exists(solver) or tflops.sha256_file(solver) != meta["solver_sha256"]:
|
|
470
|
+
lines.append(f"FAIL re-measure: the solver file {meta['solver']} is missing here or differs from the ladder's")
|
|
471
|
+
return False, lines
|
|
472
|
+
mod, derive, verify, d = tflops.load_family(meta["family"])
|
|
473
|
+
if tflops.sha256_file(os.path.join(d, "decl.py")) != meta.get("decl_sha256"):
|
|
474
|
+
lines.append("FAIL re-measure: this checkout's decl.py differs from the ladder's")
|
|
475
|
+
return False, lines
|
|
476
|
+
if instances.commit(master) != meta["commit"]:
|
|
477
|
+
lines.append("SKIP re-measure: the seed given does not match the ladder's commitment")
|
|
478
|
+
return False, lines
|
|
479
|
+
solved = [r for r in recs if r["status"] == "solved"]
|
|
480
|
+
if not solved:
|
|
481
|
+
lines.append("FAIL re-measure: the ladder has no solved record to re-measure")
|
|
482
|
+
return False, lines
|
|
483
|
+
k = len(solved) if rerun == "all" else min(int(rerun), len(solved))
|
|
484
|
+
seed = secrets.randbits(64) if sample_seed is None else sample_seed
|
|
485
|
+
pick = random.Random(seed).sample(solved, k) if k < len(solved) else solved
|
|
486
|
+
lines.append(f"re-measure: {len(pick)} of {len(solved)} solved records, once each (sample seed {seed}); agreement means within a factor of {RERUN_TOLERANCE}, not identical")
|
|
487
|
+
info = device_info(smi_fn)
|
|
488
|
+
warm = meta.get("warm", WARM_INSTANCES)
|
|
489
|
+
agree = 0
|
|
490
|
+
sampler = Sampler(smi_fn)
|
|
491
|
+
sampler.start()
|
|
492
|
+
try:
|
|
493
|
+
for r in pick:
|
|
494
|
+
dim, idx = r["id"].split(":")
|
|
495
|
+
inst = derive(master, int(dim), int(idx))
|
|
496
|
+
try:
|
|
497
|
+
session = _open_warm(session_factory, solver, derive, int(dim), warm, cap)
|
|
498
|
+
except SessionError as e:
|
|
499
|
+
lines.append(f" could not re-measure {r['id']}: {e}")
|
|
500
|
+
continue
|
|
501
|
+
try:
|
|
502
|
+
again = measure_one(session, inst, verify, cap, info, smi_fn, sampler)
|
|
503
|
+
finally:
|
|
504
|
+
session.close()
|
|
505
|
+
if again["status"] == "solved" and 1 / RERUN_TOLERANCE <= again["cpu_net"] / r["cpu_net"] <= RERUN_TOLERANCE: # cpu_net: floored at 1e-3 s, never zero
|
|
506
|
+
agree += 1
|
|
507
|
+
else:
|
|
508
|
+
lines.append(f" differs at {r['id']}: recorded {r['cpu']:.4f} s, re-measured {again['status']} {again.get('cpu')}")
|
|
509
|
+
finally:
|
|
510
|
+
sampler.stop()
|
|
511
|
+
good = agree == len(pick)
|
|
512
|
+
lines.append(f"{'PASS' if good else 'FAIL'} re-measure: {agree}/{len(pick)} agree")
|
|
513
|
+
ok = ok and good
|
|
514
|
+
return ok, lines
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
# ----------------------------------------------------------------------------------------------------------------- command line
|
|
518
|
+
def cmd_info(a):
|
|
519
|
+
try:
|
|
520
|
+
info = device_info()
|
|
521
|
+
except GpuUnavailable as e:
|
|
522
|
+
sys.exit(f"no usable GPU: {e}")
|
|
523
|
+
busy = other_gpu_processes(smi("compute-apps", "pid,used_memory"))
|
|
524
|
+
mask = throttle_mask()
|
|
525
|
+
print(f"device : {device_key(info)} (GPU index {GPU_INDEX})\nidle : {'yes' if not busy else f'NO ({busy} compute process(es))'}\nthrottle mask now: {mask if mask is None else hex(mask)} "
|
|
526
|
+
f"({'slowdown active' if mask and mask & THROTTLE_BAD else 'none that invalidates a measurement'})")
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def cmd_plan(a):
|
|
530
|
+
master = tflops._master(a)
|
|
531
|
+
try:
|
|
532
|
+
key = device_key(device_info())
|
|
533
|
+
except GpuUnavailable as e:
|
|
534
|
+
sys.exit(f"cannot plan: the plan pins this machine's GPU, and {e}")
|
|
535
|
+
plan = tflops.plan_new("gpu_time", a.family, a.dims, a.n, a.cap, master, {"cpu": key})
|
|
536
|
+
fd = os.open(a.out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) # never overwrites a plan
|
|
537
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
538
|
+
json.dump(plan, f, indent=1, sort_keys=True)
|
|
539
|
+
print(f"plan written to {a.out}\nplan hash: {tflops.plan_hash(plan)}\nPublish this hash (somewhere the operator cannot edit) BEFORE any solver is submitted.")
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def cmd_practice(a):
|
|
543
|
+
master = tflops._master(a)
|
|
544
|
+
tag = instances.commit(master)[:8]
|
|
545
|
+
if a.plan:
|
|
546
|
+
plan = tflops.load_plan(a.plan)
|
|
547
|
+
if (plan.get("family"), sorted(set(a.dims)), a.n, a.cap, plan.get("commit")) != (a.family, plan.get("dims"), plan.get("n"), plan.get("cap"), instances.commit(master)):
|
|
548
|
+
sys.exit("refused: the dims, n, budget, family or seed on the command line are not the plan's. A planned ladder is run exactly as planned.")
|
|
549
|
+
os.makedirs(os.path.join(HERE, "results"), exist_ok=True)
|
|
550
|
+
paths = []
|
|
551
|
+
for role, solver in (("baseline", a.baseline), ("submission", a.solver)):
|
|
552
|
+
p = os.path.join(HERE, "results", f"gpu_{a.family}_{tag}_{role}_{os.path.basename(solver)}.jsonl")
|
|
553
|
+
try:
|
|
554
|
+
evaluate_ladder(solver, a.family, master, a.dims, a.n, a.cap, p, a.warm,
|
|
555
|
+
progress=lambda name, rec: print(f" {name} {rec['id']}: {rec['status']} {rec.get('cpu_net', 0):.4f}s" + (f" INVALID: {rec['err']}" if rec.get("invalid") else ""), flush=True))
|
|
556
|
+
except (GpuUnavailable, SessionError) as e:
|
|
557
|
+
sys.exit(f"cannot measure: {e}")
|
|
558
|
+
paths.append(p)
|
|
559
|
+
print("ladders:\n " + "\n ".join(paths) + "\nnext: python3 gpu_meter.py claim " + " ".join(paths))
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def cmd_claim(a):
|
|
563
|
+
r = subprocess.run([sys.executable, os.path.join(HERE, "extrapolate.py"), "--baseline", a.baseline, "--submission", a.submission], capture_output=True, text=True, cwd=HERE)
|
|
564
|
+
print(r.stdout + r.stderr)
|
|
565
|
+
if r.returncode:
|
|
566
|
+
sys.exit(r.returncode)
|
|
567
|
+
with open(a.submission + ".claim.json", encoding="utf-8") as f:
|
|
568
|
+
c = json.load(f)
|
|
569
|
+
print(f"credit (difficulty-stratified lower bound): {c.get('measured')} bits\n\nWhat this does NOT show: {CLAIM_LIMITS}")
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def cmd_verify(a):
|
|
573
|
+
master = tflops._master(a)
|
|
574
|
+
plan = tflops.load_plan(a.plan) if a.plan else None
|
|
575
|
+
if a.plan_hash and not plan:
|
|
576
|
+
sys.exit("--plan-hash needs --plan")
|
|
577
|
+
allok = True
|
|
578
|
+
for p in a.ladders:
|
|
579
|
+
ok, lines = verify_ladder(p, a.rerun, master, sample_seed=a.sample_seed)
|
|
580
|
+
if plan is not None:
|
|
581
|
+
meta, recs = tflops.read_ladder(p)
|
|
582
|
+
problems = tflops.check_plan(plan, meta or {}, recs, a.plan_hash)
|
|
583
|
+
lines.append(("FAIL plan: " + "; ".join(problems)) if problems else "PASS plan: the ladder is exactly the committed plan" + ("" if a.plan_hash else " (no --plan-hash given: the plan file itself was not checked against a published hash)"))
|
|
584
|
+
ok = ok and not problems
|
|
585
|
+
print(p)
|
|
586
|
+
for ln in lines:
|
|
587
|
+
print(" " + ln)
|
|
588
|
+
allok = allok and ok
|
|
589
|
+
print(("\nVERIFIED (consistent and re-measured on this device as requested)" if a.rerun else "\nCONSISTENT ONLY: nothing was re-measured (use --rerun K or --rerun all on the same device model)") if allok else "\nNOT VERIFIED")
|
|
590
|
+
print("What this does NOT show: " + CLAIM_LIMITS)
|
|
591
|
+
sys.exit(0 if allok else 1)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def main(argv=None):
|
|
595
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
596
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
597
|
+
sub.add_parser("info").set_defaults(fn=cmd_info)
|
|
598
|
+
p = sub.add_parser("plan")
|
|
599
|
+
p.add_argument("--family", required=True)
|
|
600
|
+
p.add_argument("--dims", type=int, nargs="+", required=True)
|
|
601
|
+
p.add_argument("--n", type=int, required=True)
|
|
602
|
+
p.add_argument("--cap", type=float, required=True)
|
|
603
|
+
p.add_argument("--seed")
|
|
604
|
+
p.add_argument("--out", required=True)
|
|
605
|
+
p.set_defaults(fn=cmd_plan)
|
|
606
|
+
p = sub.add_parser("practice")
|
|
607
|
+
p.add_argument("solver")
|
|
608
|
+
p.add_argument("--baseline", required=True)
|
|
609
|
+
p.add_argument("--family", required=True)
|
|
610
|
+
p.add_argument("--dims", type=int, nargs="+", required=True)
|
|
611
|
+
p.add_argument("--n", type=int, default=6)
|
|
612
|
+
p.add_argument("--cap", type=float, default=60)
|
|
613
|
+
p.add_argument("--warm", type=int, default=WARM_INSTANCES)
|
|
614
|
+
p.add_argument("--plan")
|
|
615
|
+
p.add_argument("--seed")
|
|
616
|
+
p.set_defaults(fn=cmd_practice)
|
|
617
|
+
p = sub.add_parser("claim")
|
|
618
|
+
p.add_argument("baseline")
|
|
619
|
+
p.add_argument("submission")
|
|
620
|
+
p.set_defaults(fn=cmd_claim)
|
|
621
|
+
p = sub.add_parser("verify")
|
|
622
|
+
p.add_argument("ladders", nargs="+")
|
|
623
|
+
p.add_argument("--rerun", type=lambda s: s if s == "all" else int(s), default=0)
|
|
624
|
+
p.add_argument("--sample-seed", type=int)
|
|
625
|
+
p.add_argument("--plan")
|
|
626
|
+
p.add_argument("--plan-hash")
|
|
627
|
+
p.add_argument("--seed")
|
|
628
|
+
p.set_defaults(fn=cmd_verify)
|
|
629
|
+
a = ap.parse_args(argv)
|
|
630
|
+
a.fn(a)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
if __name__ == "__main__":
|
|
634
|
+
main()
|