guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
guldo/_src/rounds.py
ADDED
|
@@ -0,0 +1,846 @@
|
|
|
1
|
+
"""Rounds, the transparency log and the submission store of a PUBLIC competition. Storage and rules only: no HTTP here, and
|
|
2
|
+
nothing in this module ever imports or executes an uploaded file (it is parsed with `ast` by submit_check.lint, nothing more).
|
|
3
|
+
|
|
4
|
+
A ROUND is one commit-reveal event: a family, k >= 3 seed commitments published at creation, a deadline, and a state
|
|
5
|
+
open -> closed -> evaluating -> evaluated -> revealed
|
|
6
|
+
Each round has an append-only, hash-chained LOG (rounds/<id>/log.jsonl). Entry 0 contains the commitments, so they are
|
|
7
|
+
public before any submission; every accepted submission is an entry carrying its file hash. Anyone holding a copy of the
|
|
8
|
+
log can detect a dropped, reordered, altered or back-dated entry (`verify_log`), and publishing the head hash somewhere the
|
|
9
|
+
operator cannot edit anchors it. Seeds are revealed only after evaluation, and only if they hash to the commitments.
|
|
10
|
+
|
|
11
|
+
There are no accounts. A handle is bound to a secret token issued with its first submission (only the token's hash is stored),
|
|
12
|
+
so nobody else can submit under a handle you hold. Contact details and IP addresses are never stored here.
|
|
13
|
+
"""
|
|
14
|
+
import contextlib
|
|
15
|
+
import fcntl
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import secrets
|
|
21
|
+
import shutil
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
import tempfile
|
|
25
|
+
import time
|
|
26
|
+
import uuid
|
|
27
|
+
|
|
28
|
+
import declared
|
|
29
|
+
import families
|
|
30
|
+
import judge
|
|
31
|
+
import official
|
|
32
|
+
import submit_check
|
|
33
|
+
|
|
34
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
35
|
+
ROUND_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,40}$")
|
|
36
|
+
HANDLE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{1,31}$")
|
|
37
|
+
RESERVED = {"baseline", "judge", "admin", "operator", "reference", "system", "solver", "unnamed", "root", "official"}
|
|
38
|
+
MAX_UPLOAD = 200_000
|
|
39
|
+
MAX_PER_HANDLE = 3
|
|
40
|
+
MAX_PER_ROUND = 200
|
|
41
|
+
MAX_N = 200 # instances per size a round may ask for (the official minimum is families.OFFICIAL[family]["n"])
|
|
42
|
+
STATES = ("open", "closed", "evaluating", "evaluated", "revealed")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
MIN_PYTHON = (3, 9)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def require_python():
|
|
49
|
+
"""Refuse to start on an interpreter older than the supported minimum, with a message, instead of failing later with a SyntaxError
|
|
50
|
+
or an AttributeError in the middle of a submission. Called by both entry points (public_portal.py serve, ops.py)."""
|
|
51
|
+
if tuple(sys.version_info[:2]) < MIN_PYTHON:
|
|
52
|
+
raise SystemExit(f"REFUSED: Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]} or newer is required (this is "
|
|
53
|
+
f"{sys.version_info[0]}.{sys.version_info[1]}).")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class Rejected(Exception):
|
|
57
|
+
"""A submission or operation refused for a reason the caller may show to the user."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def canonical(obj):
|
|
61
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def sha256(data):
|
|
65
|
+
return hashlib.sha256(data if isinstance(data, bytes) else data.encode()).hexdigest()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@contextlib.contextmanager
|
|
69
|
+
def locked(path):
|
|
70
|
+
"""Cross-process exclusive lock (the web tier and the operator CLI both write a round). It never creates the round's directory:
|
|
71
|
+
a lock on a round that does not exist must fail, or a request for any valid id would create it (and squat the id)."""
|
|
72
|
+
fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o660) # group-writable (subject to umask): the web tier and the operator are different users
|
|
73
|
+
try:
|
|
74
|
+
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
75
|
+
yield
|
|
76
|
+
finally:
|
|
77
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
78
|
+
os.close(fd)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def rdir(base, rid):
|
|
82
|
+
if not ROUND_ID_RE.match(rid or ""):
|
|
83
|
+
raise KeyError(rid)
|
|
84
|
+
return os.path.join(base, "rounds", rid)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _require(base, rid):
|
|
88
|
+
"""KeyError unless the round exists. Called before any lock is taken, so nothing is created for a round that does not exist."""
|
|
89
|
+
if not os.path.exists(os.path.join(rdir(base, rid), "round.json")):
|
|
90
|
+
raise KeyError(rid)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _read_json(path, default=None):
|
|
94
|
+
try:
|
|
95
|
+
with open(path, encoding="utf-8") as f:
|
|
96
|
+
return json.load(f)
|
|
97
|
+
except (OSError, ValueError):
|
|
98
|
+
return default
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _write_json(path, obj):
|
|
102
|
+
tmp = path + ".tmp"
|
|
103
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
104
|
+
json.dump(obj, f, indent=1, sort_keys=True)
|
|
105
|
+
os.replace(tmp, path) # atomic: a reader never sees half a file
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ---------------------------------------------------------------- the transparency log
|
|
109
|
+
def _entry_hash(entry):
|
|
110
|
+
body = {k: v for k, v in entry.items() if k != "hash"}
|
|
111
|
+
return sha256(entry["prev"] + canonical(body))
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def read_log(d):
|
|
115
|
+
out = []
|
|
116
|
+
try:
|
|
117
|
+
with open(os.path.join(d, "log.jsonl"), encoding="utf-8") as f:
|
|
118
|
+
for line in f:
|
|
119
|
+
if line.strip():
|
|
120
|
+
out.append(json.loads(line))
|
|
121
|
+
except OSError:
|
|
122
|
+
pass
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def append_log(d, kind, data, now):
|
|
127
|
+
"""Caller holds the round lock. Refuses to extend a chain that does not verify: appending to a damaged log would bury the damage."""
|
|
128
|
+
log = read_log(d)
|
|
129
|
+
if log:
|
|
130
|
+
ok, _, _, err = verify_log(d)
|
|
131
|
+
if not ok:
|
|
132
|
+
raise Rejected(f"REFUSED: this round's log does not verify ({err}); run `ops.py verify-log` and restore it from a backup before anything else is added")
|
|
133
|
+
prev = log[-1]["hash"] if log else sha256("genesis:" + os.path.basename(d))
|
|
134
|
+
# never write a timestamp earlier than the previous entry: a clock that steps backwards (NTP) must not make the log fail its own
|
|
135
|
+
# back-dating check and freeze the round. The clamp only ever moves a timestamp forward, so it cannot hide a late submission.
|
|
136
|
+
ts = max(round(now, 3), log[-1]["ts"]) if log else round(now, 3)
|
|
137
|
+
entry = {"seq": len(log), "ts": ts, "kind": kind, "data": data, "prev": prev}
|
|
138
|
+
entry["hash"] = _entry_hash(entry)
|
|
139
|
+
with open(os.path.join(d, "log.jsonl"), "a", encoding="utf-8") as f:
|
|
140
|
+
f.write(canonical(entry) + "\n")
|
|
141
|
+
f.flush()
|
|
142
|
+
os.fsync(f.fileno())
|
|
143
|
+
return entry
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def verify_log(d, expected_commitments=None):
|
|
147
|
+
"""(ok, entries, head_hash, error). Checks sequence, the hash chain, and that entry 0 carries the commitments."""
|
|
148
|
+
log = read_log(d)
|
|
149
|
+
if not log:
|
|
150
|
+
return False, 0, None, "empty log"
|
|
151
|
+
prev = sha256("genesis:" + os.path.basename(d))
|
|
152
|
+
for i, e in enumerate(log):
|
|
153
|
+
if e.get("seq") != i:
|
|
154
|
+
return False, len(log), None, f"entry {i}: sequence number {e.get('seq')} (an entry was dropped, duplicated or reordered)"
|
|
155
|
+
if e.get("prev") != prev:
|
|
156
|
+
return False, len(log), None, f"entry {i}: does not chain to the previous entry"
|
|
157
|
+
if _entry_hash(e) != e.get("hash"):
|
|
158
|
+
return False, len(log), None, f"entry {i}: contents do not match their hash (altered)"
|
|
159
|
+
if i > 0 and e["ts"] < log[i - 1]["ts"] - 1:
|
|
160
|
+
return False, len(log), None, f"entry {i}: timestamp earlier than the previous entry (back-dated)"
|
|
161
|
+
prev = e["hash"]
|
|
162
|
+
if log[0]["kind"] != "round_created":
|
|
163
|
+
return False, len(log), None, "the first entry is not round_created"
|
|
164
|
+
if expected_commitments is not None and log[0]["data"].get("commitments") != expected_commitments:
|
|
165
|
+
return False, len(log), None, "the logged commitments differ from the round's commitments"
|
|
166
|
+
return True, len(log), log[-1]["hash"], ""
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# ---------------------------------------------------------------- the rules a round commits to
|
|
170
|
+
# Everything that decides how an instance is generated, run, metered and scored. The hashes go into entry 0 of the log, so the
|
|
171
|
+
# operator cannot change the ladder, the baseline or the scoring code after seeing submissions without it being visible.
|
|
172
|
+
CODE_FILES = ["extrapolate.py", "families.py", "declared.py", "instances.py", "instances_mlwe.py", "instances_factor.py", "harness.py", "runner.py",
|
|
173
|
+
"judge.py", "official.py", "sandbox/meter.py", "sandbox/meter_core.py", "sandbox/Dockerfile.v2"]
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
COUNT_FILES = ["count_runner.py", "counted.py"] # the counting program and its driver: rules of a COUNTED round, pinned in its fingerprint (not in CODE_FILES, so timed rounds are unchanged)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def code_fingerprint(family, count=None):
|
|
180
|
+
"""`count` (a round's counted configuration, see count_config) makes this the fingerprint of a COUNTED round: its own size ladder, and the counting code pinned."""
|
|
181
|
+
def h(rel):
|
|
182
|
+
with open(os.path.join(HERE, rel), "rb") as f: # join(HERE, absolute path) is the absolute path (a declared domain outside the project)
|
|
183
|
+
return sha256(f.read())
|
|
184
|
+
baseline = families.DEFAULT_BASELINE[family]
|
|
185
|
+
fp = {"dims": list(families.DEFAULT_DIMS[family]), "instance_version": families.VERSIONS[family], "baseline": baseline,
|
|
186
|
+
"baseline_sha256": h(baseline), "code": {rel: h(rel) for rel in CODE_FILES}}
|
|
187
|
+
fp["code"].update(declared.domain_files(families, family)) # a declared domain: its decl.py, compiled.json and reference are rules too
|
|
188
|
+
if count is not None:
|
|
189
|
+
fp["dims"] = list(count["dims"])
|
|
190
|
+
fp["cost_model"] = "count"
|
|
191
|
+
fp["code"].update({rel: h(rel) for rel in COUNT_FILES})
|
|
192
|
+
return fp
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def count_config(family, dims, n, cap):
|
|
196
|
+
"""Validate and complete the counted configuration of a round: -> {dims, n, cap, steps_per_second, python, counter_sha256}. Rejected if the family cannot be
|
|
197
|
+
counted, this interpreter cannot count, or a value is out of range. The interpreter and counter are recorded because instruction counts are only comparable
|
|
198
|
+
on the same Python and the same counting program."""
|
|
199
|
+
import certificate # (lazily: certificate imports this module)
|
|
200
|
+
import counted
|
|
201
|
+
if not counted.supported():
|
|
202
|
+
raise Rejected(f"a counted round needs Python {counted.MIN_PYTHON[0]}.{counted.MIN_PYTHON[1]} or newer on the operator's machine (this is {counted.python_version()})")
|
|
203
|
+
why = counted.family_problem(family)
|
|
204
|
+
if why:
|
|
205
|
+
raise Rejected("this family cannot be counted: " + why)
|
|
206
|
+
if not (isinstance(dims, (list, tuple)) and dims and all(isinstance(d, int) and not isinstance(d, bool) and 1 <= d <= 512 for d in dims) and len(set(dims)) == len(dims)):
|
|
207
|
+
raise Rejected("the counted sizes must be a non-empty list of distinct integers from 1 to 512")
|
|
208
|
+
for d in dims: # a size the family cannot generate would fail every instance of every submission AFTER the seeds were committed: find out now
|
|
209
|
+
try:
|
|
210
|
+
families.derive(family, b"svp-count-config-probe", d, 0)
|
|
211
|
+
except Exception as e: # (any derive() failure: a bad size, a family-specific limit)
|
|
212
|
+
raise Rejected(f"the family {family!r} cannot generate a question of size {d}: {str(e)[:120]}")
|
|
213
|
+
if not (isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= MAX_N):
|
|
214
|
+
raise Rejected(f"the counted n must be an integer from 1 to {MAX_N} (instances per size)")
|
|
215
|
+
if not (isinstance(cap, (int, float)) and not isinstance(cap, bool) and 0 < cap <= certificate.MAX_COUNTED_CAP): # `not (...)` also rejects NaN
|
|
216
|
+
raise Rejected(f"the counted cap must be a number between 0 and {certificate.MAX_COUNTED_CAP:g} counted seconds")
|
|
217
|
+
return {"dims": sorted(dims), "n": n, "cap": float(cap), "steps_per_second": counted.STEPS_PER_SECOND, "python": counted.python_version(),
|
|
218
|
+
"counter_sha256": counted.counter_sha256()}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def fingerprint_changes(old, new):
|
|
222
|
+
"""{label: (old, new)} for everything that differs; empty if the rules are as the round committed to."""
|
|
223
|
+
out = {}
|
|
224
|
+
for k in ("dims", "instance_version", "baseline", "baseline_sha256", "cost_model"):
|
|
225
|
+
if old.get(k) != new.get(k):
|
|
226
|
+
out[k] = (old.get(k), new.get(k))
|
|
227
|
+
for rel in sorted(set(old.get("code", {})) | set(new.get("code", {}))):
|
|
228
|
+
if old.get("code", {}).get(rel) != new.get("code", {}).get(rel):
|
|
229
|
+
out["code/" + rel] = (old.get("code", {}).get(rel), new.get("code", {}).get(rel))
|
|
230
|
+
return out
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# ---------------------------------------------------------------- rounds
|
|
234
|
+
def create_round(base, rid, title, family, commitments, closes_at, opens_at=None, now=None, allow_few_seeds=False, max_submissions=None, n=None, allow_private=True, count=None):
|
|
235
|
+
"""allow_private=False makes it a public, LOGGED rule of the round that every submission must publish its source (after the reveal): a private solver
|
|
236
|
+
cannot be re-run by anyone but the operator, and some rounds will not accept results that rest on the operator alone.
|
|
237
|
+
count={"dims": [...], "n": N, "cap": C} makes it a COUNTED round: every submission and the baseline are measured by deterministic step counting (counted.py) on
|
|
238
|
+
THIS ladder, which is committed in log entry 0 (with the Python version and the counting program) before any submission. Anyone can then re-execute any
|
|
239
|
+
instance and must get identical counts, so the result does not rest on the operator's machine or sandbox. Source publication is mandatory (the proof is
|
|
240
|
+
re-execution) and n and the cap are the round's own; `n` is ignored."""
|
|
241
|
+
now = now or time.time()
|
|
242
|
+
if not ROUND_ID_RE.match(rid or ""):
|
|
243
|
+
raise Rejected("round id must be 2-41 characters of a-z, 0-9 and '-'")
|
|
244
|
+
if family not in families.FAMILIES:
|
|
245
|
+
raise Rejected(f"unknown family {family!r}")
|
|
246
|
+
if family in families.DECLARED: # a compiled domain is official only if the compiler's own end-to-end controls passed
|
|
247
|
+
st = (families.DECLARED[family]["compiled"].get("selftest") or {}).get("status")
|
|
248
|
+
if st != "pass" and not allow_few_seeds:
|
|
249
|
+
raise Rejected(f"family {family!r} has not passed its compiler self-test (status {st!r}); run compile_domain.py selftest {family}")
|
|
250
|
+
if len(set(commitments)) != len(commitments) or not all(re.fullmatch(r"[0-9a-f]{64}", c) for c in commitments):
|
|
251
|
+
raise Rejected("commitments must be distinct 64-hex sha256 values")
|
|
252
|
+
if len(commitments) < judge.MIN_SEEDS and not allow_few_seeds:
|
|
253
|
+
raise Rejected(f"an official round needs at least {judge.MIN_SEEDS} seed commitments (got {len(commitments)})")
|
|
254
|
+
opens_at = opens_at if opens_at is not None else now
|
|
255
|
+
if closes_at <= opens_at:
|
|
256
|
+
raise Rejected("the deadline must be after the opening time")
|
|
257
|
+
max_submissions = MAX_PER_ROUND if max_submissions is None else max_submissions
|
|
258
|
+
if not (isinstance(max_submissions, int) and 1 <= max_submissions <= MAX_PER_ROUND):
|
|
259
|
+
raise Rejected(f"max_submissions must be an integer from 1 to {MAX_PER_ROUND}")
|
|
260
|
+
cfg = families.OFFICIAL[family]
|
|
261
|
+
if count is not None:
|
|
262
|
+
if not isinstance(count, dict):
|
|
263
|
+
raise Rejected("count must be a dict with dims, n and cap")
|
|
264
|
+
count = count_config(family, count.get("dims"), count.get("n"), count.get("cap")) # validated BEFORE anything is created
|
|
265
|
+
n, allow_private = count["n"], False
|
|
266
|
+
else:
|
|
267
|
+
n = cfg["n"] if n is None else n
|
|
268
|
+
# instances per size: at least the official minimum (the board already requires n >= it), at most MAX_N. More than the minimum is how a
|
|
269
|
+
# round certifies larger speedups: at the official n a 4-bit effect is certified 0% of the time, at 2x the instances 100%.
|
|
270
|
+
if not (isinstance(n, int) and not isinstance(n, bool) and cfg["n"] <= n <= MAX_N):
|
|
271
|
+
raise Rejected(f"n must be an integer from {cfg['n']} to {MAX_N} (instances per size)")
|
|
272
|
+
d = rdir(base, rid)
|
|
273
|
+
if os.path.exists(d):
|
|
274
|
+
raise Rejected(f"round {rid!r} already exists")
|
|
275
|
+
os.makedirs(os.path.join(d, "submissions"))
|
|
276
|
+
r = {"id": rid, "title": title[:120], "family": family, "commitments": sorted(commitments), "opens_at": opens_at,
|
|
277
|
+
# n is the round's own (logged) value and is what the worker evaluates with; the CPU cap is always the family's official one (a counted round's own cap)
|
|
278
|
+
"closes_at": closes_at, "state": "open", "n": n, "budget": cfg["budget"] if count is None else count["cap"], "created_at": now,
|
|
279
|
+
"max_submissions": max_submissions, "fingerprint": code_fingerprint(family, count) if count is not None else code_fingerprint(family), "allow_private": bool(allow_private)}
|
|
280
|
+
logged = ["id", "title", "family", "commitments", "opens_at", "closes_at", "n", "budget", "max_submissions", "fingerprint", "allow_private"]
|
|
281
|
+
if count is not None:
|
|
282
|
+
r["cost_model"], r["count"] = "count", count
|
|
283
|
+
logged += ["cost_model", "count"] # a public rule of the round, in log entry 0: the ladder every participant is counted on
|
|
284
|
+
with locked(os.path.join(d, ".lock")):
|
|
285
|
+
_write_json(os.path.join(d, "round.json"), r)
|
|
286
|
+
_write_json(os.path.join(d, "handles.json"), {})
|
|
287
|
+
append_log(d, "round_created", {k: r[k] for k in logged}, now)
|
|
288
|
+
return r
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def get_round(base, rid):
|
|
292
|
+
r = _read_json(os.path.join(rdir(base, rid), "round.json"))
|
|
293
|
+
if r is None:
|
|
294
|
+
raise KeyError(rid)
|
|
295
|
+
return r
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def list_rounds(base):
|
|
299
|
+
root = os.path.join(base, "rounds")
|
|
300
|
+
out = []
|
|
301
|
+
for rid in sorted(os.listdir(root)) if os.path.isdir(root) else []:
|
|
302
|
+
r = _read_json(os.path.join(root, rid, "round.json"))
|
|
303
|
+
if r:
|
|
304
|
+
out.append(r)
|
|
305
|
+
return sorted(out, key=lambda r: r["created_at"], reverse=True)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _set_state(d, r, state, now, kind, data=None):
|
|
309
|
+
# LOG FIRST. append_log refuses (and changes nothing) if the chain does not verify; writing round.json first used to leave a
|
|
310
|
+
# round "closed" with no log entry, and every later call then skipped the closure because the state already said so.
|
|
311
|
+
append_log(d, kind, data or {}, now)
|
|
312
|
+
r["state"] = state
|
|
313
|
+
_write_json(os.path.join(d, "round.json"), r)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def close_round(base, rid, now=None, force=False):
|
|
317
|
+
"""Close a round at its deadline (or earlier with force=True, an operator action that is logged)."""
|
|
318
|
+
now = now or time.time()
|
|
319
|
+
_require(base, rid)
|
|
320
|
+
d = rdir(base, rid)
|
|
321
|
+
with locked(os.path.join(d, ".lock")):
|
|
322
|
+
r = get_round(base, rid)
|
|
323
|
+
if r["state"] != "open":
|
|
324
|
+
return r
|
|
325
|
+
if now < r["closes_at"] and not force:
|
|
326
|
+
raise Rejected("the deadline has not passed")
|
|
327
|
+
_set_state(d, r, "closed", now, "round_closed", {"forced": bool(force and now < r["closes_at"])})
|
|
328
|
+
return r
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def close_due(base, now=None):
|
|
332
|
+
"""Close every overdue round. One round whose log is damaged must not stop the others (or, since the web tier calls this on
|
|
333
|
+
every page view, take the whole site down): it is left unchanged, and its own page reports the log problem."""
|
|
334
|
+
now = now or time.time()
|
|
335
|
+
for r in list_rounds(base):
|
|
336
|
+
if r["state"] == "open" and now >= r["closes_at"]:
|
|
337
|
+
try:
|
|
338
|
+
close_round(base, r["id"], now)
|
|
339
|
+
except Rejected:
|
|
340
|
+
continue
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
# ---------------------------------------------------------------- submissions
|
|
344
|
+
def list_submissions(base, rid):
|
|
345
|
+
root = os.path.join(rdir(base, rid), "submissions")
|
|
346
|
+
out = []
|
|
347
|
+
for sid in sorted(os.listdir(root)) if os.path.isdir(root) else []:
|
|
348
|
+
m = _read_json(os.path.join(root, sid, "meta.json"))
|
|
349
|
+
if m:
|
|
350
|
+
out.append(m)
|
|
351
|
+
return sorted(out, key=lambda m: m["seq"])
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def submit(base, rid, handle, token, source, publish_source=False, now=None):
|
|
355
|
+
"""Validate and store a submission WITHOUT running it. Returns a dict with id, seq, hash, warnings and, only when the handle
|
|
356
|
+
is new, the secret token (shown once). Raises Rejected with a message that is safe to show. Rejections store nothing."""
|
|
357
|
+
now = now or time.time()
|
|
358
|
+
handle = (handle or "").strip()
|
|
359
|
+
if not HANDLE_RE.match(handle) or handle.lower() in RESERVED:
|
|
360
|
+
raise Rejected("handle must be 2-32 characters: letters, digits, '.', '_' or '-', starting with a letter or digit (some names are reserved)")
|
|
361
|
+
if not isinstance(source, (bytes, bytearray)) or len(source) == 0:
|
|
362
|
+
raise Rejected("empty file")
|
|
363
|
+
_require(base, rid)
|
|
364
|
+
if len(source) > MAX_UPLOAD:
|
|
365
|
+
raise Rejected(f"the file is {len(source)} bytes; the limit is {MAX_UPLOAD}")
|
|
366
|
+
try:
|
|
367
|
+
text = bytes(source).decode("utf-8")
|
|
368
|
+
except UnicodeDecodeError:
|
|
369
|
+
raise Rejected("the file is not UTF-8 text")
|
|
370
|
+
if "\x00" in text:
|
|
371
|
+
raise Rejected("the file contains NUL bytes")
|
|
372
|
+
d = rdir(base, rid)
|
|
373
|
+
with locked(os.path.join(d, ".lock")):
|
|
374
|
+
r = get_round(base, rid)
|
|
375
|
+
if r["state"] == "open" and now >= r["closes_at"]:
|
|
376
|
+
_set_state(d, r, "closed", now, "round_closed", {"forced": False})
|
|
377
|
+
if r["state"] != "open":
|
|
378
|
+
raise Rejected("this round is closed")
|
|
379
|
+
if now < r["opens_at"]:
|
|
380
|
+
raise Rejected("this round has not opened yet")
|
|
381
|
+
if not publish_source and not r.get("allow_private", True): # a rule logged in the round's first entry; rounds made before it existed allow private solvers
|
|
382
|
+
raise Rejected("this round requires every solver to be published: tick 'publish source' (your source becomes public only after the seeds are revealed)")
|
|
383
|
+
subs = list_submissions(base, rid)
|
|
384
|
+
if len(subs) >= r.get("max_submissions", MAX_PER_ROUND):
|
|
385
|
+
raise Rejected("this round has reached its submission limit")
|
|
386
|
+
digest = sha256(bytes(source))
|
|
387
|
+
if any(m["sha256"] == digest for m in subs):
|
|
388
|
+
raise Rejected("an identical file was already submitted in this round")
|
|
389
|
+
handles = _read_json(os.path.join(d, "handles.json"), {})
|
|
390
|
+
key = handle.lower()
|
|
391
|
+
new_token = None
|
|
392
|
+
if key in handles:
|
|
393
|
+
if not token or not secrets.compare_digest(sha256(token), handles[key]):
|
|
394
|
+
raise Rejected("that handle is already taken in this round; supply its token, or pick another handle")
|
|
395
|
+
if sum(1 for m in subs if m["handle"].lower() == key) >= MAX_PER_HANDLE:
|
|
396
|
+
raise Rejected(f"a handle may make at most {MAX_PER_HANDLE} submissions per round")
|
|
397
|
+
else:
|
|
398
|
+
new_token = secrets.token_urlsafe(24)
|
|
399
|
+
sid = uuid.uuid4().hex[:12]
|
|
400
|
+
fname = f"{handle}-{sid}.py" # judge.py names a board entry after the file, so every submission needs a unique name
|
|
401
|
+
sdir = os.path.join(d, "submissions", sid)
|
|
402
|
+
os.makedirs(sdir)
|
|
403
|
+
path = os.path.join(sdir, fname)
|
|
404
|
+
try:
|
|
405
|
+
with open(path, "wb") as f:
|
|
406
|
+
f.write(bytes(source)) # the EXACT uploaded bytes (never a re-encoded string): the log committed to their hash
|
|
407
|
+
try:
|
|
408
|
+
warnings, fatal = submit_check.lint(path) # ast only: never imports or runs the file
|
|
409
|
+
except (SyntaxError, ValueError, RecursionError) as e:
|
|
410
|
+
warnings, fatal = [], [f"could not parse: {type(e).__name__}"]
|
|
411
|
+
if fatal:
|
|
412
|
+
raise Rejected("; ".join(fatal))
|
|
413
|
+
if r.get("cost_model") == "count":
|
|
414
|
+
import counted
|
|
415
|
+
problems = counted.source_problems(bytes(source).decode("utf-8", "replace"))
|
|
416
|
+
if problems: # the counter enforces the same rules when it loads the solver: a source outside the subset could not be counted at all
|
|
417
|
+
raise Rejected("a counted round runs solvers in a restricted subset of Python (pure computation: no sys, files, frames, getattr/type/eval, dunder attributes) "
|
|
418
|
+
"and this source is outside it: " + "; ".join(problems[:5]))
|
|
419
|
+
entry = append_log(d, "submission", {"id": sid, "handle": handle, "sha256": digest, "size": len(source),
|
|
420
|
+
"publish_source": bool(publish_source)}, now)
|
|
421
|
+
meta = {"id": sid, "handle": handle, "family": r["family"], "file": fname, "sha256": digest, "size": len(source),
|
|
422
|
+
"submitted_at": now, "publish_source": bool(publish_source), "warnings": warnings, "state": "queued",
|
|
423
|
+
"seq": entry["seq"], "log_hash": entry["hash"], "result": None, "reason": ""}
|
|
424
|
+
_write_json(os.path.join(sdir, "meta.json"), meta)
|
|
425
|
+
if new_token:
|
|
426
|
+
handles[key] = sha256(new_token)
|
|
427
|
+
_write_json(os.path.join(d, "handles.json"), handles)
|
|
428
|
+
except BaseException:
|
|
429
|
+
shutil.rmtree(sdir, ignore_errors=True) # a rejected or failed submission leaves nothing behind
|
|
430
|
+
raise
|
|
431
|
+
out = {"id": sid, "seq": meta["seq"], "hash": meta["log_hash"], "sha256": digest, "warnings": warnings, "round": rid}
|
|
432
|
+
if new_token:
|
|
433
|
+
out["token"] = new_token
|
|
434
|
+
return out
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _logged_submissions(d):
|
|
438
|
+
"""{submission id: log entry} for every well-formed submission entry; malformed entries (a damaged log) are skipped, never fatal."""
|
|
439
|
+
out = {}
|
|
440
|
+
for e in read_log(d):
|
|
441
|
+
data = e.get("data") if isinstance(e, dict) else None
|
|
442
|
+
if e.get("kind") == "submission" and isinstance(data, dict) and data.get("id") and data.get("sha256"):
|
|
443
|
+
out[data["id"]] = e
|
|
444
|
+
return out
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def check_consistency(base, rid, hash_files=False):
|
|
448
|
+
"""Cross-check the log against what is stored. Returns a list of problems (empty = consistent). The log commits to every accepted
|
|
449
|
+
file's hash, so: a logged submission with nothing stored (a crash between the log append and the metadata write), a stored
|
|
450
|
+
submission with no matching log entry, and (hash_files=True, which reads every file) a file that no longer matches its logged hash."""
|
|
451
|
+
d = rdir(base, rid)
|
|
452
|
+
logged = _logged_submissions(d)
|
|
453
|
+
stored = {m["id"]: m for m in list_submissions(base, rid)}
|
|
454
|
+
problems = []
|
|
455
|
+
for sid, e in sorted(logged.items()):
|
|
456
|
+
if sid not in stored:
|
|
457
|
+
problems.append(f"log entry #{e['seq']} (submission {sid}) has no stored submission")
|
|
458
|
+
for sid, m in sorted(stored.items()):
|
|
459
|
+
e = logged.get(sid)
|
|
460
|
+
if e is None or e["hash"] != m["log_hash"] or e["data"]["sha256"] != m["sha256"]:
|
|
461
|
+
problems.append(f"stored submission {sid} has no matching log entry")
|
|
462
|
+
elif hash_files:
|
|
463
|
+
try:
|
|
464
|
+
with open(os.path.join(d, "submissions", sid, m["file"]), "rb") as f:
|
|
465
|
+
actual = sha256(f.read())
|
|
466
|
+
except OSError:
|
|
467
|
+
actual = None
|
|
468
|
+
if actual != e["data"]["sha256"]:
|
|
469
|
+
problems.append(f"submission {sid}: the stored file does not match its logged hash (swapped or missing after acceptance)")
|
|
470
|
+
return problems
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def get_submission(base, rid, sid):
|
|
474
|
+
if not re.fullmatch(r"[0-9a-f]{12}", sid or ""):
|
|
475
|
+
raise KeyError(sid)
|
|
476
|
+
m = _read_json(os.path.join(rdir(base, rid), "submissions", sid, "meta.json"))
|
|
477
|
+
if m is None:
|
|
478
|
+
raise KeyError(sid)
|
|
479
|
+
return m
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
# ---------------------------------------------------------------- evaluation and reveal (operator side)
|
|
483
|
+
def _save_meta(base, rid, m):
|
|
484
|
+
_write_json(os.path.join(rdir(base, rid), "submissions", m["id"], "meta.json"), m)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def evaluate_round(base, rid, seed_dir, sandbox_ok, now=None, runner=subprocess.run, only_one=False, allow_code_change=False, results_dir=None, allow_unsandboxed=False):
|
|
488
|
+
"""Evaluate every queued submission of a CLOSED round in the docker sandbox, against exactly the round's committed seeds.
|
|
489
|
+
sandbox_ok: callable returning True only if the sandbox is verified (portal.sandbox_verified). Refuses otherwise.
|
|
490
|
+
Also refuses if the ladder, baseline or scoring code differ from what entry 0 of the log committed to; allow_code_change
|
|
491
|
+
proceeds anyway but writes a public `code_changed` entry with the old and new hashes.
|
|
492
|
+
A COUNTED round (create_round(count=...)) is not evaluated in the sandbox: its submissions are run, UNSANDBOXED, under step counting, so the operator must say
|
|
493
|
+
so (allow_unsandboxed) and should do it as a throwaway user or VM that cannot read anything of value. What stops a solver from cheating there is that its
|
|
494
|
+
source is public and every count can be re-executed by anyone: a solver that behaves differently on the operator's machine (reading a seed file, say) is exposed
|
|
495
|
+
by the first re-run. The interpreter and the counting program must be the ones the round committed to, whatever allow_code_change says: other ones count differently."""
|
|
496
|
+
now = now or time.time()
|
|
497
|
+
_require(base, rid)
|
|
498
|
+
r0 = get_round(base, rid)
|
|
499
|
+
if r0.get("cost_model") == "count":
|
|
500
|
+
import counted
|
|
501
|
+
c = r0["count"]
|
|
502
|
+
if not allow_unsandboxed:
|
|
503
|
+
raise Rejected("REFUSED: this is a COUNTED round: its submissions run unsandboxed under step counting (anyone re-executes the counts to check them). Run the worker as a "
|
|
504
|
+
"throwaway user or VM that holds nothing of value, then pass --unsandboxed")
|
|
505
|
+
if os.geteuid() == 0:
|
|
506
|
+
raise Rejected("REFUSED: a counted round runs submitted code as the user running the worker, and that user is root. Use an unprivileged throwaway user")
|
|
507
|
+
if not counted.supported() or counted.python_version() != c["python"]:
|
|
508
|
+
raise Rejected(f"REFUSED: this round counts on {c['python']} and this machine runs {counted.python_version()}: counts from another interpreter would not reproduce")
|
|
509
|
+
if counted.counter_sha256() != c["counter_sha256"]:
|
|
510
|
+
raise Rejected("REFUSED: count_runner.py is not the counting program this round committed to; another counter would count the same program differently")
|
|
511
|
+
elif not sandbox_ok():
|
|
512
|
+
raise Rejected("REFUSED: the sandbox is not verified (or its files changed since it was); nothing was evaluated")
|
|
513
|
+
d = rdir(base, rid)
|
|
514
|
+
# ONE worker for the whole data directory, not one per round: every evaluation writes the same global leaderboard file, and two
|
|
515
|
+
# rounds evaluating at once would race on it. The lock dies with its process, so a crashed worker never blocks a restart.
|
|
516
|
+
wfd = os.open(os.path.join(base, ".worker.lock"), os.O_CREAT | os.O_RDWR, 0o660)
|
|
517
|
+
try:
|
|
518
|
+
fcntl.flock(wfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
519
|
+
except OSError:
|
|
520
|
+
os.close(wfd)
|
|
521
|
+
raise Rejected("another worker is already evaluating this round")
|
|
522
|
+
try:
|
|
523
|
+
return _evaluate_locked(base, rid, seed_dir, now, runner, only_one, allow_code_change, results_dir)
|
|
524
|
+
finally:
|
|
525
|
+
fcntl.flock(wfd, fcntl.LOCK_UN)
|
|
526
|
+
os.close(wfd)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _evaluate_counted_submission(base, rid, r, m, path, seed_dir, results_dir, now):
|
|
530
|
+
"""Count one submission (and, once per seed, the baseline) on the round's declared ladder, then score it exactly as a certificate would. Sets m's state, result
|
|
531
|
+
and reason; never raises for a failure of the evaluation itself (that is a failed submission, with the details in a file the export never touches)."""
|
|
532
|
+
import certificate
|
|
533
|
+
import counted
|
|
534
|
+
d = rdir(base, rid)
|
|
535
|
+
results_dir = os.path.abspath(results_dir or os.path.join(HERE, "results"))
|
|
536
|
+
c = r["count"]
|
|
537
|
+
baseline = judge.default_baseline(r["family"])
|
|
538
|
+
try:
|
|
539
|
+
os.makedirs(results_dir, exist_ok=True)
|
|
540
|
+
for seed_file, h in official.verified_seeds(seed_dir):
|
|
541
|
+
stale = os.path.join(results_dir, f"judged_{r['family']}_{h[:8]}_submission_{os.path.basename(path)}.jsonl")
|
|
542
|
+
if os.path.exists(stale): # a worker that died mid-ladder: keep what it wrote (moved aside, not deleted) and count again
|
|
543
|
+
os.replace(stale, f"{stale}.stale-{int(now)}")
|
|
544
|
+
counted.evaluate_seed(r["family"], seed_file, path, baseline, c["dims"], c["n"], c["cap"], results_dir, reuse_baseline=True)
|
|
545
|
+
cert = certificate.build(r["family"], path, seed_dir, reveal_seeds=False, results_dir=results_dir, round_data=base, round_id=rid)
|
|
546
|
+
got = cert.get("count") or {}
|
|
547
|
+
if any(got.get(k) != c.get(k) for k in ("dims", "n", "cap", "steps_per_second", "python", "counter_sha256")):
|
|
548
|
+
raise certificate.CertificateError("the ladders just written do not match the round's declared counted configuration")
|
|
549
|
+
claim = cert["claim"]
|
|
550
|
+
except (OSError, SystemExit, certificate.CertificateError, KeyError, ValueError, TypeError) as e:
|
|
551
|
+
m["state"], m["reason"] = "failed", "evaluation failed; the operator has the details"
|
|
552
|
+
with open(os.path.join(d, "submissions", m["id"], "eval_error.log"), "w", encoding="utf-8", errors="replace") as f:
|
|
553
|
+
f.write(f"{type(e).__name__}: {str(e)[-300:]}\n")
|
|
554
|
+
return
|
|
555
|
+
m["result"] = {"credit": claim["credit_bits"], "status": claim["status"], "per_seed": claim["per_seed"], "reasons": claim["reasons"]}
|
|
556
|
+
m["state"], m["reason"] = "done", ""
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _evaluate_locked(base, rid, seed_dir, now, runner, only_one, allow_code_change, results_dir=None):
|
|
560
|
+
d = rdir(base, rid)
|
|
561
|
+
with locked(os.path.join(d, ".lock")):
|
|
562
|
+
r = get_round(base, rid)
|
|
563
|
+
if r["state"] == "open":
|
|
564
|
+
raise Rejected("the round is still open; close it first")
|
|
565
|
+
if r["state"] in ("evaluated", "revealed"):
|
|
566
|
+
return 0
|
|
567
|
+
cfile = _read_json(os.path.join(os.path.abspath(os.path.expanduser(seed_dir)), official.COMMITMENTS), {})
|
|
568
|
+
if sorted(cfile.get("commitments", [])) != r["commitments"]:
|
|
569
|
+
raise Rejected("the seed directory's commitments are not this round's commitments")
|
|
570
|
+
now_fp = code_fingerprint(r["family"], r["count"]) if r.get("count") else code_fingerprint(r["family"]) # (one argument for a timed round, as it always was)
|
|
571
|
+
changes = fingerprint_changes(r.get("fingerprint") or {}, now_fp)
|
|
572
|
+
if changes:
|
|
573
|
+
names = ", ".join(sorted(changes))
|
|
574
|
+
if not allow_code_change:
|
|
575
|
+
raise Rejected(f"REFUSED: the rules changed since the round was created ({names}). Evaluating would score submissions under "
|
|
576
|
+
"rules they were not submitted under. Create a new round, or pass --accept-code-change to proceed with a public log entry.")
|
|
577
|
+
already = {k for e in read_log(d) if e["kind"] == "code_changed" for k in e["data"]["changed"]}
|
|
578
|
+
if not set(changes) <= already:
|
|
579
|
+
append_log(d, "code_changed", {"changed": sorted(changes), "old": {k: v[0] for k, v in changes.items()},
|
|
580
|
+
"new": {k: v[1] for k, v in changes.items()}}, now)
|
|
581
|
+
if r["state"] == "closed":
|
|
582
|
+
_set_state(d, r, "evaluating", now, "evaluation_started")
|
|
583
|
+
done = 0
|
|
584
|
+
logged = {sid: e["data"]["sha256"] for sid, e in _logged_submissions(d).items()}
|
|
585
|
+
for m in list_submissions(base, rid):
|
|
586
|
+
if m["state"] not in ("queued", "evaluating"): # "evaluating" here means a previous worker died mid-run: we hold the only lock, so retry it
|
|
587
|
+
continue
|
|
588
|
+
path = os.path.join(d, "submissions", m["id"], m["file"])
|
|
589
|
+
# The log commits to this file's hash. Run the file ONLY if it still matches: a file swapped after acceptance (by anyone with
|
|
590
|
+
# write access to the data directory) must never be evaluated while the log still verifies.
|
|
591
|
+
try:
|
|
592
|
+
with open(path, "rb") as f:
|
|
593
|
+
actual = sha256(f.read())
|
|
594
|
+
except OSError:
|
|
595
|
+
actual = None
|
|
596
|
+
if actual is None or actual != m["sha256"] or logged.get(m["id"]) != m["sha256"]:
|
|
597
|
+
m["state"], m["reason"] = "failed", "the stored file does not match its logged hash; it was not run"
|
|
598
|
+
with open(os.path.join(d, "submissions", m["id"], "eval_error.log"), "w", encoding="utf-8", errors="replace") as f:
|
|
599
|
+
f.write(f"the stored file does not match its logged hash: stored file {actual}; submission record {m['sha256']}; log {logged.get(m['id'])}\n")
|
|
600
|
+
_save_meta(base, rid, m)
|
|
601
|
+
done += 1
|
|
602
|
+
continue
|
|
603
|
+
m["state"] = "evaluating"
|
|
604
|
+
_save_meta(base, rid, m)
|
|
605
|
+
if r.get("cost_model") == "count":
|
|
606
|
+
_evaluate_counted_submission(base, rid, r, m, path, seed_dir, results_dir, now)
|
|
607
|
+
_save_meta(base, rid, m)
|
|
608
|
+
done += 1
|
|
609
|
+
if only_one:
|
|
610
|
+
break
|
|
611
|
+
continue
|
|
612
|
+
plan = official.plan_cost(r["family"], len(r["commitments"]), n=r["n"], budget=r["budget"])
|
|
613
|
+
try:
|
|
614
|
+
p = runner([sys.executable, os.path.join(HERE, "official.py"), "evaluate", "--family", r["family"], "--submission", path,
|
|
615
|
+
"--seed-dir", seed_dir, "--sandbox", "docker", "--reuse-baseline", "--n", str(r["n"])], cwd=HERE, capture_output=True, text=True,
|
|
616
|
+
timeout=plan["worst_case_cpu_hours"] * 3600 * 1.5 + 600)
|
|
617
|
+
rc, tail = p.returncode, (p.stdout + p.stderr)[-300:]
|
|
618
|
+
except subprocess.TimeoutExpired:
|
|
619
|
+
rc, tail = 1, "evaluation exceeded its time allowance"
|
|
620
|
+
if rc == 0:
|
|
621
|
+
board = _read_json(judge.BOARD, [])
|
|
622
|
+
stem = os.path.splitext(m["file"])[0]
|
|
623
|
+
row = next((x for x in judge.leaderboard_rows(board) if x["name"] == stem and x["family"] == r["family"]), None)
|
|
624
|
+
m["result"] = ({"credit": row["credit"], "status": row["status"], "per_seed": row["per_seed"], "reasons": row["reasons"]}
|
|
625
|
+
if row else None)
|
|
626
|
+
m["state"] = "done" if row else "failed"
|
|
627
|
+
m["reason"] = "" if row else "evaluation finished but produced no board entry"
|
|
628
|
+
else:
|
|
629
|
+
# The raw output can contain the operator's seed paths or seed text (a traceback), and `reason` is shown publicly once the
|
|
630
|
+
# round is evaluated, before the reveal. Keep the output in a file the export never touches; publish only a generic reason.
|
|
631
|
+
m["state"], m["reason"] = "failed", "evaluation failed; the operator has the details"
|
|
632
|
+
with open(os.path.join(d, "submissions", m["id"], "eval_error.log"), "w", encoding="utf-8", errors="replace") as f:
|
|
633
|
+
f.write(tail)
|
|
634
|
+
_save_meta(base, rid, m)
|
|
635
|
+
done += 1
|
|
636
|
+
if only_one:
|
|
637
|
+
break
|
|
638
|
+
with locked(os.path.join(d, ".lock")):
|
|
639
|
+
r = get_round(base, rid)
|
|
640
|
+
subs = list_submissions(base, rid)
|
|
641
|
+
if r["state"] == "evaluating" and not any(m["state"] in ("queued", "evaluating") for m in subs):
|
|
642
|
+
results = {m["id"]: m["result"] for m in subs}
|
|
643
|
+
_set_state(d, r, "evaluated", now, "round_evaluated", {"results_sha256": sha256(canonical(results)), "count": len(subs)})
|
|
644
|
+
return done
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def reveal(base, rid, seed_dir, now=None):
|
|
648
|
+
"""Publish the seeds of an EVALUATED round, only if every one hashes to a commitment the round published at creation."""
|
|
649
|
+
now = now or time.time()
|
|
650
|
+
_require(base, rid)
|
|
651
|
+
d = rdir(base, rid)
|
|
652
|
+
with locked(os.path.join(d, ".lock")):
|
|
653
|
+
r = get_round(base, rid)
|
|
654
|
+
if r["state"] == "revealed":
|
|
655
|
+
return _read_json(os.path.join(d, "reveal.json"))
|
|
656
|
+
if r["state"] != "evaluated":
|
|
657
|
+
raise Rejected("seeds are revealed only after every submission has been evaluated")
|
|
658
|
+
try:
|
|
659
|
+
pairs = official.verified_seeds(seed_dir)
|
|
660
|
+
except SystemExit as e:
|
|
661
|
+
raise Rejected(str(e))
|
|
662
|
+
seeds = {}
|
|
663
|
+
for path, h in pairs:
|
|
664
|
+
with open(path, "rb") as f:
|
|
665
|
+
seeds[h] = f.read().strip().decode()
|
|
666
|
+
if sorted(seeds) != r["commitments"]:
|
|
667
|
+
raise Rejected("the seed files do not match this round's commitments one for one")
|
|
668
|
+
_write_json(os.path.join(d, "reveal.json"), seeds)
|
|
669
|
+
_set_state(d, r, "revealed", now, "seeds_revealed", {"seeds": seeds})
|
|
670
|
+
return seeds
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
# ---------------------------------------------------------------- public export (what the web tier and the static site show)
|
|
674
|
+
def public_submission(base, rid, m, r):
|
|
675
|
+
out = {"id": m["id"], "handle": m["handle"], "sha256": m["sha256"], "submitted_at": m["submitted_at"], "seq": m["seq"],
|
|
676
|
+
"log_hash": m["log_hash"], "state": m["state"],
|
|
677
|
+
# the static-check warnings quote line numbers and the modules and calls a solver uses: for a solver its owner keeps private they are shown to its
|
|
678
|
+
# owner when they submit (the receipt) and are not published
|
|
679
|
+
"warnings": m["warnings"] if m["publish_source"] else []}
|
|
680
|
+
if r["state"] in ("evaluated", "revealed"):
|
|
681
|
+
out["result"], out["reason"] = m.get("result"), m.get("reason", "")
|
|
682
|
+
if r["state"] == "revealed" and _public_cert_is_safe(base, rid, m):
|
|
683
|
+
# the public certificate is always downloadable; it carries the solver's source only if the submitter opted in (otherwise it is a private-solver
|
|
684
|
+
# copy and the full one, with the source, needs the handle's token)
|
|
685
|
+
out["certificate"] = {"path": f"/api/cert/{rid}/{m['id']}.json", "private": not m["publish_source"]}
|
|
686
|
+
if not m["publish_source"]:
|
|
687
|
+
out["certificate"]["full_path"] = f"/api/cert-full/{rid}/{m['id']}.json"
|
|
688
|
+
if r["state"] == "revealed" and m["publish_source"]:
|
|
689
|
+
with open(os.path.join(rdir(base, rid), "submissions", m["id"], m["file"]), "rb") as f:
|
|
690
|
+
raw = f.read()
|
|
691
|
+
if sha256(raw) == m["sha256"]: # publish the source only if it is still exactly what the log committed to
|
|
692
|
+
out["source"] = raw.decode("utf-8", "replace")
|
|
693
|
+
else:
|
|
694
|
+
out["source_note"] = "the stored file does not match its logged hash, so the source is not published"
|
|
695
|
+
return out
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def public_round(base, rid):
|
|
699
|
+
r = get_round(base, rid)
|
|
700
|
+
ok, n, head, err = verify_log(rdir(base, rid), r["commitments"])
|
|
701
|
+
out = {k: r[k] for k in ("id", "title", "family", "commitments", "opens_at", "closes_at", "state", "n", "budget")}
|
|
702
|
+
out["max_submissions"] = r.get("max_submissions", MAX_PER_ROUND)
|
|
703
|
+
out["allow_private"] = r.get("allow_private", True)
|
|
704
|
+
out["cost_model"] = r.get("cost_model", "time")
|
|
705
|
+
out["count"] = r.get("count")
|
|
706
|
+
out["fingerprint"] = r.get("fingerprint")
|
|
707
|
+
out["consistency_problems"] = check_consistency(base, rid) # cheap mode: compares records, never reads the source files
|
|
708
|
+
out.update({"log_ok": ok, "log_entries": n, "log_head": head, "log_error": err,
|
|
709
|
+
"submissions": [public_submission(base, rid, m, r) for m in list_submissions(base, rid)]})
|
|
710
|
+
if r["state"] == "revealed":
|
|
711
|
+
out["seeds"] = _read_json(os.path.join(rdir(base, rid), "reveal.json"), {})
|
|
712
|
+
return out
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def export_public(base, out_dir):
|
|
716
|
+
"""Write rounds.json, round_<id>.json and each round's log.jsonl for the static site and for third-party verification."""
|
|
717
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
718
|
+
index = []
|
|
719
|
+
for r in list_rounds(base):
|
|
720
|
+
pr = public_round(base, r["id"])
|
|
721
|
+
_write_json(os.path.join(out_dir, f"round_{r['id']}.json"), pr)
|
|
722
|
+
# copyfile, not copy: copy would carry over the source's group-only mode (0660) and the web server could not read the export
|
|
723
|
+
shutil.copyfile(os.path.join(rdir(base, r["id"]), "log.jsonl"), os.path.join(out_dir, f"log_{r['id']}.jsonl"))
|
|
724
|
+
index.append({k: pr[k] for k in ("id", "title", "family", "commitments", "opens_at", "closes_at", "state", "log_ok", "log_head")}
|
|
725
|
+
| {"submissions": len(pr["submissions"])})
|
|
726
|
+
if pr["state"] == "revealed": # the PUBLIC certificates: full ones (source published by its owner) and private-solver copies (no source). Never `.full.`
|
|
727
|
+
for m in list_submissions(base, r["id"]):
|
|
728
|
+
src = cert_file(base, r["id"], m["id"])
|
|
729
|
+
if _public_cert_is_safe(base, r["id"], m):
|
|
730
|
+
os.makedirs(os.path.join(out_dir, f"certs_{r['id']}"), exist_ok=True)
|
|
731
|
+
shutil.copyfile(src, os.path.join(out_dir, f"certs_{r['id']}", f"{m['id']}.cert.json"))
|
|
732
|
+
_write_json(os.path.join(out_dir, "rounds.json"), index)
|
|
733
|
+
return index
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
# ---------------------------------------------------------------- certificates (operator builds them after the reveal; submitters download them)
|
|
737
|
+
def cert_file(base, rid, sid, full=False):
|
|
738
|
+
"""The PUBLIC certificate of a submission (`full` False): the full one when the owner published their source, otherwise a private-solver copy with
|
|
739
|
+
no source and no failure text. `full` True: the full certificate of a submission whose source was NOT published (token-gated); it exists only then."""
|
|
740
|
+
return os.path.join(rdir(base, rid), "certs", f"{sid}.full.cert.json" if full else f"{sid}.cert.json")
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _public_cert_is_safe(base, rid, m):
|
|
744
|
+
"""May this submission's PUBLIC certificate be served? Only if it exists and does not leak a solver its owner kept private: for a submission whose
|
|
745
|
+
source was not published it must actually be a private-solver certificate (marked so, with no source). Checked on every release, not just when it is
|
|
746
|
+
built, so a stale file from an older version or a mistake by the operator fails CLOSED instead of publishing someone's algorithm."""
|
|
747
|
+
path = cert_file(base, rid, m["id"])
|
|
748
|
+
try:
|
|
749
|
+
st = os.stat(path)
|
|
750
|
+
except OSError:
|
|
751
|
+
return False
|
|
752
|
+
if m["publish_source"]:
|
|
753
|
+
return True
|
|
754
|
+
key = (path, st.st_mtime_ns, st.st_size) # the round description is polled: parse a certificate once per version of the FILE, not once per request
|
|
755
|
+
if key in _SAFE_CACHE:
|
|
756
|
+
return _SAFE_CACHE[key]
|
|
757
|
+
try:
|
|
758
|
+
with open(path, encoding="utf-8") as f:
|
|
759
|
+
c = json.load(f)
|
|
760
|
+
ok = isinstance(c, dict) and c.get("private") is True and isinstance(c.get("submission"), dict) and c["submission"].get("source") is None
|
|
761
|
+
except (OSError, ValueError):
|
|
762
|
+
ok = False
|
|
763
|
+
if len(_SAFE_CACHE) > 4096:
|
|
764
|
+
_SAFE_CACHE.clear()
|
|
765
|
+
_SAFE_CACHE[key] = ok
|
|
766
|
+
return ok
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
_SAFE_CACHE = {}
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
def _store_cert(dest, cert):
|
|
773
|
+
tmp = dest + ".tmp"
|
|
774
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
775
|
+
json.dump(cert, f, indent=1, sort_keys=True)
|
|
776
|
+
os.chmod(tmp, 0o644) # served by the web tier's user
|
|
777
|
+
os.replace(tmp, dest)
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def build_certificates(base, rid, seed_dir, results_dir=None):
|
|
781
|
+
"""Operator side. After the reveal, build the certificates of every evaluated submission and store them beside the round. A certificate is stored
|
|
782
|
+
only if it verifies; one that does not is reported, never published. Existing certificates are left alone.
|
|
783
|
+
A submission whose owner published their source gets one public full certificate. One who did not gets TWO: a public private-solver copy (no source,
|
|
784
|
+
no failure text: anyone can verify everything except re-running it) and the full certificate, served only to the owner.
|
|
785
|
+
-> {"built": [submission ids], "skipped": {submission id: reason}}"""
|
|
786
|
+
import certificate # certificate imports rounds, so this cannot be a module-level import
|
|
787
|
+
_require(base, rid)
|
|
788
|
+
r = get_round(base, rid)
|
|
789
|
+
if r["state"] != "revealed":
|
|
790
|
+
raise Rejected(f"round {rid} is {r['state']!r}: certificates carry the seeds, so they are built only after the reveal")
|
|
791
|
+
out = {"built": [], "skipped": {}}
|
|
792
|
+
cdir = os.path.join(rdir(base, rid), "certs")
|
|
793
|
+
os.makedirs(cdir, exist_ok=True)
|
|
794
|
+
for m in list_submissions(base, rid):
|
|
795
|
+
sid, dest = m["id"], cert_file(base, rid, m["id"])
|
|
796
|
+
if m["state"] != "done" or not m.get("result"):
|
|
797
|
+
out["skipped"][sid] = f"not evaluated successfully (state {m['state']})"
|
|
798
|
+
continue
|
|
799
|
+
if os.path.exists(dest) and _public_cert_is_safe(base, rid, m):
|
|
800
|
+
out["skipped"][sid] = "already built"
|
|
801
|
+
continue # (an existing public file that would leak a private solver is NOT "built": it is rebuilt, replacing it)
|
|
802
|
+
path = os.path.join(rdir(base, rid), "submissions", sid, m["file"])
|
|
803
|
+
try:
|
|
804
|
+
with open(path, "rb") as f:
|
|
805
|
+
if sha256(f.read()) != m["sha256"]:
|
|
806
|
+
out["skipped"][sid] = "the stored file does not match its logged hash"
|
|
807
|
+
continue
|
|
808
|
+
cert = certificate.build(r["family"], path, seed_dir, reveal_seeds=True, results_dir=results_dir, round_data=base, round_id=rid)
|
|
809
|
+
public = cert if m["publish_source"] else certificate.make_private(cert)
|
|
810
|
+
bad = []
|
|
811
|
+
for label, c in (("full", cert), ("public", public)) if public is not cert else (("full", cert),):
|
|
812
|
+
res = certificate.verify(c)
|
|
813
|
+
if not res["ok"]:
|
|
814
|
+
bad.append(f"the {label} certificate does not verify: " + "; ".join(x["check"] for x in res["checks"] if x["status"] == "fail"))
|
|
815
|
+
if bad:
|
|
816
|
+
out["skipped"][sid] = "; ".join(bad)
|
|
817
|
+
continue
|
|
818
|
+
except (certificate.CertificateError, OSError, SystemExit, KeyError, ValueError, TypeError) as e:
|
|
819
|
+
out["skipped"][sid] = f"could not be built: {str(e)[:200]}"
|
|
820
|
+
continue
|
|
821
|
+
if public is not cert:
|
|
822
|
+
_store_cert(cert_file(base, rid, sid, full=True), cert) # first: the public copy's presence is what marks the submission as done
|
|
823
|
+
_store_cert(dest, public)
|
|
824
|
+
out["built"].append(sid)
|
|
825
|
+
return out
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
def cert_for_download(base, rid, sid, token=None, full=False):
|
|
829
|
+
"""The path of a submission's certificate if the caller may have it. `full` False: the PUBLIC certificate, for anyone (no source unless the owner
|
|
830
|
+
published it). `full` True: the full certificate with the solver's source: anyone when the owner published it, otherwise only with the handle's token.
|
|
831
|
+
KeyError: no such certificate. Rejected: not allowed."""
|
|
832
|
+
m = get_submission(base, rid, sid)
|
|
833
|
+
public = cert_file(base, rid, sid)
|
|
834
|
+
if not os.path.exists(public):
|
|
835
|
+
raise KeyError(sid)
|
|
836
|
+
if not full or m["publish_source"]:
|
|
837
|
+
if not _public_cert_is_safe(base, rid, m):
|
|
838
|
+
raise Rejected("this certificate is not ready for public release (it would show a solver its owner kept private): the operator must rebuild it with ops.py certs")
|
|
839
|
+
return public
|
|
840
|
+
stored = _read_json(os.path.join(rdir(base, rid), "handles.json"), {}).get(m["handle"].lower())
|
|
841
|
+
if token and stored and secrets.compare_digest(sha256(token), stored):
|
|
842
|
+
path = cert_file(base, rid, sid, full=True)
|
|
843
|
+
if os.path.exists(path):
|
|
844
|
+
return path
|
|
845
|
+
raise KeyError(sid)
|
|
846
|
+
raise Rejected("the full certificate contains the submitter's solver source, which was not published: it is available only with the handle's token")
|