guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,750 @@
|
|
|
1
|
+
"""Export a compiled domain as a platform-neutral CHALLENGE BUNDLE: everything another leaderboard needs to host the challenge and to say how
|
|
2
|
+
much of a measured speedup is real, with no copy of this repository.
|
|
3
|
+
|
|
4
|
+
python3 export_bundle.py export NAME [--out DIR] [--field FIELD] [--no-selftest] (also: compile_domain.py export NAME)
|
|
5
|
+
python3 export_bundle.py verify DIR [--selftest] re-check a bundle's hashes (and run its self-test)
|
|
6
|
+
|
|
7
|
+
A bundle holds the declaration (derive/verify), the reference solver, the compiled ladder and power analysis, a standalone checker
|
|
8
|
+
(`checker/check.py`: derive instances, verify answers, self-test), a standalone scorer (`checker/score.py`: the protocol's difficulty-stratified
|
|
9
|
+
credit, and a head-to-head `compare` of two solvers), a byte copy of the estimator, a manifest of every file's sha256, a README, and the
|
|
10
|
+
intake text a platform like Yukon asks a proposer for (PROPOSAL.md). Export refuses a stale compile, and refuses to emit a bundle that fails
|
|
11
|
+
its own self-test in a clean interpreter (`python3 -I`, empty directory): a bundle that only works inside this checkout is not a bundle.
|
|
12
|
+
|
|
13
|
+
Export executes the declaration and the reference solver (the self-test does), unsandboxed, like `compile`: export only declarations you
|
|
14
|
+
have read. It never overwrites an existing directory.
|
|
15
|
+
"""
|
|
16
|
+
import argparse
|
|
17
|
+
import ast
|
|
18
|
+
import hashlib
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import shutil
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
|
|
27
|
+
import declared
|
|
28
|
+
|
|
29
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
30
|
+
TEMPLATES = os.path.join(HERE, "bundle_templates")
|
|
31
|
+
FORMAT = "guldo-challenge-bundle-v1"
|
|
32
|
+
RESERVED = {"decl", "reference", "estimator", "check", "score", "challenge", "quality_core", "qcheck", "qscore"} # module names the bundle itself uses
|
|
33
|
+
MAX_VENDORED = 12
|
|
34
|
+
SELFTEST_TIMEOUT = 900
|
|
35
|
+
STD = set(sys.stdlib_module_names)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ExportError(Exception):
|
|
39
|
+
"""Nothing was written; the message says what to fix."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def sha256_file(path):
|
|
43
|
+
return declared.sha256_file(path)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def imports_of(path):
|
|
47
|
+
with open(path, encoding="utf-8") as f:
|
|
48
|
+
tree = ast.parse(f.read(), path)
|
|
49
|
+
names = set()
|
|
50
|
+
for node in ast.walk(tree):
|
|
51
|
+
if isinstance(node, ast.Import):
|
|
52
|
+
names.update(a.name.split(".")[0] for a in node.names)
|
|
53
|
+
elif isinstance(node, ast.ImportFrom):
|
|
54
|
+
if node.level:
|
|
55
|
+
raise ExportError(f"{os.path.basename(path)} uses a relative import; a bundle cannot resolve it")
|
|
56
|
+
if node.module:
|
|
57
|
+
names.add(node.module.split(".")[0])
|
|
58
|
+
return names
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def resolve_imports(entry, search_dirs, seen=None, vendored=None, third=None):
|
|
62
|
+
"""Walk the imports of the declaration. A module that lives beside it or in the project is VENDORED (copied byte for byte); anything
|
|
63
|
+
else that is not standard library is a third-party requirement the bundle names but cannot ship. Imports made through importlib with a
|
|
64
|
+
string are invisible here: the clean-interpreter self-test is what catches those."""
|
|
65
|
+
seen = set() if seen is None else seen
|
|
66
|
+
vendored = {} if vendored is None else vendored
|
|
67
|
+
third = set() if third is None else third
|
|
68
|
+
for name in sorted(imports_of(entry)):
|
|
69
|
+
if name in STD or name in seen:
|
|
70
|
+
continue
|
|
71
|
+
seen.add(name)
|
|
72
|
+
src = next((os.path.join(d, name + ".py") for d in search_dirs if os.path.isfile(os.path.join(d, name + ".py"))), None)
|
|
73
|
+
if src is None:
|
|
74
|
+
third.add(name)
|
|
75
|
+
continue
|
|
76
|
+
if name in RESERVED:
|
|
77
|
+
raise ExportError(f"the declaration imports a module called {name!r}, which the bundle uses for itself; rename it")
|
|
78
|
+
vendored[name] = src
|
|
79
|
+
if len(vendored) > MAX_VENDORED:
|
|
80
|
+
raise ExportError(f"the declaration pulls in more than {MAX_VENDORED} project modules ({sorted(vendored)}...): it is not portable as written")
|
|
81
|
+
resolve_imports(src, search_dirs, seen, vendored, third)
|
|
82
|
+
return vendored, third
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
PROBE = r'''
|
|
86
|
+
import importlib.util, json, os, sys
|
|
87
|
+
d, here, sizes = sys.argv[1], sys.argv[2], [int(x) for x in sys.argv[3:]]
|
|
88
|
+
opened = set()
|
|
89
|
+
def hook(event, args):
|
|
90
|
+
if event == "open" and args and isinstance(args[0], (str, bytes, os.PathLike)):
|
|
91
|
+
opened.add(os.fsdecode(args[0]))
|
|
92
|
+
sys.addaudithook(hook)
|
|
93
|
+
sys.path[:0] = [d, here]
|
|
94
|
+
spec = importlib.util.spec_from_file_location("probe_decl", os.path.join(d, "decl.py"))
|
|
95
|
+
decl = importlib.util.module_from_spec(spec)
|
|
96
|
+
spec.loader.exec_module(decl)
|
|
97
|
+
for size in sizes:
|
|
98
|
+
for idx in range(3):
|
|
99
|
+
inst = dict(decl.derive(b"practice-seed-public", size, idx))
|
|
100
|
+
for ans in [decl.witness(inst) if hasattr(decl, "witness") else inst["planted"], None, 0, [], "x"]:
|
|
101
|
+
try:
|
|
102
|
+
decl.verify(inst, ans)
|
|
103
|
+
except Exception:
|
|
104
|
+
pass
|
|
105
|
+
rd, rh = os.path.realpath(d), os.path.realpath(here)
|
|
106
|
+
found, data, foreign = {}, set(), set()
|
|
107
|
+
for name, mod in list(sys.modules.items()):
|
|
108
|
+
f = getattr(mod, "__file__", None)
|
|
109
|
+
if f and "." not in name and name != "probe_decl":
|
|
110
|
+
rp = os.path.realpath(f)
|
|
111
|
+
if rp == os.path.join(rd, name + ".py") or rp == os.path.join(rh, name + ".py"):
|
|
112
|
+
found[name] = rp
|
|
113
|
+
for p in opened:
|
|
114
|
+
if not os.path.isabs(p):
|
|
115
|
+
continue
|
|
116
|
+
rp = os.path.realpath(p)
|
|
117
|
+
if not os.path.isfile(rp) or rp.endswith((".py", ".pyc")) or "__pycache__" in rp:
|
|
118
|
+
continue
|
|
119
|
+
if rp.startswith(rd + os.sep):
|
|
120
|
+
data.add(os.path.relpath(rp, rd))
|
|
121
|
+
elif rp.startswith(rh + os.sep):
|
|
122
|
+
foreign.add(rp)
|
|
123
|
+
print(json.dumps({"modules": found, "data": sorted(data), "foreign": sorted(foreign)}))
|
|
124
|
+
'''
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def probe_imports(domain_dir, sizes):
|
|
128
|
+
"""Run the declaration (import, then derive and verify several practice instances at every practice size, the witness and a few junk answers) and
|
|
129
|
+
report what it ACTUALLY touched: project or sibling modules it loaded (importlib.import_module("x"), __import__, imports inside functions that
|
|
130
|
+
run), and data files it opened (an audit hook sees every open()). The static import walk cannot see either. What this still misses: a code path
|
|
131
|
+
none of those calls takes. The clean-interpreter self-test is the backstop."""
|
|
132
|
+
with tempfile.TemporaryDirectory() as empty:
|
|
133
|
+
try:
|
|
134
|
+
r = subprocess.run([sys.executable, "-B", "-c", PROBE, domain_dir, HERE] + [str(s) for s in sizes], cwd=empty, capture_output=True, text=True,
|
|
135
|
+
timeout=SELFTEST_TIMEOUT)
|
|
136
|
+
except subprocess.TimeoutExpired:
|
|
137
|
+
raise ExportError("the declaration did not finish deriving its practice instances while probing it")
|
|
138
|
+
if r.returncode != 0:
|
|
139
|
+
tail = "\n".join(r.stderr.strip().splitlines()[-8:])
|
|
140
|
+
hint = ("\n(a data file must be opened by a path relative to the declaration itself, e.g. os.path.join(os.path.dirname(__file__), 'x.json'), "
|
|
141
|
+
"not relative to the working directory)" if "FileNotFoundError" in tail else "")
|
|
142
|
+
raise ExportError("the declaration failed while probing it:\n" + tail + hint)
|
|
143
|
+
return json.loads(r.stdout.strip().splitlines()[-1])
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def simulation_block(compiled):
|
|
147
|
+
"""What `score.py` needs to re-run the power simulation at a different noise or threshold: the compiled profile."""
|
|
148
|
+
plan = compiled.get("plan") or {}
|
|
149
|
+
try:
|
|
150
|
+
return {"cap": plan["cap"], "sizes": [int(s) for s in plan["sizes"]], "median_log2": {str(k): v for k, v in plan["median_log2"].items()}, "sigma": plan["sigma"]}
|
|
151
|
+
except (KeyError, TypeError, ValueError, AttributeError):
|
|
152
|
+
return None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def read_template(name):
|
|
156
|
+
with open(os.path.join(TEMPLATES, name), "rb") as f:
|
|
157
|
+
return f.read()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def write_bytes(path, data, mode=0o644):
|
|
161
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
162
|
+
with open(path, "wb") as f:
|
|
163
|
+
f.write(data)
|
|
164
|
+
os.chmod(path, mode)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def copy_exact(src, dst):
|
|
168
|
+
with open(src, "rb") as f:
|
|
169
|
+
write_bytes(dst, f.read())
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def power_block(compiled):
|
|
173
|
+
cp = compiled.get("power") or {}
|
|
174
|
+
return {
|
|
175
|
+
"threshold_bits": cp.get("threshold", 1.0),
|
|
176
|
+
"target_power": cp.get("target_power", 0.8),
|
|
177
|
+
"target_effect_bits": cp.get("target_effect"),
|
|
178
|
+
"pair_noise_bits": cp.get("pair_noise"),
|
|
179
|
+
"min_certifiable_bits": cp.get("min_certifiable_effect"),
|
|
180
|
+
"max_certifiable_bits": cp.get("max_certifiable_effect"),
|
|
181
|
+
"curve": {str(e): {"single": v["single"], "all_seeds": v["all_three"], "inconclusive": v.get("inconclusive")}
|
|
182
|
+
for e, v in (cp.get("curve") or {}).items()},
|
|
183
|
+
"notes": list(cp.get("notes") or []),
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def status_of(compiled, power):
|
|
188
|
+
reasons = []
|
|
189
|
+
st = (compiled.get("selftest") or {}).get("status", "pending")
|
|
190
|
+
if st != "pass":
|
|
191
|
+
reasons.append(f"the compiler's own self-test is {st!r}, not 'pass'")
|
|
192
|
+
if power["min_certifiable_bits"] is None:
|
|
193
|
+
reasons.append("the power analysis found no speedup this ladder can certify")
|
|
194
|
+
if not compiled.get("reference_note"):
|
|
195
|
+
reasons.append("the declaration states no REFERENCE_NOTE: nobody can tell how strong the baseline is")
|
|
196
|
+
if not compiled.get("plan", {}).get("confirmed", True):
|
|
197
|
+
reasons.append("the ladder's predicted costs were not confirmed by measurement")
|
|
198
|
+
return ("READY" if not reasons else "DRAFT"), reasons
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def table(power):
|
|
202
|
+
rows = ["| true speedup (bits) | P(one seed credited) | P(all seeds credited) |", "|---|---|---|"]
|
|
203
|
+
rows += [f"| {e} | {100 * v['single']:.0f}% | {100 * v['all_seeds']:.0f}% |" for e, v in power["curve"].items()]
|
|
204
|
+
return "\n".join(rows)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def readme(ch):
|
|
208
|
+
pw, lad, name = ch["power"], ch["ladder"], ch["name"]
|
|
209
|
+
rng = (f"from about {pw['min_certifiable_bits']} to about {pw['max_certifiable_bits']} bits" if pw["min_certifiable_bits"] is not None
|
|
210
|
+
else "NONE: this challenge is too weak to certify anything")
|
|
211
|
+
reasons = "".join(f"\n* {r}" for r in ch["status_reasons"])
|
|
212
|
+
return f"""# {name} ({ch['version']}): challenge bundle, status {ch['status']}
|
|
213
|
+
{('The bundle is a DRAFT because:' + reasons) if reasons else ''}
|
|
214
|
+
|
|
215
|
+
{ch['problem']['doc']}
|
|
216
|
+
|
|
217
|
+
* solver: {ch['solver_contract']}
|
|
218
|
+
* sizes {lad['dims']}; {lad['n']} instances per size; CPU cap {lad['budget_seconds']} s per instance; credit needs {ch['protocol']['seeds_required']} committed seeds
|
|
219
|
+
* reference: {ch['reference']['note'] or 'NOT STATED by the declaration author'}
|
|
220
|
+
* this ladder can certify speedups {rng} (simulated through the real estimator; assumed per-instance pair noise {pw['pair_noise_bits']} bit)
|
|
221
|
+
|
|
222
|
+
{table(pw)}
|
|
223
|
+
|
|
224
|
+
## What is in here
|
|
225
|
+
|
|
226
|
+
| path | what |
|
|
227
|
+
|---|---|
|
|
228
|
+
| `challenge.json` | the manifest: problem, ladder, protocol, power, provenance, requirements, and the sha256 of every pinned file |
|
|
229
|
+
| `checker/decl.py`, `checker/reference.py` | the declaration (instances are a pure function of seed, size, index; `verify` is exact) and the baseline, byte-for-byte what was compiled |
|
|
230
|
+
| `checker/check.py` | `info`, `selftest`, `commit`, `derive`, `verify`, `rerun` (sandboxed re-measurement of a claimed result; `--repeat`, `--save`), `spread` (merge several machines' saved measurements into a tolerance), `keygen`, `attest-check`, `log append|verify|checkpoint`: the operator's instance generator and answer checker, a verifier's tools, and signed attestations. `selftest`, `derive` and `verify` run sandboxed by default. Standard library only. `selftest` tests the verifier on a fresh random seed (printed), including answers derived from the instance itself and random answers of the witness's shape |
|
|
231
|
+
| `checker/score.py` | `score` (one solver against the reference), `compare` (is solver A really faster than solver B?), `power` (re-simulable), `recheck`. Needs numpy |
|
|
232
|
+
| `checker/estimator.py` | byte copy of the protocol's estimator (`extrapolate.py`); its hash is in the manifest |
|
|
233
|
+
| `checker/vendor/` | project modules the declaration imports, copied byte-for-byte (empty if none) |
|
|
234
|
+
| `compiled.json` | the compiler's full record, for audit |
|
|
235
|
+
| `MANIFEST.sha256` | `sha256sum -c MANIFEST.sha256` (macOS: `shasum -a 256 -c MANIFEST.sha256`), run in this directory |
|
|
236
|
+
| `PROPOSAL.md` | the text a platform's proposal form asks for, pre-filled from this record |
|
|
237
|
+
|
|
238
|
+
## Before you run anything here
|
|
239
|
+
|
|
240
|
+
`checker/check.py derive|verify|selftest` import and run `checker/decl.py`, and `selftest` runs `checker/reference.py`. That is the author's code, and the manifest
|
|
241
|
+
hashes prove only that the files are the ones listed, not that they are safe. All three therefore re-run themselves INSIDE an OS sandbox by default (macOS
|
|
242
|
+
`sandbox-exec`, tested; Linux `bwrap`, tested on one real machine (Ubuntu 26.04, kernel 7.0, bubblewrap 0.11.1, as root: an attack probe and 63 unit tests passed). On Linux it
|
|
243
|
+
gives no network for any process, a read-only root, a writable scratch directory and masked credential directories, but does NOT restrict which programs may run. Ubuntu
|
|
244
|
+
restricts unprivileged user namespaces (AppArmor), so bwrap may be unusable for a normal user unless installed from the distribution package, which carries the exemption;
|
|
245
|
+
the tools say so when that happens): no network, no writes outside a scratch directory, no running any program except the Python interpreter,
|
|
246
|
+
`~/.ssh`, `~/.aws`, `~/.gnupg` and the keychain unreadable. Where there is no sandbox they warn and run as you; `export_bundle.py verify --selftest` instead refuses unless you pass `--unsandboxed`. A sandbox stops
|
|
247
|
+
the practical damage, not a determined attacker with a kernel exploit, and your own `--sandbox off` runs are as you. `check.py rerun` sandboxes a COMPETITOR's solver the same
|
|
248
|
+
way. (`score.py` runs nothing from the ladders it reads.) Export runs its own self-test sandboxed too, so a declaration that secretly needs to write a file or reach the
|
|
249
|
+
network fails at export, not for the next person. Cost: the sandbox adds about 75 ms to each call (measured on `verify`: 57 ms unsandboxed, 130 ms sandboxed), which is
|
|
250
|
+
nothing for a few hundred verifications per submission; for bulk verification of a bundle you have read, pass `--sandbox off` or run the whole batch inside your own VM.
|
|
251
|
+
|
|
252
|
+
## Hosting it on a board
|
|
253
|
+
|
|
254
|
+
A board needs three things from its own side; the bundle supplies the rest.
|
|
255
|
+
|
|
256
|
+
1. **Seeds.** Pick at least {ch['protocol']['seeds_required']} secret seeds, publish `python3 checker/check.py commit --seed-file S` for each BEFORE the deadline, reveal after. Instances are
|
|
257
|
+
`python3 checker/check.py derive --seed-file S --size N --idx I`; nothing can be precomputed.
|
|
258
|
+
2. **A meter.** Run each submission once per instance, one at a time on one core, charge failures the cap, and write a ladder file: a first line
|
|
259
|
+
`{{"meta": true, "commit": "<the seed's sha256>", "budget": {lad['budget_seconds']}}}`, then one line per instance, ids `SIZE:0` .. `SIZE:{lad['n'] - 1}` for every size:
|
|
260
|
+
`{{"id": "28:0", "dim": 28, "status": "solved", "cost": 3.41}}` (cost > 0; statuses solved, timeout, crash, failed, wrong). Do the same for the reference
|
|
261
|
+
(`checker/reference.py`). Answers are checked with `python3 checker/check.py verify --seed-file S --size N --idx I --answer @answer.json`.
|
|
262
|
+
3. **A scorer run.** `python3 checker/score.py score --pair REF_SEED1.jsonl SUB_SEED1.jsonl --pair ... --pair ... --out result.json` prints the credit and writes JSON.
|
|
263
|
+
`python3 checker/score.py compare --a A1 A2 A3 --b B1 B2 B3` answers "is #1 really ahead of #2, or is that noise?". Anyone can then run
|
|
264
|
+
`python3 checker/score.py recheck result.json`, and a verifier with the revealed seed can run
|
|
265
|
+
`python3 checker/check.py rerun --seed-file S --solver sub.py --claimed SUB_SEED1.jsonl --baseline REF_SEED1.jsonl` on their own machine.
|
|
266
|
+
|
|
267
|
+
Competitors can try the contract with `python3 checker/check.py derive --practice --size {lad['practice_dims'][0]} --idx 0`.
|
|
268
|
+
|
|
269
|
+
## What a score does not show (read before publishing one)
|
|
270
|
+
|
|
271
|
+
* **`score.py` does not know how the costs were measured**, but `check.py rerun` can test them. `rerun` takes the seed, the solver and the two claimed ladders,
|
|
272
|
+
picks a random sample of instances (the verifier's own choice: the operator must not know it in advance), re-runs the reference and the solver on THIS machine,
|
|
273
|
+
checks the answers, and compares the re-measured speedup with the claimed one instance by instance. Costs typed into a file, or a solver slower than claimed, show as a
|
|
274
|
+
gap; a baseline ladder that was not measured on one machine shows as a spread. What it does not show: a meter whose bias is the same on every instance; ladders
|
|
275
|
+
from a different hardware class (it reports the machine factor, which cancels in a ratio only if it is the same at every size); instances you did not sample. The
|
|
276
|
+
default tolerance (2 bits) is loose on purpose, because machine-to-machine spread has been measured only once: `rerun --repeat N` measures THIS machine's own timing noise
|
|
277
|
+
(real dlog claims: 0.01 bit), and `rerun --sample-seed S --save F` on two or more machines followed by `check.py spread F1 F2 ...` turns their disagreement into a recommended
|
|
278
|
+
tolerance (needs 2+ distinct machines and 3+ shared instances; one machine is refused, not faked; never below 1 bit). The one real measurement: an Apple M4 Pro under macOS
|
|
279
|
+
and a Hetzner x86 server under Linux (about 0.5 bit apart in raw speed) agreed on the re-measured speedup to a median 0.01 bit, worst 0.03 bit, over 4 instances of one
|
|
280
|
+
pure-Python domain: encouraging, not a bound (one domain, one solver family; BLAS-heavy work may differ). Under bubblewrap, `rerun` also checks that the CPU meter works at
|
|
281
|
+
all: it refuses if an empty Python start measures ~0, and fails if the reference reads ~0 where its published cost is seconds. It also fails a claim when a run took over
|
|
282
|
+
half a second of wall time (the scorer ignores costs below 0.5 s anyway) but its CPU share is under 60% of the share the reference got on the same instance (30% absolute when the reference ran under 0.5 s): a solver that waits, or that forks its work and returns without waiting (on Linux that child's CPU escapes the kernel's
|
|
283
|
+
accounting; measured: 2.0 s of work read as 0.02 s without the sandbox, and a solver padding its CPU to 34% still faked 1.0 bit against a fixed 30% rule). Judged against the reference, a
|
|
284
|
+
loaded machine does not fail an honest solver, and the most a solver can still hide is about 0.7 bit. `--sandbox off` is for solvers you trust. It runs the solver and the reference in the OS sandbox described above (and additionally hides the seed and
|
|
285
|
+
your signing key from them) and refuses to run at all where there is none unless you pass `--sandbox off`. It is not a proof of isolation: CPU, memory and the kernel are
|
|
286
|
+
shared, and `sandbox-exec` is deprecated by Apple. (On macOS the profile also denies every Mach service lookup except user-name lookups, so a solver using `ctypes` cannot
|
|
287
|
+
talk to LaunchServices, the pasteboard or the preferences daemon: measured, all three were reachable before.)
|
|
288
|
+
Run the meter you trust, on one machine, one instance at a time, the same CPU for the reference and the submission.
|
|
289
|
+
* **The credit is the speedup over THIS reference**, in bits, on the instances the reference finds hard (cost above 0.5 s). A weak reference makes
|
|
290
|
+
big speedups cheap. The declaration author's statement of what the reference is: {ch['reference']['note'] or 'not stated'}
|
|
291
|
+
* **It measures speed, not quality.** The cost is one number per instance (seconds, or any unit that plays the role of seconds against the 0.5 s floor, for
|
|
292
|
+
example counted bytecode steps scaled to seconds). A challenge whose score is accuracy, size or a trade-off is not expressible here; the verifier is
|
|
293
|
+
all-or-nothing per instance.
|
|
294
|
+
* **A speedup beyond the certifiable range cannot be certified**: a very fast solver costs less than the 0.5 s floor on the instances that matter, so
|
|
295
|
+
they stop counting. Above the range the result is inconclusive or flat, not a measure of how fast. `score.py` says so in those words, gives the raw
|
|
296
|
+
median gap as description (never as a bound), and says what it would take: the slower solver must cost about 0.5 s x 2^gap on the hard instances, and if that is
|
|
297
|
+
above the cap, no ladder under this cap can resolve it (raise the cap or use a stronger reference, and recompile).
|
|
298
|
+
* **The power table assumed a per-instance noise; `score.py` measures it.** Each pair's within-bin noise is reported; if it is clearly above the assumed value the
|
|
299
|
+
scorer says the table is optimistic and re-simulates the certifiable range at the observed value. `python3 checker/score.py power --pair-noise X --min-gap G` re-simulates
|
|
300
|
+
on demand from the compiled profile (Monte-Carlo error is printed). It is a simulation of ladders drawn from the compile-time profile, not of your solver.
|
|
301
|
+
* **`compare` resolves only gaps on instances where the slower solver costs at least 0.5 s** (and the faster one does too). It prints its own simulated power at the
|
|
302
|
+
observed noise, including how often a true gap of 0 would be declared. The `--min-gap` threshold itself (default 0.5 bit) is a convention, not a pre-registered test.
|
|
303
|
+
* **A result can be re-checked, not trusted.** `score.py --out` records the sha256 of every input ladder, of this challenge, of the estimator and of the scorer;
|
|
304
|
+
`score.py recheck RESULT.json` recomputes from the ladders and fails on any difference: tamper-evidence for a published number.
|
|
305
|
+
* **Attestations and a log, with their honest limits.** `check.py keygen` makes an Ed25519 key (pure Python, checked against the RFC 8032 vectors); `rerun --sign KEY
|
|
306
|
+
--attest-out F` signs the outcome (sample, per-instance gaps, solver and ladder hashes, machine, sandboxed or not); `attest-check` verifies signatures and counts only
|
|
307
|
+
keys YOU pinned (`--trusted`, obtained out of band; an operator can make five keys) as vouched, while a valid signed dissent from ANY key fails the check.
|
|
308
|
+
`check.py log append|verify` is an append-only hash chain of submissions, results or attestations: it detects an edited, removed or reordered entry. Appending holds a file
|
|
309
|
+
lock from reading the log to writing the entry (POSIX; concurrent appends queue up instead of corrupting the chain, tested with 16 at once; elsewhere it refuses). A truncated copy
|
|
310
|
+
still verifies ALONE, so anyone (the operator, a competitor, a watcher) can `log checkpoint LOG --key K --out C`, a signed (head, count); `log verify LOG --checkpoint C`
|
|
311
|
+
then fails for a log that no longer contains it, whether truncated or forked into a different but internally valid history. Checkpoints only help if someone other
|
|
312
|
+
than the operator keeps them (and, for it to mean independence, you pinned their key). The time in an entry is the logger's clock, not evidence. None of this is a
|
|
313
|
+
trusted clock or a trusted-hardware proof; for claims that must hold against an adversarial operator, the full protocol (guldo certificates, rounds) is still stronger.
|
|
314
|
+
* **Malformed or hostile input is refused with a message, not crashed on**: ladders, result files, attestations, measurements, checkpoints, logs, answers and every numeric
|
|
315
|
+
command-line option (fuzz-tested in the guldo project). Options have bounds (`--boot` 1..5000, `-k` 1..1000, `--repeat` 1..50, `derive`/`verify` sizes inside the
|
|
316
|
+
challenge's declared range, ...), so no option can request an unbounded computation or hand the author's `derive()` a size it was never meant for. Files named on the
|
|
317
|
+
command line must be REGULAR files under a size cap (1 MB for seeds and key lists, 16 MB for JSON, 32 MB for ladders, 64 MB for logs and answers): a FIFO, a device such as
|
|
318
|
+
`/dev/zero` (also behind a symlink), a socket or a directory is refused in one line, because opening a FIFO blocks for ever and a device never ends. A result file that asks
|
|
319
|
+
for an unreasonable bootstrap count or leaves an input ladder unbound is refused up front.
|
|
320
|
+
* **The verifier is tested, not proved.** `check.py selftest` tries a fixed list of junk answers AND answers derived from each instance (its arguments and their
|
|
321
|
+
neighbours, other instances' witnesses, that instances really differ between seeds, witnesses with every part damaged; a witness with one part changed only warns, because a verifier with a
|
|
322
|
+
tolerance accepts it by design, and 100 random answers of the witness's type, length and magnitude per instance) on the public practice seed and on a fresh random seed.
|
|
323
|
+
Reuse checks are rates over many pairs: more than 50% fails and more than 25% warns, because at small sizes instances can share factors and chance hits cluster. So a
|
|
324
|
+
mildly small answer space is a warning; a lookup-table verifier (accepts nearly every other instance's answer) fails. A verifier that accepts only something outside
|
|
325
|
+
all of that still passes.
|
|
326
|
+
* **Not tied to any particular platform.** `PROPOSAL.md` is pre-filled intake text; no platform has reviewed or accepted this bundle.
|
|
327
|
+
|
|
328
|
+
Provenance: compiled {ch['provenance']['compiled_at']}, compiler self-test {ch['provenance']['selftest']}, decl {ch['files']['checker/decl.py'][:16]}..., reference {ch['files']['checker/reference.py'][:16]}...
|
|
329
|
+
"""
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def proposal(ch, field):
|
|
333
|
+
pw, lad = ch["power"], ch["ladder"]
|
|
334
|
+
rng = (f"about {pw['min_certifiable_bits']} to {pw['max_certifiable_bits']} bits" if pw["min_certifiable_bits"] is not None else "none yet")
|
|
335
|
+
return f"""# Proposal text for {ch['name']} (pre-filled; edit before sending)
|
|
336
|
+
|
|
337
|
+
Written from the bundle's own record. Nothing here has been reviewed by any platform. Bundle status: {ch['status']}.
|
|
338
|
+
{('Fix first: ' + '; '.join(ch['status_reasons'])) if ch['status_reasons'] else ''}
|
|
339
|
+
|
|
340
|
+
**Name:** [you]
|
|
341
|
+
**Email:** [you]
|
|
342
|
+
**Institution / affiliation:** [you]
|
|
343
|
+
**Field:** {field}
|
|
344
|
+
|
|
345
|
+
## The Problem
|
|
346
|
+
{ch['problem']['doc']}
|
|
347
|
+
|
|
348
|
+
Goal metric: speedup over a stated reference solver, in bits (log2 of the cost ratio), measured on instances the reference finds hard
|
|
349
|
+
(cost above 0.5 s), reported as a one-sided 95% lower bound and, across {ch['protocol']['seeds_required']} committed seeds, the minimum. The reference is: {ch['reference']['note'] or '[state what the reference is and what is known to beat it]'}
|
|
350
|
+
Why it matters: [one or two sentences about why a faster solver for this problem is worth having].
|
|
351
|
+
|
|
352
|
+
## The Verifier
|
|
353
|
+
* Instances are a pure function of (seed, size, index), derived by the declaration's `derive`; the seed's sha256 is published before the deadline and the seed after, so nothing can be precomputed.
|
|
354
|
+
* `verify(instance, answer)` is exact and rejects junk (checked: None, 0, 1, -1, booleans, empty and nested lists, strings, floats, huge ints).
|
|
355
|
+
* Cost is CPU seconds of the whole process tree, one instance at a time on one core, with a cap of {lad['budget_seconds']} s per instance; a failure is charged the cap.
|
|
356
|
+
* Sizes {lad['dims']}, {lad['n']} instances per size. By simulation through the real estimator this ladder can certify speedups of {rng}; the full power table is in the bundle.
|
|
357
|
+
* A standalone checker, a scorer and the estimator are in the bundle, with a sha256 manifest. `python3 checker/check.py selftest` runs in seconds, on a fresh random seed.
|
|
358
|
+
* Claimed results can be re-measured by a verifier (`check.py rerun`, random sample, own machine) and re-computed by anyone (`score.py recheck`).
|
|
359
|
+
|
|
360
|
+
## Link
|
|
361
|
+
[URL of the published bundle]
|
|
362
|
+
|
|
363
|
+
## Honest limits to state up front
|
|
364
|
+
* The bundle measures speed only, and the credit is only as meaningful as the reference.
|
|
365
|
+
* The scorer takes measured costs as input; it does not itself provide a sandbox or meter (the re-measurement tool samples; it does not replace a meter).
|
|
366
|
+
* Speedups beyond the certifiable range read as inconclusive, with a statement of what cap would be needed to resolve them.
|
|
367
|
+
"""
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def build(arg, out=None, field="Computer Science", selftest=True):
|
|
371
|
+
import compile_domain # lazy: it imports the whole project, and `verify` must not need that
|
|
372
|
+
d = compile_domain._resolve(arg)
|
|
373
|
+
mod = compile_domain._load(d, arg)
|
|
374
|
+
compiled = declared.read_compiled(d)
|
|
375
|
+
problems = declared.freshness_problems(d, mod, compiled)
|
|
376
|
+
if problems:
|
|
377
|
+
raise ExportError("REFUSED: " + "; ".join(problems) + " (compile it again)")
|
|
378
|
+
is_q = compiled.get("kind") == "quality" # answers are scored, not the clock: a different scorer, power analysis and README (see COMPILER.md, "Quality challenges")
|
|
379
|
+
d = os.path.abspath(d)
|
|
380
|
+
ref_path = declared.reference_path(d, mod)
|
|
381
|
+
safe_version = re.sub(r"[^A-Za-z0-9._-]", "_", mod.VERSION) # the author's string becomes a directory name
|
|
382
|
+
out = os.path.abspath(out or os.path.join(d, "out", f"bundle-{safe_version}-{compiled['decl_sha256'][:8]}"))
|
|
383
|
+
if os.path.exists(out):
|
|
384
|
+
raise ExportError(f"{out} exists; this tool never overwrites (choose another --out, or remove it yourself)")
|
|
385
|
+
os.makedirs(os.path.dirname(out), exist_ok=True)
|
|
386
|
+
|
|
387
|
+
vendored, third = resolve_imports(os.path.join(d, declared.DECL), [d, HERE])
|
|
388
|
+
dynamic = {}
|
|
389
|
+
probed = probe_imports(d, compiled["practice_dims"][:3]) # what the walk could not see: imports made at run time, and data files opened
|
|
390
|
+
if probed["foreign"]:
|
|
391
|
+
raise ExportError("the declaration reads project files that cannot travel with a bundle: " + ", ".join(os.path.relpath(p, HERE) for p in probed["foreign"])
|
|
392
|
+
+ ". Put the data beside decl.py (it is then copied), or remove the dependency.")
|
|
393
|
+
data_files = sorted(probed["data"])
|
|
394
|
+
for name, path in probed["modules"].items():
|
|
395
|
+
if name not in vendored:
|
|
396
|
+
if name in RESERVED:
|
|
397
|
+
raise ExportError(f"the declaration loads a module called {name!r} at run time, which the bundle uses for itself; rename it")
|
|
398
|
+
dynamic[name] = path
|
|
399
|
+
for name, path in sorted(dynamic.items()):
|
|
400
|
+
vendored[name] = path
|
|
401
|
+
for path in list(dynamic.values()):
|
|
402
|
+
resolve_imports(path, [d, HERE], set(vendored) | STD, vendored, third)
|
|
403
|
+
ref_vendored, ref_third = resolve_imports(ref_path, [d, HERE])
|
|
404
|
+
if ref_vendored:
|
|
405
|
+
raise ExportError(f"the reference solver imports project modules {sorted(ref_vendored)}; it runs in a sandbox that has only the standard library, "
|
|
406
|
+
"numpy and fpylll, so it must not")
|
|
407
|
+
third |= ref_third
|
|
408
|
+
|
|
409
|
+
stage = tempfile.mkdtemp(prefix=".bundle-stage-", dir=os.path.dirname(out))
|
|
410
|
+
try:
|
|
411
|
+
files = {}
|
|
412
|
+
|
|
413
|
+
def put(rel, src):
|
|
414
|
+
copy_exact(src, os.path.join(stage, rel))
|
|
415
|
+
files[rel] = sha256_file(os.path.join(stage, rel)) # the hash of what was written, not of a source that could change meanwhile
|
|
416
|
+
|
|
417
|
+
put("checker/decl.py", os.path.join(d, declared.DECL))
|
|
418
|
+
put("checker/reference.py", ref_path)
|
|
419
|
+
put("checker/quality_core.py" if is_q else "checker/estimator.py", os.path.join(HERE, "quality_core.py" if is_q else "extrapolate.py"))
|
|
420
|
+
put("compiled.json", os.path.join(d, declared.COMPILED))
|
|
421
|
+
for name, src in sorted(vendored.items()):
|
|
422
|
+
put(f"checker/vendor/{name}.py", src)
|
|
423
|
+
for rel in data_files: # beside decl.py in the domain, beside checker/decl.py in the bundle: a path relative to __file__ keeps working
|
|
424
|
+
if rel in ("compiled.json",) or rel.split(os.sep)[0] in ("out",):
|
|
425
|
+
continue
|
|
426
|
+
put("checker/" + rel.replace(os.sep, "/"), os.path.join(d, rel))
|
|
427
|
+
write_bytes(os.path.join(stage, "checker", "check.py"), read_template("check.py"), 0o755)
|
|
428
|
+
write_bytes(os.path.join(stage, "checker", "score.py"), read_template("score.py"), 0o755)
|
|
429
|
+
if is_q:
|
|
430
|
+
write_bytes(os.path.join(stage, "checker", "qscore.py"), read_template("qscore.py"), 0o755)
|
|
431
|
+
write_bytes(os.path.join(stage, "checker", "qcheck.py"), read_template("qcheck.py"), 0o755)
|
|
432
|
+
|
|
433
|
+
import extrapolate as E
|
|
434
|
+
power = power_block(compiled)
|
|
435
|
+
status, reasons = status_of(compiled, power)
|
|
436
|
+
ch = {
|
|
437
|
+
"format": FORMAT, "name": mod.NAME, "version": mod.VERSION, "status": status, "status_reasons": reasons,
|
|
438
|
+
"problem": {"doc": getattr(mod, "DOC", "(the declaration gives no DOC string)"), "signature": getattr(mod, "SIGNATURE", None)},
|
|
439
|
+
"solver_contract": "one Python file defining a top-level solve(*args) returning an int, a list of ints, or None; deterministic, one process, no files, no network",
|
|
440
|
+
"reference": {"file": "checker/reference.py", "note": compiled.get("reference_note")},
|
|
441
|
+
"ladder": {"dims": compiled["dims"], "practice_dims": compiled["practice_dims"], "size_range": compiled.get("size_range"),
|
|
442
|
+
"size_step": compiled.get("size_step"), "n": compiled["official"]["n"], "budget_seconds": compiled["official"]["budget"]},
|
|
443
|
+
"protocol": {"seeds_required": compiled_seeds(), "commitment": "sha256 of the seed file's bytes (surrounding whitespace stripped)",
|
|
444
|
+
"instances": "a pure function of (seed, size, idx): checker/check.py derive",
|
|
445
|
+
"cost_floor_seconds": E.FLOOR_ABS, "min_instances_per_bin": E.MIN_BIN, "top_bins_pooled": E.TOP_K,
|
|
446
|
+
"credit": "one-sided 95% joint-bootstrap lower bound of the pooled median log2 speedup over the hardest usable difficulty bins; min over seeds; inconclusive = 0"},
|
|
447
|
+
"power": power,
|
|
448
|
+
"provenance": {"compiled_at": compiled.get("compiled_at"), "compiler_sha256": compiled.get("compiler_sha256"),
|
|
449
|
+
"selftest": (compiled.get("selftest") or {}).get("status", "pending"), "cpu_hours": compiled.get("cpu_hours"),
|
|
450
|
+
"exported_with": f"python {sys.version.split()[0]}"},
|
|
451
|
+
"requires": {"third_party": sorted(third), "vendored": sorted(vendored), "vendored_found_only_at_run_time": sorted(dynamic), "data_files": [x for x in files if x.startswith("checker/") and not x.startswith(("checker/vendor/", "checker/decl.py", "checker/reference.py", "checker/estimator.py", "checker/quality_core.py"))],
|
|
452
|
+
"checker": "standard library", "scorer": "numpy"},
|
|
453
|
+
"simulation": simulation_block(compiled),
|
|
454
|
+
"files": dict(sorted(files.items())),
|
|
455
|
+
}
|
|
456
|
+
if is_q:
|
|
457
|
+
ch = quality_challenge(ch, compiled)
|
|
458
|
+
status, reasons = ch["status"], ch["status_reasons"] # what build() returns and prints: the speed record's reasons above do not apply
|
|
459
|
+
write_bytes(os.path.join(stage, "challenge.json"), (json.dumps(ch, indent=1, sort_keys=True) + "\n").encode())
|
|
460
|
+
write_bytes(os.path.join(stage, "README.md"), (readme_quality(ch) if is_q else readme(ch)).encode())
|
|
461
|
+
write_bytes(os.path.join(stage, "PROPOSAL.md"), (proposal_quality(ch, field) if is_q else proposal(ch, field)).encode())
|
|
462
|
+
write_manifest(stage)
|
|
463
|
+
if selftest: # a fixed fresh seed (from the declaration's hash) keeps an export reproducible; `check.py selftest` run later draws a new one each time
|
|
464
|
+
run_selftest(stage, hashlib.sha256((compiled["decl_sha256"] + ":fresh").encode()).hexdigest()[:32])
|
|
465
|
+
os.chmod(stage, 0o755) # mkdtemp makes it 0700; a bundle is meant to be copied and shared
|
|
466
|
+
os.rename(stage, out)
|
|
467
|
+
except BaseException:
|
|
468
|
+
shutil.rmtree(stage, ignore_errors=True)
|
|
469
|
+
raise
|
|
470
|
+
return out, status, reasons
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def quality_challenge(ch, compiled):
|
|
474
|
+
"""The manifest of a QUALITY bundle: `ch` is the speed-shaped record built above; everything that describes speed is replaced."""
|
|
475
|
+
import compile_quality
|
|
476
|
+
qc = compiled["quality"]
|
|
477
|
+
pw = qc["power"]
|
|
478
|
+
reasons = []
|
|
479
|
+
st = (compiled.get("selftest") or {}).get("status", "pending")
|
|
480
|
+
if st != "pass":
|
|
481
|
+
reasons.append(f"the compiler's own self-test is {st!r}, not 'pass'")
|
|
482
|
+
if pw["min_certifiable"] is None:
|
|
483
|
+
reasons.append("the power analysis found no gain this ladder can certify")
|
|
484
|
+
if not compiled.get("reference_note"):
|
|
485
|
+
reasons.append("the declaration states no REFERENCE_NOTE: nobody can tell how strong the baseline is")
|
|
486
|
+
out = dict(ch)
|
|
487
|
+
out.update({
|
|
488
|
+
"metric": "quality", "status": "READY" if not reasons else "DRAFT", "status_reasons": reasons,
|
|
489
|
+
"solver_contract": "one Python file defining a top-level solve(*args) returning a JSON value the declaration's verify() accepts; deterministic or not, one process, no files, no network; "
|
|
490
|
+
f"it gets {qc['budget']:g} CPU seconds per instance",
|
|
491
|
+
"protocol": {"seeds_required": compiled_seeds(), "commitment": "sha256 of the seed file's bytes (surrounding whitespace stripped)",
|
|
492
|
+
"instances": "a pure function of (seed, size, idx): checker/check.py derive",
|
|
493
|
+
"credit": "one-sided 95% lower bound of the mean paired quality difference (solver minus reference), bootstrapped within each size, averaged over sizes; "
|
|
494
|
+
"max(0, .) per seed; min over seeds. An instance without a verified answer scores 0."},
|
|
495
|
+
"power": {"threshold": pw["threshold"], "target_power": pw["target_power"], "target_effect": pw["target_effect"], "noise": pw["noise"], "noise_assumed": pw["noise_assumed"],
|
|
496
|
+
"min_certifiable": pw["min_certifiable"], "max_certifiable": pw["max_certifiable"], "curve": pw["curve"], "notes": list(pw["notes"]) + list(qc.get("notes", []))},
|
|
497
|
+
"quality": {"budget_seconds": qc["budget"], "min_gain": qc["min_gain"], "headroom_mean_cap": qc["headroom_mean_cap"],
|
|
498
|
+
"reference_profile": {s: {"mean": v["mean"], "sd": v["sd"], "timeouts": v["timeouts"]} for s, v in qc["profile"]["sizes"].items()}},
|
|
499
|
+
"simulation": qc["simulation"],
|
|
500
|
+
})
|
|
501
|
+
out["provenance"] = dict(out["provenance"], cpu_hours=qc["cpu_hours"], quality_compiler_sha256=compiled.get("quality_compiler_sha256"))
|
|
502
|
+
out["requires"] = dict(out["requires"], scorer="standard library", checker="standard library")
|
|
503
|
+
return out
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def readme_quality(ch):
|
|
507
|
+
pw, lad, name = ch["power"], ch["ladder"], ch["name"]
|
|
508
|
+
reasons = "".join(f"\n* {r}" for r in ch["status_reasons"])
|
|
509
|
+
rng = (f"from about {pw['min_certifiable']:g} to about {pw['max_certifiable']:g}" if pw["min_certifiable"] is not None else "NONE: this challenge is too weak to certify anything")
|
|
510
|
+
rows = ["| true mean gain (quality points) | P(one seed credited) | P(all seeds credited) |", "|---|---|---|"]
|
|
511
|
+
rows += [f"| {float(e):g} | {100 * v['single']:.0f}% | {100 * v['all_seeds']:.0f}% |" for e, v in pw["curve"].items()]
|
|
512
|
+
prof = "\n".join(f" * size {s}: reference mean quality {v['mean']:.3f} (sd {v['sd']:.3f}, {v['timeouts']} timeouts in the profile)" for s, v in ch["quality"]["reference_profile"].items()
|
|
513
|
+
if int(s) in set(lad["dims"]))
|
|
514
|
+
return f"""# {name} ({ch['version']}): QUALITY challenge bundle, status {ch['status']}
|
|
515
|
+
{('The bundle is a DRAFT because:' + reasons) if reasons else ''}
|
|
516
|
+
|
|
517
|
+
{ch['problem']['doc']}
|
|
518
|
+
|
|
519
|
+
* **What is scored:** the QUALITY of each answer (0 to 1, from the declaration's `quality(inst, ans)`), at the same CPU budget for everyone: {lad['budget_seconds']} s per instance. A solver that is
|
|
520
|
+
faster but no better gets nothing; one that finds better answers inside the budget gets the difference. An instance with no verified answer scores 0.
|
|
521
|
+
* solver: {ch['solver_contract']}
|
|
522
|
+
* sizes {lad['dims']}; {lad['n']} instances per size; credit needs {ch['protocol']['seeds_required']} committed seeds
|
|
523
|
+
* reference: {ch['reference']['note'] or 'NOT STATED by the declaration author'}
|
|
524
|
+
{prof}
|
|
525
|
+
* this ladder can certify mean gains {rng} quality points over the reference (threshold {pw['threshold']:g}, simulated; assumed per-instance noise {pw['noise']:g}, an ASSUMPTION that `qscore.py` replaces by the value it measures)
|
|
526
|
+
|
|
527
|
+
{chr(10).join(rows)}
|
|
528
|
+
|
|
529
|
+
## What is in here
|
|
530
|
+
|
|
531
|
+
| path | what |
|
|
532
|
+
|---|---|
|
|
533
|
+
| `challenge.json` | the manifest (`"metric": "quality"`): problem, ladder, protocol, power, provenance, requirements, the sha256 of every pinned file |
|
|
534
|
+
| `checker/decl.py`, `checker/reference.py` | the declaration (`derive`, `verify`, `quality`) and the baseline, byte-for-byte what was compiled |
|
|
535
|
+
| `checker/check.py` | `info`, `selftest`, `commit`, `derive`, `keygen`, `attest-check`, `log`: the speed bundle's tools, which work here. `selftest` tests `verify` AND `quality()` (range, determinism, that no answer needing no search scores above 0.9). `rerun` and `spread` are speed tools and refuse |
|
|
536
|
+
| `checker/qcheck.py` | `run` (run a solver on every instance, sandboxed, and write the ladder), `verify` (valid? and its quality), `qrerun` (re-run a random sample of a CLAIMED result; `--sign`) |
|
|
537
|
+
| `checker/qscore.py` | `score` (one solver against the reference), `compare` (is A really better than B?), `power` (re-simulable), `recheck`. Standard library only |
|
|
538
|
+
| `checker/quality_core.py` | the statistics (paired difference, stratified bootstrap bound, power simulation); its hash is in the manifest |
|
|
539
|
+
| `checker/score.py` | the speed scorer's readers, reused by `qscore.py`; its speed commands refuse on this bundle |
|
|
540
|
+
| `checker/vendor/` | project modules the declaration imports, copied byte-for-byte (empty if none) |
|
|
541
|
+
| `compiled.json` | the compiler's full record, for audit |
|
|
542
|
+
| `MANIFEST.sha256` | `sha256sum -c MANIFEST.sha256` (macOS: `shasum -a 256 -c MANIFEST.sha256`), run in this directory |
|
|
543
|
+
| `PROPOSAL.md` | intake text pre-filled from this record |
|
|
544
|
+
|
|
545
|
+
## Before you run anything here
|
|
546
|
+
|
|
547
|
+
`check.py derive|selftest` and `qcheck.py verify` import and run `checker/decl.py` (and `selftest` the reference): that is the author's code, and the manifest hashes prove only that the files are the
|
|
548
|
+
ones listed, not that they are safe. They re-run themselves inside an OS sandbox by default (macOS `sandbox-exec`; Linux `bwrap`: no network, no writes outside a scratch directory, credentials
|
|
549
|
+
unreadable) and warn where there is none. `qcheck.py run` and `qrerun` execute a COMPETITOR's solver the same way and REFUSE to run without a sandbox unless you pass `--sandbox off`. They also
|
|
550
|
+
call `decl.verify()` and `decl.quality()` in the parent process, unsandboxed: read `decl.py` first. A sandbox stops the practical damage, not a determined attacker.
|
|
551
|
+
|
|
552
|
+
## Hosting it on a board
|
|
553
|
+
|
|
554
|
+
1. **Seeds.** Pick at least {ch['protocol']['seeds_required']} secret seeds, publish `python3 checker/check.py commit --seed-file S` for each BEFORE the deadline, reveal after.
|
|
555
|
+
2. **A meter.** `python3 checker/qcheck.py run --seed-file S --solver sub.py --out SUB_S.jsonl` runs the solver on every instance (one at a time, CPU-limited to the budget, sandboxed), writes the
|
|
556
|
+
ladder, and prints each size's mean quality. Do the same with `--solver checker/reference.py` for the baseline ladder. A ladder is JSON lines: a first line
|
|
557
|
+
`{{"meta": true, "commit": "<the seed's sha256>", "budget": {lad['budget_seconds']}}}`, then `{{"id": "SIZE:IDX", "dim": SIZE, "status": "solved", "quality": 0.873}}` for every instance
|
|
558
|
+
(statuses solved, timeout, crash, failed, wrong; anything but solved scores 0).
|
|
559
|
+
3. **A scorer run.** `python3 checker/qscore.py score --pair REF_S1.jsonl SUB_S1.jsonl --pair ... --pair ... --out result.json` prints the credit in quality points and writes JSON;
|
|
560
|
+
`python3 checker/qscore.py compare --a A1 A2 A3 --b B1 B2 B3` answers "is #1 really ahead of #2?"; anyone can `qscore.py recheck result.json`; a verifier with the revealed seed can
|
|
561
|
+
`python3 checker/qcheck.py qrerun --seed-file S --solver sub.py --claimed SUB_S1.jsonl --baseline REF_S1.jsonl --sign KEY --attest-out A.json` on their own machine, and
|
|
562
|
+
`check.py attest-check` counts attestations from keys you pinned.
|
|
563
|
+
|
|
564
|
+
Competitors can try the contract with `python3 checker/qcheck.py run --practice --solver my.py` (public practice seed, sizes {lad['practice_dims']}).
|
|
565
|
+
|
|
566
|
+
## What a score does not show (read before publishing one)
|
|
567
|
+
|
|
568
|
+
* **The credit is only as meaningful as `quality()`.** The scorer averages whatever the declaration returns. If quality() rewards the wrong thing, the score does too. It is checked
|
|
569
|
+
to be a finite number in [0, 1], deterministic, and not high for answers that need no search; whether it measures what you care about is not checkable by a tool.
|
|
570
|
+
* **The credit is the gain over THIS reference, in quality points, at THIS budget.** A weak reference makes gains cheap; a saturated one (the reference already near 1) makes every solver tie:
|
|
571
|
+
the compiler drops sizes where the reference averages above {ch['quality']['headroom_mean_cap']}, and `qscore.py` warns when half the reference's answers are at the top. Declaration author's statement of
|
|
572
|
+
the reference: {ch['reference']['note'] or 'not stated'}
|
|
573
|
+
* **Equal weight per size, equal budget for everyone.** The mean is over sizes, so a gain on the largest size counts as much as one on the smallest. A solver that does better by
|
|
574
|
+
using more CPU than the budget is not credited: `run` scores an answer that came after the cap as 0 and `qrerun` fails a solver whose CPU exceeds it. Wall-clock, memory and
|
|
575
|
+
parallelism are NOT limited beyond the sandbox: the budget is CPU seconds of the process tree, and a solver that hides its CPU (waits, or forks work it never waits for) is detected only
|
|
576
|
+
when more than 70% is hidden.
|
|
577
|
+
* **The power table assumed a per-instance noise; `qscore.py` measures it** (pooled within-size sd of the paired differences) and, if the ladders show clearly more, re-simulates the
|
|
578
|
+
certifiable range at the observed value. The assumption itself is a share of the reference's own spread, not measured on any competitor.
|
|
579
|
+
* **Randomised solvers do not reproduce exactly.** `qrerun` compares claimed and re-measured qualities within a tolerance (default 0.05); a solver whose own run-to-run spread is larger
|
|
580
|
+
will fail it, which is honest: its claimed number was not reproducible.
|
|
581
|
+
* **Machine speed matters here in a way it does not for a speed bundle's ratios:** an anytime solver does more work per CPU second on a faster machine, so the same budget gives a better
|
|
582
|
+
answer. Run the reference and every submission on the same machine (`run` records no machine), or re-run with `qrerun` on yours. Cross-machine spread of quality has NOT been measured.
|
|
583
|
+
* **A result can be re-checked, not trusted.** `qscore.py --out` records the sha256 of every input ladder, of this challenge and of the scorer; `recheck` recomputes and fails on any difference.
|
|
584
|
+
* **Hostile input is refused, not crashed on** (ladders, result files, answers, options), with the speed bundle's size caps and bounds. The quality code paths were added later than the
|
|
585
|
+
speed ones and have had less testing.
|
|
586
|
+
* **Not tied to any particular platform.** `PROPOSAL.md` is intake text; no platform has reviewed or accepted this bundle.
|
|
587
|
+
|
|
588
|
+
Provenance: compiled {ch['provenance']['compiled_at']}, compiler self-test {ch['provenance']['selftest']}, decl {ch['files']['checker/decl.py'][:16]}..., reference {ch['files']['checker/reference.py'][:16]}...
|
|
589
|
+
"""
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def proposal_quality(ch, field):
|
|
593
|
+
pw, lad = ch["power"], ch["ladder"]
|
|
594
|
+
rng = (f"about {pw['min_certifiable']:g} to {pw['max_certifiable']:g} quality points" if pw["min_certifiable"] is not None else "none yet")
|
|
595
|
+
return f"""# Proposal text for {ch['name']} (pre-filled; edit before sending)
|
|
596
|
+
|
|
597
|
+
Written from the bundle's own record. Nothing here has been reviewed by any platform. Bundle status: {ch['status']}.
|
|
598
|
+
{('Fix first: ' + '; '.join(ch['status_reasons'])) if ch['status_reasons'] else ''}
|
|
599
|
+
|
|
600
|
+
**Name:** [you]
|
|
601
|
+
**Email:** [you]
|
|
602
|
+
**Institution / affiliation:** [you]
|
|
603
|
+
**Field:** {field}
|
|
604
|
+
|
|
605
|
+
## The Problem
|
|
606
|
+
{ch['problem']['doc']}
|
|
607
|
+
|
|
608
|
+
Goal metric: the QUALITY of the answer (0 to 1) at a fixed CPU budget of {lad['budget_seconds']} s per instance, as a gain over a stated reference solver. The score is the one-sided 95% lower bound
|
|
609
|
+
of the mean paired quality difference, bootstrapped within each size, and across {ch['protocol']['seeds_required']} committed seeds the minimum. The reference is: {ch['reference']['note'] or '[state what the reference is and what is known to beat it]'}
|
|
610
|
+
Why it matters: [one or two sentences about why better answers to this problem are worth having].
|
|
611
|
+
|
|
612
|
+
## The Verifier
|
|
613
|
+
* Instances are a pure function of (seed, size, index); the seed's sha256 is published before the deadline and the seed after, so nothing can be precomputed.
|
|
614
|
+
* `verify(instance, answer)` decides validity; `quality(instance, answer)` scores a valid answer in [0, 1] (checked: finite, in range, deterministic, and no answer that needs no search scores above 0.9). An instance without a verified answer scores 0.
|
|
615
|
+
* CPU seconds of the whole process tree, one instance at a time, a cap of {lad['budget_seconds']} s per instance, the same for the reference and every submission.
|
|
616
|
+
* Sizes {lad['dims']}, {lad['n']} instances per size. By simulation this ladder can certify mean gains of {rng}; the full power table is in the bundle.
|
|
617
|
+
* A standalone checker (`check.py`, `qcheck.py`), a scorer (`qscore.py`) and the statistics module are in the bundle, with a sha256 manifest. `python3 checker/check.py selftest` runs in seconds on a fresh random seed.
|
|
618
|
+
* Claimed results can be re-run by a verifier on their own machine (`qcheck.py qrerun`, random sample, signed attestation) and re-computed by anyone (`qscore.py recheck`).
|
|
619
|
+
|
|
620
|
+
## Link
|
|
621
|
+
[URL of the published bundle]
|
|
622
|
+
|
|
623
|
+
## Honest limits to state up front
|
|
624
|
+
* The score is only as meaningful as `quality()`; the tools check its range and determinism, not its meaning.
|
|
625
|
+
* The scorer takes measured qualities as input; `qcheck.py run` is a sandboxed local meter, not a hosted, isolated one.
|
|
626
|
+
* Cross-machine reproducibility of quality (an anytime solver does more per CPU second on a faster machine) has not been measured.
|
|
627
|
+
"""
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def compiled_seeds():
|
|
631
|
+
import compile_domain
|
|
632
|
+
return compile_domain.SEEDS
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def write_manifest(root):
|
|
636
|
+
lines = []
|
|
637
|
+
for dp, dns, fns in os.walk(root):
|
|
638
|
+
dns[:] = sorted(x for x in dns if x != "__pycache__")
|
|
639
|
+
for fn in sorted(fns):
|
|
640
|
+
if fn == "MANIFEST.sha256" or fn.endswith(".pyc"):
|
|
641
|
+
continue
|
|
642
|
+
p = os.path.join(dp, fn)
|
|
643
|
+
lines.append(f"{sha256_file(p)} {os.path.relpath(p, root)}")
|
|
644
|
+
write_bytes(os.path.join(root, "MANIFEST.sha256"), ("\n".join(sorted(lines, key=lambda s: s.split(" ", 1)[1])) + "\n").encode())
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def manifest_problems(root):
|
|
648
|
+
path = os.path.join(root, "MANIFEST.sha256")
|
|
649
|
+
if not os.path.isfile(path):
|
|
650
|
+
return ["MANIFEST.sha256 is missing"]
|
|
651
|
+
listed, out = {}, []
|
|
652
|
+
with open(path, encoding="utf-8") as f:
|
|
653
|
+
for line in f:
|
|
654
|
+
if line.strip():
|
|
655
|
+
h, _, rel = line.rstrip("\n").partition(" ")
|
|
656
|
+
listed[rel] = h
|
|
657
|
+
for rel, h in sorted(listed.items()):
|
|
658
|
+
p = os.path.join(root, rel)
|
|
659
|
+
if not os.path.isfile(p):
|
|
660
|
+
out.append(f"missing: {rel}")
|
|
661
|
+
elif sha256_file(p) != h:
|
|
662
|
+
out.append(f"changed: {rel}")
|
|
663
|
+
for dp, dns, fns in os.walk(root):
|
|
664
|
+
dns[:] = [x for x in dns if x != "__pycache__"]
|
|
665
|
+
for fn in fns:
|
|
666
|
+
rel = os.path.relpath(os.path.join(dp, fn), root)
|
|
667
|
+
if rel != "MANIFEST.sha256" and not fn.endswith(".pyc") and rel not in listed:
|
|
668
|
+
out.append(f"not in the manifest: {rel}")
|
|
669
|
+
return out
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def run_selftest(root, fresh_seed=None, sandbox="auto", require=False):
|
|
673
|
+
"""check.py selftest in a clean interpreter (-I: no PYTHONPATH, no user site, no script directory; -B: leave no bytecode in the bundle) from an empty working
|
|
674
|
+
directory, inside the OS sandbox unless sandbox="off" (so a declaration that secretly needs to write files or reach the network fails HERE, at export, not for
|
|
675
|
+
the next person). require=True refuses to run at all where no sandbox exists."""
|
|
676
|
+
with tempfile.TemporaryDirectory() as empty:
|
|
677
|
+
env = {"PATH": os.environ.get("PATH", ""), "HOME": empty, "PYTHONHASHSEED": "0", "PYTHONDONTWRITEBYTECODE": "1"}
|
|
678
|
+
if require:
|
|
679
|
+
env["GULDO_REQUIRE_SANDBOX"] = "1"
|
|
680
|
+
try:
|
|
681
|
+
cmd = ([sys.executable, "-I", "-B", os.path.join(root, "checker", "check.py"), "selftest", "--sandbox", sandbox]
|
|
682
|
+
+ (["--fresh-seed", fresh_seed] if fresh_seed else []))
|
|
683
|
+
r = subprocess.run(cmd, cwd=empty, env=env,
|
|
684
|
+
capture_output=True, text=True, timeout=SELFTEST_TIMEOUT)
|
|
685
|
+
except subprocess.TimeoutExpired:
|
|
686
|
+
raise ExportError(f"the bundle's self-test did not finish in {SELFTEST_TIMEOUT} s")
|
|
687
|
+
if r.returncode != 0:
|
|
688
|
+
tail = "\n".join((r.stdout + r.stderr).strip().splitlines()[-15:])
|
|
689
|
+
raise ExportError("the bundle FAILS its own self-test in a clean interpreter, so it is not portable as written:\n" + tail)
|
|
690
|
+
return r.stdout
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def verify_bundle(root, selftest=False, unsandboxed=False):
|
|
694
|
+
"""Hashes and pins, and with selftest=True the bundle's own self-test. That runs the bundle's decl.py and reference, which are its AUTHOR's code, so it runs
|
|
695
|
+
inside the OS sandbox and refuses to run at all where there is none, unless unsandboxed=True."""
|
|
696
|
+
problems = manifest_problems(root)
|
|
697
|
+
if not problems:
|
|
698
|
+
try:
|
|
699
|
+
with open(os.path.join(root, "challenge.json"), encoding="utf-8") as f:
|
|
700
|
+
ch = json.load(f)
|
|
701
|
+
if ch.get("format") != FORMAT:
|
|
702
|
+
problems.append(f"challenge.json has format {ch.get('format')!r}, expected {FORMAT!r}")
|
|
703
|
+
for rel, h in ch.get("files", {}).items():
|
|
704
|
+
p = os.path.join(root, rel)
|
|
705
|
+
if not os.path.isfile(p) or sha256_file(p) != h:
|
|
706
|
+
problems.append(f"challenge.json pins {rel} but it differs")
|
|
707
|
+
except (OSError, ValueError) as e:
|
|
708
|
+
problems.append(f"challenge.json unreadable: {e}")
|
|
709
|
+
if selftest and not problems:
|
|
710
|
+
try:
|
|
711
|
+
run_selftest(root, None, "off" if unsandboxed else "auto", require=not unsandboxed)
|
|
712
|
+
except ExportError as e:
|
|
713
|
+
problems.append(str(e))
|
|
714
|
+
return problems
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def main(argv=None):
|
|
718
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
719
|
+
if argv and argv[0] not in ("export", "verify") and not argv[0].startswith("-"):
|
|
720
|
+
argv = ["export"] + argv # `guldo export NAME` (the form the docs show) means `export NAME`; `guldo export export NAME` also still works
|
|
721
|
+
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__)
|
|
722
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
723
|
+
p = sub.add_parser("export", help="write a bundle for a compiled, fresh domain")
|
|
724
|
+
p.add_argument("name", metavar="NAME_or_dir")
|
|
725
|
+
p.add_argument("--out", help="bundle directory (default domains/NAME/out/bundle-VERSION-DECLHASH); never overwritten")
|
|
726
|
+
p.add_argument("--field", default="Computer Science", help="the Field entry PROPOSAL.md pre-fills (default: Computer Science)")
|
|
727
|
+
p.add_argument("--no-selftest", action="store_true", help="skip the clean-interpreter self-test (the bundle is then not known to be portable)")
|
|
728
|
+
p = sub.add_parser("verify", help="re-check a bundle's hashes (and optionally run its self-test)")
|
|
729
|
+
p.add_argument("dir")
|
|
730
|
+
p.add_argument("--selftest", action="store_true", help="ALSO run the bundle's self-test, which executes its decl.py and reference (its author's code): inside the OS sandbox, "
|
|
731
|
+
"and refusing to run where there is none")
|
|
732
|
+
p.add_argument("--unsandboxed", action="store_true", help="with --selftest: run it as you, without the sandbox (only for a bundle you have read)")
|
|
733
|
+
a = ap.parse_args(argv)
|
|
734
|
+
try:
|
|
735
|
+
if a.cmd == "export":
|
|
736
|
+
out, status, reasons = build(a.name, a.out, a.field, not a.no_selftest)
|
|
737
|
+
print(f"wrote {out}")
|
|
738
|
+
print(f"status: {status}" + "".join(f"\n - {r}" for r in reasons))
|
|
739
|
+
print("next: python3 " + os.path.join(out, "checker", "check.py") + " selftest (and read " + os.path.join(out, "README.md") + ")")
|
|
740
|
+
return 0
|
|
741
|
+
bad = verify_bundle(a.dir, a.selftest, a.unsandboxed)
|
|
742
|
+
print("bundle OK" if not bad else "bundle PROBLEMS:\n " + "\n ".join(bad))
|
|
743
|
+
return 1 if bad else 0
|
|
744
|
+
except ExportError as e:
|
|
745
|
+
print(str(e), file=sys.stderr)
|
|
746
|
+
return 1
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
if __name__ == "__main__":
|
|
750
|
+
sys.exit(main())
|