guldo-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guldo/__init__.py +0 -0
- guldo/_src/COMPILER.md +329 -0
- guldo/_src/DEPLOY.md +200 -0
- guldo/_src/LAUNCH.md +85 -0
- guldo/_src/PROTOCOL.md +294 -0
- guldo/_src/RULES.md +25 -0
- guldo/_src/SUBMIT.md +170 -0
- guldo/_src/__init__.py +0 -0
- guldo/_src/attest.py +257 -0
- guldo/_src/build_site.py +958 -0
- guldo/_src/bundle_templates/check.py +1382 -0
- guldo/_src/bundle_templates/qcheck.py +253 -0
- guldo/_src/bundle_templates/qscore.py +402 -0
- guldo/_src/bundle_templates/score.py +649 -0
- guldo/_src/certificate.py +828 -0
- guldo/_src/certs_site.py +108 -0
- guldo/_src/claims.py +343 -0
- guldo/_src/compile_domain.py +1312 -0
- guldo/_src/compile_quality.py +286 -0
- guldo/_src/count_runner.py +248 -0
- guldo/_src/counted.py +274 -0
- guldo/_src/declared.py +215 -0
- guldo/_src/deploy/Caddyfile +48 -0
- guldo/_src/deploy/ledger-backup.service +9 -0
- guldo/_src/deploy/ledger-backup.sh +73 -0
- guldo/_src/deploy/ledger-backup.timer +10 -0
- guldo/_src/deploy/ledger-export.service +12 -0
- guldo/_src/deploy/ledger-export.timer +9 -0
- guldo/_src/deploy/ledger-web.service +38 -0
- guldo/_src/deploy/pin_cpu.sh +34 -0
- guldo/_src/deploy/provision_eval.sh +128 -0
- guldo/_src/deploy_smoke.py +148 -0
- guldo/_src/dlog_experiment.py +165 -0
- guldo/_src/domains/aidecode/compiled.json +586 -0
- guldo/_src/domains/aidecode/decl.py +134 -0
- guldo/_src/domains/aidecode/reference.py +73 -0
- guldo/_src/domains/aiinfer/compiled.json +722 -0
- guldo/_src/domains/aiinfer/decl.py +91 -0
- guldo/_src/domains/aiinfer/reference.py +42 -0
- guldo/_src/domains/dlog/compiled.json +248 -0
- guldo/_src/domains/dlog/decl.py +41 -0
- guldo/_src/domains/dlog/reference.py +10 -0
- guldo/_src/domains/maxcut/compiled.json +291 -0
- guldo/_src/domains/maxcut/decl.py +46 -0
- guldo/_src/domains/maxcut/reference.py +16 -0
- guldo/_src/drift_check.py +89 -0
- guldo/_src/export_bundle.py +750 -0
- guldo/_src/extrapolate.py +326 -0
- guldo/_src/families.py +53 -0
- guldo/_src/gen_cheat_table.py +36 -0
- guldo/_src/gpu_meter.py +634 -0
- guldo/_src/gpu_runner.py +63 -0
- guldo/_src/harness.py +273 -0
- guldo/_src/instances.py +86 -0
- guldo/_src/instances_factor.py +40 -0
- guldo/_src/instances_mlwe.py +69 -0
- guldo/_src/judge.py +278 -0
- guldo/_src/limits_plan.py +158 -0
- guldo/_src/official.py +189 -0
- guldo/_src/ops.py +281 -0
- guldo/_src/portal.py +313 -0
- guldo/_src/portal_core.py +57 -0
- guldo/_src/power.py +174 -0
- guldo/_src/prereg.py +173 -0
- guldo/_src/public_portal.py +500 -0
- guldo/_src/quality_core.py +126 -0
- guldo/_src/recheck.py +236 -0
- guldo/_src/remeasure.py +88 -0
- guldo/_src/report_html.py +493 -0
- guldo/_src/results/.keep +0 -0
- guldo/_src/rounds.py +846 -0
- guldo/_src/runner.py +52 -0
- guldo/_src/sandbox/Dockerfile +11 -0
- guldo/_src/sandbox/Dockerfile.v2 +19 -0
- guldo/_src/sandbox/README.md +40 -0
- guldo/_src/sandbox/linux_selftest.py +117 -0
- guldo/_src/sandbox/meter.py +89 -0
- guldo/_src/sandbox/meter_core.py +28 -0
- guldo/_src/site_assets.py +176 -0
- guldo/_src/site_live_check.py +79 -0
- guldo/_src/solvers/aidecode_flops_kv.py +73 -0
- guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
- guldo/_src/solvers/aidecode_flops_ref.py +64 -0
- guldo/_src/solvers/aidecode_kv.py +76 -0
- guldo/_src/solvers/aidecode_kv_f32.py +77 -0
- guldo/_src/solvers/aidecode_lossy.py +76 -0
- guldo/_src/solvers/aidecode_repeat.py +5 -0
- guldo/_src/solvers/aiinfer_batched.py +40 -0
- guldo/_src/solvers/aiinfer_f32.py +41 -0
- guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
- guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
- guldo/_src/solvers/aiinfer_lossy.py +42 -0
- guldo/_src/solvers/baseline_bkz.py +14 -0
- guldo/_src/solvers/cheat_lookup.py +24 -0
- guldo/_src/solvers/cheat_table.json +1 -0
- guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
- guldo/_src/solvers/daemon_cpu_hider.py +17 -0
- guldo/_src/solvers/dlog_bsgs.py +19 -0
- guldo/_src/solvers/dlog_stride4.py +18 -0
- guldo/_src/solvers/egg_base.py +11 -0
- guldo/_src/solvers/egg_eightaxis.py +68 -0
- guldo/_src/solvers/egg_template.py +11 -0
- guldo/_src/solvers/factor_brent.py +34 -0
- guldo/_src/solvers/factor_floyd.py +19 -0
- guldo/_src/solvers/factor_trial_fast_py.py +21 -0
- guldo/_src/solvers/factor_trial_np.py +21 -0
- guldo/_src/solvers/factor_trial_py.py +12 -0
- guldo/_src/solvers/fixed_overhead_ref.py +21 -0
- guldo/_src/solvers/fork_bomb.py +18 -0
- guldo/_src/solvers/gpu_noop.py +6 -0
- guldo/_src/solvers/hostile_probe.py +36 -0
- guldo/_src/solvers/kill_meter.py +27 -0
- guldo/_src/solvers/memory_bomb.py +10 -0
- guldo/_src/solvers/noop.py +5 -0
- guldo/_src/solvers/orphan_cpu_hider.py +16 -0
- guldo/_src/solvers/planted_peek.py +23 -0
- guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
- guldo/_src/solvers/small_n_trick.py +22 -0
- guldo/_src/solvers/starter_template.py +21 -0
- guldo/_src/solvers/stdout_flood.py +9 -0
- guldo/_src/solvers/tmp_fill.py +10 -0
- guldo/_src/solvers/tuned_bkz.py +25 -0
- guldo/_src/static_check.py +214 -0
- guldo/_src/submit_check.py +166 -0
- guldo/_src/svp.py +889 -0
- guldo/_src/tensorops.py +304 -0
- guldo/_src/tflops.py +453 -0
- guldo/_src/tflops_runner.py +134 -0
- guldo/cli.py +60 -0
- guldo_cli-0.1.0.dist-info/METADATA +30 -0
- guldo_cli-0.1.0.dist-info/RECORD +134 -0
- guldo_cli-0.1.0.dist-info/WHEEL +5 -0
- guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
- guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# The public web tier. Runs as an unprivileged user that CANNOT read the seeds or reach Docker.
|
|
2
|
+
# Install: cp deploy/ledger-web.service /etc/systemd/system/ && systemctl daemon-reload && systemctl enable --now ledger-web
|
|
3
|
+
[Unit]
|
|
4
|
+
Description=Speedup Ledger submission service (web tier)
|
|
5
|
+
After=network.target
|
|
6
|
+
|
|
7
|
+
[Service]
|
|
8
|
+
User=ledger-web
|
|
9
|
+
Group=ledger-web
|
|
10
|
+
WorkingDirectory=/opt/speedup-verification
|
|
11
|
+
ExecStart=/usr/bin/python3 public_portal.py serve --data /srv/ledger/data --host ledger.example.org --bind 127.0.0.1 --port 8080 --trust-proxy --site-url https://ledger.example.org
|
|
12
|
+
Restart=on-failure
|
|
13
|
+
RestartSec=3
|
|
14
|
+
# the operator user (group "ledger") also writes the data directory (closing rounds, evaluation results), so files must be group-writable
|
|
15
|
+
UMask=0007
|
|
16
|
+
SupplementaryGroups=ledger
|
|
17
|
+
|
|
18
|
+
# Containment: the process only ever parses uploads, but assume a bug in the parser.
|
|
19
|
+
NoNewPrivileges=true
|
|
20
|
+
ProtectSystem=strict
|
|
21
|
+
ReadWritePaths=/srv/ledger/data
|
|
22
|
+
ProtectHome=true
|
|
23
|
+
PrivateTmp=true
|
|
24
|
+
PrivateDevices=true
|
|
25
|
+
ProtectKernelTunables=true
|
|
26
|
+
ProtectControlGroups=true
|
|
27
|
+
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
|
|
28
|
+
RestrictNamespaces=true
|
|
29
|
+
LockPersonality=true
|
|
30
|
+
MemoryMax=512M
|
|
31
|
+
TasksMax=128
|
|
32
|
+
LimitNOFILE=1024
|
|
33
|
+
CapabilityBoundingSet=
|
|
34
|
+
SystemCallFilter=@system-service
|
|
35
|
+
SystemCallErrorNumber=EPERM
|
|
36
|
+
|
|
37
|
+
[Install]
|
|
38
|
+
WantedBy=multi-user.target
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Run a command (normally the evaluation worker) so that every `docker run` it starts is pinned to the given CPU core(s).
|
|
3
|
+
#
|
|
4
|
+
# bash deploy/pin_cpu.sh 1 -- python3 ops.py worker --data /srv/ledger/data --id lattice-1 --seed-dir /srv/ledger/seeds/lattice-1
|
|
5
|
+
# bash deploy/pin_cpu.sh 2,3 -- ... (a list or a range such as 2-3 also works)
|
|
6
|
+
#
|
|
7
|
+
# Why a wrapper: the harness builds its docker command in harness.py, which is one of the hashed files a round's fingerprint pins, so it must not change
|
|
8
|
+
# while a round is open. The wrapper changes nothing hashed: it puts a tiny `docker` shim first in PATH, and the shim adds `--cpuset-cpus <set>` right
|
|
9
|
+
# after `run` and otherwise passes everything through to the real docker unchanged. Pin to a core the web service and the operating system do not
|
|
10
|
+
# need (on a 2-core machine, core 1 leaves core 0 for them), and say which core you used when you publish the round's results: the harness records the
|
|
11
|
+
# CPU model but not the pinning.
|
|
12
|
+
set -euo pipefail
|
|
13
|
+
|
|
14
|
+
[ "$#" -ge 3 ] && [ "$2" = "--" ] || { sed -n '2,12p' "$0"; exit 2; }
|
|
15
|
+
SET="$1"; shift 2
|
|
16
|
+
[[ "$SET" =~ ^[0-9]+([,-][0-9]+)*$ ]] || { echo "bad CPU set: $SET (use e.g. 1, 2,3 or 2-3)"; exit 2; }
|
|
17
|
+
|
|
18
|
+
REAL="$(command -v docker || true)"
|
|
19
|
+
[ -n "$REAL" ] || { echo "docker not found in PATH"; exit 1; }
|
|
20
|
+
SHIM_DIR="$(mktemp -d)"
|
|
21
|
+
trap 'rm -rf "$SHIM_DIR"' EXIT
|
|
22
|
+
|
|
23
|
+
cat > "$SHIM_DIR/docker" <<EOF
|
|
24
|
+
#!/usr/bin/env bash
|
|
25
|
+
if [ "\${1:-}" = "run" ]; then
|
|
26
|
+
shift
|
|
27
|
+
exec "$REAL" run --cpuset-cpus "$SET" "\$@"
|
|
28
|
+
fi
|
|
29
|
+
exec "$REAL" "\$@"
|
|
30
|
+
EOF
|
|
31
|
+
chmod +x "$SHIM_DIR/docker"
|
|
32
|
+
|
|
33
|
+
echo "pin_cpu: every docker run is pinned to CPU set $SET" >&2
|
|
34
|
+
PATH="$SHIM_DIR:$PATH" "$@"
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Provision a DISPOSABLE evaluation machine for timed rounds, from a fresh Ubuntu (22.04 or newer) or Debian server, in one command.
|
|
3
|
+
# Run it AS ROOT, from a copy of this project that is on the machine (rsync the project there first; EVAL_BOX.md has the exact command):
|
|
4
|
+
#
|
|
5
|
+
# sudo bash deploy/provision_eval.sh [--no-firewall] [--no-ssh-hardening] [--skip-gates]
|
|
6
|
+
#
|
|
7
|
+
# What it does, in order (every step is safe to repeat):
|
|
8
|
+
# packages (Docker, Python, numpy, cryptography, rsync) -> swap if the machine has little memory -> the unprivileged operator user `ledger-ops`
|
|
9
|
+
# (in the docker group on a disposable machine; with --shared-with-web never) -> the data and seed directories ->
|
|
10
|
+
# the code under /opt/speedup-verification (root-owned, not writable by others) -> the firewall (only ssh) and key-only ssh ->
|
|
11
|
+
# the sandbox image -> the Linux meter self-test -> the whole sandbox test suite and the SANDBOX_VERIFIED marker the worker requires.
|
|
12
|
+
# It stops at the first failure and says which step failed. It never touches seeds, DNS, or any other machine.
|
|
13
|
+
set -euo pipefail
|
|
14
|
+
|
|
15
|
+
FIREWALL=1; HARDEN_SSH=1; GATES=1; SHARED=0
|
|
16
|
+
for a in "$@"; do
|
|
17
|
+
case "$a" in
|
|
18
|
+
--shared-with-web) SHARED=1 ;; # this machine ALSO runs the web portal: keep its users, data-directory permissions and seeds exactly as they are
|
|
19
|
+
--no-firewall) FIREWALL=0 ;;
|
|
20
|
+
--no-ssh-hardening) HARDEN_SSH=0 ;;
|
|
21
|
+
--skip-gates) GATES=0 ;;
|
|
22
|
+
-h|--help) sed -n '2,15p' "$0"; exit 0 ;;
|
|
23
|
+
*) echo "unknown option: $a"; exit 2 ;;
|
|
24
|
+
esac
|
|
25
|
+
done
|
|
26
|
+
|
|
27
|
+
step() { printf '\n== %s\n' "$*"; }
|
|
28
|
+
die() { printf '\nFAILED at: %s\n' "$*" >&2; exit 1; }
|
|
29
|
+
trap 'die "${STEP:-startup} (line $LINENO)"' ERR
|
|
30
|
+
|
|
31
|
+
[ "$(id -u)" = 0 ] || { echo "run as root (sudo bash deploy/provision_eval.sh)"; exit 1; }
|
|
32
|
+
command -v apt-get >/dev/null || { echo "this script supports apt-based systems (Ubuntu, Debian)"; exit 1; }
|
|
33
|
+
SRC="$(cd "$(dirname "$0")/.." && pwd)"
|
|
34
|
+
DEST=/opt/speedup-verification
|
|
35
|
+
DATA=/srv/ledger/data
|
|
36
|
+
[ -f "$SRC/harness.py" ] && [ -f "$SRC/sandbox/Dockerfile.v2" ] || { echo "run it from a copy of the project (harness.py and sandbox/ not found in $SRC)"; exit 1; }
|
|
37
|
+
|
|
38
|
+
STEP="packages"; step "$STEP"
|
|
39
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
40
|
+
apt-get update -qq
|
|
41
|
+
apt-get install -y -qq python3 python3-numpy python3-cryptography rsync ca-certificates curl >/dev/null
|
|
42
|
+
if ! command -v docker >/dev/null; then apt-get install -y -qq docker.io >/dev/null; fi
|
|
43
|
+
systemctl enable --now docker >/dev/null 2>&1 || true
|
|
44
|
+
docker version --format 'docker {{.Server.Version}}' || die "docker is not answering"
|
|
45
|
+
|
|
46
|
+
STEP="swap"; step "$STEP"
|
|
47
|
+
mem_mb=$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)
|
|
48
|
+
if [ "$mem_mb" -lt 3500 ] && ! swapon --show | grep -q .; then
|
|
49
|
+
fallocate -l 4G /swapfile && chmod 600 /swapfile && mkswap /swapfile >/dev/null && swapon /swapfile
|
|
50
|
+
grep -q '^/swapfile' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab
|
|
51
|
+
echo "added 4 GB of swap (RAM is ${mem_mb} MB)"
|
|
52
|
+
else
|
|
53
|
+
echo "no swap needed (RAM ${mem_mb} MB, or swap already present)"
|
|
54
|
+
fi
|
|
55
|
+
|
|
56
|
+
STEP="users and directories"; step "$STEP"
|
|
57
|
+
id ledger-ops >/dev/null 2>&1 || useradd -m -s /bin/bash ledger-ops
|
|
58
|
+
# Membership of the docker group is root-equivalent. On a disposable evaluation machine that is acceptable. On a machine that also serves the portal it is
|
|
59
|
+
# NOT: ledger-ops is the account that evaluates strangers' counted solvers, so it is never put in the group there. Docker is reached instead by running a
|
|
60
|
+
# command as root with `sudo -u ledger-ops -g docker ...`, which gives that one process the docker group as its primary group and nothing persistent.
|
|
61
|
+
if [ "$SHARED" = 0 ]; then usermod -aG docker ledger-ops; fi
|
|
62
|
+
if [ "$SHARED" = 1 ]; then
|
|
63
|
+
# the web tier writes this directory through the group `ledger` (setgid, 2770): do not tighten it, or submissions stop working
|
|
64
|
+
[ -d "$DATA/rounds" ] || { echo "--shared-with-web expects the web layout from DEPLOY.md already in place ($DATA/rounds missing)"; exit 1; }
|
|
65
|
+
install -d -o ledger-ops -g ledger-ops -m 0755 /srv/ledger/results
|
|
66
|
+
echo "shared machine: left $DATA and /srv/ledger/seeds exactly as they are"
|
|
67
|
+
else
|
|
68
|
+
install -d -o ledger-ops -g ledger-ops -m 0755 /srv/ledger
|
|
69
|
+
install -d -o ledger-ops -g ledger-ops -m 0750 "$DATA" "$DATA/rounds"
|
|
70
|
+
install -d -o ledger-ops -g ledger-ops -m 0700 /srv/ledger/seeds
|
|
71
|
+
install -d -o ledger-ops -g ledger-ops -m 0755 /srv/ledger/results
|
|
72
|
+
fi
|
|
73
|
+
echo "operator user ledger-ops; data $DATA; seeds /srv/ledger/seeds (0700)"
|
|
74
|
+
# round_sync.py logs in as ledger-ops (not root) so that copied files get the right owner; it uses the same key you used for root
|
|
75
|
+
if [ -s /root/.ssh/authorized_keys ]; then
|
|
76
|
+
install -d -o ledger-ops -g ledger-ops -m 0700 /home/ledger-ops/.ssh
|
|
77
|
+
install -o ledger-ops -g ledger-ops -m 0600 /root/.ssh/authorized_keys /home/ledger-ops/.ssh/authorized_keys
|
|
78
|
+
echo "ssh key installed for ledger-ops"
|
|
79
|
+
fi
|
|
80
|
+
|
|
81
|
+
STEP="code"; step "$STEP"
|
|
82
|
+
mkdir -p "$DEST"
|
|
83
|
+
if [ "$SRC" != "$DEST" ]; then
|
|
84
|
+
rsync -a --delete --exclude .git --exclude __pycache__ --exclude /site --exclude /dist --exclude /ledger --exclude /ledger-seeds \
|
|
85
|
+
--exclude /ledger-submissions --exclude /results --exclude .vercel "$SRC"/ "$DEST"/
|
|
86
|
+
fi
|
|
87
|
+
install -d -o ledger-ops -g ledger-ops -m 0755 "$DEST/results"
|
|
88
|
+
chown -R root:root "$DEST"; chown ledger-ops:ledger-ops "$DEST/results"; chmod -R go-w "$DEST"
|
|
89
|
+
echo "code installed in $DEST"
|
|
90
|
+
|
|
91
|
+
if [ "$FIREWALL" = 1 ]; then
|
|
92
|
+
STEP="firewall"; step "$STEP"
|
|
93
|
+
apt-get install -y -qq ufw fail2ban >/dev/null
|
|
94
|
+
ufw allow OpenSSH >/dev/null; ufw --force enable >/dev/null
|
|
95
|
+
ufw status | head -4
|
|
96
|
+
fi
|
|
97
|
+
|
|
98
|
+
if [ "$HARDEN_SSH" = 1 ]; then
|
|
99
|
+
STEP="ssh hardening"; step "$STEP"
|
|
100
|
+
if [ -s /root/.ssh/authorized_keys ]; then
|
|
101
|
+
printf 'PasswordAuthentication no\nPermitRootLogin prohibit-password\n' > /etc/ssh/sshd_config.d/10-hardening.conf
|
|
102
|
+
sshd -t && (systemctl reload ssh 2>/dev/null || systemctl reload sshd)
|
|
103
|
+
echo "password logins are off; root may log in with a key only"
|
|
104
|
+
else
|
|
105
|
+
echo "SKIPPED: /root/.ssh/authorized_keys is empty, and turning passwords off now would lock you out. Add your key, then rerun."
|
|
106
|
+
fi
|
|
107
|
+
fi
|
|
108
|
+
|
|
109
|
+
STEP="sandbox image"; step "$STEP"
|
|
110
|
+
cd "$DEST"
|
|
111
|
+
BUILD="docker build"; docker buildx version >/dev/null 2>&1 && BUILD="docker build --load" # Ubuntu's docker.io has no buildx and no --load flag
|
|
112
|
+
$BUILD -t svp-solver -f sandbox/Dockerfile.v2 . >/tmp/provision-build.log 2>&1 || { tail -20 /tmp/provision-build.log; die "docker build (log above)"; }
|
|
113
|
+
docker image inspect svp-solver --format 'image {{.Id}}' | cut -c1-40
|
|
114
|
+
|
|
115
|
+
if [ "$GATES" = 1 ]; then
|
|
116
|
+
STEP="Linux meter self-test"; step "$STEP"
|
|
117
|
+
python3 sandbox/linux_selftest.py
|
|
118
|
+
STEP="sandbox tests and marker"; step "$STEP"
|
|
119
|
+
mkdir -p "$DATA"; [ "$SHARED" = 1 ] || chown ledger-ops:ledger-ops "$DATA"
|
|
120
|
+
sudo -u ledger-ops -g docker python3 launch_gates.py --only env,docker --with-docker --data "$DATA"
|
|
121
|
+
sudo -u ledger-ops python3 -c "import portal_core, sys; sys.exit(0 if portal_core.sandbox_verified('$DATA') else 1)" || die "the SANDBOX_VERIFIED marker is missing or stale"
|
|
122
|
+
echo "sandbox VERIFIED on this machine: the worker will now evaluate timed rounds here"
|
|
123
|
+
fi
|
|
124
|
+
|
|
125
|
+
if [ "$SHARED" = 1 ]; then
|
|
126
|
+
printf '\nNote: ledger-ops is NOT in the docker group on this shared machine, on purpose. Run the timed worker as root with\n sudo -u ledger-ops -g docker python3 ops.py worker ...\nso that only that process has Docker access (EVAL_BOX.md).\n'
|
|
127
|
+
fi
|
|
128
|
+
printf '\nDone. Next (EVAL_BOX.md): push a closed round here with round_sync.py, run ops.py worker as ledger-ops, then pull it back.\n'
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Smoke test for a RUNNING submission service: is what is actually listening what DEPLOY.md says it is?
|
|
2
|
+
|
|
3
|
+
python3 deploy_smoke.py https://ledger.example.org [--submit FILE --round ID --handle NAME]
|
|
4
|
+
|
|
5
|
+
Read-only by default (a few GETs and two requests that must be REFUSED). It checks, against the live address:
|
|
6
|
+
* /healthz answers, over TLS if the address is https (a bad certificate fails here, not silently);
|
|
7
|
+
* the security headers the page code promises are really on the wire (a proxy that strips them is a finding);
|
|
8
|
+
* a request with a foreign Host header and a cross-site Origin are refused (DNS-rebinding and CSRF controls);
|
|
9
|
+
* a submission without the rules accepted is refused with a JSON error and stores nothing;
|
|
10
|
+
* every round's public log verifies (hash chain, commitments in entry 0) with YOUR copy of the checker, and matches the round's description;
|
|
11
|
+
* the JSON API does not leak a seed before the reveal.
|
|
12
|
+
With --submit it also makes ONE real submission through svp.py (a real, logged, permanent entry: use a test round).
|
|
13
|
+
Cost to know: the refusal probe is a POST, and the portal rate-limits POSTs before it reads the form, so each OPEN round spends one of your
|
|
14
|
+
address's submission tokens (burst 3, 6 per hour). Run it from somewhere that is not also the address you submit from, or wait an hour.
|
|
15
|
+
Exit status: 0 all passed, 1 something failed, 2 could not reach the service.
|
|
16
|
+
"""
|
|
17
|
+
import argparse
|
|
18
|
+
import http.client
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
import urllib.parse
|
|
22
|
+
|
|
23
|
+
import certificate
|
|
24
|
+
import rounds
|
|
25
|
+
import svp
|
|
26
|
+
|
|
27
|
+
REQUIRED_HEADERS = {"x-content-type-options": "nosniff", "x-frame-options": "DENY", "referrer-policy": "no-referrer"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _conn(server):
|
|
31
|
+
u = urllib.parse.urlsplit(server)
|
|
32
|
+
cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection
|
|
33
|
+
return cls(u.hostname, u.port or (443 if u.scheme == "https" else 80), timeout=20)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _raw(server, method, path, headers=None, body=None):
|
|
37
|
+
c = _conn(server)
|
|
38
|
+
try:
|
|
39
|
+
c.request(method, path, body=body, headers=headers or {})
|
|
40
|
+
r = c.getresponse()
|
|
41
|
+
return r.status, {k.lower(): v for k, v in r.getheaders()}, r.read(svp.MAX_JSON + 1)
|
|
42
|
+
finally:
|
|
43
|
+
c.close()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def smoke(server, submit=None):
|
|
47
|
+
"""-> [(status, check, detail)] with status in pass / fail / warn. A connection that drops or a server that answers nonsense half-way through
|
|
48
|
+
is a FINDING in the report, never a traceback."""
|
|
49
|
+
out = []
|
|
50
|
+
try:
|
|
51
|
+
_smoke(server, submit, out)
|
|
52
|
+
except (OSError, http.client.HTTPException, ValueError) as e:
|
|
53
|
+
out.append(("fail", "connection", f"the service stopped answering properly mid-test ({type(e).__name__}: {svp.clean(e, 100)})"))
|
|
54
|
+
except (AttributeError, TypeError, KeyError, IndexError) as e: # JSON of an unexpected shape (a log whose entries are not objects, ...)
|
|
55
|
+
out.append(("fail", "response shape", f"the service sent data in an unexpected shape ({type(e).__name__}: {svp.clean(e, 100)})"))
|
|
56
|
+
return out
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _smoke(server, submit, out):
|
|
60
|
+
add = lambda st, name, detail="": out.append((st, name, detail))
|
|
61
|
+
u = urllib.parse.urlsplit(server)
|
|
62
|
+
try:
|
|
63
|
+
st, hdr, body = _raw(server, "GET", "/healthz")
|
|
64
|
+
except (OSError, http.client.HTTPException) as e:
|
|
65
|
+
out.append(("fail", "reachable", f"cannot reach {server}: {type(e).__name__}: {svp.clean(e, 100)}"))
|
|
66
|
+
return
|
|
67
|
+
add("pass" if st == 200 and body.strip() == b"ok" else "fail", "healthz", f"HTTP {st}")
|
|
68
|
+
add("pass" if u.scheme == "https" else "warn", "transport",
|
|
69
|
+
"https (the certificate verified, or the connection would have failed)" if u.scheme == "https" else
|
|
70
|
+
"plain http: fine on loopback, but a public service must be https (tokens and sources travel in the clear otherwise)")
|
|
71
|
+
st, hdr, body = _raw(server, "GET", "/")
|
|
72
|
+
missing = [k for k, v in REQUIRED_HEADERS.items() if hdr.get(k) != v]
|
|
73
|
+
add("pass" if st == 200 and not missing else "fail", "security headers", "present" if not missing else "missing or wrong: " + ", ".join(missing))
|
|
74
|
+
csp = hdr.get("content-security-policy", "")
|
|
75
|
+
add("pass" if "default-src 'none'" in csp and "frame-ancestors 'none'" in csp else "fail", "content security policy", svp.clean(csp, 80) or "absent")
|
|
76
|
+
if u.scheme == "https":
|
|
77
|
+
add("pass" if "max-age" in hdr.get("strict-transport-security", "") else "warn", "HSTS", hdr.get("strict-transport-security", "absent (the proxy may add it)"))
|
|
78
|
+
st, _, _ = _raw(server, "GET", "/", headers={"Host": "evil.invalid"})
|
|
79
|
+
add("pass" if st == 403 else "fail", "foreign Host refused", f"HTTP {st} (want 403)")
|
|
80
|
+
st, _, _ = _raw(server, "POST", "/round/nonexistent/submit", headers={"Origin": "https://evil.invalid", "Content-Type": "multipart/form-data; boundary=x",
|
|
81
|
+
"Content-Length": "0"})
|
|
82
|
+
add("pass" if st == 403 else "fail", "cross-site POST refused", f"HTTP {st} (want 403)")
|
|
83
|
+
try:
|
|
84
|
+
data = svp.get_json(server, "/api/rounds.json")
|
|
85
|
+
except svp.SvpError as e:
|
|
86
|
+
add("fail", "rounds api", str(e))
|
|
87
|
+
return
|
|
88
|
+
if not isinstance(data, list) or not all(isinstance(r, dict) for r in data):
|
|
89
|
+
add("fail", "rounds api", "the rounds list is malformed")
|
|
90
|
+
return
|
|
91
|
+
add("pass", "rounds api", f"{len(data)} round(s)") # only after the shape is known good: one finding per check, never a PASS and a FAIL
|
|
92
|
+
for r in data:
|
|
93
|
+
rid = r.get("id", "?")
|
|
94
|
+
try:
|
|
95
|
+
rj = svp._round(server, rid)
|
|
96
|
+
status, logbody = svp.request(server, f"/api/log/{rid}.jsonl")
|
|
97
|
+
entries = svp.parse_log(logbody) if status == 200 else []
|
|
98
|
+
ok, why = certificate.commit_published(entries, rj.get("commitments", []))
|
|
99
|
+
same = entries and entries[0].get("data", {}).get("commitments") == rj.get("commitments")
|
|
100
|
+
add("pass" if ok and same and rj.get("log_ok") else "fail", f"round {rid}: log", why if ok else f"does not verify: {why}")
|
|
101
|
+
leaked = rj.get("state") not in ("revealed",) and bool(rj.get("seeds"))
|
|
102
|
+
add("fail" if leaked else "pass", f"round {rid}: no seed before the reveal", f"state {rj.get('state')}")
|
|
103
|
+
if rj.get("state") == "open":
|
|
104
|
+
body, ctype = svp.multipart({"handle": "smoketest"}, "x.py", b"def solve(*a):\n return None\n") # no `agree` field on purpose
|
|
105
|
+
st, _, resp = _raw(server, "POST", f"/round/{rid}/submit", headers={"Content-Type": ctype, "Accept": "application/json"}, body=body)
|
|
106
|
+
try:
|
|
107
|
+
parsed = json.loads(resp)
|
|
108
|
+
msg = parsed.get("error", "") if isinstance(parsed, dict) else "" # a service answering with a list is a failed check, not a crash
|
|
109
|
+
except ValueError:
|
|
110
|
+
msg = ""
|
|
111
|
+
add("pass" if st == 422 and "rules" in msg else "fail", f"round {rid}: refuses without the rules accepted", f"HTTP {st}: {svp.clean(msg, 80)}")
|
|
112
|
+
except svp.SvpError as e:
|
|
113
|
+
add("fail", f"round {rid}", str(e))
|
|
114
|
+
if submit:
|
|
115
|
+
import argparse as _ap
|
|
116
|
+
file, rid, handle, home = submit
|
|
117
|
+
a = _ap.Namespace(server=server, file=file, round=rid, handle=handle, accept_rules=True, publish_source=False, token=None, no_check=False,
|
|
118
|
+
allow_insecure_http=False, home=home, json=False)
|
|
119
|
+
try:
|
|
120
|
+
rc = svp.cmd_submit(a)
|
|
121
|
+
except svp.SvpError as e:
|
|
122
|
+
add("fail", "real submission", str(e))
|
|
123
|
+
else:
|
|
124
|
+
add("pass" if rc == 0 else "fail", "real submission", "accepted and confirmed in the public log" if rc == 0 else f"svp exit {rc}")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def main(argv=None):
|
|
128
|
+
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
129
|
+
ap.add_argument("server")
|
|
130
|
+
ap.add_argument("--submit", metavar="FILE", help="make ONE real submission with svp.py (permanent: use a test round)")
|
|
131
|
+
ap.add_argument("--round")
|
|
132
|
+
ap.add_argument("--handle")
|
|
133
|
+
ap.add_argument("--home", help="where svp keeps the receipt")
|
|
134
|
+
a = ap.parse_args(argv)
|
|
135
|
+
if a.submit and not (a.round and a.handle):
|
|
136
|
+
raise SystemExit("--submit needs --round and --handle")
|
|
137
|
+
res = smoke(svp.normalize_server(a.server), (a.submit, a.round, a.handle, a.home) if a.submit else None)
|
|
138
|
+
for st, name, detail in res:
|
|
139
|
+
print(f" {st.upper():4s} {name}: {detail}")
|
|
140
|
+
if res and res[0][1] == "reachable":
|
|
141
|
+
return 2
|
|
142
|
+
bad = [r for r in res if r[0] == "fail"]
|
|
143
|
+
print("\nALL CHECKS PASSED" if not bad else f"\n{len(bad)} CHECK(S) FAILED: do not open a round until these are fixed")
|
|
144
|
+
return 1 if bad else 0
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
sys.exit(main())
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Judge the pre-registered dlog experiment (results/PREREGISTERED_dlog_realsolver.txt) mechanically, with no reinterpretation.
|
|
2
|
+
|
|
3
|
+
python3 dlog_experiment.py print each criterion as PASS / FAIL / PENDING, and the numbers behind it
|
|
4
|
+
python3 dlog_experiment.py --json the same, as JSON
|
|
5
|
+
|
|
6
|
+
Reads only saved files: the three baseline ladders, the two submissions' ladders and the primary estimator's claim file next to each (written
|
|
7
|
+
by judge.py at evaluation time). It also re-hashes every code file the registration lists and says so if any changed since, because a result
|
|
8
|
+
measured with different code is not the registered experiment.
|
|
9
|
+
|
|
10
|
+
Definitions fixed here, in code, before the data were read: a claim is the MINIMUM of the three seeds' credits (judge.py's rule: any seed with
|
|
11
|
+
no conclusion makes the claim inconclusive); the "pooled paired median" is the median over all three seeds of log2(baseline cost / solver cost)
|
|
12
|
+
on every instance both solvers solved whose BASELINE cost is at least the estimator's 0.5 s floor (the same instances the credit may use).
|
|
13
|
+
The all-pairs median (no floor) is reported beside it and is not used by any criterion.
|
|
14
|
+
"""
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import math
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import statistics
|
|
21
|
+
import sys
|
|
22
|
+
|
|
23
|
+
import extrapolate as E
|
|
24
|
+
|
|
25
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
26
|
+
PREREG = "PREREGISTERED_dlog_realsolver.txt"
|
|
27
|
+
TAGS = ("e5dab2c8", "d00100a7", "e70be0d8") # the first 8 hex of the three registered seed commitments
|
|
28
|
+
A_SOLVER, B_SOLVER = "dlog_stride4.py", "dlog_bsgs.py"
|
|
29
|
+
BASELINE = "reference.py"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _ladder(results, tag, role, name):
|
|
33
|
+
return os.path.join(results, f"judged_dlog_{tag}_{role}_{name}.jsonl")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _records(path):
|
|
37
|
+
with open(path, encoding="utf-8") as f:
|
|
38
|
+
return [r for r in map(json.loads, (ln for ln in f if ln.strip())) if not r.get("meta")]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _claim(path):
|
|
42
|
+
p = path + ".claim.json"
|
|
43
|
+
if not os.path.exists(p):
|
|
44
|
+
return "missing"
|
|
45
|
+
with open(p, encoding="utf-8") as f:
|
|
46
|
+
return json.load(f)["measured"] # None = inconclusive
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def registered_hashes(results):
|
|
50
|
+
with open(os.path.join(results, PREREG), encoding="utf-8") as f:
|
|
51
|
+
return re.findall(r"^([0-9a-f]{64}) (\S+)$", f.read(), re.M)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def code_changed(results):
|
|
55
|
+
bad = []
|
|
56
|
+
for want, rel in registered_hashes(results):
|
|
57
|
+
p = os.path.join(HERE, rel)
|
|
58
|
+
got = None
|
|
59
|
+
if os.path.exists(p):
|
|
60
|
+
with open(p, "rb") as f:
|
|
61
|
+
got = hashlib.sha256(f.read()).hexdigest()
|
|
62
|
+
if got != want:
|
|
63
|
+
bad.append(rel)
|
|
64
|
+
return bad
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def paired_medians(results, name):
|
|
68
|
+
"""(median over floor-eligible pairs, median over all pairs, number of eligible pairs); None when a ladder is missing."""
|
|
69
|
+
elig, allp = [], []
|
|
70
|
+
for tag in TAGS:
|
|
71
|
+
bp, sp = _ladder(results, tag, "baseline", BASELINE), _ladder(results, tag, "submission", name)
|
|
72
|
+
if not (os.path.exists(bp) and os.path.exists(sp)):
|
|
73
|
+
return None
|
|
74
|
+
sub = {r["id"]: r for r in _records(sp)}
|
|
75
|
+
for b in _records(bp):
|
|
76
|
+
s = sub.get(b["id"])
|
|
77
|
+
if s and b["status"] == "solved" and s["status"] == "solved":
|
|
78
|
+
x = math.log2(b["cpu_net"] / max(s["cpu_net"], 1e-3))
|
|
79
|
+
allp.append(x)
|
|
80
|
+
if b["cpu_net"] >= E.FLOOR_ABS:
|
|
81
|
+
elig.append(x)
|
|
82
|
+
return (statistics.median(elig) if elig else None, statistics.median(allp) if allp else None, len(elig))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def seed_claims(results, name):
|
|
86
|
+
out = []
|
|
87
|
+
for tag in TAGS:
|
|
88
|
+
sp = _ladder(results, tag, "submission", name)
|
|
89
|
+
out.append("missing" if not os.path.exists(sp) else _claim(sp))
|
|
90
|
+
return out
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _fmt(c):
|
|
94
|
+
return "pending" if c == "missing" else "inconclusive" if c is None else f"{c:.2f}"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def judge(results=None):
|
|
98
|
+
results = results or os.path.join(HERE, "results")
|
|
99
|
+
out = []
|
|
100
|
+
changed = code_changed(results)
|
|
101
|
+
out.append({"criterion": "code unchanged", "status": "fail" if changed else "pass",
|
|
102
|
+
"detail": ("changed since registration: " + ", ".join(changed) + ". Results measured with different code are not the registered experiment") if changed
|
|
103
|
+
else f"all {len(registered_hashes(results))} registered code hashes still match"})
|
|
104
|
+
for key, name in (("A", A_SOLVER), ("B", B_SOLVER)):
|
|
105
|
+
claims = seed_claims(results, name)
|
|
106
|
+
pm = paired_medians(results, name)
|
|
107
|
+
if "missing" in claims or pm is None:
|
|
108
|
+
out.append({"criterion": f"{key}: all seeds evaluated", "status": "pending", "detail": f"per-seed claims so far: {[_fmt(c) for c in claims]}"})
|
|
109
|
+
continue
|
|
110
|
+
conclusive = all(c is not None for c in claims)
|
|
111
|
+
claim = min(claims) if conclusive else None
|
|
112
|
+
elig, allm, n = pm
|
|
113
|
+
shown = f"claim {_fmt(claim) if conclusive else 'inconclusive'} bits (per seed {[_fmt(c) for c in claims]}); pooled paired median {'n/a' if elig is None else f'{elig:.2f}'} bits over {n} eligible pairs (all-pairs median {allm:.2f})"
|
|
114
|
+
if key == "A":
|
|
115
|
+
out.append({"criterion": "A1: every seed conclusive", "status": "pass" if conclusive else "fail", "detail": shown})
|
|
116
|
+
if conclusive:
|
|
117
|
+
ok2 = 0.7 <= claim <= 1.7 and (elig is None or claim <= elig + 0.3)
|
|
118
|
+
out.append({"criterion": "A2: claim in [0.7, 1.7] and not above the paired median + 0.3", "status": "pass" if ok2 else "fail", "detail": shown})
|
|
119
|
+
out.append({"criterion": "A3: claim >= 1.0 bit", "status": "pass" if claim >= 1.0 else "fail",
|
|
120
|
+
"detail": shown + ("" if claim >= 1.0 else ". A miss, not explained away: A's true effect sits at the lower edge of the stated range")})
|
|
121
|
+
else:
|
|
122
|
+
out += [{"criterion": c, "status": "fail", "detail": "an inconclusive seed: " + shown} for c in ("A2: claim in [0.7, 1.7] and not above the paired median + 0.3", "A3: claim >= 1.0 bit")]
|
|
123
|
+
else:
|
|
124
|
+
big = conclusive and claim >= 5.0
|
|
125
|
+
out.append({"criterion": "B1: not credited at 5 bits or more (the ceiling is real)", "status": "fail" if big else "pass", "detail": shown})
|
|
126
|
+
out.append({"criterion": "B2: paired median reported beside the claim", "status": "pass",
|
|
127
|
+
"detail": shown + ". The gap between them is the ceiling's size, not a failure"})
|
|
128
|
+
return out
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def overall(res):
|
|
132
|
+
st = {r["status"] for r in res}
|
|
133
|
+
return "FAIL" if "fail" in st else "PENDING" if "pending" in st else "PASS"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def card_html(results=None):
|
|
137
|
+
"""A Results-page card, or "" when the registration is absent. Pending criteria are shown as pending, never hidden."""
|
|
138
|
+
results = results or os.path.join(HERE, "results")
|
|
139
|
+
if not os.path.exists(os.path.join(results, PREREG)):
|
|
140
|
+
return ""
|
|
141
|
+
import html
|
|
142
|
+
res = judge(results)
|
|
143
|
+
label = {"pass": "Met", "fail": "Not met", "pending": "Not run yet"}
|
|
144
|
+
rows = "".join(f'<tr><td>{html.escape(r["criterion"])}</td><td><strong>{label[r["status"]]}</strong></td><td class="fine">{html.escape(r["detail"])}</td></tr>' for r in res)
|
|
145
|
+
return ('<section class="card"><h2>Does the compiler\'s prediction match real solvers? (dlog)</h2>'
|
|
146
|
+
'<p class="fine">Two real solvers were run through the full pipeline on three fresh seeds. What would count as success was written down before any data existed '
|
|
147
|
+
f'(<a href="prereg/{PREREG}">the registration</a>), and each criterion below is applied exactly as written, with no reinterpretation. '
|
|
148
|
+
'Local mode, one computer, no locked box: even a full pass is provisional.</p>'
|
|
149
|
+
f'<div class="scroll"><table class="lb"><thead><tr><th>Criterion</th><th>Outcome</th><th>The numbers</th></tr></thead><tbody>{rows}</tbody></table></div>'
|
|
150
|
+
f'<p class="fine">Overall: <strong>{overall(res)}</strong>.</p></section>')
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def main(argv=None):
|
|
154
|
+
res = judge()
|
|
155
|
+
if "--json" in (argv if argv is not None else sys.argv[1:]):
|
|
156
|
+
print(json.dumps(res, indent=1))
|
|
157
|
+
else:
|
|
158
|
+
for r in res:
|
|
159
|
+
print(f" {r['status'].upper():7s} {r['criterion']}: {r['detail']}")
|
|
160
|
+
print(f"\noverall: {overall(res)}")
|
|
161
|
+
sys.exit(0 if overall(res) == "PASS" else 1)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
if __name__ == "__main__":
|
|
165
|
+
main()
|