guldo-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. guldo/__init__.py +0 -0
  2. guldo/_src/COMPILER.md +329 -0
  3. guldo/_src/DEPLOY.md +200 -0
  4. guldo/_src/LAUNCH.md +85 -0
  5. guldo/_src/PROTOCOL.md +294 -0
  6. guldo/_src/RULES.md +25 -0
  7. guldo/_src/SUBMIT.md +170 -0
  8. guldo/_src/__init__.py +0 -0
  9. guldo/_src/attest.py +257 -0
  10. guldo/_src/build_site.py +958 -0
  11. guldo/_src/bundle_templates/check.py +1382 -0
  12. guldo/_src/bundle_templates/qcheck.py +253 -0
  13. guldo/_src/bundle_templates/qscore.py +402 -0
  14. guldo/_src/bundle_templates/score.py +649 -0
  15. guldo/_src/certificate.py +828 -0
  16. guldo/_src/certs_site.py +108 -0
  17. guldo/_src/claims.py +343 -0
  18. guldo/_src/compile_domain.py +1312 -0
  19. guldo/_src/compile_quality.py +286 -0
  20. guldo/_src/count_runner.py +248 -0
  21. guldo/_src/counted.py +274 -0
  22. guldo/_src/declared.py +215 -0
  23. guldo/_src/deploy/Caddyfile +48 -0
  24. guldo/_src/deploy/ledger-backup.service +9 -0
  25. guldo/_src/deploy/ledger-backup.sh +73 -0
  26. guldo/_src/deploy/ledger-backup.timer +10 -0
  27. guldo/_src/deploy/ledger-export.service +12 -0
  28. guldo/_src/deploy/ledger-export.timer +9 -0
  29. guldo/_src/deploy/ledger-web.service +38 -0
  30. guldo/_src/deploy/pin_cpu.sh +34 -0
  31. guldo/_src/deploy/provision_eval.sh +128 -0
  32. guldo/_src/deploy_smoke.py +148 -0
  33. guldo/_src/dlog_experiment.py +165 -0
  34. guldo/_src/domains/aidecode/compiled.json +586 -0
  35. guldo/_src/domains/aidecode/decl.py +134 -0
  36. guldo/_src/domains/aidecode/reference.py +73 -0
  37. guldo/_src/domains/aiinfer/compiled.json +722 -0
  38. guldo/_src/domains/aiinfer/decl.py +91 -0
  39. guldo/_src/domains/aiinfer/reference.py +42 -0
  40. guldo/_src/domains/dlog/compiled.json +248 -0
  41. guldo/_src/domains/dlog/decl.py +41 -0
  42. guldo/_src/domains/dlog/reference.py +10 -0
  43. guldo/_src/domains/maxcut/compiled.json +291 -0
  44. guldo/_src/domains/maxcut/decl.py +46 -0
  45. guldo/_src/domains/maxcut/reference.py +16 -0
  46. guldo/_src/drift_check.py +89 -0
  47. guldo/_src/export_bundle.py +750 -0
  48. guldo/_src/extrapolate.py +326 -0
  49. guldo/_src/families.py +53 -0
  50. guldo/_src/gen_cheat_table.py +36 -0
  51. guldo/_src/gpu_meter.py +634 -0
  52. guldo/_src/gpu_runner.py +63 -0
  53. guldo/_src/harness.py +273 -0
  54. guldo/_src/instances.py +86 -0
  55. guldo/_src/instances_factor.py +40 -0
  56. guldo/_src/instances_mlwe.py +69 -0
  57. guldo/_src/judge.py +278 -0
  58. guldo/_src/limits_plan.py +158 -0
  59. guldo/_src/official.py +189 -0
  60. guldo/_src/ops.py +281 -0
  61. guldo/_src/portal.py +313 -0
  62. guldo/_src/portal_core.py +57 -0
  63. guldo/_src/power.py +174 -0
  64. guldo/_src/prereg.py +173 -0
  65. guldo/_src/public_portal.py +500 -0
  66. guldo/_src/quality_core.py +126 -0
  67. guldo/_src/recheck.py +236 -0
  68. guldo/_src/remeasure.py +88 -0
  69. guldo/_src/report_html.py +493 -0
  70. guldo/_src/results/.keep +0 -0
  71. guldo/_src/rounds.py +846 -0
  72. guldo/_src/runner.py +52 -0
  73. guldo/_src/sandbox/Dockerfile +11 -0
  74. guldo/_src/sandbox/Dockerfile.v2 +19 -0
  75. guldo/_src/sandbox/README.md +40 -0
  76. guldo/_src/sandbox/linux_selftest.py +117 -0
  77. guldo/_src/sandbox/meter.py +89 -0
  78. guldo/_src/sandbox/meter_core.py +28 -0
  79. guldo/_src/site_assets.py +176 -0
  80. guldo/_src/site_live_check.py +79 -0
  81. guldo/_src/solvers/aidecode_flops_kv.py +73 -0
  82. guldo/_src/solvers/aidecode_flops_lossy.py +73 -0
  83. guldo/_src/solvers/aidecode_flops_ref.py +64 -0
  84. guldo/_src/solvers/aidecode_kv.py +76 -0
  85. guldo/_src/solvers/aidecode_kv_f32.py +77 -0
  86. guldo/_src/solvers/aidecode_lossy.py +76 -0
  87. guldo/_src/solvers/aidecode_repeat.py +5 -0
  88. guldo/_src/solvers/aiinfer_batched.py +40 -0
  89. guldo/_src/solvers/aiinfer_f32.py +41 -0
  90. guldo/_src/solvers/aiinfer_gpu_batched.py +42 -0
  91. guldo/_src/solvers/aiinfer_gpu_ref.py +45 -0
  92. guldo/_src/solvers/aiinfer_lossy.py +42 -0
  93. guldo/_src/solvers/baseline_bkz.py +14 -0
  94. guldo/_src/solvers/cheat_lookup.py +24 -0
  95. guldo/_src/solvers/cheat_table.json +1 -0
  96. guldo/_src/solvers/cheat_table_v1_stale.json +1 -0
  97. guldo/_src/solvers/daemon_cpu_hider.py +17 -0
  98. guldo/_src/solvers/dlog_bsgs.py +19 -0
  99. guldo/_src/solvers/dlog_stride4.py +18 -0
  100. guldo/_src/solvers/egg_base.py +11 -0
  101. guldo/_src/solvers/egg_eightaxis.py +68 -0
  102. guldo/_src/solvers/egg_template.py +11 -0
  103. guldo/_src/solvers/factor_brent.py +34 -0
  104. guldo/_src/solvers/factor_floyd.py +19 -0
  105. guldo/_src/solvers/factor_trial_fast_py.py +21 -0
  106. guldo/_src/solvers/factor_trial_np.py +21 -0
  107. guldo/_src/solvers/factor_trial_py.py +12 -0
  108. guldo/_src/solvers/fixed_overhead_ref.py +21 -0
  109. guldo/_src/solvers/fork_bomb.py +18 -0
  110. guldo/_src/solvers/gpu_noop.py +6 -0
  111. guldo/_src/solvers/hostile_probe.py +36 -0
  112. guldo/_src/solvers/kill_meter.py +27 -0
  113. guldo/_src/solvers/memory_bomb.py +10 -0
  114. guldo/_src/solvers/noop.py +5 -0
  115. guldo/_src/solvers/orphan_cpu_hider.py +16 -0
  116. guldo/_src/solvers/planted_peek.py +23 -0
  117. guldo/_src/solvers/sleeping_daemon_hang.py +12 -0
  118. guldo/_src/solvers/small_n_trick.py +22 -0
  119. guldo/_src/solvers/starter_template.py +21 -0
  120. guldo/_src/solvers/stdout_flood.py +9 -0
  121. guldo/_src/solvers/tmp_fill.py +10 -0
  122. guldo/_src/solvers/tuned_bkz.py +25 -0
  123. guldo/_src/static_check.py +214 -0
  124. guldo/_src/submit_check.py +166 -0
  125. guldo/_src/svp.py +889 -0
  126. guldo/_src/tensorops.py +304 -0
  127. guldo/_src/tflops.py +453 -0
  128. guldo/_src/tflops_runner.py +134 -0
  129. guldo/cli.py +60 -0
  130. guldo_cli-0.1.0.dist-info/METADATA +30 -0
  131. guldo_cli-0.1.0.dist-info/RECORD +134 -0
  132. guldo_cli-0.1.0.dist-info/WHEEL +5 -0
  133. guldo_cli-0.1.0.dist-info/entry_points.txt +2 -0
  134. guldo_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,38 @@
1
+ # The public web tier. Runs as an unprivileged user that CANNOT read the seeds or reach Docker.
2
+ # Install: cp deploy/ledger-web.service /etc/systemd/system/ && systemctl daemon-reload && systemctl enable --now ledger-web
3
+ [Unit]
4
+ Description=Speedup Ledger submission service (web tier)
5
+ After=network.target
6
+
7
+ [Service]
8
+ User=ledger-web
9
+ Group=ledger-web
10
+ WorkingDirectory=/opt/speedup-verification
11
+ ExecStart=/usr/bin/python3 public_portal.py serve --data /srv/ledger/data --host ledger.example.org --bind 127.0.0.1 --port 8080 --trust-proxy --site-url https://ledger.example.org
12
+ Restart=on-failure
13
+ RestartSec=3
14
+ # the operator user (group "ledger") also writes the data directory (closing rounds, evaluation results), so files must be group-writable
15
+ UMask=0007
16
+ SupplementaryGroups=ledger
17
+
18
+ # Containment: the process only ever parses uploads, but assume a bug in the parser.
19
+ NoNewPrivileges=true
20
+ ProtectSystem=strict
21
+ ReadWritePaths=/srv/ledger/data
22
+ ProtectHome=true
23
+ PrivateTmp=true
24
+ PrivateDevices=true
25
+ ProtectKernelTunables=true
26
+ ProtectControlGroups=true
27
+ RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
28
+ RestrictNamespaces=true
29
+ LockPersonality=true
30
+ MemoryMax=512M
31
+ TasksMax=128
32
+ LimitNOFILE=1024
33
+ CapabilityBoundingSet=
34
+ SystemCallFilter=@system-service
35
+ SystemCallErrorNumber=EPERM
36
+
37
+ [Install]
38
+ WantedBy=multi-user.target
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env bash
2
+ # Run a command (normally the evaluation worker) so that every `docker run` it starts is pinned to the given CPU core(s).
3
+ #
4
+ # bash deploy/pin_cpu.sh 1 -- python3 ops.py worker --data /srv/ledger/data --id lattice-1 --seed-dir /srv/ledger/seeds/lattice-1
5
+ # bash deploy/pin_cpu.sh 2,3 -- ... (a list or a range such as 2-3 also works)
6
+ #
7
+ # Why a wrapper: the harness builds its docker command in harness.py, which is one of the hashed files a round's fingerprint pins, so it must not change
8
+ # while a round is open. The wrapper changes nothing hashed: it puts a tiny `docker` shim first in PATH, and the shim adds `--cpuset-cpus <set>` right
9
+ # after `run` and otherwise passes everything through to the real docker unchanged. Pin to a core the web service and the operating system do not
10
+ # need (on a 2-core machine, core 1 leaves core 0 for them), and say which core you used when you publish the round's results: the harness records the
11
+ # CPU model but not the pinning.
12
+ set -euo pipefail
13
+
14
+ [ "$#" -ge 3 ] && [ "$2" = "--" ] || { sed -n '2,12p' "$0"; exit 2; }
15
+ SET="$1"; shift 2
16
+ [[ "$SET" =~ ^[0-9]+([,-][0-9]+)*$ ]] || { echo "bad CPU set: $SET (use e.g. 1, 2,3 or 2-3)"; exit 2; }
17
+
18
+ REAL="$(command -v docker || true)"
19
+ [ -n "$REAL" ] || { echo "docker not found in PATH"; exit 1; }
20
+ SHIM_DIR="$(mktemp -d)"
21
+ trap 'rm -rf "$SHIM_DIR"' EXIT
22
+
23
+ cat > "$SHIM_DIR/docker" <<EOF
24
+ #!/usr/bin/env bash
25
+ if [ "\${1:-}" = "run" ]; then
26
+ shift
27
+ exec "$REAL" run --cpuset-cpus "$SET" "\$@"
28
+ fi
29
+ exec "$REAL" "\$@"
30
+ EOF
31
+ chmod +x "$SHIM_DIR/docker"
32
+
33
+ echo "pin_cpu: every docker run is pinned to CPU set $SET" >&2
34
+ PATH="$SHIM_DIR:$PATH" "$@"
@@ -0,0 +1,128 @@
1
+ #!/usr/bin/env bash
2
+ # Provision a DISPOSABLE evaluation machine for timed rounds, from a fresh Ubuntu (22.04 or newer) or Debian server, in one command.
3
+ # Run it AS ROOT, from a copy of this project that is on the machine (rsync the project there first; EVAL_BOX.md has the exact command):
4
+ #
5
+ # sudo bash deploy/provision_eval.sh [--no-firewall] [--no-ssh-hardening] [--skip-gates]
6
+ #
7
+ # What it does, in order (every step is safe to repeat):
8
+ # packages (Docker, Python, numpy, cryptography, rsync) -> swap if the machine has little memory -> the unprivileged operator user `ledger-ops`
9
+ # (in the docker group on a disposable machine; with --shared-with-web never) -> the data and seed directories ->
10
+ # the code under /opt/speedup-verification (root-owned, not writable by others) -> the firewall (only ssh) and key-only ssh ->
11
+ # the sandbox image -> the Linux meter self-test -> the whole sandbox test suite and the SANDBOX_VERIFIED marker the worker requires.
12
+ # It stops at the first failure and says which step failed. It never touches seeds, DNS, or any other machine.
13
+ set -euo pipefail
14
+
15
+ FIREWALL=1; HARDEN_SSH=1; GATES=1; SHARED=0
16
+ for a in "$@"; do
17
+ case "$a" in
18
+ --shared-with-web) SHARED=1 ;; # this machine ALSO runs the web portal: keep its users, data-directory permissions and seeds exactly as they are
19
+ --no-firewall) FIREWALL=0 ;;
20
+ --no-ssh-hardening) HARDEN_SSH=0 ;;
21
+ --skip-gates) GATES=0 ;;
22
+ -h|--help) sed -n '2,15p' "$0"; exit 0 ;;
23
+ *) echo "unknown option: $a"; exit 2 ;;
24
+ esac
25
+ done
26
+
27
+ step() { printf '\n== %s\n' "$*"; }
28
+ die() { printf '\nFAILED at: %s\n' "$*" >&2; exit 1; }
29
+ trap 'die "${STEP:-startup} (line $LINENO)"' ERR
30
+
31
+ [ "$(id -u)" = 0 ] || { echo "run as root (sudo bash deploy/provision_eval.sh)"; exit 1; }
32
+ command -v apt-get >/dev/null || { echo "this script supports apt-based systems (Ubuntu, Debian)"; exit 1; }
33
+ SRC="$(cd "$(dirname "$0")/.." && pwd)"
34
+ DEST=/opt/speedup-verification
35
+ DATA=/srv/ledger/data
36
+ [ -f "$SRC/harness.py" ] && [ -f "$SRC/sandbox/Dockerfile.v2" ] || { echo "run it from a copy of the project (harness.py and sandbox/ not found in $SRC)"; exit 1; }
37
+
38
+ STEP="packages"; step "$STEP"
39
+ export DEBIAN_FRONTEND=noninteractive
40
+ apt-get update -qq
41
+ apt-get install -y -qq python3 python3-numpy python3-cryptography rsync ca-certificates curl >/dev/null
42
+ if ! command -v docker >/dev/null; then apt-get install -y -qq docker.io >/dev/null; fi
43
+ systemctl enable --now docker >/dev/null 2>&1 || true
44
+ docker version --format 'docker {{.Server.Version}}' || die "docker is not answering"
45
+
46
+ STEP="swap"; step "$STEP"
47
+ mem_mb=$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)
48
+ if [ "$mem_mb" -lt 3500 ] && ! swapon --show | grep -q .; then
49
+ fallocate -l 4G /swapfile && chmod 600 /swapfile && mkswap /swapfile >/dev/null && swapon /swapfile
50
+ grep -q '^/swapfile' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab
51
+ echo "added 4 GB of swap (RAM is ${mem_mb} MB)"
52
+ else
53
+ echo "no swap needed (RAM ${mem_mb} MB, or swap already present)"
54
+ fi
55
+
56
+ STEP="users and directories"; step "$STEP"
57
+ id ledger-ops >/dev/null 2>&1 || useradd -m -s /bin/bash ledger-ops
58
+ # Membership of the docker group is root-equivalent. On a disposable evaluation machine that is acceptable. On a machine that also serves the portal it is
59
+ # NOT: ledger-ops is the account that evaluates strangers' counted solvers, so it is never put in the group there. Docker is reached instead by running a
60
+ # command as root with `sudo -u ledger-ops -g docker ...`, which gives that one process the docker group as its primary group and nothing persistent.
61
+ if [ "$SHARED" = 0 ]; then usermod -aG docker ledger-ops; fi
62
+ if [ "$SHARED" = 1 ]; then
63
+ # the web tier writes this directory through the group `ledger` (setgid, 2770): do not tighten it, or submissions stop working
64
+ [ -d "$DATA/rounds" ] || { echo "--shared-with-web expects the web layout from DEPLOY.md already in place ($DATA/rounds missing)"; exit 1; }
65
+ install -d -o ledger-ops -g ledger-ops -m 0755 /srv/ledger/results
66
+ echo "shared machine: left $DATA and /srv/ledger/seeds exactly as they are"
67
+ else
68
+ install -d -o ledger-ops -g ledger-ops -m 0755 /srv/ledger
69
+ install -d -o ledger-ops -g ledger-ops -m 0750 "$DATA" "$DATA/rounds"
70
+ install -d -o ledger-ops -g ledger-ops -m 0700 /srv/ledger/seeds
71
+ install -d -o ledger-ops -g ledger-ops -m 0755 /srv/ledger/results
72
+ fi
73
+ echo "operator user ledger-ops; data $DATA; seeds /srv/ledger/seeds (0700)"
74
+ # round_sync.py logs in as ledger-ops (not root) so that copied files get the right owner; it uses the same key you used for root
75
+ if [ -s /root/.ssh/authorized_keys ]; then
76
+ install -d -o ledger-ops -g ledger-ops -m 0700 /home/ledger-ops/.ssh
77
+ install -o ledger-ops -g ledger-ops -m 0600 /root/.ssh/authorized_keys /home/ledger-ops/.ssh/authorized_keys
78
+ echo "ssh key installed for ledger-ops"
79
+ fi
80
+
81
+ STEP="code"; step "$STEP"
82
+ mkdir -p "$DEST"
83
+ if [ "$SRC" != "$DEST" ]; then
84
+ rsync -a --delete --exclude .git --exclude __pycache__ --exclude /site --exclude /dist --exclude /ledger --exclude /ledger-seeds \
85
+ --exclude /ledger-submissions --exclude /results --exclude .vercel "$SRC"/ "$DEST"/
86
+ fi
87
+ install -d -o ledger-ops -g ledger-ops -m 0755 "$DEST/results"
88
+ chown -R root:root "$DEST"; chown ledger-ops:ledger-ops "$DEST/results"; chmod -R go-w "$DEST"
89
+ echo "code installed in $DEST"
90
+
91
+ if [ "$FIREWALL" = 1 ]; then
92
+ STEP="firewall"; step "$STEP"
93
+ apt-get install -y -qq ufw fail2ban >/dev/null
94
+ ufw allow OpenSSH >/dev/null; ufw --force enable >/dev/null
95
+ ufw status | head -4
96
+ fi
97
+
98
+ if [ "$HARDEN_SSH" = 1 ]; then
99
+ STEP="ssh hardening"; step "$STEP"
100
+ if [ -s /root/.ssh/authorized_keys ]; then
101
+ printf 'PasswordAuthentication no\nPermitRootLogin prohibit-password\n' > /etc/ssh/sshd_config.d/10-hardening.conf
102
+ sshd -t && (systemctl reload ssh 2>/dev/null || systemctl reload sshd)
103
+ echo "password logins are off; root may log in with a key only"
104
+ else
105
+ echo "SKIPPED: /root/.ssh/authorized_keys is empty, and turning passwords off now would lock you out. Add your key, then rerun."
106
+ fi
107
+ fi
108
+
109
+ STEP="sandbox image"; step "$STEP"
110
+ cd "$DEST"
111
+ BUILD="docker build"; docker buildx version >/dev/null 2>&1 && BUILD="docker build --load" # Ubuntu's docker.io has no buildx and no --load flag
112
+ $BUILD -t svp-solver -f sandbox/Dockerfile.v2 . >/tmp/provision-build.log 2>&1 || { tail -20 /tmp/provision-build.log; die "docker build (log above)"; }
113
+ docker image inspect svp-solver --format 'image {{.Id}}' | cut -c1-40
114
+
115
+ if [ "$GATES" = 1 ]; then
116
+ STEP="Linux meter self-test"; step "$STEP"
117
+ python3 sandbox/linux_selftest.py
118
+ STEP="sandbox tests and marker"; step "$STEP"
119
+ mkdir -p "$DATA"; [ "$SHARED" = 1 ] || chown ledger-ops:ledger-ops "$DATA"
120
+ sudo -u ledger-ops -g docker python3 launch_gates.py --only env,docker --with-docker --data "$DATA"
121
+ sudo -u ledger-ops python3 -c "import portal_core, sys; sys.exit(0 if portal_core.sandbox_verified('$DATA') else 1)" || die "the SANDBOX_VERIFIED marker is missing or stale"
122
+ echo "sandbox VERIFIED on this machine: the worker will now evaluate timed rounds here"
123
+ fi
124
+
125
+ if [ "$SHARED" = 1 ]; then
126
+ printf '\nNote: ledger-ops is NOT in the docker group on this shared machine, on purpose. Run the timed worker as root with\n sudo -u ledger-ops -g docker python3 ops.py worker ...\nso that only that process has Docker access (EVAL_BOX.md).\n'
127
+ fi
128
+ printf '\nDone. Next (EVAL_BOX.md): push a closed round here with round_sync.py, run ops.py worker as ledger-ops, then pull it back.\n'
@@ -0,0 +1,148 @@
1
+ """Smoke test for a RUNNING submission service: is what is actually listening what DEPLOY.md says it is?
2
+
3
+ python3 deploy_smoke.py https://ledger.example.org [--submit FILE --round ID --handle NAME]
4
+
5
+ Read-only by default (a few GETs and two requests that must be REFUSED). It checks, against the live address:
6
+ * /healthz answers, over TLS if the address is https (a bad certificate fails here, not silently);
7
+ * the security headers the page code promises are really on the wire (a proxy that strips them is a finding);
8
+ * a request with a foreign Host header and a cross-site Origin are refused (DNS-rebinding and CSRF controls);
9
+ * a submission without the rules accepted is refused with a JSON error and stores nothing;
10
+ * every round's public log verifies (hash chain, commitments in entry 0) with YOUR copy of the checker, and matches the round's description;
11
+ * the JSON API does not leak a seed before the reveal.
12
+ With --submit it also makes ONE real submission through svp.py (a real, logged, permanent entry: use a test round).
13
+ Cost to know: the refusal probe is a POST, and the portal rate-limits POSTs before it reads the form, so each OPEN round spends one of your
14
+ address's submission tokens (burst 3, 6 per hour). Run it from somewhere that is not also the address you submit from, or wait an hour.
15
+ Exit status: 0 all passed, 1 something failed, 2 could not reach the service.
16
+ """
17
+ import argparse
18
+ import http.client
19
+ import json
20
+ import sys
21
+ import urllib.parse
22
+
23
+ import certificate
24
+ import rounds
25
+ import svp
26
+
27
+ REQUIRED_HEADERS = {"x-content-type-options": "nosniff", "x-frame-options": "DENY", "referrer-policy": "no-referrer"}
28
+
29
+
30
+ def _conn(server):
31
+ u = urllib.parse.urlsplit(server)
32
+ cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection
33
+ return cls(u.hostname, u.port or (443 if u.scheme == "https" else 80), timeout=20)
34
+
35
+
36
+ def _raw(server, method, path, headers=None, body=None):
37
+ c = _conn(server)
38
+ try:
39
+ c.request(method, path, body=body, headers=headers or {})
40
+ r = c.getresponse()
41
+ return r.status, {k.lower(): v for k, v in r.getheaders()}, r.read(svp.MAX_JSON + 1)
42
+ finally:
43
+ c.close()
44
+
45
+
46
+ def smoke(server, submit=None):
47
+ """-> [(status, check, detail)] with status in pass / fail / warn. A connection that drops or a server that answers nonsense half-way through
48
+ is a FINDING in the report, never a traceback."""
49
+ out = []
50
+ try:
51
+ _smoke(server, submit, out)
52
+ except (OSError, http.client.HTTPException, ValueError) as e:
53
+ out.append(("fail", "connection", f"the service stopped answering properly mid-test ({type(e).__name__}: {svp.clean(e, 100)})"))
54
+ except (AttributeError, TypeError, KeyError, IndexError) as e: # JSON of an unexpected shape (a log whose entries are not objects, ...)
55
+ out.append(("fail", "response shape", f"the service sent data in an unexpected shape ({type(e).__name__}: {svp.clean(e, 100)})"))
56
+ return out
57
+
58
+
59
+ def _smoke(server, submit, out):
60
+ add = lambda st, name, detail="": out.append((st, name, detail))
61
+ u = urllib.parse.urlsplit(server)
62
+ try:
63
+ st, hdr, body = _raw(server, "GET", "/healthz")
64
+ except (OSError, http.client.HTTPException) as e:
65
+ out.append(("fail", "reachable", f"cannot reach {server}: {type(e).__name__}: {svp.clean(e, 100)}"))
66
+ return
67
+ add("pass" if st == 200 and body.strip() == b"ok" else "fail", "healthz", f"HTTP {st}")
68
+ add("pass" if u.scheme == "https" else "warn", "transport",
69
+ "https (the certificate verified, or the connection would have failed)" if u.scheme == "https" else
70
+ "plain http: fine on loopback, but a public service must be https (tokens and sources travel in the clear otherwise)")
71
+ st, hdr, body = _raw(server, "GET", "/")
72
+ missing = [k for k, v in REQUIRED_HEADERS.items() if hdr.get(k) != v]
73
+ add("pass" if st == 200 and not missing else "fail", "security headers", "present" if not missing else "missing or wrong: " + ", ".join(missing))
74
+ csp = hdr.get("content-security-policy", "")
75
+ add("pass" if "default-src 'none'" in csp and "frame-ancestors 'none'" in csp else "fail", "content security policy", svp.clean(csp, 80) or "absent")
76
+ if u.scheme == "https":
77
+ add("pass" if "max-age" in hdr.get("strict-transport-security", "") else "warn", "HSTS", hdr.get("strict-transport-security", "absent (the proxy may add it)"))
78
+ st, _, _ = _raw(server, "GET", "/", headers={"Host": "evil.invalid"})
79
+ add("pass" if st == 403 else "fail", "foreign Host refused", f"HTTP {st} (want 403)")
80
+ st, _, _ = _raw(server, "POST", "/round/nonexistent/submit", headers={"Origin": "https://evil.invalid", "Content-Type": "multipart/form-data; boundary=x",
81
+ "Content-Length": "0"})
82
+ add("pass" if st == 403 else "fail", "cross-site POST refused", f"HTTP {st} (want 403)")
83
+ try:
84
+ data = svp.get_json(server, "/api/rounds.json")
85
+ except svp.SvpError as e:
86
+ add("fail", "rounds api", str(e))
87
+ return
88
+ if not isinstance(data, list) or not all(isinstance(r, dict) for r in data):
89
+ add("fail", "rounds api", "the rounds list is malformed")
90
+ return
91
+ add("pass", "rounds api", f"{len(data)} round(s)") # only after the shape is known good: one finding per check, never a PASS and a FAIL
92
+ for r in data:
93
+ rid = r.get("id", "?")
94
+ try:
95
+ rj = svp._round(server, rid)
96
+ status, logbody = svp.request(server, f"/api/log/{rid}.jsonl")
97
+ entries = svp.parse_log(logbody) if status == 200 else []
98
+ ok, why = certificate.commit_published(entries, rj.get("commitments", []))
99
+ same = entries and entries[0].get("data", {}).get("commitments") == rj.get("commitments")
100
+ add("pass" if ok and same and rj.get("log_ok") else "fail", f"round {rid}: log", why if ok else f"does not verify: {why}")
101
+ leaked = rj.get("state") not in ("revealed",) and bool(rj.get("seeds"))
102
+ add("fail" if leaked else "pass", f"round {rid}: no seed before the reveal", f"state {rj.get('state')}")
103
+ if rj.get("state") == "open":
104
+ body, ctype = svp.multipart({"handle": "smoketest"}, "x.py", b"def solve(*a):\n return None\n") # no `agree` field on purpose
105
+ st, _, resp = _raw(server, "POST", f"/round/{rid}/submit", headers={"Content-Type": ctype, "Accept": "application/json"}, body=body)
106
+ try:
107
+ parsed = json.loads(resp)
108
+ msg = parsed.get("error", "") if isinstance(parsed, dict) else "" # a service answering with a list is a failed check, not a crash
109
+ except ValueError:
110
+ msg = ""
111
+ add("pass" if st == 422 and "rules" in msg else "fail", f"round {rid}: refuses without the rules accepted", f"HTTP {st}: {svp.clean(msg, 80)}")
112
+ except svp.SvpError as e:
113
+ add("fail", f"round {rid}", str(e))
114
+ if submit:
115
+ import argparse as _ap
116
+ file, rid, handle, home = submit
117
+ a = _ap.Namespace(server=server, file=file, round=rid, handle=handle, accept_rules=True, publish_source=False, token=None, no_check=False,
118
+ allow_insecure_http=False, home=home, json=False)
119
+ try:
120
+ rc = svp.cmd_submit(a)
121
+ except svp.SvpError as e:
122
+ add("fail", "real submission", str(e))
123
+ else:
124
+ add("pass" if rc == 0 else "fail", "real submission", "accepted and confirmed in the public log" if rc == 0 else f"svp exit {rc}")
125
+
126
+
127
+ def main(argv=None):
128
+ ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
129
+ ap.add_argument("server")
130
+ ap.add_argument("--submit", metavar="FILE", help="make ONE real submission with svp.py (permanent: use a test round)")
131
+ ap.add_argument("--round")
132
+ ap.add_argument("--handle")
133
+ ap.add_argument("--home", help="where svp keeps the receipt")
134
+ a = ap.parse_args(argv)
135
+ if a.submit and not (a.round and a.handle):
136
+ raise SystemExit("--submit needs --round and --handle")
137
+ res = smoke(svp.normalize_server(a.server), (a.submit, a.round, a.handle, a.home) if a.submit else None)
138
+ for st, name, detail in res:
139
+ print(f" {st.upper():4s} {name}: {detail}")
140
+ if res and res[0][1] == "reachable":
141
+ return 2
142
+ bad = [r for r in res if r[0] == "fail"]
143
+ print("\nALL CHECKS PASSED" if not bad else f"\n{len(bad)} CHECK(S) FAILED: do not open a round until these are fixed")
144
+ return 1 if bad else 0
145
+
146
+
147
+ if __name__ == "__main__":
148
+ sys.exit(main())
@@ -0,0 +1,165 @@
1
+ """Judge the pre-registered dlog experiment (results/PREREGISTERED_dlog_realsolver.txt) mechanically, with no reinterpretation.
2
+
3
+ python3 dlog_experiment.py print each criterion as PASS / FAIL / PENDING, and the numbers behind it
4
+ python3 dlog_experiment.py --json the same, as JSON
5
+
6
+ Reads only saved files: the three baseline ladders, the two submissions' ladders and the primary estimator's claim file next to each (written
7
+ by judge.py at evaluation time). It also re-hashes every code file the registration lists and says so if any changed since, because a result
8
+ measured with different code is not the registered experiment.
9
+
10
+ Definitions fixed here, in code, before the data were read: a claim is the MINIMUM of the three seeds' credits (judge.py's rule: any seed with
11
+ no conclusion makes the claim inconclusive); the "pooled paired median" is the median over all three seeds of log2(baseline cost / solver cost)
12
+ on every instance both solvers solved whose BASELINE cost is at least the estimator's 0.5 s floor (the same instances the credit may use).
13
+ The all-pairs median (no floor) is reported beside it and is not used by any criterion.
14
+ """
15
+ import hashlib
16
+ import json
17
+ import math
18
+ import os
19
+ import re
20
+ import statistics
21
+ import sys
22
+
23
+ import extrapolate as E
24
+
25
+ HERE = os.path.dirname(os.path.abspath(__file__))
26
+ PREREG = "PREREGISTERED_dlog_realsolver.txt"
27
+ TAGS = ("e5dab2c8", "d00100a7", "e70be0d8") # the first 8 hex of the three registered seed commitments
28
+ A_SOLVER, B_SOLVER = "dlog_stride4.py", "dlog_bsgs.py"
29
+ BASELINE = "reference.py"
30
+
31
+
32
+ def _ladder(results, tag, role, name):
33
+ return os.path.join(results, f"judged_dlog_{tag}_{role}_{name}.jsonl")
34
+
35
+
36
+ def _records(path):
37
+ with open(path, encoding="utf-8") as f:
38
+ return [r for r in map(json.loads, (ln for ln in f if ln.strip())) if not r.get("meta")]
39
+
40
+
41
+ def _claim(path):
42
+ p = path + ".claim.json"
43
+ if not os.path.exists(p):
44
+ return "missing"
45
+ with open(p, encoding="utf-8") as f:
46
+ return json.load(f)["measured"] # None = inconclusive
47
+
48
+
49
+ def registered_hashes(results):
50
+ with open(os.path.join(results, PREREG), encoding="utf-8") as f:
51
+ return re.findall(r"^([0-9a-f]{64}) (\S+)$", f.read(), re.M)
52
+
53
+
54
+ def code_changed(results):
55
+ bad = []
56
+ for want, rel in registered_hashes(results):
57
+ p = os.path.join(HERE, rel)
58
+ got = None
59
+ if os.path.exists(p):
60
+ with open(p, "rb") as f:
61
+ got = hashlib.sha256(f.read()).hexdigest()
62
+ if got != want:
63
+ bad.append(rel)
64
+ return bad
65
+
66
+
67
+ def paired_medians(results, name):
68
+ """(median over floor-eligible pairs, median over all pairs, number of eligible pairs); None when a ladder is missing."""
69
+ elig, allp = [], []
70
+ for tag in TAGS:
71
+ bp, sp = _ladder(results, tag, "baseline", BASELINE), _ladder(results, tag, "submission", name)
72
+ if not (os.path.exists(bp) and os.path.exists(sp)):
73
+ return None
74
+ sub = {r["id"]: r for r in _records(sp)}
75
+ for b in _records(bp):
76
+ s = sub.get(b["id"])
77
+ if s and b["status"] == "solved" and s["status"] == "solved":
78
+ x = math.log2(b["cpu_net"] / max(s["cpu_net"], 1e-3))
79
+ allp.append(x)
80
+ if b["cpu_net"] >= E.FLOOR_ABS:
81
+ elig.append(x)
82
+ return (statistics.median(elig) if elig else None, statistics.median(allp) if allp else None, len(elig))
83
+
84
+
85
+ def seed_claims(results, name):
86
+ out = []
87
+ for tag in TAGS:
88
+ sp = _ladder(results, tag, "submission", name)
89
+ out.append("missing" if not os.path.exists(sp) else _claim(sp))
90
+ return out
91
+
92
+
93
+ def _fmt(c):
94
+ return "pending" if c == "missing" else "inconclusive" if c is None else f"{c:.2f}"
95
+
96
+
97
+ def judge(results=None):
98
+ results = results or os.path.join(HERE, "results")
99
+ out = []
100
+ changed = code_changed(results)
101
+ out.append({"criterion": "code unchanged", "status": "fail" if changed else "pass",
102
+ "detail": ("changed since registration: " + ", ".join(changed) + ". Results measured with different code are not the registered experiment") if changed
103
+ else f"all {len(registered_hashes(results))} registered code hashes still match"})
104
+ for key, name in (("A", A_SOLVER), ("B", B_SOLVER)):
105
+ claims = seed_claims(results, name)
106
+ pm = paired_medians(results, name)
107
+ if "missing" in claims or pm is None:
108
+ out.append({"criterion": f"{key}: all seeds evaluated", "status": "pending", "detail": f"per-seed claims so far: {[_fmt(c) for c in claims]}"})
109
+ continue
110
+ conclusive = all(c is not None for c in claims)
111
+ claim = min(claims) if conclusive else None
112
+ elig, allm, n = pm
113
+ shown = f"claim {_fmt(claim) if conclusive else 'inconclusive'} bits (per seed {[_fmt(c) for c in claims]}); pooled paired median {'n/a' if elig is None else f'{elig:.2f}'} bits over {n} eligible pairs (all-pairs median {allm:.2f})"
114
+ if key == "A":
115
+ out.append({"criterion": "A1: every seed conclusive", "status": "pass" if conclusive else "fail", "detail": shown})
116
+ if conclusive:
117
+ ok2 = 0.7 <= claim <= 1.7 and (elig is None or claim <= elig + 0.3)
118
+ out.append({"criterion": "A2: claim in [0.7, 1.7] and not above the paired median + 0.3", "status": "pass" if ok2 else "fail", "detail": shown})
119
+ out.append({"criterion": "A3: claim >= 1.0 bit", "status": "pass" if claim >= 1.0 else "fail",
120
+ "detail": shown + ("" if claim >= 1.0 else ". A miss, not explained away: A's true effect sits at the lower edge of the stated range")})
121
+ else:
122
+ out += [{"criterion": c, "status": "fail", "detail": "an inconclusive seed: " + shown} for c in ("A2: claim in [0.7, 1.7] and not above the paired median + 0.3", "A3: claim >= 1.0 bit")]
123
+ else:
124
+ big = conclusive and claim >= 5.0
125
+ out.append({"criterion": "B1: not credited at 5 bits or more (the ceiling is real)", "status": "fail" if big else "pass", "detail": shown})
126
+ out.append({"criterion": "B2: paired median reported beside the claim", "status": "pass",
127
+ "detail": shown + ". The gap between them is the ceiling's size, not a failure"})
128
+ return out
129
+
130
+
131
+ def overall(res):
132
+ st = {r["status"] for r in res}
133
+ return "FAIL" if "fail" in st else "PENDING" if "pending" in st else "PASS"
134
+
135
+
136
+ def card_html(results=None):
137
+ """A Results-page card, or "" when the registration is absent. Pending criteria are shown as pending, never hidden."""
138
+ results = results or os.path.join(HERE, "results")
139
+ if not os.path.exists(os.path.join(results, PREREG)):
140
+ return ""
141
+ import html
142
+ res = judge(results)
143
+ label = {"pass": "Met", "fail": "Not met", "pending": "Not run yet"}
144
+ rows = "".join(f'<tr><td>{html.escape(r["criterion"])}</td><td><strong>{label[r["status"]]}</strong></td><td class="fine">{html.escape(r["detail"])}</td></tr>' for r in res)
145
+ return ('<section class="card"><h2>Does the compiler\'s prediction match real solvers? (dlog)</h2>'
146
+ '<p class="fine">Two real solvers were run through the full pipeline on three fresh seeds. What would count as success was written down before any data existed '
147
+ f'(<a href="prereg/{PREREG}">the registration</a>), and each criterion below is applied exactly as written, with no reinterpretation. '
148
+ 'Local mode, one computer, no locked box: even a full pass is provisional.</p>'
149
+ f'<div class="scroll"><table class="lb"><thead><tr><th>Criterion</th><th>Outcome</th><th>The numbers</th></tr></thead><tbody>{rows}</tbody></table></div>'
150
+ f'<p class="fine">Overall: <strong>{overall(res)}</strong>.</p></section>')
151
+
152
+
153
+ def main(argv=None):
154
+ res = judge()
155
+ if "--json" in (argv if argv is not None else sys.argv[1:]):
156
+ print(json.dumps(res, indent=1))
157
+ else:
158
+ for r in res:
159
+ print(f" {r['status'].upper():7s} {r['criterion']}: {r['detail']}")
160
+ print(f"\noverall: {overall(res)}")
161
+ sys.exit(0 if overall(res) == "PASS" else 1)
162
+
163
+
164
+ if __name__ == "__main__":
165
+ main()