visualproof 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +16 -7
  2. package/dist/assets/tips/tour-agent.webp +0 -0
  3. package/dist/assets/tips/tour-arrange.webp +0 -0
  4. package/dist/assets/tips/tour-artwork.webp +0 -0
  5. package/dist/assets/tips/tour-checks.webp +0 -0
  6. package/dist/assets/tips/tour-keys.webp +0 -0
  7. package/dist/assets/tips/tour-library.webp +0 -0
  8. package/dist/assets/tips/tour-project.webp +0 -0
  9. package/dist/assets/tips/tour-review.webp +0 -0
  10. package/dist/assets/tips/tour-share.webp +0 -0
  11. package/dist/assets/tips/tour-source.webp +0 -0
  12. package/dist/assets/tips/tour-target.webp +0 -0
  13. package/dist/assets/tips/tour-view.webp +0 -0
  14. package/dist/assets/tips/tour.json +117 -112
  15. package/dist/assets/tips/tour.webp +0 -0
  16. package/dist/server/access.js +3 -1
  17. package/dist/server/accesspage.js +130 -0
  18. package/dist/server/agent.js +6 -1
  19. package/dist/server/buildwatch.js +133 -0
  20. package/dist/server/deploy.js +844 -0
  21. package/dist/server/doctor.js +15 -3
  22. package/dist/server/film.js +1 -1
  23. package/dist/server/historyapi.js +100 -50
  24. package/dist/server/historythread.js +206 -0
  25. package/dist/server/hover.js +8 -0
  26. package/dist/server/live.js +187 -36
  27. package/dist/server/main.js +256 -30
  28. package/dist/server/review.js +27 -7
  29. package/dist/server/status.js +119 -46
  30. package/dist/server/usage.js +51 -0
  31. package/dist/shared/board.js +2 -1
  32. package/dist/shared/deploy.js +35 -0
  33. package/dist/shared/layout.js +297 -70
  34. package/dist/shared/libm.js +366 -0
  35. package/dist/shared/palette.js +1 -1
  36. package/dist/shared/separate.js +556 -32
  37. package/dist/web/bundle/app.js +1972 -826
  38. package/dist/web/bundle/edges.js +2 -1
  39. package/dist/web/bundle/layout.js +1026 -80
  40. package/dist/web/src/canvas.js +47 -34
  41. package/dist/web/src/overview.js +4 -1
  42. package/dist/web/src/scene.js +13 -12
  43. package/dist/web/src/status.js +28 -0
  44. package/dist/web/src/theme.js +108 -0
  45. package/dist/web/style.css +102 -35
  46. package/docs/agent-api/history.md +3 -0
  47. package/docs/agent-api/requests.md +4 -3
  48. package/package.json +1 -1
@@ -0,0 +1,844 @@
1
+ // Following a branch of a git repository (SPEC/DeploySpec.md): `visualproof serve <git URL>`.
2
+ // The server keeps a clone of its own in the cache, one per URL and branch, never anyone's
3
+ // workspace; asks the remote for the branch's head every so often (`git ls-remote`); and builds
4
+ // each new head itself — fetched, checked out detached, Mathlib's cache got where the project is on
5
+ // Mathlib, `lake build` — one build at a time, in programs of their own (never the server's
6
+ // process), the newest head seen meanwhile the only one built after it. The live loop
7
+ // (server/live.ts) reads the build as it reads a workspace's, but only between builds: while one
8
+ // changes the tree, nothing is read from it, and the board on show stays the last build's.
9
+ //
10
+ // What is kept, in the deployment's folder of the cache (`<cache>/deploy/<repo>-<hash>/`): the
11
+ // clone, `state.json` (the commit the tree's outputs are of, the build going on, the last one
12
+ // that ended, the newest head seen), each build's log (`logs/`), and `.lock`: one server follows a
13
+ // branch with a cache at a time. A build cut short (a restart into a patch, a stop) is done again
14
+ // by the next server, Lake keeping what it finished.
15
+ //
16
+ // Git and Lake are run with argument arrays, never a shell; the address is checked against the
17
+ // schemes followed, the branch by git (`check-ref-format --branch`); nothing that git or Lake starts
18
+ // waits on a prompt; the credentials are git's own (an SSH key, a deploy key, a credential helper),
19
+ // and VisualProof keeps none.
20
+ import { spawn, spawnSync } from "node:child_process";
21
+ import { createHash } from "node:crypto";
22
+ import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
23
+ import { homedir, tmpdir } from "node:os";
24
+ import { join } from "node:path";
25
+ import { shortCommit, slimDeploy } from "../shared/deploy.js";
26
+ import { childEnv, DETACH, end, grouped } from "./children.js";
27
+ import { dataHome, lockSite, SiteBusy } from "./dirs.js";
28
+ import { sourceAddress } from "./history.js";
29
+ import { openProject, ProjectError } from "./project.js";
30
+ /** How often the remote is asked by default (DeploySpec decision 3). */
31
+ export const DEFAULT_POLL_MINUTES = 2;
32
+ /** the least `--poll` takes, in minutes */
33
+ export const MIN_POLL_MINUTES = 0.05;
34
+ /** a question to the remote is given up after this long */
35
+ const ASK_MS = 60_000;
36
+ /** a fetch, a checkout */
37
+ const GIT_MS = 10 * 60_000;
38
+ /** Mathlib's cache (`lake exe cache get`): it compiles the cache program first */
39
+ const CACHE_MS = 30 * 60_000;
40
+ /** `lake build`, past which the build is stopped and failed (proposed: DeploySpec decision 14) */
41
+ export const BUILD_MS = 6 * 3600_000;
42
+ /** the most of a build's log kept for the pages, in lines and characters */
43
+ const TAIL_LINES = 60;
44
+ const TAIL_CHARS = 8000;
45
+ /** the lines of the build going on the pages are shown */
46
+ const LIVE_LINES = 6;
47
+ /** build logs kept, the newest */
48
+ const LOGS_KEPT = 30;
49
+ /** Whether a project's place on the command line is a git address rather than a folder: a URL
50
+ * (`scheme://…`), or git's scp-like `[user@]host:path` (a colon before any slash). Anything else is
51
+ * a folder; a folder whose name has a colon is written `./a:b`. */
52
+ export function isGitAddress(arg) {
53
+ if (/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(arg))
54
+ return true;
55
+ return /^[^/\\:\s]+:/.test(arg) && !/^[A-Za-z]:[\\/]/.test(arg);
56
+ }
57
+ const HOST = /^(?:[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)*$/;
58
+ const PATH = /^[\w.~/@+-]+$/;
59
+ /** A git address checked against what is followed (DeploySpec decision 10): `https://` without a
60
+ * user name or password in it, `ssh://` and scp-like (`git@host:path`), and `file://` for a
61
+ * repository on this machine; never `http://` (nothing checks what comes), `git://`, `ext::` or any
62
+ * other. The address, or why it is refused. */
63
+ export function parseRemote(arg) {
64
+ const no = (why) => ({ error: `${arg.slice(0, 200)}: ${why}` });
65
+ if (!arg || arg.length > 2000 || /[\s\0-\x1f\x7f]/.test(arg))
66
+ return no("an address has no spaces or control characters");
67
+ if (arg.startsWith("-"))
68
+ return no("an address does not start with -");
69
+ if (/(^|[/:])\.\.?(\/|$)/.test(arg))
70
+ return no("not a repository's path: no . or .. in it");
71
+ const scheme = /^([A-Za-z][A-Za-z0-9+.-]*):\/\//.exec(arg)?.[1].toLowerCase();
72
+ let kind;
73
+ let host = "";
74
+ let path;
75
+ if (scheme) {
76
+ if (scheme === "http")
77
+ return no("http:// is not followed: nothing checks what comes over it. Give the https:// address");
78
+ if (scheme !== "https" && scheme !== "ssh" && scheme !== "file")
79
+ return no(`${scheme}:// is not followed: give an https:// or ssh:// address, git's scp-like one (git@host:owner/repo.git), or file:// for a repository on this machine`);
80
+ let u;
81
+ try {
82
+ u = new URL(arg);
83
+ }
84
+ catch {
85
+ return no("not an address");
86
+ }
87
+ if (u.search || u.hash)
88
+ return no("an address has no query or fragment");
89
+ if (scheme === "file") {
90
+ if (u.host)
91
+ return no("file:// names a repository on this machine: file:///its/absolute/path");
92
+ kind = "file";
93
+ path = decodeURIComponent(u.pathname);
94
+ if (!path.startsWith("/") || path === "/")
95
+ return no("file:// takes the repository's absolute path");
96
+ }
97
+ else {
98
+ if (u.password || (scheme === "https" && u.username))
99
+ return no("no user name, password or token in the address: VisualProof keeps no credential. Let git find it (an SSH key, a deploy key, a credential helper)");
100
+ if (u.username && !/^[A-Za-z0-9._-]+$/.test(u.username))
101
+ return no("not a user name git takes");
102
+ host = u.hostname;
103
+ if (!HOST.test(host) && !/^\[[0-9A-Fa-f:.]+\]$/.test(host))
104
+ return no("not a host name");
105
+ kind = scheme === "https" ? "https" : "ssh";
106
+ path = decodeURIComponent(u.pathname);
107
+ }
108
+ }
109
+ else {
110
+ // scp-like, as git reads it: [user@]host:path
111
+ const m = /^(?:([A-Za-z0-9._][A-Za-z0-9._-]*)@)?([^@/:]+):(.+)$/.exec(arg);
112
+ if (!m)
113
+ return no("not a git address: give https://host/owner/repo, ssh://… or git@host:owner/repo");
114
+ if (!HOST.test(m[2]))
115
+ return no("not a host name");
116
+ host = m[2];
117
+ path = m[3];
118
+ if (path.startsWith("-"))
119
+ return no("a path does not start with -");
120
+ kind = "ssh";
121
+ }
122
+ const parts = path.replace(/\/+$/, "").split("/").filter(Boolean);
123
+ if (!parts.length || !PATH.test(path) || parts.some((p) => p === ".."))
124
+ return no("not a repository's path");
125
+ const name = (parts[parts.length - 1].replace(/\.git$/, "").replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^[.-]+/, "") || "repository").slice(0, 60);
126
+ const gh = kind !== "file" && /^(www\.)?github\.com$/i.test(host) && parts.length === 2 ? `${parts[0]}/${parts[1].replace(/\.git$/, "")}` : null;
127
+ return { url: arg, kind, source: kind === "file" ? null : sourceAddress(arg), github: gh, name };
128
+ }
129
+ /** Why `name` is no branch to follow, or null: git's own check (`git check-ref-format --branch`),
130
+ * after refusing what that check would read otherwise (`@{-1}`, the branch checked out before). */
131
+ export function branchProblem(name) {
132
+ const no = `${name.slice(0, 200)} is not a branch name git takes (git check-ref-format --branch)`;
133
+ if (!name || name.length > 255 || name.startsWith("-") || name.includes("@{") || /[\s\0-\x1f\x7f]/.test(name))
134
+ return no;
135
+ const r = spawnSync("git", ["check-ref-format", "--branch", name], { cwd: tmpdir(), encoding: "utf8", timeout: 10_000, env: gitBase() });
136
+ if (r.error)
137
+ return `git could not be run (${r.error.message}): following a branch needs git`;
138
+ return r.status === 0 && r.stdout.trim() === name ? null : no;
139
+ }
140
+ // ---------------------------------------------------------------- where it is kept
141
+ /** A deployment's name among the others: the repository's name and a hash of the address and the
142
+ * branch. */
143
+ export function deployKey(remote, branch) {
144
+ return `${remote.name}-${createHash("sha1").update(`${remote.url}\0${branch}`).digest("hex").slice(0, 8)}`;
145
+ }
146
+ export function deployFolder(cacheRoot, remote, branch) {
147
+ const dir = join(cacheRoot, "deploy", deployKey(remote, branch));
148
+ return { dir, checkout: join(dir, remote.name), state: join(dir, "state.json"), logs: join(dir, "logs") };
149
+ }
150
+ /** The site of a deployment, without `--site`: under the data home by the address and the branch,
151
+ * so that the review stays where it is whatever `--cache` says. */
152
+ export function deploySite(remote, branch, h) {
153
+ return join(dataHome(h), "sites", deployKey(remote, branch));
154
+ }
155
+ /** The remotes' default branches as last asked (`<cache>/deploy/remotes.json`): a server started
156
+ * without `--branch` and without the network follows the branch it followed. */
157
+ function remotesFile(cacheRoot) {
158
+ return join(cacheRoot, "deploy", "remotes.json");
159
+ }
160
+ export function recordedDefault(cacheRoot, url) {
161
+ try {
162
+ const all = JSON.parse(readFileSync(remotesFile(cacheRoot), "utf8"));
163
+ const b = Object.hasOwn(all, url) ? all[url].branch : null;
164
+ return typeof b === "string" && !branchProblem(b) ? b : null;
165
+ }
166
+ catch {
167
+ return null;
168
+ }
169
+ }
170
+ function recordDefault(cacheRoot, url, branch) {
171
+ const file = remotesFile(cacheRoot);
172
+ let all = {};
173
+ try {
174
+ all = JSON.parse(readFileSync(file, "utf8"));
175
+ }
176
+ catch {
177
+ // none yet
178
+ }
179
+ all[url] = { branch, at: new Date().toISOString() };
180
+ try {
181
+ mkdirSync(join(cacheRoot, "deploy"), { recursive: true });
182
+ writeFileSync(`${file}.tmp-${process.pid}`, JSON.stringify(all, null, 1) + "\n");
183
+ renameSync(`${file}.tmp-${process.pid}`, file);
184
+ }
185
+ catch {
186
+ // a record to spare the network at the next start; not a reason to fail
187
+ }
188
+ }
189
+ // ---------------------------------------------------------------- the programs' environment
190
+ /** What tells git which repository to work on, where the server's own environment has it (a hook's,
191
+ * a test's): never handed on, so that git works on the clone alone. */
192
+ const REPO_VARS = ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_NAMESPACE", "GIT_PREFIX", "GIT_QUARANTINE_PATH", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT"];
193
+ /** The server's environment without what is the server's alone (`VISUALPROOF_*`, server/children.ts)
194
+ * and without what would point git elsewhere. */
195
+ function gitBase() {
196
+ const env = childEnv();
197
+ for (const k of REPO_VARS)
198
+ delete env[k];
199
+ for (const k of Object.keys(env))
200
+ if (/^GIT_CONFIG_(KEY|VALUE)_\d+$/.test(k))
201
+ delete env[k];
202
+ return env;
203
+ }
204
+ let sshConfigured;
205
+ /** The ssh command git is configured with (`core.sshCommand`, the user's or the system's), read once. */
206
+ function configuredSsh() {
207
+ if (sshConfigured !== undefined)
208
+ return sshConfigured;
209
+ const r = spawnSync("git", ["config", "--get", "core.sshCommand"], { cwd: tmpdir(), encoding: "utf8", timeout: 10_000, env: gitBase() });
210
+ return (sshConfigured = r.status === 0 && r.stdout.trim() ? r.stdout.trim() : null);
211
+ }
212
+ /** No prompt is ever waited on (DeploySpec decision 9): git asks no terminal and runs no askpass
213
+ * (`GIT_TERMINAL_PROMPT=0`, an empty `GIT_ASKPASS`), the credential manager is not interactive, and
214
+ * ssh runs in batch mode — the ssh command git would run (`GIT_SSH_COMMAND`, else `core.sshCommand`,
215
+ * else `ssh`) with `-o BatchMode=yes` after it; a `GIT_SSH` program is left as it is. Credentials
216
+ * are git's own: helpers, keys and agents work as for any git. The programs run without a
217
+ * terminal of their own besides (server/children.ts, `grouped`). */
218
+ export function noPrompts(env, ssh = configuredSsh()) {
219
+ const out = { ...env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "", SSH_ASKPASS_REQUIRE: "never", GCM_INTERACTIVE: "never" };
220
+ const own = env.GIT_SSH_COMMAND || ssh || (env.GIT_SSH ? null : "ssh");
221
+ if (own)
222
+ out.GIT_SSH_COMMAND = `${own} -o BatchMode=yes`;
223
+ return out;
224
+ }
225
+ /** git's environment, for the remote `remote`: no prompts, and only its protocol allowed
226
+ * (`GIT_ALLOW_PROTOCOL`), whatever a redirect or a configuration says. */
227
+ export function gitEnv(remote) {
228
+ return { ...noPrompts(gitBase()), GIT_ALLOW_PROTOCOL: remote.kind };
229
+ }
230
+ /** Lake's environment for a build: the server's without what is its own — no access key or token
231
+ * of VisualProof's is in it — and no prompts for the git it runs for the dependencies. */
232
+ export function buildEnv() {
233
+ return noPrompts(gitBase());
234
+ }
235
+ /** Run `cmd` with `args` — an array, never through a shell — as a group of its own (what it starts
236
+ * ends with it: server/children.ts), stopped past `timeoutMs`; what it writes goes to `onText` as
237
+ * it comes. Never rejects. */
238
+ export function run(cmd, args, o) {
239
+ return new Promise((done) => {
240
+ const keep = o.keep ?? 20_000;
241
+ let stdout = "", out = "", timedOut = false, finished = false;
242
+ let child;
243
+ try {
244
+ child = grouped(spawn(cmd, args, { cwd: o.cwd, env: o.env, stdio: ["ignore", "pipe", "pipe"], detached: DETACH }));
245
+ }
246
+ catch (e) {
247
+ return done({ code: null, signal: null, stdout, out, timedOut, error: e instanceof Error ? e.message : String(e) });
248
+ }
249
+ const take = (b, own) => {
250
+ const t = b.toString();
251
+ if (own)
252
+ stdout = (stdout + t).slice(-keep);
253
+ out = (out + t).slice(-keep);
254
+ o.onText?.(t);
255
+ };
256
+ child.stdout.on("data", (b) => take(b, true));
257
+ child.stderr.on("data", (b) => take(b, false));
258
+ const timer = setTimeout(() => {
259
+ timedOut = true;
260
+ end(child);
261
+ setTimeout(() => end(child, "SIGKILL"), 5000).unref();
262
+ }, o.timeoutMs);
263
+ const finish = (r) => {
264
+ if (finished)
265
+ return;
266
+ finished = true;
267
+ clearTimeout(timer);
268
+ done(r);
269
+ };
270
+ child.on("error", (e) => finish({ code: null, signal: null, stdout, out, timedOut, error: e.message }));
271
+ child.on("close", (code, signal) => finish({ code, signal, stdout, out, timedOut }));
272
+ });
273
+ }
274
+ /** What git said went wrong, in a line: its last `fatal:` or `error:` line, else its last line. */
275
+ function gitWhy(r) {
276
+ if (r.error)
277
+ return `git could not be run: ${r.error}`;
278
+ if (r.timedOut)
279
+ return "git did not finish in time";
280
+ const lines = r.out.split("\n").map((l) => l.trim()).filter(Boolean);
281
+ const said = [...lines].reverse().find((l) => /^(fatal|error):/.test(l)) ?? lines[lines.length - 1] ?? `git exited with ${r.code}`;
282
+ return said.replace(/^(fatal|error):\s*/, "").slice(0, 400);
283
+ }
284
+ const git = (args, env, ms = GIT_MS, onText) => run("git", args, { env, timeoutMs: ms, cwd: tmpdir(), onText });
285
+ const SHA = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
286
+ /** The head of `branch` on the remote; without a branch, the remote's default branch and its head
287
+ * (`git ls-remote --symref HEAD`). Only that branch is ever asked of: never a pull request's ref, nor
288
+ * a fork's. */
289
+ export async function askRemote(remote, branch, env) {
290
+ const r = await git(branch ? ["ls-remote", "--", remote.url, `refs/heads/${branch}`] : ["ls-remote", "--symref", "--", remote.url, "HEAD"], env, ASK_MS);
291
+ if (r.code !== 0 || r.error || r.timedOut)
292
+ return { error: r.timedOut ? "the remote did not answer within a minute" : gitWhy(r) };
293
+ const lines = r.stdout.split("\n");
294
+ if (!branch) {
295
+ const ref = lines.map((l) => /^ref: refs\/heads\/(\S+)\tHEAD$/.exec(l)).find(Boolean)?.[1];
296
+ if (!ref || branchProblem(ref))
297
+ return { error: "the remote names no default branch: say which to follow, --branch <name>" };
298
+ const head = lines.map((l) => /^([0-9a-f]+)\tHEAD$/.exec(l)).find(Boolean)?.[1];
299
+ return head && SHA.test(head) ? { head, branch: ref } : { error: `the remote's ${ref} has no commit` };
300
+ }
301
+ // the pattern matches refs that end so: the branch's own ref alone
302
+ const head = lines.map((l) => l.split("\t")).find((p) => p[1] === `refs/heads/${branch}`)?.[0];
303
+ return head && SHA.test(head) ? { head, branch } : { error: `the remote has no branch ${branch}` };
304
+ }
305
+ /** The branch to follow: `given`, else the remote's default as it says it now, else as it said it
306
+ * last (the network may be away at a restart). */
307
+ export async function branchToFollow(remote, given, cacheRoot) {
308
+ if (given)
309
+ return given;
310
+ const a = await askRemote(remote, null, gitEnv(remote));
311
+ if ("head" in a) {
312
+ recordDefault(cacheRoot, remote.url, a.branch);
313
+ return a.branch;
314
+ }
315
+ return recordedDefault(cacheRoot, remote.url) ?? { error: `cannot learn the remote's default branch (${a.error}): give --branch <name>` };
316
+ }
317
+ /** The deployment's clone, made at the first start (DeploySpec decision 2): an empty repository
318
+ * whose `origin` is the address, the branch fetched alone, its head checked out detached. It is
319
+ * made in a folder of its own and moved into place once whole, so that a start cut short leaves
320
+ * nothing half made. The commit checked out, or why there is none. */
321
+ export async function ensureClone(remote, branch, folder, log) {
322
+ const env = gitEnv(remote);
323
+ if (existsSync(join(folder.checkout, ".git"))) {
324
+ const r = await git(["-C", folder.checkout, "rev-parse", "--verify", "HEAD^{commit}"], env);
325
+ const c = r.stdout.trim();
326
+ if (r.code === 0 && SHA.test(c))
327
+ return { commit: c, made: false };
328
+ return { error: `the clone ${folder.checkout} cannot be read (${gitWhy(r)}): remove it, and it is made again` };
329
+ }
330
+ mkdirSync(folder.dir, { recursive: true });
331
+ // what a start that died midway left
332
+ for (const f of readdirSync(folder.dir)) {
333
+ const m = new RegExp(`^${remote.name.replace(/[.]/g, "\\.")}\\.tmp-(\\d+)$`).exec(f);
334
+ if (m && !alive(Number(m[1])))
335
+ rmSync(join(folder.dir, f), { recursive: true, force: true });
336
+ }
337
+ const tmp = `${folder.checkout}.tmp-${process.pid}`;
338
+ rmSync(tmp, { recursive: true, force: true });
339
+ const t0 = Date.now();
340
+ log(`cloning ${remote.source ?? remote.url} (${branch}) into ${folder.checkout}…`);
341
+ const steps = [
342
+ ["init", "--quiet", tmp],
343
+ ["-C", tmp, "remote", "add", "origin", "--", remote.url],
344
+ ["-C", tmp, "fetch", "--no-tags", "--no-recurse-submodules", "--quiet", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`],
345
+ ];
346
+ for (const s of steps) {
347
+ const r = await git(s, env);
348
+ if (r.code !== 0) {
349
+ rmSync(tmp, { recursive: true, force: true });
350
+ return { error: `cannot clone ${remote.source ?? remote.url} (${branch}): ${gitWhy(r)}` };
351
+ }
352
+ }
353
+ const rp = await git(["-C", tmp, "rev-parse", "--verify", `refs/remotes/origin/${branch}^{commit}`], env);
354
+ const commit = rp.stdout.trim();
355
+ const co = SHA.test(commit) ? await git(["-C", tmp, "-c", "advice.detachedHead=false", "checkout", "--quiet", "--force", "--detach", commit], env) : rp;
356
+ if (co.code !== 0 || !SHA.test(commit)) {
357
+ rmSync(tmp, { recursive: true, force: true });
358
+ return { error: `cannot check out ${branch} of ${remote.source ?? remote.url}: ${gitWhy(co)}` };
359
+ }
360
+ renameSync(tmp, folder.checkout);
361
+ log(`cloned in ${((Date.now() - t0) / 1000).toFixed(1)} s, at ${shortCommit(commit)}`);
362
+ return { commit, made: true };
363
+ }
364
+ function alive(pid) {
365
+ try {
366
+ process.kill(pid, 0);
367
+ return true;
368
+ }
369
+ catch (e) {
370
+ return e.code === "EPERM";
371
+ }
372
+ }
373
+ /** Another server follows the branch with this cache. */
374
+ export class DeployBusy extends Error {
375
+ }
376
+ /** Take the deployment for this process (`.lock`, as a site is taken: server/dirs.ts). */
377
+ export function lockDeployment(folder, remote, branch) {
378
+ mkdirSync(folder.dir, { recursive: true });
379
+ try {
380
+ return lockSite(folder.dir);
381
+ }
382
+ catch (e) {
383
+ if (e instanceof SiteBusy)
384
+ throw new DeployBusy(`another VisualProof server (process ${e.pid}) follows ${branch} of ${remote.source ?? remote.url} with this cache (${folder.dir}): stop it first, or give this one a --cache of its own`);
385
+ throw e;
386
+ }
387
+ }
388
+ /** Whether the project is on Mathlib: its manifest names it (DeploySpec decision 4). */
389
+ export function onMathlib(root) {
390
+ try {
391
+ const m = JSON.parse(readFileSync(join(root, "lake-manifest.json"), "utf8"));
392
+ return Array.isArray(m.packages) && m.packages.some((p) => typeof p?.name === "string" && p.name.toLowerCase() === "mathlib");
393
+ }
394
+ catch {
395
+ return false;
396
+ }
397
+ }
398
+ // ---------------------------------------------------------------- a build's log
399
+ /** A build's log: everything its programs wrote, in `logs/<commit>.log` of the deployment (a build
400
+ * done again writes it again), and its end for the pages, where the clone's folder reads `.` and the
401
+ * home `~`, colours and redrawn lines taken out. */
402
+ export class BuildLog {
403
+ file;
404
+ fd = null;
405
+ lines = [];
406
+ partial = "";
407
+ /** the clone's path as the log may write it, the longest first: Lean prints the path resolved
408
+ * (macOS's /var is /private/var), git and Lake the one given */
409
+ roots;
410
+ constructor(dir, commit, root) {
411
+ mkdirSync(dir, { recursive: true });
412
+ this.file = join(dir, `${commit}.log`);
413
+ let real = root;
414
+ try {
415
+ real = realpathSync(root);
416
+ }
417
+ catch {
418
+ // not there yet: the path as given
419
+ }
420
+ this.roots = [...new Set([real, root])].sort((a, b) => b.length - a.length);
421
+ try {
422
+ this.fd = openSync(this.file, "w", 0o600);
423
+ }
424
+ catch {
425
+ // a log that cannot be written: its end is still kept for the pages
426
+ }
427
+ prune(dir, LOGS_KEPT);
428
+ }
429
+ /** a line of VisualProof's own: the step, the command */
430
+ say(line) {
431
+ this.text(`${line}\n`);
432
+ }
433
+ text(t) {
434
+ if (this.fd !== null) {
435
+ try {
436
+ writeSync(this.fd, t);
437
+ }
438
+ catch {
439
+ this.fd = null;
440
+ }
441
+ }
442
+ const parts = (this.partial + t).split("\n");
443
+ this.partial = parts.pop() ?? "";
444
+ for (const p of parts)
445
+ this.push(p);
446
+ }
447
+ push(raw) {
448
+ // a line redrawn in place (a progress bar) is its last state
449
+ const line = raw.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "").split("\r").pop().trimEnd();
450
+ if (!line)
451
+ return;
452
+ this.lines.push(line);
453
+ if (this.lines.length > TAIL_LINES * 2)
454
+ this.lines.splice(0, this.lines.length - TAIL_LINES);
455
+ }
456
+ /** its end, as the pages are given it: its last `n` lines, the one being written included */
457
+ tail(n = TAIL_LINES) {
458
+ const last = this.partial.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "").split("\r").pop().trimEnd();
459
+ const home = homedir();
460
+ let t = [...this.lines, ...(last ? [last] : [])].slice(-n).join("\n");
461
+ for (const r of this.roots)
462
+ t = t.split(r).join(".");
463
+ if (home && home.length > 1)
464
+ t = t.split(home).join("~");
465
+ return t.length > TAIL_CHARS ? `…${t.slice(-TAIL_CHARS)}` : t;
466
+ }
467
+ close() {
468
+ if (this.fd !== null)
469
+ closeSync(this.fd);
470
+ this.fd = null;
471
+ }
472
+ }
473
+ /** Keep the newest `keep` files of `dir`. */
474
+ function prune(dir, keep) {
475
+ try {
476
+ const files = readdirSync(dir)
477
+ .filter((f) => f.endsWith(".log"))
478
+ .map((f) => ({ f, t: statSync(join(dir, f)).mtimeMs }))
479
+ .sort((a, b) => b.t - a.t);
480
+ for (const { f } of files.slice(keep))
481
+ unlinkSync(join(dir, f));
482
+ }
483
+ catch {
484
+ // housekeeping
485
+ }
486
+ }
487
+ /** A branch followed and built (DeploySpec decisions 3–8): the remote asked every `everyMs`, each
488
+ * new head built in turn, the newest seen meanwhile the only one built after the build going on. */
489
+ export class Deployment {
490
+ remote;
491
+ branch;
492
+ folder;
493
+ o;
494
+ state;
495
+ /** the newest head seen: what is built next */
496
+ want;
497
+ /** the last commit tried */
498
+ done;
499
+ /** a commit whose try could not be made: tried again at the next look at the remote, not before */
500
+ later = null;
501
+ /** the tree is not one build's: a build was cut short, and nothing is read until one ends */
502
+ interrupted;
503
+ building = null;
504
+ asked = null;
505
+ timer = null;
506
+ retry = null;
507
+ stopped = false;
508
+ asking = null;
509
+ /** the build going on */
510
+ running = null;
511
+ env;
512
+ lakeEnv;
513
+ /** every build tried since the start, in order: what the tests read */
514
+ tried = [];
515
+ /** the log of the build going on */
516
+ log = null;
517
+ constructor(o) {
518
+ this.o = o;
519
+ this.remote = o.remote;
520
+ this.branch = o.branch;
521
+ this.folder = o.folder;
522
+ this.env = gitEnv(o.remote);
523
+ this.lakeEnv = buildEnv();
524
+ this.state = this.read();
525
+ const head = this.headNow();
526
+ // the tree is one build's only where the outputs are of the commit checked out and no build
527
+ // was going on
528
+ this.interrupted = this.state.building !== null || (this.state.outputs !== null && head !== null && head !== this.state.outputs.commit);
529
+ this.want = this.state.head?.commit ?? this.state.building?.commit ?? o.head ?? head;
530
+ this.done = this.interrupted ? null : this.state.done;
531
+ if (this.state.building)
532
+ o.hooks.log(`build: the build of ${shortCommit(this.state.building.commit)} was cut short (a restart, a stop): it is done again`);
533
+ }
534
+ read() {
535
+ const fresh = { version: 1, url: this.remote.url, branch: this.branch, outputs: null, building: null, done: null, last: null, head: null };
536
+ try {
537
+ const s = JSON.parse(readFileSync(this.folder.state, "utf8"));
538
+ return { ...fresh, ...s, version: 1, url: this.remote.url, branch: this.branch };
539
+ }
540
+ catch {
541
+ return fresh;
542
+ }
543
+ }
544
+ save() {
545
+ const tmp = `${this.folder.state}.tmp-${process.pid}`;
546
+ try {
547
+ writeFileSync(tmp, JSON.stringify(this.state, null, 1) + "\n");
548
+ renameSync(tmp, this.folder.state);
549
+ }
550
+ catch (e) {
551
+ this.o.hooks.log(`build: ${this.folder.state} cannot be written: ${e instanceof Error ? e.message : String(e)}`);
552
+ }
553
+ }
554
+ /** the commit checked out in the clone */
555
+ headNow() {
556
+ const r = spawnSync("git", ["-C", this.folder.checkout, "rev-parse", "--verify", "HEAD^{commit}"], { encoding: "utf8", timeout: 10_000, env: this.env });
557
+ const c = r.status === 0 ? r.stdout.trim() : "";
558
+ return SHA.test(c) ? c : null;
559
+ }
560
+ /** Ask the remote at once, then every `everyMs`. */
561
+ start() {
562
+ void this.poll();
563
+ this.timer = setInterval(() => void this.poll(), this.o.everyMs);
564
+ this.timer.unref?.();
565
+ }
566
+ /** No more asking or building; a build going on is stopped with the server's programs
567
+ * (server/children.ts) and recorded as cut short, to be done again at the next start. */
568
+ stop() {
569
+ this.stopped = true;
570
+ if (this.timer)
571
+ clearInterval(this.timer);
572
+ if (this.retry)
573
+ clearTimeout(this.retry);
574
+ this.timer = this.retry = null;
575
+ }
576
+ /** Whether the tree is not to be read now: a build changes it, one was cut short, or none was ever
577
+ * made. */
578
+ holds() {
579
+ return this.building !== null || this.interrupted || this.state.outputs === null;
580
+ }
581
+ /** a build is going on, which a restart would cut short (UpdateSpec decision 6) */
582
+ get busy() {
583
+ return this.running !== null;
584
+ }
585
+ /** The commit what the tree holds was built from, and the branch: the history's for each build it
586
+ * reads (HistorySpec decision 4). */
587
+ commit() {
588
+ return this.state.outputs ? { hash: this.state.outputs.commit, branch: this.branch } : null;
589
+ }
590
+ /** What the pages and agents are told; without the build's log for a read-only reader. */
591
+ status(full) {
592
+ const s = {
593
+ source: this.remote.source,
594
+ github: this.remote.github,
595
+ branch: this.branch,
596
+ every: Math.round(this.o.everyMs / 1000),
597
+ asked: this.asked,
598
+ // what the build going on says now (the cache's download, Lake's last lines)
599
+ building: this.building && { ...this.building, ...(this.log ? { tail: this.log.tail(LIVE_LINES) } : {}) },
600
+ // a commit waiting for its try to be made again is said by the last build's
601
+ next: this.building && this.want && this.want !== this.done && this.want !== this.building.commit ? this.want : null,
602
+ last: this.state.last,
603
+ built: this.state.outputs?.commit ?? null,
604
+ };
605
+ return full ? s : slimDeploy(s);
606
+ }
607
+ /** Ask the remote for the branch's head; a new one is built once the build going on ends. A
608
+ * question asked while one is on its way waits for it. */
609
+ poll() {
610
+ this.asking ??= (async () => {
611
+ try {
612
+ const a = this.o.ask ? await this.o.ask() : await askRemote(this.remote, this.branch, this.env);
613
+ const at = new Date().toISOString();
614
+ if (this.stopped)
615
+ return;
616
+ if ("error" in a) {
617
+ if (this.asked?.error !== a.error)
618
+ this.o.hooks.log(`build: the remote cannot be asked: ${a.error}`);
619
+ this.asked = { at, head: this.asked?.head ?? null, error: a.error };
620
+ }
621
+ else {
622
+ this.asked = { at, head: a.head };
623
+ if (a.head !== this.state.head?.commit) {
624
+ this.state.head = { commit: a.head, at };
625
+ this.save();
626
+ }
627
+ if (a.head !== this.want) {
628
+ if (this.want)
629
+ this.o.hooks.log(`build: ${this.branch} is at ${shortCommit(a.head)} now${this.running ? `: built once the build of ${shortCommit(this.building?.commit ?? "")} ends` : ""}`);
630
+ this.want = a.head;
631
+ }
632
+ }
633
+ this.later = null;
634
+ this.o.hooks.changed();
635
+ this.kick();
636
+ }
637
+ finally {
638
+ this.asking = null;
639
+ }
640
+ })();
641
+ return this.asking;
642
+ }
643
+ /** Start the build of the newest head where none is going on and it is not built yet; while the
644
+ * server reads the tree, a moment later. */
645
+ kick() {
646
+ if (this.stopped || this.running || !this.want || this.want === this.done || this.want === this.later)
647
+ return;
648
+ if (this.o.hooks.reading()) {
649
+ this.retry ??= setTimeout(() => {
650
+ this.retry = null;
651
+ this.kick();
652
+ }, 300);
653
+ return;
654
+ }
655
+ const commit = this.want;
656
+ this.tried.push(commit);
657
+ this.running = this.attempt(commit)
658
+ .catch((e) => this.o.hooks.log(`build: ${shortCommit(commit)}: ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`))
659
+ .finally(() => {
660
+ this.running = null;
661
+ this.building = null;
662
+ this.o.hooks.changed();
663
+ this.kick();
664
+ });
665
+ }
666
+ /** Until the build going on and those waiting are done: what the tests wait for. */
667
+ async settled() {
668
+ while (this.running || this.asking || this.retry)
669
+ await (this.running ?? this.asking ?? new Promise((ok) => setTimeout(ok, 50)));
670
+ }
671
+ step(s) {
672
+ if (this.building)
673
+ this.building = { ...this.building, step: s };
674
+ this.o.hooks.changed();
675
+ }
676
+ /** The build of `commit` (DeploySpec decisions 4–7): fetched where the clone has it not, checked
677
+ * out detached and cleaned (the build outputs kept), the project read again as it says it,
678
+ * Mathlib's cache got where it is on Mathlib, `lake build`. A try that could not change or build
679
+ * the tree leaves it as the last build had it. */
680
+ async attempt(commit) {
681
+ const t0 = Date.now();
682
+ const since = new Date(t0).toISOString();
683
+ // before anything is awaited: from here the tree is not read (holds)
684
+ this.building = { commit, since, step: "fetch" };
685
+ this.o.hooks.changed();
686
+ const short = shortCommit(commit);
687
+ const say = this.o.hooks.log;
688
+ const log = (this.log = new BuildLog(this.folder.logs, commit, this.folder.checkout));
689
+ const g = (args, ms = GIT_MS) => {
690
+ log.say(`$ git ${args.join(" ")}`);
691
+ return git(["-C", this.folder.checkout, ...args], this.env, ms, (t) => log.text(t));
692
+ };
693
+ const ended = (b) => this.ended(commit, t0, { ...b, tail: log.tail() }, log);
694
+ try {
695
+ say(`build: ${short}: ${this.state.outputs ? "" : "the first build: "}fetching`);
696
+ if (!(await this.has(commit))) {
697
+ const f = await g(["fetch", "--no-tags", "--no-recurse-submodules", "--quiet", "origin", `+refs/heads/${this.branch}:refs/remotes/origin/${this.branch}`]);
698
+ if (this.stopped)
699
+ return;
700
+ if (f.code !== 0)
701
+ return ended({ ok: false, retry: true, error: `it could not be fetched: ${gitWhy(f)}` });
702
+ if (!(await this.has(commit)))
703
+ return ended({ ok: false, retry: true, error: "the branch moved on before it could be fetched: its newest commit is built next" });
704
+ }
705
+ // a graph being made ready to serve as the fetch went on (the usage pass's) is let finish: the
706
+ // tree changes from here
707
+ while (this.o.hooks.reading() && !this.stopped)
708
+ await new Promise((ok) => setTimeout(ok, 100));
709
+ if (this.stopped)
710
+ return;
711
+ // the build is recorded as going on, so that one cut short is done again (DeploySpec
712
+ // decision 8)
713
+ this.step("checkout");
714
+ this.o.hooks.starting();
715
+ this.state.building = { commit, since };
716
+ this.save();
717
+ const co = await g(["-c", "advice.detachedHead=false", "checkout", "--quiet", "--force", "--detach", commit]);
718
+ const cl = co.code === 0 ? await g(["clean", "-ffdq", "-e", ".lake"]) : co;
719
+ if (this.stopped)
720
+ return;
721
+ if (cl.code !== 0) {
722
+ await this.restore(log);
723
+ return ended({ ok: false, retry: true, error: `it could not be checked out: ${gitWhy(cl)}` });
724
+ }
725
+ try {
726
+ this.adopt(openProject(this.folder.checkout, this.o.roots, false));
727
+ }
728
+ catch (e) {
729
+ if (!(e instanceof ProjectError))
730
+ throw e;
731
+ await this.restore(log);
732
+ return ended({ ok: false, error: `it is no Lean project VisualProof can read: ${e.message}` });
733
+ }
734
+ const lake = this.o.lake ?? "lake";
735
+ if (onMathlib(this.folder.checkout)) {
736
+ this.step("cache");
737
+ say(`build: ${short}: getting Mathlib's cache`);
738
+ for (let n = 1; n <= 2 && !this.stopped; n++) {
739
+ log.say(`$ lake exe cache get`);
740
+ const c = await run(lake, ["exe", "cache", "get"], { cwd: this.folder.checkout, env: this.lakeEnv, timeoutMs: CACHE_MS, onText: (t) => log.text(t) });
741
+ if (c.code === 0)
742
+ break;
743
+ // Lake builds what the cache did not give
744
+ log.say(`Mathlib's cache could not be had (${c.error ?? (c.timedOut ? "it took too long" : `exit ${c.code}`)})${n === 1 ? ": trying again" : ": lake build builds what it lacks"}`);
745
+ }
746
+ }
747
+ if (this.stopped)
748
+ return;
749
+ this.step("build");
750
+ say(`build: ${short}: lake build${this.o.roots.length ? ` ${this.o.roots.join(" ")}` : ""}`);
751
+ log.say(`$ lake build${this.o.roots.length ? ` ${this.o.roots.join(" ")}` : ""}`);
752
+ const b = await run(lake, ["build", ...this.o.roots], { cwd: this.folder.checkout, env: this.lakeEnv, timeoutMs: BUILD_MS, onText: (t) => log.text(t) });
753
+ // cut short by the server's stop: done again at the next start
754
+ if (this.stopped)
755
+ return;
756
+ if (b.error) {
757
+ await this.restore(log);
758
+ return ended({ ok: false, retry: true, error: `lake could not be run: ${b.error}` });
759
+ }
760
+ // the tree is this build's, whatever came of it
761
+ this.state.outputs = { commit, at: new Date().toISOString() };
762
+ this.state.building = null;
763
+ this.interrupted = false;
764
+ // the modules a library's globs give, now that they are built
765
+ try {
766
+ this.adopt(openProject(this.folder.checkout, this.o.roots, false));
767
+ }
768
+ catch {
769
+ // read as it was after the checkout
770
+ }
771
+ const ok = b.code === 0 && !b.timedOut;
772
+ return ended({ ok, exit: b.code, ...(b.timedOut ? { error: `it took longer than ${BUILD_MS / 3600_000} hours: stopped` } : {}) });
773
+ }
774
+ catch (e) {
775
+ if (this.stopped)
776
+ return;
777
+ if (this.state.building)
778
+ await this.restore(log);
779
+ return ended({ ok: false, retry: true, error: e instanceof Error ? e.message : String(e) });
780
+ }
781
+ finally {
782
+ log.close();
783
+ this.log = null;
784
+ }
785
+ }
786
+ /** Whether the clone has `commit`. */
787
+ async has(commit) {
788
+ const r = await git(["-C", this.folder.checkout, "cat-file", "-e", `${commit}^{commit}`], this.env, ASK_MS);
789
+ return r.code === 0;
790
+ }
791
+ /** The tree as the last build left it, after a try that changed it and did not build it: checked
792
+ * out at that build's commit again, so that what is read of it, and a restart, are that build's.
793
+ * Where that cannot be done, the tree stays held until a build ends. */
794
+ async restore(log) {
795
+ const was = this.state.outputs?.commit;
796
+ if (was) {
797
+ log.say(`back to ${shortCommit(was)}, the last build's commit`);
798
+ const co = await git(["-C", this.folder.checkout, "-c", "advice.detachedHead=false", "checkout", "--quiet", "--force", "--detach", was], this.env);
799
+ if (co.code !== 0) {
800
+ log.say(`it could not be checked out again: ${gitWhy(co)}`);
801
+ this.interrupted = true;
802
+ return;
803
+ }
804
+ try {
805
+ this.adopt(openProject(this.folder.checkout, this.o.roots, false));
806
+ }
807
+ catch {
808
+ // it was read at that build
809
+ }
810
+ }
811
+ // no build to go back to: nothing is read until one is made (holds)
812
+ this.state.building = null;
813
+ this.interrupted = false;
814
+ }
815
+ /** The project as the commit checked out says it, taken into the server's own in place: its
816
+ * toolchain, its name, its roots. */
817
+ adopt(fresh) {
818
+ const p = this.o.project;
819
+ const was = p.toolchain;
820
+ Object.assign(p, { ...fresh, root: p.root });
821
+ if (fresh.toolchain !== was) {
822
+ this.o.hooks.log(`build: the toolchain is ${fresh.toolchain} now (it was ${was})`);
823
+ this.o.hooks.toolchainChanged();
824
+ }
825
+ }
826
+ /** A try ended: recorded, said, and — unless it could not be made — not tried again. */
827
+ ended(commit, t0, b, log) {
828
+ const ms = Date.now() - t0;
829
+ const build = { commit, at: new Date().toISOString(), ms, ...b };
830
+ this.state.last = build;
831
+ if (!b.retry)
832
+ this.state.done = this.done = commit;
833
+ else
834
+ this.later = commit;
835
+ this.save();
836
+ this.o.hooks.ended?.();
837
+ const secs = ms < 60_000 ? `${(ms / 1000).toFixed(1)} s` : `${Math.round(ms / 60_000)} min`;
838
+ const short = shortCommit(commit);
839
+ if (b.ok)
840
+ this.o.hooks.log(`build: ${short} built in ${secs}`);
841
+ else
842
+ this.o.hooks.log(`build: ${short} failed after ${secs}: ${b.error ?? `lake build exited with ${b.exit}`}${b.retry ? "; tried again at the next look at the remote" : ""} (its log: ${log.file})`);
843
+ }
844
+ }