visualproof 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -7
- package/dist/assets/latex/visualproof.sty +19 -9
- package/dist/assets/tips/tour-agent.webp +0 -0
- package/dist/assets/tips/tour-arrange.webp +0 -0
- package/dist/assets/tips/tour-artwork.webp +0 -0
- package/dist/assets/tips/tour-checks.webp +0 -0
- package/dist/assets/tips/tour-keys.webp +0 -0
- package/dist/assets/tips/tour-library.webp +0 -0
- package/dist/assets/tips/tour-project.webp +0 -0
- package/dist/assets/tips/tour-review.webp +0 -0
- package/dist/assets/tips/tour-share.webp +0 -0
- package/dist/assets/tips/tour-source.webp +0 -0
- package/dist/assets/tips/tour-target.webp +0 -0
- package/dist/assets/tips/tour-view.webp +0 -0
- package/dist/assets/tips/tour.json +117 -112
- package/dist/assets/tips/tour.webp +0 -0
- package/dist/mcp/visualproof-mcp.mjs +7 -3
- package/dist/server/access.js +37 -1
- package/dist/server/accesspage.js +130 -0
- package/dist/server/agent.js +11 -5
- package/dist/server/agentlibrary.js +1 -1
- package/dist/server/agentviews.js +1 -1
- package/dist/server/artcmd.js +1 -1
- package/dist/server/askfollow.js +1 -4
- package/dist/server/buildwatch.js +133 -0
- package/dist/server/deploy.js +844 -0
- package/dist/server/doctor.js +15 -3
- package/dist/server/events.js +20 -5
- package/dist/server/export.js +17 -4
- package/dist/server/fetchurl.js +30 -13
- package/dist/server/film.js +1 -1
- package/dist/server/history.js +3 -1
- package/dist/server/historyapi.js +100 -50
- package/dist/server/historythread.js +206 -0
- package/dist/server/hover.js +8 -0
- package/dist/server/identity.js +25 -1
- package/dist/server/library.js +10 -3
- package/dist/server/live.js +206 -28
- package/dist/server/lostfound.js +3 -2
- package/dist/server/main.js +361 -50
- package/dist/server/poppler.js +48 -10
- package/dist/server/renames.js +10 -5
- package/dist/server/review.js +32 -2
- package/dist/server/reviewapi.js +1 -1
- package/dist/server/status.js +144 -50
- package/dist/server/talk.js +7 -2
- package/dist/server/usage.js +51 -0
- package/dist/server/views.js +21 -7
- package/dist/shared/board.js +2 -1
- package/dist/shared/collab.js +14 -8
- package/dist/shared/deploy.js +35 -0
- package/dist/shared/doc/latex.js +5 -3
- package/dist/shared/doc/outline.js +6 -3
- package/dist/shared/layout.js +297 -70
- package/dist/shared/libm.js +366 -0
- package/dist/shared/library.js +11 -6
- package/dist/shared/palette.js +1 -1
- package/dist/shared/review.js +7 -5
- package/dist/shared/search.js +7 -3
- package/dist/shared/separate.js +556 -32
- package/dist/shared/views.js +8 -1
- package/dist/web/bundle/app.js +3595 -1520
- package/dist/web/bundle/edges.js +44 -16
- package/dist/web/bundle/layout.js +1026 -80
- package/dist/web/index.html +2 -2
- package/dist/web/src/artwork.js +25 -4
- package/dist/web/src/axiomring.js +42 -0
- package/dist/web/src/bundles.js +52 -16
- package/dist/web/src/canvas.js +109 -54
- package/dist/web/src/overview.js +4 -1
- package/dist/web/src/past.js +28 -0
- package/dist/web/src/scene.js +13 -12
- package/dist/web/src/selection.js +47 -4
- package/dist/web/src/status.js +32 -2
- package/dist/web/src/theme.js +108 -0
- package/dist/web/style.css +181 -45
- package/docs/agent-api/following.md +1 -1
- package/docs/agent-api/history.md +3 -0
- package/docs/agent-api/library.md +1 -1
- package/docs/agent-api/requests.md +4 -1
- package/docs/agent-api/views.md +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,844 @@
|
|
|
1
|
+
// Following a branch of a git repository (SPEC/DeploySpec.md): `visualproof serve <git URL>`.
|
|
2
|
+
// The server keeps a clone of its own in the cache, one per URL and branch, never anyone's
|
|
3
|
+
// workspace; asks the remote for the branch's head every so often (`git ls-remote`); and builds
|
|
4
|
+
// each new head itself — fetched, checked out detached, Mathlib's cache got where the project is on
|
|
5
|
+
// Mathlib, `lake build` — one build at a time, in programs of their own (never the server's
|
|
6
|
+
// process), the newest head seen meanwhile the only one built after it. The live loop
|
|
7
|
+
// (server/live.ts) reads the build as it reads a workspace's, but only between builds: while one
|
|
8
|
+
// changes the tree, nothing is read from it, and the board on show stays the last build's.
|
|
9
|
+
//
|
|
10
|
+
// What is kept, in the deployment's folder of the cache (`<cache>/deploy/<repo>-<hash>/`): the
|
|
11
|
+
// clone, `state.json` (the commit the tree's outputs are of, the build going on, the last one
|
|
12
|
+
// that ended, the newest head seen), each build's log (`logs/`), and `.lock`: one server follows a
|
|
13
|
+
// branch with a cache at a time. A build cut short (a restart into a patch, a stop) is done again
|
|
14
|
+
// by the next server, Lake keeping what it finished.
|
|
15
|
+
//
|
|
16
|
+
// Git and Lake are run with argument arrays, never a shell; the address is checked against the
|
|
17
|
+
// schemes followed, the branch by git (`check-ref-format --branch`); nothing that git or Lake starts
|
|
18
|
+
// waits on a prompt; the credentials are git's own (an SSH key, a deploy key, a credential helper),
|
|
19
|
+
// and VisualProof keeps none.
|
|
20
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
21
|
+
import { createHash } from "node:crypto";
|
|
22
|
+
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
23
|
+
import { homedir, tmpdir } from "node:os";
|
|
24
|
+
import { join } from "node:path";
|
|
25
|
+
import { shortCommit, slimDeploy } from "../shared/deploy.js";
|
|
26
|
+
import { childEnv, DETACH, end, grouped } from "./children.js";
|
|
27
|
+
import { dataHome, lockSite, SiteBusy } from "./dirs.js";
|
|
28
|
+
import { sourceAddress } from "./history.js";
|
|
29
|
+
import { openProject, ProjectError } from "./project.js";
|
|
30
|
+
/** How often the remote is asked by default (DeploySpec decision 3). */
|
|
31
|
+
export const DEFAULT_POLL_MINUTES = 2;
|
|
32
|
+
/** the least `--poll` takes, in minutes */
|
|
33
|
+
export const MIN_POLL_MINUTES = 0.05;
|
|
34
|
+
/** a question to the remote is given up after this long */
|
|
35
|
+
const ASK_MS = 60_000;
|
|
36
|
+
/** a fetch, a checkout */
|
|
37
|
+
const GIT_MS = 10 * 60_000;
|
|
38
|
+
/** Mathlib's cache (`lake exe cache get`): it compiles the cache program first */
|
|
39
|
+
const CACHE_MS = 30 * 60_000;
|
|
40
|
+
/** `lake build`, past which the build is stopped and failed (proposed: DeploySpec decision 14) */
|
|
41
|
+
export const BUILD_MS = 6 * 3600_000;
|
|
42
|
+
/** the most of a build's log kept for the pages, in lines and characters */
|
|
43
|
+
const TAIL_LINES = 60;
|
|
44
|
+
const TAIL_CHARS = 8000;
|
|
45
|
+
/** the lines of the build going on the pages are shown */
|
|
46
|
+
const LIVE_LINES = 6;
|
|
47
|
+
/** build logs kept, the newest */
|
|
48
|
+
const LOGS_KEPT = 30;
|
|
49
|
+
/** Whether a project's place on the command line is a git address rather than a folder: a URL
|
|
50
|
+
* (`scheme://…`), or git's scp-like `[user@]host:path` (a colon before any slash). Anything else is
|
|
51
|
+
* a folder; a folder whose name has a colon is written `./a:b`. */
|
|
52
|
+
export function isGitAddress(arg) {
|
|
53
|
+
if (/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(arg))
|
|
54
|
+
return true;
|
|
55
|
+
return /^[^/\\:\s]+:/.test(arg) && !/^[A-Za-z]:[\\/]/.test(arg);
|
|
56
|
+
}
|
|
57
|
+
const HOST = /^(?:[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)*$/;
|
|
58
|
+
const PATH = /^[\w.~/@+-]+$/;
|
|
59
|
+
/** A git address checked against what is followed (DeploySpec decision 10): `https://` without a
|
|
60
|
+
* user name or password in it, `ssh://` and scp-like (`git@host:path`), and `file://` for a
|
|
61
|
+
* repository on this machine; never `http://` (nothing checks what comes), `git://`, `ext::` or any
|
|
62
|
+
* other. The address, or why it is refused. */
|
|
63
|
+
export function parseRemote(arg) {
|
|
64
|
+
const no = (why) => ({ error: `${arg.slice(0, 200)}: ${why}` });
|
|
65
|
+
if (!arg || arg.length > 2000 || /[\s\0-\x1f\x7f]/.test(arg))
|
|
66
|
+
return no("an address has no spaces or control characters");
|
|
67
|
+
if (arg.startsWith("-"))
|
|
68
|
+
return no("an address does not start with -");
|
|
69
|
+
if (/(^|[/:])\.\.?(\/|$)/.test(arg))
|
|
70
|
+
return no("not a repository's path: no . or .. in it");
|
|
71
|
+
const scheme = /^([A-Za-z][A-Za-z0-9+.-]*):\/\//.exec(arg)?.[1].toLowerCase();
|
|
72
|
+
let kind;
|
|
73
|
+
let host = "";
|
|
74
|
+
let path;
|
|
75
|
+
if (scheme) {
|
|
76
|
+
if (scheme === "http")
|
|
77
|
+
return no("http:// is not followed: nothing checks what comes over it. Give the https:// address");
|
|
78
|
+
if (scheme !== "https" && scheme !== "ssh" && scheme !== "file")
|
|
79
|
+
return no(`${scheme}:// is not followed: give an https:// or ssh:// address, git's scp-like one (git@host:owner/repo.git), or file:// for a repository on this machine`);
|
|
80
|
+
let u;
|
|
81
|
+
try {
|
|
82
|
+
u = new URL(arg);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return no("not an address");
|
|
86
|
+
}
|
|
87
|
+
if (u.search || u.hash)
|
|
88
|
+
return no("an address has no query or fragment");
|
|
89
|
+
if (scheme === "file") {
|
|
90
|
+
if (u.host)
|
|
91
|
+
return no("file:// names a repository on this machine: file:///its/absolute/path");
|
|
92
|
+
kind = "file";
|
|
93
|
+
path = decodeURIComponent(u.pathname);
|
|
94
|
+
if (!path.startsWith("/") || path === "/")
|
|
95
|
+
return no("file:// takes the repository's absolute path");
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
if (u.password || (scheme === "https" && u.username))
|
|
99
|
+
return no("no user name, password or token in the address: VisualProof keeps no credential. Let git find it (an SSH key, a deploy key, a credential helper)");
|
|
100
|
+
if (u.username && !/^[A-Za-z0-9._-]+$/.test(u.username))
|
|
101
|
+
return no("not a user name git takes");
|
|
102
|
+
host = u.hostname;
|
|
103
|
+
if (!HOST.test(host) && !/^\[[0-9A-Fa-f:.]+\]$/.test(host))
|
|
104
|
+
return no("not a host name");
|
|
105
|
+
kind = scheme === "https" ? "https" : "ssh";
|
|
106
|
+
path = decodeURIComponent(u.pathname);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
// scp-like, as git reads it: [user@]host:path
|
|
111
|
+
const m = /^(?:([A-Za-z0-9._][A-Za-z0-9._-]*)@)?([^@/:]+):(.+)$/.exec(arg);
|
|
112
|
+
if (!m)
|
|
113
|
+
return no("not a git address: give https://host/owner/repo, ssh://… or git@host:owner/repo");
|
|
114
|
+
if (!HOST.test(m[2]))
|
|
115
|
+
return no("not a host name");
|
|
116
|
+
host = m[2];
|
|
117
|
+
path = m[3];
|
|
118
|
+
if (path.startsWith("-"))
|
|
119
|
+
return no("a path does not start with -");
|
|
120
|
+
kind = "ssh";
|
|
121
|
+
}
|
|
122
|
+
const parts = path.replace(/\/+$/, "").split("/").filter(Boolean);
|
|
123
|
+
if (!parts.length || !PATH.test(path) || parts.some((p) => p === ".."))
|
|
124
|
+
return no("not a repository's path");
|
|
125
|
+
const name = (parts[parts.length - 1].replace(/\.git$/, "").replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^[.-]+/, "") || "repository").slice(0, 60);
|
|
126
|
+
const gh = kind !== "file" && /^(www\.)?github\.com$/i.test(host) && parts.length === 2 ? `${parts[0]}/${parts[1].replace(/\.git$/, "")}` : null;
|
|
127
|
+
return { url: arg, kind, source: kind === "file" ? null : sourceAddress(arg), github: gh, name };
|
|
128
|
+
}
|
|
129
|
+
/** Why `name` is no branch to follow, or null: git's own check (`git check-ref-format --branch`),
|
|
130
|
+
* after refusing what that check would read otherwise (`@{-1}`, the branch checked out before). */
|
|
131
|
+
export function branchProblem(name) {
|
|
132
|
+
const no = `${name.slice(0, 200)} is not a branch name git takes (git check-ref-format --branch)`;
|
|
133
|
+
if (!name || name.length > 255 || name.startsWith("-") || name.includes("@{") || /[\s\0-\x1f\x7f]/.test(name))
|
|
134
|
+
return no;
|
|
135
|
+
const r = spawnSync("git", ["check-ref-format", "--branch", name], { cwd: tmpdir(), encoding: "utf8", timeout: 10_000, env: gitBase() });
|
|
136
|
+
if (r.error)
|
|
137
|
+
return `git could not be run (${r.error.message}): following a branch needs git`;
|
|
138
|
+
return r.status === 0 && r.stdout.trim() === name ? null : no;
|
|
139
|
+
}
|
|
140
|
+
// ---------------------------------------------------------------- where it is kept
|
|
141
|
+
/** A deployment's name among the others: the repository's name and a hash of the address and the
|
|
142
|
+
* branch. */
|
|
143
|
+
export function deployKey(remote, branch) {
|
|
144
|
+
return `${remote.name}-${createHash("sha1").update(`${remote.url}\0${branch}`).digest("hex").slice(0, 8)}`;
|
|
145
|
+
}
|
|
146
|
+
export function deployFolder(cacheRoot, remote, branch) {
|
|
147
|
+
const dir = join(cacheRoot, "deploy", deployKey(remote, branch));
|
|
148
|
+
return { dir, checkout: join(dir, remote.name), state: join(dir, "state.json"), logs: join(dir, "logs") };
|
|
149
|
+
}
|
|
150
|
+
/** The site of a deployment, without `--site`: under the data home by the address and the branch,
|
|
151
|
+
* so that the review stays where it is whatever `--cache` says. */
|
|
152
|
+
export function deploySite(remote, branch, h) {
|
|
153
|
+
return join(dataHome(h), "sites", deployKey(remote, branch));
|
|
154
|
+
}
|
|
155
|
+
/** The remotes' default branches as last asked (`<cache>/deploy/remotes.json`): a server started
|
|
156
|
+
* without `--branch` and without the network follows the branch it followed. */
|
|
157
|
+
function remotesFile(cacheRoot) {
|
|
158
|
+
return join(cacheRoot, "deploy", "remotes.json");
|
|
159
|
+
}
|
|
160
|
+
export function recordedDefault(cacheRoot, url) {
|
|
161
|
+
try {
|
|
162
|
+
const all = JSON.parse(readFileSync(remotesFile(cacheRoot), "utf8"));
|
|
163
|
+
const b = Object.hasOwn(all, url) ? all[url].branch : null;
|
|
164
|
+
return typeof b === "string" && !branchProblem(b) ? b : null;
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
function recordDefault(cacheRoot, url, branch) {
|
|
171
|
+
const file = remotesFile(cacheRoot);
|
|
172
|
+
let all = {};
|
|
173
|
+
try {
|
|
174
|
+
all = JSON.parse(readFileSync(file, "utf8"));
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
// none yet
|
|
178
|
+
}
|
|
179
|
+
all[url] = { branch, at: new Date().toISOString() };
|
|
180
|
+
try {
|
|
181
|
+
mkdirSync(join(cacheRoot, "deploy"), { recursive: true });
|
|
182
|
+
writeFileSync(`${file}.tmp-${process.pid}`, JSON.stringify(all, null, 1) + "\n");
|
|
183
|
+
renameSync(`${file}.tmp-${process.pid}`, file);
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
// a record to spare the network at the next start; not a reason to fail
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// ---------------------------------------------------------------- the programs' environment
|
|
190
|
+
/** What tells git which repository to work on, where the server's own environment has it (a hook's,
|
|
191
|
+
* a test's): never handed on, so that git works on the clone alone. */
|
|
192
|
+
const REPO_VARS = ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_NAMESPACE", "GIT_PREFIX", "GIT_QUARANTINE_PATH", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT"];
|
|
193
|
+
/** The server's environment without what is the server's alone (`VISUALPROOF_*`, server/children.ts)
|
|
194
|
+
* and without what would point git elsewhere. */
|
|
195
|
+
function gitBase() {
|
|
196
|
+
const env = childEnv();
|
|
197
|
+
for (const k of REPO_VARS)
|
|
198
|
+
delete env[k];
|
|
199
|
+
for (const k of Object.keys(env))
|
|
200
|
+
if (/^GIT_CONFIG_(KEY|VALUE)_\d+$/.test(k))
|
|
201
|
+
delete env[k];
|
|
202
|
+
return env;
|
|
203
|
+
}
|
|
204
|
+
let sshConfigured;
|
|
205
|
+
/** The ssh command git is configured with (`core.sshCommand`, the user's or the system's), read once. */
|
|
206
|
+
function configuredSsh() {
|
|
207
|
+
if (sshConfigured !== undefined)
|
|
208
|
+
return sshConfigured;
|
|
209
|
+
const r = spawnSync("git", ["config", "--get", "core.sshCommand"], { cwd: tmpdir(), encoding: "utf8", timeout: 10_000, env: gitBase() });
|
|
210
|
+
return (sshConfigured = r.status === 0 && r.stdout.trim() ? r.stdout.trim() : null);
|
|
211
|
+
}
|
|
212
|
+
/** No prompt is ever waited on (DeploySpec decision 9): git asks no terminal and runs no askpass
|
|
213
|
+
* (`GIT_TERMINAL_PROMPT=0`, an empty `GIT_ASKPASS`), the credential manager is not interactive, and
|
|
214
|
+
* ssh runs in batch mode — the ssh command git would run (`GIT_SSH_COMMAND`, else `core.sshCommand`,
|
|
215
|
+
* else `ssh`) with `-o BatchMode=yes` after it; a `GIT_SSH` program is left as it is. Credentials
|
|
216
|
+
* are git's own: helpers, keys and agents work as for any git. The programs run without a
|
|
217
|
+
* terminal of their own besides (server/children.ts, `grouped`). */
|
|
218
|
+
export function noPrompts(env, ssh = configuredSsh()) {
|
|
219
|
+
const out = { ...env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "", SSH_ASKPASS_REQUIRE: "never", GCM_INTERACTIVE: "never" };
|
|
220
|
+
const own = env.GIT_SSH_COMMAND || ssh || (env.GIT_SSH ? null : "ssh");
|
|
221
|
+
if (own)
|
|
222
|
+
out.GIT_SSH_COMMAND = `${own} -o BatchMode=yes`;
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
/** git's environment, for the remote `remote`: no prompts, and only its protocol allowed
|
|
226
|
+
* (`GIT_ALLOW_PROTOCOL`), whatever a redirect or a configuration says. */
|
|
227
|
+
export function gitEnv(remote) {
|
|
228
|
+
return { ...noPrompts(gitBase()), GIT_ALLOW_PROTOCOL: remote.kind };
|
|
229
|
+
}
|
|
230
|
+
/** Lake's environment for a build: the server's without what is its own — no access key or token
|
|
231
|
+
* of VisualProof's is in it — and no prompts for the git it runs for the dependencies. */
|
|
232
|
+
export function buildEnv() {
|
|
233
|
+
return noPrompts(gitBase());
|
|
234
|
+
}
|
|
235
|
+
/** Run `cmd` with `args` — an array, never through a shell — as a group of its own (what it starts
|
|
236
|
+
* ends with it: server/children.ts), stopped past `timeoutMs`; what it writes goes to `onText` as
|
|
237
|
+
* it comes. Never rejects. */
|
|
238
|
+
export function run(cmd, args, o) {
|
|
239
|
+
return new Promise((done) => {
|
|
240
|
+
const keep = o.keep ?? 20_000;
|
|
241
|
+
let stdout = "", out = "", timedOut = false, finished = false;
|
|
242
|
+
let child;
|
|
243
|
+
try {
|
|
244
|
+
child = grouped(spawn(cmd, args, { cwd: o.cwd, env: o.env, stdio: ["ignore", "pipe", "pipe"], detached: DETACH }));
|
|
245
|
+
}
|
|
246
|
+
catch (e) {
|
|
247
|
+
return done({ code: null, signal: null, stdout, out, timedOut, error: e instanceof Error ? e.message : String(e) });
|
|
248
|
+
}
|
|
249
|
+
const take = (b, own) => {
|
|
250
|
+
const t = b.toString();
|
|
251
|
+
if (own)
|
|
252
|
+
stdout = (stdout + t).slice(-keep);
|
|
253
|
+
out = (out + t).slice(-keep);
|
|
254
|
+
o.onText?.(t);
|
|
255
|
+
};
|
|
256
|
+
child.stdout.on("data", (b) => take(b, true));
|
|
257
|
+
child.stderr.on("data", (b) => take(b, false));
|
|
258
|
+
const timer = setTimeout(() => {
|
|
259
|
+
timedOut = true;
|
|
260
|
+
end(child);
|
|
261
|
+
setTimeout(() => end(child, "SIGKILL"), 5000).unref();
|
|
262
|
+
}, o.timeoutMs);
|
|
263
|
+
const finish = (r) => {
|
|
264
|
+
if (finished)
|
|
265
|
+
return;
|
|
266
|
+
finished = true;
|
|
267
|
+
clearTimeout(timer);
|
|
268
|
+
done(r);
|
|
269
|
+
};
|
|
270
|
+
child.on("error", (e) => finish({ code: null, signal: null, stdout, out, timedOut, error: e.message }));
|
|
271
|
+
child.on("close", (code, signal) => finish({ code, signal, stdout, out, timedOut }));
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
/** What git said went wrong, in a line: its last `fatal:` or `error:` line, else its last line. */
|
|
275
|
+
function gitWhy(r) {
|
|
276
|
+
if (r.error)
|
|
277
|
+
return `git could not be run: ${r.error}`;
|
|
278
|
+
if (r.timedOut)
|
|
279
|
+
return "git did not finish in time";
|
|
280
|
+
const lines = r.out.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
281
|
+
const said = [...lines].reverse().find((l) => /^(fatal|error):/.test(l)) ?? lines[lines.length - 1] ?? `git exited with ${r.code}`;
|
|
282
|
+
return said.replace(/^(fatal|error):\s*/, "").slice(0, 400);
|
|
283
|
+
}
|
|
284
|
+
const git = (args, env, ms = GIT_MS, onText) => run("git", args, { env, timeoutMs: ms, cwd: tmpdir(), onText });
|
|
285
|
+
const SHA = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
|
|
286
|
+
/** The head of `branch` on the remote; without a branch, the remote's default branch and its head
|
|
287
|
+
* (`git ls-remote --symref HEAD`). Only that branch is ever asked of: never a pull request's ref, nor
|
|
288
|
+
* a fork's. */
|
|
289
|
+
export async function askRemote(remote, branch, env) {
|
|
290
|
+
const r = await git(branch ? ["ls-remote", "--", remote.url, `refs/heads/${branch}`] : ["ls-remote", "--symref", "--", remote.url, "HEAD"], env, ASK_MS);
|
|
291
|
+
if (r.code !== 0 || r.error || r.timedOut)
|
|
292
|
+
return { error: r.timedOut ? "the remote did not answer within a minute" : gitWhy(r) };
|
|
293
|
+
const lines = r.stdout.split("\n");
|
|
294
|
+
if (!branch) {
|
|
295
|
+
const ref = lines.map((l) => /^ref: refs\/heads\/(\S+)\tHEAD$/.exec(l)).find(Boolean)?.[1];
|
|
296
|
+
if (!ref || branchProblem(ref))
|
|
297
|
+
return { error: "the remote names no default branch: say which to follow, --branch <name>" };
|
|
298
|
+
const head = lines.map((l) => /^([0-9a-f]+)\tHEAD$/.exec(l)).find(Boolean)?.[1];
|
|
299
|
+
return head && SHA.test(head) ? { head, branch: ref } : { error: `the remote's ${ref} has no commit` };
|
|
300
|
+
}
|
|
301
|
+
// the pattern matches refs that end so: the branch's own ref alone
|
|
302
|
+
const head = lines.map((l) => l.split("\t")).find((p) => p[1] === `refs/heads/${branch}`)?.[0];
|
|
303
|
+
return head && SHA.test(head) ? { head, branch } : { error: `the remote has no branch ${branch}` };
|
|
304
|
+
}
|
|
305
|
+
/** The branch to follow: `given`, else the remote's default as it says it now, else as it said it
|
|
306
|
+
* last (the network may be away at a restart). */
|
|
307
|
+
export async function branchToFollow(remote, given, cacheRoot) {
|
|
308
|
+
if (given)
|
|
309
|
+
return given;
|
|
310
|
+
const a = await askRemote(remote, null, gitEnv(remote));
|
|
311
|
+
if ("head" in a) {
|
|
312
|
+
recordDefault(cacheRoot, remote.url, a.branch);
|
|
313
|
+
return a.branch;
|
|
314
|
+
}
|
|
315
|
+
return recordedDefault(cacheRoot, remote.url) ?? { error: `cannot learn the remote's default branch (${a.error}): give --branch <name>` };
|
|
316
|
+
}
|
|
317
|
+
/** The deployment's clone, made at the first start (DeploySpec decision 2): an empty repository
|
|
318
|
+
* whose `origin` is the address, the branch fetched alone, its head checked out detached. It is
|
|
319
|
+
* made in a folder of its own and moved into place once whole, so that a start cut short leaves
|
|
320
|
+
* nothing half made. The commit checked out, or why there is none. */
|
|
321
|
+
export async function ensureClone(remote, branch, folder, log) {
|
|
322
|
+
const env = gitEnv(remote);
|
|
323
|
+
if (existsSync(join(folder.checkout, ".git"))) {
|
|
324
|
+
const r = await git(["-C", folder.checkout, "rev-parse", "--verify", "HEAD^{commit}"], env);
|
|
325
|
+
const c = r.stdout.trim();
|
|
326
|
+
if (r.code === 0 && SHA.test(c))
|
|
327
|
+
return { commit: c, made: false };
|
|
328
|
+
return { error: `the clone ${folder.checkout} cannot be read (${gitWhy(r)}): remove it, and it is made again` };
|
|
329
|
+
}
|
|
330
|
+
mkdirSync(folder.dir, { recursive: true });
|
|
331
|
+
// what a start that died midway left
|
|
332
|
+
for (const f of readdirSync(folder.dir)) {
|
|
333
|
+
const m = new RegExp(`^${remote.name.replace(/[.]/g, "\\.")}\\.tmp-(\\d+)$`).exec(f);
|
|
334
|
+
if (m && !alive(Number(m[1])))
|
|
335
|
+
rmSync(join(folder.dir, f), { recursive: true, force: true });
|
|
336
|
+
}
|
|
337
|
+
const tmp = `${folder.checkout}.tmp-${process.pid}`;
|
|
338
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
339
|
+
const t0 = Date.now();
|
|
340
|
+
log(`cloning ${remote.source ?? remote.url} (${branch}) into ${folder.checkout}…`);
|
|
341
|
+
const steps = [
|
|
342
|
+
["init", "--quiet", tmp],
|
|
343
|
+
["-C", tmp, "remote", "add", "origin", "--", remote.url],
|
|
344
|
+
["-C", tmp, "fetch", "--no-tags", "--no-recurse-submodules", "--quiet", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`],
|
|
345
|
+
];
|
|
346
|
+
for (const s of steps) {
|
|
347
|
+
const r = await git(s, env);
|
|
348
|
+
if (r.code !== 0) {
|
|
349
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
350
|
+
return { error: `cannot clone ${remote.source ?? remote.url} (${branch}): ${gitWhy(r)}` };
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
const rp = await git(["-C", tmp, "rev-parse", "--verify", `refs/remotes/origin/${branch}^{commit}`], env);
|
|
354
|
+
const commit = rp.stdout.trim();
|
|
355
|
+
const co = SHA.test(commit) ? await git(["-C", tmp, "-c", "advice.detachedHead=false", "checkout", "--quiet", "--force", "--detach", commit], env) : rp;
|
|
356
|
+
if (co.code !== 0 || !SHA.test(commit)) {
|
|
357
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
358
|
+
return { error: `cannot check out ${branch} of ${remote.source ?? remote.url}: ${gitWhy(co)}` };
|
|
359
|
+
}
|
|
360
|
+
renameSync(tmp, folder.checkout);
|
|
361
|
+
log(`cloned in ${((Date.now() - t0) / 1000).toFixed(1)} s, at ${shortCommit(commit)}`);
|
|
362
|
+
return { commit, made: true };
|
|
363
|
+
}
|
|
364
|
+
function alive(pid) {
|
|
365
|
+
try {
|
|
366
|
+
process.kill(pid, 0);
|
|
367
|
+
return true;
|
|
368
|
+
}
|
|
369
|
+
catch (e) {
|
|
370
|
+
return e.code === "EPERM";
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
/** Another server follows the branch with this cache. */
|
|
374
|
+
export class DeployBusy extends Error {
|
|
375
|
+
}
|
|
376
|
+
/** Take the deployment for this process (`.lock`, as a site is taken: server/dirs.ts). */
|
|
377
|
+
export function lockDeployment(folder, remote, branch) {
|
|
378
|
+
mkdirSync(folder.dir, { recursive: true });
|
|
379
|
+
try {
|
|
380
|
+
return lockSite(folder.dir);
|
|
381
|
+
}
|
|
382
|
+
catch (e) {
|
|
383
|
+
if (e instanceof SiteBusy)
|
|
384
|
+
throw new DeployBusy(`another VisualProof server (process ${e.pid}) follows ${branch} of ${remote.source ?? remote.url} with this cache (${folder.dir}): stop it first, or give this one a --cache of its own`);
|
|
385
|
+
throw e;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
/** Whether the project is on Mathlib: its manifest names it (DeploySpec decision 4). */
|
|
389
|
+
export function onMathlib(root) {
|
|
390
|
+
try {
|
|
391
|
+
const m = JSON.parse(readFileSync(join(root, "lake-manifest.json"), "utf8"));
|
|
392
|
+
return Array.isArray(m.packages) && m.packages.some((p) => typeof p?.name === "string" && p.name.toLowerCase() === "mathlib");
|
|
393
|
+
}
|
|
394
|
+
catch {
|
|
395
|
+
return false;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
// ---------------------------------------------------------------- a build's log
|
|
399
|
+
/** A build's log: everything its programs wrote, in `logs/<commit>.log` of the deployment (a build
|
|
400
|
+
* done again writes it again), and its end for the pages, where the clone's folder reads `.` and the
|
|
401
|
+
* home `~`, colours and redrawn lines taken out. */
|
|
402
|
+
export class BuildLog {
|
|
403
|
+
file;
|
|
404
|
+
fd = null;
|
|
405
|
+
lines = [];
|
|
406
|
+
partial = "";
|
|
407
|
+
/** the clone's path as the log may write it, the longest first: Lean prints the path resolved
|
|
408
|
+
* (macOS's /var is /private/var), git and Lake the one given */
|
|
409
|
+
roots;
|
|
410
|
+
constructor(dir, commit, root) {
|
|
411
|
+
mkdirSync(dir, { recursive: true });
|
|
412
|
+
this.file = join(dir, `${commit}.log`);
|
|
413
|
+
let real = root;
|
|
414
|
+
try {
|
|
415
|
+
real = realpathSync(root);
|
|
416
|
+
}
|
|
417
|
+
catch {
|
|
418
|
+
// not there yet: the path as given
|
|
419
|
+
}
|
|
420
|
+
this.roots = [...new Set([real, root])].sort((a, b) => b.length - a.length);
|
|
421
|
+
try {
|
|
422
|
+
this.fd = openSync(this.file, "w", 0o600);
|
|
423
|
+
}
|
|
424
|
+
catch {
|
|
425
|
+
// a log that cannot be written: its end is still kept for the pages
|
|
426
|
+
}
|
|
427
|
+
prune(dir, LOGS_KEPT);
|
|
428
|
+
}
|
|
429
|
+
/** a line of VisualProof's own: the step, the command */
|
|
430
|
+
say(line) {
|
|
431
|
+
this.text(`${line}\n`);
|
|
432
|
+
}
|
|
433
|
+
text(t) {
|
|
434
|
+
if (this.fd !== null) {
|
|
435
|
+
try {
|
|
436
|
+
writeSync(this.fd, t);
|
|
437
|
+
}
|
|
438
|
+
catch {
|
|
439
|
+
this.fd = null;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
const parts = (this.partial + t).split("\n");
|
|
443
|
+
this.partial = parts.pop() ?? "";
|
|
444
|
+
for (const p of parts)
|
|
445
|
+
this.push(p);
|
|
446
|
+
}
|
|
447
|
+
push(raw) {
|
|
448
|
+
// a line redrawn in place (a progress bar) is its last state
|
|
449
|
+
const line = raw.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "").split("\r").pop().trimEnd();
|
|
450
|
+
if (!line)
|
|
451
|
+
return;
|
|
452
|
+
this.lines.push(line);
|
|
453
|
+
if (this.lines.length > TAIL_LINES * 2)
|
|
454
|
+
this.lines.splice(0, this.lines.length - TAIL_LINES);
|
|
455
|
+
}
|
|
456
|
+
/** its end, as the pages are given it: its last `n` lines, the one being written included */
|
|
457
|
+
tail(n = TAIL_LINES) {
|
|
458
|
+
const last = this.partial.replace(/\x1b\[[0-9;?]*[ -/]*[@-~]/g, "").split("\r").pop().trimEnd();
|
|
459
|
+
const home = homedir();
|
|
460
|
+
let t = [...this.lines, ...(last ? [last] : [])].slice(-n).join("\n");
|
|
461
|
+
for (const r of this.roots)
|
|
462
|
+
t = t.split(r).join(".");
|
|
463
|
+
if (home && home.length > 1)
|
|
464
|
+
t = t.split(home).join("~");
|
|
465
|
+
return t.length > TAIL_CHARS ? `…${t.slice(-TAIL_CHARS)}` : t;
|
|
466
|
+
}
|
|
467
|
+
close() {
|
|
468
|
+
if (this.fd !== null)
|
|
469
|
+
closeSync(this.fd);
|
|
470
|
+
this.fd = null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
/** Keep the newest `keep` files of `dir`. */
|
|
474
|
+
function prune(dir, keep) {
|
|
475
|
+
try {
|
|
476
|
+
const files = readdirSync(dir)
|
|
477
|
+
.filter((f) => f.endsWith(".log"))
|
|
478
|
+
.map((f) => ({ f, t: statSync(join(dir, f)).mtimeMs }))
|
|
479
|
+
.sort((a, b) => b.t - a.t);
|
|
480
|
+
for (const { f } of files.slice(keep))
|
|
481
|
+
unlinkSync(join(dir, f));
|
|
482
|
+
}
|
|
483
|
+
catch {
|
|
484
|
+
// housekeeping
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
/** A branch followed and built (DeploySpec decisions 3–8): the remote asked every `everyMs`, each
|
|
488
|
+
* new head built in turn, the newest seen meanwhile the only one built after the build going on. */
|
|
489
|
+
export class Deployment {
|
|
490
|
+
remote;
|
|
491
|
+
branch;
|
|
492
|
+
folder;
|
|
493
|
+
o;
|
|
494
|
+
state;
|
|
495
|
+
/** the newest head seen: what is built next */
|
|
496
|
+
want;
|
|
497
|
+
/** the last commit tried */
|
|
498
|
+
done;
|
|
499
|
+
/** a commit whose try could not be made: tried again at the next look at the remote, not before */
|
|
500
|
+
later = null;
|
|
501
|
+
/** the tree is not one build's: a build was cut short, and nothing is read until one ends */
|
|
502
|
+
interrupted;
|
|
503
|
+
building = null;
|
|
504
|
+
asked = null;
|
|
505
|
+
timer = null;
|
|
506
|
+
retry = null;
|
|
507
|
+
stopped = false;
|
|
508
|
+
asking = null;
|
|
509
|
+
/** the build going on */
|
|
510
|
+
running = null;
|
|
511
|
+
env;
|
|
512
|
+
lakeEnv;
|
|
513
|
+
/** every build tried since the start, in order: what the tests read */
|
|
514
|
+
tried = [];
|
|
515
|
+
/** the log of the build going on */
|
|
516
|
+
log = null;
|
|
517
|
+
constructor(o) {
|
|
518
|
+
this.o = o;
|
|
519
|
+
this.remote = o.remote;
|
|
520
|
+
this.branch = o.branch;
|
|
521
|
+
this.folder = o.folder;
|
|
522
|
+
this.env = gitEnv(o.remote);
|
|
523
|
+
this.lakeEnv = buildEnv();
|
|
524
|
+
this.state = this.read();
|
|
525
|
+
const head = this.headNow();
|
|
526
|
+
// the tree is one build's only where the outputs are of the commit checked out and no build
|
|
527
|
+
// was going on
|
|
528
|
+
this.interrupted = this.state.building !== null || (this.state.outputs !== null && head !== null && head !== this.state.outputs.commit);
|
|
529
|
+
this.want = this.state.head?.commit ?? this.state.building?.commit ?? o.head ?? head;
|
|
530
|
+
this.done = this.interrupted ? null : this.state.done;
|
|
531
|
+
if (this.state.building)
|
|
532
|
+
o.hooks.log(`build: the build of ${shortCommit(this.state.building.commit)} was cut short (a restart, a stop): it is done again`);
|
|
533
|
+
}
|
|
534
|
+
read() {
|
|
535
|
+
const fresh = { version: 1, url: this.remote.url, branch: this.branch, outputs: null, building: null, done: null, last: null, head: null };
|
|
536
|
+
try {
|
|
537
|
+
const s = JSON.parse(readFileSync(this.folder.state, "utf8"));
|
|
538
|
+
return { ...fresh, ...s, version: 1, url: this.remote.url, branch: this.branch };
|
|
539
|
+
}
|
|
540
|
+
catch {
|
|
541
|
+
return fresh;
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
save() {
|
|
545
|
+
const tmp = `${this.folder.state}.tmp-${process.pid}`;
|
|
546
|
+
try {
|
|
547
|
+
writeFileSync(tmp, JSON.stringify(this.state, null, 1) + "\n");
|
|
548
|
+
renameSync(tmp, this.folder.state);
|
|
549
|
+
}
|
|
550
|
+
catch (e) {
|
|
551
|
+
this.o.hooks.log(`build: ${this.folder.state} cannot be written: ${e instanceof Error ? e.message : String(e)}`);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
/** the commit checked out in the clone */
|
|
555
|
+
headNow() {
|
|
556
|
+
const r = spawnSync("git", ["-C", this.folder.checkout, "rev-parse", "--verify", "HEAD^{commit}"], { encoding: "utf8", timeout: 10_000, env: this.env });
|
|
557
|
+
const c = r.status === 0 ? r.stdout.trim() : "";
|
|
558
|
+
return SHA.test(c) ? c : null;
|
|
559
|
+
}
|
|
560
|
+
/** Ask the remote at once, then every `everyMs`. */
|
|
561
|
+
start() {
|
|
562
|
+
void this.poll();
|
|
563
|
+
this.timer = setInterval(() => void this.poll(), this.o.everyMs);
|
|
564
|
+
this.timer.unref?.();
|
|
565
|
+
}
|
|
566
|
+
/** No more asking or building; a build going on is stopped with the server's programs
|
|
567
|
+
* (server/children.ts) and recorded as cut short, to be done again at the next start. */
|
|
568
|
+
stop() {
|
|
569
|
+
this.stopped = true;
|
|
570
|
+
if (this.timer)
|
|
571
|
+
clearInterval(this.timer);
|
|
572
|
+
if (this.retry)
|
|
573
|
+
clearTimeout(this.retry);
|
|
574
|
+
this.timer = this.retry = null;
|
|
575
|
+
}
|
|
576
|
+
/** Whether the tree is not to be read now: a build changes it, one was cut short, or none was ever
|
|
577
|
+
* made. */
|
|
578
|
+
holds() {
|
|
579
|
+
return this.building !== null || this.interrupted || this.state.outputs === null;
|
|
580
|
+
}
|
|
581
|
+
/** a build is going on, which a restart would cut short (UpdateSpec decision 6) */
|
|
582
|
+
get busy() {
|
|
583
|
+
return this.running !== null;
|
|
584
|
+
}
|
|
585
|
+
/** The commit what the tree holds was built from, and the branch: the history's for each build it
|
|
586
|
+
* reads (HistorySpec decision 4). */
|
|
587
|
+
commit() {
|
|
588
|
+
return this.state.outputs ? { hash: this.state.outputs.commit, branch: this.branch } : null;
|
|
589
|
+
}
|
|
590
|
+
/** What the pages and agents are told; without the build's log for a read-only reader. */
|
|
591
|
+
status(full) {
|
|
592
|
+
const s = {
|
|
593
|
+
source: this.remote.source,
|
|
594
|
+
github: this.remote.github,
|
|
595
|
+
branch: this.branch,
|
|
596
|
+
every: Math.round(this.o.everyMs / 1000),
|
|
597
|
+
asked: this.asked,
|
|
598
|
+
// what the build going on says now (the cache's download, Lake's last lines)
|
|
599
|
+
building: this.building && { ...this.building, ...(this.log ? { tail: this.log.tail(LIVE_LINES) } : {}) },
|
|
600
|
+
// a commit waiting for its try to be made again is said by the last build's
|
|
601
|
+
next: this.building && this.want && this.want !== this.done && this.want !== this.building.commit ? this.want : null,
|
|
602
|
+
last: this.state.last,
|
|
603
|
+
built: this.state.outputs?.commit ?? null,
|
|
604
|
+
};
|
|
605
|
+
return full ? s : slimDeploy(s);
|
|
606
|
+
}
|
|
607
|
+
/** Ask the remote for the branch's head; a new one is built once the build going on ends. A
|
|
608
|
+
* question asked while one is on its way waits for it. */
|
|
609
|
+
poll() {
|
|
610
|
+
this.asking ??= (async () => {
|
|
611
|
+
try {
|
|
612
|
+
const a = this.o.ask ? await this.o.ask() : await askRemote(this.remote, this.branch, this.env);
|
|
613
|
+
const at = new Date().toISOString();
|
|
614
|
+
if (this.stopped)
|
|
615
|
+
return;
|
|
616
|
+
if ("error" in a) {
|
|
617
|
+
if (this.asked?.error !== a.error)
|
|
618
|
+
this.o.hooks.log(`build: the remote cannot be asked: ${a.error}`);
|
|
619
|
+
this.asked = { at, head: this.asked?.head ?? null, error: a.error };
|
|
620
|
+
}
|
|
621
|
+
else {
|
|
622
|
+
this.asked = { at, head: a.head };
|
|
623
|
+
if (a.head !== this.state.head?.commit) {
|
|
624
|
+
this.state.head = { commit: a.head, at };
|
|
625
|
+
this.save();
|
|
626
|
+
}
|
|
627
|
+
if (a.head !== this.want) {
|
|
628
|
+
if (this.want)
|
|
629
|
+
this.o.hooks.log(`build: ${this.branch} is at ${shortCommit(a.head)} now${this.running ? `: built once the build of ${shortCommit(this.building?.commit ?? "")} ends` : ""}`);
|
|
630
|
+
this.want = a.head;
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
this.later = null;
|
|
634
|
+
this.o.hooks.changed();
|
|
635
|
+
this.kick();
|
|
636
|
+
}
|
|
637
|
+
finally {
|
|
638
|
+
this.asking = null;
|
|
639
|
+
}
|
|
640
|
+
})();
|
|
641
|
+
return this.asking;
|
|
642
|
+
}
|
|
643
|
+
/** Start the build of the newest head where none is going on and it is not built yet; while the
|
|
644
|
+
* server reads the tree, a moment later. */
|
|
645
|
+
kick() {
|
|
646
|
+
if (this.stopped || this.running || !this.want || this.want === this.done || this.want === this.later)
|
|
647
|
+
return;
|
|
648
|
+
if (this.o.hooks.reading()) {
|
|
649
|
+
this.retry ??= setTimeout(() => {
|
|
650
|
+
this.retry = null;
|
|
651
|
+
this.kick();
|
|
652
|
+
}, 300);
|
|
653
|
+
return;
|
|
654
|
+
}
|
|
655
|
+
const commit = this.want;
|
|
656
|
+
this.tried.push(commit);
|
|
657
|
+
this.running = this.attempt(commit)
|
|
658
|
+
.catch((e) => this.o.hooks.log(`build: ${shortCommit(commit)}: ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`))
|
|
659
|
+
.finally(() => {
|
|
660
|
+
this.running = null;
|
|
661
|
+
this.building = null;
|
|
662
|
+
this.o.hooks.changed();
|
|
663
|
+
this.kick();
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
/** Until the build going on and those waiting are done: what the tests wait for. */
|
|
667
|
+
async settled() {
|
|
668
|
+
while (this.running || this.asking || this.retry)
|
|
669
|
+
await (this.running ?? this.asking ?? new Promise((ok) => setTimeout(ok, 50)));
|
|
670
|
+
}
|
|
671
|
+
step(s) {
|
|
672
|
+
if (this.building)
|
|
673
|
+
this.building = { ...this.building, step: s };
|
|
674
|
+
this.o.hooks.changed();
|
|
675
|
+
}
|
|
676
|
+
/** The build of `commit` (DeploySpec decisions 4–7): fetched where the clone has it not, checked
|
|
677
|
+
* out detached and cleaned (the build outputs kept), the project read again as it says it,
|
|
678
|
+
* Mathlib's cache got where it is on Mathlib, `lake build`. A try that could not change or build
|
|
679
|
+
* the tree leaves it as the last build had it. */
|
|
680
|
+
async attempt(commit) {
|
|
681
|
+
const t0 = Date.now();
|
|
682
|
+
const since = new Date(t0).toISOString();
|
|
683
|
+
// before anything is awaited: from here the tree is not read (holds)
|
|
684
|
+
this.building = { commit, since, step: "fetch" };
|
|
685
|
+
this.o.hooks.changed();
|
|
686
|
+
const short = shortCommit(commit);
|
|
687
|
+
const say = this.o.hooks.log;
|
|
688
|
+
const log = (this.log = new BuildLog(this.folder.logs, commit, this.folder.checkout));
|
|
689
|
+
const g = (args, ms = GIT_MS) => {
|
|
690
|
+
log.say(`$ git ${args.join(" ")}`);
|
|
691
|
+
return git(["-C", this.folder.checkout, ...args], this.env, ms, (t) => log.text(t));
|
|
692
|
+
};
|
|
693
|
+
const ended = (b) => this.ended(commit, t0, { ...b, tail: log.tail() }, log);
|
|
694
|
+
try {
|
|
695
|
+
say(`build: ${short}: ${this.state.outputs ? "" : "the first build: "}fetching`);
|
|
696
|
+
if (!(await this.has(commit))) {
|
|
697
|
+
const f = await g(["fetch", "--no-tags", "--no-recurse-submodules", "--quiet", "origin", `+refs/heads/${this.branch}:refs/remotes/origin/${this.branch}`]);
|
|
698
|
+
if (this.stopped)
|
|
699
|
+
return;
|
|
700
|
+
if (f.code !== 0)
|
|
701
|
+
return ended({ ok: false, retry: true, error: `it could not be fetched: ${gitWhy(f)}` });
|
|
702
|
+
if (!(await this.has(commit)))
|
|
703
|
+
return ended({ ok: false, retry: true, error: "the branch moved on before it could be fetched: its newest commit is built next" });
|
|
704
|
+
}
|
|
705
|
+
// a graph being made ready to serve as the fetch went on (the usage pass's) is let finish: the
|
|
706
|
+
// tree changes from here
|
|
707
|
+
while (this.o.hooks.reading() && !this.stopped)
|
|
708
|
+
await new Promise((ok) => setTimeout(ok, 100));
|
|
709
|
+
if (this.stopped)
|
|
710
|
+
return;
|
|
711
|
+
// the build is recorded as going on, so that one cut short is done again (DeploySpec
|
|
712
|
+
// decision 8)
|
|
713
|
+
this.step("checkout");
|
|
714
|
+
this.o.hooks.starting();
|
|
715
|
+
this.state.building = { commit, since };
|
|
716
|
+
this.save();
|
|
717
|
+
const co = await g(["-c", "advice.detachedHead=false", "checkout", "--quiet", "--force", "--detach", commit]);
|
|
718
|
+
const cl = co.code === 0 ? await g(["clean", "-ffdq", "-e", ".lake"]) : co;
|
|
719
|
+
if (this.stopped)
|
|
720
|
+
return;
|
|
721
|
+
if (cl.code !== 0) {
|
|
722
|
+
await this.restore(log);
|
|
723
|
+
return ended({ ok: false, retry: true, error: `it could not be checked out: ${gitWhy(cl)}` });
|
|
724
|
+
}
|
|
725
|
+
try {
|
|
726
|
+
this.adopt(openProject(this.folder.checkout, this.o.roots, false));
|
|
727
|
+
}
|
|
728
|
+
catch (e) {
|
|
729
|
+
if (!(e instanceof ProjectError))
|
|
730
|
+
throw e;
|
|
731
|
+
await this.restore(log);
|
|
732
|
+
return ended({ ok: false, error: `it is no Lean project VisualProof can read: ${e.message}` });
|
|
733
|
+
}
|
|
734
|
+
const lake = this.o.lake ?? "lake";
|
|
735
|
+
if (onMathlib(this.folder.checkout)) {
|
|
736
|
+
this.step("cache");
|
|
737
|
+
say(`build: ${short}: getting Mathlib's cache`);
|
|
738
|
+
for (let n = 1; n <= 2 && !this.stopped; n++) {
|
|
739
|
+
log.say(`$ lake exe cache get`);
|
|
740
|
+
const c = await run(lake, ["exe", "cache", "get"], { cwd: this.folder.checkout, env: this.lakeEnv, timeoutMs: CACHE_MS, onText: (t) => log.text(t) });
|
|
741
|
+
if (c.code === 0)
|
|
742
|
+
break;
|
|
743
|
+
// Lake builds what the cache did not give
|
|
744
|
+
log.say(`Mathlib's cache could not be had (${c.error ?? (c.timedOut ? "it took too long" : `exit ${c.code}`)})${n === 1 ? ": trying again" : ": lake build builds what it lacks"}`);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
if (this.stopped)
|
|
748
|
+
return;
|
|
749
|
+
this.step("build");
|
|
750
|
+
say(`build: ${short}: lake build${this.o.roots.length ? ` ${this.o.roots.join(" ")}` : ""}`);
|
|
751
|
+
log.say(`$ lake build${this.o.roots.length ? ` ${this.o.roots.join(" ")}` : ""}`);
|
|
752
|
+
const b = await run(lake, ["build", ...this.o.roots], { cwd: this.folder.checkout, env: this.lakeEnv, timeoutMs: BUILD_MS, onText: (t) => log.text(t) });
|
|
753
|
+
// cut short by the server's stop: done again at the next start
|
|
754
|
+
if (this.stopped)
|
|
755
|
+
return;
|
|
756
|
+
if (b.error) {
|
|
757
|
+
await this.restore(log);
|
|
758
|
+
return ended({ ok: false, retry: true, error: `lake could not be run: ${b.error}` });
|
|
759
|
+
}
|
|
760
|
+
// the tree is this build's, whatever came of it
|
|
761
|
+
this.state.outputs = { commit, at: new Date().toISOString() };
|
|
762
|
+
this.state.building = null;
|
|
763
|
+
this.interrupted = false;
|
|
764
|
+
// the modules a library's globs give, now that they are built
|
|
765
|
+
try {
|
|
766
|
+
this.adopt(openProject(this.folder.checkout, this.o.roots, false));
|
|
767
|
+
}
|
|
768
|
+
catch {
|
|
769
|
+
// read as it was after the checkout
|
|
770
|
+
}
|
|
771
|
+
const ok = b.code === 0 && !b.timedOut;
|
|
772
|
+
return ended({ ok, exit: b.code, ...(b.timedOut ? { error: `it took longer than ${BUILD_MS / 3600_000} hours: stopped` } : {}) });
|
|
773
|
+
}
|
|
774
|
+
catch (e) {
|
|
775
|
+
if (this.stopped)
|
|
776
|
+
return;
|
|
777
|
+
if (this.state.building)
|
|
778
|
+
await this.restore(log);
|
|
779
|
+
return ended({ ok: false, retry: true, error: e instanceof Error ? e.message : String(e) });
|
|
780
|
+
}
|
|
781
|
+
finally {
|
|
782
|
+
log.close();
|
|
783
|
+
this.log = null;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
/** Whether the clone has `commit`. */
|
|
787
|
+
async has(commit) {
|
|
788
|
+
const r = await git(["-C", this.folder.checkout, "cat-file", "-e", `${commit}^{commit}`], this.env, ASK_MS);
|
|
789
|
+
return r.code === 0;
|
|
790
|
+
}
|
|
791
|
+
/** The tree as the last build left it, after a try that changed it and did not build it: checked
|
|
792
|
+
* out at that build's commit again, so that what is read of it, and a restart, are that build's.
|
|
793
|
+
* Where that cannot be done, the tree stays held until a build ends. */
|
|
794
|
+
async restore(log) {
|
|
795
|
+
const was = this.state.outputs?.commit;
|
|
796
|
+
if (was) {
|
|
797
|
+
log.say(`back to ${shortCommit(was)}, the last build's commit`);
|
|
798
|
+
const co = await git(["-C", this.folder.checkout, "-c", "advice.detachedHead=false", "checkout", "--quiet", "--force", "--detach", was], this.env);
|
|
799
|
+
if (co.code !== 0) {
|
|
800
|
+
log.say(`it could not be checked out again: ${gitWhy(co)}`);
|
|
801
|
+
this.interrupted = true;
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
804
|
+
try {
|
|
805
|
+
this.adopt(openProject(this.folder.checkout, this.o.roots, false));
|
|
806
|
+
}
|
|
807
|
+
catch {
|
|
808
|
+
// it was read at that build
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
// no build to go back to: nothing is read until one is made (holds)
|
|
812
|
+
this.state.building = null;
|
|
813
|
+
this.interrupted = false;
|
|
814
|
+
}
|
|
815
|
+
/** The project as the commit checked out says it, taken into the server's own in place: its
|
|
816
|
+
* toolchain, its name, its roots. */
|
|
817
|
+
adopt(fresh) {
|
|
818
|
+
const p = this.o.project;
|
|
819
|
+
const was = p.toolchain;
|
|
820
|
+
Object.assign(p, { ...fresh, root: p.root });
|
|
821
|
+
if (fresh.toolchain !== was) {
|
|
822
|
+
this.o.hooks.log(`build: the toolchain is ${fresh.toolchain} now (it was ${was})`);
|
|
823
|
+
this.o.hooks.toolchainChanged();
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
/** A try ended: recorded, said, and — unless it could not be made — not tried again. */
|
|
827
|
+
ended(commit, t0, b, log) {
|
|
828
|
+
const ms = Date.now() - t0;
|
|
829
|
+
const build = { commit, at: new Date().toISOString(), ms, ...b };
|
|
830
|
+
this.state.last = build;
|
|
831
|
+
if (!b.retry)
|
|
832
|
+
this.state.done = this.done = commit;
|
|
833
|
+
else
|
|
834
|
+
this.later = commit;
|
|
835
|
+
this.save();
|
|
836
|
+
this.o.hooks.ended?.();
|
|
837
|
+
const secs = ms < 60_000 ? `${(ms / 1000).toFixed(1)} s` : `${Math.round(ms / 60_000)} min`;
|
|
838
|
+
const short = shortCommit(commit);
|
|
839
|
+
if (b.ok)
|
|
840
|
+
this.o.hooks.log(`build: ${short} built in ${secs}`);
|
|
841
|
+
else
|
|
842
|
+
this.o.hooks.log(`build: ${short} failed after ${secs}: ${b.error ?? `lake build exited with ${b.exit}`}${b.retry ? "; tried again at the next look at the remote" : ""} (its log: ${log.file})`);
|
|
843
|
+
}
|
|
844
|
+
}
|