visualproof 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -7
- package/dist/assets/latex/visualproof.sty +19 -9
- package/dist/assets/tips/tour-agent.webp +0 -0
- package/dist/assets/tips/tour-arrange.webp +0 -0
- package/dist/assets/tips/tour-artwork.webp +0 -0
- package/dist/assets/tips/tour-checks.webp +0 -0
- package/dist/assets/tips/tour-keys.webp +0 -0
- package/dist/assets/tips/tour-library.webp +0 -0
- package/dist/assets/tips/tour-project.webp +0 -0
- package/dist/assets/tips/tour-review.webp +0 -0
- package/dist/assets/tips/tour-share.webp +0 -0
- package/dist/assets/tips/tour-source.webp +0 -0
- package/dist/assets/tips/tour-target.webp +0 -0
- package/dist/assets/tips/tour-view.webp +0 -0
- package/dist/assets/tips/tour.json +117 -112
- package/dist/assets/tips/tour.webp +0 -0
- package/dist/mcp/visualproof-mcp.mjs +7 -3
- package/dist/server/access.js +37 -1
- package/dist/server/accesspage.js +130 -0
- package/dist/server/agent.js +11 -5
- package/dist/server/agentlibrary.js +1 -1
- package/dist/server/agentviews.js +1 -1
- package/dist/server/artcmd.js +1 -1
- package/dist/server/askfollow.js +1 -4
- package/dist/server/buildwatch.js +133 -0
- package/dist/server/deploy.js +844 -0
- package/dist/server/doctor.js +15 -3
- package/dist/server/events.js +20 -5
- package/dist/server/export.js +17 -4
- package/dist/server/fetchurl.js +30 -13
- package/dist/server/film.js +1 -1
- package/dist/server/history.js +3 -1
- package/dist/server/historyapi.js +100 -50
- package/dist/server/historythread.js +206 -0
- package/dist/server/hover.js +8 -0
- package/dist/server/identity.js +25 -1
- package/dist/server/library.js +10 -3
- package/dist/server/live.js +206 -28
- package/dist/server/lostfound.js +3 -2
- package/dist/server/main.js +361 -50
- package/dist/server/poppler.js +48 -10
- package/dist/server/renames.js +10 -5
- package/dist/server/review.js +32 -2
- package/dist/server/reviewapi.js +1 -1
- package/dist/server/status.js +144 -50
- package/dist/server/talk.js +7 -2
- package/dist/server/usage.js +51 -0
- package/dist/server/views.js +21 -7
- package/dist/shared/board.js +2 -1
- package/dist/shared/collab.js +14 -8
- package/dist/shared/deploy.js +35 -0
- package/dist/shared/doc/latex.js +5 -3
- package/dist/shared/doc/outline.js +6 -3
- package/dist/shared/layout.js +297 -70
- package/dist/shared/libm.js +366 -0
- package/dist/shared/library.js +11 -6
- package/dist/shared/palette.js +1 -1
- package/dist/shared/review.js +7 -5
- package/dist/shared/search.js +7 -3
- package/dist/shared/separate.js +556 -32
- package/dist/shared/views.js +8 -1
- package/dist/web/bundle/app.js +3595 -1520
- package/dist/web/bundle/edges.js +44 -16
- package/dist/web/bundle/layout.js +1026 -80
- package/dist/web/index.html +2 -2
- package/dist/web/src/artwork.js +25 -4
- package/dist/web/src/axiomring.js +42 -0
- package/dist/web/src/bundles.js +52 -16
- package/dist/web/src/canvas.js +109 -54
- package/dist/web/src/overview.js +4 -1
- package/dist/web/src/past.js +28 -0
- package/dist/web/src/scene.js +13 -12
- package/dist/web/src/selection.js +47 -4
- package/dist/web/src/status.js +32 -2
- package/dist/web/src/theme.js +108 -0
- package/dist/web/style.css +181 -45
- package/docs/agent-api/following.md +1 -1
- package/docs/agent-api/history.md +3 -0
- package/docs/agent-api/library.md +1 -1
- package/docs/agent-api/requests.md +4 -1
- package/docs/agent-api/views.md +1 -1
- package/package.json +1 -1
package/dist/server/doctor.js
CHANGED
|
@@ -5,13 +5,14 @@
|
|
|
5
5
|
import { spawnSync } from "node:child_process";
|
|
6
6
|
import { accessSync, constants, existsSync, readFileSync, statSync } from "node:fs";
|
|
7
7
|
import { filmPin } from "./film.js";
|
|
8
|
-
import { tmpdir } from "node:os";
|
|
8
|
+
import { freemem, tmpdir, totalmem } from "node:os";
|
|
9
9
|
import { dirname, join, relative, resolve } from "node:path";
|
|
10
10
|
import { cacheHome, projectDirs, siteHolder } from "./dirs.js";
|
|
11
11
|
import { cmp, packageVersion, parseVersion, satisfies } from "./version.js";
|
|
12
12
|
import { choose, FROM_PACKAGE, installCommand, newerThan, newest, notesOf, npmCommand, PACKAGE_ROOT, readInfo, sameMinor, Versions } from "./updates.js";
|
|
13
13
|
export { packageVersion, satisfies };
|
|
14
14
|
import { ProjectError, openProject, targetsInWords } from "./project.js";
|
|
15
|
+
import { DEFAULT_JOBS, inGB, jobsAdvice, memoryFloor, onMathlib, WORKER_PEAK } from "./usage.js";
|
|
15
16
|
export const run = (cmd, args, cwd) => {
|
|
16
17
|
const r = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000, stdio: ["ignore", "pipe", "pipe"], ...(cwd ? { cwd } : {}) });
|
|
17
18
|
return r.error || r.status !== 0 ? null : `${r.stdout}${r.stderr}`;
|
|
@@ -179,7 +180,7 @@ export function doctor(o) {
|
|
|
179
180
|
? { level: "ok", what: `XeLaTeX (${/TeX Live \d+|MiKTeX[^\s)]*/.exec(xelatex)?.[0] ?? "found"})` }
|
|
180
181
|
: { level: "info", what: "XeLaTeX not found: needed only for the printed document (Export)", fix: install("xelatex", platform) });
|
|
181
182
|
if (exec("git", ["--version"]) === null) {
|
|
182
|
-
checks.push({ level: "info", what: "git not found: builds are kept without their commit, and past sources only as the builds kept them" });
|
|
183
|
+
checks.push({ level: "info", what: "git not found: builds are kept without their commit, and past sources only as the builds kept them; a branch cannot be followed (serve <git URL>)" });
|
|
183
184
|
}
|
|
184
185
|
const film = filmLine(o.cache ? resolve(o.cache) : cacheHome(), o.updates ?? process.env.VISUALPROOF_UPDATES);
|
|
185
186
|
const updates = updatesLine(exec, node, o.fromPackage ?? FROM_PACKAGE, o.cache ? resolve(o.cache) : cacheHome(), o.updates ?? process.env.VISUALPROOF_UPDATES, o.path === null ? null : projectDirs(resolve(o.path), o.site, o.cache).site);
|
|
@@ -211,7 +212,7 @@ export function doctor(o) {
|
|
|
211
212
|
}
|
|
212
213
|
const from = o.roots.length ? "--root" : existsSync(join(root, "lakefile.toml")) ? "lakefile.toml" : "lakefile.lean";
|
|
213
214
|
const missing = roots.filter((r) => !existsSync(join(libDir, ...r.split(".")) + ".olean"));
|
|
214
|
-
const mathlib =
|
|
215
|
+
const mathlib = onMathlib(root);
|
|
215
216
|
checks.push(missing.length === 0
|
|
216
217
|
? { level: "ok", what: `built: ${targetsInWords(project)} (from ${from})` }
|
|
217
218
|
: {
|
|
@@ -219,6 +220,17 @@ export function doctor(o) {
|
|
|
219
220
|
what: `not built: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ` and ${missing.length - 5} more` : ""} (${relative(root, join(libDir, ...missing[0].split("."))) + ".olean"} is missing; from ${from})`,
|
|
220
221
|
fix: `in ${root}: ${mathlib ? "lake exe cache get && " : ""}lake build${o.roots.length === 0 && from === "lakefile.lean" ? ", or name the modules to read with --root" : ""}`,
|
|
221
222
|
});
|
|
223
|
+
// what the usage pass's workers may hold, at the default --jobs, against the memory available
|
|
224
|
+
const mem = o.memory ?? { available: freemem(), total: totalmem() };
|
|
225
|
+
const advice = jobsAdvice(DEFAULT_JOBS, mathlib, mem.available);
|
|
226
|
+
const had = `memory: ${inGB(mem.available)} available of ${inGB(mem.total)}`;
|
|
227
|
+
checks.push(advice
|
|
228
|
+
? {
|
|
229
|
+
level: "warn",
|
|
230
|
+
what: `${had}: on Mathlib the usage pass's workers may hold up to ${inGB(WORKER_PEAK)} each, ${inGB(advice.need)} at ${DEFAULT_JOBS} at a time (the default --jobs); it starts fewer while less than ${inGB(memoryFloor(mem.total))} is available`,
|
|
231
|
+
fix: `serve with --jobs ${advice.fits}`,
|
|
232
|
+
}
|
|
233
|
+
: { level: "ok", what: `${had}${mathlib ? `: enough for the usage pass at ${DEFAULT_JOBS} at a time` : ""}` });
|
|
222
234
|
const dirs = projectDirs(root, o.site, o.cache);
|
|
223
235
|
for (const [what, dir] of [["site", dirs.site], ["cache", dirs.cache]]) {
|
|
224
236
|
if (!writable(dir))
|
package/dist/server/events.js
CHANGED
|
@@ -129,26 +129,41 @@ export class Timeline {
|
|
|
129
129
|
/** A new graph: every target's TCB is compared with the one it had, and a change is an event.
|
|
130
130
|
* `renamed`: the renames found against the build before (HistorySpec.md), old name → new: a
|
|
131
131
|
* declaration renamed is the same one, neither removed nor added, and a target renamed keeps its
|
|
132
|
-
* baseline.
|
|
132
|
+
* baseline. So are the renames in force (the review's `renamed`, followed on to the name the
|
|
133
|
+
* graph has): a build's first graph waits for the usage pass and is not compared, so the graph
|
|
134
|
+
* that is (the usage pass's, the build read again) finds no renames of its own; and a move made
|
|
135
|
+
* by hand renames too. */
|
|
133
136
|
graphChanged(prep, review, renamed = new Map()) {
|
|
134
137
|
const meta = JSON.parse(gunzipSync(prep.metaGz).toString("utf8"));
|
|
135
138
|
const raw = gunzipSync(prep.edgesGz);
|
|
136
139
|
const edges = new Int32Array(new Uint8Array(raw).buffer);
|
|
137
|
-
this.index = new TcbIndex(meta.nodes, edges);
|
|
140
|
+
const index = (this.index = new TcbIndex(meta.nodes, edges));
|
|
138
141
|
// a baseline kept before versions holds the kernel's hashes (server/versions.ts): one that is
|
|
139
142
|
// the kernel's hash of the content there now is that content's version
|
|
140
143
|
const kernel = new Map();
|
|
141
144
|
for (const v of meta.nodes)
|
|
142
145
|
if (v.kernel && v.hash)
|
|
143
146
|
kernel.set(v.name, v.kernel);
|
|
144
|
-
|
|
147
|
+
const ahead = new Map(renamed);
|
|
148
|
+
for (const [to, from] of Object.entries(review.renamed ?? {}))
|
|
149
|
+
if (!ahead.has(from))
|
|
150
|
+
ahead.set(from, to);
|
|
151
|
+
/** a name of a baseline as the graph has it now: renamed on where it is gone (a loop, which a log
|
|
152
|
+
* edited by hand could make, is cut short) */
|
|
153
|
+
const nameNow = (k) => {
|
|
154
|
+
let n = k;
|
|
155
|
+
for (let i = 0; i < 1000 && !index.byName.has(n) && ahead.has(n); i++)
|
|
156
|
+
n = ahead.get(n);
|
|
157
|
+
return index.byName.has(n) ? n : k;
|
|
158
|
+
};
|
|
159
|
+
if (ahead.size) {
|
|
145
160
|
for (const t of Object.keys(this.baselines)) {
|
|
146
161
|
const b = this.baselines[t];
|
|
147
162
|
const now = byName();
|
|
148
163
|
for (const k of Object.keys(b))
|
|
149
|
-
now[
|
|
164
|
+
now[nameNow(k)] = b[k];
|
|
150
165
|
delete this.baselines[t];
|
|
151
|
-
this.baselines[
|
|
166
|
+
this.baselines[nameNow(t)] = now;
|
|
152
167
|
}
|
|
153
168
|
}
|
|
154
169
|
for (const target of Object.keys(review.targets)) {
|
package/dist/server/export.js
CHANGED
|
@@ -90,7 +90,7 @@ export class Exporter {
|
|
|
90
90
|
delete j.notes;
|
|
91
91
|
return j;
|
|
92
92
|
};
|
|
93
|
-
const known = this.
|
|
93
|
+
const known = this.known(id);
|
|
94
94
|
if (known && known.state !== "failed")
|
|
95
95
|
return note(known);
|
|
96
96
|
const kept = this.fromCache(id);
|
|
@@ -121,11 +121,22 @@ export class Exporter {
|
|
|
121
121
|
job(id) {
|
|
122
122
|
if (!/^[0-9a-f]{32}$/.test(id))
|
|
123
123
|
return null;
|
|
124
|
-
const j = this.
|
|
124
|
+
const j = this.known(id) ?? this.fromCache(id);
|
|
125
125
|
if (j)
|
|
126
126
|
this.remember(j);
|
|
127
127
|
return j;
|
|
128
128
|
}
|
|
129
|
+
/** A job this server knows, unless it is done and its files have left the cache since (newer
|
|
130
|
+
* documents took their place): that one is forgotten, so that the same document asked for again
|
|
131
|
+
* is typeset again rather than answered with files that are gone. */
|
|
132
|
+
known(id) {
|
|
133
|
+
const j = this.jobs.get(id);
|
|
134
|
+
if (j?.state !== "done" || existsSync(join(this.cacheDir, id, "doc.pdf")))
|
|
135
|
+
return j;
|
|
136
|
+
this.jobs.delete(id);
|
|
137
|
+
this.ends.delete(id);
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
129
140
|
/** The job once it is over. */
|
|
130
141
|
wait(id) {
|
|
131
142
|
return this.ends.get(id) ?? Promise.resolve(this.job(id));
|
|
@@ -317,14 +328,16 @@ export class Exporter {
|
|
|
317
328
|
return zip(files);
|
|
318
329
|
}
|
|
319
330
|
}
|
|
320
|
-
|
|
331
|
+
/** The sources' README: how to typeset them again, and the macros a document is written in, each
|
|
332
|
+
* one the style file defines (a test checks it). */
|
|
333
|
+
export const README = `This is a document VisualProof exported, as LaTeX.
|
|
321
334
|
|
|
322
335
|
Typeset it with XeLaTeX, twice (the second run resolves page references and bookmarks):
|
|
323
336
|
|
|
324
337
|
xelatex doc.tex
|
|
325
338
|
xelatex doc.tex
|
|
326
339
|
|
|
327
|
-
doc.tex holds the content only, as semantic macros (\\
|
|
340
|
+
doc.tex holds the content only, as semantic macros (\\vpSection, \\begin{vpDecl}, \\vpFullName, \\begin{vpCode}, ...).
|
|
328
341
|
How it looks is visualproof.sty's: change it, replace it, or load a package of your own after it
|
|
329
342
|
and \\renewcommand what you want different. Its head lists every macro a document uses.
|
|
330
343
|
|
package/dist/server/fetchurl.js
CHANGED
|
@@ -12,7 +12,7 @@ import { request as httpRequest } from "node:http";
|
|
|
12
12
|
import { request as httpsRequest } from "node:https";
|
|
13
13
|
import { isIP } from "node:net";
|
|
14
14
|
import { networkInterfaces } from "node:os";
|
|
15
|
-
import { MAX_FILE } from "../shared/library.js";
|
|
15
|
+
import { MAX_FILE, sizeWords } from "../shared/library.js";
|
|
16
16
|
const DEFAULTS = { maxBytes: MAX_FILE, timeoutMs: 60_000, maxRedirects: 5 };
|
|
17
17
|
/** the most of a page that is read, and the longest it may take: an HTML page is far smaller than a
|
|
18
18
|
* file, and what it says of one is not worth a long wait */
|
|
@@ -64,13 +64,17 @@ function blockedV4(b) {
|
|
|
64
64
|
(a === 172 && c >= 16 && c <= 31) || // private
|
|
65
65
|
(a === 192 && c === 168) || // private
|
|
66
66
|
(a === 192 && c === 0 && b[2] === 0) || // IETF protocol assignments
|
|
67
|
+
(a === 192 && c === 0 && b[2] === 2) || // documentation (TEST-NET-1)
|
|
68
|
+
(a === 192 && c === 88 && b[2] === 99) || // 6to4 relays, deprecated
|
|
67
69
|
(a === 198 && (c === 18 || c === 19)) || // benchmarking
|
|
70
|
+
(a === 198 && c === 51 && b[2] === 100) || // documentation (TEST-NET-2)
|
|
71
|
+
(a === 203 && c === 0 && b[2] === 113) || // documentation (TEST-NET-3)
|
|
68
72
|
a >= 224 // multicast, reserved, broadcast
|
|
69
73
|
);
|
|
70
74
|
}
|
|
71
75
|
/** Whether the server refuses to fetch from an address: loopback, private, link-local, unique-local,
|
|
72
|
-
* multicast, reserved or unspecified, an IPv4 one written as IPv6 (mapped, NAT64
|
|
73
|
-
* that is not an address at all is refused too. */
|
|
76
|
+
* multicast, reserved, documentation or unspecified, an IPv4 one written as IPv6 (mapped, NAT64,
|
|
77
|
+
* 6to4) included. Anything that is not an address at all is refused too. */
|
|
74
78
|
export function blockedAddress(address) {
|
|
75
79
|
const kind = isIP(address.replace(/%.*$/, ""));
|
|
76
80
|
if (kind === 4)
|
|
@@ -91,6 +95,8 @@ export function blockedAddress(address) {
|
|
|
91
95
|
return true; // IPv4-compatible, deprecated
|
|
92
96
|
if (p[0] === 0x64 && p[1] === 0xff9b && p.slice(2, 6).every((x) => x === 0))
|
|
93
97
|
return blockedV4(v4(p[6], p[7])); // NAT64
|
|
98
|
+
if (p[0] === 0x2002)
|
|
99
|
+
return blockedV4(v4(p[1], p[2])); // 6to4: the IPv4 address it carries
|
|
94
100
|
if ((p[0] & 0xfe00) === 0xfc00)
|
|
95
101
|
return true; // unique local
|
|
96
102
|
if ((p[0] & 0xffc0) === 0xfe80)
|
|
@@ -211,7 +217,7 @@ async function guardedFetch(raw, kind, o, allowed) {
|
|
|
211
217
|
return { ok: false, status: 400, error: `${host} cannot be found.` };
|
|
212
218
|
const refused = addresses.find((a) => !allowed(a.address));
|
|
213
219
|
if (refused)
|
|
214
|
-
return { ok: false, status: 403, error: `${host} is ${refused.address}, an address of a private or local network or of this server: the server does not fetch from there.` };
|
|
220
|
+
return { ok: false, status: 403, error: `${host === refused.address ? host : `${host} is ${refused.address}, which`} is an address of a private or local network or of this server: the server does not fetch from there.` };
|
|
215
221
|
const left = deadline - Date.now();
|
|
216
222
|
if (left <= 0)
|
|
217
223
|
return { ok: false, status: 504, error: "The address took too long to answer." };
|
|
@@ -224,8 +230,11 @@ async function guardedFetch(raw, kind, o, allowed) {
|
|
|
224
230
|
return { ok: false, status: 502, error: `The address could not be fetched: ${e instanceof Error ? e.message : String(e)}.` };
|
|
225
231
|
}
|
|
226
232
|
const status = res.statusCode ?? 0;
|
|
233
|
+
// what is not the file is never read: a redirect's or an error's body, drained, could run on
|
|
234
|
+
// past the bytes and the time a fetch may take, the fetch answered long since
|
|
235
|
+
if (status !== 200)
|
|
236
|
+
res.destroy();
|
|
227
237
|
if ([301, 302, 303, 307, 308].includes(status)) {
|
|
228
|
-
res.resume();
|
|
229
238
|
const to = res.headers.location;
|
|
230
239
|
if (!to)
|
|
231
240
|
return { ok: false, status: 502, error: `The address answered ${status} without saying where to.` };
|
|
@@ -239,26 +248,28 @@ async function guardedFetch(raw, kind, o, allowed) {
|
|
|
239
248
|
}
|
|
240
249
|
continue;
|
|
241
250
|
}
|
|
242
|
-
if (status !== 200)
|
|
243
|
-
res.resume();
|
|
251
|
+
if (status !== 200)
|
|
244
252
|
return { ok: false, status: 502, error: `The address answered ${status}, not the ${noun}.` };
|
|
245
|
-
}
|
|
246
253
|
const length = Number(res.headers["content-length"] ?? NaN);
|
|
247
254
|
if (Number.isFinite(length) && length > o.maxBytes) {
|
|
248
255
|
res.destroy();
|
|
249
|
-
return { ok: false, status: 413, error: `The ${noun} is ${
|
|
256
|
+
return { ok: false, status: 413, error: `The ${noun} is ${sizeWords(length)}: ${most} is at most ${sizeWords(o.maxBytes)}.` };
|
|
250
257
|
}
|
|
251
258
|
const body = await readCapped(res, o.maxBytes, deadline);
|
|
252
259
|
if (typeof body === "string")
|
|
253
|
-
return { ok: false, status: body === "large" ? 413 : 504, error: body === "large" ? `The ${noun} is larger than ${
|
|
260
|
+
return { ok: false, status: body === "large" ? 413 : 504, error: body === "large" ? `The ${noun} is larger than ${sizeWords(o.maxBytes)}, the most ${most} may be.` : `The address took too long to send the ${noun}.` };
|
|
254
261
|
const disposition = res.headers["content-disposition"];
|
|
255
262
|
return { ok: true, bytes: body, url, disposition };
|
|
256
263
|
}
|
|
257
264
|
}
|
|
258
|
-
/** A request to `url`, connected to the address checked for it and to no other; its answer's head
|
|
265
|
+
/** A request to `url`, connected to the address checked for it and to no other; its answer's head,
|
|
266
|
+
* within `ms` in all. */
|
|
259
267
|
function connect(url, at, ms, accept) {
|
|
260
268
|
return new Promise((ok, no) => {
|
|
261
269
|
const make = url.protocol === "https:" ? httpsRequest : httpRequest;
|
|
270
|
+
// the socket's `timeout` is idleness alone: a head sent a byte at a time is never idle, and
|
|
271
|
+
// would hold the fetch past its time for as long as its sender likes
|
|
272
|
+
const late = setTimeout(() => req.destroy(new Error("no answer in time")), ms);
|
|
262
273
|
const req = make(url, {
|
|
263
274
|
method: "GET",
|
|
264
275
|
headers: { "user-agent": "VisualProof (adds a file to a project's library)", accept },
|
|
@@ -271,9 +282,15 @@ function connect(url, at, ms, accept) {
|
|
|
271
282
|
cb(null, at.address, at.family);
|
|
272
283
|
}),
|
|
273
284
|
timeout: ms,
|
|
274
|
-
},
|
|
285
|
+
}, (res) => {
|
|
286
|
+
clearTimeout(late);
|
|
287
|
+
ok(res);
|
|
288
|
+
});
|
|
275
289
|
req.on("timeout", () => req.destroy(new Error("no answer in time")));
|
|
276
|
-
req.on("error",
|
|
290
|
+
req.on("error", (e) => {
|
|
291
|
+
clearTimeout(late);
|
|
292
|
+
no(e);
|
|
293
|
+
});
|
|
277
294
|
req.end();
|
|
278
295
|
});
|
|
279
296
|
}
|
package/dist/server/film.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// The tips' tour film (UpdateSpec decision 14, the owner's of 2026-10-17): 3
|
|
1
|
+
// The tips' tour film (UpdateSpec decision 14, the owner's of 2026-10-17): about 3½ minutes at
|
|
2
2
|
// 2560×1600, too large for the package and for git, so it is an asset of the repository's GitHub
|
|
3
3
|
// prerelease `tour-1`, and the package pins it (assets/tips/tour.json: its file, sha256 and size).
|
|
4
4
|
// The server fetches it the first time a page asks for it (GET /tips/tour.mp4), one download for
|
package/dist/server/history.js
CHANGED
|
@@ -32,6 +32,7 @@ import { childEnv } from "./children.js";
|
|
|
32
32
|
import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
33
33
|
import { join } from "node:path";
|
|
34
34
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
35
|
+
import { endsLine } from "./lostfound.js";
|
|
35
36
|
import { SourceStore, summarize } from "./sourcestore.js";
|
|
36
37
|
import { ownName, refIdentity, versionOf } from "./versions.js";
|
|
37
38
|
const sha = (s) => createHash("sha256").update(s).digest("hex").slice(0, 32);
|
|
@@ -493,7 +494,8 @@ export class History {
|
|
|
493
494
|
}
|
|
494
495
|
save(info) {
|
|
495
496
|
this.infos.set(info.id, info);
|
|
496
|
-
|
|
497
|
+
// after a line cut short, on a line of its own: the line cut short alone is lost
|
|
498
|
+
appendFileSync(this.indexFile, (endsLine(this.indexFile) ? "" : "\n") + JSON.stringify(info) + "\n");
|
|
497
499
|
}
|
|
498
500
|
readGz(file) {
|
|
499
501
|
return JSON.parse(gunzipSync(readFileSync(file)).toString("utf8"));
|
|
@@ -12,58 +12,77 @@
|
|
|
12
12
|
// in 0.6 s): the page takes it in two parts, its declarations as JSON and its
|
|
13
13
|
// edges packed as the graph on show's are (shared/edgecodec.ts), each compressed once and kept for
|
|
14
14
|
// the last few snapshots asked, under the renames in force, and tagged so that a page that has one
|
|
15
|
-
// is answered 304.
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
// is answered 304. The parts, a snapshot's whole graph and what changed between two snapshots are
|
|
16
|
+
// made off the server's thread (server/historythread.ts), which meanwhile answers others: on
|
|
17
|
+
// NavierStokesAndEuler the Time Machine's first look at a build held it 4 s.
|
|
18
|
+
import { dirname } from "node:path";
|
|
18
19
|
import { MAX_ARCHIVE_KEYS, validKey } from "./lostfound.js";
|
|
19
|
-
import {
|
|
20
|
+
import { offThread, runJob } from "./historythread.js";
|
|
20
21
|
const no = (status, error) => ({ status, body: { error } });
|
|
21
22
|
const ok = (body) => ({ status: 200, body });
|
|
22
23
|
/** the parts of the last snapshots asked, per history, by snapshot, build read and renames */
|
|
23
24
|
const kept = new WeakMap();
|
|
24
25
|
/** how many snapshots' parts are kept: about a megabyte each, compressed, on a large project (Sphere-Packing-Lean's:
|
|
25
|
-
* 0.7 MB of declarations, 0.5 MB of edges) */
|
|
26
|
+
* 0.7 MB of declarations, 0.5 MB of edges; NavierStokesAndEuler's 5.2 and 4.0 MB) */
|
|
26
27
|
const KEEP_PARTS = 8;
|
|
27
28
|
/** what changed between two snapshots, as answered last (a step on the Time Machine's timeline asks
|
|
28
29
|
* it of each build against the latest, every reader the same), by the builds as read and the renames
|
|
29
30
|
* in force; a few hundred KB at most each */
|
|
30
31
|
const diffsKept = new WeakMap();
|
|
31
32
|
const KEEP_DIFFS = 32;
|
|
33
|
+
/** what is being made off the thread, by what it will be kept under: the Time Machine asks a
|
|
34
|
+
* snapshot's two parts at once, made by one job */
|
|
35
|
+
const making = new WeakMap();
|
|
36
|
+
const mapOf = (m, k) => {
|
|
37
|
+
let x = m.get(k);
|
|
38
|
+
if (!x)
|
|
39
|
+
m.set(k, (x = new Map()));
|
|
40
|
+
return x;
|
|
41
|
+
};
|
|
42
|
+
/** Keep `v` under `key`, the most recently asked last: the oldest goes first. */
|
|
43
|
+
function keep(byKey, key, v, most) {
|
|
44
|
+
byKey.delete(key);
|
|
45
|
+
byKey.set(key, v);
|
|
46
|
+
while (byKey.size > most)
|
|
47
|
+
byKey.delete(byKey.keys().next().value);
|
|
48
|
+
}
|
|
49
|
+
/** What a snapshot's parts are kept under: the snapshot, its build as read, the renames in force,
|
|
50
|
+
* and its texts (once backfilled, they change what the declarations' part says). */
|
|
51
|
+
const partsKey = (info, tag) => `${info.id}\u0000${info.fingerprint}\u0000${info.read}\u0000${tag}\u0000${JSON.stringify(info.sources ?? null)}`;
|
|
52
|
+
const diffKey = (fa, fb, most, tag) => [fa.id, fa.fingerprint, fa.read, fb.id, fb.fingerprint, fb.read, most, tag].join("\u0000");
|
|
53
|
+
/** A job, made once however many ask for it at once (`key`): in the history's worker where the
|
|
54
|
+
* world says its renames, else on this thread. */
|
|
55
|
+
function made(w, key, job) {
|
|
56
|
+
const h = w.history;
|
|
57
|
+
const byKey = mapOf(making, h);
|
|
58
|
+
let p = byKey.get(key);
|
|
59
|
+
if (!p) {
|
|
60
|
+
const here = () => runJob(h, w.identity(), job);
|
|
61
|
+
const renamed = w.renamed?.();
|
|
62
|
+
p = (renamed === undefined ? Promise.resolve().then(here) : offThread(dirname(h.dir), renamed, job, here)).finally(() => byKey.delete(key));
|
|
63
|
+
byKey.set(key, p);
|
|
64
|
+
}
|
|
65
|
+
return p;
|
|
66
|
+
}
|
|
32
67
|
/** Snapshot `id`'s graph as the page takes it (`part`): its declarations — the answer of /graph
|
|
33
|
-
* without the edges, and how many there are — or its edges, packed. Made together, once
|
|
34
|
-
|
|
68
|
+
* without the edges, and how many there are — or its edges, packed. Made together, once, off the
|
|
69
|
+
* server's thread. */
|
|
70
|
+
export async function snapshotParts(w, id) {
|
|
35
71
|
const h = w.history;
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
const
|
|
39
|
-
let byKey = kept.get(h);
|
|
40
|
-
if (!byKey)
|
|
41
|
-
kept.set(h, (byKey = new Map()));
|
|
72
|
+
const tag = w.identityTag?.() ?? "";
|
|
73
|
+
const key = partsKey(h.get(id), tag);
|
|
74
|
+
const byKey = mapOf(kept, h);
|
|
42
75
|
const have = byKey.get(key);
|
|
43
76
|
if (have) {
|
|
44
|
-
|
|
45
|
-
byKey.delete(key);
|
|
46
|
-
byKey.set(key, have);
|
|
77
|
+
keep(byKey, key, have, KEEP_PARTS);
|
|
47
78
|
return have;
|
|
48
79
|
}
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
while (byKey.size > KEEP_PARTS)
|
|
56
|
-
byKey.delete(byKey.keys().next().value);
|
|
57
|
-
return parts;
|
|
58
|
-
}
|
|
59
|
-
/** The modules whose text snapshot `id` keeps, each with how it was matched to its build
|
|
60
|
-
* (server/sourcestore.ts): what the page says of a declaration's source then before it asks. */
|
|
61
|
-
function keptTexts(h, id) {
|
|
62
|
-
const out = {};
|
|
63
|
-
for (const [m, r] of Object.entries(h.sourcesOf(id) ?? {}))
|
|
64
|
-
if (r.blob && r.check)
|
|
65
|
-
out[m] = r.check;
|
|
66
|
-
return out;
|
|
80
|
+
const d = await made(w, `parts\u0000${key}`, { kind: "parts", id });
|
|
81
|
+
if (d.kind !== "parts")
|
|
82
|
+
throw new Error("not the parts asked for");
|
|
83
|
+
// kept as the job read the snapshot: one read again meanwhile is made again for whoever asks next
|
|
84
|
+
keep(byKey, partsKey(d.info, tag), d.parts, KEEP_PARTS);
|
|
85
|
+
return d.parts;
|
|
67
86
|
}
|
|
68
87
|
/** Module `module`'s text at snapshot `id`'s build (GET /source; HistorySpec decision 23): the
|
|
69
88
|
* text its build was compiled from, with its Lake hash and how it was matched to the build, or
|
|
@@ -98,7 +117,7 @@ function snapshotOf(w, a, key, dflt) {
|
|
|
98
117
|
}
|
|
99
118
|
/** A request under /api/history (or /api/agent/history). `refuse`: why the caller may not move
|
|
100
119
|
* things (an agent below level full), or null. */
|
|
101
|
-
export function historyAnswer(w, a, refuse = () => null) {
|
|
120
|
+
export async function historyAnswer(w, a, refuse = () => null) {
|
|
102
121
|
const h = w.history;
|
|
103
122
|
if (!h)
|
|
104
123
|
return no(404, "This server serves a saved graph: it reads no builds, and keeps no history.");
|
|
@@ -149,7 +168,10 @@ export function historyAnswer(w, a, refuse = () => null) {
|
|
|
149
168
|
}
|
|
150
169
|
if (path === "") {
|
|
151
170
|
const latest = h.latest();
|
|
152
|
-
|
|
171
|
+
// agents are told the branch's builds too (DeploySpec decision 11): the page has them with the
|
|
172
|
+
// build status
|
|
173
|
+
const following = a.path.startsWith("/api/agent/") ? w.following?.() : null;
|
|
174
|
+
return ok({ snapshots: h.list(), current: latest?.id ?? null, diskBytes: h.diskBytes(), ...(following ? { following } : {}) });
|
|
153
175
|
}
|
|
154
176
|
if (path === "/graph") {
|
|
155
177
|
const id = snapshotOf(w, a, "id", true);
|
|
@@ -157,12 +179,17 @@ export function historyAnswer(w, a, refuse = () => null) {
|
|
|
157
179
|
return id;
|
|
158
180
|
const part = a.query.get("part");
|
|
159
181
|
if (part === "nodes")
|
|
160
|
-
return { status: 200, body: null, gz: { type: "application/json", body: snapshotParts(w, id).nodes } };
|
|
182
|
+
return { status: 200, body: null, gz: { type: "application/json", body: (await snapshotParts(w, id)).nodes } };
|
|
161
183
|
if (part === "edges")
|
|
162
|
-
return { status: 200, body: null, gz: { type: "application/octet-stream", body: snapshotParts(w, id).edges } };
|
|
184
|
+
return { status: 200, body: null, gz: { type: "application/octet-stream", body: (await snapshotParts(w, id)).edges } };
|
|
163
185
|
if (part !== null)
|
|
164
186
|
return no(400, "part is nodes or edges; leave it out for the whole graph.");
|
|
165
|
-
|
|
187
|
+
// the whole graph, which agents ask (105 MB of JSON on NavierStokesAndEuler): made and
|
|
188
|
+
// compressed off the thread, each time
|
|
189
|
+
const d = await made(w, `graph\u0000${partsKey(h.get(id), w.identityTag?.() ?? "")}`, { kind: "graph", id });
|
|
190
|
+
if (d.kind !== "graph")
|
|
191
|
+
throw new Error("not the graph asked for");
|
|
192
|
+
return { status: 200, body: null, gz: { type: "application/json", body: d.gz } };
|
|
166
193
|
}
|
|
167
194
|
if (path === "/source") {
|
|
168
195
|
const id = snapshotOf(w, a, "id", false);
|
|
@@ -181,7 +208,7 @@ export function historyAnswer(w, a, refuse = () => null) {
|
|
|
181
208
|
if (typeof to !== "number")
|
|
182
209
|
return to;
|
|
183
210
|
const most = Math.min(MOST_DIFF, Math.max(0, Number(a.query.get("most") ?? 2000) || 0));
|
|
184
|
-
return ok(
|
|
211
|
+
return ok(await snapshotDiffOff(w, from, to, most));
|
|
185
212
|
}
|
|
186
213
|
return no(404, "not found");
|
|
187
214
|
}
|
|
@@ -190,23 +217,43 @@ export function historyAnswer(w, a, refuse = () => null) {
|
|
|
190
217
|
export const MOST_DIFF = 20000;
|
|
191
218
|
/** What changed from snapshot `from` to snapshot `to` (both in the history), at most `most` names of
|
|
192
219
|
* each kind, as GET /diff answers it: made once, then kept for every reader and for the export of a
|
|
193
|
-
* view at a past build (server/pastdoc.ts), which makes the same past graph the page does.
|
|
220
|
+
* view at a past build (server/pastdoc.ts), which makes the same past graph the page does. Made
|
|
221
|
+
* here, on the server's thread, where it is not kept. */
|
|
194
222
|
export function snapshotDiff(w, from, to, most) {
|
|
195
223
|
const h = w.history;
|
|
196
|
-
const
|
|
197
|
-
const
|
|
198
|
-
let byKey = diffsKept.get(h);
|
|
199
|
-
if (!byKey)
|
|
200
|
-
diffsKept.set(h, (byKey = new Map()));
|
|
224
|
+
const key = diffKey(h.get(from), h.get(to), most, w.identityTag?.() ?? "");
|
|
225
|
+
const byKey = mapOf(diffsKept, h);
|
|
201
226
|
let body = byKey.get(key);
|
|
202
227
|
if (body === undefined) {
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
228
|
+
const d = runJob(h, w.identity(), diffJob(w, from, to, most));
|
|
229
|
+
if (d.kind !== "diff")
|
|
230
|
+
throw new Error("not the diff asked for");
|
|
231
|
+
body = d.diff;
|
|
207
232
|
}
|
|
233
|
+
keep(byKey, key, body, KEEP_DIFFS);
|
|
208
234
|
return body;
|
|
209
235
|
}
|
|
236
|
+
/** The same, made off the server's thread where it is not kept (GET /diff). */
|
|
237
|
+
async function snapshotDiffOff(w, from, to, most) {
|
|
238
|
+
const h = w.history;
|
|
239
|
+
const tag = w.identityTag?.() ?? "";
|
|
240
|
+
const key = diffKey(h.get(from), h.get(to), most, tag);
|
|
241
|
+
const byKey = mapOf(diffsKept, h);
|
|
242
|
+
const have = byKey.get(key);
|
|
243
|
+
if (have) {
|
|
244
|
+
keep(byKey, key, have, KEEP_DIFFS);
|
|
245
|
+
return have;
|
|
246
|
+
}
|
|
247
|
+
const d = await made(w, `diff\u0000${key}`, diffJob(w, from, to, most));
|
|
248
|
+
if (d.kind !== "diff")
|
|
249
|
+
throw new Error("not the diff asked for");
|
|
250
|
+
keep(byKey, diffKey(d.infos[0], d.infos[1], most, tag), d.diff, KEEP_DIFFS);
|
|
251
|
+
return d.diff;
|
|
252
|
+
}
|
|
253
|
+
/** The job of what changed from `from` to `to`, with the renames recorded between them. */
|
|
254
|
+
function diffJob(w, from, to, most) {
|
|
255
|
+
return { kind: "diff", from, to, most, renames: [...(w.renamesBetween?.(from, to) ?? new Map())] };
|
|
256
|
+
}
|
|
210
257
|
/** What the history's answers read, from what the agents' API has (server/agent.ts, World): what
|
|
211
258
|
* is not there answers as a server with nothing of it would. */
|
|
212
259
|
export function historyWorld(world) {
|
|
@@ -215,6 +262,8 @@ export function historyWorld(world) {
|
|
|
215
262
|
return world.history ?? null;
|
|
216
263
|
},
|
|
217
264
|
identity: () => world.identity?.() ?? ((u) => u),
|
|
265
|
+
// the renames, where the world says them or has no identity of its own (each name its own)
|
|
266
|
+
...(world.renamed || !world.identity ? { renamed: () => world.renamed?.() ?? null } : {}),
|
|
218
267
|
identityTag: () => world.identityTag?.() ?? "",
|
|
219
268
|
renamesBetween: (from, to) => world.renamesBetween?.(from, to) ?? new Map(),
|
|
220
269
|
moves: () => world.moves?.() ?? { migrations: [], suggestions: [] },
|
|
@@ -223,6 +272,7 @@ export function historyWorld(world) {
|
|
|
223
272
|
lostAndFound: () => world.lostAndFound?.() ?? { lost: [], migrations: [], archived: [], aside: { entries: 0, modules: [] }, count: 0 },
|
|
224
273
|
lastHad: (name) => world.lastHad?.(name) ?? null,
|
|
225
274
|
archiveEntries: (keys, on, who) => world.archiveEntries?.(keys, on, who) ?? { status: 404, body: { error: "not found" } },
|
|
275
|
+
following: () => world.following?.() ?? null,
|
|
226
276
|
};
|
|
227
277
|
}
|
|
228
278
|
/** The same, as endpoints of the agents' API (server/agent.ts): reading only, at every level. */
|