visualproof 0.4.4 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +16 -7
  2. package/dist/assets/tips/phone-fingers.light.mp4 +0 -0
  3. package/dist/assets/tips/phone-fingers.light.webp +0 -0
  4. package/dist/assets/tips/tour-agent.webp +0 -0
  5. package/dist/assets/tips/tour-arrange.webp +0 -0
  6. package/dist/assets/tips/tour-artwork.webp +0 -0
  7. package/dist/assets/tips/tour-checks.webp +0 -0
  8. package/dist/assets/tips/tour-keys.webp +0 -0
  9. package/dist/assets/tips/tour-library.webp +0 -0
  10. package/dist/assets/tips/tour-project.webp +0 -0
  11. package/dist/assets/tips/tour-review.webp +0 -0
  12. package/dist/assets/tips/tour-share.webp +0 -0
  13. package/dist/assets/tips/tour-source.webp +0 -0
  14. package/dist/assets/tips/tour-target.webp +0 -0
  15. package/dist/assets/tips/tour-view.webp +0 -0
  16. package/dist/assets/tips/tour.json +132 -117
  17. package/dist/assets/tips/tour.webp +0 -0
  18. package/dist/mcp/visualproof-mcp.mjs +20 -9
  19. package/dist/server/access.js +31 -1
  20. package/dist/server/accesspage.js +130 -0
  21. package/dist/server/agent.js +143 -50
  22. package/dist/server/askfollow.js +1 -1
  23. package/dist/server/buildwatch.js +133 -0
  24. package/dist/server/deploy.js +995 -0
  25. package/dist/server/doctor.js +15 -3
  26. package/dist/server/film.js +1 -1
  27. package/dist/server/historyapi.js +100 -50
  28. package/dist/server/historythread.js +206 -0
  29. package/dist/server/hover.js +8 -0
  30. package/dist/server/invites.js +113 -20
  31. package/dist/server/live.js +308 -60
  32. package/dist/server/main.js +450 -77
  33. package/dist/server/mcptools.js +33 -20
  34. package/dist/server/presence.js +3 -3
  35. package/dist/server/prompt.js +8 -3
  36. package/dist/server/readonlygate.js +4 -4
  37. package/dist/server/review.js +27 -7
  38. package/dist/server/reviewapi.js +51 -17
  39. package/dist/server/status.js +119 -46
  40. package/dist/server/talk.js +231 -115
  41. package/dist/server/tokens.js +10 -2
  42. package/dist/server/usage.js +51 -0
  43. package/dist/shared/board.js +2 -1
  44. package/dist/shared/build.js +35 -1
  45. package/dist/shared/collab.js +8 -0
  46. package/dist/shared/deploy.js +35 -0
  47. package/dist/shared/keys.js +66 -37
  48. package/dist/shared/layout.js +297 -70
  49. package/dist/shared/libm.js +366 -0
  50. package/dist/shared/lostfound.js +1 -0
  51. package/dist/shared/mentions.js +83 -0
  52. package/dist/shared/palette.js +1 -1
  53. package/dist/shared/query.js +13 -4
  54. package/dist/shared/review.js +91 -21
  55. package/dist/shared/reviewqueue.js +6 -1
  56. package/dist/shared/separate.js +556 -32
  57. package/dist/web/bundle/app.js +5022 -2736
  58. package/dist/web/bundle/edges.js +2 -1
  59. package/dist/web/bundle/layout.js +1026 -80
  60. package/dist/web/index.html +45 -9
  61. package/dist/web/src/canvas.js +47 -34
  62. package/dist/web/src/overview.js +4 -1
  63. package/dist/web/src/reviewqueue.js +52 -19
  64. package/dist/web/src/scene.js +13 -12
  65. package/dist/web/src/status.js +28 -0
  66. package/dist/web/src/theme.js +108 -0
  67. package/dist/web/src/trust.js +271 -0
  68. package/dist/web/style.css +283 -105
  69. package/docs/agent-api/README.md +13 -11
  70. package/docs/agent-api/advice.md +8 -2
  71. package/docs/agent-api/export.md +1 -1
  72. package/docs/agent-api/following.md +11 -12
  73. package/docs/agent-api/history.md +6 -2
  74. package/docs/agent-api/mcp.md +15 -15
  75. package/docs/agent-api/notes.md +1 -1
  76. package/docs/agent-api/reading.md +2 -2
  77. package/docs/agent-api/requests.md +16 -18
  78. package/docs/agent-api/talking.md +60 -55
  79. package/docs/agent-api/tcb.md +21 -21
  80. package/docs/agent-api/views.md +3 -4
  81. package/docs/agent-api/words.md +8 -8
  82. package/package.json +2 -1
@@ -5,13 +5,14 @@
5
5
  import { spawnSync } from "node:child_process";
6
6
  import { accessSync, constants, existsSync, readFileSync, statSync } from "node:fs";
7
7
  import { filmPin } from "./film.js";
8
- import { tmpdir } from "node:os";
8
+ import { freemem, tmpdir, totalmem } from "node:os";
9
9
  import { dirname, join, relative, resolve } from "node:path";
10
10
  import { cacheHome, projectDirs, siteHolder } from "./dirs.js";
11
11
  import { cmp, packageVersion, parseVersion, satisfies } from "./version.js";
12
12
  import { choose, FROM_PACKAGE, installCommand, newerThan, newest, notesOf, npmCommand, PACKAGE_ROOT, readInfo, sameMinor, Versions } from "./updates.js";
13
13
  export { packageVersion, satisfies };
14
14
  import { ProjectError, openProject, targetsInWords } from "./project.js";
15
+ import { DEFAULT_JOBS, inGB, jobsAdvice, memoryFloor, onMathlib, WORKER_PEAK } from "./usage.js";
15
16
  export const run = (cmd, args, cwd) => {
16
17
  const r = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000, stdio: ["ignore", "pipe", "pipe"], ...(cwd ? { cwd } : {}) });
17
18
  return r.error || r.status !== 0 ? null : `${r.stdout}${r.stderr}`;
@@ -179,7 +180,7 @@ export function doctor(o) {
179
180
  ? { level: "ok", what: `XeLaTeX (${/TeX Live \d+|MiKTeX[^\s)]*/.exec(xelatex)?.[0] ?? "found"})` }
180
181
  : { level: "info", what: "XeLaTeX not found: needed only for the printed document (Export)", fix: install("xelatex", platform) });
181
182
  if (exec("git", ["--version"]) === null) {
182
- checks.push({ level: "info", what: "git not found: builds are kept without their commit, and past sources only as the builds kept them" });
183
+ checks.push({ level: "info", what: "git not found: builds are kept without their commit, and past sources only as the builds kept them; a branch cannot be followed (serve <git URL>)" });
183
184
  }
184
185
  const film = filmLine(o.cache ? resolve(o.cache) : cacheHome(), o.updates ?? process.env.VISUALPROOF_UPDATES);
185
186
  const updates = updatesLine(exec, node, o.fromPackage ?? FROM_PACKAGE, o.cache ? resolve(o.cache) : cacheHome(), o.updates ?? process.env.VISUALPROOF_UPDATES, o.path === null ? null : projectDirs(resolve(o.path), o.site, o.cache).site);
@@ -211,7 +212,7 @@ export function doctor(o) {
211
212
  }
212
213
  const from = o.roots.length ? "--root" : existsSync(join(root, "lakefile.toml")) ? "lakefile.toml" : "lakefile.lean";
213
214
  const missing = roots.filter((r) => !existsSync(join(libDir, ...r.split(".")) + ".olean"));
214
- const mathlib = ["lakefile.toml", "lakefile.lean", "lake-manifest.json"].some((f) => existsSync(join(root, f)) && /mathlib/i.test(readFileSync(join(root, f), "utf8")));
215
+ const mathlib = onMathlib(root);
215
216
  checks.push(missing.length === 0
216
217
  ? { level: "ok", what: `built: ${targetsInWords(project)} (from ${from})` }
217
218
  : {
@@ -219,6 +220,17 @@ export function doctor(o) {
219
220
  what: `not built: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ` and ${missing.length - 5} more` : ""} (${relative(root, join(libDir, ...missing[0].split("."))) + ".olean"} is missing; from ${from})`,
220
221
  fix: `in ${root}: ${mathlib ? "lake exe cache get && " : ""}lake build${o.roots.length === 0 && from === "lakefile.lean" ? ", or name the modules to read with --root" : ""}`,
221
222
  });
223
+ // what the usage pass's workers may hold, at the default --jobs, against the memory available
224
+ const mem = o.memory ?? { available: freemem(), total: totalmem() };
225
+ const advice = jobsAdvice(DEFAULT_JOBS, mathlib, mem.available);
226
+ const had = `memory: ${inGB(mem.available)} available of ${inGB(mem.total)}`;
227
+ checks.push(advice
228
+ ? {
229
+ level: "warn",
230
+ what: `${had}: on Mathlib the usage pass's workers may hold up to ${inGB(WORKER_PEAK)} each, ${inGB(advice.need)} at ${DEFAULT_JOBS} at a time (the default --jobs); it starts fewer while less than ${inGB(memoryFloor(mem.total))} is available`,
231
+ fix: `serve with --jobs ${advice.fits}`,
232
+ }
233
+ : { level: "ok", what: `${had}${mathlib ? `: enough for the usage pass at ${DEFAULT_JOBS} at a time` : ""}` });
222
234
  const dirs = projectDirs(root, o.site, o.cache);
223
235
  for (const [what, dir] of [["site", dirs.site], ["cache", dirs.cache]]) {
224
236
  if (!writable(dir))
@@ -1,4 +1,4 @@
1
- // The tips' tour film (UpdateSpec decision 14, the owner's of 2026-10-17): 3½–4 minutes at
1
+ // The tips' tour film (UpdateSpec decision 14, the owner's of 2026-10-17): about 3½ minutes at
2
2
  // 2560×1600, too large for the package and for git, so it is an asset of the repository's GitHub
3
3
  // prerelease `tour-1`, and the package pins it (assets/tips/tour.json: its file, sha256 and size).
4
4
  // The server fetches it the first time a page asks for it (GET /tips/tour.mp4), one download for
@@ -12,58 +12,77 @@
12
12
  // in 0.6 s): the page takes it in two parts, its declarations as JSON and its
13
13
  // edges packed as the graph on show's are (shared/edgecodec.ts), each compressed once and kept for
14
14
  // the last few snapshots asked, under the renames in force, and tagged so that a page that has one
15
- // is answered 304.
16
- import { gzipSync } from "node:zlib";
17
- import { packEdges } from "../shared/edgecodec.js";
15
+ // is answered 304. The parts, a snapshot's whole graph and what changed between two snapshots are
16
+ // made off the server's thread (server/historythread.ts), which meanwhile answers others: on
17
+ // NavierStokesAndEuler the Time Machine's first look at a build held it 4 s.
18
+ import { dirname } from "node:path";
18
19
  import { MAX_ARCHIVE_KEYS, validKey } from "./lostfound.js";
19
- import { diffGraphs, pageGraph } from "./history.js";
20
+ import { offThread, runJob } from "./historythread.js";
20
21
  const no = (status, error) => ({ status, body: { error } });
21
22
  const ok = (body) => ({ status: 200, body });
22
23
  /** the parts of the last snapshots asked, per history, by snapshot, build read and renames */
23
24
  const kept = new WeakMap();
24
25
  /** how many snapshots' parts are kept: about a megabyte each, compressed, on a large project (Sphere-Packing-Lean's:
25
- * 0.7 MB of declarations, 0.5 MB of edges) */
26
+ * 0.7 MB of declarations, 0.5 MB of edges; NavierStokesAndEuler's 5.2 and 4.0 MB) */
26
27
  const KEEP_PARTS = 8;
27
28
  /** what changed between two snapshots, as answered last (a step on the Time Machine's timeline asks
28
29
  * it of each build against the latest, every reader the same), by the builds as read and the renames
29
30
  * in force; a few hundred KB at most each */
30
31
  const diffsKept = new WeakMap();
31
32
  const KEEP_DIFFS = 32;
33
+ /** what is being made off the thread, by what it will be kept under: the Time Machine asks a
34
+ * snapshot's two parts at once, made by one job */
35
+ const making = new WeakMap();
36
+ const mapOf = (m, k) => {
37
+ let x = m.get(k);
38
+ if (!x)
39
+ m.set(k, (x = new Map()));
40
+ return x;
41
+ };
42
+ /** Keep `v` under `key`, the most recently asked last: the oldest goes first. */
43
+ function keep(byKey, key, v, most) {
44
+ byKey.delete(key);
45
+ byKey.set(key, v);
46
+ while (byKey.size > most)
47
+ byKey.delete(byKey.keys().next().value);
48
+ }
49
+ /** What a snapshot's parts are kept under: the snapshot, its build as read, the renames in force,
50
+ * and its texts (once backfilled, they change what the declarations' part says). */
51
+ const partsKey = (info, tag) => `${info.id}\u0000${info.fingerprint}\u0000${info.read}\u0000${tag}\u0000${JSON.stringify(info.sources ?? null)}`;
52
+ const diffKey = (fa, fb, most, tag) => [fa.id, fa.fingerprint, fa.read, fb.id, fb.fingerprint, fb.read, most, tag].join("\u0000");
53
+ /** A job, made once however many ask for it at once (`key`): in the history's worker where the
54
+ * world says its renames, else on this thread. */
55
+ function made(w, key, job) {
56
+ const h = w.history;
57
+ const byKey = mapOf(making, h);
58
+ let p = byKey.get(key);
59
+ if (!p) {
60
+ const here = () => runJob(h, w.identity(), job);
61
+ const renamed = w.renamed?.();
62
+ p = (renamed === undefined ? Promise.resolve().then(here) : offThread(dirname(h.dir), renamed, job, here)).finally(() => byKey.delete(key));
63
+ byKey.set(key, p);
64
+ }
65
+ return p;
66
+ }
32
67
  /** Snapshot `id`'s graph as the page takes it (`part`): its declarations — the answer of /graph
33
- * without the edges, and how many there are — or its edges, packed. Made together, once. */
34
- export function snapshotParts(w, id) {
68
+ * without the edges, and how many there are — or its edges, packed. Made together, once, off the
69
+ * server's thread. */
70
+ export async function snapshotParts(w, id) {
35
71
  const h = w.history;
36
- const info = h.get(id);
37
- // its texts, once backfilled, change what the declarations' part says
38
- const key = `${id}\u0000${info.fingerprint}\u0000${info.read}\u0000${w.identityTag?.() ?? ""}\u0000${JSON.stringify(info.sources ?? null)}`;
39
- let byKey = kept.get(h);
40
- if (!byKey)
41
- kept.set(h, (byKey = new Map()));
72
+ const tag = w.identityTag?.() ?? "";
73
+ const key = partsKey(h.get(id), tag);
74
+ const byKey = mapOf(kept, h);
42
75
  const have = byKey.get(key);
43
76
  if (have) {
44
- // most recently asked last: the oldest goes first
45
- byKey.delete(key);
46
- byKey.set(key, have);
77
+ keep(byKey, key, have, KEEP_PARTS);
47
78
  return have;
48
79
  }
49
- const g = h.graph(id);
50
- const { edges: _flat, ...rest } = pageGraph({ ...g, edges: [] }, w.identity());
51
- const E = new Int32Array(3 * g.edges.length);
52
- g.edges.forEach(([s, d, m], k) => E.set([s, d, m], 3 * k));
53
- const parts = { nodes: gzipSync(JSON.stringify({ ...rest, edgeCount: g.edges.length, sources: keptTexts(h, id) })), edges: gzipSync(packEdges(E, g.nodes.length, 2), { level: 9 }) };
54
- byKey.set(key, parts);
55
- while (byKey.size > KEEP_PARTS)
56
- byKey.delete(byKey.keys().next().value);
57
- return parts;
58
- }
59
- /** The modules whose text snapshot `id` keeps, each with how it was matched to its build
60
- * (server/sourcestore.ts): what the page says of a declaration's source then before it asks. */
61
- function keptTexts(h, id) {
62
- const out = {};
63
- for (const [m, r] of Object.entries(h.sourcesOf(id) ?? {}))
64
- if (r.blob && r.check)
65
- out[m] = r.check;
66
- return out;
80
+ const d = await made(w, `parts\u0000${key}`, { kind: "parts", id });
81
+ if (d.kind !== "parts")
82
+ throw new Error("not the parts asked for");
83
+ // kept as the job read the snapshot: one read again meanwhile is made again for whoever asks next
84
+ keep(byKey, partsKey(d.info, tag), d.parts, KEEP_PARTS);
85
+ return d.parts;
67
86
  }
68
87
  /** Module `module`'s text at snapshot `id`'s build (GET /source; HistorySpec decision 23): the
69
88
  * text its build was compiled from, with its Lake hash and how it was matched to the build, or
@@ -98,7 +117,7 @@ function snapshotOf(w, a, key, dflt) {
98
117
  }
99
118
  /** A request under /api/history (or /api/agent/history). `refuse`: why the caller may not move
100
119
  * things (an agent below level full), or null. */
101
- export function historyAnswer(w, a, refuse = () => null) {
120
+ export async function historyAnswer(w, a, refuse = () => null) {
102
121
  const h = w.history;
103
122
  if (!h)
104
123
  return no(404, "This server serves a saved graph: it reads no builds, and keeps no history.");
@@ -149,7 +168,10 @@ export function historyAnswer(w, a, refuse = () => null) {
149
168
  }
150
169
  if (path === "") {
151
170
  const latest = h.latest();
152
- return ok({ snapshots: h.list(), current: latest?.id ?? null, diskBytes: h.diskBytes() });
171
+ // agents are told the branch's builds too (DeploySpec decision 11): the page has them with the
172
+ // build status
173
+ const following = a.path.startsWith("/api/agent/") ? w.following?.() : null;
174
+ return ok({ snapshots: h.list(), current: latest?.id ?? null, diskBytes: h.diskBytes(), ...(following ? { following } : {}) });
153
175
  }
154
176
  if (path === "/graph") {
155
177
  const id = snapshotOf(w, a, "id", true);
@@ -157,12 +179,17 @@ export function historyAnswer(w, a, refuse = () => null) {
157
179
  return id;
158
180
  const part = a.query.get("part");
159
181
  if (part === "nodes")
160
- return { status: 200, body: null, gz: { type: "application/json", body: snapshotParts(w, id).nodes } };
182
+ return { status: 200, body: null, gz: { type: "application/json", body: (await snapshotParts(w, id)).nodes } };
161
183
  if (part === "edges")
162
- return { status: 200, body: null, gz: { type: "application/octet-stream", body: snapshotParts(w, id).edges } };
184
+ return { status: 200, body: null, gz: { type: "application/octet-stream", body: (await snapshotParts(w, id)).edges } };
163
185
  if (part !== null)
164
186
  return no(400, "part is nodes or edges; leave it out for the whole graph.");
165
- return ok(pageGraph(h.graph(id), w.identity()));
187
+ // the whole graph, which agents ask (105 MB of JSON on NavierStokesAndEuler): made and
188
+ // compressed off the thread, each time
189
+ const d = await made(w, `graph\u0000${partsKey(h.get(id), w.identityTag?.() ?? "")}`, { kind: "graph", id });
190
+ if (d.kind !== "graph")
191
+ throw new Error("not the graph asked for");
192
+ return { status: 200, body: null, gz: { type: "application/json", body: d.gz } };
166
193
  }
167
194
  if (path === "/source") {
168
195
  const id = snapshotOf(w, a, "id", false);
@@ -181,7 +208,7 @@ export function historyAnswer(w, a, refuse = () => null) {
181
208
  if (typeof to !== "number")
182
209
  return to;
183
210
  const most = Math.min(MOST_DIFF, Math.max(0, Number(a.query.get("most") ?? 2000) || 0));
184
- return ok(snapshotDiff(w, from, to, most));
211
+ return ok(await snapshotDiffOff(w, from, to, most));
185
212
  }
186
213
  return no(404, "not found");
187
214
  }
@@ -190,23 +217,43 @@ export function historyAnswer(w, a, refuse = () => null) {
190
217
  export const MOST_DIFF = 20000;
191
218
  /** What changed from snapshot `from` to snapshot `to` (both in the history), at most `most` names of
192
219
  * each kind, as GET /diff answers it: made once, then kept for every reader and for the export of a
193
- * view at a past build (server/pastdoc.ts), which makes the same past graph the page does. */
220
+ * view at a past build (server/pastdoc.ts), which makes the same past graph the page does. Made
221
+ * here, on the server's thread, where it is not kept. */
194
222
  export function snapshotDiff(w, from, to, most) {
195
223
  const h = w.history;
196
- const [fa, fb] = [h.get(from), h.get(to)];
197
- const key = [from, fa.fingerprint, fa.read, to, fb.fingerprint, fb.read, most, w.identityTag?.() ?? ""].join("\u0000");
198
- let byKey = diffsKept.get(h);
199
- if (!byKey)
200
- diffsKept.set(h, (byKey = new Map()));
224
+ const key = diffKey(h.get(from), h.get(to), most, w.identityTag?.() ?? "");
225
+ const byKey = mapOf(diffsKept, h);
201
226
  let body = byKey.get(key);
202
227
  if (body === undefined) {
203
- body = { from, to, ...diffGraphs(h.graph(from), h.graph(to), w.identity(), w.renamesBetween?.(from, to) ?? new Map(), most) };
204
- byKey.set(key, body);
205
- while (byKey.size > KEEP_DIFFS)
206
- byKey.delete(byKey.keys().next().value);
228
+ const d = runJob(h, w.identity(), diffJob(w, from, to, most));
229
+ if (d.kind !== "diff")
230
+ throw new Error("not the diff asked for");
231
+ body = d.diff;
207
232
  }
233
+ keep(byKey, key, body, KEEP_DIFFS);
208
234
  return body;
209
235
  }
236
+ /** The same, made off the server's thread where it is not kept (GET /diff). */
237
+ async function snapshotDiffOff(w, from, to, most) {
238
+ const h = w.history;
239
+ const tag = w.identityTag?.() ?? "";
240
+ const key = diffKey(h.get(from), h.get(to), most, tag);
241
+ const byKey = mapOf(diffsKept, h);
242
+ const have = byKey.get(key);
243
+ if (have) {
244
+ keep(byKey, key, have, KEEP_DIFFS);
245
+ return have;
246
+ }
247
+ const d = await made(w, `diff\u0000${key}`, diffJob(w, from, to, most));
248
+ if (d.kind !== "diff")
249
+ throw new Error("not the diff asked for");
250
+ keep(byKey, diffKey(d.infos[0], d.infos[1], most, tag), d.diff, KEEP_DIFFS);
251
+ return d.diff;
252
+ }
253
+ /** The job of what changed from `from` to `to`, with the renames recorded between them. */
254
+ function diffJob(w, from, to, most) {
255
+ return { kind: "diff", from, to, most, renames: [...(w.renamesBetween?.(from, to) ?? new Map())] };
256
+ }
210
257
  /** What the history's answers read, from what the agents' API has (server/agent.ts, World): what
211
258
  * is not there answers as a server with nothing of it would. */
212
259
  export function historyWorld(world) {
@@ -215,6 +262,8 @@ export function historyWorld(world) {
215
262
  return world.history ?? null;
216
263
  },
217
264
  identity: () => world.identity?.() ?? ((u) => u),
265
+ // the renames, where the world says them or has no identity of its own (each name its own)
266
+ ...(world.renamed || !world.identity ? { renamed: () => world.renamed?.() ?? null } : {}),
218
267
  identityTag: () => world.identityTag?.() ?? "",
219
268
  renamesBetween: (from, to) => world.renamesBetween?.(from, to) ?? new Map(),
220
269
  moves: () => world.moves?.() ?? { migrations: [], suggestions: [] },
@@ -223,6 +272,7 @@ export function historyWorld(world) {
223
272
  lostAndFound: () => world.lostAndFound?.() ?? { lost: [], migrations: [], archived: [], aside: { entries: 0, modules: [] }, count: 0 },
224
273
  lastHad: (name) => world.lastHad?.(name) ?? null,
225
274
  archiveEntries: (keys, on, who) => world.archiveEntries?.(keys, on, who) ?? { status: 404, body: { error: "not found" } },
275
+ following: () => world.following?.() ?? null,
226
276
  };
227
277
  }
228
278
  /** The same, as endpoints of the agents' API (server/agent.ts): reading only, at every level. */
@@ -0,0 +1,206 @@
1
+ // What the Time Machine and the agents read of the history (server/historyapi.ts) — a snapshot's
2
+ // graph in the page's two parts, its whole graph, what changed between two snapshots — made off the
3
+ // server's thread. On NavierStokesAndEuler (52,648 declarations, 5 M edges) the Time Machine's first
4
+ // look at a build held every request for 4 s: the snapshot read from its files (1 s), its two parts
5
+ // made and compressed (1.6 s), what changed since (0.8 s). A worker reads the history from the site's
6
+ // files as the server's own does, and only reads them; it keeps what it read while the history's
7
+ // index stays as it was (every snapshot written, read again or given its texts adds a line to it),
8
+ // answers one job at a time, and goes after a while without one, its memory with it. Should no worker
9
+ // start, or one fail, the job is done here: slower to answer others meanwhile, never wrong. This
10
+ // module is its own worker, as server/layoutthread.ts is.
11
+ import { statSync } from "node:fs";
12
+ import { join } from "node:path";
13
+ import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads";
14
+ import { gzipSync } from "node:zlib";
15
+ import { packEdges } from "../shared/edgecodec.js";
16
+ import { identityIn } from "../shared/moves.js";
17
+ import { History, diffGraphs, pageGraph } from "./history.js";
18
+ import { ownName } from "./versions.js";
19
+ /** The modules whose text snapshot `id` keeps, each with how it was matched to its build
20
+ * (server/sourcestore.ts): what the page says of a declaration's source then before it asks. */
21
+ function keptTexts(h, id) {
22
+ const out = {};
23
+ for (const [m, r] of Object.entries(h.sourcesOf(id) ?? {}))
24
+ if (r.blob && r.check)
25
+ out[m] = r.check;
26
+ return out;
27
+ }
28
+ /** Do a job on history `h`, under `identity` (the renames in force), here. */
29
+ export function runJob(h, identity, job) {
30
+ if (job.kind === "diff") {
31
+ const [a, b] = [h.graph(job.from), h.graph(job.to)];
32
+ if (!a || !b)
33
+ throw new Error(`There is no snapshot ${a ? job.to : job.from}.`);
34
+ const diff = { from: job.from, to: job.to, ...diffGraphs(a, b, identity, new Map(job.renames), job.most) };
35
+ return { kind: "diff", infos: [a.info, b.info], diff };
36
+ }
37
+ const g = h.graph(job.id);
38
+ if (!g)
39
+ throw new Error(`There is no snapshot ${job.id}.`);
40
+ if (job.kind === "graph")
41
+ return { kind: "graph", info: g.info, gz: gzipSync(JSON.stringify(pageGraph(g, identity))) };
42
+ const { edges: _flat, ...rest } = pageGraph({ ...g, edges: [] }, identity);
43
+ const E = new Int32Array(3 * g.edges.length);
44
+ for (let k = 0; k < g.edges.length; k++) {
45
+ const [s, d, m] = g.edges[k];
46
+ E[3 * k] = s;
47
+ E[3 * k + 1] = d;
48
+ E[3 * k + 2] = m;
49
+ }
50
+ const nodes = gzipSync(JSON.stringify({ ...rest, edgeCount: g.edges.length, sources: keptTexts(h, job.id) }));
51
+ return { kind: "parts", info: g.info, parts: { nodes, edges: gzipSync(packEdges(E, g.nodes.length, 2), { level: 9 }) } };
52
+ }
53
+ /** The identity of a unit under renames `renamed` (shared/moves.ts), as the server's own is
54
+ * (server/live.ts, identity): to → from, made again without a prototype. */
55
+ export function identityOf(renamed) {
56
+ if (!renamed)
57
+ return ownName;
58
+ const r = Object.create(null);
59
+ for (const [to, from] of renamed)
60
+ r[to] = from;
61
+ return (u) => identityIn(r, u);
62
+ }
63
+ /** how long the worker stays without a job: the Time Machine steps from build to build, each a job
64
+ * or three */
65
+ const IDLE_MS = 60_000;
66
+ let worker = null;
67
+ let idleTimer = null;
68
+ let asked = 0;
69
+ /** how many jobs were done in the worker and how many here, since the start (the tests read it) */
70
+ export const jobsDone = { worker: 0, here: 0 };
71
+ /** the jobs out, by number: each with what does it here, should the worker fail */
72
+ const out = new Map();
73
+ /** Every job out done here instead, and the worker forgotten: it stopped, or failed. */
74
+ function lost(w) {
75
+ if (worker !== w)
76
+ return;
77
+ worker = null;
78
+ const jobs = [...out.values()];
79
+ out.clear();
80
+ for (const j of jobs) {
81
+ try {
82
+ j.ok(j.here());
83
+ }
84
+ catch (e) {
85
+ j.fail(e);
86
+ }
87
+ }
88
+ }
89
+ function spawn() {
90
+ if (worker)
91
+ return worker;
92
+ let w;
93
+ try {
94
+ w = new Worker(new URL(import.meta.url), { workerData: { vpHistory: true } });
95
+ }
96
+ catch {
97
+ return null;
98
+ }
99
+ w.on("message", (a) => {
100
+ const j = out.get(a.n);
101
+ out.delete(a.n);
102
+ if (j) {
103
+ if ("done" in a) {
104
+ jobsDone.worker++;
105
+ j.ok(revive(a.done));
106
+ }
107
+ else {
108
+ // what the worker could not do is no worker's failure: tried here, it says why
109
+ try {
110
+ j.ok(j.here());
111
+ }
112
+ catch (e) {
113
+ j.fail(e);
114
+ }
115
+ }
116
+ }
117
+ if (!out.size)
118
+ rest(w);
119
+ });
120
+ w.on("error", () => lost(w));
121
+ w.on("exit", () => lost(w));
122
+ // held only while a job is out (offThread): an idle worker keeps no process running. After its
123
+ // listeners, which hold it again as they are added
124
+ w.unref();
125
+ worker = w;
126
+ return w;
127
+ }
128
+ /** The worker holds nothing any more, and goes after IDLE_MS without a job. */
129
+ function rest(w) {
130
+ w.unref();
131
+ if (idleTimer)
132
+ clearTimeout(idleTimer);
133
+ idleTimer = setTimeout(() => {
134
+ idleTimer = null;
135
+ if (out.size || !worker)
136
+ return;
137
+ const w = worker;
138
+ worker = null;
139
+ void w.terminate();
140
+ }, IDLE_MS);
141
+ idleTimer.unref();
142
+ }
143
+ /** Buffers come back from the worker as bytes: Buffers again. */
144
+ function revive(d) {
145
+ const buf = (u) => Buffer.from(u.buffer, u.byteOffset, u.byteLength);
146
+ if (d.kind === "parts")
147
+ return { ...d, parts: { nodes: buf(d.parts.nodes), edges: buf(d.parts.edges) } };
148
+ if (d.kind === "graph")
149
+ return { ...d, gz: buf(d.gz) };
150
+ return d;
151
+ }
152
+ /** Do a job on the history of site `site` in the worker, under the renames in force `renamed`;
153
+ * `here` does it on this thread, where no worker can. */
154
+ export function offThread(site, renamed, job, doHere) {
155
+ const here = () => (jobsDone.here++, doHere());
156
+ return new Promise((ok, fail) => {
157
+ const w = spawn();
158
+ if (!w) {
159
+ try {
160
+ return ok(here());
161
+ }
162
+ catch (e) {
163
+ return fail(e);
164
+ }
165
+ }
166
+ if (idleTimer)
167
+ clearTimeout(idleTimer);
168
+ idleTimer = null;
169
+ const n = ++asked;
170
+ out.set(n, { ok, fail, here });
171
+ w.ref();
172
+ w.postMessage({ n, site, renamed, job });
173
+ });
174
+ }
175
+ // in a worker: the jobs asked, one at a time, on the history as its files are now
176
+ if (!isMainThread && workerData?.vpHistory) {
177
+ let held = null;
178
+ /** the history of `site`, read again when its index changed since the last job */
179
+ const historyOf = (site) => {
180
+ let stamp = "";
181
+ try {
182
+ const st = statSync(join(site, "history", "index.jsonl"));
183
+ stamp = `${st.size}:${st.mtimeMs}:${st.ino}`;
184
+ }
185
+ catch {
186
+ // no history yet: nothing to keep
187
+ }
188
+ if (!held || held.site !== site || held.stamp !== stamp || !stamp)
189
+ held = { site, stamp, h: new History(site) };
190
+ return held.h;
191
+ };
192
+ parentPort.on("message", (a) => {
193
+ let answer;
194
+ try {
195
+ answer = { n: a.n, done: runJob(historyOf(a.site), identityOf(a.renamed), a.job) };
196
+ }
197
+ catch (e) {
198
+ answer = { n: a.n, error: e instanceof Error ? e.message : String(e) };
199
+ }
200
+ const d = "done" in answer ? answer.done : null;
201
+ const bytes = d?.kind === "parts" ? [d.parts.nodes, d.parts.edges] : d?.kind === "graph" ? [d.gz] : [];
202
+ // the compressed bytes are handed over, not copied, where they are a buffer of their own
203
+ const transfer = bytes.filter((b) => b.byteOffset === 0 && b.byteLength === b.buffer.byteLength).map((b) => b.buffer);
204
+ parentPort.postMessage(answer, transfer);
205
+ });
206
+ }
@@ -591,6 +591,14 @@ export class LeanHover {
591
591
  if (p)
592
592
  end(p);
593
593
  }
594
+ /** The project's toolchain changed (a new commit of the branch the server follows:
595
+ * server/deploy.ts): Lean stops, and starts at the next question with the new toolchain's `lean`
596
+ * and LEAN_PATH. */
597
+ toolchainChanged() {
598
+ this.stop();
599
+ this.env = null;
600
+ this.failure = null;
601
+ }
594
602
  }
595
603
  /** Null after `ms`: what a question for Lean is raced against. */
596
604
  function after(ms) {