visualproof 0.4.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +16 -7
  2. package/dist/assets/latex/visualproof.sty +19 -9
  3. package/dist/assets/tips/tour-agent.webp +0 -0
  4. package/dist/assets/tips/tour-arrange.webp +0 -0
  5. package/dist/assets/tips/tour-artwork.webp +0 -0
  6. package/dist/assets/tips/tour-checks.webp +0 -0
  7. package/dist/assets/tips/tour-keys.webp +0 -0
  8. package/dist/assets/tips/tour-library.webp +0 -0
  9. package/dist/assets/tips/tour-project.webp +0 -0
  10. package/dist/assets/tips/tour-review.webp +0 -0
  11. package/dist/assets/tips/tour-share.webp +0 -0
  12. package/dist/assets/tips/tour-source.webp +0 -0
  13. package/dist/assets/tips/tour-target.webp +0 -0
  14. package/dist/assets/tips/tour-view.webp +0 -0
  15. package/dist/assets/tips/tour.json +117 -112
  16. package/dist/assets/tips/tour.webp +0 -0
  17. package/dist/mcp/visualproof-mcp.mjs +7 -3
  18. package/dist/server/access.js +37 -1
  19. package/dist/server/accesspage.js +130 -0
  20. package/dist/server/agent.js +11 -5
  21. package/dist/server/agentlibrary.js +1 -1
  22. package/dist/server/agentviews.js +1 -1
  23. package/dist/server/artcmd.js +1 -1
  24. package/dist/server/askfollow.js +1 -4
  25. package/dist/server/buildwatch.js +133 -0
  26. package/dist/server/deploy.js +844 -0
  27. package/dist/server/doctor.js +15 -3
  28. package/dist/server/events.js +20 -5
  29. package/dist/server/export.js +17 -4
  30. package/dist/server/fetchurl.js +30 -13
  31. package/dist/server/film.js +1 -1
  32. package/dist/server/history.js +3 -1
  33. package/dist/server/historyapi.js +100 -50
  34. package/dist/server/historythread.js +206 -0
  35. package/dist/server/hover.js +8 -0
  36. package/dist/server/identity.js +25 -1
  37. package/dist/server/library.js +10 -3
  38. package/dist/server/live.js +206 -28
  39. package/dist/server/lostfound.js +3 -2
  40. package/dist/server/main.js +361 -50
  41. package/dist/server/poppler.js +48 -10
  42. package/dist/server/renames.js +10 -5
  43. package/dist/server/review.js +32 -2
  44. package/dist/server/reviewapi.js +1 -1
  45. package/dist/server/status.js +144 -50
  46. package/dist/server/talk.js +7 -2
  47. package/dist/server/usage.js +51 -0
  48. package/dist/server/views.js +21 -7
  49. package/dist/shared/board.js +2 -1
  50. package/dist/shared/collab.js +14 -8
  51. package/dist/shared/deploy.js +35 -0
  52. package/dist/shared/doc/latex.js +5 -3
  53. package/dist/shared/doc/outline.js +6 -3
  54. package/dist/shared/layout.js +297 -70
  55. package/dist/shared/libm.js +366 -0
  56. package/dist/shared/library.js +11 -6
  57. package/dist/shared/palette.js +1 -1
  58. package/dist/shared/review.js +7 -5
  59. package/dist/shared/search.js +7 -3
  60. package/dist/shared/separate.js +556 -32
  61. package/dist/shared/views.js +8 -1
  62. package/dist/web/bundle/app.js +3595 -1520
  63. package/dist/web/bundle/edges.js +44 -16
  64. package/dist/web/bundle/layout.js +1026 -80
  65. package/dist/web/index.html +2 -2
  66. package/dist/web/src/artwork.js +25 -4
  67. package/dist/web/src/axiomring.js +42 -0
  68. package/dist/web/src/bundles.js +52 -16
  69. package/dist/web/src/canvas.js +109 -54
  70. package/dist/web/src/overview.js +4 -1
  71. package/dist/web/src/past.js +28 -0
  72. package/dist/web/src/scene.js +13 -12
  73. package/dist/web/src/selection.js +47 -4
  74. package/dist/web/src/status.js +32 -2
  75. package/dist/web/src/theme.js +108 -0
  76. package/dist/web/style.css +181 -45
  77. package/docs/agent-api/following.md +1 -1
  78. package/docs/agent-api/history.md +3 -0
  79. package/docs/agent-api/library.md +1 -1
  80. package/docs/agent-api/requests.md +4 -1
  81. package/docs/agent-api/views.md +1 -1
  82. package/package.json +1 -1
@@ -5,13 +5,14 @@
5
5
  import { spawnSync } from "node:child_process";
6
6
  import { accessSync, constants, existsSync, readFileSync, statSync } from "node:fs";
7
7
  import { filmPin } from "./film.js";
8
- import { tmpdir } from "node:os";
8
+ import { freemem, tmpdir, totalmem } from "node:os";
9
9
  import { dirname, join, relative, resolve } from "node:path";
10
10
  import { cacheHome, projectDirs, siteHolder } from "./dirs.js";
11
11
  import { cmp, packageVersion, parseVersion, satisfies } from "./version.js";
12
12
  import { choose, FROM_PACKAGE, installCommand, newerThan, newest, notesOf, npmCommand, PACKAGE_ROOT, readInfo, sameMinor, Versions } from "./updates.js";
13
13
  export { packageVersion, satisfies };
14
14
  import { ProjectError, openProject, targetsInWords } from "./project.js";
15
+ import { DEFAULT_JOBS, inGB, jobsAdvice, memoryFloor, onMathlib, WORKER_PEAK } from "./usage.js";
15
16
  export const run = (cmd, args, cwd) => {
16
17
  const r = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000, stdio: ["ignore", "pipe", "pipe"], ...(cwd ? { cwd } : {}) });
17
18
  return r.error || r.status !== 0 ? null : `${r.stdout}${r.stderr}`;
@@ -179,7 +180,7 @@ export function doctor(o) {
179
180
  ? { level: "ok", what: `XeLaTeX (${/TeX Live \d+|MiKTeX[^\s)]*/.exec(xelatex)?.[0] ?? "found"})` }
180
181
  : { level: "info", what: "XeLaTeX not found: needed only for the printed document (Export)", fix: install("xelatex", platform) });
181
182
  if (exec("git", ["--version"]) === null) {
182
- checks.push({ level: "info", what: "git not found: builds are kept without their commit, and past sources only as the builds kept them" });
183
+ checks.push({ level: "info", what: "git not found: builds are kept without their commit, and past sources only as the builds kept them; a branch cannot be followed (serve <git URL>)" });
183
184
  }
184
185
  const film = filmLine(o.cache ? resolve(o.cache) : cacheHome(), o.updates ?? process.env.VISUALPROOF_UPDATES);
185
186
  const updates = updatesLine(exec, node, o.fromPackage ?? FROM_PACKAGE, o.cache ? resolve(o.cache) : cacheHome(), o.updates ?? process.env.VISUALPROOF_UPDATES, o.path === null ? null : projectDirs(resolve(o.path), o.site, o.cache).site);
@@ -211,7 +212,7 @@ export function doctor(o) {
211
212
  }
212
213
  const from = o.roots.length ? "--root" : existsSync(join(root, "lakefile.toml")) ? "lakefile.toml" : "lakefile.lean";
213
214
  const missing = roots.filter((r) => !existsSync(join(libDir, ...r.split(".")) + ".olean"));
214
- const mathlib = ["lakefile.toml", "lakefile.lean", "lake-manifest.json"].some((f) => existsSync(join(root, f)) && /mathlib/i.test(readFileSync(join(root, f), "utf8")));
215
+ const mathlib = onMathlib(root);
215
216
  checks.push(missing.length === 0
216
217
  ? { level: "ok", what: `built: ${targetsInWords(project)} (from ${from})` }
217
218
  : {
@@ -219,6 +220,17 @@ export function doctor(o) {
219
220
  what: `not built: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ` and ${missing.length - 5} more` : ""} (${relative(root, join(libDir, ...missing[0].split("."))) + ".olean"} is missing; from ${from})`,
220
221
  fix: `in ${root}: ${mathlib ? "lake exe cache get && " : ""}lake build${o.roots.length === 0 && from === "lakefile.lean" ? ", or name the modules to read with --root" : ""}`,
221
222
  });
223
+ // what the usage pass's workers may hold, at the default --jobs, against the memory available
224
+ const mem = o.memory ?? { available: freemem(), total: totalmem() };
225
+ const advice = jobsAdvice(DEFAULT_JOBS, mathlib, mem.available);
226
+ const had = `memory: ${inGB(mem.available)} available of ${inGB(mem.total)}`;
227
+ checks.push(advice
228
+ ? {
229
+ level: "warn",
230
+ what: `${had}: on Mathlib the usage pass's workers may hold up to ${inGB(WORKER_PEAK)} each, ${inGB(advice.need)} at ${DEFAULT_JOBS} at a time (the default --jobs); it starts fewer while less than ${inGB(memoryFloor(mem.total))} is available`,
231
+ fix: `serve with --jobs ${advice.fits}`,
232
+ }
233
+ : { level: "ok", what: `${had}${mathlib ? `: enough for the usage pass at ${DEFAULT_JOBS} at a time` : ""}` });
222
234
  const dirs = projectDirs(root, o.site, o.cache);
223
235
  for (const [what, dir] of [["site", dirs.site], ["cache", dirs.cache]]) {
224
236
  if (!writable(dir))
@@ -129,26 +129,41 @@ export class Timeline {
129
129
  /** A new graph: every target's TCB is compared with the one it had, and a change is an event.
130
130
  * `renamed`: the renames found against the build before (HistorySpec.md), old name → new: a
131
131
  * declaration renamed is the same one, neither removed nor added, and a target renamed keeps its
132
- * baseline. */
132
+ * baseline. So are the renames in force (the review's `renamed`, followed on to the name the
133
+ * graph has): a build's first graph waits for the usage pass and is not compared, so the graph
134
+ * that is (the usage pass's, the build read again) finds no renames of its own; and a move made
135
+ * by hand renames too. */
133
136
  graphChanged(prep, review, renamed = new Map()) {
134
137
  const meta = JSON.parse(gunzipSync(prep.metaGz).toString("utf8"));
135
138
  const raw = gunzipSync(prep.edgesGz);
136
139
  const edges = new Int32Array(new Uint8Array(raw).buffer);
137
- this.index = new TcbIndex(meta.nodes, edges);
140
+ const index = (this.index = new TcbIndex(meta.nodes, edges));
138
141
  // a baseline kept before versions holds the kernel's hashes (server/versions.ts): one that is
139
142
  // the kernel's hash of the content there now is that content's version
140
143
  const kernel = new Map();
141
144
  for (const v of meta.nodes)
142
145
  if (v.kernel && v.hash)
143
146
  kernel.set(v.name, v.kernel);
144
- if (renamed.size) {
147
+ const ahead = new Map(renamed);
148
+ for (const [to, from] of Object.entries(review.renamed ?? {}))
149
+ if (!ahead.has(from))
150
+ ahead.set(from, to);
151
+ /** a name of a baseline as the graph has it now: renamed on where it is gone (a loop, which a log
152
+ * edited by hand could make, is cut short) */
153
+ const nameNow = (k) => {
154
+ let n = k;
155
+ for (let i = 0; i < 1000 && !index.byName.has(n) && ahead.has(n); i++)
156
+ n = ahead.get(n);
157
+ return index.byName.has(n) ? n : k;
158
+ };
159
+ if (ahead.size) {
145
160
  for (const t of Object.keys(this.baselines)) {
146
161
  const b = this.baselines[t];
147
162
  const now = byName();
148
163
  for (const k of Object.keys(b))
149
- now[renamed.get(k) ?? k] = b[k];
164
+ now[nameNow(k)] = b[k];
150
165
  delete this.baselines[t];
151
- this.baselines[renamed.get(t) ?? t] = now;
166
+ this.baselines[nameNow(t)] = now;
152
167
  }
153
168
  }
154
169
  for (const target of Object.keys(review.targets)) {
@@ -90,7 +90,7 @@ export class Exporter {
90
90
  delete j.notes;
91
91
  return j;
92
92
  };
93
- const known = this.jobs.get(id);
93
+ const known = this.known(id);
94
94
  if (known && known.state !== "failed")
95
95
  return note(known);
96
96
  const kept = this.fromCache(id);
@@ -121,11 +121,22 @@ export class Exporter {
121
121
  job(id) {
122
122
  if (!/^[0-9a-f]{32}$/.test(id))
123
123
  return null;
124
- const j = this.jobs.get(id) ?? this.fromCache(id);
124
+ const j = this.known(id) ?? this.fromCache(id);
125
125
  if (j)
126
126
  this.remember(j);
127
127
  return j;
128
128
  }
129
+ /** A job this server knows, unless it is done and its files have left the cache since (newer
130
+ * documents took their place): that one is forgotten, so that the same document asked for again
131
+ * is typeset again rather than answered with files that are gone. */
132
+ known(id) {
133
+ const j = this.jobs.get(id);
134
+ if (j?.state !== "done" || existsSync(join(this.cacheDir, id, "doc.pdf")))
135
+ return j;
136
+ this.jobs.delete(id);
137
+ this.ends.delete(id);
138
+ return undefined;
139
+ }
129
140
  /** The job once it is over. */
130
141
  wait(id) {
131
142
  return this.ends.get(id) ?? Promise.resolve(this.job(id));
@@ -317,14 +328,16 @@ export class Exporter {
317
328
  return zip(files);
318
329
  }
319
330
  }
320
- const README = `This is a document VisualProof exported, as LaTeX.
331
+ /** The sources' README: how to typeset them again, and the macros a document is written in, each
332
+ * one the style file defines (a test checks it). */
333
+ export const README = `This is a document VisualProof exported, as LaTeX.
321
334
 
322
335
  Typeset it with XeLaTeX, twice (the second run resolves page references and bookmarks):
323
336
 
324
337
  xelatex doc.tex
325
338
  xelatex doc.tex
326
339
 
327
- doc.tex holds the content only, as semantic macros (\\vpPart, \\begin{vpDecl}, \\vpFullName, \\begin{vpCode}, ...).
340
+ doc.tex holds the content only, as semantic macros (\\vpSection, \\begin{vpDecl}, \\vpFullName, \\begin{vpCode}, ...).
328
341
  How it looks is visualproof.sty's: change it, replace it, or load a package of your own after it
329
342
  and \\renewcommand what you want different. Its head lists every macro a document uses.
330
343
 
@@ -12,7 +12,7 @@ import { request as httpRequest } from "node:http";
12
12
  import { request as httpsRequest } from "node:https";
13
13
  import { isIP } from "node:net";
14
14
  import { networkInterfaces } from "node:os";
15
- import { MAX_FILE } from "../shared/library.js";
15
+ import { MAX_FILE, sizeWords } from "../shared/library.js";
16
16
  const DEFAULTS = { maxBytes: MAX_FILE, timeoutMs: 60_000, maxRedirects: 5 };
17
17
  /** the most of a page that is read, and the longest it may take: an HTML page is far smaller than a
18
18
  * file, and what it says of one is not worth a long wait */
@@ -64,13 +64,17 @@ function blockedV4(b) {
64
64
  (a === 172 && c >= 16 && c <= 31) || // private
65
65
  (a === 192 && c === 168) || // private
66
66
  (a === 192 && c === 0 && b[2] === 0) || // IETF protocol assignments
67
+ (a === 192 && c === 0 && b[2] === 2) || // documentation (TEST-NET-1)
68
+ (a === 192 && c === 88 && b[2] === 99) || // 6to4 relays, deprecated
67
69
  (a === 198 && (c === 18 || c === 19)) || // benchmarking
70
+ (a === 198 && c === 51 && b[2] === 100) || // documentation (TEST-NET-2)
71
+ (a === 203 && c === 0 && b[2] === 113) || // documentation (TEST-NET-3)
68
72
  a >= 224 // multicast, reserved, broadcast
69
73
  );
70
74
  }
71
75
  /** Whether the server refuses to fetch from an address: loopback, private, link-local, unique-local,
72
- * multicast, reserved or unspecified, an IPv4 one written as IPv6 (mapped, NAT64) included. Anything
73
- * that is not an address at all is refused too. */
76
+ * multicast, reserved, documentation or unspecified, an IPv4 one written as IPv6 (mapped, NAT64,
77
+ * 6to4) included. Anything that is not an address at all is refused too. */
74
78
  export function blockedAddress(address) {
75
79
  const kind = isIP(address.replace(/%.*$/, ""));
76
80
  if (kind === 4)
@@ -91,6 +95,8 @@ export function blockedAddress(address) {
91
95
  return true; // IPv4-compatible, deprecated
92
96
  if (p[0] === 0x64 && p[1] === 0xff9b && p.slice(2, 6).every((x) => x === 0))
93
97
  return blockedV4(v4(p[6], p[7])); // NAT64
98
+ if (p[0] === 0x2002)
99
+ return blockedV4(v4(p[1], p[2])); // 6to4: the IPv4 address it carries
94
100
  if ((p[0] & 0xfe00) === 0xfc00)
95
101
  return true; // unique local
96
102
  if ((p[0] & 0xffc0) === 0xfe80)
@@ -211,7 +217,7 @@ async function guardedFetch(raw, kind, o, allowed) {
211
217
  return { ok: false, status: 400, error: `${host} cannot be found.` };
212
218
  const refused = addresses.find((a) => !allowed(a.address));
213
219
  if (refused)
214
- return { ok: false, status: 403, error: `${host} is ${refused.address}, an address of a private or local network or of this server: the server does not fetch from there.` };
220
+ return { ok: false, status: 403, error: `${host === refused.address ? host : `${host} is ${refused.address}, which`} is an address of a private or local network or of this server: the server does not fetch from there.` };
215
221
  const left = deadline - Date.now();
216
222
  if (left <= 0)
217
223
  return { ok: false, status: 504, error: "The address took too long to answer." };
@@ -224,8 +230,11 @@ async function guardedFetch(raw, kind, o, allowed) {
224
230
  return { ok: false, status: 502, error: `The address could not be fetched: ${e instanceof Error ? e.message : String(e)}.` };
225
231
  }
226
232
  const status = res.statusCode ?? 0;
233
+ // what is not the file is never read: a redirect's or an error's body, drained, could run on
234
+ // past the bytes and the time a fetch may take, the fetch answered long since
235
+ if (status !== 200)
236
+ res.destroy();
227
237
  if ([301, 302, 303, 307, 308].includes(status)) {
228
- res.resume();
229
238
  const to = res.headers.location;
230
239
  if (!to)
231
240
  return { ok: false, status: 502, error: `The address answered ${status} without saying where to.` };
@@ -239,26 +248,28 @@ async function guardedFetch(raw, kind, o, allowed) {
239
248
  }
240
249
  continue;
241
250
  }
242
- if (status !== 200) {
243
- res.resume();
251
+ if (status !== 200)
244
252
  return { ok: false, status: 502, error: `The address answered ${status}, not the ${noun}.` };
245
- }
246
253
  const length = Number(res.headers["content-length"] ?? NaN);
247
254
  if (Number.isFinite(length) && length > o.maxBytes) {
248
255
  res.destroy();
249
- return { ok: false, status: 413, error: `The ${noun} is ${Math.round(length / 1e6)} MB: ${most} is at most ${Math.round(o.maxBytes / 1e6)} MB.` };
256
+ return { ok: false, status: 413, error: `The ${noun} is ${sizeWords(length)}: ${most} is at most ${sizeWords(o.maxBytes)}.` };
250
257
  }
251
258
  const body = await readCapped(res, o.maxBytes, deadline);
252
259
  if (typeof body === "string")
253
- return { ok: false, status: body === "large" ? 413 : 504, error: body === "large" ? `The ${noun} is larger than ${Math.round(o.maxBytes / 1e6)} MB, the most ${most} may be.` : `The address took too long to send the ${noun}.` };
260
+ return { ok: false, status: body === "large" ? 413 : 504, error: body === "large" ? `The ${noun} is larger than ${sizeWords(o.maxBytes)}, the most ${most} may be.` : `The address took too long to send the ${noun}.` };
254
261
  const disposition = res.headers["content-disposition"];
255
262
  return { ok: true, bytes: body, url, disposition };
256
263
  }
257
264
  }
258
- /** A request to `url`, connected to the address checked for it and to no other; its answer's head. */
265
+ /** A request to `url`, connected to the address checked for it and to no other; its answer's head,
266
+ * within `ms` in all. */
259
267
  function connect(url, at, ms, accept) {
260
268
  return new Promise((ok, no) => {
261
269
  const make = url.protocol === "https:" ? httpsRequest : httpRequest;
270
+ // the socket's `timeout` is idleness alone: a head sent a byte at a time is never idle, and
271
+ // would hold the fetch past its time for as long as its sender likes
272
+ const late = setTimeout(() => req.destroy(new Error("no answer in time")), ms);
262
273
  const req = make(url, {
263
274
  method: "GET",
264
275
  headers: { "user-agent": "VisualProof (adds a file to a project's library)", accept },
@@ -271,9 +282,15 @@ function connect(url, at, ms, accept) {
271
282
  cb(null, at.address, at.family);
272
283
  }),
273
284
  timeout: ms,
274
- }, ok);
285
+ }, (res) => {
286
+ clearTimeout(late);
287
+ ok(res);
288
+ });
275
289
  req.on("timeout", () => req.destroy(new Error("no answer in time")));
276
- req.on("error", no);
290
+ req.on("error", (e) => {
291
+ clearTimeout(late);
292
+ no(e);
293
+ });
277
294
  req.end();
278
295
  });
279
296
  }
@@ -1,4 +1,4 @@
1
- // The tips' tour film (UpdateSpec decision 14, the owner's of 2026-10-17): 3½–4 minutes at
1
+ // The tips' tour film (UpdateSpec decision 14, the owner's of 2026-10-17): about 3½ minutes at
2
2
  // 2560×1600, too large for the package and for git, so it is an asset of the repository's GitHub
3
3
  // prerelease `tour-1`, and the package pins it (assets/tips/tour.json: its file, sha256 and size).
4
4
  // The server fetches it the first time a page asks for it (GET /tips/tour.mp4), one download for
@@ -32,6 +32,7 @@ import { childEnv } from "./children.js";
32
32
  import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
33
33
  import { join } from "node:path";
34
34
  import { gunzipSync, gzipSync } from "node:zlib";
35
+ import { endsLine } from "./lostfound.js";
35
36
  import { SourceStore, summarize } from "./sourcestore.js";
36
37
  import { ownName, refIdentity, versionOf } from "./versions.js";
37
38
  const sha = (s) => createHash("sha256").update(s).digest("hex").slice(0, 32);
@@ -493,7 +494,8 @@ export class History {
493
494
  }
494
495
  save(info) {
495
496
  this.infos.set(info.id, info);
496
- appendFileSync(this.indexFile, JSON.stringify(info) + "\n");
497
+ // after a line cut short, on a line of its own: the line cut short alone is lost
498
+ appendFileSync(this.indexFile, (endsLine(this.indexFile) ? "" : "\n") + JSON.stringify(info) + "\n");
497
499
  }
498
500
  readGz(file) {
499
501
  return JSON.parse(gunzipSync(readFileSync(file)).toString("utf8"));
@@ -12,58 +12,77 @@
12
12
  // in 0.6 s): the page takes it in two parts, its declarations as JSON and its
13
13
  // edges packed as the graph on show's are (shared/edgecodec.ts), each compressed once and kept for
14
14
  // the last few snapshots asked, under the renames in force, and tagged so that a page that has one
15
- // is answered 304.
16
- import { gzipSync } from "node:zlib";
17
- import { packEdges } from "../shared/edgecodec.js";
15
+ // is answered 304. The parts, a snapshot's whole graph and what changed between two snapshots are
16
+ // made off the server's thread (server/historythread.ts), which meanwhile answers others: on
17
+ // NavierStokesAndEuler the Time Machine's first look at a build held it 4 s.
18
+ import { dirname } from "node:path";
18
19
  import { MAX_ARCHIVE_KEYS, validKey } from "./lostfound.js";
19
- import { diffGraphs, pageGraph } from "./history.js";
20
+ import { offThread, runJob } from "./historythread.js";
20
21
  const no = (status, error) => ({ status, body: { error } });
21
22
  const ok = (body) => ({ status: 200, body });
22
23
  /** the parts of the last snapshots asked, per history, by snapshot, build read and renames */
23
24
  const kept = new WeakMap();
24
25
  /** how many snapshots' parts are kept: about a megabyte each, compressed, on a large project (Sphere-Packing-Lean's:
25
- * 0.7 MB of declarations, 0.5 MB of edges) */
26
+ * 0.7 MB of declarations, 0.5 MB of edges; NavierStokesAndEuler's 5.2 and 4.0 MB) */
26
27
  const KEEP_PARTS = 8;
27
28
  /** what changed between two snapshots, as answered last (a step on the Time Machine's timeline asks
28
29
  * it of each build against the latest, every reader the same), by the builds as read and the renames
29
30
  * in force; a few hundred KB at most each */
30
31
  const diffsKept = new WeakMap();
31
32
  const KEEP_DIFFS = 32;
33
+ /** what is being made off the thread, by what it will be kept under: the Time Machine asks a
34
+ * snapshot's two parts at once, made by one job */
35
+ const making = new WeakMap();
36
+ const mapOf = (m, k) => {
37
+ let x = m.get(k);
38
+ if (!x)
39
+ m.set(k, (x = new Map()));
40
+ return x;
41
+ };
42
+ /** Keep `v` under `key`, the most recently asked last: the oldest goes first. */
43
+ function keep(byKey, key, v, most) {
44
+ byKey.delete(key);
45
+ byKey.set(key, v);
46
+ while (byKey.size > most)
47
+ byKey.delete(byKey.keys().next().value);
48
+ }
49
+ /** What a snapshot's parts are kept under: the snapshot, its build as read, the renames in force,
50
+ * and its texts (once backfilled, they change what the declarations' part says). */
51
+ const partsKey = (info, tag) => `${info.id}\u0000${info.fingerprint}\u0000${info.read}\u0000${tag}\u0000${JSON.stringify(info.sources ?? null)}`;
52
+ const diffKey = (fa, fb, most, tag) => [fa.id, fa.fingerprint, fa.read, fb.id, fb.fingerprint, fb.read, most, tag].join("\u0000");
53
+ /** A job, made once however many ask for it at once (`key`): in the history's worker where the
54
+ * world says its renames, else on this thread. */
55
+ function made(w, key, job) {
56
+ const h = w.history;
57
+ const byKey = mapOf(making, h);
58
+ let p = byKey.get(key);
59
+ if (!p) {
60
+ const here = () => runJob(h, w.identity(), job);
61
+ const renamed = w.renamed?.();
62
+ p = (renamed === undefined ? Promise.resolve().then(here) : offThread(dirname(h.dir), renamed, job, here)).finally(() => byKey.delete(key));
63
+ byKey.set(key, p);
64
+ }
65
+ return p;
66
+ }
32
67
  /** Snapshot `id`'s graph as the page takes it (`part`): its declarations — the answer of /graph
33
- * without the edges, and how many there are — or its edges, packed. Made together, once. */
34
- export function snapshotParts(w, id) {
68
+ * without the edges, and how many there are — or its edges, packed. Made together, once, off the
69
+ * server's thread. */
70
+ export async function snapshotParts(w, id) {
35
71
  const h = w.history;
36
- const info = h.get(id);
37
- // its texts, once backfilled, change what the declarations' part says
38
- const key = `${id}\u0000${info.fingerprint}\u0000${info.read}\u0000${w.identityTag?.() ?? ""}\u0000${JSON.stringify(info.sources ?? null)}`;
39
- let byKey = kept.get(h);
40
- if (!byKey)
41
- kept.set(h, (byKey = new Map()));
72
+ const tag = w.identityTag?.() ?? "";
73
+ const key = partsKey(h.get(id), tag);
74
+ const byKey = mapOf(kept, h);
42
75
  const have = byKey.get(key);
43
76
  if (have) {
44
- // most recently asked last: the oldest goes first
45
- byKey.delete(key);
46
- byKey.set(key, have);
77
+ keep(byKey, key, have, KEEP_PARTS);
47
78
  return have;
48
79
  }
49
- const g = h.graph(id);
50
- const { edges: _flat, ...rest } = pageGraph({ ...g, edges: [] }, w.identity());
51
- const E = new Int32Array(3 * g.edges.length);
52
- g.edges.forEach(([s, d, m], k) => E.set([s, d, m], 3 * k));
53
- const parts = { nodes: gzipSync(JSON.stringify({ ...rest, edgeCount: g.edges.length, sources: keptTexts(h, id) })), edges: gzipSync(packEdges(E, g.nodes.length, 2), { level: 9 }) };
54
- byKey.set(key, parts);
55
- while (byKey.size > KEEP_PARTS)
56
- byKey.delete(byKey.keys().next().value);
57
- return parts;
58
- }
59
- /** The modules whose text snapshot `id` keeps, each with how it was matched to its build
60
- * (server/sourcestore.ts): what the page says of a declaration's source then before it asks. */
61
- function keptTexts(h, id) {
62
- const out = {};
63
- for (const [m, r] of Object.entries(h.sourcesOf(id) ?? {}))
64
- if (r.blob && r.check)
65
- out[m] = r.check;
66
- return out;
80
+ const d = await made(w, `parts\u0000${key}`, { kind: "parts", id });
81
+ if (d.kind !== "parts")
82
+ throw new Error("not the parts asked for");
83
+ // kept as the job read the snapshot: one read again meanwhile is made again for whoever asks next
84
+ keep(byKey, partsKey(d.info, tag), d.parts, KEEP_PARTS);
85
+ return d.parts;
67
86
  }
68
87
  /** Module `module`'s text at snapshot `id`'s build (GET /source; HistorySpec decision 23): the
69
88
  * text its build was compiled from, with its Lake hash and how it was matched to the build, or
@@ -98,7 +117,7 @@ function snapshotOf(w, a, key, dflt) {
98
117
  }
99
118
  /** A request under /api/history (or /api/agent/history). `refuse`: why the caller may not move
100
119
  * things (an agent below level full), or null. */
101
- export function historyAnswer(w, a, refuse = () => null) {
120
+ export async function historyAnswer(w, a, refuse = () => null) {
102
121
  const h = w.history;
103
122
  if (!h)
104
123
  return no(404, "This server serves a saved graph: it reads no builds, and keeps no history.");
@@ -149,7 +168,10 @@ export function historyAnswer(w, a, refuse = () => null) {
149
168
  }
150
169
  if (path === "") {
151
170
  const latest = h.latest();
152
- return ok({ snapshots: h.list(), current: latest?.id ?? null, diskBytes: h.diskBytes() });
171
+ // agents are told the branch's builds too (DeploySpec decision 11): the page has them with the
172
+ // build status
173
+ const following = a.path.startsWith("/api/agent/") ? w.following?.() : null;
174
+ return ok({ snapshots: h.list(), current: latest?.id ?? null, diskBytes: h.diskBytes(), ...(following ? { following } : {}) });
153
175
  }
154
176
  if (path === "/graph") {
155
177
  const id = snapshotOf(w, a, "id", true);
@@ -157,12 +179,17 @@ export function historyAnswer(w, a, refuse = () => null) {
157
179
  return id;
158
180
  const part = a.query.get("part");
159
181
  if (part === "nodes")
160
- return { status: 200, body: null, gz: { type: "application/json", body: snapshotParts(w, id).nodes } };
182
+ return { status: 200, body: null, gz: { type: "application/json", body: (await snapshotParts(w, id)).nodes } };
161
183
  if (part === "edges")
162
- return { status: 200, body: null, gz: { type: "application/octet-stream", body: snapshotParts(w, id).edges } };
184
+ return { status: 200, body: null, gz: { type: "application/octet-stream", body: (await snapshotParts(w, id)).edges } };
163
185
  if (part !== null)
164
186
  return no(400, "part is nodes or edges; leave it out for the whole graph.");
165
- return ok(pageGraph(h.graph(id), w.identity()));
187
+ // the whole graph, which agents ask (105 MB of JSON on NavierStokesAndEuler): made and
188
+ // compressed off the thread, each time
189
+ const d = await made(w, `graph\u0000${partsKey(h.get(id), w.identityTag?.() ?? "")}`, { kind: "graph", id });
190
+ if (d.kind !== "graph")
191
+ throw new Error("not the graph asked for");
192
+ return { status: 200, body: null, gz: { type: "application/json", body: d.gz } };
166
193
  }
167
194
  if (path === "/source") {
168
195
  const id = snapshotOf(w, a, "id", false);
@@ -181,7 +208,7 @@ export function historyAnswer(w, a, refuse = () => null) {
181
208
  if (typeof to !== "number")
182
209
  return to;
183
210
  const most = Math.min(MOST_DIFF, Math.max(0, Number(a.query.get("most") ?? 2000) || 0));
184
- return ok(snapshotDiff(w, from, to, most));
211
+ return ok(await snapshotDiffOff(w, from, to, most));
185
212
  }
186
213
  return no(404, "not found");
187
214
  }
@@ -190,23 +217,43 @@ export function historyAnswer(w, a, refuse = () => null) {
190
217
  export const MOST_DIFF = 20000;
191
218
  /** What changed from snapshot `from` to snapshot `to` (both in the history), at most `most` names of
192
219
  * each kind, as GET /diff answers it: made once, then kept for every reader and for the export of a
193
- * view at a past build (server/pastdoc.ts), which makes the same past graph the page does. */
220
+ * view at a past build (server/pastdoc.ts), which makes the same past graph the page does. Made
221
+ * here, on the server's thread, where it is not kept. */
194
222
  export function snapshotDiff(w, from, to, most) {
195
223
  const h = w.history;
196
- const [fa, fb] = [h.get(from), h.get(to)];
197
- const key = [from, fa.fingerprint, fa.read, to, fb.fingerprint, fb.read, most, w.identityTag?.() ?? ""].join("\u0000");
198
- let byKey = diffsKept.get(h);
199
- if (!byKey)
200
- diffsKept.set(h, (byKey = new Map()));
224
+ const key = diffKey(h.get(from), h.get(to), most, w.identityTag?.() ?? "");
225
+ const byKey = mapOf(diffsKept, h);
201
226
  let body = byKey.get(key);
202
227
  if (body === undefined) {
203
- body = { from, to, ...diffGraphs(h.graph(from), h.graph(to), w.identity(), w.renamesBetween?.(from, to) ?? new Map(), most) };
204
- byKey.set(key, body);
205
- while (byKey.size > KEEP_DIFFS)
206
- byKey.delete(byKey.keys().next().value);
228
+ const d = runJob(h, w.identity(), diffJob(w, from, to, most));
229
+ if (d.kind !== "diff")
230
+ throw new Error("not the diff asked for");
231
+ body = d.diff;
207
232
  }
233
+ keep(byKey, key, body, KEEP_DIFFS);
208
234
  return body;
209
235
  }
236
+ /** The same, made off the server's thread where it is not kept (GET /diff). */
237
+ async function snapshotDiffOff(w, from, to, most) {
238
+ const h = w.history;
239
+ const tag = w.identityTag?.() ?? "";
240
+ const key = diffKey(h.get(from), h.get(to), most, tag);
241
+ const byKey = mapOf(diffsKept, h);
242
+ const have = byKey.get(key);
243
+ if (have) {
244
+ keep(byKey, key, have, KEEP_DIFFS);
245
+ return have;
246
+ }
247
+ const d = await made(w, `diff\u0000${key}`, diffJob(w, from, to, most));
248
+ if (d.kind !== "diff")
249
+ throw new Error("not the diff asked for");
250
+ keep(byKey, diffKey(d.infos[0], d.infos[1], most, tag), d.diff, KEEP_DIFFS);
251
+ return d.diff;
252
+ }
253
+ /** The job of what changed from `from` to `to`, with the renames recorded between them. */
254
+ function diffJob(w, from, to, most) {
255
+ return { kind: "diff", from, to, most, renames: [...(w.renamesBetween?.(from, to) ?? new Map())] };
256
+ }
210
257
  /** What the history's answers read, from what the agents' API has (server/agent.ts, World): what
211
258
  * is not there answers as a server with nothing of it would. */
212
259
  export function historyWorld(world) {
@@ -215,6 +262,8 @@ export function historyWorld(world) {
215
262
  return world.history ?? null;
216
263
  },
217
264
  identity: () => world.identity?.() ?? ((u) => u),
265
+ // the renames, where the world says them or has no identity of its own (each name its own)
266
+ ...(world.renamed || !world.identity ? { renamed: () => world.renamed?.() ?? null } : {}),
218
267
  identityTag: () => world.identityTag?.() ?? "",
219
268
  renamesBetween: (from, to) => world.renamesBetween?.(from, to) ?? new Map(),
220
269
  moves: () => world.moves?.() ?? { migrations: [], suggestions: [] },
@@ -223,6 +272,7 @@ export function historyWorld(world) {
223
272
  lostAndFound: () => world.lostAndFound?.() ?? { lost: [], migrations: [], archived: [], aside: { entries: 0, modules: [] }, count: 0 },
224
273
  lastHad: (name) => world.lastHad?.(name) ?? null,
225
274
  archiveEntries: (keys, on, who) => world.archiveEntries?.(keys, on, who) ?? { status: 404, body: { error: "not found" } },
275
+ following: () => world.following?.() ?? null,
226
276
  };
227
277
  }
228
278
  /** The same, as endpoints of the agents' API (server/agent.ts): reading only, at every level. */