visualproof 0.4.3 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +16 -7
  2. package/dist/assets/latex/visualproof.sty +19 -9
  3. package/dist/assets/tips/tour-agent.webp +0 -0
  4. package/dist/assets/tips/tour-arrange.webp +0 -0
  5. package/dist/assets/tips/tour-artwork.webp +0 -0
  6. package/dist/assets/tips/tour-checks.webp +0 -0
  7. package/dist/assets/tips/tour-keys.webp +0 -0
  8. package/dist/assets/tips/tour-library.webp +0 -0
  9. package/dist/assets/tips/tour-project.webp +0 -0
  10. package/dist/assets/tips/tour-review.webp +0 -0
  11. package/dist/assets/tips/tour-share.webp +0 -0
  12. package/dist/assets/tips/tour-source.webp +0 -0
  13. package/dist/assets/tips/tour-target.webp +0 -0
  14. package/dist/assets/tips/tour-view.webp +0 -0
  15. package/dist/assets/tips/tour.json +117 -112
  16. package/dist/assets/tips/tour.webp +0 -0
  17. package/dist/mcp/visualproof-mcp.mjs +7 -3
  18. package/dist/server/access.js +37 -1
  19. package/dist/server/accesspage.js +130 -0
  20. package/dist/server/agent.js +11 -5
  21. package/dist/server/agentlibrary.js +1 -1
  22. package/dist/server/agentviews.js +1 -1
  23. package/dist/server/artcmd.js +1 -1
  24. package/dist/server/askfollow.js +1 -4
  25. package/dist/server/buildwatch.js +133 -0
  26. package/dist/server/deploy.js +844 -0
  27. package/dist/server/doctor.js +15 -3
  28. package/dist/server/events.js +20 -5
  29. package/dist/server/export.js +17 -4
  30. package/dist/server/fetchurl.js +30 -13
  31. package/dist/server/film.js +1 -1
  32. package/dist/server/history.js +3 -1
  33. package/dist/server/historyapi.js +100 -50
  34. package/dist/server/historythread.js +206 -0
  35. package/dist/server/hover.js +8 -0
  36. package/dist/server/identity.js +25 -1
  37. package/dist/server/library.js +10 -3
  38. package/dist/server/live.js +206 -28
  39. package/dist/server/lostfound.js +3 -2
  40. package/dist/server/main.js +361 -50
  41. package/dist/server/poppler.js +48 -10
  42. package/dist/server/renames.js +10 -5
  43. package/dist/server/review.js +32 -2
  44. package/dist/server/reviewapi.js +1 -1
  45. package/dist/server/status.js +144 -50
  46. package/dist/server/talk.js +7 -2
  47. package/dist/server/usage.js +51 -0
  48. package/dist/server/views.js +21 -7
  49. package/dist/shared/board.js +2 -1
  50. package/dist/shared/collab.js +14 -8
  51. package/dist/shared/deploy.js +35 -0
  52. package/dist/shared/doc/latex.js +5 -3
  53. package/dist/shared/doc/outline.js +6 -3
  54. package/dist/shared/layout.js +297 -70
  55. package/dist/shared/libm.js +366 -0
  56. package/dist/shared/library.js +11 -6
  57. package/dist/shared/palette.js +1 -1
  58. package/dist/shared/review.js +7 -5
  59. package/dist/shared/search.js +7 -3
  60. package/dist/shared/separate.js +556 -32
  61. package/dist/shared/views.js +8 -1
  62. package/dist/web/bundle/app.js +3595 -1520
  63. package/dist/web/bundle/edges.js +44 -16
  64. package/dist/web/bundle/layout.js +1026 -80
  65. package/dist/web/index.html +2 -2
  66. package/dist/web/src/artwork.js +25 -4
  67. package/dist/web/src/axiomring.js +42 -0
  68. package/dist/web/src/bundles.js +52 -16
  69. package/dist/web/src/canvas.js +109 -54
  70. package/dist/web/src/overview.js +4 -1
  71. package/dist/web/src/past.js +28 -0
  72. package/dist/web/src/scene.js +13 -12
  73. package/dist/web/src/selection.js +47 -4
  74. package/dist/web/src/status.js +32 -2
  75. package/dist/web/src/theme.js +108 -0
  76. package/dist/web/style.css +181 -45
  77. package/docs/agent-api/following.md +1 -1
  78. package/docs/agent-api/history.md +3 -0
  79. package/docs/agent-api/library.md +1 -1
  80. package/docs/agent-api/requests.md +4 -1
  81. package/docs/agent-api/views.md +1 -1
  82. package/package.json +1 -1
@@ -11,10 +11,10 @@
11
11
  // most two at once.
12
12
  import { execFile } from "node:child_process";
13
13
  import { tracked } from "./children.js";
14
- import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
14
+ import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
15
15
  import { join } from "node:path";
16
16
  import { gunzipSync, gzipSync } from "node:zlib";
17
- import { wordsText } from "../shared/library.js";
17
+ import { plainPunctuation, wordsText } from "../shared/library.js";
18
18
  /** the longest each tool may run, in ms */
19
19
  const INFO_MS = 20_000;
20
20
  const TEXT_MS = 120_000;
@@ -111,16 +111,17 @@ export function parseBboxLayout(xhtml) {
111
111
  export function pageText(p) {
112
112
  return wordsText(p.words, p.lines);
113
113
  }
114
- /** Find `q` in every page's text: case and runs of white space read loosely; at most `most`
115
- * matches, in page order. */
114
+ /** Find `q` in every page's text: case, runs of white space and typographic quotes and dashes read
115
+ * loosely; at most `most` matches, in page order. */
116
116
  export function searchWords(pages, q, most = 100) {
117
- const want = q.replace(/\s+/g, " ").trim().toLowerCase();
117
+ // typed as a keyboard types it: ' finds ’, - finds – (shared/library.ts, plainPunctuation)
118
+ const want = plainPunctuation(q.replace(/\s+/g, " ").trim().toLowerCase());
118
119
  const matches = [];
119
120
  if (!want)
120
121
  return { matches, more: false };
121
122
  for (const p of pages) {
122
123
  const { text, starts } = pageText(p);
123
- const flat = text.replace(/\n/g, " ").toLowerCase();
124
+ const flat = plainPunctuation(text.replace(/\n/g, " ").toLowerCase());
124
125
  for (let at = flat.indexOf(want); at >= 0; at = flat.indexOf(want, at + 1)) {
125
126
  if (matches.length >= most)
126
127
  return { matches, more: true };
@@ -150,13 +151,20 @@ function upper(starts, at) {
150
151
  /** The widths a page or a clip is drawn at: a few, so that what is kept stays small. */
151
152
  const WIDTHS = [400, 800, 1200, 1600, 2000];
152
153
  export const clampWidth = (w) => WIDTHS.find((x) => x >= w) ?? WIDTHS[WIDTHS.length - 1];
154
+ /** The most clips kept of a file, the oldest going first. A page is kept at a few widths, bounded by
155
+ * the file; but anyone who reads — a read-only reader, whoever comes to a `--public` site — may ask
156
+ * for any region of any page, and each was kept for good, filling the disk. The clips notes carry
157
+ * are asked again and again, and stay. */
158
+ export const MAX_CLIPS = 500;
153
159
  /** A PDF's words, pages and images, kept in `cacheDir/<sha>/`. */
154
160
  export class PdfReader {
155
161
  cacheDir;
156
162
  words = new Map();
157
163
  infos = new Map();
158
- constructor(cacheDir) {
164
+ maxClips;
165
+ constructor(cacheDir, maxClips = MAX_CLIPS) {
159
166
  this.cacheDir = cacheDir;
167
+ this.maxClips = maxClips;
160
168
  }
161
169
  dir(sha) {
162
170
  const d = join(this.cacheDir, sha);
@@ -220,8 +228,17 @@ export class PdfReader {
220
228
  const w = clampWidth(width);
221
229
  const r = region.map((x) => Math.round(x * 1e4) / 1e4);
222
230
  const kept = join(this.dir(sha), `clip-${page}-${r.join("_")}-w${w}.png`);
223
- if (existsSync(kept))
224
- return readFileSync(kept);
231
+ if (existsSync(kept)) {
232
+ try {
233
+ // asked for again: the last to go (atime is not kept on every system)
234
+ const now = new Date();
235
+ utimesSync(kept, now, now);
236
+ return readFileSync(kept);
237
+ }
238
+ catch {
239
+ // pruned meanwhile: drawn again
240
+ }
241
+ }
225
242
  // the resolution at which the region is `w` pixels wide, within reason
226
243
  const dpi = Math.min(600, Math.max(36, (w / ((r[2] - r[0]) * size.w)) * 72));
227
244
  const px = (v, of) => Math.round((v * of * dpi) / 72);
@@ -235,7 +252,28 @@ export class PdfReader {
235
252
  finally {
236
253
  rmSync(`${tmp}.png`, { force: true });
237
254
  }
238
- return readFileSync(kept);
255
+ const out = readFileSync(kept);
256
+ this.pruneClips(sha);
257
+ return out;
258
+ }
259
+ /** The clips kept of a file beyond the most there may be go, those drawn or asked for longest
260
+ * ago first (a clip asked for again is touched). */
261
+ pruneClips(sha) {
262
+ const d = join(this.cacheDir, sha);
263
+ let clips;
264
+ try {
265
+ clips = readdirSync(d)
266
+ .filter((f) => f.startsWith("clip-"))
267
+ .map((f) => ({ f, t: statSync(join(d, f)).mtimeMs }));
268
+ }
269
+ catch {
270
+ return;
271
+ }
272
+ if (clips.length <= this.maxClips)
273
+ return;
274
+ clips.sort((a, b) => a.t - b.t);
275
+ for (const c of clips.slice(0, clips.length - this.maxClips))
276
+ rmSync(join(d, c.f), { force: true });
239
277
  }
240
278
  /** What is kept of a file goes (its bytes purged). */
241
279
  forget(sha) {
@@ -16,6 +16,7 @@
16
16
  // What a module of either build that could not be read held is set aside: it did not vanish.
17
17
  import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
18
18
  import { join } from "node:path";
19
+ import { endsLine } from "./lostfound.js";
19
20
  import { refIdentity, versionOf } from "./versions.js";
20
21
  /** What stands for the declaration itself in its own references, so that a declaration can be
21
22
  * paired before its own rename is known. */
@@ -202,8 +203,7 @@ export class Renames {
202
203
  record(snap, pairs) {
203
204
  if (!pairs.length)
204
205
  return;
205
- mkdirSync(join(this.pairFile, ".."), { recursive: true });
206
- appendFileSync(this.pairFile, JSON.stringify({ snap, at: new Date().toISOString(), pairs }) + "\n");
206
+ append(this.pairFile, { snap, at: new Date().toISOString(), pairs });
207
207
  this.add(snap, pairs);
208
208
  }
209
209
  get(id) {
@@ -212,8 +212,7 @@ export class Renames {
212
212
  /** A migration made: kept, under the next id. */
213
213
  migrated(m) {
214
214
  const full = { id: `m${this.migrations.length + 1}`, ...m };
215
- mkdirSync(join(this.migFile, ".."), { recursive: true });
216
- appendFileSync(this.migFile, JSON.stringify(full) + "\n");
215
+ append(this.migFile, full);
217
216
  this.migrations.push(full);
218
217
  this.byId.set(full.id, full);
219
218
  return full;
@@ -225,7 +224,7 @@ export class Renames {
225
224
  return;
226
225
  const at = new Date().toISOString();
227
226
  const who = { by, ...(agent ? { agent: true } : {}), ...(person ? { person } : {}) };
228
- appendFileSync(this.migFile, JSON.stringify({ undo: id, ...who, at }) + "\n");
227
+ append(this.migFile, { undo: id, ...who, at });
229
228
  m.undone = { ...who, at };
230
229
  }
231
230
  /** The renames of high confidence found from snapshot `from` to snapshot `to`, composed along the
@@ -250,6 +249,12 @@ export class Renames {
250
249
  return out;
251
250
  }
252
251
  }
252
+ /** A line appended to one of the files: after a line cut short (the server stopped while writing),
253
+ * on a line of its own, so that the line cut short alone is lost, never the one written after it. */
254
+ function append(file, o) {
255
+ mkdirSync(join(file, ".."), { recursive: true });
256
+ appendFileSync(file, (endsLine(file) ? "" : "\n") + JSON.stringify(o) + "\n");
257
+ }
253
258
  function lines(file) {
254
259
  if (!existsSync(file))
255
260
  return [];
@@ -20,7 +20,8 @@
20
20
  //
21
21
  // What changed is told as a change (`ReviewChange`): the events that came after everything replayed
22
22
  // so far, which a page applies to its own review — or, when an event came in between (older events
23
- // restored from a backup, a rewrite), the whole review again.
23
+ // restored from a backup, a rewrite), none: the page reads the whole review again, which its tag
24
+ // (the run and the revision) keeps from being sent to a page that has it.
24
25
  import { createHash, randomBytes } from "node:crypto";
25
26
  import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, statSync, writeFileSync, writeSync } from "node:fs";
26
27
  import { isAbsolute, join, resolve } from "node:path";
@@ -70,6 +71,29 @@ export class ReviewStore {
70
71
  whole() {
71
72
  return { epoch: this.epoch, rev: this.rev, data: this.data, problems: this.problems, error: this.error };
72
73
  }
74
+ /** the review's data as JSON, and the whole review, each with the revision it is of */
75
+ dataJson = null;
76
+ wholeJson = null;
77
+ /** The review's data as JSON, written once a revision. */
78
+ dataText() {
79
+ if (this.dataJson?.rev !== this.rev)
80
+ this.dataJson = { rev: this.rev, text: JSON.stringify(this.data) };
81
+ return this.dataJson.text;
82
+ }
83
+ /** The whole review as JSON (`whole()`), written once a revision: every page that loads asks for
84
+ * it, and every page whose review fell behind (GET /api/review), and with thousands of opinions it
85
+ * is megabytes (7.6 MB for 5,000 on Sphere-Packing-Lean: 50 ms each time, a hundred pages opening
86
+ * held the server 5 s). */
87
+ wholeText() {
88
+ if (this.wholeJson?.rev !== this.rev)
89
+ this.wholeJson = { rev: this.rev, text: `{"epoch":${JSON.stringify(this.epoch)},"rev":${this.rev},"data":${this.dataText()},"problems":${JSON.stringify(this.problems)},"error":${JSON.stringify(this.error)}}` };
90
+ return this.wholeJson.text;
91
+ }
92
+ /** The tag of the whole review as it is now (GET /api/review's ETag): its run and its revision,
93
+ * which every change moves. */
94
+ tag() {
95
+ return `"r${this.epoch}.${this.rev}"`;
96
+ }
73
97
  /** The file a writer's events go to (shared/review.ts, logFileName). */
74
98
  fileOf(w) {
75
99
  return w.server ? SERVER_LOG : logFileName(w.name, w.agent, w.person ?? null);
@@ -187,13 +211,17 @@ export class ReviewStore {
187
211
  /** Replay what was read, and say what changed: the new events alone where they all come after
188
212
  * what was replayed before, else the whole review. A log read whole again has events of its own
189
213
  * (new objects), and one gone has none: either way what was replayed is no longer where it was,
190
- * and the review is replayed whole. */
214
+ * and the review is replayed whole. A replay whole that comes to the review it replaced (a new
215
+ * graph's reading that binds every opinion as before, a log rewritten as it was) is told as a
216
+ * change of no events: no page reads the whole review again for it. */
191
217
  commit() {
192
218
  const logs = [...this.files.values()].map((f) => ({ file: f.file, events: f.tail?.event ? [...f.events, f.tail.event] : f.events }));
193
219
  const next = merged(logs).map((x) => this.readAs(x.e));
194
220
  const before = this.order;
195
221
  const appended = !this.again && next.length >= before.length && before.every((e, k) => next[k] === e);
196
222
  this.again = false;
223
+ // what the review was, where it is replayed whole: compared with what the replay makes
224
+ const was = appended ? null : this.dataText();
197
225
  const base = this.rev;
198
226
  this.rev++;
199
227
  this.order = next;
@@ -207,6 +235,8 @@ export class ReviewStore {
207
235
  this.data = emptyReview();
208
236
  for (const e of next)
209
237
  applyEvent(this.data, e);
238
+ if (this.dataText() === was)
239
+ events = [];
210
240
  }
211
241
  const c = { epoch: this.epoch, base, rev: this.rev, events, problems: this.problems };
212
242
  this.onChange?.(c);
@@ -199,7 +199,7 @@ export class ReviewApi {
199
199
  ...Object.values(d.targets).map((t) => ({ by: t.by, agent: !!t.agent, person: !!t.person })),
200
200
  ...Object.values(d.keys).flatMap((marks) => Object.values(marks).map((m) => ({ by: m.by, agent: !!m.agent, person: !!m.person }))),
201
201
  ];
202
- if (writers.some((w) => w.agent && sameName(w.by, name)))
202
+ if (this.world.agentNamed?.(name) || writers.some((w) => w.agent && sameName(w.by, name)))
203
203
  return refuse(400, `${name} is an agent: nobody records an opinion for an agent.`, "Record it for a person, or as your own.");
204
204
  if (held)
205
205
  return { for: held.name, forPerson: held.id };
@@ -11,7 +11,7 @@
11
11
  // changes, not when the mtime does, so "stale" here compares the same hash (`lakeTextHash`).
12
12
  import { execFile } from "node:child_process";
13
13
  import { tracked } from "./children.js";
14
- import { existsSync, readFileSync, statSync } from "node:fs";
14
+ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
15
15
  import { join } from "node:path";
16
16
  const IMPORT_RE = /^\s*(?:public\s+|private\s+)?(?:meta\s+)?import\s+(?:all\s+)?([^\s-]+)/;
17
17
  /** The imports in a file's header: everything up to the first line that is not blank, a
@@ -82,27 +82,88 @@ export function scanModules(p) {
82
82
  }
83
83
  return mods;
84
84
  }
85
- /** Modules a lean process is compiling right now, for this project's build directory. */
85
+ /** Modules a lean process is compiling right now, for this project's build directory: a process
86
+ * `lean <src> -o <libDir>/Foo.olean …` (docs/LAKE.md). Asked every second the server runs. */
86
87
  export function compiling(p) {
88
+ const fromProc = commandsInProc();
89
+ if (fromProc)
90
+ return Promise.resolve(compiledBy(p, fromProc));
87
91
  return new Promise((resolve) => {
88
92
  tracked(execFile("ps", ["axo", "command"], { maxBuffer: 16 << 20 }, (err, stdout) => {
89
- const out = new Set();
90
93
  if (err)
91
- return resolve(out);
92
- const marker = ` -o ${p.libDir}/`;
93
- for (const line of stdout.split("\n")) {
94
- const at = line.indexOf(marker);
95
- if (at < 0)
96
- continue;
97
- const rest = line.slice(at + marker.length);
98
- const path = rest.slice(0, rest.indexOf(".olean"));
99
- if (path)
100
- out.add(path.split("/").join("."));
101
- }
102
- resolve(out);
94
+ return resolve(new Set());
95
+ resolve(compiledBy(p, stdout.split("\n")));
103
96
  }));
104
97
  });
105
98
  }
99
+ /** The modules the command lines `cmds` compile into the project's build directory. */
100
+ function compiledBy(p, cmds) {
101
+ const out = new Set();
102
+ const marker = ` -o ${p.libDir}/`;
103
+ for (const line of cmds) {
104
+ const at = line.indexOf(marker);
105
+ if (at < 0)
106
+ continue;
107
+ const rest = line.slice(at + marker.length);
108
+ const path = rest.slice(0, rest.indexOf(".olean"));
109
+ if (path)
110
+ out.add(path.split("/").join("."));
111
+ }
112
+ return out;
113
+ }
114
+ /** The processes /proc lists, by pid: when each was first seen, and its command line where it
115
+ * could be a compile (one with ` -o `), else "". */
116
+ const procs = new Map();
117
+ /** a process this young is read again: one forked a moment ago may not have run its program yet */
118
+ const YOUNG_MS = 2500;
119
+ /** whether /proc can be read here (Linux): null until tried */
120
+ let procOk = null;
121
+ /** The command lines of the processes running, from /proc, where there is one (Linux): a listing
122
+ * of it, and the command line of each process new since the last look, rather than `ps` every
123
+ * second — which read every process's files, in a process of its own, 1.5 % of a core with 600
124
+ * processes running. Null where there is no /proc. */
125
+ export function commandsInProc(now = Date.now()) {
126
+ if (procOk === false)
127
+ return null;
128
+ let names;
129
+ try {
130
+ names = readdirSync("/proc");
131
+ }
132
+ catch {
133
+ procOk = false;
134
+ return null;
135
+ }
136
+ const alive = new Set();
137
+ const cmds = [];
138
+ for (const n of names) {
139
+ const c = n.charCodeAt(0);
140
+ if (c < 48 || c > 57)
141
+ continue;
142
+ const pid = Number(n);
143
+ alive.add(pid);
144
+ let e = procs.get(pid);
145
+ if (!e || now - e.seen < YOUNG_MS) {
146
+ let cmd = "";
147
+ try {
148
+ // NUL between its arguments, as ps prints them: spaces
149
+ cmd = readFileSync(`/proc/${n}/cmdline`, "utf8").replace(/\0+$/, "").replaceAll("\0", " ");
150
+ }
151
+ catch {
152
+ // gone meanwhile, or not ours to read
153
+ }
154
+ e = { seen: e?.seen ?? now, cmd: cmd.includes(" -o ") ? cmd : "" };
155
+ procs.set(pid, e);
156
+ }
157
+ if (e.cmd)
158
+ cmds.push(e.cmd);
159
+ }
160
+ for (const pid of procs.keys())
161
+ if (!alive.has(pid))
162
+ procs.delete(pid);
163
+ if (procOk === null)
164
+ procOk = alive.size > 0;
165
+ return procOk ? cmds : null;
166
+ }
106
167
  const M = 0xc6a4a7935bd1e995n;
107
168
  const MASK = (1n << 64n) - 1n;
108
169
  /** Lean's `String.hash`: MurmurHash64A over the UTF-8 bytes, seed 11. */
@@ -142,17 +203,19 @@ export function lakeTextHash(text) {
142
203
  return h.toString(16).padStart(16, "0");
143
204
  }
144
205
  const srcHashCache = new Map();
145
- /** The Lake hash of a source file, recomputed only when its mtime or size moves. */
206
+ /** The Lake hash of a source file, recomputed only when its mtime or size moves, and its mtime. */
146
207
  function sourceHash(file) {
147
208
  try {
148
- const st = statSync(file);
209
+ const st = statSync(file, { throwIfNoEntry: false });
210
+ if (!st)
211
+ return null;
149
212
  const key = `${st.mtimeMs}:${st.size}`;
150
213
  const hit = srcHashCache.get(file);
151
214
  if (hit && hit.key === key)
152
- return hit.hash;
215
+ return { hash: hit.hash, at: st.mtimeMs };
153
216
  const hash = lakeTextHash(readFileSync(file, "utf8"));
154
217
  srcHashCache.set(file, { key, hash });
155
- return hash;
218
+ return { hash, at: st.mtimeMs };
156
219
  }
157
220
  catch {
158
221
  return null;
@@ -160,7 +223,7 @@ function sourceHash(file) {
160
223
  }
161
224
  function mtime(f) {
162
225
  try {
163
- return statSync(f).mtimeMs;
226
+ return statSync(f, { throwIfNoEntry: false })?.mtimeMs ?? null;
164
227
  }
165
228
  catch {
166
229
  return null;
@@ -195,7 +258,31 @@ function probe(p, m) {
195
258
  const base = join(p.libDir, ...m.name.split("."));
196
259
  const trace = mtime(base + ".trace");
197
260
  const t = trace !== null ? readTrace(base + ".trace", trace, "/" + m.file) : { warnings: 0, sorry: false, builtFrom: null };
198
- return { olean: mtime(base + ".olean"), trace, source: sourceHash(join(p.root, m.file)), ...t };
261
+ const src = sourceHash(join(p.root, m.file));
262
+ return { olean: mtime(base + ".olean"), trace, source: src?.hash ?? null, sourceAt: src?.at ?? null, ...t };
263
+ }
264
+ /** A look at every module's files: three times each (its outputs', its source's), read once. */
265
+ export function probeAll(p, mods) {
266
+ const out = new Map();
267
+ for (const m of mods.values())
268
+ out.set(m.name, probe(p, m));
269
+ return out;
270
+ }
271
+ /** A fingerprint of what a look found of the outputs and the sources: an edit is not a build, but it
272
+ * changes what the status says. */
273
+ export function stampOf(probes) {
274
+ const parts = [];
275
+ for (const [name, pr] of probes)
276
+ parts.push(`${name}:${pr.olean}:${pr.trace}:${pr.sourceAt}`);
277
+ return parts.join("|");
278
+ }
279
+ /** The same of the outputs alone: what a build wrote, which is all a followed branch's tree is read
280
+ * for (DeploySpec decision 5). */
281
+ export function outputsOf(probes) {
282
+ const parts = [];
283
+ for (const [name, pr] of probes)
284
+ parts.push(`${name}:${pr.olean}:${pr.trace}`);
285
+ return parts.join("|");
199
286
  }
200
287
  /** The Lake hash of the text a module's present build was compiled from, if it has a build: what
201
288
  * the graph says about the module's declarations (their places in the file among them) is about
@@ -207,47 +294,31 @@ export function compiledTextHash(p, module, file) {
207
294
  return null;
208
295
  return readTrace(base + ".trace", trace, "/" + file).builtFrom;
209
296
  }
210
- /** A fingerprint of the build outputs, cheap enough to take every second. */
211
- export function outputStamp(p, mods) {
212
- const parts = [];
213
- for (const m of mods.values()) {
214
- const base = join(p.libDir, ...m.name.split("."));
215
- parts.push(`${m.name}:${mtime(base + ".olean")}:${mtime(base + ".trace")}`);
216
- }
217
- return parts.join("|");
218
- }
219
- /** The same for the sources: an edit is not a build, but it changes what the status says. */
220
- export function sourceStamp(p, mods) {
221
- const parts = [];
222
- for (const m of mods.values())
223
- parts.push(`${m.name}:${mtime(join(p.root, m.file))}`);
224
- return parts.join("|");
225
- }
226
297
  export class BuildTracker {
227
298
  p;
228
299
  /** when outputs were last seen changing, to bridge the gaps between compile processes */
229
300
  lastChange = 0;
230
- lastStamp = "";
301
+ /** each module's outputs as last seen (outputsMoved), and when; null before the first look */
302
+ seen = null;
303
+ seenAt = 0;
231
304
  buildStart = 0;
232
305
  expected = null;
233
306
  constructor(p) {
234
307
  this.p = p;
235
308
  }
236
- async status(mods) {
309
+ /** The status of every module: from a look at their files (`probes`, made now where not given)
310
+ * and the compiles running (`running`, asked now where not given). */
311
+ async status(mods, probes = probeAll(this.p, mods), running) {
237
312
  const p = this.p;
313
+ running ??= await compiling(p);
314
+ // a module the look did not find (one the list gained since): looked at now
315
+ for (const m of mods.values())
316
+ if (!probes.has(m.name))
317
+ probes.set(m.name, probe(p, m));
238
318
  const now = Date.now();
239
- const running = await compiling(p);
240
- // only outputs moving count as a build: an edit to a source is not one
241
- const st = outputStamp(p, mods);
242
- if (st !== this.lastStamp) {
243
- if (this.lastStamp)
244
- this.lastChange = now;
245
- this.lastStamp = st;
246
- }
319
+ if (this.outputsMoved(mods, now, probes))
320
+ this.lastChange = now;
247
321
  const active = running.size > 0 || now - this.lastChange < 2500;
248
- const probes = new Map();
249
- for (const m of mods.values())
250
- probes.set(m.name, probe(p, m));
251
322
  // own state first, then what imports say about it
252
323
  const own = new Map();
253
324
  for (const m of mods.values()) {
@@ -341,6 +412,29 @@ export class BuildTracker {
341
412
  counts[f.state]++;
342
413
  return { active, progress, files, counts };
343
414
  }
415
+ /** Whether the build's outputs moved since the last look: only outputs moving count as a build.
416
+ * An edit to a source is not one, nor a file the project gains or loses while nothing is written
417
+ * (an import added of a file not built yet, one taken out): a module new to the list moved only
418
+ * where what it has was written since the last look. The first look is no change. */
419
+ outputsMoved(mods, now, probes) {
420
+ const was = this.seen;
421
+ const since = this.seenAt;
422
+ const next = new Map();
423
+ let moved = false;
424
+ for (const m of mods.values()) {
425
+ const { olean, trace } = probes.get(m.name);
426
+ const s = `${olean}:${trace}`;
427
+ next.set(m.name, s);
428
+ if (!was)
429
+ continue;
430
+ const before = was.get(m.name);
431
+ if (before === undefined ? Math.max(olean ?? 0, trace ?? 0) > since : before !== s)
432
+ moved = true;
433
+ }
434
+ this.seen = next;
435
+ this.seenAt = now;
436
+ return moved;
437
+ }
344
438
  }
345
439
  /** The modules that can be imported right now: built, with everything under them built. */
346
440
  export function importable(p, mods) {
@@ -147,9 +147,13 @@ export class Talk {
147
147
  }
148
148
  /** The agent said it leaves (POST /api/agent/view {leave: true}; its MCP server does as it
149
149
  * stops): offline at once, its wait ended, until it waits or asks again; its next wait is a new
150
- * session's. */
151
- left(token, now = Date.now()) {
150
+ * session's. A leave from another session than the one that waited last (`session`, as
151
+ * X-VisualProof-Session says it) is one a newer session replaced, whose client goes now: it
152
+ * leaves nothing, the newer one listening on (false). */
153
+ left(token, session = null, now = Date.now()) {
152
154
  const a = this.agent(token);
155
+ if (session !== null && a.session !== null && a.session !== session)
156
+ return false;
153
157
  a.wait?.end({ gone: true });
154
158
  a.lastRequest = 0;
155
159
  a.graceUntil = 0;
@@ -158,6 +162,7 @@ export class Talk {
158
162
  a.away = true;
159
163
  a.before = undefined;
160
164
  this.update(token, now);
165
+ return true;
161
166
  }
162
167
  /** A request of the agent's: it is not offline, and `doing` (when it says something) is what it
163
168
  * does now. */
@@ -21,6 +21,7 @@ import { spawn } from "node:child_process";
21
21
  import { childEnv, DETACH, grouped } from "./children.js";
22
22
  import { createHash } from "node:crypto";
23
23
  import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
24
+ import { freemem, totalmem } from "node:os";
24
25
  import { join } from "node:path";
25
26
  import { SITE_BODY, SITE_GRAMMAR, SITE_PROOF, SITE_TCB } from "../shared/graph.js";
26
27
  import { USAGE, USAGE_PROGRAM, leanProgram, prune } from "./extract.js";
@@ -31,6 +32,43 @@ import { compiledTextHash, lakeTextHash } from "./status.js";
31
32
  export const DEFAULT_JOBS = 8;
32
33
  /** A worker still running after this long is stopped, and its module reported as failed. */
33
34
  const WORKER_TIMEOUT_MS = 10 * 60_000;
35
+ const GB = 2 ** 30;
36
+ /** What a worker may hold at most, the `.olean` files it maps counted, on a large project on
37
+ * Mathlib: the showcase's (NavierStokesAndEuler, 2,657 modules) at 6 jobs peaked at 39 GB resident,
38
+ * 15 GB of it shared. Off Mathlib, a worker holds far less. */
39
+ export const WORKER_PEAK = 6 * GB;
40
+ /** What the machine keeps for itself and the rest while the pass runs: with less available, no new
41
+ * worker starts while another runs (`roomForWorker`). */
42
+ export function memoryFloor(total = totalmem()) {
43
+ return Math.max(2 * GB, total / 20);
44
+ }
45
+ /** Whether one more worker may start, `running` running: the first always; another while the
46
+ * machine has more than the floor available (os.freemem: MemAvailable on Linux). Workers come and
47
+ * go a module at a time, so that the pass holds back to what the memory allows. */
48
+ export function roomForWorker(running, available = freemem(), floor = memoryFloor()) {
49
+ return running === 0 || available >= floor;
50
+ }
51
+ /** Whether a project is on Mathlib, as its lakefile or manifest says: whose workers each map
52
+ * gigabytes. */
53
+ export function onMathlib(root) {
54
+ return ["lakefile.toml", "lakefile.lean", "lake-manifest.json"].some((f) => {
55
+ try {
56
+ return /mathlib/i.test(readFileSync(join(root, f), "utf8"));
57
+ }
58
+ catch {
59
+ return false;
60
+ }
61
+ });
62
+ }
63
+ /** What `jobs` workers may hold on a project on Mathlib, against `available` bytes: null where it
64
+ * fits (or off Mathlib), else what they may need and how many fit. */
65
+ export function jobsAdvice(jobs, mathlib, available = freemem()) {
66
+ if (!mathlib || jobs <= 1 || jobs * WORKER_PEAK <= available)
67
+ return null;
68
+ return { need: jobs * WORKER_PEAK, fits: Math.max(1, Math.floor(available / WORKER_PEAK)) };
69
+ }
70
+ /** Bytes in GB, in words. */
71
+ export const inGB = (bytes) => `${(bytes / GB).toFixed(bytes < 10 * GB ? 1 : 0)} GB`;
34
72
  /** Mixed into the fingerprint of a merged graph: bumped when the merge changes, so that graphs
35
73
  * merged before are not served as if made by this one. 3: the units' proof spans. (A change to
36
74
  * extractor/Usage.lean changes every module's key by itself: its text is in the salt.) */
@@ -93,6 +131,8 @@ export class UsagePass {
93
131
  status;
94
132
  /** the run going on, to stop when a newer build supersedes it */
95
133
  current = null;
134
+ /** the machine's memory available, as the pass asks it before each worker (a test's own) */
135
+ available = freemem;
96
136
  constructor(project, dirs, jobs, log) {
97
137
  this.project = project;
98
138
  this.dir = join(dirs.cache, "usage");
@@ -236,8 +276,19 @@ export class UsagePass {
236
276
  const modsFile = join(this.dir, "project-modules.txt");
237
277
  writeFileSync(modsFile, plan.modules.join("\n") + "\n");
238
278
  const queue = plan.todo.slice();
279
+ const floor = memoryFloor();
280
+ let held = false;
239
281
  const worker = async () => {
240
282
  while (queue.length && !run.cancelled) {
283
+ // memory short, another worker running: this one waits for it (said once a pass)
284
+ while (!roomForWorker(run.children.size, this.available(), floor) && !run.cancelled) {
285
+ if (!held)
286
+ this.log(`usage pass: ${inGB(this.available())} of memory available, less than ${inGB(floor)}: fewer workers at a time until there is more`);
287
+ held = true;
288
+ await new Promise((ok) => setTimeout(ok, 250));
289
+ }
290
+ if (!queue.length || run.cancelled)
291
+ return;
241
292
  const t = queue.shift();
242
293
  const e = await this.one(run, bin, env, modsFile, t);
243
294
  if (run.cancelled)