visualproof 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +16 -7
  2. package/dist/assets/tips/tour-agent.webp +0 -0
  3. package/dist/assets/tips/tour-arrange.webp +0 -0
  4. package/dist/assets/tips/tour-artwork.webp +0 -0
  5. package/dist/assets/tips/tour-checks.webp +0 -0
  6. package/dist/assets/tips/tour-keys.webp +0 -0
  7. package/dist/assets/tips/tour-library.webp +0 -0
  8. package/dist/assets/tips/tour-project.webp +0 -0
  9. package/dist/assets/tips/tour-review.webp +0 -0
  10. package/dist/assets/tips/tour-share.webp +0 -0
  11. package/dist/assets/tips/tour-source.webp +0 -0
  12. package/dist/assets/tips/tour-target.webp +0 -0
  13. package/dist/assets/tips/tour-view.webp +0 -0
  14. package/dist/assets/tips/tour.json +117 -112
  15. package/dist/assets/tips/tour.webp +0 -0
  16. package/dist/server/access.js +3 -1
  17. package/dist/server/accesspage.js +130 -0
  18. package/dist/server/agent.js +6 -1
  19. package/dist/server/buildwatch.js +133 -0
  20. package/dist/server/deploy.js +844 -0
  21. package/dist/server/doctor.js +15 -3
  22. package/dist/server/film.js +1 -1
  23. package/dist/server/historyapi.js +100 -50
  24. package/dist/server/historythread.js +206 -0
  25. package/dist/server/hover.js +8 -0
  26. package/dist/server/live.js +187 -36
  27. package/dist/server/main.js +256 -30
  28. package/dist/server/review.js +27 -7
  29. package/dist/server/status.js +119 -46
  30. package/dist/server/usage.js +51 -0
  31. package/dist/shared/board.js +2 -1
  32. package/dist/shared/deploy.js +35 -0
  33. package/dist/shared/layout.js +297 -70
  34. package/dist/shared/libm.js +366 -0
  35. package/dist/shared/palette.js +1 -1
  36. package/dist/shared/separate.js +556 -32
  37. package/dist/web/bundle/app.js +1972 -826
  38. package/dist/web/bundle/edges.js +2 -1
  39. package/dist/web/bundle/layout.js +1026 -80
  40. package/dist/web/src/canvas.js +47 -34
  41. package/dist/web/src/overview.js +4 -1
  42. package/dist/web/src/scene.js +13 -12
  43. package/dist/web/src/status.js +28 -0
  44. package/dist/web/src/theme.js +108 -0
  45. package/dist/web/style.css +102 -35
  46. package/docs/agent-api/history.md +3 -0
  47. package/docs/agent-api/requests.md +4 -3
  48. package/package.json +1 -1
@@ -11,7 +11,7 @@
11
11
  // changes, not when the mtime does, so "stale" here compares the same hash (`lakeTextHash`).
12
12
  import { execFile } from "node:child_process";
13
13
  import { tracked } from "./children.js";
14
- import { existsSync, readFileSync, statSync } from "node:fs";
14
+ import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
15
15
  import { join } from "node:path";
16
16
  const IMPORT_RE = /^\s*(?:public\s+|private\s+)?(?:meta\s+)?import\s+(?:all\s+)?([^\s-]+)/;
17
17
  /** The imports in a file's header: everything up to the first line that is not blank, a
@@ -82,27 +82,88 @@ export function scanModules(p) {
82
82
  }
83
83
  return mods;
84
84
  }
85
- /** Modules a lean process is compiling right now, for this project's build directory. */
85
+ /** Modules a lean process is compiling right now, for this project's build directory: a process
86
+ * `lean <src> -o <libDir>/Foo.olean …` (docs/LAKE.md). Asked every second the server runs. */
86
87
  export function compiling(p) {
88
+ const fromProc = commandsInProc();
89
+ if (fromProc)
90
+ return Promise.resolve(compiledBy(p, fromProc));
87
91
  return new Promise((resolve) => {
88
92
  tracked(execFile("ps", ["axo", "command"], { maxBuffer: 16 << 20 }, (err, stdout) => {
89
- const out = new Set();
90
93
  if (err)
91
- return resolve(out);
92
- const marker = ` -o ${p.libDir}/`;
93
- for (const line of stdout.split("\n")) {
94
- const at = line.indexOf(marker);
95
- if (at < 0)
96
- continue;
97
- const rest = line.slice(at + marker.length);
98
- const path = rest.slice(0, rest.indexOf(".olean"));
99
- if (path)
100
- out.add(path.split("/").join("."));
101
- }
102
- resolve(out);
94
+ return resolve(new Set());
95
+ resolve(compiledBy(p, stdout.split("\n")));
103
96
  }));
104
97
  });
105
98
  }
99
+ /** The modules the command lines `cmds` compile into the project's build directory. */
100
+ function compiledBy(p, cmds) {
101
+ const out = new Set();
102
+ const marker = ` -o ${p.libDir}/`;
103
+ for (const line of cmds) {
104
+ const at = line.indexOf(marker);
105
+ if (at < 0)
106
+ continue;
107
+ const rest = line.slice(at + marker.length);
108
+ const path = rest.slice(0, rest.indexOf(".olean"));
109
+ if (path)
110
+ out.add(path.split("/").join("."));
111
+ }
112
+ return out;
113
+ }
114
+ /** The processes /proc lists, by pid: when each was first seen, and its command line where it
115
+ * could be a compile (one with ` -o `), else "". */
116
+ const procs = new Map();
117
+ /** a process this young is read again: one forked a moment ago may not have run its program yet */
118
+ const YOUNG_MS = 2500;
119
+ /** whether /proc can be read here (Linux): null until tried */
120
+ let procOk = null;
121
+ /** The command lines of the processes running, from /proc, where there is one (Linux): a listing
122
+ * of it, and the command line of each process new since the last look, rather than `ps` every
123
+ * second — which read every process's files, in a process of its own, 1.5 % of a core with 600
124
+ * processes running. Null where there is no /proc. */
125
+ export function commandsInProc(now = Date.now()) {
126
+ if (procOk === false)
127
+ return null;
128
+ let names;
129
+ try {
130
+ names = readdirSync("/proc");
131
+ }
132
+ catch {
133
+ procOk = false;
134
+ return null;
135
+ }
136
+ const alive = new Set();
137
+ const cmds = [];
138
+ for (const n of names) {
139
+ const c = n.charCodeAt(0);
140
+ if (c < 48 || c > 57)
141
+ continue;
142
+ const pid = Number(n);
143
+ alive.add(pid);
144
+ let e = procs.get(pid);
145
+ if (!e || now - e.seen < YOUNG_MS) {
146
+ let cmd = "";
147
+ try {
148
+ // NUL between its arguments, as ps prints them: spaces
149
+ cmd = readFileSync(`/proc/${n}/cmdline`, "utf8").replace(/\0+$/, "").replaceAll("\0", " ");
150
+ }
151
+ catch {
152
+ // gone meanwhile, or not ours to read
153
+ }
154
+ e = { seen: e?.seen ?? now, cmd: cmd.includes(" -o ") ? cmd : "" };
155
+ procs.set(pid, e);
156
+ }
157
+ if (e.cmd)
158
+ cmds.push(e.cmd);
159
+ }
160
+ for (const pid of procs.keys())
161
+ if (!alive.has(pid))
162
+ procs.delete(pid);
163
+ if (procOk === null)
164
+ procOk = alive.size > 0;
165
+ return procOk ? cmds : null;
166
+ }
106
167
  const M = 0xc6a4a7935bd1e995n;
107
168
  const MASK = (1n << 64n) - 1n;
108
169
  /** Lean's `String.hash`: MurmurHash64A over the UTF-8 bytes, seed 11. */
@@ -142,17 +203,19 @@ export function lakeTextHash(text) {
142
203
  return h.toString(16).padStart(16, "0");
143
204
  }
144
205
  const srcHashCache = new Map();
145
- /** The Lake hash of a source file, recomputed only when its mtime or size moves. */
206
+ /** The Lake hash of a source file, recomputed only when its mtime or size moves, and its mtime. */
146
207
  function sourceHash(file) {
147
208
  try {
148
- const st = statSync(file);
209
+ const st = statSync(file, { throwIfNoEntry: false });
210
+ if (!st)
211
+ return null;
149
212
  const key = `${st.mtimeMs}:${st.size}`;
150
213
  const hit = srcHashCache.get(file);
151
214
  if (hit && hit.key === key)
152
- return hit.hash;
215
+ return { hash: hit.hash, at: st.mtimeMs };
153
216
  const hash = lakeTextHash(readFileSync(file, "utf8"));
154
217
  srcHashCache.set(file, { key, hash });
155
- return hash;
218
+ return { hash, at: st.mtimeMs };
156
219
  }
157
220
  catch {
158
221
  return null;
@@ -160,7 +223,7 @@ function sourceHash(file) {
160
223
  }
161
224
  function mtime(f) {
162
225
  try {
163
- return statSync(f).mtimeMs;
226
+ return statSync(f, { throwIfNoEntry: false })?.mtimeMs ?? null;
164
227
  }
165
228
  catch {
166
229
  return null;
@@ -195,7 +258,31 @@ function probe(p, m) {
195
258
  const base = join(p.libDir, ...m.name.split("."));
196
259
  const trace = mtime(base + ".trace");
197
260
  const t = trace !== null ? readTrace(base + ".trace", trace, "/" + m.file) : { warnings: 0, sorry: false, builtFrom: null };
198
- return { olean: mtime(base + ".olean"), trace, source: sourceHash(join(p.root, m.file)), ...t };
261
+ const src = sourceHash(join(p.root, m.file));
262
+ return { olean: mtime(base + ".olean"), trace, source: src?.hash ?? null, sourceAt: src?.at ?? null, ...t };
263
+ }
264
+ /** A look at every module's files: three times each (its outputs', its source's), read once. */
265
+ export function probeAll(p, mods) {
266
+ const out = new Map();
267
+ for (const m of mods.values())
268
+ out.set(m.name, probe(p, m));
269
+ return out;
270
+ }
271
+ /** A fingerprint of what a look found of the outputs and the sources: an edit is not a build, but it
272
+ * changes what the status says. */
273
+ export function stampOf(probes) {
274
+ const parts = [];
275
+ for (const [name, pr] of probes)
276
+ parts.push(`${name}:${pr.olean}:${pr.trace}:${pr.sourceAt}`);
277
+ return parts.join("|");
278
+ }
279
+ /** The same of the outputs alone: what a build wrote, which is all a followed branch's tree is read
280
+ * for (DeploySpec decision 5). */
281
+ export function outputsOf(probes) {
282
+ const parts = [];
283
+ for (const [name, pr] of probes)
284
+ parts.push(`${name}:${pr.olean}:${pr.trace}`);
285
+ return parts.join("|");
199
286
  }
200
287
  /** The Lake hash of the text a module's present build was compiled from, if it has a build: what
201
288
  * the graph says about the module's declarations (their places in the file among them) is about
@@ -207,22 +294,6 @@ export function compiledTextHash(p, module, file) {
207
294
  return null;
208
295
  return readTrace(base + ".trace", trace, "/" + file).builtFrom;
209
296
  }
210
- /** A fingerprint of the build outputs, cheap enough to take every second. */
211
- export function outputStamp(p, mods) {
212
- const parts = [];
213
- for (const m of mods.values()) {
214
- const base = join(p.libDir, ...m.name.split("."));
215
- parts.push(`${m.name}:${mtime(base + ".olean")}:${mtime(base + ".trace")}`);
216
- }
217
- return parts.join("|");
218
- }
219
- /** The same for the sources: an edit is not a build, but it changes what the status says. */
220
- export function sourceStamp(p, mods) {
221
- const parts = [];
222
- for (const m of mods.values())
223
- parts.push(`${m.name}:${mtime(join(p.root, m.file))}`);
224
- return parts.join("|");
225
- }
226
297
  export class BuildTracker {
227
298
  p;
228
299
  /** when outputs were last seen changing, to bridge the gaps between compile processes */
@@ -235,16 +306,19 @@ export class BuildTracker {
235
306
  constructor(p) {
236
307
  this.p = p;
237
308
  }
238
- async status(mods) {
309
+ /** The status of every module: from a look at their files (`probes`, made now where not given)
310
+ * and the compiles running (`running`, asked now where not given). */
311
+ async status(mods, probes = probeAll(this.p, mods), running) {
239
312
  const p = this.p;
313
+ running ??= await compiling(p);
314
+ // a module the look did not find (one the list gained since): looked at now
315
+ for (const m of mods.values())
316
+ if (!probes.has(m.name))
317
+ probes.set(m.name, probe(p, m));
240
318
  const now = Date.now();
241
- const running = await compiling(p);
242
- if (this.outputsMoved(mods, now))
319
+ if (this.outputsMoved(mods, now, probes))
243
320
  this.lastChange = now;
244
321
  const active = running.size > 0 || now - this.lastChange < 2500;
245
- const probes = new Map();
246
- for (const m of mods.values())
247
- probes.set(m.name, probe(p, m));
248
322
  // own state first, then what imports say about it
249
323
  const own = new Map();
250
324
  for (const m of mods.values()) {
@@ -342,14 +416,13 @@ export class BuildTracker {
342
416
  * An edit to a source is not one, nor a file the project gains or loses while nothing is written
343
417
  * (an import added of a file not built yet, one taken out): a module new to the list moved only
344
418
  * where what it has was written since the last look. The first look is no change. */
345
- outputsMoved(mods, now) {
419
+ outputsMoved(mods, now, probes) {
346
420
  const was = this.seen;
347
421
  const since = this.seenAt;
348
422
  const next = new Map();
349
423
  let moved = false;
350
424
  for (const m of mods.values()) {
351
- const base = join(this.p.libDir, ...m.name.split("."));
352
- const olean = mtime(base + ".olean"), trace = mtime(base + ".trace");
425
+ const { olean, trace } = probes.get(m.name);
353
426
  const s = `${olean}:${trace}`;
354
427
  next.set(m.name, s);
355
428
  if (!was)
@@ -21,6 +21,7 @@ import { spawn } from "node:child_process";
21
21
  import { childEnv, DETACH, grouped } from "./children.js";
22
22
  import { createHash } from "node:crypto";
23
23
  import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
24
+ import { freemem, totalmem } from "node:os";
24
25
  import { join } from "node:path";
25
26
  import { SITE_BODY, SITE_GRAMMAR, SITE_PROOF, SITE_TCB } from "../shared/graph.js";
26
27
  import { USAGE, USAGE_PROGRAM, leanProgram, prune } from "./extract.js";
@@ -31,6 +32,43 @@ import { compiledTextHash, lakeTextHash } from "./status.js";
31
32
  export const DEFAULT_JOBS = 8;
32
33
  /** A worker still running after this long is stopped, and its module reported as failed. */
33
34
  const WORKER_TIMEOUT_MS = 10 * 60_000;
35
+ const GB = 2 ** 30;
36
+ /** What a worker may hold at most, the `.olean` files it maps counted, on a large project on
37
+ * Mathlib: the showcase's (NavierStokesAndEuler, 2,657 modules) at 6 jobs peaked at 39 GB resident,
38
+ * 15 GB of it shared. Off Mathlib, a worker holds far less. */
39
+ export const WORKER_PEAK = 6 * GB;
40
+ /** What the machine keeps for itself and the rest while the pass runs: with less available, no new
41
+ * worker starts while another runs (`roomForWorker`). */
42
+ export function memoryFloor(total = totalmem()) {
43
+ return Math.max(2 * GB, total / 20);
44
+ }
45
+ /** Whether one more worker may start, `running` running: the first always; another while the
46
+ * machine has more than the floor available (os.freemem: MemAvailable on Linux). Workers come and
47
+ * go a module at a time, so that the pass holds back to what the memory allows. */
48
+ export function roomForWorker(running, available = freemem(), floor = memoryFloor()) {
49
+ return running === 0 || available >= floor;
50
+ }
51
+ /** Whether a project is on Mathlib, as its lakefile or manifest says: whose workers each map
52
+ * gigabytes. */
53
+ export function onMathlib(root) {
54
+ return ["lakefile.toml", "lakefile.lean", "lake-manifest.json"].some((f) => {
55
+ try {
56
+ return /mathlib/i.test(readFileSync(join(root, f), "utf8"));
57
+ }
58
+ catch {
59
+ return false;
60
+ }
61
+ });
62
+ }
63
+ /** What `jobs` workers may hold on a project on Mathlib, against `available` bytes: null where it
64
+ * fits (or off Mathlib), else what they may need and how many fit. */
65
+ export function jobsAdvice(jobs, mathlib, available = freemem()) {
66
+ if (!mathlib || jobs <= 1 || jobs * WORKER_PEAK <= available)
67
+ return null;
68
+ return { need: jobs * WORKER_PEAK, fits: Math.max(1, Math.floor(available / WORKER_PEAK)) };
69
+ }
70
+ /** Bytes in GB, in words. */
71
+ export const inGB = (bytes) => `${(bytes / GB).toFixed(bytes < 10 * GB ? 1 : 0)} GB`;
34
72
  /** Mixed into the fingerprint of a merged graph: bumped when the merge changes, so that graphs
35
73
  * merged before are not served as if made by this one. 3: the units' proof spans. (A change to
36
74
  * extractor/Usage.lean changes every module's key by itself: its text is in the salt.) */
@@ -93,6 +131,8 @@ export class UsagePass {
93
131
  status;
94
132
  /** the run going on, to stop when a newer build supersedes it */
95
133
  current = null;
134
+ /** the machine's memory available, as the pass asks it before each worker (a test's own) */
135
+ available = freemem;
96
136
  constructor(project, dirs, jobs, log) {
97
137
  this.project = project;
98
138
  this.dir = join(dirs.cache, "usage");
@@ -236,8 +276,19 @@ export class UsagePass {
236
276
  const modsFile = join(this.dir, "project-modules.txt");
237
277
  writeFileSync(modsFile, plan.modules.join("\n") + "\n");
238
278
  const queue = plan.todo.slice();
279
+ const floor = memoryFloor();
280
+ let held = false;
239
281
  const worker = async () => {
240
282
  while (queue.length && !run.cancelled) {
283
+ // memory short, another worker running: this one waits for it (said once a pass)
284
+ while (!roomForWorker(run.children.size, this.available(), floor) && !run.cancelled) {
285
+ if (!held)
286
+ this.log(`usage pass: ${inGB(this.available())} of memory available, less than ${inGB(floor)}: fewer workers at a time until there is more`);
287
+ held = true;
288
+ await new Promise((ok) => setTimeout(ok, 250));
289
+ }
290
+ if (!queue.length || run.cancelled)
291
+ return;
241
292
  const t = queue.shift();
242
293
  const e = await this.one(run, bin, env, modsFile, t);
243
294
  if (run.cancelled)
@@ -28,6 +28,7 @@
28
28
  // definition on a board of a target's review set — is still a path. So what is lit is exactly what a view of it holds
29
29
  // (`lightUp`, `zoomScope`), and Zoom in shows what was lit.
30
30
  import { clusterTree, computeLayout, memoryOf, regroup, stabilize, withLocal } from "./layout.js";
31
+ import { hypot } from "./libm.js";
31
32
  import { evaluate, within } from "./query.js";
32
33
  import { Graph } from "./viewgraph.js";
33
34
  /** The declarations `set` holds, as a graph of their own (edges among them only). */
@@ -272,7 +273,7 @@ export function reframePins(g, before, after, pins, mode) {
272
273
  return w;
273
274
  };
274
275
  const r = (v) => Math.round(v * 10) / 10;
275
- const away = (x1, y1, x2, y2) => Math.hypot(x1 - x2, y1 - y2) > 0.5;
276
+ const away = (x1, y1, x2, y2) => hypot(x1 - x2, y1 - y2) > 0.5;
276
277
  let extra = 0;
277
278
  // the same after, parents first, with the pins as they are made
278
279
  const wa = [];
@@ -0,0 +1,35 @@
1
+ // A branch of a git repository the server follows and builds itself (SPEC/DeploySpec.md), as the
2
+ // pages and agents are told it: where it comes from, the build going on, the newest commit waiting
3
+ // for it, the last build that ended; and the links into GitHub a page makes of it.
4
+ /** The same without what is for those let in fully: the builds' logs. */
5
+ export function slimDeploy(d) {
6
+ const drop = (x) => {
7
+ if (!x || x.tail === undefined)
8
+ return x;
9
+ const { tail: _, ...rest } = x;
10
+ return rest;
11
+ };
12
+ return { ...d, building: drop(d.building), last: drop(d.last) };
13
+ }
14
+ /** A commit as a page writes it. */
15
+ export const shortCommit = (c) => c.slice(0, 7);
16
+ /** What a step is doing, in words. */
17
+ export const STEP_WORDS = {
18
+ fetch: "fetching it",
19
+ checkout: "checking it out",
20
+ cache: "getting Mathlib's cache",
21
+ build: "lake build",
22
+ };
23
+ const enc = (p) => p.split("/").map(encodeURIComponent).join("/");
24
+ /** A file of the repository at a commit on GitHub, lines `from` to `to` (from 1) marked. */
25
+ export function githubBlob(repo, commit, file, from, to) {
26
+ return `https://github.com/${repo}/blob/${commit}/${enc(file)}#L${from}${to > from ? `-L${to}` : ""}`;
27
+ }
28
+ /** A commit on GitHub. */
29
+ export function githubCommit(repo, commit) {
30
+ return `https://github.com/${repo}/commit/${commit}`;
31
+ }
32
+ /** A branch on GitHub. */
33
+ export function githubTree(repo, branch) {
34
+ return `https://github.com/${repo}/tree/${enc(branch)}`;
35
+ }