visualproof 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -7
- package/dist/assets/latex/visualproof.sty +19 -9
- package/dist/assets/tips/tour-agent.webp +0 -0
- package/dist/assets/tips/tour-arrange.webp +0 -0
- package/dist/assets/tips/tour-artwork.webp +0 -0
- package/dist/assets/tips/tour-checks.webp +0 -0
- package/dist/assets/tips/tour-keys.webp +0 -0
- package/dist/assets/tips/tour-library.webp +0 -0
- package/dist/assets/tips/tour-project.webp +0 -0
- package/dist/assets/tips/tour-review.webp +0 -0
- package/dist/assets/tips/tour-share.webp +0 -0
- package/dist/assets/tips/tour-source.webp +0 -0
- package/dist/assets/tips/tour-target.webp +0 -0
- package/dist/assets/tips/tour-view.webp +0 -0
- package/dist/assets/tips/tour.json +117 -112
- package/dist/assets/tips/tour.webp +0 -0
- package/dist/mcp/visualproof-mcp.mjs +7 -3
- package/dist/server/access.js +37 -1
- package/dist/server/accesspage.js +130 -0
- package/dist/server/agent.js +11 -5
- package/dist/server/agentlibrary.js +1 -1
- package/dist/server/agentviews.js +1 -1
- package/dist/server/artcmd.js +1 -1
- package/dist/server/askfollow.js +1 -4
- package/dist/server/buildwatch.js +133 -0
- package/dist/server/deploy.js +844 -0
- package/dist/server/doctor.js +15 -3
- package/dist/server/events.js +20 -5
- package/dist/server/export.js +17 -4
- package/dist/server/fetchurl.js +30 -13
- package/dist/server/film.js +1 -1
- package/dist/server/history.js +3 -1
- package/dist/server/historyapi.js +100 -50
- package/dist/server/historythread.js +206 -0
- package/dist/server/hover.js +8 -0
- package/dist/server/identity.js +25 -1
- package/dist/server/library.js +10 -3
- package/dist/server/live.js +206 -28
- package/dist/server/lostfound.js +3 -2
- package/dist/server/main.js +361 -50
- package/dist/server/poppler.js +48 -10
- package/dist/server/renames.js +10 -5
- package/dist/server/review.js +32 -2
- package/dist/server/reviewapi.js +1 -1
- package/dist/server/status.js +144 -50
- package/dist/server/talk.js +7 -2
- package/dist/server/usage.js +51 -0
- package/dist/server/views.js +21 -7
- package/dist/shared/board.js +2 -1
- package/dist/shared/collab.js +14 -8
- package/dist/shared/deploy.js +35 -0
- package/dist/shared/doc/latex.js +5 -3
- package/dist/shared/doc/outline.js +6 -3
- package/dist/shared/layout.js +297 -70
- package/dist/shared/libm.js +366 -0
- package/dist/shared/library.js +11 -6
- package/dist/shared/palette.js +1 -1
- package/dist/shared/review.js +7 -5
- package/dist/shared/search.js +7 -3
- package/dist/shared/separate.js +556 -32
- package/dist/shared/views.js +8 -1
- package/dist/web/bundle/app.js +3595 -1520
- package/dist/web/bundle/edges.js +44 -16
- package/dist/web/bundle/layout.js +1026 -80
- package/dist/web/index.html +2 -2
- package/dist/web/src/artwork.js +25 -4
- package/dist/web/src/axiomring.js +42 -0
- package/dist/web/src/bundles.js +52 -16
- package/dist/web/src/canvas.js +109 -54
- package/dist/web/src/overview.js +4 -1
- package/dist/web/src/past.js +28 -0
- package/dist/web/src/scene.js +13 -12
- package/dist/web/src/selection.js +47 -4
- package/dist/web/src/status.js +32 -2
- package/dist/web/src/theme.js +108 -0
- package/dist/web/style.css +181 -45
- package/docs/agent-api/following.md +1 -1
- package/docs/agent-api/history.md +3 -0
- package/docs/agent-api/library.md +1 -1
- package/docs/agent-api/requests.md +4 -1
- package/docs/agent-api/views.md +1 -1
- package/package.json +1 -1
package/dist/server/poppler.js
CHANGED
|
@@ -11,10 +11,10 @@
|
|
|
11
11
|
// most two at once.
|
|
12
12
|
import { execFile } from "node:child_process";
|
|
13
13
|
import { tracked } from "./children.js";
|
|
14
|
-
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
14
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
17
|
-
import { wordsText } from "../shared/library.js";
|
|
17
|
+
import { plainPunctuation, wordsText } from "../shared/library.js";
|
|
18
18
|
/** the longest each tool may run, in ms */
|
|
19
19
|
const INFO_MS = 20_000;
|
|
20
20
|
const TEXT_MS = 120_000;
|
|
@@ -111,16 +111,17 @@ export function parseBboxLayout(xhtml) {
|
|
|
111
111
|
export function pageText(p) {
|
|
112
112
|
return wordsText(p.words, p.lines);
|
|
113
113
|
}
|
|
114
|
-
/** Find `q` in every page's text: case
|
|
115
|
-
* matches, in page order. */
|
|
114
|
+
/** Find `q` in every page's text: case, runs of white space and typographic quotes and dashes read
|
|
115
|
+
* loosely; at most `most` matches, in page order. */
|
|
116
116
|
export function searchWords(pages, q, most = 100) {
|
|
117
|
-
|
|
117
|
+
// typed as a keyboard types it: ' finds ’, - finds – (shared/library.ts, plainPunctuation)
|
|
118
|
+
const want = plainPunctuation(q.replace(/\s+/g, " ").trim().toLowerCase());
|
|
118
119
|
const matches = [];
|
|
119
120
|
if (!want)
|
|
120
121
|
return { matches, more: false };
|
|
121
122
|
for (const p of pages) {
|
|
122
123
|
const { text, starts } = pageText(p);
|
|
123
|
-
const flat = text.replace(/\n/g, " ").toLowerCase();
|
|
124
|
+
const flat = plainPunctuation(text.replace(/\n/g, " ").toLowerCase());
|
|
124
125
|
for (let at = flat.indexOf(want); at >= 0; at = flat.indexOf(want, at + 1)) {
|
|
125
126
|
if (matches.length >= most)
|
|
126
127
|
return { matches, more: true };
|
|
@@ -150,13 +151,20 @@ function upper(starts, at) {
|
|
|
150
151
|
/** The widths a page or a clip is drawn at: a few, so that what is kept stays small. */
|
|
151
152
|
const WIDTHS = [400, 800, 1200, 1600, 2000];
|
|
152
153
|
export const clampWidth = (w) => WIDTHS.find((x) => x >= w) ?? WIDTHS[WIDTHS.length - 1];
|
|
154
|
+
/** The most clips kept of a file, the oldest going first. A page is kept at a few widths, bounded by
|
|
155
|
+
* the file; but anyone who reads — a read-only reader, whoever comes to a `--public` site — may ask
|
|
156
|
+
* for any region of any page, and each was kept for good, filling the disk. The clips notes carry
|
|
157
|
+
* are asked again and again, and stay. */
|
|
158
|
+
export const MAX_CLIPS = 500;
|
|
153
159
|
/** A PDF's words, pages and images, kept in `cacheDir/<sha>/`. */
|
|
154
160
|
export class PdfReader {
|
|
155
161
|
cacheDir;
|
|
156
162
|
words = new Map();
|
|
157
163
|
infos = new Map();
|
|
158
|
-
|
|
164
|
+
maxClips;
|
|
165
|
+
constructor(cacheDir, maxClips = MAX_CLIPS) {
|
|
159
166
|
this.cacheDir = cacheDir;
|
|
167
|
+
this.maxClips = maxClips;
|
|
160
168
|
}
|
|
161
169
|
dir(sha) {
|
|
162
170
|
const d = join(this.cacheDir, sha);
|
|
@@ -220,8 +228,17 @@ export class PdfReader {
|
|
|
220
228
|
const w = clampWidth(width);
|
|
221
229
|
const r = region.map((x) => Math.round(x * 1e4) / 1e4);
|
|
222
230
|
const kept = join(this.dir(sha), `clip-${page}-${r.join("_")}-w${w}.png`);
|
|
223
|
-
if (existsSync(kept))
|
|
224
|
-
|
|
231
|
+
if (existsSync(kept)) {
|
|
232
|
+
try {
|
|
233
|
+
// asked for again: the last to go (atime is not kept on every system)
|
|
234
|
+
const now = new Date();
|
|
235
|
+
utimesSync(kept, now, now);
|
|
236
|
+
return readFileSync(kept);
|
|
237
|
+
}
|
|
238
|
+
catch {
|
|
239
|
+
// pruned meanwhile: drawn again
|
|
240
|
+
}
|
|
241
|
+
}
|
|
225
242
|
// the resolution at which the region is `w` pixels wide, within reason
|
|
226
243
|
const dpi = Math.min(600, Math.max(36, (w / ((r[2] - r[0]) * size.w)) * 72));
|
|
227
244
|
const px = (v, of) => Math.round((v * of * dpi) / 72);
|
|
@@ -235,7 +252,28 @@ export class PdfReader {
|
|
|
235
252
|
finally {
|
|
236
253
|
rmSync(`${tmp}.png`, { force: true });
|
|
237
254
|
}
|
|
238
|
-
|
|
255
|
+
const out = readFileSync(kept);
|
|
256
|
+
this.pruneClips(sha);
|
|
257
|
+
return out;
|
|
258
|
+
}
|
|
259
|
+
/** The clips kept of a file beyond the most there may be go, those drawn or asked for longest
|
|
260
|
+
* ago first (a clip asked for again is touched). */
|
|
261
|
+
pruneClips(sha) {
|
|
262
|
+
const d = join(this.cacheDir, sha);
|
|
263
|
+
let clips;
|
|
264
|
+
try {
|
|
265
|
+
clips = readdirSync(d)
|
|
266
|
+
.filter((f) => f.startsWith("clip-"))
|
|
267
|
+
.map((f) => ({ f, t: statSync(join(d, f)).mtimeMs }));
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
if (clips.length <= this.maxClips)
|
|
273
|
+
return;
|
|
274
|
+
clips.sort((a, b) => a.t - b.t);
|
|
275
|
+
for (const c of clips.slice(0, clips.length - this.maxClips))
|
|
276
|
+
rmSync(join(d, c.f), { force: true });
|
|
239
277
|
}
|
|
240
278
|
/** What is kept of a file goes (its bytes purged). */
|
|
241
279
|
forget(sha) {
|
package/dist/server/renames.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
// What a module of either build that could not be read held is set aside: it did not vanish.
|
|
17
17
|
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
18
18
|
import { join } from "node:path";
|
|
19
|
+
import { endsLine } from "./lostfound.js";
|
|
19
20
|
import { refIdentity, versionOf } from "./versions.js";
|
|
20
21
|
/** What stands for the declaration itself in its own references, so that a declaration can be
|
|
21
22
|
* paired before its own rename is known. */
|
|
@@ -202,8 +203,7 @@ export class Renames {
|
|
|
202
203
|
record(snap, pairs) {
|
|
203
204
|
if (!pairs.length)
|
|
204
205
|
return;
|
|
205
|
-
|
|
206
|
-
appendFileSync(this.pairFile, JSON.stringify({ snap, at: new Date().toISOString(), pairs }) + "\n");
|
|
206
|
+
append(this.pairFile, { snap, at: new Date().toISOString(), pairs });
|
|
207
207
|
this.add(snap, pairs);
|
|
208
208
|
}
|
|
209
209
|
get(id) {
|
|
@@ -212,8 +212,7 @@ export class Renames {
|
|
|
212
212
|
/** A migration made: kept, under the next id. */
|
|
213
213
|
migrated(m) {
|
|
214
214
|
const full = { id: `m${this.migrations.length + 1}`, ...m };
|
|
215
|
-
|
|
216
|
-
appendFileSync(this.migFile, JSON.stringify(full) + "\n");
|
|
215
|
+
append(this.migFile, full);
|
|
217
216
|
this.migrations.push(full);
|
|
218
217
|
this.byId.set(full.id, full);
|
|
219
218
|
return full;
|
|
@@ -225,7 +224,7 @@ export class Renames {
|
|
|
225
224
|
return;
|
|
226
225
|
const at = new Date().toISOString();
|
|
227
226
|
const who = { by, ...(agent ? { agent: true } : {}), ...(person ? { person } : {}) };
|
|
228
|
-
|
|
227
|
+
append(this.migFile, { undo: id, ...who, at });
|
|
229
228
|
m.undone = { ...who, at };
|
|
230
229
|
}
|
|
231
230
|
/** The renames of high confidence found from snapshot `from` to snapshot `to`, composed along the
|
|
@@ -250,6 +249,12 @@ export class Renames {
|
|
|
250
249
|
return out;
|
|
251
250
|
}
|
|
252
251
|
}
|
|
252
|
+
/** A line appended to one of the files: after a line cut short (the server stopped while writing),
|
|
253
|
+
* on a line of its own, so that the line cut short alone is lost, never the one written after it. */
|
|
254
|
+
function append(file, o) {
|
|
255
|
+
mkdirSync(join(file, ".."), { recursive: true });
|
|
256
|
+
appendFileSync(file, (endsLine(file) ? "" : "\n") + JSON.stringify(o) + "\n");
|
|
257
|
+
}
|
|
253
258
|
function lines(file) {
|
|
254
259
|
if (!existsSync(file))
|
|
255
260
|
return [];
|
package/dist/server/review.js
CHANGED
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
//
|
|
21
21
|
// What changed is told as a change (`ReviewChange`): the events that came after everything replayed
|
|
22
22
|
// so far, which a page applies to its own review — or, when an event came in between (older events
|
|
23
|
-
// restored from a backup, a rewrite), the whole review again
|
|
23
|
+
// restored from a backup, a rewrite), none: the page reads the whole review again, which its tag
|
|
24
|
+
// (the run and the revision) keeps from being sent to a page that has it.
|
|
24
25
|
import { createHash, randomBytes } from "node:crypto";
|
|
25
26
|
import { closeSync, existsSync, fstatSync, mkdirSync, openSync, readdirSync, readFileSync, statSync, writeFileSync, writeSync } from "node:fs";
|
|
26
27
|
import { isAbsolute, join, resolve } from "node:path";
|
|
@@ -70,6 +71,29 @@ export class ReviewStore {
|
|
|
70
71
|
whole() {
|
|
71
72
|
return { epoch: this.epoch, rev: this.rev, data: this.data, problems: this.problems, error: this.error };
|
|
72
73
|
}
|
|
74
|
+
/** the review's data as JSON, and the whole review, each with the revision it is of */
|
|
75
|
+
dataJson = null;
|
|
76
|
+
wholeJson = null;
|
|
77
|
+
/** The review's data as JSON, written once a revision. */
|
|
78
|
+
dataText() {
|
|
79
|
+
if (this.dataJson?.rev !== this.rev)
|
|
80
|
+
this.dataJson = { rev: this.rev, text: JSON.stringify(this.data) };
|
|
81
|
+
return this.dataJson.text;
|
|
82
|
+
}
|
|
83
|
+
/** The whole review as JSON (`whole()`), written once a revision: every page that loads asks for
|
|
84
|
+
* it, and every page whose review fell behind (GET /api/review), and with thousands of opinions it
|
|
85
|
+
* is megabytes (7.6 MB for 5,000 on Sphere-Packing-Lean: 50 ms each time, a hundred pages opening
|
|
86
|
+
* held the server 5 s). */
|
|
87
|
+
wholeText() {
|
|
88
|
+
if (this.wholeJson?.rev !== this.rev)
|
|
89
|
+
this.wholeJson = { rev: this.rev, text: `{"epoch":${JSON.stringify(this.epoch)},"rev":${this.rev},"data":${this.dataText()},"problems":${JSON.stringify(this.problems)},"error":${JSON.stringify(this.error)}}` };
|
|
90
|
+
return this.wholeJson.text;
|
|
91
|
+
}
|
|
92
|
+
/** The tag of the whole review as it is now (GET /api/review's ETag): its run and its revision,
|
|
93
|
+
* which every change moves. */
|
|
94
|
+
tag() {
|
|
95
|
+
return `"r${this.epoch}.${this.rev}"`;
|
|
96
|
+
}
|
|
73
97
|
/** The file a writer's events go to (shared/review.ts, logFileName). */
|
|
74
98
|
fileOf(w) {
|
|
75
99
|
return w.server ? SERVER_LOG : logFileName(w.name, w.agent, w.person ?? null);
|
|
@@ -187,13 +211,17 @@ export class ReviewStore {
|
|
|
187
211
|
/** Replay what was read, and say what changed: the new events alone where they all come after
|
|
188
212
|
* what was replayed before, else the whole review. A log read whole again has events of its own
|
|
189
213
|
* (new objects), and one gone has none: either way what was replayed is no longer where it was,
|
|
190
|
-
* and the review is replayed whole.
|
|
214
|
+
* and the review is replayed whole. A replay whole that comes to the review it replaced (a new
|
|
215
|
+
* graph's reading that binds every opinion as before, a log rewritten as it was) is told as a
|
|
216
|
+
* change of no events: no page reads the whole review again for it. */
|
|
191
217
|
commit() {
|
|
192
218
|
const logs = [...this.files.values()].map((f) => ({ file: f.file, events: f.tail?.event ? [...f.events, f.tail.event] : f.events }));
|
|
193
219
|
const next = merged(logs).map((x) => this.readAs(x.e));
|
|
194
220
|
const before = this.order;
|
|
195
221
|
const appended = !this.again && next.length >= before.length && before.every((e, k) => next[k] === e);
|
|
196
222
|
this.again = false;
|
|
223
|
+
// what the review was, where it is replayed whole: compared with what the replay makes
|
|
224
|
+
const was = appended ? null : this.dataText();
|
|
197
225
|
const base = this.rev;
|
|
198
226
|
this.rev++;
|
|
199
227
|
this.order = next;
|
|
@@ -207,6 +235,8 @@ export class ReviewStore {
|
|
|
207
235
|
this.data = emptyReview();
|
|
208
236
|
for (const e of next)
|
|
209
237
|
applyEvent(this.data, e);
|
|
238
|
+
if (this.dataText() === was)
|
|
239
|
+
events = [];
|
|
210
240
|
}
|
|
211
241
|
const c = { epoch: this.epoch, base, rev: this.rev, events, problems: this.problems };
|
|
212
242
|
this.onChange?.(c);
|
package/dist/server/reviewapi.js
CHANGED
|
@@ -199,7 +199,7 @@ export class ReviewApi {
|
|
|
199
199
|
...Object.values(d.targets).map((t) => ({ by: t.by, agent: !!t.agent, person: !!t.person })),
|
|
200
200
|
...Object.values(d.keys).flatMap((marks) => Object.values(marks).map((m) => ({ by: m.by, agent: !!m.agent, person: !!m.person }))),
|
|
201
201
|
];
|
|
202
|
-
if (writers.some((w) => w.agent && sameName(w.by, name)))
|
|
202
|
+
if (this.world.agentNamed?.(name) || writers.some((w) => w.agent && sameName(w.by, name)))
|
|
203
203
|
return refuse(400, `${name} is an agent: nobody records an opinion for an agent.`, "Record it for a person, or as your own.");
|
|
204
204
|
if (held)
|
|
205
205
|
return { for: held.name, forPerson: held.id };
|
package/dist/server/status.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// changes, not when the mtime does, so "stale" here compares the same hash (`lakeTextHash`).
|
|
12
12
|
import { execFile } from "node:child_process";
|
|
13
13
|
import { tracked } from "./children.js";
|
|
14
|
-
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
14
|
+
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
const IMPORT_RE = /^\s*(?:public\s+|private\s+)?(?:meta\s+)?import\s+(?:all\s+)?([^\s-]+)/;
|
|
17
17
|
/** The imports in a file's header: everything up to the first line that is not blank, a
|
|
@@ -82,27 +82,88 @@ export function scanModules(p) {
|
|
|
82
82
|
}
|
|
83
83
|
return mods;
|
|
84
84
|
}
|
|
85
|
-
/** Modules a lean process is compiling right now, for this project's build directory
|
|
85
|
+
/** Modules a lean process is compiling right now, for this project's build directory: a process
|
|
86
|
+
* `lean <src> -o <libDir>/Foo.olean …` (docs/LAKE.md). Asked every second the server runs. */
|
|
86
87
|
export function compiling(p) {
|
|
88
|
+
const fromProc = commandsInProc();
|
|
89
|
+
if (fromProc)
|
|
90
|
+
return Promise.resolve(compiledBy(p, fromProc));
|
|
87
91
|
return new Promise((resolve) => {
|
|
88
92
|
tracked(execFile("ps", ["axo", "command"], { maxBuffer: 16 << 20 }, (err, stdout) => {
|
|
89
|
-
const out = new Set();
|
|
90
93
|
if (err)
|
|
91
|
-
return resolve(
|
|
92
|
-
|
|
93
|
-
for (const line of stdout.split("\n")) {
|
|
94
|
-
const at = line.indexOf(marker);
|
|
95
|
-
if (at < 0)
|
|
96
|
-
continue;
|
|
97
|
-
const rest = line.slice(at + marker.length);
|
|
98
|
-
const path = rest.slice(0, rest.indexOf(".olean"));
|
|
99
|
-
if (path)
|
|
100
|
-
out.add(path.split("/").join("."));
|
|
101
|
-
}
|
|
102
|
-
resolve(out);
|
|
94
|
+
return resolve(new Set());
|
|
95
|
+
resolve(compiledBy(p, stdout.split("\n")));
|
|
103
96
|
}));
|
|
104
97
|
});
|
|
105
98
|
}
|
|
99
|
+
/** The modules the command lines `cmds` compile into the project's build directory. */
|
|
100
|
+
function compiledBy(p, cmds) {
|
|
101
|
+
const out = new Set();
|
|
102
|
+
const marker = ` -o ${p.libDir}/`;
|
|
103
|
+
for (const line of cmds) {
|
|
104
|
+
const at = line.indexOf(marker);
|
|
105
|
+
if (at < 0)
|
|
106
|
+
continue;
|
|
107
|
+
const rest = line.slice(at + marker.length);
|
|
108
|
+
const path = rest.slice(0, rest.indexOf(".olean"));
|
|
109
|
+
if (path)
|
|
110
|
+
out.add(path.split("/").join("."));
|
|
111
|
+
}
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
/** The processes /proc lists, by pid: when each was first seen, and its command line where it
|
|
115
|
+
* could be a compile (one with ` -o `), else "". */
|
|
116
|
+
const procs = new Map();
|
|
117
|
+
/** a process this young is read again: one forked a moment ago may not have run its program yet */
|
|
118
|
+
const YOUNG_MS = 2500;
|
|
119
|
+
/** whether /proc can be read here (Linux): null until tried */
|
|
120
|
+
let procOk = null;
|
|
121
|
+
/** The command lines of the processes running, from /proc, where there is one (Linux): a listing
|
|
122
|
+
* of it, and the command line of each process new since the last look, rather than `ps` every
|
|
123
|
+
* second — which read every process's files, in a process of its own, 1.5 % of a core with 600
|
|
124
|
+
* processes running. Null where there is no /proc. */
|
|
125
|
+
export function commandsInProc(now = Date.now()) {
|
|
126
|
+
if (procOk === false)
|
|
127
|
+
return null;
|
|
128
|
+
let names;
|
|
129
|
+
try {
|
|
130
|
+
names = readdirSync("/proc");
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
procOk = false;
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
const alive = new Set();
|
|
137
|
+
const cmds = [];
|
|
138
|
+
for (const n of names) {
|
|
139
|
+
const c = n.charCodeAt(0);
|
|
140
|
+
if (c < 48 || c > 57)
|
|
141
|
+
continue;
|
|
142
|
+
const pid = Number(n);
|
|
143
|
+
alive.add(pid);
|
|
144
|
+
let e = procs.get(pid);
|
|
145
|
+
if (!e || now - e.seen < YOUNG_MS) {
|
|
146
|
+
let cmd = "";
|
|
147
|
+
try {
|
|
148
|
+
// NUL between its arguments, as ps prints them: spaces
|
|
149
|
+
cmd = readFileSync(`/proc/${n}/cmdline`, "utf8").replace(/\0+$/, "").replaceAll("\0", " ");
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
// gone meanwhile, or not ours to read
|
|
153
|
+
}
|
|
154
|
+
e = { seen: e?.seen ?? now, cmd: cmd.includes(" -o ") ? cmd : "" };
|
|
155
|
+
procs.set(pid, e);
|
|
156
|
+
}
|
|
157
|
+
if (e.cmd)
|
|
158
|
+
cmds.push(e.cmd);
|
|
159
|
+
}
|
|
160
|
+
for (const pid of procs.keys())
|
|
161
|
+
if (!alive.has(pid))
|
|
162
|
+
procs.delete(pid);
|
|
163
|
+
if (procOk === null)
|
|
164
|
+
procOk = alive.size > 0;
|
|
165
|
+
return procOk ? cmds : null;
|
|
166
|
+
}
|
|
106
167
|
const M = 0xc6a4a7935bd1e995n;
|
|
107
168
|
const MASK = (1n << 64n) - 1n;
|
|
108
169
|
/** Lean's `String.hash`: MurmurHash64A over the UTF-8 bytes, seed 11. */
|
|
@@ -142,17 +203,19 @@ export function lakeTextHash(text) {
|
|
|
142
203
|
return h.toString(16).padStart(16, "0");
|
|
143
204
|
}
|
|
144
205
|
const srcHashCache = new Map();
|
|
145
|
-
/** The Lake hash of a source file, recomputed only when its mtime or size moves. */
|
|
206
|
+
/** The Lake hash of a source file, recomputed only when its mtime or size moves, and its mtime. */
|
|
146
207
|
function sourceHash(file) {
|
|
147
208
|
try {
|
|
148
|
-
const st = statSync(file);
|
|
209
|
+
const st = statSync(file, { throwIfNoEntry: false });
|
|
210
|
+
if (!st)
|
|
211
|
+
return null;
|
|
149
212
|
const key = `${st.mtimeMs}:${st.size}`;
|
|
150
213
|
const hit = srcHashCache.get(file);
|
|
151
214
|
if (hit && hit.key === key)
|
|
152
|
-
return hit.hash;
|
|
215
|
+
return { hash: hit.hash, at: st.mtimeMs };
|
|
153
216
|
const hash = lakeTextHash(readFileSync(file, "utf8"));
|
|
154
217
|
srcHashCache.set(file, { key, hash });
|
|
155
|
-
return hash;
|
|
218
|
+
return { hash, at: st.mtimeMs };
|
|
156
219
|
}
|
|
157
220
|
catch {
|
|
158
221
|
return null;
|
|
@@ -160,7 +223,7 @@ function sourceHash(file) {
|
|
|
160
223
|
}
|
|
161
224
|
function mtime(f) {
|
|
162
225
|
try {
|
|
163
|
-
return statSync(f)
|
|
226
|
+
return statSync(f, { throwIfNoEntry: false })?.mtimeMs ?? null;
|
|
164
227
|
}
|
|
165
228
|
catch {
|
|
166
229
|
return null;
|
|
@@ -195,7 +258,31 @@ function probe(p, m) {
|
|
|
195
258
|
const base = join(p.libDir, ...m.name.split("."));
|
|
196
259
|
const trace = mtime(base + ".trace");
|
|
197
260
|
const t = trace !== null ? readTrace(base + ".trace", trace, "/" + m.file) : { warnings: 0, sorry: false, builtFrom: null };
|
|
198
|
-
|
|
261
|
+
const src = sourceHash(join(p.root, m.file));
|
|
262
|
+
return { olean: mtime(base + ".olean"), trace, source: src?.hash ?? null, sourceAt: src?.at ?? null, ...t };
|
|
263
|
+
}
|
|
264
|
+
/** A look at every module's files: three times each (its outputs', its source's), read once. */
|
|
265
|
+
export function probeAll(p, mods) {
|
|
266
|
+
const out = new Map();
|
|
267
|
+
for (const m of mods.values())
|
|
268
|
+
out.set(m.name, probe(p, m));
|
|
269
|
+
return out;
|
|
270
|
+
}
|
|
271
|
+
/** A fingerprint of what a look found of the outputs and the sources: an edit is not a build, but it
|
|
272
|
+
* changes what the status says. */
|
|
273
|
+
export function stampOf(probes) {
|
|
274
|
+
const parts = [];
|
|
275
|
+
for (const [name, pr] of probes)
|
|
276
|
+
parts.push(`${name}:${pr.olean}:${pr.trace}:${pr.sourceAt}`);
|
|
277
|
+
return parts.join("|");
|
|
278
|
+
}
|
|
279
|
+
/** The same of the outputs alone: what a build wrote, which is all a followed branch's tree is read
|
|
280
|
+
* for (DeploySpec decision 5). */
|
|
281
|
+
export function outputsOf(probes) {
|
|
282
|
+
const parts = [];
|
|
283
|
+
for (const [name, pr] of probes)
|
|
284
|
+
parts.push(`${name}:${pr.olean}:${pr.trace}`);
|
|
285
|
+
return parts.join("|");
|
|
199
286
|
}
|
|
200
287
|
/** The Lake hash of the text a module's present build was compiled from, if it has a build: what
|
|
201
288
|
* the graph says about the module's declarations (their places in the file among them) is about
|
|
@@ -207,47 +294,31 @@ export function compiledTextHash(p, module, file) {
|
|
|
207
294
|
return null;
|
|
208
295
|
return readTrace(base + ".trace", trace, "/" + file).builtFrom;
|
|
209
296
|
}
|
|
210
|
-
/** A fingerprint of the build outputs, cheap enough to take every second. */
|
|
211
|
-
export function outputStamp(p, mods) {
|
|
212
|
-
const parts = [];
|
|
213
|
-
for (const m of mods.values()) {
|
|
214
|
-
const base = join(p.libDir, ...m.name.split("."));
|
|
215
|
-
parts.push(`${m.name}:${mtime(base + ".olean")}:${mtime(base + ".trace")}`);
|
|
216
|
-
}
|
|
217
|
-
return parts.join("|");
|
|
218
|
-
}
|
|
219
|
-
/** The same for the sources: an edit is not a build, but it changes what the status says. */
|
|
220
|
-
export function sourceStamp(p, mods) {
|
|
221
|
-
const parts = [];
|
|
222
|
-
for (const m of mods.values())
|
|
223
|
-
parts.push(`${m.name}:${mtime(join(p.root, m.file))}`);
|
|
224
|
-
return parts.join("|");
|
|
225
|
-
}
|
|
226
297
|
export class BuildTracker {
|
|
227
298
|
p;
|
|
228
299
|
/** when outputs were last seen changing, to bridge the gaps between compile processes */
|
|
229
300
|
lastChange = 0;
|
|
230
|
-
|
|
301
|
+
/** each module's outputs as last seen (outputsMoved), and when; null before the first look */
|
|
302
|
+
seen = null;
|
|
303
|
+
seenAt = 0;
|
|
231
304
|
buildStart = 0;
|
|
232
305
|
expected = null;
|
|
233
306
|
constructor(p) {
|
|
234
307
|
this.p = p;
|
|
235
308
|
}
|
|
236
|
-
|
|
309
|
+
/** The status of every module: from a look at their files (`probes`, made now where not given)
|
|
310
|
+
* and the compiles running (`running`, asked now where not given). */
|
|
311
|
+
async status(mods, probes = probeAll(this.p, mods), running) {
|
|
237
312
|
const p = this.p;
|
|
313
|
+
running ??= await compiling(p);
|
|
314
|
+
// a module the look did not find (one the list gained since): looked at now
|
|
315
|
+
for (const m of mods.values())
|
|
316
|
+
if (!probes.has(m.name))
|
|
317
|
+
probes.set(m.name, probe(p, m));
|
|
238
318
|
const now = Date.now();
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
const st = outputStamp(p, mods);
|
|
242
|
-
if (st !== this.lastStamp) {
|
|
243
|
-
if (this.lastStamp)
|
|
244
|
-
this.lastChange = now;
|
|
245
|
-
this.lastStamp = st;
|
|
246
|
-
}
|
|
319
|
+
if (this.outputsMoved(mods, now, probes))
|
|
320
|
+
this.lastChange = now;
|
|
247
321
|
const active = running.size > 0 || now - this.lastChange < 2500;
|
|
248
|
-
const probes = new Map();
|
|
249
|
-
for (const m of mods.values())
|
|
250
|
-
probes.set(m.name, probe(p, m));
|
|
251
322
|
// own state first, then what imports say about it
|
|
252
323
|
const own = new Map();
|
|
253
324
|
for (const m of mods.values()) {
|
|
@@ -341,6 +412,29 @@ export class BuildTracker {
|
|
|
341
412
|
counts[f.state]++;
|
|
342
413
|
return { active, progress, files, counts };
|
|
343
414
|
}
|
|
415
|
+
/** Whether the build's outputs moved since the last look: only outputs moving count as a build.
|
|
416
|
+
* An edit to a source is not one, nor a file the project gains or loses while nothing is written
|
|
417
|
+
* (an import added of a file not built yet, one taken out): a module new to the list moved only
|
|
418
|
+
* where what it has was written since the last look. The first look is no change. */
|
|
419
|
+
outputsMoved(mods, now, probes) {
|
|
420
|
+
const was = this.seen;
|
|
421
|
+
const since = this.seenAt;
|
|
422
|
+
const next = new Map();
|
|
423
|
+
let moved = false;
|
|
424
|
+
for (const m of mods.values()) {
|
|
425
|
+
const { olean, trace } = probes.get(m.name);
|
|
426
|
+
const s = `${olean}:${trace}`;
|
|
427
|
+
next.set(m.name, s);
|
|
428
|
+
if (!was)
|
|
429
|
+
continue;
|
|
430
|
+
const before = was.get(m.name);
|
|
431
|
+
if (before === undefined ? Math.max(olean ?? 0, trace ?? 0) > since : before !== s)
|
|
432
|
+
moved = true;
|
|
433
|
+
}
|
|
434
|
+
this.seen = next;
|
|
435
|
+
this.seenAt = now;
|
|
436
|
+
return moved;
|
|
437
|
+
}
|
|
344
438
|
}
|
|
345
439
|
/** The modules that can be imported right now: built, with everything under them built. */
|
|
346
440
|
export function importable(p, mods) {
|
package/dist/server/talk.js
CHANGED
|
@@ -147,9 +147,13 @@ export class Talk {
|
|
|
147
147
|
}
|
|
148
148
|
/** The agent said it leaves (POST /api/agent/view {leave: true}; its MCP server does as it
|
|
149
149
|
* stops): offline at once, its wait ended, until it waits or asks again; its next wait is a new
|
|
150
|
-
* session's.
|
|
151
|
-
|
|
150
|
+
* session's. A leave from another session than the one that waited last (`session`, as
|
|
151
|
+
* X-VisualProof-Session says it) is one a newer session replaced, whose client goes now: it
|
|
152
|
+
* leaves nothing, the newer one listening on (false). */
|
|
153
|
+
left(token, session = null, now = Date.now()) {
|
|
152
154
|
const a = this.agent(token);
|
|
155
|
+
if (session !== null && a.session !== null && a.session !== session)
|
|
156
|
+
return false;
|
|
153
157
|
a.wait?.end({ gone: true });
|
|
154
158
|
a.lastRequest = 0;
|
|
155
159
|
a.graceUntil = 0;
|
|
@@ -158,6 +162,7 @@ export class Talk {
|
|
|
158
162
|
a.away = true;
|
|
159
163
|
a.before = undefined;
|
|
160
164
|
this.update(token, now);
|
|
165
|
+
return true;
|
|
161
166
|
}
|
|
162
167
|
/** A request of the agent's: it is not offline, and `doing` (when it says something) is what it
|
|
163
168
|
* does now. */
|
package/dist/server/usage.js
CHANGED
|
@@ -21,6 +21,7 @@ import { spawn } from "node:child_process";
|
|
|
21
21
|
import { childEnv, DETACH, grouped } from "./children.js";
|
|
22
22
|
import { createHash } from "node:crypto";
|
|
23
23
|
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
24
|
+
import { freemem, totalmem } from "node:os";
|
|
24
25
|
import { join } from "node:path";
|
|
25
26
|
import { SITE_BODY, SITE_GRAMMAR, SITE_PROOF, SITE_TCB } from "../shared/graph.js";
|
|
26
27
|
import { USAGE, USAGE_PROGRAM, leanProgram, prune } from "./extract.js";
|
|
@@ -31,6 +32,43 @@ import { compiledTextHash, lakeTextHash } from "./status.js";
|
|
|
31
32
|
export const DEFAULT_JOBS = 8;
|
|
32
33
|
/** A worker still running after this long is stopped, and its module reported as failed. */
|
|
33
34
|
const WORKER_TIMEOUT_MS = 10 * 60_000;
|
|
35
|
+
const GB = 2 ** 30;
|
|
36
|
+
/** What a worker may hold at most, the `.olean` files it maps counted, on a large project on
|
|
37
|
+
* Mathlib: the showcase's (NavierStokesAndEuler, 2,657 modules) at 6 jobs peaked at 39 GB resident,
|
|
38
|
+
* 15 GB of it shared. Off Mathlib, a worker holds far less. */
|
|
39
|
+
export const WORKER_PEAK = 6 * GB;
|
|
40
|
+
/** What the machine keeps for itself and the rest while the pass runs: with less available, no new
|
|
41
|
+
* worker starts while another runs (`roomForWorker`). */
|
|
42
|
+
export function memoryFloor(total = totalmem()) {
|
|
43
|
+
return Math.max(2 * GB, total / 20);
|
|
44
|
+
}
|
|
45
|
+
/** Whether one more worker may start, `running` running: the first always; another while the
|
|
46
|
+
* machine has more than the floor available (os.freemem: MemAvailable on Linux). Workers come and
|
|
47
|
+
* go a module at a time, so that the pass holds back to what the memory allows. */
|
|
48
|
+
export function roomForWorker(running, available = freemem(), floor = memoryFloor()) {
|
|
49
|
+
return running === 0 || available >= floor;
|
|
50
|
+
}
|
|
51
|
+
/** Whether a project is on Mathlib, as its lakefile or manifest says: whose workers each map
|
|
52
|
+
* gigabytes. */
|
|
53
|
+
export function onMathlib(root) {
|
|
54
|
+
return ["lakefile.toml", "lakefile.lean", "lake-manifest.json"].some((f) => {
|
|
55
|
+
try {
|
|
56
|
+
return /mathlib/i.test(readFileSync(join(root, f), "utf8"));
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
/** What `jobs` workers may hold on a project on Mathlib, against `available` bytes: null where it
|
|
64
|
+
* fits (or off Mathlib), else what they may need and how many fit. */
|
|
65
|
+
export function jobsAdvice(jobs, mathlib, available = freemem()) {
|
|
66
|
+
if (!mathlib || jobs <= 1 || jobs * WORKER_PEAK <= available)
|
|
67
|
+
return null;
|
|
68
|
+
return { need: jobs * WORKER_PEAK, fits: Math.max(1, Math.floor(available / WORKER_PEAK)) };
|
|
69
|
+
}
|
|
70
|
+
/** Bytes in GB, in words. */
|
|
71
|
+
export const inGB = (bytes) => `${(bytes / GB).toFixed(bytes < 10 * GB ? 1 : 0)} GB`;
|
|
34
72
|
/** Mixed into the fingerprint of a merged graph: bumped when the merge changes, so that graphs
|
|
35
73
|
* merged before are not served as if made by this one. 3: the units' proof spans. (A change to
|
|
36
74
|
* extractor/Usage.lean changes every module's key by itself: its text is in the salt.) */
|
|
@@ -93,6 +131,8 @@ export class UsagePass {
|
|
|
93
131
|
status;
|
|
94
132
|
/** the run going on, to stop when a newer build supersedes it */
|
|
95
133
|
current = null;
|
|
134
|
+
/** the machine's memory available, as the pass asks it before each worker (a test's own) */
|
|
135
|
+
available = freemem;
|
|
96
136
|
constructor(project, dirs, jobs, log) {
|
|
97
137
|
this.project = project;
|
|
98
138
|
this.dir = join(dirs.cache, "usage");
|
|
@@ -236,8 +276,19 @@ export class UsagePass {
|
|
|
236
276
|
const modsFile = join(this.dir, "project-modules.txt");
|
|
237
277
|
writeFileSync(modsFile, plan.modules.join("\n") + "\n");
|
|
238
278
|
const queue = plan.todo.slice();
|
|
279
|
+
const floor = memoryFloor();
|
|
280
|
+
let held = false;
|
|
239
281
|
const worker = async () => {
|
|
240
282
|
while (queue.length && !run.cancelled) {
|
|
283
|
+
// memory short, another worker running: this one waits for it (said once a pass)
|
|
284
|
+
while (!roomForWorker(run.children.size, this.available(), floor) && !run.cancelled) {
|
|
285
|
+
if (!held)
|
|
286
|
+
this.log(`usage pass: ${inGB(this.available())} of memory available, less than ${inGB(floor)}: fewer workers at a time until there is more`);
|
|
287
|
+
held = true;
|
|
288
|
+
await new Promise((ok) => setTimeout(ok, 250));
|
|
289
|
+
}
|
|
290
|
+
if (!queue.length || run.cancelled)
|
|
291
|
+
return;
|
|
241
292
|
const t = queue.shift();
|
|
242
293
|
const e = await this.one(run, bin, env, modsFile, t);
|
|
243
294
|
if (run.cancelled)
|