papilot 0.8.6 → 0.8.7

package/src/server/pdf.ts

@@ -1,27 +1,180 @@
+import fs from 'node:fs';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { ENGINE } from './paths.ts';
-import { reviewClient, taskRoute } from './review.ts';
-
-const pending = new Map<string, Promise<string>>();
-
-export async function fullPaperPdf(root: string, name: string, round: number, side: 'before' | 'after'): Promise<string> {
-  const key = JSON.stringify([root, name, round, side]);
-  const existing = pending.get(key);
-  if (existing) return existing;
-  const build = (async () => {
-    const client = await reviewClient(root);
-    const detail = await client.get<{ before: string; after: string }>(`${taskRoute(name)}/rounds/${round}`);
-    const [{ Builder }, { Repo }, { loadProject }] = await Promise.all([
-      import(pathToFileURL(path.join(ENGINE, 'src/core/build.ts')).href),
-      import(pathToFileURL(path.join(ENGINE, 'src/core/git.ts')).href),
-      import(pathToFileURL(path.join(ENGINE, 'src/core/config.ts')).href),
-    ]);
-    // A diff can use a partial build; explicitly compile the entire immutable
-    // round revision here. The engine reuses its normal full-build cache.
-    const result = await new Builder(loadProject(root), new Repo(root)).build(detail[side], ['full']);
-    return result.pdf as string;
-  })();
-  pending.set(key, build);
-  try { return await build; } finally { if (pending.get(key) === build) pending.delete(key); }
+import { taskRoute } from './review.ts';
+
+type Connect = (root: string) => Promise<{ get<T>(route: string): Promise<T> }>;
+
+type Priority = 'foreground' | 'normal' | 'background';
+const RANK: Record<Priority, number> = { background: 0, normal: 1, foreground: 2 };
+
+// The complete PDF of one version of a paper, for the review's full paper. A
+// round may compile only the changed chapters; this compiles them all, once.
+// It can start before anyone asks, as soon as a round's new version exists;
+// whoever then waits for it has it compile first and sees each TeX pass.
+interface Paper {
+  priority: Priority;
+  following: boolean;
+  // The passes shown so far, by number, while the PDF compiled.
+  passes: Map<number, string>;
+  pdf: string | null;
+  failure: Error | null;
+  done: Promise<void>;
+  changed: Set<() => void>;
+}
+const papers = new Map<string, Paper>();
+// Compiled papers are remembered for the passes they showed, a few at a time.
+const REMEMBERED = 32;
+
+async function builder(root: string) {
+  const [{ Builder }, { Repo }, { loadProject }] = await Promise.all([
+    import(pathToFileURL(path.join(ENGINE, 'src/core/build.ts')).href),
+    import(pathToFileURL(path.join(ENGINE, 'src/core/git.ts')).href),
+    import(pathToFileURL(path.join(ENGINE, 'src/core/config.ts')).href),
+  ]);
+  return new Builder(loadProject(root), new Repo(root));
+}
+
+const notFound = () => Object.assign(new Error('Review resource not found.'), { statusCode: 404 });
+const changed = (paper: Paper) => { for (const notify of [...paper.changed]) notify(); };
+
+/** Starts compiling the complete PDF of `revision`, or has the build already
+ *  under way run at `priority` and, with `follow`, show its passes. */
+export function preparePaper(root: string, revision: string, priority: Priority = 'normal', follow = false): Paper {
+  const key = JSON.stringify([root, revision]);
+  let paper = papers.get(key);
+  const compile = (target: Paper, wanted: Priority, watch: boolean) => builder(root).then(b => b.build(revision, ['full'], null, {
+    priority: wanted,
+    ...(watch ? {
+      pass: (preview: { pass: number; pdf: string }) => {
+        if (target.pdf || target.passes.has(preview.pass)) return;
+        target.passes.set(preview.pass, preview.pdf);
+        changed(target);
+      },
+    } : {}),
+  })) as Promise<{ pdf: string }>;
+  if (paper) {
+    papers.delete(key);
+    papers.set(key, paper);
+    if (!paper.pdf && !paper.failure && (RANK[priority] > RANK[paper.priority] || (follow && !paper.following))) {
+      if (RANK[priority] > RANK[paper.priority]) paper.priority = priority;
+      paper.following ||= follow;
+      void compile(paper, paper.priority, follow).catch(() => undefined);
+    }
+    return paper;
+  }
+  const created: Paper = { priority, following: follow, passes: new Map(), pdf: null, failure: null, done: Promise.resolve(), changed: new Set() };
+  created.done = compile(created, priority, follow).then(
+    result => { created.pdf = result.pdf; },
+    error => {
+      created.failure = error instanceof Error ? error : new Error(String(error));
+      // A later request compiles it again.
+      if (papers.get(key) === created) papers.delete(key);
+    },
+  ).finally(() => changed(created));
+  papers.set(key, created);
+  for (const old of papers.keys()) {
+    if (papers.size <= REMEMBERED) break;
+    if (papers.get(old)!.pdf) papers.delete(old);
+  }
+  return created;
+}
+
+// A round does not change; what the full paper needs of it is looked up once.
+interface RoundInfo { before: string; after: string; build?: { key: string; targets: string[] } }
+const rounds = new Map<string, Promise<RoundInfo>>();
+function roundInfo(connect: Connect, root: string, name: string, round: number) {
+  const key = JSON.stringify([root, name, round]);
+  let found = rounds.get(key);
+  if (!found) {
+    found = connect(root)
+      .then(client => client.get<RoundInfo>(`${taskRoute(name)}/rounds/${round}`))
+      .then(detail => {
+        if (typeof detail?.before !== 'string' || typeof detail.after !== 'string') throw notFound();
+        return { before: detail.before, after: detail.after, ...(detail.build ? { build: detail.build } : {}) };
+      });
+    rounds.set(key, found);
+    found.catch(() => rounds.delete(key));
+    if (rounds.size > 256) rounds.delete(rounds.keys().next().value!);
+  }
+  return found;
+}
+
+// The round's own PDF when it compiled only the changed chapters: their pages
+// with the changes, exactly where its strips come from, at once.
+async function chaptersPdf(root: string, info: RoundInfo) {
+  if (!info.build || !/^[0-9a-f]{32}$/.test(info.build.key) || JSON.stringify(info.build.targets) === '["full"]') return null;
+  const { loadProject } = await import(pathToFileURL(path.join(ENGINE, 'src/core/config.ts')).href);
+  try {
+    const result = JSON.parse(await fs.promises.readFile(path.join(loadProject(root).state, 'cache/builds', info.build.key, 'result.json'), 'utf8')) as { pdf?: unknown };
+    if (typeof result.pdf !== 'string') return null;
+    await fs.promises.access(result.pdf);
+    return result.pdf;
+  } catch { return null; }
 }
+
+async function ready(root: string, revision: string, follow = false) {
+  for (let attempt = 0; ; attempt++) {
+    const paper = preparePaper(root, revision, 'foreground', follow);
+    await paper.done;
+    if (paper.failure) throw paper.failure;
+    // The build cache may have removed an old PDF meanwhile.
+    if (attempt || await fs.promises.access(paper.pdf!).then(() => true, () => false)) return paper.pdf!;
+    papers.delete(JSON.stringify([root, revision]));
+  }
+}
+
+/** The complete PDF of a round's version, once it has compiled. */
+export async function fullPaperPdf(connect: Connect, root: string, name: string, round: number, side: 'before' | 'after'): Promise<string> {
+  return ready(root, (await roundInfo(connect, root, name, round))[side]);
+}
+
+/** What the review's full paper shows next, after version `after`: a later
+ *  version, `null` once the complete PDF is ready, or nothing when `wait`
+ *  ends first. Until then it shows the round's changed chapters (version 1
+ *  when the round compiled only those), then each TeX pass of the whole. */
+export async function nextPaperPass(connect: Connect, root: string, name: string, round: number, after: number, wait = 20000): Promise<{ pass?: number | null }> {
+  const info = await roundInfo(connect, root, name, round), chapters = await chaptersPdf(root, info), offset = chapters ? 1 : 0;
+  if (chapters && after < 1) {
+    // A complete PDF that is already there, or nearly, comes first.
+    const paper = preparePaper(root, info.after, 'foreground', true);
+    await Promise.race([paper.done, new Promise(resolve => setTimeout(resolve, 300))]);
+    if (paper.pdf) return { pass: null };
+    return { pass: 1 };
+  }
+  const next = await nextPass(root, info.after, Math.max(0, after - offset), wait);
+  return typeof next.pass === 'number' ? { pass: next.pass + offset } : next;
+}
+export async function nextPass(root: string, revision: string, after: number, wait = 20000): Promise<{ pass?: number | null }> {
+  const paper = preparePaper(root, revision, 'foreground', true);
+  const deadline = Date.now() + wait;
+  for (;;) {
+    if (paper.failure) throw paper.failure;
+    if (paper.pdf) return { pass: null };
+    const newest = Math.max(0, ...paper.passes.keys());
+    if (newest > after) return { pass: newest };
+    const left = deadline - Date.now();
+    if (left <= 0) return {};
+    await new Promise<void>(resolve => {
+      const done = () => { clearTimeout(timer); paper.changed.delete(done); resolve(); };
+      const timer = setTimeout(done, left);
+      paper.changed.add(done);
+    });
+  }
+}
+
+/** The PDF of a version that `nextPaperPass` named. */
+export async function paperPassPdf(connect: Connect, root: string, name: string, round: number, pass: number) {
+  const info = await roundInfo(connect, root, name, round), chapters = await chaptersPdf(root, info);
+  if (chapters && pass === 1) return chapters;
+  return passPdf(root, info.after, pass - (chapters ? 1 : 0));
+}
+export function passPdf(root: string, revision: string, pass: number) {
+  const pdf = papers.get(JSON.stringify([root, revision]))?.passes.get(pass);
+  if (!pdf) throw notFound();
+  return pdf;
+}
+
+/** The complete PDF of `revision`, once it has compiled. */
+export const paperPdf = ready;

package/src/server/references.ts

@@ -20,7 +20,7 @@ interface Engine {
   entryInputs(entries: unknown[]): Input[];
   ReferenceChecker: new (file: string) => Checker;
   Confirmations: new (file: string) => Confirmations;
-  Repo: new (root: string) => { gitDir(): string };
+  Repo: new (root: string) => { gitDir(): string; textAsync(args: string[]): Promise<string> };
 }
 let loading: Promise<Engine> | undefined;
 export function engine(): Promise<Engine> {
@@ -38,9 +38,17 @@ export async function checker(file = path.join(DATA, 'references.json')) {
 }
 // Confirmations live beside the review engine's state in the repository, so
 // a task's review, the uncommitted changes and this panel agree.
+const gitDirs = new Map<string, Promise<string>>();
 export async function confirmations(root: string) {
   const { Confirmations, Repo } = await engine();
-  return new Confirmations(path.join(new Repo(root).gitDir(), 'latex-review', 'references-confirmed.json'));
+  // The panel polls while a check runs; the Git directory is looked up once.
+  let dir = gitDirs.get(root);
+  if (!dir) {
+    const found = new Repo(root).textAsync(['rev-parse', '--absolute-git-dir']).then(folder => fs.promises.realpath(folder));
+    found.catch(() => gitDirs.delete(root));
+    gitDirs.set(root, dir = found);
+  }
+  return new Confirmations(path.join(await dir, 'latex-review', 'references-confirmed.json'));
 }
 
 // A file of the paper, read only from inside its folder.

package/src/server/review.ts

@@ -4,11 +4,15 @@ import { ENGINE } from './paths.ts';
 import type { Task } from '../shared.ts';
 
 export interface ReviewClient { url: string; token: string; get<T = any>(route: string): Promise<T>; post<T = any>(route: string, body?: unknown, timeout?: number): Promise<T> }
-let modulePromise: Promise<{ connect: (root: string, start?: boolean) => Promise<ReviewClient | null> }> | null = null;
+let modulePromise: Promise<{ connect: (root: string, start?: boolean, options?: { owned?: boolean }) => Promise<ReviewClient | null> }> | null = null;
+// The desktop app owns the review servers it starts, so they stop with it,
+// even after a crash. The web server leaves them running to finish builds
+// across a restart.
+const owned = process.env.PAPILOT_DESKTOP === '1';
 export function reviewClient(root: string, reuse = false): Promise<ReviewClient> {
   modulePromise ??= import(pathToFileURL(path.join(ENGINE, 'src/client.ts')).href);
   // Polling an existing job must not replace a healthy server of another version.
-  return modulePromise!.then(async module => (reuse && await module.connect(root, false)) || (await module.connect(root))!);
+  return modulePromise!.then(async module => (reuse && await module.connect(root, false)) || (await module.connect(root, true, { owned }))!);
 }
 export const taskRoute = (name: string) => `/api/tasks/${encodeURIComponent(name)}`;
 export function reviewConnectionError(error: unknown): boolean {

package/src/server/start.ts

@@ -39,6 +39,9 @@ const localListener = { kind: 'local' as const, port: PORT, publicOrigin: PUBLIC
 const apps = [await buildApp({ ...shared, listener: localListener })];
 if (ACCOUNT_ORIGIN) apps.push(await buildApp({ ...shared, listener: { kind: 'account', origin: ACCOUNT_ORIGIN, hsts: accountHsts(ACCOUNT_ORIGIN, PUBLIC_ORIGIN) } }));
 
+// Agent SDKs and the PDF engine load before the first request, not in the
+// middle of one.
+if (!process.env.PAPILOT_UPDATE_PROBE) await Promise.race([Promise.all([workspace.preload(), manuscripts.warm()]), new Promise(resolve => setTimeout(resolve, 5000).unref())]);
 const localUrl = await apps[0].listen({ host: '127.0.0.1', port: PORT }).catch(error => {
   if (!desktop || error.code !== 'EADDRINUSE') throw error;
   return apps[0].listen({ host: '127.0.0.1', port: 0 });
@@ -62,7 +65,7 @@ for (const task of store.tasks()) {
 setInterval(() => accounts.purge(), 3600000).unref();
 let shuttingDown = false;
 function shutdown() {
-  if (shuttingDown) return; shuttingDown = true; updates.close(); workspace.close();
+  if (shuttingDown) return; shuttingDown = true; updates.close(); workspace.close(); void manuscripts.stop().catch(() => undefined);
   void Promise.all(apps.map(app => app.close())).finally(() => { store.close(); process.exit(0); });
   setTimeout(() => process.exit(0), 2500).unref();
 }

package/src/server/workspace.ts

@@ -161,6 +161,16 @@ export class Workspace extends EventEmitter {
     this.emit('change');
   }
   snapshot(): Snapshot { return { version: VERSION, projects: this.store.projects(), tasks: this.store.tasks(), requests: [...this.requests.values()].map(({ rpcId: _, agent: __, ...r }) => r), runtime: this.runtime, runtimes: this.runtimes }; }
+  /** Loads the SDKs of the agents this host has before the server answers
+   *  requests. Evaluating one blocks for up to a few hundred milliseconds,
+   *  which would otherwise hold up whatever the app asked for while the
+   *  agents connect. SDKs not installed yet are fetched when first used. */
+  async preload() {
+    await Promise.all(AGENTS.map(async id => {
+      const agent = this.agents[id];
+      try { if (agent.preload && (!agent.detect || agent.detect())) await agent.preload(); } catch { /* Connecting reports it. */ }
+    }));
+  }
   async connect() {
     if (this.connecting) return this.connecting;
     this.connecting = Promise.all(AGENTS.map(id => this.connectAgent(id))).then(() => { this.emit('change'); this.schedule(); }).finally(() => { this.connecting = null; });
@@ -780,6 +790,7 @@ ${this.agentOf(task) === 'antigravity' ? 'Shell commands are not available to yo
     let pending = this.store.task(taskId).pendingBuild!;
     let client: ReviewClient | undefined;
     let reuse = resume;
+    let compiling: string | undefined;
     // Only connection failures are retried. Every submission has a persisted key,
     // so losing its response cannot submit a second build.
     const request = async <T>(work: (client: ReviewClient) => Promise<T>, submitting = false): Promise<T> => {
@@ -810,6 +821,11 @@ ${this.agentOf(task) === 'antigravity' ? 'Shell commands are not available to yo
           this.store.updateTask(taskId, { pendingBuild: pending });
         }
         const current = await request(client => client.get(`/api/jobs/${pending.job}`));
+        // The round's new version exists; it compiles only some chapters here.
+        if (typeof current.revision === 'string' && current.revision !== compiling) {
+          compiling = current.revision;
+          this.emit('publishing', taskId, compiling);
+        }
         const summary = await request(client => client.get(taskRoute(task.name)));
         this.store.updateTask(taskId, { review: { ...this.store.task(taskId).review, state: summary.state, round: summary.round, counts: summary.counts, job: { ...current, id: pending.job } } });
         if (current.status === 'done' || current.status === 'completed' || current.status === 'succeeded') {
@@ -838,6 +854,8 @@ ${this.agentOf(task) === 'antigravity' ? 'Shell commands are not available to yo
         await delay(1500);
       }
       await this.refresh(taskId);
+      // What applying the new round will write can now be compiled ahead.
+      this.emit('reviewed', taskId);
     } catch (e) {
       if (this.closing) return;
       if (reviewConnectionError(e)) {