papilot 0.6.0 → 0.8.0

package/src/server/manuscript.ts

@@ -1,3 +1,4 @@
+import type { ReviewSnapshot } from './ai-review.ts';
 import { execFile } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { EventEmitter } from 'node:events';
@@ -7,15 +8,17 @@ import path from 'node:path';
 import { promisify } from 'node:util';
 import { pathToFileURL } from 'node:url';
 import { ENGINE } from './paths.ts';
-import type { Annotation, GitFile, GitStatus, ManuscriptBuild } from '../shared.ts';
+import type { Annotation, BuildProblem, GitFile, GitStatus, ManuscriptBuild, ManuscriptProblems } from '../shared.ts';
 import { notesInstructions } from '../notes.ts';
 import { ignorePattern } from '../commit.ts';
+import { AnnotationPages, placeAnnotation } from './annotation-placement.ts';
+import { collectProblems, countProblems, locateProblems, scrubLog, sortProblems, type StoredProblem } from './build-problems.ts';
 
 type Rect = [number, number, number, number];
 
 const exec = promisify(execFile);
 async function git(root: string, args: string[], env: NodeJS.ProcessEnv = {}) {
-  return (await exec('git', args, { cwd: root, env: { ...process.env, ...env }, maxBuffer: 64 * 1024 * 1024 })).stdout;
+  return (await exec('git', args, { cwd: root, timeout: 30_000, env: { ...process.env, ...env }, maxBuffer: 64 * 1024 * 1024 })).stdout;
 }
 
 // Porcelain v2 keeps paths with spaces intact under -z and reports branch,
@@ -53,7 +56,7 @@ export function parseStatus(output: string, numstat: string): Omit<GitStatus, 'h
 
 export async function gitStatus(root: string): Promise<GitStatus> {
   const [status, numstat, head, remotes] = await Promise.all([
-    git(root, ['status', '--porcelain=v2', '--branch', '-z', '--untracked-files=all']),
+    git(root, ['status', '--porcelain=v2', '--branch', '-z', '--untracked-files=all'], { GIT_OPTIONAL_LOCKS: '0' }),
     git(root, ['diff', '--numstat', '-z', 'HEAD']).catch(() => ''),
     git(root, ['log', '-1', '--format=%h%x00%s%x00%ct']).catch(() => ''),
     git(root, ['remote']).catch(() => ''),
@@ -339,44 +342,241 @@ export async function pdfWords(pdf: string, page: number) {
   } finally { document.close(); }
 }
 
-interface Build { tree: string; status: ManuscriptBuild['status']; key?: string; pdf?: string; root?: string; sourceDir?: string; pages?: [number, number][]; builtAt?: number; error?: string }
+// `problems` come from the last finished build. They are placed on the PDF
+// named `problemsKey`; a failed build has no PDF of its own to place them on.
+interface Build {
+  tree: string; status: ManuscriptBuild['status']; key?: string; pdf?: string; root?: string; sourceDir?: string; pages?: [number, number][]; builtAt?: number; error?: string;
+  problems?: StoredProblem[]; problemsKey?: string; log?: string; logFolder?: string;
+}
+
+
 
 export class Manuscripts extends EventEmitter {
   private builds = new Map<string, Build>();
+  private oldPdfs = new Map<string, string | null>();
+  private placements = new Map<string, (Pick<Annotation, 'key' | 'page' | 'rect' | 'rects' | 'source'>) | null>();
+  // Placed problems per problem list; a rebuild in progress shares its predecessor's list.
+  private located = new WeakMap<StoredProblem[], Promise<BuildProblem[]>>();
+  get busyForUpdate() { return [...this.builds.values()].some(build => build.status === 'building'); }
+  busyForProject(id: string) { return this.builds.get(id)?.status === 'building'; }
+  forgetProject(id: string) { this.builds.delete(id); }
 
   async state(projectId: string, root: string, rebuild = false): Promise<{ git: GitStatus; build: ManuscriptBuild }> {
     const [status, tree] = await Promise.all([gitStatus(root), workingTree(root)]);
     let build = this.builds.get(projectId);
-    if (!build || (build.status !== 'building' && (build.tree !== tree || (rebuild && build.status === 'failed')))) build = this.start(projectId, root, tree, build);
+    if (!build || (build.status !== 'building' && (build.tree !== tree || rebuild))) build = this.start(projectId, root, tree, build, rebuild);
     const { status: state, key, pages, builtAt, error } = build;
-    return { git: status, build: { status: state, key, pages, builtAt, error, current: build.tree === tree && state === 'ready' } };
+    return { git: status, build: { status: state, key, pages, builtAt, error, current: build.tree === tree && state === 'ready', problems: build.problems && countProblems(build.problems) } };
   }
 
   // Keep the previous pages visible while a newer working tree compiles.
-  private start(projectId: string, root: string, tree: string, previous?: Build) {
-    const build: Build = { tree, status: 'building', key: previous?.key, pdf: previous?.pdf, pages: previous?.pages, builtAt: previous?.builtAt };
+  private start(projectId: string, root: string, tree: string, previous?: Build, force = false) {
+    if (previous?.key && previous.pdf) this.oldPdfs.set(`${root}:${previous.key}`, previous.pdf);
+    const build: Build = { tree, status: 'building', key: previous?.key, pdf: previous?.pdf, root: previous?.root, sourceDir: previous?.sourceDir, pages: previous?.pages, builtAt: previous?.builtAt, problems: previous?.problems, problemsKey: previous?.problemsKey, log: previous?.log, logFolder: previous?.logFolder };
     this.builds.set(projectId, build);
     void (async () => {
+      let project: { source_dir?: string; doc_dir?: string; state: string } | undefined;
       try {
         const { Builder, Repo, loadProject, mupdf } = await engine();
-        const project = loadProject(root), result = await new Builder(project, new Repo(root)).build(tree, ['full']);
+        project = loadProject(root);
+        const result = await new Builder(project, new Repo(root)).build(tree, ['full'], null, { force });
         const document = mupdf.Document.openDocument(fs.readFileSync(result.pdf), 'application/pdf');
         const pages: [number, number][] = [];
         for (let i = 0; i < document.countPages(); i++) { const [x0, y0, x1, y1] = document.loadPage(i).getBounds(); pages.push([Math.round(x1 - x0), Math.round(y1 - y0)]); }
         document.destroy?.();
         const key = createHash('sha256').update(result.key + result.pdf).digest('hex').slice(0, 16);
         if (this.builds.get(projectId) !== build) return;
-        Object.assign(build, { status: 'ready', key, pdf: result.pdf, root: result.root, sourceDir: project.source_dir || '', pages, builtAt: Date.now(), error: undefined });
+        const log = String(result.pdf).replace(/\.pdf$/, '.log');
+        let problems: StoredProblem[] = [];
+        try { problems = collectProblems(fs.readFileSync(log), { cwd: result.cwd, root: result.root, sourceDir: project!.source_dir || '', outDir: path.dirname(result.pdf) }); } catch { /* A build without a readable log reports no problems. */ }
+        Object.assign(build, { status: 'ready', key, pdf: result.pdf, root: result.root, sourceDir: project!.source_dir || '', pages, builtAt: Date.now(), error: undefined, problems, problemsKey: key, log: fs.existsSync(log) ? log : undefined, logFolder: path.dirname(result.root) });
+        // Place the problems now, so the list is ready when someone opens it.
+        if (problems.length) void this.problems(projectId).catch(() => undefined);
       } catch (error) {
         if (this.builds.get(projectId) !== build) return;
-        const detail = error as { message?: string; details?: string[] };
-        Object.assign(build, { status: 'failed', error: [detail.message, ...(detail.details || [])].filter(Boolean).join('\n') || 'The manuscript could not be compiled.' });
+        const detail = error as { message?: string; details?: string[]; log?: string };
+        const message = [detail.message, ...(detail.details || [])].filter(Boolean).join('\n') || 'The manuscript could not be compiled.';
+        // The engine keeps latexmk's transcript and the TeX log of a failed
+        // build; its sources are gone, so files resolve in the working tree.
+        let problems: StoredProblem[] = [];
+        if (detail.log && project) try {
+          const kept = fs.readFileSync(detail.log), marker = Buffer.from('\n\n--- TeX log ---\n'), at = kept.indexOf(marker);
+          // TeX stopped in its first pass, before references resolve, so only
+          // the errors of a failed run mean anything.
+          if (at >= 0) problems = collectProblems(kept.subarray(at + marker.length), { cwd: path.join(root, project.doc_dir || '.'), root: path.join(root, project.source_dir || '.'), sourceDir: project.source_dir || '' })
+            .filter(problem => problem.severity === 'error').map(({ source: _source, ...problem }) => problem);
+        } catch { /* Fall back to the compiler's own message below. */ }
+        // BibTeX, Biber, latexmk and helper programs fail outside the TeX log;
+        // whatever the compiler said that the parsed errors do not cover stays.
+        const plain = (text: string) => text.replace(/\s+/g, ' ').trim();
+        const unexplained = (detail.details || []).filter(line => line.trim() && !/==> Fatal error occurred|^Emergency stop/.test(line) && !problems.some(problem => plain(line).includes(plain(problem.message).replace(/^[^:]+:\d+: /, ''))));
+        const logFolder = detail.log && project ? path.join(project.state, 'cache/builds', path.basename(detail.log, '.log')) : undefined;
+        if (!problems.length || unexplained.length) problems.push({ kind: 'error', severity: 'error', message: scrubLog([...(problems.length ? [] : [detail.message || 'The manuscript could not be compiled.']), ...unexplained].join('\n'), logFolder) });
+        // The previous PDF stays on screen, and so do its warnings and places.
+        const shown = build.problemsKey && build.problemsKey === build.key ? (build.problems || []).filter(problem => problem.severity !== 'error') : [];
+        Object.assign(build, { status: 'failed', error: message, problems: [...problems, ...shown], problemsKey: shown.length ? build.problemsKey : undefined, log: detail.log && fs.existsSync(detail.log) ? detail.log : undefined, logFolder });
       }
       this.emit('change', projectId);
     })();
     return build;
   }
 
+  // Old annotations keep their saved content. Only their proven PDF position
+  // changes; ambiguous or missing content stays on the earlier version.
+  async reanchor(projectId: string, root: string, notes: Annotation[]): Promise<Annotation[]> {
+    const build = this.builds.get(projectId);
+    if (!build?.pdf || !build.key || build.status !== 'ready' || !notes.some(note => note.key !== build.key)) return notes;
+    const { PdfDocument, loadProject } = await engine();
+    const current = new PdfDocument(build.pdf), target = new AnnotationPages(current);
+    const old = new Map<string, { document: InstanceType<typeof PdfDocument>; pages: AnnotationPages }>();
+    try {
+      const output: Annotation[] = [];
+      for (const note of notes) {
+        if (note.key === build.key) { output.push(note); continue; }
+        const cacheKey = JSON.stringify([projectId, build.key, note.key, note.page, note.rect, note.rects, note.quote]);
+        if (!this.placements.has(cacheKey)) {
+          let placement: Pick<Annotation, 'key' | 'page' | 'rect' | 'rects' | 'source'> | null = null;
+          try {
+            const pdf = await this.cachedPdf(root, note.key);
+            if (pdf) {
+              if (!old.has(pdf)) { const document = new PdfDocument(pdf); old.set(pdf, { document, pages: new AnnotationPages(document) }); }
+              const match = placeAnnotation(note, old.get(pdf)!.pages, target);
+              if (match) {
+                const { source } = await this.locate(projectId, build.key, match.page, match.rect, match.rects, note.quote);
+                placement = { ...match, key: build.key, source };
+              }
+            }
+          } catch { /* Keep the annotation intact when its position cannot be verified. */ }
+          this.placements.set(cacheKey, placement);
+        }
+        const placement = this.placements.get(cacheKey);
+        output.push(placement ? { ...note, ...placement } : note);
+      }
+      if (this.placements.size > 2000) this.placements.clear();
+      if (this.oldPdfs.size > 500) this.oldPdfs.clear();
+      return this.builds.get(projectId)?.key === build.key ? output : notes;
+    } finally { current.close(); for (const entry of old.values()) entry.document.close(); }
+  }
+
+  // Every problem of the last finished build, placed on its PDF when there is
+  // one. Placing runs SyncTeX per problem, once per build.
+  async problems(projectId: string): Promise<ManuscriptProblems> {
+    const build = this.builds.get(projectId);
+    if (!build?.problems) return { status: build?.status || 'building', problems: [], log: Boolean(build?.log) };
+    const { problems, problemsKey: key, pdf } = build;
+    if (!key || key !== build.key || !pdf) return { status: build.status, problems: sortProblems(problems.map(({ source: _source, occurrence: _occurrence, ...problem }) => problem)), log: Boolean(build.log) };
+    let placed = this.located.get(problems);
+    if (!placed) {
+      placed = (async () => {
+        const { PdfDocument } = await engine();
+        const document = new PdfDocument(pdf), pages = new Map<number, ReturnType<typeof document.page>>();
+        try {
+          return sortProblems(await locateProblems(problems, pdf, page => {
+            if (!Number.isInteger(page) || page < 0 || page >= document.length) return null;
+            if (!pages.has(page)) pages.set(page, document.page(page));
+            return pages.get(page)!;
+          }));
+        } finally { for (const page of pages.values()) page.close(); document.close(); }
+      })();
+      placed.catch(() => this.located.delete(problems));
+      this.located.set(problems, placed);
+    }
+    return { status: build.status, key, problems: await placed, log: Boolean(build.log) };
+  }
+
+  log(projectId: string) {
+    const build = this.builds.get(projectId);
+    if (!build?.log || !fs.existsSync(build.log)) throw Object.assign(new Error('There is no compilation log yet.'), { statusCode: 404 });
+    return scrubLog(fs.readFileSync(build.log, 'utf8'), build.logFolder);
+  }
+
+  // Compile the task's frozen source tree, then retain its PDF outside the build cache.
+  async captureReview(root: string, worktree: string, taskId: string): Promise<ReviewSnapshot> {
+    const { Builder, Repo, loadProject } = await engine();
+    const tree = await workingTree(worktree);
+    const result = await new Builder(loadProject(root), new Repo(root)).build(tree, ['full']);
+    const pdf = await this.preserveReviewPdf(root, taskId, result.pdf);
+    const key = createHash('sha256').update(fs.readFileSync(pdf)).digest('hex').slice(0, 16);
+    return { ...await this.snapshotPdf(pdf, key), pdf };
+  }
+
+  private async reviewPdfPath(root: string, taskId: string) {
+    const { loadProject } = await engine();
+    return path.join(loadProject(root).state, 'ai-reviews', createHash('sha256').update(taskId).digest('hex'), 'review.pdf');
+  }
+
+  async savedReviewPdf(root: string, taskId: string): Promise<string | null> {
+    const pdf = await this.reviewPdfPath(root, taskId);
+    return fs.existsSync(pdf) ? pdf : null;
+  }
+
+  async preserveReviewPdf(root: string, taskId: string, source: string) {
+    const pdf = await this.reviewPdfPath(root, taskId), folder = path.dirname(pdf);
+    fs.mkdirSync(folder, { recursive: true });
+    // A retry always uses the original task copy, even after its build cache is pruned.
+    if (!fs.existsSync(pdf)) { const temporary = path.join(folder, 'review.tmp'); fs.copyFileSync(source, temporary); fs.renameSync(temporary, pdf); }
+    return pdf;
+  }
+
+  async cachedPdf(root: string, key: string): Promise<string | null> {
+    const lookup = `${root}:${key}`;
+    if (this.oldPdfs.has(lookup)) return this.oldPdfs.get(lookup)!;
+    const { loadProject } = await engine();
+    const cache = path.join(loadProject(root).state, 'cache/builds');
+    let found: string | null = null;
+    if (fs.existsSync(cache)) for (const entry of fs.readdirSync(cache, { withFileTypes: true })) {
+      if (!entry.isDirectory() || entry.name.endsWith('.lock')) continue;
+      try {
+        const folder = path.join(cache, entry.name), saved = JSON.parse(fs.readFileSync(path.join(folder, 'result.json'), 'utf8'));
+        if (typeof saved.pdf !== 'string' || !path.resolve(saved.pdf).startsWith(path.resolve(folder) + path.sep)) continue;
+        if (createHash('sha256').update(saved.key + saved.pdf).digest('hex').slice(0, 16) === key && fs.existsSync(saved.pdf)) { found = saved.pdf; break; }
+      } catch { /* Incomplete builds have no usable PDF. */ }
+    }
+    this.oldPdfs.set(lookup, found); return found;
+  }
+
+  async placeReviewNote(projectId: string, root: string, note: Annotation, pdf: string): Promise<Annotation> {
+    const state = await this.state(projectId, root);
+    if (!state.build.current || !state.build.key) throw new Error('Wait for the current manuscript PDF to finish compiling, then copy the suggestion again.');
+    const key = state.build.key, { PdfDocument } = await engine();
+    const original = new PdfDocument(pdf), current = new PdfDocument(this.pdf(projectId, key));
+    try {
+      const match = placeAnnotation(note, new AnnotationPages(original), new AnnotationPages(current));
+      if (!match) throw new Error('This passage could not be located reliably in the current manuscript. The suggestion remains in the review task.');
+      const { source } = await this.locate(projectId, key, match.page, match.rect, match.rects, note.quote);
+      const latest = await this.state(projectId, root);
+      if (!latest.build.current || latest.build.key !== key) throw new Error('The manuscript changed while copying this suggestion. Try again.');
+      return { ...note, ...match, key, source };
+    } finally { original.close(); current.close(); }
+  }
+
+  async reviewSnapshot(projectId: string, key: string): Promise<ReviewSnapshot> {
+    return this.snapshotPdf(this.pdf(projectId, key), key);
+  }
+
+  private async snapshotPdf(pdf: string, key: string): Promise<ReviewSnapshot> {
+    const { PdfDocument } = await engine();
+    const document = new PdfDocument(pdf), lines: ReviewSnapshot['lines'] = [];
+    let length = 0;
+    try {
+      for (let index = 0; index < document.length; index++) {
+        const page = document.page(index), [left, top, right, bottom] = page.bounds;
+        for (const [number, line] of page.lines.entries()) {
+          const text = line.chars.map((char: { text: string }) => char.text).join('').trim();
+          if (!text) continue;
+          length += text.length;
+          if (length > 180000) throw new Error('This PDF is too long for one AI review (180,000 characters maximum).');
+          const [x0, y0, x1, y1] = line.rect;
+          const rect = [(x0 - left) / (right - left), (y0 - top) / (bottom - top), (x1 - left) / (right - left), (y1 - top) / (bottom - top)].map(value => Math.max(0, Math.min(1, value))) as Rect;
+          if (rect[2] <= rect[0] || rect[3] <= rect[1]) continue;
+          lines.push({ id: `p${index + 1}-l${number + 1}`, page: index, text, rect });
+        }
+      }
+      if (!lines.length) throw new Error('This PDF has no selectable text to review.');
+      return { key, lines };
+    } finally { document.close(); }
+  }
+
   pdf(projectId: string, key: string) {
     const build = this.builds.get(projectId);
     if (!build?.pdf || build.key !== key) throw Object.assign(new Error('This manuscript preview is out of date. Refresh it and try again.'), { statusCode: 404 });

package/src/server/note-results.ts

@@ -0,0 +1,65 @@
+import type { NoteChange, NoteResult, NoteSummary, Run, Task, TimelineItem } from '../shared.ts';
+import { noteAnchor, readNotesInstructions } from '../notes.ts';
+import { taskRoute, type ReviewClient } from './review.ts';
+
+// Only the fields needed to trace notes; the review engine owns the full format.
+export interface NoteRound {
+  round: number;
+  units: { id: string; file: string; old_start: number; old_end: number; old_text: string; new_text: string }[];
+  sections: { blocks: NoteBlock[] }[];
+  loose: NoteBlock[];
+}
+interface NoteBlock { kind: string; units?: string[]; before?: { text?: { text: string }[] }[]; identical?: boolean }
+const normalize = (text: string) => text.normalize('NFKC').toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
+
+export function matchNoteChanges(note: NoteSummary, detail: NoteRound, decisions: Record<string, string>): NoteChange[] {
+  const anchor = noteAnchor(note), quote = normalize(note.quote);
+  const units = new Map(detail.units.map(unit => [unit.id, unit]));
+  const blocks = [...detail.sections.flatMap(section => section.blocks.filter(block => block.kind === 'change')), ...detail.loose];
+  const matches = blocks.flatMap<NoteChange>((block, stop) => {
+    const known = (block.units || []).map(id => units.get(id)).filter(unit => Boolean(unit) && unit!.old_text !== unit!.new_text) as NoteRound['units'];
+    // Manuscript positions refer to the source before this revision. Review
+    // selections refer to an older proposal, so prefer their actual PDF text.
+    const source = anchor.round === undefined ? known.filter(unit => anchor.sources.some(place => place.file === unit.file && Math.max(place.lines[0], unit.old_start) <= Math.min(place.lines[1], unit.old_end))) : [];
+    const sameFiles = anchor.sources.length ? known.filter(unit => anchor.sources.some(place => place.file === unit.file)) : known;
+    const textMatches = quote.length >= 4 && sameFiles.length && (
+      normalize((block.before || []).flatMap(strip => (strip.text || []).map(line => line.text)).join(' ')).includes(quote)
+      || sameFiles.some(unit => normalize(unit.old_text).includes(quote))
+    );
+    // Region notes can lack text; their location is the remaining evidence.
+    const region = !quote && anchor.round !== undefined ? sameFiles.filter(unit => anchor.sources.some(place => place.file === unit.file && Math.max(place.lines[0], unit.old_start) <= Math.min(place.lines[1], unit.old_end))) : [];
+    const matched = source.length ? source : textMatches ? sameFiles : region;
+    if (!matched.length) return [];
+    const states = new Set(matched.map(unit => decisions[unit.id] || 'pending'));
+    const state = states.size === 1 ? [...states][0] : 'mixed';
+    return [{ round: detail.round, stop, units: matched.map(unit => unit.id), state: (['accepted', 'rejected', 'commented', 'mixed'].includes(state) ? state : 'pending') as NoteChange['state'], match: source.length || region.length ? 'source' : 'quote' }];
+  });
+  // A short word needs a unique location. This also covers titles and macros:
+  // SyncTeX may point at their use while the changed definition lives elsewhere.
+  return quote.length < 8 && matches.filter(match => match.match === 'quote').length > 1 ? matches.filter(match => match.match === 'source') : matches;
+}
+
+export async function taskNoteResults(task: Task, items: TimelineItem[], runs: Run[], client: ReviewClient): Promise<NoteResult[]> {
+  const byId = new Map(runs.map(run => [run.id, run]));
+  const results: NoteResult[] = items.filter(item => item.type === 'user').flatMap(item => (item.meta?.notes || readNotesInstructions(item.text) || []).map((note, index) => {
+    const run = byId.get(item.id), origin = noteAnchor(note).round || 0;
+    // Older runs predate recorded round IDs. Only their finished review is a
+    // fallback; a queued follow-up must never point at an earlier proposal.
+    const legacy = (!run || run.status === 'completed') && !['running', 'queued', 'building', 'waiting_input'].includes(task.status) && (task.review?.round || 0) > origin;
+    const round = run?.reviewRound !== undefined ? run.reviewRound || null : legacy ? task.review?.round || null : null;
+    return { key: `${item.id}:${index}`, itemId: item.id, index, note, round, state: round ? 'unmatched' : run?.reviewRound === null ? 'unmatched' : 'waiting', changes: [] } as NoteResult;
+  }));
+  const rounds = [...new Set(results.flatMap(result => result.round ? [result.round] : []))];
+  await Promise.all(rounds.map(async round => {
+    const [detail, state] = await Promise.all([
+      client.get<NoteRound>(`${taskRoute(task.name)}/rounds/${round}`),
+      client.get<{ decisions: Record<string, string> }>(`${taskRoute(task.name)}/state?round=${round}`),
+    ]);
+    if (detail.round !== round || !Array.isArray(detail.units) || !Array.isArray(detail.sections) || !Array.isArray(detail.loose) || !state.decisions) throw new Error('The review could not be loaded. Try again.');
+    for (const result of results.filter(result => result.round === round)) {
+      result.changes = matchNoteChanges(result.note, detail, state.decisions);
+      result.state = result.changes.length ? 'matched' : 'unmatched';
+    }
+  }));
+  return results;
+}

package/src/server/paths.ts

@@ -12,4 +12,4 @@ export const OWNER = process.env.PAPILOT_TAILSCALE_LOGIN || '';
 // port its proxy (such as Tailscale Funnel) forwards to.
 export const ACCOUNT_ORIGIN = (process.env.PAPILOT_ACCOUNT_ORIGIN || '').replace(/\/+$/, '');
 export const ACCOUNT_PORT = Number(process.env.PAPILOT_ACCOUNT_PORT || 7275);
-export const CODEX = process.env.PAPILOT_CODEX_BIN || (fs.existsSync(path.join(ROOT, 'node_modules/.bin/codex')) ? path.join(ROOT, 'node_modules/.bin/codex') : 'codex');
+export const CODEX = process.env.PAPILOT_CODEX_BIN || [path.join(ROOT, 'node_modules/.bin/codex'), path.join(DATA, 'agents/node_modules/.bin/codex')].find(file => fs.existsSync(file)) || 'codex';

package/src/server/references.ts

@@ -0,0 +1,70 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { DATA, ENGINE } from './paths.ts';
+import type { BibliographyReport, ReferenceCheck } from '../shared.ts';
+
+// The paper's bibliography as it is on disk: every entry, where it is cited,
+// what is wrong with it without looking anything up, and, when asked, what
+// Crossref, DBLP and arXiv know of it. The review engine does the reading
+// and the checking; this is where Papilot asks it for a whole paper.
+
+type Input = { id: string; type: string; fields: Record<string, string> };
+interface Checker { lookup(inputs: Input[], start?: boolean, retry?: boolean): { checks: Record<string, ReferenceCheck>; pending: number } }
+interface Confirmations { of(inputs: Input[]): Record<string, number>; set(input: Input, on: boolean): void }
+interface Engine {
+  bibliographyReport(doc: { read(file: string): string | null; main: string; outline: unknown[] }): Omit<BibliographyReport, 'checks' | 'pending' | 'confirmed'>;
+  buildOutline(read: (file: string) => string | null, main: string): { nodes: unknown[] };
+  loadProject(root: string): { main: string };
+  checkInputs(references: unknown): Input[];
+  entryInputs(entries: unknown[]): Input[];
+  ReferenceChecker: new (file: string) => Checker;
+  Confirmations: new (file: string) => Confirmations;
+  Repo: new (root: string) => { gitDir(): string };
+}
+let loading: Promise<Engine> | undefined;
+export function engine(): Promise<Engine> {
+  const load = (file: string) => import(pathToFileURL(path.join(ENGINE, file)).href);
+  loading ??= Promise.all(['src/core/bib.ts', 'src/core/outline.ts', 'src/core/config.ts', 'src/core/verify.ts', 'src/core/git.ts'].map(load))
+    .then(modules => Object.assign({}, ...modules) as Engine);
+  return loading;
+}
+// One checker for Papilot's process, so that the panel and the review of
+// uncommitted changes share their results and their file.
+let shared: Checker | undefined;
+export async function checker(file = path.join(DATA, 'references.json')) {
+  const { ReferenceChecker } = await engine();
+  return shared ??= new ReferenceChecker(file);
+}
+// Confirmations live beside the review engine's state in the repository, so
+// a task's review, the uncommitted changes and this panel agree.
+export async function confirmations(root: string) {
+  const { Confirmations, Repo } = await engine();
+  return new Confirmations(path.join(new Repo(root).gitDir(), 'latex-review', 'references-confirmed.json'));
+}
+
+// A file of the paper, read only from inside its folder.
+function reader(root: string) {
+  const base = fs.realpathSync(root);
+  return (file: string) => {
+    const target = path.resolve(base, file);
+    if (target !== base && !target.startsWith(base + path.sep)) return null;
+    try { return fs.statSync(target).isFile() ? fs.readFileSync(target, 'utf8') : null; } catch { return null; }
+  };
+}
+async function report(root: string) {
+  const mods = await engine(), read = reader(root), { main } = mods.loadProject(root);
+  const result = mods.bibliographyReport({ read, main, outline: mods.buildOutline(read, main).nodes });
+  return { result, inputs: mods.entryInputs(result.entries) };
+}
+
+export async function bibliography(root: string, check: boolean, retry: boolean): Promise<BibliographyReport> {
+  const { result, inputs } = await report(root);
+  return { ...result, ...(await checker()).lookup(inputs, check, retry), confirmed: (await confirmations(root)).of(inputs) };
+}
+export async function confirmEntry(root: string, id: string, on: boolean) {
+  const { inputs } = await report(root), input = inputs.find(item => item.id === id);
+  if (!input) throw Object.assign(new Error('This entry is no longer in the bibliography. Refresh the panel.'), { statusCode: 404 });
+  (await confirmations(root)).set(input, on);
+  return bibliography(root, false, false);
+}

package/src/server/review.ts

@@ -3,7 +3,7 @@ import path from 'node:path';
 import { ENGINE } from './paths.ts';
 import type { Task } from '../shared.ts';
 
-export interface ReviewClient { url: string; token: string; get<T = any>(route: string): Promise<T>; post<T = any>(route: string, body?: unknown): Promise<T> }
+export interface ReviewClient { url: string; token: string; get<T = any>(route: string): Promise<T>; post<T = any>(route: string, body?: unknown, timeout?: number): Promise<T> }
 let modulePromise: Promise<{ connect: (root: string) => Promise<ReviewClient> }> | null = null;
 export function reviewClient(root: string): Promise<ReviewClient> {
   modulePromise ??= import(pathToFileURL(path.join(ENGINE, 'src/client.ts')).href);
@@ -34,14 +34,15 @@ export function annotateReviewState(value: unknown, tasks: readonly Task[]): unk
   else if (task.status === 'building') decisionLock = 'Wait for the PDF build before accepting, rejecting, or undoing changes. You can still add comments.';
   else if (['running', 'waiting_input', 'queued'].includes(task.status)) decisionLock = 'Pause this task in Papilot before accepting, rejecting, or undoing changes. You can still add comments.';
   else if (task.pendingReview) decisionLock = 'Wait for the current review action before accepting, rejecting, or undoing changes. You can still add comments.';
+  else if (task.delivery?.pending) decisionLock = 'Retry the pending delivery before changing review decisions.';
   // Changes the task has committed are in the manuscript too.
-  return { ...state, decisionLock, commentLock, ...(task ? { workspace: { status: task.status, unsent: task.review?.unsent ?? 0, comments: task.review?.comments ?? 0, accepted: task.review?.files?.length || (task.committed ? 1 : 0) } } : {}) };
+  return { ...state, decisionLock, commentLock, ...(task ? { workspace: { status: task.status, unsent: task.review?.unsent ?? 0, comments: task.review?.comments ?? 0, accepted: task.review?.files?.length || (task.committed || task.delivery?.pending ? 1 : 0), ...(task.deferred ? { deferred: true, destination: task.delivery?.pending ? undefined : task.delivery?.mode } : {}) } } : {}) };
 }
 
 export function allowedReviewRoute(route: string, method: string): boolean {
   if (route.includes('..') || route.includes('\\')) return false;
-  if (method === 'GET') return route === '/' || /^\/(assets|fonts)\/[\w./-]+$/.test(route) || /^\/files\/[a-z0-9-]+\/\d+\/[\w.-]+\.(svg|png)$/.test(route) || /^\/api\/tasks(?:\/[a-z0-9-]+(?:\/(?:state|rounds\/\d+(?:\/effects\/c\d+)?))?)?$/.test(route);
-  return method === 'POST' && /^\/api\/tasks\/[a-z0-9-]+\/(decide|undo|note)$/.test(route);
+  if (method === 'GET') return route === '/' || /^\/(assets|fonts)\/[\w./-]+$/.test(route) || /^\/files\/[a-z0-9-]+\/\d+\/[\w.-]+\.(svg|png)$/.test(route) || /^\/api\/tasks(?:\/[a-z0-9-]+(?:\/(?:state|rounds\/\d+(?:\/effects\/c\d+|\/references)?))?)?$/.test(route);
+  return method === 'POST' && /^\/api\/tasks\/[a-z0-9-]+\/(decide|undo|note|rounds\/\d+\/references\/confirm)$/.test(route);
 }
 export function rewriteReviewJson(value: any, prefix: string): any {
   if (Array.isArray(value)) return value.map(v => rewriteReviewJson(v, prefix));
@@ -49,6 +50,6 @@ export function rewriteReviewJson(value: any, prefix: string): any {
   return typeof value === 'string' && value.startsWith('/files/') ? prefix + value : value;
 }
 export function rewriteReviewHtml(html: string, prefix: string): string {
-  const script = `<script>(()=>{const prefix=${JSON.stringify(prefix)};if(!crypto.randomUUID)crypto.randomUUID=()=>Array.from(crypto.getRandomValues(new Uint8Array(20)),b=>b.toString(16).padStart(2,'0')).join('');const original=window.fetch;window.fetch=(input,init)=>{if(typeof input==='string'&&input.startsWith('/api/'))input=prefix+input;return original.call(window,input,init)};})();</script>`;
+  const script = `<script>(()=>{const prefix=${JSON.stringify(prefix)};if(!crypto.randomUUID)crypto.randomUUID=()=>Array.from(crypto.getRandomValues(new Uint8Array(20)),b=>b.toString(16).padStart(2,'0')).join('');const original=window.fetch;window.fetch=(input,init)=>{if(typeof input==='string'&&input.startsWith('/api/'))input=prefix+input;const response=original.call(window,input,init);if(typeof input==='string'&&/\\/rounds\\/\\d+$/.test(input))response.then(r=>{if(r.ok)r.clone().json().then(value=>{if(Number.isSafeInteger(value.round)){document.documentElement.dataset.papilotRound=String(value.round);window.dispatchEvent(new Event('papilot:round'));}}).catch(()=>{});}).catch(()=>{});return response};})();</script>`;
   return html.replace(/(src|href)="\/(?!\/)/g, `$1="${prefix}/`).replace('<head>', '<head>' + script);
 }

package/src/server/skill-library.ts

@@ -0,0 +1,113 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash, randomUUID } from 'node:crypto';
+import type { Skill } from '../shared.ts';
+import type { LibrarySkill, SkillCatalog, SkillDocument } from '../skill-library.ts';
+import { listedSkill, loadSkills, parseSkill, skillFiles, skillRoots, SKILL_NAME } from './skills.ts';
+
+const fail = (message: string, statusCode = 400): never => { throw Object.assign(new Error(message), { statusCode }); };
+const revision = (content: string) => createHash('sha256').update(content).digest('hex');
+
+export class SkillLibrary {
+  constructor(readonly roots = skillRoots) {}
+
+  private root(source: unknown, paper?: string): string {
+    if (source === 'builtin' || source === 'user') return this.roots[source];
+    if (source === 'paper' && paper) return path.join(paper, '.papilot', 'skills');
+    return fail('Choose a valid skill location.');
+  }
+
+  private id(id: unknown): asserts id is string {
+    if (typeof id !== 'string' || !SKILL_NAME.test(id)) fail('Use 1–48 lowercase letters, numbers or hyphens for the skill ID.');
+  }
+
+  // A paper can contain links checked into Git. Never follow those links when
+  // editing, nor replace a linked SKILL.md or a hard link to another file.
+  private safe(file: string, root: string, paper?: string) {
+    const boundary = path.resolve(paper || root), target = path.resolve(file);
+    if (!target.startsWith(boundary + path.sep)) fail('The skill must stay inside its skills folder.');
+    let cursor = boundary;
+    for (const part of ['', ...path.relative(boundary, target).split(path.sep)]) {
+      if (part) cursor = path.join(cursor, part);
+      try {
+        const stat = fs.lstatSync(cursor);
+        if (stat.isSymbolicLink() || (stat.isFile() && stat.nlink > 1)) fail('Linked skill files cannot be edited here.');
+      } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
+    }
+  }
+
+  private locate(source: unknown, id: unknown, paper?: string) {
+    this.id(id);
+    const root = this.root(source, paper);
+    const matches = skillFiles(root).filter(file => file.id === id && fs.existsSync(file.file));
+    if (matches.length > 1) fail('Several files use this skill ID. Give them distinct names before editing.');
+    const found = matches[0];
+    const file = found?.file || path.join(root, id, 'SKILL.md');
+    this.safe(file, root, source === 'paper' ? paper : undefined);
+    return { root, file, found };
+  }
+
+  list(paper?: string): SkillCatalog {
+    const effective = new Map(loadSkills(paper, this.roots).map(skill => [skill.id, skill.source]));
+    const skills: LibrarySkill[] = [];
+    const sources: Skill['source'][] = ['user', ...(paper ? ['paper' as const] : []), 'builtin'];
+    for (const source of sources) {
+      const root = this.root(source, paper);
+      const unique = new Map<string, LibrarySkill>();
+      for (const { id, file } of skillFiles(root)) {
+        try {
+          this.safe(file, root, source === 'paper' ? paper : undefined);
+          const skill = parseSkill(fs.readFileSync(file, 'utf8'), id, source);
+          if (skill) unique.set(id, { ...listedSkill(skill), location: file, active: effective.get(id) === source });
+        } catch { /* A broken or linked file is not editable from the library. */ }
+      }
+      skills.push(...unique.values());
+    }
+    return { skills, locations: { user: this.roots.user, ...(paper ? { paper: this.root('paper', paper) } : {}) } };
+  }
+
+  read(source: unknown, id: unknown, paper?: string): SkillDocument {
+    const { file, found } = this.locate(source, id, paper);
+    if (!found) return fail('This skill no longer exists. Refresh the list.', 404);
+    const content = fs.readFileSync(file, 'utf8');
+    const skill = parseSkill(content, id as string, source as Skill['source']);
+    if (!skill) return fail('The skill must contain instructions.');
+    return { ...listedSkill(skill), location: file, active: loadSkills(paper, this.roots).find(item => item.id === id)?.source === source, content, revision: revision(content) };
+  }
+
+  save(source: unknown, id: unknown, content: unknown, expected: unknown, paper?: string): SkillDocument {
+    if (source === 'builtin') fail('Built-in skills are read-only. Make a custom copy first.', 403);
+    if (typeof content !== 'string' || content.length > 20000 || !content.trim() || content.includes('\0')) fail('Skill content must contain 1–20,000 characters.');
+    if (!parseSkill(content as string, String(id), source as Skill['source'])) fail('The skill must contain instructions.');
+    const { file, found } = this.locate(source, id, paper);
+    if (found) {
+      if (expected === undefined) fail('A skill with this ID already exists in this location.', 409);
+      if (typeof expected !== 'string' || revision(fs.readFileSync(file, 'utf8')) !== expected) fail('This skill changed on disk. Reload it before saving.', 409);
+    } else if (expected !== undefined) fail('This skill was removed on disk. Reload the list before saving.', 409);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    if (!found) {
+      // Exclusive creation also prevents a retry from silently overwriting a
+      // file created by another browser since the catalog was loaded.
+      try { fs.writeFileSync(file, content as string, { flag: 'wx', mode: 0o600 }); }
+      catch (error) { if ((error as NodeJS.ErrnoException).code === 'EEXIST') fail('A skill with this ID already exists in this location.', 409); throw error; }
+    } else {
+      const temporary = path.join(path.dirname(file), `.skill-${randomUUID()}.tmp`);
+      try {
+        fs.writeFileSync(temporary, content as string, { flag: 'wx', mode: fs.statSync(file).mode & 0o777 });
+        fs.renameSync(temporary, file);
+      } finally { fs.rmSync(temporary, { force: true }); }
+    }
+    return this.read(source, id, paper);
+  }
+
+  remove(source: unknown, id: unknown, expected: unknown, paper?: string) {
+    if (source === 'builtin') fail('Built-in skills are read-only.', 403);
+    const { file, found } = this.locate(source, id, paper);
+    if (!found) return fail('This skill no longer exists. Refresh the list.', 404);
+    if (typeof expected !== 'string' || revision(fs.readFileSync(file, 'utf8')) !== expected) fail('This skill changed on disk. Reload it before deleting.', 409);
+    fs.unlinkSync(file);
+    // Keep supporting scripts and references. Only remove an empty skill folder.
+    if (found.folder && fs.readdirSync(found.folder).length === 0) fs.rmdirSync(found.folder);
+    return { ok: true };
+  }
+}

package/src/server/skill-routes.ts

@@ -0,0 +1,19 @@
+import type { FastifyInstance } from 'fastify';
+import type { Store } from './store.ts';
+import { SkillLibrary } from './skill-library.ts';
+
+export function registerSkillRoutes(app: FastifyInstance, store: Store, library = new SkillLibrary()) {
+  const owner = { config: { access: 'owner' as const } };
+  function paper(id: unknown) {
+    if (id === undefined) return undefined;
+    if (typeof id !== 'string' || !id || id.length > 64) throw new Error('Select a valid paper.');
+    return store.project(id).path;
+  }
+  type Query = { projectId?: string };
+  type Params = { source: string; id: string };
+  type Body = { projectId?: string; content?: unknown; revision?: unknown };
+  app.get<{ Querystring: Query }>('/api/skills', owner, request => library.list(paper(request.query.projectId)));
+  app.get<{ Params: Params; Querystring: Query }>('/api/skills/:source/:id', owner, request => library.read(request.params.source, request.params.id, paper(request.query.projectId)));
+  app.post<{ Params: Params; Body: Body }>('/api/skills/:source/:id', owner, request => library.save(request.params.source, request.params.id, request.body?.content, request.body?.revision, paper(request.body?.projectId)));
+  app.post<{ Params: Params; Body: Body }>('/api/skills/:source/:id/remove', owner, request => library.remove(request.params.source, request.params.id, request.body?.revision, paper(request.body?.projectId)));
+}