papilot 0.7.0 → 0.8.1

package/src/server/app.ts

@@ -11,13 +11,14 @@ import type { Workspace } from './workspace.ts';
 import { reviewClient, allowedReviewRoute, rewriteReviewHtml, rewriteReviewJson, confirmedReviewResponse, annotateReviewState } from './review.ts';
 import { fullPaperPdf } from './pdf.ts';
 import { type ChangeAction, Changes } from './changes.ts';
+import { bibliography, confirmEntry } from './references.ts';
 import { type Manuscripts, annotationTask, gitAbortMerge, gitCommit, gitDiff, gitFinishMerge, gitIgnore, gitRemote, gitStage, gitStatus } from './manuscript.ts';
 import { gitOverview } from './git-overview.ts';
 import { findSkill, listedSkill, loadSkills } from './skills.ts';
 import { registerSkillRoutes } from './skill-routes.ts';
 import { type Accounts, deviceLabel } from './accounts.ts';
 import { type Listener, Limiter, SESSION_COOKIE, author, clientAddress, cookie, equals, guard, viewerInfo, visibleSnapshot } from './access.ts';
-import type { AgentId, NoteSummary } from '../shared.ts';
+import { isTaskReviewNote, isAgentId, type AgentId, type NoteSummary } from '../shared.ts';
 import { parseGroupInstructions } from '../note-groups.ts';
 import { taskNoteResults } from './note-results.ts';
 import type { Updates } from './updates.ts';
@@ -26,6 +27,8 @@ import { attachmentIds } from './attachments.ts';
 import { MAX_ATTACHMENT_BYTES } from '../shared.ts';
 import { PaperImports } from './imports.ts';
 import { DATA } from './paths.ts';
+import { registerAiReviewRoutes } from './ai-review-routes.ts';
+import { detectLatex } from './latex-installation.ts';
 
 const importsByWorkspace = new WeakMap<Workspace, PaperImports>();
 
@@ -46,6 +49,7 @@ export interface AppOptions {
   connectReview?: typeof reviewClient;
   updates?: Updates; updateInstance?: string;
   diagnostics?: Diagnostics;
+  latexInstallation?: typeof detectLatex;
 }
 
 // One Fastify instance per listener: the local one for this host and Tailscale,
@@ -82,6 +86,7 @@ export async function buildApp(options: AppOptions) {
   app.setErrorHandler((error, _request, reply) => { reply.code((error as any).statusCode || 400).send({ error: (error as Error).message }); });
   const any = { config: { access: 'any' as const } }, owner = { config: { access: 'owner' as const } };
   registerSkillRoutes(app, store);
+  registerAiReviewRoutes(app, store, manuscripts);
   const project = (param = 'id') => ({ config: { access: { project: param } } }), task = { config: { access: { task: 'id' } } };
   app.addContentTypeParser('application/octet-stream', { parseAs: 'buffer' }, (_request, body, done) => done(null, body));
   app.post<{ Params: { id: string } }>('/api/projects/:id/attachments', { ...project(), bodyLimit: MAX_ATTACHMENT_BYTES }, async request => {
@@ -104,7 +109,7 @@ export async function buildApp(options: AppOptions) {
   });
   async function manuscript(id: string, rebuild = false) {
     const root = store.project(id).path, state = await manuscripts.state(id, root, rebuild);
-    const notes = store.annotations(id).filter(note => !note.taskId);
+    const notes = store.annotations(id).filter(note => !note.taskId && !isTaskReviewNote(note));
     const placed = await manuscripts.reanchor(id, root, notes).catch(() => notes);
     for (let index = 0; index < notes.length; index++) {
       const next = placed[index], previous = notes[index];
@@ -115,11 +120,12 @@ export async function buildApp(options: AppOptions) {
       if (current.deletedAt !== undefined || current.taskId || workspace.annotationSending(current.id) || current.key !== previous.key) continue;
       store.saveAnnotation({ ...current, key: next.key, page: next.page, rect: next.rect, rects: next.rects, source: next.source });
     }
-    return { ...state, annotations: store.annotations(id).filter(note => !note.taskId) };
+    return { ...state, annotations: store.annotations(id).filter(note => !note.taskId && !isTaskReviewNote(note)) };
   }
 
   app.get('/api/health', { config: { access: 'public' } }, () => listener.kind === 'local' ? { ok: true, name: 'Papilot', version: options.version, ...(options.updateInstance ? { updateInstance: options.updateInstance } : {}) } : { ok: true });
   // Environment reports include host paths, so they are available only to the owner.
+  app.get('/api/runtime/latex', owner, () => (options.latexInstallation || detectLatex)());
   app.post('/api/diagnostics', owner, request => {
     const projectId = (request.body as { projectId?: unknown } | null)?.projectId;
     if (projectId !== undefined && (typeof projectId !== 'string' || !projectId || projectId.length > 64)) throw new Error('Select a valid paper to check.');
@@ -129,6 +135,7 @@ export async function buildApp(options: AppOptions) {
   if (options.updates) {
     app.get('/api/updates', owner, () => options.updates!.snapshot());
     app.post('/api/updates/check', owner, () => options.updates!.check());
+    app.post('/api/updates/channel', owner, request => options.updates!.setChannel((request.body as any)?.channel));
     app.post('/api/updates/install', owner, request => options.updates!.install((request.body as any)?.version));
   }
   app.get('/api/state', any, request => {
@@ -162,7 +169,7 @@ export async function buildApp(options: AppOptions) {
   });
   app.post('/api/reconnect', any, async request => { await workspace.connect(); return visibleSnapshot(workspace.snapshot(), request.viewer!, accounts).runtime; });
   // Codex names reasoning efforts with short lowercase words (low, medium, xhigh).
-  const agent = (value: unknown): AgentId => value === 'claude' || value === 'antigravity' ? value : 'codex';
+  const agent = (value: unknown): AgentId => isAgentId(value) ? value : 'codex';
   const effort = (value: unknown) => typeof value === 'string' && /^[a-z]{1,16}$/.test(value) ? value : undefined;
   const string = (value: unknown, name: string, max = 20000) => { if (typeof value !== 'string' || !value.trim() || value.length > max) throw new Error(`${name} is required and must be no longer than ${max} characters.`); return value.trim(); };
   // Adding a paper names a folder on this host, so only the owner can.
@@ -193,7 +200,17 @@ export async function buildApp(options: AppOptions) {
     const b = request.body as any, skill = skillName(b.skill), attachments = attachmentIds(b.attachments);
     if (b.annotations !== undefined && (!Array.isArray(b.annotations) || b.annotations.length > 500 || b.annotations.some((id: unknown) => typeof id !== 'string' || !id || id.length > 64))) throw new Error('Select valid PDF notes for this task.');
     const annotations: string[] | undefined = b.annotations, groupInstructions = parseGroupInstructions(b.groupInstructions);
-    return workspace.createTask({ id: string(b.id, 'Task ID', 64), projectId: string(b.projectId, 'Paper ID', 64), title: string(b.title, 'Task title', 120), instructions: skill || annotations?.length || attachments.length ? optional(b.instructions, 'Instructions') : string(b.instructions, 'Instructions'), skill, annotations, groupInstructions, attachments, agent: agent(b.agent), model: typeof b.model === 'string' ? b.model : undefined, effort: effort(b.effort), serviceTier: b.serviceTier === undefined ? undefined : string(b.serviceTier, 'Service tier', 64), author: author(request.viewer) });
+    const id = string(b.id, 'Task ID', 64), projectId = string(b.projectId, 'Paper ID', 64);
+    // A skill submission uses the same task endpoint and retry identity as editing.
+    const existing = store.tasks().find(task => task.id === id);
+    if (existing) { if (existing.projectId !== projectId) throw new Error('Invalid task ID.'); return existing; }
+    let reviewInput;
+    if (skill && findSkill(store.project(projectId).path, skill).output === 'annotations') {
+      const state = await manuscripts.state(projectId, store.project(projectId).path);
+      if (!state.build.current || !state.build.key) throw new Error('Wait for the current PDF to finish compiling, then run the AI review skill.');
+      reviewInput = await manuscripts.reviewSnapshot(projectId, state.build.key);
+    }
+    return workspace.createTask({ id, projectId, reviewInput, ...(reviewInput ? { prepareReview: (worktree: string) => manuscripts.captureReview(store.project(projectId).path, worktree, id) } : {}), title: string(b.title, 'Task title', 120), instructions: skill || annotations?.length || attachments.length ? optional(b.instructions, 'Instructions') : string(b.instructions, 'Instructions'), skill, annotations, groupInstructions, attachments, agent: agent(b.agent), model: typeof b.model === 'string' ? b.model : undefined, effort: effort(b.effort), serviceTier: b.serviceTier === undefined ? undefined : string(b.serviceTier, 'Service tier', 64), author: author(request.viewer) });
   });
   app.get<{ Params: { id: string } }>('/api/tasks/:id/items', task, request => { store.task(request.params.id); return { items: store.items(request.params.id) }; });
   app.get<{ Params: { id: string } }>('/api/tasks/:id/note-results', task, async request => {
@@ -238,8 +255,9 @@ export async function buildApp(options: AppOptions) {
     return serialGit(task.projectId, () => workspace.commitTask(task.id, string(b.message, 'Commit message', 5000), b.push === true));
   });
   app.post<{ Params: { id: string } }>('/api/tasks/:id/apply', task, request => {
-    const task = store.task(request.params.id);
-    return serialGit(task.projectId, () => workspace.deliverTask(task.id, 'workspace'));
+    const task = store.task(request.params.id), b = (request.body || {}) as any;
+    if (b.mode !== undefined && b.mode !== 'workspace' && b.mode !== 'commit') throw new Error('Invalid save action.');
+    return serialGit(task.projectId, () => workspace.applyTask(task.id, b.keepCurrent === true, b.mode === 'commit' ? { message: string(b.message, 'Commit message', 5000), ...(b.confirmation ? { confirmation: string(b.confirmation, 'Commit confirmation', 100) } : {}) } : undefined));
   });
   app.post<{ Params: { id: string } }>('/api/tasks/:id/commit-message', task, request => {
     const task = store.task(request.params.id);
@@ -247,7 +265,7 @@ export async function buildApp(options: AppOptions) {
   });
   app.post<{ Params: { id: string } }>('/api/tasks/:id/rename', task, request => workspace.renameTask(request.params.id, string((request.body as any).title, 'Task title', 120)));
   app.post<{ Params: { id: string } }>('/api/tasks/:id/title', task, request => workspace.requestTaskTitle(request.params.id));
-  app.post<{ Params: { id: string } }>('/api/tasks/:id/archive', task, request => workspace.archiveTask(request.params.id, (request.body as any)?.archived));
+  app.post<{ Params: { id: string } }>('/api/tasks/:id/archive', task, request => workspace.archiveTask(request.params.id, (request.body as any)?.archived, (request.body as any)?.finished === true));
   app.post<{ Params: { id: string } }>('/api/tasks/:id/interrupt', task, request => workspace.interrupt(request.params.id).then(() => ({ ok: true })));
   // Deleting a task that is already gone succeeds, so a lost response can be retried; a hidden one answers the same.
   app.post<{ Params: { id: string } }>('/api/tasks/:id/delete', { config: { access: { task: 'id' }, hidden: 'ok' } }, request => {
@@ -262,6 +280,14 @@ export async function buildApp(options: AppOptions) {
   // The manuscript preview compiles each paper's working tree and reports its Git status.
   app.get<{ Params: { id: string } }>('/api/projects/:id/manuscript', project(), request => manuscript(request.params.id));
   app.get<{ Params: { id: string } }>('/api/projects/:id/git', project(), request => gitStatus(store.project(request.params.id).path));
+  // The paper's bibliography: every entry, its problems and its online check.
+  app.get<{ Params: { id: string }; Querystring: { check?: string; retry?: string } }>('/api/projects/:id/references', project(), request =>
+    bibliography(store.project(request.params.id).path, request.query.check === '1', request.query.retry === '1'));
+  app.post<{ Params: { id: string } }>('/api/projects/:id/references/confirm', project(), request => {
+    const b = request.body as any;
+    if (typeof b?.id !== 'string' || typeof b?.confirmed !== 'boolean') throw new Error('Choose an entry to confirm.');
+    return confirmEntry(store.project(request.params.id).path, b.id, b.confirmed);
+  });
   app.get<{ Params: { id: string } }>('/api/projects/:id/git/overview', project(), request => gitOverview(store.project(request.params.id).path, store.tasks().filter(task => task.projectId === request.params.id)));
   app.get<{ Params: { id: string }; Querystring: { path?: string } }>('/api/projects/:id/git/diff', project(), request => gitDiff(store.project(request.params.id).path, string(request.query.path, 'File path', 4096)));
   const paths = (value: unknown): string[] => Array.isArray(value) ? value.filter((p: unknown): p is string => typeof p === 'string').slice(0, 5000) : [];
@@ -311,7 +337,7 @@ export async function buildApp(options: AppOptions) {
     return serialGit(project.id, async () => { await gitRemote(project.path, action); return { git: await gitStatus(project.path) }; });
   });
   app.post<{ Params: { id: string } }>('/api/projects/:id/manuscript', project(), request => manuscript(request.params.id, true));
-  // Notes marked on the manuscript PDF become one agent task when sent.
+  // Notes are saved for reading; only explicitly attached IDs reach an agent task.
   app.post<{ Params: { id: string } }>('/api/projects/:id/annotations', project(), async request => {
     const project = store.project(request.params.id), b = request.body as any;
     const isRect = (value: unknown): value is [number, number, number, number] => Array.isArray(value) && value.length === 4 && value.every(v => typeof v === 'number' && v >= 0 && v <= 1) && value[2] > value[0] && value[3] > value[1];
@@ -326,6 +352,7 @@ export async function buildApp(options: AppOptions) {
   });
   app.post<{ Params: { id: string; note: string } }>('/api/projects/:id/annotations/:note', project(), request => {
     const note = store.annotation(request.params.id, request.params.note);
+    if (isTaskReviewNote(note)) throw new Error('Manage this suggestion in its AI review task, or copy it to the manuscript first.');
     if (workspace.annotationSending(note.id)) throw new Error('This note is being sent. Wait for the task to finish creating.');
     if (note.taskId) throw new Error('This note has already been sent.');
     if (note.deletedAt !== undefined) throw new Error('This note has been deleted. Undo the deletion first.');
@@ -333,28 +360,34 @@ export async function buildApp(options: AppOptions) {
   });
   app.post<{ Params: { id: string; note: string } }>('/api/projects/:id/annotations/:note/delete', project(), request => {
     const note = store.annotation(request.params.id, request.params.note);
+    if (isTaskReviewNote(note)) throw new Error('Manage this suggestion in its AI review task, or copy it to the manuscript first.');
     if (workspace.annotationSending(note.id)) throw new Error('This note is being sent. Wait for the task to finish creating.');
     store.deleteAnnotation(request.params.id, request.params.note); return { ok: true };
   });
   app.post<{ Params: { id: string; note: string } }>('/api/projects/:id/annotations/:note/restore', project(), request => {
     const note = store.annotation(request.params.id, request.params.note);
+    if (isTaskReviewNote(note)) throw new Error('Manage this suggestion in its AI review task, or copy it to the manuscript first.');
     if (workspace.annotationSending(note.id)) throw new Error('This note is being sent. Wait for the task to finish creating.');
     if (note.taskId) throw new Error('This note has already been sent.');
     const { deletedAt: _, ...restored } = note;
     store.saveAnnotation(restored);
     // Return the original order, including any notes added since the deletion.
-    return { annotations: store.annotations(request.params.id).filter(note => !note.taskId) };
+    return { annotations: store.annotations(request.params.id).filter(note => !note.taskId && !isTaskReviewNote(note)) };
   });
   app.post<{ Params: { id: string } }>('/api/projects/:id/annotations/send', project(), async request => {
     const project = store.project(request.params.id), b = request.body as any, id = string(b.id, 'Task ID', 64);
     // A retried send returns its task; an ID from another paper is refused.
     const existing = store.tasks().find(task => task.id === id);
     if (existing) { if (existing.projectId !== project.id) throw new Error('Invalid task ID.'); return existing; }
-    const notes = store.annotations(project.id).filter(note => !note.taskId);
-    if (!notes.length) throw new Error('Mark a region on the PDF and add a note first.');
+    if (!Array.isArray(b.annotations) || !b.annotations.length || b.annotations.length > 500 || b.annotations.some((id: unknown) => typeof id !== 'string' || !id || id.length > 64)) throw new Error('Select valid PDF notes for this task.');
+    const notes = [...new Set<string>(b.annotations)].map(id => store.annotation(project.id, id));
+    if (notes.some(note => note.taskId || isTaskReviewNote(note) || note.deletedAt !== undefined)) throw new Error('Select unsent PDF notes for this task.');
     const { title } = annotationTask(notes);
     return workspace.createTask({ id, projectId: project.id, title, instructions: '', annotations: notes.map(note => note.id), agent: agent(b.agent), model: typeof b.model === 'string' ? b.model : undefined, effort: effort(b.effort), serviceTier: b.serviceTier === undefined ? undefined : string(b.serviceTier, 'Service tier', 64), author: author(request.viewer) });
   });
+  // What the last build reported, placed on its PDF, and the TeX log itself.
+  app.get<{ Params: { id: string } }>('/api/projects/:id/manuscript/problems', project(), request => manuscripts.problems(store.project(request.params.id).id));
+  app.get<{ Params: { id: string } }>('/api/projects/:id/manuscript/log', project(), (request, reply) => reply.type('text/plain; charset=utf-8').header('Cache-Control', 'no-store').send(manuscripts.log(store.project(request.params.id).id)));
   // Exact word places for the PDF view's selection layer; immutable per build key.
   app.get<{ Params: { id: string }; Querystring: { key?: string; page?: string } }>('/api/projects/:id/manuscript/words', project(), async (request, reply) => {
     const project = store.project(request.params.id);
@@ -423,6 +456,16 @@ export async function buildApp(options: AppOptions) {
   app.get<{ Params: { projectId: string; number: string } }>('/review/:projectId/api/changes/rounds/:number', changesAccess, request => changes.detail(store.project(request.params.projectId).id, Number(request.params.number)));
   app.get<{ Params: { projectId: string; number: string; unit: string } }>('/review/:projectId/api/changes/rounds/:number/effects/:unit', changesAccess, request =>
     changes.effects(store.project(request.params.projectId).id, Number(request.params.number), request.params.unit));
+  app.get<{ Params: { projectId: string; number: string }; Querystring: { check?: string; retry?: string } }>('/review/:projectId/api/changes/rounds/:number/references', changesAccess, request => {
+    const project = store.project(request.params.projectId);
+    return changes.references(project.id, project.path, Number(request.params.number), request.query.check === '1', request.query.retry === '1');
+  });
+  app.post<{ Params: { projectId: string; number: string } }>('/review/:projectId/api/changes/rounds/:number/references/confirm', changesAccess, request => {
+    changesToken(request);
+    const project = store.project(request.params.projectId), b = request.body as any;
+    if (typeof b?.id !== 'string' || typeof b?.confirmed !== 'boolean') throw new Error('Choose an entry to confirm.');
+    return changes.confirmReference(project.id, project.path, Number(request.params.number), b.id, b.confirmed);
+  });
   app.get<{ Params: { projectId: string; number: string; file: string } }>('/review/:projectId/changes/files/:number/:file', changesAccess, (request, reply) => {
     const file = changes.file(store.project(request.params.projectId).id, Number(request.params.number), request.params.file);
     return reply.header('Cache-Control', 'private, max-age=31536000, immutable').type(file.endsWith('.svg') ? 'image/svg+xml' : 'image/png').send(fs.createReadStream(file));
@@ -471,12 +514,11 @@ export async function buildApp(options: AppOptions) {
             const task = store.tasks().find(t => t.projectId === project.id && t.name === route.split('/')[3]);
             if (task) store.updateTask(task.id, { error: 'Review action failed: ' + (value as any).error });
           }
-          if (response.ok && request.method === 'POST') {
+          if (response.ok && request.method === 'POST' && !route.endsWith('/references/confirm')) {
             const task = store.tasks().find(t => t.projectId === project.id && t.name === route.split('/')[3]);
             if (task) {
               store.updateTask(task.id, { error: null });
               await workspace.refresh(task.id);
-              void workspace.prepareCommitMessage(task.id);
             }
           }
           return rewriteReviewJson(value, prefix);

package/src/server/build-problems.ts

@@ -0,0 +1,329 @@
+import { execFile } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { countProblems, type BuildProblem } from '../shared.ts';
+
+const exec = promisify(execFile);
+
+// A problem as read from the log: `input` is the file as TeX named it, relative
+// to the directory TeX ran in. It never leaves the server.
+export interface LogProblem extends Omit<BuildProblem, 'file' | 'page' | 'rect'> { input?: string; occurrence?: number }
+export interface StoredProblem extends BuildProblem { source?: string; occurrence?: number }
+export { countProblems };
+
+// Overfull lines narrower than this are invisible in print.
+export const OVERFULL_MIN_PT = 1;
+// Locating runs SyncTeX once or twice per problem; a paper with more problems
+// lists the rest without a page, starting with the least important.
+export const LOCATE_LIMIT = 300;
+const WRAP = 79;
+const utf8Length = (lead: number) => lead >= 0xf0 ? 4 : lead >= 0xe0 ? 3 : lead >= 0xc0 ? 2 : 1;
+
+// TeX breaks every log line at 79 columns, even inside file names. pdfTeX
+// counts bytes and may cut a character in two, LuaTeX breaks early rather than
+// cut one, and XeTeX counts characters. Rejoin the pieces on the raw bytes,
+// before decoding splits a character for good.
+export function unwrapLog(input: string | Buffer) {
+  const bytes = typeof input === 'string' ? Buffer.from(input, 'utf8') : input, raw: Buffer[] = [];
+  for (let start = 0, i = 0; i <= bytes.length; i++) {
+    if (i < bytes.length && bytes[i] !== 0x0a) continue;
+    raw.push(bytes.subarray(start, i > start && bytes[i - 1] === 0x0d ? i - 1 : i)); start = i + 1;
+  }
+  const out: string[] = [];
+  let parts: Buffer[] = [];
+  for (let k = 0; k < raw.length; k++) {
+    const line = raw[k], next = raw[k + 1], text = line.toString('utf8');
+    parts.push(line);
+    const wrapped = next !== undefined && (line.length === WRAP || [...text].length === WRAP
+      || (line.length < WRAP && next.length > 0 && next[0] >= 0xc0 && line.length + utf8Length(next[0]) > WRAP)
+      // Text decoded earlier keeps each half of a cut character as U+FFFD.
+      || (text.endsWith('�') && next.toString('utf8').startsWith('�')));
+    if (!wrapped) { out.push(parts.length === 1 ? text : Buffer.concat(parts).toString('utf8')); parts = []; }
+  }
+  if (parts.length) out.push(Buffer.concat(parts).toString('utf8'));
+  return out;
+}
+
+const looksLikeFile = (name: string) => /^(?:\.{1,2}\/|\/|[A-Za-z]:[\\/])/.test(name) || /\.[A-Za-z][\w-]*$/.test(name);
+const IGNORED = [
+  /^There (?:were|was) (?:undefined|multiply)/, /^Rerun to get/, /Label\(s\) may have changed/, /^Please (?:\(re\))?run/i,
+  /^Writing or overwriting file/, /^File [`'][^']+' already exists on the system/, /^Unused global option/,
+];
+// The source context TeX prints after an error or a pdfTeX warning: a line
+// starting "<to be read again> " or "l.12 " quotes the source up to where TeX
+// stopped, cut on the left, so its parentheses may be unmatched. The indented
+// line below it continues the source and then TeX's own output, which can
+// close and open files, so that one is read as usual.
+const CONTEXT = /^(?:<[a-z*][a-z ]*>|l\.\d+) /;
+
+// TeX reports which file it is reading with "(name" and ")" around everything
+// that file prints. Only a "(" followed by a file that exists opens a file, so
+// parentheses in messages and wrapped text keep the stack balanced but never
+// change the current file. `opened` collects every file TeX read.
+export function parseLatexLog(text: string | Buffer, isFile: (name: string) => boolean = () => true, opened?: Set<string>): LogProblem[] {
+  const lines = unwrapLog(text), stack: (string | null)[] = [], found: LogProblem[] = [], seen = new Set<string>(), occurrences = new Map<string, number>();
+  const current = () => { for (let i = stack.length - 1; i >= 0; i--) if (stack[i]) return stack[i]!; return undefined; };
+  const add = (problem: LogProblem) => {
+    // Keep the position among all marks on the line, before repeated keys
+    // collapse into one row. In a,a,b, b is still the third printed mark.
+    if (problem.kind === 'reference' || problem.kind === 'citation') {
+      const line = JSON.stringify([problem.input, problem.line, problem.kind]);
+      problem.occurrence = occurrences.get(line) || 0;
+      occurrences.set(line, problem.occurrence + 1);
+    }
+    const id = JSON.stringify([problem.kind, problem.key, problem.input, problem.line, problem.end, problem.message]);
+    if (!seen.has(id)) { seen.add(id); found.push(problem); }
+  };
+  // pdfTeX prints a name with spaces unquoted, so the longest run of the line
+  // that names an existing file wins.
+  const fileAt = (rest: string): [string, number] | null => {
+    const quoted = /^"([^"]+)"/.exec(rest);
+    if (quoted) return isFile(quoted[1]) ? [quoted[1], quoted[0].length] : null;
+    const run = /^[^()[\]{}<>"]*/.exec(rest.slice(0, 512))![0];
+    if (!run || /^\s/.test(run)) return null;
+    const cuts = [run.length];
+    for (let at = run.lastIndexOf(' '); at > 0; at = run.lastIndexOf(' ', at - 1)) cuts.push(at);
+    for (const cut of cuts) {
+      const name = run.slice(0, cut).trimEnd();
+      if (name && looksLikeFile(name) && isFile(name)) return [name, name.length];
+    }
+    return null;
+  };
+  const scan = (line: string) => {
+    for (let j = 0; j < line.length; j++) {
+      if (line[j] === ')') { stack.pop(); continue; }
+      if (line[j] !== '(') continue;
+      const file = fileAt(line.slice(j + 1));
+      if (file) { stack.push(file[0]); opened?.add(file[0]); j += file[1]; } else stack.push(null);
+    }
+  };
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    // file:line: message (from -file-line-error) or "! message". The file may
+    // be one TeX generated, such as the .bbl, which a failed build has removed.
+    const located = /^(.+?):(\d+): (.*)$/.exec(line);
+    const errorFile = located && (located[1] === current() || looksLikeFile(located[1])) ? located : null;
+    if (errorFile || line.startsWith('! ')) {
+      let message = errorFile ? errorFile[3] : line.slice(2), last = line, end = i + 1;
+      for (; end < lines.length && end < i + 20; end++) {
+        const next = lines[end];
+        if (!next.trim() || next.startsWith('! ') || /^(?:See |Type |l\.\d+ |\s*\.\.\.|\s*<)/.test(next) || /^(.+?):(\d+): /.test(next)) break;
+        message += (last.length >= WRAP ? '' : ' ') + next; last = next;
+      }
+      let context = -1;
+      for (let k = end; k < Math.min(lines.length, end + 12); k++) if (/^l\.\d+ /.test(lines[k])) { context = k; break; }
+      message = message.replace(/\s+/g, ' ').trim();
+      if (!/==> Fatal error occurred|^Emergency stop/.test(message)) {
+        const missing = /File [`']([^']+)' not found/.exec(message);
+        add({
+          kind: missing ? 'missing-file' : 'error', severity: 'error', message, key: missing?.[1],
+          input: errorFile ? errorFile[1] : current(), line: errorFile ? Number(errorFile[2]) : context >= 0 ? Number(/^l\.(\d+)/.exec(lines[context])![1]) : undefined,
+        });
+      }
+      // Help text and the quoted source are TeX's commentary, not file events.
+      i = context >= 0 ? context : end - 1;
+      continue;
+    }
+    if (CONTEXT.test(line)) continue;
+    const warning = /^(LaTeX|Package|Class|Module)(?: (\S+))? Warning: (.*)$|^(pdfTeX) warning(?: \([^)]*\))?: (.*)$/i.exec(line);
+    if (warning) {
+      const owner = warning[2] || '', isLatex = warning[1] === 'LaTeX';
+      let message = warning[3] ?? warning[5], end = i + 1;
+      // Continuation lines start with "(package)" or, for LaTeX, an indent.
+      for (; end < lines.length; end++) {
+        const next = lines[end], prefix = owner ? `(${owner})` : '';
+        if (prefix && next.startsWith(prefix)) message += ' ' + next.slice(prefix.length).trim();
+        else if (isLatex && !owner && /^\s{4,}\S/.test(next)) message += ' ' + next.trim();
+        else break;
+      }
+      i = end - 1;
+      message = message.replace(/\s+/g, ' ').trim();
+      if (owner === 'Font' || IGNORED.some(pattern => pattern.test(message))) continue;
+      const inputLine = /on input line (\d+)/.exec(message), at = inputLine ? Number(inputLine[1]) : undefined;
+      const reference = /^Reference [`']([^']+)' on page \S+ undefined/.exec(message);
+      const citation = /Citation [`']([^']+)'(?: on page \S+)? undefined/.exec(message);
+      const label = /^Label [`']([^']+)' multiply defined/.exec(message);
+      if (reference) add({ kind: 'reference', severity: 'warning', message, key: reference[1], input: current(), line: at });
+      else if (citation) add({ kind: 'citation', severity: 'warning', message, key: citation[1], input: current(), line: at });
+      // cleveref repeats every label as `name@cref`; the user's label is reported already.
+      else if (label) { if (!label[1].includes('@')) add({ kind: 'label', severity: 'warning', message, key: label[1] }); }
+      else add({ kind: 'other', severity: 'minor', message: `${owner && !isLatex ? owner + ': ' : ''}${message}`, input: current(), line: at });
+      continue;
+    }
+    const box = /^(Overfull|Underfull) \\([hv])box \((?:badness \d+|([\d.]+)pt too (?:wide|high))\)(?: (?:in paragraph|in alignment|detected) at lines? (\d+)(?:--(\d+))?)?/.exec(line);
+    if (box) {
+      const amount = Number(box[3]);
+      if (box[1] === 'Overfull' && amount >= OVERFULL_MIN_PT && box[4]) {
+        const start = Number(box[4]), stop = box[5] ? Number(box[5]) : start;
+        add({ kind: 'overfull', severity: 'minor', message: line.trim(), amount: Math.round(amount * 10) / 10, input: current(), line: start, end: stop > start ? stop : undefined });
+      }
+      // The box's contents follow until a blank line; their text has stray parentheses.
+      while (i + 1 < lines.length && lines[i + 1].trim()) i++;
+      continue;
+    }
+    const glyph = /^Missing character: There is no (.+?) in font (.+?)!/.exec(line);
+    if (glyph) { add({ kind: 'glyph', severity: 'warning', message: line.trim(), key: glyph[1], input: current() }); continue; }
+    scan(line);
+  }
+  // A remark repeated for many objects, such as pdfTeX's duplicate link
+  // targets for every equation, is listed once with how often it occurred.
+  const remarks = new Map<string, LogProblem>();
+  return found.filter(problem => {
+    if (problem.kind !== 'other') return true;
+    const shape = problem.message.replace(/\{[^{}]*\}/g, '{}').replace(/[`'][^`']*'/g, "''").replace(/\d+/g, '#');
+    const first = remarks.get(shape);
+    if (!first) { remarks.set(shape, problem); return true; }
+    first.count = (first.count || 1) + 1;
+    return false;
+  });
+}
+
+// The log without the build folder's path or the home directory.
+export function scrubLog(text: string, folder?: string, home = os.homedir()) {
+  if (folder && folder.length > 1) text = text.split(folder).join('…');
+  if (home && home.length > 1) text = text.split(home + path.sep).join('~' + path.sep);
+  return text;
+}
+
+const stripComments = (line: string) => line.replace(/(^|[^\\])%.*$/, '$1');
+// Files TeX writes for itself: the build folder and auxiliary files. A problem
+// reported in one has no place in the manuscript to show.
+const GENERATED = new Set(['.aux', '.bbl', '.toc', '.lof', '.lot', '.loa', '.out', '.ind', '.idx', '.gls', '.glo', '.nls', '.brf', '.nav', '.snm', '.vrb', '.bcf']);
+
+// Where every problem is in the manuscript. TeX names files relative to the
+// directory it ran in; the paper shows them relative to its repository, like
+// note sources. A label defined twice is listed at each definition among the
+// files TeX read.
+export function collectProblems(log: string | Buffer, { cwd, root, sourceDir, outDir = path.join(cwd, 'build') }: { cwd: string; root: string; sourceDir: string; outDir?: string }): StoredProblem[] {
+  const resolve = (name: string) => path.resolve(cwd, name), known = new Map<string, boolean>(), opened = new Set<string>();
+  const isFile = (name: string) => {
+    let result = known.get(name);
+    if (result === undefined) { try { result = fs.statSync(resolve(name)).isFile(); } catch { result = false; } known.set(name, result); }
+    return result;
+  };
+  const parsed = parseLatexLog(log, isFile, opened);
+  const display = (absolute: string) => {
+    const relative = path.relative(root, absolute);
+    return relative.startsWith('..') || path.isAbsolute(relative) ? undefined : path.posix.join(sourceDir || '', relative.split(path.sep).join('/'));
+  };
+  const generated = (absolute: string) => {
+    const inside = path.relative(outDir, absolute);
+    return (!inside.startsWith('..') && !path.isAbsolute(inside)) || GENERATED.has(path.extname(absolute)) || absolute.endsWith('.run.xml');
+  };
+  let sources: { file: string; lines: string[] }[] | null = null;
+  const output: StoredProblem[] = [];
+  for (const { input, ...problem } of parsed) {
+    if (problem.kind === 'label' && problem.key) {
+      sources ??= [...opened].map(resolve).filter(file => file.endsWith('.tex') && display(file) !== undefined && !generated(file)).flatMap(file => {
+        try { return [{ file, lines: fs.readFileSync(file, 'utf8').split('\n') }]; } catch { return []; }
+      });
+      const needle = `\\label{${problem.key}}`, places: StoredProblem[] = [];
+      for (const { file, lines } of sources) lines.forEach((text, index) => {
+        if (stripComments(text).includes(needle)) places.push({ ...problem, file: display(file), line: index + 1, source: file });
+      });
+      output.push(...(places.length ? places : [problem]));
+      continue;
+    }
+    const absolute = input ? resolve(input) : undefined;
+    if (absolute && generated(absolute)) {
+      // An error in, say, the .bbl names that file so its cause can be traced.
+      const named = problem.severity === 'error' ? { message: `${path.basename(absolute)}${problem.line ? ':' + problem.line : ''}: ${problem.message}` } : {};
+      output.push({ ...problem, ...named, line: undefined, end: undefined });
+      continue;
+    }
+    const file = absolute ? display(absolute) : undefined;
+    output.push(file ? { ...problem, file, source: absolute } : { ...problem, line: undefined, end: undefined });
+  }
+  return output;
+}
+
+interface SyncBox { page: number; x: number; y: number; h: number; v: number; W: number; H: number }
+// `synctex view` lists one record per box: the point it names (x, y) and the
+// box itself, h from the left and v at its baseline, W wide and H tall.
+export function parseSynctexView(output: string): SyncBox[] {
+  const boxes: Partial<SyncBox>[] = [];
+  for (const line of output.split('\n')) {
+    const match = /^(Page|x|y|h|v|W|H):(-?[\d.]+)\s*$/.exec(line);
+    if (!match) continue;
+    if (match[1] === 'Page') boxes.push({ page: Number(match[2]) - 1 });
+    else if (boxes.length) (boxes[boxes.length - 1] as Record<string, number>)[match[1]] = Number(match[2]);
+  }
+  return boxes.filter((box): box is SyncBox => ['page', 'x', 'y', 'h', 'v', 'W', 'H'].every(key => Number.isFinite((box as Record<string, number>)[key])));
+}
+
+interface PageText { width: number; height: number; bounds: number[]; search(needle: string): number[][] }
+// Put each problem on the page SyncTeX names. A missing reference or citation
+// is narrowed to its "??" or "?" in that line; everything else marks its lines.
+// Which problems get a place when there are more than `limit`: errors, then
+// warnings, then typesetting, each in the order TeX reported them.
+export function placementOrder(problems: Pick<BuildProblem, 'severity'>[], limit = LOCATE_LIMIT) {
+  const rank = { error: 0, warning: 1, minor: 2 };
+  return new Set(problems.map((problem, index) => index).sort((a, b) => rank[problems[a].severity] - rank[problems[b].severity] || a - b).slice(0, limit));
+}
+
+export async function locateProblems(problems: StoredProblem[], pdf: string, openPage: (page: number) => PageText | null, limit = LOCATE_LIMIT): Promise<BuildProblem[]> {
+  // SyncTeX reads its whole file on every call, so each source line is asked once.
+  const views = new Map<string, Promise<SyncBox[]>>();
+  const view = (file: string, line: number) => {
+    const id = `${line}:${file}`;
+    if (!views.has(id)) views.set(id, exec('synctex', ['view', '-i', `${line}:0:${file}`, '-o', pdf], { timeout: 10000 }).then(result => parseSynctexView(result.stdout), () => []));
+    return views.get(id)!;
+  };
+  // TeX warns about the missing references of one source line in reading
+  // order, so the second warning for a line belongs to its second "??".
+  const ordinal = new Map<StoredProblem, number>(), counted = new Map<string, number>();
+  for (const problem of problems) {
+    const id = JSON.stringify([problem.source, problem.line, problem.kind]), seen = counted.get(id) || 0;
+    ordinal.set(problem, problem.occurrence ?? seen); counted.set(id, (problem.occurrence ?? seen) + 1);
+  }
+  const place = async (problem: StoredProblem): Promise<BuildProblem> => {
+    const { source, occurrence: _occurrence, ...shown } = problem;
+    if (!source || !problem.line) return shown;
+    const lines = problem.end && problem.end !== problem.line ? [problem.line, problem.end] : [problem.line];
+    const boxes = (await Promise.all(lines.map(line => view(source, line)))).flat();
+    if (!boxes.length) return shown;
+    const page = boxes[0].page, text = openPage(page);
+    if (!text) return shown;
+    const [x0, y0] = text.bounds;
+    const onPage = boxes.filter(box => box.page === page), tidy = onPage.filter(box => box.H > 0 && box.H < text.height * 0.4);
+    let rect: number[] = tidy.length
+      ? [Math.min(...tidy.map(b => b.h)), Math.min(...tidy.map(b => b.v - b.H)), Math.max(...tidy.map(b => b.h + b.W)), Math.max(...tidy.map(b => b.v))]
+      : [onPage[0].x - 6, onPage[0].y - 10, onPage[0].x + 6, onPage[0].y + 2];
+    rect = [rect[0] + x0, rect[1] + y0, rect[2] + x0, rect[3] + y0];
+    if (problem.kind === 'reference' || problem.kind === 'citation') {
+      // A citation prints "?" or its key; the "?" of a missing reference's "??" is not one.
+      const pairs = text.search('??'), inside = (mark: number[], outer: number[]) => mark[0] >= outer[0] - 0.5 && mark[2] <= outer[2] + 0.5 && mark[1] >= outer[1] - 0.5 && mark[3] <= outer[3] + 0.5;
+      const marks = (problem.kind === 'reference' ? pairs : [...text.search('?').filter(mark => !pairs.some(pair => inside(mark, pair))), ...(problem.key ? text.search(problem.key) : [])])
+        .filter(mark => { const middle = (mark[1] + mark[3]) / 2; return onPage.some(box => middle >= box.v - box.H + y0 - 3 && middle <= box.v + y0 + 3); })
+        .sort((a, b) => Math.abs(a[1] - b[1]) > 2 ? a[1] - b[1] : a[0] - b[0]);
+      // SyncTeX's points sit on the nodes this source line printed, so a mark
+      // that starts beside one came from this line, not from its neighbours.
+      const gap = (mark: number[]) => Math.min(...onPage.map(box => Math.abs(mark[0] - (box.x + x0))));
+      const own = marks.filter(mark => gap(mark) <= 6);
+      let best: number[] | undefined = own[ordinal.get(problem) || 0];
+      if (!best && marks.length) best = marks.reduce((a, b) => gap(b) < gap(a) ? b : a);
+      if (best) rect = [best[0] - 2, best[1] - 1, best[2] + 2, best[3] + 1];
+    }
+    const unit = (value: number) => Math.min(1, Math.max(0, Math.round(value * 10000) / 10000));
+    return { ...shown, page, rect: [unit((rect[0] - x0) / text.width), unit((rect[1] - y0) / text.height), unit((rect[2] - x0) / text.width), unit((rect[3] - y0) / text.height)] };
+  };
+  const output: BuildProblem[] = new Array(problems.length), chosen = placementOrder(problems, limit);
+  let next = 0;
+  await Promise.all(Array.from({ length: 6 }, async () => {
+    for (let index = next++; index < problems.length; index = next++) {
+      if (!chosen.has(index)) { const { source: _source, occurrence: _occurrence, ...shown } = problems[index]; output[index] = shown; continue; }
+      output[index] = await place(problems[index]);
+    }
+  }));
+  return output;
+}
+
+// Errors first, then the paper's order: page, then position on the page.
+export function sortProblems(problems: BuildProblem[]) {
+  const rank = { error: 0, warning: 1, minor: 2 };
+  return [...problems].sort((a, b) => rank[a.severity] - rank[b.severity]
+    || (a.page ?? Infinity) - (b.page ?? Infinity) || (a.rect?.[1] ?? 0) - (b.rect?.[1] ?? 0)
+    || (a.file || '').localeCompare(b.file || '') || (a.line ?? 0) - (b.line ?? 0));
+}

package/src/server/changes.ts

@@ -5,6 +5,7 @@ import { pathToFileURL } from 'node:url';
 import { DATA, ENGINE } from './paths.ts';
 import { mergeInProgress, workingTree } from './manuscript.ts';
 import { ignorePattern } from '../commit.ts';
+import { checker, confirmations, engine as referenceEngine } from './references.ts';
 
 // The manuscript's uncommitted changes as a PDF comparison: the last commit
 // against the working tree, rendered by the review engine like a task's round
@@ -197,14 +198,36 @@ export class Changes {
   detail(projectId: string, number: number) {
     const round = this.comparison(projectId).rounds.find(item => item.number === number);
     if (!round) throw Object.assign(new Error('This comparison is out of date. Refresh the page.'), { statusCode: 404 });
-    let render: { units: Record<string, object>; sections: unknown[]; loose: unknown[] } = { units: {}, sections: [], loose: [] };
-    try { render = JSON.parse(fs.readFileSync(path.join(round.folder, 'round.json'), 'utf8')); } catch { /* An empty comparison has no render. */ }
+    const render = this.rendered(round);
     return {
       task: 'changes', round: round.number, before: round.before, after: round.after,
       units: round.units.map(unit => ({ ...unit, ...render.units[unit.id] })), sections: render.sections, loose: render.loose,
       warnings: round.warnings, files: `/review/${projectId}/changes/files/${round.number}/`,
+      ...(render.references ? { references: render.references } : {}),
     };
   }
+  private rendered(round: Round): { units: Record<string, object>; sections: unknown[]; loose: unknown[]; references?: unknown } {
+    try { return JSON.parse(fs.readFileSync(path.join(round.folder, 'round.json'), 'utf8')); } catch { return { units: {}, sections: [], loose: [] }; } // An empty comparison has no render.
+  }
+
+  // Online checks of the bibliography entries the uncommitted changes add or
+  // modify, as for a task's round; check asks for lookups to start.
+  async references(projectId: string, root: string, number: number, check: boolean, retry: boolean) {
+    const inputs = await this.entryInputs(projectId, number);
+    return { ...(await checker()).lookup(inputs, check, retry), confirmed: (await confirmations(root)).of(inputs) };
+  }
+  // The author checked an entry by hand, or takes that back.
+  async confirmReference(projectId: string, root: string, number: number, id: string, on: boolean) {
+    const inputs = await this.entryInputs(projectId, number), input = inputs.find(item => item.id === id);
+    if (!input) throw Object.assign(new Error('This entry is no longer among the changes. Refresh the page.'), { statusCode: 404 });
+    (await confirmations(root)).set(input, on);
+    return this.references(projectId, root, number, false, false);
+  }
+  private async entryInputs(projectId: string, number: number) {
+    const round = this.comparison(projectId).rounds.find(item => item.number === number);
+    if (!round) throw Object.assign(new Error('This comparison is out of date. Refresh the page.'), { statusCode: 404 });
+    return (await referenceEngine()).checkInputs(this.rendered(round).references);
+  }
 
   // The places a change alters away from its own lines, past those the
   // comparison rendered, rendered into its folder when first asked for.

package/src/server/codex.ts

@@ -34,7 +34,8 @@ export class Codex extends EventEmitter {
     };
     child.once('error', failed);
     child.once('exit', (code, signal) => failed(new Error(`Codex exited (${signal || code})`)));
-    await this.request('initialize', { clientInfo: { name: 'papilot', title: 'Papilot', version: VERSION } });
+    // The experimental API carries the tools Papilot gives its threads (see VIEW_PDF_TOOL).
+    await this.request('initialize', { clientInfo: { name: 'papilot', title: 'Papilot', version: VERSION }, capabilities: { experimentalApi: true } });
     this.notify('initialized', {}); this.ready = true;
   }
   private receive(message: any) {