papilot 0.7.0 → 0.8.0

package/vendor/latex-review/src/core/view.ts

@@ -0,0 +1,204 @@
+import fs from "node:fs";
+import path from "node:path";
+import { randomBytes } from "node:crypto";
+import { Builder, BuildError } from "./build.ts";
+import { cleanPath, within } from "./git.ts";
+import { run } from "./process.ts";
+import { mupdf } from "../render/pdf.ts";
+import type { App } from "../server.ts";
+import type { BuildInfo } from "../types.ts";
+// A task's agent looks at its typeset pages: the task worktree as it is now,
+// or the accepted text, compiled like a review round (only the chapters the
+// edits and the asked-for places need, when the paper allows partial builds,
+// so a later publication of the same text reuses the build) and drawn as PNG
+// files the agent opens with its own image viewer. Places are source lines
+// or printed page numbers.
+export interface ViewRequest {
+  locations?: string[];
+  pages?: string[];
+  accepted?: boolean;
+}
+export interface ViewedPage {
+  /** The page's position in the PDF that was built, from 1. */
+  index: number;
+  /** The PDF's page label, or null when no numbering metadata is available. */
+  label: string | null;
+  /** What asked for this page. */
+  shows: string[];
+  /** The drawn page. */
+  file: string;
+}
+export interface ViewResult {
+  pages: ViewedPage[];
+  /** Requests no page answered, with the reason. */
+  missing: string[];
+  targets: string[];
+  error?: string;
+  details?: string[];
+}
+// Pages drawn per request, and how far from an asked-for line SyncTeX may
+// look for one that printed something (blank lines and comments print nothing).
+const PAGES = 4,
+  NEARBY = 10,
+  SCALE = 1.5,
+  // Drawn pages are kept this long, for an agent that opens them later in its turn.
+  KEEP = 6 * 3600 * 1000;
+// A new folder for one view's pages, after removing the task's old ones.
+function folder(app: App, name: string) {
+  const views = path.join(app.project.state, "views", name);
+  for (const entry of fs.existsSync(views) ? fs.readdirSync(views) : []) {
+    const old = path.join(views, entry);
+    if (Date.now() - fs.statSync(old).mtimeMs > KEEP)
+      fs.rmSync(old, { recursive: true, force: true });
+  }
+  const created = path.join(views, `${Date.now()}-${randomBytes(3).toString("hex")}`);
+  fs.mkdirSync(created, { recursive: true });
+  return created;
+}
+interface Location {
+  asked: string;
+  file: string;
+  line: number;
+}
+function location(app: App, worktree: string, value: string): Location {
+  const match = /^(.*?):(\d+)$/.exec(value.trim());
+  if (!match) throw new Error(`Give a location as file:line, such as main.tex:12: ${value}`);
+  let file = match[1];
+  if (path.isAbsolute(file)) file = path.relative(worktree, file);
+  file = cleanPath(file);
+  if (!within(file, [app.project.source_dir]))
+    throw new Error(`${file} is outside the paper's source folder ${app.project.source_dir}.`);
+  return { asked: value.trim(), file, line: Number(match[2]) };
+}
+// The pages SyncTeX gives for a source line, or for the nearest line that
+// printed something.
+function pagesOf(pdf: string, file: string, line: number) {
+  for (let distance = 0; distance <= NEARBY; distance++)
+    for (const candidate of distance ? [line - distance, line + distance] : [line]) {
+      if (candidate < 1) continue;
+      const output = run(
+        "synctex",
+        ["view", "-i", `${candidate}:0:${file}`, "-o", pdf],
+        { check: false, timeout: 20000 },
+      ).stdout.toString();
+      const pages = [
+        ...new Set(
+          [...output.matchAll(/^Page:(\d+)$/gm)].map((m) => Number(m[1])),
+        ),
+      ];
+      if (pages.length) return pages;
+    }
+  return [];
+}
+export async function view(
+  app: App,
+  name: string,
+  request: ViewRequest,
+): Promise<ViewResult> {
+  const task = app.tasks.get(name),
+    project = app.project;
+  if (!fs.existsSync(task.worktree))
+    throw new Error(`The task worktree ${task.worktree} no longer exists.`);
+  const locations = (request.locations ?? []).map((l) =>
+      location(app, task.worktree, l),
+    ),
+    labels = (request.pages ?? []).map((p) => p.trim()).filter(Boolean);
+  if (!locations.length && !labels.length)
+    throw new Error("Ask for at least one source location or page number.");
+  const after = app.tasks.snapshot(task),
+    before = task.accepted,
+    builder = new Builder(project, app.repo),
+    targets = builder.plan(before, after, [
+      ...new Set([
+        ...app.repo.changedFiles(before, after),
+        ...locations.map((l) => l.file),
+      ]),
+    ]);
+  let build: BuildInfo;
+  try {
+    const seed = targets[0] !== "full" ? await builder.build(task.base) : null;
+    build = await builder.build(request.accepted ? before : after, targets, seed);
+  } catch (error) {
+    if (!(error instanceof BuildError)) throw error;
+    return {
+      pages: [],
+      missing: [],
+      targets,
+      error: request.accepted
+        ? "The accepted text could not be compiled."
+        : "The paper in the task worktree could not be compiled.",
+      details: error.details,
+    };
+  }
+  const pdf = build.pdf,
+    document = mupdf.Document.openDocument(fs.readFileSync(pdf), "application/pdf");
+  try {
+    // MuPDF synthesizes 1, 2, ... when /PageLabels is absent. Those are
+    // physical positions, not the printed numbers (front matter may be i, ii).
+    const trailer = document.asPDF()?.getTrailer(),
+      pageLabels = trailer?.get("Root", "PageLabels"),
+      hasLabels = pageLabels?.isDictionary() ?? false;
+    pageLabels?.destroy();
+    trailer?.destroy();
+    const wanted = new Map<number, string[]>(),
+      missing: string[] = [],
+      ask = (index: number, asked: string) =>
+        wanted.set(index, [...(wanted.get(index) ?? []), asked]);
+    for (const l of locations) {
+      // The build compiles an export of the source folder.
+      const file = path.join(
+          build.root,
+          path.posix.relative(project.source_dir || ".", l.file),
+        ),
+        found = fs.existsSync(file) ? pagesOf(pdf, file, l.line) : [];
+      if (found.length) found.forEach((page) => ask(page, l.asked));
+      else missing.push(`${l.asked}: SyncTeX places no typeset text at or near this line.`);
+    }
+    if (labels.length && !hasLabels) {
+      missing.push("This PDF has no page-label metadata, so printed page numbers cannot be located reliably. Ask for source locations (file:line) instead.");
+    } else if (labels.length) {
+      const printed = Array.from({ length: document.countPages() }, (_, i) => {
+        const page = document.loadPage(i),
+          label = page.getLabel();
+        page.destroy();
+        return label;
+      });
+      for (const label of labels) {
+        const index = printed.findIndex(
+          (p) => p.toLowerCase() === label.toLowerCase(),
+        );
+        if (index >= 0) ask(index + 1, label);
+        else
+          missing.push(
+            targets[0] === "full"
+              ? `Page ${label}: no page has this printed number.`
+              : `Page ${label}: not in the chapters compiled for this view (${targets.join(", ")}). Ask for a source line on that page instead.`,
+          );
+      }
+    }
+    const indices = [...wanted.keys()].sort((a, b) => a - b);
+    if (indices.length > PAGES)
+      missing.push(
+        `Only the first ${PAGES} pages are shown; ask for more specific source locations to see the remaining pages (physical positions ${indices.slice(PAGES).join(", ")} in this build).`,
+      );
+    const out = indices.length ? folder(app, name) : "";
+    const pages = indices.slice(0, PAGES).map((index) => {
+      const page = document.loadPage(index - 1),
+        pixmap = page.toPixmap(
+          [SCALE, 0, 0, SCALE, 0, 0],
+          mupdf.ColorSpace.DeviceRGB,
+          false,
+          true,
+        ),
+        label = hasLabels ? page.getLabel() || null : null,
+        file = path.join(out, `page-${label && /^[\w-]{1,16}$/u.test(label) ? label : index}-${index}.png`);
+      fs.writeFileSync(file, pixmap.asPNG());
+      pixmap.destroy();
+      page.destroy();
+      return { index, label, shows: wanted.get(index)!, file };
+    });
+    return { pages, missing, targets };
+  } finally {
+    document.destroy();
+  }
+}

package/vendor/latex-review/src/render/round.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 import { readJson, writeJson } from "../core/files.ts";
 import { within } from "../core/git.ts";
 import { deepestNode } from "../core/outline.ts";
+import { referenceChanges } from "../core/bib.ts";
 import { changesOf, placesOf, renderPlace } from "./effects.ts";
 import { Build } from "./locate.ts";
 import { renderSection, snap, Strips } from "./inline.ts";
@@ -54,8 +55,11 @@ export function classify(
     /\\(?:verb\b|lstinline\b|mintinline\b|begin\{(?:verbatim\*?|Verbatim|lstlisting|minted)\})/.test(
       (old.read(name) ?? "") + (next.read(name) ?? ""),
     );
+  // A .bib file is not typeset where it is written, and % in a URL is not
+  // a comment there: its edits are compared in the PDF.
   if (
     unit.kind !== "binary" &&
+    !name.endsWith(".bib") &&
     !whitespaceSensitive &&
     !unit.truncated &&
     unit.status === "modified" &&
@@ -286,6 +290,13 @@ export function renderRound(
       if (!placed.has(index))
         result.loose.push(looseBlock(card, strips, old, next));
     });
+    try {
+      const references = referenceChanges(request.units, old, next);
+      if (references) result.references = references;
+    } catch (error) {
+      // The PDF review stands without the bibliography's entry view.
+      console.error("Could not read the bibliography changes:", error);
+    }
     fs.writeFileSync(path.join(out, "round.json"), JSON.stringify(result));
     return result;
   } finally {

package/vendor/latex-review/src/server.ts

@@ -12,8 +12,10 @@ import { Tasks } from "./core/tasks.ts";
 import { Review } from "./core/review.ts";
 import { Deliveries } from "./core/delivery.ts";
 import { more, PublishError, Publisher } from "./core/publish.ts";
+import { view, type ViewRequest } from "./core/view.ts";
 import { acquireLock, now, readJson, writeJson } from "./core/files.ts";
 import { stopAll } from "./core/process.ts";
+import { checkInputs, Confirmations, ReferenceChecker } from "./core/verify.ts";
 import type { ActionKind } from "./types.ts";
 export class App {
   root: string;
@@ -28,6 +30,8 @@ export class App {
   review: Review;
   delivery: Deliveries;
   publisher: Publisher;
+  references: ReferenceChecker;
+  confirmations: Confirmations;
   private cachedProject?: Project;
   private stamp?: number;
   constructor(root: string) {
@@ -39,6 +43,12 @@ export class App {
     this.review = new Review(this);
     this.delivery = new Deliveries(this);
     this.publisher = new Publisher(this);
+    this.references = new ReferenceChecker(
+      path.join(this.state, "references.json"),
+    );
+    this.confirmations = new Confirmations(
+      path.join(this.state, "references-confirmed.json"),
+    );
     this.review.recover();
     this.delivery.recover();
   }
@@ -210,6 +220,55 @@ export function createServer(app: App) {
         }
       },
     );
+    // Checks of the bibliography entries a round adds or modifies. The page
+    // asks with check=1 to look up entries online, and keeps asking until
+    // none is pending; without it only results already known are returned.
+    server.get<{
+      Params: { name: string; number: string };
+      Querystring: { check?: string; retry?: string };
+    }>("/api/tasks/:name/rounds/:number/references", (request) => {
+      const inputs = checkInputs(
+        app.review.roundDetail(request.params.name, Number(request.params.number))
+          .references,
+      );
+      return {
+        ...app.references.lookup(
+          inputs,
+          request.query.check === "1",
+          request.query.retry === "1",
+        ),
+        confirmed: app.confirmations.of(inputs),
+      };
+    });
+    // The author confirms, or takes back, that they checked an entry by hand.
+    server.post<{
+      Params: { name: string; number: string };
+      Body: { id: string; confirmed: boolean };
+    }>(
+      "/api/tasks/:name/rounds/:number/references/confirm",
+      {
+        schema: {
+          body: {
+            ...{ type: "object" as const },
+            required: ["id", "confirmed"],
+            properties: { id: { type: "string" }, confirmed: { type: "boolean" } },
+          },
+        },
+      },
+      (request, reply) => {
+        const inputs = checkInputs(
+            app.review.roundDetail(request.params.name, Number(request.params.number))
+              .references,
+          ),
+          input = inputs.find((i) => i.id === request.body.id);
+        if (!input) return reply.code(404).send({ error: "No such entry in this round." });
+        app.confirmations.set(input, request.body.confirmed);
+        return {
+          ...app.references.lookup(inputs),
+          confirmed: app.confirmations.of(inputs),
+        };
+      },
+    );
     server.get<{ Params: { name: string; number: string; file: string } }>(
       "/files/:name/:number/:file",
       (request, reply) => {
@@ -272,6 +331,12 @@ export function createServer(app: App) {
         return { job: job.id, status: job.status };
       },
     );
+    // The task's typeset pages as images, for its agent to look at.
+    server.post<{ Params: { name: string }; Body: ViewRequest }>(
+      "/api/tasks/:name/view",
+      { schema: { body: { ...objectSchema, additionalProperties: false, properties: { locations: { type: "array", maxItems: 8, items: { type: "string", maxLength: 1000 } }, pages: { type: "array", maxItems: 8, items: { type: "string", maxLength: 40 } }, accepted: { type: "boolean" } } } } },
+      (request) => view(app, request.params.name, request.body),
+    );
     server.post<{ Params: { name: string }; Body: { mode: "workspace" | "branch"; message?: string } }>(
       "/api/tasks/:name/deliver",
       { schema: { body: { ...objectSchema, required: ["mode"], properties: { mode: { enum: ["workspace", "branch"] }, message: { type: "string", maxLength: 5000 } } } } },

package/vendor/latex-review/src/types.ts

@@ -239,6 +239,130 @@ export interface RenderedRound {
   units: Record<string, VisualUnit>;
   sections: { node: string; number: string; title: string; blocks: Block[] }[];
   loose: ChangeBlock[];
+  references?: References;
+}
+/** One field of a changed bibliography entry, before and after; null
+ * where the entry lacks it. `resolved` is the new value with @string
+ * macros expanded, when that differs; `oldResolved` is its old expansion. */
+export interface ReferenceField {
+  name: string;
+  old: string | null;
+  new: string | null;
+  resolved?: string;
+  oldResolved?: string;
+}
+export interface ReferenceCitation {
+  file: string;
+  line: number;
+  section: string | null;
+}
+/** still-cited: a removed key that the document still cites.
+ * not-in-bibliography: the document does not read this .bib file. */
+export type ReferenceProblem =
+  | "still-cited"
+  | "duplicate-key"
+  | "unreadable"
+  | "not-in-bibliography";
+/** A bibliography entry that a round adds, removes or modifies, shown whole
+ * under the edit it belongs to. */
+export interface ReferenceEntry {
+  id: string;
+  unit: string;
+  /** All edits affecting this entry when macros span multiple files. */
+  units?: string[];
+  file: string;
+  key: string;
+  old_key?: string;
+  type: string;
+  old_type?: string;
+  status: "added" | "removed" | "modified";
+  fields: ReferenceField[];
+  lines: Span;
+  /** Where the revised document cites the key. */
+  cited: ReferenceCitation[];
+  /** Where it still cites a renamed entry's old key. */
+  old_cited?: ReferenceCitation[];
+  problems: ReferenceProblem[];
+  error?: string;
+}
+/** A key the revised document newly cites with no bibliography entry. */
+export interface MissingCitation {
+  key: string;
+  file: string;
+  line: number;
+  unit: string | null;
+  section: string | null;
+}
+export interface References {
+  entries: ReferenceEntry[];
+  missing: MissingCitation[];
+}
+/** A published record a bibliography entry was compared with. */
+export interface CheckRecord {
+  source: "doi" | "crossref" | "arxiv" | "dblp";
+  title: string;
+  authors: string[];
+  year: number | null;
+  venue: string | null;
+  doi: string | null;
+  url: string;
+}
+export interface CheckDifference {
+  field: "title" | "author" | "year" | "doi" | "arxiv";
+  bib: string;
+  record: string;
+  /** Shown, but not a reason to doubt the entry: punctuation in a title,
+   * or a preprint's year before the published version's. */
+  minor?: boolean;
+}
+/** verified: a record agrees. mismatch: a record of the same work differs,
+ * or the DOI names another work. unverified: no record was found.
+ * failed: a source could not be reached. skipped: nothing to look up. */
+export interface ReferenceCheck {
+  status: "verified" | "mismatch" | "unverified" | "failed" | "skipped";
+  record?: CheckRecord;
+  differences: CheckDifference[];
+  /** What the entry was looked up by. */
+  via: "doi" | "arxiv" | "title" | null;
+  reason?: "doi-not-found" | "arxiv-not-found" | "no-match" | "nothing-to-check";
+  /** Sources that could not be reached for this check. */
+  unavailable: string[];
+  checked: number;
+}
+export interface ReferenceChecks {
+  checks: Record<string, ReferenceCheck>;
+  /** Entries still being looked up. */
+  pending: number;
+  /** Entries the author has checked by hand, with when (seconds). A
+   * confirmation holds while the entry's looked-up fields stay the same. */
+  confirmed?: Record<string, number>;
+}
+/** An entry of the paper's bibliography as it is now. */
+export interface BibliographyEntry {
+  id: string;
+  file: string;
+  key: string;
+  type: string;
+  fields: { name: string; value: string; resolved?: string }[];
+  lines: Span;
+  cited: ReferenceCitation[];
+  error?: string;
+}
+/** The whole bibliography of a paper: its entries, where each is cited,
+ * and what is wrong with them without looking anything up. */
+export interface Bibliography {
+  /** The .bib files the paper names, and those of them that do not exist. */
+  bibs: string[];
+  absent: string[];
+  entries: BibliographyEntry[];
+  /** Citations of keys no entry defines, each place. */
+  missing: (ReferenceCitation & { key: string })[];
+  /** Entries sharing a key. */
+  duplicates: string[][];
+  /** Entries under different keys with the same DOI or title. */
+  similar: { ids: string[]; by: "doi" | "title" }[];
+  /** \nocite{*} puts every entry in the bibliography. */
+  all_cited: boolean;
 }
 export interface RenderRequest {
   main: string;
@@ -273,6 +397,7 @@ export interface RoundDetail extends Omit<RenderedRound, "units"> {
   units: (Unit & Partial<VisualUnit> & { history?: History[] })[];
   warnings: string[];
   files: string;
+  references?: References;
 }
 export type TaskState =
   "editing" | "building" | "build_failed" | "reviewing" | "revise" | "done";

package/vendor/latex-review/src/web/effect-preferences.ts

@@ -0,0 +1,10 @@
+// Shared with the host settings; storage events also update standalone tabs.
+export const EFFECTS_STORAGE = "latex-review:effects";
+export const EFFECTS_EVENT = "latex-review:effects-changed";
+export function readExpandedEffects() {
+  try {
+    return localStorage.getItem(EFFECTS_STORAGE) === "on";
+  } catch {
+    return false;
+  }
+}