papilot 0.0.0-stage → 0.1.0

package/vendor/latex-review/src/render/round.ts

@@ -0,0 +1,236 @@
+import fs from "node:fs";
+import path from "node:path";
+import { within } from "../core/git.ts";
+import { deepestNode } from "../core/outline.ts";
+import { Build } from "./locate.ts";
+import { renderSection, snap, Strips } from "./inline.ts";
+import { pixelDifference, textChanges } from "./pdfdiff.ts";
+import type {
+  Card,
+  ChangeBlock,
+  Pair,
+  RenderedRound,
+  RenderRequest,
+  Strip,
+  Unit,
+} from "../types.ts";
+export function typesetWords(text: string) {
+  // TeX comments consume the end-of-line too. Keeping that newline would
+  // turn a standalone comment into a paragraph break, or split foo%\nbar.
+  const uncommented = (text.match(/[^\n]*\n|[^\n]+$/g) ?? [])
+    .map((line) => {
+      line = line.replace(/^[ \t]+/, "");
+      for (let i = 0; i < line.length; i++) {
+        if (line[i] === "\\") {
+          i++;
+          continue;
+        }
+        if (line[i] === "%") return line.slice(0, i);
+      }
+      return line;
+    })
+    .join("");
+  return uncommented
+    .split(/\n[ \t]*\n/)
+    .filter((p) => p.trim())
+    .map((p) => p.trim().replace(/\s+/g, " "));
+}
+export function classify(
+  old: Build,
+  next: Build,
+  unit: Unit,
+  body: Set<string>,
+  figureFiles: (file: string) => boolean,
+): Pick<Card, "pairs" | "comparison" | "own"> {
+  const name = unit.file;
+  // A changed line may be inside a whitespace-sensitive environment without
+  // including its \begin in the hunk. Inspect the owning file before using
+  // the source-only no-op shortcut; the PDF is the authority for these files.
+  const whitespaceSensitive =
+    /\\(?:verb\b|lstinline\b|mintinline\b|begin\{(?:verbatim\*?|Verbatim|lstlisting|minted)\})/.test(
+      (old.read(name) ?? "") + (next.read(name) ?? ""),
+    );
+  if (
+    unit.kind !== "binary" &&
+    !whitespaceSensitive &&
+    !unit.truncated &&
+    unit.status === "modified" &&
+    JSON.stringify(typesetWords(unit.old_text)) ===
+      JSON.stringify(typesetWords(unit.new_text))
+  )
+    return { pairs: [], comparison: "identical" };
+  const located =
+      name.endsWith(".tex") && unit.kind !== "binary" && body.has(name),
+    whole = unit.whole_file || figureFiles(name),
+    left =
+      located && unit.status !== "added"
+        ? old.locator.locate(name, unit.old_start, unit.old_end, whole)
+        : [],
+    right =
+      located && unit.status !== "deleted"
+        ? next.locator.locate(name, unit.new_start, unit.new_end, whole)
+        : [],
+    analysis = textChanges(old.document, next.document, left, right),
+    pairs: Pair[] = [];
+  let pixels = 0;
+  for (let i = 0; i < Math.max(left.length, right.length); i++) {
+    const a = left[i] ?? null,
+      b = right[i] ?? null;
+    pairs.push({
+      old: a,
+      new: b,
+      before_marks: analysis.old_marks[i] ?? [],
+      after_marks: analysis.new_marks[i] ?? [],
+    });
+    if (!analysis.text_changed)
+      pixels += pixelDifference(
+        a ? old.document.page(a.page).raster(a.clip) : null,
+        b ? next.document.page(b.page).raster(b.clip) : null,
+      );
+  }
+  const own = {
+      old:
+        left.length && !whole
+          ? old.locator.boxes(name, unit.old_start, unit.old_end)
+          : [],
+      new:
+        right.length && !whole
+          ? next.locator.boxes(name, unit.new_start, unit.new_end)
+          : [],
+    },
+    moved = left.some(
+      (a, i) =>
+        right[i] &&
+        (a.page !== right[i].page ||
+          a.label !== right[i].label ||
+          a.clip.some((v, j) => Math.abs(v - right[i].clip[j]) > 1)),
+    );
+  return {
+    pairs,
+    comparison: !pairs.length
+      ? "unmapped"
+      : analysis.text_changed
+        ? "text"
+        : pixels
+          ? "layout"
+          : moved
+            ? "position"
+            : "identical",
+    own,
+  };
+}
+function looseBlock(
+  card: Card,
+  strips: Strips,
+  old: Build,
+  next: Build,
+): ChangeBlock {
+  const before: Strip[] = [],
+    after: Strip[] = [];
+  for (const pair of card.pairs)
+    for (const [side, build, images, marks] of [
+      ["old", old, before, pair.before_marks],
+      ["new", next, after, pair.after_marks],
+    ] as const) {
+      const position = pair[side];
+      if (position) {
+        const image = strips.write(
+          build.document,
+          position.page,
+          ...snap(
+            build.document,
+            position.page,
+            position.clip[1],
+            position.clip[3],
+          ),
+          marks,
+          side,
+        );
+        if (image) images.push(image);
+      }
+    }
+  return {
+    kind: "change",
+    units: [card.id],
+    identical: card.comparison === "identical",
+    before,
+    after,
+  };
+}
+export function renderRound(
+  request: RenderRequest,
+  out: string,
+): RenderedRound {
+  fs.mkdirSync(out, { recursive: true });
+  const figureFiles = (name: string) => within(name, request.figures),
+    old = new Build(request.old, request.main, request.source_dir),
+    next = new Build(request.new, request.main, request.source_dir);
+  try {
+    const body = new Set(
+        [old, next].flatMap((b) => b.outline.flatMap((n) => n.files)),
+      ),
+      cards: Card[] = [],
+      result: RenderedRound = { units: {}, sections: [], loose: [] };
+    for (const unit of request.units) {
+      const card: Card = {
+          id: unit.id,
+          file: unit.file,
+          ...classify(old, next, unit, body, figureFiles),
+        },
+        node = card.pairs.length
+          ? unit.status === "deleted"
+            ? deepestNode(old.outline, unit.file, unit.old_start, unit.old_end)
+            : deepestNode(next.outline, unit.file, unit.new_start, unit.new_end)
+          : null;
+      card.context_node = node?.id ?? null;
+      cards.push(card);
+      result.units[unit.id] = {
+        visual: card.comparison,
+        pages: [
+          ...new Set(card.pairs.map((p) => (p.new ?? p.old)!.label)),
+        ].sort(),
+        section: node ? `${node.number} ${node.title}`.trim() : null,
+      };
+    }
+    const strips = new Strips(out),
+      placed = new Set<number>();
+    for (const node of next.outline.filter((n) =>
+      cards.some((c) => c.context_node === n.id),
+    )) {
+      const members = cards.flatMap((c, i): [number, Card][] =>
+        c.context_node === node.id ? [[i, c]] : [],
+      );
+      let rendered: ReturnType<typeof renderSection>;
+      try {
+        rendered = renderSection(old, next, node, members, strips, figureFiles);
+      } catch (error) {
+        if (
+          !(error as Error).message.includes("PDF outline") &&
+          !(error as Error).message.includes("outside this PDF")
+        )
+          throw error;
+        continue;
+      }
+      const [blocks, unlocated] = rendered;
+      if (!blocks.some((b) => b.kind === "change")) continue;
+      result.sections.push({
+        node: node.id,
+        number: node.number,
+        title: node.title,
+        blocks,
+      });
+      members.forEach(([i]) => {
+        if (!unlocated.includes(i)) placed.add(i);
+      });
+    }
+    cards.forEach((card, index) => {
+      if (!placed.has(index))
+        result.loose.push(looseBlock(card, strips, old, next));
+    });
+    fs.writeFileSync(path.join(out, "round.json"), JSON.stringify(result));
+    return result;
+  } finally {
+    old.close();
+    next.close();
+  }
+}

package/vendor/latex-review/src/render/worker.ts

@@ -0,0 +1,10 @@
+import fs from "node:fs";
+import path from "node:path";
+import { renderRound } from "./round.ts";
+import type { RenderRequest } from "../types.ts";
+const file = process.argv[2];
+if (!file) throw new Error("Expected a renderer request file.");
+renderRound(
+  JSON.parse(fs.readFileSync(file, "utf8")) as RenderRequest,
+  path.dirname(file),
+);

package/vendor/latex-review/src/runtime.ts

@@ -0,0 +1,23 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { createHash } from "node:crypto";
+import { listFiles } from "./core/files.ts";
+export const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+export const WEB = path.join(ROOT, "dist/web");
+let version: string | undefined;
+export function codeVersion() {
+  if (!version) {
+    const digest = createHash("sha256");
+    for (const file of [
+      ...listFiles(path.join(ROOT, "src")),
+      ...listFiles(WEB),
+      path.join(ROOT, "package-lock.json"),
+    ].sort())
+      digest
+        .update(path.relative(ROOT, file) + "\0")
+        .update(fs.readFileSync(file));
+    version = digest.digest("hex").slice(0, 16);
+  }
+  return version;
+}

package/vendor/latex-review/src/server.ts

@@ -0,0 +1,401 @@
+import fs from "node:fs";
+import path from "node:path";
+import { randomBytes, timingSafeEqual } from "node:crypto";
+import Fastify from "fastify";
+import staticFiles from "@fastify/static";
+import compress from "@fastify/compress";
+import { WEB, codeVersion } from "./runtime.ts";
+import { Repo } from "./core/git.ts";
+import { CONFIG_NAME, loadProject, type Project } from "./core/config.ts";
+import { Store } from "./core/store.ts";
+import { Tasks } from "./core/tasks.ts";
+import { Review } from "./core/review.ts";
+import { Publisher } from "./core/publish.ts";
+import { acquireLock, now, readJson, writeJson } from "./core/files.ts";
+import { stopAll } from "./core/process.ts";
+import type { ActionKind } from "./types.ts";
+export class App {
+  root: string;
+  repo: Repo;
+  state: string;
+  store: Store;
+  csrf = randomBytes(32).toString("base64url");
+  viewers = new Map<string, number>();
+  url = "";
+  activity = now();
+  tasks: Tasks;
+  review: Review;
+  publisher: Publisher;
+  private cachedProject?: Project;
+  private stamp?: number;
+  constructor(root: string) {
+    this.root = fs.realpathSync(root);
+    this.repo = new Repo(this.root);
+    this.state = path.join(this.repo.gitDir(), "latex-review");
+    this.store = new Store(path.join(this.state, "state.sqlite3"));
+    this.tasks = new Tasks(this);
+    this.review = new Review(this);
+    this.publisher = new Publisher(this);
+    this.review.recover();
+  }
+  get project() {
+    const config = path.join(this.root, CONFIG_NAME),
+      stamp = fs.existsSync(config) ? fs.statSync(config).mtimeMs : 0;
+    if (!this.cachedProject || this.stamp !== stamp) {
+      this.cachedProject = loadProject(this.root);
+      this.stamp = stamp;
+    }
+    return this.cachedProject;
+  }
+  page(name: string) {
+    return `${this.url}/?task=${encodeURIComponent(name)}`;
+  }
+  taskInfo(name: string) {
+    const task = this.tasks.get(name),
+      scope = task.scope;
+    return {
+      ...this.review.summary(name),
+      worktree: task.worktree,
+      url: this.page(name),
+      viewing: now() - (this.viewers.get(name) ?? 0) < 15,
+      scope: {
+        sections: scope.sections.map((s) => `${s.number} ${s.title}`.trim()),
+        files: scope.ranges
+          ? Object.entries(scope.ranges).flatMap(([file, spans]) =>
+              spans.map(([a, b]) => `${file}:${a}-${b}`),
+            )
+          : scope.paths,
+      },
+    };
+  }
+  tasksOverview() {
+    return this.tasks.list().map((task) => {
+      const { next: _next, ...summary } = this.review.summary(task.name);
+      return { ...summary, url: this.page(task.name) };
+    });
+  }
+  async close() {
+    this.publisher.stop();
+    await this.publisher.join();
+    this.store.close();
+  }
+}
+export function createServer(app: App) {
+  const server = Fastify({ logger: false, bodyLimit: 999999 });
+  server.register(compress, { threshold: 1024 });
+  server.addHook("onRequest", async (request, reply) => {
+    app.activity = now();
+    const port = new URL(app.url || "http://127.0.0.1:80").port || "80",
+      hosts = new Set([`127.0.0.1:${port}`, `localhost:${port}`]);
+    reply
+      .header("Cache-Control", "no-store")
+      .header("X-Content-Type-Options", "nosniff")
+      .header("X-Frame-Options", "DENY");
+    if (
+      !hosts.has(request.headers.host ?? "") ||
+      (request.headers.origin &&
+        ![...hosts].some((h) => request.headers.origin === "http://" + h))
+    )
+      return reply
+        .code(403)
+        .send({ error: "Only same-origin localhost requests are accepted." });
+    if (request.method === "POST") {
+      const token = Buffer.from(
+          String(request.headers["x-review-token"] ?? ""),
+        ),
+        expected = Buffer.from(app.csrf);
+      if (token.length !== expected.length || !timingSafeEqual(token, expected))
+        return reply
+          .code(403)
+          .send({ error: "This page is out of date. Reload it." });
+    }
+  });
+  server.setErrorHandler((error, _request, reply) => {
+    const e = error as Error & { statusCode?: number };
+    reply.code(e.statusCode ?? 400).send({ error: e.message });
+  });
+  server.after(() => {
+    server.get("/api/info", () => ({
+      root: app.root,
+      version: codeVersion(),
+      pid: process.pid,
+      busy: app.publisher.busy(),
+      build_workers: app.project.build_workers,
+      csrf: app.csrf,
+      runtime: "typescript",
+    }));
+    server.get("/api/tasks", () => ({
+      tasks: app.tasksOverview(),
+      csrf: app.csrf,
+    }));
+    server.get<{ Params: { id: string } }>("/api/jobs/:id", (request) => {
+      const j = app.publisher.job(request.params.id);
+      return {
+        id: j.id,
+        task: j.task,
+        status: j.status,
+        phase: j.phase,
+        ...j.result,
+      };
+    });
+    server.get<{ Params: { name: string } }>("/api/tasks/:name", (request) =>
+      app.taskInfo(request.params.name),
+    );
+    server.get<{ Params: { name: string }; Querystring: { round?: string } }>(
+      "/api/tasks/:name/state",
+      (request) =>
+        app.review.pageState(
+          request.params.name,
+          request.query.round && /^\d+$/.test(request.query.round)
+            ? Number(request.query.round)
+            : undefined,
+        ),
+    );
+    server.get<{ Params: { name: string }; Querystring: { after?: string } }>(
+      "/api/tasks/:name/feedback",
+      (request) =>
+        app.review.feedback(
+          request.params.name,
+          /^\d+$/.test(request.query.after ?? "")
+            ? Number(request.query.after)
+            : 0,
+        ),
+    );
+    server.get<{ Params: { name: string; number: string } }>(
+      "/api/tasks/:name/rounds/:number",
+      (request) =>
+        app.review.roundDetail(
+          request.params.name,
+          Number(request.params.number),
+        ),
+    );
+    server.get<{ Params: { name: string; number: string; file: string } }>(
+      "/files/:name/:number/:file",
+      (request, reply) => {
+        const { name, number, file } = request.params;
+        app.tasks.get(name, false);
+        if (!/^\d+$/.test(number) || !/^[\w.-]+\.(svg|png)$/.test(file))
+          return reply.code(404).send({ error: "No such file" });
+        const target = path.join(
+          app.state,
+          "rounds",
+          name,
+          String(Number(number)).padStart(3, "0"),
+          file,
+        );
+        if (!fs.existsSync(target))
+          return reply.code(404).send({ error: "No such file" });
+        reply
+          .header("Cache-Control", "private, max-age=31536000, immutable")
+          .type(file.endsWith(".svg") ? "image/svg+xml" : "image/png");
+        return fs.createReadStream(target);
+      },
+    );
+    const objectSchema = { type: "object" as const };
+    server.post<{
+      Body: {
+        name?: string;
+        title?: string;
+        sections?: string[];
+        files?: string[];
+      };
+    }>(
+      "/api/tasks",
+      {
+        schema: {
+          body: {
+            ...objectSchema,
+            properties: {
+              name: { type: "string" },
+              title: { type: "string" },
+              sections: { type: "array", items: { type: "string" } },
+              files: { type: "array", items: { type: "string" } },
+            },
+          },
+        },
+      },
+      (request) => {
+        const d = request.body;
+        return app.taskInfo(
+          app.tasks.create(d.name, d.title, d.sections, d.files).name,
+        );
+      },
+    );
+    server.post<{ Params: { name: string } }>(
+      "/api/tasks/:name/publish",
+      (request) => {
+        const job = app.publisher.submit(request.params.name);
+        return { job: job.id, status: job.status };
+      },
+    );
+    server.post<{
+      Params: { name: string };
+      Body: {
+        action: ActionKind;
+        id?: string;
+        changes: string[];
+        round: number;
+        body?: string;
+      };
+    }>(
+      "/api/tasks/:name/decide",
+      {
+        schema: {
+          body: {
+            ...objectSchema,
+            required: ["action", "changes", "round"],
+            properties: {
+              action: { enum: ["accept", "reject", "comment"] },
+              id: { type: "string" },
+              changes: { type: "array", items: { type: "string" } },
+              round: { type: "integer", minimum: 1 },
+              body: { type: "string" },
+            },
+          },
+        },
+      },
+      (request) => app.review.decide(request.params.name, request.body),
+    );
+    server.post<{ Params: { name: string }; Body: { id: string } }>(
+      "/api/tasks/:name/undo",
+      {
+        schema: {
+          body: {
+            ...objectSchema,
+            required: ["id"],
+            properties: { id: { type: "string" } },
+          },
+        },
+      },
+      (request) => app.review.undo(request.params.name, request.body.id),
+    );
+    server.post<{ Params: { name: string }; Body: { body: string } }>(
+      "/api/tasks/:name/note",
+      {
+        schema: {
+          body: {
+            ...objectSchema,
+            required: ["body"],
+            properties: { body: { type: "string" } },
+          },
+        },
+      },
+      (request) => app.review.note(request.params.name, request.body.body),
+    );
+    server.post<{ Params: { name: string }; Body: { force?: boolean } }>(
+      "/api/tasks/:name/finish",
+      {
+        schema: {
+          body: { ...objectSchema, properties: { force: { type: "boolean" } } },
+        },
+      },
+      async (request) => {
+        const result = await app.tasks.finish(
+          request.params.name,
+          request.body.force,
+        );
+        return {
+          task: request.params.name,
+          finished: true,
+          changed_files: result.changed,
+          discarded: Object.fromEntries(
+            Object.entries(result.discarded).filter(
+              ([k, n]) => ["pending", "commented"].includes(k) && n,
+            ),
+          ),
+        };
+      },
+    );
+    server.post<{ Body: { force?: boolean } }>(
+      "/api/shutdown",
+      {
+        schema: {
+          body: { ...objectSchema, properties: { force: { type: "boolean" } } },
+        },
+      },
+      (request, reply) => {
+        if (app.publisher.busy() && !request.body.force)
+          throw new Error(
+            "A round is being built. Stop after it finishes, or force the stop.",
+          );
+        reply.send({ ok: true });
+        setImmediate(() => {
+          app.publisher.stop();
+          stopAll();
+          void server.close();
+        });
+      },
+    );
+    if (fs.existsSync(WEB)) {
+      server.register(staticFiles, {
+        root: WEB,
+        prefix: "/",
+        cacheControl: false,
+        setHeaders: (response) => response.header("Cache-Control", "no-cache"),
+      });
+    } else
+      server.get("/", (_req, reply) =>
+        reply
+          .type("text/plain")
+          .send("Build the review UI with npm run build."),
+      );
+  });
+  return server;
+}
+export async function serve(root: string, port = 0, idle = 12 * 3600) {
+  root = new Repo(root).toplevel();
+  process.chdir(root);
+  const home = path.join(new Repo(root).gitDir(), "latex-review/server"),
+    release = await acquireLock(path.join(home, "server.lock"), false),
+    app = new App(root),
+    server = createServer(app);
+  let timer: NodeJS.Timeout | undefined;
+  try {
+    try {
+      await server.listen({ host: "127.0.0.1", port });
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error;
+      await server.listen({ host: "127.0.0.1", port: 0 });
+    }
+    const address = server.server.address();
+    if (!address || typeof address === "string")
+      throw new Error("No server address");
+    app.url = `http://127.0.0.1:${address.port}`;
+    const info = {
+      url: app.url,
+      port: address.port,
+      pid: process.pid,
+      version: codeVersion(),
+      root,
+      runtime: "typescript",
+    };
+    writeJson(path.join(home, "connection.json"), info, 0o600);
+    writeJson(path.join(home, "last.json"), { port: address.port });
+    const shutdown = () => {
+      app.publisher.stop();
+      stopAll();
+      void server.close();
+    };
+    process.once("SIGTERM", shutdown);
+    process.once("SIGINT", shutdown);
+    timer = setInterval(
+      () => {
+        if (now() - app.activity > idle && !app.publisher.busy()) shutdown();
+      },
+      Math.min(30000, idle * 1000),
+    );
+    console.log(JSON.stringify(info));
+    await new Promise<void>((resolve) => server.server.once("close", resolve));
+  } finally {
+    if (timer) clearInterval(timer);
+    app.publisher.stop();
+    stopAll();
+    await app.close();
+    if (
+      readJson<{ pid?: number }>(path.join(home, "connection.json"), {}).pid ===
+      process.pid
+    )
+      fs.rmSync(path.join(home, "connection.json"), { force: true });
+    await release();
+  }
+}