papilot 0.0.0-stage → 0.1.0

package/vendor/latex-review/src/client.ts

@@ -0,0 +1,198 @@
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { ROOT, codeVersion } from "./runtime.ts";
+import { Repo } from "./core/git.ts";
+import { readJson, sleep, withLock } from "./core/files.ts";
+export interface Connection {
+  url: string;
+  port: number;
+  pid: number;
+  root?: string;
+  version?: string;
+}
+export interface ServerInfo {
+  root: string;
+  version: string;
+  pid: number;
+  busy: boolean;
+  csrf: string;
+  build_workers: number;
+  runtime?: string;
+}
+export class ServerError extends Error {}
+export async function call<T>(
+  url: string,
+  route: string,
+  data?: unknown,
+  token = "",
+  timeout = 30000,
+): Promise<T> {
+  const response = await fetch(url + route, {
+    method: data === undefined ? "GET" : "POST",
+    headers: { "Content-Type": "application/json", "X-Review-Token": token },
+    ...(data === undefined ? {} : { body: JSON.stringify(data) }),
+    signal: AbortSignal.timeout(timeout),
+    redirect: "error",
+  });
+  const value = (await response.json()) as T & { error?: string };
+  if (!response.ok)
+    throw new ServerError(value.error || `HTTP ${response.status}`);
+  return value;
+}
+export class Client {
+  constructor(
+    public url: string,
+    public token: string,
+  ) {}
+  get<T>(route: string, timeout = 30000) {
+    return call<T>(this.url, route, undefined, "", timeout);
+  }
+  post<T>(route: string, data: unknown = {}, timeout = 120000) {
+    return call<T>(this.url, route, data, this.token, timeout);
+  }
+}
+export const serverHome = (root: string) =>
+  path.join(new Repo(root).gitDir(), "latex-review/server");
+export async function probe(root: string, info: Partial<Connection> | null) {
+  if (!info?.url) return null;
+  const address = new URL(info.url);
+  if (address.protocol !== "http:" || address.hostname !== "127.0.0.1")
+    return null;
+  try {
+    const answer = await call<ServerInfo>(
+      info.url,
+      "/api/info",
+      undefined,
+      "",
+      5000,
+    );
+    return path.resolve(answer.root) === path.resolve(root) ? answer : null;
+  } catch {
+    return null;
+  }
+}
+async function waitForExit(url: string, pid: number) {
+  const deadline = Date.now() + 25000;
+  while (Date.now() < deadline) {
+    try {
+      const info = await call<ServerInfo>(
+        url,
+        "/api/info",
+        undefined,
+        "",
+        1000,
+      );
+      if (info.pid !== pid) return;
+    } catch {
+      return;
+    }
+    await sleep(100);
+  }
+  throw new ServerError(
+    "The review server has not stopped yet. Retry after it finishes.",
+  );
+}
+async function launch(root: string, port: number): Promise<Client> {
+  const home = serverHome(root);
+  fs.mkdirSync(home, { recursive: true });
+  const log = path.join(home, "server.log");
+  if (fs.existsSync(log) && fs.statSync(log).size > 2000000)
+    fs.renameSync(log, log + ".1");
+  const fd = fs.openSync(log, "a"),
+    child = spawn(
+      process.execPath,
+      [
+        "--import",
+        "tsx",
+        path.join(ROOT, "src/cli.ts"),
+        "serve",
+        "--root",
+        root,
+        "--port",
+        String(port),
+      ],
+      {
+        cwd: ROOT,
+        detached: true,
+        stdio: ["ignore", fd, fd],
+        env: process.env,
+      },
+    );
+  fs.closeSync(fd);
+  let launchError: Error | undefined;
+  child.on("error", (error) => {
+    launchError = error;
+  });
+  child.unref();
+  const deadline = Date.now() + 60000;
+  while (Date.now() < deadline) {
+    if (launchError) throw new ServerError(launchError.message);
+    const info = readJson<Connection | null>(
+      path.join(home, "connection.json"),
+      null,
+    );
+    if (info && info.pid === child.pid) {
+      const answer = await probe(root, info);
+      if (answer) return new Client(info.url, answer.csrf);
+    }
+    if (child.exitCode !== null) break;
+    await sleep(100);
+  }
+  throw new ServerError(
+    "The review server failed to start:\n" +
+      fs.readFileSync(log, "utf8").slice(-1500),
+  );
+}
+export async function connect(
+  root: string,
+  start: false,
+): Promise<Client | null>;
+export async function connect(root: string, start?: true): Promise<Client>;
+export async function connect(
+  root: string,
+  start = true,
+): Promise<Client | null> {
+  root = path.resolve(root);
+  const home = serverHome(root);
+  return withLock(path.join(home, "start.lock"), async () => {
+    const last = readJson<{ port: number } | null>(
+      path.join(home, "last.json"),
+      null,
+    );
+    let info = readJson<Connection | null>(
+      path.join(home, "connection.json"),
+      null,
+    );
+    if (!info && last)
+      info = { url: `http://127.0.0.1:${last.port}`, port: last.port, pid: 0 };
+    let answer = await probe(root, info);
+    if (answer && (!start || answer.version === codeVersion()))
+      return new Client(info!.url, answer.csrf);
+    if (!start) return null;
+    if (answer && info) {
+      const deadline = Date.now() + 900000;
+      while (answer?.busy && Date.now() < deadline) {
+        await sleep(1000);
+        answer = await probe(root, info);
+      }
+      if (answer?.busy)
+        throw new ServerError(
+          "A round is still building. Wait for it before upgrading the server.",
+        );
+      if (answer) {
+        await call(info.url, "/api/shutdown", {}, answer.csrf, 5000);
+        await waitForExit(info.url, answer.pid);
+      }
+    }
+    return launch(root, info?.port ?? last?.port ?? 0);
+  });
+}
+export async function stop(root: string, force = false) {
+  const client = await connect(root, false);
+  if (!client) return null;
+  const answer = await client.get<ServerInfo>("/api/info");
+  await client.post("/api/shutdown", { force });
+  await waitForExit(client.url, answer.pid);
+  return client.url;
+}

package/vendor/latex-review/src/core/build.ts

@@ -0,0 +1,296 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import { Repo } from "./git.ts";
+import { listFiles, readJson, writeJson, withLock, now } from "./files.ts";
+import { stripComments } from "./outline.ts";
+import { signature, type Project } from "./config.ts";
+import { run, runAsync } from "./process.ts";
+import type { BuildInfo } from "../types.ts";
+export class BuildError extends Error {
+  constructor(
+    message: string,
+    public details: string[] = [],
+    public log?: string,
+  ) {
+    super(message);
+  }
+}
+const AUXILIARY = new Set([
+  ".aux",
+  ".bbl",
+  ".brf",
+  ".toc",
+  ".lof",
+  ".lot",
+  ".loe",
+  ".out",
+  ".thm",
+]);
+const hash = (value: string | Buffer) =>
+  createHash("sha256").update(value).digest("hex");
+export const includes = (text: string) =>
+  [...stripComments(text).matchAll(/^\s*\\include\{([^}]+)\}/gm)].map((m) =>
+    m[1].trim(),
+  );
+export function affectedTargets(
+  texts: Record<string, string>,
+  changed: string[],
+  targets: string[],
+) {
+  const owners: Record<string, Set<string>> = {};
+  function visit(file: string, target: string, seen: Set<string>) {
+    if (!path.posix.extname(file)) file += ".tex";
+    if (seen.has(file) || !(file in texts)) return;
+    seen.add(file);
+    (owners[file] ??= new Set()).add(target);
+    for (const m of stripComments(texts[file]).matchAll(
+      /\\(?:input|include|subfile)\{([^}]+)\}/g,
+    ))
+      visit(path.posix.normalize(m[1].trim()), target, seen);
+  }
+  targets.forEach((t) => visit(t, t, new Set()));
+  const selected = new Set<string>();
+  for (const file of changed) {
+    if (!owners[file]) return ["full"];
+    for (const owner of owners[file]) selected.add(owner);
+  }
+  return [...selected].sort();
+}
+export function errors(text: string) {
+  const lines = text.split("\n"),
+    result: string[] = [];
+  for (const [i, line] of lines.entries())
+    if (/^(\S[^:\n]*):(\d+): (.+)$/.test(line) || line.startsWith("! ")) {
+      const context = lines
+        .slice(i + 1, i + 8)
+        .find((l) => /^l\.\d+ /.test(l))
+        ?.trim();
+      result.push(
+        line.replace(/^! |^\.\//, "") + (context ? `  [${context}]` : ""),
+      );
+      if (result.length === 8) break;
+    }
+  return result;
+}
+export function warnings(text: string) {
+  const patterns: [RegExp, string][] = [
+    [/LaTeX Warning: Reference `([^']+)' on page/g, "Undefined reference: "],
+    [/Warning: Citation [`']([^']+)' on page/g, "Undefined citation: "],
+    [
+      /LaTeX Warning: Label `([^']+)' multiply defined/g,
+      "Label defined twice: ",
+    ],
+    [/Missing character: There is no (\S+)/g, "Missing glyph: "],
+  ];
+  return [
+    ...new Set(
+      patterns.flatMap(([re, label]) =>
+        [...text.matchAll(re)].map((m) => label + m[1]),
+      ),
+    ),
+  ].sort();
+}
+export class Builder {
+  cache: string;
+  constructor(
+    public project: Project,
+    public repo: Repo,
+  ) {
+    this.cache = path.join(project.state, "cache/builds");
+  }
+  documentTexts(rev: string) {
+    const files = this.repo
+        .files(rev, this.project.doc_dir)
+        .filter((f) => f.endsWith(".tex")),
+      blobs = this.repo.blobs(rev, files);
+    return Object.fromEntries(
+      files.map((f) => [
+        path.posix.relative(this.project.doc_dir || ".", f),
+        blobs[f]?.toString() ?? "",
+      ]),
+    );
+  }
+  plan(before: string, after: string, changed: string[]) {
+    const p = this.project,
+      old = this.documentTexts(before),
+      next = this.documentTexts(after),
+      main = path.posix.basename(p.main),
+      targets = includes(next[main] ?? "");
+    if (
+      !p.partial_builds ||
+      targets.length < 2 ||
+      JSON.stringify(targets) !== JSON.stringify(includes(old[main] ?? ""))
+    )
+      return ["full"];
+    if (p.doc_dir && changed.some((f) => !f.startsWith(p.doc_dir + "/")))
+      return ["full"];
+    const local = changed.map((f) => path.posix.relative(p.doc_dir || ".", f)),
+      selected = new Set([
+        ...affectedTargets(old, local, targets),
+        ...affectedTargets(next, local, targets),
+      ]);
+    if (selected.has("full") || !selected.size) return ["full"];
+    for (const name of [...selected])
+      for (const dependency of p.dependencies[name] ?? [])
+        selected.add(dependency);
+    return targets.filter((t) => selected.has(t));
+  }
+  async build(
+    rev: string,
+    targets = ["full"],
+    seed: BuildInfo | null = null,
+  ): Promise<BuildInfo> {
+    const p = this.project,
+      partial = JSON.stringify(targets) !== '["full"]';
+    if (partial && !seed)
+      throw new BuildError(
+        "A partial build needs a completed full build to start from.",
+      );
+    let version: string;
+    try {
+      version = run(p.engine, ["--version"]).stdout.toString().split("\n")[0];
+    } catch {
+      throw new BuildError(
+        `${p.engine} was not found. Install TeX Live (or MacTeX) with latexmk.`,
+      );
+    }
+    const tree = this.repo.treeId(rev, p.source_dir),
+      key = hash(
+        JSON.stringify([
+          tree,
+          targets,
+          seed?.key ?? null,
+          signature(p),
+          version,
+          hash(fs.readFileSync(fileURLToPath(import.meta.url))),
+        ]),
+      ).slice(0, 32),
+      folder = path.join(this.cache, key);
+    return withLock(path.join(this.cache, key + ".lock"), async () => {
+      const cached = readJson<BuildInfo | null>(
+        path.join(folder, "result.json"),
+        null,
+      );
+      if (cached && fs.existsSync(cached.pdf)) {
+        fs.utimesSync(folder, new Date(), new Date());
+        return cached;
+      }
+      fs.rmSync(folder, { recursive: true, force: true });
+      const source = path.join(folder, "src");
+      this.repo.export(rev, p.source_dir, source);
+      const cwd = path.join(
+          source,
+          path.posix.relative(p.source_dir || ".", p.doc_dir || "."),
+        ),
+        main = path.posix.basename(p.main),
+        job = path.posix.basename(main, ".tex"),
+        out = path.join(cwd, "build");
+      fs.mkdirSync(out, { recursive: true });
+      for (const target of includes(
+        fs.readFileSync(path.join(cwd, main), "utf8"),
+      ))
+        fs.mkdirSync(path.join(out, path.dirname(target)), { recursive: true });
+      const args = [
+        "-norc",
+        ...(p.latexmkrc
+          ? ["-r", path.posix.relative(p.doc_dir || ".", p.latexmkrc)]
+          : []),
+        { pdflatex: "-pdf", xelatex: "-xelatex", lualatex: "-lualatex" }[
+          p.engine
+        ],
+        "-outdir=build",
+        `-jobname=${job}`,
+        "-synctex=1",
+        "-interaction=nonstopmode",
+        "-halt-on-error",
+        "-file-line-error",
+      ];
+      if (partial) {
+        const seedOut = path.dirname(seed!.pdf);
+        for (const file of listFiles(seedOut))
+          if (AUXILIARY.has(path.extname(file))) {
+            const target = path.join(out, path.relative(seedOut, file));
+            fs.mkdirSync(path.dirname(target), { recursive: true });
+            if (!fs.existsSync(target)) fs.copyFileSync(file, target);
+          }
+        const preload = Object.entries(p.preload)
+          .filter(
+            ([name, file]) =>
+              !targets.includes(name) && fs.existsSync(path.join(cwd, file)),
+          )
+          .map(([, file]) => `\\input{${file}}`)
+          .join("");
+        args.push(
+          `-usepretex=\\includeonly{${targets.join(",")}}` +
+            (preload ? `\\AtBeginDocument{${preload}}` : ""),
+        );
+      }
+      args.push(main);
+      const transcript = path.join(folder, "latexmk.log"),
+        fd = fs.openSync(transcript, "w"),
+        started = now();
+      let failure: Error | null = null;
+      try {
+        await runAsync("latexmk", args, {
+          cwd,
+          timeout: 1800000,
+          output: (chunk) => fs.writeSync(fd, chunk),
+        });
+      } catch (error) {
+        failure = error as Error;
+      } finally {
+        fs.closeSync(fd);
+      }
+      const pdf = path.join(out, job + ".pdf"),
+        log = path.join(out, job + ".log"),
+        logText = fs.existsSync(log) ? fs.readFileSync(log, "utf8") : "";
+      if (failure || !fs.existsSync(pdf)) {
+        const details = errors(logText),
+          kept = path.join(p.state, "logs", key + ".log");
+        fs.mkdirSync(path.dirname(kept), { recursive: true });
+        fs.copyFileSync(fs.existsSync(log) ? log : transcript, kept);
+        const tail = fs.readFileSync(transcript, "utf8").split("\n").slice(-25);
+        fs.rmSync(folder, { recursive: true, force: true });
+        throw new BuildError(
+          failure?.message.includes("ENOENT")
+            ? "latexmk was not found. Install TeX Live or MacTeX."
+            : "LaTeX failed to compile.",
+          details.length ? details : tail,
+          kept,
+        );
+      }
+      const result: BuildInfo = {
+        key,
+        tree,
+        targets,
+        root: source,
+        cwd,
+        pdf,
+        seconds: Math.round((now() - started) * 10) / 10,
+        warnings: warnings(logText),
+      };
+      writeJson(path.join(folder, "result.json"), result);
+      return result;
+    });
+  }
+  async prune(keep: Set<string>, maxAge = 3 * 86400) {
+    if (!fs.existsSync(this.cache)) return;
+    for (const entry of fs.readdirSync(this.cache, { withFileTypes: true })) {
+      const folder = path.join(this.cache, entry.name);
+      if (
+        entry.isDirectory() &&
+        !entry.name.endsWith(".lock") &&
+        !keep.has(entry.name) &&
+        now() - fs.statSync(folder).mtimeMs / 1000 > maxAge
+      )
+        await withLock(
+          path.join(this.cache, entry.name + ".lock"),
+          async () => {
+            fs.rmSync(folder, { recursive: true, force: true });
+          },
+        );
+    }
+  }
+}

package/vendor/latex-review/src/core/changes.ts

@@ -0,0 +1,126 @@
+import { Repo, within } from "./git.ts";
+import { groups, opcodes } from "./diff.ts";
+import type { Scope, Unit, Op } from "../types.ts";
+export const EXCERPT = 6000;
+export function textOf(data: Buffer | null): string | null {
+  if (data === null) return "";
+  if (data.includes(0)) return null;
+  try {
+    return new TextDecoder("utf-8", { fatal: true }).decode(data);
+  } catch {
+    return null;
+  }
+}
+export const splitLines = (text: string) =>
+  text.match(/[^\n]*\n|[^\n]+$/g) ?? [];
+export function diffUnits(
+  repo: Repo,
+  before: string,
+  after: string,
+  isFigure: (p: string) => boolean = () => false,
+): Unit[] {
+  const files = repo.changedFiles(before, after),
+    oldBlobs = repo.blobs(before, files),
+    newBlobs = repo.blobs(after, files),
+    units: Unit[] = [];
+  for (const file of files) {
+    const old = oldBlobs[file],
+      next = newBlobs[file],
+      a = textOf(old),
+      b = textOf(next);
+    const status =
+        old === null ? "added" : next === null ? "deleted" : "modified",
+      binary = a === null || b === null;
+    const left = binary ? [] : splitLines(a!),
+      right = binary ? [] : splitLines(b!);
+    let hunks = binary ? [] : groups(opcodes(left, right));
+    const whole =
+      binary || status !== "modified" || isFigure(file) || !hunks.length;
+    if (whole) hunks = [[["replace", 0, left.length, 0, right.length]]];
+    for (const edits of hunks) {
+      const [, i, , x] = edits[0],
+        [, , j, , y] = edits.at(-1)!;
+      const oldText = left.slice(i, j).join(""),
+        newText = right.slice(x, y).join("");
+      units.push({
+        id: `c${String(units.length + 1).padStart(3, "0")}`,
+        file,
+        status,
+        kind: binary ? "binary" : isFigure(file) ? "figure" : "text",
+        whole_file: whole,
+        ops: whole ? [] : edits,
+        old_start: i + 1,
+        old_end: Math.max(i + 1, j),
+        new_start: x + 1,
+        new_end: Math.max(x + 1, y),
+        old_text: binary ? "" : oldText.slice(0, EXCERPT),
+        new_text: binary ? "" : newText.slice(0, EXCERPT),
+        truncated: oldText.length > EXCERPT || newText.length > EXCERPT,
+      });
+    }
+  }
+  return units;
+}
+function edits(units: Unit[], index: 1 | 3): Op[] {
+  return units.flatMap((u) => u.ops).sort((a, b) => b[index] - a[index]);
+}
+export function applyUnits(
+  old: Buffer | null,
+  next: Buffer | null,
+  units: Unit[],
+) {
+  if (units.some((u) => u.whole_file)) return next;
+  const a = splitLines(old!.toString()),
+    b = splitLines(next!.toString());
+  for (const [, i, j, x, y] of edits(units, 1))
+    a.splice(i, j - i, ...b.slice(x, y));
+  return Buffer.from(a.join(""));
+}
+export function revertUnits(
+  old: Buffer | null,
+  next: Buffer | null,
+  units: Unit[],
+) {
+  if (units.some((u) => u.whole_file)) return old;
+  const a = splitLines(old!.toString()),
+    b = splitLines(next!.toString());
+  for (const [, i, j, x, y] of edits(units, 3))
+    b.splice(x, y - x, ...a.slice(i, j));
+  return Buffer.from(b.join(""));
+}
+export function scopeViolations(
+  repo: Repo,
+  base: string,
+  revision: string,
+  scope: Scope,
+) {
+  const files = repo.changedFiles(base, revision);
+  if (scope.ranges === null)
+    return files.filter((f) => !within(f, scope.paths));
+  const old = repo.blobs(base, files),
+    next = repo.blobs(revision, files),
+    violations: string[] = [];
+  for (const file of files) {
+    const a = textOf(old[file]),
+      b = textOf(next[file]),
+      allowed = scope.ranges[file];
+    if (!allowed || old[file] === null || a === null || b === null) {
+      violations.push(file);
+      continue;
+    }
+    for (const [tag, i, j] of opcodes(splitLines(a), splitLines(b))) {
+      if (tag === "equal") continue;
+      const permitted =
+        i === j
+          ? allowed.some(([s, e]) => s - 1 <= i && i <= e)
+          : Array.from({ length: j - i }, (_, k) => i + k + 1).every((n) =>
+              allowed.some(([s, e]) => s <= n && n <= e),
+            );
+      if (!permitted) {
+        violations.push(`${file}:${i + 1}`);
+        break;
+      }
+    }
+  }
+  return violations;
+}