token-goat 2.9.13 → 2.9.15

package/dist/token-goat-chunk-NEI4NC54.mjs

@@ -0,0 +1,424 @@
+import { createRequire as __cjsRequire } from 'node:module';
+const require = __cjsRequire(import.meta.url);
+import {
+  createLazyModuleLoader
+} from "./token-goat-chunk-AH6QILZM.mjs";
+import {
+  DocumentRefusedError,
+  MAX_DOCUMENT_WORK_MILLIS
+} from "./token-goat-chunk-Y4AFKTHK.mjs";
+import {
+  compileGuardedRegex
+} from "./token-goat-chunk-GMOUBOX4.mjs";
+import {
+  init_define_import_meta_env
+} from "./token-goat-chunk-A37V4PBF.mjs";
+
+// src/pdf_extract.ts
+init_define_import_meta_env();
+import * as fs from "node:fs";
+var loadPdfjs = createLazyModuleLoader(async () => {
+  const mod = await import("pdfjs-dist/legacy/build/pdf.mjs");
+  if (typeof import.meta.resolve === "function") {
+    try {
+      mod.GlobalWorkerOptions.workerSrc = await import.meta.resolve("pdfjs-dist/legacy/build/pdf.worker.mjs");
+    } catch {
+    }
+  }
+  return mod;
+}, "pdf-extract disabled (pdfjs-dist unavailable)");
+var MAX_PDF_TEXT_BYTES = 8 * 1024 * 1024;
+var MAX_PDF_INPUT_BYTES = 50 * 1024 * 1024;
+var MAX_PDF_TEXT_ITEMS = 2e5;
+var MAX_OUTLINE_ENTRIES = 2e4;
+var MAX_OUTLINE_TITLE_CHARS = 500;
+var MAX_LOCATE_MATCHES = 1e3;
+var MAX_LOCATE_CONTEXT_CHARS = 4e3;
+var MAX_PDF_WORK_MILLIS = MAX_DOCUMENT_WORK_MILLIS;
+var PdfRefusedError = class extends DocumentRefusedError {
+};
+var PdfTooLargeError = class extends PdfRefusedError {
+  constructor(message) {
+    super(message, "PdfTooLargeError");
+  }
+};
+var PdfTookTooLongError = class extends PdfRefusedError {
+  constructor(message) {
+    super(message, "PdfTookTooLongError", true);
+  }
+};
+function pdfWorkDeadline() {
+  return Date.now() + MAX_PDF_WORK_MILLIS;
+}
+function pdfWorkTookTooLong() {
+  return new PdfTookTooLongError(`reading this PDF's text passed the ${MAX_PDF_WORK_MILLIS}ms limit. Narrow the read with --pages, or use a smaller document.`);
+}
+function assertPdfIsARegularFileWithinBounds(stat, file) {
+  if (!stat.isFile()) {
+    throw new PdfTooLargeError(`${file} is not a regular file, so its size cannot be checked before reading it.`);
+  }
+  if (stat.size > MAX_PDF_INPUT_BYTES) {
+    throw new PdfTooLargeError(`${file} is ${stat.size} bytes, past the ${MAX_PDF_INPUT_BYTES}-byte limit for a PDF. Split it, or extract from a smaller copy.`);
+  }
+}
+function pdfTextBudgetExceeded() {
+  return new PdfTooLargeError(`this PDF's text passes the ${MAX_PDF_TEXT_BYTES}-byte extraction limit. Narrow the read with --pages, or use a smaller document.`);
+}
+function pdfPageTooDenseToRead() {
+  return new PdfTooLargeError(`a page of this PDF holds more than ${MAX_PDF_TEXT_ITEMS} separate pieces of text, past what extraction will hold whatever they weigh. Use a smaller document.`);
+}
+function assertPdfTextWithinBounds(text) {
+  if (Buffer.byteLength(text, "utf8") > MAX_PDF_TEXT_BYTES) throw pdfTextBudgetExceeded();
+}
+async function readPdfFileWithinBounds(file) {
+  const handle = await fs.promises.open(file, "r");
+  try {
+    const stat = await handle.stat();
+    assertPdfIsARegularFileWithinBounds(stat, file);
+    return await readAllWithinReportedSize(handle, stat.size, file);
+  } finally {
+    await handle.close();
+  }
+}
+async function readAllWithinReportedSize(handle, reportedSize, file) {
+  const buffer = Buffer.alloc(reportedSize + 1);
+  let read = 0;
+  for (; ; ) {
+    const { bytesRead } = await handle.read(buffer, read, buffer.length - read, read);
+    if (bytesRead === 0) break;
+    read += bytesRead;
+    if (read === buffer.length) {
+      throw new PdfTooLargeError(`${file} grew past the ${reportedSize} bytes it reported while it was being read. Extract from a copy that is not being written to.`);
+    }
+  }
+  return read === reportedSize ? new Uint8Array(buffer.buffer, buffer.byteOffset, read) : new Uint8Array(buffer.subarray(0, read));
+}
+async function* pageTextItems(page, deadline) {
+  const keep = (items) => items.filter((item) => item !== null && typeof item === "object" && "str" in item);
+  if (typeof page.streamTextContent !== "function") {
+    const content = await raceDeadline(page.getTextContent(), deadline);
+    yield keep(content.items);
+    return;
+  }
+  const reader = page.streamTextContent().getReader();
+  for (; ; ) {
+    const { done, value } = await raceDeadline(reader.read(), deadline);
+    if (done) return;
+    yield keep(value?.items ?? []);
+  }
+}
+async function getPageWithinDeadline(doc, pageNum, deadline) {
+  if (Date.now() > deadline) throw pdfWorkTookTooLong();
+  return raceDeadline(doc.getPage(pageNum), deadline);
+}
+async function readPageTextItems(page, budget, deadline) {
+  let items = [];
+  let spent = 0;
+  let count = 0;
+  let over = null;
+  for await (const chunk of pageTextItems(page, deadline)) {
+    if (Date.now() > deadline) throw pdfWorkTookTooLong();
+    if (over) continue;
+    for (const item of chunk) spent += item.str.length + 1;
+    count += chunk.length;
+    if (spent > budget || count > MAX_PDF_TEXT_ITEMS) {
+      over = count > MAX_PDF_TEXT_ITEMS ? "items" : "bytes";
+      items = [];
+      continue;
+    }
+    for (const item of chunk) items.push(item);
+  }
+  if (over) throw over === "items" ? pdfPageTooDenseToRead() : pdfTextBudgetExceeded();
+  return items;
+}
+function parsePageRange(spec, pageCount) {
+  if (!spec) return null;
+  const m = /^(\d+)(?:-(\d+))?$/.exec(spec.trim());
+  if (!m) throw new Error(`invalid --pages spec: ${spec} (expected "N" or "N-M")`);
+  const start = parseInt(m[1], 10);
+  const end = m[2] ? parseInt(m[2], 10) : start;
+  if (start < 1 || end < start) throw new Error(`invalid --pages spec: ${spec}`);
+  if (start > pageCount) throw new Error(`invalid --pages spec: ${spec} (page ${start} is past end of document with ${pageCount} pages)`);
+  return { start, end: Math.min(end, pageCount) };
+}
+var Y_EPSILON = 2;
+function reconstructLayout(items, deadline) {
+  const rows = [];
+  const bucketOf = (y) => Math.floor(y / Y_EPSILON);
+  const byBucket = /* @__PURE__ */ new Map();
+  const bucketOfRow = [];
+  const place = (bucket, row) => {
+    const list = byBucket.get(bucket);
+    if (list) list.push(row);
+    else byBucket.set(bucket, [row]);
+    bucketOfRow[row] = bucket;
+  };
+  let checked = 0;
+  for (const item of items) {
+    if ((checked++ & 1023) === 0 && Date.now() > deadline) throw pdfWorkTookTooLong();
+    const y = item.transform[5];
+    if (!Number.isFinite(y)) {
+      rows.push([item]);
+      continue;
+    }
+    const home = bucketOf(y);
+    let found = -1;
+    for (const bucket of [home - 1, home, home + 1]) {
+      for (const row of byBucket.get(bucket) ?? []) {
+        const last = rows[row][rows[row].length - 1];
+        if (Math.abs(last.transform[5] - y) < Y_EPSILON && (found === -1 || row < found)) found = row;
+      }
+    }
+    if (found === -1) {
+      rows.push([item]);
+      place(home, rows.length - 1);
+      continue;
+    }
+    ;
+    rows[found].push(item);
+    const was = bucketOfRow[found];
+    if (was !== home) {
+      const list = byBucket.get(was);
+      const at = list.indexOf(found);
+      if (at >= 0) list.splice(at, 1);
+      place(home, found);
+    }
+  }
+  rows.sort((a, b) => {
+    const ya = a[0].transform[5];
+    const yb = b[0].transform[5];
+    const finiteA = Number.isFinite(ya);
+    const finiteB = Number.isFinite(yb);
+    if (finiteA && finiteB) return yb - ya;
+    if (finiteA) return -1;
+    if (finiteB) return 1;
+    return 0;
+  });
+  const lines = [];
+  for (const row of rows) {
+    row.sort((a, b) => a.transform[4] - b.transform[4]);
+    let line = "";
+    let prevEndX = null;
+    for (const item of row) {
+      const x = item.transform[4];
+      if (prevEndX !== null) {
+        const gap = x - prevEndX;
+        line += gap > 20 ? "   " : gap > 4 ? " " : "";
+      }
+      line += item.str;
+      prevEndX = x + (item.width ?? 0);
+    }
+    lines.push(line);
+  }
+  return lines.join("\n");
+}
+async function withPdfDocument(pdfjs, data, fn) {
+  const deadline = pdfWorkDeadline();
+  const loadingTask = pdfjs.getDocument({ data, useWorkerFetch: false, disableFontFace: true, verbosity: 0 });
+  try {
+    return await fn(await raceDeadline(loadingTask.promise, deadline), deadline);
+  } finally {
+    await raceDeadline(loadingTask.destroy(), Date.now() + PDF_TEARDOWN_MILLIS).catch(() => void 0);
+  }
+}
+var PDF_TEARDOWN_MILLIS = 5e3;
+async function raceDeadline(work, deadline) {
+  let timer;
+  try {
+    return await Promise.race([
+      work,
+      new Promise((_, reject) => {
+        timer = setTimeout(() => reject(pdfWorkTookTooLong()), Math.max(0, deadline - Date.now()));
+        timer.unref();
+      })
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+function withCrashGuard(fn, wrapCrash) {
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const cleanup = () => {
+      process.removeListener("uncaughtException", onCrash);
+      process.removeListener("unhandledRejection", onCrash);
+    };
+    const onCrash = (err) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(wrapCrash(err));
+    };
+    process.on("uncaughtException", onCrash);
+    process.on("unhandledRejection", onCrash);
+    fn().then(
+      (result) => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        resolve(result);
+      },
+      (err) => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        reject(err instanceof Error ? err : new Error(String(err)));
+      }
+    );
+  });
+}
+var MAX_OUTLINE_DEPTH = 500;
+async function extractPdfText(data, pagesSpec, layout = false) {
+  const pdfjs = await loadPdfjs();
+  if (!pdfjs) throw new Error("pdfjs-dist is not installed; run `npm install pdfjs-dist` to enable pdf-extract");
+  return withPdfDocument(pdfjs, data, async (doc, deadline) => {
+    const range = parsePageRange(pagesSpec, doc.numPages);
+    const start = range ? range.start : 1;
+    const end = range ? range.end : doc.numPages;
+    const pages = [];
+    let spent = 0;
+    for (let i = start; i <= end; i++) {
+      const page = await getPageWithinDeadline(doc, i, deadline);
+      const textItems = await readPageTextItems(page, MAX_PDF_TEXT_BYTES - spent, deadline);
+      const pageText = layout ? reconstructLayout(textItems, deadline) : textItems.map((item) => item.str).join(" ");
+      spent += pageText.length;
+      pages.push(pageText.trim());
+    }
+    const text = pages.join("\n\n");
+    assertPdfTextWithinBounds(text);
+    return { text, pageCount: doc.numPages, pagesExtracted: end - start + 1 };
+  });
+}
+async function locatePdfPages(data, pattern, opts) {
+  const guarded = compileGuardedRegex(pattern, opts.ignoreCase === true ? "i" : "");
+  if (!guarded.ok) throw new Error(`invalid regex pattern: ${pattern} (${guarded.reason})`);
+  const re = guarded.re;
+  const pdfjs = await loadPdfjs();
+  if (!pdfjs) throw new Error("pdfjs-dist is not installed; run `npm install pdfjs-dist` to enable pdf-extract");
+  const maxMatches = Math.min(opts.maxMatches ?? 50, MAX_LOCATE_MATCHES);
+  const context = Math.min(opts.context ?? 80, MAX_LOCATE_CONTEXT_CHARS);
+  return withPdfDocument(pdfjs, data, async (doc, deadline) => {
+    const range = parsePageRange(opts.pages, doc.numPages);
+    const start = range ? range.start : 1;
+    const end = range ? range.end : doc.numPages;
+    const matches = [];
+    let i = start;
+    for (; i <= end && matches.length < maxMatches; i++) {
+      const page = await getPageWithinDeadline(doc, i, deadline);
+      const textItems = await readPageTextItems(page, MAX_PDF_TEXT_BYTES, deadline);
+      const pageText = textItems.map((item) => item.str).join(" ");
+      const m = re.exec(pageText);
+      if (m === null) continue;
+      matches.push({ page: i, snippet: locateSnippet(pageText, m.index, m[0].length, context) });
+    }
+    const truncated = matches.length >= maxMatches && i <= end;
+    return { matches, truncated };
+  });
+}
+function locateSnippet(text, index, matchLen, context) {
+  const kept = Math.min(matchLen, context);
+  const pad = Math.max(0, context - kept);
+  const from = Math.max(0, index - Math.floor(pad / 2));
+  const to = Math.min(text.length, index + kept + Math.ceil(pad / 2));
+  const snippet = text.slice(from, to).replace(/\s+/g, " ").trim();
+  return matchLen > kept ? `${snippet}...` : snippet;
+}
+async function resolveDestPage(doc, dest, deadline) {
+  let explicitDest = dest;
+  if (typeof explicitDest === "string") {
+    explicitDest = await raceDeadline(doc.getDestination(explicitDest), deadline);
+  }
+  if (!Array.isArray(explicitDest) || explicitDest.length === 0) return null;
+  try {
+    const pageIndex = await raceDeadline(doc.getPageIndex(explicitDest[0]), deadline);
+    return pageIndex + 1;
+  } catch (err) {
+    if (err instanceof PdfTookTooLongError) throw err;
+    return null;
+  }
+}
+async function extractPdfOutline(data) {
+  const pdfjs = await loadPdfjs();
+  if (!pdfjs) throw new Error("pdfjs-dist is not installed; run `npm install pdfjs-dist` to enable pdf-outline");
+  return withCrashGuard(
+    () => withPdfDocument(pdfjs, data, async (doc, deadline) => {
+      const outline = await raceDeadline(doc.getOutline(), deadline);
+      if (!outline) return [];
+      const entries = [];
+      async function walk(items, level) {
+        if (level >= MAX_OUTLINE_DEPTH) return;
+        for (const item of items) {
+          if (entries.length >= MAX_OUTLINE_ENTRIES) return;
+          if (Date.now() > deadline) throw pdfWorkTookTooLong();
+          const page = await resolveDestPage(doc, item.dest, deadline);
+          const title = item.title.trim();
+          entries.push({ level, title: title.length > MAX_OUTLINE_TITLE_CHARS ? `${title.slice(0, MAX_OUTLINE_TITLE_CHARS)}...` : title, page });
+          if (item.items.length > 0) await walk(item.items, level + 1);
+        }
+      }
+      await walk(outline, 0);
+      return entries;
+    }),
+    (err) => new Error(`PDF outline is too deeply nested to read safely (pdfjs-dist crashed marshaling it): ${err instanceof Error ? err.message : String(err)}`)
+  );
+}
+async function pageHasText(doc, pageNum, deadline) {
+  const page = await getPageWithinDeadline(doc, pageNum, deadline);
+  let found = false;
+  for await (const chunk of pageTextItems(page, deadline)) {
+    if (Date.now() > deadline) throw pdfWorkTookTooLong();
+    found ||= chunk.some((item) => item.str.trim().length > 0);
+  }
+  return found;
+}
+async function extractPdfMeta(data) {
+  const pdfjs = await loadPdfjs();
+  if (!pdfjs) throw new Error("pdfjs-dist is not installed; run `npm install pdfjs-dist` to enable pdf-meta");
+  return withPdfDocument(pdfjs, data, async (doc, deadline) => {
+    const { info } = await raceDeadline(doc.getMetadata(), deadline);
+    const infoDict = info;
+    const sampleNums = Array.from(new Set([1, Math.ceil(doc.numPages / 2), doc.numPages].filter((n) => n >= 1 && n <= doc.numPages)));
+    let hasTextLayer = false;
+    for (const n of sampleNums) {
+      if (await pageHasText(doc, n, deadline)) {
+        hasTextLayer = true;
+        break;
+      }
+    }
+    return {
+      pageCount: doc.numPages,
+      title: typeof infoDict["Title"] === "string" && infoDict["Title"].trim().length > 0 ? infoDict["Title"] : null,
+      author: typeof infoDict["Author"] === "string" && infoDict["Author"].trim().length > 0 ? infoDict["Author"] : null,
+      hasTextLayer
+    };
+  });
+}
+
+export {
+  MAX_PDF_TEXT_BYTES,
+  MAX_PDF_INPUT_BYTES,
+  MAX_PDF_TEXT_ITEMS,
+  MAX_OUTLINE_ENTRIES,
+  MAX_OUTLINE_TITLE_CHARS,
+  MAX_LOCATE_MATCHES,
+  MAX_LOCATE_CONTEXT_CHARS,
+  MAX_PDF_WORK_MILLIS,
+  PdfRefusedError,
+  PdfTooLargeError,
+  PdfTookTooLongError,
+  pdfWorkDeadline,
+  assertPdfIsARegularFileWithinBounds,
+  assertPdfTextWithinBounds,
+  readPdfFileWithinBounds,
+  readAllWithinReportedSize,
+  readPageTextItems,
+  parsePageRange,
+  reconstructLayout,
+  withPdfDocument,
+  PDF_TEARDOWN_MILLIS,
+  extractPdfText,
+  locatePdfPages,
+  resolveDestPage,
+  extractPdfOutline,
+  extractPdfMeta
+};