ppt-template-reuse 0.4.0-rc.2

package/src/core/source-extractor.mjs

@@ -0,0 +1,412 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import JSZip from "jszip";
+import { DOMParser } from "@xmldom/xmldom";
+
+import { fingerprint, sha256File } from "./io.mjs";
+
+const parser = new DOMParser();
+const TARGET_UNIT_CHARS = 300;
+const MAX_UNIT_CHARS = 400;
+
+function localName(node) {
+  return String(node?.localName || node?.nodeName || "").split(":").pop();
+}
+
+function descendants(node, wanted) {
+  const result = [];
+  const visit = (current) => {
+    for (const child of Array.from(current?.childNodes || [])) {
+      if (child.nodeType === 1 && localName(child) === wanted) result.push(child);
+      visit(child);
+    }
+  };
+  visit(node);
+  return result;
+}
+
+function attribute(node, wanted) {
+  for (const item of Array.from(node?.attributes || [])) {
+    if (localName(item) === wanted) return item.value;
+  }
+  return undefined;
+}
+
+function nodeText(node) {
+  return descendants(node, "t")
+    .map((item) => item.textContent || "")
+    .join("")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function compactText(value) {
+  return String(value || "").replace(/\s+/g, " ").trim();
+}
+
+function splitOversizedToken(token, maximum = MAX_UNIT_CHARS) {
+  const value = compactText(token);
+  if (value.length <= maximum) return value ? [value] : [];
+  const chunks = [];
+  let remaining = value;
+  while (remaining.length > maximum) {
+    const window = remaining.slice(0, maximum + 1);
+    const breakAt = Math.max(
+      window.lastIndexOf("，"),
+      window.lastIndexOf(","),
+      window.lastIndexOf("、"),
+      window.lastIndexOf(" "),
+    );
+    const boundary =
+      breakAt >= Math.floor(maximum * 0.55) ? breakAt + 1 : maximum;
+    chunks.push(compactText(remaining.slice(0, boundary)));
+    remaining = compactText(remaining.slice(boundary));
+  }
+  if (remaining) chunks.push(remaining);
+  return chunks;
+}
+
+export function splitSourceText(
+  value,
+  {
+    targetChars = TARGET_UNIT_CHARS,
+    maxChars = MAX_UNIT_CHARS,
+  } = {},
+) {
+  const normalized = compactText(value);
+  if (!normalized) return [];
+  if (normalized.length <= maxChars) return [normalized];
+  const sentences =
+    normalized.match(/[^。！？!?；;\n]+[。！？!?；;]?/gu)?.flatMap((sentence) =>
+      splitOversizedToken(sentence, maxChars),
+    ) || splitOversizedToken(normalized, maxChars);
+  const chunks = [];
+  let current = "";
+  for (const sentence of sentences) {
+    const candidate = compactText(`${current} ${sentence}`);
+    if (current && candidate.length > maxChars) {
+      chunks.push(current);
+      current = sentence;
+      continue;
+    }
+    current = candidate;
+    if (current.length >= targetChars) {
+      chunks.push(current);
+      current = "";
+    }
+  }
+  if (current) {
+    const previous = chunks.at(-1);
+    if (
+      previous &&
+      current.length < Math.floor(targetChars * 0.35) &&
+      previous.length + current.length + 1 <= maxChars
+    ) {
+      chunks[chunks.length - 1] = compactText(`${previous} ${current}`);
+    } else {
+      chunks.push(current);
+    }
+  }
+  return chunks;
+}
+
+function splitTableUnit(unit) {
+  const rows = unit.metadata?.rows;
+  if (!Array.isArray(rows) || rows.length <= 1) return [unit];
+  const chunks = [];
+  let current = [];
+  let currentChars = 0;
+  for (const row of rows) {
+    const rowText = row.map((cell) => compactText(cell)).join(" | ");
+    if (current.length && currentChars + rowText.length > MAX_UNIT_CHARS) {
+      chunks.push(current);
+      current = [];
+      currentChars = 0;
+    }
+    current.push(row);
+    currentChars += rowText.length;
+  }
+  if (current.length) chunks.push(current);
+  if (chunks.length === 1) return [unit];
+  return chunks.map((chunk, index) => ({
+    ...unit,
+    text: chunk.map((row) => row.join(" | ")).join("\n"),
+    anchor: `${unit.anchor}:segment:${index + 1}`,
+    parentAnchor: unit.anchor,
+    segmentIndex: index + 1,
+    segmentCount: chunks.length,
+    metadata: { ...unit.metadata, rows: chunk },
+  }));
+}
+
+function segmentRawUnit(unit) {
+  if (unit.kind === "heading") return [unit];
+  if (unit.kind === "table") return splitTableUnit(unit);
+  const segments = splitSourceText(unit.text);
+  if (segments.length <= 1) return [unit];
+  return segments.map((text, index) => ({
+    ...unit,
+    text,
+    anchor: `${unit.anchor}:segment:${index + 1}`,
+    parentAnchor: unit.anchor,
+    segmentIndex: index + 1,
+    segmentCount: segments.length,
+  }));
+}
+
+function makeUnit({
+  index,
+  kind,
+  text,
+  anchor,
+  parentAnchor,
+  segmentIndex,
+  segmentCount,
+  title,
+  page,
+  metadata = {},
+}) {
+  const normalized = compactText(text);
+  return {
+    id: `U${String(index).padStart(4, "0")}`,
+    anchor,
+    parentAnchor: parentAnchor || undefined,
+    segmentIndex: segmentIndex || undefined,
+    segmentCount: segmentCount || undefined,
+    extractionKind: kind,
+    sourcePage: page || undefined,
+    topicTitle: title || undefined,
+    text: normalized,
+    summary: normalized.slice(0, 180),
+    detail: normalized,
+    ...metadata,
+  };
+}
+
+async function extractTextLike(sourcePath, extension) {
+  const text = await fs.readFile(sourcePath, "utf8");
+  const lines = text.split(/\r?\n/);
+  const units = [];
+  let heading = "";
+  let buffer = [];
+  const flush = () => {
+    const value = buffer.join(" ").replace(/\s+/g, " ").trim();
+    if (value) {
+      units.push({
+        kind: "paragraph",
+        text: value,
+        title: heading,
+        anchor: `${path.basename(sourcePath)}#paragraph:${units.length + 1}`,
+      });
+    }
+    buffer = [];
+  };
+  for (const raw of lines) {
+    const line = raw.trim();
+    const markdownHeading = extension === ".md" && line.match(/^#{1,6}\s+(.+)$/);
+    if (markdownHeading) {
+      flush();
+      heading = markdownHeading[1].trim();
+      units.push({
+        kind: "heading",
+        text: heading,
+        title: heading,
+        anchor: `${path.basename(sourcePath)}#heading:${units.length + 1}`,
+      });
+    } else if (!line) {
+      flush();
+    } else {
+      buffer.push(line.replace(/^[-*+]\s+/, ""));
+    }
+  }
+  flush();
+  return { units, scannedPages: [] };
+}
+
+async function extractDocx(sourcePath) {
+  const zip = await JSZip.loadAsync(await fs.readFile(sourcePath));
+  const xml = await zip.file("word/document.xml")?.async("string");
+  if (!xml) throw new Error("DOCX is missing word/document.xml");
+  const document = parser.parseFromString(xml, "application/xml");
+  const body = descendants(document, "body")[0];
+  const units = [];
+  let heading = "";
+  let paragraphIndex = 0;
+  let tableIndex = 0;
+  for (const child of Array.from(body?.childNodes || [])) {
+    if (child.nodeType !== 1) continue;
+    if (localName(child) === "p") {
+      const text = nodeText(child);
+      if (!text) continue;
+      paragraphIndex += 1;
+      const style = descendants(child, "pStyle")
+        .map((item) => attribute(item, "val"))
+        .find(Boolean);
+      const isHeading = /heading|标题/i.test(String(style || ""));
+      if (isHeading) heading = text;
+      units.push({
+        kind: isHeading ? "heading" : "paragraph",
+        text,
+        title: heading,
+        anchor: `${path.basename(sourcePath)}#paragraph:${paragraphIndex}`,
+        metadata: { paragraphStyle: style || undefined },
+      });
+    } else if (localName(child) === "tbl") {
+      tableIndex += 1;
+      const rows = descendants(child, "tr").map((row) =>
+        descendants(row, "tc").map((cell) => nodeText(cell)),
+      );
+      const text = rows.map((row) => row.join(" | ")).join("\n").trim();
+      if (!text) continue;
+      units.push({
+        kind: "table",
+        text,
+        title: heading,
+        anchor: `${path.basename(sourcePath)}#table:${tableIndex}`,
+        metadata: { rows },
+      });
+    }
+  }
+  return { units, scannedPages: [] };
+}
+
+function relationshipMap(xml) {
+  const document = parser.parseFromString(xml, "application/xml");
+  return new Map(
+    descendants(document, "Relationship").map((node) => [
+      attribute(node, "Id"),
+      attribute(node, "Target"),
+    ]),
+  );
+}
+
+async function extractPptx(sourcePath) {
+  const zip = await JSZip.loadAsync(await fs.readFile(sourcePath));
+  const presentationXml = await zip.file("ppt/presentation.xml")?.async("string");
+  const relsXml = await zip
+    .file("ppt/_rels/presentation.xml.rels")
+    ?.async("string");
+  if (!presentationXml || !relsXml) {
+    throw new Error("PPTX is missing presentation relationships");
+  }
+  const rels = relationshipMap(relsXml);
+  const presentation = parser.parseFromString(
+    presentationXml,
+    "application/xml",
+  );
+  const slideIds = descendants(presentation, "sldId");
+  const units = [];
+  for (let index = 0; index < slideIds.length; index += 1) {
+    const relId =
+      slideIds[index].getAttribute("r:id") ||
+      Array.from(slideIds[index].attributes || []).find(
+        (item) => item.name === "r:id",
+      )?.value;
+    const target = rels.get(relId);
+    if (!target) continue;
+    const slidePath = path.posix.normalize(path.posix.join("ppt", target));
+    const slideXml = await zip.file(slidePath)?.async("string");
+    if (!slideXml) continue;
+    const slide = parser.parseFromString(slideXml, "application/xml");
+    const paragraphs = descendants(slide, "p")
+      .map((item) => nodeText(item))
+      .filter(Boolean);
+    if (!paragraphs.length) continue;
+    units.push({
+      kind: "slide",
+      text: paragraphs.join("\n"),
+      title: paragraphs[0],
+      page: index + 1,
+      anchor: `${path.basename(sourcePath)}#slide:${index + 1}`,
+      metadata: { paragraphs },
+    });
+  }
+  return { units, scannedPages: [] };
+}
+
+async function extractPdf(sourcePath) {
+  const canvas = await import("@napi-rs/canvas").catch(() => null);
+  for (const name of ["DOMMatrix", "ImageData", "Path2D"]) {
+    if (!globalThis[name] && canvas?.[name]) globalThis[name] = canvas[name];
+  }
+  const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
+  const bytes = new Uint8Array(await fs.readFile(sourcePath));
+  const loadingTask = pdfjs.getDocument({
+    data: bytes,
+    disableFontFace: true,
+    isEvalSupported: false,
+    useSystemFonts: true,
+  });
+  const document = await loadingTask.promise;
+  const units = [];
+  const scannedPages = [];
+  for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {
+    const page = await document.getPage(pageNumber);
+    const content = await page.getTextContent();
+    const text = content.items
+      .map((item) => String(item.str || ""))
+      .join(" ")
+      .replace(/\s+/g, " ")
+      .trim();
+    if (text.replace(/\s+/g, "").length < 20) {
+      scannedPages.push(pageNumber);
+      continue;
+    }
+    units.push({
+      kind: "page",
+      text,
+      title: text.slice(0, 80),
+      page: pageNumber,
+      anchor: `${path.basename(sourcePath)}#page:${pageNumber}`,
+    });
+  }
+  return { units, scannedPages, pageCount: document.numPages };
+}
+
+export async function extractSource(sourcePath) {
+  const resolved = path.resolve(sourcePath);
+  const extension = path.extname(resolved).toLowerCase();
+  let extracted;
+  if ([".md", ".txt"].includes(extension)) {
+    extracted = await extractTextLike(resolved, extension);
+  } else if (extension === ".docx") {
+    extracted = await extractDocx(resolved);
+  } else if (extension === ".pptx") {
+    extracted = await extractPptx(resolved);
+  } else if (extension === ".pdf") {
+    extracted = await extractPdf(resolved);
+  } else {
+    throw new Error(
+      `Unsupported source format ${extension || "(none)"}; use DOCX, PDF, PPTX, Markdown, or text.`,
+    );
+  }
+  const units = extracted.units
+    .flatMap(segmentRawUnit)
+    .filter((item) => String(item.text || "").trim())
+    .map((item, index) => makeUnit({ ...item, index: index + 1 }));
+  const result = {
+    schemaVersion: 1,
+    source: {
+      path: resolved,
+      sha256: await sha256File(resolved),
+      extension,
+    },
+    units,
+    scannedPages: extracted.scannedPages || [],
+    pageCount: extracted.pageCount,
+  };
+  result.extractionFingerprint = fingerprint(result);
+  result.actionRequired = {
+    type: "source-semantics",
+    instructions: [
+      "Read every extracted unit and retain its id and anchor.",
+      "Label each unit as background, fact, chronology, issue, rule, method, evidence, data, application, analysis, counterargument, rebuttal, conclusion, or recommendation.",
+      "Assign critical, supporting, or optional priority and core, appendix, or exclude presentation relevance.",
+      "Add summary, detail, claim, evidence, whyItMatters, and explicit counterargument/rebuttal links where applicable.",
+      "Add short source-grounded displayFragments for dates, labels, steps, metric labels, and descriptions when a structured template layout needs them.",
+      "Do not omit, merge away, or invent source units. Segmented units retain parentAnchor and segmentIndex so long source blocks remain traceable.",
+    ],
+    scannedPages: result.scannedPages,
+  };
+  return result;
+}