paper-drift 0.0.0-stage → 0.1.0

package/src/markdown/converter.js

@@ -0,0 +1,254 @@
+/**
+ * @file src/markdown/converter.js
+ * @description Direct WebAssembly PDF to GitHub Flavored Markdown (GFM) conversion engine.
+ * Converts academic papers and enterprise documents directly into clean, token-efficient Markdown
+ * with spatial reading order, GFM pipe tables, LaTeX math restoration, and typography detection.
+ * Part of paper-drift - https://github.com/Sooriya04/Paper-Drift
+ */
+
+import mupdf from 'mupdf';
+import fs from 'fs';
+import path from 'path';
+import { parsePageRange } from '../converter.js';
+import { resolveReadingOrder } from '../spatial/reading-order.js';
+import { reconstructTableFromLines } from '../tables/reconstructor.js';
+import { mapUnicodeToLatex, normalizeMathIndices } from '../math/index.js';
+
+/**
+ * Converts a structured page JSON representation into clean, structured Markdown.
+ *
+ * @param {Object} pageJson - Parsed JSON from `page.toStructuredText().asJSON()`
+ * @param {number} widthPt - Page width in points
+ * @param {number} heightPt - Page height in points
+ * @param {Object} [options={}]
+ * @returns {{ markdown: string, tablesCount: number, headingsCount: number }}
+ */
+export function pageToMarkdown(pageJson, widthPt, heightPt, options = {}) {
+  const spatial = resolveReadingOrder(pageJson, widthPt, heightPt);
+  const blocks = spatial.blocks || [];
+
+  if (blocks.length === 0) {
+    return { markdown: '', tablesCount: 0, headingsCount: 0 };
+  }
+
+  // Calculate median body font size across all lines on this page
+  const fontSizes = [];
+  for (const b of blocks) {
+    for (const l of (b.lines || [])) {
+      if (l.font?.size) fontSizes.push(l.font.size);
+    }
+  }
+  fontSizes.sort((a, b) => a - b);
+  const medianFontSize = fontSizes.length > 0 ? fontSizes[Math.floor(fontSizes.length / 2)] : 10.0;
+
+  const mdSections = [];
+  let tablesCount = 0;
+  let headingsCount = 0;
+
+  let i = 0;
+  while (i < blocks.length) {
+    const block = blocks[i];
+    const lines = block.lines || [];
+    if (lines.length === 0) {
+      i++;
+      continue;
+    }
+
+    const blockText = lines.map(l => l.text).join(' ').trim();
+
+    // 1. Check for Table Caption (e.g. "Table 1: ...", "Table 2. ...", "TABLE I: ...")
+    if (/^(Table|TABLE)\s+([0-9]+|[IVXLCDM]+)[:.]/i.test(blockText)) {
+      mdSections.push(`**${blockText}**`);
+
+      // Look ahead to collect consecutive table rows/blocks directly beneath this caption
+      const tableLines = [];
+      let j = i + 1;
+      let lastBlockBottom = block.bbox.y + block.bbox.h;
+
+      while (j < blocks.length) {
+        const nextB = blocks[j];
+        const nextLines = nextB.lines || [];
+        if (nextLines.length === 0) { j++; continue; }
+
+        // Stop if vertical gap between blocks exceeds standard table row spacing (> 20pt)
+        const gap = nextB.bbox.y - lastBlockBottom;
+        if (tableLines.length > 0 && gap > 20) {
+          break;
+        }
+
+        // Stop if next block is a full-width paragraph sentence (> 240pt with > 8 words)
+        const isParagraph = nextLines.some(l => (l.bbox?.w || 0) > 240 && (l.text || '').split(' ').length > 8);
+        if (isParagraph) {
+          break;
+        }
+
+        tableLines.push(...nextLines);
+        lastBlockBottom = nextB.bbox.y + nextB.bbox.h;
+        j++;
+      }
+
+      if (tableLines.length >= 4) {
+        const tableMd = reconstructTableFromLines(tableLines);
+        if (tableMd) {
+          mdSections.push(tableMd);
+          tablesCount++;
+          i = j;
+          continue;
+        }
+      }
+
+      i++;
+      continue;
+    }
+
+    // 2. Check for Table Grid within a single block
+    if (lines.length >= 4 && !lines.some(l => (l.bbox?.w || 0) > 250 && (l.text || '').split(' ').length > 8)) {
+      const tableMd = reconstructTableFromLines(lines);
+      if (tableMd) {
+        mdSections.push(tableMd);
+        tablesCount++;
+        i++;
+        continue;
+      }
+    }
+
+    // 3. Check for Display Math Equations (e.g. PE(pos, 2i) = sin(...) or expressions ending in equation tags (1))
+    const isFormula = (
+      lines.length <= 3 &&
+      (/=\s*(sin|cos|tan|softmax|exp|\sum|\int|arg)/i.test(blockText) ||
+       /PE\(/i.test(blockText) ||
+       /\(\d+\)$/.test(blockText) && /[=+\-·×/]/.test(blockText))
+    );
+
+    if (isFormula) {
+      let formulaText = mapUnicodeToLatex(blockText);
+      formulaText = formulaText.replace(/\((\d+)\)$/, '\\tag{$1}');
+      mdSections.push(`$$\n${formulaText}\n$$`);
+      i++;
+      continue;
+    }
+
+    // 4. Check for Document Typography & Headings
+    const firstLine = lines[0];
+    const fontSize = firstLine.font?.size || medianFontSize;
+    const isBold = firstLine.font?.weight >= 700 || /bold/i.test(firstLine.font?.family || '');
+    const isNumberedHeading = /^(\d+(\.\d+)*)\s+[A-Z]/.test(blockText);
+
+    if (fontSize >= medianFontSize * 1.45) {
+      mdSections.push(`# ${blockText}`);
+      headingsCount++;
+    } else if (fontSize >= medianFontSize * 1.15 || isNumberedHeading) {
+      mdSections.push(`## ${blockText}`);
+      headingsCount++;
+    } else if (fontSize >= medianFontSize * 1.05 && isBold && blockText.length < 80) {
+      mdSections.push(`### ${blockText}`);
+      headingsCount++;
+    } else if (/^[•–\-*]\s+/.test(blockText)) {
+      // Unordered list item
+      mdSections.push(`- ${blockText.replace(/^[•–\-*]\s+/, '')}`);
+    } else {
+      // 5. Standard body paragraph
+      const formattedPara = lines
+        .map(l => l.text)
+        .join(' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+
+      if (formattedPara) {
+        mdSections.push(formattedPara);
+      }
+    }
+
+    i++;
+  }
+
+  return {
+    markdown: mdSections.join('\n\n'),
+    tablesCount,
+    headingsCount
+  };
+}
+
+/**
+ * Converts a PDF buffer or file path directly into GitHub Flavored Markdown (GFM).
+ *
+ * @param {string|Buffer|Uint8Array} input - File path or binary buffer of the PDF.
+ * @param {Object} [options={}] - Conversion options.
+ * @param {string|number[]} [options.pageRange='all'] - Page range to process (e.g. "1-5", "2,4", "all").
+ * @param {function} [options.onProgress] - Optional progress callback `({ current, total, pageNumber }) => void`.
+ * @returns {Promise<{ markdown: string, pages: Array<Object>, totalPages: number, pagesProcessed: number, durationMs: number }>}
+ */
+export async function convertPdfToMarkdown(input, options = {}) {
+  const startTime = Date.now();
+  const pageRangeStr = options.pageRange || 'all';
+
+  let pdfBuffer;
+  if (typeof input === 'string') {
+    if (!fs.existsSync(input)) {
+      throw new Error(`File not found: ${input}`);
+    }
+    pdfBuffer = fs.readFileSync(input);
+  } else if (Buffer.isBuffer(input) || input instanceof Uint8Array) {
+    pdfBuffer = input;
+  } else {
+    throw new TypeError('Input must be a valid file path string, Buffer, or Uint8Array.');
+  }
+
+  const doc = mupdf.Document.openDocument(pdfBuffer, 'application/pdf');
+  const totalPages = doc.countPages();
+  const targetIndices = parsePageRange(pageRangeStr, totalPages);
+
+  if (targetIndices.length === 0) {
+    throw new Error(`No matching pages found for page range: "${pageRangeStr}" (Document has ${totalPages} pages).`);
+  }
+
+  const pagesMarkdown = [];
+  const processedPages = [];
+
+  for (let step = 0; step < targetIndices.length; step++) {
+    const pageIndex = targetIndices[step];
+    const pageNumber = pageIndex + 1;
+
+    if (typeof options.onProgress === 'function') {
+      options.onProgress({
+        current: step + 1,
+        total: targetIndices.length,
+        pageNumber,
+        totalPages
+      });
+    }
+
+    const page = doc.loadPage(pageIndex);
+    const bounds = page.getBounds();
+    const widthPt = bounds[2] - bounds[0];
+    const heightPt = bounds[3] - bounds[1];
+
+    const structuredText = page.toStructuredText();
+    const pageJson = JSON.parse(structuredText.asJSON());
+
+    const result = pageToMarkdown(pageJson, widthPt, heightPt, options);
+
+    processedPages.push({
+      pageNumber,
+      width: widthPt,
+      height: heightPt,
+      tablesCount: result.tablesCount,
+      headingsCount: result.headingsCount,
+      markdown: result.markdown
+    });
+
+    if (result.markdown) {
+      pagesMarkdown.push(result.markdown);
+    }
+  }
+
+  const fullMarkdown = pagesMarkdown.join('\n\n---\n\n');
+
+  return {
+    markdown: fullMarkdown,
+    pages: processedPages,
+    totalPages,
+    pagesProcessed: targetIndices.length,
+    durationMs: Date.now() - startTime
+  };
+}

package/src/markdown/index.js

@@ -0,0 +1,10 @@
+/**
+ * @file src/markdown/index.js
+ * @description Direct Markdown conversion module entrypoint.
+ * Part of paper-drift - https://github.com/Sooriya04/Paper-Drift
+ */
+
+export {
+  convertPdfToMarkdown,
+  pageToMarkdown
+} from './converter.js';

package/src/math/index.js

@@ -0,0 +1,26 @@
+/**
+ * @file src/math/index.js
+ * @description Main export for the paper-drift LaTeX math restoration engine.
+ * Provides symbol mapping, sub/superscript resolution, equation tagging, and math block formatting.
+ * Part of paper-drift - https://github.com/Sooriya04/Paper-Drift
+ */
+
+export {
+  restoreLineMath,
+  restoreBlockMath,
+  formatInlineMath,
+  normalizeMathIndices,
+  isMathFont,
+  containsMathSymbols
+} from './restorer.js';
+
+export {
+  ALL_MATH_SYMBOLS,
+  mapUnicodeToLatex,
+  GREEK_LOWERCASE,
+  GREEK_UPPERCASE,
+  MATH_OPERATORS,
+  MATH_RELATIONS,
+  ARROWS_AND_LOGIC,
+  BLACKBOARD_SETS
+} from './symbols.js';

package/src/math/restorer.js

@@ -0,0 +1,221 @@
+/**
+ * @file src/math/restorer.js
+ * @description Advanced LaTeX math heuristic restoration engine for academic and scientific PDFs.
+ * Reconstructs sub/superscripts, fractions, square roots, Greek/symbol glyphs,
+ * and frames expressions into inline ($...$) or display ($$...$$) LaTeX equations.
+ * Part of paper-drift - https://github.com/Sooriya04/Paper-Drift
+ */
+
+import { ALL_MATH_SYMBOLS, mapUnicodeToLatex } from './symbols.js';
+
+const MATH_FONT_PATTERNS = [
+  /cmr/i,
+  /cmsy/i,
+  /cmmi/i,
+  /math/i,
+  /symbol/i,
+  /stix/i,
+  /euler/i,
+  /msam/i,
+  /msbm/i
+];
+
+/**
+ * Checks if a font name indicates a LaTeX or mathematical typography font.
+ * @param {string} fontName
+ * @returns {boolean}
+ */
+export function isMathFont(fontName) {
+  if (!fontName) return false;
+  return MATH_FONT_PATTERNS.some(p => p.test(fontName));
+}
+
+/**
+ * Checks if a string contains mathematical or Greek symbols.
+ * @param {string} str
+ * @returns {boolean}
+ */
+export function containsMathSymbols(str) {
+  if (!str) return false;
+  for (const sym of Object.keys(ALL_MATH_SYMBOLS)) {
+    if (str.includes(sym)) return true;
+  }
+  return false;
+}
+
+/**
+ * Normalizes math variable indices (e.g. dk -> d_k, dmodel -> d_{model}, headi -> head_i).
+ * @param {string} str
+ * @returns {string}
+ */
+export function normalizeMathIndices(str) {
+  if (!str) return '';
+
+  // Common academic math variable patterns
+  return str
+    .replace(/\b([dD])([kqv])\b/g, '$1_{$2}')
+    .replace(/\b(d)(model)\b/g, '$1_{\\text{$2}}')
+    .replace(/\b(head)([i-k])\b/g, '\\text{$1}_{$2}')
+    .replace(/\b([Ww])([0-9a-zA-Z])\b/g, '$1_{$2}')
+    .replace(/\b([XxYyZz])([i-k0-9])\b/g, '$1_{$2}');
+}
+
+/**
+ * Restores sub/superscripts, symbols, and operators in a horizontal line of text tokens.
+ *
+ * @param {Array<{ str: string, x: number, y: number, fontSize: number, fontName?: string, w?: number, h?: number }>} tokens
+ * @param {Object} [options={}]
+ * @param {boolean} [options.bottomUp=false] - True if coordinates are bottom-up (PDF standard) vs top-down (DOM)
+ * @returns {string} Reconstructed LaTeX expression
+ */
+export function restoreLineMath(tokens, options = {}) {
+  const { bottomUp = false } = options;
+  if (!tokens || tokens.length === 0) return '';
+
+  // Sort primarily by X coordinate
+  const sorted = [...tokens]
+    .filter(t => t.str && t.str.trim() !== '')
+    .sort((a, b) => a.x - b.x);
+
+  if (sorted.length === 0) return '';
+
+  let out = '';
+  let prevToken = null;
+
+  for (let i = 0; i < sorted.length; i++) {
+    const curr = sorted[i];
+    let tokenText = mapUnicodeToLatex(curr.str);
+
+    // Apply index normalization for math variables
+    if (isMathFont(curr.fontName) || curr.fontSize <= 9) {
+      tokenText = normalizeMathIndices(tokenText);
+    }
+
+    if (prevToken) {
+      const isSmaller = curr.fontSize < prevToken.fontSize * 0.85;
+      const yDiff = bottomUp ? (curr.y - prevToken.y) : (prevToken.y - curr.y);
+
+      // Superscript check: smaller font and elevated baseline
+      if (isSmaller && yDiff > 1.0) {
+        out = out.trimEnd() + `^{${tokenText}}`;
+        prevToken = curr;
+        continue;
+      }
+
+      // Subscript check: smaller font and lowered baseline
+      if (isSmaller && yDiff < -1.0) {
+        out = out.trimEnd() + `_{${tokenText}}`;
+        prevToken = curr;
+        continue;
+      }
+
+      // Spacing between distinct mathematical terms
+      const xGap = curr.x - (prevToken.x + (prevToken.w || prevToken.fontSize * 0.45));
+      if (xGap > 2.0 && !out.endsWith(' ') && !out.endsWith('{') && !out.endsWith('_') && !out.endsWith('^') && !out.endsWith('\\')) {
+        out += ' ';
+      }
+    }
+
+    // Format square roots: √x -> \sqrt{x}
+    if (tokenText.startsWith('\\sqrt')) {
+      const remainder = tokenText.slice(5).trim();
+      tokenText = remainder.length > 0 ? `\\sqrt{${remainder}}` : '\\sqrt';
+    }
+
+    out += tokenText;
+    prevToken = curr;
+  }
+
+  return out.replace(/\s+/g, ' ').trim();
+}
+
+/**
+ * Reconstructs a full mathematical block, framing as display ($$...$$) or inline ($...$).
+ *
+ * @param {Object} block - Text block with lines of tokens
+ * @param {Object} [options={}]
+ * @returns {{ isMath: boolean, latex: string }}
+ */
+export function restoreBlockMath(block, options = {}) {
+  if (!block || !block.lines || block.lines.length === 0) {
+    return { isMath: false, latex: '' };
+  }
+
+  // Extract all tokens across lines
+  const allTokens = [];
+  let mathTokensCount = 0;
+
+  for (const line of block.lines) {
+    const tokens = line.tokens || [{
+      str: line.text,
+      x: line.bbox?.x ?? line.x ?? 0,
+      y: line.bbox?.y ?? line.y ?? 0,
+      w: line.bbox?.w,
+      h: line.bbox?.h,
+      fontSize: line.font?.size || line.bbox?.h || 10,
+      fontName: line.font?.name || ''
+    }];
+
+    for (const t of tokens) {
+      allTokens.push(t);
+      if (isMathFont(t.fontName) || containsMathSymbols(t.str)) {
+        mathTokensCount++;
+      }
+    }
+  }
+
+  const rawText = block.lines.map(l => l.text).join(' ').trim();
+  const mathDensity = allTokens.length > 0 ? (mathTokensCount / allTokens.length) : 0;
+
+  // Check for equation tag pattern at the end: (1), (2.4), (12)
+  const equationLabelMatch = rawText.match(/\((\d+(\.\d+)?)\)\s*$/);
+  const hasEquationLabel = Boolean(equationLabelMatch);
+
+  // High math likelihood triggers
+  const isEquation = mathDensity >= 0.25 ||
+                     hasEquationLabel ||
+                     (containsMathSymbols(rawText) && (rawText.includes('=') || rawText.includes('+')));
+
+  if (!isEquation) {
+    return { isMath: false, latex: rawText };
+  }
+
+  // Reconstruct math expression
+  let restored = restoreLineMath(allTokens, options);
+
+  // If there's an equation tag at the end, convert to \tag{N}
+  if (equationLabelMatch) {
+    const label = equationLabelMatch[1];
+    restored = restored.replace(/\(\d+(\.\d+)?\)\s*$/, '').trim();
+    return {
+      isMath: true,
+      latex: `$$\n${restored} \\tag{${label}}\n$$`
+    };
+  }
+
+  return {
+    isMath: true,
+    latex: `$$\n${restored}\n$$`
+  };
+}
+
+/**
+ * Scans regular text paragraphs and converts inline math expressions (e.g. d_k, x^2, \alpha)
+ * into standard inline LaTeX ($...$).
+ *
+ * @param {string} text
+ * @returns {string} Text with inline math wrapped in $...$
+ */
+export function formatInlineMath(text) {
+  if (!text) return '';
+
+  let result = mapUnicodeToLatex(text);
+  result = normalizeMathIndices(result);
+
+  // Wrap isolated sub/superscript tokens or Greek symbols in $...$
+  result = result.replace(/(^|\s)([a-zA-Z0-9\\]+[\^_]\{[^{}]+\})($|\s|[.,;:])/g, (match, prefix, math, suffix) => {
+    return `${prefix}$${math}$${suffix}`;
+  });
+
+  return result;
+}

package/src/math/symbols.js

@@ -0,0 +1,153 @@
+/**
+ * @file src/math/symbols.js
+ * @description Comprehensive Unicode-to-LaTeX mathematical symbol and Greek alphabet mapping dictionary.
+ * Translates raw glyphs from academic PDF fonts (CMR, CMSY, CMMI, Symbol) into clean LaTeX expressions.
+ * Part of paper-drift - https://github.com/Sooriya04/Paper-Drift
+ */
+
+export const GREEK_LOWERCASE = {
+  'α': '\\alpha',
+  'β': '\\beta',
+  'γ': '\\gamma',
+  'δ': '\\delta',
+  'ε': '\\epsilon',
+  'ϵ': '\\varepsilon',
+  'ζ': '\\zeta',
+  'η': '\\eta',
+  'θ': '\\theta',
+  'ϑ': '\\vartheta',
+  'ι': '\\iota',
+  'κ': '\\kappa',
+  'λ': '\\lambda',
+  'μ': '\\mu',
+  'ν': '\\nu',
+  'ξ': '\\xi',
+  'π': '\\pi',
+  'ϖ': '\\varpi',
+  'ρ': '\\rho',
+  'ϱ': '\\varrho',
+  'σ': '\\sigma',
+  'ς': '\\varsigma',
+  'τ': '\\tau',
+  'υ': '\\upsilon',
+  'φ': '\\phi',
+  'ϕ': '\\varphi',
+  'χ': '\\chi',
+  'ψ': '\\psi',
+  'ω': '\\omega'
+};
+
+export const GREEK_UPPERCASE = {
+  'Γ': '\\Gamma',
+  'Δ': '\\Delta',
+  'Θ': '\\Theta',
+  'Λ': '\\Lambda',
+  'Ξ': '\\Xi',
+  'Π': '\\Pi',
+  'Σ': '\\Sigma',
+  'Υ': '\\Upsilon',
+  'Φ': '\\Phi',
+  'Ψ': '\\Psi',
+  'Ω': '\\Omega'
+};
+
+export const MATH_OPERATORS = {
+  '∑': '\\sum',
+  '∏': '\\prod',
+  '∐': '\\coprod',
+  '∫': '\\int',
+  '∬': '\\iint',
+  '∭': '\\iiint',
+  '∮': '\\oint',
+  '√': '\\sqrt',
+  '±': '\\pm',
+  '∓': '\\mp',
+  '×': '\\times',
+  '÷': '\\div',
+  '·': '\\cdot',
+  '∘': '\\circ',
+  '∙': '\\bullet',
+  '⊗': '\\otimes',
+  '⊕': '\\oplus',
+  '⊙': '\\odot',
+  '∗': '*',
+  '⋆': '\\star'
+};
+
+export const MATH_RELATIONS = {
+  '≤': '\\le',
+  '≥': '\\ge',
+  '≠': '\\neq',
+  '≈': '\\approx',
+  '≡': '\\equiv',
+  '∼': '\\sim',
+  '≃': '\\simeq',
+  '∝': '\\propto',
+  '≪': '\\ll',
+  '≫': '\\gg',
+  '∈': '\\in',
+  '∉': '\\notin',
+  '∋': '\\owns',
+  '⊂': '\\subset',
+  '⊃': '\\supset',
+  '⊆': '\\subseteq',
+  '⊇': '\\supseteq',
+  '∩': '\\cap',
+  '∪': '\\cup'
+};
+
+export const ARROWS_AND_LOGIC = {
+  '→': '\\to',
+  '←': '\\leftarrow',
+  '↔': '\\leftrightarrow',
+  '⇒': '\\implies',
+  '⇐': '\\impliedby',
+  '⇔': '\\iff',
+  '↑': '\\uparrow',
+  '↓': '\\downarrow',
+  '∀': '\\forall',
+  '∃': '\\exists',
+  '∄': '\\nexists',
+  '∇': '\\nabla',
+  '∂': '\\partial',
+  '∞': '\\infty',
+  '∠': '\\angle',
+  '⊥': '\\perp'
+};
+
+export const BLACKBOARD_SETS = {
+  'ℝ': '\\mathbb{R}',
+  'ℂ': '\\mathbb{C}',
+  'ℕ': '\\mathbb{N}',
+  'ℤ': '\\mathbb{Z}',
+  'ℚ': '\\mathbb{Q}',
+  '𝔼': '\\mathbb{E}',
+  'ℙ': '\\mathbb{P}',
+  '𝟙': '\\mathbb{1}'
+};
+
+// Unified lookup map
+export const ALL_MATH_SYMBOLS = {
+  ...GREEK_LOWERCASE,
+  ...GREEK_UPPERCASE,
+  ...MATH_OPERATORS,
+  ...MATH_RELATIONS,
+  ...ARROWS_AND_LOGIC,
+  ...BLACKBOARD_SETS
+};
+
+/**
+ * Replaces unicode math symbols with their LaTeX equivalent.
+ * @param {string} text
+ * @returns {string}
+ */
+export function mapUnicodeToLatex(text) {
+  if (!text) return '';
+  let out = text;
+  for (const [sym, latex] of Object.entries(ALL_MATH_SYMBOLS)) {
+    if (out.includes(sym)) {
+      out = out.split(sym).join(` ${latex} `);
+    }
+  }
+  return out.replace(/\s+/g, ' ').trim();
+}