scribe.js-ocr 0.7.4 → 0.9.0

package/build-deno-compile.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+## Note: 
+## The Windows build does not work and the Mac build has never been tested.
+## The Linux build should work.
+
+# Extract version from package.json using grep and sed
+VERSION=$(grep '"version"' package.json | sed -E 's/.*"version": *"([^"]+)".*/\1/')
+if [ -z "$VERSION" ]; then
+    echo "Failed to extract version from package.json"
+    exit 1
+fi
+
+# Create build directory
+mkdir -p build
+
+# Build for different platforms
+echo "Building for Linux x64..."
+deno compile --allow-sys --allow-read --allow-net --allow-write --target x86_64-unknown-linux-gnu --output build/scribe-linux-x64 --include mupdf --include fonts --include js/worker cli/scribe.js
+
+echo "Building for macOS x64..."
+deno compile --allow-sys --allow-read --allow-net --allow-write --target x86_64-apple-darwin --output build/scribe-macos-x64 --include mupdf --include fonts --include js/worker cli/scribe.js
+
+echo "Building for Windows x64..."
+deno compile --allow-sys --allow-read --allow-net --allow-write --target x86_64-pc-windows-msvc --output build/scribe-windows-x64.exe --include mupdf --include fonts --include js/worker cli/scribe.js
+
+# Create checksums
+cd build
+sha256sum * > checksums.txt
+cd ..

package/cli/cli.js

@@ -7,18 +7,36 @@ import {
   evalInternal, overlay, recognize,
 } from './main.js';
 
-export const confCLI = async (ocrFile) => {
-  await conf(ocrFile);
+/**
+ * Print confidence of Abbyy .xml file.
+ *
+ * @param {string[]} files - Paths to input files.
+ */
+export const confCLI = async (files) => {
+  await conf(files);
   process.exitCode = 0;
 };
 
-export const checkCLI = async (pdfFile, ocrFile, options) => {
-  await check(pdfFile, ocrFile, options);
+/**
+ *
+ * @param {string[]} files - Paths to input files.
+ * @param {Object} options
+ * @param {number} [options.workers]
+ */
+export const checkCLI = async (files, options) => {
+  await check(files, options);
   process.exitCode = 0;
 };
 
-export const evalInternalCLI = async (pdfFile, ocrFile, options) => {
-  const { evalMetrics } = await evalInternal(pdfFile, ocrFile, options);
+/**
+ * Evaluate internal OCR engine.
+ *
+ * @param {string[]} files - Paths to input files.
+ * @param {Object} options
+ * @param {number} [options.workers]
+ */
+export const evalInternalCLI = async (files, options) => {
+  const { evalMetrics } = await evalInternal(files, options);
 
   const ignoreExtra = true;
   let metricWER;
@@ -34,14 +52,14 @@ export const evalInternalCLI = async (pdfFile, ocrFile, options) => {
 
 /**
  *
- * @param {string} pdfFile - Path to PDF file.
+ * @param {string} inputFile - Path to PDF file.
  * @param {?string} [outputDir='.'] - Output directory.
  * @param {Object} [options]
  * @param {"pdf" | "hocr" | "docx" | "xlsx" | "txt" | "text" | "html"} [options.format]
  * @param {boolean} [options.reflow]
  */
-export const extractCLI = async (pdfFile, outputDir, options) => {
-  await extract(pdfFile, outputDir, options);
+export const extractCLI = async (inputFile, outputDir, options) => {
+  await extract(inputFile, outputDir, options);
   process.exitCode = 0;
 };
 
@@ -57,28 +75,38 @@ export const detectPDFTypeCLI = async (pdfFile, outputPath) => {
 
 /**
  *
- * @param {string} pdfFile - Path to PDF file.
- * @param {*} ocrFile
- * @param {*} outputDir
+ * @param {string[]} files - Paths to input files.
  * @param {Object} options
+ * @param {string} [options.output] - Output directory for the resulting PDF.
  * @param {boolean} [options.robust]
  * @param {boolean} [options.conf]
  * @param {boolean} [options.vis]
  * @param {number} [options.workers]
  */
-export const overlayCLI = async (pdfFile, ocrFile, outputDir, options) => {
+export const overlayCLI = async (files, options) => {
   options.overlayMode = options.vis ? 'proof' : 'invis';
-  await overlay(pdfFile, ocrFile, outputDir, options);
+  await overlay(files, options.output, options);
   process.exitCode = 0;
 };
 
-export const recognizeCLI = async (pdfFile, options) => {
+/**
+ *
+ * @param {string[]} files - Paths to input files.
+ * @param {*} options
+ */
+export const recognizeCLI = async (files, options) => {
   options.overlayMode = options.vis ? 'proof' : 'invis';
-  await recognize(pdfFile, options);
+  await recognize(files, options);
   process.exitCode = 0;
 };
 
-export const debugCLI = async (pdfFile, outputDir, options) => {
-  await debug(pdfFile, outputDir, options);
+/**
+ *
+ * @param {string[]} files - Paths to input files.
+ * @param {*} outputDir
+ * @param {*} options
+ */
+export const debugCLI = async (files, outputDir, options) => {
+  await debug(files, outputDir, options);
   process.exitCode = 0;
 };

package/cli/detectPDFType.js

@@ -1,4 +1,4 @@
-import fs from 'fs';
+import fs from 'node:fs';
 import scribe from '../scribe.js';
 
 /**
@@ -30,5 +30,4 @@ export const detectPDFType = async (pdfFile, outputPath) => {
   console.log('PDF Type:', type);
 
   mupdfScheduler.scheduler.terminate();
-
 };

package/cli/extract.js

@@ -1,28 +1,35 @@
-import fs from 'fs';
-import path from 'path';
+import fs from 'node:fs';
+import path from 'node:path';
 import scribe from '../scribe.js';
 
+// TODO: Consider whether this should exist and whether it should be combined into a larger CLI utility.
+// This was originally created to provide a simple interface to extract existing text from a PDF file,
+// however it now does other things, and this should likely be part of a larger `convert` utility.
+
 /**
  *
- * @param {string} pdfFile - Path to PDF file.
+ * @param {string} inputFile - Path to input file.
  * @param {?string} [output='.'] - Output file or directory.
  * @param {Object} [options]
  * @param {Parameters<typeof scribe.download>[0]} [options.format]
  * @param {boolean} [options.reflow]
  */
-export const extract = async (pdfFile, output, options) => {
+export const extract = async (inputFile, output, options) => {
   const format = options?.format || 'txt';
 
   output = output || '.';
   const outputDir = path.dirname(output);
-  const outputFile = outputDir === output ? `${path.basename(pdfFile).replace(/\.\w{1,5}$/i, `.${format}`)}` : path.basename(output);
+  const outputFile = outputDir === output ? `${path.basename(inputFile).replace(/\.\w{1,6}$/i, `.${format}`)}` : path.basename(output);
   const outputPath = `${outputDir}/${outputFile}`;
 
   scribe.opt.reflow = true;
   scribe.opt.extractText = true;
+  scribe.opt.displayMode = 'ebook';
 
-  await scribe.init();
-  await scribe.importFiles([pdfFile]);
+  // TODO: Fonts do not need to be loaded for .txt output, but are needed for .pdf output.
+  // so a more robust implementation would consider the arguments and only load fonts if necessary.
+  await scribe.init({ font: true });
+  await scribe.importFiles([inputFile]);
 
   if (outputDir) fs.mkdirSync(outputDir, { recursive: true });
 

package/cli/main.js

@@ -1,8 +1,8 @@
 // Code for adding visualization to OCR output
 // Use: `node addOverlay.js [PDF file] [OCR data file] [output directory]`
 
-import fs from 'fs';
-import path from 'path';
+import fs from 'node:fs';
+import path from 'node:path';
 
 import scribe from '../scribe.js';
 
@@ -13,8 +13,7 @@ scribe.opt.saveDebugImages = debugMode;
 /**
  * @param {string} func
  * @param {Object} params
- * @param {string} [params.pdfFile]
- * @param {string} [params.ocrFile]
+ * @param {string[]} [params.files]
  * @param {string} [params.outputDir]
  * @param {Array<string>} [params.list]
  * @param {boolean} [params.robustConfMode]
@@ -26,6 +25,10 @@ scribe.opt.saveDebugImages = debugMode;
 async function main(func, params) {
   scribe.opt.workerN = params.workerN || null;
 
+  if (!params.files || params.files.length === 0) {
+    throw new Error('No input files provided.');
+  }
+
   await scribe.init({
     pdf: true,
     ocr: true,
@@ -39,15 +42,9 @@ async function main(func, params) {
 
   const output = {};
 
-  const files = [];
-  if (params.pdfFile) files.push(params.pdfFile);
-  if (params.ocrFile) files.push(params.ocrFile);
-  await scribe.importFiles(files);
+  await scribe.importFiles(params.files);
 
-  const backgroundArg = params.pdfFile;
-  const backgroundStem = backgroundArg ? path.basename(backgroundArg).replace(/\.\w{1,5}$/i, '') : undefined;
-  const ocrStem = params.ocrFile ? path.basename(params.ocrFile).replace(/\.\w{1,5}$/i, '') : undefined;
-  const outputStem = backgroundStem || ocrStem || 'output';
+  const outputStem = scribe.inputData.defaultDownloadFileName.replace(/\.\w{1,6}$/i, '') || 'output';
 
   const outputDir = params.outputDir || '.';
 
@@ -75,23 +72,29 @@ async function main(func, params) {
     }
   }
 
-  if (['overlay', 'recognize'].includes(func) && backgroundArg) {
+  if (['overlay', 'recognize'].includes(func) && (scribe.inputData.pdfMode || scribe.inputData.imageMode)) {
     let outputSuffix = '';
     if (scribe.opt.displayMode === 'proof') {
       outputSuffix = '_vis';
     } else if (scribe.opt.displayMode === 'invis') {
-      const resolvedInputFile = path.dirname(path.resolve(backgroundArg));
-      const resolvedOutputDir = path.resolve(outputDir);
-      if (resolvedInputFile === resolvedOutputDir) {
-        outputSuffix = '_ocr';
+      // Check if output file would overwrite any input file, and if so, add a suffix to avoid overwriting.
+      // This software is still in development--nobody should be ovewriting input files.
+      const resolvedOutputFileTmp = path.resolve(`${outputDir}/${outputStem}.pdf`);
+      for (let i = 0; i < params.files.length; i++) {
+        const resolvedInputFile = path.resolve(params.files[i]);
+        if (resolvedInputFile === resolvedOutputFileTmp) {
+          outputSuffix = '_ocr';
+          console.log(`Saving output with ${outputSuffix} suffix to avoid overwriting input: ${resolvedInputFile}`);
+          break;
+        }
       }
     }
 
-    const outputPath = `${outputDir}/${path.basename(backgroundArg).replace(/\.\w{1,5}$/i, `${outputSuffix}.pdf`)}`;
+    const outputPath = path.resolve(`${outputDir}/${outputStem}${outputSuffix}.pdf`);
     await scribe.download('pdf', outputPath);
 
     if (params.hocr) {
-      const outputPathHocr = `${outputDir}/${path.basename(backgroundArg).replace(/\.\w{1,5}$/i, '.hocr')}`;
+      const outputPathHocr = path.resolve(`${outputDir}/${outputStem}.hocr`);
       await scribe.download('hocr', outputPathHocr);
     }
   }
@@ -100,7 +103,7 @@ async function main(func, params) {
     const debugDir = `${outputDir}/${outputStem}_debug`;
     fs.mkdirSync(debugDir, { recursive: true });
     const outputPathCsv = `${debugDir}/_debug.csv`;
-    scribe.utils.writeDebugCsv(scribe.data.ocr.active, outputPathCsv);
+    scribe.utils.writeDebugCsv({ pages: scribe.data.ocr.active, fileName: outputPathCsv });
 
     scribe.utils.dumpDebugImages(debugDir);
     scribe.utils.dumpHOCR(debugDir);
@@ -123,33 +126,30 @@ async function main(func, params) {
 /**
  * Print confidence of Abbyy .xml file.
  *
- * @param {string} ocrFile
+ * @param {string[]} files - Paths to input files.
  */
-export const conf = async (ocrFile) => (main('conf', { ocrFile }));
+export const conf = async (files) => (main('conf', { files }));
 
 /**
  *
- * @param {string} pdfFile - Path to PDF file.
- * @param {string} ocrFile
+ * @param {string[]} files - Paths to input files.
  * @param {Object} options
  * @param {number} [options.workers]
  */
-export const check = async (pdfFile, ocrFile, options) => (main('check', { pdfFile, ocrFile, workerN: options?.workers }));
+export const check = async (files, options) => (main('check', { files, workerN: options?.workers }));
 
 /**
  * Evaluate internal OCR engine.
  *
- * @param {string} pdfFile - Path to PDF file.
- * @param {string} ocrFile - Path to OCR file containing ground truth.
+ * @param {string[]} files - Paths to input files.
  * @param {Object} options
  * @param {number} [options.workers]
  */
-export const evalInternal = async (pdfFile, ocrFile, options) => (main('eval', { pdfFile, ocrFile, workerN: options?.workers }));
+export const evalInternal = async (files, options) => (main('eval', { files, workerN: options?.workers }));
 
 /**
  *
- * @param {string} pdfFile - Path to PDF file.
- * @param {*} ocrFile
+ * @param {string[]} files - Paths to input files.
  * @param {*} outputDir
  * @param {Object} options
  * @param {boolean} [options.robust]
@@ -157,29 +157,29 @@ export const evalInternal = async (pdfFile, ocrFile, options) => (main('eval', {
  * @param {"eval" | "ebook" | "proof" | "invis"} [options.overlayMode]
  * @param {number} [options.workers]
  */
-export const overlay = async (pdfFile, ocrFile, outputDir, options) => (main('overlay', {
-  pdfFile, ocrFile, outputDir, robustConfMode: options?.robust || false, printConf: options?.conf || false, overlayMode: options?.overlayMode || 'invis', workerN: options?.workers,
+export const overlay = async (files, outputDir, options) => (main('overlay', {
+  files, outputDir, robustConfMode: options?.robust || false, printConf: options?.conf || false, overlayMode: options?.overlayMode || 'invis', workerN: options?.workers,
 }));
 
 /**
  *
- * @param {string} pdfFile - Path to PDF file.
+ * @param {string[]} files - Paths to input files.
  * @param {Object} options
+ * @param {string} [options.output]
  * @param {"eval" | "ebook" | "proof" | "invis"} [options.overlayMode]
  * @param {boolean} [options.hocr]
  * @param {number} [options.workers]
  */
-export const recognize = async (pdfFile, options) => (main('recognize', {
-  pdfFile, overlayMode: options?.overlayMode || 'invis', workerN: options?.workers, hocr: options?.hocr,
+export const recognize = async (files, options) => (main('recognize', {
+  files, overlayMode: options?.overlayMode || 'invis', workerN: options?.workers, hocr: options?.hocr, outputDir: options?.output || '.',
 }));
 
 /**
  *
- * @param {string} pdfFile - Path to PDF file.
+ * @param {string[]} files - Paths to input files.
  * @param {*} outputDir
  * @param {*} options
- * @returns
  */
-export const debug = async (pdfFile, outputDir, options) => (main('debug', {
-  pdfFile, outputDir, list: options?.list,
+export const debug = async (files, outputDir, options) => (main('debug', {
+  files, outputDir, list: options?.list,
 }));

package/cli/require.js

@@ -2,6 +2,6 @@
 // While we will switch everything to worker modules eventually, Firefox still does not support them:
 // https://developer.mozilla.org/en-US/docs/Web/API/Worker#browser_compatibility
 // Therefore, we use a dynamic import statement to run the following code only in the Node.js version.
-const { createRequire } = await import('module');
+const { createRequire } = await import('node:module');
 globalThis.require = createRequire(import.meta.url);
 globalThis.__dirname = import.meta.url;

package/cli/scribe.js

@@ -1,3 +1,5 @@
+#!/usr/bin/env node
+
 import { Command, Option } from 'commander';
 
 import {
@@ -18,23 +20,21 @@ program
 
 program
   .command('check')
-  .argument('<pdf_file>', 'Input PDF file.')
-  .argument('<ocr_file>', 'Input OCR file.  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
   .option('-w, --workers <number>', 'Number of workers to use. Default is up to 8.')
+  .argument('<files...>', 'Input PDF file and OCR file(s).  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
   .description('Calculate confidence metric for OCR data by running Tesseract OCR and comparing results.')
   .action(checkCLI);
 
 program
   .command('eval')
-  .argument('<pdf_file>', 'Input PDF file.')
-  .argument('<ocr_file>', 'Input OCR file.  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
   .option('-w, --workers <number>', 'Number of workers to use. Default is up to 8.')
+  .argument('<files...>', 'Input PDF file and OCR file(s).  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
   .description('Evaluate internal OCR engine by recognizing document (provided PDF file), and comparing to ground truth (provided OCR file).')
   .action(evalInternalCLI);
 
 program
   .command('extract')
-  .argument('<pdf_file>', 'Input PDF file.')
+  .argument('<input_file>', 'Input PDF file.')
   .argument('[output]', 'Output directory or file to save results.', '.')
   .addOption(new Option('-f, --format <ext>', 'Output format.').choices(['pdf', 'hocr', 'docx', 'xlsx', 'txt', 'text', 'html']).default('txt'))
   .option('-r, --reflow', 'Reflow text by combining lines into paragraphs.')
@@ -43,23 +43,23 @@ program
 
 program
   .command('overlay')
-  .argument('<pdf_file>', 'Input PDF file.')
-  .argument('<ocr_file>', 'Input OCR file.  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
-  .argument('[output_dir]', 'Directory for output file(s).', '.')
+  .option('-o, --output <directory>', 'Directory for output file(s). Default is current directory.')
   .option('-v, --vis', 'Print OCR text visibly over provided PDF file with colors coded by confidence.')
   .option('-c, --conf', 'Print average confidence metric for document.')
   .option('-r, --robust', 'Generate confidence metrics by running Tesseract OCR and comparing, rather than using confidence info in provided data.')
   .option('-w, --workers <number>', 'Number of workers to use. Default is up to 8.')
+  .argument('<files...>', 'Input PDF file and OCR file(s).  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
   .description('Add OCR data to provided PDF file and save result as PDF.')
   .action(overlayCLI);
 
 program
   .command('recognize')
-  .argument('<pdf_file>', 'Input PDF file.')
-  .description('Recognize text in PDF file using internal OCR engine.')
+  .option('-o, --output <directory>', 'Directory for output file(s). Default is current directory.')
   .option('-v, --vis', 'Print OCR text visibly over provided PDF file with colors coded by confidence.')
   .option('-h, --hocr', 'Output .hocr intermediate data in addition to .pdf.')
   .option('-w, --workers <number>', 'Number of workers to use. Default is up to 8.')
+  .argument('<files...>', 'Input PDF file and OCR file(s).  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
+  .description('Recognize text in PDF file using internal OCR engine.')
   .action(recognizeCLI);
 
 program
@@ -71,9 +71,10 @@ program
 
 program
   .command('debug')
-  .argument('<pdf_file>', 'Input PDF file.')
+  .option('-o, --output <directory>', 'Directory for output file(s). Default is current directory.')
   .argument('[output_dir]', 'Directory for output file(s).', '.')
   .option('--list <items>', 'Comma separated list of visualizations to include.', (value) => value.split(','))
+  .argument('<files...>', 'Input PDF file and OCR file(s).  Accepts .hocr and Abbyy .xml (with character-level data enabled).')
   .description('Generate and write Tesseract debugging images.')
   .action(debugCLI);
 

package/js/canvasAdapter.js

@@ -28,6 +28,9 @@ export class ca {
     if (typeof process === 'undefined') {
       return new OffscreenCanvas(width, height);
     }
+    if (!width || !height || width <= 0 || height <= 0) {
+      throw new Error(`Invalid canvas size: ${width}x${height}`);
+    }
     const canvasKit = await ca.getCanvasKit();
     return canvasKit.MakeCanvas(width, height);
   };
@@ -80,7 +83,7 @@ export class ca {
     } else {
       const dummyCanvas = await ca.dummyCanvasPromise;
 
-      const fs = await import('fs');
+      const fs = await import('node:fs');
       const fontBuffer = typeof fontObj.src === 'string' ? fs.readFileSync(fontObj.src) : fontObj.src;
 
       dummyCanvas.loadFont(fontBuffer, {

package/js/clear.js

@@ -1,27 +1,26 @@
 import { inputData } from './containers/app.js';
 import {
   convertPageWarn,
-  fontMetricsObj,
   layoutDataTables,
   layoutRegions,
   ocrAll,
   ocrAllRaw,
-  pageMetricsArr,
+  pageMetricsAll,
 } from './containers/dataContainer.js';
 import { FontCont } from './containers/fontContainer.js';
 import { ImageCache } from './containers/imageContainer.js';
-import { replaceObjectProperties } from './utils/miscUtils.js';
+import { clearObjectProperties } from './utils/miscUtils.js';
 
 export function clearData() {
   inputData.clear();
-  replaceObjectProperties(ocrAll, { active: [] });
-  replaceObjectProperties(ocrAllRaw, { active: [] });
+  clearObjectProperties(ocrAll);
+  ocrAll.active = [];
+  clearObjectProperties(ocrAllRaw);
+  ocrAllRaw.active = [];
   layoutRegions.pages.length = 0;
   layoutDataTables.pages.length = 0;
-  pageMetricsArr.length = 0;
+  pageMetricsAll.length = 0;
   convertPageWarn.length = 0;
   ImageCache.clear();
-  // Clear optimized font data and reset fontAll to raw data.
-  replaceObjectProperties(fontMetricsObj);
   FontCont.clear();
 }

package/js/containers/app.js

@@ -31,6 +31,8 @@ export class opt {
 
   static removeMargins = false;
 
+  static includeImages = false;
+
   static pageBreaks = true;
 
   /** @type {("invis"|"ebook"|"eval"|"proof")} */

package/js/containers/dataContainer.js

@@ -1,9 +1,6 @@
 // This file contains various objects that are imported by other modules.
 // Everything here is essentially a global variable; none of them are technically "containers".
 
-/** @type {Object.<string, FontMetricsFamily>} */
-export const fontMetricsObj = {};
-
 export class layoutRegions {
   /** @type {Array<LayoutPage>} */
   static pages = [];
@@ -66,7 +63,7 @@ export const ocrAll = { active: [] };
 export const ocrAllRaw = { active: [] };
 
 /** @type {Array<PageMetrics>} */
-export const pageMetricsArr = [];
+export const pageMetricsAll = [];
 
 /**
  * Class that stores various debug data.