totem-llm 0.1.0

package/collector/utils/OCRLoader/index.js

@@ -0,0 +1,355 @@
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { VALID_LANGUAGE_CODES } = require("./validLangs");
+
+class OCRLoader {
+  /**
+   * The language code(s) to use for the OCR.
+   * @type {string[]}
+   */
+  language;
+  /**
+   * The cache directory for the OCR.
+   * @type {string}
+   */
+  cacheDir;
+
+  /**
+   * The constructor for the OCRLoader.
+   * @param {Object} options - The options for the OCRLoader.
+   * @param {string} options.targetLanguages - The target languages to use for the OCR as a comma separated string. eg: "eng,deu,..."
+   */
+  constructor({ targetLanguages = "eng" } = {}) {
+    this.language = this.parseLanguages(targetLanguages);
+    this.cacheDir = path.resolve(
+      process.env.STORAGE_DIR
+        ? path.resolve(process.env.STORAGE_DIR, `models`, `tesseract`)
+        : path.resolve(__dirname, `../../../server/storage/models/tesseract`)
+    );
+
+    // Ensure the cache directory exists or else Tesseract will persist the cache in the default location.
+    if (!fs.existsSync(this.cacheDir))
+      fs.mkdirSync(this.cacheDir, { recursive: true });
+    this.log(
+      `OCRLoader initialized with language support for:`,
+      this.language.map((lang) => VALID_LANGUAGE_CODES[lang]).join(", ")
+    );
+  }
+
+  /**
+   * Parses the language code from a provided comma separated string of language codes.
+   * @param {string} language - The language code to parse.
+   * @returns {string[]} The parsed language code.
+   */
+  parseLanguages(language = null) {
+    try {
+      if (!language || typeof language !== "string") return ["eng"];
+      const langList = language
+        .split(",")
+        .map((lang) => (lang.trim() !== "" ? lang.trim() : null))
+        .filter(Boolean)
+        .filter((lang) => VALID_LANGUAGE_CODES.hasOwnProperty(lang));
+      if (langList.length === 0) return ["eng"];
+      return langList;
+    } catch (e) {
+      this.log(`Error parsing languages: ${e.message}`, e.stack);
+      return ["eng"];
+    }
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[OCRLoader]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   * Loads a PDF file and returns an array of documents.
+   * This function is reserved to parsing for SCANNED documents - digital documents are not supported in this function
+   * @returns {Promise<{pageContent: string, metadata: object}[]>} An array of documents with page content and metadata.
+   */
+  async ocrPDF(
+    filePath,
+    { maxExecutionTime = 300_000, batchSize = 10, maxWorkers = null } = {}
+  ) {
+    if (
+      !filePath ||
+      !fs.existsSync(filePath) ||
+      !fs.statSync(filePath).isFile()
+    ) {
+      this.log(`File ${filePath} does not exist. Skipping OCR.`);
+      return [];
+    }
+
+    const documentTitle = path.basename(filePath);
+    this.log(`Starting OCR of ${documentTitle}`);
+    const pdfjs = await import("pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js");
+    let buffer = fs.readFileSync(filePath);
+
+    const pdfDocument = await pdfjs.getDocument({ data: buffer });
+
+    const documents = [];
+    const meta = await pdfDocument.getMetadata().catch(() => null);
+    const metadata = {
+      source: filePath,
+      pdf: {
+        version: "v2.0.550",
+        info: meta?.info,
+        metadata: meta?.metadata,
+        totalPages: pdfDocument.numPages,
+      },
+    };
+
+    const pdfSharp = new PDFSharp({
+      validOps: [
+        pdfjs.OPS.paintJpegXObject,
+        pdfjs.OPS.paintImageXObject,
+        pdfjs.OPS.paintInlineImageXObject,
+      ],
+    });
+    await pdfSharp.init();
+
+    const { createWorker, OEM } = require("tesseract.js");
+    const BATCH_SIZE = batchSize;
+    const MAX_EXECUTION_TIME = maxExecutionTime;
+    const NUM_WORKERS = maxWorkers ?? Math.min(os.cpus().length, 4);
+    const totalPages = pdfDocument.numPages;
+    const workerPool = await Promise.all(
+      Array(NUM_WORKERS)
+        .fill(0)
+        .map(() =>
+          createWorker(this.language, OEM.LSTM_ONLY, {
+            cachePath: this.cacheDir,
+          })
+        )
+    );
+
+    const startTime = Date.now();
+    try {
+      this.log("Bootstrapping OCR completed successfully!", {
+        MAX_EXECUTION_TIME_MS: MAX_EXECUTION_TIME,
+        BATCH_SIZE,
+        MAX_CONCURRENT_WORKERS: NUM_WORKERS,
+        TOTAL_PAGES: totalPages,
+      });
+      const timeoutPromise = new Promise((_, reject) => {
+        setTimeout(() => {
+          reject(
+            new Error(
+              `OCR job took too long to complete (${
+                MAX_EXECUTION_TIME / 1000
+              } seconds)`
+            )
+          );
+        }, MAX_EXECUTION_TIME);
+      });
+
+      const processPages = async () => {
+        for (
+          let startPage = 1;
+          startPage <= totalPages;
+          startPage += BATCH_SIZE
+        ) {
+          const endPage = Math.min(startPage + BATCH_SIZE - 1, totalPages);
+          const pageNumbers = Array.from(
+            { length: endPage - startPage + 1 },
+            (_, i) => startPage + i
+          );
+          this.log(`Working on pages ${startPage} - ${endPage}`);
+
+          const pageQueue = [...pageNumbers];
+          const results = [];
+          const workerPromises = workerPool.map(async (worker, workerIndex) => {
+            while (pageQueue.length > 0) {
+              const pageNum = pageQueue.shift();
+              this.log(
+                `\x1b[34m[Worker ${
+                  workerIndex + 1
+                }]\x1b[0m assigned pg${pageNum}`
+              );
+              const page = await pdfDocument.getPage(pageNum);
+              const imageBuffer = await pdfSharp.pageToBuffer({ page });
+              if (!imageBuffer) continue;
+              const { data } = await worker.recognize(imageBuffer, {}, "text");
+              this.log(
+                `✅ \x1b[34m[Worker ${
+                  workerIndex + 1
+                }]\x1b[0m completed pg${pageNum}`
+              );
+              results.push({
+                pageContent: data.text,
+                metadata: {
+                  ...metadata,
+                  loc: { pageNumber: pageNum },
+                },
+              });
+            }
+          });
+
+          await Promise.all(workerPromises);
+          documents.push(
+            ...results.sort(
+              (a, b) => a.metadata.loc.pageNumber - b.metadata.loc.pageNumber
+            )
+          );
+        }
+        return documents;
+      };
+
+      await Promise.race([timeoutPromise, processPages()]);
+    } catch (e) {
+      this.log(`Error: ${e.message}`, e.stack);
+    } finally {
+      global.Image = undefined;
+      await Promise.all(workerPool.map((worker) => worker.terminate()));
+    }
+
+    this.log(`Completed OCR of ${documentTitle}!`, {
+      documentsParsed: documents.length,
+      totalPages: totalPages,
+      executionTime: `${((Date.now() - startTime) / 1000).toFixed(2)}s`,
+    });
+    return documents;
+  }
+
+  /**
+   * Loads an image file and returns the OCRed text.
+   * @param {string} filePath - The path to the image file.
+   * @param {Object} options - The options for the OCR.
+   * @param {number} options.maxExecutionTime - The maximum execution time of the OCR in milliseconds.
+   * @returns {Promise<string>} The OCRed text.
+   */
+  async ocrImage(filePath, { maxExecutionTime = 300_000 } = {}) {
+    let content = "";
+    let worker = null;
+    if (
+      !filePath ||
+      !fs.existsSync(filePath) ||
+      !fs.statSync(filePath).isFile()
+    ) {
+      this.log(`File ${filePath} does not exist. Skipping OCR.`);
+      return null;
+    }
+
+    const documentTitle = path.basename(filePath);
+    try {
+      this.log(`Starting OCR of ${documentTitle}`);
+      const startTime = Date.now();
+      const { createWorker, OEM } = require("tesseract.js");
+      worker = await createWorker(this.language, OEM.LSTM_ONLY, {
+        cachePath: this.cacheDir,
+      });
+
+      // Race the timeout with the OCR
+      const timeoutPromise = new Promise((_, reject) => {
+        setTimeout(() => {
+          reject(
+            new Error(
+              `OCR job took too long to complete (${
+                maxExecutionTime / 1000
+              } seconds)`
+            )
+          );
+        }, maxExecutionTime);
+      });
+
+      const processImage = async () => {
+        const { data } = await worker.recognize(filePath, {}, "text");
+        content = data.text;
+      };
+
+      await Promise.race([timeoutPromise, processImage()]);
+      this.log(`Completed OCR of ${documentTitle}!`, {
+        executionTime: `${((Date.now() - startTime) / 1000).toFixed(2)}s`,
+      });
+
+      return content;
+    } catch (e) {
+      this.log(`Error: ${e.message}`);
+      return null;
+    } finally {
+      //eslint-disable-next-line
+      if (!worker) return;
+      await worker.terminate();
+    }
+  }
+}
+
+/**
+ * Converts a PDF page to a buffer using Sharp.
+ * @param {Object} options - The options for the Sharp PDF page object.
+ * @param {Object} options.page - The PDFJS page proxy object.
+ * @returns {Promise<Buffer>} The buffer of the page.
+ */
+class PDFSharp {
+  constructor({ validOps = [] } = {}) {
+    this.sharp = null;
+    this.validOps = validOps;
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[PDFSharp]\x1b[0m ${text}`, ...args);
+  }
+
+  async init() {
+    this.sharp = (await import("sharp")).default;
+  }
+
+  /**
+   * Converts a PDF page to a buffer.
+   * @param {Object} options - The options for the Sharp PDF page object.
+   * @param {Object} options.page - The PDFJS page proxy object.
+   * @returns {Promise<Buffer>} The buffer of the page.
+   */
+  async pageToBuffer({ page }) {
+    if (!this.sharp) await this.init();
+    try {
+      this.log(`Converting page ${page.pageNumber} to image...`);
+      const ops = await page.getOperatorList();
+      const pageImages = ops.fnArray.length;
+
+      for (let i = 0; i < pageImages; i++) {
+        try {
+          if (!this.validOps.includes(ops.fnArray[i])) continue;
+
+          const name = ops.argsArray[i][0];
+          const img = await page.objs.get(name);
+          const { width, height } = img;
+          const size = img.data.length;
+          const channels = size / width / height;
+          const targetDPI = 70;
+          const targetWidth = Math.floor(width * (targetDPI / 72));
+          const targetHeight = Math.floor(height * (targetDPI / 72));
+
+          const image = this.sharp(img.data, {
+            raw: { width, height, channels },
+            density: targetDPI,
+          })
+            .resize({
+              width: targetWidth,
+              height: targetHeight,
+              fit: "fill",
+            })
+            .withMetadata({
+              density: targetDPI,
+              resolution: targetDPI,
+            })
+            .png();
+
+          // For debugging purposes
+          // await image.toFile(path.resolve(__dirname, `../../storage/`, `pg${page.pageNumber}.png`));
+          return await image.toBuffer();
+        } catch (error) {
+          this.log(`Iteration error: ${error.message}`, error.stack);
+          continue;
+        }
+      }
+      this.log(`No valid images found on page ${page.pageNumber}`);
+      return null;
+    } catch (error) {
+      this.log(`Error: ${error.message}`, error.stack);
+      return null;
+    }
+  }
+}
+
+module.exports = OCRLoader;

package/collector/utils/OCRLoader/validLangs.js

@@ -0,0 +1,155 @@
+/*
+
+To get the list of valid language codes - do the following:
+Open the following URL in your browser: https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html
+
+Check this element is the proper table tbody with all the codes via console:
+document.getElementsByTagName('table').item(0).children.item(1)
+
+Now, copy the following code and paste it into the console:
+function parseLangs() {
+let langs = {};
+  Array.from(document.getElementsByTagName('table').item(0).children.item(1).children).forEach((el) => {
+    const [codeEl, languageEl, ...rest] = el.children
+    const code = codeEl.innerText.trim()
+    const language = languageEl.innerText.trim()
+    if (!!code && !!language) langs[code] = language
+  })
+  return langs;
+}
+
+now, run the function:
+copy(parseLangs())
+*/
+
+const VALID_LANGUAGE_CODES = {
+  afr: "Afrikaans",
+  amh: "Amharic",
+  ara: "Arabic",
+  asm: "Assamese",
+  aze: "Azerbaijani",
+  aze_cyrl: "Azerbaijani - Cyrilic",
+  bel: "Belarusian",
+  ben: "Bengali",
+  bod: "Tibetan",
+  bos: "Bosnian",
+  bre: "Breton",
+  bul: "Bulgarian",
+  cat: "Catalan; Valencian",
+  ceb: "Cebuano",
+  ces: "Czech",
+  chi_sim: "Chinese - Simplified",
+  chi_tra: "Chinese - Traditional",
+  chr: "Cherokee",
+  cos: "Corsican",
+  cym: "Welsh",
+  dan: "Danish",
+  dan_frak: "Danish - Fraktur (contrib)",
+  deu: "German",
+  deu_frak: "German - Fraktur (contrib)",
+  deu_latf: "German (Fraktur Latin)",
+  dzo: "Dzongkha",
+  ell: "Greek, Modern (1453-)",
+  eng: "English",
+  enm: "English, Middle (1100-1500)",
+  epo: "Esperanto",
+  equ: "Math / equation detection module",
+  est: "Estonian",
+  eus: "Basque",
+  fao: "Faroese",
+  fas: "Persian",
+  fil: "Filipino (old - Tagalog)",
+  fin: "Finnish",
+  fra: "French",
+  frk: "German - Fraktur (now deu_latf)",
+  frm: "French, Middle (ca.1400-1600)",
+  fry: "Western Frisian",
+  gla: "Scottish Gaelic",
+  gle: "Irish",
+  glg: "Galician",
+  grc: "Greek, Ancient (to 1453) (contrib)",
+  guj: "Gujarati",
+  hat: "Haitian; Haitian Creole",
+  heb: "Hebrew",
+  hin: "Hindi",
+  hrv: "Croatian",
+  hun: "Hungarian",
+  hye: "Armenian",
+  iku: "Inuktitut",
+  ind: "Indonesian",
+  isl: "Icelandic",
+  ita: "Italian",
+  ita_old: "Italian - Old",
+  jav: "Javanese",
+  jpn: "Japanese",
+  kan: "Kannada",
+  kat: "Georgian",
+  kat_old: "Georgian - Old",
+  kaz: "Kazakh",
+  khm: "Central Khmer",
+  kir: "Kirghiz; Kyrgyz",
+  kmr: "Kurmanji (Kurdish - Latin Script)",
+  kor: "Korean",
+  kor_vert: "Korean (vertical)",
+  kur: "Kurdish (Arabic Script)",
+  lao: "Lao",
+  lat: "Latin",
+  lav: "Latvian",
+  lit: "Lithuanian",
+  ltz: "Luxembourgish",
+  mal: "Malayalam",
+  mar: "Marathi",
+  mkd: "Macedonian",
+  mlt: "Maltese",
+  mon: "Mongolian",
+  mri: "Maori",
+  msa: "Malay",
+  mya: "Burmese",
+  nep: "Nepali",
+  nld: "Dutch; Flemish",
+  nor: "Norwegian",
+  oci: "Occitan (post 1500)",
+  ori: "Oriya",
+  osd: "Orientation and script detection module",
+  pan: "Panjabi; Punjabi",
+  pol: "Polish",
+  por: "Portuguese",
+  pus: "Pushto; Pashto",
+  que: "Quechua",
+  ron: "Romanian; Moldavian; Moldovan",
+  rus: "Russian",
+  san: "Sanskrit",
+  sin: "Sinhala; Sinhalese",
+  slk: "Slovak",
+  slk_frak: "Slovak - Fraktur (contrib)",
+  slv: "Slovenian",
+  snd: "Sindhi",
+  spa: "Spanish; Castilian",
+  spa_old: "Spanish; Castilian - Old",
+  sqi: "Albanian",
+  srp: "Serbian",
+  srp_latn: "Serbian - Latin",
+  sun: "Sundanese",
+  swa: "Swahili",
+  swe: "Swedish",
+  syr: "Syriac",
+  tam: "Tamil",
+  tat: "Tatar",
+  tel: "Telugu",
+  tgk: "Tajik",
+  tgl: "Tagalog (new - Filipino)",
+  tha: "Thai",
+  tir: "Tigrinya",
+  ton: "Tonga",
+  tur: "Turkish",
+  uig: "Uighur; Uyghur",
+  ukr: "Ukrainian",
+  urd: "Urdu",
+  uzb: "Uzbek",
+  uzb_cyrl: "Uzbek - Cyrilic",
+  vie: "Vietnamese",
+  yid: "Yiddish",
+  yor: "Yoruba",
+};
+
+module.exports.VALID_LANGUAGE_CODES = VALID_LANGUAGE_CODES;

package/collector/utils/WhisperProviders/OpenAiWhisper.js

@@ -0,0 +1,49 @@
+const fs = require("fs");
+
+class OpenAiWhisper {
+  constructor({ options }) {
+    const { OpenAI: OpenAIApi } = require("openai");
+    if (!options.openAiKey) throw new Error("No OpenAI API key was set.");
+
+    this.openai = new OpenAIApi({
+      apiKey: options.openAiKey,
+    });
+    this.model = "whisper-1";
+    this.temperature = 0;
+    this.#log("Initialized.");
+  }
+
+  #log(text, ...args) {
+    console.log(`\x1b[32m[OpenAiWhisper]\x1b[0m ${text}`, ...args);
+  }
+
+  async processFile(fullFilePath) {
+    return await this.openai.audio.transcriptions
+      .create({
+        file: fs.createReadStream(fullFilePath),
+        model: this.model,
+        temperature: this.temperature,
+      })
+      .then((response) => {
+        if (!response) {
+          return {
+            content: "",
+            error: "No content was able to be transcribed.",
+          };
+        }
+
+        return { content: response.text, error: null };
+      })
+      .catch((error) => {
+        this.#log(
+          `Could not get any response from openai whisper`,
+          error.message
+        );
+        return { content: "", error: error.message };
+      });
+  }
+}
+
+module.exports = {
+  OpenAiWhisper,
+};

package/collector/utils/WhisperProviders/ffmpeg/index.js

@@ -0,0 +1,114 @@
+const fs = require("fs");
+const path = require("path");
+const { execSync, spawnSync } = require("child_process");
+const { patchShellEnvironmentPath } = require("../../shell");
+/**
+ * Custom FFMPEG wrapper class for audio file conversion.
+ * Replaces deprecated fluent-ffmpeg package.
+ * Locates ffmpeg binary and converts audio files to required
+ * WAV format (16k hz mono 32f) for Whisper transcription.
+ *
+ * @class FFMPEGWrapper
+ */
+class FFMPEGWrapper {
+  static _instance;
+
+  constructor() {
+    if (FFMPEGWrapper._instance) return FFMPEGWrapper._instance;
+    FFMPEGWrapper._instance = this;
+    this._ffmpegPath = null;
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[35m[FFMPEG]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   * Locates ffmpeg binary.
+   * Uses fix-path on non-Windows platforms to ensure we can find ffmpeg.
+   *
+   * @returns {Promise<string>} Path to ffmpeg binary
+   * @throws {Error}
+   */
+  async ffmpegPath() {
+    if (this._ffmpegPath) return this._ffmpegPath;
+    await patchShellEnvironmentPath();
+
+    try {
+      const which = process.platform === "win32" ? "where" : "which";
+      const result = execSync(`${which} ffmpeg`, { encoding: "utf8" }).trim();
+      const candidatePath = result?.split("\n")?.[0]?.trim();
+      if (!candidatePath) throw new Error("FFMPEG candidate path not found.");
+      if (!this.isValidFFMPEG(candidatePath))
+        throw new Error("FFMPEG candidate path is not valid ffmpeg binary.");
+
+      this.log(`Found FFMPEG binary at ${candidatePath}`);
+      this._ffmpegPath = candidatePath;
+      return this._ffmpegPath;
+    } catch (error) {
+      this.log(error.message);
+    }
+
+    throw new Error("FFMPEG binary not found.");
+  }
+
+  /**
+   * Validates that path points to a valid ffmpeg binary.
+   * Runs ffmpeg -version command.
+   *
+   * @param {string} pathToTest - Path of ffmpeg binary
+   * @returns {boolean}
+   */
+  isValidFFMPEG(pathToTest) {
+    try {
+      if (!pathToTest || !fs.existsSync(pathToTest)) return false;
+      execSync(`"${pathToTest}" -version`, { encoding: "utf8", stdio: "pipe" });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Converts audio file to WAV format with required parameters for Whisper.
+   * Output: 16k hz, mono, 32bit float.
+   *
+   * @param {string} inputPath - Input path for audio file (any format supported by ffmpeg)
+   * @param {string} outputPath - Output path for converted file
+   * @returns {Promise<boolean>}
+   * @throws {Error} If ffmpeg binary cannot be found or conversion fails
+   */
+  async convertAudioToWav(inputPath, outputPath) {
+    if (!fs.existsSync(inputPath))
+      throw new Error(`Input file ${inputPath} does not exist.`);
+    const outputDir = path.dirname(outputPath);
+    if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true });
+
+    this.log(`Converting ${path.basename(inputPath)} to WAV format...`);
+    // Convert to 16k hz mono 32f
+    const result = spawnSync(
+      await this.ffmpegPath(),
+      [
+        "-i",
+        inputPath,
+        "-ar",
+        "16000",
+        "-ac",
+        "1",
+        "-acodec",
+        "pcm_f32le",
+        "-y",
+        outputPath,
+      ],
+      { encoding: "utf8" }
+    );
+
+    // ffmpeg writes progress to stderr
+    if (result.stderr) this.log(result.stderr.trim());
+    if (result.status !== 0) throw new Error(`FFMPEG conversion failed`);
+    this.log(`Conversion complete: ${path.basename(outputPath)}`);
+    return true;
+  }
+}
+
+module.exports = { FFMPEGWrapper };