totem-llm 0.1.0

package/collector/utils/WhisperProviders/localWhisper.js

@@ -0,0 +1,197 @@
+const fs = require("fs");
+const path = require("path");
+const { v4 } = require("uuid");
+const defaultWhisper = "Xenova/whisper-small"; // Model Card: https://huggingface.co/Xenova/whisper-small
+const fileSize = {
+  "Xenova/whisper-small": "250mb",
+  "Xenova/whisper-large": "1.56GB",
+};
+
+class LocalWhisper {
+  constructor({ options }) {
+    this.model = options?.WhisperModelPref ?? defaultWhisper;
+    this.fileSize = fileSize[this.model];
+    this.cacheDir = path.resolve(
+      process.env.STORAGE_DIR
+        ? path.resolve(process.env.STORAGE_DIR, `models`)
+        : path.resolve(__dirname, `../../../server/storage/models`)
+    );
+
+    this.modelPath = path.resolve(this.cacheDir, ...this.model.split("/"));
+    // Make directory when it does not exist in existing installations
+    if (!fs.existsSync(this.cacheDir))
+      fs.mkdirSync(this.cacheDir, { recursive: true });
+
+    this.#log("Initialized.");
+  }
+
+  #log(text, ...args) {
+    console.log(`\x1b[32m[LocalWhisper]\x1b[0m ${text}`, ...args);
+  }
+
+  #validateAudioFile(wavFile) {
+    const sampleRate = wavFile.fmt.sampleRate;
+    const duration = wavFile.data.samples / sampleRate;
+
+    // Most speech recognition systems expect minimum 8kHz
+    // But we'll set it lower to be safe
+    if (sampleRate < 4000) {
+      // 4kHz minimum
+      throw new Error(
+        "Audio file sample rate is too low for accurate transcription. Minimum required is 4kHz."
+      );
+    }
+
+    // Typical audio file duration limits
+    const MAX_DURATION_SECONDS = 4 * 60 * 60; // 4 hours
+    if (duration > MAX_DURATION_SECONDS) {
+      throw new Error("Audio file duration exceeds maximum limit of 4 hours.");
+    }
+
+    // Check final sample count after upsampling to prevent memory issues
+    const targetSampleRate = 16000;
+    const upsampledSamples = duration * targetSampleRate;
+    const MAX_SAMPLES = 230_400_000; // ~4 hours at 16kHz
+
+    if (upsampledSamples > MAX_SAMPLES) {
+      throw new Error("Audio file exceeds maximum allowed length.");
+    }
+
+    return true;
+  }
+
+  async #convertToWavAudioData(sourcePath) {
+    try {
+      let buffer;
+      const wavefile = require("wavefile");
+      const { FFMPEGWrapper } = require("./ffmpeg");
+      const ffmpeg = new FFMPEGWrapper();
+      const outFolder = path.resolve(__dirname, `../../storage/tmp`);
+      if (!fs.existsSync(outFolder))
+        fs.mkdirSync(outFolder, { recursive: true });
+
+      const outputFile = path.resolve(outFolder, `${v4()}.wav`);
+      const success = await ffmpeg.convertAudioToWav(sourcePath, outputFile);
+      if (!success)
+        throw new Error(
+          "[Conversion Failed]: Could not convert file to .wav format!"
+        );
+
+      buffer = fs.readFileSync(outputFile);
+      fs.rmSync(outputFile);
+
+      const wavFile = new wavefile.WaveFile(buffer);
+      try {
+        this.#validateAudioFile(wavFile);
+      } catch (error) {
+        this.#log(`Audio validation failed: ${error.message}`);
+        throw new Error(`Invalid audio file: ${error.message}`);
+      }
+
+      // Although we use ffmpeg to convert to the correct format (16k hz 32f),
+      // different versions of ffmpeg produce different results based on the
+      // environment. To ensure consistency, we convert to the correct format again.
+      wavFile.toBitDepth("32f");
+      wavFile.toSampleRate(16000);
+
+      let audioData = wavFile.getSamples();
+      if (Array.isArray(audioData)) {
+        if (audioData.length > 1) {
+          const SCALING_FACTOR = Math.sqrt(2);
+
+          // Merge channels into first channel to save memory
+          for (let i = 0; i < audioData[0].length; ++i) {
+            audioData[0][i] =
+              (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2;
+          }
+        }
+        audioData = audioData[0];
+      }
+
+      return audioData;
+    } catch (error) {
+      console.error(`convertToWavAudioData`, error);
+      return null;
+    }
+  }
+
+  async client() {
+    if (!fs.existsSync(this.modelPath)) {
+      this.#log(
+        `The native whisper model has never been run and will be downloaded right now. Subsequent runs will be faster. (~${this.fileSize})`
+      );
+    }
+
+    try {
+      // Convert ESM to CommonJS via import so we can load this library.
+      const pipeline = (...args) =>
+        import("@xenova/transformers").then(({ pipeline }) => {
+          return pipeline(...args);
+        });
+      return await pipeline("automatic-speech-recognition", this.model, {
+        cache_dir: this.cacheDir,
+        ...(!fs.existsSync(this.modelPath)
+          ? {
+              // Show download progress if we need to download any files
+              progress_callback: (data) => {
+                if (!data.hasOwnProperty("progress")) return;
+                console.log(
+                  `\x1b[34m[ONNXWhisper - Downloading Model Files]\x1b[0m ${
+                    data.file
+                  } ${~~data?.progress}%`
+                );
+              },
+            }
+          : {}),
+      });
+    } catch (error) {
+      let errMsg = error.message;
+      if (errMsg.includes("Could not locate file")) {
+        errMsg =
+          "The native whisper model failed to download from the huggingface.co CDN. Your internet connection may be unstable or blocked by Huggingface.co - you will need to download the model manually and place it in the storage/models folder to use local Whisper transcription.";
+      }
+
+      this.#log(
+        `Failed to load the native whisper model: ${errMsg}`,
+        error.stack
+      );
+      throw new Error(errMsg);
+    }
+  }
+
+  async processFile(fullFilePath, filename) {
+    try {
+      const audioDataPromise = new Promise((resolve) =>
+        this.#convertToWavAudioData(fullFilePath).then((audioData) =>
+          resolve(audioData)
+        )
+      );
+      const [audioData, transcriber] = await Promise.all([
+        audioDataPromise,
+        this.client(),
+      ]);
+
+      if (!audioData) {
+        this.#log(`Failed to parse content from ${filename}.`);
+        return {
+          content: null,
+          error: `Failed to parse content from ${filename}.`,
+        };
+      }
+
+      this.#log(`Transcribing audio data to text...`);
+      const { text } = await transcriber(audioData, {
+        chunk_length_s: 30,
+        stride_length_s: 5,
+      });
+
+      return { content: text, error: null };
+    } catch (error) {
+      return { content: null, error: error.message };
+    }
+  }
+}
+
+module.exports = {
+  LocalWhisper,
+};

package/collector/utils/comKey/index.js

@@ -0,0 +1,54 @@
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const keyPath =
+  process.env.NODE_ENV === "development"
+    ? path.resolve(__dirname, `../../../server/storage/comkey`)
+    : path.resolve(
+        process.env.STORAGE_DIR ??
+          path.resolve(__dirname, `../../../server/storage`),
+        `comkey`
+      );
+
+class CommunicationKey {
+  #pubKeyName = "ipc-pub.pem";
+  #storageLoc = keyPath;
+
+  constructor() {}
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[CommunicationKeyVerify]\x1b[0m ${text}`, ...args);
+  }
+
+  #readPublicKey() {
+    return fs.readFileSync(path.resolve(this.#storageLoc, this.#pubKeyName));
+  }
+
+  // Given a signed payload from private key from /app/server/ this signature should
+  // decode to match the textData provided. This class does verification only in collector.
+  // Note: The textData is typically the JSON stringified body sent to the document processor API.
+  verify(signature = "", textData = "") {
+    try {
+      let data = textData;
+      if (typeof textData !== "string") data = JSON.stringify(data);
+      return crypto.verify(
+        "RSA-SHA256",
+        Buffer.from(data),
+        this.#readPublicKey(),
+        Buffer.from(signature, "hex")
+      );
+    } catch {}
+    return false;
+  }
+
+  // Use the rolling public-key to decrypt arbitrary data that was encrypted via the private key on the server side CommunicationKey class
+  // that we know was done with the same key-pair and the given input is in base64 format already.
+  // Returns plaintext string of the data that was encrypted.
+  decrypt(base64String = "") {
+    return crypto
+      .publicDecrypt(this.#readPublicKey(), Buffer.from(base64String, "base64"))
+      .toString();
+  }
+}
+
+module.exports = { CommunicationKey };

package/collector/utils/constants.js

@@ -0,0 +1,92 @@
+const WATCH_DIRECTORY = require("path").resolve(__dirname, "../hotdir");
+
+const ACCEPTED_MIMES = {
+  "text/plain": [".txt", ".md", ".org", ".adoc", ".rst"],
+  "text/html": [".html"],
+  "text/csv": [".csv"],
+  "application/json": [".json"],
+  // TODO: Create asDoc.js that works for standard MS Word files.
+  // "application/msword": [".doc"],
+
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
+    ".docx",
+  ],
+  "application/vnd.openxmlformats-officedocument.presentationml.presentation": [
+    ".pptx",
+  ],
+
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [
+    ".xlsx",
+  ],
+
+  "application/vnd.oasis.opendocument.text": [".odt"],
+  "application/vnd.oasis.opendocument.presentation": [".odp"],
+
+  "application/pdf": [".pdf"],
+  "application/mbox": [".mbox"],
+
+  "audio/wav": [".wav"],
+  "audio/mpeg": [".mp3"],
+  "audio/ogg": [".ogg", ".oga"],
+  "audio/opus": [".opus"],
+  "audio/mp4": [".m4a"],
+  "audio/x-m4a": [".m4a"],
+  "audio/webm": [".webm"],
+
+  "video/mp4": [".mp4"],
+  "video/mpeg": [".mpeg"],
+  "application/epub+zip": [".epub"],
+  "image/png": [".png"],
+  "image/jpeg": [".jpg"],
+  "image/jpg": [".jpg"],
+  "image/webp": [".webp"],
+};
+
+const SUPPORTED_FILETYPE_CONVERTERS = {
+  ".txt": "./convert/asTxt.js",
+  ".md": "./convert/asTxt.js",
+  ".org": "./convert/asTxt.js",
+  ".adoc": "./convert/asTxt.js",
+  ".rst": "./convert/asTxt.js",
+  ".csv": "./convert/asTxt.js",
+  ".json": "./convert/asTxt.js",
+
+  ".html": "./convert/asTxt.js",
+  ".pdf": "./convert/asPDF/index.js",
+
+  ".docx": "./convert/asDocx.js",
+  // TODO: Create asDoc.js that works for standard MS Word files.
+  // ".doc": "./convert/asDoc.js",
+
+  ".pptx": "./convert/asOfficeMime.js",
+
+  ".odt": "./convert/asOfficeMime.js",
+  ".odp": "./convert/asOfficeMime.js",
+
+  ".xlsx": "./convert/asXlsx.js",
+
+  ".mbox": "./convert/asMbox.js",
+
+  ".epub": "./convert/asEPub.js",
+
+  ".mp3": "./convert/asAudio.js",
+  ".wav": "./convert/asAudio.js",
+  ".mp4": "./convert/asAudio.js",
+  ".mpeg": "./convert/asAudio.js",
+  ".ogg": "./convert/asAudio.js",
+  ".oga": "./convert/asAudio.js",
+  ".opus": "./convert/asAudio.js",
+  ".m4a": "./convert/asAudio.js",
+  ".webm": "./convert/asAudio.js",
+
+  ".png": "./convert/asImage.js",
+  ".jpg": "./convert/asImage.js",
+  ".jpeg": "./convert/asImage.js",
+  ".webp": "./convert/asImage.js",
+};
+
+module.exports = {
+  SUPPORTED_FILETYPE_CONVERTERS,
+  WATCH_DIRECTORY,
+  ACCEPTED_MIMES,
+};

package/collector/utils/downloadURIToFile/index.js

@@ -0,0 +1,90 @@
+const { WATCH_DIRECTORY, ACCEPTED_MIMES } = require("../constants");
+const fs = require("fs");
+const path = require("path");
+const { pipeline } = require("stream/promises");
+const { validURL } = require("../url");
+const { default: slugify } = require("slugify");
+
+// Add a custom slugify extension for slashing to handle URLs with paths.
+slugify.extend({ "/": "-" });
+
+/**
+ * Maps a MIME type to the preferred file extension using ACCEPTED_MIMES.
+ * Returns null if the MIME type is not recognized or if there are no possible extensions.
+ * @param {string} mimeType - The MIME type to resolve (e.g., "application/pdf")
+ * @returns {string|null} - The file extension (e.g., ".pdf") or null
+ */
+function mimeToExtension(mimeType) {
+  if (!mimeType || !ACCEPTED_MIMES.hasOwnProperty(mimeType)) return null;
+  const possibleExtensions = ACCEPTED_MIMES[mimeType] ?? [];
+  if (possibleExtensions.length === 0) return null;
+  return possibleExtensions[0];
+}
+
+/**
+ * Download a file to the hotdir
+ * @param {string} url - The URL of the file to download
+ * @param {number} maxTimeout - The maximum timeout in milliseconds
+ * @returns {Promise<{success: boolean, fileLocation: string|null, reason: string|null}>} - The path to the downloaded file
+ */
+async function downloadURIToFile(url, maxTimeout = 10_000) {
+  if (!url || typeof url !== "string" || !validURL(url))
+    return { success: false, reason: "Not a valid URL.", fileLocation: null };
+
+  try {
+    const abortController = new AbortController();
+    const timeout = setTimeout(() => {
+      abortController.abort();
+      console.error(
+        `Timeout ${maxTimeout}ms reached while downloading file for URL:`,
+        url.toString()
+      );
+    }, maxTimeout);
+
+    const res = await fetch(url, { signal: abortController.signal })
+      .then((res) => {
+        if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
+        return res;
+      })
+      .finally(() => clearTimeout(timeout));
+
+    const urlObj = new URL(url);
+    const sluggedPath = slugify(urlObj.pathname, { lower: true });
+    let filename = `${urlObj.hostname}-${sluggedPath}`;
+
+    const existingExt = path.extname(filename).toLowerCase();
+    const { SUPPORTED_FILETYPE_CONVERTERS } = require("../constants");
+
+    // If the filename does not already have a supported file extension,
+    // try to infer one from the response Content-Type header.
+    // This handles URLs like https://arxiv.org/pdf/2307.10265 where the
+    // path has no explicit extension but the server responds with
+    // Content-Type: application/pdf.
+    if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(existingExt)) {
+      const { parseContentType } = require("../../processLink/helpers");
+      const contentType = parseContentType(res.headers.get("Content-Type"));
+      const inferredExt = mimeToExtension(contentType);
+      if (inferredExt) {
+        console.log(
+          `[Collector] URL path has no recognized extension. Inferred ${inferredExt} from Content-Type: ${contentType}`
+        );
+        filename += inferredExt;
+      }
+    }
+
+    const localFilePath = path.join(WATCH_DIRECTORY, filename);
+    const writeStream = fs.createWriteStream(localFilePath);
+    await pipeline(res.body, writeStream);
+
+    console.log(`[SUCCESS]: File ${localFilePath} downloaded to hotdir.`);
+    return { success: true, fileLocation: localFilePath, reason: null };
+  } catch (error) {
+    console.error(`Error writing to hotdir: ${error} for URL: ${url}`);
+    return { success: false, reason: error.message, fileLocation: null };
+  }
+}
+
+module.exports = {
+  downloadURIToFile,
+  mimeToExtension,
+};

package/collector/utils/extensions/Confluence/ConfluenceLoader/index.js

@@ -0,0 +1,154 @@
+/*
+ * This is a custom implementation of the Confluence langchain loader. There was an issue where
+ * code blocks were not being extracted. This is a temporary fix until this issue is resolved.*/
+
+const { htmlToText } = require("html-to-text");
+
+class ConfluencePagesLoader {
+  constructor({
+    baseUrl,
+    spaceKey,
+    username,
+    accessToken,
+    limit = 25,
+    expand = "body.storage,version",
+    personalAccessToken,
+    cloud = true,
+    bypassSSL = false,
+  }) {
+    this.baseUrl = baseUrl;
+    this.spaceKey = spaceKey;
+    this.username = username;
+    this.accessToken = accessToken;
+    this.limit = limit;
+    this.expand = expand;
+    this.personalAccessToken = personalAccessToken;
+    this.cloud = cloud;
+    this.bypassSSL = bypassSSL;
+    this.log("Initialized Confluence Loader");
+    if (this.bypassSSL)
+      this.log("!!SSL bypass is enabled!! Use at your own risk!!");
+  }
+
+  log(message, ...args) {
+    console.log(`\x1b[36m[Confluence Loader]\x1b[0m ${message}`, ...args);
+  }
+
+  get authorizationHeader() {
+    if (this.personalAccessToken) {
+      return `Bearer ${this.personalAccessToken}`;
+    } else if (this.username && this.accessToken) {
+      const authToken = Buffer.from(
+        `${this.username}:${this.accessToken}`
+      ).toString("base64");
+      return `Basic ${authToken}`;
+    }
+    return undefined;
+  }
+
+  async load(options) {
+    try {
+      const pages = await this.fetchAllPagesInSpace(
+        options?.start,
+        options?.limit
+      );
+      return pages.map((page) => this.createDocumentFromPage(page));
+    } catch (error) {
+      this.log("Error:", error);
+      return [];
+    }
+  }
+
+  async fetchConfluenceData(url) {
+    try {
+      const initialHeaders = {
+        "Content-Type": "application/json",
+        Accept: "application/json",
+      };
+      const authHeader = this.authorizationHeader;
+      if (authHeader) initialHeaders.Authorization = authHeader;
+
+      // If SSL bypass is enabled, set the NODE_TLS_REJECT_UNAUTHORIZED environment variable
+      if (this.bypassSSL) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+      const response = await fetch(url, { headers: initialHeaders });
+      if (!response.ok) {
+        throw new Error(
+          `Failed to fetch ${url} from Confluence: ${response.status}`
+        );
+      }
+      return await response.json();
+    } catch (error) {
+      this.log("Error:", error);
+      throw new Error(error.message);
+    } finally {
+      if (this.bypassSSL) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1";
+    }
+  }
+
+  // https://developer.atlassian.com/cloud/confluence/rest/v2/intro/#auth
+  async fetchAllPagesInSpace(start = 0, limit = this.limit) {
+    const url = `${this.baseUrl}${
+      this.cloud ? "/wiki" : ""
+    }/rest/api/content?spaceKey=${
+      this.spaceKey
+    }&limit=${limit}&start=${start}&expand=${this.expand}`;
+    const data = await this.fetchConfluenceData(url);
+    if (data.size === 0) {
+      return [];
+    }
+    const nextPageStart = start + data.size;
+    const nextPageResults = await this.fetchAllPagesInSpace(
+      nextPageStart,
+      limit
+    );
+    return data.results.concat(nextPageResults);
+  }
+
+  createDocumentFromPage(page) {
+    // Function to extract code blocks
+    const extractCodeBlocks = (content) => {
+      const codeBlockRegex =
+        /<ac:structured-macro ac:name="code"[^>]*>[\s\S]*?<ac:plain-text-body><!\[CDATA\[([\s\S]*?)\]\]><\/ac:plain-text-body>[\s\S]*?<\/ac:structured-macro>/g;
+      const languageRegex =
+        /<ac:parameter ac:name="language">(.*?)<\/ac:parameter>/;
+
+      return content.replace(codeBlockRegex, (match) => {
+        const language = match.match(languageRegex)?.[1] || "";
+        const code =
+          match.match(
+            /<ac:plain-text-body><!\[CDATA\[([\s\S]*?)\]\]><\/ac:plain-text-body>/
+          )?.[1] || "";
+        return `\n\`\`\`${language}\n${code.trim()}\n\`\`\`\n`;
+      });
+    };
+
+    const contentWithCodeBlocks = extractCodeBlocks(page.body.storage.value);
+    const plainTextContent = htmlToText(contentWithCodeBlocks, {
+      wordwrap: false,
+      preserveNewlines: true,
+    });
+    const textWithPreservedStructure = plainTextContent.replace(
+      /\n{3,}/g,
+      "\n\n"
+    );
+    const pageUrl = `${this.baseUrl}${this.cloud ? "/wiki" : ""}/spaces/${
+      this.spaceKey
+    }/pages/${page.id}`;
+
+    return {
+      pageContent: textWithPreservedStructure,
+      metadata: {
+        id: page.id,
+        status: page.status,
+        title: page.title,
+        type: page.type,
+        url: pageUrl,
+        version: page.version?.number,
+        updated_by: page.version?.by?.displayName,
+        updated_at: page.version?.when,
+      },
+    };
+  }
+}
+
+module.exports = { ConfluencePagesLoader };