totem-llm 0.1.0

package/collector/processLink/helpers/index.js

@@ -0,0 +1,190 @@
+const path = require("path");
+const { validURL } = require("../../utils/url");
+const { processSingleFile } = require("../../processSingleFile");
+const { downloadURIToFile } = require("../../utils/downloadURIToFile");
+const { ACCEPTED_MIMES } = require("../../utils/constants");
+const { validYoutubeVideoUrl } = require("../../utils/url");
+
+/**
+ * Parse a Content-Type header value and return the MIME type without charset or other parameters.
+ * @param {string|null} contentTypeHeader - The raw Content-Type header value
+ * @returns {string|null} - The MIME type (e.g., "application/pdf") or null
+ */
+function parseContentType(contentTypeHeader) {
+  if (!contentTypeHeader) return null;
+  return contentTypeHeader.toLowerCase().split(";")[0].trim() || null;
+}
+
+/**
+ * Get the content type of a resource
+ * - Sends a HEAD request to the URL and returns the Content-Type header with a 5 second timeout
+ * @param {string} url - The URL to get the content type of
+ * @returns {Promise<{success: boolean, reason: string|null, contentType: string|null}>} - The content type of the resource
+ */
+async function getContentTypeFromURL(url) {
+  try {
+    if (!url || typeof url !== "string" || !validURL(url))
+      return { success: false, reason: "Not a valid URL.", contentType: null };
+
+    const abortController = new AbortController();
+    const timeout = setTimeout(() => {
+      abortController.abort();
+      console.error("Timeout fetching content type for URL:", url.toString());
+    }, 5_000);
+
+    const res = await fetch(url, {
+      method: "HEAD",
+      signal: abortController.signal,
+    }).finally(() => clearTimeout(timeout));
+
+    if (!res.ok)
+      return {
+        success: false,
+        reason: `HTTP ${res.status}: ${res.statusText}`,
+        contentType: null,
+      };
+
+    const contentTypeWithoutCharset = parseContentType(
+      res.headers.get("Content-Type")
+    );
+    if (!contentTypeWithoutCharset)
+      return {
+        success: false,
+        reason: "No Content-Type found.",
+        contentType: null,
+      };
+    return {
+      success: true,
+      reason: null,
+      contentType: contentTypeWithoutCharset,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      reason: `Error: ${error.message}`,
+      contentType: null,
+    };
+  }
+}
+
+/**
+ * Normalize the result object based on the saveAsDocument flag
+ * @param {Object} result - The result object to normalize
+ * @param {boolean} result.success - Whether the result is successful
+ * @param {string|null} result.reason - The reason for the result
+ * @param {Object[]} result.documents - The documents from the result
+ * @param {string|null} result.content - The content of the result
+ * @param {boolean} result.saveAsDocument - Whether to save the content as a document. Default is true
+ * @returns {{success: boolean, reason: string|null, documents: Object[], content: string|null}} - The normalized result object
+ */
+function returnResult({
+  success,
+  reason,
+  documents,
+  content,
+  saveAsDocument = true,
+} = {}) {
+  if (!saveAsDocument) {
+    return {
+      success,
+      content,
+    };
+  } else return { success, reason, documents };
+}
+
+/**
+ * Determine the content type of a link - should be a URL
+ * @param {string} uri - The link to determine the content type of
+ * @returns {Promise<{contentType: string|null, processVia: 'web' | 'file' | 'youtube'}>} - The content type of the link
+ */
+async function determineContentType(uri) {
+  let processVia = "web";
+
+  // Dont check for content type if it is a YouTube video URL
+  if (validYoutubeVideoUrl(uri))
+    return { contentType: "text/html", processVia: "youtube" };
+
+  return await getContentTypeFromURL(uri)
+    .then((result) => {
+      if (!!result.reason) console.error(result.reason);
+
+      // If the content type is not text/html or text/plain, and it is in the ACCEPTED_MIMES,
+      // then we can process it as a file
+      if (
+        !!result.contentType &&
+        !["text/html", "text/plain"].includes(result.contentType) &&
+        result.contentType in ACCEPTED_MIMES
+      )
+        processVia = "file";
+
+      return { contentType: result.contentType, processVia };
+    })
+    .catch((error) => {
+      console.error("Error getting content type from URL", error);
+      return { contentType: null, processVia };
+    });
+}
+
+/**
+ * Process a link as a file
+ * @param {string} uri - The link to process as a file
+ * @param {boolean} saveAsDocument - Whether to save the content as a document. Default is true
+ * @returns {Promise<{success: boolean, reason: string|null, documents: Object[], content: string|null, saveAsDocument: boolean}>} - The content of the file
+ */
+async function processAsFile({ uri, saveAsDocument = true }) {
+  const fileContentResult = await downloadURIToFile(uri);
+  if (!fileContentResult.success)
+    return returnResult({
+      success: false,
+      reason: fileContentResult.reason,
+      documents: [],
+      content: null,
+      saveAsDocument,
+    });
+
+  const fileFilePath = fileContentResult.fileLocation;
+  const targetFilename = path.basename(fileFilePath);
+
+  /**
+   * If the saveAsDocument is false, we are only interested in the text content
+   * and can ignore the file as a document by using `parseOnly` in the options.
+   * This will send the file to the Direct Uploads folder instead of the Documents folder.
+   * that will be deleted by the cleanup-orphan-documents job that runs frequently. The trade off
+   * is that since it still is in FS we can debug its output or even potentially reuse it for other purposes.
+   *
+   * TODO: Improve this process via a new option that will instantly delete the file after processing
+   * if we find we dont need this file ever after processing.
+   */
+  const processSingleFileResult = await processSingleFile(targetFilename, {
+    parseOnly: saveAsDocument === false,
+  });
+  if (!processSingleFileResult.success) {
+    return returnResult({
+      success: false,
+      reason: processSingleFileResult.reason,
+      documents: [],
+      content: null,
+      saveAsDocument,
+    });
+  }
+
+  // If we intend to return only the text content, return the content from the file
+  // and then delete the file - otherwise it will be saved as a document
+  if (!saveAsDocument) {
+    return returnResult({
+      success: true,
+      content: processSingleFileResult.documents[0].pageContent,
+      saveAsDocument,
+    });
+  }
+
+  return processSingleFileResult;
+}
+
+module.exports = {
+  parseContentType,
+  returnResult,
+  getContentTypeFromURL,
+  determineContentType,
+  processAsFile,
+};

package/collector/processLink/index.js

@@ -0,0 +1,47 @@
+const { validURL } = require("../utils/url");
+const { scrapeGenericUrl } = require("./convert/generic");
+const { validateURL } = require("../utils/url");
+
+/**
+ * Process a link and return the text content. This util will save the link as a document
+ * so it can be used for embedding later.
+ * @param {string} link - The link to process
+ * @param {{[key: string]: string}} scraperHeaders - Custom headers to apply when scraping the link
+ * @param {Object} metadata - Optional metadata to attach to the document
+ * @returns {Promise<{success: boolean, content: string}>} - Response from collector
+ */
+async function processLink(link, scraperHeaders = {}, metadata = {}) {
+  const validatedLink = validateURL(link);
+  if (!validURL(validatedLink))
+    return { success: false, reason: "Not a valid URL." };
+  return await scrapeGenericUrl({
+    link: validatedLink,
+    captureAs: "text",
+    scraperHeaders,
+    metadata,
+    saveAsDocument: true,
+  });
+}
+
+/**
+ * Get the text content of a link - does not save the link as a document
+ * Mostly used in agentic flows/tools calls to get the text content of a link
+ * @param {string} link - The link to get the text content of
+ * @param {('html' | 'text' | 'json')} captureAs - The format to capture the page content as
+ * @returns {Promise<{success: boolean, content: string}>} - Response from collector
+ */
+async function getLinkText(link, captureAs = "text") {
+  const validatedLink = validateURL(link);
+  if (!validURL(validatedLink))
+    return { success: false, reason: "Not a valid URL." };
+  return await scrapeGenericUrl({
+    link: validatedLink,
+    captureAs,
+    saveAsDocument: false,
+  });
+}
+
+module.exports = {
+  processLink,
+  getLinkText,
+};

package/collector/processRawText/index.js

@@ -0,0 +1,83 @@
+const { v4 } = require("uuid");
+const { writeToServerDocuments } = require("../utils/files");
+const { tokenizeString } = require("../utils/tokenizer");
+const { default: slugify } = require("slugify");
+
+// Will remove the last .extension from the input
+// and stringify the input + move to lowercase.
+function stripAndSlug(input) {
+  if (!input.includes(".")) return slugify(input, { lower: true });
+  return slugify(input.split(".").slice(0, -1).join("-"), { lower: true });
+}
+
+const METADATA_KEYS = {
+  possible: {
+    url: ({ url, title }) => {
+      let validUrl;
+      try {
+        const u = new URL(url);
+        validUrl = ["https:", "http:"].includes(u.protocol);
+      } catch {}
+
+      if (validUrl) return `web://${url.toLowerCase()}.website`;
+      return `file://${stripAndSlug(title)}.txt`;
+    },
+    title: ({ title }) => `${stripAndSlug(title)}.txt`,
+    docAuthor: ({ docAuthor }) => {
+      return typeof docAuthor === "string" ? docAuthor : "no author specified";
+    },
+    description: ({ description }) => {
+      return typeof description === "string"
+        ? description
+        : "no description found";
+    },
+    docSource: ({ docSource }) => {
+      return typeof docSource === "string" ? docSource : "no source set";
+    },
+    chunkSource: ({ chunkSource, title }) => {
+      return typeof chunkSource === "string"
+        ? chunkSource
+        : `${stripAndSlug(title)}.txt`;
+    },
+    published: ({ published }) => {
+      if (isNaN(Number(published))) return new Date().toLocaleString();
+      return new Date(Number(published)).toLocaleString();
+    },
+  },
+};
+
+async function processRawText(textContent, metadata) {
+  console.log(`-- Working Raw Text doc ${metadata.title} --`);
+  if (!textContent || textContent.length === 0) {
+    return {
+      success: false,
+      reason: "textContent was empty - nothing to process.",
+      documents: [],
+    };
+  }
+
+  const data = {
+    id: v4(),
+    url: METADATA_KEYS.possible.url(metadata),
+    title: METADATA_KEYS.possible.title(metadata),
+    docAuthor: METADATA_KEYS.possible.docAuthor(metadata),
+    description: METADATA_KEYS.possible.description(metadata),
+    docSource: METADATA_KEYS.possible.docSource(metadata),
+    chunkSource: METADATA_KEYS.possible.chunkSource(metadata),
+    published: METADATA_KEYS.possible.published(metadata),
+    wordCount: textContent.split(" ").length,
+    pageContent: textContent,
+    token_count_estimate: tokenizeString(textContent),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `raw-${stripAndSlug(metadata.title)}-${data.id}`,
+  });
+  console.log(
+    `[SUCCESS]: Raw text and metadata saved & ready for embedding.\n`
+  );
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = { processRawText };

package/collector/processSingleFile/convert/asAudio.js

@@ -0,0 +1,79 @@
+const { v4 } = require("uuid");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../utils/files");
+const { tokenizeString } = require("../../utils/tokenizer");
+const { default: slugify } = require("slugify");
+const { LocalWhisper } = require("../../utils/WhisperProviders/localWhisper");
+const { OpenAiWhisper } = require("../../utils/WhisperProviders/OpenAiWhisper");
+
+const WHISPER_PROVIDERS = {
+  openai: OpenAiWhisper,
+  local: LocalWhisper,
+};
+
+async function asAudio({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  const WhisperProvider = WHISPER_PROVIDERS.hasOwnProperty(
+    options?.whisperProvider
+  )
+    ? WHISPER_PROVIDERS[options?.whisperProvider]
+    : WHISPER_PROVIDERS.local;
+
+  console.log(`-- Working ${filename} --`);
+  const whisper = new WhisperProvider({ options });
+  const { content, error } = await whisper.processFile(fullFilePath, filename);
+
+  if (!!error) {
+    console.error(`Error encountered for parsing of ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: error,
+      documents: [],
+    };
+  }
+
+  if (!content?.length) {
+    console.error(`Resulting text content was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No text content found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  const data = {
+    id: v4(),
+    url: "file://" + fullFilePath,
+    title: metadata.title || filename,
+    docAuthor: metadata.docAuthor || "no author found",
+    description: metadata.description || "No description found.",
+    docSource: metadata.docSource || "audio file uploaded by the user.",
+    chunkSource: metadata.chunkSource || "",
+    published: createdDate(fullFilePath),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `${slugify(filename)}-${data.id}`,
+    options: { parseOnly: options.parseOnly },
+  });
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(
+    `[SUCCESS]: ${filename} transcribed, converted & ready for embedding.\n`
+  );
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = asAudio;

package/collector/processSingleFile/convert/asDocx.js

@@ -0,0 +1,63 @@
+const { v4 } = require("uuid");
+const { DocxLoader } = require("langchain/document_loaders/fs/docx");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../utils/files");
+const { tokenizeString } = require("../../utils/tokenizer");
+const { default: slugify } = require("slugify");
+
+async function asDocX({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  const loader = new DocxLoader(fullFilePath);
+
+  console.log(`-- Working ${filename} --`);
+  let pageContent = [];
+  const docs = await loader.load();
+  for (const doc of docs) {
+    console.log(`-- Parsing content from docx page --`);
+    if (!doc.pageContent.length) continue;
+    pageContent.push(doc.pageContent);
+  }
+
+  if (!pageContent.length) {
+    console.error(`Resulting text content was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No text content found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  const content = pageContent.join("");
+  const data = {
+    id: v4(),
+    url: "file://" + fullFilePath,
+    title: metadata.title || filename,
+    docAuthor: metadata.docAuthor || "no author found",
+    description: metadata.description || "No description found.",
+    docSource: metadata.docSource || "docx file uploaded by the user.",
+    chunkSource: metadata.chunkSource || "",
+    published: createdDate(fullFilePath),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `${slugify(filename)}-${data.id}`,
+    options: { parseOnly: options.parseOnly },
+  });
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = asDocX;

package/collector/processSingleFile/convert/asEPub.js

@@ -0,0 +1,61 @@
+const { v4 } = require("uuid");
+const { EPubLoader } = require("langchain/document_loaders/fs/epub");
+const { tokenizeString } = require("../../utils/tokenizer");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../utils/files");
+const { default: slugify } = require("slugify");
+
+async function asEPub({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  let content = "";
+  try {
+    const loader = new EPubLoader(fullFilePath, { splitChapters: false });
+    const docs = await loader.load();
+    docs.forEach((doc) => (content += doc.pageContent));
+  } catch (err) {
+    console.error("Could not read epub file!", err);
+  }
+
+  if (!content?.length) {
+    console.error(`Resulting text content was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No text content found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  console.log(`-- Working ${filename} --`);
+  const data = {
+    id: v4(),
+    url: "file://" + fullFilePath,
+    title: metadata.title || filename,
+    docAuthor: metadata.docAuthor || "Unknown",
+    description: metadata.description || "Unknown",
+    docSource: metadata.docSource || "epub file uploaded by the user.",
+    chunkSource: metadata.chunkSource || "",
+    published: createdDate(fullFilePath),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `${slugify(filename)}-${data.id}`,
+    options: { parseOnly: options.parseOnly },
+  });
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = asEPub;

package/collector/processSingleFile/convert/asImage.js

@@ -0,0 +1,56 @@
+const { v4 } = require("uuid");
+const { tokenizeString } = require("../../utils/tokenizer");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../utils/files");
+const OCRLoader = require("../../utils/OCRLoader");
+const { default: slugify } = require("slugify");
+
+async function asImage({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  let content = await new OCRLoader({
+    targetLanguages: options?.ocr?.langList,
+  }).ocrImage(fullFilePath);
+
+  if (!content?.length) {
+    console.error(`Resulting text content was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No text content found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  console.log(`-- Working ${filename} --`);
+  const data = {
+    id: v4(),
+    url: "file://" + fullFilePath,
+    title: metadata.title || filename,
+    docAuthor: metadata.docAuthor || "Unknown",
+    description: metadata.description || "Unknown",
+    docSource: metadata.docSource || "image file uploaded by the user.",
+    chunkSource: metadata.chunkSource || "",
+    published: createdDate(fullFilePath),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `${slugify(filename)}-${data.id}`,
+    options: { parseOnly: options.parseOnly },
+  });
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = asImage;

package/collector/processSingleFile/convert/asMbox.js

@@ -0,0 +1,83 @@
+const { v4 } = require("uuid");
+const fs = require("fs");
+const { mboxParser } = require("mbox-parser");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../utils/files");
+const { tokenizeString } = require("../../utils/tokenizer");
+const { default: slugify } = require("slugify");
+
+async function asMbox({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  console.log(`-- Working ${filename} --`);
+
+  const mails = await mboxParser(fs.createReadStream(fullFilePath))
+    .then((mails) => mails)
+    .catch((error) => {
+      console.log(`Could not parse mail items`, error);
+      return [];
+    });
+
+  if (!mails.length) {
+    console.error(`Resulting mail items was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No mail items found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  let item = 1;
+  const documents = [];
+  for (const mail of mails) {
+    if (!mail.hasOwnProperty("text")) continue;
+
+    const content = mail.text;
+    if (!content) continue;
+    console.log(
+      `-- Working on message "${mail.subject || "Unknown subject"}" --`
+    );
+
+    const data = {
+      id: v4(),
+      url: "file://" + fullFilePath,
+      title:
+        metadata.title ||
+        (mail?.subject
+          ? slugify(mail?.subject?.replace(".", "")) + ".mbox"
+          : `msg_${item}-${filename}`),
+      docAuthor: metadata.docAuthor || mail?.from?.text,
+      description: metadata.description || "No description found.",
+      docSource:
+        metadata.docSource || "Mbox message file uploaded by the user.",
+      chunkSource: metadata.chunkSource || "",
+      published: createdDate(fullFilePath),
+      wordCount: content.split(" ").length,
+      pageContent: content,
+      token_count_estimate: tokenizeString(content),
+    };
+
+    item++;
+    const document = writeToServerDocuments({
+      data,
+      filename: `${slugify(filename)}-${data.id}-msg-${item}`,
+      options: { parseOnly: options.parseOnly },
+    });
+    documents.push(document);
+  }
+
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(
+    `[SUCCESS]: ${filename} messages converted & ready for embedding.\n`
+  );
+  return { success: true, reason: null, documents };
+}
+
+module.exports = asMbox;