totem-llm 0.1.0

package/collector/index.js

@@ -0,0 +1,202 @@
+process.env.NODE_ENV === "development"
+  ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` })
+  : require("dotenv").config();
+
+require("./utils/logger")();
+const express = require("express");
+const bodyParser = require("body-parser");
+const cors = require("cors");
+const path = require("path");
+const { ACCEPTED_MIMES } = require("./utils/constants");
+const { reqBody, getCollectorPort } = require("./utils/http");
+const { processSingleFile } = require("./processSingleFile");
+const { processLink, getLinkText } = require("./processLink");
+const { wipeCollectorStorage } = require("./utils/files");
+const extensions = require("./extensions");
+const { processRawText } = require("./processRawText");
+const { verifyPayloadIntegrity } = require("./middleware/verifyIntegrity");
+const { httpLogger } = require("./middleware/httpLogger");
+const app = express();
+const FILE_LIMIT = "3GB";
+const COLLECTOR_PORT = getCollectorPort();
+
+// Only log HTTP requests in development mode and if the ENABLE_HTTP_LOGGER environment variable is set to true
+if (
+  process.env.NODE_ENV === "development" &&
+  !!process.env.ENABLE_HTTP_LOGGER
+) {
+  app.use(
+    httpLogger({
+      enableTimestamps: !!process.env.ENABLE_HTTP_LOGGER_TIMESTAMPS,
+    })
+  );
+}
+app.use(cors({ origin: true }));
+app.use(
+  bodyParser.text({ limit: FILE_LIMIT }),
+  bodyParser.json({ limit: FILE_LIMIT }),
+  bodyParser.urlencoded({
+    limit: FILE_LIMIT,
+    extended: true,
+  })
+);
+
+app.post(
+  "/process",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { filename, options = {}, metadata = {} } = reqBody(request);
+    try {
+      const targetFilename = path
+        .normalize(filename)
+        .replace(/^(\.\.(\/|\\|$))+/, "");
+      const {
+        success,
+        reason,
+        documents = [],
+      } = await processSingleFile(targetFilename, options, metadata);
+      response
+        .status(200)
+        .json({ filename: targetFilename, success, reason, documents });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        filename: filename,
+        success: false,
+        reason: "A processing error occurred.",
+        documents: [],
+      });
+    }
+    return;
+  }
+);
+
+app.post(
+  "/parse",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { filename, options = {} } = reqBody(request);
+    try {
+      const targetFilename = path
+        .normalize(filename)
+        .replace(/^(\.\.(\/|\\|$))+/, "");
+      const {
+        success,
+        reason,
+        documents = [],
+      } = await processSingleFile(targetFilename, {
+        ...options,
+        parseOnly: true,
+        absolutePath: options.absolutePath || null,
+      });
+      response
+        .status(200)
+        .json({ filename: targetFilename, success, reason, documents });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        filename: filename,
+        success: false,
+        reason: "A processing error occurred.",
+        documents: [],
+      });
+    }
+    return;
+  }
+);
+
+app.post(
+  "/process-link",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { link, scraperHeaders = {}, metadata = {} } = reqBody(request);
+    try {
+      const {
+        success,
+        reason,
+        documents = [],
+      } = await processLink(link, scraperHeaders, metadata);
+      response.status(200).json({ url: link, success, reason, documents });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        url: link,
+        success: false,
+        reason: "A processing error occurred.",
+        documents: [],
+      });
+    }
+    return;
+  }
+);
+
+app.post(
+  "/util/get-link",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { link, captureAs = "text" } = reqBody(request);
+    try {
+      const { success, content = null } = await getLinkText(link, captureAs);
+      response.status(200).json({ url: link, success, content });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        url: link,
+        success: false,
+        content: null,
+      });
+    }
+    return;
+  }
+);
+
+app.post(
+  "/process-raw-text",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { textContent, metadata } = reqBody(request);
+    try {
+      const {
+        success,
+        reason,
+        documents = [],
+      } = await processRawText(textContent, metadata);
+      response
+        .status(200)
+        .json({ filename: metadata.title, success, reason, documents });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        filename: metadata?.title || "Unknown-doc.txt",
+        success: false,
+        reason: "A processing error occurred.",
+        documents: [],
+      });
+    }
+    return;
+  }
+);
+
+extensions(app);
+
+app.get("/accepts", function (_, response) {
+  response.status(200).json(ACCEPTED_MIMES);
+});
+
+app.all("*", function (_, response) {
+  response.sendStatus(200);
+});
+
+app
+  .listen(COLLECTOR_PORT, async () => {
+    await wipeCollectorStorage();
+    console.log(`Document processor app listening on port ${COLLECTOR_PORT}`);
+  })
+  .on("error", function (_) {
+    process.once("SIGUSR2", function () {
+      process.kill(process.pid, "SIGUSR2");
+    });
+    process.on("SIGINT", function () {
+      process.kill(process.pid, "SIGINT");
+    });
+  });

package/collector/middleware/httpLogger.js

@@ -0,0 +1,29 @@
+const httpLogger =
+  ({ enableTimestamps = false }) =>
+  (req, res, next) => {
+    // Capture the original res.end to log response status
+    const originalEnd = res.end;
+
+    res.end = function (chunk, encoding) {
+      // Log the request method, status code, and path
+      const statusColor = res.statusCode >= 400 ? "\x1b[31m" : "\x1b[32m"; // Red for errors, green for success
+      console.log(
+        `\x1b[32m[HTTP]\x1b[0m ${statusColor}${res.statusCode}\x1b[0m ${
+          req.method
+        } -> ${req.path} ${
+          enableTimestamps
+            ? `@ ${new Date().toLocaleTimeString("en-US", { hour12: true })}`
+            : ""
+        }`.trim()
+      );
+
+      // Call the original end method
+      return originalEnd.call(this, chunk, encoding);
+    };
+
+    next();
+  };
+
+module.exports = {
+  httpLogger,
+};

package/collector/middleware/setDataSigner.js

@@ -0,0 +1,44 @@
+const { EncryptionWorker } = require("../utils/EncryptionWorker");
+const { CommunicationKey } = require("../utils/comKey");
+
+/**
+ * Express Response Object interface with defined encryptionWorker attached to locals property.
+ * @typedef {import("express").Response & import("express").Response['locals'] & {encryptionWorker: EncryptionWorker} } ResponseWithSigner
+ */
+
+// You can use this middleware to assign the EncryptionWorker to the response locals
+// property so that if can be used to encrypt/decrypt arbitrary data via response object.
+// eg: Encrypting API keys in chunk sources.
+
+// The way this functions is that the rolling RSA Communication Key is used server-side to private-key encrypt the raw
+// key of the persistent EncryptionManager credentials. Since EncryptionManager credentials do _not_ roll, we should not send them
+// even between server<>collector in plaintext because if the user configured the server/collector to be public they could technically
+// be exposing the key in transit via the X-Payload-Signer header. Even if this risk is minimal we should not do this.
+
+// This middleware uses the CommunicationKey public key to first decrypt the base64 representation of the EncryptionManager credentials
+// and then loads that in to the EncryptionWorker as a buffer so we can use the same credentials across the system. Should we ever break the
+// collector out into its own service this would still work without SSL/TLS.
+
+/**
+ *
+ * @param {import("express").Request} request
+ * @param {import("express").Response} response
+ * @param {import("express").NextFunction} next
+ */
+function setDataSigner(request, response, next) {
+  const comKey = new CommunicationKey();
+  const encryptedPayloadSigner = request.header("X-Payload-Signer");
+  if (!encryptedPayloadSigner)
+    console.log(
+      "Failed to find signed-payload to set encryption worker! Encryption calls will fail."
+    );
+
+  const decryptedPayloadSignerKey = comKey.decrypt(encryptedPayloadSigner);
+  const encryptionWorker = new EncryptionWorker(decryptedPayloadSignerKey);
+  response.locals.encryptionWorker = encryptionWorker;
+  next();
+}
+
+module.exports = {
+  setDataSigner,
+};

package/collector/middleware/verifyIntegrity.js

@@ -0,0 +1,32 @@
+const { CommunicationKey } = require("../utils/comKey");
+const RuntimeSettings = require("../utils/runtimeSettings");
+const runtimeSettings = new RuntimeSettings();
+
+function verifyPayloadIntegrity(request, response, next) {
+  const comKey = new CommunicationKey();
+  if (process.env.NODE_ENV === "development") {
+    comKey.log("verifyPayloadIntegrity is skipped in development.");
+    runtimeSettings.parseOptionsFromRequest(request);
+    next();
+    return;
+  }
+
+  const signature = request.header("X-Integrity");
+  if (!signature)
+    return response
+      .status(400)
+      .json({ msg: "Failed integrity signature check." });
+
+  const validSignedPayload = comKey.verify(signature, request.body);
+  if (!validSignedPayload)
+    return response
+      .status(400)
+      .json({ msg: "Failed integrity signature check." });
+
+  runtimeSettings.parseOptionsFromRequest(request);
+  next();
+}
+
+module.exports = {
+  verifyPayloadIntegrity,
+};

package/collector/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "anything-llm-document-collector",
+  "version": "1.13.0",
+  "description": "Document collector server endpoints",
+  "main": "index.js",
+  "author": "Timothy Carambat (Mintplex Labs)",
+  "license": "MIT",
+  "private": false,
+  "engines": {
+    "node": ">=18.12.1"
+  },
+  "scripts": {
+    "dev": "cross-env NODE_ENV=development nodemon --ignore hotdir --ignore storage --trace-warnings index.js",
+    "start": "cross-env NODE_ENV=production node index.js",
+    "lint": "eslint --fix .",
+    "lint:check": "eslint ."
+  },
+  "dependencies": {
+    "@langchain/community": "^0.2.23",
+    "@xenova/transformers": "^2.14.0",
+    "body-parser": "^1.20.3",
+    "cors": "^2.8.5",
+    "dotenv": "^16.0.3",
+    "epub2": "git+https://github.com/Mintplex-Labs/epub2-static.git#main",
+    "express": "^4.21.2",
+    "fix-path": "^4.0.0",
+    "html-to-text": "^9.0.5",
+    "ignore": "^5.3.0",
+    "js-tiktoken": "^1.0.8",
+    "langchain": "0.1.36",
+    "mammoth": "^1.6.0",
+    "mbox-parser": "^1.0.1",
+    "mime": "^3.0.0",
+    "moment": "^2.29.4",
+    "node-html-parser": "^6.1.13",
+    "node-xlsx": "^0.24.0",
+    "officeparser": "^4.0.5",
+    "openai": "4.95.1",
+    "pdf-parse": "^1.1.1",
+    "puppeteer": "~21.5.2",
+    "sharp": "^0.33.5",
+    "slugify": "^1.6.6",
+    "strip-ansi": "^7.1.2",
+    "tesseract.js": "^6.0.0",
+    "url-pattern": "^1.0.3",
+    "uuid": "^9.0.0",
+    "wavefile": "^11.0.0",
+    "winston": "^3.13.0",
+    "youtube-transcript-plus": "^1.1.2",
+    "youtubei.js": "^9.1.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.0.0",
+    "cross-env": "^7.0.3",
+    "eslint": "^9.0.0",
+    "eslint-config-prettier": "^9.0.0",
+    "eslint-plugin-prettier": "^5.0.0",
+    "eslint-plugin-unused-imports": "^4.0.0",
+    "globals": "^17.4.0",
+    "nodemon": "^2.0.22",
+    "prettier": "^2.4.1"
+  },
+  "resolutions": {
+    "string-width": "^4.2.3",
+    "wrap-ansi": "^7.0.0"
+  }
+}

package/collector/processLink/convert/generic.js

@@ -0,0 +1,240 @@
+const { v4 } = require("uuid");
+const {
+  PuppeteerWebBaseLoader,
+} = require("langchain/document_loaders/web/puppeteer");
+const { writeToServerDocuments } = require("../../utils/files");
+const { tokenizeString } = require("../../utils/tokenizer");
+const { default: slugify } = require("slugify");
+const {
+  returnResult,
+  determineContentType,
+  processAsFile,
+} = require("../helpers");
+const {
+  loadYouTubeTranscript,
+} = require("../../utils/extensions/YoutubeTranscript");
+const RuntimeSettings = require("../../utils/runtimeSettings");
+
+/**
+ * Scrape a generic URL and return the content in the specified format
+ * @param {Object} config - The configuration object
+ * @param {string} config.link - The URL to scrape
+ * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
+ * @param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request
+ * @param {{[key: string]: string}} config.metadata - Metadata to use when creating the document
+ * @param {boolean} config.saveAsDocument - Whether to save the content as a document. Default is true
+ * @returns {Promise<Object>} - The content of the page
+ */
+async function scrapeGenericUrl({
+  link,
+  captureAs = "text",
+  scraperHeaders = {},
+  metadata = {},
+  saveAsDocument = true,
+}) {
+  /** @type {'web' | 'file' | 'youtube'} */
+  console.log(`-- Working URL ${link} => (captureAs: ${captureAs}) --`);
+  let { contentType, processVia } = await determineContentType(link);
+  console.log(`-- URL determined to be ${contentType} (${processVia}) --`);
+
+  /**
+   * When the content is a file or a YouTube video, we can use the existing processing functions
+   * These are self-contained and will return the correct response based on the saveAsDocument flag already
+   * so we can return the content immediately.
+   */
+  if (processVia === "file")
+    return await processAsFile({ uri: link, saveAsDocument });
+  else if (processVia === "youtube")
+    return await loadYouTubeTranscript(
+      { url: link },
+      { parseOnly: saveAsDocument === false }
+    );
+
+  // Otherwise, assume the content is a webpage and scrape the content from the webpage
+  const content = await getPageContent({
+    link,
+    captureAs,
+    headers: scraperHeaders,
+  });
+  if (!content || !content.length) {
+    console.error(`Resulting URL content was empty at ${link}.`);
+    return returnResult({
+      success: false,
+      reason: `No URL content found at ${link}.`,
+      documents: [],
+      content: null,
+      saveAsDocument,
+    });
+  }
+
+  // If the captureAs is text, return the content as a string immediately
+  // so that we dont save the content as a document
+  if (!saveAsDocument)
+    return returnResult({
+      success: true,
+      content,
+      saveAsDocument,
+    });
+
+  // Save the content as a document from the URL
+  const url = new URL(link);
+  const decodedPathname = decodeURIComponent(url.pathname);
+  const filename = `${url.hostname}${decodedPathname.replace(/\//g, "_")}`;
+  const data = {
+    id: v4(),
+    url: "file://" + slugify(filename) + ".html",
+    title: metadata.title || slugify(filename) + ".html",
+    docAuthor: metadata.docAuthor || "no author found",
+    description: metadata.description || "No description found.",
+    docSource: metadata.docSource || "URL link uploaded by the user.",
+    chunkSource: `link://${link}`,
+    published: new Date().toLocaleString(),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `url-${slugify(filename)}-${data.id}`,
+  });
+  console.log(`[SUCCESS]: URL ${link} converted & ready for embedding.\n`);
+  return { success: true, reason: null, documents: [document] };
+}
+
+/**
+ * Validate the headers object
+ * - Keys & Values must be strings and not empty
+ * - Assemble a new object with only the valid keys and values
+ * @param {{[key: string]: string}} headers - The headers object to validate
+ * @returns {{[key: string]: string}} - The validated headers object
+ */
+function validatedHeaders(headers = {}) {
+  try {
+    if (Object.keys(headers).length === 0) return {};
+    let validHeaders = {};
+    for (const key of Object.keys(headers)) {
+      if (!key?.trim()) continue;
+      if (typeof headers[key] !== "string" || !headers[key]?.trim()) continue;
+      validHeaders[key] = headers[key].trim();
+    }
+    return validHeaders;
+  } catch (error) {
+    console.error("Error validating headers", error);
+    return {};
+  }
+}
+
+/**
+ * Get the content of a page
+ * @param {Object} config - The configuration object
+ * @param {string} config.link - The URL to get the content of
+ * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
+ * @param {{[key: string]: string}} config.headers - Custom headers to use when making the request
+ * @returns {Promise<string>} - The content of the page
+ */
+async function getPageContent({ link, captureAs = "text", headers = {} }) {
+  try {
+    let pageContents = [];
+    const runtimeSettings = new RuntimeSettings();
+
+    /** @type {import('puppeteer').PuppeteerLaunchOptions} */
+    let launchConfig = { headless: "new" };
+
+    /* On MacOS 15.1, the headless=new option causes the browser to crash immediately.
+     * It is not clear why this is the case, but it is reproducible. Since AnythinglLM
+     * in production runs in a container, we can disable headless mode to workaround the issue for development purposes.
+     *
+     * This may show a popup window when scraping a page in development mode.
+     * This is expected behavior if seen in development mode on MacOS 15+
+     */
+    if (
+      process.platform === "darwin" &&
+      process.env.NODE_ENV === "development"
+    ) {
+      console.log(
+        "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing."
+      );
+      launchConfig.headless = "false";
+    }
+
+    const loader = new PuppeteerWebBaseLoader(link, {
+      launchOptions: {
+        headless: launchConfig.headless,
+        ignoreHTTPSErrors: true,
+        args: runtimeSettings.get("browserLaunchArgs"),
+      },
+      gotoOptions: {
+        waitUntil: "networkidle2",
+      },
+      async evaluate(page, browser) {
+        const result = await page.evaluate((captureAs) => {
+          if (captureAs === "text") return document.body.innerText;
+          if (captureAs === "html") return document.documentElement.innerHTML;
+          return document.body.innerText;
+        }, captureAs);
+        await browser.close();
+        return result;
+      },
+    });
+
+    // Override scrape method if headers are available
+    let overrideHeaders = validatedHeaders(headers);
+    if (Object.keys(overrideHeaders).length > 0) {
+      loader.scrape = async function () {
+        const { launch } = await PuppeteerWebBaseLoader.imports();
+        const browser = await launch({
+          headless: "new",
+          defaultViewport: null,
+          ignoreDefaultArgs: ["--disable-extensions"],
+          ...this.options?.launchOptions,
+        });
+        const page = await browser.newPage();
+        await page.setExtraHTTPHeaders(overrideHeaders);
+
+        await page.goto(this.webPath, {
+          timeout: 180000,
+          waitUntil: "networkidle2",
+          ...this.options?.gotoOptions,
+        });
+
+        const bodyHTML = this.options?.evaluate
+          ? await this.options.evaluate(page, browser)
+          : await page.evaluate(() => document.body.innerHTML);
+
+        await browser.close();
+        return bodyHTML;
+      };
+    }
+
+    const docs = await loader.load();
+    for (const doc of docs) pageContents.push(doc.pageContent);
+    return pageContents.join(" ");
+  } catch (error) {
+    console.error(
+      "getPageContent failed to be fetched by puppeteer - falling back to fetch!",
+      error
+    );
+  }
+
+  try {
+    const pageText = await fetch(link, {
+      method: "GET",
+      headers: {
+        "Content-Type": "text/plain",
+        "User-Agent":
+          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)",
+        ...validatedHeaders(headers),
+      },
+    }).then((res) => res.text());
+    return pageText;
+  } catch (error) {
+    console.error("getPageContent failed to be fetched by any method.", error);
+  }
+
+  return null;
+}
+
+module.exports = {
+  scrapeGenericUrl,
+};