totem-llm 0.1.0

package/collector/utils/runtimeSettings/index.js

@@ -0,0 +1,97 @@
+const { reqBody } = require("../http");
+
+/**
+ * Runtime settings are used to configure the collector per-request.
+ * These settings are persisted across requests, but can be overridden per-request.
+ *
+ * The settings are passed in the request body via `options.runtimeSettings`
+ * which is set in the backend #attachOptions function in CollectorApi.
+ *
+ * We do this so that the collector and backend can share the same ENV variables
+ * but only pass the relevant settings to the collector per-request and be able to
+ * access them across the collector via a single instance of RuntimeSettings.
+ *
+ * TODO: We may want to set all options passed from backend to collector here,
+ * but for now - we are only setting the runtime settings specifically for backwards
+ * compatibility with existing CollectorApi usage.
+ */
+class RuntimeSettings {
+  static _instance = null;
+  settings = {};
+
+  // Any settings here will be persisted across requests
+  // and must be explicitly defined here.
+  settingConfigs = {
+    seenAnyIpWarning: {
+      default: false,
+      validate: (value) => String(value) === "true",
+    },
+    allowAnyIp: {
+      default: false,
+      // Value must be explicitly "true" or "false" as a string
+      validate: (value) => String(value) === "true",
+    },
+    browserLaunchArgs: {
+      default: [],
+      validate: (value) => {
+        let args = [];
+        if (Array.isArray(value)) args = value.map((arg) => String(arg.trim()));
+        if (typeof value === "string")
+          args = value.split(",").map((arg) => arg.trim());
+        return args;
+      },
+    },
+  };
+
+  constructor() {
+    if (RuntimeSettings._instance) return RuntimeSettings._instance;
+    RuntimeSettings._instance = this;
+    return this;
+  }
+
+  /**
+   * Parse the runtime settings from the request body options body
+   * see #attachOptions https://github.com/Mintplex-Labs/anything-llm/blob/ebf112007e0d579af3d2b43569db95bdfc59074b/server/utils/collectorApi/index.js#L18
+   * @param {import('express').Request} request
+   * @returns {void}
+   */
+  parseOptionsFromRequest(request = {}) {
+    const options = reqBody(request)?.options?.runtimeSettings || {};
+    for (const [key, value] of Object.entries(options)) {
+      if (!this.settingConfigs.hasOwnProperty(key)) continue;
+      this.set(key, value);
+    }
+    return;
+  }
+
+  /**
+   * Get a runtime setting
+   * - Will throw an error if the setting requested is not a supported runtime setting key
+   * - Will return the default value if the setting requested is not set at all
+   * @param {string} key
+   * @returns {any}
+   */
+  get(key) {
+    if (!this.settingConfigs[key])
+      throw new Error(`Invalid runtime setting: ${key}`);
+    return this.settings.hasOwnProperty(key)
+      ? this.settings[key]
+      : this.settingConfigs[key].default;
+  }
+
+  /**
+   * Set a runtime setting
+   * - Will throw an error if the setting requested is not a supported runtime setting key
+   * - Will validate the value against the setting's validate function
+   * @param {string} key
+   * @param {any} value
+   * @returns {void}
+   */
+  set(key, value = null) {
+    if (!this.settingConfigs[key])
+      throw new Error(`Invalid runtime setting: ${key}`);
+    this.settings[key] = this.settingConfigs[key].validate(value);
+  }
+}
+
+module.exports = RuntimeSettings;

package/collector/utils/shell.js

@@ -0,0 +1,25 @@
+/**
+ * Patch the shell environment path to ensure the PATH is properly set for the current platform.
+ * On Docker, we are on Node v18 and cannot support fix-path v5.
+ * So we need to use the ESM-style import() to import the fix-path module + add the strip-ansi call to patch the PATH, which is the only change between v4 and v5.
+ * https://github.com/sindresorhus/fix-path/issues/6
+ * @returns {Promise<{[key: string]: string}>} - Environment variables from shell
+ */
+async function patchShellEnvironmentPath() {
+  try {
+    if (process.platform === "win32") return process.env;
+    const { default: fixPath } = await import("fix-path");
+    const { default: stripAnsi } = await import("strip-ansi");
+    fixPath();
+    if (process.env.PATH) process.env.PATH = stripAnsi(process.env.PATH);
+    console.log("Shell environment path patched successfully.");
+    return process.env;
+  } catch (error) {
+    console.error("Failed to patch shell environment path:", error);
+    return process.env;
+  }
+}
+
+module.exports = {
+  patchShellEnvironmentPath,
+};

package/collector/utils/tokenizer/index.js

@@ -0,0 +1,66 @@
+const { getEncoding } = require("js-tiktoken");
+
+class TikTokenTokenizer {
+  static MAX_KB_ESTIMATE = 10;
+  static DIVISOR = 8;
+
+  constructor() {
+    if (TikTokenTokenizer.instance) {
+      this.log(
+        "Singleton instance already exists. Returning existing instance."
+      );
+      return TikTokenTokenizer.instance;
+    }
+
+    this.encoder = getEncoding("cl100k_base");
+    TikTokenTokenizer.instance = this;
+    this.log("Initialized new TikTokenTokenizer instance.");
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[35m[TikTokenTokenizer]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   * Check if the input is too long to encode
+   * this is more of a rough estimate and a sanity check to prevent
+   * CPU issues from encoding too large of strings
+   * Assumes 1 character = 2 bytes in JS
+   * @param {string} input
+   * @returns {boolean}
+   */
+  #isTooLong(input) {
+    const bytesEstimate = input.length * 2;
+    const kbEstimate = Math.floor(bytesEstimate / 1024);
+    return kbEstimate >= TikTokenTokenizer.MAX_KB_ESTIMATE;
+  }
+
+  /**
+   * Encode a string into tokens for rough token count estimation.
+   * @param {string} input
+   * @returns {number}
+   */
+  tokenizeString(input = "") {
+    try {
+      if (this.#isTooLong(input)) {
+        this.log("Input will take too long to encode - estimating");
+        return Math.ceil(input.length / TikTokenTokenizer.DIVISOR);
+      }
+
+      return this.encoder.encode(input).length;
+    } catch (e) {
+      this.log("Could not tokenize string! Estimating...", e.message, e.stack);
+      return Math.ceil(input?.length / TikTokenTokenizer.DIVISOR) || 0;
+    }
+  }
+}
+
+const tokenizer = new TikTokenTokenizer();
+module.exports = {
+  /**
+   * Encode a string into tokens for rough token count estimation.
+   * @param {string} input
+   * @returns {number}
+   */
+  tokenizeString: (input) => tokenizer.tokenizeString(input),
+};

package/collector/utils/url/index.js

@@ -0,0 +1,137 @@
+const RuntimeSettings = require("../runtimeSettings");
+/**  ATTN: SECURITY RESEARCHERS
+ * To Security researchers about to submit an SSRF report CVE - please don't.
+ * We are aware that the code below is does not defend against any of the thousands of ways
+ * you can map a hostname to another IP via tunneling, hosts editing, etc. The code below does not have intention of blocking this
+ * and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects
+ * since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles.
+ * If an attacker has those roles then the system is already vulnerable and this is not a primary concern.
+ *
+ * We have gotten this report may times, marked them as duplicate or information and continue to get them. We communicate
+ * already that deployment (and security) of an instance is on the deployer and system admin deploying it. This would include
+ * isolation, firewalls, and the general security of the instance.
+ */
+
+const VALID_PROTOCOLS = ["https:", "http:"];
+const INVALID_OCTETS = [192, 172, 10, 127, 169];
+const runtimeSettings = new RuntimeSettings();
+
+/**
+ * If an ip address is passed in the user is attempting to collector some internal service running on internal/private IP.
+ * This is not a security feature and simply just prevents the user from accidentally entering invalid IP addresses.
+ * Can be bypassed via COLLECTOR_ALLOW_ANY_IP environment variable.
+ * @param {URL} param0
+ * @param {URL['hostname']} param0.hostname
+ * @returns {boolean}
+ */
+function isInvalidIp({ hostname }) {
+  if (runtimeSettings.get("allowAnyIp")) {
+    if (!runtimeSettings.get("seenAnyIpWarning")) {
+      console.log(
+        "\x1b[33mURL IP local address restrictions have been disabled by administrator!\x1b[0m"
+      );
+      runtimeSettings.set("seenAnyIpWarning", true);
+    }
+    return false;
+  }
+
+  const IPRegex = new RegExp(
+    /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/gi
+  );
+
+  // Not an IP address at all - passthrough
+  if (!IPRegex.test(hostname)) return false;
+  const [octetOne, ..._rest] = hostname.split(".");
+
+  // If fails to validate to number - abort and return as invalid.
+  if (isNaN(Number(octetOne))) return true;
+
+  // Allow localhost loopback and 0.0.0.0 for scraping convenience
+  // for locally hosted services or websites
+  if (["127.0.0.1", "0.0.0.0"].includes(hostname)) return false;
+
+  return INVALID_OCTETS.includes(Number(octetOne));
+}
+
+/**
+ * Validates a URL strictly
+ * - Checks the URL forms a valid URL
+ * - Checks the URL is at least HTTP(S)
+ * - Checks the URL is not an internal IP - can be bypassed via COLLECTOR_ALLOW_ANY_IP
+ * @param {string} url
+ * @returns {boolean}
+ */
+function validURL(url) {
+  try {
+    const destination = new URL(url);
+    if (!VALID_PROTOCOLS.includes(destination.protocol)) return false;
+    if (isInvalidIp(destination)) return false;
+    return true;
+  } catch {}
+  return false;
+}
+
+/**
+ * Modifies a URL to be valid:
+ * - Checks the URL is at least HTTP(S) so that protocol exists
+ * - Checks the URL forms a valid URL
+ * @param {string} url
+ * @returns {string}
+ */
+function validateURL(url) {
+  try {
+    let destination = url.trim();
+    // If the URL has a protocol, just pass through
+    // If the URL doesn't have a protocol, assume https://
+    if (destination.includes("://"))
+      destination = new URL(destination).toString();
+    else destination = new URL(`https://${destination}`).toString();
+
+    // If the URL ends with a slash, remove it
+    return destination.endsWith("/") ? destination.slice(0, -1) : destination;
+  } catch {
+    if (typeof url !== "string") return "";
+    return url.trim();
+  }
+}
+
+/**
+ * Validate if a link is a valid YouTube video URL
+ * - Checks youtu.be, youtube.com, m.youtube.com, music.youtube.com
+ * - Embed video URLs
+ * - Short URLs
+ * - Live URLs
+ * - Regular watch URLs
+ * - Optional query parameters (including ?v parameter)
+ *
+ * Can be used to extract the video ID from a YouTube video URL via the returnVideoId parameter.
+ * @param {string} link - The link to validate
+ * @param {boolean} returnVideoId - Whether to return the video ID if the link is a valid YouTube video URL
+ * @returns {boolean|string} - Whether the link is a valid YouTube video URL or the video ID if returnVideoId is true
+ */
+function validYoutubeVideoUrl(link, returnVideoId = false) {
+  try {
+    if (!link || typeof link !== "string") return false;
+    let urlToValidate = link;
+
+    if (!link.startsWith("http://") && !link.startsWith("https://")) {
+      urlToValidate = "https://" + link;
+      urlToValidate = new URL(urlToValidate).toString();
+    }
+
+    const regex =
+      /^(?:https?:\/\/)?(?:www\.|m\.|music\.)?(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?(?:.*&)?v=|(?:live\/)?|shorts\/))([\w-]{11})(?:\S+)?$/;
+    const match = urlToValidate.match(regex);
+    if (returnVideoId) return match?.[1] ?? null;
+    return !!match?.[1];
+  } catch (error) {
+    console.error("Error validating YouTube video URL", error);
+    return returnVideoId ? null : false;
+  }
+}
+
+module.exports = {
+  validURL,
+  validateURL,
+  validYoutubeVideoUrl,
+};