totem-llm 0.1.0

package/server/utils/files/logo.js

@@ -0,0 +1,117 @@
+const path = require("path");
+const fs = require("fs");
+const { getType } = require("mime");
+const { v4 } = require("uuid");
+const { SystemSettings } = require("../../models/systemSettings");
+const { normalizePath, isWithin } = require(".");
+const LOGO_FILENAME = "anything-llm.png";
+const LOGO_FILENAME_DARK = "anything-llm-dark.png";
+
+/**
+ * Checks if the filename is the default logo filename for dark or light mode.
+ * @param {string} filename - The filename to check.
+ * @returns {boolean} Whether the filename is the default logo filename.
+ */
+function isDefaultFilename(filename) {
+  return [LOGO_FILENAME, LOGO_FILENAME_DARK].includes(filename);
+}
+
+function validFilename(newFilename = "") {
+  return !isDefaultFilename(newFilename);
+}
+
+/**
+ * Shows the logo for the current theme. In dark mode, it shows the light logo
+ * and vice versa.
+ * @param {boolean} darkMode - Whether the logo should be for dark mode.
+ * @returns {string} The filename of the logo.
+ */
+function getDefaultFilename(darkMode = true) {
+  return darkMode ? LOGO_FILENAME : LOGO_FILENAME_DARK;
+}
+
+async function determineLogoFilepath(defaultFilename = LOGO_FILENAME) {
+  const currentLogoFilename = await SystemSettings.currentLogoFilename();
+  const basePath = process.env.STORAGE_DIR
+    ? path.join(process.env.STORAGE_DIR, "assets")
+    : path.join(__dirname, "../../storage/assets");
+  const defaultFilepath = path.join(basePath, defaultFilename);
+
+  if (currentLogoFilename && validFilename(currentLogoFilename)) {
+    const customLogoPath = path.join(
+      basePath,
+      normalizePath(currentLogoFilename)
+    );
+    if (!isWithin(path.resolve(basePath), path.resolve(customLogoPath)))
+      return defaultFilepath;
+    return fs.existsSync(customLogoPath) ? customLogoPath : defaultFilepath;
+  }
+
+  return defaultFilepath;
+}
+
+function fetchLogo(logoPath) {
+  if (!fs.existsSync(logoPath)) {
+    return {
+      found: false,
+      buffer: null,
+      size: 0,
+      mime: "none/none",
+    };
+  }
+
+  const mime = getType(logoPath);
+  const buffer = fs.readFileSync(logoPath);
+  return {
+    found: true,
+    buffer,
+    size: buffer.length,
+    mime,
+  };
+}
+
+async function renameLogoFile(originalFilename = null) {
+  const extname = path.extname(originalFilename) || ".png";
+  const newFilename = `${v4()}${extname}`;
+  const assetsDirectory = process.env.STORAGE_DIR
+    ? path.join(process.env.STORAGE_DIR, "assets")
+    : path.join(__dirname, `../../storage/assets`);
+  const originalFilepath = path.join(
+    assetsDirectory,
+    normalizePath(originalFilename)
+  );
+  if (!isWithin(path.resolve(assetsDirectory), path.resolve(originalFilepath)))
+    throw new Error("Invalid file path.");
+
+  // The output always uses a random filename.
+  const outputFilepath = process.env.STORAGE_DIR
+    ? path.join(process.env.STORAGE_DIR, "assets", normalizePath(newFilename))
+    : path.join(__dirname, `../../storage/assets`, normalizePath(newFilename));
+
+  fs.renameSync(originalFilepath, outputFilepath);
+  return newFilename;
+}
+
+async function removeCustomLogo(logoFilename = LOGO_FILENAME) {
+  if (!logoFilename || !validFilename(logoFilename)) return false;
+  const assetsDirectory = process.env.STORAGE_DIR
+    ? path.join(process.env.STORAGE_DIR, "assets")
+    : path.join(__dirname, `../../storage/assets`);
+
+  const logoPath = path.join(assetsDirectory, normalizePath(logoFilename));
+  if (!isWithin(path.resolve(assetsDirectory), path.resolve(logoPath)))
+    throw new Error("Invalid file path.");
+  if (fs.existsSync(logoPath)) fs.unlinkSync(logoPath);
+  return true;
+}
+
+module.exports = {
+  fetchLogo,
+  renameLogoFile,
+  removeCustomLogo,
+  validFilename,
+  getDefaultFilename,
+  determineLogoFilepath,
+  isDefaultFilename,
+  LOGO_FILENAME,
+};

package/server/utils/files/multer.js

@@ -0,0 +1,178 @@
+const multer = require("multer");
+const path = require("path");
+const fs = require("fs");
+const { v4 } = require("uuid");
+const { normalizePath, sanitizeFileName } = require(".");
+
+/**
+ * Handle File uploads for auto-uploading.
+ * Mostly used for internal GUI/API uploads.
+ */
+const fileUploadStorage = multer.diskStorage({
+  destination: function (_, __, cb) {
+    const uploadOutput =
+      process.env.NODE_ENV === "development"
+        ? path.resolve(__dirname, `../../../collector/hotdir`)
+        : path.resolve(process.env.STORAGE_DIR, `../../collector/hotdir`);
+    cb(null, uploadOutput);
+  },
+  filename: function (_, file, cb) {
+    file.originalname = sanitizeFileName(
+      normalizePath(Buffer.from(file.originalname, "latin1").toString("utf8"))
+    );
+    cb(null, file.originalname);
+  },
+});
+
+/**
+ * Handle API file upload as documents - this does not manipulate the filename
+ * at all for encoding/charset reasons.
+ */
+const fileAPIUploadStorage = multer.diskStorage({
+  destination: function (_, __, cb) {
+    const uploadOutput =
+      process.env.NODE_ENV === "development"
+        ? path.resolve(__dirname, `../../../collector/hotdir`)
+        : path.resolve(process.env.STORAGE_DIR, `../../collector/hotdir`);
+    cb(null, uploadOutput);
+  },
+  filename: function (_, file, cb) {
+    file.originalname = sanitizeFileName(
+      normalizePath(Buffer.from(file.originalname, "latin1").toString("utf8"))
+    );
+    cb(null, file.originalname);
+  },
+});
+
+// Asset storage for logos
+const assetUploadStorage = multer.diskStorage({
+  destination: function (_, __, cb) {
+    const uploadOutput =
+      process.env.NODE_ENV === "development"
+        ? path.resolve(__dirname, `../../storage/assets`)
+        : path.resolve(process.env.STORAGE_DIR, "assets");
+    fs.mkdirSync(uploadOutput, { recursive: true });
+    return cb(null, uploadOutput);
+  },
+  filename: function (_, file, cb) {
+    file.originalname = sanitizeFileName(
+      normalizePath(Buffer.from(file.originalname, "latin1").toString("utf8"))
+    );
+    cb(null, file.originalname);
+  },
+});
+
+/**
+ * Handle PFP file upload as logos
+ */
+const pfpUploadStorage = multer.diskStorage({
+  destination: function (_, __, cb) {
+    const uploadOutput =
+      process.env.NODE_ENV === "development"
+        ? path.resolve(__dirname, `../../storage/assets/pfp`)
+        : path.resolve(process.env.STORAGE_DIR, "assets/pfp");
+    fs.mkdirSync(uploadOutput, { recursive: true });
+    return cb(null, uploadOutput);
+  },
+  filename: function (req, file, cb) {
+    const randomFileName = `${v4()}${path.extname(
+      normalizePath(file.originalname)
+    )}`;
+    req.randomFileName = randomFileName;
+    cb(null, randomFileName);
+  },
+});
+
+/**
+ * Handle Generic file upload as documents from the GUI
+ * @param {Request} request
+ * @param {Response} response
+ * @param {NextFunction} next
+ */
+function handleFileUpload(request, response, next) {
+  const upload = multer({ storage: fileUploadStorage }).single("file");
+  upload(request, response, function (err) {
+    if (err) {
+      response
+        .status(500)
+        .json({
+          success: false,
+          error: `Invalid file upload. ${err.message}`,
+        })
+        .end();
+      return;
+    }
+    next();
+  });
+}
+
+/**
+ * Handle API file upload as documents - this does not manipulate the filename
+ * at all for encoding/charset reasons.
+ * @param {Request} request
+ * @param {Response} response
+ * @param {NextFunction} next
+ */
+function handleAPIFileUpload(request, response, next) {
+  const upload = multer({ storage: fileAPIUploadStorage }).single("file");
+  upload(request, response, function (err) {
+    if (err) {
+      response
+        .status(500)
+        .json({
+          success: false,
+          error: `Invalid file upload. ${err.message}`,
+        })
+        .end();
+      return;
+    }
+    next();
+  });
+}
+
+/**
+ * Handle logo asset uploads
+ */
+function handleAssetUpload(request, response, next) {
+  const upload = multer({ storage: assetUploadStorage }).single("logo");
+  upload(request, response, function (err) {
+    if (err) {
+      response
+        .status(500)
+        .json({
+          success: false,
+          error: `Invalid file upload. ${err.message}`,
+        })
+        .end();
+      return;
+    }
+    next();
+  });
+}
+
+/**
+ * Handle PFP file upload as logos
+ */
+function handlePfpUpload(request, response, next) {
+  const upload = multer({ storage: pfpUploadStorage }).single("file");
+  upload(request, response, function (err) {
+    if (err) {
+      response
+        .status(500)
+        .json({
+          success: false,
+          error: `Invalid file upload. ${err.message}`,
+        })
+        .end();
+      return;
+    }
+    next();
+  });
+}
+
+module.exports = {
+  handleFileUpload,
+  handleAPIFileUpload,
+  handleAssetUpload,
+  handlePfpUpload,
+};

package/server/utils/files/pfp.js

@@ -0,0 +1,63 @@
+const path = require("path");
+const fs = require("fs");
+const { getType } = require("mime");
+const { User } = require("../../models/user");
+const { normalizePath, isWithin } = require(".");
+const { Workspace } = require("../../models/workspace");
+
+function fetchPfp(pfpPath) {
+  if (!fs.existsSync(pfpPath)) {
+    return {
+      found: false,
+      buffer: null,
+      size: 0,
+      mime: "none/none",
+    };
+  }
+
+  const mime = getType(pfpPath);
+  const buffer = fs.readFileSync(pfpPath);
+  return {
+    found: true,
+    buffer,
+    size: buffer.length,
+    mime,
+  };
+}
+
+async function determinePfpFilepath(id) {
+  const numberId = Number(id);
+  const user = await User.get({ id: numberId });
+  const pfpFilename = user?.pfpFilename || null;
+  if (!pfpFilename) return null;
+
+  const basePath = process.env.STORAGE_DIR
+    ? path.join(process.env.STORAGE_DIR, "assets/pfp")
+    : path.join(__dirname, "../../storage/assets/pfp");
+  const pfpFilepath = path.join(basePath, normalizePath(pfpFilename));
+
+  if (!isWithin(path.resolve(basePath), path.resolve(pfpFilepath))) return null;
+  if (!fs.existsSync(pfpFilepath)) return null;
+  return pfpFilepath;
+}
+
+async function determineWorkspacePfpFilepath(slug) {
+  const workspace = await Workspace.get({ slug });
+  const pfpFilename = workspace?.pfpFilename || null;
+  if (!pfpFilename) return null;
+
+  const basePath = process.env.STORAGE_DIR
+    ? path.join(process.env.STORAGE_DIR, "assets/pfp")
+    : path.join(__dirname, "../../storage/assets/pfp");
+  const pfpFilepath = path.join(basePath, normalizePath(pfpFilename));
+
+  if (!isWithin(path.resolve(basePath), path.resolve(pfpFilepath))) return null;
+  if (!fs.existsSync(pfpFilepath)) return null;
+  return pfpFilepath;
+}
+
+module.exports = {
+  fetchPfp,
+  determinePfpFilepath,
+  determineWorkspacePfpFilepath,
+};

package/server/utils/files/purgeDocument.js

@@ -0,0 +1,91 @@
+const fs = require("fs");
+const path = require("path");
+const {
+  purgeVectorCache,
+  purgeSourceDocument,
+  normalizePath,
+  isWithin,
+  documentsPath,
+} = require(".");
+const { Document } = require("../../models/documents");
+const { Workspace } = require("../../models/workspace");
+
+async function purgeDocument(filename = null) {
+  if (!filename || !normalizePath(filename)) return;
+
+  await purgeVectorCache(filename);
+  await purgeSourceDocument(filename);
+  const workspaces = await Workspace.where();
+  for (const workspace of workspaces) {
+    await Document.removeDocuments(workspace, [filename]);
+  }
+  return;
+}
+
+/**
+ * Purge a folder and all its contents. This will also remove all vector-cache files and workspace document associations
+ * for the documents within the folder.
+ * @notice This function is not recursive. It only purges the contents of the specified folder.
+ * @notice You cannot purge the `custom-documents` folder.
+ * @param {string} folderName - The name/path of the folder to purge.
+ * @returns {Promise<void>}
+ */
+async function purgeFolder(folderName = null) {
+  if (!folderName) return;
+  const subFolder = normalizePath(folderName);
+  const subFolderPath = path.resolve(documentsPath, subFolder);
+  const validRemovableSubFolders = fs
+    .readdirSync(documentsPath)
+    .map((folder) => {
+      // Filter out any results which are not folders or
+      // are the protected custom-documents folder.
+      if (folder === "custom-documents") return null;
+      const subfolderPath = path.resolve(documentsPath, folder);
+      if (!fs.lstatSync(subfolderPath).isDirectory()) return null;
+      return folder;
+    })
+    .filter((subFolder) => !!subFolder);
+
+  if (
+    !validRemovableSubFolders.includes(subFolder) ||
+    !fs.existsSync(subFolderPath) ||
+    !isWithin(documentsPath, subFolderPath)
+  )
+    return;
+
+  const filenames = fs
+    .readdirSync(subFolderPath)
+    .map((file) =>
+      path.join(subFolderPath, file).replace(documentsPath + "/", "")
+    );
+  const workspaces = await Workspace.where();
+
+  const purgePromises = [];
+  // Remove associated Vector-cache files
+  for (const filename of filenames) {
+    const rmVectorCache = () =>
+      new Promise((resolve) =>
+        purgeVectorCache(filename).then(() => resolve(true))
+      );
+    purgePromises.push(rmVectorCache);
+  }
+
+  // Remove workspace document associations
+  for (const workspace of workspaces) {
+    const rmWorkspaceDoc = () =>
+      new Promise((resolve) =>
+        Document.removeDocuments(workspace, filenames).then(() => resolve(true))
+      );
+    purgePromises.push(rmWorkspaceDoc);
+  }
+
+  await Promise.all(purgePromises.flat().map((f) => f()));
+  fs.rmSync(subFolderPath, { recursive: true }); // Delete target document-folder and source files.
+
+  return;
+}
+
+module.exports = {
+  purgeDocument,
+  purgeFolder,
+};

package/server/utils/helpers/admin/index.js

@@ -0,0 +1,56 @@
+const { User } = require("../../../models/user");
+const { ROLES } = require("../../middleware/multiUserProtected");
+
+// When a user is updating or creating a user in multi-user, we need to check if they
+// are allowed to do this and that the new or existing user will be at or below their permission level.
+// the user executing this function should be an admin or manager.
+function validRoleSelection(currentUser = {}, newUserParams = {}) {
+  if (!newUserParams.hasOwnProperty("role"))
+    return { valid: true, error: null }; // not updating role, so skip.
+  if (currentUser.role === ROLES.admin) return { valid: true, error: null };
+  if (currentUser.role === ROLES.manager) {
+    const validRoles = [ROLES.manager, ROLES.default];
+    if (!validRoles.includes(newUserParams.role))
+      return { valid: false, error: "Invalid role selection for user." };
+    return { valid: true, error: null };
+  }
+  return { valid: false, error: "Invalid condition for caller." };
+}
+
+// Check to make sure with this update that includes a role change to an existing admin to a non-admin
+// that we still have at least one admin left or else they will lock themselves out.
+async function canModifyAdmin(userToModify, updates) {
+  // if updates don't include role property
+  // or the user being modified isn't an admin currently
+  // or the updates role is equal to the users current role.
+  // skip validation.
+  if (!updates.hasOwnProperty("role")) return { valid: true, error: null };
+  if (userToModify.role !== ROLES.admin) return { valid: true, error: null };
+  if (updates.role === userToModify.role) return { valid: true, error: null };
+
+  const adminCount = await User.count({ role: ROLES.admin });
+  if (adminCount - 1 <= 0)
+    return {
+      valid: false,
+      error: "No system admins will remain if you do this. Update failed.",
+    };
+  return { valid: true, error: null };
+}
+
+function validCanModify(currentUser, existingUser) {
+  if (currentUser.role === ROLES.admin) return { valid: true, error: null };
+  if (currentUser.role === ROLES.manager) {
+    const validRoles = [ROLES.manager, ROLES.default];
+    if (!validRoles.includes(existingUser.role))
+      return { valid: false, error: "Cannot perform that action on user." };
+    return { valid: true, error: null };
+  }
+
+  return { valid: false, error: "Invalid condition for caller." };
+}
+
+module.exports = {
+  validCanModify,
+  validRoleSelection,
+  canModifyAdmin,
+};

package/server/utils/helpers/agents.js

@@ -0,0 +1,35 @@
+const chalk = require("chalk");
+
+/**
+ * Checks if a skill is auto-approved by the ENV variable AGENT_AUTO_APPROVED_SKILLS.
+ * which is a comma-separated list of skill names. This property applies globally to all users
+ * so that all invocations of the skill are auto-approved without user interaction.
+ * @param {Object} options - The options object
+ * @param {string} options.skillName - The name of the skill
+ * @returns {boolean} True if the skill is auto-approved, false otherwise
+ */
+function skillIsAutoApproved({ skillName }) {
+  if (!("AGENT_AUTO_APPROVED_SKILLS" in process.env)) return false;
+  const autoApprovedSkills = String(process.env.AGENT_AUTO_APPROVED_SKILLS)
+    .split(",")
+    .map((skill) => skill.trim())
+    .filter((skill) => !!skill);
+
+  // If the list contains <all>, then all skills are auto-approved
+  // This is a special case and overrides any other items in the list.
+  if (autoApprovedSkills.includes("<all>")) return true;
+
+  if (!autoApprovedSkills.length || !autoApprovedSkills.includes(skillName))
+    return false;
+
+  console.log(
+    chalk.green(
+      `Skill ${skillName} is auto-approved by the ENV variable AGENT_AUTO_APPROVED_SKILLS.`
+    )
+  );
+  return true;
+}
+
+module.exports = {
+  skillIsAutoApproved,
+};

package/server/utils/helpers/camelcase.js

@@ -0,0 +1,143 @@
+const UPPERCASE = /[\p{Lu}]/u;
+const LOWERCASE = /[\p{Ll}]/u;
+const LEADING_CAPITAL = /^[\p{Lu}](?![\p{Lu}])/gu;
+const IDENTIFIER = /([\p{Alpha}\p{N}_]|$)/u;
+const SEPARATORS = /[_.\- ]+/;
+
+const LEADING_SEPARATORS = new RegExp("^" + SEPARATORS.source);
+const SEPARATORS_AND_IDENTIFIER = new RegExp(
+  SEPARATORS.source + IDENTIFIER.source,
+  "gu"
+);
+const NUMBERS_AND_IDENTIFIER = new RegExp("\\d+" + IDENTIFIER.source, "gu");
+
+const preserveCamelCase = (
+  string,
+  toLowerCase,
+  toUpperCase,
+  preserveConsecutiveUppercase
+) => {
+  let isLastCharLower = false;
+  let isLastCharUpper = false;
+  let isLastLastCharUpper = false;
+  let isLastLastCharPreserved = false;
+
+  for (let index = 0; index < string.length; index++) {
+    const character = string[index];
+    isLastLastCharPreserved = index > 2 ? string[index - 3] === "-" : true;
+
+    if (isLastCharLower && UPPERCASE.test(character)) {
+      string = string.slice(0, index) + "-" + string.slice(index);
+      isLastCharLower = false;
+      isLastLastCharUpper = isLastCharUpper;
+      isLastCharUpper = true;
+      index++;
+    } else if (
+      isLastCharUpper &&
+      isLastLastCharUpper &&
+      LOWERCASE.test(character) &&
+      (!isLastLastCharPreserved || preserveConsecutiveUppercase)
+    ) {
+      string = string.slice(0, index - 1) + "-" + string.slice(index - 1);
+      isLastLastCharUpper = isLastCharUpper;
+      isLastCharUpper = false;
+      isLastCharLower = true;
+    } else {
+      isLastCharLower =
+        toLowerCase(character) === character &&
+        toUpperCase(character) !== character;
+      isLastLastCharUpper = isLastCharUpper;
+      isLastCharUpper =
+        toUpperCase(character) === character &&
+        toLowerCase(character) !== character;
+    }
+  }
+
+  return string;
+};
+
+const preserveConsecutiveUppercase = (input, toLowerCase) => {
+  LEADING_CAPITAL.lastIndex = 0;
+
+  return input.replace(LEADING_CAPITAL, (m1) => toLowerCase(m1));
+};
+
+const postProcess = (input, toUpperCase) => {
+  SEPARATORS_AND_IDENTIFIER.lastIndex = 0;
+  NUMBERS_AND_IDENTIFIER.lastIndex = 0;
+
+  return input
+    .replace(SEPARATORS_AND_IDENTIFIER, (_, identifier) =>
+      toUpperCase(identifier)
+    )
+    .replace(NUMBERS_AND_IDENTIFIER, (m) => toUpperCase(m));
+};
+
+function camelCase(input, options) {
+  if (!(typeof input === "string" || Array.isArray(input))) {
+    throw new TypeError("Expected the input to be `string | string[]`");
+  }
+
+  options = {
+    pascalCase: true,
+    preserveConsecutiveUppercase: false,
+    ...options,
+  };
+
+  if (Array.isArray(input)) {
+    input = input
+      .map((x) => x.trim())
+      .filter((x) => x.length)
+      .join("-");
+  } else {
+    input = input.trim();
+  }
+
+  if (input.length === 0) {
+    return "";
+  }
+
+  const toLowerCase =
+    options.locale === false
+      ? (string) => string.toLowerCase()
+      : (string) => string.toLocaleLowerCase(options.locale);
+
+  const toUpperCase =
+    options.locale === false
+      ? (string) => string.toUpperCase()
+      : (string) => string.toLocaleUpperCase(options.locale);
+
+  if (input.length === 1) {
+    if (SEPARATORS.test(input)) {
+      return "";
+    }
+
+    return options.pascalCase ? toUpperCase(input) : toLowerCase(input);
+  }
+
+  const hasUpperCase = input !== toLowerCase(input);
+
+  if (hasUpperCase) {
+    input = preserveCamelCase(
+      input,
+      toLowerCase,
+      toUpperCase,
+      options.preserveConsecutiveUppercase
+    );
+  }
+
+  input = input.replace(LEADING_SEPARATORS, "");
+  input = options.preserveConsecutiveUppercase
+    ? preserveConsecutiveUppercase(input, toLowerCase)
+    : toLowerCase(input);
+
+  if (options.pascalCase) {
+    input = toUpperCase(input.charAt(0)) + input.slice(1);
+  }
+
+  return postProcess(input, toUpperCase);
+}
+
+module.exports = {
+  camelCase,
+};