totem-llm 0.1.0

package/server/utils/AiProviders/minimax/index.js

@@ -0,0 +1,165 @@
+const { NativeEmbedder } = require("../../EmbeddingEngines/native");
+const {
+  LLMPerformanceMonitor,
+} = require("../../helpers/chat/LLMPerformanceMonitor");
+const {
+  handleDefaultStreamResponseV2,
+} = require("../../helpers/chat/responses");
+const { MODEL_MAP } = require("../modelMap");
+
+class MinimaxLLM {
+  constructor(embedder = null, modelPreference = null) {
+    const { OpenAI: OpenAIApi } = require("openai");
+    if (!process.env.MINIMAX_API_KEY)
+      throw new Error("No Minimax API key was set.");
+    this.className = "MinimaxLLM";
+
+    this.openai = new OpenAIApi({
+      baseURL: "https://api.minimax.io/v1",
+      apiKey: process.env.MINIMAX_API_KEY,
+    });
+    this.model =
+      modelPreference || process.env.MINIMAX_MODEL_PREF || "MiniMax-M2.7";
+    this.limits = {
+      history: this.promptWindowLimit() * 0.15,
+      system: this.promptWindowLimit() * 0.15,
+      user: this.promptWindowLimit() * 0.7,
+    };
+
+    this.embedder = embedder ?? new NativeEmbedder();
+    this.defaultTemp = 0.7;
+    this.log(
+      `Initialized ${this.model} with context window ${this.promptWindowLimit()}`
+    );
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
+  }
+
+  #appendContext(contextTexts = []) {
+    if (!contextTexts || !contextTexts.length) return "";
+    return (
+      "\nContext:\n" +
+      contextTexts
+        .map((text, i) => {
+          return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+        })
+        .join("")
+    );
+  }
+
+  streamingEnabled() {
+    return "streamGetChatCompletion" in this;
+  }
+
+  static promptWindowLimit(modelName) {
+    return MODEL_MAP.get("minimax", modelName) ?? 196000;
+  }
+
+  promptWindowLimit() {
+    return MODEL_MAP.get("minimax", this.model) ?? 196000;
+  }
+
+  async isValidChatCompletionModel(modelName = "") {
+    const models = await this.openai.models.list().catch(() => ({ data: [] }));
+    return models.data.some((model) => model.id === modelName);
+  }
+
+  constructPrompt({
+    systemPrompt = "",
+    contextTexts = [],
+    chatHistory = [],
+    userPrompt = "",
+  }) {
+    const prompt = {
+      role: "system",
+      content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
+    };
+    return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
+  }
+
+  async getChatCompletion(messages = null, { temperature = 0.7 }) {
+    if (!(await this.isValidChatCompletionModel(this.model)))
+      throw new Error(
+        `Minimax chat: ${this.model} is not valid for chat completion!`
+      );
+
+    const result = await LLMPerformanceMonitor.measureAsyncFunction(
+      this.openai.chat.completions
+        .create({
+          model: this.model,
+          messages,
+          temperature,
+        })
+        .catch((e) => {
+          throw new Error(e.message);
+        })
+    );
+
+    if (
+      !result?.output?.hasOwnProperty("choices") ||
+      result?.output?.choices?.length === 0
+    )
+      throw new Error(
+        `Invalid response body returned from Minimax: ${JSON.stringify(result.output)}`
+      );
+
+    return {
+      textResponse: result.output.choices[0].message.content,
+      metrics: {
+        prompt_tokens: result.output.usage.prompt_tokens || 0,
+        completion_tokens: result.output.usage.completion_tokens || 0,
+        total_tokens: result.output.usage.total_tokens || 0,
+        outputTps: result.output.usage.completion_tokens / result.duration,
+        duration: result.duration,
+        model: this.model,
+        provider: this.className,
+        timestamp: new Date(),
+      },
+    };
+  }
+
+  async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
+    if (!(await this.isValidChatCompletionModel(this.model)))
+      throw new Error(
+        `Minimax stream: ${this.model} is not valid for chat completion!`
+      );
+
+    const measuredStreamRequest = await LLMPerformanceMonitor.measureStream({
+      func: this.openai.chat.completions.create({
+        model: this.model,
+        stream: true,
+        messages,
+        temperature,
+      }),
+      messages,
+      runPromptTokenCalculation: false,
+      modelTag: this.model,
+      provider: this.className,
+    });
+
+    return measuredStreamRequest;
+  }
+
+  handleStream(response, stream, responseProps) {
+    return handleDefaultStreamResponseV2(response, stream, responseProps);
+  }
+
+  async embedTextInput(textInput) {
+    return await this.embedder.embedTextInput(textInput);
+  }
+  async embedChunks(textChunks = []) {
+    return await this.embedder.embedChunks(textChunks);
+  }
+
+  async compressMessages(promptArgs = {}, rawHistory = []) {
+    const { messageArrayCompressor } = require("../../helpers/chat");
+    const messageArray = this.constructPrompt(promptArgs);
+    return await messageArrayCompressor(this, messageArray, rawHistory);
+  }
+}
+
+module.exports = {
+  MinimaxLLM,
+};

package/server/utils/AiProviders/mistral/index.js

@@ -0,0 +1,191 @@
+const { NativeEmbedder } = require("../../EmbeddingEngines/native");
+const {
+  LLMPerformanceMonitor,
+} = require("../../helpers/chat/LLMPerformanceMonitor");
+const {
+  handleDefaultStreamResponseV2,
+  formatChatHistory,
+} = require("../../helpers/chat/responses");
+
+class MistralLLM {
+  constructor(embedder = null, modelPreference = null) {
+    if (!process.env.MISTRAL_API_KEY)
+      throw new Error("No Mistral API key was set.");
+
+    this.className = "MistralLLM";
+    const { OpenAI: OpenAIApi } = require("openai");
+    this.openai = new OpenAIApi({
+      baseURL: "https://api.mistral.ai/v1",
+      apiKey: process.env.MISTRAL_API_KEY ?? null,
+    });
+    this.model =
+      modelPreference || process.env.MISTRAL_MODEL_PREF || "mistral-tiny";
+    this.limits = {
+      history: this.promptWindowLimit() * 0.15,
+      system: this.promptWindowLimit() * 0.15,
+      user: this.promptWindowLimit() * 0.7,
+    };
+
+    this.embedder = embedder ?? new NativeEmbedder();
+    this.defaultTemp = 0.0;
+    this.log("Initialized with model:", this.model);
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[${this.className}]\x1b[0m ${text}`, ...args);
+  }
+
+  #appendContext(contextTexts = []) {
+    if (!contextTexts || !contextTexts.length) return "";
+    return (
+      "\nContext:\n" +
+      contextTexts
+        .map((text, i) => {
+          return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+        })
+        .join("")
+    );
+  }
+
+  streamingEnabled() {
+    return "streamGetChatCompletion" in this;
+  }
+
+  static promptWindowLimit() {
+    return 32000;
+  }
+
+  promptWindowLimit() {
+    return 32000;
+  }
+
+  async isValidChatCompletionModel(_modelName = "") {
+    return true;
+  }
+
+  /**
+   * Generates appropriate content array for a message + attachments.
+   * @param {{userPrompt:string, attachments: import("../../helpers").Attachment[]}}
+   * @returns {string|object[]}
+   */
+  #generateContent({ userPrompt, attachments = [] }) {
+    if (!attachments.length) return userPrompt;
+
+    const content = [{ type: "text", text: userPrompt }];
+    for (let attachment of attachments) {
+      content.push({
+        type: "image_url",
+        image_url: attachment.contentString,
+      });
+    }
+    return content.flat();
+  }
+
+  /**
+   * Construct the user prompt for this model.
+   * @param {{attachments: import("../../helpers").Attachment[]}} param0
+   * @returns
+   */
+  constructPrompt({
+    systemPrompt = "",
+    contextTexts = [],
+    chatHistory = [],
+    userPrompt = "",
+    attachments = [], // This is the specific attachment for only this prompt
+  }) {
+    const prompt = {
+      role: "system",
+      content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
+    };
+    return [
+      prompt,
+      ...formatChatHistory(chatHistory, this.#generateContent),
+      {
+        role: "user",
+        content: this.#generateContent({ userPrompt, attachments }),
+      },
+    ];
+  }
+
+  async getChatCompletion(messages = null, { temperature = 0.7 }) {
+    if (!(await this.isValidChatCompletionModel(this.model)))
+      throw new Error(
+        `Mistral chat: ${this.model} is not valid for chat completion!`
+      );
+
+    const result = await LLMPerformanceMonitor.measureAsyncFunction(
+      this.openai.chat.completions
+        .create({
+          model: this.model,
+          messages,
+          temperature,
+        })
+        .catch((e) => {
+          throw new Error(e.message);
+        })
+    );
+
+    if (
+      !result.output.hasOwnProperty("choices") ||
+      result.output.choices.length === 0
+    )
+      return null;
+
+    return {
+      textResponse: result.output.choices[0].message.content,
+      metrics: {
+        prompt_tokens: result.output.usage.prompt_tokens || 0,
+        completion_tokens: result.output.usage.completion_tokens || 0,
+        total_tokens: result.output.usage.total_tokens || 0,
+        outputTps: result.output.usage.completion_tokens / result.duration,
+        duration: result.duration,
+        model: this.model,
+        provider: this.className,
+        timestamp: new Date(),
+      },
+    };
+  }
+
+  async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
+    if (!(await this.isValidChatCompletionModel(this.model)))
+      throw new Error(
+        `Mistral chat: ${this.model} is not valid for chat completion!`
+      );
+
+    const measuredStreamRequest = await LLMPerformanceMonitor.measureStream({
+      func: this.openai.chat.completions.create({
+        model: this.model,
+        stream: true,
+        messages,
+        temperature,
+      }),
+      messages,
+      runPromptTokenCalculation: false,
+      modelTag: this.model,
+      provider: this.className,
+    });
+    return measuredStreamRequest;
+  }
+
+  handleStream(response, stream, responseProps) {
+    return handleDefaultStreamResponseV2(response, stream, responseProps);
+  }
+
+  // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+  async embedTextInput(textInput) {
+    return await this.embedder.embedTextInput(textInput);
+  }
+  async embedChunks(textChunks = []) {
+    return await this.embedder.embedChunks(textChunks);
+  }
+
+  async compressMessages(promptArgs = {}, rawHistory = []) {
+    const { messageArrayCompressor } = require("../../helpers/chat");
+    const messageArray = this.constructPrompt(promptArgs);
+    return await messageArrayCompressor(this, messageArray, rawHistory);
+  }
+}
+
+module.exports = {
+  MistralLLM,
+};

package/server/utils/AiProviders/modelMap/index.js

@@ -0,0 +1,212 @@
+const path = require("path");
+const fs = require("fs");
+const LEGACY_MODEL_MAP = require("./legacy");
+
+class ContextWindowFinder {
+  static instance = null;
+  static modelMap = LEGACY_MODEL_MAP;
+
+  /**
+   * Mapping for AnythingLLM provider <> LiteLLM provider
+   * @type {Record<string, string>}
+   */
+  static trackedProviders = {
+    anthropic: "anthropic",
+    openai: "openai",
+    cohere: "cohere_chat",
+    gemini: "vertex_ai-language-models",
+    groq: "groq",
+    xai: "xai",
+    deepseek: "deepseek",
+    moonshot: "moonshot",
+    zai: "vercel_ai_gateway", // Vercel has correct context windows for Z.AI models
+    sambanova: "sambanova",
+    minimax: "minimax",
+    cerebras: "cerebras",
+  };
+  static expiryMs = 1000 * 60 * 60 * 24 * 3; // 3 days
+  static remoteUrl =
+    "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
+
+  cacheLocation = path.resolve(
+    process.env.STORAGE_DIR
+      ? path.resolve(process.env.STORAGE_DIR, "models", "context-windows")
+      : path.resolve(__dirname, `../../../storage/models/context-windows`)
+  );
+  cacheFilePath = path.resolve(this.cacheLocation, "context-windows.json");
+  cacheFileExpiryPath = path.resolve(this.cacheLocation, ".cached_at");
+  seenStaleCacheWarning = false;
+
+  constructor() {
+    if (ContextWindowFinder.instance) return ContextWindowFinder.instance;
+    ContextWindowFinder.instance = this;
+    if (!fs.existsSync(this.cacheLocation))
+      fs.mkdirSync(this.cacheLocation, { recursive: true });
+
+    // If the cache is stale or not found at all, pull the model map from remote
+    if (this.isCacheStale || !fs.existsSync(this.cacheFilePath)) {
+      this.#pullRemoteModelMap().catch((err) =>
+        this.log("Background model map pull failed:", err)
+      );
+    }
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[33m[ContextWindowFinder]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   * Checks if the cache is stale by checking if the cache file exists and if the cache file is older than the expiry time.
+   * @returns {boolean}
+   */
+  get isCacheStale() {
+    if (!fs.existsSync(this.cacheFileExpiryPath)) return true;
+    const cachedAt = fs.readFileSync(this.cacheFileExpiryPath, "utf8");
+    return Date.now() - cachedAt > ContextWindowFinder.expiryMs;
+  }
+
+  /**
+   * Gets the cached model map.
+   *
+   * Always returns the available model map - even if it is expired since re-pulling
+   * the model map only occurs on container start/system start.
+   * @returns {Record<string, Record<string, number>> | null} - The cached model map
+   */
+  get cachedModelMap() {
+    if (!fs.existsSync(this.cacheFilePath)) {
+      this.log(`\x1b[33m
+--------------------------------
+[WARNING] Model map cache is not found!
+Invalid context windows will be returned leading to inaccurate model responses
+or smaller context windows than expected.
+You can fix this by restarting AnythingLLM so the model map is re-pulled.
+--------------------------------\x1b[0m`);
+      return null;
+    }
+
+    if (this.isCacheStale && !this.seenStaleCacheWarning) {
+      this.log(
+        "Model map cache is stale - some model context windows may be incorrect. This is OK and the model map will be re-pulled on next boot."
+      );
+      this.seenStaleCacheWarning = true;
+    }
+
+    return JSON.parse(
+      fs.readFileSync(this.cacheFilePath, { encoding: "utf8" })
+    );
+  }
+
+  /**
+   * Pulls the remote model map from the remote URL, formats it and caches it.
+   * @returns {Record<string, Record<string, number>>} - The formatted model map
+   */
+  async #pullRemoteModelMap() {
+    try {
+      this.log("Pulling remote model map...");
+      const response = await fetch(ContextWindowFinder.remoteUrl);
+      if (response.status !== 200) {
+        throw new Error(
+          "Failed to fetch remote model map - non 200 status code"
+        );
+      }
+
+      const data = await response.json();
+      const modelMap = this.#validateModelMap(this.#formatModelMap(data));
+      await Promise.all([
+        fs.promises.writeFile(
+          this.cacheFilePath,
+          JSON.stringify(modelMap, null, 2)
+        ),
+        fs.promises.writeFile(this.cacheFileExpiryPath, Date.now().toString()),
+      ]);
+
+      this.log("Remote model map synced and cached");
+      return modelMap;
+    } catch (error) {
+      this.log("Error syncing remote model map", error);
+      return null;
+    }
+  }
+
+  #validateModelMap(modelMap = {}) {
+    for (const [provider, models] of Object.entries(modelMap)) {
+      // If the models is null/falsey or has no keys, throw an error
+      if (typeof models !== "object")
+        throw new Error(
+          `Invalid model map for ${provider} - models is not an object`
+        );
+      if (!models || Object.keys(models).length === 0)
+        throw new Error(`Invalid model map for ${provider} - no models found!`);
+
+      // Validate that the context window is a number
+      for (const [model, contextWindow] of Object.entries(models)) {
+        if (isNaN(contextWindow) || contextWindow <= 0) {
+          this.log(
+            `${provider}:${model} - context window is not a positive number. Got ${contextWindow}.`
+          );
+          delete models[model];
+          continue;
+        }
+      }
+    }
+    return modelMap;
+  }
+
+  /**
+   * Formats the remote model map to a format that is compatible with how we store the model map
+   * for all providers who use it.
+   * @param {Record<string, any>} modelMap - The remote model map
+   * @returns {Record<string, Record<string, number>>} - The formatted model map
+   */
+  #formatModelMap(modelMap = {}) {
+    const formattedModelMap = {};
+
+    for (const [provider, liteLLMProviderTag] of Object.entries(
+      ContextWindowFinder.trackedProviders
+    )) {
+      formattedModelMap[provider] = {};
+      const matches = Object.entries(modelMap).filter(
+        ([_key, config]) => config.litellm_provider === liteLLMProviderTag
+      );
+      for (const [key, config] of matches) {
+        const contextWindow = Number(config.max_input_tokens);
+        if (isNaN(contextWindow)) continue;
+
+        // Some models have a provider/model-tag format, so we need to get the last part since we dont do paths
+        // for names with the exception of some router-providers like OpenRouter or Together.
+        const modelName = key.split("/").pop();
+        formattedModelMap[provider][modelName] = contextWindow;
+      }
+    }
+    return formattedModelMap;
+  }
+
+  /**
+   * Gets the context window for a given provider and model.
+   *
+   * If the provider is not found, null is returned.
+   * If the model is not found, the provider's entire model map is returned.
+   *
+   * if both provider and model are provided, the context window for the given model is returned.
+   * @param {string|null} provider - The provider to get the context window for
+   * @param {string|null} model - The model to get the context window for
+   * @returns {number|null} - The context window for the given provider and model
+   */
+  get(provider = null, model = null) {
+    if (!provider || !this.cachedModelMap || !this.cachedModelMap[provider])
+      return null;
+    if (!model) return this.cachedModelMap[provider];
+
+    const modelContextWindow = this.cachedModelMap[provider][model];
+    if (!modelContextWindow) {
+      this.log("Invalid access to model context window - not found in cache", {
+        provider,
+        model,
+      });
+      return null;
+    }
+    return Number(modelContextWindow);
+  }
+}
+
+module.exports = { MODEL_MAP: new ContextWindowFinder() };