totem-llm 0.1.0

package/server/utils/agents/aibitat/providers/lemonade.js

@@ -0,0 +1,213 @@
+const OpenAI = require("openai");
+const Provider = require("./ai-provider.js");
+const InheritMultiple = require("./helpers/classes.js");
+const UnTooled = require("./helpers/untooled.js");
+const { tooledStream, tooledComplete } = require("./helpers/tooled.js");
+const { RetryError } = require("../error.js");
+const {
+  LemonadeLLM,
+  parseLemonadeServerEndpoint,
+} = require("../../../AiProviders/lemonade/index.js");
+
+/**
+ * The agent provider for the Lemonade.
+ */
+class LemonadeProvider extends InheritMultiple([Provider, UnTooled]) {
+  model;
+
+  /**
+   *
+   * @param {{model?: string}} config
+   */
+  constructor(config = {}) {
+    super();
+    const model = config?.model || process.env.LEMONADE_LLM_MODEL_PREF || null;
+    const client = new OpenAI({
+      baseURL: parseLemonadeServerEndpoint(
+        process.env.LEMONADE_LLM_BASE_PATH,
+        "openai"
+      ),
+      apiKey: process.env.LEMONADE_LLM_API_KEY || null,
+      maxRetries: 3,
+    });
+
+    this._client = client;
+    this.model = model;
+    this.verbose = true;
+    this.preloaded = false;
+    this._supportsToolCalling = null;
+  }
+
+  get client() {
+    return this._client;
+  }
+
+  get supportsAgentStreaming() {
+    return true;
+  }
+
+  async preloadModel() {
+    if (this.preloaded) return;
+    await LemonadeLLM.loadModel(this.model);
+    this.preloaded = true;
+  }
+
+  /**
+   * Whether this provider supports native OpenAI-compatible tool calling.
+   * - Since Lemonade models vary in tool calling support, we check the ENV.
+   * - If the ENV is not set and the capabilities are not set, we default to false.
+   * - To enable tool calling for a model, set the ENV flag for `PROVIDER_SUPPORTS_NATIVE_TOOL_CALLING` to include `lemonade`.
+   * - or update the label in the Lemonade server to include `tool-calling`.
+   * @returns {boolean|Promise<boolean>}
+   */
+  async supportsNativeToolCalling() {
+    if (this._supportsToolCalling !== null) return this._supportsToolCalling;
+    const lemonade = new LemonadeLLM(null, this.model);
+
+    // Labels can be missing for tool calling models, so we also check if ENV flag is set
+    const supportsToolCallingFlag =
+      this.supportsNativeToolCallingViaEnv("lemonade");
+    if (supportsToolCallingFlag) {
+      this.providerLog(
+        "Lemonade supports native tool calling is ENABLED via ENV."
+      );
+      this._supportsToolCalling = true;
+      return this._supportsToolCalling;
+    }
+
+    const capabilities = await lemonade.getModelCapabilities();
+    this._supportsToolCalling = capabilities.tools === true;
+    return this._supportsToolCalling;
+  }
+
+  async #handleFunctionCallChat({ messages = [] }) {
+    return await this.client.chat.completions
+      .create({
+        model: this.model,
+        messages,
+      })
+      .then((result) => {
+        if (!result.hasOwnProperty("choices"))
+          throw new Error("Lemonade chat: No results!");
+        if (result.choices.length === 0)
+          throw new Error("Lemonade chat: No results length!");
+        return result.choices[0].message.content;
+      })
+      .catch((_) => {
+        return null;
+      });
+  }
+
+  async #handleFunctionCallStream({ messages = [] }) {
+    return await this.client.chat.completions.create({
+      model: this.model,
+      stream: true,
+      messages,
+    });
+  }
+
+  /**
+   * Stream a chat completion with tool calling support.
+   * Uses native tool calling when supported, otherwise falls back to UnTooled.
+   */
+  async stream(messages, functions = [], eventHandler = null) {
+    await this.preloadModel();
+    const useNative =
+      functions.length > 0 && (await this.supportsNativeToolCalling());
+
+    if (!useNative) {
+      return await UnTooled.prototype.stream.call(
+        this,
+        messages,
+        functions,
+        this.#handleFunctionCallStream.bind(this),
+        eventHandler
+      );
+    }
+
+    this.providerLog(
+      "LemonadeProvider.stream (tooled) - will process this chat completion."
+    );
+
+    try {
+      return await tooledStream(
+        this.client,
+        this.model,
+        messages,
+        functions,
+        eventHandler,
+        { provider: this }
+      );
+    } catch (error) {
+      console.error(error.message, error);
+      if (error instanceof OpenAI.AuthenticationError) throw error;
+      if (
+        error instanceof OpenAI.RateLimitError ||
+        error instanceof OpenAI.InternalServerError ||
+        error instanceof OpenAI.APIError
+      ) {
+        throw new RetryError(error.message);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Create a non-streaming completion with tool calling support.
+   * Uses native tool calling when supported, otherwise falls back to UnTooled.
+   */
+  async complete(messages, functions = []) {
+    await this.preloadModel();
+    const useNative =
+      functions.length > 0 && (await this.supportsNativeToolCalling());
+
+    if (!useNative) {
+      return await UnTooled.prototype.complete.call(
+        this,
+        messages,
+        functions,
+        this.#handleFunctionCallChat.bind(this)
+      );
+    }
+
+    try {
+      const result = await tooledComplete(
+        this.client,
+        this.model,
+        messages,
+        functions,
+        this.getCost.bind(this),
+        { provider: this }
+      );
+
+      if (result.retryWithError) {
+        return this.complete([...messages, result.retryWithError], functions);
+      }
+
+      return result;
+    } catch (error) {
+      if (error instanceof OpenAI.AuthenticationError) throw error;
+      if (
+        error instanceof OpenAI.RateLimitError ||
+        error instanceof OpenAI.InternalServerError ||
+        error instanceof OpenAI.APIError
+      ) {
+        throw new RetryError(error.message);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Get the cost of the completion.
+   *
+   * @param _usage The completion to get the cost for.
+   * @returns The cost of the completion.
+   * Stubbed since Lemonade has no cost basis.
+   */
+  getCost(_usage) {
+    return 0;
+  }
+}
+
+module.exports = LemonadeProvider;

package/server/utils/agents/aibitat/providers/litellm.js

@@ -0,0 +1,186 @@
+const OpenAI = require("openai");
+const Provider = require("./ai-provider.js");
+const InheritMultiple = require("./helpers/classes.js");
+const UnTooled = require("./helpers/untooled.js");
+const { tooledStream, tooledComplete } = require("./helpers/tooled.js");
+const { RetryError } = require("../error.js");
+
+/**
+ * The agent provider for LiteLLM.
+ * Supports true OpenAI-compatible tool calling when enabled via ENV,
+ * falling back to the UnTooled prompt-based approach otherwise.
+ */
+class LiteLLMProvider extends InheritMultiple([Provider, UnTooled]) {
+  model;
+
+  constructor(config = {}) {
+    super();
+    const { model = null } = config;
+    const client = new OpenAI({
+      baseURL: process.env.LITE_LLM_BASE_PATH,
+      apiKey: process.env.LITE_LLM_API_KEY ?? null,
+      maxRetries: 3,
+    });
+
+    this._client = client;
+    this.model = model || process.env.LITE_LLM_MODEL_PREF;
+    this.verbose = true;
+    this._supportsToolCalling = null;
+  }
+
+  get client() {
+    return this._client;
+  }
+
+  get supportsAgentStreaming() {
+    return true;
+  }
+
+  /**
+   * Whether this provider supports native OpenAI-compatible tool calling.
+   * - Since LiteLLM models vary in tool calling support, we check the ENV.
+   * - If the ENV is not set, we default to false.
+   * @returns {boolean}
+   */
+  supportsNativeToolCalling() {
+    if (this._supportsToolCalling !== null) return this._supportsToolCalling;
+    const supportsToolCalling = this.supportsNativeToolCallingViaEnv("litellm");
+    if (supportsToolCalling)
+      this.providerLog(
+        "LiteLLM supports native tool calling is ENABLED via ENV."
+      );
+    else
+      this.providerLog(
+        "LiteLLM supports native tool calling is DISABLED via ENV. Will use UnTooled instead."
+      );
+    this._supportsToolCalling = supportsToolCalling;
+    return supportsToolCalling;
+  }
+
+  async #handleFunctionCallChat({ messages = [] }) {
+    return await this.client.chat.completions
+      .create({
+        model: this.model,
+        messages,
+      })
+      .then((result) => {
+        if (!result.hasOwnProperty("choices"))
+          throw new Error("LiteLLM chat: No results!");
+        if (result.choices.length === 0)
+          throw new Error("LiteLLM chat: No results length!");
+        return result.choices[0].message.content;
+      })
+      .catch((_) => {
+        return null;
+      });
+  }
+
+  async #handleFunctionCallStream({ messages = [] }) {
+    return await this.client.chat.completions.create({
+      model: this.model,
+      stream: true,
+      messages,
+    });
+  }
+
+  /**
+   * Stream a chat completion with tool calling support.
+   * Uses native tool calling when enabled via ENV, otherwise falls back to UnTooled.
+   */
+  async stream(messages, functions = [], eventHandler = null) {
+    const useNative =
+      functions.length > 0 && (await this.supportsNativeToolCalling());
+
+    if (!useNative) {
+      return await UnTooled.prototype.stream.call(
+        this,
+        messages,
+        functions,
+        this.#handleFunctionCallStream.bind(this),
+        eventHandler
+      );
+    }
+
+    this.providerLog(
+      "Provider.stream (tooled) - will process this chat completion."
+    );
+
+    try {
+      return await tooledStream(
+        this.client,
+        this.model,
+        messages,
+        functions,
+        eventHandler,
+        { provider: this }
+      );
+    } catch (error) {
+      console.error(error.message, error);
+      if (error instanceof OpenAI.AuthenticationError) throw error;
+      if (
+        error instanceof OpenAI.RateLimitError ||
+        error instanceof OpenAI.InternalServerError ||
+        error instanceof OpenAI.APIError
+      ) {
+        throw new RetryError(error.message);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Create a non-streaming completion with tool calling support.
+   * Uses native tool calling when enabled via ENV, otherwise falls back to UnTooled.
+   */
+  async complete(messages, functions = []) {
+    const useNative =
+      functions.length > 0 && (await this.supportsNativeToolCalling());
+
+    if (!useNative) {
+      return await UnTooled.prototype.complete.call(
+        this,
+        messages,
+        functions,
+        this.#handleFunctionCallChat.bind(this)
+      );
+    }
+
+    try {
+      const result = await tooledComplete(
+        this.client,
+        this.model,
+        messages,
+        functions,
+        this.getCost.bind(this),
+        { provider: this }
+      );
+
+      if (result.retryWithError) {
+        return this.complete([...messages, result.retryWithError], functions);
+      }
+
+      return result;
+    } catch (error) {
+      if (error instanceof OpenAI.AuthenticationError) throw error;
+      if (
+        error instanceof OpenAI.RateLimitError ||
+        error instanceof OpenAI.InternalServerError ||
+        error instanceof OpenAI.APIError
+      ) {
+        throw new RetryError(error.message);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Get the cost of the completion.
+   * @param _usage The completion to get the cost for.
+   * @returns The cost of the completion.
+   */
+  getCost(_usage) {
+    return 0;
+  }
+}
+
+module.exports = LiteLLMProvider;

package/server/utils/agents/aibitat/providers/lmstudio.js

@@ -0,0 +1,195 @@
+const OpenAI = require("openai");
+const Provider = require("./ai-provider.js");
+const InheritMultiple = require("./helpers/classes.js");
+const UnTooled = require("./helpers/untooled.js");
+const { tooledStream, tooledComplete } = require("./helpers/tooled.js");
+const { RetryError } = require("../error.js");
+const {
+  LMStudioLLM,
+  parseLMStudioBasePath,
+} = require("../../../AiProviders/lmStudio/index.js");
+
+/**
+ * The agent provider for the LMStudio.
+ * Supports true OpenAI-compatible tool calling when the model supports it,
+ * falling back to the UnTooled prompt-based approach otherwise.
+ */
+class LMStudioProvider extends InheritMultiple([Provider, UnTooled]) {
+  model;
+
+  /**
+   * @param {{model?: string}} config
+   */
+  constructor(config = {}) {
+    super();
+    const model = config?.model || process.env.LMSTUDIO_MODEL_PREF;
+    if (!model) throw new Error("LMStudio must have a valid model set.");
+
+    const apiKey = process.env.LMSTUDIO_AUTH_TOKEN ?? null;
+    const client = new OpenAI({
+      baseURL: parseLMStudioBasePath(process.env.LMSTUDIO_BASE_PATH),
+      apiKey,
+      maxRetries: 3,
+    });
+
+    this._client = client;
+    this.model = model;
+    this.verbose = true;
+    this._supportsToolCalling = null;
+  }
+
+  get client() {
+    return this._client;
+  }
+
+  get supportsAgentStreaming() {
+    return true;
+  }
+
+  /**
+   * Whether the loaded model supports native OpenAI-compatible tool calling.
+   * Checks the LMStudio /api/v1/models endpoint for the model's capabilities.
+   * @returns {Promise<boolean>}
+   */
+  async supportsNativeToolCalling() {
+    if (this._supportsToolCalling !== null) return this._supportsToolCalling;
+    const lmstudio = new LMStudioLLM(null, this.model);
+    const capabilities = await lmstudio.getModelCapabilities();
+    this._supportsToolCalling = capabilities.tools === true;
+    return this._supportsToolCalling;
+  }
+
+  // ---- UnTooled callbacks (used when native tool calling is not supported) ----
+
+  async #handleFunctionCallChat({ messages = [] }) {
+    await LMStudioLLM.cacheContextWindows();
+    return await this.client.chat.completions
+      .create({
+        model: this.model,
+        messages,
+      })
+      .then((result) => {
+        if (!result.hasOwnProperty("choices"))
+          throw new Error("LMStudio chat: No results!");
+        if (result.choices.length === 0)
+          throw new Error("LMStudio chat: No results length!");
+        return result.choices[0].message.content;
+      })
+      .catch((_) => {
+        return null;
+      });
+  }
+
+  async #handleFunctionCallStream({ messages = [] }) {
+    await LMStudioLLM.cacheContextWindows();
+    return await this.client.chat.completions.create({
+      model: this.model,
+      stream: true,
+      messages,
+    });
+  }
+
+  /**
+   * Stream a chat completion with tool calling support.
+   * Uses native tool calling when supported, otherwise falls back to UnTooled.
+   */
+  async stream(messages, functions = [], eventHandler = null) {
+    const useNative =
+      functions.length > 0 && (await this.supportsNativeToolCalling());
+
+    if (!useNative) {
+      return await UnTooled.prototype.stream.call(
+        this,
+        messages,
+        functions,
+        this.#handleFunctionCallStream.bind(this),
+        eventHandler
+      );
+    }
+
+    this.providerLog(
+      "Provider.stream (tooled) - will process this chat completion."
+    );
+
+    try {
+      await LMStudioLLM.cacheContextWindows();
+      return await tooledStream(
+        this.client,
+        this.model,
+        messages,
+        functions,
+        eventHandler,
+        { provider: this }
+      );
+    } catch (error) {
+      console.error(error.message, error);
+      if (error instanceof OpenAI.AuthenticationError) throw error;
+      if (
+        error instanceof OpenAI.RateLimitError ||
+        error instanceof OpenAI.InternalServerError ||
+        error instanceof OpenAI.APIError
+      ) {
+        throw new RetryError(error.message);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Create a non-streaming completion with tool calling support.
+   * Uses native tool calling when supported, otherwise falls back to UnTooled.
+   */
+  async complete(messages, functions = []) {
+    const useNative =
+      functions.length > 0 && (await this.supportsNativeToolCalling());
+
+    if (!useNative) {
+      return await UnTooled.prototype.complete.call(
+        this,
+        messages,
+        functions,
+        this.#handleFunctionCallChat.bind(this)
+      );
+    }
+
+    try {
+      await LMStudioLLM.cacheContextWindows();
+      const result = await tooledComplete(
+        this.client,
+        this.model,
+        messages,
+        functions,
+        this.getCost.bind(this),
+        { provider: this }
+      );
+
+      if (result.retryWithError) {
+        return this.complete([...messages, result.retryWithError], functions);
+      }
+
+      return result;
+    } catch (error) {
+      if (error instanceof OpenAI.AuthenticationError) throw error;
+      if (
+        error instanceof OpenAI.RateLimitError ||
+        error instanceof OpenAI.InternalServerError ||
+        error instanceof OpenAI.APIError
+      ) {
+        throw new RetryError(error.message);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Get the cost of the completion.
+   * Stubbed since LMStudio has no cost basis.
+   * @param _usage The completion to get the cost for.
+   * @returns The cost of the completion.
+   */
+  getCost(_usage) {
+    return 0;
+  }
+}
+
+module.exports = LMStudioProvider;