totem-llm 0.1.0

package/server/utils/TextSplitter/index.js

@@ -0,0 +1,206 @@
+/**
+ * @typedef {object} DocumentMetadata
+ * @property {string} id - eg; "123e4567-e89b-12d3-a456-426614174000"
+ * @property {string} url - eg; "file://example.com/index.html"
+ * @property {string} title - eg; "example.com/index.html"
+ * @property {string} docAuthor - eg; "no author found"
+ * @property {string} description - eg; "No description found."
+ * @property {string} docSource - eg; "URL link uploaded by the user."
+ * @property {string} chunkSource - eg; link://https://example.com
+ * @property {string} published - ISO 8601 date string
+ * @property {number} wordCount - Number of words in the document
+ * @property {string} pageContent - The raw text content of the document
+ * @property {number} token_count_estimate - Number of tokens in the document
+ */
+
+function isNullOrNaN(value) {
+  if (value === null) return true;
+  return isNaN(value);
+}
+
+class TextSplitter {
+  #splitter;
+
+  /**
+   * Creates a new TextSplitter instance.
+   * @param {Object} config
+   * @param {string} [config.chunkPrefix = ""] - Prefix to be added to the start of each chunk.
+   * @param {number} [config.chunkSize = 1000] - The size of each chunk.
+   * @param {number} [config.chunkOverlap = 20] - The overlap between chunks.
+   * @param {Object} [config.chunkHeaderMeta = null] - Metadata to be added to the start of each chunk - will come after the prefix.
+   */
+  constructor(config = {}) {
+    this.config = config;
+    this.#splitter = this.#setSplitter(config);
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[35m[TextSplitter]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   *  Does a quick check to determine the text chunk length limit.
+   * Embedder models have hard-set limits that cannot be exceeded, just like an LLM context
+   * so here we want to allow override of the default 1000, but up to the models maximum, which is
+   * sometimes user defined.
+   */
+  static determineMaxChunkSize(preferred = null, embedderLimit = 1000) {
+    const prefValue = isNullOrNaN(preferred)
+      ? Number(embedderLimit)
+      : Number(preferred);
+    const limit = Number(embedderLimit);
+    if (prefValue > limit)
+      console.log(
+        `\x1b[43m[WARN]\x1b[0m Text splitter chunk length of ${prefValue} exceeds embedder model max of ${embedderLimit}. Will use ${embedderLimit}.`
+      );
+    return prefValue > limit ? limit : prefValue;
+  }
+
+  /**
+   *  Creates a string of metadata to be prepended to each chunk.
+   * @param {DocumentMetadata} metadata - Metadata to be prepended to each chunk.
+   * @returns {{[key: ('title' | 'published' | 'source')]: string}} Object of metadata that will be prepended to each chunk.
+   */
+  static buildHeaderMeta(metadata = {}) {
+    if (!metadata || Object.keys(metadata).length === 0) return null;
+    const PLUCK_MAP = {
+      title: {
+        as: "sourceDocument",
+        pluck: (metadata) => {
+          return metadata?.title || null;
+        },
+      },
+      published: {
+        as: "published",
+        pluck: (metadata) => {
+          return metadata?.published || null;
+        },
+      },
+      chunkSource: {
+        as: "source",
+        pluck: (metadata) => {
+          const validPrefixes = ["link://", "youtube://"];
+          // If the chunkSource is a link or youtube link, we can add the URL
+          // as its source in the metadata so the LLM can use it for context.
+          // eg prompt: Where did you get this information? -> answer: "from https://example.com"
+          if (
+            !metadata?.chunkSource || // Exists
+            !metadata?.chunkSource.length || // Is not empty
+            typeof metadata.chunkSource !== "string" || // Is a string
+            !validPrefixes.some(
+              (prefix) => metadata.chunkSource.startsWith(prefix) // Has a valid prefix we respect
+            )
+          )
+            return null;
+
+          // We know a prefix is present, so we can split on it and return the rest.
+          // If nothing is found, return null and it will not be added to the metadata.
+          let source = null;
+          for (const prefix of validPrefixes) {
+            source = metadata.chunkSource.split(prefix)?.[1] || null;
+            if (source) break;
+          }
+
+          return source;
+        },
+      },
+    };
+
+    const pluckedData = {};
+    Object.entries(PLUCK_MAP).forEach(([key, value]) => {
+      if (!(key in metadata)) return; // Skip if the metadata key is not present.
+      const pluckedValue = value.pluck(metadata);
+      if (!pluckedValue) return; // Skip if the plucked value is null/empty.
+      pluckedData[value.as] = pluckedValue;
+    });
+
+    return pluckedData;
+  }
+
+  /**
+   * Apply the chunk prefix to the text if it is present.
+   * @param {string} text - The text to apply the prefix to.
+   * @returns {string} The text with the embedder model prefix applied.
+   */
+  #applyPrefix(text = "") {
+    if (!this.config.chunkPrefix) return text;
+    return `${this.config.chunkPrefix}${text}`;
+  }
+
+  /**
+   * Creates a string of metadata to be prepended to each chunk.
+   * Will additionally prepend a prefix to the text if it was provided (requirement for some embedders).
+   * @returns {string} The text with the embedder model prefix applied.
+   */
+  stringifyHeader() {
+    let content = "";
+    if (!this.config.chunkHeaderMeta) return this.#applyPrefix(content);
+    Object.entries(this.config.chunkHeaderMeta).map(([key, value]) => {
+      if (!key || !value) return;
+      content += `${key}: ${value}\n`;
+    });
+
+    if (!content) return this.#applyPrefix(content);
+    return this.#applyPrefix(
+      `<document_metadata>\n${content}</document_metadata>\n\n`
+    );
+  }
+
+  /**
+   * Sets the splitter to use a defined config passes to other subclasses.
+   * @param {Object} config
+   * @param {string} [config.chunkPrefix = ""] - Prefix to be added to the start of each chunk.
+   * @param {number} [config.chunkSize = 1000] - The size of each chunk.
+   * @param {number} [config.chunkOverlap = 20] - The overlap between chunks.
+   */
+  #setSplitter(config = {}) {
+    // if (!config?.splitByFilename) {// TODO do something when specific extension is present? }
+    return new RecursiveSplitter({
+      chunkSize: isNaN(config?.chunkSize) ? 1_000 : Number(config?.chunkSize),
+      chunkOverlap: isNaN(config?.chunkOverlap)
+        ? 20
+        : Number(config?.chunkOverlap),
+      chunkHeader: this.stringifyHeader(),
+    });
+  }
+
+  async splitText(documentText) {
+    return this.#splitter._splitText(documentText);
+  }
+}
+
+// Wrapper for Langchain default RecursiveCharacterTextSplitter class.
+class RecursiveSplitter {
+  constructor({ chunkSize, chunkOverlap, chunkHeader = null }) {
+    const {
+      RecursiveCharacterTextSplitter,
+    } = require("@langchain/textsplitters");
+    this.log(`Will split with`, {
+      chunkSize,
+      chunkOverlap,
+      chunkHeader: chunkHeader ? `${chunkHeader?.slice(0, 50)}...` : null,
+    });
+    this.chunkHeader = chunkHeader;
+    this.engine = new RecursiveCharacterTextSplitter({
+      chunkSize,
+      chunkOverlap,
+    });
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[35m[RecursiveSplitter]\x1b[0m ${text}`, ...args);
+  }
+
+  async _splitText(documentText) {
+    if (!this.chunkHeader) return this.engine.splitText(documentText);
+    const strings = await this.engine.splitText(documentText);
+    const documents = await this.engine.createDocuments(strings, [], {
+      chunkHeader: this.chunkHeader,
+    });
+    return documents
+      .filter((doc) => !!doc.pageContent)
+      .map((doc) => doc.pageContent);
+  }
+}
+
+module.exports.TextSplitter = TextSplitter;

package/server/utils/TextToSpeech/elevenLabs/index.js

@@ -0,0 +1,54 @@
+const { ElevenLabsClient } = require("elevenlabs");
+
+class ElevenLabsTTS {
+  constructor() {
+    if (!process.env.TTS_ELEVEN_LABS_KEY)
+      throw new Error("No ElevenLabs API key was set.");
+    this.elevenLabs = new ElevenLabsClient({
+      apiKey: process.env.TTS_ELEVEN_LABS_KEY,
+    });
+
+    // Rachel as default voice
+    // https://api.elevenlabs.io/v1/voices
+    this.voiceId =
+      process.env.TTS_ELEVEN_LABS_VOICE_MODEL ?? "21m00Tcm4TlvDq8ikWAM";
+    this.modelId = "eleven_multilingual_v2";
+  }
+
+  static async voices(apiKey = null) {
+    try {
+      const client = new ElevenLabsClient({
+        apiKey: apiKey ?? process.env.TTS_ELEVEN_LABS_KEY ?? null,
+      });
+      return (await client.voices.getAll())?.voices ?? [];
+    } catch {}
+    return [];
+  }
+
+  #stream2buffer(stream) {
+    return new Promise((resolve, reject) => {
+      const _buf = [];
+      stream.on("data", (chunk) => _buf.push(chunk));
+      stream.on("end", () => resolve(Buffer.concat(_buf)));
+      stream.on("error", (err) => reject(err));
+    });
+  }
+
+  async ttsBuffer(textInput) {
+    try {
+      const audio = await this.elevenLabs.generate({
+        voice: this.voiceId,
+        text: textInput,
+        model_id: "eleven_multilingual_v2",
+      });
+      return Buffer.from(await this.#stream2buffer(audio));
+    } catch (e) {
+      console.error(e);
+    }
+    return null;
+  }
+}
+
+module.exports = {
+  ElevenLabsTTS,
+};

package/server/utils/TextToSpeech/index.js

@@ -0,0 +1,18 @@
+function getTTSProvider() {
+  const provider = process.env.TTS_PROVIDER || "openai";
+  switch (provider) {
+    case "openai":
+      const { OpenAiTTS } = require("./openAi");
+      return new OpenAiTTS();
+    case "elevenlabs":
+      const { ElevenLabsTTS } = require("./elevenLabs");
+      return new ElevenLabsTTS();
+    case "generic-openai":
+      const { GenericOpenAiTTS } = require("./openAiGeneric");
+      return new GenericOpenAiTTS();
+    default:
+      throw new Error("ENV: No TTS_PROVIDER value found in environment!");
+  }
+}
+
+module.exports = { getTTSProvider };

package/server/utils/TextToSpeech/openAi/index.js

@@ -0,0 +1,29 @@
+class OpenAiTTS {
+  constructor() {
+    if (!process.env.TTS_OPEN_AI_KEY)
+      throw new Error("No OpenAI API key was set.");
+    const { OpenAI: OpenAIApi } = require("openai");
+    this.openai = new OpenAIApi({
+      apiKey: process.env.TTS_OPEN_AI_KEY,
+    });
+    this.voice = process.env.TTS_OPEN_AI_VOICE_MODEL ?? "alloy";
+  }
+
+  async ttsBuffer(textInput) {
+    try {
+      const result = await this.openai.audio.speech.create({
+        model: "tts-1",
+        voice: this.voice,
+        input: textInput,
+      });
+      return Buffer.from(await result.arrayBuffer());
+    } catch (e) {
+      console.error(e);
+    }
+    return null;
+  }
+}
+
+module.exports = {
+  OpenAiTTS,
+};

package/server/utils/TextToSpeech/openAiGeneric/index.js

@@ -0,0 +1,58 @@
+class GenericOpenAiTTS {
+  constructor() {
+    if (!process.env.TTS_OPEN_AI_COMPATIBLE_KEY)
+      this.#log(
+        "No OpenAI compatible API key was set. You might need to set this to use your OpenAI compatible TTS service."
+      );
+    if (!process.env.TTS_OPEN_AI_COMPATIBLE_MODEL)
+      this.#log(
+        "No OpenAI compatible TTS model was set. We will use the default voice model 'tts-1'. This may not exist or be valid your selected endpoint."
+      );
+    if (!process.env.TTS_OPEN_AI_COMPATIBLE_VOICE_MODEL)
+      this.#log(
+        "No OpenAI compatible voice model was set. We will use the default voice model 'alloy'. This may not exist for your selected endpoint."
+      );
+    if (!process.env.TTS_OPEN_AI_COMPATIBLE_ENDPOINT)
+      throw new Error(
+        "No OpenAI compatible endpoint was set. Please set this to use your OpenAI compatible TTS service."
+      );
+
+    const { OpenAI: OpenAIApi } = require("openai");
+    this.openai = new OpenAIApi({
+      apiKey: process.env.TTS_OPEN_AI_COMPATIBLE_KEY || null,
+      baseURL: process.env.TTS_OPEN_AI_COMPATIBLE_ENDPOINT,
+    });
+    this.model = process.env.TTS_OPEN_AI_COMPATIBLE_MODEL ?? "tts-1";
+    this.voice = process.env.TTS_OPEN_AI_COMPATIBLE_VOICE_MODEL ?? "alloy";
+    this.#log(
+      `Service (${process.env.TTS_OPEN_AI_COMPATIBLE_ENDPOINT}) with model: ${this.model} and voice: ${this.voice}`
+    );
+  }
+
+  #log(text, ...args) {
+    console.log(`\x1b[32m[OpenAiGenericTTS]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   * Generates a buffer from the given text input using the OpenAI compatible TTS service.
+   * @param {string} textInput - The text to be converted to audio.
+   * @returns {Promise<Buffer>} A buffer containing the audio data.
+   */
+  async ttsBuffer(textInput) {
+    try {
+      const result = await this.openai.audio.speech.create({
+        model: this.model,
+        voice: this.voice,
+        input: textInput,
+      });
+      return Buffer.from(await result.arrayBuffer());
+    } catch (e) {
+      console.error(e);
+    }
+    return null;
+  }
+}
+
+module.exports = {
+  GenericOpenAiTTS,
+};

package/server/utils/agentFlows/executor.js

@@ -0,0 +1,233 @@
+const { FLOW_TYPES } = require("./flowTypes");
+const executeApiCall = require("./executors/api-call");
+const executeLLMInstruction = require("./executors/llm-instruction");
+const executeWebScraping = require("./executors/web-scraping");
+const { safeJsonParse } = require("../http");
+
+class FlowExecutor {
+  constructor() {
+    this.variables = {};
+    this.introspect = (...args) => console.log("[introspect] ", ...args);
+    this.logger = console.info;
+    this.aibitat = null;
+  }
+
+  attachLogging(introspectFn = null, loggerFn = null) {
+    this.introspect =
+      introspectFn || ((...args) => console.log("[introspect] ", ...args));
+    this.logger = loggerFn || console.info;
+  }
+
+  /**
+   * Resolves nested values from objects using dot notation and array indices
+   * Supports paths like "data.items[0].name" or "response.users[2].address.city"
+   * Returns undefined for invalid paths or errors
+   * @param {Object|string} obj - The object to resolve the value from
+   * @param {string} path - The path to the value
+   * @returns {string} The resolved value
+   */
+  getValueFromPath(obj = {}, path = "") {
+    if (typeof obj === "string") obj = safeJsonParse(obj, {});
+
+    if (
+      !obj ||
+      !path ||
+      typeof obj !== "object" ||
+      Object.keys(obj).length === 0 ||
+      typeof path !== "string"
+    )
+      return "";
+
+    // First split by dots that are not inside brackets
+    const parts = [];
+    let currentPart = "";
+    let inBrackets = false;
+
+    for (let i = 0; i < path.length; i++) {
+      const char = path[i];
+      if (char === "[") {
+        inBrackets = true;
+        if (currentPart) {
+          parts.push(currentPart);
+          currentPart = "";
+        }
+        currentPart += char;
+      } else if (char === "]") {
+        inBrackets = false;
+        currentPart += char;
+        parts.push(currentPart);
+        currentPart = "";
+      } else if (char === "." && !inBrackets) {
+        if (currentPart) {
+          parts.push(currentPart);
+          currentPart = "";
+        }
+      } else {
+        currentPart += char;
+      }
+    }
+
+    if (currentPart) parts.push(currentPart);
+    let current = obj;
+
+    for (const part of parts) {
+      if (current === null || typeof current !== "object") return undefined;
+
+      // Handle bracket notation
+      if (part.startsWith("[") && part.endsWith("]")) {
+        const key = part.slice(1, -1);
+        const cleanKey = key.replace(/^['"]|['"]$/g, "");
+
+        if (!isNaN(cleanKey)) {
+          if (!Array.isArray(current)) return undefined;
+          current = current[parseInt(cleanKey)];
+        } else {
+          if (!(cleanKey in current)) return undefined;
+          current = current[cleanKey];
+        }
+      } else {
+        // Handle dot notation
+        if (!(part in current)) return undefined;
+        current = current[part];
+      }
+
+      if (current === undefined || current === null) return undefined;
+    }
+
+    return typeof current === "object" ? JSON.stringify(current) : current;
+  }
+
+  /**
+   * Replaces variables in the config with their values
+   * @param {Object} config - The config to replace variables in
+   * @returns {Object} The config with variables replaced
+   */
+  replaceVariables(config) {
+    const deepReplace = (obj) => {
+      if (typeof obj === "string") {
+        return obj.replace(/\${([^}]+)}/g, (match, varName) => {
+          const value = this.getValueFromPath(this.variables, varName);
+          return value !== undefined ? value : match;
+        });
+      }
+
+      if (Array.isArray(obj)) return obj.map((item) => deepReplace(item));
+
+      if (obj && typeof obj === "object") {
+        const result = {};
+        for (const [key, value] of Object.entries(obj)) {
+          result[key] = deepReplace(value);
+        }
+        return result;
+      }
+      return obj;
+    };
+
+    return deepReplace(config);
+  }
+
+  /**
+   * Executes a single step of the flow
+   * @param {Object} step - The step to execute
+   * @returns {Promise<Object>} The result of the step
+   */
+  async executeStep(step) {
+    const config = this.replaceVariables(step.config);
+    let result;
+    // Create execution context with introspect
+    const context = {
+      introspect: this.introspect,
+      variables: this.variables,
+      logger: this.logger,
+      aibitat: this.aibitat,
+    };
+
+    switch (step.type) {
+      case FLOW_TYPES.START.type:
+        // For start blocks, we just initialize variables if they're not already set
+        if (config.variables) {
+          config.variables.forEach((v) => {
+            if (v.name && !this.variables[v.name]) {
+              this.variables[v.name] = v.value || "";
+            }
+          });
+        }
+        result = this.variables;
+        break;
+      case FLOW_TYPES.API_CALL.type:
+        result = await executeApiCall(config, context);
+        break;
+      case FLOW_TYPES.LLM_INSTRUCTION.type:
+        result = await executeLLMInstruction(config, context);
+        break;
+      case FLOW_TYPES.WEB_SCRAPING.type:
+        result = await executeWebScraping(config, context);
+        break;
+      default:
+        throw new Error(`Unknown flow type: ${step.type}`);
+    }
+
+    // Store result in variable if specified
+    if (config.resultVariable || config.responseVariable) {
+      const varName = config.resultVariable || config.responseVariable;
+      this.variables[varName] = result;
+    }
+
+    // If directOutput is true, mark this result for direct output
+    if (config.directOutput) result = { directOutput: true, result };
+    return result;
+  }
+
+  /**
+   * Execute entire flow
+   * @param {Object} flow - The flow to execute
+   * @param {Object} initialVariables - Initial variables for the flow
+   * @param {Object} aibitat - The aibitat instance from the agent handler
+   */
+  async executeFlow(flow, initialVariables = {}, aibitat) {
+
+    // Initialize variables with both initial values and any passed-in values
+    this.variables = {
+      ...(
+        flow.config.steps.find((s) => s.type === "start")?.config?.variables ||
+        []
+      ).reduce((acc, v) => ({ ...acc, [v.name]: v.value }), {}),
+      ...initialVariables, // This will override any default values with passed-in values
+    };
+
+    this.aibitat = aibitat;
+    this.attachLogging(aibitat?.introspect, aibitat?.handlerProps?.log);
+    const results = [];
+    let directOutputResult = null;
+
+    for (const step of flow.config.steps) {
+      try {
+        const result = await this.executeStep(step);
+
+        // If the step has directOutput, stop processing and return the result
+        // so that no other steps are executed or processed
+        if (result?.directOutput) {
+          directOutputResult = result.result;
+          break;
+        }
+
+        results.push({ success: true, result });
+      } catch (error) {
+        results.push({ success: false, error: error.message });
+        break;
+      }
+    }
+
+    return {
+      success: results.every((r) => r.success),
+      results,
+      variables: this.variables,
+      directOutput: directOutputResult,
+    };
+  }
+}
+
+module.exports = {
+  FlowExecutor,
+  FLOW_TYPES,
+};

package/server/utils/agentFlows/executors/api-call.js

@@ -0,0 +1,60 @@
+const { safeJsonParse } = require("../../http");
+
+/**
+ * Execute an API call flow step
+ * @param {Object} config Flow step configuration
+ * @param {Object} context Execution context with introspect function
+ * @returns {Promise<string>} Response data
+ */
+async function executeApiCall(config, context) {
+  const { url, method, headers = [], body, bodyType, formData } = config;
+  const { introspect, logger } = context;
+  logger(`\x1b[43m[AgentFlowToolExecutor]\x1b[0m - executing API Call block`);
+  introspect(`Making ${method} request to external API...`);
+
+  const requestConfig = {
+    method,
+    headers: headers.reduce((acc, h) => ({ ...acc, [h.key]: h.value }), {}),
+  };
+
+  if (["POST", "PUT", "PATCH"].includes(method)) {
+    if (bodyType === "form") {
+      const formDataObj = new URLSearchParams();
+      formData.forEach(({ key, value }) => formDataObj.append(key, value));
+      requestConfig.body = formDataObj.toString();
+      requestConfig.headers["Content-Type"] =
+        "application/x-www-form-urlencoded";
+    } else if (bodyType === "json") {
+      const parsedBody = safeJsonParse(body, null);
+      if (parsedBody !== null) {
+        requestConfig.body = JSON.stringify(parsedBody);
+      }
+      requestConfig.headers["Content-Type"] = "application/json";
+    } else if (bodyType === "text") {
+      requestConfig.body = String(body);
+    } else {
+      requestConfig.body = body;
+    }
+  }
+
+  try {
+    introspect(`Sending body to ${url}: ${requestConfig?.body || "No body"}`);
+    const response = await fetch(url, requestConfig);
+    if (!response.ok) {
+      introspect(`Request failed with status ${response.status}`);
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    introspect(`API call completed`);
+    return await response
+      .text()
+      .then((text) =>
+        safeJsonParse(text, "Failed to parse output from API call block")
+      );
+  } catch (error) {
+    console.error(error);
+    throw new Error(`API Call failed: ${error.message}`);
+  }
+}
+
+module.exports = executeApiCall;