totem-llm 0.1.0

package/server/utils/helpers/chat/LLMPerformanceMonitor.js

@@ -0,0 +1,113 @@
+const { TokenManager } = require("../tiktoken");
+
+/**
+ * @typedef {import("openai/streaming").Stream<import("openai").OpenAI.ChatCompletionChunk>} OpenAICompatibleStream
+ * @typedef {(reportedUsage: {[key: string]: number, completion_tokens?: number, prompt_tokens?: number}) => StreamMetrics} EndMeasurementFunction
+ * @typedef {Array<{content: string}>} Messages
+ */
+
+/**
+ * @typedef {Object} StreamMetrics
+ * @property {number} prompt_tokens - the number of tokens in the prompt
+ * @property {number} completion_tokens - the number of tokens in the completion
+ * @property {number} total_tokens - the total number of tokens
+ * @property {number} outputTps - the tokens per second of the output
+ * @property {number} duration - the duration of the stream
+ */
+
+/**
+ * @typedef {Object} MonitoredStream
+ * @property {number} start - the start time of the stream
+ * @property {number} duration - the duration of the stream
+ * @property {StreamMetrics} metrics - the metrics of the stream
+ * @property {EndMeasurementFunction} endMeasurement - the method to end the stream and calculate the metrics
+ */
+
+class LLMPerformanceMonitor {
+  static tokenManager = new TokenManager();
+  /**
+   * Counts the tokens in the messages.
+   * @param {Array<{content: string}>} messages - the messages sent to the LLM so we can calculate the prompt tokens since most providers do not return this on stream
+   * @returns {number}
+   */
+  static countTokens(messages = []) {
+    try {
+      return this.tokenManager.statsFrom(messages);
+    } catch {
+      return 0;
+    }
+  }
+  /**
+   * Wraps a function and logs the duration (in seconds) of the function call.
+   * If the output contains a `usage.duration` property, it will be used instead of the calculated duration.
+   * This allows providers to supply more accurate timing information.
+   * @param {Function} func
+   * @returns {Promise<{output: any, duration: number}>}
+   */
+  static measureAsyncFunction(func) {
+    return (async () => {
+      const start = Date.now();
+      const output = await func; // is a promise
+      const end = Date.now();
+      const duration = output?.usage?.duration ?? (end - start) / 1000;
+      return { output, duration };
+    })();
+  }
+
+  /**
+   * Wraps a completion stream and and attaches a start time and duration property to the stream.
+   * Also attaches an `endMeasurement` method to the stream that will calculate the duration of the stream and metrics.
+   * @param {Object} opts
+   * @param {Promise<OpenAICompatibleStream>} opts.func
+   * @param {Messages} [opts.messages=[]] - the messages sent to the LLM so we can calculate the prompt tokens since most providers do not return this on stream
+   * @param {boolean} [opts.runPromptTokenCalculation=true] - whether to run the prompt token calculation to estimate the `prompt_tokens` metric. This is useful for providers that do not return this on stream.
+   * @param {string} [opts.modelTag=""] - the tag of the model that was used to generate the stream (eg: gpt-4o, claude-3-5-sonnet, qwen3/72b-instruct, etc.)
+   * @param {string} [opts.provider=""] - the class name of the LLM that was used to generate the stream (eg: OpenAI, Anthropic, LMStudio, ApiPie, etc.)
+   * @returns {Promise<MonitoredStream>}
+   */
+  static async measureStream({
+    func,
+    messages = [],
+    runPromptTokenCalculation = true,
+    modelTag = "",
+    provider = "",
+  }) {
+    const stream = await func;
+    stream.start = Date.now();
+    stream.duration = 0;
+    stream.metrics = {
+      completion_tokens: 0,
+      prompt_tokens: runPromptTokenCalculation ? this.countTokens(messages) : 0,
+      total_tokens: 0,
+      outputTps: 0,
+      duration: 0,
+      ...(modelTag ? { model: modelTag } : {}),
+      ...(provider ? { provider: provider } : {}),
+    };
+
+    stream.endMeasurement = (reportedUsage = {}) => {
+      const end = Date.now();
+      const estimatedDuration = (end - stream.start) / 1000;
+
+      // Merge the reported usage with the existing metrics
+      // so the math in the metrics object is correct when calculating
+      stream.metrics = {
+        ...stream.metrics,
+        ...reportedUsage,
+        duration: reportedUsage?.duration ?? estimatedDuration,
+        timestamp: new Date(),
+      };
+
+      stream.metrics.total_tokens =
+        stream.metrics.prompt_tokens + (stream.metrics.completion_tokens || 0);
+      stream.metrics.outputTps =
+        stream.metrics.completion_tokens / stream.metrics.duration;
+      return stream.metrics;
+    };
+    return stream;
+  }
+}
+
+module.exports = {
+  LLMPerformanceMonitor,
+};

package/server/utils/helpers/chat/convertTo.js

@@ -0,0 +1,256 @@
+// Helpers that convert workspace chats to some supported format
+// for external use by the user.
+
+const { WorkspaceChats } = require("../../../models/workspaceChats");
+const { EmbedChats } = require("../../../models/embedChats");
+const { safeJsonParse } = require("../../http");
+const { SystemSettings } = require("../../../models/systemSettings");
+
+async function convertToCSV(preparedData) {
+  const headers = new Set(["id", "workspace", "prompt", "response", "sent_at"]);
+  preparedData.forEach((item) =>
+    Object.keys(item).forEach((key) => headers.add(key))
+  );
+
+  const rows = [Array.from(headers).join(",")];
+
+  for (const item of preparedData) {
+    const record = Array.from(headers)
+      .map((header) => {
+        const value = item[header] ?? "";
+        return escapeCsv(String(value));
+      })
+      .join(",");
+    rows.push(record);
+  }
+  return rows.join("\n");
+}
+
+async function convertToJSON(preparedData) {
+  return JSON.stringify(preparedData, null, 4);
+}
+
+// ref: https://raw.githubusercontent.com/gururise/AlpacaDataCleaned/main/alpaca_data.json
+async function convertToJSONAlpaca(preparedData) {
+  return JSON.stringify(preparedData, null, 4);
+}
+
+// You can validate JSONL outputs on https://jsonlines.org/validator/
+async function convertToJSONL(workspaceChatsMap) {
+  return Object.values(workspaceChatsMap)
+    .map((workspaceChats) => JSON.stringify(workspaceChats))
+    .join("\n");
+}
+
+async function prepareChatsForExport(format = "jsonl", chatType = "workspace") {
+  if (!exportMap.hasOwnProperty(format))
+    throw new Error(`Invalid export type: ${format}`);
+
+  let chats;
+  if (chatType === "workspace") {
+    chats = await WorkspaceChats.whereWithData({}, null, null, {
+      id: "asc",
+    });
+  } else if (chatType === "embed") {
+    chats = await EmbedChats.whereWithEmbedAndWorkspace(
+      {},
+      null,
+      {
+        id: "asc",
+      },
+      null
+    );
+  } else {
+    throw new Error(`Invalid chat type: ${chatType}`);
+  }
+
+  if (format === "csv" || format === "json") {
+    const preparedData = chats.map((chat) => {
+      const responseJson = safeJsonParse(chat.response, {});
+      const baseData = {
+        id: chat.id,
+        prompt: chat.prompt,
+        response: responseJson.text,
+        sent_at: chat.createdAt,
+        // Only add attachments to the json format since we cannot arrange attachments in csv format
+        ...(format === "json"
+          ? {
+              attachments:
+                responseJson.attachments?.length > 0
+                  ? responseJson.attachments.map((attachment) => ({
+                      type: "image",
+                      image: attachmentToDataUrl(attachment),
+                    }))
+                  : [],
+            }
+          : {}),
+      };
+
+      if (chatType === "embed") {
+        return {
+          ...baseData,
+          workspace: chat.embed_config
+            ? chat.embed_config.workspace.name
+            : "unknown workspace",
+        };
+      }
+
+      return {
+        ...baseData,
+        workspace: chat.workspace ? chat.workspace.name : "unknown workspace",
+        username: chat.user
+          ? chat.user.username
+          : chat.api_session_id !== null
+            ? "API"
+            : "unknown user",
+        rating:
+          chat.feedbackScore === null
+            ? "--"
+            : chat.feedbackScore
+              ? "GOOD"
+              : "BAD",
+      };
+    });
+
+    return preparedData;
+  }
+
+  // jsonAlpaca format does not support array outputs
+  if (format === "jsonAlpaca") {
+    const preparedData = chats.map((chat) => {
+      const responseJson = safeJsonParse(chat.response, {});
+      return {
+        instruction: buildSystemPrompt(
+          chat,
+          chat.workspace ? chat.workspace.openAiPrompt : null
+        ),
+        input: chat.prompt,
+        output: responseJson.text,
+      };
+    });
+
+    return preparedData;
+  }
+
+  // Export to JSONL format (recommended for fine-tuning)
+  const workspaceChatsMap = chats.reduce((acc, chat) => {
+    const { prompt, response, workspaceId } = chat;
+    const responseJson = safeJsonParse(response, { attachments: [] });
+    const attachments = responseJson.attachments;
+
+    if (!acc[workspaceId]) {
+      acc[workspaceId] = {
+        messages: [
+          {
+            role: "system",
+            content: [
+              {
+                type: "text",
+                text:
+                  chat.workspace?.openAiPrompt ??
+                  SystemSettings.saneDefaultSystemPrompt,
+              },
+            ],
+          },
+        ],
+      };
+    }
+
+    acc[workspaceId].messages.push(
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: prompt,
+          },
+          ...(attachments?.length > 0
+            ? attachments.map((attachment) => ({
+                type: "image",
+                image: attachmentToDataUrl(attachment),
+              }))
+            : []),
+        ],
+      },
+      {
+        role: "assistant",
+        content: [
+          {
+            type: "text",
+            text: responseJson.text,
+          },
+        ],
+      }
+    );
+
+    return acc;
+  }, {});
+
+  return workspaceChatsMap;
+}
+
+const exportMap = {
+  json: {
+    contentType: "application/json",
+    func: convertToJSON,
+  },
+  csv: {
+    contentType: "text/csv",
+    func: convertToCSV,
+  },
+  jsonl: {
+    contentType: "application/jsonl",
+    func: convertToJSONL,
+  },
+  jsonAlpaca: {
+    contentType: "application/json",
+    func: convertToJSONAlpaca,
+  },
+};
+
+function escapeCsv(str) {
+  if (str === null || str === undefined) return '""';
+  return `"${str.replace(/"/g, '""').replace(/\n/g, " ")}"`;
+}
+
+async function exportChatsAsType(format = "jsonl", chatType = "workspace") {
+  const { contentType, func } = exportMap.hasOwnProperty(format)
+    ? exportMap[format]
+    : exportMap.jsonl;
+  const chats = await prepareChatsForExport(format, chatType);
+  return {
+    contentType,
+    data: await func(chats),
+  };
+}
+
+function buildSystemPrompt(chat, prompt = null) {
+  const sources = safeJsonParse(chat.response)?.sources || [];
+  const contextTexts = sources.map((source) => source.text);
+  const context =
+    sources.length > 0
+      ? "\nContext:\n" +
+        contextTexts
+          .map((text, i) => {
+            return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+          })
+          .join("")
+      : "";
+  return `${prompt ?? SystemSettings.saneDefaultSystemPrompt}${context}`;
+}
+
+/**
+ * Converts an attachment's content string to a proper data URL format if needed
+ * @param {Object} attachment - The attachment object containing contentString and mime type
+ * @returns {string} The properly formatted data URL
+ */
+function attachmentToDataUrl(attachment) {
+  return attachment.contentString.startsWith("data:")
+    ? attachment.contentString
+    : `data:${attachment.mime};base64,${attachment.contentString}`;
+}
+
+module.exports = {
+  prepareChatsForExport,
+  exportChatsAsType,
+};