totem-llm 0.1.0

package/server/jobs/sync-watched-documents.js

@@ -0,0 +1,209 @@
+const { Document } = require("../models/documents.js");
+const { DocumentSyncQueue } = require("../models/documentSyncQueue.js");
+const { CollectorApi } = require("../utils/collectorApi");
+const { fileData } = require("../utils/files");
+const { log, conclude, updateSourceDocument } = require("./helpers/index.js");
+const { getVectorDbClass } = require("../utils/helpers/index.js");
+const { DocumentSyncRun } = require("../models/documentSyncRun.js");
+
+(async () => {
+  try {
+    const queuesToProcess = await DocumentSyncQueue.staleDocumentQueues();
+    if (queuesToProcess.length === 0) {
+      log("No outstanding documents to sync. Exiting.");
+      return;
+    }
+
+    const collector = new CollectorApi();
+    if (!(await collector.online())) {
+      log("Could not reach collector API. Exiting.");
+      return;
+    }
+
+    log(
+      `${queuesToProcess.length} watched documents have been found to be stale and will be updated now.`
+    );
+    for (const queue of queuesToProcess) {
+      let newContent = null;
+      const document = queue.workspaceDoc;
+      const workspace = document.workspace;
+      const { metadata, type, source } =
+        Document.parseDocumentTypeAndSource(document);
+
+      if (!metadata || !DocumentSyncQueue.validFileTypes.includes(type)) {
+        // Document is either broken, invalid, or not supported so drop it from future queues.
+        log(
+          `Document ${document.filename} has no metadata, is broken, or invalid and has been removed from all future runs.`
+        );
+        await DocumentSyncQueue.unwatch(document);
+        continue;
+      }
+
+      if (["link", "youtube"].includes(type)) {
+        const response = await collector.forwardExtensionRequest({
+          endpoint: "/ext/resync-source-document",
+          method: "POST",
+          body: JSON.stringify({
+            type,
+            options: { link: source },
+          }),
+        });
+        newContent = response?.content;
+      }
+
+      if (["confluence", "github", "gitlab", "drupalwiki"].includes(type)) {
+        const response = await collector.forwardExtensionRequest({
+          endpoint: "/ext/resync-source-document",
+          method: "POST",
+          body: JSON.stringify({
+            type,
+            options: { chunkSource: metadata.chunkSource },
+          }),
+        });
+        newContent = response?.content;
+      }
+
+      if (!newContent) {
+        // Check if the last "x" runs were all failures (not exits!). If so - remove the job entirely since it is broken.
+        const failedRunCount = (
+          await DocumentSyncRun.where(
+            { queueId: queue.id },
+            DocumentSyncQueue.maxRepeatFailures,
+            { createdAt: "desc" }
+          )
+        ).filter(
+          (run) => run.status === DocumentSyncRun.statuses.failed
+        ).length;
+        if (failedRunCount >= DocumentSyncQueue.maxRepeatFailures) {
+          log(
+            `Document ${document.filename} has failed to refresh ${failedRunCount} times continuously and will now be removed from the watched document set.`
+          );
+          await DocumentSyncQueue.unwatch(document);
+          continue;
+        }
+
+        log(
+          `Failed to get a new content response from collector for source ${source}. Skipping, but will retry next worker interval. Attempt ${failedRunCount === 0 ? 1 : failedRunCount}/${DocumentSyncQueue.maxRepeatFailures}`
+        );
+        await DocumentSyncQueue.saveRun(
+          queue.id,
+          DocumentSyncRun.statuses.failed,
+          {
+            filename: document.filename,
+            workspacesModified: [],
+            reason: "No content found.",
+          }
+        );
+        continue;
+      }
+
+      const currentDocumentData = await fileData(document.docpath);
+      if (currentDocumentData.pageContent === newContent) {
+        const nextSync = DocumentSyncQueue.calcNextSync(queue);
+        log(
+          `Source ${source} is unchanged and will be skipped. Next sync will be ${nextSync.toLocaleString()}.`
+        );
+        await DocumentSyncQueue._update(queue.id, {
+          lastSyncedAt: new Date().toISOString(),
+          nextSyncAt: nextSync.toISOString(),
+        });
+        await DocumentSyncQueue.saveRun(
+          queue.id,
+          DocumentSyncRun.statuses.exited,
+          {
+            filename: document.filename,
+            workspacesModified: [],
+            reason: "Content unchanged.",
+          }
+        );
+        continue;
+      }
+
+      // update the defined document and workspace vectorDB with the latest information
+      // it will skip cache and create a new vectorCache file.
+      const vectorDatabase = getVectorDbClass();
+      await vectorDatabase.deleteDocumentFromNamespace(
+        workspace.slug,
+        document.docId
+      );
+      await vectorDatabase.addDocumentToNamespace(
+        workspace.slug,
+        {
+          ...currentDocumentData,
+          pageContent: newContent,
+          docId: document.docId,
+        },
+        document.docpath,
+        true
+      );
+      updateSourceDocument(document.docpath, {
+        ...currentDocumentData,
+        pageContent: newContent,
+        docId: document.docId,
+        published: new Date().toLocaleString(),
+        // Todo: Update word count and token_estimate?
+      });
+      log(
+        `Workspace "${workspace.name}" vectors of ${source} updated. Document and vector cache updated.`
+      );
+
+      // Now we can bloom the results to all matching documents in all other workspaces
+      const workspacesModified = [workspace.slug];
+      const moreReferences = await Document.where(
+        {
+          id: { not: document.id },
+          filename: document.filename,
+        },
+        null,
+        null,
+        { workspace: true }
+      );
+
+      if (moreReferences.length !== 0) {
+        log(
+          `${source} is referenced in ${moreReferences.length} other workspaces. Updating those workspaces as well...`
+        );
+        for (const additionalDocumentRef of moreReferences) {
+          const additionalWorkspace = additionalDocumentRef.workspace;
+          workspacesModified.push(additionalWorkspace.slug);
+
+          await vectorDatabase.deleteDocumentFromNamespace(
+            additionalWorkspace.slug,
+            additionalDocumentRef.docId
+          );
+          await vectorDatabase.addDocumentToNamespace(
+            additionalWorkspace.slug,
+            {
+              ...currentDocumentData,
+              pageContent: newContent,
+              docId: additionalDocumentRef.docId,
+            },
+            additionalDocumentRef.docpath
+          );
+          log(
+            `Workspace "${additionalWorkspace.name}" vectors for ${source} was also updated with the new content from cache.`
+          );
+        }
+      }
+
+      const nextRefresh = DocumentSyncQueue.calcNextSync(queue);
+      log(
+        `${source} has been refreshed in all workspaces it is currently referenced in. Next refresh will be ${nextRefresh.toLocaleString()}.`
+      );
+      await DocumentSyncQueue._update(queue.id, {
+        lastSyncedAt: new Date().toISOString(),
+        nextSyncAt: nextRefresh.toISOString(),
+      });
+      await DocumentSyncQueue.saveRun(
+        queue.id,
+        DocumentSyncRun.statuses.success,
+        { filename: document.filename, workspacesModified }
+      );
+    }
+  } catch (e) {
+    console.error(e);
+    log(`errored with ${e.message}`);
+  } finally {
+    conclude();
+  }
+})();

package/server/jsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "target": "ES2020"
+  },
+  "include": [
+    "./endpoints/**/*",
+    "./models/**/*",
+    "./utils/**/*",
+    "./swagger/**/*",
+    "index.js"
+  ],
+  "exclude": ["node_modules", "storage"]
+}

package/server/middleware/httpLogger.js

@@ -0,0 +1,23 @@
+const httpLogger =
+  ({ enableTimestamps = false }) =>
+  (req, res, next) => {
+    // Capture the original res.end to log response status
+    const originalEnd = res.end;
+
+    res.end = function (chunk, encoding) {
+      // Log the request method, status code, and path
+      const statusColor = res.statusCode >= 400 ? "\x1b[31m" : "\x1b[32m"; // Red for errors, green for success
+      console.log(
+        `\x1b[32m[HTTP]\x1b[0m ${statusColor}${res.statusCode}\x1b[0m ${req.method} -> ${req.path} ${enableTimestamps ? `@ ${new Date().toLocaleTimeString("en-US", { hour12: true })}` : ""}`.trim()
+      );
+
+      // Call the original end method
+      return originalEnd.call(this, chunk, encoding);
+    };
+
+    next();
+  };
+
+module.exports = {
+  httpLogger,
+};

package/server/models/agentSkillWhitelist.js

@@ -0,0 +1,100 @@
+const prisma = require("../utils/prisma");
+const { safeJsonParse } = require("../utils/http");
+
+const AgentSkillWhitelist = {
+  SINGLE_USER_LABEL: "whitelisted_agent_skills",
+
+  /**
+   * Get the label for storing whitelist in system_settings
+   * @param {number|null} userId - User ID in multi-user mode, null for single-user
+   * @returns {string}
+   */
+  _getLabel: function (userId = null) {
+    if (userId) return `user_${userId}_whitelisted_agent_skills`;
+    return this.SINGLE_USER_LABEL;
+  },
+
+  /**
+   * Get the whitelisted skills for a user or the system
+   * @param {number|null} userId - User ID in multi-user mode, null for single-user
+   * @returns {Promise<string[]>} Array of whitelisted skill names
+   */
+  get: async function (userId = null) {
+    try {
+      const label = this._getLabel(userId);
+      const setting = await prisma.system_settings.findFirst({
+        where: { label },
+      });
+      return safeJsonParse(setting?.value, []);
+    } catch (error) {
+      console.error("AgentSkillWhitelist.get error:", error.message);
+      return [];
+    }
+  },
+
+  /**
+   * Add a skill to the whitelist
+   * @param {string} skillName - The skill name to whitelist
+   * @param {number|null} userId - User ID in multi-user mode, null for single-user
+   * @returns {Promise<{success: boolean, error: string|null}>}
+   */
+  add: async function (skillName, userId = null) {
+    try {
+      if (!skillName || typeof skillName !== "string") {
+        return { success: false, error: "Invalid skill name" };
+      }
+
+      const label = this._getLabel(userId);
+      const currentList = await this.get(userId);
+
+      if (currentList.includes(skillName)) {
+        return { success: true, error: null };
+      }
+
+      const newList = [...currentList, skillName];
+
+      await prisma.system_settings.upsert({
+        where: { label },
+        update: { value: JSON.stringify(newList) },
+        create: { label, value: JSON.stringify(newList) },
+      });
+
+      return { success: true, error: null };
+    } catch (error) {
+      console.error("AgentSkillWhitelist.add error:", error.message);
+      return { success: false, error: error.message };
+    }
+  },
+
+  /**
+   * Check if a skill is whitelisted
+   * @param {string} skillName - The skill name to check
+   * @param {number|null} userId - User ID in multi-user mode, null for single-user
+   * @returns {Promise<boolean>}
+   */
+  isWhitelisted: async function (skillName, userId = null) {
+    const whitelist = await this.get(userId);
+    return whitelist.includes(skillName);
+  },
+
+  /**
+   * Clear the single-user whitelist (used when switching to multi-user mode)
+   * @returns {Promise<{success: boolean, error: string|null}>}
+   */
+  clearSingleUserWhitelist: async function () {
+    try {
+      await prisma.system_settings.deleteMany({
+        where: { label: this.SINGLE_USER_LABEL },
+      });
+      return { success: true, error: null };
+    } catch (error) {
+      console.error(
+        "AgentSkillWhitelist.clearSingleUserWhitelist error:",
+        error.message
+      );
+      return { success: false, error: error.message };
+    }
+  },
+};
+
+module.exports = { AgentSkillWhitelist };

package/server/models/apiKeys.js

@@ -0,0 +1,99 @@
+const prisma = require("../utils/prisma");
+
+const ApiKey = {
+  tablename: "api_keys",
+  writable: ["name"],
+
+  makeSecret: () => {
+    const uuidAPIKey = require("uuid-apikey");
+    return uuidAPIKey.create().apiKey;
+  },
+
+  create: async function (createdByUserId = null, name = null) {
+    try {
+      const normalizedName =
+        typeof name === "string" && name.trim().length > 0 ? name.trim() : null;
+      const apiKey = await prisma.api_keys.create({
+        data: {
+          name: normalizedName,
+          secret: this.makeSecret(),
+          createdBy: createdByUserId,
+        },
+      });
+
+      return { apiKey, error: null };
+    } catch (error) {
+      console.error("FAILED TO CREATE API KEY.", error.message);
+      return { apiKey: null, error: error.message };
+    }
+  },
+
+  get: async function (clause = {}) {
+    try {
+      const apiKey = await prisma.api_keys.findFirst({ where: clause });
+      return apiKey;
+    } catch (error) {
+      console.error("FAILED TO GET API KEY.", error.message);
+      return null;
+    }
+  },
+
+  count: async function (clause = {}) {
+    try {
+      const count = await prisma.api_keys.count({ where: clause });
+      return count;
+    } catch (error) {
+      console.error("FAILED TO COUNT API KEYS.", error.message);
+      return 0;
+    }
+  },
+
+  delete: async function (clause = {}) {
+    try {
+      await prisma.api_keys.deleteMany({ where: clause });
+      return true;
+    } catch (error) {
+      console.error("FAILED TO DELETE API KEY.", error.message);
+      return false;
+    }
+  },
+
+  where: async function (clause = {}, limit) {
+    try {
+      const apiKeys = await prisma.api_keys.findMany({
+        where: clause,
+        take: limit,
+      });
+      return apiKeys;
+    } catch (error) {
+      console.error("FAILED TO GET API KEYS.", error.message);
+      return [];
+    }
+  },
+
+  whereWithUser: async function (clause = {}, limit) {
+    try {
+      const { User } = require("./user");
+      const apiKeys = await this.where(clause, limit);
+
+      for (const apiKey of apiKeys) {
+        if (!apiKey.createdBy) continue;
+        const user = await User.get({ id: apiKey.createdBy });
+        if (!user) continue;
+
+        apiKey.createdBy = {
+          id: user.id,
+          username: user.username,
+          role: user.role,
+        };
+      }
+
+      return apiKeys;
+    } catch (error) {
+      console.error("FAILED TO GET API KEYS WITH USER.", error.message);
+      return [];
+    }
+  },
+};
+
+module.exports = { ApiKey };

package/server/models/browserExtensionApiKey.js

@@ -0,0 +1,190 @@
+const prisma = require("../utils/prisma");
+const { SystemSettings } = require("./systemSettings");
+const { ROLES } = require("../utils/middleware/multiUserProtected");
+
+const BrowserExtensionApiKey = {
+  /**
+   * Creates a new secret for a browser extension API key.
+   * @returns {string} brx-*** API key to use with extension
+   */
+  makeSecret: () => {
+    const uuidAPIKey = require("uuid-apikey");
+    return `brx-${uuidAPIKey.create().apiKey}`;
+  },
+
+  /**
+   * Creates a new api key for the browser Extension
+   * @param {number|null} userId - User id to associate creation of key with.
+   * @returns {Promise<{apiKey: import("@prisma/client").browser_extension_api_keys|null, error:string|null}>}
+   */
+  create: async function (userId = null) {
+    try {
+      const apiKey = await prisma.browser_extension_api_keys.create({
+        data: {
+          key: this.makeSecret(),
+          user_id: userId,
+        },
+      });
+      return { apiKey, error: null };
+    } catch (error) {
+      console.error("Failed to create browser extension API key", error);
+      return { apiKey: null, error: error.message };
+    }
+  },
+
+  /**
+   * Validated existing API key
+   * @param {string} key
+   * @returns {Promise<{apiKey: import("@prisma/client").browser_extension_api_keys|boolean}>}
+   */
+  validate: async function (key) {
+    if (!key.startsWith("brx-")) return false;
+    const apiKey = await prisma.browser_extension_api_keys.findUnique({
+      where: { key: key.toString() },
+      include: { user: true },
+    });
+    if (!apiKey) return false;
+
+    const multiUserMode = await SystemSettings.isMultiUserMode();
+    if (!multiUserMode) return apiKey; // In single-user mode, all keys are valid
+
+    // In multi-user mode, check if the key is associated with a user
+    return apiKey.user_id ? apiKey : false;
+  },
+
+  /**
+   * Fetches browser api key by params.
+   * @param {object} clause - Prisma props for search
+   * @returns {Promise<{apiKey: import("@prisma/client").browser_extension_api_keys|boolean}>}
+   */
+  get: async function (clause = {}) {
+    try {
+      const apiKey = await prisma.browser_extension_api_keys.findFirst({
+        where: clause,
+      });
+      return apiKey;
+    } catch (error) {
+      console.error("FAILED TO GET BROWSER EXTENSION API KEY.", error.message);
+      return null;
+    }
+  },
+
+  /**
+   * Deletes browser api key by db id.
+   * @param {number} id - database id of browser key
+   * @returns {Promise<{success: boolean, error:string|null}>}
+   */
+  delete: async function (id) {
+    try {
+      await prisma.browser_extension_api_keys.delete({
+        where: { id: parseInt(id) },
+      });
+      return { success: true, error: null };
+    } catch (error) {
+      console.error("Failed to delete browser extension API key", error);
+      return { success: false, error: error.message };
+    }
+  },
+
+  /**
+   * Deletes all browser extension API keys for a user.
+   * Should be called when a user is deleted to revoke all their keys.
+   * @param {number} userId - The user ID whose keys should be deleted
+   * @returns {Promise<{success: boolean, error: string|null}>}
+   */
+  deleteAllForUser: async function (userId) {
+    try {
+      if (!userId) return { success: false, error: "User ID is required" };
+      await prisma.browser_extension_api_keys.deleteMany({
+        where: { user_id: parseInt(userId) },
+      });
+      return { success: true, error: null };
+    } catch (error) {
+      console.error(
+        "Failed to delete browser extension API keys for user",
+        error
+      );
+      return { success: false, error: error.message };
+    }
+  },
+
+  /**
+   * Gets browser keys by params
+   * @param {object} clause
+   * @param {number|null} limit
+   * @param {object|null} orderBy
+   * @returns {Promise<import("@prisma/client").browser_extension_api_keys[]>}
+   */
+  where: async function (clause = {}, limit = null, orderBy = null) {
+    try {
+      const apiKeys = await prisma.browser_extension_api_keys.findMany({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+        ...(orderBy !== null ? { orderBy } : {}),
+        include: { user: true },
+      });
+      return apiKeys;
+    } catch (error) {
+      console.error("FAILED TO GET BROWSER EXTENSION API KEYS.", error.message);
+      return [];
+    }
+  },
+
+  /**
+   * Get browser API keys for user
+   * @param {import("@prisma/client").users} user
+   * @param {object} clause
+   * @param {number|null} limit
+   * @param {object|null} orderBy
+   * @returns {Promise<import("@prisma/client").browser_extension_api_keys[]>}
+   */
+  whereWithUser: async function (
+    user,
+    clause = {},
+    limit = null,
+    orderBy = null
+  ) {
+    // Admin can view and use any keys
+    if ([ROLES.admin].includes(user.role))
+      return await this.where(clause, limit, orderBy);
+
+    try {
+      const apiKeys = await prisma.browser_extension_api_keys.findMany({
+        where: {
+          ...clause,
+          user_id: user.id,
+        },
+        include: { user: true },
+        ...(limit !== null ? { take: limit } : {}),
+        ...(orderBy !== null ? { orderBy } : {}),
+      });
+      return apiKeys;
+    } catch (error) {
+      console.error(error.message);
+      return [];
+    }
+  },
+
+  /**
+   * Updates owner of all DB ids to new admin.
+   * @param {number} userId
+   * @returns {Promise<void>}
+   */
+  migrateApiKeysToMultiUser: async function (userId) {
+    try {
+      await prisma.browser_extension_api_keys.updateMany({
+        where: {
+          user_id: null,
+        },
+        data: {
+          user_id: userId,
+        },
+      });
+      console.log("Successfully migrated API keys to multi-user mode");
+    } catch (error) {
+      console.error("Error migrating API keys to multi-user mode:", error);
+    }
+  },
+};
+
+module.exports = { BrowserExtensionApiKey };

package/server/models/cacheData.js

@@ -0,0 +1,69 @@
+const prisma = require("../utils/prisma");
+
+const CacheData = {
+  new: async function (inputs = {}) {
+    try {
+      const cache = await prisma.cache_data.create({
+        data: inputs,
+      });
+      return { cache, message: null };
+    } catch (error) {
+      console.error(error.message);
+      return { cache: null, message: error.message };
+    }
+  },
+
+  get: async function (clause = {}, limit = null, orderBy = null) {
+    try {
+      const cache = await prisma.cache_data.findFirst({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+        ...(orderBy !== null ? { orderBy } : {}),
+      });
+      return cache || null;
+    } catch (error) {
+      console.error(error.message);
+      return null;
+    }
+  },
+
+  delete: async function (clause = {}) {
+    try {
+      await prisma.cache_data.deleteMany({
+        where: clause,
+      });
+      return true;
+    } catch (error) {
+      console.error(error.message);
+      return false;
+    }
+  },
+
+  where: async function (clause = {}, limit = null, orderBy = null) {
+    try {
+      const caches = await prisma.cache_data.findMany({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+        ...(orderBy !== null ? { orderBy } : {}),
+      });
+      return caches;
+    } catch (error) {
+      console.error(error.message);
+      return [];
+    }
+  },
+
+  count: async function (clause = {}) {
+    try {
+      const count = await prisma.cache_data.count({
+        where: clause,
+      });
+      return count;
+    } catch (error) {
+      console.error(error.message);
+      return 0;
+    }
+  },
+};
+
+module.exports = { CacheData };