totem-llm 0.1.0

package/server/endpoints/workspacesParsedFiles.js

@@ -0,0 +1,208 @@
+const { reqBody, multiUserMode, userFromSession } = require("../utils/http");
+const { handleFileUpload } = require("../utils/files/multer");
+const { validatedRequest } = require("../utils/middleware/validatedRequest");
+const {
+  flexUserRoleValid,
+  ROLES,
+} = require("../utils/middleware/multiUserProtected");
+const { EventLogs } = require("../models/eventLogs");
+const { validWorkspaceSlug } = require("../utils/middleware/validWorkspace");
+const { CollectorApi } = require("../utils/collectorApi");
+const { WorkspaceThread } = require("../models/workspaceThread");
+const { WorkspaceParsedFiles } = require("../models/workspaceParsedFiles");
+
+function workspaceParsedFilesEndpoints(app) {
+  if (!app) return;
+
+  app.get(
+    "/workspace/:slug/parsed-files",
+    [validatedRequest, flexUserRoleValid([ROLES.all]), validWorkspaceSlug],
+    async (request, response) => {
+      try {
+        const threadSlug = request.query.threadSlug || null;
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+        const thread = threadSlug
+          ? await WorkspaceThread.get({ slug: String(threadSlug) })
+          : null;
+        const { files, contextWindow, currentContextTokenCount } =
+          await WorkspaceParsedFiles.getContextMetadataAndLimits(
+            workspace,
+            thread || null,
+            multiUserMode(response) ? user : null
+          );
+
+        return response
+          .status(200)
+          .json({ files, contextWindow, currentContextTokenCount });
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.delete(
+    "/workspace/:slug/delete-parsed-files",
+    [validatedRequest, flexUserRoleValid([ROLES.all]), validWorkspaceSlug],
+    async function (request, response) {
+      try {
+        const { fileIds = [] } = reqBody(request);
+        if (!fileIds.length) return response.sendStatus(400).end();
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+        const success = await WorkspaceParsedFiles.delete({
+          id: {
+            in: fileIds.map((id) => parseInt(id)),
+          },
+          ...(user ? { userId: user.id } : {}),
+          workspaceId: workspace.id,
+        });
+        return response.status(success ? 200 : 403).end();
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.post(
+    "/workspace/:slug/embed-parsed-file/:fileId",
+    [
+      validatedRequest,
+      // Embed is still an admin/manager only feature
+      flexUserRoleValid([ROLES.admin, ROLES.manager]),
+      validWorkspaceSlug,
+    ],
+    async function (request, response) {
+      const { fileId = null } = request.params;
+      try {
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+
+        if (!fileId) return response.sendStatus(400).end();
+        const { success, error, document } =
+          await WorkspaceParsedFiles.moveToDocumentsAndEmbed(
+            user,
+            fileId,
+            workspace
+          );
+
+        if (!success) {
+          return response.status(500).json({
+            success: false,
+            error: error || "Failed to embed file",
+          });
+        }
+
+        await EventLogs.logEvent(
+          "document_embedded",
+          {
+            documentName: document?.name || "unknown",
+            workspaceId: workspace.id,
+          },
+          user?.id
+        );
+
+        return response.status(200).json({
+          success: true,
+          error: null,
+          document,
+        });
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      } finally {
+        // eslint-disable-next-line
+        if (!fileId) return;
+        await WorkspaceParsedFiles.delete({ id: parseInt(fileId) });
+      }
+    }
+  );
+
+  app.post(
+    "/workspace/:slug/parse",
+    [
+      validatedRequest,
+      flexUserRoleValid([ROLES.all]),
+      handleFileUpload,
+      validWorkspaceSlug,
+    ],
+    async function (request, response) {
+      try {
+        const user = await userFromSession(request, response);
+        const workspace = response.locals.workspace;
+        const Collector = new CollectorApi();
+        const { originalname } = request.file;
+        const processingOnline = await Collector.online();
+
+        if (!processingOnline) {
+          return response.status(500).json({
+            success: false,
+            error: `Document processing API is not online. Document ${originalname} will not be parsed.`,
+          });
+        }
+
+        const { success, reason, documents } =
+          await Collector.parseDocument(originalname);
+        if (!success || !documents?.[0]) {
+          return response.status(500).json({
+            success: false,
+            error: reason || "No document returned from collector",
+          });
+        }
+
+        // Get thread ID if we have a slug
+        const { threadSlug = null } = reqBody(request);
+        const thread = threadSlug
+          ? await WorkspaceThread.get({
+              slug: String(threadSlug),
+              workspace_id: workspace.id,
+              user_id: user?.id || null,
+            })
+          : null;
+        const files = await Promise.all(
+          documents.map(async (doc) => {
+            const metadata = { ...doc };
+            // Strip out pageContent
+            delete metadata.pageContent;
+            const filename = `${originalname}-${doc.id}.json`;
+            const { file, error: dbError } = await WorkspaceParsedFiles.create({
+              filename,
+              workspaceId: workspace.id,
+              userId: user?.id || null,
+              threadId: thread?.id || null,
+              metadata: JSON.stringify(metadata),
+              tokenCountEstimate: doc.token_count_estimate || 0,
+            });
+
+            if (dbError) throw new Error(dbError);
+            return file;
+          })
+        );
+
+        Collector.log(`Document ${originalname} parsed successfully.`);
+        await EventLogs.logEvent(
+          "document_uploaded_to_chat",
+          {
+            documentName: originalname,
+            workspace: workspace.slug,
+            thread: thread?.name || null,
+          },
+          user?.id
+        );
+
+        return response.status(200).json({
+          success: true,
+          error: null,
+          files,
+        });
+      } catch (e) {
+        console.error(e.message, e);
+        return response.sendStatus(500).end();
+      }
+    }
+  );
+}
+
+module.exports = { workspaceParsedFilesEndpoints };

package/server/index.js

@@ -0,0 +1,178 @@
+process.env.NODE_ENV === "development"
+  ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` })
+  : require("dotenv").config();
+
+require("./utils/logger")();
+const express = require("express");
+const bodyParser = require("body-parser");
+const cors = require("cors");
+const path = require("path");
+const { reqBody } = require("./utils/http");
+const { systemEndpoints } = require("./endpoints/system");
+const { workspaceEndpoints } = require("./endpoints/workspaces");
+const { chatEndpoints } = require("./endpoints/chat");
+const { embeddedEndpoints } = require("./endpoints/embed");
+const { embedManagementEndpoints } = require("./endpoints/embedManagement");
+const { getVectorDbClass } = require("./utils/helpers");
+const { adminEndpoints } = require("./endpoints/admin");
+const { modelRouterEndpoints } = require("./endpoints/modelRouter");
+const { inviteEndpoints } = require("./endpoints/invite");
+const { utilEndpoints } = require("./endpoints/utils");
+const { developerEndpoints } = require("./endpoints/api");
+const { extensionEndpoints } = require("./endpoints/extensions");
+const { bootHTTP, bootSSL } = require("./utils/boot");
+const { workspaceThreadEndpoints } = require("./endpoints/workspaceThreads");
+const { documentEndpoints } = require("./endpoints/document");
+const { agentWebsocket } = require("./endpoints/agentWebsocket");
+const {
+  agentSkillWhitelistEndpoints,
+} = require("./endpoints/agentSkillWhitelist");
+const { agentFileServerEndpoints } = require("./endpoints/agentFileServer");
+const { experimentalEndpoints } = require("./endpoints/experimental");
+const { browserExtensionEndpoints } = require("./endpoints/browserExtension");
+const { communityHubEndpoints } = require("./endpoints/communityHub");
+const { agentFlowEndpoints } = require("./endpoints/agentFlows");
+const { mcpServersEndpoints } = require("./endpoints/mcpServers");
+const { mobileEndpoints } = require("./endpoints/mobile");
+const { webPushEndpoints } = require("./endpoints/webPush");
+const { telegramEndpoints } = require("./endpoints/telegram");
+const { scheduledJobEndpoints } = require("./endpoints/scheduledJobs");
+const {
+  outlookAgentEndpoints,
+} = require("./endpoints/utils/outlookAgentUtils");
+const {
+  googleAgentSkillEndpoints,
+} = require("./endpoints/utils/googleAgentSkillEndpoints");
+const { memoryEndpoints } = require("./endpoints/memory");
+const { httpLogger } = require("./middleware/httpLogger");
+const app = express();
+const apiRouter = express.Router();
+const FILE_LIMIT = "3GB";
+
+// Only log HTTP requests in development mode and if the ENABLE_HTTP_LOGGER environment variable is set to true
+if (
+  process.env.NODE_ENV === "development" &&
+  !!process.env.ENABLE_HTTP_LOGGER
+) {
+  app.use(
+    httpLogger({
+      enableTimestamps: !!process.env.ENABLE_HTTP_LOGGER_TIMESTAMPS,
+    })
+  );
+}
+app.use(cors({ origin: true }));
+app.use(bodyParser.text({ limit: FILE_LIMIT }));
+app.use(bodyParser.json({ limit: FILE_LIMIT }));
+app.use(
+  bodyParser.urlencoded({
+    limit: FILE_LIMIT,
+    extended: true,
+  })
+);
+
+if (!!process.env.ENABLE_HTTPS) {
+  bootSSL(app, process.env.SERVER_PORT || 3001);
+} else {
+  require("@mintplex-labs/express-ws").default(app); // load WebSockets in non-SSL mode.
+}
+
+app.use("/api", apiRouter);
+systemEndpoints(apiRouter);
+extensionEndpoints(apiRouter);
+workspaceEndpoints(apiRouter);
+workspaceThreadEndpoints(apiRouter);
+chatEndpoints(apiRouter);
+adminEndpoints(apiRouter);
+modelRouterEndpoints(apiRouter);
+inviteEndpoints(apiRouter);
+embedManagementEndpoints(apiRouter);
+utilEndpoints(apiRouter);
+documentEndpoints(apiRouter);
+agentWebsocket(apiRouter);
+agentSkillWhitelistEndpoints(apiRouter);
+agentFileServerEndpoints(apiRouter);
+experimentalEndpoints(apiRouter);
+developerEndpoints(app, apiRouter);
+communityHubEndpoints(apiRouter);
+agentFlowEndpoints(apiRouter);
+mcpServersEndpoints(apiRouter);
+mobileEndpoints(apiRouter);
+webPushEndpoints(apiRouter);
+telegramEndpoints(apiRouter);
+scheduledJobEndpoints(apiRouter);
+outlookAgentEndpoints(apiRouter);
+googleAgentSkillEndpoints(apiRouter);
+memoryEndpoints(apiRouter);
+// Externally facing embedder endpoints
+embeddedEndpoints(apiRouter);
+
+// Externally facing browser extension endpoints
+browserExtensionEndpoints(apiRouter);
+
+if (process.env.NODE_ENV !== "development") {
+  const { MetaGenerator } = require("./utils/boot/MetaGenerator");
+  const IndexPage = new MetaGenerator();
+
+  app.use(
+    express.static(path.resolve(__dirname, "public"), {
+      extensions: ["js"],
+      setHeaders: (res) => {
+        // Disable I-framing of entire site UI
+        res.removeHeader("X-Powered-By");
+        res.setHeader("X-Frame-Options", "DENY");
+      },
+    })
+  );
+
+  app.get("/robots.txt", function (_, response) {
+    response.type("text/plain");
+    response.send("User-agent: *\nDisallow: /").end();
+  });
+
+  app.get("/manifest.json", async function (_, response) {
+    IndexPage.generateManifest(response);
+    return;
+  });
+
+  app.use("/", function (_, response) {
+    IndexPage.generate(response);
+    return;
+  });
+} else {
+  // Debug route for development connections to vectorDBs
+  apiRouter.post("/v/:command", async (request, response) => {
+    try {
+      const VectorDb = getVectorDbClass();
+      const { command } = request.params;
+      if (!Object.getOwnPropertyNames(VectorDb).includes(command)) {
+        response.status(500).json({
+          message: "invalid interface command",
+          commands: Object.getOwnPropertyNames(VectorDb),
+        });
+        return;
+      }
+
+      try {
+        const body = reqBody(request);
+        const resBody = await VectorDb[command](body);
+        response.status(200).json({ ...resBody });
+      } catch (e) {
+        // console.error(e)
+        console.error(JSON.stringify(e));
+        response.status(500).json({ error: e.message });
+      }
+      return;
+    } catch (e) {
+      console.error(e.message, e);
+      response.sendStatus(500).end();
+    }
+  });
+}
+
+app.all("*", function (_, response) {
+  response.sendStatus(404);
+});
+
+// In non-https mode we need to boot at the end since the server has not yet
+// started and is `.listen`ing.
+if (!process.env.ENABLE_HTTPS) bootHTTP(app, process.env.SERVER_PORT || 3001);

package/server/jobs/cleanup-generated-files.js

@@ -0,0 +1,160 @@
+const { log, conclude } = require("./helpers/index.js");
+const { WorkspaceChats } = require("../models/workspaceChats.js");
+const { ScheduledJobRun } = require("../models/scheduledJobRun.js");
+const createFilesLib = require("../utils/agents/aibitat/plugins/create-files/lib.js");
+const { safeJsonParse } = require("../utils/http/index.js");
+
+(async () => {
+  try {
+    const fs = require("fs");
+    const path = require("path");
+    const storageDirectory = await createFilesLib.getOutputDirectory();
+    if (!fs.existsSync(storageDirectory)) return;
+
+    const files = fs.readdirSync(storageDirectory);
+    if (files.length === 0) return;
+
+    // Get all storage filenames referenced in active (include: true) chats
+    const activeFileRefs = await getActiveStorageFilenames();
+    const filesToDelete = [];
+    for (const filename of files) {
+      const fullPath = path.join(storageDirectory, filename);
+      const stat = fs.statSync(fullPath);
+
+      // Skip files/folders that don't match our naming pattern and add to deletion list
+      if (!filename.match(/^[a-z]+-[a-f0-9-]{36}(\.\w+)?$/i)) {
+        filesToDelete.push({ path: fullPath, isDirectory: stat.isDirectory() });
+        continue;
+      }
+
+      // If file/folder is not referenced in any active chat, add to deletion list
+      if (!activeFileRefs.has(filename))
+        filesToDelete.push({ path: fullPath, isDirectory: stat.isDirectory() });
+    }
+
+    if (filesToDelete.length === 0) return;
+
+    log(`Found ${filesToDelete.length} orphaned files/folders to delete.`);
+    let deletedCount = 0;
+    let failedCount = 0;
+    for (const { path: itemPath, isDirectory } of filesToDelete) {
+      try {
+        if (isDirectory) fs.rmSync(itemPath, { recursive: true });
+        else fs.unlinkSync(itemPath);
+        deletedCount++;
+      } catch (error) {
+        failedCount++;
+        log(`Failed to delete ${itemPath}: ${error.message}`);
+      }
+    }
+
+    log(
+      `Cleanup complete: deleted ${deletedCount} items, ${failedCount} failures.`
+    );
+  } catch (error) {
+    console.error(error);
+    log(`Error during cleanup: ${error.message}`);
+  } finally {
+    conclude();
+  }
+})();
+
+/**
+ * Retrieves all storage filenames referenced in active (include: true) workspace chats.
+ * Searches through the outputs array in chat responses.
+ * Uses pagination to avoid loading all chats into memory at once.
+ * @param {number} batchSize - Number of chats to process per batch (default: 50)
+ * @returns {Promise<Set<string>>}
+ */
+async function getActiveStorageFilenames(batchSize = 50) {
+  const [workspaceChats, scheduledJobRuns] = await Promise.all([
+    workspaceChatGeneratedFilenames(batchSize),
+    scheduledJobRunGeneratedFilenames(batchSize),
+  ]);
+
+  return new Set([...workspaceChats, ...scheduledJobRuns]);
+}
+
+async function workspaceChatGeneratedFilenames(batchSize = 50) {
+  const storageFilenames = new Set();
+  try {
+    let offset = 0;
+    let hasMore = true;
+
+    while (hasMore) {
+      const chats = await WorkspaceChats.where(
+        { include: true },
+        batchSize,
+        { id: "asc" },
+        offset
+      );
+
+      if (chats.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      for (const chat of chats) {
+        try {
+          const response = safeJsonParse(chat.response, { outputs: [] });
+          for (const output of response.outputs) {
+            if (!output || !output.payload || !output.payload.storageFilename)
+              continue;
+            storageFilenames.add(output.payload.storageFilename);
+          }
+        } catch {
+          continue;
+        }
+      }
+
+      offset += chats.length;
+      hasMore = chats.length === batchSize;
+    }
+  } catch (error) {
+    console.error("[workspaceChatGeneratedFilenames] Error:", error.message);
+  }
+
+  return storageFilenames;
+}
+
+async function scheduledJobRunGeneratedFilenames(batchSize = 50) {
+  const storageFilenames = new Set();
+  try {
+    let offset = 0;
+    let hasMore = true;
+
+    while (hasMore) {
+      const runs = await ScheduledJobRun.where(
+        { status: "completed" },
+        batchSize,
+        { id: "asc" },
+        {},
+        offset
+      );
+
+      if (runs.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      for (const run of runs) {
+        try {
+          const response = safeJsonParse(run.result, { outputs: [] });
+          for (const output of response.outputs) {
+            if (!output?.payload?.storageFilename) continue;
+            storageFilenames.add(output.payload.storageFilename);
+          }
+        } catch {
+          continue;
+        }
+      }
+
+      offset += runs.length;
+      hasMore = runs.length === batchSize;
+    }
+  } catch (error) {
+    console.error("[scheduledJobRunGeneratedFilenames] Error:", error.message);
+  }
+
+  return storageFilenames;
+}

package/server/jobs/cleanup-orphan-documents.js

@@ -0,0 +1,79 @@
+const fs = require("fs");
+const path = require("path");
+const { default: slugify } = require("slugify");
+const { log, conclude } = require("./helpers/index.js");
+const { WorkspaceParsedFiles } = require("../models/workspaceParsedFiles.js");
+const { directUploadsPath } = require("../utils/files");
+
+async function batchDeleteFiles(filesToDelete, batchSize = 500) {
+  let deletedCount = 0;
+  let failedCount = 0;
+
+  for (let i = 0; i < filesToDelete.length; i += batchSize) {
+    const batch = filesToDelete.slice(i, i + batchSize);
+
+    try {
+      await Promise.all(
+        batch.map((filePath) => {
+          if (!fs.existsSync(filePath)) return;
+          if (fs.lstatSync(filePath).isDirectory())
+            fs.rmSync(filePath, { recursive: true });
+          else fs.unlinkSync(filePath);
+        })
+      );
+      deletedCount += batch.length;
+
+      log(
+        `Deleted batch ${Math.floor(i / batchSize) + 1}: ${batch.length} files`
+      );
+    } catch {
+      // If batch fails, try individual files sync
+      for (const filePath of batch) {
+        try {
+          fs.unlinkSync(filePath);
+          deletedCount++;
+        } catch (fileErr) {
+          failedCount++;
+          log(`Failed to delete ${filePath}: ${fileErr.message}`);
+        }
+      }
+    }
+  }
+
+  return { deletedCount, failedCount };
+}
+
+(async () => {
+  try {
+    const filesToDelete = [];
+    const knownFiles = await WorkspaceParsedFiles.where({}, null, null, {
+      filename: true,
+    })
+      // Slugify the filename to match the direct uploads naming convention otherwise
+      // files with spaces will not result in a match and will be pruned when attached to a thread.
+      // This could then result in files showing "Attached" but the model not seeing them during chat.
+      .then((files) => new Set(files.map((f) => slugify(f.filename))));
+
+    if (!fs.existsSync(directUploadsPath))
+      return log("No direct uploads path found - exiting.");
+    const filesInDirectUploadsPath = fs.readdirSync(directUploadsPath);
+    if (filesInDirectUploadsPath.length === 0) return;
+
+    for (let i = 0; i < filesInDirectUploadsPath.length; i++) {
+      const file = filesInDirectUploadsPath[i];
+      if (knownFiles.has(file)) continue;
+      filesToDelete.push(path.resolve(directUploadsPath, file));
+    }
+
+    if (filesToDelete.length === 0) return; // No orphaned files to delete
+    log(`Found ${filesToDelete.length} orphaned files to delete`);
+    const { deletedCount, failedCount } = await batchDeleteFiles(filesToDelete);
+    log(`Deleted ${deletedCount} orphaned files`);
+    if (failedCount > 0) log(`Failed to delete ${failedCount} files`);
+  } catch (e) {
+    console.error(e);
+    log(`errored with ${e.message}`);
+  } finally {
+    conclude();
+  }
+})();