totem-llm 0.1.0

package/LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2026 Fred Terzi
+Copyright (c) Mintplex Labs Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,81 @@
+# Totem LLM - Your Private AI
+
+Totem LLM is a sovereign AI solution, aimed at optimizing for consumer hardware and ensuring data privacy. It is designed to run efficiently on consumer-grade hardware, such as laptops and desktops, without the need for cloud-based processing. This allows users to have full control over their data and AI interactions, without relying on third-party services.
+
+## Quick Start (Build from Source)
+
+### Prerequisites
+
+- [Node.js](https://nodejs.org/) v18 or later
+- [Yarn](https://yarnpkg.com/) v1.x (`npm install -g yarn`)
+- [Ollama](https://ollama.com/) (recommended local LLM provider) or an [OpenRouter](https://openrouter.ai/) API key
+- Linux build machine - current build relies on `cp` command with `--reflink` option for efficient file copying. Building on non-Linux platforms may require additional setup (e.g. WSL on Windows).
+
+### 1. Clone the repository
+
+```bash
+git clone https://github.com/fred-terzi/totem-llm.git
+cd totem-llm
+```
+
+### 2. Install dependencies and initialise the database
+
+```bash
+yarn setup
+```
+
+This installs dependencies for all three packages (`server`, `collector`, `frontend`), copies the example `.env` files, generates the Prisma client, and runs database migrations.
+
+### 3. Configure environment variables
+
+Open `server/.env.development` and set at minimum:
+
+```env
+STORAGE_DIR="/absolute/path/to/your/storage"
+```
+
+All other settings (LLM provider, embedding model, etc.) can be changed later through the UI.
+
+### 4. Start in development mode
+
+```bash
+yarn dev:all
+```
+
+This starts the server (port `3001`), collector (port `8888`), and frontend dev server (port `5173`) concurrently. Open [http://localhost:5173](http://localhost:5173) to access the UI.
+
+> Alternatively, run each service in its own terminal: `yarn dev:server`, `yarn dev:collector`, `yarn dev:frontend`.
+
+### 5. Build for production
+
+Choose a build profile and run the frontend build + deploy script:
+
+```bash
+# Available profiles: npm | desktop-free | desktop-premium | source
+TOTEM_BUILD_PROFILE=source node scripts/build-frontend.mjs
+```
+
+This builds the Vite frontend and copies the output into `server/public/`. Then start the server:
+
+```bash
+node bin/totem-llm.js start
+```
+
+The app will be available at [http://localhost:3001](http://localhost:3001).
+
+#### Build profiles
+
+| Profile | Community Hub | Model Router | LLM Providers | Branding & Whitelabeling |
+|---|---|---|---|---|
+| `npm` | ✗ | ✗ | Ollama, OpenRouter only | ✗ |
+| `desktop-free` | ✗ | ✓ | All | ✗ |
+| `desktop-premium` | ✓ | ✓ | All | ✓ |
+| `source` | ✓ | ✓ | All | ✓ |
+
+---
+
+## Thanks
+
+Massive thank you and appreciation to AnythingLLM. Totem LLM is a fork of AnythingLLM, and we are grateful for the foundation they have provided. We will continue to build on their work and contribute back to the community.
+
+https://github.com/Mintplex-Labs/anything-llm

package/bin/totem-llm.js

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import { setup } from "../lib/setup.js";
+import { launch } from "../lib/launcher.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packageRoot = join(__dirname, "..");
+
+const args = process.argv.slice(2);
+const isHelp = args.includes("--help") || args.includes("-h");
+const command = isHelp
+  ? "help"
+  : (args.find((a) => !a.startsWith("--")) ?? "start");
+const noCollector = args.includes("--no-collector");
+const portArg = args.find((a) => a.startsWith("--port="));
+const port = portArg ? portArg.split("=")[1] : undefined;
+
+async function main() {
+  switch (command) {
+    case "start":
+      await setup({ packageRoot });
+      await launch({ packageRoot, noCollector, port });
+      break;
+
+    case "setup":
+      await setup({ packageRoot, force: args.includes("--force") });
+      console.log(
+        '\nSetup complete. Run "totem-llm start" to launch the server.'
+      );
+      break;
+
+    case "help":
+    case "--help":
+    case "-h":
+      printHelp();
+      break;
+
+    default:
+      console.error(`Unknown command: "${command}"`);
+      printHelp();
+      process.exit(1);
+  }
+}
+
+function printHelp() {
+  console.log(`
+Totem LLM – Your Private AI
+
+Usage:
+  totem-llm [command] [options]
+
+Commands:
+  start         Start Totem LLM (default)
+  setup         Run first-time setup without starting
+  help          Show this help message
+
+Options:
+  --no-collector   Start without the document collector (skips Puppeteer/ffmpeg)
+  --port=<port>    Override the server port (default: 3001)
+  --force          (setup only) Regenerate configuration even if it already exists
+
+Environment Variables:
+  TOTEM_STORAGE_DIR    Override storage directory (default: ~/.totem-llm)
+  SERVER_PORT          Override server port
+  COLLECTOR_PORT       Override collector port (default: 8888)
+
+Examples:
+  totem-llm                        Start with defaults
+  totem-llm --no-collector         Start without document ingestion
+  totem-llm --port=8080            Start server on port 8080
+  TOTEM_STORAGE_DIR=/data totem-llm start
+`);
+}
+
+main().catch((err) => {
+  console.error("Fatal error:", err.message);
+  process.exit(1);
+});

package/collector/.env

@@ -0,0 +1,9 @@
+# Placeholder .env file for collector runtime
+
+# Port the collector listens on. The server must use the same COLLECTOR_PORT value when running services separately.
+# COLLECTOR_PORT=8888
+
+# This enables HTTP request/response logging in development. Set value to truthy string to enable, leave empty value or comment out to disable
+# ENABLE_HTTP_LOGGER=""
+# This enables timestamps for the HTTP Logger. Set value to true to enable, leave empty or comment out to disable
+# ENABLE_HTTP_LOGGER_TIMESTAMPS=""

package/collector/.env.example

@@ -0,0 +1,9 @@
+# Placeholder .env file for collector runtime
+
+# Port the collector listens on. The server must use the same COLLECTOR_PORT value when running services separately.
+# COLLECTOR_PORT=8888
+
+# This enables HTTP request/response logging in development. Set value to truthy string to enable, leave empty value or comment out to disable
+# ENABLE_HTTP_LOGGER=""
+# This enables timestamps for the HTTP Logger. Set value to true to enable, leave empty or comment out to disable
+# ENABLE_HTTP_LOGGER_TIMESTAMPS=""

package/collector/.nvmrc

@@ -0,0 +1 @@
+v18.18.0

package/collector/extensions/index.js

@@ -0,0 +1,239 @@
+const { setDataSigner } = require("../middleware/setDataSigner");
+const { verifyPayloadIntegrity } = require("../middleware/verifyIntegrity");
+const {
+  resolveRepoLoader,
+  resolveRepoLoaderFunction,
+} = require("../utils/extensions/RepoLoader");
+const { reqBody } = require("../utils/http");
+const { validURL, validateURL } = require("../utils/url");
+const RESYNC_METHODS = require("./resync");
+const { loadObsidianVault } = require("../utils/extensions/ObsidianVault");
+
+function extensions(app) {
+  if (!app) return;
+
+  app.post(
+    "/ext/resync-source-document",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const { type, options } = reqBody(request);
+        if (!RESYNC_METHODS.hasOwnProperty(type))
+          throw new Error(`Type "${type}" is not a valid type to sync.`);
+        return await RESYNC_METHODS[type](options, response);
+      } catch (e) {
+        console.error(e);
+        response.status(200).json({
+          success: false,
+          content: null,
+          reason: e.message || "A processing error occurred.",
+        });
+      }
+      return;
+    }
+  );
+
+  app.post(
+    "/ext/:repo_platform-repo",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const loadRepo = resolveRepoLoaderFunction(
+          request.params.repo_platform
+        );
+        const { success, reason, data } = await loadRepo(
+          reqBody(request),
+          response
+        );
+        response.status(200).json({
+          success,
+          reason,
+          data,
+        });
+      } catch (e) {
+        console.error(e);
+        response.status(200).json({
+          success: false,
+          reason: e.message || "A processing error occurred.",
+          data: {},
+        });
+      }
+      return;
+    }
+  );
+
+  // gets all branches for a specific repo
+  app.post(
+    "/ext/:repo_platform-repo/branches",
+    [verifyPayloadIntegrity],
+    async function (request, response) {
+      try {
+        const RepoLoader = resolveRepoLoader(request.params.repo_platform);
+        const allBranches = await new RepoLoader(
+          reqBody(request)
+        ).getRepoBranches();
+        response.status(200).json({
+          success: true,
+          reason: null,
+          data: {
+            branches: allBranches,
+          },
+        });
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: {
+            branches: [],
+          },
+        });
+      }
+      return;
+    }
+  );
+
+  app.post(
+    "/ext/youtube-transcript",
+    [verifyPayloadIntegrity],
+    async function (request, response) {
+      try {
+        const {
+          loadYouTubeTranscript,
+        } = require("../utils/extensions/YoutubeTranscript");
+        const { success, reason, data } = await loadYouTubeTranscript(
+          reqBody(request)
+        );
+        response.status(200).json({ success, reason, data });
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: {
+            title: null,
+            author: null,
+          },
+        });
+      }
+      return;
+    }
+  );
+
+  app.post(
+    "/ext/website-depth",
+    [verifyPayloadIntegrity],
+    async function (request, response) {
+      try {
+        const websiteDepth = require("../utils/extensions/WebsiteDepth");
+        const { url, depth = 1, maxLinks = 20 } = reqBody(request);
+        const validatedUrl = validateURL(url);
+        if (!validURL(validatedUrl)) throw new Error("Not a valid URL.");
+        const scrapedData = await websiteDepth(validatedUrl, depth, maxLinks);
+        response.status(200).json({ success: true, data: scrapedData });
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({ success: false, reason: e.message });
+      }
+      return;
+    }
+  );
+
+  app.post(
+    "/ext/confluence",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const { loadConfluence } = require("../utils/extensions/Confluence");
+        const { success, reason, data } = await loadConfluence(
+          reqBody(request),
+          response
+        );
+        response.status(200).json({ success, reason, data });
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: {
+            title: null,
+            author: null,
+          },
+        });
+      }
+      return;
+    }
+  );
+
+  app.post(
+    "/ext/drupalwiki",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const {
+          loadAndStoreSpaces,
+        } = require("../utils/extensions/DrupalWiki");
+        const { success, reason, data } = await loadAndStoreSpaces(
+          reqBody(request),
+          response
+        );
+        response.status(200).json({ success, reason, data });
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: {
+            title: null,
+            author: null,
+          },
+        });
+      }
+      return;
+    }
+  );
+
+  app.post(
+    "/ext/obsidian/vault",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const { files } = reqBody(request);
+        const result = await loadObsidianVault({ files });
+        response.status(200).json(result);
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: null,
+        });
+      }
+      return;
+    }
+  );
+
+  app.post(
+    "/ext/paperless-ngx",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const {
+          loadPaperlessNgx,
+        } = require("../utils/extensions/PaperlessNgx");
+        const result = await loadPaperlessNgx(reqBody(request), response);
+        response.status(200).json(result);
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: null,
+        });
+      }
+      return;
+    }
+  );
+}
+
+module.exports = extensions;

package/collector/extensions/resync/index.js

@@ -0,0 +1,197 @@
+const { getLinkText } = require("../../processLink");
+
+/**
+ * Fetches the content of a raw link. Returns the content as a text string of the link in question.
+ * @param {object} data - metadata from document (eg: link)
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncLink({ link }, response) {
+  if (!link) throw new Error("Invalid link provided");
+  try {
+    const { success, content = null, reason } = await getLinkText(link);
+    if (!success) throw new Error(`Failed to sync link content. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a YouTube link. Returns the content as a text string of the video in question.
+ * We offer this as there may be some videos where a transcription could be manually edited after initial scraping
+ * but in general - transcriptions often never change.
+ * @param {object} data - metadata from document (eg: link)
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncYouTube({ link }, response) {
+  if (!link) throw new Error("Invalid link provided");
+  try {
+    const {
+      fetchVideoTranscriptContent,
+    } = require("../../utils/extensions/YoutubeTranscript");
+    const { success, reason, content } = await fetchVideoTranscriptContent({
+      url: link,
+    });
+    if (!success)
+      throw new Error(`Failed to sync YouTube video transcript. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a specific confluence page via its chunkSource.
+ * Returns the content as a text string of the page in question and only that page.
+ * @param {object} data - metadata from document (eg: chunkSource)
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncConfluence({ chunkSource }, response) {
+  if (!chunkSource) throw new Error("Invalid source property provided");
+  try {
+    // Confluence data is `payload` encrypted. So we need to expand its
+    // encrypted payload back into query params so we can reFetch the page with same access token/params.
+    const source = response.locals.encryptionWorker.expandPayload(chunkSource);
+    const {
+      fetchConfluencePage,
+    } = require("../../utils/extensions/Confluence");
+    const { success, reason, content } = await fetchConfluencePage({
+      pageUrl: `https:${source.pathname}`, // need to add back the real protocol
+      baseUrl: source.searchParams.get("baseUrl"),
+      spaceKey: source.searchParams.get("spaceKey"),
+      accessToken: source.searchParams.get("token"),
+      username: source.searchParams.get("username"),
+      cloud: source.searchParams.get("cloud") === "true",
+      bypassSSL: source.searchParams.get("bypassSSL") === "true",
+    });
+
+    if (!success)
+      throw new Error(`Failed to sync Confluence page content. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a specific confluence page via its chunkSource.
+ * Returns the content as a text string of the page in question and only that page.
+ * @param {object} data - metadata from document (eg: chunkSource)
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncGithub({ chunkSource }, response) {
+  if (!chunkSource) throw new Error("Invalid source property provided");
+  try {
+    // Github file data is `payload` encrypted (might contain PAT). So we need to expand its
+    // encrypted payload back into query params so we can reFetch the page with same access token/params.
+    const source = response.locals.encryptionWorker.expandPayload(chunkSource);
+    const {
+      fetchGithubFile,
+    } = require("../../utils/extensions/RepoLoader/GithubRepo");
+    const { success, reason, content } = await fetchGithubFile({
+      repoUrl: `https:${source.pathname}`, // need to add back the real protocol
+      branch: source.searchParams.get("branch"),
+      accessToken: source.searchParams.get("pat"),
+      sourceFilePath: source.searchParams.get("path"),
+    });
+
+    if (!success)
+      throw new Error(`Failed to sync GitHub file content. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a specific DrupalWiki page via its chunkSource.
+ * Returns the content as a text string of the page in question and only that page.
+ * @param {object} data - metadata from document (eg: chunkSource)
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncDrupalWiki({ chunkSource }, response) {
+  if (!chunkSource) throw new Error("Invalid source property provided");
+  try {
+    // DrupalWiki data is `payload` encrypted. So we need to expand its
+    // encrypted payload back into query params so we can reFetch the page with same access token/params.
+    const source = response.locals.encryptionWorker.expandPayload(chunkSource);
+    const { loadPage } = require("../../utils/extensions/DrupalWiki");
+    const { success, reason, content } = await loadPage({
+      baseUrl: source.searchParams.get("baseUrl"),
+      pageId: source.searchParams.get("pageId"),
+      accessToken: source.searchParams.get("accessToken"),
+    });
+
+    if (!success) {
+      console.error(`Failed to sync DrupalWiki page content. ${reason}`);
+      response.status(200).json({
+        success: false,
+        content: null,
+      });
+    } else {
+      response.status(200).json({ success, content });
+    }
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a specific Paperless-ngx document via its chunkSource.
+ * Returns the content as a text string of the document.
+ * @param {object} data - metadata from document (eg: chunkSource)
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncPaperlessNgx({ chunkSource }, response) {
+  if (!chunkSource) throw new Error("Invalid source property provided");
+  try {
+    const source = response.locals.encryptionWorker.expandPayload(chunkSource);
+    const {
+      PaperlessNgxLoader,
+    } = require("../../utils/extensions/PaperlessNgx/PaperlessNgxLoader");
+    const loader = new PaperlessNgxLoader({
+      baseUrl: source.searchParams.get("baseUrl"),
+      apiToken: source.searchParams.get("token"),
+    });
+    const documentId = source.pathname.split("//")[1];
+    const content = await loader.fetchDocumentContent(documentId);
+
+    if (!content) throw new Error("Failed to fetch document content");
+    response.status(200).json({ success: true, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+module.exports = {
+  link: resyncLink,
+  youtube: resyncYouTube,
+  confluence: resyncConfluence,
+  github: resyncGithub,
+  drupalwiki: resyncDrupalWiki,
+  "paperless-ngx": resyncPaperlessNgx,
+};

package/collector/hotdir/__HOTDIR__.md

@@ -0,0 +1,3 @@
+### What is the "Hot directory"
+
+This is a pre-set file location that documents will be written to when uploaded by AnythingLLM. There is really no need to touch it.