totem-llm 0.1.0

package/lib/config.js

@@ -0,0 +1,79 @@
+import { join } from "path";
+import { homedir } from "os";
+import { existsSync, readFileSync } from "fs";
+
+/**
+ * Returns the storage directory where user data (database, documents, models, etc.) lives.
+ * Controlled by TOTEM_STORAGE_DIR env var; defaults to ~/.totem-llm.
+ */
+export function getStorageDir() {
+  return process.env.TOTEM_STORAGE_DIR ?? join(homedir(), ".totem-llm");
+}
+
+/** Path to the persisted .env file inside the storage directory. */
+export function getEnvPath() {
+  return join(getStorageDir(), ".env");
+}
+
+/** True if the storage directory has already been configured. */
+export function isConfigured() {
+  return existsSync(getEnvPath());
+}
+
+/**
+ * Parse a .env file into a plain object.
+ * Handles quoted values and ignores comments / blank lines.
+ */
+function parseEnvFile(contents) {
+  const env = {};
+  for (const raw of contents.split("\n")) {
+    const line = raw.trim();
+    if (!line || line.startsWith("#")) continue;
+    const idx = line.indexOf("=");
+    if (idx === -1) continue;
+    const key = line.slice(0, idx).trim();
+    let value = line.slice(idx + 1).trim();
+    // Strip surrounding single or double quotes
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+    env[key] = value;
+  }
+  return env;
+}
+
+/**
+ * Build the merged environment object that is passed to the server and collector
+ * child processes.  Order of precedence (lowest → highest):
+ *   1. Current process.env
+ *   2. Values from the storage .env file
+ *   3. Computed values (STORAGE_DIR, DATABASE_URL, NODE_ENV)
+ *
+ * @param {{ storageDir?: string }} [opts]
+ * @returns {NodeJS.ProcessEnv}
+ */
+export function getEnv({ storageDir } = {}) {
+  const storage = storageDir ?? getStorageDir();
+  const envPath = join(storage, ".env");
+
+  let fileEnv = {};
+  if (existsSync(envPath)) {
+    try {
+      fileEnv = parseEnvFile(readFileSync(envPath, "utf8"));
+    } catch {
+      // Non-fatal; missing or unreadable env file is handled during setup
+    }
+  }
+
+  return {
+    ...process.env,
+    ...fileEnv,
+    // Always override these so they point at the correct storage dir
+    STORAGE_DIR: storage,
+    DATABASE_URL: `file:${join(storage, "anythingllm.db")}`,
+    NODE_ENV: "production",
+  };
+}

package/lib/launcher.js

@@ -0,0 +1,116 @@
+import { spawn } from "child_process";
+import { join } from "path";
+import { getEnv } from "./config.js";
+
+/**
+ * Spawn a Node.js service and prefix every log line with a fixed-width label.
+ *
+ * @param {string}   nodeExe     Absolute path to the Node binary (process.execPath)
+ * @param {string}   cwd         Working directory for the child process
+ * @param {object}   env         Environment variables to pass to the child
+ * @param {string}   label       Log prefix, e.g. "[server]   "
+ * @returns {{ process: import('child_process').ChildProcess }}
+ */
+function spawnService(nodeExe, cwd, env, label) {
+  const child = spawn(nodeExe, ["index.js"], {
+    cwd,
+    env,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  const pad = label.padEnd(13);
+  let buf = "";
+
+  function flush(stream, chunk) {
+    buf += chunk.toString();
+    const lines = buf.split("\n");
+    buf = lines.pop(); // keep any incomplete trailing line
+    for (const line of lines) {
+      if (line.trim()) stream.write(`${pad} ${line}\n`);
+    }
+  }
+
+  child.stdout.on("data", (c) => flush(process.stdout, c));
+  child.stderr.on("data", (c) => flush(process.stderr, c));
+
+  return child;
+}
+
+/**
+ * Launch the server (and optionally the collector), wire up graceful shutdown,
+ * and keep the process alive until the user presses Ctrl+C.
+ *
+ * @param {{ packageRoot: string, noCollector?: boolean, port?: string }} opts
+ */
+export async function launch({ packageRoot, noCollector = false, port } = {}) {
+  const env = getEnv();
+  if (port) env.SERVER_PORT = port;
+
+  const serverPort = env.SERVER_PORT ?? "3001";
+  const collectorPort = env.COLLECTOR_PORT ?? "8888";
+  const nodeExe = process.execPath; // cross-platform: always the right Node binary
+
+  const serverDir = join(packageRoot, "server");
+  const collectorDir = join(packageRoot, "collector");
+
+  console.log("Starting Totem LLM…");
+  console.log(`  Server     → http://localhost:${serverPort}`);
+  if (!noCollector) {
+    console.log(`  Collector  → http://localhost:${collectorPort}`);
+  }
+  console.log(`  Storage    → ${env.STORAGE_DIR}`);
+  console.log("\nPress Ctrl+C to stop.\n");
+
+  const serverProc = spawnService(nodeExe, serverDir, env, "[server]");
+
+  let collectorProc = null;
+  if (!noCollector) {
+    collectorProc = spawnService(nodeExe, collectorDir, env, "[collector]");
+  }
+
+  // ── Graceful shutdown ────────────────────────────────────────────────────
+  let shuttingDown = false;
+
+  function shutdown(signal) {
+    if (shuttingDown) return;
+    shuttingDown = true;
+
+    console.log(`\nShutting down (${signal})…`);
+    serverProc.kill("SIGTERM");
+    if (collectorProc) collectorProc.kill("SIGTERM");
+
+    // Force-kill after 8 s if processes are still running
+    const forceKill = setTimeout(() => {
+      if (!serverProc.killed) serverProc.kill("SIGKILL");
+      if (collectorProc && !collectorProc.killed) collectorProc.kill("SIGKILL");
+      process.exit(0);
+    }, 8000);
+
+    forceKill.unref(); // Don't prevent Node from exiting naturally
+  }
+
+  process.on("SIGINT", () => shutdown("SIGINT"));
+  process.on("SIGTERM", () => shutdown("SIGTERM"));
+
+  // Exit this process when the server exits unexpectedly
+  serverProc.on("close", (code) => {
+    if (!shuttingDown && code !== null && code !== 0) {
+      console.error(`[server] exited unexpectedly with code ${code}`);
+      shutdown("server-exit");
+      setTimeout(() => process.exit(1), 500);
+    }
+  });
+
+  if (collectorProc) {
+    collectorProc.on("close", (code) => {
+      if (!shuttingDown && code !== null && code !== 0) {
+        console.error(
+          `[collector] exited with code ${code} — document ingestion unavailable`
+        );
+      }
+    });
+  }
+
+  // Keep the main process alive indefinitely
+  await new Promise(() => {});
+}

package/lib/setup.js

@@ -0,0 +1,143 @@
+import { join } from "path";
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { randomBytes } from "crypto";
+import { spawn } from "child_process";
+import { getStorageDir, getEnvPath, getEnv } from "./config.js";
+
+/** Subdirectories that must exist inside the storage directory. */
+const STORAGE_SUBDIRS = [
+  "documents",
+  "vector-cache",
+  "models",
+  "direct-uploads",
+  "generated-files",
+  "comkey",
+  "tmp",
+];
+
+function generateSecret(bytes = 32) {
+  return randomBytes(bytes).toString("hex");
+}
+
+function buildEnvFile(storageDir) {
+  return [
+    "# Totem LLM configuration",
+    `# Generated ${new Date().toISOString()}`,
+    "# Edit this file to configure your instance, then restart.",
+    "",
+    `STORAGE_DIR="${storageDir}"`,
+    "SERVER_PORT=3001",
+    "COLLECTOR_PORT=8888",
+    "",
+    "# Security secrets (auto-generated — do not share these)",
+    `JWT_SECRET="${generateSecret(24)}"`,
+    `SIG_KEY="${generateSecret(32)}"`,
+    `SIG_SALT="${generateSecret(32)}"`,
+    "",
+    "# LLM provider — configure via the UI after first launch, or set here.",
+    "# Example (Ollama running locally):",
+    "# LLM_PROVIDER=ollama",
+    "# OLLAMA_BASE_PATH=http://127.0.0.1:11434",
+    "",
+    "# Vector database (lancedb = zero-setup file-based default)",
+    "VECTOR_DB=lancedb",
+    "",
+    "# Embedding engine (native = built-in, no external service needed)",
+    "EMBEDDING_ENGINE=native",
+    "",
+  ].join("\n");
+}
+
+/**
+ * Run a command and stream its output to the parent's stdio.
+ * Rejects with a descriptive error if the process exits non-zero.
+ */
+function runCommand(cmd, args, { cwd, env } = {}) {
+  return new Promise((resolve, reject) => {
+    // On Windows, executables like "npx" need the .cmd wrapper
+    const isWin = process.platform === "win32";
+    const bin = isWin && !cmd.endsWith(".cmd") ? `${cmd}.cmd` : cmd;
+
+    const child = spawn(bin, args, {
+      cwd,
+      env,
+      stdio: "inherit",
+      shell: false,
+    });
+
+    child.on("close", (code) => {
+      if (code === 0) return resolve();
+      reject(
+        new Error(`"${cmd} ${args.join(" ")}" exited with code ${code}`)
+      );
+    });
+    child.on("error", (err) => {
+      reject(new Error(`Failed to run "${cmd}": ${err.message}`));
+    });
+  });
+}
+
+/**
+ * Ensure the storage directory and all required subdirectories exist.
+ */
+function ensureStorageDirs(storageDir) {
+  if (!existsSync(storageDir)) {
+    console.log(`  Creating storage directory: ${storageDir}`);
+    mkdirSync(storageDir, { recursive: true });
+  }
+  for (const sub of STORAGE_SUBDIRS) {
+    const dir = join(storageDir, sub);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  }
+}
+
+/**
+ * Run Prisma database migrations against the storage-dir SQLite file.
+ * Safe to call on every startup — only applies pending migrations.
+ */
+async function runMigrations(serverDir, env) {
+  await runCommand("npx", ["prisma", "migrate", "deploy", "--schema", join(serverDir, "prisma", "schema.prisma")], {
+    cwd: serverDir,
+    env,
+  });
+}
+
+/**
+ * First-run (and subsequent startup) setup:
+ *  1. Creates storage directory + subdirectories
+ *  2. Generates secrets and writes ~/.totem-llm/.env (skipped if already present)
+ *  3. Runs Prisma migrations (idempotent)
+ *
+ * @param {{ packageRoot: string, force?: boolean }} opts
+ *   packageRoot – absolute path to the installed package root
+ *   force       – if true, regenerate the .env file even if it already exists
+ */
+export async function setup({ packageRoot, force = false }) {
+  const storageDir = getStorageDir();
+  const envPath = getEnvPath();
+
+  console.log("\nTotem LLM – setup");
+  console.log("─────────────────────────────────────────");
+
+  // 1. Storage directory
+  ensureStorageDirs(storageDir);
+
+  // 2. Configuration file
+  if (!existsSync(envPath) || force) {
+    console.log("  Generating configuration...");
+    writeFileSync(envPath, buildEnvFile(storageDir), "utf8");
+    console.log(`  Config saved to: ${envPath}`);
+  } else {
+    console.log(`  Config found: ${envPath}`);
+  }
+
+  // 3. Database migrations
+  const serverDir = join(packageRoot, "server");
+  const env = getEnv({ storageDir });
+
+  console.log("  Running database migrations...");
+  await runMigrations(serverDir, env);
+  console.log("  Database ready.");
+
+  console.log("─────────────────────────────────────────\n");
+}

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "totem-llm",
+  "version": "0.1.0",
+  "description": "Totem LLM – Your Private AI. Run a self-hosted AI assistant locally on Linux, macOS, or Windows.",
+  "main": "bin/totem-llm.js",
+  "type": "module",
+  "author": "Fred Terzi",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fred-terzi/totem-llm.git"
+  },
+  "bugs": {
+    "url": "https://github.com/fred-terzi/totem-llm/issues"
+  },
+  "homepage": "https://github.com/fred-terzi/totem-llm#readme",
+  "engines": {
+    "node": ">=18"
+  },
+  "bin": {
+    "totem-llm": "bin/totem-llm.js"
+  },
+  "files": [
+    "bin/",
+    "lib/",
+    "scripts/postinstall.mjs",
+    "server/",
+    "collector/"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "test": "jest",
+    "lint": "cd server && yarn lint && cd ../frontend && yarn lint && cd ../collector && yarn lint",
+    "lint:ci": "cd server && yarn lint:check && cd ../frontend && yarn lint:check && cd ../collector && yarn lint:check",
+    "setup": "cd server && yarn && cd ../collector && yarn && cd ../frontend && yarn && cd .. && yarn setup:envs && yarn prisma:setup && echo \"Please run yarn dev:server, yarn dev:collector, and yarn dev:frontend in separate terminal tabs.\"",
+    "setup:envs": "node scripts/setup-envs.mjs",
+    "dev:server": "cd server && yarn dev",
+    "dev:collector": "cd collector && yarn dev",
+    "dev:frontend": "cd frontend && yarn dev",
+    "dev:all": "npx concurrently \"yarn dev:server\" \"yarn dev:frontend\" \"yarn dev:collector\"",
+    "prisma:generate": "cd server && npx prisma generate",
+    "prisma:migrate": "cd server && npx prisma migrate dev --name init",
+    "prisma:seed": "cd server && npx prisma db seed",
+    "prisma:setup": "yarn prisma:generate && yarn prisma:migrate && yarn prisma:seed",
+    "prisma:reset": "node -e \"const{writeFileSync,existsSync}=require('fs');const p='server/storage/anythingllm.db';if(existsSync(p)){writeFileSync(p,'');console.log('Database cleared.')}\" && yarn prisma:migrate",
+    "prod:server": "cd server && yarn start",
+    "prod:frontend": "cd frontend && yarn build",
+    "generate:cloudformation": "node cloud-deployments/aws/cloudformation/generate.mjs",
+    "generate::gcp_deployment": "node cloud-deployments/gcp/deployment/generate.mjs",
+    "translations:verify": "cd frontend/src/locales && node verifyTranslations.mjs",
+    "translations:normalize": "cd frontend/src/locales && node normalizeEn.mjs && cd ../../.. && cd frontend && yarn lint && cd .. && yarn translations:verify",
+    "translations:prune": "cd frontend/src/locales && node findUnusedTranslations.mjs --delete && cd ../../../ && yarn translations:normalize",
+    "translations:create": "cd extras/translator && node index.mjs --all && cd ../../frontend && yarn lint && cd ../..",
+    "prepublishOnly": "node scripts/build-frontend.mjs",
+    "postinstall": "node scripts/postinstall.mjs"
+  },
+  "private": false,
+  "devDependencies": {
+    "concurrently": "^9.1.2",
+    "jest": "^29.7.0"
+  }
+}

package/scripts/postinstall.mjs

@@ -0,0 +1,117 @@
+#!/usr/bin/env node
+/**
+ * postinstall.mjs
+ *
+ * Runs after `npm install -g totem-llm`.
+ * Installs production dependencies for the server and collector sub-packages,
+ * then generates the Prisma client for the current platform.
+ *
+ * All errors are caught and treated as non-fatal so that a transient network
+ * issue doesn't leave the overall install in a broken state.  Users can re-run
+ * this script manually via `node node_modules/totem-llm/scripts/postinstall.mjs`.
+ */
+
+import { spawn } from "child_process";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import { existsSync } from "fs";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const packageRoot = join(__dirname, "..");
+
+const isWin = process.platform === "win32";
+
+/** Wrap a command name so it resolves correctly on Windows (.cmd suffix). */
+function bin(name) {
+  return isWin ? `${name}.cmd` : name;
+}
+
+/** Run a command, streaming output, resolving on exit-0, rejecting otherwise. */
+function run(cmd, args, cwd) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(bin(cmd), args, { cwd, stdio: "inherit", shell: false });
+    child.on("close", (code) =>
+      code === 0
+        ? resolve()
+        : reject(new Error(`"${cmd} ${args.join(" ")}" failed (exit ${code})`))
+    );
+    child.on("error", (err) =>
+      reject(new Error(`Could not run "${cmd}": ${err.message}`))
+    );
+  });
+}
+
+async function installDeps(label, dir) {
+  if (existsSync(join(dir, "node_modules"))) {
+    console.log(`  [${label}] dependencies already installed — skipping`);
+    return;
+  }
+  console.log(`  [${label}] installing production dependencies…`);
+  // Prefer npm (always available in npm-installed contexts) over yarn
+  await run("npm", ["install", "--omit=dev", "--no-audit", "--no-fund"], dir);
+}
+
+async function main() {
+  // Skip in CI environments where dependency management is handled externally
+  if (process.env.CI) return;
+
+  // Skip if running inside the development workspace (i.e. a developer has
+  // cloned the repo and run `yarn install` at the root). We detect this by
+  // checking whether the root package.json name matches AND we are NOT being
+  // installed as a dependency of something else.
+  const isLocalDev =
+    process.env.npm_config_local_prefix &&
+    process.env.npm_config_local_prefix === process.env.INIT_CWD;
+  if (isLocalDev) return;
+
+  console.log("\nTotem LLM – post-install setup");
+  console.log("─────────────────────────────────────────");
+
+  const serverDir = join(packageRoot, "server");
+  const collectorDir = join(packageRoot, "collector");
+
+  try {
+    await installDeps("server", serverDir);
+  } catch (err) {
+    console.warn(`  [server] dependency install failed: ${err.message}`);
+    console.warn("  Run: npm install --omit=dev inside the server/ directory manually.");
+  }
+
+  try {
+    // Only regenerate if the client doesn't exist yet (avoid re-running on
+    // every install in already-configured environments)
+    const prismaClientExists = existsSync(
+      join(serverDir, "node_modules", ".prisma", "client", "index.js")
+    );
+    if (!prismaClientExists) {
+      console.log("  [server] generating Prisma client…");
+      await run(
+        "npx",
+        ["prisma", "generate", "--schema", join(serverDir, "prisma", "schema.prisma")],
+        serverDir
+      );
+    } else {
+      console.log("  [server] Prisma client already generated — skipping");
+    }
+  } catch (err) {
+    console.warn(`  [server] Prisma generate failed: ${err.message}`);
+    console.warn("  Run: npx prisma generate inside the server/ directory manually.");
+  }
+
+  try {
+    await installDeps("collector", collectorDir);
+  } catch (err) {
+    console.warn(`  [collector] dependency install failed: ${err.message}`);
+    console.warn("  Run: npm install --omit=dev inside the collector/ directory manually.");
+  }
+
+  console.log("─────────────────────────────────────────");
+  console.log('  Done. Run "totem-llm" to start.\n');
+}
+
+main().catch((err) => {
+  // Non-fatal — print a warning but let the npm install succeed
+  console.warn("Post-install script encountered an error:", err.message);
+  console.warn('Run "totem-llm setup" after installation to complete setup.');
+});