stellar-memory 0.5.0

package/dist/utils/tokenizer.js

@@ -0,0 +1,46 @@
+/**
+ * Lightweight token estimator for mixed English/Korean text.
+ *
+ * Unicode-aware estimation:
+ *   - ASCII/Latin text: ~1.3 tokens per whitespace-delimited word
+ *   - CJK characters (Korean/Chinese/Japanese): ~2.0 tokens per character
+ *   - Other Unicode: ~1.5 tokens per word
+ *
+ * This replaces the old 0.75 tokens/word estimate which underestimated
+ * by ~50%, causing sun token budget overflows.
+ */
+export function estimateTokens(text) {
+    if (!text || text.length === 0)
+        return 0;
+    let tokens = 0;
+    // CJK Unicode ranges:
+    //   Hangul Syllables: U+AC00–U+D7AF
+    //   Hangul Jamo:      U+1100–U+11FF
+    //   CJK Unified:      U+4E00–U+9FFF
+    //   Katakana/Hiragana: U+3040–U+30FF
+    const CJK_REGEX = /[\u1100-\u11FF\u3040-\u30FF\u4E00-\u9FFF\uAC00-\uD7AF]/;
+    // Split into whitespace-delimited segments
+    const segments = text.split(/\s+/).filter(w => w.length > 0);
+    for (const segment of segments) {
+        if (CJK_REGEX.test(segment)) {
+            // For CJK-containing segments, count chars individually
+            for (const char of segment) {
+                if (CJK_REGEX.test(char)) {
+                    tokens += 2.0; // CJK chars typically split into 2+ tokens
+                }
+                else {
+                    tokens += 0.5; // Punctuation/ASCII mixed in with CJK
+                }
+            }
+        }
+        else {
+            // English/Latin word: ~1.3 tokens per word
+            tokens += 1.3;
+        }
+    }
+    // Account for whitespace tokens (roughly 1 token per 4 spaces/newlines)
+    const whitespace = text.length - text.replace(/\s/g, '').length;
+    tokens += whitespace * 0.25;
+    return Math.ceil(tokens);
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/utils/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/utils/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEzC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,sBAAsB;IACtB,oCAAoC;IACpC,oCAAoC;IACpC,oCAAoC;IACpC,qCAAqC;IACrC,MAAM,SAAS,GAAG,wDAAwD,CAAC;IAE3E,2CAA2C;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE7D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,wDAAwD;YACxD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBAC3B,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,GAAG,CAAC,CAAE,2CAA2C;gBAC7D,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,GAAG,CAAC,CAAE,sCAAsC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,2CAA2C;YAC3C,MAAM,IAAI,GAAG,CAAC;QAChB,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,IAAI,UAAU,GAAG,IAAI,CAAC;IAE5B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC"}

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "stellar-memory",
+  "version": "0.5.0",
+  "description": "AI memory system using celestial mechanics metaphor - MCP server for persistent context across sessions",
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "stellar-memory": "dist/cli/index.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/sangjun0000/stellar-memory-core.git"
+  },
+  "homepage": "https://stellar-memory.com",
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "dist/**/*.js.map",
+    "scripts/check-node.mjs",
+    "scripts/setup.mjs",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "scripts": {
+    "preinstall": "node scripts/check-node.mjs",
+    "postinstall": "node scripts/setup.mjs || true",
+    "setup": "node scripts/setup.mjs",
+    "init": "node dist/cli/init.js",
+    "build": "tsc",
+    "build:web": "cd web && npx vite build",
+    "build:electron": "tsc -p electron/tsconfig.json && node --input-type=commonjs -e \"require('fs').writeFileSync('dist-electron/package.json','{\\\"type\\\":\\\"commonjs\\\"}')\"",
+    "build:all": "npm run build && npm run build:web && npm run build:electron",
+    "dev": "tsx src/index.ts",
+    "start": "node dist/index.js",
+    "api": "tsx src/api/server.ts",
+    "api:start": "node dist/api/server.js",
+    "electron:dev": "npm run build:all && electron dist-electron/main.js",
+    "electron:pack": "npm run build:all && electron-builder --config electron-builder.yml",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "daemon": "tsx src/service/daemon.ts",
+    "daemon:start": "node dist/service/daemon.js",
+    "docker:build": "docker build -t stellar-memory .",
+    "docker:run": "docker compose up -d",
+    "benchmark": "npx tsx benchmarks/index.ts",
+    "benchmark:skip-embeddings": "npx tsx benchmarks/index.ts --skip-embeddings"
+  },
+  "keywords": [
+    "mcp",
+    "memory",
+    "ai",
+    "context",
+    "stellar"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@hono/node-server": "^1.19.9",
+    "@modelcontextprotocol/sdk": "^1.27.1",
+    "@xenova/transformers": "^2.17.2",
+    "hono": "^4.12.3",
+    "sqlite-vec": "^0.1.7-alpha.2",
+    "ws": "^8.19.0",
+    "zod": "^3.25.76"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.13",
+    "@types/ws": "^8.18.1",
+    "electron": "^40.6.1",
+    "electron-builder": "^26.8.1",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "^2.1.9"
+  }
+}

package/scripts/check-node.mjs

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+/**
+ * check-node.mjs — Preinstall script that validates Node.js version.
+ * Runs before `npm install` via the "preinstall" hook in package.json.
+ */
+
+const REQUIRED_MAJOR = 22;
+
+const [major] = process.versions.node.split('.').map(Number);
+
+if (major < REQUIRED_MAJOR) {
+  console.error(`
+╔══════════════════════════════════════════════════════════════╗
+║              Stellar Memory — Node.js Version Error          ║
+╠══════════════════════════════════════════════════════════════╣
+║                                                              ║
+║  Required:  Node.js 22 or higher                             ║
+║  Detected:  Node.js ${process.versions.node.padEnd(40)}║
+║                                                              ║
+║  Stellar Memory uses the built-in node:sqlite module which   ║
+║  was introduced in Node.js 22. It will not work on older     ║
+║  versions.                                                   ║
+║                                                              ║
+║  How to upgrade:                                             ║
+║                                                              ║
+║  Option A — nvm (recommended):                               ║
+║    nvm install 22                                            ║
+║    nvm use 22                                                ║
+║                                                              ║
+║  Option B — Direct download:                                 ║
+║    https://nodejs.org/en/download                            ║
+║                                                              ║
+╚══════════════════════════════════════════════════════════════╝
+`);
+  process.exit(1);
+}

package/scripts/setup.mjs

@@ -0,0 +1,157 @@
+#!/usr/bin/env node
+/**
+ * setup.mjs — Pre-downloads the embedding model so first use is instant.
+ *
+ * Run once after installing:  npm run setup
+ *
+ * What this does:
+ *   1. Downloads Xenova/all-MiniLM-L6-v2 (~90 MB, quantized) from HuggingFace.
+ *   2. Caches it in ~/.cache/huggingface (standard HuggingFace cache location).
+ *   3. Shows real-time download progress so you know it's working.
+ *
+ * After this completes, Stellar Memory starts instantly without any network delay.
+ */
+
+const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2';
+const TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
+
+console.log('Stellar Memory — Model Setup');
+console.log('============================');
+console.log(`Model: ${MODEL_NAME}`);
+console.log('Size:  ~90 MB (quantized int8)');
+console.log('Cache: ~/.cache/huggingface');
+console.log('');
+console.log('Downloading embedding model...');
+console.log('(This only happens once — subsequent starts are instant)');
+console.log('');
+
+let lastFile = '';
+let startTime = Date.now();
+let downloadedBytes = 0;
+let totalBytes = 0;
+
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+function formatTime(seconds) {
+  if (seconds < 60) return `${Math.round(seconds)}s`;
+  return `${Math.floor(seconds / 60)}m ${Math.round(seconds % 60)}s`;
+}
+
+function renderProgressBar(percent, width = 30) {
+  const filled = Math.round((percent / 100) * width);
+  const empty = width - filled;
+  return '[' + '#'.repeat(filled) + '-'.repeat(empty) + ']';
+}
+
+function onProgress(info) {
+  // info: { status, name, file, progress, loaded, total }
+  const { status, file, progress, loaded, total } = info;
+
+  if (status === 'initiate') {
+    if (file && file !== lastFile) {
+      lastFile = file;
+      const fileName = file.split('/').pop() ?? file;
+      process.stdout.write(`  Fetching: ${fileName}\n`);
+    }
+    return;
+  }
+
+  if (status === 'download') {
+    if (loaded != null) downloadedBytes = loaded;
+    if (total != null) totalBytes = total;
+    return;
+  }
+
+  if (status === 'progress' && progress != null) {
+    const pct = Math.min(100, Math.round(progress));
+    const bar = renderProgressBar(pct);
+    const elapsed = (Date.now() - startTime) / 1000;
+    const eta = pct > 0 ? (elapsed / pct) * (100 - pct) : 0;
+    const sizeInfo = totalBytes > 0
+      ? `${formatBytes(downloadedBytes)} / ${formatBytes(totalBytes)}`
+      : (downloadedBytes > 0 ? formatBytes(downloadedBytes) : '');
+    const etaInfo = eta > 1 ? `  ETA: ${formatTime(eta)}` : '';
+
+    process.stdout.write(
+      `\r  ${bar} ${String(pct).padStart(3)}%${sizeInfo ? '  ' + sizeInfo : ''}${etaInfo}   `
+    );
+    return;
+  }
+
+  if (status === 'done') {
+    // Clear the progress line and print completion
+    process.stdout.write('\r' + ' '.repeat(80) + '\r');
+    const fileName = (file ?? lastFile).split('/').pop() ?? '';
+    if (fileName) {
+      console.log(`  [OK] ${fileName}`);
+    }
+    return;
+  }
+
+  if (status === 'ready') {
+    return; // Pipeline ready — handled below
+  }
+}
+
+// Race setup against a timeout
+const timeoutPromise = new Promise((_, reject) =>
+  setTimeout(() => reject(new Error('timeout')), TIMEOUT_MS)
+);
+
+const setupPromise = (async () => {
+  const { pipeline, env } = await import('@xenova/transformers');
+
+  // Respect custom cache dir if set, otherwise use HuggingFace default
+  env.cacheDir = process.env['TRANSFORMERS_CACHE'] ?? undefined;
+
+  const pipe = await pipeline('feature-extraction', MODEL_NAME, {
+    quantized: true,
+    progress_callback: onProgress,
+  });
+
+  // Run a quick smoke test to make sure the model is functional
+  const testOutput = await pipe('test', { pooling: 'mean', normalize: true });
+  const dims = testOutput.data.length;
+
+  return dims;
+})();
+
+try {
+  const dims = await Promise.race([setupPromise, timeoutPromise]);
+
+  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  console.log('');
+  console.log(`Setup complete in ${elapsed}s`);
+  console.log(`Model output: ${dims}-dimensional embeddings`);
+  console.log('');
+  console.log('Stellar Memory is ready. Start with:');
+  console.log('  npm run dev    (MCP server)');
+  console.log('  npm run api    (REST API)');
+} catch (err) {
+  if (err.message === 'timeout') {
+    console.error('');
+    console.error('ERROR: Download timed out after 5 minutes.');
+    console.error('');
+    console.error('Possible causes:');
+    console.error('  - Slow internet connection');
+    console.error('  - HuggingFace servers unreachable');
+    console.error('  - Firewall blocking the download');
+    console.error('');
+    console.error('To retry:  npm run setup');
+    console.error('');
+    console.error('If the problem persists, you can set a custom cache directory:');
+    console.error('  TRANSFORMERS_CACHE=/path/to/cache npm run setup');
+  } else {
+    console.error('');
+    console.error('ERROR: Model download failed.');
+    console.error('');
+    console.error('Details:', err.message);
+    console.error('');
+    console.error('To retry:  npm run setup');
+  }
+  process.exit(1);
+}