wolverine-ai 1.0.0

package/src/brain/embedder.js

@@ -0,0 +1,131 @@
+const { getClient, aiCall } = require("../core/ai-client");
+const { getModel } = require("../core/models");
+
+/**
+ * Embedder — converts text to vector embeddings using TEXT_EMBEDDING_MODEL.
+ *
+ * Optimized for speed:
+ * - Batch embedding support (multiple texts in one API call)
+ * - LRU cache for repeated queries
+ * - Dimensionality matches text-embedding-3-small (1536) by default
+ */
+
+// Simple LRU cache for embedding results
+const CACHE_SIZE = 500;
+const _cache = new Map();
+
+function _cacheGet(key) {
+  if (_cache.has(key)) {
+    const val = _cache.get(key);
+    // Move to end (most recently used)
+    _cache.delete(key);
+    _cache.set(key, val);
+    return val;
+  }
+  return null;
+}
+
+function _cacheSet(key, val) {
+  if (_cache.size >= CACHE_SIZE) {
+    // Evict oldest (first entry)
+    const oldest = _cache.keys().next().value;
+    _cache.delete(oldest);
+  }
+  _cache.set(key, val);
+}
+
+/**
+ * Embed a single text string. Returns number[].
+ */
+async function embed(text) {
+  const cached = _cacheGet(text);
+  if (cached) return cached;
+
+  const openai = getClient();
+  const model = getModel("embedding");
+
+  const response = await openai.embeddings.create({
+    model,
+    input: text,
+  });
+
+  const embedding = response.data[0].embedding;
+  _cacheSet(text, embedding);
+  return embedding;
+}
+
+/**
+ * Embed multiple texts in a single API call. Returns number[][].
+ * Much faster than calling embed() in a loop.
+ */
+async function embedBatch(texts) {
+  if (texts.length === 0) return [];
+
+  // Check cache for all
+  const results = new Array(texts.length);
+  const uncached = [];
+  const uncachedIndices = [];
+
+  for (let i = 0; i < texts.length; i++) {
+    const cached = _cacheGet(texts[i]);
+    if (cached) {
+      results[i] = cached;
+    } else {
+      uncached.push(texts[i]);
+      uncachedIndices.push(i);
+    }
+  }
+
+  if (uncached.length === 0) return results;
+
+  const openai = getClient();
+  const model = getModel("embedding");
+
+  const response = await openai.embeddings.create({
+    model,
+    input: uncached,
+  });
+
+  // Sort by index to maintain order
+  const sorted = response.data.sort((a, b) => a.index - b.index);
+
+  for (let i = 0; i < sorted.length; i++) {
+    const embedding = sorted[i].embedding;
+    const originalIdx = uncachedIndices[i];
+    results[originalIdx] = embedding;
+    _cacheSet(uncached[i], embedding);
+  }
+
+  return results;
+}
+
+/**
+ * Compact text using UTILITY_MODEL before embedding.
+ * Reduces token count while preserving semantic meaning.
+ * This makes embeddings more focused and search more accurate.
+ */
+async function compact(text) {
+  // Skip compaction for short texts
+  if (text.length < 200) return text;
+
+  const result = await aiCall({
+    model: getModel("utility"),
+    systemPrompt: "Compress the following text into a dense, semantically rich summary. Keep all technical terms, function names, file paths, and error messages. Remove filler words. Output ONLY the compressed text, nothing else.",
+    userPrompt: text,
+    maxTokens: 256,
+    category: "brain",
+  });
+
+  return result.content || text;
+}
+
+/**
+ * Full pipeline: compact → embed. Returns { compacted, embedding }.
+ */
+async function compactAndEmbed(text) {
+  const compacted = await compact(text);
+  const embedding = await embed(compacted);
+  return { compacted, embedding };
+}
+
+module.exports = { embed, embedBatch, compact, compactAndEmbed };

package/src/brain/function-map.js

@@ -0,0 +1,263 @@
+const fs = require("fs");
+const path = require("path");
+
+/**
+ * Live Function Map — scans the server project on startup to build
+ * a complete map of routes, exports, functions, and modules.
+ *
+ * This gives the agent instant context about what the server IS and
+ * what it's capable of, without reading every file at heal time.
+ *
+ * Scans for:
+ * - HTTP route handlers (express, http, fastify, koa patterns)
+ * - Module exports (module.exports, exports.X)
+ * - Function declarations and arrow functions
+ * - Class definitions
+ * - Package.json dependencies
+ * - Config files (.env, .json, .yaml)
+ */
+
+const SKIP_DIRS = new Set(["node_modules", ".wolverine", ".git", "dist", "build", "coverage", "src", "bin", "tests"]);
+const CODE_EXTENSIONS = new Set([".js", ".ts", ".mjs", ".cjs", ".jsx", ".tsx"]);
+const CONFIG_EXTENSIONS = new Set([".json", ".yaml", ".yml", ".toml", ".env"]);
+
+/**
+ * Scan the project and return a structured function map.
+ */
+function scanProject(projectRoot) {
+  const root = path.resolve(projectRoot);
+  const map = {
+    routes: [],
+    exports: [],
+    functions: [],
+    classes: [],
+    files: [],
+    configs: [],
+    dependencies: {},
+    summary: "",
+  };
+
+  // Scan package.json for dependencies
+  const pkgPath = path.join(root, "package.json");
+  if (fs.existsSync(pkgPath)) {
+    try {
+      const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
+      map.dependencies = {
+        ...(pkg.dependencies || {}),
+        ...(pkg.devDependencies || {}),
+      };
+    } catch { /* skip */ }
+  }
+
+  // Recursive scan
+  _scanDir(root, root, map);
+
+  // Build summary
+  map.summary = _buildSummary(map);
+
+  return map;
+}
+
+function _scanDir(dir, root, map) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch { return; }
+
+  for (const entry of entries) {
+    if (SKIP_DIRS.has(entry.name)) continue;
+    if (entry.name.startsWith(".") && entry.name !== ".env") continue;
+
+    const fullPath = path.join(dir, entry.name);
+    const relPath = path.relative(root, fullPath).replace(/\\/g, "/");
+
+    if (entry.isDirectory()) {
+      _scanDir(fullPath, root, map);
+      continue;
+    }
+
+    const ext = path.extname(entry.name);
+
+    if (CONFIG_EXTENSIONS.has(ext) || entry.name.startsWith(".env")) {
+      map.configs.push(relPath);
+      map.files.push({ path: relPath, type: "config" });
+      continue;
+    }
+
+    if (!CODE_EXTENSIONS.has(ext)) continue;
+
+    map.files.push({ path: relPath, type: "code" });
+
+    // Parse the file for patterns
+    let content;
+    try {
+      content = fs.readFileSync(fullPath, "utf-8");
+    } catch { continue; }
+
+    _extractRoutes(content, relPath, map);
+    _extractExports(content, relPath, map);
+    _extractFunctions(content, relPath, map);
+    _extractClasses(content, relPath, map);
+  }
+}
+
+function _extractRoutes(content, filePath, map) {
+  // Express-style routes: app.get('/path', ...), router.post('/path', ...)
+  const expressPattern = /(?:app|router)\.(get|post|put|delete|patch|use|all)\s*\(\s*['"`]([^'"`]+)['"`]/gi;
+  let match;
+  while ((match = expressPattern.exec(content)) !== null) {
+    map.routes.push({ method: match[1].toUpperCase(), path: match[2], file: filePath });
+  }
+
+  // http.createServer with URL checks: req.url === '/path'
+  const httpPattern = /req\.url\s*===?\s*['"`]([^'"`]+)['"`]/gi;
+  while ((match = httpPattern.exec(content)) !== null) {
+    // Try to find method from nearby req.method check
+    const nearby = content.slice(Math.max(0, match.index - 200), match.index + 200);
+    const methodMatch = nearby.match(/req\.method\s*===?\s*['"`]([^'"`]+)['"`]/i);
+    map.routes.push({
+      method: methodMatch ? methodMatch[1].toUpperCase() : "*",
+      path: match[1],
+      file: filePath,
+    });
+  }
+
+  // Fastify-style: fastify.get('/path', ...)
+  const fastifyPattern = /fastify\.(get|post|put|delete|patch)\s*\(\s*['"`]([^'"`]+)['"`]/gi;
+  while ((match = fastifyPattern.exec(content)) !== null) {
+    map.routes.push({ method: match[1].toUpperCase(), path: match[2], file: filePath });
+  }
+}
+
+function _extractExports(content, filePath, map) {
+  // module.exports = { ... } or module.exports = functionName
+  const moduleExport = content.match(/module\.exports\s*=\s*(?:\{([^}]+)\}|(\w+))/);
+  if (moduleExport) {
+    if (moduleExport[1]) {
+      // Object exports: { foo, bar, baz }
+      const names = moduleExport[1].split(",").map(s => s.trim().split(":")[0].trim()).filter(Boolean);
+      for (const name of names) {
+        map.exports.push({ name, file: filePath });
+      }
+    } else if (moduleExport[2]) {
+      map.exports.push({ name: moduleExport[2], file: filePath });
+    }
+  }
+
+  // exports.name = ...
+  const namedExports = content.matchAll(/exports\.(\w+)\s*=/g);
+  for (const m of namedExports) {
+    map.exports.push({ name: m[1], file: filePath });
+  }
+}
+
+function _extractFunctions(content, filePath, map) {
+  // function declarations
+  const funcDecls = content.matchAll(/(?:async\s+)?function\s+(\w+)\s*\(/g);
+  for (const m of funcDecls) {
+    map.functions.push({ name: m[0].includes("async") ? `async ${m[1]}` : m[1], file: filePath });
+  }
+
+  // Named arrow functions: const foo = (...) => or const foo = async (...) =>
+  const arrowFuncs = content.matchAll(/(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[a-zA-Z_]\w*)\s*=>/g);
+  for (const m of arrowFuncs) {
+    map.functions.push({ name: m[1], file: filePath });
+  }
+}
+
+function _extractClasses(content, filePath, map) {
+  const classDecls = content.matchAll(/class\s+(\w+)(?:\s+extends\s+(\w+))?\s*\{/g);
+  for (const m of classDecls) {
+    map.classes.push({ name: m[1], extends: m[2] || null, file: filePath });
+  }
+}
+
+function _buildSummary(map) {
+  const lines = [];
+  lines.push(`Project: ${map.files.length} files (${map.files.filter(f => f.type === "code").length} code, ${map.configs.length} config)`);
+  lines.push(`Dependencies: ${Object.keys(map.dependencies).length}`);
+
+  if (map.routes.length > 0) {
+    lines.push(`Routes (${map.routes.length}):`);
+    for (const r of map.routes.slice(0, 20)) {
+      lines.push(`  ${r.method.padEnd(7)} ${r.path} → ${r.file}`);
+    }
+    if (map.routes.length > 20) lines.push(`  ... and ${map.routes.length - 20} more`);
+  }
+
+  if (map.classes.length > 0) {
+    lines.push(`Classes: ${map.classes.map(c => c.name).join(", ")}`);
+  }
+
+  if (map.exports.length > 0) {
+    lines.push(`Exports (${map.exports.length}): ${map.exports.slice(0, 15).map(e => e.name).join(", ")}${map.exports.length > 15 ? "..." : ""}`);
+  }
+
+  if (map.functions.length > 0) {
+    lines.push(`Functions (${map.functions.length}): ${map.functions.slice(0, 15).map(f => f.name).join(", ")}${map.functions.length > 15 ? "..." : ""}`);
+  }
+
+  return lines.join("\n");
+}
+
+/**
+ * Convert the function map to embeddable text chunks.
+ * Each chunk is focused on one aspect for precise vector search.
+ */
+function mapToChunks(map) {
+  const chunks = [];
+
+  // Routes chunk
+  if (map.routes.length > 0) {
+    chunks.push({
+      namespace: "functions",
+      text: `HTTP Routes:\n${map.routes.map(r => `${r.method} ${r.path} (${r.file})`).join("\n")}`,
+      metadata: { type: "routes", count: map.routes.length },
+    });
+  }
+
+  // Per-file function chunks
+  const byFile = {};
+  for (const fn of map.functions) {
+    if (!byFile[fn.file]) byFile[fn.file] = [];
+    byFile[fn.file].push(fn.name);
+  }
+  for (const [file, fns] of Object.entries(byFile)) {
+    chunks.push({
+      namespace: "functions",
+      text: `File ${file} functions: ${fns.join(", ")}`,
+      metadata: { type: "file-functions", file },
+    });
+  }
+
+  // Dependencies chunk
+  if (Object.keys(map.dependencies).length > 0) {
+    chunks.push({
+      namespace: "functions",
+      text: `Dependencies: ${Object.entries(map.dependencies).map(([k, v]) => `${k}@${v}`).join(", ")}`,
+      metadata: { type: "dependencies" },
+    });
+  }
+
+  // Classes chunk
+  if (map.classes.length > 0) {
+    chunks.push({
+      namespace: "functions",
+      text: `Classes: ${map.classes.map(c => `${c.name}${c.extends ? ` extends ${c.extends}` : ""} (${c.file})`).join(", ")}`,
+      metadata: { type: "classes" },
+    });
+  }
+
+  // Config files chunk
+  if (map.configs.length > 0) {
+    chunks.push({
+      namespace: "functions",
+      text: `Config files: ${map.configs.join(", ")}`,
+      metadata: { type: "configs" },
+    });
+  }
+
+  return chunks;
+}
+
+module.exports = { scanProject, mapToChunks };

package/src/brain/vector-store.js

@@ -0,0 +1,267 @@
+const fs = require("fs");
+const path = require("path");
+
+/**
+ * In-memory vector store with file persistence.
+ *
+ * Design priorities:
+ * 1. SPEED — everything in RAM, cosine similarity is just dot products
+ * 2. Persistence — saved to .wolverine/brain/vectors.bin for restart survival
+ * 3. No dependencies — pure JS, no external vector DB needed
+ *
+ * Storage: each entry is { id, namespace, text, metadata, embedding: Float32Array }
+ * Namespaces partition the store: "docs", "errors", "fixes", "functions", "learnings"
+ */
+
+const BRAIN_DIR = ".wolverine/brain";
+const STORE_FILE = "vectors.json";
+
+class VectorStore {
+  constructor(projectRoot) {
+    this.projectRoot = path.resolve(projectRoot);
+    this.brainDir = path.join(this.projectRoot, BRAIN_DIR);
+    this.storePath = path.join(this.brainDir, STORE_FILE);
+
+    // In-memory entries: Map<id, Entry>
+    this._entries = new Map();
+    // Namespace index for fast filtered search: Map<namespace, Set<id>>
+    this._nsIndex = new Map();
+    // Auto-increment ID
+    this._nextId = 1;
+
+    this._ensureDir();
+    this._load();
+  }
+
+  /**
+   * Add an entry to the store. Returns the entry ID.
+   *
+   * @param {string} namespace - Category: "docs", "errors", "fixes", "functions", "learnings"
+   * @param {string} text - The compacted text (what gets searched against)
+   * @param {number[]} embedding - Float array from the embedding model
+   * @param {object} metadata - Arbitrary metadata (timestamps, file paths, etc.)
+   */
+  add(namespace, text, embedding, metadata = {}) {
+    const id = `${namespace}-${(this._nextId++).toString(36)}`;
+    const entry = {
+      id,
+      namespace,
+      text,
+      metadata: { ...metadata, createdAt: Date.now() },
+      embedding: new Float32Array(embedding),
+    };
+
+    this._entries.set(id, entry);
+
+    if (!this._nsIndex.has(namespace)) {
+      this._nsIndex.set(namespace, new Set());
+    }
+    this._nsIndex.get(namespace).add(id);
+
+    return id;
+  }
+
+  /**
+   * Semantic search — find the top-k most similar entries.
+   *
+   * @param {number[]} queryEmbedding - Embedding of the search query
+   * @param {object} options
+   * @param {number} options.topK - Max results (default: 5)
+   * @param {string} options.namespace - Filter to a specific namespace
+   * @param {number} options.minScore - Minimum similarity score (default: 0.3)
+   * @returns {Array<{ id, namespace, text, metadata, score }>}
+   */
+  search(queryEmbedding, { topK = 5, namespace, minScore = 0.3 } = {}) {
+    const queryVec = new Float32Array(queryEmbedding);
+    const results = [];
+
+    // Determine which entries to search
+    let entryIds;
+    if (namespace && this._nsIndex.has(namespace)) {
+      entryIds = this._nsIndex.get(namespace);
+    } else if (namespace) {
+      return []; // namespace doesn't exist
+    } else {
+      entryIds = this._entries.keys();
+    }
+
+    for (const id of entryIds) {
+      const entry = this._entries.get(id);
+      if (!entry) continue;
+
+      const score = cosineSimilarity(queryVec, entry.embedding);
+      if (score >= minScore) {
+        results.push({
+          id: entry.id,
+          namespace: entry.namespace,
+          text: entry.text,
+          metadata: entry.metadata,
+          score,
+        });
+      }
+    }
+
+    // Sort by score descending, take topK
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, topK);
+  }
+
+  /**
+   * Fast keyword search — no embedding API call, instant.
+   * Tokenizes query and scores entries by keyword overlap.
+   * Use as first-pass before expensive semantic search.
+   */
+  keywordSearch(query, { topK = 5, namespace, minTokens = 2 } = {}) {
+    const tokens = query.toLowerCase()
+      .replace(/[^a-z0-9\s]/g, " ")
+      .split(/\s+/)
+      .filter(t => t.length > 2);
+
+    if (tokens.length === 0) return [];
+
+    const results = [];
+    let entryIds;
+    if (namespace && this._nsIndex.has(namespace)) {
+      entryIds = this._nsIndex.get(namespace);
+    } else {
+      entryIds = this._entries.keys();
+    }
+
+    for (const id of entryIds) {
+      const entry = this._entries.get(id);
+      if (!entry) continue;
+
+      const textLower = entry.text.toLowerCase();
+      let score = 0;
+      for (const token of tokens) {
+        if (textLower.includes(token)) score++;
+      }
+
+      if (score >= minTokens) {
+        results.push({
+          id: entry.id,
+          namespace: entry.namespace,
+          text: entry.text,
+          metadata: entry.metadata,
+          score: score / tokens.length, // normalize 0-1
+        });
+      }
+    }
+
+    results.sort((a, b) => b.score - a.score);
+    return results.slice(0, topK);
+  }
+
+  /**
+   * Get all entries in a namespace.
+   */
+  getNamespace(namespace) {
+    const ids = this._nsIndex.get(namespace);
+    if (!ids) return [];
+    return Array.from(ids).map(id => {
+      const e = this._entries.get(id);
+      return { id: e.id, namespace: e.namespace, text: e.text, metadata: e.metadata };
+    });
+  }
+
+  /**
+   * Delete an entry by ID.
+   */
+  delete(id) {
+    const entry = this._entries.get(id);
+    if (!entry) return false;
+    this._entries.delete(id);
+    const nsSet = this._nsIndex.get(entry.namespace);
+    if (nsSet) nsSet.delete(id);
+    return true;
+  }
+
+  /**
+   * Get store stats.
+   */
+  getStats() {
+    const nsCounts = {};
+    for (const [ns, ids] of this._nsIndex) {
+      nsCounts[ns] = ids.size;
+    }
+    return { totalEntries: this._entries.size, namespaces: nsCounts };
+  }
+
+  /**
+   * Persist to disk. Call periodically or after batch operations.
+   */
+  save() {
+    const data = {
+      version: 1,
+      nextId: this._nextId,
+      entries: [],
+    };
+
+    for (const entry of this._entries.values()) {
+      data.entries.push({
+        id: entry.id,
+        namespace: entry.namespace,
+        text: entry.text,
+        metadata: entry.metadata,
+        embedding: Array.from(entry.embedding),
+      });
+    }
+
+    // Atomic write: write to temp file, then rename (prevents corruption on kill)
+    const tmpPath = this.storePath + ".tmp";
+    fs.writeFileSync(tmpPath, JSON.stringify(data), "utf-8");
+    fs.renameSync(tmpPath, this.storePath);
+  }
+
+  // -- Private --
+
+  _ensureDir() {
+    fs.mkdirSync(this.brainDir, { recursive: true });
+  }
+
+  _load() {
+    if (!fs.existsSync(this.storePath)) return;
+
+    try {
+      const data = JSON.parse(fs.readFileSync(this.storePath, "utf-8"));
+      this._nextId = data.nextId || 1;
+
+      for (const entry of data.entries) {
+        const stored = {
+          id: entry.id,
+          namespace: entry.namespace,
+          text: entry.text,
+          metadata: entry.metadata,
+          embedding: new Float32Array(entry.embedding),
+        };
+        this._entries.set(stored.id, stored);
+
+        if (!this._nsIndex.has(stored.namespace)) {
+          this._nsIndex.set(stored.namespace, new Set());
+        }
+        this._nsIndex.get(stored.namespace).add(stored.id);
+      }
+    } catch {
+      // Corrupt store — start fresh
+      this._entries.clear();
+      this._nsIndex.clear();
+    }
+  }
+}
+
+/**
+ * Cosine similarity between two Float32Arrays.
+ * Returns value between -1 and 1 (higher = more similar).
+ */
+function cosineSimilarity(a, b) {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+module.exports = { VectorStore, cosineSimilarity };