promptgraph-mcp 1.1.0 → 1.3.0

package/README.md

@@ -128,7 +128,8 @@ promptgraph-mcp reindex   # Re-index all skills
 |---|---|---|
 | Skills in context | All 40+ | 1 (router) |
 | Tokens per session | ~20,000–50,000 | ~150 + 1 skill |
-| Scales to | ~50 skills | 5,000+ skills |
+
+> **Search:** Uses HNSW index (via [vectra](https://github.com/Stevenic/vectra)) for O(log N) approximate nearest neighbor search. Falls back to brute-force on first run before index is built.
 
 ## File Structure
 

package/ann.js

@@ -0,0 +1,50 @@
+import { LocalIndex } from 'vectra';
+import path from 'path';
+import os from 'os';
+import { getDb } from './db.js';
+
+const INDEX_PATH = path.join(os.homedir(), '.claude', '.promptgraph', 'hnsw-index');
+
+let _index = null;
+
+async function getIndex() {
+  if (_index) return _index;
+  _index = new LocalIndex(INDEX_PATH);
+  if (!await _index.isIndexCreated()) {
+    await _index.createIndex({ version: 1, deleteIfExists: true });
+  }
+  return _index;
+}
+
+export async function buildAnnIndex() {
+  const index = await getIndex();
+  await index.createIndex({ version: 1, deleteIfExists: true });
+
+  const db = getDb();
+  const chunks = db.prepare('SELECT skill_id, chunk_index, embedding FROM chunks').all();
+
+  for (const chunk of chunks) {
+    const vec = JSON.parse(chunk.embedding);
+    await index.insertItem({
+      vector: vec,
+      metadata: { skill_id: chunk.skill_id, chunk_index: chunk.chunk_index },
+    });
+  }
+
+  console.error(`[PromptGraph] ANN index built: ${chunks.length} chunks`);
+}
+
+export async function annSearch(queryVec, topK = 20) {
+  try {
+    const index = await getIndex();
+    if (!await index.isIndexCreated()) return null;
+
+    const results = await index.queryItems(queryVec, topK);
+    return results.map(r => ({
+      skill_id: r.item.metadata.skill_id,
+      score: r.score,
+    }));
+  } catch {
+    return null;
+  }
+}

package/db.js

@@ -1,40 +1,44 @@
-import Database from 'better-sqlite3';
-import path from 'path';
-import os from 'os';
-import fs from 'fs';
-
-const DB_PATH = path.join(os.homedir(), '.claude', '.promptgraph', 'promptgraph.db');
-
-export function getDb() {
-  fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
-  const db = new Database(DB_PATH);
-  db.pragma('journal_mode = WAL');
-
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS skills (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      name TEXT UNIQUE NOT NULL,
-      description TEXT,
-      path TEXT NOT NULL,
-      source TEXT NOT NULL,
-      content TEXT NOT NULL
-    );
-
-    CREATE TABLE IF NOT EXISTS chunks (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      skill_name TEXT NOT NULL,
-      chunk_index INTEGER NOT NULL,
-      text TEXT NOT NULL,
-      embedding TEXT NOT NULL,
-      UNIQUE(skill_name, chunk_index)
-    );
-
-    CREATE TABLE IF NOT EXISTS edges (
-      from_skill TEXT NOT NULL,
-      to_skill TEXT NOT NULL,
-      PRIMARY KEY (from_skill, to_skill)
-    );
-  `);
-
-  return db;
-}
+import Database from 'better-sqlite3';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+
+const DB_PATH = path.join(os.homedir(), '.claude', '.promptgraph', 'promptgraph.db');
+
+export function getDb() {
+  fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
+  const db = new Database(DB_PATH);
+  db.pragma('journal_mode = WAL');
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS skills (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      description TEXT,
+      path TEXT NOT NULL,
+      source TEXT NOT NULL,
+      content TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS chunks (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      skill_id TEXT NOT NULL,
+      chunk_index INTEGER NOT NULL,
+      text TEXT NOT NULL,
+      embedding TEXT NOT NULL,
+      UNIQUE(skill_id, chunk_index)
+    );
+
+    CREATE TABLE IF NOT EXISTS edges (
+      from_skill TEXT NOT NULL,
+      to_skill TEXT NOT NULL,
+      PRIMARY KEY (from_skill, to_skill)
+    );
+  `);
+
+  return db;
+}
+
+export function skillId(source, name) {
+  return `${source}::${name}`;
+}

package/github-import.js

@@ -2,6 +2,7 @@ import { execSync } from 'child_process';
 import path from 'path';
 import os from 'os';
 import fs from 'fs';
+import { globSync } from 'glob';
 import { indexAll } from './indexer.js';
 import { loadConfig, saveConfig } from './config.js';
 
@@ -27,10 +28,10 @@ export async function importFromGitHub(repoUrl) {
     execSync(`git clone --depth=1 ${url} "${dest}"`, { stdio: 'inherit' });
   }
 
-  const mdCount = execSync(`find "${dest}" -name "*.md" | wc -l`).toString().trim();
-  console.log(`Found ${mdCount} .md files`);
+  const mdFiles = globSync(`${dest}/**/*.md`);
+  console.log(`Found ${mdFiles.length} .md files`);
 
-  if (parseInt(mdCount) < 2) {
+  if (mdFiles.length < 2) {
     console.warn('Warning: repo has fewer than 2 .md files — may be empty');
   }
 
@@ -44,5 +45,5 @@ export async function importFromGitHub(repoUrl) {
 
   console.log('\nReindexing...');
   await indexAll();
-  console.log(`Done! Imported from ${repoName}`);
+  console.log(`Done! Imported ${mdFiles.length} files from ${repoName}`);
 }

package/indexer.js

@@ -1,37 +1,37 @@
 import { globSync } from 'glob';
 import { parseSkillFile } from './parser.js';
 import { embed } from './embedder.js';
-import { getDb } from './db.js';
+import { getDb, skillId } from './db.js';
 import { loadConfig } from './config.js';
 import { chunkText } from './chunker.js';
+import { buildAnnIndex } from './ann.js';
 
 async function indexSkill(db, skill) {
+  const id = skillId(skill.source, skill.name);
+
   db.prepare(`
-    INSERT INTO skills (name, description, path, source, content)
-    VALUES (@name, @description, @path, @source, @content)
-    ON CONFLICT(name) DO UPDATE SET
+    INSERT INTO skills (id, name, description, path, source, content)
+    VALUES (@id, @name, @description, @path, @source, @content)
+    ON CONFLICT(id) DO UPDATE SET
+      name = excluded.name,
       description = excluded.description,
       path = excluded.path,
-      source = excluded.source,
       content = excluded.content
-  `).run({ name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content });
+  `).run({ id, name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content });
 
-  db.prepare('DELETE FROM chunks WHERE skill_name = ?').run(skill.name);
+  db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(id);
 
   const chunks = chunkText(skill.name + ' ' + skill.description + '\n' + skill.content);
-  const upsertChunk = db.prepare(`
-    INSERT OR REPLACE INTO chunks (skill_name, chunk_index, text, embedding)
-    VALUES (?, ?, ?, ?)
-  `);
+  const upsertChunk = db.prepare(`INSERT OR REPLACE INTO chunks (skill_id, chunk_index, text, embedding) VALUES (?, ?, ?, ?)`);
   for (let i = 0; i < chunks.length; i++) {
     const vec = await embed(chunks[i]);
-    upsertChunk.run(skill.name, i, chunks[i], JSON.stringify(vec));
+    upsertChunk.run(id, i, chunks[i], JSON.stringify(vec));
   }
 
-  db.prepare('DELETE FROM edges WHERE from_skill = ?').run(skill.name);
+  db.prepare('DELETE FROM edges WHERE from_skill = ?').run(id);
   const upsertEdge = db.prepare('INSERT OR IGNORE INTO edges (from_skill, to_skill) VALUES (?, ?)');
   for (const called of skill.calls) {
-    upsertEdge.run(skill.name, called);
+    upsertEdge.run(id, called);
   }
 }
 
@@ -54,6 +54,8 @@ export async function indexAll() {
       }
     }
   }
+  process.stdout.write('\r  Building ANN index...');
+  await buildAnnIndex();
   console.log(`\n  Done. ${count} skills indexed.`);
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "main": "index.js",
   "type": "module",
   "bin": {
@@ -37,6 +37,7 @@
     "chokidar": "^5.0.0",
     "fastembed": "^2.1.0",
     "glob": "^13.0.6",
-    "gray-matter": "^4.0.3"
+    "gray-matter": "^4.0.3",
+    "vectra": "^0.15.0"
   }
 }

package/parser.js

@@ -1,47 +1,36 @@
-import fs from 'fs';
-import path from 'path';
-
-const SKILL_REF_RE = /\/([a-z0-9][a-z0-9-]+)/g;
-const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---/;
-
-export function parseSkillFile(filePath, source) {
-  const raw = fs.readFileSync(filePath, 'utf8');
-
-  let name = null;
-  let description = null;
-  let content = raw;
-
-  const fm = raw.match(FRONTMATTER_RE);
-  if (fm) {
-    try {
-      const nameMatch = fm[1].match(/^name:\s*(.+)$/m);
-      const descMatch = fm[1].match(/^description:\s*(.+)$/m);
-      if (nameMatch) name = nameMatch[1].trim().replace(/^["']|["']$/g, '');
-      if (descMatch) description = descMatch[1].trim().replace(/^["']|["']$/g, '');
-    } catch {}
-  }
-
-  name = name || path.basename(filePath, '.md');
-  description = description || extractFirstParagraph(raw);
-
-  const calls = new Set();
-  for (const match of raw.matchAll(SKILL_REF_RE)) {
-    const ref = match[1];
-    if (ref !== name && ref.length > 2) calls.add(ref);
-  }
-
-  return {
-    name,
-    description,
-    path: filePath,
-    source,
-    content: raw,
-    calls: [...calls],
-  };
-}
-
-function extractFirstParagraph(content) {
-  const lines = content.replace(FRONTMATTER_RE, '').split('\n')
-    .filter(l => l.trim() && !l.startsWith('#') && !l.startsWith('---'));
-  return lines[0]?.trim().slice(0, 200) || '';
-}
+import fs from 'fs';
+import path from 'path';
+import matter from 'gray-matter';
+
+const SKILL_REF_RE = /\/([a-z0-9][a-z0-9-]+)/g;
+
+export function parseSkillFile(filePath, source) {
+  const raw = fs.readFileSync(filePath, 'utf8');
+
+  let name, description, content;
+
+  try {
+    const { data, content: body } = matter(raw);
+    name = data.name;
+    description = data.description;
+    content = body;
+  } catch {
+    content = raw;
+  }
+
+  name = (name && String(name).trim()) || path.basename(filePath, '.md');
+  description = (description && String(description).trim()) || extractFirstParagraph(content || raw);
+
+  const calls = new Set();
+  for (const match of raw.matchAll(SKILL_REF_RE)) {
+    const ref = match[1];
+    if (ref !== name && ref.length > 2) calls.add(ref);
+  }
+
+  return { name, description, path: filePath, source, content: raw, calls: [...calls] };
+}
+
+function extractFirstParagraph(content) {
+  const lines = content.split('\n').filter(l => l.trim() && !l.startsWith('#') && !l.startsWith('---'));
+  return lines[0]?.trim().slice(0, 200) || '';
+}

package/search.js

@@ -1,54 +1,72 @@
 import { embed, cosineSimilarity } from './embedder.js';
 import { getDb } from './db.js';
+import { annSearch } from './ann.js';
 
 export async function search(query, topK = 5) {
   const db = getDb();
   const queryVec = await embed(query);
 
-  // RAG: search over chunks, deduplicate by skill
-  const chunks = db.prepare('SELECT skill_name, embedding FROM chunks').all();
+  // Try ANN index first (fast, O(log N))
+  const annResults = await annSearch(queryVec, topK * 4);
 
+  if (annResults && annResults.length > 0) {
+    const bestBySkill = new Map();
+    for (const r of annResults) {
+      const prev = bestBySkill.get(r.skill_id);
+      if (!prev || r.score > prev) bestBySkill.set(r.skill_id, r.score);
+    }
+    return [...bestBySkill.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, topK)
+      .map(([id, score]) => {
+        const skill = db.prepare('SELECT id, name, description, path, source FROM skills WHERE id = ?').get(id);
+        return skill ? { ...skill, score } : null;
+      })
+      .filter(Boolean);
+  }
+
+  // Fallback: brute force (used before first reindex)
+  const chunks = db.prepare('SELECT skill_id, embedding FROM chunks').all();
   const bestBySkill = new Map();
   for (const chunk of chunks) {
     const score = cosineSimilarity(queryVec, JSON.parse(chunk.embedding));
-    const prev = bestBySkill.get(chunk.skill_name);
-    if (!prev || score > prev) bestBySkill.set(chunk.skill_name, score);
+    const prev = bestBySkill.get(chunk.skill_id);
+    if (!prev || score > prev) bestBySkill.set(chunk.skill_id, score);
   }
-
-  const skillNames = [...bestBySkill.entries()]
+  return [...bestBySkill.entries()]
     .sort((a, b) => b[1] - a[1])
     .slice(0, topK)
-    .map(([name]) => name);
-
-  return skillNames.map(name => {
-    const skill = db.prepare('SELECT name, description, path, source FROM skills WHERE name = ?').get(name);
-    return { ...skill, score: bestBySkill.get(name) };
-  });
+    .map(([id, score]) => {
+      const skill = db.prepare('SELECT id, name, description, path, source FROM skills WHERE id = ?').get(id);
+      return skill ? { ...skill, score } : null;
+    })
+    .filter(Boolean);
 }
 
-export function getContext(name) {
+export function getContext(id) {
   const db = getDb();
-  const skill = db.prepare('SELECT * FROM skills WHERE name = ?').get(name);
+  const skill = db.prepare('SELECT * FROM skills WHERE id = ?').get(id)
+    || db.prepare('SELECT * FROM skills WHERE name = ? ORDER BY id LIMIT 1').get(id);
   if (!skill) return null;
-  const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(name).map(r => r.to_skill);
-  const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(name).map(r => r.from_skill);
+  const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(skill.id).map(r => r.to_skill);
+  const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(skill.id).map(r => r.from_skill);
   return { ...skill, callees, callers };
 }
 
-export function getCallers(name) {
+export function getCallers(id) {
   const db = getDb();
-  return db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(name).map(r => r.from_skill);
+  return db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(id).map(r => r.from_skill);
 }
 
-export function getCallees(name) {
+export function getCallees(id) {
   const db = getDb();
-  return db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(name).map(r => r.to_skill);
+  return db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(id).map(r => r.to_skill);
 }
 
-export function getImpact(name) {
+export function getImpact(id) {
   const db = getDb();
   const visited = new Set();
-  const queue = [name];
+  const queue = [id];
   while (queue.length) {
     const cur = queue.shift();
     if (visited.has(cur)) continue;
@@ -56,11 +74,11 @@ export function getImpact(name) {
     const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(cur).map(r => r.from_skill);
     queue.push(...callers);
   }
-  visited.delete(name);
+  visited.delete(id);
   return [...visited];
 }
 
 export function listAll() {
   const db = getDb();
-  return db.prepare('SELECT name, description, source FROM skills ORDER BY name').all();
+  return db.prepare('SELECT id, name, description, source FROM skills ORDER BY source, name').all();
 }