promptgraph-mcp 2.4.7 → 2.5.0

package/README.md

@@ -154,12 +154,9 @@ Claude uses these automatically when the MCP server is running:
 
 ## Search modes
 
-| Mode | How | Quality |
-|---|---|---|
-| After `pg reindex` | Cosine similarity on 384-dim BGE vectors | Semantic — finds by meaning |
-| After `pg reindex --fast` | SQLite FTS5 BM25 | Keyword — exact word match |
-
-Search falls back to FTS5 automatically if no embeddings exist.
+Search falls back to FTS5 automatically if no embeddings exist. When embeddings are
+available, results are a hybrid of semantic similarity (embedding cosine) and keyword
+relevance (BM25) — giving you the best of both approaches.
 
 ---
 

package/ann.js

@@ -1,61 +1,33 @@
 import { getDb, blobToVec } from './db.js';
+import { getStore, resetStore } from './src/store/index.js';
 
-// In-memory flat index — no external dependency.
-// For typical skill counts (<5000) this is faster than vectra's disk-based HNSW
-// because all data fits in RAM and no I/O is needed per query.
-let _cache = null;
-let _cacheChunkCount = -1;
-
-function loadCache(db) {
-  const count = db.prepare('SELECT COUNT(*) as n FROM chunks').get().n;
-  if (_cache && count === _cacheChunkCount) return _cache;
-  const rows = db.prepare('SELECT skill_id, embedding FROM chunks').all();
-  _cache = rows.map(r => ({
-    skill_id: r.skill_id,
-    vec: new Float32Array(blobToVec(r.embedding)),
-  }));
-  _cacheChunkCount = count;
-  return _cache;
-}
-
-function cosineSim(a, b) {
-  let dot = 0, na = 0, nb = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot += a[i] * b[i];
-    na += a[i] * a[i];
-    nb += b[i] * b[i];
-  }
-  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-8);
-}
-
-// Called after reindex — invalidate cache so next search reloads
 export async function buildAnnIndex() {
-  _cache = null;
-  _cacheChunkCount = -1;
-  const db = getDb();
-  const count = db.prepare('SELECT COUNT(*) as n FROM chunks').get().n;
-  console.error(`[PromptGraph] In-memory index ready: ${count} chunks`);
+  try {
+    const db = getDb();
+    const rows = db.prepare('SELECT skill_id, embedding FROM chunks').all();
+    const entries = rows.map(r => ({
+      skill_id: r.skill_id,
+      vector: blobToVec(r.embedding),
+    }));
+    const store = getStore();
+    await store.build(entries);
+    console.error(`[PromptGraph] ANN index ready: ${store.size} vectors`);
+  } catch (e) {
+    console.error(`[PromptGraph] ANN build failed: ${e.message}`);
+  }
 }
 
 export async function annSearch(queryVec, topK = 20) {
   try {
-    const db = getDb();
-    const cache = loadCache(db);
-    if (!cache.length) return null;
-
-    const qArr = new Float32Array(queryVec);
-    const bestBySkill = new Map();
-    for (const entry of cache) {
-      const score = cosineSim(qArr, entry.vec);
-      const prev = bestBySkill.get(entry.skill_id);
-      if (!prev || score > prev) bestBySkill.set(entry.skill_id, score);
-    }
-
-    return [...bestBySkill.entries()]
-      .sort((a, b) => b[1] - a[1])
-      .slice(0, topK)
-      .map(([skill_id, score]) => ({ skill_id, score }));
-  } catch {
+    const store = getStore();
+    if (store.size === 0) return null;
+    return await store.search(queryVec, topK);
+  } catch (e) {
+    console.error(`[PromptGraph] ANN search error: ${e.message}`);
     return null;
   }
 }
+
+export function resetAnnIndex() {
+  resetStore();
+}

package/api.js

@@ -0,0 +1,200 @@
+import { search as internalSearch, getContext, listAll as internalList } from './search.js';
+import { indexAll, indexSource } from './indexer.js';
+import { getDb } from './db.js';
+import { loadConfig } from './config.js';
+
+/**
+ * Search for skills by query text.
+ * @param {string} query - Natural language search query
+ * @param {{ topK?: number, embedWeight?: number, bm25Weight?: number, source?: string, category?: string }} [options]
+ * @returns {Promise<Array<{id: string, name: string, description: string, source: string, score: number, snippet: string}>>}
+ */
+export async function search(query, options = {}) {
+  try {
+    const { topK = 5, source } = options;
+    const results = await internalSearch(query, source ? topK * 3 : topK);
+    if (source) {
+      return results.filter(r => r.source === source).slice(0, topK);
+    }
+    return results;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Index all .md files in a directory under a source name.
+ * @param {string} sourceDir - Path to directory containing .md skill files
+ * @param {string} sourceName - Source label stored with each skill
+ * @returns {Promise<{indexed: number, skipped: number, errors: number}>}
+ */
+export async function index(sourceDir, sourceName) {
+  try {
+    if (sourceDir.includes('..')) {
+      return { indexed: 0, skipped: 0, errors: 1, error: 'Path traversal detected in sourceDir' };
+    }
+    const db = getDb();
+    const { globSync } = await import('glob');
+    const fs = await import('fs');
+    const path = await import('path');
+    const { createHash } = await import('crypto');
+    const { parseSkillFile, isSkillFile } = await import('./parser.js');
+    const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
+    const { skillId, vecToBlob } = await import('./db.js');
+    const { chunkText } = await import('./chunker.js');
+
+    const files = globSync(`${sourceDir}/**/*.md`);
+    let indexed = 0, skipped = 0, errors = 0;
+    const batch = [];
+
+    for (const file of files) {
+      try {
+        const stat = fs.statSync(file);
+        if (stat.size > 5 * 1024 * 1024) { skipped++; continue; }
+        const raw = fs.readFileSync(file, 'utf8');
+        const hash = createHash('md5').update(raw).digest('hex');
+        if (!isSkillFile(file, raw)) { skipped++; continue; }
+        const parsed = parseSkillFile(file, sourceName, { raw });
+        batch.push({ ...parsed, hash });
+        if (batch.length >= BATCH_SIZE) {
+          await indexBatch(db, batch);
+          indexed += batch.length;
+          batch.length = 0;
+        }
+      } catch { errors++; }
+    }
+    if (batch.length > 0) {
+      await indexBatch(db, batch);
+      indexed += batch.length;
+    }
+    return { indexed, skipped, errors };
+  } catch (e) {
+    return { indexed: 0, skipped: 0, errors: 1, error: e.message };
+  }
+}
+
+/**
+ * Remove a skill by its ID.
+ * @param {string} skillId - Skill ID (format: "source::name")
+ * @returns {{ok: boolean, error?: string}}
+ */
+export function remove(skillId) {
+  try {
+    const db = getDb();
+    db.prepare('DELETE FROM skills WHERE id = ?').run(skillId);
+    db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(skillId);
+    db.prepare('DELETE FROM edges WHERE from_skill = ? OR to_skill = ?').run(skillId, skillId);
+    db.prepare('DELETE FROM ratings WHERE skill_id = ?').run(skillId);
+    return { ok: true };
+  } catch (e) {
+    return { ok: false, error: e.message };
+  }
+}
+
+/**
+ * Reindex changed files only (hash-based). Scans all configured sources,
+ * updates files whose content hash changed, and removes stale skills.
+ * @returns {Promise<{updated: number, removed: number, errors: number}>}
+ */
+export async function update() {
+  try {
+    const db = getDb();
+    const config = loadConfig();
+    const { globSync } = await import('glob');
+    const fs = await import('fs');
+    const path = await import('path');
+    const { createHash } = await import('crypto');
+    const { parseSkillFile, isSkillFile } = await import('./parser.js');
+    const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
+    const { skillId, vecToBlob } = await import('./db.js');
+    const { chunkText } = await import('./chunker.js');
+    const { indexBatch } = await import('./indexer.js');
+
+    const normalizedSources = config.sources.map(s => ({ ...s, normDir: path.resolve(s.dir) }))
+      .sort((a, b) => b.normDir.length - a.normDir.length);
+
+    const seenFiles = new Set();
+    const allFiles = [];
+    for (const { dir, source } of normalizedSources) {
+      const files = globSync(`${dir}/**/*.md`);
+      for (const f of files) {
+        const norm = path.resolve(f);
+        if (!seenFiles.has(norm)) { seenFiles.add(norm); allFiles.push({ file: norm, source }); }
+      }
+    }
+
+    const dbByPath = new Map();
+    for (const row of db.prepare('SELECT id, path, hash FROM skills').all()) {
+      dbByPath.set(row.path, row);
+    }
+
+    const existingPaths = new Set(allFiles.map(f => f.file));
+    let removed = 0;
+    for (const [filePath, row] of dbByPath) {
+      if (!existingPaths.has(filePath)) {
+        db.prepare('DELETE FROM skills WHERE id = ?').run(row.id);
+        db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(row.id);
+        db.prepare('DELETE FROM edges WHERE from_skill = ? OR to_skill = ?').run(row.id, row.id);
+        db.prepare('DELETE FROM ratings WHERE skill_id = ?').run(row.id);
+        removed++;
+      }
+    }
+
+    let updated = 0, errors = 0, batch = [];
+    for (const { file, source } of allFiles) {
+      try {
+        const raw = fs.readFileSync(file, 'utf8');
+        const hash = createHash('md5').update(raw).digest('hex');
+        const dbRow = dbByPath.get(file);
+        if (dbRow?.hash === hash) continue;
+        if (!isSkillFile(file, raw)) continue;
+        const parsed = parseSkillFile(file, source, { raw });
+        batch.push({ ...parsed, hash });
+        if (batch.length >= BATCH_SIZE) {
+          await indexBatch(db, batch);
+          updated += batch.length;
+          batch.length = 0;
+        }
+      } catch { errors++; }
+    }
+    if (batch.length > 0) {
+      await indexBatch(db, batch);
+      updated += batch.length;
+    }
+
+    return { updated, removed, errors };
+  } catch (e) {
+    return { updated: 0, removed: 0, errors: 1, error: e.message };
+  }
+}
+
+/**
+ * Get full skill details by ID, including edges.
+ * @param {string} skillId
+ * @returns {object|null}
+ */
+export function get(skillId) {
+  try {
+    return getContext(skillId);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * List all skills with optional filtering.
+ * @param {{ source?: string, category?: string, limit?: number, offset?: number }} [options]
+ * @returns {Array<object>}
+ */
+export function list(options = {}) {
+  try {
+    const { source, limit, offset } = options;
+    let results = internalList();
+    if (source) results = results.filter(r => r.source === source);
+    if (offset) results = results.slice(offset);
+    if (limit) results = results.slice(0, limit);
+    return results;
+  } catch {
+    return [];
+  }
+}

package/config.js

@@ -8,6 +8,13 @@ export const PROMPTGRAPH_DIR = path.join(CLAUDE_DIR, '.promptgraph');
 export const SKILLS_STORE_DIR = path.join(CLAUDE_DIR, 'skills-store');
 const CONFIG_PATH = path.join(PROMPTGRAPH_DIR, 'config.json');
 
+export const MAX_DOWNLOAD_SIZE = 50 * 1024 * 1024  // 50 MB per file
+export const MAX_FILE_COUNT = 50000                  // max files per repo
+export const MAX_REPO_SIZE = 500 * 1024 * 1024      // 500 MB per repo
+export const RATE_LIMIT_REQUESTS = 30               // requests per window
+export const RATE_LIMIT_WINDOW_MS = 60000           // 1 minute window
+export const BATCH_SIZE = 100                       // batch indexing size
+
 const DEFAULTS = {
   sources: [
     { dir: path.join(CLAUDE_DIR, 'skills-store'), source: 'skills-store' },
@@ -56,3 +63,10 @@ export async function promptConfig() {
   console.log(`\nConfig saved to ${CONFIG_PATH}`);
   return config;
 }
+
+export function sanitizePath(inputPath) {
+  if (inputPath.includes('..')) {
+    throw new Error(`Path traversal blocked: "${inputPath}"`);
+  }
+  return path.resolve(inputPath);
+}

package/db.js

@@ -63,6 +63,54 @@ export function getDb() {
     db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
   }
 
+  // migrate: add registry metadata columns
+  if (!cols.includes('version')) {
+    db.exec('ALTER TABLE skills ADD COLUMN version TEXT');
+  }
+  if (!cols.includes('author')) {
+    db.exec('ALTER TABLE skills ADD COLUMN author TEXT');
+  }
+  if (!cols.includes('license')) {
+    db.exec('ALTER TABLE skills ADD COLUMN license TEXT');
+  }
+  if (!cols.includes('updated_at')) {
+    db.exec('ALTER TABLE skills ADD COLUMN updated_at TEXT');
+  }
+  if (!cols.includes('downloads')) {
+    db.exec('ALTER TABLE skills ADD COLUMN downloads INTEGER DEFAULT 0');
+  }
+  if (!cols.includes('verified')) {
+    db.exec('ALTER TABLE skills ADD COLUMN verified INTEGER DEFAULT 0');
+  }
+  if (!cols.includes('trust_level')) {
+    db.exec('ALTER TABLE skills ADD COLUMN trust_level TEXT DEFAULT \'unknown\'');
+  }
+  if (!cols.includes('rating')) {
+    db.exec('ALTER TABLE skills ADD COLUMN rating REAL DEFAULT 0');
+  }
+  if (!cols.includes('rating_count')) {
+    db.exec('ALTER TABLE skills ADD COLUMN rating_count INTEGER DEFAULT 0');
+  }
+  if (!cols.includes('popularity')) {
+    db.exec('ALTER TABLE skills ADD COLUMN popularity REAL DEFAULT 0');
+  }
+  if (!cols.includes('last_update')) {
+    db.exec('ALTER TABLE skills ADD COLUMN last_update TEXT');
+  }
+
+  // migrate: registry entries metadata table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS registry_entries (
+      id TEXT PRIMARY KEY,
+      trust_level TEXT DEFAULT 'unknown',
+      downloads INTEGER DEFAULT 0,
+      rating REAL DEFAULT 0,
+      rating_count INTEGER DEFAULT 0,
+      popularity REAL DEFAULT 0,
+      last_update TEXT
+    );
+  `);
+
   // migrate: convert JSON text embeddings to Float32 BLOB (one-time, ~10x smaller)
   const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();
   if (textEmbeddings?.n > 0) {

package/embedder.js

@@ -4,6 +4,11 @@ import os from 'os';
 
 const CACHE_DIR = path.join(os.homedir(), '.claude', '.promptgraph', 'model-cache');
 const BATCH_SIZE = 256;
+const MAX_EMBEDDING_CALLS = 10000;
+let embedCallCount = 0;
+
+export function getEmbedCallCount() { return embedCallCount; }
+export function resetEmbedCallCount() { embedCallCount = 0; }
 
 let model = null;
 
@@ -18,6 +23,7 @@ async function getModel() {
 }
 
 export async function embed(text) {
+  embedCallCount++;
   const m = await getModel();
   const results = [];
   for await (const batch of m.embed([text])) {
@@ -27,6 +33,10 @@ export async function embed(text) {
 }
 
 export async function embedBatch(texts) {
+  if (embedCallCount + texts.length > MAX_EMBEDDING_CALLS) {
+    throw new Error(`Embedding queue limit exceeded (max ${MAX_EMBEDDING_CALLS} chunks per session). Use --fast or reindex incrementally.`);
+  }
+  embedCallCount += texts.length;
   const m = await getModel();
   const all = [];
   for await (const batch of m.embed(texts)) {