promptgraph-mcp 1.2.0 → 1.4.0

package/README.md

@@ -129,7 +129,7 @@ promptgraph-mcp reindex   # Re-index all skills
 | Skills in context | All 40+ | 1 (router) |
 | Tokens per session | ~20,000–50,000 | ~150 + 1 skill |
 
-> **Note:** Search is currently linear O(N) over all chunks. Works well up to ~1,000 skills. ANN index (HNSW/LanceDB) planned for larger collections.
+> **Search:** Uses HNSW index (via [vectra](https://github.com/Stevenic/vectra)) for O(log N) approximate nearest neighbor search. Falls back to brute-force on first run before index is built.
 
 ## File Structure
 

package/ann.js

@@ -0,0 +1,50 @@
+import { LocalIndex } from 'vectra';
+import path from 'path';
+import os from 'os';
+import { getDb } from './db.js';
+
+const INDEX_PATH = path.join(os.homedir(), '.claude', '.promptgraph', 'hnsw-index');
+
+let _index = null;
+
+async function getIndex() {
+  if (_index) return _index;
+  _index = new LocalIndex(INDEX_PATH);
+  if (!await _index.isIndexCreated()) {
+    await _index.createIndex({ version: 1, deleteIfExists: true });
+  }
+  return _index;
+}
+
+export async function buildAnnIndex() {
+  const index = await getIndex();
+  await index.createIndex({ version: 1, deleteIfExists: true });
+
+  const db = getDb();
+  const chunks = db.prepare('SELECT skill_id, chunk_index, embedding FROM chunks').all();
+
+  for (const chunk of chunks) {
+    const vec = JSON.parse(chunk.embedding);
+    await index.insertItem({
+      vector: vec,
+      metadata: { skill_id: chunk.skill_id, chunk_index: chunk.chunk_index },
+    });
+  }
+
+  console.error(`[PromptGraph] ANN index built: ${chunks.length} chunks`);
+}
+
+export async function annSearch(queryVec, topK = 20) {
+  try {
+    const index = await getIndex();
+    if (!await index.isIndexCreated()) return null;
+
+    const results = await index.queryItems(queryVec, topK);
+    return results.map(r => ({
+      skill_id: r.item.metadata.skill_id,
+      score: r.score,
+    }));
+  } catch {
+    return null;
+  }
+}

package/chunker.js

@@ -1,5 +1,5 @@
-const CHUNK_SIZE = 400;
-const CHUNK_OVERLAP = 80;
+const CHUNK_SIZE = 800;
+const CHUNK_OVERLAP = 100;
 
 export function chunkText(text) {
   const words = text.split(/\s+/);

package/db.js

@@ -17,7 +17,8 @@ export function getDb() {
       description TEXT,
       path TEXT NOT NULL,
       source TEXT NOT NULL,
-      content TEXT NOT NULL
+      content TEXT NOT NULL,
+      hash TEXT
     );
 
     CREATE TABLE IF NOT EXISTS chunks (
@@ -34,8 +35,21 @@ export function getDb() {
       to_skill TEXT NOT NULL,
       PRIMARY KEY (from_skill, to_skill)
     );
+
+    CREATE TABLE IF NOT EXISTS ratings (
+      skill_id TEXT PRIMARY KEY,
+      uses INTEGER DEFAULT 0,
+      success INTEGER DEFAULT 0,
+      fail INTEGER DEFAULT 0
+    );
   `);
 
+  // migrate: add hash column if missing
+  const cols = db.pragma('table_info(skills)').map(c => c.name);
+  if (!cols.includes('hash')) {
+    db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
+  }
+
   return db;
 }
 

package/embedder.js

@@ -1,32 +1,44 @@
-import { EmbeddingModel, FlagEmbedding } from 'fastembed';
-import path from 'path';
-import os from 'os';
-
-const CACHE_DIR = path.join(os.homedir(), '.claude', '.promptgraph', 'model-cache');
-
-let model = null;
-
-async function getModel() {
-  if (!model) {
-    model = await FlagEmbedding.init({
-      model: EmbeddingModel.BGESmallENV15,
-      cacheDir: CACHE_DIR,
-    });
-  }
-  return model;
-}
-
-export async function embed(text) {
-  const m = await getModel();
-  const results = [];
-  for await (const batch of m.embed([text])) {
-    results.push(...batch);
-  }
-  return Array.from(results[0]);
-}
-
-export function cosineSimilarity(a, b) {
-  let dot = 0;
-  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
-  return dot;
-}
+import { EmbeddingModel, FlagEmbedding } from 'fastembed';
+import path from 'path';
+import os from 'os';
+
+const CACHE_DIR = path.join(os.homedir(), '.claude', '.promptgraph', 'model-cache');
+const BATCH_SIZE = 64;
+
+let model = null;
+
+async function getModel() {
+  if (!model) {
+    model = await FlagEmbedding.init({
+      model: EmbeddingModel.BGESmallENV15,
+      cacheDir: CACHE_DIR,
+    });
+  }
+  return model;
+}
+
+export async function embed(text) {
+  const m = await getModel();
+  const results = [];
+  for await (const batch of m.embed([text])) {
+    results.push(...batch);
+  }
+  return Array.from(results[0]);
+}
+
+export async function embedBatch(texts) {
+  const m = await getModel();
+  const all = [];
+  for await (const batch of m.embed(texts)) {
+    all.push(...batch);
+  }
+  return all.map(v => Array.from(v));
+}
+
+export function cosineSimilarity(a, b) {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+
+export { BATCH_SIZE };

package/index.js

@@ -8,6 +8,7 @@ import { startWatcher } from './watcher.js';
 import { promptConfig } from './config.js';
 import { importFromGitHub } from './github-import.js';
 import { detectPlatforms, PLATFORMS } from './platform.js';
+import { browseMarketplace, installSkill, publishSkill, getTopRated, recordUse, recordSuccess, recordFail } from './marketplace.js';
 
 const args = process.argv.slice(2);
 
@@ -134,6 +135,52 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
         required: ['name'],
       },
     },
+    {
+      name: 'pg_rate',
+      description: 'Record skill usage outcome. Call after applying a skill: outcome="success" if it helped, "fail" if it did not.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          skill_id: { type: 'string' },
+          outcome: { type: 'string', enum: ['use', 'success', 'fail'] },
+        },
+        required: ['skill_id', 'outcome'],
+      },
+    },
+    {
+      name: 'pg_top_rated',
+      description: 'Get top rated skills by success rate.',
+      inputSchema: {
+        type: 'object',
+        properties: { top_k: { type: 'number' } },
+      },
+    },
+    {
+      name: 'pg_marketplace_browse',
+      description: 'Browse top skills from the PromptGraph marketplace.',
+      inputSchema: {
+        type: 'object',
+        properties: { top_k: { type: 'number' } },
+      },
+    },
+    {
+      name: 'pg_marketplace_install',
+      description: 'Install a skill from the marketplace by ID.',
+      inputSchema: {
+        type: 'object',
+        properties: { skill_id: { type: 'string' } },
+        required: ['skill_id'],
+      },
+    },
+    {
+      name: 'pg_marketplace_publish',
+      description: 'Publish a local skill file to the marketplace via GitHub Gist.',
+      inputSchema: {
+        type: 'object',
+        properties: { file_path: { type: 'string' } },
+        required: ['file_path'],
+      },
+    },
   ],
 }));
 
@@ -149,6 +196,16 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       case 'pg_callers': result = getCallers(args.name); break;
       case 'pg_callees': result = getCallees(args.name); break;
       case 'pg_impact': result = getImpact(args.name); break;
+      case 'pg_rate':
+        if (args.outcome === 'use') recordUse(args.skill_id);
+        else if (args.outcome === 'success') recordSuccess(args.skill_id);
+        else if (args.outcome === 'fail') recordFail(args.skill_id);
+        result = { ok: true };
+        break;
+      case 'pg_top_rated': result = getTopRated(args.top_k || 10); break;
+      case 'pg_marketplace_browse': result = await browseMarketplace(args.top_k || 20); break;
+      case 'pg_marketplace_install': result = await installSkill(args.skill_id); break;
+      case 'pg_marketplace_publish': result = await publishSkill(args.file_path); break;
       default: throw new Error(`Unknown tool: ${name}`);
     }
     return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };

package/indexer.js

@@ -1,37 +1,64 @@
 import { globSync } from 'glob';
+import { createHash } from 'crypto';
+import fs from 'fs';
 import { parseSkillFile } from './parser.js';
-import { embed } from './embedder.js';
+import { embedBatch, BATCH_SIZE } from './embedder.js';
 import { getDb, skillId } from './db.js';
 import { loadConfig } from './config.js';
 import { chunkText } from './chunker.js';
+import { buildAnnIndex } from './ann.js';
 
-async function indexSkill(db, skill) {
-  const id = skillId(skill.source, skill.name);
+function fileHash(filePath) {
+  const content = fs.readFileSync(filePath);
+  return createHash('md5').update(content).digest('hex');
+}
 
-  db.prepare(`
-    INSERT INTO skills (id, name, description, path, source, content)
-    VALUES (@id, @name, @description, @path, @source, @content)
+async function indexBatch(db, skills) {
+  const upsertSkill = db.prepare(`
+    INSERT INTO skills (id, name, description, path, source, content, hash)
+    VALUES (@id, @name, @description, @path, @source, @content, @hash)
     ON CONFLICT(id) DO UPDATE SET
       name = excluded.name,
       description = excluded.description,
       path = excluded.path,
-      content = excluded.content
-  `).run({ id, name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content });
-
-  db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(id);
+      content = excluded.content,
+      hash = excluded.hash
+  `);
+  const deleteChunks = db.prepare('DELETE FROM chunks WHERE skill_id = ?');
+  const deleteEdges = db.prepare('DELETE FROM edges WHERE from_skill = ?');
+  const upsertChunk = db.prepare('INSERT OR REPLACE INTO chunks (skill_id, chunk_index, text, embedding) VALUES (?, ?, ?, ?)');
+  const upsertEdge = db.prepare('INSERT OR IGNORE INTO edges (from_skill, to_skill) VALUES (?, ?)');
 
-  const chunks = chunkText(skill.name + ' ' + skill.description + '\n' + skill.content);
-  const upsertChunk = db.prepare(`INSERT OR REPLACE INTO chunks (skill_id, chunk_index, text, embedding) VALUES (?, ?, ?, ?)`);
-  for (let i = 0; i < chunks.length; i++) {
-    const vec = await embed(chunks[i]);
-    upsertChunk.run(id, i, chunks[i], JSON.stringify(vec));
+  // collect all chunks across skills in batch
+  const allChunks = [];
+  for (const skill of skills) {
+    const id = skillId(skill.source, skill.name);
+    const chunks = chunkText(skill.name + ' ' + skill.description + '\n' + skill.content);
+    for (let i = 0; i < chunks.length; i++) {
+      allChunks.push({ id, skill, chunkIndex: i, text: chunks[i] });
+    }
   }
 
-  db.prepare('DELETE FROM edges WHERE from_skill = ?').run(id);
-  const upsertEdge = db.prepare('INSERT OR IGNORE INTO edges (from_skill, to_skill) VALUES (?, ?)');
-  for (const called of skill.calls) {
-    upsertEdge.run(id, called);
-  }
+  // embed all chunks in one batch call
+  const texts = allChunks.map(c => c.text);
+  const embeddings = await embedBatch(texts);
+
+  const txn = db.transaction(() => {
+    for (const skill of skills) {
+      const id = skillId(skill.source, skill.name);
+      upsertSkill.run({ id, name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content, hash: skill.hash || null });
+      deleteChunks.run(id);
+      deleteEdges.run(id);
+      for (const called of skill.calls) {
+        upsertEdge.run(id, called);
+      }
+    }
+    for (let i = 0; i < allChunks.length; i++) {
+      const { id, chunkIndex, text } = allChunks[i];
+      upsertChunk.run(id, chunkIndex, text, JSON.stringify(embeddings[i]));
+    }
+  });
+  txn();
 }
 
 export async function indexAll() {
@@ -39,25 +66,67 @@ export async function indexAll() {
   const db = getDb();
   db.prepare('DELETE FROM edges').run();
 
-  let count = 0;
+  // pre-count total files
+  let total = 0;
+  const allFiles = [];
   for (const { dir, source } of config.sources) {
     const files = globSync(`${dir}/**/*.md`);
-    for (const file of files) {
-      try {
-        const skill = parseSkillFile(file, source);
-        await indexSkill(db, skill);
+    files.forEach(f => allFiles.push({ file: f, source }));
+    total += files.length;
+  }
+  console.log(`  Total files: ${total}`);
+
+  let count = 0;
+  let errors = 0;
+  let batch = [];
+  const start = Date.now();
+
+  const getHash = db.prepare('SELECT hash FROM skills WHERE id = ?');
+
+  let skipped = 0;
+  for (const { file, source } of allFiles) {
+    try {
+      const hash = fileHash(file);
+      const parsed = parseSkillFile(file, source);
+      const id = skillId(source, parsed.name);
+      const existing = getHash.get(id);
+      if (existing?.hash === hash) {
+        skipped++;
         count++;
-        process.stdout.write(`\r  Indexed: ${count} skills`);
-      } catch (e) {
-        console.error(`\n  Error indexing ${file}: ${e.message}`);
+        if (count % 100 === 0) {
+          const pct = Math.round(count / total * 100);
+          process.stdout.write(`\r  [${pct}%] ${count}/${total} | skipped: ${skipped} | errors: ${errors}  `);
+        }
+        continue;
+      }
+      const skill = { ...parsed, hash };
+      batch.push(skill);
+      if (batch.length >= BATCH_SIZE) {
+        await indexBatch(db, batch);
+        count += batch.length;
+        batch = [];
+        const pct = Math.round(count / total * 100);
+        const elapsed = ((Date.now() - start) / 1000).toFixed(0);
+        const eta = count > 0 ? Math.round((total - count) * (Date.now() - start) / count / 1000) : '?';
+        process.stdout.write(`\r  [${pct}%] ${count}/${total} skills | ${elapsed}s elapsed | ETA: ${eta}s | errors: ${errors}  `);
       }
+    } catch (e) {
+      errors++;
     }
   }
+
+  if (batch.length > 0) {
+    await indexBatch(db, batch);
+    count += batch.length;
+  }
+
+  process.stdout.write('\r  Building ANN index...');
+  await buildAnnIndex();
   console.log(`\n  Done. ${count} skills indexed.`);
 }
 
 export async function indexFile(filePath, source) {
   const db = getDb();
   const skill = parseSkillFile(filePath, source);
-  await indexSkill(db, skill);
+  await indexBatch(db, [skill]);
 }

package/marketplace.js

@@ -0,0 +1,104 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { execSync } from 'child_process';
+import { getDb } from './db.js';
+
+const REGISTRY_URL = 'https://raw.githubusercontent.com/NeiP4n/promptgraph-registry/main/registry.json';
+const SKILLS_DIR = path.join(os.homedir(), '.claude', 'skills-store', 'marketplace');
+
+export async function browseMarketplace(topK = 20) {
+  try {
+    const res = await fetch(REGISTRY_URL);
+    const registry = await res.json();
+    return registry.skills
+      .sort((a, b) => (b.stars || 0) - (a.stars || 0))
+      .slice(0, topK);
+  } catch {
+    return { error: 'Registry unavailable. Check https://github.com/NeiP4n/promptgraph-registry' };
+  }
+}
+
+export async function installSkill(skillId) {
+  try {
+    const res = await fetch(REGISTRY_URL);
+    const registry = await res.json();
+    const skill = registry.skills.find(s => s.id === skillId);
+    if (!skill) return { error: `Skill "${skillId}" not found in registry` };
+
+    fs.mkdirSync(SKILLS_DIR, { recursive: true });
+    const dest = path.join(SKILLS_DIR, `${skillId}.md`);
+
+    const content = await fetch(skill.raw_url);
+    const text = await content.text();
+    fs.writeFileSync(dest, text);
+
+    return { success: true, path: dest, name: skill.name };
+  } catch (e) {
+    return { error: e.message };
+  }
+}
+
+export async function publishSkill(filePath) {
+  if (!fs.existsSync(filePath)) return { error: `File not found: ${filePath}` };
+
+  const content = fs.readFileSync(filePath, 'utf8');
+  const name = path.basename(filePath, '.md');
+
+  try {
+    const result = execSync(
+      `gh gist create "${filePath}" --desc "PromptGraph skill: ${name}" --public`,
+      { encoding: 'utf8' }
+    ).trim();
+    return {
+      success: true,
+      url: result,
+      message: `Published! Submit to registry: https://github.com/NeiP4n/promptgraph-registry/issues/new`,
+    };
+  } catch {
+    return { error: 'gh CLI not found or not authenticated. Run: gh auth login' };
+  }
+}
+
+export function getTopRated(topK = 10) {
+  const db = getDb();
+  return db.prepare(`
+    SELECT s.id, s.name, s.description, s.source,
+           r.uses, r.success, r.fail,
+           CASE WHEN (r.success + r.fail) > 0
+                THEN ROUND(CAST(r.success AS FLOAT) / (r.success + r.fail), 2)
+                ELSE NULL END as rating
+    FROM skills s
+    LEFT JOIN ratings r ON s.id = r.skill_id
+    WHERE r.uses > 0
+    ORDER BY rating DESC, r.uses DESC
+    LIMIT ?
+  `).all(topK);
+}
+
+export function recordUse(skillId) {
+  const db = getDb();
+  db.prepare(`
+    INSERT INTO ratings (skill_id, uses, success, fail)
+    VALUES (?, 1, 0, 0)
+    ON CONFLICT(skill_id) DO UPDATE SET uses = uses + 1
+  `).run(skillId);
+}
+
+export function recordSuccess(skillId) {
+  const db = getDb();
+  db.prepare(`
+    INSERT INTO ratings (skill_id, uses, success, fail)
+    VALUES (?, 0, 1, 0)
+    ON CONFLICT(skill_id) DO UPDATE SET success = success + 1
+  `).run(skillId);
+}
+
+export function recordFail(skillId) {
+  const db = getDb();
+  db.prepare(`
+    INSERT INTO ratings (skill_id, uses, success, fail)
+    VALUES (?, 0, 0, 1)
+    ON CONFLICT(skill_id) DO UPDATE SET fail = fail + 1
+  `).run(skillId);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "main": "index.js",
   "type": "module",
   "bin": {
@@ -37,6 +37,7 @@
     "chokidar": "^5.0.0",
     "fastembed": "^2.1.0",
     "glob": "^13.0.6",
-    "gray-matter": "^4.0.3"
+    "gray-matter": "^4.0.3",
+    "vectra": "^0.15.0"
   }
 }

package/search.js

@@ -1,34 +1,61 @@
 import { embed, cosineSimilarity } from './embedder.js';
 import { getDb } from './db.js';
+import { annSearch } from './ann.js';
+
+function applyRatingBoost(db, id, score) {
+  const r = db.prepare('SELECT success, fail FROM ratings WHERE skill_id = ?').get(id);
+  if (r && (r.success + r.fail) > 3) {
+    const rating = r.success / (r.success + r.fail);
+    return score * (0.85 + 0.15 * rating);
+  }
+  return score;
+}
 
 export async function search(query, topK = 5) {
   const db = getDb();
   const queryVec = await embed(query);
 
-  const chunks = db.prepare('SELECT skill_id, embedding FROM chunks').all();
+  // Try ANN index first (fast, O(log N))
+  const annResults = await annSearch(queryVec, topK * 4);
 
+  if (annResults && annResults.length > 0) {
+    const bestBySkill = new Map();
+    for (const r of annResults) {
+      const prev = bestBySkill.get(r.skill_id);
+      if (!prev || r.score > prev) bestBySkill.set(r.skill_id, r.score);
+    }
+    return [...bestBySkill.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, topK)
+      .map(([id, score]) => {
+        const skill = db.prepare('SELECT id, name, description, path, source FROM skills WHERE id = ?').get(id);
+        return skill ? { ...skill, score: applyRatingBoost(db, id, score) } : null;
+      })
+      .filter(Boolean);
+  }
+
+  // Fallback: brute force (used before first reindex)
+  const chunks = db.prepare('SELECT skill_id, embedding FROM chunks').all();
   const bestBySkill = new Map();
   for (const chunk of chunks) {
     const score = cosineSimilarity(queryVec, JSON.parse(chunk.embedding));
     const prev = bestBySkill.get(chunk.skill_id);
     if (!prev || score > prev) bestBySkill.set(chunk.skill_id, score);
   }
-
-  const topIds = [...bestBySkill.entries()]
+  return [...bestBySkill.entries()]
     .sort((a, b) => b[1] - a[1])
     .slice(0, topK)
-    .map(([id]) => id);
-
-  return topIds.map(id => {
-    const skill = db.prepare('SELECT id, name, description, path, source FROM skills WHERE id = ?').get(id);
-    return { ...skill, score: bestBySkill.get(id) };
-  });
+    .map(([id, score]) => {
+      const skill = db.prepare('SELECT id, name, description, path, source FROM skills WHERE id = ?').get(id);
+      return skill ? { ...skill, score } : null;
+    })
+    .filter(Boolean);
 }
 
 export function getContext(id) {
   const db = getDb();
   const skill = db.prepare('SELECT * FROM skills WHERE id = ?').get(id)
-    || db.prepare("SELECT * FROM skills WHERE name = ? ORDER BY id LIMIT 1").get(id);
+    || db.prepare('SELECT * FROM skills WHERE name = ? ORDER BY id LIMIT 1').get(id);
   if (!skill) return null;
   const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(skill.id).map(r => r.to_skill);
   const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(skill.id).map(r => r.from_skill);