promptgraph-mcp 2.4.7 → 2.4.8

package/indexer.js

@@ -11,16 +11,29 @@ import { buildAnnIndex } from './ann.js';
 import { progress, progressDone, success, info, spinner } from './cli.js';
 import chalk from 'chalk';
 
+const MAX_FILE_SIZE = 5 * 1024 * 1024;   // 5 MB per file
+const MAX_FILE_COUNT = 100000;            // 100k files per reindex
+
+function sanitizePath(filePath) {
+  return path.resolve(filePath);
+}
+
 async function indexBatch(db, skills, { fast = false } = {}) {
   const upsertSkill = db.prepare(`
-    INSERT INTO skills (id, name, description, path, source, content, hash)
-    VALUES (@id, @name, @description, @path, @source, @content, @hash)
+    INSERT INTO skills (id, name, description, path, source, content, hash, version, author, license, updated_at, downloads, verified)
+    VALUES (@id, @name, @description, @path, @source, @content, @hash, @version, @author, @license, @updated_at, @downloads, @verified)
     ON CONFLICT(id) DO UPDATE SET
       name = excluded.name,
       description = excluded.description,
       path = excluded.path,
       content = excluded.content,
-      hash = excluded.hash
+      hash = excluded.hash,
+      version = excluded.version,
+      author = excluded.author,
+      license = excluded.license,
+      updated_at = excluded.updated_at,
+      downloads = excluded.downloads,
+      verified = excluded.verified
   `);
   const deleteChunks = db.prepare('DELETE FROM chunks WHERE skill_id = ?');
   const deleteEdges = db.prepare('DELETE FROM edges WHERE from_skill = ?');
@@ -51,7 +64,7 @@ async function indexBatch(db, skills, { fast = false } = {}) {
   db.transaction(() => {
     for (const skill of skills) {
       const id = skillId(skill.source, skill.name);
-      upsertSkill.run({ id, name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content, hash: skill.hash || null });
+      upsertSkill.run({ id, name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content, hash: skill.hash || null, version: skill.version || null, author: skill.author || null, license: skill.license || null, updated_at: skill.updated_at || null, downloads: skill.downloads ?? 0, verified: skill.verified ? 1 : 0 });
       upsertFts.run(id, skill.name, skill.description || '', skill.content || '');
       if (!fast) {
         deleteChunks.run(id);
@@ -93,19 +106,30 @@ export async function indexAll({ fast = false } = {}) {
     normDir: path.resolve(s.dir),
   })).sort((a, b) => b.normDir.length - a.normDir.length); // longest first
 
+  console.error('DEBUG normalizedSources count:', normalizedSources.length);
+  console.error('DEBUG first dir:', normalizedSources[0]?.dir);
+
   const seenFiles = new Set();
   const allFiles = [];
   for (const { dir, source } of normalizedSources) {
     const files = globSync(`${dir}/**/*.md`);
+    console.error('DEBUG source:', source, 'dir:', dir, 'count:', files.length);
     for (const f of files) {
-      const norm = path.resolve(f);
+      const norm = sanitizePath(f);
       if (!seenFiles.has(norm)) {
         seenFiles.add(norm);
-        allFiles.push({ file: f, source });
+        allFiles.push({ file: norm, source });
+      } else {
+        console.error('DEBUG duplicate:', norm);
       }
     }
+    if (allFiles.length > MAX_FILE_COUNT) {
+      info(chalk.yellow(`Reached max file count (${MAX_FILE_COUNT}) — truncating`));
+      break;
+    }
   }
   const total = allFiles.length;
+  console.error('DEBUG total files found:', total);
   info(`Found ${chalk.white.bold(total)} files`);
 
   // reconcile: remove skills whose files no longer exist OR whose name changed
@@ -150,9 +174,13 @@ export async function indexAll({ fast = false } = {}) {
 
   for (const { file, source } of allFiles) {
     try {
-      // 1. Read file once
+      // 1. Read file once (with size gate)
       let raw;
-      try { raw = fs.readFileSync(file, 'utf8'); } catch { skipped++; count++; continue; }
+      try {
+        const stat = fs.statSync(file);
+        if (stat.size > MAX_FILE_SIZE) { skipped++; count++; continue; }
+        raw = fs.readFileSync(file, 'utf8');
+      } catch { skipped++; count++; continue; }
 
       // 2. Hash first — cheapest check
       const hash = createHash('md5').update(raw).digest('hex');
@@ -222,10 +250,13 @@ export async function indexAll({ fast = false } = {}) {
 }
 
 export async function indexFile(filePath, source) {
+  const safe = sanitizePath(filePath);
+  const stat = fs.statSync(safe);
+  if (stat.size > MAX_FILE_SIZE) throw new Error(`File too large: ${filePath}`);
   const db = getDb();
-  const raw = fs.readFileSync(filePath, 'utf8');
+  const raw = fs.readFileSync(safe, 'utf8');
   const hash = createHash('md5').update(raw).digest('hex');
-  const skill = parseSkillFile(filePath, source, { raw });
+  const skill = parseSkillFile(safe, source, { raw });
   await indexBatch(db, [{ ...skill, hash }]);
 }
 
@@ -256,7 +287,10 @@ export async function indexSource(dir, sourceName) {
 
   for (const file of files) {
     try {
-      const raw = fs.readFileSync(file, 'utf8');
+      const norm = sanitizePath(file);
+      const stat = fs.statSync(norm);
+      if (stat.size > MAX_FILE_SIZE) { skipped++; count++; continue; }
+      const raw = fs.readFileSync(norm, 'utf8');
       const hash = createHash('md5').update(raw).digest('hex');
       const dbRow = dbByPath.get(file);
       if (dbRow?.hash === hash) { skipped++; count++; continue; }

package/marketplace.js

@@ -42,6 +42,32 @@ const _require = createRequire(import.meta.url);
 const PKG_VERSION = (() => { try { return _require('./package.json').version; } catch { return '1.x'; } })();
 const UA = `promptgraph-mcp/${PKG_VERSION}`;
 
+const REGISTRY_ID_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
+
+function validateRegistryEntry(item) {
+  const errors = [];
+  if (!item.id || typeof item.id !== 'string') errors.push('missing required id');
+  else if (!REGISTRY_ID_RE.test(item.id)) errors.push(`invalid id "${item.id}" (must match /^[a-z0-9][a-z0-9-]{1,63}$/)`);
+  if (!item.name || typeof item.name !== 'string') errors.push('missing required name');
+  else if (item.name.length < 3) errors.push(`name "${item.name}" too short (min 3 chars)`);
+  if (!item.description || typeof item.description !== 'string') errors.push('missing required description');
+  else if (item.description.length < 15) errors.push(`description too short (min 15 chars)`);
+  if (item.version !== undefined && typeof item.version !== 'string') errors.push('version must be a string');
+  if (item.author !== undefined && typeof item.author !== 'string') errors.push('author must be a string');
+  if (item.license !== undefined && typeof item.license !== 'string') errors.push('license must be a string');
+  if (item.updated_at !== undefined) {
+    if (typeof item.updated_at !== 'string') errors.push('updated_at must be a string');
+    else if (isNaN(Date.parse(item.updated_at))) errors.push(`updated_at "${item.updated_at}" is not a valid ISO date`);
+  }
+  if (item.downloads !== undefined && typeof item.downloads !== 'number') errors.push('downloads must be a number');
+  if (item.verified !== undefined && typeof item.verified !== 'boolean') errors.push('verified must be a boolean');
+  if (errors.length > 0) {
+    console.warn(`[PromptGraph] Skipping invalid registry entry "${item.id || item.name || '(unnamed)'}": ${errors.join('; ')}`);
+    return { ok: false };
+  }
+  return { ok: true };
+}
+
 // Deterministic short code from an id. Same id always yields the same code,
 // so codes auto-generate — no need to assign them by hand.
 export function codeFor(id) {
@@ -137,6 +163,8 @@ export async function browseMarketplace(topK = 20) {
     if (!Array.isArray(registry.skills)) return { error: 'Invalid registry format' };
     return registry.skills
       .map(s => ({ ...s, code: s.code || codeFor(s.id) })) // auto-fill code
+      .filter(s => validateRegistryEntry(s).ok)
+      .filter(s => s.raw_url)
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
       .slice(0, topK);
   } catch (e) {
@@ -154,6 +182,10 @@ export async function installSkillFromUrl(url) {
     // derive filename from URL
     const urlName = rawUrl.split('/').pop().replace(/[^a-z0-9-_.]/gi, '-');
     const dest = path.join(SKILLS_DIR, urlName.endsWith('.md') ? urlName : urlName + '.md');
+    const resolvedDest = path.resolve(dest);
+    if (!resolvedDest.startsWith(path.resolve(SKILLS_DIR))) {
+      return { error: 'Path traversal blocked: destination outside marketplace directory' };
+    }
     const v = writeSkillAtomic(dest, content);
     if (!v.ok) return { error: 'Downloaded skill failed validation', issues: v.errors };
     return { success: true, path: dest, url: rawUrl };
@@ -172,8 +204,9 @@ export async function installSkill(query) {
     const text = await fetchText(REGISTRY_URL);
     const registry = JSON.parse(text);
     const q = String(query).trim().toLowerCase();
+    const validSkills = (registry.skills || []).filter(s => validateRegistryEntry(s).ok);
     // match by code (stored OR auto-generated), id, or name
-    const skill = registry.skills?.find(s =>
+    const skill = validSkills.find(s =>
       (s.code || codeFor(s.id)).toLowerCase() === q ||
       s.id?.toLowerCase() === q ||
       s.name?.toLowerCase() === q
@@ -191,6 +224,10 @@ export async function installSkill(query) {
     fs.mkdirSync(SKILLS_DIR, { recursive: true });
     ensureMarketplaceSource();
     const dest = path.join(SKILLS_DIR, `${skillId}.md`);
+    const resolvedDest = path.resolve(dest);
+    if (!resolvedDest.startsWith(path.resolve(SKILLS_DIR))) {
+      return { error: 'Path traversal blocked: destination outside marketplace directory' };
+    }
 
     const content = await fetchText(skill.raw_url);
     const v = writeSkillAtomic(dest, content);
@@ -222,7 +259,7 @@ export async function browseBundles(topK = 20) {
   try {
     const text = await fetchText(REGISTRY_URL);
     const registry = JSON.parse(text);
-    const bundles = registry.bundles || [];
+    const bundles = (registry.bundles || []).filter(b => validateRegistryEntry(b).ok);
     const cache = readSkillCountCache();
     const now = Date.now();
     let changed = false;
@@ -273,7 +310,8 @@ export async function installBundle(bundleId) {
     const text = await fetchText(REGISTRY_URL);
     const registry = JSON.parse(text);
     const q = String(bundleId).trim().toLowerCase();
-    const bundle = (registry.bundles || []).find(b =>
+    const validSkills = (registry.skills || []).filter(s => validateRegistryEntry(s).ok);
+    const bundle = (registry.bundles || []).filter(b => validateRegistryEntry(b).ok).find(b =>
       (b.code || codeFor(b.id)).toLowerCase() === q || b.id?.toLowerCase() === q || b.name?.toLowerCase() === q
     );
     if (!bundle) return { error: `No bundle matching "${bundleId}"` };
@@ -290,12 +328,16 @@ export async function installBundle(bundleId) {
 
     const delay = (ms) => new Promise(r => setTimeout(r, ms));
     for (const skillId of bundle.skills || []) {
-      const skill = registry.skills?.find(s => s.id === skillId);
+      const skill = validSkills.find(s => s.id === skillId);
       if (!skill?.raw_url) { failed.push(skillId); continue; }
       try {
         if (installed.length > 0) await delay(300); // rate limit: 300ms between requests
         const content = await fetchText(skill.raw_url);
         const dest = path.join(SKILLS_DIR, `${skillId}.md`);
+        const resolvedDest = path.resolve(dest);
+        if (!resolvedDest.startsWith(path.resolve(SKILLS_DIR))) {
+          failed.push(skillId); continue;
+        }
         const v = writeSkillAtomic(dest, content);
         if (!v.ok) { failed.push(skillId); continue; }
         installed.push(skillId);
@@ -478,7 +520,7 @@ export function pruneInvalidRepos() {
       continue;
     }
 
-    const mdFiles = globSync(`${src.dir}/**/*.md`);
+    const mdFiles = globSync(`${src.dir}/**/*.md`).map(fp => path.resolve(fp));
     if (mdFiles.length === 0) {
       removed.push({ repo: repoName, reason: 'no .md files' });
       config.sources = config.sources.filter(s => s !== src);

package/package.json

@@ -1,6 +1,11 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.4.7",
+  "version": "2.4.8",
+  "files": [
+    "*.js",
+    "src/**/*.js",
+    "README.md"
+  ],
   "main": "index.js",
   "type": "module",
   "bin": {
@@ -42,13 +47,14 @@
     "fastembed": "^2.1.0",
     "glob": "^13.0.6",
     "gray-matter": "^4.0.3",
+    "hnswlib-node": "^3.0.0",
     "ora": "^9.4.0"
   },
   "devDependencies": {
+    "@vitest/coverage-v8": "^4.1.8",
     "vitest": "^4.1.8"
   },
   "overrides": {
-    "tar": "^6.2.1",
-    "js-yaml": "^4.1.0"
+    "tar": "^6.2.1"
   }
 }

package/search.js

@@ -2,6 +2,26 @@ import { embed, cosineSimilarity } from './embedder.js';
 import { getDb, blobToVec } from './db.js';
 import { annSearch } from './ann.js';
 
+let embedWeight = 0.7;
+let bm25Weight = 0.3;
+
+export function setHybridWeights(ew, bw) {
+  embedWeight = ew;
+  bm25Weight = bw;
+}
+
+function adaptWeights(query) {
+  // Technical terms (caps, digits) → BM25 matters more
+  const hasTechTerms = /[A-Z]|\d/.test(query);
+  if (hasTechTerms) {
+    embedWeight = 0.5;
+    bm25Weight = 0.5;
+  } else {
+    embedWeight = 0.7;
+    bm25Weight = 0.3;
+  }
+}
+
 function applyRatingBoost(db, id, score) {
   const r = db.prepare('SELECT success, fail FROM ratings WHERE skill_id = ?').get(id);
   if (r && (r.success + r.fail) > 3) {
@@ -11,13 +31,13 @@ function applyRatingBoost(db, id, score) {
   return score;
 }
 
-export async function search(query, topK = 5) {
-  const db = getDb();
-  const queryVec = await embed(query);
-
-  // Try ANN index first (fast, O(log N))
-  const annResults = await annSearch(queryVec, topK * 4);
+function normalizeBM25(raw) {
+  return Math.max(0, 1 + raw / 10);
+}
 
+async function runEmbeddingSearch(db, queryVec, topK) {
+  // Try ANN index first (fast)
+  const annResults = await annSearch(queryVec, topK);
   if (annResults && annResults.length > 0) {
     const bestBySkill = new Map();
     for (const r of annResults) {
@@ -25,11 +45,8 @@ export async function search(query, topK = 5) {
       if (!prev || r.score > prev) bestBySkill.set(r.skill_id, r.score);
     }
     return [...bestBySkill.entries()]
-      .map(([id, score]) => ({ id, score: applyRatingBoost(db, id, score) }))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, topK)
-      .map(({ id, score }) => skillWithSnippet(db, id, score))
-      .filter(Boolean);
+      .map(([id, score]) => ({ id, score: Math.max(0, score) }))
+      .sort((a, b) => b.score - a.score);
   }
 
   // Fallback: brute force cosine (used before first reindex)
@@ -42,14 +59,14 @@ export async function search(query, topK = 5) {
       if (!prev || score > prev) bestBySkill.set(chunk.skill_id, score);
     }
     return [...bestBySkill.entries()]
-      .map(([id, score]) => ({ id, score: applyRatingBoost(db, id, score) }))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, topK)
-      .map(({ id, score }) => skillWithSnippet(db, id, score))
-      .filter(Boolean);
+      .map(([id, score]) => ({ id, score: Math.max(0, score) }))
+      .sort((a, b) => b.score - a.score);
   }
 
-  // Fast-mode fallback: FTS5 keyword search (no embeddings)
+  return [];
+}
+
+function runBM25Search(db, query, topK) {
   try {
     const terms = query.replace(/[^\w\s]/g, ' ').trim().split(/\s+/).filter(Boolean).join(' OR ');
     const rows = db.prepare(
@@ -57,14 +74,61 @@ export async function search(query, topK = 5) {
        JOIN skills s ON skills_fts.id = s.id
        WHERE skills_fts MATCH ? ORDER BY score LIMIT ?`
     ).all(terms, topK);
-    return rows.map(r => skillWithSnippet(db, r.id, Math.max(0, 1 + r.score / 10))).filter(Boolean);
+    return rows.map(r => ({ id: r.id, score: normalizeBM25(r.score) }));
   } catch {
     return [];
   }
 }
 
+export async function search(query, topK = 5) {
+  const db = getDb();
+  adaptWeights(query);
+  const queryVec = await embed(query);
+
+  const embedResults = await runEmbeddingSearch(db, queryVec, topK * 4);
+  const bm25Results = runBM25Search(db, query, topK * 4);
+
+  // Fallback: pure BM25 if embeddings unavailable
+  if (!embedResults.length) {
+    return bm25Results.slice(0, topK)
+      .map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score)))
+      .filter(Boolean);
+  }
+
+  // Fallback: pure embedding if BM25 returns nothing
+  if (!bm25Results.length) {
+    return embedResults.slice(0, topK)
+      .map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
+      .filter(Boolean);
+  }
+
+  // Hybrid: combine normalized scores from both signals
+  const combined = new Map();
+  for (const { id, score } of embedResults) {
+    combined.set(id, { embedScore: score, bm25Score: 0 });
+  }
+  for (const { id, score } of bm25Results) {
+    const entry = combined.get(id);
+    if (entry) {
+      entry.bm25Score = score;
+    } else {
+      combined.set(id, { embedScore: 0, bm25Score: score });
+    }
+  }
+
+  return [...combined.entries()]
+    .map(([id, s]) => ({
+      id,
+      score: embedWeight * s.embedScore + bm25Weight * s.bm25Score,
+    }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, topK)
+    .map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
+    .filter(Boolean);
+}
+
 function skillWithSnippet(db, id, score) {
-  const skill = db.prepare('SELECT id, name, description, path, source, content FROM skills WHERE id = ?').get(id);
+  const skill = db.prepare('SELECT id, name, description, path, source, content, version, author, license, updated_at, downloads, verified FROM skills WHERE id = ?').get(id);
   if (!skill) return null;
   const { content, ...rest } = skill;
   const snippet = content?.replace(/^---[\s\S]*?---\n?/, '').trim().slice(0, 400) || '';
@@ -131,5 +195,5 @@ export function getImpact(nameOrId) {
 
 export function listAll() {
   const db = getDb();
-  return db.prepare('SELECT id, name, description, source FROM skills ORDER BY source, name').all();
+  return db.prepare('SELECT id, name, description, source, version, author, license, updated_at, downloads, verified FROM skills ORDER BY source, name').all();
 }

package/src/filter/hard-filter.js

@@ -19,6 +19,8 @@ const SKIP_DIRS = new Set([
   'tests', 'test', '__tests__', 'spec', 'fixtures', 'assets', 'images',
   'img', 'screenshots', 'media', 'static', 'public', 'dist', 'build',
   'node_modules', 'vendor', 'third_party',
+  'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
+  'cheatsheets', 'resources',
 ]);
 
 const BADGE_RE = /!\[.*\]\(https?:\/\/(img\.shields\.io|badge\.fury|travis-ci|github\.com\/[^)]+\/badge)/i;

package/src/filter/train.js

@@ -1,10 +1,12 @@
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
+import { fileURLToPath } from 'url';
 import { globSync } from 'glob';
 import { embedBatch } from '../../embedder.js';
 
-const TRAINING_DIR = path.resolve(new URL('../../registry/training', import.meta.url).pathname);
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const TRAINING_DIR = path.resolve(__dirname, '../../registry/training');
 const MODEL_PATH = path.join(os.homedir(), '.claude', '.promptgraph', 'model.json');
 
 function readAllMd(dir) {

package/src/store/flat-store.js

@@ -0,0 +1,61 @@
+import { VectorStore } from './vector-store.js';
+
+export class FlatVectorStore extends VectorStore {
+  constructor() {
+    super();
+    this._items = [];
+  }
+
+  async add(id, vector) {
+    this._items.push({ skill_id: id, vec: new Float32Array(vector) });
+  }
+
+  async addBatch(entries) {
+    for (const { id, vector } of entries) {
+      this._items.push({ skill_id: id, vec: new Float32Array(vector) });
+    }
+  }
+
+  async remove(id) {
+    this._items = this._items.filter(item => item.skill_id !== id);
+  }
+
+  async search(vector, topK = 20) {
+    const qArr = new Float32Array(vector);
+    const bestBySkill = new Map();
+    for (const entry of this._items) {
+      const score = cosineSim(qArr, entry.vec);
+      const prev = bestBySkill.get(entry.skill_id);
+      if (!prev || score > prev) bestBySkill.set(entry.skill_id, score);
+    }
+    return [...bestBySkill.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, topK)
+      .map(([skill_id, score]) => ({ skill_id, score }));
+  }
+
+  async build(entries) {
+    this._items = [];
+    for (const { skill_id, vector } of entries) {
+      this._items.push({ skill_id, vec: new Float32Array(vector) });
+    }
+  }
+
+  async clear() {
+    this._items = [];
+  }
+
+  get size() {
+    return this._items.length;
+  }
+}
+
+function cosineSim(a, b) {
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-8);
+}