promptgraph-mcp 2.4.7 → 2.5.0

package/marketplace.js

@@ -42,6 +42,42 @@ const _require = createRequire(import.meta.url);
 const PKG_VERSION = (() => { try { return _require('./package.json').version; } catch { return '1.x'; } })();
 const UA = `promptgraph-mcp/${PKG_VERSION}`;
 
+const REGISTRY_ID_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
+
+function validateRegistryEntry(item) {
+  const errors = [];
+  if (!item.id || typeof item.id !== 'string') errors.push('missing required id');
+  else if (!REGISTRY_ID_RE.test(item.id)) errors.push(`invalid id "${item.id}" (must match /^[a-z0-9][a-z0-9-]{1,63}$/)`);
+  if (!item.name || typeof item.name !== 'string') errors.push('missing required name');
+  else if (item.name.length < 3) errors.push(`name "${item.name}" too short (min 3 chars)`);
+  if (!item.description || typeof item.description !== 'string') errors.push('missing required description');
+  else if (item.description.length < 15) errors.push(`description too short (min 15 chars)`);
+  if (item.version !== undefined && typeof item.version !== 'string') errors.push('version must be a string');
+  if (item.author !== undefined && typeof item.author !== 'string') errors.push('author must be a string');
+  if (item.license !== undefined && typeof item.license !== 'string') errors.push('license must be a string');
+  if (item.updated_at !== undefined) {
+    if (typeof item.updated_at !== 'string') errors.push('updated_at must be a string');
+    else if (isNaN(Date.parse(item.updated_at))) errors.push(`updated_at "${item.updated_at}" is not a valid ISO date`);
+  }
+  if (item.downloads !== undefined && typeof item.downloads !== 'number') errors.push('downloads must be a number');
+  if (item.verified !== undefined && typeof item.verified !== 'boolean') errors.push('verified must be a boolean');
+  if (item.trustLevel !== undefined && typeof item.trustLevel !== 'string') errors.push('trustLevel must be a string');
+  if (item.rating !== undefined) {
+    if (typeof item.rating !== 'number') errors.push('rating must be a number');
+    else if (item.rating < 0 || item.rating > 5) errors.push('rating must be between 0 and 5');
+  }
+  if (item.popularity !== undefined && typeof item.popularity !== 'number') errors.push('popularity must be a number');
+  if (item.lastUpdate !== undefined) {
+    if (typeof item.lastUpdate !== 'string') errors.push('lastUpdate must be a string');
+    else if (isNaN(Date.parse(item.lastUpdate))) errors.push(`lastUpdate "${item.lastUpdate}" is not a valid ISO date`);
+  }
+  if (errors.length > 0) {
+    console.warn(`[PromptGraph] Skipping invalid registry entry "${item.id || item.name || '(unnamed)'}": ${errors.join('; ')}`);
+    return { ok: false };
+  }
+  return { ok: true };
+}
+
 // Deterministic short code from an id. Same id always yields the same code,
 // so codes auto-generate — no need to assign them by hand.
 export function codeFor(id) {
@@ -136,7 +172,9 @@ export async function browseMarketplace(topK = 20) {
     const registry = JSON.parse(text);
     if (!Array.isArray(registry.skills)) return { error: 'Invalid registry format' };
     return registry.skills
-      .map(s => ({ ...s, code: s.code || codeFor(s.id) })) // auto-fill code
+      .map(s => Object.assign(Object.create(null), s, { code: s.code || codeFor(s.id) }))
+      .filter(s => validateRegistryEntry(s).ok)
+      .filter(s => s.raw_url)
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
       .slice(0, topK);
   } catch (e) {
@@ -154,6 +192,10 @@ export async function installSkillFromUrl(url) {
     // derive filename from URL
     const urlName = rawUrl.split('/').pop().replace(/[^a-z0-9-_.]/gi, '-');
     const dest = path.join(SKILLS_DIR, urlName.endsWith('.md') ? urlName : urlName + '.md');
+    const resolvedDest = path.resolve(dest);
+    if (!resolvedDest.startsWith(path.resolve(SKILLS_DIR))) {
+      return { error: 'Path traversal blocked: destination outside marketplace directory' };
+    }
     const v = writeSkillAtomic(dest, content);
     if (!v.ok) return { error: 'Downloaded skill failed validation', issues: v.errors };
     return { success: true, path: dest, url: rawUrl };
@@ -172,8 +214,9 @@ export async function installSkill(query) {
     const text = await fetchText(REGISTRY_URL);
     const registry = JSON.parse(text);
     const q = String(query).trim().toLowerCase();
+    const validSkills = (registry.skills || []).filter(s => validateRegistryEntry(s).ok);
     // match by code (stored OR auto-generated), id, or name
-    const skill = registry.skills?.find(s =>
+    const skill = validSkills.find(s =>
       (s.code || codeFor(s.id)).toLowerCase() === q ||
       s.id?.toLowerCase() === q ||
       s.name?.toLowerCase() === q
@@ -191,6 +234,10 @@ export async function installSkill(query) {
     fs.mkdirSync(SKILLS_DIR, { recursive: true });
     ensureMarketplaceSource();
     const dest = path.join(SKILLS_DIR, `${skillId}.md`);
+    const resolvedDest = path.resolve(dest);
+    if (!resolvedDest.startsWith(path.resolve(SKILLS_DIR))) {
+      return { error: 'Path traversal blocked: destination outside marketplace directory' };
+    }
 
     const content = await fetchText(skill.raw_url);
     const v = writeSkillAtomic(dest, content);
@@ -222,7 +269,7 @@ export async function browseBundles(topK = 20) {
   try {
     const text = await fetchText(REGISTRY_URL);
     const registry = JSON.parse(text);
-    const bundles = registry.bundles || [];
+    const bundles = (registry.bundles || []).filter(b => validateRegistryEntry(b).ok);
     const cache = readSkillCountCache();
     const now = Date.now();
     let changed = false;
@@ -250,7 +297,7 @@ export async function browseBundles(topK = 20) {
 
     return bundles
       .filter(b => !b.repo_url || b.skillCount > 0)
-      .map(b => ({ ...b, code: b.code || codeFor(b.id) }))
+      .map(b => Object.assign(Object.create(null), b, { code: b.code || codeFor(b.id) }))
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
       .slice(0, topK);
   } catch (e) {
@@ -273,7 +320,8 @@ export async function installBundle(bundleId) {
     const text = await fetchText(REGISTRY_URL);
     const registry = JSON.parse(text);
     const q = String(bundleId).trim().toLowerCase();
-    const bundle = (registry.bundles || []).find(b =>
+    const validSkills = (registry.skills || []).filter(s => validateRegistryEntry(s).ok);
+    const bundle = (registry.bundles || []).filter(b => validateRegistryEntry(b).ok).find(b =>
       (b.code || codeFor(b.id)).toLowerCase() === q || b.id?.toLowerCase() === q || b.name?.toLowerCase() === q
     );
     if (!bundle) return { error: `No bundle matching "${bundleId}"` };
@@ -290,12 +338,16 @@ export async function installBundle(bundleId) {
 
     const delay = (ms) => new Promise(r => setTimeout(r, ms));
     for (const skillId of bundle.skills || []) {
-      const skill = registry.skills?.find(s => s.id === skillId);
+      const skill = validSkills.find(s => s.id === skillId);
       if (!skill?.raw_url) { failed.push(skillId); continue; }
       try {
         if (installed.length > 0) await delay(300); // rate limit: 300ms between requests
         const content = await fetchText(skill.raw_url);
         const dest = path.join(SKILLS_DIR, `${skillId}.md`);
+        const resolvedDest = path.resolve(dest);
+        if (!resolvedDest.startsWith(path.resolve(SKILLS_DIR))) {
+          failed.push(skillId); continue;
+        }
         const v = writeSkillAtomic(dest, content);
         if (!v.ok) { failed.push(skillId); continue; }
         installed.push(skillId);
@@ -478,7 +530,7 @@ export function pruneInvalidRepos() {
       continue;
     }
 
-    const mdFiles = globSync(`${src.dir}/**/*.md`);
+    const mdFiles = globSync(`${src.dir}/**/*.md`).map(fp => path.resolve(fp));
     if (mdFiles.length === 0) {
       removed.push({ repo: repoName, reason: 'no .md files' });
       config.sources = config.sources.filter(s => s !== src);
@@ -506,3 +558,74 @@ export function pruneInvalidRepos() {
   saveConfig(config);
   return { removed, kept, errors };
 }
+
+// ── Trust level system ─────────────────────────────────────────────────────────
+const VALID_TRUST_LEVELS = ['verified', 'official', 'community', 'trusted', 'unknown']
+
+function calcPopularity(downloads = 0, rating = 0) {
+  return Math.round((downloads || 0) * ((rating || 0) + 1) * 100) / 100
+}
+
+export async function setTrustLevel(name, level) {
+  const db = getDb()
+  if (!VALID_TRUST_LEVELS.includes(level)) {
+    return { ok: false, error: `Invalid trust level "${level}". Must be one of: ${VALID_TRUST_LEVELS.join(', ')}` }
+  }
+  const id = String(name)
+  db.prepare(`
+    INSERT INTO registry_entries (id, trust_level, last_update)
+    VALUES (?, ?, datetime('now'))
+    ON CONFLICT(id) DO UPDATE SET trust_level = excluded.trust_level, last_update = excluded.last_update
+  `).run(id, level)
+  return { ok: true }
+}
+
+export async function getByTrustLevel(level) {
+  const db = getDb()
+  if (level && !VALID_TRUST_LEVELS.includes(level)) {
+    return { error: `Invalid trust level "${level}". Must be one of: ${VALID_TRUST_LEVELS.join(', ')}` }
+  }
+  if (level) {
+    return db.prepare('SELECT * FROM registry_entries WHERE trust_level = ? ORDER BY id').all(level)
+  }
+  return db.prepare('SELECT * FROM registry_entries ORDER BY id').all()
+}
+
+export async function incrementDownloads(name) {
+  const db = getDb()
+  const id = String(name)
+  const existing = db.prepare('SELECT downloads, rating FROM registry_entries WHERE id = ?').get(id)
+  if (existing) {
+    const newDownloads = (existing.downloads || 0) + 1
+    const pop = calcPopularity(newDownloads, existing.rating || 0)
+    db.prepare('UPDATE registry_entries SET downloads = ?, popularity = ?, last_update = datetime(\'now\') WHERE id = ?')
+      .run(newDownloads, pop, id)
+  } else {
+    const pop = calcPopularity(1, 0)
+    db.prepare('INSERT INTO registry_entries (id, downloads, popularity, last_update) VALUES (?, 1, ?, datetime(\'now\'))')
+      .run(id, pop)
+  }
+  return { ok: true }
+}
+
+export async function rateSkill(name, rating) {
+  const db = getDb()
+  if (typeof rating !== 'number' || rating < 0 || rating > 5) {
+    return { ok: false, error: 'Rating must be a number between 0 and 5' }
+  }
+  const id = String(name)
+  const existing = db.prepare('SELECT rating, rating_count, downloads FROM registry_entries WHERE id = ?').get(id)
+  if (existing) {
+    const count = existing.rating_count || 0
+    const newRating = Math.round(((existing.rating || 0) * count + rating) / (count + 1) * 100) / 100
+    const newCount = count + 1
+    const pop = calcPopularity(existing.downloads || 0, newRating)
+    db.prepare('UPDATE registry_entries SET rating = ?, rating_count = ?, popularity = ?, last_update = datetime(\'now\') WHERE id = ?')
+      .run(newRating, newCount, pop, id)
+  } else {
+    const pop = calcPopularity(0, rating)
+    db.prepare('INSERT INTO registry_entries (id, rating, rating_count, popularity, last_update) VALUES (?, ?, 1, ?, datetime(\'now\'))')
+      .run(id, rating, pop)
+  }
+  return { ok: true }
+}

package/package.json

@@ -1,6 +1,11 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.4.7",
+  "version": "2.5.0",
+  "files": [
+    "*.js",
+    "src/**/*.js",
+    "README.md"
+  ],
   "main": "index.js",
   "type": "module",
   "bin": {
@@ -42,13 +47,14 @@
     "fastembed": "^2.1.0",
     "glob": "^13.0.6",
     "gray-matter": "^4.0.3",
+    "hnswlib-node": "^3.0.0",
     "ora": "^9.4.0"
   },
   "devDependencies": {
+    "@vitest/coverage-v8": "^4.1.8",
     "vitest": "^4.1.8"
   },
   "overrides": {
-    "tar": "^6.2.1",
-    "js-yaml": "^4.1.0"
+    "tar": "^7.5.11"
   }
 }

package/search.js

@@ -2,6 +2,12 @@ import { embed, cosineSimilarity } from './embedder.js';
 import { getDb, blobToVec } from './db.js';
 import { annSearch } from './ann.js';
 
+function getHybridWeights(query) {
+  const hasTechTerms = /[A-Z]|\d/.test(query);
+  if (hasTechTerms) return { embedWeight: 0.5, bm25Weight: 0.5 }
+  return { embedWeight: 0.7, bm25Weight: 0.3 }
+}
+
 function applyRatingBoost(db, id, score) {
   const r = db.prepare('SELECT success, fail FROM ratings WHERE skill_id = ?').get(id);
   if (r && (r.success + r.fail) > 3) {
@@ -11,13 +17,13 @@ function applyRatingBoost(db, id, score) {
   return score;
 }
 
-export async function search(query, topK = 5) {
-  const db = getDb();
-  const queryVec = await embed(query);
-
-  // Try ANN index first (fast, O(log N))
-  const annResults = await annSearch(queryVec, topK * 4);
+function normalizeBM25(raw) {
+  return Math.max(0, 1 + raw / 10);
+}
 
+async function runEmbeddingSearch(db, queryVec, topK) {
+  // Try ANN index first (fast)
+  const annResults = await annSearch(queryVec, topK);
   if (annResults && annResults.length > 0) {
     const bestBySkill = new Map();
     for (const r of annResults) {
@@ -25,11 +31,8 @@ export async function search(query, topK = 5) {
       if (!prev || r.score > prev) bestBySkill.set(r.skill_id, r.score);
     }
     return [...bestBySkill.entries()]
-      .map(([id, score]) => ({ id, score: applyRatingBoost(db, id, score) }))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, topK)
-      .map(({ id, score }) => skillWithSnippet(db, id, score))
-      .filter(Boolean);
+      .map(([id, score]) => ({ id, score: Math.max(0, score) }))
+      .sort((a, b) => b.score - a.score);
   }
 
   // Fallback: brute force cosine (used before first reindex)
@@ -42,14 +45,14 @@ export async function search(query, topK = 5) {
       if (!prev || score > prev) bestBySkill.set(chunk.skill_id, score);
     }
     return [...bestBySkill.entries()]
-      .map(([id, score]) => ({ id, score: applyRatingBoost(db, id, score) }))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, topK)
-      .map(({ id, score }) => skillWithSnippet(db, id, score))
-      .filter(Boolean);
+      .map(([id, score]) => ({ id, score: Math.max(0, score) }))
+      .sort((a, b) => b.score - a.score);
   }
 
-  // Fast-mode fallback: FTS5 keyword search (no embeddings)
+  return [];
+}
+
+function runBM25Search(db, query, topK) {
   try {
     const terms = query.replace(/[^\w\s]/g, ' ').trim().split(/\s+/).filter(Boolean).join(' OR ');
     const rows = db.prepare(
@@ -57,14 +60,77 @@ export async function search(query, topK = 5) {
        JOIN skills s ON skills_fts.id = s.id
        WHERE skills_fts MATCH ? ORDER BY score LIMIT ?`
     ).all(terms, topK);
-    return rows.map(r => skillWithSnippet(db, r.id, Math.max(0, 1 + r.score / 10))).filter(Boolean);
+    return rows.map(r => ({ id: r.id, score: normalizeBM25(r.score) }));
   } catch {
     return [];
   }
 }
 
+export async function search(query, topK = 5) {
+  const db = getDb();
+  const { embedWeight, bm25Weight } = getHybridWeights(query);
+  const queryVec = await embed(query);
+
+  const embedResults = await runEmbeddingSearch(db, queryVec, topK * 4);
+  const bm25Results = runBM25Search(db, query, topK * 4);
+
+  // Fallback: pure BM25 if embeddings unavailable
+  if (!embedResults.length) {
+    return bm25Results.slice(0, topK)
+      .map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score)))
+      .filter(Boolean);
+  }
+
+  // Fallback: pure embedding if BM25 returns nothing
+  if (!bm25Results.length) {
+    return embedResults.slice(0, topK)
+      .map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
+      .filter(Boolean);
+  }
+
+  // Hybrid: combine normalized scores from both signals
+  const combined = new Map();
+  for (const { id, score } of embedResults) {
+    combined.set(id, { embedScore: score, bm25Score: 0 });
+  }
+  for (const { id, score } of bm25Results) {
+    const entry = combined.get(id);
+    if (entry) {
+      entry.bm25Score = score;
+    } else {
+      combined.set(id, { embedScore: 0, bm25Score: score });
+    }
+  }
+
+  const ordered = [...combined.entries()]
+    .map(([id, s]) => ({
+      id,
+      score: embedWeight * s.embedScore + bm25Weight * s.bm25Score,
+    }))
+    .sort((a, b) => b.score - a.score)
+
+  const rerankerEnabled = process.env.PG_RERANKER !== '0'
+
+  if (rerankerEnabled) {
+    const { Reranker } = await import('./src/reranker/reranker.js')
+    const reranker = new Reranker()
+    const topN = ordered.slice(0, 20)
+      .map(({ id, score }) => {
+        const s = skillWithSnippet(db, id, score)
+        return s ? { id, text: s.snippet, score } : null
+      })
+      .filter(Boolean)
+    const reranked = await reranker.rerank(query, topN)
+    return reranked.map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score))).filter(Boolean)
+  }
+
+  return ordered.slice(0, topK)
+    .map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
+    .filter(Boolean);
+}
+
 function skillWithSnippet(db, id, score) {
-  const skill = db.prepare('SELECT id, name, description, path, source, content FROM skills WHERE id = ?').get(id);
+  const skill = db.prepare('SELECT id, name, description, path, source, content, version, author, license, updated_at, downloads, verified FROM skills WHERE id = ?').get(id);
   if (!skill) return null;
   const { content, ...rest } = skill;
   const snippet = content?.replace(/^---[\s\S]*?---\n?/, '').trim().slice(0, 400) || '';
@@ -131,5 +197,5 @@ export function getImpact(nameOrId) {
 
 export function listAll() {
   const db = getDb();
-  return db.prepare('SELECT id, name, description, source FROM skills ORDER BY source, name').all();
+  return db.prepare('SELECT id, name, description, source, version, author, license, updated_at, downloads, verified FROM skills ORDER BY source, name').all();
 }

package/src/filter/hard-filter.js

@@ -19,6 +19,8 @@ const SKIP_DIRS = new Set([
   'tests', 'test', '__tests__', 'spec', 'fixtures', 'assets', 'images',
   'img', 'screenshots', 'media', 'static', 'public', 'dist', 'build',
   'node_modules', 'vendor', 'third_party',
+  'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
+  'cheatsheets', 'resources',
 ]);
 
 const BADGE_RE = /!\[.*\]\(https?:\/\/(img\.shields\.io|badge\.fury|travis-ci|github\.com\/[^)]+\/badge)/i;

package/src/filter/train.js

@@ -1,10 +1,12 @@
 import fs from 'fs';
 import path from 'path';
 import os from 'os';
+import { fileURLToPath } from 'url';
 import { globSync } from 'glob';
 import { embedBatch } from '../../embedder.js';
 
-const TRAINING_DIR = path.resolve(new URL('../../registry/training', import.meta.url).pathname);
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const TRAINING_DIR = path.resolve(__dirname, '../../registry/training');
 const MODEL_PATH = path.join(os.homedir(), '.claude', '.promptgraph', 'model.json');
 
 function readAllMd(dir) {

package/src/reranker/reranker.js

@@ -0,0 +1,27 @@
+export class Reranker {
+  constructor(modelName = 'default', device = 'cpu') {
+    this.modelName = modelName
+    this.device = device
+  }
+
+  // Cross-encoder style reranker
+  // Plug-in point for BGE Reranker / MiniLM Reranker via ONNX
+  async rerank(query, results) {
+    if (!results || results.length === 0) return []
+
+    const queryTerms = (query.toLowerCase().match(/\w+/g) || [])
+    const queryTermCount = queryTerms.length || 1
+
+    const scored = results.map(r => {
+      const text = (r.text || '').toLowerCase()
+      const termOverlap = queryTerms.filter(t => text.includes(t)).length / queryTermCount
+
+      return {
+        ...r,
+        score: 0.8 * (r.score || 0) + 0.2 * termOverlap,
+      }
+    })
+
+    return scored.sort((a, b) => b.score - a.score).slice(0, 5)
+  }
+}

package/src/store/flat-store.js

@@ -0,0 +1,61 @@
+import { VectorStore } from './vector-store.js';
+
+export class FlatVectorStore extends VectorStore {
+  constructor() {
+    super();
+    this._items = [];
+  }
+
+  async add(id, vector) {
+    this._items.push({ skill_id: id, vec: new Float32Array(vector) });
+  }
+
+  async addBatch(entries) {
+    for (const { id, vector } of entries) {
+      this._items.push({ skill_id: id, vec: new Float32Array(vector) });
+    }
+  }
+
+  async remove(id) {
+    this._items = this._items.filter(item => item.skill_id !== id);
+  }
+
+  async search(vector, topK = 20) {
+    const qArr = new Float32Array(vector);
+    const bestBySkill = new Map();
+    for (const entry of this._items) {
+      const score = cosineSim(qArr, entry.vec);
+      const prev = bestBySkill.get(entry.skill_id);
+      if (!prev || score > prev) bestBySkill.set(entry.skill_id, score);
+    }
+    return [...bestBySkill.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, topK)
+      .map(([skill_id, score]) => ({ skill_id, score }));
+  }
+
+  async build(entries) {
+    this._items = [];
+    for (const { skill_id, vector } of entries) {
+      this._items.push({ skill_id, vec: new Float32Array(vector) });
+    }
+  }
+
+  async clear() {
+    this._items = [];
+  }
+
+  get size() {
+    return this._items.length;
+  }
+}
+
+function cosineSim(a, b) {
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-8);
+}